WIP refactor dequeue_write/continue_write into a single "coroutine"

.gitea/workflows/buildenv.Dockerfile

@@ -1,29 +1,28 @@
-FROM node:16-bookworm
+FROM node:16-bullseye
 
 WORKDIR /root
 
-ADD ./docker/etc/apt/trusted.gpg.d /etc/apt/trusted.gpg.d
+ADD ./docker/vitastor.gpg /etc/apt/trusted.gpg.d
 
-RUN echo 'deb http://deb.debian.org/debian bookworm-backports main' >> /etc/apt/sources.list; \
+RUN echo 'deb http://deb.debian.org/debian bullseye-backports main' >> /etc/apt/sources.list; \
-    echo 'deb http://vitastor.io/debian bookworm main' >> /etc/apt/sources.list; \
+    echo 'deb http://vitastor.io/debian bullseye main' >> /etc/apt/sources.list; \
     echo >> /etc/apt/preferences; \
     echo 'Package: *' >> /etc/apt/preferences; \
-    echo 'Pin: release n=bookworm-backports' >> /etc/apt/preferences; \
+    echo 'Pin: release a=bullseye-backports' >> /etc/apt/preferences; \
     echo 'Pin-Priority: 500' >> /etc/apt/preferences; \
     echo >> /etc/apt/preferences; \
     echo 'Package: *' >> /etc/apt/preferences; \
     echo 'Pin: origin "vitastor.io"' >> /etc/apt/preferences; \
     echo 'Pin-Priority: 1000' >> /etc/apt/preferences; \
-    perl -i -pe 's/Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/debian.sources; \
     grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb/deb-src/' >> /etc/apt/sources.list; \
     echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf; \
     echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
 
 RUN apt-get update
-RUN apt-get -y install etcd qemu-system-x86 qemu-block-extra qemu-utils fio libasan8 \
+RUN apt-get -y install etcd qemu-system-x86 qemu-block-extra qemu-utils fio libasan5 \
     libgoogle-perftools-dev devscripts libjerasure-dev cmake libibverbs-dev libisal-dev
 RUN apt-get -y build-dep fio qemu=`dpkg -s qemu-system-x86|grep ^Version:|awk '{print $2}'`
-RUN apt-get update && apt-get -y install jq lp-solve sudo nfs-common fdisk parted libc-ares-dev udev
+RUN apt-get update && apt-get -y install jq lp-solve sudo nfs-common fdisk parted
 RUN apt-get --download-only source fio qemu=`dpkg -s qemu-system-x86|grep ^Version:|awk '{print $2}'`
 
 RUN set -ex; \

.gitea/workflows/test.yml

@@ -63,7 +63,7 @@ jobs:
     container: ${{env.TEST_IMAGE}}:${{github.sha}}
     steps:
     # leak sanitizer sometimes crashes
-    - run: cd /root/vitastor/build && ASAN_OPTIONS=detect_leaks=0 make -j16 build_tests test
+    - run: cd /root/vitastor/build && ASAN_OPTIONS=detect_leaks=0 make -j16 test
 
   npm_lint:
     runs-on: ubuntu-latest
@@ -306,78 +306,6 @@ jobs:
           echo ""
         done
 
-  test_dump_load:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_dump_load.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_dump_load_32k:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: TEST_NAME=32k OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k" OFFSET_ARGS="$OSD_ARGS" /root/vitastor/tests/test_dump_load.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_old_dump_load:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 /root/vitastor/tests/test_dump_load.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_dump_load_old_32k:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: TEST_NAME=old_32k OLD=1 OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k" OFFSET_ARGS="$OSD_ARGS" /root/vitastor/tests/test_dump_load.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
   test_old_interrupted_rebalance:
     runs-on: ubuntu-latest
     needs: build
@@ -1278,78 +1206,6 @@ jobs:
           echo ""
         done
 
-  test_checksum:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_checksum.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_old_checksum:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 /root/vitastor/tests/test_checksum.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_corrupt_all:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_corrupt_all.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_old_corrupt_all:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 /root/vitastor/tests/test_corrupt_all.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
   test_reweight_half:
     runs-on: ubuntu-latest
     needs: build
@@ -1530,24 +1386,6 @@ jobs:
           echo ""
         done
 
-  test_resize_last:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_resize_last.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
   test_resize_auto:
     runs-on: ubuntu-latest
     needs: build
@@ -1584,24 +1422,6 @@ jobs:
           echo ""
         done
 
-  test_old_resize_last:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 /root/vitastor/tests/test_resize_last.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
   test_old_resize_auto:
     runs-on: ubuntu-latest
     needs: build

CMakeLists.txt

@@ -1,20 +1,20 @@
-cmake_minimum_required(VERSION 2.8...3.30)
+cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "3.0.14")
+set(VITASTOR_VERSION "3.0.2")
 
 include(CTest)
 
 add_custom_target(build_tests)
-set_property(TEST PROPERTY ENVIRONMENT LSAN_OPTIONS=suppressions=${CMAKE_CURRENT_BINARY_DIR}/lsan-suppress.txt)
+add_custom_target(test
-add_test(gen_lsan_suppress
+	COMMAND
-    ${CMAKE_COMMAND} -E echo leak:tcmalloc > "${CMAKE_CURRENT_BINARY_DIR}/lsan-suppress.txt"
+	echo leak:tcmalloc > ${CMAKE_CURRENT_BINARY_DIR}/lsan-suppress.txt &&
+	env LSAN_OPTIONS=suppressions=${CMAKE_CURRENT_BINARY_DIR}/lsan-suppress.txt ${CMAKE_CTEST_COMMAND}
 )
-set_tests_properties(gen_lsan_suppress PROPERTIES FIXTURES_SETUP f_lsan_suppress)
-set_property(TEST PROPERTY FIXTURES_REQUIRED f_lsan_suppress)
 # make -j16 -C ../../build test_heap && ../../build/src/test/test_heap
 # make -j16 -C ../../build test_heap && rm -f $(find ../../build -name '*.gcda') && ctest -V -T test -T coverage -R heap --test-dir ../../build && (cd ../../build; gcovr -f ../src --html --html-nested -o coverage/index.html; cd ../src/test)
 # make -j16 -C ../../build test_blockstore && rm -f $(find ../../build -name '*.gcda') && ctest -V -T test -T coverage -R blockstore --test-dir ../../build && (cd ../../build; gcovr -f ../src --html --html-nested -o coverage/index.html; cd ../src/test)
 # kcov --include-path=../../../src ../../kcov ./test_blockstore
+add_dependencies(test build_tests)
 add_subdirectory(src)

csi/Dockerfile

@@ -1,5 +1,5 @@
 # Compile stage
-FROM golang:trixie AS build
+FROM golang:bookworm AS build
 
 ADD go.sum go.mod /app/
 RUN cd /app; CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go mod download -x
@@ -9,7 +9,7 @@ RUN perl -i -e '$/ = undef; while(<>) { s/\n\s*(\{\s*\n)/$1\n/g; s/\}(\s*\n\s*)e
     CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o vitastor-csi
 
 # Final stage
-FROM debian:trixie
+FROM debian:bookworm
 
 LABEL maintainers="Vitaliy Filippov <vitalif@yourcmc.ru>"
 LABEL description="Vitastor CSI Driver"
@@ -25,20 +25,20 @@ RUN apt-get update && \
         # NFS mount dependencies
         nfs-common netbase \
         # dependencies of qemu-storage-daemon
-        libaio1t64 libc6 libfuse3-4 libglib2.0-0t64 libgmp10 libgnutls30t64 \
+        libnuma1 liburing2 libglib2.0-0 libfuse3-3 libaio1 libzstd1 libnettle8 \
-        libhogweed6t64 libnettle8t64 libnuma1 libselinux1 liburing2 libzstd1 zlib1g && \
+        libgmp10 libhogweed6 libp11-kit0 libidn2-0 libunistring2 libtasn1-6 libpcre2-8-0 libffi8 && \
     apt-get clean && \
     (echo options nbd nbds_max=128 > /etc/modprobe.d/nbd.conf)
 
 COPY --from=build /app/vitastor-csi /bin/
 
-RUN (echo deb http://vitastor.io/debian trixie main > /etc/apt/sources.list.d/vitastor.list) && \
+RUN (echo deb http://vitastor.io/debian bookworm main > /etc/apt/sources.list.d/vitastor.list) && \
     ((echo 'Package: *'; echo 'Pin: origin "vitastor.io"'; echo 'Pin-Priority: 1000') > /etc/apt/preferences.d/vitastor.pref) && \
     wget -q -O /etc/apt/trusted.gpg.d/vitastor.gpg https://vitastor.io/debian/pubkey.gpg && \
     apt-get update && \
     apt-get install -y vitastor-client ibverbs-providers && \
-    wget https://vitastor.io/archive/qemu/qemu-trixie-10.0.2%2Bds-2%2Bvitastor1/qemu-utils_10.0.2%2Bds-2%2Bvitastor1_amd64.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-utils_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
-    wget https://vitastor.io/archive/qemu/qemu-trixie-10.0.2%2Bds-2%2Bvitastor1/qemu-block-extra_10.0.2%2Bds-2%2Bvitastor1_amd64.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-block-extra_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
     dpkg -x qemu-utils*.deb tmp1 && \
     dpkg -x qemu-block-extra*.deb tmp1 && \
     cp -a tmp1/usr/bin/qemu-storage-daemon /usr/bin/ && \

csi/Dockerfile.local

@@ -1,5 +1,5 @@
 # Compile stage
-FROM golang:trixie AS build
+FROM golang:bookworm AS build
 
 ADD go.sum go.mod /app/
 RUN cd /app; CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go mod download -x
@@ -9,7 +9,7 @@ RUN perl -i -e '$/ = undef; while(<>) { s/\n\s*(\{\s*\n)/$1\n/g; s/\}(\s*\n\s*)e
     CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o vitastor-csi
 
 # Final stage
-FROM debian:trixie
+FROM debian:bookworm
 
 LABEL maintainers="Vitaliy Filippov <vitalif@yourcmc.ru>"
 LABEL description="Vitastor CSI Driver"
@@ -36,8 +36,8 @@ ADD deb /deb
 
 RUN apt-get update && \
     apt-get -y install /deb/vitastor-client_*.deb && \
-    wget https://vitastor.io/archive/qemu/qemu-trixie-9.2.2%2Bds-1%2Bvitastor4/qemu-utils_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-utils_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
-    wget https://vitastor.io/archive/qemu/qemu-trixie-9.2.2%2Bds-1%2Bvitastor4/qemu-block-extra_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-block-extra_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
     dpkg -x qemu-utils*.deb tmp1 && \
     dpkg -x qemu-block-extra*.deb tmp1 && \
     cp -a tmp1/usr/bin/qemu-storage-daemon /usr/bin/ && \

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.14
+VITASTOR_VERSION ?= v3.0.2
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v3.0.14
+          image: vitalif/vitastor-csi:v3.0.2
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v3.0.14
+          image: vitalif/vitastor-csi:v3.0.2
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "3.0.14"
+    vitastorCSIDriverVersion = "3.0.2"
 )
 
 // Config struct fills the parameters of request or user input

debian/build-vitastor-ubuntu-resolute.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-# 26.04 Resolute Raccoon
-
-docker build --build-arg DISTRO=ubuntu --build-arg REL=resolute -t vitastor-buildenv:resolute -f vitastor-buildenv.Dockerfile .
-docker run -it --rm -e REL=resolute -v `dirname $0`/../:/root/vitastor vitastor-buildenv:resolute /root/vitastor/debian/vitastor-build.sh

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (3.0.14-1) unstable; urgency=medium
+vitastor (3.0.2-1) unstable; urgency=medium
 
   * Bugfixes
 

debian/vitastor-build.sh

@@ -44,7 +44,7 @@ curl -s https://git.yourcmc.ru/vitalif/antietcd/archive/master.tar.gz | tar -zx
 curl -s https://git.yourcmc.ru/vitalif/tinyraft/archive/master.tar.gz | tar -zx
 
 cd /root/vitastor/packages/vitastor-$REL
-if [[ ( "$REL" = "trixie" || "$REL" = "resolute" ) && -e ../vitastor-bookworm/vitastor_$VER.orig.tar.xz ]]; then
+if [[ "$REL" = "trixie" && -e ../vitastor-bookworm/vitastor_$VER.orig.tar.xz ]]; then
     # Fucking shit, archives differ between bookworm (xz 5.4.1) and trixie (xz 5.8.1)
     cp ../vitastor-bookworm/vitastor_$VER.orig.tar.xz .
 else

docker/Dockerfile

@@ -1,6 +1,6 @@
 # Build Docker image with Vitastor packages
 
-FROM debian:trixie
+FROM debian:bookworm
 
 ADD etc/apt /etc/apt/
 RUN apt-get update && apt-get -y install vitastor ibverbs-providers udev systemd qemu-system-x86 qemu-system-common qemu-block-extra qemu-utils jq nfs-common && apt-get clean

docker/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.14
+VITASTOR_VERSION ?= v3.0.2
 
 all: build push
 

docker/etc/apt/preferences

@@ -1,3 +1,3 @@
 Package: *
-Pin: release n=trixie-backports
+Pin: release n=bookworm-backports
 Pin-Priority: 500

docker/etc/apt/sources.list.d/vitastor.list

@@ -1,2 +1,2 @@
-deb http://vitastor.io/debian trixie main
+deb http://vitastor.io/debian bookworm main
-#deb http://http.debian.net/debian/ trixie-backports main
+deb http://http.debian.net/debian/ bookworm-backports main

docker/etc/systemd/system/vitastor-host.service

@@ -7,7 +7,7 @@ PartOf=vitastor.target
 [Service]
 Restart=always
 EnvironmentFile=/etc/vitastor/docker.conf
-ExecStart=bash -c 'docker run --rm -i -v /etc/vitastor:/etc/vitastor -v /dev:/dev -v /run:/run -e SYSTEMD_IN_CHROOT=0 \
+ExecStart=bash -c 'docker run --rm -i -v /etc/vitastor:/etc/vitastor -v /dev:/dev -v /run:/run \
     --security-opt seccomp=unconfined --privileged --pid=host --log-driver none --network host --name vitastor vitastor:$VITASTOR_VERSION \
     sleep.sh'
 ExecStartPost=udevadm trigger

docker/etc/vitastor/docker.conf

@@ -4,7 +4,7 @@
 #
 
 # Desired Vitastor version
-VITASTOR_VERSION=v3.0.14
+VITASTOR_VERSION=v3.0.2
 
 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here

docker/install.sh

@@ -2,7 +2,8 @@
 
 set -e
 
-cp -urv /etc/systemd/system/vitastor* /host-etc/systemd/system/
+cp -urv /etc/default /host-etc/
-cp -urv /etc/udev/rules.d /host-etc/udev/
+cp -urv /etc/systemd /host-etc/
+cp -urv /etc/udev /host-etc/
 cp -urnv /etc/vitastor /host-etc/
 cp -urnv /opt/scripts/* /host-bin/

docs/config/network.en.md

@@ -22,6 +22,7 @@ between clients, OSDs and etcd.
 - [rdma_max_msg](#rdma_max_msg)
 - [rdma_max_recv](#rdma_max_recv)
 - [rdma_max_send](#rdma_max_send)
+- [rdma_odp](#rdma_odp)
 - [peer_connect_interval](#peer_connect_interval)
 - [peer_connect_timeout](#peer_connect_timeout)
 - [osd_idle_timeout](#osd_idle_timeout)
@@ -101,6 +102,11 @@ found or if `osd_network` is not specified. Auto-selection is also
 unsupported with old libibverbs < v32, like in Debian 10 Buster or
 CentOS 7.
 
+Vitastor supports all adapters, even ones without ODP support, like
+Mellanox ConnectX-3 and non-Mellanox cards. Versions up to Vitastor
+1.2.0 required ODP which is only present in Mellanox ConnectX >= 4.
+See also [rdma_odp](#rdma_odp).
+
 Run `ibv_devinfo -v` as root to list available RDMA devices and their
 features.
 
@@ -110,23 +116,6 @@ the manual of your network vendor for details about setting up the switch
 for RoCEv2 correctly. Usually it means setting up Lossless Ethernet with
 PFC (Priority Flow Control) and ECN (Explicit Congestion Notification).
 
-Vitastor supports all adapters, even ones without ODP (On-Demand Paging)
-support, like Mellanox ConnectX-3 and non-Mellanox cards. ODP is only present
-in Mellanox ConnectX >= 4 adapters and allows to skip memory registration
-for RDMA and thus, in theory, avoid memory copying.
-
-Versions up to Vitastor 1.2.0 required ODP, then it was disabled by default,
-but it was still supported up to 3.0.3. Now ODP support is removed because it
-actually only hurts performance: an example 3-node cluster with 8 NVMe in each
-node and 2*25 GBit/s ConnectX-6 RDMA network pushed 3950000 read iops without
-ODP, but only 239000 iops with ODP.
-
-This happens because Mellanox ODP implementation seems to be based on
-message retransmissions when the adapter doesn't know about the buffer yet -
-it likely uses standard "RNR retransmissions" (RNR = receiver not ready)
-which is generally slow in RDMA/RoCE networks. Here's a presentation about
-it from ISPASS-2021 conference: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
-
 ## rdma_port_num
 
 - Type: integer
@@ -198,6 +187,28 @@ less than `rdma_max_recv` so the receiving side doesn't run out of buffers.
 Doesn't affect memory usage - additional memory isn't allocated for send
 operations.
 
+## rdma_odp
+
+- Type: boolean
+- Default: false
+
+Use RDMA with On-Demand Paging. ODP is currently only available on Mellanox
+ConnectX-4 and newer adapters. ODP allows to not register memory explicitly
+for RDMA adapter to be able to use it. This, in turn, allows to skip memory
+copying during sending. One would think this should improve performance, but
+**in reality** RDMA performance with ODP is **drastically** worse. Example
+3-node cluster with 8 NVMe in each node and 2*25 GBit/s ConnectX-6 RDMA network
+without ODP pushes 3950000 read iops, but only 239000 iops with ODP...
+
+This happens because Mellanox ODP implementation seems to be based on
+message retransmissions when the adapter doesn't know about the buffer yet -
+it likely uses standard "RNR retransmissions" (RNR = receiver not ready)
+which is generally slow in RDMA/RoCE networks. Here's a presentation about
+it from ISPASS-2021 conference: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
+
+ODP support is retained in the code just in case a good ODP implementation
+appears one day.
+
 ## peer_connect_interval
 
 - Type: seconds

docs/config/network.ru.md

@@ -22,6 +22,7 @@
 - [rdma_max_msg](#rdma_max_msg)
 - [rdma_max_recv](#rdma_max_recv)
 - [rdma_max_send](#rdma_max_send)
+- [rdma_odp](#rdma_odp)
 - [peer_connect_interval](#peer_connect_interval)
 - [peer_connect_timeout](#peer_connect_timeout)
 - [osd_idle_timeout](#osd_idle_timeout)
@@ -100,6 +101,12 @@ RoCEv1/RoCEv2, и даже позволяет полностью отключи
 не задана. Также автовыбор не поддерживается со старыми версиями библиотеки
 libibverbs < v32, например в Debian 10 Buster или CentOS 7.
 
+Vitastor поддерживает все модели адаптеров, включая те, у которых
+нет поддержки ODP, то есть вы можете использовать RDMA с ConnectX-3 и
+картами производства не Mellanox. Версии Vitastor до 1.2.0 включительно
+требовали ODP, который есть только на Mellanox ConnectX 4 и более новых.
+См. также [rdma_odp](#rdma_odp).
+
 Запустите `ibv_devinfo -v` от имени суперпользователя, чтобы посмотреть
 список доступных RDMA-устройств, их параметры и возможности.
 
@@ -110,24 +117,6 @@ libibverbs < v32, например в Debian 10 Buster или CentOS 7.
 подразумевает настройку сети без потерь на основе PFC (Priority Flow
 Control) и ECN (Explicit Congestion Notification).
 
-Vitastor поддерживает все модели адаптеров, включая те, у которых нет
-поддержки ODP (On-Demand Paging), например, ConnectX-3 и карты производства
-не Mellanox. Функция ODP доступна только на адаптерах Mellanox ConnectX-4 и
-более новых и позволяет не регистрировать память для её использования RDMA-картой,
-благодаря чему в теории можно избежать лишних копирований памяти.
-
-Версии Vitastor до 1.2.0 включительно требовали ODP, потом функция был отключена
-по умолчанию, но поддерживалась вплоть до версии 3.0.3. Сейчас поддержка ODP
-полностью удалена, так как на самом деле она только портит производительность:
-например, на 3-узловом кластере с 8 NVMe в каждом узле и сетью 2*25 Гбит/с на
-чтение с RDMA без ODP удаётся снять 3950000 iops, а с ODP - всего 239000 iops.
-
-Это происходит из-за того, что реализация ODP у Mellanox неоптимальная и
-основана на повторной передаче сообщений, когда карте не известен буфер -
-вероятно, на стандартных "RNR retransmission" (RNR = receiver not ready).
-А данные повторные передачи в RDMA/RoCE - всегда очень медленная штука.
-Презентация на эту тему с конференции ISPASS-2021: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
-
 ## rdma_port_num
 
 - Тип: целое число
@@ -203,6 +192,29 @@ OSD в любом случае согласовывают реальное зн
 Не влияет на потребление памяти - дополнительная память на операции отправки
 не выделяется.
 
+## rdma_odp
+
+- Тип: булево (да/нет)
+- Значение по умолчанию: false
+
+Использовать RDMA с On-Demand Paging. ODP - функция, доступная пока что
+исключительно на адаптерах Mellanox ConnectX-4 и более новых. ODP позволяет
+не регистрировать память для её использования RDMA-картой. Благодаря этому
+можно не копировать данные при отправке их в сеть и, казалось бы, это должно
+улучшать производительность - но **по факту** получается так, что
+производительность только ухудшается, причём сильно. Пример - на 3-узловом
+кластере с 8 NVMe в каждом узле и сетью 2*25 Гбит/с на чтение с RDMA без ODP
+удаётся снять 3950000 iops, а с ODP - всего 239000 iops...
+
+Это происходит из-за того, что реализация ODP у Mellanox неоптимальная и
+основана на повторной передаче сообщений, когда карте не известен буфер -
+вероятно, на стандартных "RNR retransmission" (RNR = receiver not ready).
+А данные повторные передачи в RDMA/RoCE - всегда очень медленная штука.
+Презентация на эту тему с конференции ISPASS-2021: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
+
+Возможность использования ODP сохранена в коде на случай, если вдруг в один
+прекрасный день появится хорошая реализация ODP.
+
 ## peer_connect_interval
 
 - Тип: секунды

docs/config/osd.en.md

@@ -38,7 +38,6 @@ with an OSD restart or, for some of them, even without restarting by updating co
 - [journal_io](#journal_io)
 - [journal_sector_buffer_count](#journal_sector_buffer_count)
 - [journal_no_same_sector_overwrites](#journal_no_same_sector_overwrites)
-- [skip_corrupted_meta_entries](#skip_corrupted_meta_entries)
 - [throttle_small_writes](#throttle_small_writes)
 - [throttle_target_iops](#throttle_target_iops)
 - [throttle_target_mbs](#throttle_target_mbs)
@@ -70,7 +69,6 @@ with an OSD restart or, for some of them, even without restarting by updating co
 - [use_atomic_flag](#use_atomic_flag)
 - [pg_reshard_chunk_size](#pg_reshard_chunk_size)
 - [pg_reshard_chunk_pause_ms](#pg_reshard_chunk_pause_ms)
-- [gc_on_start](#gc_on_start)
 
 ## bind_address
 
@@ -281,19 +279,13 @@ Maximum number of journal flushers (see above min_flusher_count).
 - Type: boolean
 - Default: true
 
-Only for the old store ([meta_format](layout-osd.en.md#meta_format) 2).
+This parameter makes Vitastor always keep metadata area of the block device
-
+in memory. It's required for good performance because it allows to avoid
-This parameter makes Vitastor keep a copy of metadata area in memory as it is
+additional read-modify-write cycles during metadata modifications. Metadata
-on disk, in addition to the metadata database. When the option is enabled, every
+area size is currently roughly 224 MB per 1 TB of data. You can turn it off
-metadata entry is effectively stored in RAM twice. It's required for good performance
+to reduce memory usage by this value, but it will hurt performance. This
-because it allows to avoid additional read-modify-write cycles during metadata
+restriction is likely to be removed in the future along with the upgrade
-modifications. Metadata area size with the old store is roughly 224 MB per 1 TB
+of the metadata storage scheme.
-of data. You can turn the option off to reduce memory usage by this value, but
-it will reduce performance.
-
-For the new store ([meta_format](layout-osd.en.md#meta_format) 3), the option
-may be changed in the future to support operation without loading full metadata
-database in memory.
 
 ## inmemory_journal
 
@@ -372,8 +364,6 @@ blocks. The only situation when you should increase it to a larger value
 is when you enable journal_no_same_sector_overwrites. In this case set
 it to, for example, 1024.
 
-Not applicable to the new store ([meta_format](layout-osd.en.md#meta_format) 3).
-
 ## journal_no_same_sector_overwrites
 
 - Type: boolean
@@ -387,17 +377,6 @@ journal after writing it instead of possibly overwriting it the second time.
 
 Most (99%) other SSDs don't need this option.
 
-Not applicable to the new store ([meta_format](layout-osd.en.md#meta_format) 3).
-
-## skip_corrupted_meta_entries
-
-- Type: boolean
-- Default: false
-
-Only for the new store ([meta_format](layout-osd.en.md#meta_format) 3).
-Allow OSD to start when some metadata entries or blocks are corrupted by
-skipping them. Should be only used as an emergency measure.
-
 ## throttle_small_writes
 
 - Type: boolean
@@ -754,9 +733,3 @@ This option sets the maximum number of object is a chunk. Moving 100k objects us
 - Default: 100
 
 This option sets the interval between handling two PG count change chunks.
-
-## gc_on_start
-
-- Type: boolean
-
-Forcibly clean all garbage entries in the new store on every OSD restart.

docs/config/osd.ru.md

@@ -39,7 +39,6 @@
 - [journal_io](#journal_io)
 - [journal_sector_buffer_count](#journal_sector_buffer_count)
 - [journal_no_same_sector_overwrites](#journal_no_same_sector_overwrites)
-- [skip_corrupted_meta_entries](#skip_corrupted_meta_entries)
 - [throttle_small_writes](#throttle_small_writes)
 - [throttle_target_iops](#throttle_target_iops)
 - [throttle_target_mbs](#throttle_target_mbs)
@@ -71,7 +70,6 @@
 - [use_atomic_flag](#use_atomic_flag)
 - [pg_reshard_chunk_size](#pg_reshard_chunk_size)
 - [pg_reshard_chunk_pause_ms](#pg_reshard_chunk_pause_ms)
-- [gc_on_start](#gc_on_start)
 
 ## bind_address
 
@@ -289,19 +287,13 @@ Flusher - это микро-поток (корутина), которая коп
 - Тип: булево (да/нет)
 - Значение по умолчанию: true
 
-Только для старого хранилища ([meta_format](layout-osd.en.md#meta_format) 2).
+Данный параметр заставляет Vitastor всегда держать область метаданных диска
-
+в памяти. Это нужно, чтобы избегать дополнительных операций чтения с диска
-Данный параметр заставляет Vitastor всегда держать копию области метаданных
+при записи. Размер области метаданных на данный момент составляет примерно
-в памяти в том же виде, как она лежит на диске, в дополнение к БД метаданных.
+224 МБ на 1 ТБ данных. При включении потребление памяти снизится примерно
-То есть, с включённой опцией каждая запись метаданных хранится в памяти дважды.
+на эту величину, но при этом также снизится и производительность. В будущем,
-Это нужно, чтобы избегать дополнительных операций чтения с диска при записи.
+после обновления схемы хранения метаданных, это ограничение, скорее всего,
-Размер области метаданных в старом хранилище составляет примерно 224 МБ на
+будет ликвидировано.
-1 ТБ данных. Вы можете отключить опцию, чтобы снизить потребление памяти
-примерно на эту величину, но при этом также снизится и производительность.
-
-Для нового хранилища ([meta_format](layout-osd.en.md#meta_format) 3) опция,
-возможно, будет переработана в будущем для поддержки работы без полной
-загрузки метаданных в памяти.
 
 ## inmemory_journal
 
@@ -384,8 +376,6 @@ fsync небезопасным даже с режимом "directsync".
 нужно менять - это если вы включаете journal_no_same_sector_overwrites. В
 этом случае установите данный параметр, например, в 1024.
 
-Неприменимо к новому хранилищу ([meta_format](layout-osd.en.md#meta_format) 3).
-
 ## journal_no_same_sector_overwrites
 
 - Тип: булево (да/нет)
@@ -401,18 +391,6 @@ fsync небезопасным даже с режимом "directsync".
 
 Почти все другие SSD (99% моделей) не требуют данной опции.
 
-Неприменимо к новому хранилищу ([meta_format](layout-osd.en.md#meta_format) 3).
-
-## skip_corrupted_meta_entries
-
-- Тип: булево (да/нет)
-- Значение по умолчанию: false
-
-Только для нового хранилища ([meta_format](layout-osd.en.md#meta_format) 3).
-Разрешить OSD запускаться, даже если часть блоков или записей метаданных
-повреждена, пропуская их. Опция предназначена для использования только в
-целях аварийного восстановления.
-
 ## throttle_small_writes
 
 - Тип: булево (да/нет)
@@ -794,9 +772,3 @@ pg_minsize OSD во время переключений, что может по
 - Значение по умолчанию: 100
 
 Данная опция задаёт интервал между обработкой двух порций изменения числа PG пулов.
-
-## gc_on_start
-
-- Тип: булево (да/нет)
-
-Принудительно очищать все мусорные записи в новом хранилище при каждом запуске OSD.

docs/config/src/network.yml

@@ -84,6 +84,11 @@
     unsupported with old libibverbs < v32, like in Debian 10 Buster or
     CentOS 7.
 
+    Vitastor supports all adapters, even ones without ODP support, like
+    Mellanox ConnectX-3 and non-Mellanox cards. Versions up to Vitastor
+    1.2.0 required ODP which is only present in Mellanox ConnectX >= 4.
+    See also [rdma_odp](#rdma_odp).
+
     Run `ibv_devinfo -v` as root to list available RDMA devices and their
     features.
 
@@ -92,23 +97,6 @@
     the manual of your network vendor for details about setting up the switch
     for RoCEv2 correctly. Usually it means setting up Lossless Ethernet with
     PFC (Priority Flow Control) and ECN (Explicit Congestion Notification).
-
-    Vitastor supports all adapters, even ones without ODP (On-Demand Paging)
-    support, like Mellanox ConnectX-3 and non-Mellanox cards. ODP is only present
-    in Mellanox ConnectX >= 4 adapters and allows to skip memory registration
-    for RDMA and thus, in theory, avoid memory copying.
-
-    Versions up to Vitastor 1.2.0 required ODP, then it was disabled by default,
-    but it was still supported up to 3.0.3. Now ODP support is removed because it
-    actually only hurts performance: an example 3-node cluster with 8 NVMe in each
-    node and 2*25 GBit/s ConnectX-6 RDMA network pushed 3950000 read iops without
-    ODP, but only 239000 iops with ODP.
-
-    This happens because Mellanox ODP implementation seems to be based on
-    message retransmissions when the adapter doesn't know about the buffer yet -
-    it likely uses standard "RNR retransmissions" (RNR = receiver not ready)
-    which is generally slow in RDMA/RoCE networks. Here's a presentation about
-    it from ISPASS-2021 conference: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
   info_ru: |
     Название RDMA-устройства для связи с Vitastor OSD (например, "rocep5s0f0").
     Если не указано, Vitastor попробует найти RoCE-устройство, соответствующее
@@ -117,6 +105,12 @@
     не задана. Также автовыбор не поддерживается со старыми версиями библиотеки
     libibverbs < v32, например в Debian 10 Buster или CentOS 7.
 
+    Vitastor поддерживает все модели адаптеров, включая те, у которых
+    нет поддержки ODP, то есть вы можете использовать RDMA с ConnectX-3 и
+    картами производства не Mellanox. Версии Vitastor до 1.2.0 включительно
+    требовали ODP, который есть только на Mellanox ConnectX 4 и более новых.
+    См. также [rdma_odp](#rdma_odp).
+
     Запустите `ibv_devinfo -v` от имени суперпользователя, чтобы посмотреть
     список доступных RDMA-устройств, их параметры и возможности.
 
@@ -126,24 +120,6 @@
     коммутатора для RoCEv2 ищите в документации производителя. Обычно это
     подразумевает настройку сети без потерь на основе PFC (Priority Flow
     Control) и ECN (Explicit Congestion Notification).
-
-    Vitastor поддерживает все модели адаптеров, включая те, у которых нет
-    поддержки ODP (On-Demand Paging), например, ConnectX-3 и карты производства
-    не Mellanox. Функция ODP доступна только на адаптерах Mellanox ConnectX-4 и
-    более новых и позволяет не регистрировать память для её использования RDMA-картой,
-    благодаря чему в теории можно избежать лишних копирований памяти.
-
-    Версии Vitastor до 1.2.0 включительно требовали ODP, потом функция был отключена
-    по умолчанию, но поддерживалась вплоть до версии 3.0.3. Сейчас поддержка ODP
-    полностью удалена, так как на самом деле она только портит производительность:
-    например, на 3-узловом кластере с 8 NVMe в каждом узле и сетью 2*25 Гбит/с на
-    чтение с RDMA без ODP удаётся снять 3950000 iops, а с ODP - всего 239000 iops.
-
-    Это происходит из-за того, что реализация ODP у Mellanox неоптимальная и
-    основана на повторной передаче сообщений, когда карте не известен буфер -
-    вероятно, на стандартных "RNR retransmission" (RNR = receiver not ready).
-    А данные повторные передачи в RDMA/RoCE - всегда очень медленная штука.
-    Презентация на эту тему с конференции ISPASS-2021: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
 - name: rdma_port_num
   type: int
   info: |
@@ -242,6 +218,45 @@
     у принимающей стороны в процессе работы не заканчивались буферы на приём.
     Не влияет на потребление памяти - дополнительная память на операции отправки
     не выделяется.
+- name: rdma_odp
+  type: bool
+  default: false
+  online: false
+  info: |
+    Use RDMA with On-Demand Paging. ODP is currently only available on Mellanox
+    ConnectX-4 and newer adapters. ODP allows to not register memory explicitly
+    for RDMA adapter to be able to use it. This, in turn, allows to skip memory
+    copying during sending. One would think this should improve performance, but
+    **in reality** RDMA performance with ODP is **drastically** worse. Example
+    3-node cluster with 8 NVMe in each node and 2*25 GBit/s ConnectX-6 RDMA network
+    without ODP pushes 3950000 read iops, but only 239000 iops with ODP...
+
+    This happens because Mellanox ODP implementation seems to be based on
+    message retransmissions when the adapter doesn't know about the buffer yet -
+    it likely uses standard "RNR retransmissions" (RNR = receiver not ready)
+    which is generally slow in RDMA/RoCE networks. Here's a presentation about
+    it from ISPASS-2021 conference: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
+
+    ODP support is retained in the code just in case a good ODP implementation
+    appears one day.
+  info_ru: |
+    Использовать RDMA с On-Demand Paging. ODP - функция, доступная пока что
+    исключительно на адаптерах Mellanox ConnectX-4 и более новых. ODP позволяет
+    не регистрировать память для её использования RDMA-картой. Благодаря этому
+    можно не копировать данные при отправке их в сеть и, казалось бы, это должно
+    улучшать производительность - но **по факту** получается так, что
+    производительность только ухудшается, причём сильно. Пример - на 3-узловом
+    кластере с 8 NVMe в каждом узле и сетью 2*25 Гбит/с на чтение с RDMA без ODP
+    удаётся снять 3950000 iops, а с ODP - всего 239000 iops...
+
+    Это происходит из-за того, что реализация ODP у Mellanox неоптимальная и
+    основана на повторной передаче сообщений, когда карте не известен буфер -
+    вероятно, на стандартных "RNR retransmission" (RNR = receiver not ready).
+    А данные повторные передачи в RDMA/RoCE - всегда очень медленная штука.
+    Презентация на эту тему с конференции ISPASS-2021: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
+
+    Возможность использования ODP сохранена в коде на случай, если вдруг в один
+    прекрасный день появится хорошая реализация ODP.
 - name: peer_connect_interval
   type: sec
   min: 1

docs/config/src/osd.yml

@@ -253,33 +253,21 @@
   type: bool
   default: true
   info: |
-    Only for the old store ([meta_format](layout-osd.en.md#meta_format) 2).
+    This parameter makes Vitastor always keep metadata area of the block device
-
+    in memory. It's required for good performance because it allows to avoid
-    This parameter makes Vitastor keep a copy of metadata area in memory as it is
+    additional read-modify-write cycles during metadata modifications. Metadata
-    on disk, in addition to the metadata database. When the option is enabled, every
+    area size is currently roughly 224 MB per 1 TB of data. You can turn it off
-    metadata entry is effectively stored in RAM twice. It's required for good performance
+    to reduce memory usage by this value, but it will hurt performance. This
-    because it allows to avoid additional read-modify-write cycles during metadata
+    restriction is likely to be removed in the future along with the upgrade
-    modifications. Metadata area size with the old store is roughly 224 MB per 1 TB
+    of the metadata storage scheme.
-    of data. You can turn the option off to reduce memory usage by this value, but
-    it will reduce performance.
-
-    For the new store ([meta_format](layout-osd.en.md#meta_format) 3), the option
-    may be changed in the future to support operation without loading full metadata
-    database in memory.
   info_ru: |
-    Только для старого хранилища ([meta_format](layout-osd.en.md#meta_format) 2).
+    Данный параметр заставляет Vitastor всегда держать область метаданных диска
-
+    в памяти. Это нужно, чтобы избегать дополнительных операций чтения с диска
-    Данный параметр заставляет Vitastor всегда держать копию области метаданных
+    при записи. Размер области метаданных на данный момент составляет примерно
-    в памяти в том же виде, как она лежит на диске, в дополнение к БД метаданных.
+    224 МБ на 1 ТБ данных. При включении потребление памяти снизится примерно
-    То есть, с включённой опцией каждая запись метаданных хранится в памяти дважды.
+    на эту величину, но при этом также снизится и производительность. В будущем,
-    Это нужно, чтобы избегать дополнительных операций чтения с диска при записи.
+    после обновления схемы хранения метаданных, это ограничение, скорее всего,
-    Размер области метаданных в старом хранилище составляет примерно 224 МБ на
+    будет ликвидировано.
-    1 ТБ данных. Вы можете отключить опцию, чтобы снизить потребление памяти
-    примерно на эту величину, но при этом также снизится и производительность.
-
-    Для нового хранилища ([meta_format](layout-osd.en.md#meta_format) 3) опция,
-    возможно, будет переработана в будущем для поддержки работы без полной
-    загрузки метаданных в памяти.
 - name: inmemory_journal
   type: bool
   default: true
@@ -398,15 +386,11 @@
     blocks. The only situation when you should increase it to a larger value
     is when you enable journal_no_same_sector_overwrites. In this case set
     it to, for example, 1024.
-
-    Not applicable to the new store ([meta_format](layout-osd.en.md#meta_format) 3).
   info_ru: |
     Максимальное число буферов, разрешённых для использования под записываемые
     в журнал блоки метаданных. Единственная ситуация, в которой этот параметр
     нужно менять - это если вы включаете journal_no_same_sector_overwrites. В
     этом случае установите данный параметр, например, в 1024.
-
-    Неприменимо к новому хранилищу ([meta_format](layout-osd.en.md#meta_format) 3).
 - name: journal_no_same_sector_overwrites
   type: bool
   default: false
@@ -418,8 +402,6 @@
     journal after writing it instead of possibly overwriting it the second time.
 
     Most (99%) other SSDs don't need this option.
-
-    Not applicable to the new store ([meta_format](layout-osd.en.md#meta_format) 3).
   info_ru: |
     Включайте данную опцию для SSD вроде Intel D3-S4510 и D3-S4610, которые
     ОЧЕНЬ не любят, когда ПО перезаписывает один и тот же сектор несколько раз
@@ -430,20 +412,6 @@
     самого сектора.
 
     Почти все другие SSD (99% моделей) не требуют данной опции.
-
-    Неприменимо к новому хранилищу ([meta_format](layout-osd.en.md#meta_format) 3).
-- name: skip_corrupted_meta_entries
-  type: bool
-  default: false
-  info: |
-    Only for the new store ([meta_format](layout-osd.en.md#meta_format) 3).
-    Allow OSD to start when some metadata entries or blocks are corrupted by
-    skipping them. Should be only used as an emergency measure.
-  info_ru: |
-    Только для нового хранилища ([meta_format](layout-osd.en.md#meta_format) 3).
-    Разрешить OSD запускаться, даже если часть блоков или записей метаданных
-    повреждена, пропуская их. Опция предназначена для использования только в
-    целях аварийного восстановления.
 - name: throttle_small_writes
   type: bool
   default: false
@@ -938,7 +906,3 @@
     This option sets the interval between handling two PG count change chunks.
   info_ru: |
     Данная опция задаёт интервал между обработкой двух порций изменения числа PG пулов.
-- name: gc_on_start
-  type: bool
-  info: Forcibly clean all garbage entries in the new store on every OSD restart.
-  info_ru: Принудительно очищать все мусорные записи в новом хранилище при каждом запуске OSD.

docs/installation/docker.en.md

@@ -26,37 +26,13 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.
 
 1. Download a Docker image of the desired version: \
-   `docker pull vitalif/vitastor:v3.0.14`
+   `docker pull vitalif/vitastor:v3.0.2`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.14 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.2 install.sh`
 3. Reload udev rules: \
    `udevadm control --reload-rules`
-4. Enable the vitastor-host service: \
-   `systemctl enable --now vitastor-host`
 
-After these steps, you can return to [Quick Start](../intro/quickstart.en.md).
+And you can return to [Quick Start](../intro/quickstart.en.md).
-
-## Podman
-
-If you use Podman, run the following commands as root before installing Vitastor containers:
-
-```
-ln -s podman /usr/bin/docker
-
-mkdir -p /etc/systemd/system/systemd-udevd.service.d
-
-cat >/etc/systemd/system/systemd-udevd.service.d/override.conf <<EOF
-[Service]
-CapabilityBoundingSet=~
-SystemCallFilter=@mount capset
-EOF
-
-systemctl daemon-reload
-
-systemctl restart systemd-udevd
-```
-
-Without it, udev fails to do calls into a Podman container and Vitastor disk detection doesn't work.
 
 ## Upgrading Containers
 

docs/installation/docker.ru.md

@@ -25,39 +25,14 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.
 
 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitalif/vitastor:v3.0.14`
+   `docker pull vitalif/vitastor:v3.0.2`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.14 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.2 install.sh`
 3. Перезагрузите правила udev: \
    `udevadm control --reload-rules`
-4. Включите сервис vitastor-host: \
-   `systemctl enable --now vitastor-host`
 
 После этого вы можете возвращаться к разделу [Быстрый старт](../intro/quickstart.ru.md).
 
-## Podman
-
-Если вы используете Podman, перед установкой контейнеров Vitastor выполните следующие
-команды от имени суперпользователя:
-
-```
-ln -s podman /usr/bin/docker
-
-mkdir -p /etc/systemd/system/systemd-udevd.service.d
-
-cat >/etc/systemd/system/systemd-udevd.service.d/override.conf <<EOF
-[Service]
-CapabilityBoundingSet=~
-SystemCallFilter=@mount capset
-EOF
-
-systemctl daemon-reload
-
-systemctl restart systemd-udevd
-```
-
-Без этих настроек udev не может делать вызовы внутрь Podman-контейнеров и определение дисков Vitastor не работает.
-
 ## Обновление контейнеров
 
 Сначала обязательно проверьте раздел [Обновление Vitastor](../usage/admin.ru.md#обновление-vitastor),

docs/installation/packages.en.md

@@ -17,7 +17,6 @@
   - Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
   - Ubuntu 22.04 (Jammy): `deb https://vitastor.io/debian jammy main`
   - Ubuntu 24.04 (Noble): `deb https://vitastor.io/debian noble main`
-  - Ubuntu 26.04 (Resolute): `deb https://vitastor.io/debian resolute main`
   - Add `-oldstable` to bookworm/bullseye/buster in this line to install the last
     stable version from 0.9.x branch instead of 1.x
 - To always prefer vitastor-patched QEMU and Libvirt versions, add the following to `/etc/apt/preferences`:
@@ -34,17 +33,15 @@
   - CentOS 7: `yum install https://vitastor.io/rpms/centos/7/vitastor-release.rpm`
   - CentOS 8: `dnf install https://vitastor.io/rpms/centos/8/vitastor-release.rpm`
   - AlmaLinux 9 and other RHEL 9 clones (Rocky, Oracle...): `dnf install https://vitastor.io/rpms/centos/9/vitastor-release.rpm`
-  - AlmaLinux 10 and other RHEL 10 clones: `dnf install https://vitastor.io/rpms/centos/10/vitastor-release.rpm`
 - Enable EPEL: `yum/dnf install epel-release`
 - Enable additional CentOS repositories:
   - CentOS 7: `yum install centos-release-scl`
   - CentOS 8: `dnf install centos-release-advanced-virtualization`
-  - RHEL 9/10 clones: not required
+  - RHEL 9 clones: not required
 - Enable elrepo-kernel:
   - CentOS 7: `yum install https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm`
   - CentOS 8: `dnf install https://www.elrepo.org/elrepo-release-8.el8.elrepo.noarch.rpm`
   - RHEL 9 clones: `dnf install https://www.elrepo.org/elrepo-release-9.el9.elrepo.noarch.rpm`
-  - RHEL 10 clones: not required
 - Install packages: `yum/dnf install vitastor lpsolve etcd kernel-ml qemu-kvm`
 
 ## Installation requirements

docs/installation/packages.ru.md

@@ -17,7 +17,6 @@
   - Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
   - Ubuntu 22.04 (Jammy): `deb https://vitastor.io/debian jammy main`
   - Ubuntu 24.04 (Noble): `deb https://vitastor.io/debian noble main`
-  - Ubuntu 26.04 (Resolute): `deb https://vitastor.io/debian resolute main`
   - Добавьте `-oldstable` к слову bookworm/bullseye/buster в этой строке, чтобы
     установить последнюю стабильную версию из ветки 0.9.x вместо 1.x
 - Чтобы всегда предпочитались версии пакетов QEMU и Libvirt с патчами Vitastor, добавьте в `/etc/apt/preferences`:
@@ -34,17 +33,15 @@
   - CentOS 7: `yum install https://vitastor.io/rpms/centos/7/vitastor-release.rpm`
   - CentOS 8: `dnf install https://vitastor.io/rpms/centos/8/vitastor-release.rpm`
   - AlmaLinux 9 и другие клоны RHEL 9 (Rocky, Oracle...): `dnf install https://vitastor.io/rpms/centos/9/vitastor-release.rpm`
-  - AlmaLinux 10 и другие клоны RHEL 10: `dnf install https://vitastor.io/rpms/centos/10/vitastor-release.rpm`
 - Включите EPEL: `yum/dnf install epel-release`
 - Включите дополнительные репозитории CentOS:
   - CentOS 7: `yum install centos-release-scl`
   - CentOS 8: `dnf install centos-release-advanced-virtualization`
-  - Клоны RHEL 9/10: не нужно
+  - Клоны RHEL 9: не нужно
 - Включите elrepo-kernel:
   - CentOS 7: `yum install https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm`
   - CentOS 8: `dnf install https://www.elrepo.org/elrepo-release-8.el8.elrepo.noarch.rpm`
   - Клоны RHEL 9: `dnf install https://www.elrepo.org/elrepo-release-9.el9.elrepo.noarch.rpm`
-  - Клоны RHEL 10: не нужно
 - Установите пакеты: `yum/dnf install vitastor lpsolve etcd kernel-ml qemu-kvm`
 
 ## Установочные требования

docs/installation/source.en.md

@@ -16,7 +16,8 @@
   designated initializers support from C++20
 - CMake
 - jerasure headers and libraries
-- ISA-L, libibverbs, librdmacm, libnl3 headers and libraries (optional)
+- ISA-L, libibverbs and librdmacm headers and libraries (optional)
+- tcmalloc (google-perftools-dev)
 
 ## Basic instructions
 

docs/installation/source.ru.md

@@ -16,7 +16,8 @@
   назначенных инициализаторов (designated initializers) из C++20
 - CMake
 - Заголовки и библиотеки jerasure
-- Опционально - заголовки и библиотеки ISA-L, libibverbs, librdmacm, libnl3
+- Опционально - заголовки и библиотеки ISA-L, libibverbs, librdmacm
+- tcmalloc (google-perftools-dev)
 
 ## Базовая инструкция
 

docs/usage/nfs.en.md

@@ -262,4 +262,3 @@ Options:
 | `--logfile <FILE>` | log to the specified file                                |
 | `--enforce 1`      | enforce permissions at the server side (no by default)   |
 | `--foreground 1`   | stay in foreground, do not daemonize                     |
-| `--trace`          | trace all NFS requests                                   |

docs/usage/nfs.ru.md

@@ -274,4 +274,3 @@ VitastorFS из GPUDirect.
 | `--logfile <FILE>` | записывать логи в заданный файл                         |
 | `--enforce 1`      | проверять права доступа на стороне сервера (по умолчанию нет) |
 | `--foreground 1`   | не уходить в фон после запуска                          |
-| `--trace`          | логгировать все запросы NFS                             |

mon/http_server.js

@@ -16,7 +16,7 @@ async function create_http_server(cfg, handler)
         };
         if (cfg.mon_https_ca)
         {
-            tls.ca = await fsp.readFile(cfg.mon_https_ca);
+            tls.mon_https_ca = await fsp.readFile(cfg.mon_https_ca);
         }
         if (cfg.mon_https_client_auth)
         {

mon/lp_optimizer/lp_optimizer.js

@@ -10,19 +10,16 @@ const NO_OSD = 'Z';
 async function lp_solve(text)
 {
     const cp = child_process.spawn('lp_solve');
-    let stdout = '', stderr = '', finish_cb, finished = 0;
+    let stdout = '', stderr = '', finish_cb;
     cp.stdout.on('data', buf => stdout += buf.toString());
     cp.stderr.on('data', buf => stderr += buf.toString());
-    cp.stdout.on('end', () => finish_cb());
+    cp.on('exit', () => finish_cb && finish_cb());
-    cp.stderr.on('end', () => finish_cb());
     cp.stdin.write(text);
     cp.stdin.end();
-    await new Promise(ok => (finish_cb = () =>
+    if (cp.exitCode == null)
     {
-        finished++;
+        await new Promise(ok => finish_cb = ok);
-        if (finished == 2)
+    }
-            ok();
-    }));
     if (!stdout.trim())
     {
         return null;
@@ -112,10 +109,9 @@ function make_cyclic(pgs, parity_space)
 {
     if (parity_space > 1)
     {
-        for (const id in pgs)
+        for (const pg in pgs)
         {
-            const pg = pgs[id];
+            for (let i = 1; i < pg.size; i++)
-            for (let i = 1; i < pg.length; i++)
             {
                 const cyclic = [ ...pg.slice(i), ...pg.slice(0, i) ];
                 pgs['pg_'+cyclic.join('_')] = cyclic;

mon/mon.js

@@ -627,7 +627,7 @@ class Mon
                 if (this.state.pg.history[pool_id] &&
                     this.state.pg.history[pool_id][pg])
                 {
-                    pg_history[pg-1] = JSON.parse(JSON.stringify(this.state.pg.history[pool_id][pg]));
+                    pg_history[pg-1] = this.state.pg.history[pool_id][pg];
                 }
             }
             const real_prev_pgs = [];
@@ -719,7 +719,7 @@ class Mon
                 this.next_recheck_timer = null;
                 this.next_recheck_at = 0;
                 this.schedule_recheck();
-            }, (this.next_recheck_at-now)*1000);
+            }, now-this.next_recheck_at);
         }
     }
 

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "3.0.14",
+  "version": "3.0.2",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {
@@ -9,7 +9,7 @@
   "author": "Vitaliy Filippov",
   "license": "UNLICENSED",
   "dependencies": {
-    "antietcd": "^1.2.4",
+    "antietcd": "^1.2.2",
     "sprintf-js": "^1.1.2",
     "ws": "^7.2.5"
   },

mon/pg_gen.js

@@ -195,7 +195,7 @@ async function generate_pool_pgs(state, global_config, pool_id, osd_tree, levels
     const folded_tree = make_hier_tree(global_config, folded.nodes.reduce((a, c) => { a[c.id] = c; return a; }, {}));
     const old_pg_count = prev_pgs.length;
     const optimize_cfg = {
-        osd_weights: folded.nodes.reduce((a, c) => { if (/^\d+$/.exec(c.id) && c.size != null) { a[c.id] = c.size||0; } return a; }, {}),
+        osd_weights: folded.nodes.reduce((a, c) => { if (Number(c.id)) { a[c.id] = c.size; } return a; }, {}),
         combinator: use_rules
             // new algorithm:
             ? new RuleCombinator(folded_tree, rules, pool_cfg.max_osd_combinations)

mon/pg_utils.js

@@ -84,7 +84,7 @@ function scale_pg_history(prev_pg_history, prev_pgs, new_pgs)
         finish_pg_history(merged_history[1]);
         for (let i = 0; i < new_pg_count; i++)
         {
-            new_pg_history[i] = JSON.parse(JSON.stringify(merged_history[1]));
+            new_pg_history[i] = { ...merged_history[1] };
         }
     }
     // Mark history keys for removed PGs as removed
@@ -102,7 +102,7 @@ function scale_pg_count(prev_pgs, new_pg_count)
     {
         for (let i = prev_pgs.length; i < new_pg_count; i++)
         {
-            prev_pgs[i] = [ ...prev_pgs[i % prev_pgs.length] ];
+            prev_pgs[i] = prev_pgs[i % prev_pgs.length];
         }
     }
     else if (prev_pgs.length > new_pg_count)

mon/stats.js

@@ -37,7 +37,6 @@ function derive_osd_stats(st, prev, prev_diff)
         const n = c.count - BigInt(pr && pr.count||0);
         diff.recovery_stats[op] = { ...c, bps: n > 0 ? b*1000n/timediff : 0n, iops: n > 0 ? n*1000n/timediff : 0n };
     }
-    diff.inode_stats = {};
     for (const pool_id in st.inode_stats||{})
     {
         diff.inode_stats[pool_id] = {};

node-binding/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor",
-  "version": "3.0.14",
+  "version": "3.0.2",
   "description": "Low-level native bindings to Vitastor client library",
   "main": "index.js",
   "keywords": [

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '3.0.14'
+VITASTOR_VERSION = '3.0.2'
 
 LOG = logging.getLogger(__name__)
 
@@ -275,7 +275,7 @@ class VitastorDriver(driver.CloneableImageVD,
             LOG.exception('error getting vitastor pool stats: '+str(e))
 
         self._stats = stats
-
+        
     def get_volume_stats(self, refresh=False):
         """Get volume stats.
         If 'refresh' is True, run update the stats first.
@@ -291,14 +291,6 @@ class VitastorDriver(driver.CloneableImageVD,
         else:
             return (1 + resp['kvs'][0]['value'], resp['kvs'][0]['mod_revision'])
 
-    def _cli(self, descr, *args):
-        args = [ 'vitastor-cli', *args, *(self._vitastor_args()) ]
-        try:
-            self._execute(*args)
-        except processutils.ProcessExecutionError as exc:
-            LOG.error("Failed to "+descr+": "+exc)
-            raise exception.VolumeBackendAPIException(data = exc.stderr)
-
     def create_volume(self, volume):
         """Creates a logical volume."""
 
@@ -310,7 +302,7 @@ class VitastorDriver(driver.CloneableImageVD,
 
         LOG.debug("creating volume '%s'", vol_name)
 
-        self._cli('create volume', 'create', vol_name, '--size', size)
+        self._create_image(vol_name, { 'size': size })
 
         if volume.encryption_key_id:
             self._create_encrypted_volume(volume, volume.obj_context)
@@ -354,7 +346,7 @@ class VitastorDriver(driver.CloneableImageVD,
         snap_name = utils.convert_str(snapshot.name)
         if snap_name.find('@') >= 0 or snap_name.find('/') >= 0:
             raise exception.VolumeBackendAPIException(data = '@ and / are forbidden in volume and snapshot names')
-        self._cli('create snapshot', 'snap-create', vol_name+'@'+snap_name)
+        self._create_snapshot(vol_name, vol_name+'@'+snap_name)
 
     def snapshot_revert_use_temp_snapshot(self):
         """Disable the use of a temporary snapshot on revert."""
@@ -367,8 +359,21 @@ class VitastorDriver(driver.CloneableImageVD,
         snap_name = utils.convert_str(snapshot.name)
 
         # Delete the image and recreate it from the snapshot
-        self._cli('delete image', 'rm', vol_name)
+        args = [ 'vitastor-cli', 'rm', vol_name, *(self._vitastor_args()) ]
-        self._cli('recreate image', 'create', '--parent', vol_name+'@'+snap_name, vol_name)
+        try:
+            self._execute(*args)
+        except processutils.ProcessExecutionError as exc:
+            LOG.error("Failed to delete image "+vol_name+": "+exc)
+            raise exception.VolumeBackendAPIException(data = exc.stderr)
+        args = [
+            'vitastor-cli', 'create', '--parent', vol_name+'@'+snap_name,
+            vol_name, *(self._vitastor_args())
+        ]
+        try:
+            self._execute(*args)
+        except processutils.ProcessExecutionError as exc:
+            LOG.error("Failed to recreate image "+vol_name+" from "+vol_name+"@"+snap_name+": "+exc)
+            raise exception.VolumeBackendAPIException(data = exc.stderr)
 
     def delete_snapshot(self, snapshot):
         """Deletes a snapshot."""
@@ -376,7 +381,15 @@ class VitastorDriver(driver.CloneableImageVD,
         vol_name = utils.convert_str(snapshot.volume_name)
         snap_name = utils.convert_str(snapshot.name)
 
-        self._cli('remove snapshot', 'rm', vol_name+'@'+snap_name)
+        args = [
+            'vitastor-cli', 'rm', vol_name+'@'+snap_name,
+            *(self._vitastor_args())
+        ]
+        try:
+            self._execute(*args)
+        except processutils.ProcessExecutionError as exc:
+            LOG.error("Failed to remove snapshot "+vol_name+'@'+snap_name+": "+exc)
+            raise exception.VolumeBackendAPIException(data = exc.stderr)
 
     def _child_count(self, parents):
         children = 0
@@ -414,7 +427,13 @@ class VitastorDriver(driver.CloneableImageVD,
         if src_vref.admin_metadata.get('readonly') == 'True':
             # source volume is a volume-image cache entry or other readonly volume
             # clone without intermediate snapshot
-            self._cli('create clone', 'create', '--parent', src_name, '--size', size, dest_name)
+            src = self._get_image(src_name)
+            LOG.debug("creating image '%s' from '%s'", dest_name, src_name)
+            new_cfg = self._create_image(dest_name, {
+                'size': size,
+                'parent_id': src['idx']['id'],
+                'parent_pool_id': src['idx']['pool_id'],
+            })
             return {}
 
         clone_snap = "%s@%s.clone_snap" % (src_name, dest_name)
@@ -427,12 +446,15 @@ class VitastorDriver(driver.CloneableImageVD,
             clone_snap = dest_name
             make_img = False
 
-        LOG.debug("creating snapshot '%s'", clone_snap)
+        LOG.debug("creating layer '%s' under '%s'", clone_snap, src_name)
-        self._cli('create base snapshot', 'snap-create', '--allow-existing', '1', clone_snap)
+        new_cfg = self._create_snapshot(src_name, clone_snap, True)
-
         if make_img:
             # Then create a clone from it
-            self._cli('create clone', 'create', '--parent', clone_snap, '--size', size, dest_name)
+            new_cfg = self._create_image(dest_name, {
+                'size': size,
+                'parent_id': new_cfg['parent_id'],
+                'parent_pool_id': new_cfg['parent_pool_id'],
+            })
 
         return {}
 
@@ -442,8 +464,7 @@ class VitastorDriver(driver.CloneableImageVD,
         vol_name = utils.convert_str(volume.name)
         snap_name = utils.convert_str(snapshot.name)
 
-        src_snap = 'volume-'+snapshot.volume_id+'@'+snap_name
+        snap = self._get_image('volume-'+snapshot.volume_id+'@'+snap_name)
-        snap = self._get_image(src_snap)
         if not snap:
             raise exception.SnapshotNotFound(snapshot_id = snap_name)
         snap_inode_id = int(resp['responses'][0]['kvs'][0]['value']['id'])
@@ -452,8 +473,12 @@ class VitastorDriver(driver.CloneableImageVD,
         size = snap['cfg']['size']
         if int(volume.size):
             size = int(volume.size) * units.Gi
+        new_cfg = self._create_image(vol_name, {
+            'size': size,
+            'parent_id': snap['idx']['id'],
+            'parent_pool_id': snap['idx']['pool_id'],
+        })
 
-        self._cli('create clone', 'create', vol_name, '--size', size, '--parent', src_snap)
         return {}
 
     def _vitastor_args(self):
@@ -480,7 +505,49 @@ class VitastorDriver(driver.CloneableImageVD,
         """Deletes a logical volume."""
 
         vol_name = utils.convert_str(volume.name)
-        self._cli('delete volume', 'rm', '--matching', vol_name, vol_name+'@*', '--progress', '0')
+
+        # Find the volume and all its snapshots
+        range_end = b'index/image/' + vol_name.encode('utf-8')
+        range_end = range_end[0 : len(range_end)-1] + six.int2byte(range_end[len(range_end)-1] + 1)
+        resp = self._etcd_txn({ 'success': [
+            { 'request_range': { 'key': 'index/image/'+vol_name, 'range_end': range_end } },
+        ] })
+        if len(resp['responses'][0]['kvs']) == 0:
+            # already deleted
+            LOG.info("volume %s no longer exists in backend", vol_name)
+            return
+        layers = resp['responses'][0]['kvs']
+        layer_ids = {}
+        for kv in layers:
+            inode_id = int(kv['value']['id'])
+            pool_id = int(kv['value']['pool_id'])
+            inode_pool_id = (pool_id << 48) | (inode_id & 0xffffffffffff)
+            layer_ids[inode_pool_id] = True
+
+        # Check if the volume has clones and raise 'busy' if so
+        children = self._child_count(layer_ids)
+        if children > 0:
+            raise exception.VolumeIsBusy(volume_name = vol_name)
+
+        # Clear data
+        for kv in layers:
+            args = [
+                'vitastor-cli', 'rm-data', '--pool', str(kv['value']['pool_id']),
+                '--inode', str(kv['value']['id']), '--progress', '0',
+                *(self._vitastor_args())
+            ]
+            try:
+                self._execute(*args)
+            except processutils.ProcessExecutionError as exc:
+                LOG.error("Failed to remove layer "+kv['key']+": "+exc)
+                raise exception.VolumeBackendAPIException(data = exc.stderr)
+
+        # Delete all layers from etcd
+        requests = []
+        for kv in layers:
+            requests.append({ 'request_delete_range': { 'key': kv['key'] } })
+            requests.append({ 'request_delete_range': { 'key': 'config/inode/'+str(kv['value']['pool_id'])+'/'+str(kv['value']['id']) } })
+        self._etcd_txn({ 'success': requests })
 
     def retype(self, context, volume, new_type, diff, host):
         """Change extra type specifications for a volume."""
@@ -500,6 +567,98 @@ class VitastorDriver(driver.CloneableImageVD,
         """Removes an export for a logical volume."""
         pass
 
+    def _create_image(self, vol_name, cfg):
+        pool_s = str(self.cfg['pool_id'])
+        image_id = 0
+        while image_id == 0:
+            # check if the image already exists and find a free ID
+            resp = self._etcd_txn({ 'success': [
+                { 'request_range': { 'key': 'index/image/'+vol_name } },
+                { 'request_range': { 'key': 'index/maxid/'+pool_s } },
+            ] })
+            if len(resp['responses'][0]['kvs']) > 0:
+                # already exists
+                raise exception.VolumeBackendAPIException(data = 'Volume '+vol_name+' already exists')
+            image_id, id_mod = self._next_id(resp['responses'][1])
+            # try to create the image
+            resp = self._etcd_txn({ 'compare': [
+                { 'target': 'MOD', 'mod_revision': id_mod, 'key': 'index/maxid/'+pool_s },
+                { 'target': 'VERSION', 'version': 0, 'key': 'index/image/'+vol_name },
+                { 'target': 'VERSION', 'version': 0, 'key': 'config/inode/'+pool_s+'/'+str(image_id) },
+            ], 'success': [
+                { 'request_put': { 'key': 'index/maxid/'+pool_s, 'value': image_id } },
+                { 'request_put': { 'key': 'index/image/'+vol_name, 'value': json.dumps({
+                    'id': image_id, 'pool_id': self.cfg['pool_id']
+                }) } },
+                { 'request_put': { 'key': 'config/inode/'+pool_s+'/'+str(image_id), 'value': json.dumps({
+                    **cfg, 'name': vol_name,
+                }) } },
+            ] })
+            if not resp.get('succeeded'):
+                # repeat
+                image_id = 0
+
+    def _create_snapshot(self, vol_name, snap_vol_name, allow_existing = False):
+        while True:
+            # check if the image already exists and snapshot doesn't
+            resp = self._etcd_txn({ 'success': [
+                { 'request_range': { 'key': 'index/image/'+vol_name } },
+                { 'request_range': { 'key': 'index/image/'+snap_vol_name } },
+            ] })
+            if len(resp['responses'][0]['kvs']) == 0:
+                raise exception.VolumeBackendAPIException(data = 'Volume '+vol_name+' does not exist')
+            if len(resp['responses'][1]['kvs']) > 0:
+                if allow_existing:
+                    snap_idx = resp['responses'][1]['kvs'][0]['value']
+                    resp = self._etcd_txn({ 'success': [
+                        { 'request_range': { 'key': 'config/inode/'+str(snap_idx['pool_id'])+'/'+str(snap_idx['id']) } },
+                    ] })
+                    if len(resp['responses'][0]['kvs']) == 0:
+                        raise exception.VolumeBackendAPIException(data =
+                            'Volume '+snap_vol_name+' is already indexed, but does not exist'
+                        )
+                    return resp['responses'][0]['kvs'][0]['value']
+                raise exception.VolumeBackendAPIException(
+                    data = 'Volume '+snap_vol_name+' already exists'
+                )
+            vol_idx = resp['responses'][0]['kvs'][0]['value']
+            vol_idx_mod = resp['responses'][0]['kvs'][0]['mod_revision']
+            # get image inode config and find a new ID
+            resp = self._etcd_txn({ 'success': [
+                { 'request_range': { 'key': 'config/inode/'+str(vol_idx['pool_id'])+'/'+str(vol_idx['id']) } },
+                { 'request_range': { 'key': 'index/maxid/'+str(self.cfg['pool_id']) } },
+            ] })
+            if len(resp['responses'][0]['kvs']) == 0:
+                raise exception.VolumeBackendAPIException(data = 'Volume '+vol_name+' does not exist')
+            vol_cfg = resp['responses'][0]['kvs'][0]['value']
+            vol_mod = resp['responses'][0]['kvs'][0]['mod_revision']
+            new_id, id_mod = self._next_id(resp['responses'][1])
+            # try to redirect image to the new inode
+            new_cfg = {
+                **vol_cfg, 'name': vol_name, 'parent_id': vol_idx['id'], 'parent_pool_id': vol_idx['pool_id']
+            }
+            resp = self._etcd_txn({ 'compare': [
+                { 'target': 'MOD', 'mod_revision': vol_idx_mod, 'key': 'index/image/'+vol_name },
+                { 'target': 'MOD', 'mod_revision': vol_mod, 'key': 'config/inode/'+str(vol_idx['pool_id'])+'/'+str(vol_idx['id']) },
+                { 'target': 'MOD', 'mod_revision': id_mod, 'key': 'index/maxid/'+str(self.cfg['pool_id']) },
+                { 'target': 'VERSION', 'version': 0, 'key': 'index/image/'+snap_vol_name },
+                { 'target': 'VERSION', 'version': 0, 'key': 'config/inode/'+str(self.cfg['pool_id'])+'/'+str(new_id) },
+            ], 'success': [
+                { 'request_put': { 'key': 'index/maxid/'+str(self.cfg['pool_id']), 'value': new_id } },
+                { 'request_put': { 'key': 'index/image/'+vol_name, 'value': json.dumps({
+                    'id': new_id, 'pool_id': self.cfg['pool_id']
+                }) } },
+                { 'request_put': { 'key': 'config/inode/'+str(self.cfg['pool_id'])+'/'+str(new_id), 'value': json.dumps(new_cfg) } },
+                { 'request_put': { 'key': 'index/image/'+snap_vol_name, 'value': json.dumps({
+                    'id': vol_idx['id'], 'pool_id': vol_idx['pool_id']
+                }) } },
+                { 'request_put': { 'key': 'config/inode/'+str(vol_idx['pool_id'])+'/'+str(vol_idx['id']), 'value': json.dumps({
+                    **vol_cfg, 'name': snap_vol_name, 'readonly': True
+                }) } }
+            ] })
+            if resp.get('succeeded'):
+                return new_cfg
+
     def initialize_connection(self, volume, connector):
         data = {
             'driver_volume_type': 'vitastor',
@@ -538,9 +697,13 @@ class VitastorDriver(driver.CloneableImageVD,
                     size = int(volume.size) * units.Gi
                     dest_name = utils.convert_str(volume.name)
                     # Find or create the base snapshot
-                    self._cli('create base snapshot', 'create', '--allow-existing', '1', base_vol.name+'@.clone_snap')
+                    snap_cfg = self._create_snapshot(base_vol.name, base_vol.name+'@.clone_snap', True)
                     # Then create a clone from it
-                    self._cli('create clone', 'create', dest_name, '--size', size, '--parent', base_vol.name+'@.clone_snap')
+                    new_cfg = self._create_image(dest_name, {
+                        'size': size,
+                        'parent_id': snap_cfg['parent_id'],
+                        'parent_pool_id': snap_cfg['parent_pool_id'],
+                    })
                     return ({}, True)
         return ({}, False)
 
@@ -607,8 +770,26 @@ class VitastorDriver(driver.CloneableImageVD,
     def extend_volume(self, volume, new_size):
         """Extend an existing volume."""
         vol_name = utils.convert_str(volume.name)
-        size = int(new_size) * units.Gi
+        while True:
-        self._cli('extend volume', 'modify', vol_name, '--resize', new_size)
+            vol = self._get_image(vol_name)
+            if not vol:
+                raise exception.VolumeBackendAPIException(data = 'Volume '+vol_name+' does not exist')
+            # change size
+            size = int(new_size) * units.Gi
+            if size == vol['cfg']['size']:
+                break
+            resp = self._etcd_txn({ 'compare': [ {
+                'target': 'MOD',
+                'mod_revision': vol['cfg_mod'],
+                'key': 'config/inode/'+str(vol['idx']['pool_id'])+'/'+str(vol['idx']['id']),
+            } ], 'success': [
+                { 'request_put': {
+                    'key': 'config/inode/'+str(vol['idx']['pool_id'])+'/'+str(vol['idx']['id']),
+                    'value': json.dumps({ **vol['cfg'], 'size': size }),
+                } },
+            ] })
+            if resp.get('succeeded'):
+                break
         LOG.debug(
             "Extend volume from %(old_size)s GB to %(new_size)s GB.",
             {'old_size': volume.size, 'new_size': new_size}
@@ -681,7 +862,28 @@ class VitastorDriver(driver.CloneableImageVD,
         """
         from_name = self._get_existing_name(existing_ref)
         to_name = utils.convert_str(volume.name)
-        self._cli('rename', 'modify', from_name, '--rename', to_name)
+        self._rename(from_name, to_name)
+
+    def _rename(self, from_name, to_name):
+        while True:
+            vol = self._get_image(from_name)
+            if not vol:
+                raise exception.VolumeBackendAPIException(data = 'Volume '+from_name+' does not exist')
+            to = self._get_image(to_name)
+            if to:
+                raise exception.VolumeBackendAPIException(data = 'Volume '+to_name+' already exists')
+            resp = self._etcd_txn({ 'compare': [
+                { 'target': 'MOD', 'mod_revision': vol['idx_mod'], 'key': 'index/image/'+vol['cfg']['name'] },
+                { 'target': 'MOD', 'mod_revision': vol['cfg_mod'], 'key': 'config/inode/'+str(vol['idx']['pool_id'])+'/'+str(vol['idx']['id']) },
+                { 'target': 'VERSION', 'version': 0, 'key': 'index/image/'+to_name },
+            ], 'success': [
+                { 'request_delete_range': { 'key': 'index/image/'+vol['cfg']['name'] } },
+                { 'request_put': { 'key': 'index/image/'+to_name, 'value': json.dumps(vol['idx']) } },
+                { 'request_put': { 'key': 'config/inode/'+str(vol['idx']['pool_id'])+'/'+str(vol['idx']['id']),
+                    'value': json.dumps({ **vol['cfg'], 'name': to_name }) } },
+            ] })
+            if resp.get('succeeded'):
+                break
 
     def unmanage(self, volume):
         pass
@@ -754,7 +956,7 @@ class VitastorDriver(driver.CloneableImageVD,
         snap_name = self._get_existing_name(existing_ref)
         from_name = vol_name+'@'+snap_name
         to_name = vol_name+'@'+utils.convert_str(snapshot.name)
-        self._cli('rename', 'modify', from_name, '--rename', to_name)
+        self._rename(from_name, to_name)
 
     def unmanage_snapshot(self, snapshot):
         """Removes the specified snapshot from Cinder management."""

patches/libvirt-12.0-vitastor.diff

@@ -1,637 +0,0 @@
-diff --git a/include/libvirt/libvirt-storage.h b/include/libvirt/libvirt-storage.h
-index aaad4a3da1..5f5daa8341 100644
---- a/include/libvirt/libvirt-storage.h
-+++ b/include/libvirt/libvirt-storage.h
-@@ -326,6 +326,7 @@ typedef enum {
-     VIR_CONNECT_LIST_STORAGE_POOLS_ZFS           = 1 << 17, /* (Since: 1.2.8) */
-     VIR_CONNECT_LIST_STORAGE_POOLS_VSTORAGE      = 1 << 18, /* (Since: 3.1.0) */
-     VIR_CONNECT_LIST_STORAGE_POOLS_ISCSI_DIRECT  = 1 << 19, /* (Since: 5.6.0) */
-+    VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR      = 1 << 20, /* (Since: 5.0.0) */
- } virConnectListAllStoragePoolsFlags;
- 
- int                     virConnectListAllStoragePools(virConnectPtr conn,
-diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
-index 9ca5c2450c..cc52f00c0c 100644
---- a/src/conf/domain_conf.c
-+++ b/src/conf/domain_conf.c
-@@ -7453,7 +7453,8 @@ virDomainDiskSourceNetworkParse(xmlNodePtr node,
-     src->configFile = virXPathString("string(./config/@file)", ctxt);
- 
-     if (src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTP ||
--        src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTPS)
-+        src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTPS ||
-+        src->protocol == VIR_STORAGE_NET_PROTOCOL_VITASTOR)
-         src->query = virXMLPropString(node, "query");
- 
-     if (virDomainStorageNetworkParseHosts(node, ctxt, &src->hosts, &src->nhosts) < 0)
-@@ -32187,6 +32188,7 @@ virDomainStorageSourceTranslateSourcePool(virStorageSource *src,
- 
-     case VIR_STORAGE_POOL_MPATH:
-     case VIR_STORAGE_POOL_RBD:
-+    case VIR_STORAGE_POOL_VITASTOR:
-     case VIR_STORAGE_POOL_SHEEPDOG:
-     case VIR_STORAGE_POOL_GLUSTER:
-     case VIR_STORAGE_POOL_LAST:
-diff --git a/src/conf/domain_validate.c b/src/conf/domain_validate.c
-index 7346a61731..83e94d762e 100644
---- a/src/conf/domain_validate.c
-+++ b/src/conf/domain_validate.c
-@@ -520,6 +520,7 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
-         case VIR_STORAGE_NET_PROTOCOL_RBD:
-             break;
- 
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-         case VIR_STORAGE_NET_PROTOCOL_NBD:
-         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-@@ -592,7 +593,7 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
-         }
-     }
- 
--    /* internal snapshots and config files are currently supported only with rbd: */
-+    /* internal snapshots are currently supported only with rbd: */
-     if (virStorageSourceGetActualType(src) != VIR_STORAGE_TYPE_NETWORK &&
-         src->protocol != VIR_STORAGE_NET_PROTOCOL_RBD) {
-         if (src->snapshot) {
-@@ -600,10 +601,14 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
-                            _("<snapshot> element is currently supported only with 'rbd' disks"));
-             return -1;
-         }
--
-+    }
-+    /* config files are currently supported only with rbd and vitastor: */
-+    if (virStorageSourceGetActualType(src) != VIR_STORAGE_TYPE_NETWORK &&
-+        src->protocol != VIR_STORAGE_NET_PROTOCOL_RBD &&
-+        src->protocol != VIR_STORAGE_NET_PROTOCOL_VITASTOR) {
-         if (src->configFile) {
-             virReportError(VIR_ERR_XML_ERROR, "%s",
--                           _("<config> element is currently supported only with 'rbd' disks"));
-+                           _("<config> element is currently supported only with 'rbd' and 'vitastor' disks"));
-             return -1;
-         }
-     }
-diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng
-index 114dd3f96f..c71f9a3277 100644
---- a/src/conf/schemas/domaincommon.rng
-+++ b/src/conf/schemas/domaincommon.rng
-@@ -2093,6 +2093,35 @@
-     </element>
-   </define>
- 
-+  <define name="diskSourceNetworkProtocolVitastor">
-+    <element name="source">
-+      <interleave>
-+        <attribute name="protocol">
-+          <value>vitastor</value>
-+        </attribute>
-+        <ref name="diskSourceCommon"/>
-+        <optional>
-+          <attribute name="name"/>
-+        </optional>
-+        <optional>
-+          <attribute name="query"/>
-+        </optional>
-+        <zeroOrMore>
-+          <ref name="diskSourceNetworkHost"/>
-+        </zeroOrMore>
-+        <optional>
-+          <element name="config">
-+            <attribute name="file">
-+              <ref name="absFilePath"/>
-+            </attribute>
-+            <empty/>
-+          </element>
-+        </optional>
-+        <empty/>
-+      </interleave>
-+    </element>
-+  </define>
-+
-   <define name="diskSourceNetworkProtocolISCSI">
-     <element name="source">
-       <attribute name="protocol">
-@@ -2443,6 +2472,7 @@
-       <ref name="diskSourceNetworkProtocolSimple"/>
-       <ref name="diskSourceNetworkProtocolVxHS"/>
-       <ref name="diskSourceNetworkProtocolNFS"/>
-+      <ref name="diskSourceNetworkProtocolVitastor"/>
-     </choice>
-   </define>
- 
-diff --git a/src/conf/storage_conf.c b/src/conf/storage_conf.c
-index 1dc9365bf2..a8a736be81 100644
---- a/src/conf/storage_conf.c
-+++ b/src/conf/storage_conf.c
-@@ -56,7 +56,7 @@ VIR_ENUM_IMPL(virStoragePool,
-               "logical", "disk", "iscsi",
-               "iscsi-direct", "scsi", "mpath",
-               "rbd", "sheepdog", "gluster",
--              "zfs", "vstorage",
-+              "zfs", "vstorage", "vitastor",
- );
- 
- VIR_ENUM_IMPL(virStoragePoolFormatFileSystem,
-@@ -242,6 +242,18 @@ static virStoragePoolTypeInfo poolTypeInfo[] = {
-           .formatToString = virStorageFileFormatTypeToString,
-       }
-     },
-+    {.poolType = VIR_STORAGE_POOL_VITASTOR,
-+     .poolOptions = {
-+         .flags = (VIR_STORAGE_POOL_SOURCE_HOST |
-+                   VIR_STORAGE_POOL_SOURCE_NETWORK |
-+                   VIR_STORAGE_POOL_SOURCE_NAME),
-+      },
-+      .volOptions = {
-+          .defaultFormat = VIR_STORAGE_FILE_RAW,
-+          .formatFromString = virStorageVolumeFormatFromString,
-+          .formatToString = virStorageFileFormatTypeToString,
-+      }
-+    },
-     {.poolType = VIR_STORAGE_POOL_SHEEPDOG,
-      .poolOptions = {
-          .flags = (VIR_STORAGE_POOL_SOURCE_HOST |
-@@ -538,6 +550,11 @@ virStoragePoolDefParseSource(xmlXPathContextPtr ctxt,
-                        _("element 'name' is mandatory for RBD pool"));
-         return -1;
-     }
-+    if (pool_type == VIR_STORAGE_POOL_VITASTOR && source->name == NULL) {
-+        virReportError(VIR_ERR_XML_ERROR, "%s",
-+                       _("element 'name' is mandatory for Vitastor pool"));
-+        return -1;
-+    }
- 
-     if (options->formatFromString) {
-         g_autofree char *format = NULL;
-@@ -1127,6 +1144,7 @@ virStoragePoolDefFormatBuf(virBuffer *buf,
-     /* RBD, Sheepdog, Gluster and Iscsi-direct devices are not local block devs nor
-      * files, so they don't have a target */
-     if (def->type != VIR_STORAGE_POOL_RBD &&
-+        def->type != VIR_STORAGE_POOL_VITASTOR &&
-         def->type != VIR_STORAGE_POOL_SHEEPDOG &&
-         def->type != VIR_STORAGE_POOL_GLUSTER &&
-         def->type != VIR_STORAGE_POOL_ISCSI_DIRECT) {
-diff --git a/src/conf/storage_conf.h b/src/conf/storage_conf.h
-index fc67957cfe..720c07ef74 100644
---- a/src/conf/storage_conf.h
-+++ b/src/conf/storage_conf.h
-@@ -103,6 +103,7 @@ typedef enum {
-     VIR_STORAGE_POOL_GLUSTER,  /* Gluster device */
-     VIR_STORAGE_POOL_ZFS,      /* ZFS */
-     VIR_STORAGE_POOL_VSTORAGE, /* Virtuozzo Storage */
-+    VIR_STORAGE_POOL_VITASTOR, /* Vitastor */
- 
-     VIR_STORAGE_POOL_LAST,
- } virStoragePoolType;
-@@ -454,6 +455,7 @@ VIR_ENUM_DECL(virStoragePartedFs);
-                  VIR_CONNECT_LIST_STORAGE_POOLS_SCSI     | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_MPATH    | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_RBD      | \
-+                 VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER  | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_ZFS      | \
-diff --git a/src/conf/storage_source_conf.c b/src/conf/storage_source_conf.c
-index d7b9bdfecb..38aefd0dd4 100644
---- a/src/conf/storage_source_conf.c
-+++ b/src/conf/storage_source_conf.c
-@@ -90,6 +90,7 @@ VIR_ENUM_IMPL(virStorageNetProtocol,
-               "ssh",
-               "vxhs",
-               "nfs",
-+              "vitastor",
- );
- 
- 
-@@ -1317,6 +1318,7 @@ virStorageSourceNetworkDefaultPort(virStorageNetProtocol protocol)
-         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-             return 24007;
- 
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-         case VIR_STORAGE_NET_PROTOCOL_RBD:
-             /* we don't provide a default for RBD */
-             return 0;
-diff --git a/src/conf/storage_source_conf.h b/src/conf/storage_source_conf.h
-index 22c35d420d..f1e32ea83d 100644
---- a/src/conf/storage_source_conf.h
-+++ b/src/conf/storage_source_conf.h
-@@ -131,6 +131,7 @@ typedef enum {
-     VIR_STORAGE_NET_PROTOCOL_SSH,
-     VIR_STORAGE_NET_PROTOCOL_VXHS,
-     VIR_STORAGE_NET_PROTOCOL_NFS,
-+    VIR_STORAGE_NET_PROTOCOL_VITASTOR,
- 
-     VIR_STORAGE_NET_PROTOCOL_LAST
- } virStorageNetProtocol;
-diff --git a/src/conf/virstorageobj.c b/src/conf/virstorageobj.c
-index 59fa5da372..4739167f5f 100644
---- a/src/conf/virstorageobj.c
-+++ b/src/conf/virstorageobj.c
-@@ -1438,6 +1438,7 @@ virStoragePoolObjSourceFindDuplicateCb(const void *payload,
-             return 1;
-         break;
- 
-+    case VIR_STORAGE_POOL_VITASTOR:
-     case VIR_STORAGE_POOL_ISCSI_DIRECT:
-     case VIR_STORAGE_POOL_RBD:
-     case VIR_STORAGE_POOL_LAST:
-@@ -1921,6 +1922,8 @@ virStoragePoolObjMatch(virStoragePoolObj *obj,
-                (obj->def->type == VIR_STORAGE_POOL_MPATH))   ||
-               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_RBD) &&
-                (obj->def->type == VIR_STORAGE_POOL_RBD))     ||
-+              (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR) &&
-+               (obj->def->type == VIR_STORAGE_POOL_VITASTOR)) ||
-               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG) &&
-                (obj->def->type == VIR_STORAGE_POOL_SHEEPDOG)) ||
-               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER) &&
-diff --git a/src/libvirt-storage.c b/src/libvirt-storage.c
-index db7660aac4..561df34709 100644
---- a/src/libvirt-storage.c
-+++ b/src/libvirt-storage.c
-@@ -94,6 +94,7 @@ virStoragePoolGetConnect(virStoragePoolPtr pool)
-  * VIR_CONNECT_LIST_STORAGE_POOLS_SCSI
-  * VIR_CONNECT_LIST_STORAGE_POOLS_MPATH
-  * VIR_CONNECT_LIST_STORAGE_POOLS_RBD
-+ * VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR
-  * VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG
-  * VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER
-  * VIR_CONNECT_LIST_STORAGE_POOLS_ZFS
-diff --git a/src/libxl/libxl_conf.c b/src/libxl/libxl_conf.c
-index 2b988157fa..9d0eb47b25 100644
---- a/src/libxl/libxl_conf.c
-+++ b/src/libxl/libxl_conf.c
-@@ -1069,6 +1069,7 @@ libxlMakeNetworkDiskSrcStr(virStorageSource *src,
-     case VIR_STORAGE_NET_PROTOCOL_SSH:
-     case VIR_STORAGE_NET_PROTOCOL_VXHS:
-     case VIR_STORAGE_NET_PROTOCOL_NFS:
-+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-     case VIR_STORAGE_NET_PROTOCOL_LAST:
-     case VIR_STORAGE_NET_PROTOCOL_NONE:
-         virReportError(VIR_ERR_NO_SUPPORT,
-diff --git a/src/libxl/xen_xl.c b/src/libxl/xen_xl.c
-index e72e7d7f44..8482c21805 100644
---- a/src/libxl/xen_xl.c
-+++ b/src/libxl/xen_xl.c
-@@ -1461,6 +1461,7 @@ xenFormatXLDiskSrcNet(virStorageSource *src)
-     case VIR_STORAGE_NET_PROTOCOL_SSH:
-     case VIR_STORAGE_NET_PROTOCOL_VXHS:
-     case VIR_STORAGE_NET_PROTOCOL_NFS:
-+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-     case VIR_STORAGE_NET_PROTOCOL_LAST:
-     case VIR_STORAGE_NET_PROTOCOL_NONE:
-         virReportError(VIR_ERR_NO_SUPPORT,
-diff --git a/src/qemu/qemu_block.c b/src/qemu/qemu_block.c
-index 9b43279797..459d8e8a65 100644
---- a/src/qemu/qemu_block.c
-+++ b/src/qemu/qemu_block.c
-@@ -743,6 +743,38 @@ qemuBlockStorageSourceGetRBDProps(virStorageSource *src,
- }
- 
- 
-+static virJSONValue *
-+qemuBlockStorageSourceGetVitastorProps(virStorageSource *src)
-+{
-+    virJSONValue *ret = NULL;
-+    virStorageNetHostDef *host;
-+    size_t i;
-+    g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;
-+    g_autofree char *etcd = NULL;
-+
-+    for (i = 0; i < src->nhosts; i++) {
-+        host = src->hosts + i;
-+        if ((virStorageNetHostTransport)host->transport != VIR_STORAGE_NET_HOST_TRANS_TCP) {
-+            return NULL;
-+        }
-+        virBufferAsprintf(&buf, i > 0 ? ",%s:%u" : "%s:%u", host->name, host->port);
-+    }
-+    if (src->nhosts > 0) {
-+        etcd = virBufferContentAndReset(&buf);
-+    }
-+
-+    if (virJSONValueObjectAdd(&ret,
-+                              "S:etcd-host", etcd,
-+                              "S:etcd-prefix", src->query,
-+                              "S:config-path", src->configFile,
-+                              "s:image", src->path,
-+                              NULL) < 0)
-+        return NULL;
-+
-+    return ret;
-+}
-+
-+
- static virJSONValue *
- qemuBlockStorageSourceGetSshProps(virStorageSource *src)
- {
-@@ -1094,6 +1126,12 @@ qemuBlockStorageSourceGetBackendProps(virStorageSource *src,
-                 return NULL;
-             break;
- 
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-+            driver = "vitastor";
-+            if (!(fileprops = qemuBlockStorageSourceGetVitastorProps(src)))
-+                return NULL;
-+            break;
-+
-         case VIR_STORAGE_NET_PROTOCOL_SSH:
-             driver = "ssh";
-             if (!(fileprops = qemuBlockStorageSourceGetSshProps(src)))
-@@ -1997,6 +2035,7 @@ qemuBlockGetBackingStoreString(virStorageSource *src,
- 
-             case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-             case VIR_STORAGE_NET_PROTOCOL_RBD:
-+            case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-             case VIR_STORAGE_NET_PROTOCOL_VXHS:
-             case VIR_STORAGE_NET_PROTOCOL_NFS:
-             case VIR_STORAGE_NET_PROTOCOL_SSH:
-@@ -2377,6 +2416,12 @@ qemuBlockStorageSourceCreateGetStorageProps(virStorageSource *src,
-                 return -1;
-             break;
- 
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-+            driver = "vitastor";
-+            if (!(location = qemuBlockStorageSourceGetVitastorProps(src)))
-+                return -1;
-+            break;
-+
-         case VIR_STORAGE_NET_PROTOCOL_SSH:
-             if (srcPriv->nbdkitProcess) {
-                 /* disk creation not yet supported with nbdkit, and even if it
-diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
-index ac56fc7cb4..9e407b4aab 100644
---- a/src/qemu/qemu_domain.c
-+++ b/src/qemu/qemu_domain.c
-@@ -4677,7 +4677,8 @@ qemuDomainValidateStorageSource(virStorageSource *src,
-     if (src->query &&
-         (actualType != VIR_STORAGE_TYPE_NETWORK ||
-          (src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTPS &&
--          src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTP))) {
-+          src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTP &&
-+          src->protocol != VIR_STORAGE_NET_PROTOCOL_VITASTOR))) {
-         virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
-                        _("query is supported only with HTTP(S) protocols"));
-         return -1;
-@@ -9103,6 +9104,7 @@ qemuDomainPrepareStorageSourceTLS(virStorageSource *src,
-         break;
- 
-     case VIR_STORAGE_NET_PROTOCOL_RBD:
-+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-     case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-     case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-     case VIR_STORAGE_NET_PROTOCOL_ISCSI:
-diff --git a/src/qemu/qemu_snapshot.c b/src/qemu/qemu_snapshot.c
-index e738afffc3..37d64f469b 100644
---- a/src/qemu/qemu_snapshot.c
-+++ b/src/qemu/qemu_snapshot.c
-@@ -665,6 +665,7 @@ qemuSnapshotPrepareDiskExternalInactive(virDomainSnapshotDiskDef *snapdisk,
-         case VIR_STORAGE_NET_PROTOCOL_NONE:
-         case VIR_STORAGE_NET_PROTOCOL_NBD:
-         case VIR_STORAGE_NET_PROTOCOL_RBD:
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-         case VIR_STORAGE_NET_PROTOCOL_ISCSI:
-@@ -893,6 +894,7 @@ qemuSnapshotPrepareDiskInternal(virDomainDiskDef *disk,
-         case VIR_STORAGE_NET_PROTOCOL_NONE:
-         case VIR_STORAGE_NET_PROTOCOL_NBD:
-         case VIR_STORAGE_NET_PROTOCOL_RBD:
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-         case VIR_STORAGE_NET_PROTOCOL_ISCSI:
-diff --git a/src/storage/storage_driver.c b/src/storage/storage_driver.c
-index e19e032427..59f91f4710 100644
---- a/src/storage/storage_driver.c
-+++ b/src/storage/storage_driver.c
-@@ -1626,6 +1626,7 @@ storageVolLookupByPathCallback(virStoragePoolObj *obj,
- 
-         case VIR_STORAGE_POOL_GLUSTER:
-         case VIR_STORAGE_POOL_RBD:
-+        case VIR_STORAGE_POOL_VITASTOR:
-         case VIR_STORAGE_POOL_SHEEPDOG:
-         case VIR_STORAGE_POOL_ZFS:
-         case VIR_STORAGE_POOL_LAST:
-diff --git a/src/storage_file/storage_source_backingstore.c b/src/storage_file/storage_source_backingstore.c
-index 821378883c..2211f6891b 100644
---- a/src/storage_file/storage_source_backingstore.c
-+++ b/src/storage_file/storage_source_backingstore.c
-@@ -264,6 +264,75 @@ virStorageSourceParseRBDColonString(const char *rbdstr,
- }
- 
- 
-+static int
-+virStorageSourceParseVitastorColonString(const char *colonstr,
-+                                         virStorageSource *src)
-+{
-+    char *p, *e, *next;
-+    g_autofree char *options = NULL;
-+
-+    /* optionally skip the "vitastor:" prefix if provided */
-+    if (STRPREFIX(colonstr, "vitastor:"))
-+        colonstr += strlen("vitastor:");
-+
-+    options = g_strdup(colonstr);
-+
-+    p = options;
-+    while (*p) {
-+        /* find : delimiter or end of string */
-+        for (e = p; *e && *e != ':'; ++e) {
-+            if (*e == '\\') {
-+                e++;
-+                if (*e == '\0')
-+                    break;
-+            }
-+        }
-+        if (*e == '\0') {
-+            next = e;    /* last kv pair */
-+        } else {
-+            next = e + 1;
-+            *e = '\0';
-+        }
-+
-+        if (STRPREFIX(p, "image=")) {
-+            src->path = g_strdup(p + strlen("image="));
-+        } else if (STRPREFIX(p, "etcd-prefix=")) {
-+            src->query = g_strdup(p + strlen("etcd-prefix="));
-+        } else if (STRPREFIX(p, "config-path=")) {
-+            src->configFile = g_strdup(p + strlen("config-path="));
-+        } else if (STRPREFIX(p, "etcd-host=")) {
-+            char *h, *sep;
-+
-+            h = p + strlen("etcd-host=");
-+            while (h < e) {
-+                for (sep = h; sep < e; ++sep) {
-+                    if (*sep == '\\' && (sep[1] == ',' ||
-+                                         sep[1] == ';' ||
-+                                         sep[1] == ' ')) {
-+                        *sep = '\0';
-+                        sep += 2;
-+                        break;
-+                    }
-+                }
-+
-+                if (virStorageSourceRBDAddHost(src, h) < 0)
-+                    return -1;
-+
-+                h = sep;
-+            }
-+        }
-+
-+        p = next;
-+    }
-+
-+    if (!src->path) {
-+        return -1;
-+    }
-+
-+    return 0;
-+}
-+
-+
- static int
- virStorageSourceParseNBDColonString(const char *nbdstr,
-                                     virStorageSource *src)
-@@ -379,6 +448,11 @@ virStorageSourceParseBackingColon(virStorageSource *src,
-             return -1;
-         break;
- 
-+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-+        if (virStorageSourceParseVitastorColonString(path, src) < 0)
-+            return -1;
-+        break;
-+
-     case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-     case VIR_STORAGE_NET_PROTOCOL_LAST:
-     case VIR_STORAGE_NET_PROTOCOL_NONE:
-@@ -953,6 +1027,54 @@ virStorageSourceParseBackingJSONRBD(virStorageSource *src,
-     return 0;
- }
- 
-+static int
-+virStorageSourceParseBackingJSONVitastor(virStorageSource *src,
-+                                         virJSONValue *json,
-+                                         const char *jsonstr G_GNUC_UNUSED,
-+                                         int opaque G_GNUC_UNUSED)
-+{
-+    const char *filename;
-+    const char *image = virJSONValueObjectGetString(json, "image");
-+    const char *conf = virJSONValueObjectGetString(json, "config-path");
-+    const char *etcd_prefix = virJSONValueObjectGetString(json, "etcd-prefix");
-+    virJSONValue *servers = virJSONValueObjectGetArray(json, "server");
-+    size_t nservers;
-+    size_t i;
-+
-+    src->type = VIR_STORAGE_TYPE_NETWORK;
-+    src->protocol = VIR_STORAGE_NET_PROTOCOL_VITASTOR;
-+
-+    /* legacy syntax passed via 'filename' option */
-+    if ((filename = virJSONValueObjectGetString(json, "filename")))
-+        return virStorageSourceParseVitastorColonString(filename, src);
-+
-+    if (!image) {
-+        virReportError(VIR_ERR_INVALID_ARG, "%s",
-+                       _("missing image name in Vitastor backing volume "
-+                         "JSON specification"));
-+        return -1;
-+    }
-+
-+    src->path = g_strdup(image);
-+    src->configFile = g_strdup(conf);
-+    src->query = g_strdup(etcd_prefix);
-+
-+    if (servers) {
-+        nservers = virJSONValueArraySize(servers);
-+
-+        src->hosts = g_new0(virStorageNetHostDef, nservers);
-+        src->nhosts = nservers;
-+
-+        for (i = 0; i < nservers; i++) {
-+            if (virStorageSourceParseBackingJSONInetSocketAddress(src->hosts + i,
-+                                                                  virJSONValueArrayGet(servers, i)) < 0)
-+                return -1;
-+        }
-+    }
-+
-+    return 0;
-+}
-+
- static int
- virStorageSourceParseBackingJSONRaw(virStorageSource *src,
-                                     virJSONValue *json,
-@@ -1130,6 +1252,7 @@ static const struct virStorageSourceJSONDriverParser jsonParsers[] = {
-     {"sheepdog", false, virStorageSourceParseBackingJSONSheepdog, 0},
-     {"ssh", false, virStorageSourceParseBackingJSONSSH, 0},
-     {"rbd", false, virStorageSourceParseBackingJSONRBD, 0},
-+    {"vitastor", false, virStorageSourceParseBackingJSONVitastor, 0},
-     {"raw", true, virStorageSourceParseBackingJSONRaw, 0},
-     {"nfs", false, virStorageSourceParseBackingJSONNFS, 0},
-     {"vxhs", false, virStorageSourceParseBackingJSONVxHS, 0},
-diff --git a/src/test/test_driver.c b/src/test/test_driver.c
-index 1165689de7..bba846351c 100644
---- a/src/test/test_driver.c
-+++ b/src/test/test_driver.c
-@@ -7345,6 +7345,7 @@ testStorageVolumeTypeForPool(int pooltype)
-     case VIR_STORAGE_POOL_ISCSI_DIRECT:
-     case VIR_STORAGE_POOL_GLUSTER:
-     case VIR_STORAGE_POOL_RBD:
-+    case VIR_STORAGE_POOL_VITASTOR:
-         return VIR_STORAGE_VOL_NETWORK;
-     case VIR_STORAGE_POOL_LOGICAL:
-     case VIR_STORAGE_POOL_DISK:
-diff --git a/tests/storagepoolcapsschemadata/poolcaps-fs.xml b/tests/storagepoolcapsschemadata/poolcaps-fs.xml
-index eee75af746..8bd0a57bdd 100644
---- a/tests/storagepoolcapsschemadata/poolcaps-fs.xml
-+++ b/tests/storagepoolcapsschemadata/poolcaps-fs.xml
-@@ -204,4 +204,11 @@
-       </enum>
-     </volOptions>
-   </pool>
-+  <pool type='vitastor' supported='no'>
-+    <volOptions>
-+      <defaultFormat type='raw'/>
-+      <enum name='targetFormatType'>
-+      </enum>
-+    </volOptions>
-+  </pool>
- </storagepoolCapabilities>
-diff --git a/tests/storagepoolcapsschemadata/poolcaps-full.xml b/tests/storagepoolcapsschemadata/poolcaps-full.xml
-index 805950a937..852df0de16 100644
---- a/tests/storagepoolcapsschemadata/poolcaps-full.xml
-+++ b/tests/storagepoolcapsschemadata/poolcaps-full.xml
-@@ -204,4 +204,11 @@
-       </enum>
-     </volOptions>
-   </pool>
-+  <pool type='vitastor' supported='yes'>
-+    <volOptions>
-+      <defaultFormat type='raw'/>
-+      <enum name='targetFormatType'>
-+      </enum>
-+    </volOptions>
-+  </pool>
- </storagepoolCapabilities>
-diff --git a/tests/storagepoolxml2argvtest.c b/tests/storagepoolxml2argvtest.c
-index d5c2531ab8..b19308ac38 100644
---- a/tests/storagepoolxml2argvtest.c
-+++ b/tests/storagepoolxml2argvtest.c
-@@ -57,6 +57,7 @@ testCompareXMLToArgvFiles(bool shouldFail,
-     case VIR_STORAGE_POOL_GLUSTER:
-     case VIR_STORAGE_POOL_ZFS:
-     case VIR_STORAGE_POOL_VSTORAGE:
-+    case VIR_STORAGE_POOL_VITASTOR:
-     case VIR_STORAGE_POOL_LAST:
-     default:
-         VIR_TEST_DEBUG("pool type '%s' has no xml2argv test", defTypeStr);
-diff --git a/tools/virsh-pool.c b/tools/virsh-pool.c
-index 2010ef1356..072e2ff9e8 100644
---- a/tools/virsh-pool.c
-+++ b/tools/virsh-pool.c
-@@ -1187,6 +1187,9 @@ cmdPoolList(vshControl *ctl, const vshCmd *cmd G_GNUC_UNUSED)
-             case VIR_STORAGE_POOL_VSTORAGE:
-                 flags |= VIR_CONNECT_LIST_STORAGE_POOLS_VSTORAGE;
-                 break;
-+            case VIR_STORAGE_POOL_VITASTOR:
-+                flags |= VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR;
-+                break;
-             case VIR_STORAGE_POOL_LAST:
-                 break;
-             }

patches/qemu-10.1-vitastor.patch

@@ -1,172 +1,29 @@
-diff --git a/block/meson.build b/block/meson.build
+diff --git a/src/client/qemu_driver.c b/src/client/qemu_driver.c
-index 34b1b2a306..24ca0f1e52 100644
+index d8356dab..5f4cd50d 100644
---- a/block/meson.build
+--- a/src/client/qemu_driver.c
-+++ b/block/meson.build
++++ b/src/client/qemu_driver.c
-@@ -114,6 +114,7 @@ foreach m : [
+@@ -974,14 +974,21 @@ static void vitastor_co_read_bitmap_cb(void *opaque, long retval, uint8_t *bitma
-   [libnfs, 'nfs', files('nfs.c')],
+ #endif
-   [libssh, 'ssh', files('ssh.c')],
+ }
-   [rbd, 'rbd', files('rbd.c')],
-+  [vitastor, 'vitastor', files('vitastor.c')],
- ]
-   if m[0].found()
-     module_ss = ss.source_set()
-diff --git a/meson.build b/meson.build
-index 50c774a195..e5c7a3a4b1 100644
---- a/meson.build
-+++ b/meson.build
-@@ -1652,6 +1652,26 @@ if not get_option('rbd').auto() or have_block
-   endif
- endif
  
-+vitastor = not_found
+-static int coroutine_fn vitastor_co_block_status(
-+if not get_option('vitastor').auto() or have_block
+-    BlockDriverState *bs, bool want_zero, int64_t offset, int64_t bytes,
-+  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
+-    int64_t *pnum, int64_t *map, BlockDriverState **file)
-+    required: get_option('vitastor'))
++static int coroutine_fn vitastor_co_block_status(BlockDriverState *bs,
-+  if libvitastor_client.found()
++#if QEMU_VERSION_MAJOR > 10 || QEMU_VERSION_MAJOR == 10 && QEMU_VERSION_MINOR >= 1
-+    if cc.links('''
++    unsigned int mode,
-+      #include <vitastor_c.h>
++#else
-+      int main(void) {
++    bool want_zero,
-+        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
++#endif
-+        return 0;
++    int64_t offset, int64_t bytes, int64_t *pnum, int64_t *map, BlockDriverState **file)
-+      }''', dependencies: libvitastor_client)
+ {
-+      vitastor = declare_dependency(dependencies: libvitastor_client)
+     // Allocated => return BDRV_BLOCK_DATA|BDRV_BLOCK_OFFSET_VALID
-+    elif get_option('vitastor').enabled()
+     // Not allocated => return 0
-+      error('could not link libvitastor_client')
+     // Error => return -errno
-+    else
+     // Set pnum to length of the extent, `*map` = `offset`, `*file` = `bs`
-+      warning('could not link libvitastor_client, disabling')
++#if QEMU_VERSION_MAJOR > 10 || QEMU_VERSION_MAJOR == 10 && QEMU_VERSION_MINOR >= 1
-+    endif
++    int want_zero = (mode == BDRV_WANT_PRECISE);
-+  endif
++#endif
-+endif
+     VitastorRPC task;
-+
+     VitastorClient *client = bs->opaque;
- glusterfs = not_found
+     uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode;
- glusterfs_ftruncate_has_stat = false
- glusterfs_iocb_has_stat = false
-@@ -2547,6 +2567,7 @@ endif
- config_host_data.set('CONFIG_OPENGL', opengl.found())
- config_host_data.set('CONFIG_PLUGIN', get_option('plugins'))
- config_host_data.set('CONFIG_RBD', rbd.found())
-+config_host_data.set('CONFIG_VITASTOR', vitastor.found())
- config_host_data.set('CONFIG_RDMA', rdma.found())
- config_host_data.set('CONFIG_RELOCATABLE', get_option('relocatable'))
- config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack'))
-@@ -4972,6 +4993,7 @@ summary_info += {'fdt support':       fdt_opt == 'internal' ? 'internal' : fdt}
- summary_info += {'libcap-ng support': libcap_ng}
- summary_info += {'bpf support':       libbpf}
- summary_info += {'rbd support':       rbd}
-+summary_info += {'vitastor support':  vitastor}
- summary_info += {'smartcard support': cacard}
- summary_info += {'U2F support':       u2f}
- summary_info += {'libusb':            libusb}
-diff --git a/meson_options.txt b/meson_options.txt
-index fff1521e58..f0844c0e00 100644
---- a/meson_options.txt
-+++ b/meson_options.txt
-@@ -202,6 +202,8 @@ option('pvg', type: 'feature', value: 'auto',
-        description: 'macOS paravirtualized graphics support')
- option('rbd', type : 'feature', value : 'auto',
-        description: 'Ceph block device driver')
-+option('vitastor', type : 'feature', value : 'auto',
-+       description: 'Vitastor block device driver')
- option('opengl', type : 'feature', value : 'auto',
-        description: 'OpenGL support')
- option('rdma', type : 'feature', value : 'auto',
-diff --git a/qapi/block-core.json b/qapi/block-core.json
-index dc6eb4ae23..d043f4340e 100644
---- a/qapi/block-core.json
-+++ b/qapi/block-core.json
-@@ -3280,7 +3280,7 @@
-             'parallels', 'preallocate', 'qcow', 'qcow2', 'qed', 'quorum',
-             'raw', 'rbd',
-             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
--            'ssh', 'throttle', 'vdi', 'vhdx',
-+            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
-             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
-             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
-             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
-@@ -4363,6 +4363,28 @@
-             '*key-secret': 'str',
-             '*server': ['InetSocketAddressBase'] } }
- 
-+##
-+# @BlockdevOptionsVitastor:
-+#
-+# Driver specific block device options for vitastor
-+#
-+# @image:       Image name
-+# @inode:       Inode number
-+# @pool:        Pool ID
-+# @size:        Desired image size in bytes
-+# @config-path: Path to Vitastor configuration
-+# @etcd-host:   etcd connection address(es)
-+# @etcd-prefix: etcd key/value prefix
-+##
-+{ 'struct': 'BlockdevOptionsVitastor',
-+  'data': { '*inode': 'uint64',
-+            '*pool': 'uint64',
-+            '*size': 'uint64',
-+            '*image': 'str',
-+            '*config-path': 'str',
-+            '*etcd-host': 'str',
-+            '*etcd-prefix': 'str' } }
-+
- ##
- # @ReplicationMode:
- #
-@@ -4831,6 +4853,7 @@
-       'throttle':   'BlockdevOptionsThrottle',
-       'vdi':        'BlockdevOptionsGenericFormat',
-       'vhdx':       'BlockdevOptionsGenericFormat',
-+      'vitastor':   'BlockdevOptionsVitastor',
-       'virtio-blk-vfio-pci':
-                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
-                       'if': 'CONFIG_BLKIO' },
-@@ -5304,6 +5327,20 @@
-             '*cluster-size' :   'size',
-             '*encrypt' :        'RbdEncryptionCreateOptions' } }
- 
-+##
-+# @BlockdevCreateOptionsVitastor:
-+#
-+# Driver specific image creation options for Vitastor.
-+#
-+# @location: Where to store the new image file.  This location cannot
-+#     point to a snapshot.
-+#
-+# @size: Size of the virtual disk in bytes
-+##
-+{ 'struct': 'BlockdevCreateOptionsVitastor',
-+  'data': { 'location':         'BlockdevOptionsVitastor',
-+            'size':             'size' } }
-+
- ##
- # @BlockdevVmdkSubformat:
- #
-@@ -5526,6 +5563,7 @@
-       'ssh':            'BlockdevCreateOptionsSsh',
-       'vdi':            'BlockdevCreateOptionsVdi',
-       'vhdx':           'BlockdevCreateOptionsVhdx',
-+      'vitastor':       'BlockdevCreateOptionsVitastor',
-       'vmdk':           'BlockdevCreateOptionsVmdk',
-       'vpc':            'BlockdevCreateOptionsVpc'
-   } }
-diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh
-index 0ebe6bc52a..2c37ad3892 100644
---- a/scripts/meson-buildoptions.sh
-+++ b/scripts/meson-buildoptions.sh
-@@ -175,6 +175,7 @@ meson_options_help() {
-   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
-   printf "%s\n" '  qpl             Query Processing Library support'
-   printf "%s\n" '  rbd             Ceph block device driver'
-+  printf "%s\n" '  vitastor        Vitastor block device driver'
-   printf "%s\n" '  rdma            Enable RDMA-based migration'
-   printf "%s\n" '  replication     replication support'
-   printf "%s\n" '  rust            Rust support'
-@@ -459,6 +460,8 @@ _meson_option_parse() {
-     --disable-qpl) printf "%s" -Dqpl=disabled ;;
-     --enable-rbd) printf "%s" -Drbd=enabled ;;
-     --disable-rbd) printf "%s" -Drbd=disabled ;;
-+    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
-+    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
-     --enable-rdma) printf "%s" -Drdma=enabled ;;
-     --disable-rdma) printf "%s" -Drdma=disabled ;;
-     --enable-relocatable) printf "%s" -Drelocatable=true ;;

patches/qemu-10.2-vitastor.patch

@@ -1,172 +0,0 @@
-diff --git a/block/meson.build b/block/meson.build
-index 34b1b2a306..24ca0f1e52 100644
---- a/block/meson.build
-+++ b/block/meson.build
-@@ -114,6 +114,7 @@ foreach m : [
-   [libnfs, 'nfs', files('nfs.c')],
-   [libssh, 'ssh', files('ssh.c')],
-   [rbd, 'rbd', files('rbd.c')],
-+  [vitastor, 'vitastor', files('vitastor.c')],
- ]
-   if m[0].found()
-     module_ss = ss.source_set()
-diff --git a/meson.build b/meson.build
-index d9293294d8..776a5becc6 100644
---- a/meson.build
-+++ b/meson.build
-@@ -1665,6 +1665,26 @@ if not get_option('rbd').auto() or have_block
-   endif
- endif
- 
-+vitastor = not_found
-+if not get_option('vitastor').auto() or have_block
-+  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
-+    required: get_option('vitastor'))
-+  if libvitastor_client.found()
-+    if cc.links('''
-+      #include <vitastor_c.h>
-+      int main(void) {
-+        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
-+        return 0;
-+      }''', dependencies: libvitastor_client)
-+      vitastor = declare_dependency(dependencies: libvitastor_client)
-+    elif get_option('vitastor').enabled()
-+      error('could not link libvitastor_client')
-+    else
-+      warning('could not link libvitastor_client, disabling')
-+    endif
-+  endif
-+endif
-+
- glusterfs = not_found
- glusterfs_ftruncate_has_stat = false
- glusterfs_iocb_has_stat = false
-@@ -2509,6 +2529,7 @@ endif
- config_host_data.set('CONFIG_OPENGL', opengl.found())
- config_host_data.set('CONFIG_PLUGIN', get_option('plugins'))
- config_host_data.set('CONFIG_RBD', rbd.found())
-+config_host_data.set('CONFIG_VITASTOR', vitastor.found())
- config_host_data.set('CONFIG_RDMA', rdma.found())
- config_host_data.set('CONFIG_RELOCATABLE', get_option('relocatable'))
- config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack'))
-@@ -4948,6 +4969,7 @@ summary_info += {'fdt support':       fdt_opt == 'internal' ? 'internal' : fdt}
- summary_info += {'libcap-ng support': libcap_ng}
- summary_info += {'bpf support':       libbpf}
- summary_info += {'rbd support':       rbd}
-+summary_info += {'vitastor support':  vitastor}
- summary_info += {'smartcard support': cacard}
- summary_info += {'U2F support':       u2f}
- summary_info += {'libusb':            libusb}
-diff --git a/meson_options.txt b/meson_options.txt
-index 2836156257..148086cc6f 100644
---- a/meson_options.txt
-+++ b/meson_options.txt
-@@ -206,6 +206,8 @@ option('pvg', type: 'feature', value: 'auto',
-        description: 'macOS paravirtualized graphics support')
- option('rbd', type : 'feature', value : 'auto',
-        description: 'Ceph block device driver')
-+option('vitastor', type : 'feature', value : 'auto',
-+       description: 'Vitastor block device driver')
- option('opengl', type : 'feature', value : 'auto',
-        description: 'OpenGL support')
- option('rdma', type : 'feature', value : 'auto',
-diff --git a/qapi/block-core.json b/qapi/block-core.json
-index b82af74256..f25a6f5ce8 100644
---- a/qapi/block-core.json
-+++ b/qapi/block-core.json
-@@ -3351,7 +3351,7 @@
-             'parallels', 'preallocate', 'qcow', 'qcow2', 'qed', 'quorum',
-             'raw', 'rbd',
-             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
--            'ssh', 'throttle', 'vdi', 'vhdx',
-+            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
-             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
-             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
-             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
-@@ -4434,6 +4434,28 @@
-             '*key-secret': 'str',
-             '*server': ['InetSocketAddressBase'] } }
- 
-+##
-+# @BlockdevOptionsVitastor:
-+#
-+# Driver specific block device options for vitastor
-+#
-+# @image:       Image name
-+# @inode:       Inode number
-+# @pool:        Pool ID
-+# @size:        Desired image size in bytes
-+# @config-path: Path to Vitastor configuration
-+# @etcd-host:   etcd connection address(es)
-+# @etcd-prefix: etcd key/value prefix
-+##
-+{ 'struct': 'BlockdevOptionsVitastor',
-+  'data': { '*inode': 'uint64',
-+            '*pool': 'uint64',
-+            '*size': 'uint64',
-+            '*image': 'str',
-+            '*config-path': 'str',
-+            '*etcd-host': 'str',
-+            '*etcd-prefix': 'str' } }
-+
- ##
- # @ReplicationMode:
- #
-@@ -4902,6 +4924,7 @@
-       'throttle':   'BlockdevOptionsThrottle',
-       'vdi':        'BlockdevOptionsGenericFormat',
-       'vhdx':       'BlockdevOptionsGenericFormat',
-+      'vitastor':   'BlockdevOptionsVitastor',
-       'virtio-blk-vfio-pci':
-                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
-                       'if': 'CONFIG_BLKIO' },
-@@ -5376,6 +5399,20 @@
-             '*cluster-size' :   'size',
-             '*encrypt' :        'RbdEncryptionCreateOptions' } }
- 
-+##
-+# @BlockdevCreateOptionsVitastor:
-+#
-+# Driver specific image creation options for Vitastor.
-+#
-+# @location: Where to store the new image file.  This location cannot
-+#     point to a snapshot.
-+#
-+# @size: Size of the virtual disk in bytes
-+##
-+{ 'struct': 'BlockdevCreateOptionsVitastor',
-+  'data': { 'location':         'BlockdevOptionsVitastor',
-+            'size':             'size' } }
-+
- ##
- # @BlockdevVmdkSubformat:
- #
-@@ -5598,6 +5635,7 @@
-       'ssh':            'BlockdevCreateOptionsSsh',
-       'vdi':            'BlockdevCreateOptionsVdi',
-       'vhdx':           'BlockdevCreateOptionsVhdx',
-+      'vitastor':       'BlockdevCreateOptionsVitastor',
-       'vmdk':           'BlockdevCreateOptionsVmdk',
-       'vpc':            'BlockdevCreateOptionsVpc'
-   } }
-diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh
-index 3d0d132344..65ee8c855e 100644
---- a/scripts/meson-buildoptions.sh
-+++ b/scripts/meson-buildoptions.sh
-@@ -177,6 +177,7 @@ meson_options_help() {
-   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
-   printf "%s\n" '  qpl             Query Processing Library support'
-   printf "%s\n" '  rbd             Ceph block device driver'
-+  printf "%s\n" '  vitastor        Vitastor block device driver'
-   printf "%s\n" '  rdma            Enable RDMA-based migration'
-   printf "%s\n" '  replication     replication support'
-   printf "%s\n" '  rust            Rust support'
-@@ -464,6 +465,8 @@ _meson_option_parse() {
-     --disable-qpl) printf "%s" -Dqpl=disabled ;;
-     --enable-rbd) printf "%s" -Drbd=enabled ;;
-     --disable-rbd) printf "%s" -Drbd=disabled ;;
-+    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
-+    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
-     --enable-rdma) printf "%s" -Drdma=enabled ;;
-     --disable-rdma) printf "%s" -Drdma=disabled ;;
-     --enable-relocatable) printf "%s" -Drelocatable=true ;;

rpm/build-tarball.sh

@@ -21,7 +21,7 @@ rpmbuild -bp fio.spec
 cd $VITASTOR
 VER=$(grep ^Version: rpm/vitastor-$REL.spec | awk '{print $2}')
 rm -rf fio
-ln -s $(ls -d ~/rpmbuild/BUILD/fio*/ | grep -v SPECPARTS) fio
+ln -s ~/rpmbuild/BUILD/fio*/ fio
 sh copy-fio-includes.sh
 rm fio
 mv fio-copy fio

rpm/vitastor-el10.Dockerfile

@@ -1,17 +0,0 @@
-# Build packages for AlmaLinux 10 inside a container
-# cd ..
-# docker pull --platform=linux/amd64/v2 quay.io/almalinuxorg/almalinux:10
-# docker build -t vitastor-buildenv:el10 -f rpm/vitastor-el10.Dockerfile .
-# docker run -i --rm -v ./:/root/vitastor vitastor-buildenv:el10 /root/vitastor/rpm/vitastor-build.sh
-
-FROM quay.io/almalinuxorg/almalinux:10
-
-WORKDIR /root
-
-RUN sed -i 's/enabled=0/enabled=1/' /etc/yum.repos.d/*.repo
-RUN dnf -y install epel-release dnf-plugins-core
-RUN dnf -y install https://vitastor.io/rpms/centos/10/vitastor-release-1.0-1.el10.noarch.rpm
-RUN dnf -y install gcc-c++ gperftools-devel fio nodejs rpm-build jerasure-devel isa-l-devel gf-complete-devel rdma-core-devel cmake libnl3-devel
-RUN dnf download --source fio
-RUN rpm --nomd5 -i fio*.src.rpm
-RUN cd ~/rpmbuild/SPECS && dnf builddep -y --spec fio.spec

rpm/vitastor-el10.spec

@@ -1,198 +0,0 @@
-Name:           vitastor
-Version:        3.0.14
-Release:        1%{?dist}
-Summary:        Vitastor, a fast software-defined clustered block storage
-
-License:        Vitastor Network Public License 1.1
-URL:            https://vitastor.io/
-Source0:        vitastor-3.0.14.el10.tar.gz
-
-BuildRequires:  gperftools-devel
-BuildRequires:  gcc-c++
-BuildRequires:  nodejs >= 10
-BuildRequires:  jerasure-devel
-BuildRequires:  isa-l-devel
-BuildRequires:  gf-complete-devel
-BuildRequires:  rdma-core-devel
-BuildRequires:  cmake
-BuildRequires:  libnl3-devel
-Requires:       vitastor-osd = %{version}-%{release}
-Requires:       vitastor-mon = %{version}-%{release}
-Requires:       vitastor-client = %{version}-%{release}
-Requires:       vitastor-client-devel = %{version}-%{release}
-Requires:       vitastor-fio = %{version}-%{release}
-
-%description
-Vitastor is a small, simple and fast clustered block storage (storage for VM drives),
-architecturally similar to Ceph which means strong consistency, primary-replication,
-symmetric clustering and automatic data distribution over any number of drives of any
-size with configurable redundancy (replication or erasure codes/XOR).
-
-
-%package -n vitastor-osd
-Summary:        Vitastor - OSD
-Requires:       vitastor-client = %{version}-%{release}
-Requires:       util-linux
-Requires:       parted
-
-
-%description -n vitastor-osd
-Vitastor object storage daemon, i.e. server program that stores data.
-
-
-%package -n vitastor-mon
-Summary:        Vitastor - monitor
-Requires:       nodejs >= 10
-Requires:       lpsolve
-
-
-%description -n vitastor-mon
-Vitastor monitor, i.e. server program responsible for watching cluster state and
-scheduling cluster-level operations.
-
-
-%package -n vitastor-client
-Summary:        Vitastor - client
-
-
-%description -n vitastor-client
-Vitastor client library and command-line interface.
-
-
-%package -n vitastor-client-devel
-Summary:        Vitastor - development files
-Group:          Development/Libraries
-Requires:       vitastor-client = %{version}-%{release}
-
-
-%description -n vitastor-client-devel
-Vitastor library headers for development.
-
-
-%package -n vitastor-fio
-Summary:        Vitastor - fio drivers
-Group:          Development/Libraries
-Requires:       vitastor-client = %{version}-%{release}
-Requires:       fio = 3.36-5.el10
-
-
-%description -n vitastor-fio
-Vitastor fio drivers for benchmarking.
-
-
-%package -n vitastor-opennebula
-Summary:        Vitastor for OpenNebula
-Group:          Development/Libraries
-Requires:       vitastor-client
-Requires:       jq
-Requires:       python3-lxml
-Requires:       patch
-Requires:       qemu-kvm-block-vitastor
-
-
-%description -n vitastor-opennebula
-Vitastor storage plugin for OpenNebula.
-
-
-%prep
-%setup -q
-
-
-%build
-%cmake
-%cmake_build
-
-
-%install
-rm -rf $RPM_BUILD_ROOT
-%cmake_install
-cd mon
-npm install --production
-cd ..
-mkdir -p %buildroot/usr/lib/vitastor
-cp -r mon %buildroot/usr/lib/vitastor
-mv %buildroot/usr/lib/vitastor/mon/scripts/make-etcd %buildroot/usr/lib/vitastor/mon/
-mkdir -p %buildroot/lib/systemd/system
-cp mon/scripts/vitastor.target mon/scripts/vitastor-mon.service mon/scripts/vitastor-osd@.service %buildroot/lib/systemd/system
-mkdir -p %buildroot/lib/udev/rules.d
-cp mon/scripts/90-vitastor.rules %buildroot/lib/udev/rules.d
-mkdir -p %buildroot/var/lib/one
-cp -r opennebula/remotes %buildroot/var/lib/one
-cp opennebula/install.sh %buildroot/var/lib/one/remotes/datastore/vitastor/
-mkdir -p %buildroot/etc/
-cp -r opennebula/sudoers.d %buildroot/etc/
-
-
-%files
-%doc GPL-2.0.txt VNPL-1.1.txt README.md README-ru.md
-
-
-%files -n vitastor-osd
-%_bindir/vitastor-osd
-%_bindir/vitastor-disk
-%_bindir/vitastor-dump-journal
-/lib/systemd/system/vitastor-osd@.service
-/lib/systemd/system/vitastor.target
-/lib/udev/rules.d/90-vitastor.rules
-
-
-%pre -n vitastor-osd
-groupadd -r -f vitastor 2>/dev/null ||:
-useradd -r -g vitastor -s /sbin/nologin -c "Vitastor daemons" -M -d /nonexistent vitastor 2>/dev/null ||:
-install -o vitastor -g vitastor -d /var/log/vitastor
-mkdir -p /etc/vitastor
-
-
-%files -n vitastor-mon
-/usr/lib/vitastor/mon
-/lib/systemd/system/vitastor-mon.service
-
-
-%pre -n vitastor-mon
-groupadd -r -f vitastor 2>/dev/null ||:
-useradd -r -g vitastor -s /sbin/nologin -c "Vitastor daemons" -M -d /nonexistent vitastor 2>/dev/null ||:
-mkdir -p /etc/vitastor
-mkdir -p /var/lib/vitastor
-chown vitastor:vitastor /var/lib/vitastor
-
-
-%files -n vitastor-client
-%_bindir/vitastor-nbd
-%_bindir/vitastor-ublk
-%_bindir/vitastor-nfs
-%_bindir/vitastor-cli
-%_bindir/vitastor-rm
-%_bindir/vitastor-kv
-%_bindir/vitastor-kv-stress
-%_bindir/vita
-%_libdir/libvitastor_client.so*
-%_libdir/libvitastor_kv.so*
-
-
-%files -n vitastor-client-devel
-%_includedir/vitastor_c.h
-%_includedir/vitastor_kv.h
-%_libdir/pkgconfig
-
-
-%files -n vitastor-fio
-%_libdir/libfio_vitastor.so
-%_libdir/libfio_vitastor_blk.so
-%_libdir/libfio_vitastor_sec.so
-
-
-%files -n vitastor-opennebula
-/var/lib/one
-/etc/sudoers.d/opennebula-vitastor
-
-
-%triggerin -n vitastor-opennebula -- opennebula
-[ $2 = 0 ] || exit 0
-/var/lib/one/remotes/datastore/vitastor/install.sh
-
-
-# Turn off the brp-python-bytecompile script
-%global __os_install_post %(echo '%{__os_install_post}' | sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g')
-
-
-%changelog

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.14
+Version:        3.0.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.14.el7.tar.gz
+Source0:        vitastor-3.0.2.el7.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  devtoolset-9-gcc-c++

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.14
+Version:        3.0.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.14.el8.tar.gz
+Source0:        vitastor-3.0.2.el8.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-toolset-9-gcc-c++

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.14
+Version:        3.0.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.14.el9.tar.gz
+Source0:        vitastor-3.0.2.el9.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++

src/CMakeLists.txt

@@ -1,8 +1,9 @@
-cmake_minimum_required(VERSION 2.8...3.30)
+cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
 include(GNUInstallDirs)
+include(CTest)
 include(CheckIncludeFile)
 
 find_package(PkgConfig)
@@ -20,7 +21,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 endif()
 set(ENABLE_COVERAGE false CACHE BOOL "Enable code coverage")
 
-add_definitions(-DVITASTOR_VERSION="3.0.14")
+add_definitions(-DVITASTOR_VERSION="3.0.2")
 add_definitions(-D_GNU_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -fvisibility=hidden -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/blockstore/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8...3.30)
+cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 

src/blockstore/blockstore.h

@@ -187,6 +187,7 @@ public:
     // MUST be called only when nobody makes any modifications to the DB for this pool
     virtual void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit) = 0;
     virtual bool reshard_continue(void *reshard_state, uint64_t chunk_limit) = 0;
+    virtual void reshard_abort(void *reshard_state) = 0;
 
     // Event loop
     virtual void loop() = 0;
@@ -228,9 +229,4 @@ public:
     virtual uint64_t get_journal_size() = 0;
 
     virtual uint32_t get_bitmap_granularity() = 0;
-
-    virtual uint64_t get_live_entries() = 0;
-    virtual uint64_t get_live_memory() = 0;
-    virtual uint64_t get_garbage_entries() = 0;
-    virtual uint64_t get_garbage_memory() = 0;
 };

src/blockstore/blockstore_disk.cpp

@@ -94,9 +94,6 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     csum_block_size = parse_size(config["csum_block_size"]);
     discard_on_start = config.find("discard_on_start") != config.end() &&
         (config["discard_on_start"] == "true" || config["discard_on_start"] == "1" || config["discard_on_start"] == "yes");
-    gc_on_start = config.find("gc_on_start") == config.end() ||
-        (config["gc_on_start"] == "true" || config["gc_on_start"] == "1" || config["gc_on_start"] == "yes");
-    skip_double_claim = (config["skip_double_claim"] == "true" || config["skip_double_claim"] == "1" || config["skip_double_claim"] == "yes");
     min_discard_size = parse_size(config["min_discard_size"]);
     if (!min_discard_size)
         min_discard_size = 1024*1024;
@@ -176,7 +173,9 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     }
     if (data_block_size / bitmap_granularity < 8)
     {
-        throw std::runtime_error("Data block size must be at least bitmap_granularity*8");
+        fprintf(stderr, "Warning: block_size (%u) / bitmap_granularity (%u) = %u bits. "
+            "Consider using larger block_size or bitmap_granularity for better performance.\n",
+            data_block_size, bitmap_granularity, data_block_size / bitmap_granularity);
     }
     if (!data_csum_type)
     {
@@ -517,7 +516,7 @@ void blockstore_disk_t::close_all()
 
 // Sadly DISCARD only works through ioctl(), but it seems to always block the device queue,
 // so it's not a big deal that we can only run it synchronously.
-int blockstore_disk_t::trim_data(std::function<bool(uint64_t)> is_used)
+int blockstore_disk_t::trim_data(std::function<bool(uint64_t)> is_free)
 {
     if (mock_mode)
     {
@@ -528,7 +527,7 @@ int blockstore_disk_t::trim_data(std::function<bool(uint64_t)> is_used)
     uint64_t discarded = 0;
     for (; i <= block_count; i++)
     {
-        if (i >= block_count || is_used(i))
+        if (i >= block_count || is_free(i))
         {
             if (i > j && (i-j)*data_block_size >= min_discard_size)
             {
@@ -541,21 +540,17 @@ int blockstore_disk_t::trim_data(std::function<bool(uint64_t)> is_used)
                     if (range[0] % discard_granularity)
                         range[0] = range[0] + discard_granularity - (range[0] % discard_granularity);
                     if (range[0] >= range[1])
-                        range[1] = 0;
+                        continue;
-                    else
+                    range[1] -= range[0];
-                        range[1] -= range[0];
                 }
-                if (range[1] > 0)
+                r = ioctl(data_fd, BLKDISCARD, &range);
+                if (r != 0)
                 {
-                    r = ioctl(data_fd, BLKDISCARD, &range);
+                    fprintf(stderr, "Failed to execute BLKDISCARD %ju+%ju on %s: %s (code %d)\n",
-                    if (r != 0)
+                        range[0], range[1], data_device.c_str(), strerror(-r), r);
-                    {
+                    return -errno;
-                        fprintf(stderr, "Failed to execute BLKDISCARD %ju+%ju on %s: %s (code %d)\n",
-                            range[0], range[1], data_device.c_str(), strerror(-r), r);
-                        return -errno;
-                    }
-                    discarded += range[1];
                 }
+                discarded += range[1];
             }
             j = i+1;
         }

src/blockstore/blockstore_disk.h

@@ -26,14 +26,14 @@ class allocator_t;
 struct blockstore_disk_t
 {
     std::string data_device, meta_device, journal_device;
-    uint64_t data_block_size;
+    uint32_t data_block_size;
     uint64_t cfg_journal_size, cfg_data_size;
     // Required write alignment and journal/metadata/data areas' location alignment
     uint32_t disk_alignment = 4096;
     // Journal block size - minimum_io_size of the journal device is the best choice
-    uint64_t journal_block_size = 4096;
+    uint32_t journal_block_size = 4096;
     // Metadata block size - minimum_io_size of the metadata device is the best choice
-    uint64_t meta_block_size = 4096;
+    uint32_t meta_block_size = 4096;
     // Atomic write size of the data block device
     uint32_t atomic_write_size = 4096;
     // Whether we should set RWF_ATOMIC on atomic writes
@@ -57,10 +57,6 @@ struct blockstore_disk_t
     bool inmemory_journal = true;
     // Data discard granularity and minimum size (for the sake of performance)
     bool discard_on_start = false;
-    // GC on start (new store)
-    bool gc_on_start = true;
-    // Skip double claim conflicts on start (new store, temporary until the bug is found)
-    bool skip_double_claim = false;
     uint64_t min_discard_size = 1024*1024;
     uint64_t discard_granularity = 0;
 
@@ -83,7 +79,7 @@ struct blockstore_disk_t
     void calc_lengths(bool skip_meta_check = false);
     void check_lengths();
     void close_all();
-    int trim_data(std::function<bool(uint64_t)> is_used);
+    int trim_data(std::function<bool(uint64_t)> is_free);
 
     inline uint64_t dirty_dyn_size(uint64_t offset, uint64_t len)
     {

src/blockstore/blockstore_flush.cpp

@@ -174,18 +174,14 @@ bool journal_flusher_co::loop()
     else if (wait_state == 19) goto resume_19;
     else if (wait_state == 20) goto resume_20;
     else if (wait_state == 21) goto resume_21;
-    else if (wait_state == 22) goto resume_22;
-    else if (wait_state == 23) goto resume_23;
-    else if (wait_state == 24) goto resume_24;
-    else if (wait_state == 25) goto resume_25;
 resume_0:
     wait_state = 0;
     wait_count = 0;
     cur_oid = {};
     res = bs->heap->get_next_compact(cur_oid);
+    // Advance fsynced_lsn every <journal_trim_interval> intent writes
     if ((bs->intent_write_counter >= bs->journal_trim_interval) && co_id == 0)
     {
-        // Advance fsynced_lsn every <journal_trim_interval> intent writes
         bs->intent_write_counter = 0;
 resume_17:
 resume_18:
@@ -200,7 +196,6 @@ resume_21:
     if (res == ENOENT && flusher->force_start > 0 && co_id == 0 &&
         (!bs->dsk.disable_journal_fsync || !bs->dsk.disable_meta_fsync || !bs->dsk.disable_data_fsync))
     {
-        // When under pressure, do an additional fsync to force entries to be marked compactable
         flusher->active_flushers++;
 resume_14:
 resume_15:
@@ -264,9 +259,11 @@ resume_1:
         if (wr->type() == BS_HEAP_SMALL_WRITE ||
             wr->type() == BS_HEAP_INTENT_WRITE && bs->dsk.csum_block_size > bs->dsk.bitmap_granularity)
         {
-            bs->prepare_read(read_vec, cur_obj, wr, 0, bs->dsk.data_block_size,
+            auto res = bs->prepare_read(read_vec, cur_obj, wr, 0, bs->dsk.data_block_size,
                 wr->type() == BS_HEAP_INTENT_WRITE && bs->dsk.csum_block_size > bs->dsk.bitmap_granularity && !bs->perfect_csum_update
                 ? COPY_BUF_SKIP_CSUM : 0);
+            if (res > 0)
+                copy_count++;
         }
     });
     if (!compact_info.compact_lsn)
@@ -276,53 +273,30 @@ resume_1:
         bs->heap->unlock_entry(cur_oid);
         goto resume_0;
     }
+    mem_or(new_bmp, compact_info.clean_wr->get_int_bitmap(bs->heap), bs->dsk.clean_entry_bitmap_size);
+    if (!bitmap_copied)
+    {
+        memcpy(new_ext_bmp, compact_info.clean_wr->get_ext_bitmap(bs->heap), bs->dsk.clean_entry_bitmap_size);
+        bitmap_copied = true;
+    }
+    if (bs->dsk.csum_block_size && bs->dsk.csum_block_size <= bs->dsk.bitmap_granularity)
+    {
+        memcpy(new_csums, compact_info.clean_wr->get_checksums(bs->heap), bs->dsk.data_block_size/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF));
+        for (size_t i = csum_copy.size(); i > 0; i--)
+        {
+            auto wr = csum_copy[i-1];
+            memcpy(new_csums + wr->small().offset/bs->dsk.csum_block_size*(bs->dsk.data_csum_type & 0xFF),
+                wr->get_checksums(bs->heap), wr->small().len/bs->dsk.csum_block_size*(bs->dsk.data_csum_type & 0xFF));
+        }
+        csum_copy.clear();
+    }
+    clean_loc = compact_info.clean_wr->big_location(bs->heap);
     flusher->active_flushers++;
-    for (i = 0; i < read_vec.size(); i++)
+    if (bs->log_level > 10)
     {
-        if ((read_vec[i].copy_flags & COPY_BUF_JOURNAL) &&
+        printf("Compacting %jx:%jx v%ju..v%ju / l%ju..l%ju (%d writes)\n", cur_oid.inode, cur_oid.stripe,
-            !(read_vec[i].copy_flags & COPY_BUF_COALESCED))
+            compact_info.clean_wr->version, compact_info.compact_version,
-        {
+            compact_info.clean_wr->lsn, compact_info.compact_lsn, copy_count);
-            copy_count++;
-        }
-    }
-    if (copy_count > 0 && !bs->dsk.disable_data_fsync)
-    {
-        init_fsync_data();
-    }
-    if (compact_info.do_delete)
-    {
-        if (bs->log_level > 10)
-        {
-            printf("Compacting %jx:%jx up to l%ju (delete)\n", cur_oid.inode, cur_oid.stripe, compact_info.compact_lsn);
-        }
-        clean_loc = UINT64_MAX;
-    }
-    else
-    {
-        if (bs->log_level > 10)
-        {
-            printf("Compacting %jx:%jx v%ju..v%ju / l%ju..l%ju (%d writes)\n", cur_oid.inode, cur_oid.stripe,
-                compact_info.clean_wr->version, compact_info.compact_version,
-                compact_info.clean_wr->lsn, compact_info.compact_lsn, copy_count);
-        }
-        mem_or(new_bmp, compact_info.clean_wr->get_int_bitmap(bs->heap), bs->dsk.clean_entry_bitmap_size);
-        if (!bitmap_copied)
-        {
-            memcpy(new_ext_bmp, compact_info.clean_wr->get_ext_bitmap(bs->heap), bs->dsk.clean_entry_bitmap_size);
-            bitmap_copied = true;
-        }
-        if (bs->dsk.csum_block_size && bs->dsk.csum_block_size <= bs->dsk.bitmap_granularity)
-        {
-            memcpy(new_csums, compact_info.clean_wr->get_checksums(bs->heap), bs->dsk.data_block_size/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF));
-            for (size_t i = csum_copy.size(); i > 0; i--)
-            {
-                auto wr = csum_copy[i-1];
-                memcpy(new_csums + wr->small().offset/bs->dsk.csum_block_size*(bs->dsk.data_csum_type & 0xFF),
-                    wr->get_checksums(bs->heap), wr->small().len/bs->dsk.csum_block_size*(bs->dsk.data_csum_type & 0xFF));
-            }
-            csum_copy.clear();
-        }
-        clean_loc = compact_info.clean_wr->big_location(bs->heap);
     }
     overwrite_start = overwrite_end = 0;
     if (read_vec.size() > 0)
@@ -362,13 +336,6 @@ resume_3:
     if (res == ENOENT || res == EDOM)
     {
         // Abort compaction
-abort_compact:
-        if (copy_count > 0 && !bs->dsk.disable_data_fsync)
-        {
-            cur_sync->member_count--;
-            if (cur_sync->member_count > 0)
-                bs->ringloop->wakeup();
-        }
         flusher->flushing.erase(cur_oid);
         bs->heap->unlock_entry(cur_oid);
         flusher->active_flushers--;
@@ -382,7 +349,10 @@ resume_4:
         if (res == ENOENT)
         {
             // Abort compaction
-            goto abort_compact;
+            flusher->flushing.erase(cur_oid);
+            bs->heap->unlock_entry(cur_oid);
+            flusher->active_flushers--;
+            goto resume_0;
         }
         if (res == EAGAIN)
         {
@@ -411,14 +381,14 @@ resume_9:
     for (i = 0; i < read_vec.size(); i++)
     {
         if ((read_vec[i].copy_flags & COPY_BUF_JOURNAL) &&
-            !(read_vec[i].copy_flags & COPY_BUF_COALESCED))
+            !(read_vec[i].copy_flags & COPY_BUF_COALESCED) ||
+            (read_vec[i].copy_flags & COPY_BUF_PADDED)) // FIXME Shit, simplify these flags
         {
             assert(read_vec[i].buf);
             await_sqe(10);
             data->iov = (struct iovec){ read_vec[i].buf + (read_vec[i].copy_flags & COPY_BUF_PADDED
                 ? read_vec[i].offset - read_vec[i].disk_offset : 0), (size_t)read_vec[i].len };
             data->callback = simple_callback_w;
-            assert(clean_loc + read_vec[i].offset + data->iov.iov_len <= bs->dsk.block_count*bs->dsk.data_block_size);
             io_uring_prep_writev(sqe, bs->dsk.data_fd, &data->iov, 1, bs->dsk.data_offset + clean_loc + read_vec[i].offset);
             wait_count++;
         }
@@ -429,17 +399,6 @@ resume_11:
         wait_state = 11;
         return false;
     }
-    if (copy_count > 0 && !bs->dsk.disable_data_fsync)
-    {
-resume_22:
-resume_23:
-resume_24:
-resume_25:
-        if (!fsync_data(22))
-        {
-            return false;
-        }
-    }
     // Lock is only needed to prevent freeing the big_write because we overwrite it...
     bs->heap->unlock_entry(cur_oid);
     // Mark the object compacted, but don't free and remove small_writes
@@ -449,14 +408,12 @@ resume_25:
     if (!cur_obj)
     {
         // Abort compaction
-        flusher->active_flushers--;
         flusher->flushing.erase(cur_oid);
         goto resume_0;
     }
     if (!calc_block_checksums())
     {
         // Abort compaction
-        flusher->active_flushers--;
         flusher->flushing.erase(cur_oid);
         goto resume_0;
     }
@@ -465,7 +422,6 @@ resume_25:
     if (res == EBUSY)
     {
         // Abort compaction, object is already overwritten by something else
-        flusher->active_flushers--;
         flusher->flushing.erase(cur_oid);
         goto resume_0;
     }
@@ -630,13 +586,13 @@ int journal_flusher_co::check_and_punch_checksums()
             bs->heap->calc_block_checksums((uint32_t*)(new_csums+csum_off), vec.buf, punch_bmp, vec.offset, vec.offset+vec.len, true, NULL);
         }
     }
-    // Modified, we should punch_holes and then write the block to disk
+    // Modified, we should add_punch_holes and then write the block to disk
     return EBUSY;
 }
 
 bool journal_flusher_co::calc_block_checksums()
 {
-    if (bs->dsk.csum_block_size <= bs->dsk.bitmap_granularity || compact_info.do_delete)
+    if (bs->dsk.csum_block_size <= bs->dsk.bitmap_granularity)
     {
         return true;
     }
@@ -743,67 +699,6 @@ resume_1:
     return true;
 }
 
-void journal_flusher_co::init_fsync_data()
-{
-    cur_sync = flusher->data_syncs.begin();
-    if (cur_sync == flusher->data_syncs.end() || cur_sync->ready_count > 0)
-    {
-        cur_sync = flusher->data_syncs.emplace(cur_sync);
-    }
-    cur_sync->member_count++;
-}
-
-bool journal_flusher_co::fsync_data(int wait_base)
-{
-    if (wait_state == wait_base)
-        goto resume_0;
-    else if (wait_state == wait_base+1)
-        goto resume_1;
-    else if (wait_state == wait_base+2)
-        goto resume_2;
-    else if (wait_state == wait_base+3)
-        goto resume_3;
-    cur_sync->ready_count++;
-resume_0:
-    if (cur_sync->ready_count < cur_sync->member_count)
-    {
-        wait_state = wait_base;
-        return false;
-    }
-    if (!cur_sync->sent)
-    {
-        // Sync batch is ready. Do it.
-        await_sqe(1);
-        data->iov = { 0 };
-        data->callback = simple_callback_w;
-        io_uring_prep_fsync(sqe, bs->dsk.data_fd, IORING_FSYNC_DATASYNC);
-        cur_sync->sent = true;
-        wait_count++;
-resume_2:
-        if (wait_count > 0)
-        {
-            wait_state = wait_base+2;
-            return false;
-        }
-        cur_sync->done = true;
-        // Wake up other flushers
-        bs->ringloop->wakeup();
-    }
-resume_3:
-    if (!cur_sync->done)
-    {
-        wait_state = wait_base+3;
-        return false;
-    }
-    cur_sync->done_count++;
-    if (cur_sync->done_count >= cur_sync->member_count)
-    {
-        flusher->data_syncs.erase(cur_sync);
-        cur_sync = flusher->data_syncs.end();
-    }
-    return true;
-}
-
 bool journal_flusher_co::fsync_meta(int wait_base)
 {
     if (wait_state == wait_base)        goto resume_0;

src/blockstore/blockstore_flush.h

@@ -25,15 +25,6 @@ struct flusher_meta_write_t
     std::map<uint64_t, meta_sector_t>::iterator it;
 };
 
-struct flusher_data_sync_t
-{
-    int member_count = 0;
-    int ready_count = 0;
-    int done_count = 0;
-    bool sent = false;
-    bool done = false;
-};
-
 class journal_flusher_t;
 
 // Journal flusher coroutine
@@ -67,7 +58,6 @@ class journal_flusher_co
     int i, res;
     bool read_to_fill_incomplete;
     int copy_count;
-    std::list<flusher_data_sync_t>::iterator cur_sync;
 
     friend class journal_flusher_t;
 
@@ -78,8 +68,6 @@ class journal_flusher_co
     bool calc_block_checksums();
     bool write_meta_block(int wait_base);
     bool read_buffered(int wait_base);
-    void init_fsync_data();
-    bool fsync_data(int wait_base);
     bool fsync_meta(int wait_base);
     bool fsync_buffer(int wait_base);
     bool trim_lsn(int wait_base);
@@ -100,7 +88,6 @@ class journal_flusher_t
 
     robin_hood::unordered_flat_set<object_id> flushing;
     int active_flushers = 0;
-    std::list<flusher_data_sync_t> data_syncs;
     int wanting_meta_fsync = 0;
     bool fsyncing_meta = false;
     int syncing_buffer = 0;

src/blockstore/blockstore_heap.h

@@ -57,11 +57,11 @@ struct __attribute__((__packed__)) heap_entry_t
     inline heap_small_write_t& small() { return *(heap_small_write_t*)this; }
     inline heap_big_write_t& big() { return *(heap_big_write_t*)this; }
     inline heap_big_intent_t& big_intent() { return *(heap_big_intent_t*)this; }
-    bool is_garbage() const;
+    bool is_garbage();
     void set_garbage();
-    bool is_overwrite() const;
+    bool is_overwrite();
-    bool is_compactable() const;
+    bool is_compactable();
-    bool is_before(const heap_entry_t *other) const;
+    bool is_before(heap_entry_t *other);
     uint32_t get_size(blockstore_heap_t *heap);
     uint8_t *get_ext_bitmap(blockstore_heap_t *heap);
     uint8_t *get_int_bitmap(blockstore_heap_t *heap);
@@ -117,13 +117,10 @@ struct heap_object_mvcc_t
 
 struct heap_block_info_t
 {
-    struct __attribute__((__packed__))
+    uint32_t used_space = 0;
-    {
-        uint32_t used_space = 0;
-        uint32_t garbage_space = 0;
-    };
     uint64_t mod_lsn = 0, mod_lsn_to = 0; // only 1 block write of LSN sequence is allowed at a moment
-    bool is_writing = false;
+    bool is_writing: 1;
+    bool has_garbage: 1;
     std::vector<heap_list_item_t*> entries;
 };
 
@@ -158,16 +155,6 @@ struct heap_li_equal
     }
 };
 
-struct heap_recheck_state_t
-{
-    heap_entry_t *obj = NULL;
-    heap_entry_t *next_wr = NULL;
-    size_t total_reads = 0;
-    size_t sent_reads = 0;
-    size_t checked_reads = 0;
-    heap_entry_t *bad_wr = NULL;
-};
-
 using i64hash_t = robin_hood::hash<uint64_t>;
 using heap_inode_map_t = robin_hood::unordered_flat_set<heap_list_item_t*, heap_li_hash, heap_li_equal, 88>;
 using heap_block_index_t = robin_hood::unordered_flat_map<uint64_t,
@@ -197,11 +184,6 @@ class blockstore_heap_t
     uint64_t buffer_area_used_space = 0;
     uint64_t data_used_space = 0;
 
-    uint64_t live_entries = 0;
-    uint64_t live_memory = 0;
-    uint64_t garbage_entries = 0;
-    uint64_t garbage_memory = 0;
-
     uint64_t next_lsn = 0;
     uint32_t last_allocated_block = UINT32_MAX;
     heap_mvcc_map_t object_mvcc;
@@ -218,12 +200,8 @@ class blockstore_heap_t
 
     bool marked_used_blocks = false;
     bool recheck_queue_filled = false;
-    std::vector<heap_list_item_t*> postponed_items;
-    std::vector<heap_list_item_t*> init_erase_items;
     std::set<uint32_t> recheck_modified_blocks;
     std::deque<heap_entry_t*> recheck_queue;
-    std::map<heap_entry_t*, heap_recheck_state_t> recheck_states;
-    size_t recheck_pending_reads = 0;
     int recheck_in_progress = 0;
     bool in_recheck = false;
     std::function<void(bool is_data, uint64_t offset, uint64_t len, uint8_t* buf, std::function<void()>)> recheck_cb;
@@ -232,22 +210,13 @@ class blockstore_heap_t
     uint64_t get_pg_id(inode_t inode, uint64_t stripe);
     bool validate_object(heap_entry_t *obj);
     void fill_recheck_queue();
-    void recheck_drop_entries(heap_entry_t *obj, heap_entry_t *bad_wr);
-    void recheck_start_reads(heap_recheck_state_t *st);
     int mark_used_blocks();
-    void init_free_bad_entry(heap_entry_t *wr);
-    void init_erase_bad_entry(heap_list_item_t *li);
-    bool init_erase_double_claim(heap_list_item_t *prev_li, heap_list_item_t *cur_li);
-    void recheck_full_gc();
     void recheck_buffer(heap_entry_t *cwr, uint8_t *buf);
     void defragment_block(uint32_t block_num);
     void reshard_add(heap_reshard_state_t *st, heap_list_item_t *li);
 
-    void gc_block(heap_block_info_t & inf);
     int allocate_entry(uint32_t entry_size, uint32_t *block_num, bool allow_last_free);
-    void insert_list_items(heap_list_item_t** v, size_t count, bool postpone);
+    void insert_list_item(heap_list_item_t *li);
-    void remove_list_item(heap_list_item_t *li);
-    void unlink_list_item(heap_list_item_t *li);
     int add_entry(uint32_t wr_size, uint32_t *modified_block, bool allow_last_free,
         bool explicit_complete, std::function<void(heap_entry_t *wr)> fill_entry);
     int add_simple(heap_entry_t *obj, uint64_t version, uint32_t *modified_block, uint32_t entry_type);
@@ -268,29 +237,31 @@ public:
         std::function<void(uint32_t, uint32_t, uint8_t*)> handle_block);
     int load_blocks(uint64_t disk_offset, uint64_t size, uint8_t *buf,
         bool allow_corrupted, uint64_t &entries_loaded);
-    // finish loading - should be called after load_blocks
+    // finish loading
-    void finish_load();
+    int finish_load(bool allow_corrupted = false);
     // get blocks which are modified during loading and should be written to the disk
     // before finishing initialization if not R/O
     std::vector<uint32_t> get_recheck_modified_blocks();
     // recheck small write data after reading the database from disk
     bool recheck_small_writes(std::function<void(bool is_data, uint64_t offset, uint64_t len, uint8_t* buf, std::function<void()>)> read_buffer, int queue_depth);
-    int finish_recheck();
     // reshard database according to the pool's PG count
     void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
     bool reshard_continue(void* reshard_state, uint64_t chunk_limit);
     bool reshard_check(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size);
+    void reshard_abort(void* reshard_state);
     void set_no_inode_stats(const std::vector<uint64_t> & pool_ids);
     void recalc_inode_space_stats(uint64_t pool_id, bool per_inode);
     // read an object entry and lock it against removal
     // in the future, may become asynchronous
     heap_entry_t *lock_and_read_entry(object_id oid);
+    // re-read a locked object entry with the given lsn (pointer may be invalidated)
+    heap_entry_t *read_locked_entry(object_id oid, uint64_t lsn);
     // read an object entry without locking it
     heap_entry_t *read_entry(object_id oid);
     // unlock an entry
     bool unlock_entry(object_id oid);
     // set or verify checksums in a write request
-    bool calc_checksums(heap_entry_t *wr, uint8_t *data, bool set, uint32_t offset = UINT32_MAX, uint32_t len = UINT32_MAX);
+    bool calc_checksums(heap_entry_t *wr, uint8_t *data, bool set, uint32_t offset = 0, uint32_t len = 0);
     // set or verify raw block checksums
     bool calc_block_checksums(uint32_t *block_csums, uint8_t *data, uint8_t *bitmap, uint32_t start, uint32_t end,
         bool set, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);
@@ -362,7 +333,7 @@ public:
 
     // get metadata block data buffer and used space
     void get_meta_block(uint32_t block_num, uint8_t *buffer);
-    void fill_block_empty_space(uint8_t *buffer, uint64_t pos);
+    void fill_block_empty_space(uint8_t *buffer, uint32_t pos);
     uint32_t get_meta_block_used_space(uint32_t block_num);
 
     // get space usage statistics
@@ -374,10 +345,6 @@ public:
     uint32_t get_compact_queue_size();
     uint32_t get_to_compact_count();
     uint64_t get_compacted_count();
-    uint64_t get_live_entries();
-    uint64_t get_live_memory();
-    uint64_t get_garbage_entries();
-    uint64_t get_garbage_memory();
 
     uint64_t entry_pos(uint32_t block_num, uint32_t offset);
     heap_entry_t *entry_from_pos(uint64_t entry_pos, bool allow_unallocated = false);

src/blockstore/blockstore_impl.cpp

@@ -101,7 +101,6 @@ void blockstore_impl_t::loop()
         unsigned initial_ring_space = ringloop->space_left();
         int op_idx = 0, new_idx = 0;
         bool has_unfinished_writes = false;
-        bool has_unfinished_sync = false;
         for (; op_idx < submit_queue.size(); op_idx++, new_idx++)
         {
             auto op = submit_queue[op_idx];
@@ -139,13 +138,7 @@ void blockstore_impl_t::loop()
             else if (op->opcode == BS_OP_SYNC)
             {
                 // syncs only completed writes, so doesn't have to be blocked by anything
-                if (!has_unfinished_sync)
+                wr_st = continue_sync(op);
-                {
-                    wr_st = continue_sync(op);
-                    has_unfinished_sync = (wr_st != 2);
-                }
-                else
-                    wr_st = 0;
             }
             else if (op->opcode == BS_OP_STABLE || op->opcode == BS_OP_ROLLBACK)
             {
@@ -161,7 +154,9 @@ void blockstore_impl_t::loop()
                     wr_st = 2;
                 }
                 else
+                {
                     wr_st = 0;
+                }
             }
             if (wr_st == 2)
             {
@@ -198,12 +193,12 @@ void blockstore_impl_t::loop()
             heap->start_block_write(block_num);
             mb.sent = true;
         }
-        pending_modified_blocks.clear();
         int ret = ringloop->submit();
         if (ret < 0)
         {
             throw std::runtime_error(std::string("io_uring_submit: ") + strerror(-ret));
         }
+        pending_modified_blocks.clear();
         if ((initial_ring_space - ringloop->space_left()) > 0)
         {
             live = true;
@@ -411,3 +406,8 @@ bool blockstore_impl_t::reshard_continue(void *reshard_state, uint64_t chunk_lim
 {
     return heap->reshard_continue(reshard_state, chunk_limit);
 }
+
+void blockstore_impl_t::reshard_abort(void *reshard_state)
+{
+    return heap->reshard_abort(reshard_state);
+}

src/blockstore/blockstore_impl.h

@@ -78,7 +78,6 @@ public:
     // Suitable only for server SSDs with capacitors, requires disabled data and journal fsyncs
     int immediate_commit = IMMEDIATE_NONE;
     bool inmemory_meta = false;
-    bool skip_corrupted_meta_entries = false;
     uint32_t meta_write_recheck_parallelism = 0;
     // Maximum and minimum flusher count
     unsigned max_flusher_count = 0, min_flusher_count = 0;
@@ -117,12 +116,9 @@ public:
 
     journal_flusher_t *flusher;
     int write_iodepth = 0;
+    int inflight_big = 0;
     int intent_write_counter = 0;
-    uint64_t data_fsync_next = 0;
+    bool fsyncing_data = false;
-    uint64_t data_fsync_cur = 0;
-    uint64_t data_fsync_sent = 0;
-    uint64_t data_fsync_done = 0;
-    std::deque<bool> data_fsyncs;
 
     bool live = false, queue_stall = false;
     ring_loop_i *ringloop = NULL;
@@ -195,6 +191,7 @@ public:
 
     void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
     bool reshard_continue(void *reshard_state, uint64_t chunk_limit);
+    void reshard_abort(void *reshard_state);
 
     // Event loop
     void loop();
@@ -232,9 +229,4 @@ public:
     uint64_t get_free_block_count();
     inline uint32_t get_bitmap_granularity() { return dsk.bitmap_granularity; }
     inline uint64_t get_journal_size() { return dsk.journal_len; }
-
-    inline uint64_t get_live_entries() { return heap->get_live_entries(); }
-    inline uint64_t get_live_memory() { return heap->get_live_memory(); }
-    inline uint64_t get_garbage_entries() { return heap->get_garbage_entries(); }
-    inline uint64_t get_garbage_memory() { return heap->get_garbage_memory(); }
 };

src/blockstore/blockstore_init.cpp

@@ -10,6 +10,7 @@
 #define INIT_META_EMPTY 0
 #define INIT_META_READING 1
 #define INIT_META_READ_DONE 2
+#define INIT_META_WRITING 3
 
 #define GET_SQE() \
     sqe = bs->get_sqe();\
@@ -22,15 +23,14 @@ blockstore_init_meta::blockstore_init_meta(blockstore_impl_t *bs)
     this->bs = bs;
 }
 
-void blockstore_init_meta::handle_event(ring_data_t *data, int buf_num, const char *op)
+void blockstore_init_meta::handle_event(ring_data_t *data, int buf_num)
 {
-    if (data->res != data->iov.iov_len)
+    if (data->res < 0)
     {
-        throw std::runtime_error(strprintf(
+        throw std::runtime_error(
-            "%s failed at offset %ju: got %s (code %d), but expected %zu",
+            std::string("read metadata failed at offset ") + std::to_string(buf_num >= 0 ? bufs[buf_num].offset : last_read_offset) +
-            op, (buf_num >= 0 ? bufs[buf_num].offset : last_read_offset), strerror(-data->res),
+            std::string(": ") + strerror(-data->res)
-            data->res, data->iov.iov_len
+        );
-        ));
     }
     if (buf_num >= 0)
     {
@@ -60,7 +60,7 @@ int blockstore_init_meta::loop()
     GET_SQE();
     last_read_offset = 0;
     data->iov = { bs->meta_superblock, (size_t)bs->dsk.meta_block_size };
-    data->callback = [this](ring_data_t *data) { handle_event(data, -1, "read metadata header"); };
+    data->callback = [this](ring_data_t *data) { handle_event(data, -1); };
     io_uring_prep_readv(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset);
     bs->ringloop->submit();
     submitted++;
@@ -72,19 +72,25 @@ resume_1:
     }
     if (is_zero((uint64_t*)bs->meta_superblock, bs->dsk.meta_block_size))
     {
-        assert(bs->dsk.meta_format == BLOCKSTORE_META_FORMAT_HEAP);
+        {
-        blockstore_meta_header_v3_t *hdr = (blockstore_meta_header_v3_t *)bs->meta_superblock;
+            blockstore_meta_header_v3_t *hdr = (blockstore_meta_header_v3_t *)bs->meta_superblock;
-        hdr->zero = 0;
+            hdr->zero = 0;
-        hdr->magic = BLOCKSTORE_META_MAGIC_V1;
+            hdr->magic = BLOCKSTORE_META_MAGIC_V1;
-        hdr->version = bs->dsk.meta_format;
+            hdr->version = bs->dsk.meta_format;
-        hdr->meta_block_size = bs->dsk.meta_block_size;
+            hdr->meta_block_size = bs->dsk.meta_block_size;
-        hdr->data_block_size = bs->dsk.data_block_size;
+            hdr->data_block_size = bs->dsk.data_block_size;
-        hdr->bitmap_granularity = bs->dsk.bitmap_granularity;
+            hdr->bitmap_granularity = bs->dsk.bitmap_granularity;
-        hdr->completed_lsn = 0;
+            if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_V2)
-        hdr->data_csum_type = bs->dsk.data_csum_type;
+            {
-        hdr->csum_block_size = bs->dsk.csum_block_size;
+                hdr->data_csum_type = bs->dsk.data_csum_type;
-        hdr->meta_area_size = bs->dsk.meta_area_size;
+                hdr->csum_block_size = bs->dsk.csum_block_size;
-        hdr->set_crc32c();
+            }
+            if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_HEAP)
+            {
+                hdr->meta_area_size = bs->dsk.meta_area_size;
+            }
+            hdr->set_crc32c();
+        }
         if (bs->readonly)
         {
             printf("Skipping metadata initialization because blockstore is readonly\n");
@@ -92,8 +98,21 @@ resume_1:
         else
         {
             printf("Initializing metadata area\n");
+            GET_SQE();
+            last_read_offset = 0;
+            data->iov = (struct iovec){ bs->meta_superblock, (size_t)bs->dsk.meta_block_size };
+            data->callback = [this](ring_data_t *data) { handle_event(data, -1); };
+            io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset);
+            bs->ringloop->submit();
+            submitted++;
+        resume_2:
+            if (submitted > 0)
+            {
+                wait_state = 2;
+                return 1;
+            }
+            zero_on_init = true;
         }
-        zero_on_init = true;
     }
     else
     {
@@ -126,7 +145,7 @@ resume_1:
             printf(
                 "Configuration stored in metadata superblock"
                 " (meta_block_size=%u, data_block_size=%u, bitmap_granularity=%u, data_csum_type=%u, csum_block_size=%u, meta_area_size=%ju)"
-                " differs from OSD configuration (%ju/%ju/%u, %u/%u, %ju).\n",
+                " differs from OSD configuration (%u/%u/%u, %u/%u, %ju).\n",
                 hdr->meta_block_size, hdr->data_block_size, hdr->bitmap_granularity,
                 hdr->data_csum_type, hdr->csum_block_size, hdr->meta_area_size,
                 bs->dsk.meta_block_size, bs->dsk.data_block_size, bs->dsk.bitmap_granularity,
@@ -134,17 +153,9 @@ resume_1:
             );
             exit(1);
         }
-        uint32_t csum = hdr->header_csum;
-        hdr->header_csum = 0;
-        if (crc32c(0, hdr, sizeof(*hdr)) != csum)
-        {
-            printf("Metadata header is corrupt (checksum mismatch).\n");
-            exit(1);
-        }
-        hdr->header_csum = csum;
     }
     bs->heap->start_load(((blockstore_meta_header_v3_t *)bs->meta_superblock)->completed_lsn);
-    if (bs->dsk.inmemory_journal && !zero_on_init)
+    if (bs->dsk.inmemory_journal)
     {
         // Read buffer area
         printf("Reading buffered data\n");
@@ -156,7 +167,7 @@ resume_1:
                 bs->buffer_area + md_offset,
                 (size_t)(bs->dsk.journal_len - md_offset < bs->metadata_buf_size ? bs->dsk.journal_len - md_offset : bs->metadata_buf_size),
             };
-            data->callback = [this](ring_data_t *data) { handle_event(data, -1, "read buffer area"); };
+            data->callback = [this](ring_data_t *data) { handle_event(data, -1); };
             io_uring_prep_readv(sqe, bs->dsk.journal_fd, &data->iov, 1, bs->dsk.journal_offset + md_offset);
             md_offset += data->iov.iov_len;
             submitted++;
@@ -175,7 +186,7 @@ resume_3:
     next_offset = md_offset;
     // Read the rest of the metadata
 resume_4:
-    if (next_offset < bs->dsk.meta_area_size && submitted == 0 && (!zero_on_init || !bs->readonly))
+    if (next_offset < bs->dsk.meta_area_size && submitted == 0)
     {
         // Submit one read
         for (int i = 0; i < 2; i++)
@@ -192,15 +203,12 @@ resume_4:
                 GET_SQE();
                 assert(bufs[i].size <= 0x7fffffff);
                 data->iov = { bufs[i].buf, (size_t)bufs[i].size };
+                data->callback = [this, i](ring_data_t *data) { handle_event(data, i); };
                 if (!zero_on_init)
-                {
-                    data->callback = [this, i](ring_data_t *data) { handle_event(data, i, "read metadata"); };
                     io_uring_prep_readv(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bufs[i].offset);
-                }
                 else
                 {
                     // Fill metadata with empty block pattern
-                    data->callback = [this, i](ring_data_t *data) { handle_event(data, i, "clear metadata"); };
                     memset(bufs[i].buf, 0, bufs[i].size);
                     for (uint64_t o = 0; o < bufs[i].size; o += bs->dsk.meta_block_size)
                         bs->heap->fill_block_empty_space(bufs[i].buf + o, 0);
@@ -216,14 +224,11 @@ resume_4:
         if (bufs[i].state == INIT_META_READ_DONE)
         {
             // Handle result
-            if (!zero_on_init)
+            uint64_t loaded = 0;
-            {
+            int r = bs->heap->load_blocks(bufs[i].offset-bs->dsk.meta_block_size, bufs[i].size, bufs[i].buf, false, loaded);
-                uint64_t loaded = 0;
+            if (r != 0)
-                int r = bs->heap->load_blocks(bufs[i].offset-bs->dsk.meta_block_size, bufs[i].size, bufs[i].buf, bs->skip_corrupted_meta_entries, loaded);
+                exit(1);
-                if (r != 0)
+            entries_loaded += loaded;
-                    exit(1);
-                entries_loaded += loaded;
-            }
             bufs[i].state = 0;
             bs->ringloop->wakeup();
         }
@@ -233,9 +238,24 @@ resume_4:
         wait_state = 4;
         return 1;
     }
-    // metadata read/clear finished
+    // metadata read finished
-    bs->heap->finish_load();
+    printf("Metadata entries loaded: %ju, used blocks: %ju / %ju\n", entries_loaded, bs->heap->get_data_used_space() / bs->dsk.data_block_size, bs->dsk.block_count);
-    printf("Metadata entries loaded: %ju, rechecking unfinished writes and garbage entries\n", entries_loaded);
+    if (zero_on_init && !bs->dsk.disable_meta_fsync)
+    {
+        GET_SQE();
+        io_uring_prep_fsync(sqe, bs->dsk.meta_fd, IORING_FSYNC_DATASYNC);
+        last_read_offset = 0;
+        data->iov = { 0 };
+        data->callback = [this](ring_data_t *data) { handle_event(data, -1); };
+        submitted++;
+        bs->ringloop->submit();
+    resume_5:
+        if (submitted > 0)
+        {
+            wait_state = 5;
+            return 1;
+        }
+    }
     // asynchronous recheck
 resume_6:
     wait_state = 6;
@@ -264,7 +284,7 @@ resume_6:
     }, bs->meta_write_recheck_parallelism);
     return 1;
 resume_7:
-    if (bs->heap->finish_recheck() != 0)
+    if (bs->heap->finish_load() != 0)
     {
         exit(1);
     }
@@ -272,11 +292,6 @@ resume_7:
     if (bs->readonly)
     {
         recheck_mod.clear();
-        printf("Actual metadata entries: %ju\n", bs->heap->get_live_entries());
-    }
-    else
-    {
-        printf("Actual metadata entries: %ju, clearing garbage in %zu metadata blocks\n", bs->heap->get_live_entries(), recheck_mod.size());
     }
     for (i = 0; i < recheck_mod.size(); i++)
     {
@@ -290,7 +305,7 @@ resume_8:
         uint32_t block_num = recheck_mod[i];
         uint64_t block_offset = bs->dsk.meta_offset + (uint64_t)(block_num+1) * bs->dsk.meta_block_size;
         data = ((ring_data_t*)sqe->user_data);
-        uint8_t *buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, bs->dsk.meta_block_size);
+        uint8_t *buf = (uint8_t*)malloc_or_die(bs->dsk.meta_block_size);
         bs->heap->get_meta_block(block_num, buf);
         data->iov = { buf, bs->dsk.meta_block_size };
         data->callback = [this, buf, block_offset](ring_data_t *data)
@@ -316,47 +331,5 @@ resume_9:
     }
     free(metadata_buffer);
     metadata_buffer = NULL;
-do_fsync:
-    if (!bs->dsk.disable_meta_fsync && !bs->readonly)
-    {
-        GET_SQE();
-        io_uring_prep_fsync(sqe, bs->dsk.meta_fd, IORING_FSYNC_DATASYNC);
-        last_read_offset = 0;
-        data->iov = { 0 };
-        data->callback = [this](ring_data_t *data) { handle_event(data, -1, "fsync metadata"); };
-        submitted++;
-        bs->ringloop->submit();
-    resume_5:
-        if (submitted > 0)
-        {
-            wait_state = 5;
-            return 1;
-        }
-    }
-    if (zero_on_init && !header_written && !bs->readonly)
-    {
-        GET_SQE();
-        header_written = true;
-        last_read_offset = 0;
-        data->iov = (struct iovec){ bs->meta_superblock, (size_t)bs->dsk.meta_block_size };
-        data->callback = [this](ring_data_t *data) { handle_event(data, -1, "write metadata header"); };
-        io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset);
-        bs->ringloop->submit();
-        submitted++;
-    resume_2:
-        if (submitted > 0)
-        {
-            wait_state = 2;
-            return 1;
-        }
-        if (!bs->dsk.disable_meta_fsync)
-        {
-            goto do_fsync;
-        }
-    }
-    printf("Loading finished. Data used: %ju / %ju bytes (%s / %s)\n",
-        bs->heap->get_data_used_space(), bs->dsk.block_count * bs->dsk.data_block_size,
-        format_size(bs->heap->get_data_used_space()).c_str(),
-        format_size(bs->dsk.block_count * bs->dsk.data_block_size).c_str());
     return 0;
 }

src/blockstore/blockstore_init.h

@@ -17,7 +17,6 @@ class blockstore_init_meta
     int wait_state = 0;
     int wait_count = 0;
     bool zero_on_init = false;
-    bool header_written = false;
     void *metadata_buffer = NULL;
     blockstore_init_meta_buf bufs[2] = {};
     int submitted = 0;
@@ -30,7 +29,7 @@ class blockstore_init_meta
     std::vector<uint32_t> recheck_mod;
     int i = 0, j = 0;
     bool handle_meta_block(uint8_t *buf, uint64_t count, uint64_t done_cnt);
-    void handle_event(ring_data_t *data, int buf_num, const char *op);
+    void handle_event(ring_data_t *data, int buf_num);
 public:
     blockstore_init_meta(blockstore_impl_t *bs);
     int loop();

src/blockstore/blockstore_mock.cpp

@@ -1,113 +0,0 @@
-// Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.1 (see README.md for details)
-
-#include "blockstore_mock.h"
-
-blockstore_mock_t::blockstore_mock_t(const blockstore_config_t & config)
-{
-}
-
-void blockstore_mock_t::parse_config(blockstore_config_t & config)
-{
-}
-
-void* blockstore_mock_t::reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit)
-{
-    return NULL;
-}
-
-bool blockstore_mock_t::reshard_continue(void *reshard_state, uint64_t chunk_limit)
-{
-    return true;
-}
-
-void blockstore_mock_t::loop()
-{
-}
-
-bool blockstore_mock_t::is_started()
-{
-    return true;
-}
-
-bool blockstore_mock_t::is_stalled()
-{
-    return false;
-}
-
-bool blockstore_mock_t::is_safe_to_stop()
-{
-    return true;
-}
-
-void blockstore_mock_t::enqueue_op(blockstore_op_t *op)
-{
-}
-
-int blockstore_mock_t::read_bitmap(object_id oid, uint64_t target_version, void *bitmap, uint64_t *result_version)
-{
-    return -EIO;
-}
-
-const std::map<uint64_t, uint64_t> & blockstore_mock_t::get_inode_space_stats()
-{
-    return inode_space;
-}
-
-void blockstore_mock_t::set_no_inode_stats(const std::vector<uint64_t> & pool_ids)
-{
-}
-
-void blockstore_mock_t::dump_diagnostics()
-{
-}
-
-std::string blockstore_mock_t::get_op_diag(blockstore_op_t *op)
-{
-    return "";
-}
-
-uint32_t blockstore_mock_t::get_block_size()
-{
-    return block_size;
-}
-
-uint64_t blockstore_mock_t::get_block_count()
-{
-    return block_count;
-}
-
-uint64_t blockstore_mock_t::get_free_block_count()
-{
-    return block_count;
-}
-
-uint64_t blockstore_mock_t::get_journal_size()
-{
-    return 32*1024*1024;
-}
-
-uint32_t blockstore_mock_t::get_bitmap_granularity()
-{
-    return bitmap_granularity;
-}
-
-uint64_t blockstore_mock_t::get_live_entries()
-{
-    return 0;
-}
-
-uint64_t blockstore_mock_t::get_live_memory()
-{
-    return 0;
-}
-
-uint64_t blockstore_mock_t::get_garbage_entries()
-{
-    return 0;
-}
-
-uint64_t blockstore_mock_t::get_garbage_memory()
-{
-    return 0;
-}

src/blockstore/blockstore_mock.h

@@ -1,39 +0,0 @@
-// Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.1 (see README.md for details)
-
-#pragma once
-
-#include "blockstore.h"
-
-class blockstore_mock_t: public blockstore_i
-{
-public:
-    uint32_t block_size = 128*1024;
-    uint32_t bitmap_granularity = 4096;
-    uint64_t block_count = 100*1024*8;
-    std::map<uint64_t, uint64_t> inode_space;
-
-    blockstore_mock_t(const blockstore_config_t & config);
-    void parse_config(blockstore_config_t & config) override;
-    void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit) override;
-    bool reshard_continue(void *reshard_state, uint64_t chunk_limit) override;
-    void loop() override;
-    bool is_started() override;
-    bool is_stalled() override;
-    bool is_safe_to_stop() override;
-    void enqueue_op(blockstore_op_t *op) override;
-    int read_bitmap(object_id oid, uint64_t target_version, void *bitmap, uint64_t *result_version = NULL) override;
-    const std::map<uint64_t, uint64_t> & get_inode_space_stats() override;
-    void set_no_inode_stats(const std::vector<uint64_t> & pool_ids) override;
-    void dump_diagnostics() override;
-    std::string get_op_diag(blockstore_op_t *op) override;
-    uint32_t get_block_size() override;
-    uint64_t get_block_count() override;
-    uint64_t get_free_block_count() override;
-    uint64_t get_journal_size() override;
-    uint32_t get_bitmap_granularity() override;
-    uint64_t get_live_entries() override;
-    uint64_t get_live_memory() override;
-    uint64_t get_garbage_entries() override;
-    uint64_t get_garbage_memory() override;
-};

src/blockstore/blockstore_open.cpp

@@ -28,7 +28,6 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init)
     throttle_target_parallelism = strtoull(config["throttle_target_parallelism"].c_str(), NULL, 10);
     throttle_threshold_us = strtoull(config["throttle_threshold_us"].c_str(), NULL, 10);
     perfect_csum_update = config["perfect_csum_update"] == "true" || config["perfect_csum_update"] == "1" || config["perfect_csum_update"] == "yes";
-    skip_corrupted_meta_entries = config["skip_corrupted_meta_entries"] == "true" || config["skip_corrupted_meta_entries"] == "1" || config["skip_corrupted_meta_entries"] == "yes";
     if (config["autosync_writes"] != "")
     {
         autosync_writes = strtoull(config["autosync_writes"].c_str(), NULL, 10);

src/blockstore/blockstore_read.cpp

@@ -462,10 +462,6 @@ int blockstore_impl_t::read_bitmap(object_id oid, uint64_t target_version, void
         {
             if (target_version >= wr->version)
             {
-                if (wr->type() == BS_HEAP_DELETE)
-                {
-                    return false;
-                }
                 found = true;
                 if (result_version)
                 {

src/blockstore/blockstore_stable.cpp

@@ -16,7 +16,6 @@ int blockstore_impl_t::dequeue_stable(blockstore_op_t *op)
     else if (priv->op_state == 5) goto resume_5;
     assert(!priv->op_state);
     op->retval = 0;
-    PRIV(op)->lsn = 0;
     priv->modified_block = priv->modified_block2 = UINT32_MAX;
     for (priv->stab_pos = 0; priv->stab_pos < op->len; priv->stab_pos++)
     {
@@ -28,7 +27,6 @@ int blockstore_impl_t::dequeue_stable(blockstore_op_t *op)
                 FINISH_OP(op);
                 return 2;
             }
-            priv->modified_block2 = UINT32_MAX;
             int res = op->opcode == BS_OP_STABLE
                 ? heap->add_commit(obj, v[priv->stab_pos].version, &priv->modified_block2)
                 : heap->add_rollback(obj, v[priv->stab_pos].version, &priv->modified_block2);
@@ -38,12 +36,6 @@ int blockstore_impl_t::dequeue_stable(blockstore_op_t *op)
                 FINISH_OP(op);
                 return 2;
             }
-            if (res == ENOENT)
-            {
-                op->retval = -ENOENT;
-                FINISH_OP(op);
-                return 2;
-            }
             if (res == ENOSPC)
             {
                 if (!heap->get_to_compact_count())
@@ -53,6 +45,11 @@ int blockstore_impl_t::dequeue_stable(blockstore_op_t *op)
                     FINISH_OP(op);
                     return 2;
                 }
+                if (priv->modified_block2 != UINT32_MAX)
+                {
+                    priv->stab_pos--;
+                    goto resume_1;
+                }
                 priv->wait_for = WAIT_COMPACTION;
                 priv->wait_detail = heap->get_compacted_count();
                 flusher->request_trim();
@@ -60,9 +57,9 @@ int blockstore_impl_t::dequeue_stable(blockstore_op_t *op)
             }
             assert(res == 0);
         }
-resume_1:
         if (priv->modified_block != UINT32_MAX && priv->modified_block2 != priv->modified_block)
         {
+resume_1:
             BS_SUBMIT_CHECK_SQES(1);
             prepare_meta_block_write(priv->modified_block);
 resume_2:

src/blockstore/blockstore_sync.cpp

@@ -9,7 +9,6 @@ int blockstore_impl_t::continue_sync(blockstore_op_t *op)
     if (!PRIV(op)->op_state)
     {
         op->retval = 0;
-        PRIV(op)->lsn = 0;
     }
     int res = do_sync(op, 0);
     if (res == 2)
@@ -29,12 +28,9 @@ bool blockstore_impl_t::has_unsynced()
 
 bool blockstore_impl_t::submit_fsyncs(int & wait_count)
 {
-    int n = (unsynced_meta_write_count > 0 && !dsk.disable_meta_fsync ? 1 : 0) +
+    int n = (unsynced_meta_write_count > 0 && !dsk.disable_meta_fsync) +
-        (unsynced_buffer_write_count > 0 && !dsk.disable_journal_fsync &&
+        (unsynced_buffer_write_count > 0 && !dsk.disable_journal_fsync && dsk.journal_fd != dsk.meta_fd) +
-            (!unsynced_meta_write_count || dsk.journal_fd != dsk.meta_fd) ? 1 : 0) +
+        (unsynced_data_write_count > 0 && !dsk.disable_data_fsync && dsk.data_fd != dsk.meta_fd && dsk.data_fd != dsk.journal_fd);
-        (unsynced_data_write_count > 0 && !dsk.disable_data_fsync &&
-            (!unsynced_meta_write_count || dsk.data_fd != dsk.meta_fd) &&
-            (!unsynced_buffer_write_count || dsk.data_fd != dsk.journal_fd) ? 1 : 0);
     if (ringloop->space_left() < n)
     {
         return false;
@@ -63,8 +59,7 @@ bool blockstore_impl_t::submit_fsyncs(int & wait_count)
         data->callback = cb;
         wait_count++;
     }
-    if (unsynced_buffer_write_count > 0 && !dsk.disable_journal_fsync &&
+    if (unsynced_buffer_write_count > 0 && !dsk.disable_journal_fsync && dsk.meta_fd != dsk.journal_fd)
-        (!unsynced_meta_write_count || dsk.journal_fd != dsk.meta_fd))
     {
         // fsync buffer
         io_uring_sqe *sqe = get_sqe();
@@ -75,9 +70,7 @@ bool blockstore_impl_t::submit_fsyncs(int & wait_count)
         data->callback = cb;
         wait_count++;
     }
-    if (unsynced_data_write_count > 0 && !dsk.disable_data_fsync &&
+    if (unsynced_data_write_count > 0 && !dsk.disable_data_fsync && dsk.data_fd != dsk.meta_fd && dsk.data_fd != dsk.journal_fd)
-        (!unsynced_meta_write_count || dsk.data_fd != dsk.meta_fd) &&
-        (!unsynced_buffer_write_count || dsk.data_fd != dsk.journal_fd))
     {
         // fsync data
         io_uring_sqe *sqe = get_sqe();
@@ -111,11 +104,9 @@ int blockstore_impl_t::do_sync(blockstore_op_t *op, int base_state)
         unsynced_data_write_count = unsynced_buffer_write_count = unsynced_meta_write_count = 0;
         return 2;
     }
-    assert(!PRIV(op)->lsn);
+    PRIV(op)->modified_block = heap->get_completed_lsn();
-    PRIV(op)->lsn = heap->get_completed_lsn();
     if (!submit_fsyncs(PRIV(op)->pending_ops))
     {
-        PRIV(op)->lsn = 0;
         PRIV(op)->wait_detail = 1;
         PRIV(op)->wait_for = WAIT_SQE;
         return 0;
@@ -127,6 +118,6 @@ resume_1:
         return 1;
     }
 resume_2:
-    heap->mark_lsn_fsynced(PRIV(op)->lsn);
+    heap->mark_lsn_fsynced(PRIV(op)->modified_block);
     return 2;
 }

src/blockstore/blockstore_write.cpp

@@ -13,37 +13,38 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
     return true;
 }
 
-void blockstore_impl_t::prepare_meta_block_write(uint32_t modified_block)
+bool blockstore_impl_t::prepare_meta_block_write(uint32_t modified_block)
 {
-    if (modified_blocks.find(modified_block) != modified_blocks.end())
+    auto mod_it = modified_blocks.find(modified_block);
-        return;
+    if (mod_it != modified_blocks.end())
+    {
+        return !mod_it->second.sent;
+    }
     io_uring_sqe *sqe = get_sqe();
     assert(sqe != NULL);
     ring_data_t *data = ((ring_data_t*)sqe->user_data);
     uint8_t *buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size);
     data->iov = (struct iovec){ buf, (size_t)dsk.meta_block_size };
-    data->callback = [this, modified_block](ring_data_t *data)
+    data->callback = [this, modified_block, buf](ring_data_t *data)
     {
+        free(buf);
         live = true;
         if (data->res != data->iov.iov_len)
         {
             // FIXME: our state becomes corrupted after a write error. maybe do something better than just die
             disk_error_abort("data write", data->res, data->iov.iov_len);
         }
-        auto it = modified_blocks.find(modified_block);
+        modified_blocks.erase(modified_block);
-        assert(it != modified_blocks.end());
-        free(it->second.buf);
-        modified_blocks.erase(it);
         heap->complete_block_write(modified_block);
         ringloop->wakeup();
     };
-    assert(((uint64_t)modified_block+2)*dsk.meta_block_size <= dsk.meta_area_size);
     io_uring_prep_writev(
-        sqe, dsk.meta_fd, &data->iov, 1, dsk.meta_offset + ((uint64_t)modified_block+1)*dsk.meta_block_size
+        sqe, dsk.meta_fd, &data->iov, 1, dsk.meta_offset + (modified_block+1)*dsk.meta_block_size
     );
     unsynced_meta_write_count++;
     pending_modified_blocks.push_back(modified_block);
     modified_blocks[modified_block] = { .sent = false, .buf = buf };
+    return true;
 }
 
 bool blockstore_impl_t::meta_block_is_pending(uint32_t modified_block)
@@ -124,6 +125,7 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
     heap_entry_t *obj = heap->read_entry(op->oid);
     if (op->opcode == BS_OP_DELETE)
     {
+        return continue_delete(op, 0);
         // Delete
         if (!obj || obj->type() == BS_HEAP_DELETE)
         {
@@ -136,17 +138,38 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
         BS_SUBMIT_CHECK_SQES(1);
         int res = heap->add_delete(obj, &PRIV(op)->modified_block);
         if (res == ENOSPC)
+        {
             goto enospc;
+        }
         assert(res == 0);
-        prepare_meta_block_write(PRIV(op)->modified_block);
-        PRIV(op)->pending_ops++;
-        PRIV(op)->op_state = 5;
         write_iodepth++;
+resume_1:
+        while (!prepare_meta_block_write(PRIV(op)->modified_block))
+        {
+            PRIV(op)->op_state = 1;
+            return 1;
+        }
+rseume_2:
+        while (meta_block_is_pending(PRIV(op)->modified_block))
+        {
+            PRIV(op)->op_state = 2;
+            return 1;
+        }
+resume_3:
+resume_4:
+        if (!throttle_write(op, 3))
+        {
+            return 1;
+        }
+        write_iodepth--;
+        ack_write(op);
+        return 2;
     }
     // FIXME: Allow to do initial writes as buffered, not redirected
     // FIXME: Allow to do direct writes over holes
     else if (!obj || obj->type() == BS_HEAP_DELETE || op->offset == 0 && op->len == dsk.data_block_size)
     {
+        return continue_big_write(op, 10);
         // Big (redirect) write
         PRIV(op)->write_type = dsk.disable_data_fsync || op->opcode != BS_OP_WRITE_STABLE ? BS_HEAP_BIG_WRITE : _REDIRECT_INTENT;
         BS_SUBMIT_CHECK_SQES(1);
@@ -166,6 +189,7 @@ enospc:
             flusher->request_trim();
             return 0;
         }
+        write_iodepth++;
         uint64_t loc = PRIV(op)->location;
 #ifdef BLOCKSTORE_DEBUG
         printf(
@@ -178,23 +202,73 @@ enospc:
         ring_data_t *data = ((ring_data_t*)sqe->user_data);
         data->iov = (struct iovec){ op->buf, op->len };
         data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
-        assert(loc+op->offset+op->len <= dsk.block_count*dsk.data_block_size);
         io_uring_prep_writev(sqe, dsk.data_fd, &data->iov, 1, dsk.data_offset + loc + op->offset);
-        if (!dsk.disable_data_fsync)
-        {
-            // use PRIV->lsn for fsync_data_id
-            PRIV(op)->lsn = ++data_fsync_next;
-            data_fsyncs.push_back(false);
-        }
-        PRIV(op)->pending_ops++;
-        write_iodepth++;
         if (PRIV(op)->write_type == BS_HEAP_BIG_WRITE)
-            PRIV(op)->op_state = 1;
+            inflight_big++;
+        PRIV(op)->pending_ops++;
+resume_10:
+        if (PRIV(op)->pending_ops > 0)
+        {
+            PRIV(op)->op_state = 10;
+            return 1;
+        }
+        if (PRIV(op)->write_type == BS_HEAP_BIG_WRITE)
+        {
+            inflight_big--;
+resume_11:
+resume_12:
+resume_13:
+            if (!fsync_big_write(op, 11))
+                return 1;
+        }
+        heap_entry_t *obj = heap->read_entry(op->oid);
+        int res = 0;
+        if (PRIV(op)->write_type == _REDIRECT_INTENT)
+        {
+            res = heap->add_redirect_intent(op->oid, &obj, op->version, op->offset, op->len,
+                PRIV(op)->location, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
+        }
         else
-            PRIV(op)->op_state = 3;
+        {
+            res = heap->add_big_write(op->oid, obj, op->opcode == BS_OP_WRITE_STABLE,
+                op->version, op->offset, op->len, PRIV(op)->location, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
+        }
+        if (res == ENOSPC)
+        {
+            if (!heap->get_to_compact_count())
+            {
+                // no space
+                heap->free_data(op->oid.inode, PRIV(op)->location);
+                write_iodepth--;
+                op->retval = -ENOSPC;
+                FINISH_OP(op);
+                return 2;
+            }
+            PRIV(op)->wait_for = WAIT_COMPACTION;
+            PRIV(op)->wait_detail = heap->get_compacted_count();
+            flusher->request_trim();
+            return 0;
+        }
+        assert(res == 0);
+resume_14:
+        while (!prepare_meta_block_write(PRIV(op)->modified_block))
+        {
+            PRIV(op)->op_state = 14;
+            return 1;
+        }
+resume_15:
+        while (meta_block_is_pending(PRIV(op)->modified_block))
+        {
+            PRIV(op)->op_state = 15;
+            return 1;
+        }
+        write_iodepth--;
+        ack_write(op);
+        return 2;
     }
     else if (intent_write_allowed(op, obj))
     {
+        return continue_intent_write(op, 20);
         // Direct intent-write
         BS_SUBMIT_CHECK_SQES(1);
         int res = 0;
@@ -230,13 +304,41 @@ enospc:
             assert(res == 0);
             PRIV(op)->lsn = obj->lsn;
         }
-        prepare_meta_block_write(PRIV(op)->modified_block);
-        PRIV(op)->pending_ops++;
-        PRIV(op)->op_state = 9;
         write_iodepth++;
+resume_20:
+        while (!prepare_meta_block_write(PRIV(op)->modified_block))
+        {
+            PRIV(op)->op_state = 20;
+            return 1;
+        }
+resume_21:
+        while (meta_block_is_pending(PRIV(op)->modified_block))
+        {
+            PRIV(op)->op_state = 21;
+            return 1;
+        }
+        // Direct intent-write
+        // LSN is not marked as completed so big_write won't be freed
+        BS_SUBMIT_GET_SQE(sqe, data);
+        data->iov = (struct iovec){ op->buf, op->len };
+        data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
+        io_uring_prep_writev(sqe, dsk.data_fd, &data->iov, 1, dsk.data_offset + PRIV(op)->location + op->offset);
+        if (dsk.use_atomic_flag)
+            sqe->rw_flags = RWF_ATOMIC;
+        PRIV(op)->pending_ops++;
+resume_22:
+        if (PRIV(op)->pending_ops > 0)
+        {
+            PRIV(op)->op_state = 22;
+            return 1;
+        }
+        write_iodepth--;
+        ack_write(op);
+        return 2;
     }
     else
     {
+        return continue_small_write(op, 30);
         // Small (buffered) overwrite
         // First check if there is free buffer space
         PRIV(op)->write_type = BS_HEAP_SMALL_WRITE;
@@ -249,19 +351,20 @@ enospc:
             return 0;
         }
         // There is sufficient space. Check SQE(s)
-        BS_SUBMIT_CHECK_SQES(1 + (op->len > 0 ? 1 : 0));
+        BS_SUBMIT_CHECK_SQES(1 + (op->len > 0 ? 1 : 0)); ---> refactor too
         int res = heap->add_small_write(op->oid, &obj, (BS_HEAP_SMALL_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0)),
             op->version, op->offset, op->len, loc, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
         if (res == ENOSPC)
             goto enospc;
         assert(res == 0);
         PRIV(op)->lsn = obj->lsn;
+        if (op->len)
+            heap->use_buffer_area(op->oid.inode, loc, op->len);
         prepare_meta_block_write(PRIV(op)->modified_block);
         PRIV(op)->pending_ops++;
         if (op->len > 0)
         {
             // Prepare buffered data write
-            heap->use_buffer_area(op->oid.inode, loc, op->len);
             if (dsk.inmemory_journal)
             {
                 memcpy((uint8_t*)buffer_area + loc, op->buf, op->len);
@@ -269,7 +372,6 @@ enospc:
             BS_SUBMIT_GET_SQE(sqe2, data2);
             data2->iov = (struct iovec){ op->buf, op->len };
             data2->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
-            assert(loc+op->len <= dsk.journal_len);
             io_uring_prep_writev(sqe2, dsk.journal_fd, &data2->iov, 1, dsk.journal_offset + loc);
             PRIV(op)->pending_ops++;
         }
@@ -300,6 +402,8 @@ again:
         goto resume_10;
     else if (op_state == 11)
         goto resume_11;
+    else if (op_state == 12)
+        goto resume_12;
     else
     {
         // In progress
@@ -316,86 +420,74 @@ again:
         goto again;
     }
 resume_2:
+
+
+resume_4:
+resume_6:
+
+resume_8:
+    ack
+    return 2;
+resume_10:
+    return 1;
+}
+
+bool blockstore_impl_t::fsync_big_write(blockstore_op_t *op, int base_state)
+{
+    if (PRIV(op)->state == base_state)
+        goto resume_0;
+    else if (PRIV(op)->state == base_state+1)
+        goto resume_1;
+    else if (PRIV(op)->state == base_state+2)
+        goto resume_2;
     // We must fsync all big writes to avoid complex write workflows
     // It's OK for all HDDs and for server SSDs, but slightly worse for desktop SSDs
     if (!dsk.disable_data_fsync)
     {
-        // Mark our data write as completed and advance data_fsync_cur
+        // fsync data in a batch
-        data_fsyncs[PRIV(op)->lsn - data_fsync_cur - 1] = true;
+resume_0:
-        while (data_fsyncs.size() > 0 && data_fsyncs.front())
+        if (inflight_big > 0)
         {
-            data_fsyncs.pop_front();
+            PRIV(op)->op_state = base_state;
-            data_fsync_cur++;
+            return false;
         }
-        PRIV(op)->op_state = 11;
+        if (fsyncing_data)
-        // Then wait for all other data writes currently in progress to do less fsync calls
-        // I.e. to fsync data in batches
-        PRIV(op)->lsn = data_fsync_cur + data_fsyncs.size();
-resume_11:
-        if (data_fsync_cur < PRIV(op)->lsn)
         {
-            return 1;
+resume_1:
-        }
+            if (fsyncing_data)
-        if (PRIV(op)->lsn > data_fsync_sent)
-        {
-            BS_SUBMIT_GET_SQE(sqe, data);
-            io_uring_prep_fsync(sqe, dsk.data_fd, IORING_FSYNC_DATASYNC);
-            data->iov = { 0 };
-            data->callback = [this, op, fs = data_fsync_cur](ring_data_t *data)
             {
-                if (fs > data_fsync_done)
+                PRIV(op)->op_state = base_state+1;
-                {
+                return false;
-                    data_fsync_done = fs;
-                    ringloop->wakeup();
-                }
-            };
-            data_fsync_sent = data_fsync_cur;
-        }
-        if (PRIV(op)->lsn > data_fsync_done)
-        {
-            return 1;
-        }
-        PRIV(op)->lsn = 0;
-    }
-resume_4:
-    {
-        BS_SUBMIT_CHECK_SQES(1);
-        auto obj = heap->read_entry(op->oid);
-        int res = 0;
-        if (PRIV(op)->write_type == _REDIRECT_INTENT)
-        {
-            res = heap->add_redirect_intent(op->oid, &obj, op->version, op->offset, op->len,
-                PRIV(op)->location, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
-        }
-        else
-        {
-            res = heap->add_big_write(op->oid, obj, op->opcode == BS_OP_WRITE_STABLE,
-                op->version, op->offset, op->len, PRIV(op)->location, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
-        }
-        if (res == ENOSPC)
-        {
-            if (!heap->get_to_compact_count())
-            {
-                // no space
-                heap->free_data(op->oid.inode, PRIV(op)->location);
-                write_iodepth--;
-                op->retval = -ENOSPC;
-                FINISH_OP(op);
-                return 2;
             }
-            PRIV(op)->wait_for = WAIT_COMPACTION;
+            return true;
-            PRIV(op)->wait_detail = heap->get_compacted_count();
-            flusher->request_trim();
-            return 0;
         }
-        assert(res == 0);
+        fsyncing_data = true;
-        prepare_meta_block_write(PRIV(op)->modified_block);
+        BS_SUBMIT_GET_SQE(sqe, data);
+        io_uring_prep_fsync(sqe, dsk.data_fd, IORING_FSYNC_DATASYNC);
+        data->iov = { 0 };
+        data->callback = [this, op](ring_data_t *data)
+        {
+            fsyncing_data = false;
+            handle_write_event(data, op);
+        };
         PRIV(op)->pending_ops++;
-        PRIV(op)->op_state = 5;
+resume_2:
-        return 1;
+        if (PRIV(op)->pending_ops > 0)
+        {
+            PRIV(op)->op_state = base_state+2;
+            return false;
+        }
     }
-resume_6:
+    return true;
+}
+
+bool blockstore_impl_t::throttle_write(blockstore_op_t *op, int base_state)
+{
     // Apply throttling to not fill the journal too quickly for the SSD+HDD case
+    if (PRIV(op)->op_state >= base_state+1)
+    {
+        return true;
+    }
     if (PRIV(op)->write_type == BS_HEAP_SMALL_WRITE && throttle_small_writes)
     {
         // Apply throttling
@@ -416,18 +508,21 @@ resume_6:
         if (ref_us > exec_us + throttle_threshold_us)
         {
             // Pause reply
-            PRIV(op)->pending_ops++;
+            PRIV(op)->op_state = base_state;
-            PRIV(op)->op_state = 7;
             // Remember that the timer can in theory be called right here
             tfd->set_timer_us(ref_us-exec_us, false, [this, op](int timer_id)
             {
-                PRIV(op)->pending_ops--;
+                PRIV(op)->op_state++;
                 ringloop->wakeup();
             });
-            return 1;
+            return false;
         }
     }
-resume_8:
+    return true;
+}
+
+void blockstore_impl_t::ack_write(blockstore_op_t *op)
+{
     // Acknowledge write
 #ifdef BLOCKSTORE_DEBUG
     printf("Ack write %jx:%jx v%ju\n", op->oid.inode, op->oid.stripe, op->version);
@@ -454,22 +549,7 @@ resume_8:
         unsynced_data_write_count++;
         intent_write_counter++;
     }
-    write_iodepth--;
     FINISH_OP(op);
-    return 2;
-resume_10:
-    // Direct intent-write
-    // LSN is not marked as completed so big_write won't be freed
-    BS_SUBMIT_GET_SQE(sqe, data);
-    data->iov = (struct iovec){ op->buf, op->len };
-    data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
-    assert(PRIV(op)->location + op->offset <= dsk.block_count*dsk.data_block_size);
-    io_uring_prep_writev(sqe, dsk.data_fd, &data->iov, 1, dsk.data_offset + PRIV(op)->location + op->offset);
-    if (dsk.use_atomic_flag)
-        sqe->rw_flags = RWF_ATOMIC;
-    PRIV(op)->pending_ops++;
-    PRIV(op)->op_state = 7;
-    return 1;
 }
 
 void blockstore_impl_t::handle_write_event(ring_data_t *data, blockstore_op_t *op)

src/blockstore/multilist.cpp

@@ -12,7 +12,7 @@ multilist_alloc_t::multilist_alloc_t(uint32_t count, uint32_t maxn):
     count(count), maxn(maxn)
 {
     // not-so-memory-efficient: 16 MB memory per 1 GB buffer space, but buffer spaces are small, so OK
-    assert(count > 1 && count < 0x80000000 && count >= maxn);
+    assert(count > 1 && count < 0x80000000);
     sizes.resize(count);
     nexts.resize(count); // nexts[i] = 0 -> area is used; nexts[i] = 1 -> no next; nexts[i] >= 2 -> next item
     prevs.resize(count);
@@ -171,7 +171,7 @@ void multilist_alloc_t::print()
     printf("\n");
 }
 
-bool multilist_alloc_t::use(uint32_t pos, uint32_t size)
+void multilist_alloc_t::use(uint32_t pos, uint32_t size)
 {
     assert(pos+size <= count && size > 0);
     if (sizes[pos] <= 0)
@@ -182,8 +182,7 @@ bool multilist_alloc_t::use(uint32_t pos, uint32_t size)
         else
             while (start > 0 && !sizes[start])
                 start--;
-        if (sizes[start] < size+(pos-start))
+        assert(sizes[start] >= size);
-            return false;
         use_full(start);
         uint32_t full = sizes[start];
         sizes[pos-1] = -pos+start;
@@ -200,8 +199,7 @@ bool multilist_alloc_t::use(uint32_t pos, uint32_t size)
     }
     else
     {
-        if (sizes[pos] < size)
+        assert(sizes[pos] >= size);
-            return false;
         use_full(pos);
         if (sizes[pos] > size)
         {
@@ -216,13 +214,12 @@ bool multilist_alloc_t::use(uint32_t pos, uint32_t size)
 #ifdef MULTILIST_TRACE
     print();
 #endif
-    return true;
 }
 
 void multilist_alloc_t::use_full(uint32_t pos)
 {
     uint32_t prevsize = sizes[pos];
-    assert(prevsize > 0);
+    assert(prevsize);
     assert(nexts[pos]);
     uint32_t pi = (prevsize < maxn ? prevsize : maxn)-1;
     if (heads[pi] == pos+1)

src/blockstore/multilist.h

@@ -17,7 +17,7 @@ struct multilist_alloc_t
     bool is_free(uint32_t pos);
     uint32_t find(uint32_t size);
     void use_full(uint32_t pos);
-    bool use(uint32_t pos, uint32_t size);
+    void use(uint32_t pos, uint32_t size);
     void do_free(uint32_t pos);
     void free(uint32_t pos);
     void verify();

src/blockstore/ondisk_formats.h

@@ -141,7 +141,7 @@ struct __attribute__((__packed__)) journal_entry
 inline uint32_t je_crc32(journal_entry *je)
 {
     // 0x48674bc7 = crc32(4 zero bytes)
-    return je->size < 4 ? 0 : crc32c(0x48674bc7, ((uint8_t*)je)+4, je->size-4);
+    return crc32c(0x48674bc7, ((uint8_t*)je)+4, je->size-4);
 }
 
 // "VITAstor"

src/blockstore/v1/flush.cpp

@@ -71,11 +71,6 @@ bool journal_flusher_t::is_active()
     return active_flushers > 0 || dequeuing;
 }
 
-size_t journal_flusher_t::get_queue_size()
-{
-    return flush_queue.size();
-}
-
 void journal_flusher_t::loop()
 {
     target_flusher_count = bs->write_iodepth*2;
@@ -389,7 +384,6 @@ stop_flusher:
         wait_state = 0;
         return true;
     }
-    copy_count = 0;
     try_trim = true;
     cur.oid = flusher->flush_queue.front();
     cur.version = flusher->flush_versions[cur.oid];
@@ -517,31 +511,6 @@ resume_2:
             {
                 uo_it->second.was_changed = true;
             }
-            if (!bs->journal.inmemory)
-            {
-                // Verify journaled data checksums (but not COALESCED)
-                for (it = v.begin(); it != v.end(); it++)
-                {
-                    if (it->copy_flags == COPY_BUF_JOURNAL)
-                    {
-                        iovec iov = { .iov_base = it->buf, .iov_len = it->len };
-                        bs->verify_journal_checksums(
-                            it->csum_buf, it->offset, &iov, 1,
-                            [&](uint32_t bad_block, uint32_t calc_csum, uint32_t stored_csum)
-                            {
-                                printf(
-                                    "Checksum mismatch in object %jx:%jx v%ju in journal at 0x%jx, checksum block #%u: got %08x, expected %08x\n",
-                                    cur.oid.inode, cur.oid.stripe, cur.version, it->disk_offset,
-                                    bad_block / bs->dsk.csum_block_size, calc_csum, stored_csum
-                                );
-                                bad_block += it->offset;
-                                assert(!(bad_block % bs->dsk.csum_block_size) && bad_block < bs->dsk.data_block_size);
-                                mangle_csum_blocks.insert(bad_block);
-                            }
-                        );
-                    }
-                }
-            }
         }
         // Submit data writes
         for (it = v.begin(); it != v.end(); it++)
@@ -551,7 +520,6 @@ resume_2:
                 await_sqe(15);
                 data->iov = (struct iovec){ it->buf, (size_t)it->len };
                 data->callback = simple_callback_w;
-                assert(clean_loc+it->offset+it->len <= bs->dsk.block_count*bs->dsk.data_block_size);
                 io_uring_prep_writev(
                     sqe, bs->dsk.data_fd, &data->iov, 1, bs->dsk.data_offset + clean_loc + it->offset
                 );
@@ -665,7 +633,6 @@ resume_2:
         }
         // All done
         flusher->active_flushers--;
-        copy_count = 0; // used by is_mutated()...
         wait_state = 0;
         goto resume_0;
     }
@@ -782,7 +749,6 @@ bool journal_flusher_co::write_meta_block(flusher_meta_write_t & meta_block, int
     await_sqe(0);
     data->iov = (struct iovec){ meta_block.buf, (size_t)bs->dsk.meta_block_size };
     data->callback = simple_callback_w;
-    assert(bs->dsk.meta_block_size + meta_block.sector + bs->dsk.meta_block_size <= bs->dsk.meta_area_size);
     io_uring_prep_writev(
         sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bs->dsk.meta_block_size + meta_block.sector
     );
@@ -847,21 +813,35 @@ bool journal_flusher_co::clear_incomplete_csum_block_bits(int wait_base)
                 bs->verify_padded_checksums(new_clean_bitmap, new_clean_bitmap + 2*bs->dsk.clean_entry_bitmap_size,
                     v[i].offset, &iov, 1, [&](uint32_t bad_block, uint32_t calc_csum, uint32_t stored_csum)
                 {
-                    printf("Checksum mismatch in object %jx:%jx v%ju in data area at offset 0x%jx+0x%x during flush: got %08x, expected %08x\n",
+                    printf("Checksum mismatch in object %jx:%jx v%ju in data area at offset 0x%jx+0x%x: got %08x, expected %08x\n",
                         cur.oid.inode, cur.oid.stripe, old_clean_ver, old_clean_loc, bad_block, calc_csum, stored_csum);
-                    assert(!(bad_block % bs->dsk.csum_block_size) && bad_block < bs->dsk.data_block_size);
+                    for (uint32_t j = 0; j < bs->dsk.csum_block_size; j += bs->dsk.bitmap_granularity)
-                    mangle_csum_blocks.insert(bad_block);
+                    {
+                        // Simplest method of mangling: flip one byte in every sector
+                        ((uint8_t*)v[i].buf)[j+bad_block-v[i].offset] ^= 0xff;
+                    }
                 });
             }
             else
             {
                 bs->verify_journal_checksums(v[i].csum_buf, v[i].offset, &iov, 1, [&](uint32_t bad_block, uint32_t calc_csum, uint32_t stored_csum)
                 {
-                    printf("Checksum mismatch in object %jx:%jx v%ju in journal at offset 0x%jx+0x%x (block offset 0x%jx) during flush: got %08x, expected %08x\n",
+                    printf("Checksum mismatch in object %jx:%jx v%ju in journal at offset 0x%jx+0x%x (block offset 0x%jx): got %08x, expected %08x\n",
                         cur.oid.inode, cur.oid.stripe, old_clean_ver,
                         v[i].disk_offset, bad_block, v[i].offset, calc_csum, stored_csum);
-                    assert(!(bad_block % bs->dsk.csum_block_size) && bad_block < bs->dsk.data_block_size);
+                    bad_block += (v[i].offset/bs->dsk.csum_block_size) * bs->dsk.csum_block_size;
-                    mangle_csum_blocks.insert(bad_block);
+                    uint32_t bad_block_end = bad_block + bs->dsk.csum_block_size + (v[i].offset/bs->dsk.csum_block_size) * bs->dsk.csum_block_size;
+                    if (bad_block < v[i].offset)
+                        bad_block = v[i].offset;
+                    if (bad_block_end > v[i].offset+v[i].len)
+                        bad_block_end = v[i].offset+v[i].len;
+                    bad_block -= v[i].offset;
+                    bad_block_end -= v[i].offset;
+                    for (uint32_t j = bad_block; j < bad_block_end; j += bs->dsk.bitmap_granularity)
+                    {
+                        // Simplest method of mangling: flip one byte in every sector
+                        ((uint8_t*)v[i].buf)[j] ^= 0xff;
+                    }
                 });
             }
         }
@@ -970,11 +950,6 @@ void journal_flusher_co::calc_block_checksums(uint32_t *new_data_csums, bool ski
     }
     // `v` should contain aligned items, possibly split into pieces
     assert(!block_done);
-    for (uint32_t mangle_block: mangle_csum_blocks)
-    {
-        // Flip 1 bit
-        new_data_csums[mangle_block / bs->dsk.csum_block_size] ^= 1;
-    }
 }
 
 void journal_flusher_co::scan_dirty()
@@ -1111,8 +1086,7 @@ void journal_flusher_co::scan_dirty()
             last--;
         read_to_fill_incomplete = bs->fill_partial_checksum_blocks(
             v, fulfilled, bmp_ptr, NULL, false, NULL, v[0].offset/bs->dsk.csum_block_size * bs->dsk.csum_block_size,
-            ((v[last].offset+v[last].len-1) / bs->dsk.csum_block_size + 1) * bs->dsk.csum_block_size,
+            ((v[last].offset+v[last].len-1) / bs->dsk.csum_block_size + 1) * bs->dsk.csum_block_size
-            0, bs->dsk.data_block_size
         );
     }
     else if (fill_incomplete && clean_init_bitmap)
@@ -1142,7 +1116,6 @@ bool journal_flusher_co::read_dirty(int wait_base)
     if (wait_state == wait_base)        goto resume_0;
     else if (wait_state == wait_base+1) goto resume_1;
     wait_count = wait_journal_count = 0;
-    mangle_csum_blocks.clear();
     if (bs->journal.inmemory && !read_to_fill_incomplete)
     {
         // Happy path: nothing to read :)
@@ -1374,7 +1347,7 @@ bool journal_flusher_co::fsync_batch(bool fsync_meta, int wait_base)
         cur_sync->ready_count++;
         flusher->syncing_flushers++;
     resume_1:
-        if (cur_sync->state == 0)
+        if (!cur_sync->state)
         {
             if (flusher->syncing_flushers >= flusher->active_flushers || !flusher->flush_queue.size())
             {
@@ -1402,12 +1375,6 @@ bool journal_flusher_co::fsync_batch(bool fsync_meta, int wait_base)
                 return false;
             }
         }
-        else if (cur_sync->state == 1)
-        {
-            // Wait for fsync completion
-            wait_state = wait_base+1;
-            return false;
-        }
         flusher->syncing_flushers--;
         cur_sync->ready_count--;
         if (cur_sync->ready_count == 0)

src/blockstore/v1/flush.h

@@ -66,7 +66,6 @@ class journal_flusher_co
     uint64_t clean_bitmap_offset, clean_bitmap_len;
     uint8_t *clean_init_dyn_ptr;
     uint8_t *new_clean_bitmap;
-    std::unordered_set<uint32_t> mangle_csum_blocks;
 
     uint64_t new_trim_pos;
 
@@ -124,7 +123,6 @@ public:
     void loop();
     bool is_trim_wanted() { return trim_wanted; }
     bool is_active();
-    size_t get_queue_size();
     void mark_trim_possible();
     void request_trim();
     void release_trim();

src/blockstore/v1/impl.cpp

@@ -6,12 +6,11 @@
 
 namespace v1 {
 
-blockstore_impl_t::blockstore_impl_t(blockstore_config_t & config, ring_loop_i *ringloop, timerfd_manager_t *tfd, bool mock_mode)
+blockstore_impl_t::blockstore_impl_t(blockstore_config_t & config, ring_loop_i *ringloop, timerfd_manager_t *tfd)
 {
     assert(sizeof(blockstore_op_private_t) <= BS_OP_PRIVATE_DATA_SIZE);
     this->tfd = tfd;
     this->ringloop = ringloop;
-    dsk.mock_mode = mock_mode;
     ring_consumer.loop = [this]() { loop(); };
     ringloop->register_consumer(&ring_consumer);
     initialized = 0;
@@ -36,11 +35,6 @@ blockstore_impl_t::blockstore_impl_t(blockstore_config_t & config, ring_loop_i *
 
 blockstore_impl_t::~blockstore_impl_t()
 {
-    for (auto& obj: dirty_db)
-    {
-        if (obj.second.dyn_data)
-            free(obj.second.dyn_data);
-    }
     delete data_alloc;
     delete flusher;
     if (zero_object)
@@ -486,6 +480,17 @@ resume_1:
     return true;
 }
 
+void blockstore_impl_t::reshard_abort(void *reshard_state)
+{
+    bs_reshard_state_t *st = (bs_reshard_state_t*)reshard_state;
+    for (auto sh_it = st->old_shards.begin(); sh_it != st->old_shards.end(); sh_it++)
+    {
+        auto & to = clean_db_shards[sh_it->first];
+        to.swap(sh_it->second);
+    }
+    delete st;
+}
+
 void blockstore_impl_t::process_list(blockstore_op_t *op)
 {
     uint32_t list_pg = op->pg_number+1;
@@ -861,29 +866,4 @@ std::string blockstore_impl_t::get_op_diag(blockstore_op_t *op)
     return std::string(buf);
 }
 
-uint64_t blockstore_impl_t::get_live_entries()
-{
-    return used_blocks;
-}
-
-uint64_t blockstore_impl_t::get_live_memory()
-{
-    uint64_t used = 0;
-    for (auto & kv: clean_db_shards)
-    {
-        used += kv.second.size() * sizeof(blockstore_clean_db_t::value_type);
-    }
-    return used;
-}
-
-uint64_t blockstore_impl_t::get_garbage_entries()
-{
-    return dirty_db.size();
-}
-
-uint64_t blockstore_impl_t::get_garbage_memory()
-{
-    return (sizeof(obj_ver_id) + sizeof(dirty_entry) + 32) * dirty_db.size();
-}
-
 } // namespace v1

src/blockstore/v1/impl.h

@@ -30,8 +30,6 @@
 
 //#define BLOCKSTORE_DEBUG
 
-struct bs_test_t;
-
 namespace v1 {
 
 #include "journal.h"
@@ -98,7 +96,7 @@ struct blockstore_op_private_t
     int op_state;
 
     // Read
-    uint64_t clean_loc_used;
+    uint64_t clean_block_used;
     std::vector<copy_buffer_t> read_vec;
 
     // Sync, write
@@ -124,7 +122,6 @@ typedef uint64_t pool_pg_id_t;
 
 class blockstore_impl_t: public blockstore_i
 {
-    friend struct ::bs_test_t;
     blockstore_disk_t dsk;
 
     /******* OPTIONS *******/
@@ -223,7 +220,6 @@ class blockstore_impl_t: public blockstore_i
 
     // Read
     int dequeue_read(blockstore_op_t *read_op);
-    void release_clean(blockstore_op_t *op);
     void find_holes(std::vector<copy_buffer_t> & read_vec, uint32_t item_start, uint32_t item_end,
         std::function<int(int, bool, uint32_t, uint32_t)> callback);
     int fulfill_read(blockstore_op_t *read_op,
@@ -234,8 +230,7 @@ class blockstore_impl_t: public blockstore_i
         uint8_t *clean_entry_bitmap, int *dyn_data,
         uint32_t item_start, uint32_t item_end, uint64_t clean_loc, uint64_t clean_ver);
     int fill_partial_checksum_blocks(std::vector<copy_buffer_t> & rv, uint64_t & fulfilled,
-        uint8_t *clean_entry_bitmap, int *dyn_data, bool from_journal, uint8_t *read_buf,
+        uint8_t *clean_entry_bitmap, int *dyn_data, bool from_journal, uint8_t *read_buf, uint64_t read_offset, uint64_t read_end);
-        uint32_t read_offset, uint32_t read_end, uint32_t item_start, uint32_t item_end);
     int pad_journal_read(std::vector<copy_buffer_t> & rv, copy_buffer_t & cp,
         uint64_t dirty_offset, uint64_t dirty_end, uint64_t dirty_loc, uint8_t *csum_ptr, int *dyn_data,
         uint64_t offset, uint64_t submit_len, uint64_t & blk_begin, uint64_t & blk_end, uint8_t* & blk_buf);
@@ -286,7 +281,7 @@ class blockstore_impl_t: public blockstore_i
 
 public:
 
-    blockstore_impl_t(blockstore_config_t & config, ring_loop_i *ringloop, timerfd_manager_t *tfd, bool mock_mode = false);
+    blockstore_impl_t(blockstore_config_t & config, ring_loop_i *ringloop, timerfd_manager_t *tfd);
     ~blockstore_impl_t();
 
     void parse_config(blockstore_config_t & config);
@@ -295,6 +290,7 @@ public:
     // Reshard database for a pool
     void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
     bool reshard_continue(void *reshard_state, uint64_t chunk_limit);
+    void reshard_abort(void *reshard_state);
 
     // Event loop
     void loop();
@@ -337,10 +333,6 @@ public:
     inline uint64_t get_free_block_count() { return dsk.block_count - used_blocks; }
     inline uint32_t get_bitmap_granularity() { return dsk.disk_alignment; }
     inline uint64_t get_journal_size() { return dsk.journal_len; }
-    uint64_t get_live_entries();
-    uint64_t get_live_memory();
-    uint64_t get_garbage_entries();
-    uint64_t get_garbage_memory();
 };
 
 } // namespace v1

src/blockstore/v1/init.cpp

@@ -1,7 +1,6 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 
-#include "str_util.h"
 #include "impl.h"
 #include "internal.h"
 
@@ -31,15 +30,14 @@ blockstore_init_meta::blockstore_init_meta(blockstore_impl_t *bs)
     this->bs = bs;
 }
 
-void blockstore_init_meta::handle_event(ring_data_t *data, int buf_num, const char *op)
+void blockstore_init_meta::handle_event(ring_data_t *data, int buf_num)
 {
-    if (data->res != data->iov.iov_len)
+    if (data->res < 0)
     {
-        throw std::runtime_error(strprintf(
+        throw std::runtime_error(
-            "%s failed at offset %ju: got %s (code %d), but expected %zu",
+            std::string("read metadata failed at offset ") + std::to_string(buf_num >= 0 ? bufs[buf_num].offset : last_read_offset) +
-            op, (buf_num >= 0 ? bufs[buf_num].offset : last_read_offset), strerror(-data->res),
+            std::string(": ") + strerror(-data->res)
-            data->res, data->iov.iov_len
+        );
-        ));
     }
     if (buf_num >= 0)
     {
@@ -67,11 +65,10 @@ int blockstore_init_meta::loop()
     if (!metadata_buffer)
         throw std::runtime_error("Failed to allocate metadata read buffer");
     // Read superblock
-    hdr = (blockstore_meta_header_v2_t *)memalign_or_die(MEM_ALIGNMENT, bs->dsk.meta_block_size);
     GET_SQE();
     last_read_offset = 0;
-    data->iov = { hdr, (size_t)bs->dsk.meta_block_size };
+    data->iov = { metadata_buffer, (size_t)bs->dsk.meta_block_size };
-    data->callback = [this](ring_data_t *data) { handle_event(data, -1, "read metadata header"); };
+    data->callback = [this](ring_data_t *data) { handle_event(data, -1); };
     io_uring_prep_readv(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset);
     bs->ringloop->submit();
     submitted++;
@@ -81,8 +78,24 @@ resume_1:
         wait_state = 1;
         return 1;
     }
-    if (iszero((uint64_t*)hdr, bs->dsk.meta_block_size / sizeof(uint64_t)))
+    if (iszero((uint64_t*)metadata_buffer, bs->dsk.meta_block_size / sizeof(uint64_t)))
     {
+        {
+            blockstore_meta_header_v2_t *hdr = (blockstore_meta_header_v2_t *)metadata_buffer;
+            hdr->zero = 0;
+            hdr->magic = BLOCKSTORE_META_MAGIC_V1;
+            hdr->version = bs->dsk.meta_format;
+            hdr->meta_block_size = bs->dsk.meta_block_size;
+            hdr->data_block_size = bs->dsk.data_block_size;
+            hdr->bitmap_granularity = bs->dsk.bitmap_granularity;
+            if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_V2)
+            {
+                hdr->data_csum_type = bs->dsk.data_csum_type;
+                hdr->csum_block_size = bs->dsk.csum_block_size;
+                hdr->header_csum = 0;
+                hdr->header_csum = crc32c(0, hdr, sizeof(*hdr));
+            }
+        }
         if (bs->readonly)
         {
             printf("Skipping metadata initialization because blockstore is readonly\n");
@@ -90,11 +103,25 @@ resume_1:
         else
         {
             printf("Initializing metadata area\n");
+            GET_SQE();
+            last_read_offset = 0;
+            data->iov = (struct iovec){ metadata_buffer, (size_t)bs->dsk.meta_block_size };
+            data->callback = [this](ring_data_t *data) { handle_event(data, -1); };
+            io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset);
+            bs->ringloop->submit();
+            submitted++;
+        resume_3:
+            if (submitted > 0)
+            {
+                wait_state = 3;
+                return 1;
+            }
+            zero_on_init = true;
         }
-        zero_on_init = true;
     }
     else
     {
+        blockstore_meta_header_v2_t *hdr = (blockstore_meta_header_v2_t *)metadata_buffer;
         if (hdr->zero != 0 || hdr->magic != BLOCKSTORE_META_MAGIC_V1 || hdr->version < BLOCKSTORE_META_FORMAT_V1)
         {
             printf(
@@ -162,7 +189,7 @@ resume_1:
             printf(
                 "Configuration stored in metadata superblock"
                 " (meta_block_size=%u, data_block_size=%u, bitmap_granularity=%u, data_csum_type=%u, csum_block_size=%u)"
-                " differs from OSD configuration (%ju/%ju/%u, %u/%u).\n",
+                " differs from OSD configuration (%u/%u/%u, %u/%u).\n",
                 hdr->meta_block_size, hdr->data_block_size, hdr->bitmap_granularity,
                 hdr->data_csum_type, hdr->csum_block_size,
                 bs->dsk.meta_block_size, bs->dsk.data_block_size, bs->dsk.bitmap_granularity,
@@ -196,15 +223,12 @@ resume_2:
                 GET_SQE();
                 assert(bufs[i].size <= 0x7fffffff);
                 data->iov = { bufs[i].buf, (size_t)bufs[i].size };
+                data->callback = [this, i](ring_data_t *data) { handle_event(data, i); };
                 if (!zero_on_init)
-                {
-                    data->callback = [this, i](ring_data_t *data) { handle_event(data, i, "read metadata"); };
                     io_uring_prep_readv(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bufs[i].offset);
-                }
                 else
                 {
                     // Fill metadata with zeroes
-                    data->callback = [this, i](ring_data_t *data) { handle_event(data, i, "clear metadata"); };
                     memset(data->iov.iov_base, 0, data->iov.iov_len);
                     io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bufs[i].offset);
                 }
@@ -232,7 +256,7 @@ resume_2:
                 GET_SQE();
                 assert(bufs[i].size <= 0x7fffffff);
                 data->iov = { bufs[i].buf, (size_t)bufs[i].size };
-                data->callback = [this, i](ring_data_t *data) { handle_event(data, i, "write metadata"); };
+                data->callback = [this, i](ring_data_t *data) { handle_event(data, i); };
                 io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bufs[i].offset);
                 bs->ringloop->submit();
                 bufs[i].state = INIT_META_WRITING;
@@ -261,7 +285,7 @@ resume_2:
             GET_SQE();
             last_read_offset = (1+next_offset)*bs->dsk.meta_block_size;
             data->iov = { metadata_buffer, (size_t)bs->dsk.meta_block_size };
-            data->callback = [this](ring_data_t *data) { handle_event(data, -1, "read metadata"); };
+            data->callback = [this](ring_data_t *data) { handle_event(data, -1); };
             io_uring_prep_readv(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + (1+next_offset)*bs->dsk.meta_block_size);
             bs->ringloop->submit();
             submitted++;
@@ -278,7 +302,7 @@ resume_5:
             }
             GET_SQE();
             data->iov = { metadata_buffer, (size_t)bs->dsk.meta_block_size };
-            data->callback = [this](ring_data_t *data) { handle_event(data, -1, "write metadata"); };
+            data->callback = [this](ring_data_t *data) { handle_event(data, -1); };
             io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + (1+next_offset)*bs->dsk.meta_block_size);
             bs->ringloop->submit();
             submitted++;
@@ -293,64 +317,27 @@ resume_6:
     }
     // metadata read finished
     printf("Metadata entries loaded: %ju, free blocks: %ju / %ju\n", entries_loaded, bs->data_alloc->get_free_count(), bs->dsk.block_count);
-    if (zero_on_init && !bs->readonly)
-    {
-do_fsync:
-        if (!bs->disable_meta_fsync)
-        {
-            GET_SQE();
-            io_uring_prep_fsync(sqe, bs->dsk.meta_fd, IORING_FSYNC_DATASYNC);
-            last_read_offset = 0;
-            data->iov = { 0 };
-            data->callback = [this](ring_data_t *data) { handle_event(data, -1, "fsync metadata"); };
-            submitted++;
-            bs->ringloop->submit();
-resume_4:
-            if (submitted > 0)
-            {
-                wait_state = 4;
-                return 1;
-            }
-        }
-        if (!header_written)
-        {
-            GET_SQE();
-            hdr->zero = 0;
-            hdr->magic = BLOCKSTORE_META_MAGIC_V1;
-            hdr->version = bs->dsk.meta_format;
-            hdr->meta_block_size = bs->dsk.meta_block_size;
-            hdr->data_block_size = bs->dsk.data_block_size;
-            hdr->bitmap_granularity = bs->dsk.bitmap_granularity;
-            if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_V2)
-            {
-                hdr->data_csum_type = bs->dsk.data_csum_type;
-                hdr->csum_block_size = bs->dsk.csum_block_size;
-                hdr->header_csum = 0;
-                hdr->header_csum = crc32c(0, hdr, sizeof(*hdr));
-            }
-            header_written = true;
-            last_read_offset = 0;
-            data->iov = (struct iovec){ hdr, (size_t)bs->dsk.meta_block_size };
-            data->callback = [this](ring_data_t *data) { handle_event(data, -1, "write metadata header"); };
-            io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset);
-            bs->ringloop->submit();
-            submitted++;
-resume_3:
-            if (submitted > 0)
-            {
-                wait_state = 3;
-                return 1;
-            }
-            goto do_fsync;
-        }
-    }
     if (!bs->inmemory_meta)
     {
         free(metadata_buffer);
         metadata_buffer = NULL;
     }
-    free(hdr);
+    if (zero_on_init && !bs->disable_meta_fsync)
-    hdr = NULL;
+    {
+        GET_SQE();
+        io_uring_prep_fsync(sqe, bs->dsk.meta_fd, IORING_FSYNC_DATASYNC);
+        last_read_offset = 0;
+        data->iov = { 0 };
+        data->callback = [this](ring_data_t *data) { handle_event(data, -1); };
+        submitted++;
+        bs->ringloop->submit();
+    resume_4:
+        if (submitted > 0)
+        {
+            wait_state = 4;
+            return 1;
+        }
+    }
     return 0;
 }
 
@@ -358,8 +345,6 @@ bool blockstore_init_meta::handle_meta_block(uint8_t *buf, uint64_t entries_per_
 {
     bool updated = false;
     uint64_t max_i = entries_per_block;
-    if (done_cnt > bs->dsk.block_count)
-        return false;
     if (max_i > bs->dsk.block_count-done_cnt)
         max_i = bs->dsk.block_count-done_cnt;
     for (uint64_t i = 0; i < max_i; i++)
@@ -470,21 +455,21 @@ blockstore_init_journal::blockstore_init_journal(blockstore_impl_t *bs)
     };
 }
 
-void blockstore_init_journal::handle_event(ring_data_t *data)
+void blockstore_init_journal::handle_event(ring_data_t *data1)
 {
-    if (data->res != data->iov.iov_len)
+    if (data1->res <= 0)
     {
-        throw std::runtime_error(strprintf(
+        throw std::runtime_error(
-            "read journal failed at offset %ju: got %s (code %d), but expected %zu",
+            std::string("read journal failed at offset ") + std::to_string(journal_pos) +
-            journal_pos, strerror(-data->res), data->res, data->iov.iov_len
+            std::string(": ") + strerror(-data1->res)
-        ));
+        );
     }
     done.push_back({
         .buf = submitted_buf,
         .pos = journal_pos,
-        .len = (uint64_t)data->res,
+        .len = (uint64_t)data1->res,
     });
-    journal_pos += data->res;
+    journal_pos += data1->res;
     if (journal_pos >= bs->journal.len)
     {
         // Continue from the beginning

src/blockstore/v1/init.h

@@ -16,9 +16,7 @@ class blockstore_init_meta
     blockstore_impl_t *bs;
     int wait_state = 0;
     bool zero_on_init = false;
-    bool header_written = false;
     void *metadata_buffer = NULL;
-    blockstore_meta_header_v2_t *hdr = NULL;
     blockstore_init_meta_buf bufs[2] = {};
     int submitted = 0;
     struct io_uring_sqe *sqe;
@@ -31,7 +29,7 @@ class blockstore_init_meta
     int i = 0, j = 0;
     std::vector<uint64_t> entries_to_zero;
     bool handle_meta_block(uint8_t *buf, uint64_t count, uint64_t done_cnt);
-    void handle_event(ring_data_t *data, int buf_num, const char *op);
+    void handle_event(ring_data_t *data, int buf_num);
 public:
     blockstore_init_meta(blockstore_impl_t *bs);
     int loop();

src/blockstore/v1/journal.cpp

@@ -193,7 +193,6 @@ void blockstore_impl_t::prepare_journal_sector_write(int cur_sector, blockstore_
             (size_t)journal.block_size
         };
         data->callback = [this, flush_id = journal.submit_id](ring_data_t *data) { handle_journal_write(data, flush_id); };
-        assert(journal.sector_info[cur_sector].offset+journal.block_size <= dsk.journal_len);
         io_uring_prep_writev(
             sqe, dsk.journal_fd, &data->iov, 1, journal.offset + journal.sector_info[cur_sector].offset
         );

src/blockstore/v1/read.cpp

@@ -101,8 +101,8 @@ int blockstore_impl_t::fulfill_read(blockstore_op_t *read_op,
                     .copy_flags = COPY_BUF_JOURNAL|COPY_BUF_CSUM_FILL,
                     .offset = blk_begin,
                     .len = blk_end-blk_begin,
-                    .csum_buf = (!csum ? NULL : (csum + (blk_begin/dsk.csum_block_size -
+                    .csum_buf = (csum + (blk_begin/dsk.csum_block_size -
-                        item_start/dsk.csum_block_size) * (dsk.data_csum_type & 0xFF))),
+                        item_start/dsk.csum_block_size) * (dsk.data_csum_type & 0xFF)),
                     .dyn_data = dyn_data,
                 });
                 if (dyn_data)
@@ -134,7 +134,7 @@ int blockstore_impl_t::fulfill_read(blockstore_op_t *read_op,
                 // If we don't track it then we may IN THEORY read another object's data:
                 // submit read -> remove the object -> flush remove -> overwrite with another object -> finish read
                 // Very improbable, but possible
-                PRIV(read_op)->clean_loc_used = UINT64_MAX;
+                PRIV(read_op)->clean_block_used = 1;
             }
             rv.insert(rv.begin() + pos, el);
             fulfilled += el.len;
@@ -167,8 +167,7 @@ uint8_t* blockstore_impl_t::get_clean_entry_bitmap(uint64_t block_loc, int offse
 }
 
 int blockstore_impl_t::fill_partial_checksum_blocks(std::vector<copy_buffer_t> & rv, uint64_t & fulfilled,
-    uint8_t *clean_entry_bitmap, int *dyn_data, bool from_journal, uint8_t *read_buf,
+    uint8_t *clean_entry_bitmap, int *dyn_data, bool from_journal, uint8_t *read_buf, uint64_t read_offset, uint64_t read_end)
-    uint32_t read_offset, uint32_t read_end, uint32_t item_start, uint32_t item_end)
 {
     if (read_end == read_offset)
         return 0;
@@ -176,38 +175,10 @@ int blockstore_impl_t::fill_partial_checksum_blocks(std::vector<copy_buffer_t> &
     read_buf -= read_offset;
     uint32_t last_block = (read_end-1)/dsk.csum_block_size;
     uint32_t start_block = read_offset/dsk.csum_block_size;
-    uint32_t item_start_block = item_start/dsk.csum_block_size;
     uint32_t end_block = 0;
-    auto zero_range = [&](int pos, bool alloc, uint32_t cur_start, uint32_t cur_end)
-    {
-        if (alloc)
-            return 0;
-        copy_buffer_t el = {
-            .copy_flags = COPY_BUF_ZERO,
-            .offset = cur_start,
-            .len = cur_end-cur_start,
-        };
-        rv.insert(rv.begin() + pos, el);
-        if (read_buf)
-            memset(read_buf + el.offset - read_offset, 0, el.len);
-        fulfilled += el.len;
-        return 1;
-    };
-    if (read_offset < item_start)
-    {
-        // Zero-fill the beginning
-        find_holes(rv, read_offset, item_start, zero_range);
-        read_offset = item_start;
-    }
-    if (read_end > item_end)
-    {
-        // Zero-fill the end
-        find_holes(rv, item_end, read_end, zero_range);
-        read_end = item_end;
-    }
     while (start_block <= last_block)
     {
-        if (read_range_fulfilled(rv, fulfilled, read_buf, from_journal ? NULL : clean_entry_bitmap,
+        if (read_range_fulfilled(rv, fulfilled, read_buf, clean_entry_bitmap,
             start_block*dsk.csum_block_size < read_offset ? read_offset : start_block*dsk.csum_block_size,
             (start_block+1)*dsk.csum_block_size > read_end ? read_end : (start_block+1)*dsk.csum_block_size))
         {
@@ -219,7 +190,7 @@ int blockstore_impl_t::fill_partial_checksum_blocks(std::vector<copy_buffer_t> &
             // Find a sequence of checksum blocks required to be read
             end_block = start_block;
             while ((end_block+1)*dsk.csum_block_size < read_end &&
-                !read_range_fulfilled(rv, fulfilled, read_buf, from_journal ? NULL : clean_entry_bitmap,
+                !read_range_fulfilled(rv, fulfilled, read_buf, clean_entry_bitmap,
                     (end_block+1)*dsk.csum_block_size < read_offset ? read_offset : (end_block+1)*dsk.csum_block_size,
                     (end_block+2)*dsk.csum_block_size > read_end ? read_end : (end_block+2)*dsk.csum_block_size))
             {
@@ -231,10 +202,8 @@ int blockstore_impl_t::fill_partial_checksum_blocks(std::vector<copy_buffer_t> &
                 .copy_flags = COPY_BUF_CSUM_FILL | (from_journal ? COPY_BUF_JOURNALED_BIG : 0),
                 .offset = start_block*dsk.csum_block_size,
                 .len = (end_block-start_block)*dsk.csum_block_size,
-                // save checksum reference if we're reading clean data from the journal
+                // save clean_entry_bitmap if we're reading clean data from the journal
-                .csum_buf = from_journal
+                .csum_buf = from_journal ? clean_entry_bitmap : NULL,
-                    ? clean_entry_bitmap + dsk.clean_entry_bitmap_size + (start_block-item_start_block)*(dsk.data_csum_type & 0xFF)
-                    : NULL,
                 .dyn_data = dyn_data,
             });
             if (dyn_data)
@@ -257,11 +226,6 @@ bool blockstore_impl_t::read_range_fulfilled(std::vector<copy_buffer_t> & rv, ui
     {
         if (alloc)
             return 0;
-        if (!clean_entry_bitmap)
-        {
-            all_done = false;
-            return 0;
-        }
         int diff = 0;
         uint32_t bmp_start = cur_start/dsk.bitmap_granularity;
         uint32_t bmp_end = cur_end/dsk.bitmap_granularity;
@@ -359,7 +323,7 @@ bool blockstore_impl_t::read_checksum_block(blockstore_op_t *op, int rv_pos, uin
             {
                 iov[n_iov++] = (struct iovec){ (uint8_t*)op->buf+cur_start-op->offset, lim_end-cur_start };
                 rv.insert(rv.begin() + pos, (copy_buffer_t){
-                    .copy_flags = COPY_BUF_DATA|COPY_BUF_COALESCED,
+                    .copy_flags = COPY_BUF_DATA,
                     .offset = cur_start,
                     .len = lim_end-cur_start,
                 });
@@ -397,10 +361,10 @@ bool blockstore_impl_t::read_checksum_block(blockstore_op_t *op, int rv_pos, uin
         PRIV(op)->pending_ops++;
         io_uring_prep_readv(sqe, submit_fd, iov + n_pos, n_cur, submit_offset + clean_loc + item_start + d_pos);
         data->callback = [this, op](ring_data_t *data) { handle_read_event(data, op); };
-        if (n_pos > 0 || n_iov > IOV_MAX)
+        if (n_pos > 0 || n_pos + IOV_MAX < n_iov)
         {
             uint32_t d_len = 0;
-            for (int i = 0; i < n_cur; i++)
+            for (int i = 0; i < IOV_MAX; i++)
                 d_len += iov[n_pos+i].iov_len;
             data->iov.iov_len = d_len;
             d_pos += d_len;
@@ -412,7 +376,7 @@ bool blockstore_impl_t::read_checksum_block(blockstore_op_t *op, int rv_pos, uin
     {
         // Reads running parallel to flushes of the same clean block may read
         // a mixture of old and new data. So we don't verify checksums for such blocks.
-        PRIV(op)->clean_loc_used = UINT64_MAX;
+        PRIV(op)->clean_block_used = 1;
     }
     return true;
 }
@@ -438,7 +402,7 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
     }
     uint64_t fulfilled = 0;
     PRIV(read_op)->pending_ops = 0;
-    PRIV(read_op)->clean_loc_used = 0;
+    PRIV(read_op)->clean_block_used = 0;
     auto & rv = PRIV(read_op)->read_vec;
     uint64_t result_version = 0;
     if (dirty_found)
@@ -551,50 +515,26 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
     return 2;
 undo_read:
     // need to wait. undo added requests, don't dequeue op
-    release_clean(read_op);
+    if (dsk.csum_block_size > dsk.bitmap_granularity)
-    for (auto & vec: rv)
     {
-        if ((vec.copy_flags & COPY_BUF_CSUM_FILL) && vec.buf)
+        for (auto & vec: rv)
         {
-            free(vec.buf);
+            if ((vec.copy_flags & COPY_BUF_CSUM_FILL) && vec.buf)
-            vec.buf = NULL;
+            {
-        }
+                free(vec.buf);
-        if (vec.dyn_data && --(*vec.dyn_data) == 0) // refcount
+                vec.buf = NULL;
-        {
+            }
-            free(vec.dyn_data);
+            if (vec.dyn_data && --(*vec.dyn_data) == 0) // refcount
-            vec.dyn_data = NULL;
+            {
+                free(vec.dyn_data);
+                vec.dyn_data = NULL;
+            }
         }
     }
     rv.clear();
     return 0;
 }
 
-void blockstore_impl_t::release_clean(blockstore_op_t *op)
-{
-    if (PRIV(op)->clean_loc_used == UINT64_MAX)
-    {
-        PRIV(op)->clean_loc_used = 0;
-    }
-    if (PRIV(op)->clean_loc_used)
-    {
-        // Release clean data block
-        auto uo_it = used_clean_objects.find(PRIV(op)->clean_loc_used - 1);
-        if (uo_it != used_clean_objects.end())
-        {
-            uo_it->second.refs--;
-            if (uo_it->second.refs <= 0)
-            {
-                if (uo_it->second.was_freed)
-                {
-                    data_alloc->set((PRIV(op)->clean_loc_used - 1) / dsk.data_block_size, false);
-                }
-                used_clean_objects.erase(uo_it);
-            }
-        }
-        PRIV(op)->clean_loc_used = 0;
-    }
-}
-
 int blockstore_impl_t::pad_journal_read(std::vector<copy_buffer_t> & rv, copy_buffer_t & cp,
     // FIXME Passing dirty_entry& would be nicer
     uint64_t dirty_offset, uint64_t dirty_end, uint64_t dirty_loc, uint8_t *csum_ptr, int *dyn_data,
@@ -658,15 +598,11 @@ bool blockstore_impl_t::fulfill_clean_read(blockstore_op_t *read_op, uint64_t &
     {
         auto & rv = PRIV(read_op)->read_vec;
         int req = fill_partial_checksum_blocks(rv, fulfilled, clean_entry_bitmap, dyn_data, from_journal,
-            (uint8_t*)read_op->buf, read_op->offset, read_op->offset+read_op->len, item_start, item_end);
+            (uint8_t*)read_op->buf, read_op->offset, read_op->offset+read_op->len);
         if (!inmemory_meta && !from_journal && req > 0)
         {
             // Read checksums from disk
             uint8_t *csum_buf = read_clean_meta_block(read_op, clean_loc, rv.size()-req);
-            if (!csum_buf)
-            {
-                return false;
-            }
             for (int i = req; i > 0; i--)
             {
                 rv[rv.size()-i].csum_buf = csum_buf;
@@ -679,12 +615,13 @@ bool blockstore_impl_t::fulfill_clean_read(blockstore_op_t *read_op, uint64_t &
                 return false;
             }
         }
-        PRIV(read_op)->clean_loc_used = req > 0 ? UINT64_MAX : 0;
+        PRIV(read_op)->clean_block_used = req > 0;
     }
     else if (from_journal)
     {
         // Don't scan bitmap - journal writes don't have holes (internal bitmap)!
-        uint8_t *csum = !dsk.csum_block_size ? 0 : (clean_entry_bitmap + dsk.clean_entry_bitmap_size);
+        uint8_t *csum = !dsk.csum_block_size ? 0 : (clean_entry_bitmap + dsk.clean_entry_bitmap_size +
+            item_start/dsk.csum_block_size*(dsk.data_csum_type & 0xFF));
         if (!fulfill_read(read_op, fulfilled, item_start, item_end,
             (BS_ST_BIG_WRITE | BS_ST_STABLE), 0, clean_loc + item_start, 0, csum, dyn_data))
         {
@@ -729,10 +666,6 @@ bool blockstore_impl_t::fulfill_clean_read(blockstore_op_t *read_op, uint64_t &
                 {
                     // Read checksums from disk
                     csum_buf = read_clean_meta_block(read_op, clean_loc, PRIV(read_op)->read_vec.size());
-                    if (!csum_buf)
-                    {
-                        return false;
-                    }
                     csum_done = true;
                 }
                 uint8_t *csum = !dsk.csum_block_size ? 0 : (csum_buf + 2*dsk.clean_entry_bitmap_size + bmp_start*(dsk.data_csum_type & 0xFF));
@@ -747,13 +680,13 @@ bool blockstore_impl_t::fulfill_clean_read(blockstore_op_t *read_op, uint64_t &
         }
     }
     // Increment reference counter if clean data is being read from the disk
-    if (PRIV(read_op)->clean_loc_used == UINT64_MAX)
+    if (PRIV(read_op)->clean_block_used)
     {
         auto & uo = used_clean_objects[clean_loc];
         uo.refs++;
         if (dsk.csum_block_size && flusher->is_mutated(clean_loc))
             uo.was_changed = true;
-        PRIV(read_op)->clean_loc_used = clean_loc + 1;
+        PRIV(read_op)->clean_block_used = clean_loc;
     }
     return true;
 }
@@ -793,18 +726,12 @@ bool blockstore_impl_t::verify_padded_checksums(uint8_t *clean_entry_bitmap, uin
         while (pos < iov[i].iov_len)
         {
             uint32_t start = pos;
-            uint8_t bit = 1;
+            uint8_t bit = (clean_entry_bitmap[bmp_pos >> 3] >> (bmp_pos & 0x7)) & 1;
-            if (clean_entry_bitmap)
+            while (pos < iov[i].iov_len && ((clean_entry_bitmap[bmp_pos >> 3] >> (bmp_pos & 0x7)) & 1) == bit)
             {
-                bit = (clean_entry_bitmap[bmp_pos >> 3] >> (bmp_pos & 0x7)) & 1;
+                pos += dsk.bitmap_granularity;
-                while (pos < iov[i].iov_len && ((clean_entry_bitmap[bmp_pos >> 3] >> (bmp_pos & 0x7)) & 1) == bit)
+                bmp_pos++;
-                {
-                    pos += dsk.bitmap_granularity;
-                    bmp_pos++;
-                }
             }
-            else
-                pos = iov[i].iov_len;
             uint32_t len = pos-start;
             auto buf = (uint8_t*)iov[i].iov_base+start;
             while (block_done+len >= dsk.csum_block_size)
@@ -881,7 +808,7 @@ bool blockstore_impl_t::verify_clean_padded_checksums(blockstore_op_t *op, uint6
 {
     uint32_t offset = clean_loc % dsk.data_block_size;
     if (from_journal)
-        return verify_padded_checksums(NULL, dyn_data, offset, iov, n_iov, bad_block_cb);
+        return verify_padded_checksums(dyn_data, dyn_data + dsk.clean_entry_bitmap_size, offset, iov, n_iov, bad_block_cb);
     clean_loc = (clean_loc / dsk.data_block_size) * dsk.data_block_size;
     if (!dyn_data)
     {
@@ -909,7 +836,7 @@ void blockstore_impl_t::handle_read_event(ring_data_t *data, blockstore_op_t *op
             void *meta_block = NULL;
             if (dsk.csum_block_size > dsk.bitmap_granularity)
             {
-                for (int i = 0; i < rv.size(); i++)
+                for (int i = rv.size()-1; i >= 0 && (rv[i].copy_flags & COPY_BUF_CSUM_FILL); i--)
                 {
                     if (rv[i].copy_flags & COPY_BUF_META_BLOCK)
                     {
@@ -919,41 +846,8 @@ void blockstore_impl_t::handle_read_event(ring_data_t *data, blockstore_op_t *op
                         rv[i].buf = NULL;
                         continue;
                     }
-                    if (rv[i].copy_flags & COPY_BUF_ZERO)
+                    struct iovec *iov = (struct iovec*)((uint8_t*)rv[i].buf + (rv[i].len & 0xFFFFFFFF));
-                    {
+                    int n_iov = rv[i].len >> 32;
-                        // Zero read
-                        continue;
-                    }
-                    if (rv[i].copy_flags & COPY_BUF_COALESCED)
-                    {
-                        // Sub-block shared with another read. Skip
-                        continue;
-                    }
-                    if ((rv[i].copy_flags & COPY_BUF_JOURNAL) && journal.inmemory)
-                    {
-                        // Do not check journal checksums in-memory
-                        continue;
-                    }
-                    iovec single_iov = {};
-                    iovec *iov = NULL;
-                    int n_iov = 0;
-                    if (rv[i].copy_flags & COPY_BUF_CSUM_FILL)
-                    {
-                        // Padded, buffer list passed using a 'creepy way'
-                        iov = (struct iovec*)((uint8_t*)rv[i].buf + (rv[i].len & 0xFFFFFFFF));
-                        n_iov = rv[i].len >> 32;
-                    }
-                    else
-                    {
-                        // Not padded, buffer is fully within the input buffer
-                        assert(op->buf);
-                        assert(rv[i].csum_buf);
-                        iov = &single_iov;
-                        n_iov = 1;
-                        assert(rv[i].offset >= op->offset);
-                        assert(rv[i].offset + rv[i].len <= op->offset + op->len);
-                        single_iov = { .iov_base = op->buf + rv[i].offset - op->offset, .iov_len = rv[i].len };
-                    }
                     bool ok = true;
                     if (rv[i].copy_flags & COPY_BUF_JOURNAL)
                     {
@@ -1051,7 +945,23 @@ void blockstore_impl_t::handle_read_event(ring_data_t *data, blockstore_op_t *op
                 meta_block = NULL;
             }
         }
-        release_clean(op);
+        if (PRIV(op)->clean_block_used)
+        {
+            // Release clean data block
+            auto uo_it = used_clean_objects.find(PRIV(op)->clean_block_used);
+            if (uo_it != used_clean_objects.end())
+            {
+                uo_it->second.refs--;
+                if (uo_it->second.refs <= 0)
+                {
+                    if (uo_it->second.was_freed)
+                    {
+                        data_alloc->set(PRIV(op)->clean_block_used, false);
+                    }
+                    used_clean_objects.erase(uo_it);
+                }
+            }
+        }
         if (!journal.inmemory)
         {
             // Release journal sector usage

src/blockstore/v1/stable.cpp

@@ -491,7 +491,7 @@ void blockstore_impl_t::mark_stable(obj_ver_id v, bool forget_dirty)
                     if (!exists)
                     {
                         uint64_t space_id = dirty_it->first.oid.inode;
-                        if (no_inode_stats.find(dirty_it->first.oid.inode >> (64-POOL_ID_BITS)) != no_inode_stats.end())
+                        if (no_inode_stats[dirty_it->first.oid.inode >> (64-POOL_ID_BITS)])
                             space_id = space_id & ~(((uint64_t)1 << (64-POOL_ID_BITS)) - 1);
                         inode_space_stats[space_id] += dsk.data_block_size;
                         used_blocks++;
@@ -501,7 +501,7 @@ void blockstore_impl_t::mark_stable(obj_ver_id v, bool forget_dirty)
                 else if (IS_DELETE(dirty_it->second.state))
                 {
                     uint64_t space_id = dirty_it->first.oid.inode;
-                    if (no_inode_stats.find(dirty_it->first.oid.inode >> (64-POOL_ID_BITS)) != no_inode_stats.end())
+                    if (no_inode_stats[dirty_it->first.oid.inode >> (64-POOL_ID_BITS)])
                         space_id = space_id & ~(((uint64_t)1 << (64-POOL_ID_BITS)) - 1);
                     auto & sp = inode_space_stats[space_id];
                     if (sp > dsk.data_block_size)

src/blockstore/v1/write.cpp

@@ -368,9 +368,9 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
         }
         data->iov.iov_len = op->len + stripe_offset + stripe_end; // to check it in the callback
         data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
-        const uint64_t write_offset = (loc * dsk.data_block_size) + op->offset - stripe_offset;
+        io_uring_prep_writev(
-        assert(write_offset+op->len+stripe_offset+stripe_end <= dsk.block_count*dsk.data_block_size);
+            sqe, dsk.data_fd, PRIV(op)->iov_zerofill, vcnt, dsk.data_offset + (loc * dsk.data_block_size) + op->offset - stripe_offset
-        io_uring_prep_writev(sqe, dsk.data_fd, PRIV(op)->iov_zerofill, vcnt, dsk.data_offset + write_offset);
+        );
         PRIV(op)->pending_ops = 1;
         if (!(dirty_it->second.state & BS_ST_INSTANT))
         {
@@ -495,8 +495,9 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
                 .op = op,
             });
             data2->callback = [this, flush_id = journal.submit_id](ring_data_t *data) { handle_journal_write(data, flush_id); };
-            assert(journal.next_free+op->len <= dsk.journal_len);
+            io_uring_prep_writev(
-            io_uring_prep_writev(sqe2, dsk.journal_fd, &data2->iov, 1, journal.offset + journal.next_free);
+                sqe2, dsk.journal_fd, &data2->iov, 1, journal.offset + journal.next_free
+            );
             PRIV(op)->pending_ops++;
         }
         else

src/client/CMakeLists.txt

@@ -1,24 +1,8 @@
-cmake_minimum_required(VERSION 2.8...3.30)
+cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
 # libvitastor_common.a
-add_library(vitastor_common STATIC
-	etcd_state_client.cpp
-	msgr_stop.cpp
-	msgr_op.cpp
-	../../json11/json11.cpp
-	osd_ops.cpp
-	pg_states.cpp
-	../util/allocator.cpp
-	../util/addr_util.cpp
-	../util/timerfd_manager.cpp
-	../util/str_util.cpp
-	../util/json_util.cpp
-)
-target_compile_options(vitastor_common PUBLIC -fPIC)
-
-# libvitastor_net.a
 set(MSGR_RDMA "")
 if (IBVERBS_LIBRARIES)
 	set(MSGR_RDMA "msgr_rdma.cpp")
@@ -27,32 +11,24 @@ set(MSGR_RDMACM "")
 if (RDMACM_LIBRARIES)
 	set(MSGR_RDMACM "msgr_rdmacm.cpp")
 endif (RDMACM_LIBRARIES)
-add_library(vitastor_net STATIC
+add_library(vitastor_common STATIC
-	../util/epoll_manager.cpp
+	../util/epoll_manager.cpp etcd_state_client.cpp messenger.cpp ../util/addr_util.cpp
-	etcd_state_client_http.cpp
+	msgr_stop.cpp msgr_op.cpp msgr_send.cpp msgr_receive.cpp ../util/ringloop.cpp ../../json11/json11.cpp
-	messenger.cpp
+	http_client.cpp osd_ops.cpp pg_states.cpp ../util/timerfd_manager.cpp ../util/str_util.cpp ../util/json_util.cpp ${MSGR_RDMA} ${MSGR_RDMACM}
-	msgr_iothread.cpp
-	msgr_send.cpp
-	msgr_receive.cpp
-	../util/ringloop.cpp
-	http_client.cpp
-	${MSGR_RDMA}
-	${MSGR_RDMACM}
 )
-target_link_libraries(vitastor_net pthread vitastor_common)
+target_link_libraries(vitastor_common pthread)
-target_compile_options(vitastor_net PUBLIC -fPIC)
+target_compile_options(vitastor_common PUBLIC -fPIC)
 
 # libvitastor_client.so
 add_library(vitastor_client SHARED
 	cluster_client.cpp
-	cluster_client_real.cpp
 	cluster_client_list.cpp
 	cluster_client_wb.cpp
 	vitastor_c.cpp
 )
 set_target_properties(vitastor_client PROPERTIES PUBLIC_HEADER "client/vitastor_c.h")
 target_link_libraries(vitastor_client
-	vitastor_net
+	vitastor_common
 	vitastor_cli
 	${LIBURING_LIBRARIES}
 	${IBVERBS_LIBRARIES}
@@ -76,6 +52,9 @@ if (${WITH_FIO})
 		../util/rw_blocking.cpp
 		../util/addr_util.cpp
 	)
+	target_link_libraries(fio_vitastor_sec
+		tcmalloc_minimal
+	)
 endif (${WITH_FIO})
 
 # vitastor-nbd
@@ -119,13 +98,10 @@ endif (${WITH_QEMU})
 add_executable(test_cluster_client
 	EXCLUDE_FROM_ALL
 	../test/test_cluster_client.cpp
-	cluster_client.cpp
+	pg_states.cpp osd_ops.cpp cluster_client.cpp cluster_client_list.cpp cluster_client_wb.cpp msgr_op.cpp ../test/mock/messenger.cpp msgr_stop.cpp
-	cluster_client_list.cpp
+	etcd_state_client.cpp ../util/timerfd_manager.cpp ../util/addr_util.cpp ../util/str_util.cpp ../util/json_util.cpp ../../json11/json11.cpp
-	cluster_client_wb.cpp
-	../test/mock/messenger.cpp
-	etcd_state_client_mock.cpp
 )
-target_link_libraries(test_cluster_client vitastor_common ${LIBURING_LIBRARIES})
+target_compile_definitions(test_cluster_client PUBLIC -D__MOCK__)
 target_include_directories(test_cluster_client BEFORE PUBLIC ${CMAKE_SOURCE_DIR}/src/test/mock)
 add_dependencies(build_tests test_cluster_client)
 add_test(NAME test_cluster_client COMMAND test_cluster_client)

src/client/cluster_client.cpp

@@ -11,7 +11,7 @@
 #define TRY_SEND_CONNECTING 1
 #define TRY_SEND_OK 2
 
-cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json config, std::unique_ptr<etcd_state_client_t> st_cli_ptr)
+cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json config)
 {
     wb = new writeback_cache_t();
 
@@ -27,7 +27,7 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
     msgr.ringloop = ringloop;
     msgr.repeer_pgs = [this](osd_num_t peer_osd)
     {
-        if (msgr.osd_peers.find(peer_osd) != msgr.osd_peers.end())
+        if (msgr.osd_peer_fds.find(peer_osd) != msgr.osd_peer_fds.end())
         {
             // peer_osd just connected
             continue_ops();
@@ -47,29 +47,29 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
     msgr.exec_op = [this](osd_op_t *op)
     {
         // Garbage in
-        fprintf(stderr, "Can't handle incoming operation from client %lu\n", op->client_id);
+        fprintf(stderr, "Incoming garbage from peer %d\n", op->peer_fd);
-        msgr.stop_client(op->client_id);
+        msgr.stop_client(op->peer_fd);
         delete op;
     };
     msgr.parse_config(config);
 
-    st_cli = std::move(st_cli_ptr);
+    st_cli.tfd = tfd;
-    st_cli->on_load_config_hook = [this](json11::Json::object & cfg) { on_load_config_hook(cfg); };
+    st_cli.on_load_config_hook = [this](json11::Json::object & cfg) { on_load_config_hook(cfg); };
-    st_cli->on_change_osd_state_hook = [this](uint64_t peer_osd) { on_change_osd_state_hook(peer_osd); };
+    st_cli.on_change_osd_state_hook = [this](uint64_t peer_osd) { on_change_osd_state_hook(peer_osd); };
-    st_cli->on_change_pool_config_hook = [this]() { on_change_pool_config_hook(); };
+    st_cli.on_change_pool_config_hook = [this]() { on_change_pool_config_hook(); };
-    st_cli->on_change_pg_config_hook = [this]() { on_change_pool_config_hook(); };
+    st_cli.on_change_pg_config_hook = [this]() { on_change_pool_config_hook(); };
-    st_cli->on_change_pg_state_hook = [this](pool_id_t pool_id, pg_num_t pg_num, osd_num_t prev_primary) { on_change_pg_state_hook(pool_id, pg_num, prev_primary); };
+    st_cli.on_change_pg_state_hook = [this](pool_id_t pool_id, pg_num_t pg_num, osd_num_t prev_primary) { on_change_pg_state_hook(pool_id, pg_num, prev_primary); };
-    st_cli->on_change_node_placement_hook = [this]() { on_change_node_placement_hook(); };
+    st_cli.on_change_node_placement_hook = [this]() { on_change_node_placement_hook(); };
-    st_cli->on_load_pgs_hook = [this](bool success) { on_load_pgs_hook(success); };
+    st_cli.on_load_pgs_hook = [this](bool success) { on_load_pgs_hook(success); };
-    st_cli->on_reload_hook = [this]() { this->st_cli->load_global_config(); };
+    st_cli.on_reload_hook = [this]() { st_cli.load_global_config(); };
 
-    st_cli->parse_config(config);
+    st_cli.parse_config(config);
-    st_cli->infinite_start = false;
+    st_cli.infinite_start = false;
     if (!config["client_infinite_start"].is_null())
     {
-        st_cli->infinite_start = config["client_infinite_start"].bool_value();
+        st_cli.infinite_start = config["client_infinite_start"].bool_value();
     }
-    st_cli->load_global_config();
+    st_cli.load_global_config();
 
     scrap_buffer_size = SCRAP_BUFFER_SIZE;
     scrap_buffer = malloc_or_die(scrap_buffer_size);
@@ -156,7 +156,7 @@ void cluster_client_t::continue_raw_ops(osd_num_t peer_osd)
     {
         auto op = it->second;
         op->op_type = OSD_OP_OUT;
-        op->client_id = msgr.osd_peers.at(peer_osd)->client_id;
+        op->peer_fd = msgr.osd_peer_fds.at(peer_osd);
         msgr.outbox_push(op);
         raw_ops.erase(it++);
     }
@@ -469,7 +469,7 @@ void cluster_client_t::on_load_config_hook(json11::Json::object & etcd_global_co
         auto etcd_report_interval = config["etcd_report_interval"].uint64_value();
         if (!etcd_report_interval)
             etcd_report_interval = 5;
-        client_wait_up_timeout = 1+etcd_report_interval+(st_cli->max_etcd_attempts*(2*st_cli->etcd_quick_timeout)+999)/1000;
+        client_wait_up_timeout = 1+etcd_report_interval+(st_cli.max_etcd_attempts*(2*st_cli.etcd_quick_timeout)+999)/1000;
     }
     // log_level
     log_level = config["log_level"].uint64_value();
@@ -482,8 +482,8 @@ void cluster_client_t::on_load_config_hook(json11::Json::object & etcd_global_co
         client_hostname = new_hostname;
     }
     msgr.parse_config(config);
-    st_cli->parse_config(config);
+    st_cli.parse_config(config);
-    st_cli->load_pgs();
+    st_cli.load_pgs();
 }
 
 osd_num_t cluster_client_t::select_random_osd(const std::vector<osd_num_t> & osds)
@@ -492,7 +492,7 @@ osd_num_t cluster_client_t::select_random_osd(const std::vector<osd_num_t> & osd
     int alive_count = 0;
     for (auto & osd_num: osds)
     {
-        if (!st_cli->peer_states[osd_num].is_null())
+        if (!st_cli.peer_states[osd_num].is_null())
             alive_set[alive_count++] = osd_num;
     }
     if (!alive_count)
@@ -509,7 +509,7 @@ osd_num_t cluster_client_t::select_nearest_osd(const std::vector<osd_num_t> & os
         while (self_tree_metrics.find(cur_id) == self_tree_metrics.end())
         {
             self_tree_metrics[cur_id] = metric++;
-            json11::Json cur_placement = st_cli->node_placement[cur_id];
+            json11::Json cur_placement = st_cli.node_placement[cur_id];
             cur_id = cur_placement["parent"].string_value();
         }
         if (cur_id != "")
@@ -529,7 +529,7 @@ osd_num_t cluster_client_t::select_nearest_osd(const std::vector<osd_num_t> & os
         }
         else
         {
-            auto & peer_state = st_cli->peer_states[osd_num];
+            auto & peer_state = st_cli.peer_states[osd_num];
             if (!peer_state.is_null())
             {
                 metric = self_tree_metrics[""];
@@ -539,7 +539,7 @@ osd_num_t cluster_client_t::select_nearest_osd(const std::vector<osd_num_t> & os
                 while (seen.find(cur_id) == seen.end())
                 {
                     seen.insert(cur_id);
-                    json11::Json cur_placement = st_cli->node_placement[cur_id];
+                    json11::Json cur_placement = st_cli.node_placement[cur_id];
                     std::string cur_parent = cur_placement["parent"].string_value();
                     cur_id = (!first || cur_parent != "" ? cur_parent : peer_state["host"].string_value());
                     first = false;
@@ -564,7 +564,7 @@ osd_num_t cluster_client_t::select_nearest_osd(const std::vector<osd_num_t> & os
 
 void cluster_client_t::on_load_pgs_hook(bool success)
 {
-    for (auto & pool_item: st_cli->pool_config)
+    for (auto & pool_item: st_cli.pool_config)
     {
         pg_counts[pool_item.first] = pool_item.second.real_pg_count;
     }
@@ -584,13 +584,13 @@ void cluster_client_t::on_load_pgs_hook(bool success)
 
 void cluster_client_t::on_change_pool_config_hook()
 {
-    for (auto & pool_item: st_cli->pool_config)
+    for (auto & pool_item: st_cli.pool_config)
     {
         if (pg_counts[pool_item.first] != pool_item.second.real_pg_count)
         {
             if (log_level > 2 && pg_counts[pool_item.first])
             {
-                fprintf(stderr, "Pool %u (%s) PG count changed from %lu to %lu\n", pool_item.first, pool_item.second.name.c_str(),
+                printf("Pool %u (%s) PG count changed from %lu to %lu\n", pool_item.first, pool_item.second.name.c_str(),
                     pg_counts[pool_item.first], pool_item.second.real_pg_count);
             }
             // At this point, all pool operations should have been suspended
@@ -612,7 +612,7 @@ void cluster_client_t::on_change_pool_config_hook()
 
 void cluster_client_t::on_change_pg_state_hook(pool_id_t pool_id, pg_num_t pg_num, osd_num_t prev_primary)
 {
-    auto & pg_cfg = st_cli->pool_config[pool_id].pg_config[pg_num];
+    auto & pg_cfg = st_cli.pool_config[pool_id].pg_config[pg_num];
     if (pg_cfg.cur_primary != prev_primary)
     {
         // Repeat this PG operations because an OSD which stopped being primary may not fsync operations
@@ -630,8 +630,8 @@ bool cluster_client_t::get_immediate_commit(uint64_t inode)
     pool_id_t pool_id = INODE_POOL(inode);
     if (!pool_id)
         return true;
-    auto pool_it = st_cli->pool_config.find(pool_id);
+    auto pool_it = st_cli.pool_config.find(pool_id);
-    if (pool_it == st_cli->pool_config.end())
+    if (pool_it == st_cli.pool_config.end())
         return true;
     return pool_it->second.immediate_commit == IMMEDIATE_ALL;
 }
@@ -641,7 +641,7 @@ void cluster_client_t::on_change_osd_state_hook(uint64_t peer_osd)
     osd_tree_metrics.erase(peer_osd);
     if (msgr.wanted_peers.find(peer_osd) != msgr.wanted_peers.end())
     {
-        msgr.connect_peer(peer_osd, st_cli->peer_states[peer_osd]);
+        msgr.connect_peer(peer_osd, st_cli.peer_states[peer_osd]);
         continue_lists();
     }
 }
@@ -871,13 +871,13 @@ void cluster_client_t::execute_cas(cluster_op_t *op)
         if (op->retval != expected && op->retval >= 0)
             op->retval = -EIO;
         op->retval = op->retval == -EPIPE ? -EINTR : op->retval;
-        auto peer_it = msgr.osd_peers.find(op->parts[0].osd_num);
+        auto peer_it = msgr.osd_peer_fds.find(op->parts[0].osd_num);
         if (op->retval != 0 || (op->flags & OP_IMMEDIATE_COMMIT))
         {
             auto cb = std::move(op->callback);
             cb(op);
         }
-        else if (peer_it == msgr.osd_peers.end())
+        else if (peer_it == msgr.osd_peer_fds.end())
         {
             // Care must be taken to make sure that the client doesn't reconnect to the OSD
             // before executing the previously completed operation callback (!)
@@ -888,10 +888,10 @@ void cluster_client_t::execute_cas(cluster_op_t *op)
         else
         {
             // CAS writes have a built-in sync
-            osd_client_t *cl = peer_it->second;
+            auto peer_fd = peer_it->second;
             *part = (osd_op_t){
                 .op_type = OSD_OP_OUT,
-                .client_id = cl->client_id,
+                .peer_fd = peer_fd,
                 .req = {
                     .hdr = {
                         .magic = SECONDARY_OSD_OP_MAGIC,
@@ -936,8 +936,8 @@ bool cluster_client_t::check_rw(cluster_op_t *op)
         cb(op);
         return false;
     }
-    auto pool_it = st_cli->pool_config.find(pool_id);
+    auto pool_it = st_cli.pool_config.find(pool_id);
-    if (pool_it == st_cli->pool_config.end() || pool_it->second.real_pg_count == 0)
+    if (pool_it == st_cli.pool_config.end() || pool_it->second.real_pg_count == 0)
     {
         // Pools are loaded, but this one is unknown
         op->retval = -EINVAL;
@@ -960,8 +960,8 @@ bool cluster_client_t::check_rw(cluster_op_t *op)
     }
     if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && !(op->flags & OSD_OP_IGNORE_READONLY))
     {
-        auto ino_it = st_cli->inode_config.find(op->inode);
+        auto ino_it = st_cli.inode_config.find(op->inode);
-        if (ino_it != st_cli->inode_config.end() && ino_it->second.readonly)
+        if (ino_it != st_cli.inode_config.end() && ino_it->second.readonly)
         {
             op->retval = -EROFS;
             auto cb = std::move(op->callback);
@@ -972,15 +972,15 @@ bool cluster_client_t::check_rw(cluster_op_t *op)
     op->deoptimise_snapshot = false;
     if (enable_writeback && (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP))
     {
-        auto ino_it = st_cli->inode_config.find(op->inode);
+        auto ino_it = st_cli.inode_config.find(op->inode);
-        if (ino_it != st_cli->inode_config.end())
+        if (ino_it != st_cli.inode_config.end())
         {
             int chain_size = 0;
-            while (ino_it != st_cli->inode_config.end() && ino_it->second.parent_id)
+            while (ino_it != st_cli.inode_config.end() && ino_it->second.parent_id)
             {
                 // Check for loops - FIXME check it in etcd_state_client
                 if (ino_it->second.parent_id == op->inode ||
-                    chain_size > st_cli->inode_config.size())
+                    chain_size > st_cli.inode_config.size())
                 {
                     op->retval = -EINVAL;
                     auto cb = std::move(op->callback);
@@ -995,7 +995,7 @@ bool cluster_client_t::check_rw(cluster_op_t *op)
                     break;
                 }
                 chain_size++;
-                ino_it = st_cli->inode_config.find(ino_it->second.parent_id);
+                ino_it = st_cli.inode_config.find(ino_it->second.parent_id);
             }
         }
     }
@@ -1004,17 +1004,17 @@ bool cluster_client_t::check_rw(cluster_op_t *op)
 
 void cluster_client_t::execute_raw(osd_num_t osd_num, osd_op_t *op)
 {
-    auto peer_it = msgr.osd_peers.find(osd_num);
+    auto fd_it = msgr.osd_peer_fds.find(osd_num);
-    if (peer_it != msgr.osd_peers.end())
+    if (fd_it != msgr.osd_peer_fds.end())
     {
         op->op_type = OSD_OP_OUT;
-        op->client_id = peer_it->second->client_id;
+        op->peer_fd = fd_it->second;
         msgr.outbox_push(op);
     }
     else
     {
         if (msgr.wanted_peers.find(osd_num) == msgr.wanted_peers.end())
-            msgr.connect_peer(osd_num, st_cli->peer_states[osd_num]);
+            msgr.connect_peer(osd_num, st_cli.peer_states[osd_num]);
         raw_ops.emplace(osd_num, op);
     }
 }
@@ -1119,35 +1119,28 @@ resume_2:
         // Finished successfully
         // Even if the PG count has changed in meanwhile we treat it as success
         // because if some operations were invalid for the new PG count we'd get errors
-        if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
-        {
-            // Copy part bitmaps only after finishing all part reads
-            for (auto & part: op->parts)
-                if ((part.flags & (PART_SENT|PART_DONE|PART_VALID)) == (PART_SENT|PART_DONE|PART_VALID))
-                    copy_part_bitmap(op, &part);
-        }
         if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
         {
             // Check parent inode
-            auto ino_it = st_cli->inode_config.find(op->cur_inode);
+            auto ino_it = st_cli.inode_config.find(op->cur_inode);
             // Skip parents from the same pool
             int skipped = 0;
             while (!op->deoptimise_snapshot &&
-                ino_it != st_cli->inode_config.end() && ino_it->second.parent_id &&
+                ino_it != st_cli.inode_config.end() && ino_it->second.parent_id &&
                 INODE_POOL(ino_it->second.parent_id) == INODE_POOL(op->cur_inode))
             {
                 // Check for loops - FIXME check it in etcd_state_client
                 if (ino_it->second.parent_id == op->inode ||
-                    skipped > st_cli->inode_config.size())
+                    skipped > st_cli.inode_config.size())
                 {
                     op->retval = -EINVAL;
                     erase_op(op);
                     return 1;
                 }
                 skipped++;
-                ino_it = st_cli->inode_config.find(ino_it->second.parent_id);
+                ino_it = st_cli.inode_config.find(ino_it->second.parent_id);
             }
-            if (ino_it != st_cli->inode_config.end() &&
+            if (ino_it != st_cli.inode_config.end() &&
                 ino_it->second.parent_id &&
                 ino_it->second.parent_id != op->inode)
             {
@@ -1161,7 +1154,7 @@ resume_2:
         op->retval = op->len;
         if (op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
         {
-            auto & pool_cfg = st_cli->pool_config.at(INODE_POOL(op->inode));
+            auto & pool_cfg = st_cli.pool_config.at(INODE_POOL(op->inode));
             op->retval = op->len / pool_cfg.bitmap_granularity;
         }
         if (op->flush_id)
@@ -1171,7 +1164,7 @@ resume_2:
         erase_op(op);
         return 1;
     }
-    else if (op->retval != 0 && op->opcode != OSD_OP_SYNC && !(op->flags & OP_FLUSH_BUFFER) &&
+    else if (op->retval != 0 && !(op->flags & OP_FLUSH_BUFFER) &&
         op->retval != -EPIPE && (op->retval != -EIO || !client_eio_retry_interval) && (op->retval != -ENOSPC || !client_retry_enospc))
     {
         // Fatal error (neither -EPIPE, -EIO nor -ENOSPC)
@@ -1247,7 +1240,7 @@ void cluster_client_t::slice_rw(cluster_op_t *op)
 {
     // Slice the request into individual object stripe requests
     // Primary OSDs still operate individual stripes, but their size is multiplied by PG minsize in case of EC
-    auto & pool_cfg = st_cli->pool_config.at(INODE_POOL(op->cur_inode));
+    auto & pool_cfg = st_cli.pool_config.at(INODE_POOL(op->cur_inode));
     uint32_t pg_data_size = (pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks);
     uint64_t pg_block_size = pool_cfg.data_block_size * pg_data_size;
     uint64_t first_stripe = (op->offset / pg_block_size) * pg_block_size;
@@ -1346,7 +1339,7 @@ bool cluster_client_t::affects_pg(uint64_t inode, uint64_t offset, uint64_t len,
     {
         return false;
     }
-    auto & pool_cfg = st_cli->pool_config.at(INODE_POOL(inode));
+    auto & pool_cfg = st_cli.pool_config.at(INODE_POOL(inode));
     uint32_t pg_data_size = (pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks);
     uint64_t pg_block_size = pool_cfg.data_block_size * pg_data_size;
     uint64_t first_stripe = (offset / pg_block_size) * pg_block_size;
@@ -1365,7 +1358,7 @@ bool cluster_client_t::affects_pg(uint64_t inode, uint64_t offset, uint64_t len,
 
 bool cluster_client_t::affects_osd(uint64_t inode, uint64_t offset, uint64_t len, osd_num_t osd)
 {
-    auto & pool_cfg = st_cli->pool_config.at(INODE_POOL(inode));
+    auto & pool_cfg = st_cli.pool_config.at(INODE_POOL(inode));
     uint32_t pg_data_size = (pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks);
     uint64_t pg_block_size = pool_cfg.data_block_size * pg_data_size;
     uint64_t first_stripe = (offset / pg_block_size) * pg_block_size;
@@ -1389,7 +1382,7 @@ int cluster_client_t::try_send(cluster_op_t *op, int i, std::function<void(osd_o
         init_msgr();
     }
     auto part = &op->parts[i];
-    auto & pool_cfg = st_cli->pool_config.at(INODE_POOL(op->cur_inode));
+    auto & pool_cfg = st_cli.pool_config.at(INODE_POOL(op->cur_inode));
     auto pg_it = pool_cfg.pg_config.find(part->pg_num);
     if (pg_it != pool_cfg.pg_config.end() &&
         !pg_it->second.pause && pg_it->second.cur_primary &&
@@ -1408,10 +1401,10 @@ int cluster_client_t::try_send(cluster_op_t *op, int i, std::function<void(osd_o
                 primary_osd = nearest_osd;
         }
         part->osd_num = primary_osd;
-        auto peer_it = msgr.osd_peers.find(primary_osd);
+        auto peer_it = msgr.osd_peer_fds.find(primary_osd);
-        if (peer_it != msgr.osd_peers.end())
+        if (peer_it != msgr.osd_peer_fds.end())
         {
-            osd_client_t *cl = peer_it->second;
+            int peer_fd = peer_it->second;
             part->flags |= PART_SENT|PART_VALID;
             op->inflight_count++;
             uint64_t pg_bitmap_size = (pool_cfg.data_block_size / pool_cfg.bitmap_granularity / 8) * (
@@ -1420,13 +1413,13 @@ int cluster_client_t::try_send(cluster_op_t *op, int i, std::function<void(osd_o
             uint64_t meta_rev = 0;
             if (op->opcode != OSD_OP_READ_BITMAP && op->opcode != OSD_OP_DELETE && !op->deoptimise_snapshot)
             {
-                auto ino_it = st_cli->inode_config.find(op->cur_inode);
+                auto ino_it = st_cli.inode_config.find(op->cur_inode);
-                if (ino_it != st_cli->inode_config.end())
+                if (ino_it != st_cli.inode_config.end())
                     meta_rev = ino_it->second.mod_revision;
             }
             part->op = (osd_op_t){
                 .op_type = OSD_OP_OUT,
-                .client_id = cl->client_id,
+                .peer_fd = peer_fd,
                 .req = { .rw = {
                     .header = {
                         .magic = SECONDARY_OSD_OP_MAGIC,
@@ -1453,7 +1446,7 @@ int cluster_client_t::try_send(cluster_op_t *op, int i, std::function<void(osd_o
         }
         else if (msgr.wanted_peers.find(primary_osd) == msgr.wanted_peers.end())
         {
-            msgr.connect_peer(primary_osd, st_cli->peer_states[primary_osd]);
+            msgr.connect_peer(primary_osd, st_cli.peer_states[primary_osd]);
             return TRY_SEND_CONNECTING;
         }
     }
@@ -1475,8 +1468,8 @@ int cluster_client_t::continue_sync(cluster_op_t *op)
     for (auto do_it = dirty_osds.begin(); do_it != dirty_osds.end(); )
     {
         osd_num_t sync_osd = *do_it;
-        auto peer_it = msgr.osd_peers.find(sync_osd);
+        auto peer_it = msgr.osd_peer_fds.find(sync_osd);
-        if (peer_it == msgr.osd_peers.end())
+        if (peer_it == msgr.osd_peer_fds.end())
             dirty_osds.erase(do_it++);
         else
             do_it++;
@@ -1529,12 +1522,12 @@ resume_1:
 
 void cluster_client_t::send_sync(cluster_op_t *op, cluster_op_part_t *part)
 {
-    osd_client_t *cl = msgr.osd_peers.at(part->osd_num);
+    auto peer_fd = msgr.osd_peer_fds.at(part->osd_num);
     part->flags |= PART_SENT;
     op->inflight_count++;
     part->op = (osd_op_t){
         .op_type = OSD_OP_OUT,
-        .client_id = cl->client_id,
+        .peer_fd = peer_fd,
         .req = {
             .hdr = {
                 .magic = SECONDARY_OSD_OP_MAGIC,
@@ -1574,10 +1567,10 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
             // Error priority: EIO > ENOSPC > ETIMEDOUT > EPIPE
             op->retval = part->op.reply.hdr.retval;
         }
-        uint64_t stop_client_id = 0;
+        int stop_fd = -1;
         if (op->retval != -EINTR && op->retval != -EIO && op->retval != -ENOSPC)
         {
-            stop_client_id = part->op.client_id;
+            stop_fd = part->op.peer_fd;
             if (op->retval != -EPIPE || log_level > 0)
             {
                 fprintf(
@@ -1604,9 +1597,9 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
             op->retry_after = op->retval != -EPIPE ? client_eio_retry_interval : client_retry_interval;
         }
         reset_retry_timer(op->retry_after);
-        if (stop_client_id)
+        if (stop_fd >= 0)
         {
-            msgr.stop_client(stop_client_id);
+            msgr.stop_client(stop_fd);
         }
         op->inflight_count--;
         if (op->inflight_count == 0 && !op->retry_after)
@@ -1637,6 +1630,13 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
         }
         if (op->inflight_count == 0 && !op->retry_after)
         {
+            // Copy part bitmaps only after finishing all part reads
+            if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
+            {
+                for (auto & part: op->parts)
+                    if (part.flags == (PART_SENT|PART_VALID|PART_DONE))
+                        copy_part_bitmap(op, &part);
+            }
             if (op->opcode == OSD_OP_SYNC)
                 continue_sync(op);
             else
@@ -1648,7 +1648,7 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
 void cluster_client_t::copy_part_bitmap(cluster_op_t *op, cluster_op_part_t *part)
 {
     // Copy (OR) bitmap
-    auto & pool_cfg = st_cli->pool_config.at(INODE_POOL(op->cur_inode));
+    auto & pool_cfg = st_cli.pool_config.at(INODE_POOL(op->cur_inode));
     uint32_t pg_block_size = pool_cfg.data_block_size * (
         pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks
     );

src/client/cluster_client.h

@@ -4,7 +4,7 @@
 #pragma once
 
 #include "messenger.h"
-#include "etcd_state_client_http.h"
+#include "etcd_state_client.h"
 
 #define DEFAULT_CLIENT_MAX_DIRTY_BYTES 32*1024*1024
 #define DEFAULT_CLIENT_MAX_DIRTY_OPS 1024
@@ -83,6 +83,9 @@ class writeback_cache_t;
 // FIXME: Split into public and private interfaces
 class __attribute__((visibility("default"))) cluster_client_t
 {
+#ifdef __MOCK__
+public:
+#endif
     timerfd_manager_t *tfd = NULL;
     ring_loop_t *ringloop = NULL;
 
@@ -131,7 +134,7 @@ class __attribute__((visibility("default"))) cluster_client_t
     bool msgr_initialized = false;
 
 public:
-    std::unique_ptr<etcd_state_client_t> st_cli;
+    etcd_state_client_t st_cli;
 
     osd_messenger_t msgr;
     void init_msgr();
@@ -139,8 +142,7 @@ public:
     json11::Json::object cli_config, file_config, etcd_global_config;
     json11::Json::object config;
 
-    static cluster_client_t* create(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json config);
+    cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json config);
-    cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json config, std::unique_ptr<etcd_state_client_t> st_cli);
     ~cluster_client_t();
     void execute(cluster_op_t *op);
     void execute_raw(osd_num_t osd_num, osd_op_t *op);
@@ -153,9 +155,15 @@ public:
     void list_inode(inode_t inode, uint64_t min_offset, uint64_t max_offset, int max_parallel_pgs, std::function<void(
         int status, int pgs_left, pg_num_t pg_num, std::set<object_id>&& objects)> pg_callback);
 
+    //inline uint32_t get_bs_bitmap_granularity() { return st_cli.global_bitmap_granularity; }
+    //inline uint64_t get_bs_block_size() { return st_cli.global_block_size; }
+
+#ifndef __MOCK__
 protected:
+#endif
     void continue_ops(int time_passed = 0);
 
+protected:
     bool affects_osd(uint64_t inode, uint64_t offset, uint64_t len, osd_num_t osd);
     bool affects_pg(uint64_t inode, uint64_t offset, uint64_t len, pool_id_t pool_id, pg_num_t pg_num);
 
@@ -196,5 +204,4 @@ protected:
     osd_num_t select_nearest_osd(const std::vector<osd_num_t> & osds);
 
     friend class writeback_cache_t;
-    friend class cluster_client_test_t;
 };

src/client/cluster_client_list.cpp

@@ -63,14 +63,14 @@ void cluster_client_t::list_inode(inode_t inode, uint64_t min_offset, uint64_t m
 {
     init_msgr();
     pool_id_t pool_id = INODE_POOL(inode);
-    if (!pool_id || st_cli->pool_config.find(pool_id) == st_cli->pool_config.end())
+    if (!pool_id || st_cli.pool_config.find(pool_id) == st_cli.pool_config.end())
     {
         if (log_level > 0)
             fprintf(stderr, "Pool %u does not exist\n", pool_id);
         pg_callback(-EINVAL, 0, 0, std::set<object_id>());
         return;
     }
-    auto pg_stripe_size = st_cli->pool_config.at(pool_id).pg_stripe_size;
+    auto pg_stripe_size = st_cli.pool_config.at(pool_id).pg_stripe_size;
     if (min_offset)
         min_offset = (min_offset/pg_stripe_size) * pg_stripe_size;
     inode_list_t *lst = new inode_list_t();
@@ -110,13 +110,13 @@ bool cluster_client_t::continue_listing(inode_list_t *lst)
 
 bool cluster_client_t::restart_listing(inode_list_t* lst)
 {
-    auto pool_it = st_cli->pool_config.find(lst->pool_id);
+    auto pool_it = st_cli.pool_config.find(lst->pool_id);
     // We want listing to be consistent. To achieve it we should:
     // 1) retry listing of each PG if its state changes
     // 2) abort listing if PG count changes during listing
     // 3) ideally, only talk to the primary OSD - this will be done separately
     // So first we add all PGs without checking their state
-    if (pool_it == st_cli->pool_config.end() ||
+    if (pool_it == st_cli.pool_config.end() ||
         lst->real_pg_count != pool_it->second.real_pg_count)
     {
         for (auto pg: lst->pgs)
@@ -136,7 +136,7 @@ bool cluster_client_t::restart_listing(inode_list_t* lst)
             fprintf(stderr, "PG count in pool %u changed during listing\n", lst->pool_id);
         }
         lst->pgs.clear();
-        if (pool_it == st_cli->pool_config.end())
+        if (pool_it == st_cli.pool_config.end())
         {
             // Unknown pool
             lst->callback(-EINVAL, 0, 0, std::set<object_id>());
@@ -248,7 +248,7 @@ void cluster_client_t::set_list_retry_timeout(int ms, timespec new_time)
 
 int cluster_client_t::start_pg_listing(inode_list_pg_t *pg)
 {
-    auto & pool_cfg = st_cli->pool_config.at(pg->lst->pool_id);
+    auto & pool_cfg = st_cli.pool_config.at(pg->lst->pool_id);
     auto pg_it = pool_cfg.pg_config.find(pg->pg_num);
     assert(pg->lst->real_pg_count == pool_cfg.real_pg_count);
     if (pg_it == pool_cfg.pg_config.end() ||
@@ -277,7 +277,7 @@ int cluster_client_t::start_pg_listing(inode_list_pg_t *pg)
         for (auto peer_it = all_peers.begin(); peer_it != all_peers.end(); )
         {
             if (*peer_it != pg_it->second.cur_primary &&
-                st_cli->peer_states[*peer_it].is_null())
+                st_cli.peer_states[*peer_it].is_null())
             {
                 pg->inactive_osds.push_back(*peer_it);
                 all_peers.erase(peer_it++);
@@ -295,14 +295,14 @@ int cluster_client_t::start_pg_listing(inode_list_pg_t *pg)
     bool conn = true;
     for (osd_num_t peer_osd: all_peers)
     {
-        if (msgr.osd_peers.find(peer_osd) == msgr.osd_peers.end())
+        if (msgr.osd_peer_fds.find(peer_osd) == msgr.osd_peer_fds.end())
         {
             // Initiate connection
-            if (st_cli->peer_states[peer_osd].is_null())
+            if (st_cli.peer_states[peer_osd].is_null())
             {
                 return LIST_PG_WAIT_ACTIVE;
             }
-            msgr.connect_peer(peer_osd, st_cli->peer_states[peer_osd]);
+            msgr.connect_peer(peer_osd, st_cli.peer_states[peer_osd]);
             conn = false;
         }
     }
@@ -336,11 +336,11 @@ void cluster_client_t::send_list(inode_list_osd_t *cur_list)
     if (!cur_list->pg->inflight_ops)
         cur_list->pg->lst->inflight_pgs++;
     cur_list->pg->inflight_ops++;
-    auto & pool_cfg = st_cli->pool_config[cur_list->pg->lst->pool_id];
+    auto & pool_cfg = st_cli.pool_config[cur_list->pg->lst->pool_id];
     osd_op_t *op = new osd_op_t();
     op->op_type = OSD_OP_OUT;
     // Already checked that it exists above, but anyway
-    op->client_id = msgr.osd_peers.at(cur_list->osd_num)->client_id;
+    op->peer_fd = msgr.osd_peer_fds.at(cur_list->osd_num);
     op->req = (osd_any_op_t){
         .sec_list = {
             .header = {

src/client/cluster_client_real.cpp

@@ -1,11 +0,0 @@
-// Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
-
-#include "cluster_client.h"
-#include "etcd_state_client_http.h"
-
-cluster_client_t* cluster_client_t::create(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json config)
-{
-    auto st_cli = new etcd_state_client_http_t(tfd);
-    return new cluster_client_t(ringloop, tfd, config, std::unique_ptr<etcd_state_client_t>(st_cli));
-}

src/client/cluster_client_wb.cpp

@@ -88,11 +88,6 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
     // ...or just save it for writeback if write buffering is enabled
     if (op->len == 0)
     {
-        // FIXME: OSD_OP_DELETEs are currently only sent by vitastor-cli rm/rm-data and
-        // actually have len=0, because delete is actually a delete of the full object
-        // containing the requested offset, not a "punch hole" operation. But here, writeback
-        // cache assumes it IS a "punch hole" operation. I should select one of these
-        // approaches and fix everything accordingly when I decide to implement TRIM.
         return;
     }
     auto dirty_it = find_dirty(op->inode, op->offset);
@@ -131,7 +126,6 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
                         writeback_bytes -= op->len;
                     }
                     writeback_queue_size++;
-                    writeback_queue.push_back({ op->inode, new_end });
                 }
                 break;
             }
@@ -166,7 +160,6 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
                 {
                     writeback_queue_size++;
                 }
-                writeback_queue.push_back({ op->inode, new_end });
             }
             auto new_dirty_it = dirty_buffers.emplace_hint(dirty_it, (object_id){
                 .inode = op->inode,
@@ -251,13 +244,12 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
             writeback_queue_size--;
         }
     }
-    if (!is_del && op->len > 0)
+    if (!is_del)
     {
         uint64_t pos = 0, len = op->len, iov_idx = 0;
-        while (iov_idx < op->iov.count)
+        while (len > 0 && iov_idx < op->iov.count)
         {
             auto & iov = op->iov.buf[iov_idx];
-            assert(pos + iov.iov_len <= len);
             memcpy(buf + pos, iov.iov_base, iov.iov_len);
             pos += iov.iov_len;
             iov_idx++;
@@ -451,7 +443,7 @@ void writeback_cache_t::start_writebacks(cluster_client_t *cli, int count)
         started++;
         assert(writeback_queue_size > 0);
         writeback_queue_size--;
-        writeback_bytes -= (is_del ? 0 : off - from_it->first.stripe);
+        writeback_bytes -= off - from_it->first.stripe;
         assert(writeback_queue_size > 0 || !writeback_bytes);
         flush_buffers(cli, from_it, to_it);
     }

src/client/etcd_state_client.cpp

@@ -1,12 +1,13 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
-#include <assert.h>
-
 #include "osd_ops.h"
 #include "pg_states.h"
 #include "etcd_state_client.h"
+#ifndef __MOCK__
 #include "addr_util.h"
+#include "http_client.h"
+#endif
 #include "str_util.h"
 
 etcd_state_client_t::~etcd_state_client_t()
@@ -16,8 +17,28 @@ etcd_state_client_t::~etcd_state_client_t()
         delete watch;
     }
     watches.clear();
+    etcd_watches_initialised = -1;
+#ifndef __MOCK__
+    stop_ws_keepalive();
+    if (etcd_watch_ws)
+    {
+        http_close(etcd_watch_ws);
+        etcd_watch_ws = NULL;
+    }
+    if (keepalive_client)
+    {
+        http_close(keepalive_client);
+        keepalive_client = NULL;
+    }
+#endif
+    if (load_pgs_timer_id >= 0)
+    {
+        tfd->clear_timer(load_pgs_timer_id);
+        load_pgs_timer_id = -1;
+    }
 }
 
+#ifndef __MOCK__
 etcd_kv_t etcd_state_client_t::parse_etcd_kv(const json11::Json & kv_json)
 {
     etcd_kv_t kv;
@@ -51,6 +72,104 @@ std::vector<std::string> etcd_state_client_t::get_addresses()
     return addrs;
 }
 
+void etcd_state_client_t::etcd_call_oneshot(std::string etcd_address, std::string api, json11::Json payload,
+    int timeout, std::function<void(std::string, json11::Json)> callback)
+{
+    std::string etcd_api_path;
+    int pos = etcd_address.find('/');
+    if (pos >= 0)
+    {
+        etcd_api_path = etcd_address.substr(pos);
+        etcd_address = etcd_address.substr(0, pos);
+    }
+    std::string req = payload.dump();
+    req = "POST "+etcd_api_path+api+" HTTP/1.1\r\n"
+        "Host: "+etcd_address+"\r\n"
+        "Content-Type: application/json\r\n"
+        "Content-Length: "+std::to_string(req.size())+"\r\n"
+        "Connection: close\r\n"
+        "\r\n"+req;
+    auto http_cli = http_init(tfd);
+    auto cb = [http_cli, callback](const http_response_t *response)
+    {
+        std::string err;
+        json11::Json data;
+        response->parse_json_response(err, data);
+        callback(err, data);
+        http_close(http_cli);
+    };
+    http_request(http_cli, etcd_address, req, { .timeout = timeout }, cb);
+}
+
+void etcd_state_client_t::etcd_call(std::string api, json11::Json payload, int timeout,
+    int retries, int interval, std::function<void(std::string, json11::Json)> callback)
+{
+    if (!etcd_addresses.size() && !etcd_local.size())
+    {
+        fprintf(stderr, "etcd_address is missing in Vitastor configuration\n");
+        exit(1);
+    }
+    pick_next_etcd();
+    std::string etcd_address = selected_etcd_address;
+    std::string etcd_api_path;
+    int pos = etcd_address.find('/');
+    if (pos >= 0)
+    {
+        etcd_api_path = etcd_address.substr(pos);
+        etcd_address = etcd_address.substr(0, pos);
+    }
+    std::string req = payload.dump();
+    req = "POST "+etcd_api_path+api+" HTTP/1.1\r\n"
+        "Host: "+etcd_address+"\r\n"
+        "Content-Type: application/json\r\n"
+        "Content-Length: "+std::to_string(req.size())+"\r\n"
+        "Connection: keep-alive\r\n"
+        "Keep-Alive: timeout="+std::to_string(etcd_keepalive_timeout)+"\r\n"
+        "\r\n"+req;
+    retries--;
+    auto cb = [this, api, payload, timeout, retries, interval, callback,
+        cur_addr = selected_etcd_address](const http_response_t *response)
+    {
+        std::string err;
+        json11::Json data;
+        response->parse_json_response(err, data);
+        if (err != "")
+        {
+            if (cur_addr == selected_etcd_address)
+                selected_etcd_address = "";
+            if (retries > 0)
+            {
+                if (this->log_level > 0)
+                {
+                    fprintf(
+                        stderr, "Warning: etcd request failed: %s, retrying %d more times\n",
+                        err.c_str(), retries
+                    );
+                }
+                if (interval > 0)
+                {
+                    // FIXME: Prevent destruction of etcd_state_client if timers or requests are active
+                    tfd->set_timer(interval, false, [this, api, payload, timeout, retries, interval, callback](int)
+                    {
+                        etcd_call(api, payload, timeout, retries, interval, callback);
+                    });
+                }
+                else
+                    etcd_call(api, payload, timeout, retries, interval, callback);
+            }
+            else
+                callback(err, data);
+        }
+        else
+            callback(err, data);
+    };
+    if (!keepalive_client)
+    {
+        keepalive_client = http_init(tfd);
+    }
+    http_request(keepalive_client, etcd_address, req, { .timeout = timeout, .keepalive = true }, cb);
+}
+
 void etcd_state_client_t::add_etcd_url(std::string addr)
 {
     if (addr.length() > 0)
@@ -137,6 +256,7 @@ void etcd_state_client_t::parse_config(const json11::Json & config)
         if (this->etcd_keepalive_timeout < 30)
             this->etcd_keepalive_timeout = 30;
     }
+    auto old_etcd_ws_keepalive_interval = this->etcd_ws_keepalive_interval;
     this->etcd_ws_keepalive_interval = config["etcd_ws_keepalive_interval"].uint64_value();
     if (this->etcd_ws_keepalive_interval <= 0)
     {
@@ -162,9 +282,291 @@ void etcd_state_client_t::parse_config(const json11::Json & config)
     {
         this->etcd_min_reload_interval = 50;
     }
+    if (this->etcd_ws_keepalive_interval != old_etcd_ws_keepalive_interval && ws_keepalive_timer >= 0)
+    {
+#ifndef __MOCK__
+        stop_ws_keepalive();
+        start_ws_keepalive();
+#endif
+    }
 }
 
-void etcd_state_client_t::load_global_config(std::function<void(const std::string & error)> cb)
+void etcd_state_client_t::pick_next_etcd()
+{
+    if (selected_etcd_address != "")
+        return;
+    if (addresses_to_try.size() == 0)
+    {
+        // Prefer local etcd, if any
+        for (int i = 0; i < etcd_local.size(); i++)
+            addresses_to_try.push_back(etcd_local[i]);
+        std::vector<int> ns;
+        for (int i = 0; i < etcd_addresses.size(); i++)
+            ns.push_back(i);
+        if (!rand_initialized)
+        {
+            timespec tv;
+            clock_gettime(CLOCK_REALTIME, &tv);
+            srand48(tv.tv_sec*1000000000 + tv.tv_nsec);
+            rand_initialized = true;
+        }
+        while (ns.size())
+        {
+            int i = lrand48() % ns.size();
+            addresses_to_try.push_back(etcd_addresses[ns[i]]);
+            ns.erase(ns.begin()+i, ns.begin()+i+1);
+        }
+    }
+    selected_etcd_address = addresses_to_try[0];
+    addresses_to_try.erase(addresses_to_try.begin(), addresses_to_try.begin()+1);
+}
+
+void etcd_state_client_t::start_etcd_watcher()
+{
+    if (!etcd_addresses.size() && !etcd_local.size())
+    {
+        fprintf(stderr, "etcd_address is missing in Vitastor configuration\n");
+        exit(1);
+    }
+    pick_next_etcd();
+    std::string etcd_address = selected_etcd_address;
+    std::string etcd_api_path;
+    int pos = etcd_address.find('/');
+    if (pos >= 0)
+    {
+        etcd_api_path = etcd_address.substr(pos);
+        etcd_address = etcd_address.substr(0, pos);
+    }
+    etcd_watches_initialised = 0;
+    ws_alive = 1;
+    if (etcd_watch_ws)
+    {
+        http_close(etcd_watch_ws);
+        etcd_watch_ws = NULL;
+    }
+    if (this->log_level > 1)
+    {
+        fprintf(stderr, "Trying to connect to etcd websocket at %s, watch from revision %ju/%ju/%ju\n", etcd_address.c_str(),
+            etcd_watch_revision_config, etcd_watch_revision_osd, etcd_watch_revision_pg);
+    }
+    etcd_watch_ws = open_websocket(tfd, etcd_address, etcd_api_path+"/watch", etcd_slow_timeout,
+        [this, cur_addr = selected_etcd_address](const http_response_t *msg)
+    {
+        if (msg->body.length())
+        {
+            ws_alive = 1;
+            std::string json_err;
+            json11::Json data = json11::Json::parse(msg->body, json_err);
+            if (json_err != "")
+            {
+                fprintf(stderr, "Bad JSON in etcd event: %s, ignoring event\n", json_err.c_str());
+            }
+            else
+            {
+                uint64_t watch_id = data["result"]["watch_id"].uint64_value();
+                if (data["result"]["created"].bool_value())
+                {
+                    if (watch_id == ETCD_CONFIG_WATCH_ID ||
+                        watch_id == ETCD_PG_STATE_WATCH_ID ||
+                        watch_id == ETCD_OSD_STATE_WATCH_ID)
+                    {
+                        etcd_watches_initialised++;
+                    }
+                    if (etcd_watches_initialised == ETCD_TOTAL_WATCHES && this->log_level > 0)
+                    {
+                        fprintf(stderr, "Successfully subscribed to etcd at %s, revision %ju/%ju/%ju\n", cur_addr.c_str(),
+                            etcd_watch_revision_config, etcd_watch_revision_osd, etcd_watch_revision_pg);
+                    }
+                }
+                if (data["result"]["canceled"].bool_value())
+                {
+                    // etcd watch canceled, maybe because the revision was compacted
+                    if (data["result"]["compact_revision"].uint64_value())
+                    {
+                        // we may miss events if we proceed
+                        // so we should restart from the beginning if we can
+                        if (on_reload_hook != NULL)
+                        {
+                            // check to not trigger on_reload_hook multiple times
+                            if (etcd_watch_ws != NULL)
+                            {
+                                fprintf(stderr, "Revisions before %ju were compacted by etcd, reloading state\n",
+                                    data["result"]["compact_revision"].uint64_value());
+                                http_close(etcd_watch_ws);
+                                etcd_watch_ws = NULL;
+                                etcd_watch_revision_config = etcd_watch_revision_osd = etcd_watch_revision_pg = 0;
+                                on_reload_hook();
+                            }
+                            return;
+                        }
+                        else
+                        {
+                            fprintf(stderr, "Revisions before %ju were compacted by etcd, exiting\n",
+                                data["result"]["compact_revision"].uint64_value());
+                            exit(1);
+                        }
+                    }
+                    else
+                    {
+                        fprintf(stderr, "Watch canceled by etcd, reason: %s, exiting\n", data["result"]["cancel_reason"].string_value().c_str());
+                        exit(1);
+                    }
+                }
+                // Save revision only if it's present in the message - because sometimes etcd sends something without a header, like:
+                // {"error": {"grpc_code": 14, "http_code": 503, "http_status": "Service Unavailable", "message": "error reading from server: EOF"}}
+                if (etcd_watches_initialised == ETCD_TOTAL_WATCHES && !data["result"]["header"]["revision"].is_null())
+                {
+                    // Restart watchers from the same revision number as in the last received message,
+                    // not from the next one to protect against revision being split into multiple messages,
+                    // even though etcd guarantees not to do that **within a single watcher** without fragment=true:
+                    // https://etcd.io/docs/v3.5/learning/api_guarantees/#watch-apis
+                    // Revision contents are ALWAYS split into separate messages for different watchers though!
+                    // So generally we have to resume each watcher from its own revision...
+                    // Progress messages may have watch_id=-1 if sent on behalf of multiple watchers though.
+                    // And antietcd has an advanced semantic which merges the same revision for all watchers
+                    // into one message and just omits watch_id.
+                    // So we also have to handle the case where watch_id is -1 or not present (0).
+                    auto watch_rev = data["result"]["header"]["revision"].uint64_value();
+                    if (!watch_id || watch_id == UINT64_MAX)
+                        etcd_watch_revision_config = etcd_watch_revision_osd = etcd_watch_revision_pg = watch_rev;
+                    else if (watch_id == ETCD_CONFIG_WATCH_ID)
+                        etcd_watch_revision_config = watch_rev;
+                    else if (watch_id == ETCD_PG_STATE_WATCH_ID)
+                        etcd_watch_revision_pg = watch_rev;
+                    else if (watch_id == ETCD_OSD_STATE_WATCH_ID)
+                        etcd_watch_revision_osd = watch_rev;
+                    addresses_to_try.clear();
+                }
+                // First gather all changes into a hash to remove multiple overwrites
+                std::map<std::string, etcd_kv_t> changes;
+                for (auto & ev: data["result"]["events"].array_items())
+                {
+                    auto kv = parse_etcd_kv(ev["kv"]);
+                    if (kv.key != "")
+                    {
+                        changes[kv.key] = kv;
+                    }
+                }
+                for (auto & kv: changes)
+                {
+                    if (this->log_level > 3)
+                    {
+                        fprintf(stderr, "Incoming event: %s -> %s\n", kv.first.c_str(), kv.second.value.dump().c_str());
+                    }
+                    parse_state(kv.second);
+                }
+                // React to changes
+                if (on_change_hook != NULL)
+                {
+                    on_change_hook(changes);
+                }
+            }
+        }
+        if (msg->eof)
+        {
+            fprintf(stderr, "Disconnected from etcd %s\n", cur_addr.c_str());
+            if (cur_addr == selected_etcd_address)
+                selected_etcd_address = "";
+            if (etcd_watch_ws)
+            {
+                http_close(etcd_watch_ws);
+                etcd_watch_ws = NULL;
+            }
+            if (etcd_watches_initialised == 0)
+            {
+                // Connection not established, retry in <etcd_quick_timeout>
+                tfd->set_timer(etcd_quick_timeout, false, [this](int)
+                {
+                    start_etcd_watcher();
+                });
+            }
+            else if (etcd_watches_initialised > 0)
+            {
+                // Connection was live, retry immediately
+                etcd_watches_initialised = 0;
+                start_etcd_watcher();
+            }
+        }
+    });
+    http_post_message(etcd_watch_ws, WS_TEXT, json11::Json(json11::Json::object {
+        { "create_request", json11::Json::object {
+            { "key", base64_encode(etcd_prefix+"/config/") },
+            { "range_end", base64_encode(etcd_prefix+"/config0") },
+            { "start_revision", etcd_watch_revision_config },
+            { "watch_id", ETCD_CONFIG_WATCH_ID },
+            { "progress_notify", true },
+        } }
+    }).dump());
+    http_post_message(etcd_watch_ws, WS_TEXT, json11::Json(json11::Json::object {
+        { "create_request", json11::Json::object {
+            { "key", base64_encode(etcd_prefix+"/osd/state/") },
+            { "range_end", base64_encode(etcd_prefix+"/osd/state0") },
+            { "start_revision", etcd_watch_revision_osd },
+            { "watch_id", ETCD_OSD_STATE_WATCH_ID },
+            { "progress_notify", true },
+        } }
+    }).dump());
+    http_post_message(etcd_watch_ws, WS_TEXT, json11::Json(json11::Json::object {
+        { "create_request", json11::Json::object {
+            { "key", base64_encode(etcd_prefix+"/pg/") },
+            { "range_end", base64_encode(etcd_prefix+"/pg0") },
+            { "start_revision", etcd_watch_revision_pg },
+            { "watch_id", ETCD_PG_STATE_WATCH_ID },
+            { "progress_notify", true },
+        } }
+    }).dump());
+    // FIXME: Do not watch /pg/history/ at all in client code (not in OSD)
+    if (on_start_watcher_hook)
+    {
+        on_start_watcher_hook(etcd_watch_ws);
+    }
+    start_ws_keepalive();
+}
+
+void etcd_state_client_t::stop_ws_keepalive()
+{
+    if (ws_keepalive_timer >= 0)
+    {
+        tfd->clear_timer(ws_keepalive_timer);
+        ws_keepalive_timer = -1;
+    }
+}
+
+void etcd_state_client_t::start_ws_keepalive()
+{
+    if (ws_keepalive_timer < 0)
+    {
+        ws_keepalive_timer = tfd->set_timer(etcd_ws_keepalive_interval*1000, true, [this](int)
+        {
+            if (!etcd_watch_ws || etcd_watches_initialised < ETCD_TOTAL_WATCHES)
+            {
+                // Do nothing
+            }
+            else if (!ws_alive)
+            {
+                if (this->log_level > 0)
+                {
+                    fprintf(stderr, "Websocket ping failed, disconnecting from etcd %s\n", selected_etcd_address.c_str());
+                }
+                if (etcd_watch_ws)
+                {
+                    http_close(etcd_watch_ws);
+                    etcd_watch_ws = NULL;
+                }
+                start_etcd_watcher();
+            }
+            else
+            {
+                ws_alive = 0;
+                http_post_message(etcd_watch_ws, WS_TEXT, json11::Json(json11::Json::object {
+                    { "progress_request", json11::Json::object { } }
+                }).dump());
+            }
+        });
+    }
+}
+
+void etcd_state_client_t::load_global_config()
 {
     json11::Json::object req = { { "success", json11::Json::array {
         json11::Json::object {
@@ -178,12 +580,22 @@ void etcd_state_client_t::load_global_config(std::function<void(const std::strin
             } }
         },
     } } };
-    etcd_txn(req, etcd_quick_timeout, max_etcd_attempts, 0, [this, cb](std::string err, json11::Json data)
+    etcd_txn(req, etcd_quick_timeout, max_etcd_attempts, 0, [this](std::string err, json11::Json data)
     {
         if (err != "")
         {
             fprintf(stderr, "Error reading configuration from etcd: %s\n", err.c_str());
-            cb(err);
+            if (infinite_start)
+            {
+                tfd->set_timer(etcd_slow_timeout, false, [this](int timer_id)
+                {
+                    load_global_config();
+                });
+            }
+            else
+            {
+                exit(1);
+            }
             return;
         }
         json11::Json config_kv = data["responses"][0]["response_range"]["kvs"][0];
@@ -214,12 +626,28 @@ void etcd_state_client_t::load_global_config(std::function<void(const std::strin
             parse_state(kv);
         }
         on_load_config_hook(global_config);
-        cb("");
     });
 }
 
-void etcd_state_client_t::load_pgs(std::function<void(const std::string &)> cb)
+void etcd_state_client_t::load_pgs()
 {
+    timespec tv;
+    clock_gettime(CLOCK_REALTIME, &tv);
+    uint64_t ms_passed = (tv.tv_sec-etcd_last_reload.tv_sec)*1000 + (tv.tv_nsec-etcd_last_reload.tv_nsec)/1000000;
+    if (ms_passed < etcd_min_reload_interval)
+    {
+        if (load_pgs_timer_id < 0)
+        {
+            load_pgs_timer_id = tfd->set_timer(etcd_min_reload_interval+50-ms_passed, false, [this](int) { load_pgs(); });
+        }
+        return;
+    }
+    etcd_last_reload = tv;
+    if (load_pgs_timer_id >= 0)
+    {
+        tfd->clear_timer(load_pgs_timer_id);
+        load_pgs_timer_id = -1;
+    }
     json11::Json::array txn = {
         json11::Json::object {
             { "request_range", json11::Json::object {
@@ -267,13 +695,16 @@ void etcd_state_client_t::load_pgs(std::function<void(const std::string &)> cb)
     {
         req["compare"] = checks;
     }
-    etcd_txn_slow(req, [this, cb](std::string err, json11::Json data)
+    etcd_txn_slow(req, [this](std::string err, json11::Json data)
     {
         if (err != "")
         {
             // Retry indefinitely
             fprintf(stderr, "Error loading PGs from etcd: %s\n", err.c_str());
-            cb(err);
+            tfd->set_timer(etcd_slow_timeout, false, [this](int timer_id)
+            {
+                load_pgs();
+            });
             return;
         }
         if (!data["succeeded"].bool_value())
@@ -301,9 +732,24 @@ void etcd_state_client_t::load_pgs(std::function<void(const std::string &)> cb)
         }
         clean_nonexistent_pgs();
         on_load_pgs_hook(true);
-        cb("");
+        start_etcd_watcher();
     });
 }
+#else
+void etcd_state_client_t::parse_config(const json11::Json & config)
+{
+}
+
+void etcd_state_client_t::load_global_config()
+{
+    json11::Json::object global_config;
+    on_load_config_hook(global_config);
+}
+
+void etcd_state_client_t::load_pgs()
+{
+}
+#endif
 
 void etcd_state_client_t::reset_pg_exists()
 {
@@ -416,8 +862,7 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
             if (pc.pg_size < 1 ||
                 pool_item.second["pg_size"].uint64_value() < 3 &&
                 (pc.scheme == POOL_SCHEME_XOR || pc.scheme == POOL_SCHEME_EC) ||
-                // limit is 64 because osd_peering_pg.cpp uses a 64-bit mask for has_roles
+                pool_item.second["pg_size"].uint64_value() > 256)
-                pool_item.second["pg_size"].uint64_value() > 64)
             {
                 fprintf(stderr, "Pool %u has invalid pg_size, skipping pool\n", pool_id);
                 continue;
@@ -737,6 +1182,7 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
                 if (i >= pg_state_bit_count)
                 {
                     fprintf(stderr, "Unexpected pool %u PG %u state keyword in etcd: %s\n", pool_id, pg_num, e.dump().c_str());
+                    return;
                 }
             }
             if (!cur_primary || !value["state"].is_array() || !state ||
@@ -745,6 +1191,7 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
                 (state & PG_INCOMPLETE) && state != PG_INCOMPLETE && state != (PG_INCOMPLETE|PG_HAS_INVALID))
             {
                 fprintf(stderr, "Unexpected pool %u PG %u state in etcd: primary=%ju, state=%s\n", pool_id, pg_num, cur_primary, value["state"].dump().c_str());
+                return;
             }
             pg_cfg.cur_primary = cur_primary;
             pg_cfg.cur_state = state;
@@ -757,14 +1204,8 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
     else if (key.substr(0, etcd_prefix.length()+11) == etcd_prefix+"/osd/state/")
     {
         // <etcd_prefix>/osd/state/%d
-        osd_num_t peer_osd = 0;
+        osd_num_t peer_osd = std::stoull(key.substr(etcd_prefix.length()+11));
-        char null_byte = 0;
+        if (peer_osd > 0)
-        int scanned = sscanf(key.c_str() + etcd_prefix.length()+11, "%ju%c", &peer_osd, &null_byte);
-        if (scanned != 1 || !peer_osd)
-        {
-            fprintf(stderr, "Bad etcd key %s, ignoring\n", key.c_str());
-        }
-        else
         {
             if (value.is_object() && value["state"] == "up")
             {

src/client/etcd_state_client.h

@@ -6,7 +6,7 @@
 #include <set>
 
 #include "json11/json11.hpp"
-#include "object_id.h"
+#include "osd_id.h"
 #include "timerfd_manager.h"
 
 #define ETCD_CONFIG_WATCH_ID 1
@@ -103,13 +103,15 @@ protected:
     std::vector<std::string> local_ips;
     std::vector<std::string> etcd_addresses;
     std::vector<std::string> etcd_local;
+    std::string selected_etcd_address;
+    std::vector<std::string> addresses_to_try;
     std::vector<inode_watch_t*> watches;
-    std::set<osd_num_t> seen_peers;
     bool new_pg_config = false;
-
+    int ws_keepalive_timer = -1;
+    int ws_alive = 0;
+    bool rand_initialized = false;
     void add_etcd_url(std::string);
-    void reset_pg_exists();
+    void pick_next_etcd();
-    void clean_nonexistent_pgs();
 public:
     int etcd_keepalive_timeout = 30;
     int etcd_ws_keepalive_interval = 5;
@@ -121,15 +123,21 @@ public:
     uint64_t global_block_size = DEFAULT_BLOCK_SIZE;
     uint32_t global_bitmap_granularity = DEFAULT_BITMAP_GRANULARITY;
     uint32_t global_immediate_commit = IMMEDIATE_NONE;
+
     std::string etcd_prefix;
     int log_level = 0;
+    timerfd_manager_t *tfd = NULL;
 
+    http_co_t *etcd_watch_ws = NULL, *keepalive_client = NULL;
+    int etcd_watches_initialised = 0;
     uint64_t etcd_watch_revision_config = 0;
     uint64_t etcd_watch_revision_osd = 0;
     uint64_t etcd_watch_revision_pg = 0;
-
+    timespec etcd_last_reload = {};
+    int load_pgs_timer_id = -1;
     std::map<pool_id_t, pool_config_t> pool_config;
     std::map<osd_num_t, json11::Json> peer_states;
+    std::set<osd_num_t> seen_peers;
     std::map<inode_t, inode_config_t> inode_config;
     std::map<std::string, inode_t> inode_by_name;
     json11::Json node_placement;
@@ -152,22 +160,24 @@ public:
     json11::Json::object serialize_inode_cfg(inode_config_t *cfg);
     etcd_kv_t parse_etcd_kv(const json11::Json & kv_json);
     std::vector<std::string> get_addresses();
-    virtual void etcd_call_oneshot(std::string etcd_address, std::string api, json11::Json payload, int timeout, std::function<void(std::string, json11::Json)> callback) = 0;
+    void etcd_call_oneshot(std::string etcd_address, std::string api, json11::Json payload, int timeout, std::function<void(std::string, json11::Json)> callback);
-    virtual void etcd_call(std::string api, json11::Json payload, int timeout, int retries, int interval, std::function<void(std::string, json11::Json)> callback) = 0;
+    void etcd_call(std::string api, json11::Json payload, int timeout, int retries, int interval, std::function<void(std::string, json11::Json)> callback);
     void etcd_txn(json11::Json txn, int timeout, int retries, int interval, std::function<void(std::string, json11::Json)> callback);
     void etcd_txn_slow(json11::Json txn, std::function<void(std::string, json11::Json)> callback);
-    virtual void etcd_add_watch(json11::Json watch) = 0;
+    void start_etcd_watcher();
-    void load_global_config(std::function<void(const std::string &)> cb);
+    void stop_ws_keepalive();
-    virtual void load_global_config() = 0;
+    void start_ws_keepalive();
-    void load_pgs(std::function<void(const std::string &)> cb);
+    void load_global_config();
-    virtual void load_pgs() = 0;
+    void load_pgs();
+    void reset_pg_exists();
+    void clean_nonexistent_pgs();
     void parse_state(const etcd_kv_t & kv);
-    virtual void parse_config(const json11::Json & config);
+    void parse_config(const json11::Json & config);
     void insert_inode_config(const inode_config_t & cfg);
     inode_watch_t* watch_inode(std::string name);
     void close_watch(inode_watch_t* watch);
     int address_count();
-    virtual ~etcd_state_client_t();
+    ~etcd_state_client_t();
 
     static uint32_t parse_immediate_commit(const std::string & immediate_commit_str, uint32_t default_value);
     static uint32_t parse_scheme(const std::string & scheme_str);

src/client/etcd_state_client_http.cpp

@@ -1,487 +0,0 @@
-// Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
-
-#include "etcd_state_client_http.h"
-#include "addr_util.h"
-#include "http_client.h"
-#include "str_util.h"
-
-etcd_state_client_http_t::etcd_state_client_http_t(timerfd_manager_t *tfd)
-{
-    this->tfd = tfd;
-}
-
-etcd_state_client_http_t::~etcd_state_client_http_t()
-{
-    stop_ws_keepalive();
-    if (etcd_watch_ws)
-    {
-        http_close(etcd_watch_ws);
-        etcd_watch_ws = NULL;
-    }
-    if (keepalive_client)
-    {
-        http_close(keepalive_client);
-        keepalive_client = NULL;
-    }
-    if (load_pgs_timer_id >= 0)
-    {
-        tfd->clear_timer(load_pgs_timer_id);
-        load_pgs_timer_id = -1;
-    }
-    etcd_watches_initialised = -1;
-}
-
-void etcd_state_client_http_t::etcd_add_watch(json11::Json watch)
-{
-    if (etcd_watch_ws)
-    {
-        http_post_message(etcd_watch_ws, WS_TEXT, watch.dump());
-    }
-}
-
-void etcd_state_client_http_t::etcd_call_oneshot(std::string etcd_address, std::string api, json11::Json payload,
-    int timeout, std::function<void(std::string, json11::Json)> callback)
-{
-    std::string etcd_api_path;
-    int pos = etcd_address.find('/');
-    if (pos >= 0)
-    {
-        etcd_api_path = etcd_address.substr(pos);
-        etcd_address = etcd_address.substr(0, pos);
-    }
-    std::string req = payload.dump();
-    req = "POST "+etcd_api_path+api+" HTTP/1.1\r\n"
-        "Host: "+etcd_address+"\r\n"
-        "Content-Type: application/json\r\n"
-        "Content-Length: "+std::to_string(req.size())+"\r\n"
-        "Connection: close\r\n"
-        "\r\n"+req;
-    auto http_cli = http_init(tfd);
-    auto cb = [http_cli, callback](const http_response_t *response)
-    {
-        std::string err;
-        json11::Json data;
-        response->parse_json_response(err, data);
-        callback(err, data);
-        http_close(http_cli);
-    };
-    http_request(http_cli, etcd_address, req, { .timeout = timeout }, cb);
-}
-
-void etcd_state_client_http_t::etcd_call(std::string api, json11::Json payload, int timeout,
-    int retries, int interval, std::function<void(std::string, json11::Json)> callback)
-{
-    if (!etcd_addresses.size() && !etcd_local.size())
-    {
-        fprintf(stderr, "etcd_address is missing in Vitastor configuration\n");
-        exit(1);
-    }
-    pick_next_etcd();
-    std::string etcd_address = selected_etcd_address;
-    std::string etcd_api_path;
-    int pos = etcd_address.find('/');
-    if (pos >= 0)
-    {
-        etcd_api_path = etcd_address.substr(pos);
-        etcd_address = etcd_address.substr(0, pos);
-    }
-    std::string req = payload.dump();
-    req = "POST "+etcd_api_path+api+" HTTP/1.1\r\n"
-        "Host: "+etcd_address+"\r\n"
-        "Content-Type: application/json\r\n"
-        "Content-Length: "+std::to_string(req.size())+"\r\n"
-        "Connection: keep-alive\r\n"
-        "Keep-Alive: timeout="+std::to_string(etcd_keepalive_timeout)+"\r\n"
-        "\r\n"+req;
-    retries--;
-    auto cb = [this, api, payload, timeout, retries, interval, callback,
-        cur_addr = selected_etcd_address](const http_response_t *response)
-    {
-        std::string err;
-        json11::Json data;
-        response->parse_json_response(err, data);
-        if (err != "")
-        {
-            if (cur_addr == selected_etcd_address)
-                selected_etcd_address = "";
-            if (retries > 0)
-            {
-                if (this->log_level > 0)
-                {
-                    fprintf(
-                        stderr, "Warning: etcd request failed: %s, retrying %d more times\n",
-                        err.c_str(), retries
-                    );
-                }
-                if (interval > 0)
-                {
-                    // FIXME: Prevent destruction of etcd_state_client if timers or requests are active
-                    tfd->set_timer(interval, false, [this, api, payload, timeout, retries, interval, callback](int)
-                    {
-                        etcd_call(api, payload, timeout, retries, interval, callback);
-                    });
-                }
-                else
-                    etcd_call(api, payload, timeout, retries, interval, callback);
-            }
-            else
-                callback(err, data);
-        }
-        else
-            callback(err, data);
-    };
-    if (!keepalive_client)
-    {
-        keepalive_client = http_init(tfd);
-    }
-    http_request(keepalive_client, etcd_address, req, { .timeout = timeout, .keepalive = true }, cb);
-}
-
-void etcd_state_client_http_t::parse_config(const json11::Json & config)
-{
-    auto old_etcd_ws_keepalive_interval = this->etcd_ws_keepalive_interval;
-    etcd_state_client_t::parse_config(config);
-    if (this->etcd_ws_keepalive_interval != old_etcd_ws_keepalive_interval && ws_keepalive_timer >= 0)
-    {
-        stop_ws_keepalive();
-        start_ws_keepalive();
-    }
-}
-
-void etcd_state_client_http_t::pick_next_etcd()
-{
-    if (selected_etcd_address != "")
-        return;
-    if (addresses_to_try.size() == 0)
-    {
-        // Prefer local etcd, if any
-        for (int i = 0; i < etcd_local.size(); i++)
-            addresses_to_try.push_back(etcd_local[i]);
-        std::vector<int> ns;
-        for (int i = 0; i < etcd_addresses.size(); i++)
-            ns.push_back(i);
-        if (!rand_initialized)
-        {
-            timespec tv;
-            clock_gettime(CLOCK_REALTIME, &tv);
-            srand48(tv.tv_sec*1000000000 + tv.tv_nsec);
-            rand_initialized = true;
-        }
-        while (ns.size())
-        {
-            int i = lrand48() % ns.size();
-            addresses_to_try.push_back(etcd_addresses[ns[i]]);
-            ns.erase(ns.begin()+i, ns.begin()+i+1);
-        }
-    }
-    selected_etcd_address = addresses_to_try[0];
-    addresses_to_try.erase(addresses_to_try.begin(), addresses_to_try.begin()+1);
-}
-
-void etcd_state_client_http_t::start_etcd_watcher()
-{
-    if (!etcd_addresses.size() && !etcd_local.size())
-    {
-        fprintf(stderr, "etcd_address is missing in Vitastor configuration\n");
-        exit(1);
-    }
-    pick_next_etcd();
-    std::string etcd_address = selected_etcd_address;
-    std::string etcd_api_path;
-    int pos = etcd_address.find('/');
-    if (pos >= 0)
-    {
-        etcd_api_path = etcd_address.substr(pos);
-        etcd_address = etcd_address.substr(0, pos);
-    }
-    etcd_watches_initialised = 0;
-    ws_alive = 1;
-    if (etcd_watch_ws)
-    {
-        http_close(etcd_watch_ws);
-        etcd_watch_ws = NULL;
-    }
-    if (this->log_level > 1)
-    {
-        fprintf(stderr, "Trying to connect to etcd websocket at %s, watch from revision %ju/%ju/%ju\n", etcd_address.c_str(),
-            etcd_watch_revision_config, etcd_watch_revision_osd, etcd_watch_revision_pg);
-    }
-    etcd_watch_ws = open_websocket(tfd, etcd_address, etcd_api_path+"/watch", etcd_slow_timeout,
-        [this, cur_addr = selected_etcd_address](const http_response_t *msg)
-    {
-        if (msg->body.length())
-        {
-            ws_alive = 1;
-            std::string json_err;
-            json11::Json data = json11::Json::parse(msg->body, json_err);
-            if (json_err != "")
-            {
-                fprintf(stderr, "Bad JSON in etcd event: %s, ignoring event\n", json_err.c_str());
-            }
-            else
-            {
-                uint64_t watch_id = data["result"]["watch_id"].uint64_value();
-                if (data["result"]["created"].bool_value())
-                {
-                    if (watch_id == ETCD_CONFIG_WATCH_ID ||
-                        watch_id == ETCD_PG_STATE_WATCH_ID ||
-                        watch_id == ETCD_OSD_STATE_WATCH_ID)
-                    {
-                        etcd_watches_initialised++;
-                    }
-                    if (etcd_watches_initialised == ETCD_TOTAL_WATCHES && this->log_level > 0)
-                    {
-                        fprintf(stderr, "Successfully subscribed to etcd at %s, revision %ju/%ju/%ju\n", cur_addr.c_str(),
-                            etcd_watch_revision_config, etcd_watch_revision_osd, etcd_watch_revision_pg);
-                    }
-                }
-                if (data["result"]["canceled"].bool_value())
-                {
-                    // etcd watch canceled, maybe because the revision was compacted
-                    if (data["result"]["compact_revision"].uint64_value())
-                    {
-                        // we may miss events if we proceed
-                        // so we should restart from the beginning if we can
-                        if (on_reload_hook != NULL)
-                        {
-                            // check to not trigger on_reload_hook multiple times
-                            if (etcd_watch_ws != NULL)
-                            {
-                                fprintf(stderr, "Revisions before %ju were compacted by etcd, reloading state\n",
-                                    data["result"]["compact_revision"].uint64_value());
-                                http_close(etcd_watch_ws);
-                                etcd_watch_ws = NULL;
-                                etcd_watch_revision_config = etcd_watch_revision_osd = etcd_watch_revision_pg = 0;
-                                on_reload_hook();
-                            }
-                            return;
-                        }
-                        else
-                        {
-                            fprintf(stderr, "Revisions before %ju were compacted by etcd, exiting\n",
-                                data["result"]["compact_revision"].uint64_value());
-                            exit(1);
-                        }
-                    }
-                    else
-                    {
-                        fprintf(stderr, "Watch canceled by etcd, reason: %s, exiting\n", data["result"]["cancel_reason"].string_value().c_str());
-                        exit(1);
-                    }
-                }
-                // Save revision only if it's present in the message - because sometimes etcd sends something without a header, like:
-                // {"error": {"grpc_code": 14, "http_code": 503, "http_status": "Service Unavailable", "message": "error reading from server: EOF"}}
-                // Also don't save revision from the initial created: true messages because they always contain the latest revision
-                if (etcd_watches_initialised == ETCD_TOTAL_WATCHES &&
-                    !data["result"]["header"]["revision"].is_null() &&
-                    !data["result"]["created"].bool_value())
-                {
-                    // Restart watchers from the same revision number as in the last received message,
-                    // not from the next one to protect against revision being split into multiple messages,
-                    // even though etcd guarantees not to do that **within a single watcher** without fragment=true:
-                    // https://etcd.io/docs/v3.5/learning/api_guarantees/#watch-apis
-                    // Revision contents are ALWAYS split into separate messages for different watchers though!
-                    // So generally we have to resume each watcher from its own revision...
-                    // Progress messages may have watch_id=-1 if sent on behalf of multiple watchers though.
-                    // And antietcd has an advanced semantic which merges the same revision for all watchers
-                    // into one message and just omits watch_id.
-                    // So we also have to handle the case where watch_id is -1 or not present (0).
-                    auto watch_rev = data["result"]["header"]["revision"].uint64_value();
-                    if (!watch_id || watch_id == UINT64_MAX)
-                        etcd_watch_revision_config = etcd_watch_revision_osd = etcd_watch_revision_pg = watch_rev;
-                    else if (watch_id == ETCD_CONFIG_WATCH_ID)
-                        etcd_watch_revision_config = watch_rev;
-                    else if (watch_id == ETCD_PG_STATE_WATCH_ID)
-                        etcd_watch_revision_pg = watch_rev;
-                    else if (watch_id == ETCD_OSD_STATE_WATCH_ID)
-                        etcd_watch_revision_osd = watch_rev;
-                    addresses_to_try.clear();
-                }
-                // First gather all changes into a hash to remove multiple overwrites
-                std::map<std::string, etcd_kv_t> changes;
-                for (auto & ev: data["result"]["events"].array_items())
-                {
-                    auto kv = parse_etcd_kv(ev["kv"]);
-                    if (kv.key != "")
-                    {
-                        changes[kv.key] = kv;
-                    }
-                }
-                for (auto & kv: changes)
-                {
-                    if (this->log_level > 3)
-                    {
-                        fprintf(stderr, "Incoming event: %s -> %s\n", kv.first.c_str(), kv.second.value.dump().c_str());
-                    }
-                    parse_state(kv.second);
-                }
-                // React to changes
-                if (on_change_hook != NULL)
-                {
-                    on_change_hook(changes);
-                }
-            }
-        }
-        if (msg->eof)
-        {
-            fprintf(stderr, "Disconnected from etcd %s\n", cur_addr.c_str());
-            if (cur_addr == selected_etcd_address)
-                selected_etcd_address = "";
-            if (etcd_watch_ws)
-            {
-                http_close(etcd_watch_ws);
-                etcd_watch_ws = NULL;
-            }
-            if (etcd_watches_initialised == 0)
-            {
-                // Connection not established, retry in <etcd_quick_timeout>
-                tfd->set_timer(etcd_quick_timeout, false, [this](int)
-                {
-                    start_etcd_watcher();
-                });
-            }
-            else if (etcd_watches_initialised > 0)
-            {
-                // Connection was live, retry immediately
-                etcd_watches_initialised = 0;
-                start_etcd_watcher();
-            }
-        }
-    });
-    http_post_message(etcd_watch_ws, WS_TEXT, json11::Json(json11::Json::object {
-        { "create_request", json11::Json::object {
-            { "key", base64_encode(etcd_prefix+"/config/") },
-            { "range_end", base64_encode(etcd_prefix+"/config0") },
-            { "start_revision", etcd_watch_revision_config },
-            { "watch_id", ETCD_CONFIG_WATCH_ID },
-            { "progress_notify", true },
-        } }
-    }).dump());
-    http_post_message(etcd_watch_ws, WS_TEXT, json11::Json(json11::Json::object {
-        { "create_request", json11::Json::object {
-            { "key", base64_encode(etcd_prefix+"/osd/state/") },
-            { "range_end", base64_encode(etcd_prefix+"/osd/state0") },
-            { "start_revision", etcd_watch_revision_osd },
-            { "watch_id", ETCD_OSD_STATE_WATCH_ID },
-            { "progress_notify", true },
-        } }
-    }).dump());
-    http_post_message(etcd_watch_ws, WS_TEXT, json11::Json(json11::Json::object {
-        { "create_request", json11::Json::object {
-            { "key", base64_encode(etcd_prefix+"/pg/") },
-            { "range_end", base64_encode(etcd_prefix+"/pg0") },
-            { "start_revision", etcd_watch_revision_pg },
-            { "watch_id", ETCD_PG_STATE_WATCH_ID },
-            { "progress_notify", true },
-        } }
-    }).dump());
-    // FIXME: Do not watch /pg/history/ at all in client code (not in OSD)
-    if (on_start_watcher_hook)
-    {
-        on_start_watcher_hook(etcd_watch_ws);
-    }
-    start_ws_keepalive();
-}
-
-void etcd_state_client_http_t::stop_ws_keepalive()
-{
-    if (ws_keepalive_timer >= 0)
-    {
-        tfd->clear_timer(ws_keepalive_timer);
-        ws_keepalive_timer = -1;
-    }
-}
-
-void etcd_state_client_http_t::start_ws_keepalive()
-{
-    if (ws_keepalive_timer < 0)
-    {
-        ws_keepalive_timer = tfd->set_timer(etcd_ws_keepalive_interval*1000, true, [this](int)
-        {
-            if (!etcd_watch_ws || etcd_watches_initialised < ETCD_TOTAL_WATCHES)
-            {
-                // Do nothing
-            }
-            else if (!ws_alive)
-            {
-                if (this->log_level > 0)
-                {
-                    fprintf(stderr, "Websocket ping failed, disconnecting from etcd %s\n", selected_etcd_address.c_str());
-                }
-                if (etcd_watch_ws)
-                {
-                    http_close(etcd_watch_ws);
-                    etcd_watch_ws = NULL;
-                }
-                start_etcd_watcher();
-            }
-            else
-            {
-                ws_alive = 0;
-                http_post_message(etcd_watch_ws, WS_TEXT, json11::Json(json11::Json::object {
-                    { "progress_request", json11::Json::object { } }
-                }).dump());
-            }
-        });
-    }
-}
-
-void etcd_state_client_http_t::load_global_config()
-{
-    etcd_state_client_t::load_global_config([this](const std::string & err)
-    {
-        if (err != "")
-        {
-            fprintf(stderr, "Error reading configuration from etcd: %s\n", err.c_str());
-            if (infinite_start)
-            {
-                tfd->set_timer(etcd_slow_timeout, false, [this](int timer_id)
-                {
-                    load_global_config();
-                });
-            }
-            else
-            {
-                exit(1);
-            }
-        }
-    });
-}
-
-void etcd_state_client_http_t::load_pgs()
-{
-    timespec tv;
-    clock_gettime(CLOCK_REALTIME, &tv);
-    uint64_t ms_passed = (tv.tv_sec-etcd_last_reload.tv_sec)*1000 + (tv.tv_nsec-etcd_last_reload.tv_nsec)/1000000;
-    if (ms_passed < etcd_min_reload_interval)
-    {
-        if (load_pgs_timer_id < 0)
-        {
-            load_pgs_timer_id = tfd->set_timer(etcd_min_reload_interval+50-ms_passed, false, [this](int) { load_pgs(); });
-        }
-        return;
-    }
-    etcd_last_reload = tv;
-    if (load_pgs_timer_id >= 0)
-    {
-        tfd->clear_timer(load_pgs_timer_id);
-        load_pgs_timer_id = -1;
-    }
-    etcd_state_client_t::load_pgs([this](const std::string & err)
-    {
-        if (err != "")
-        {
-            // Retry indefinitely
-            fprintf(stderr, "Error loading PGs from etcd: %s\n", err.c_str());
-            tfd->set_timer(etcd_slow_timeout, false, [this](int timer_id)
-            {
-                load_pgs();
-            });
-        }
-        else
-        {
-            start_etcd_watcher();
-        }
-    });
-}

src/client/etcd_state_client_http.h

@@ -1,36 +0,0 @@
-// Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
-
-#pragma once
-
-#include "etcd_state_client.h"
-
-struct __attribute__((visibility("default"))) etcd_state_client_http_t: public etcd_state_client_t
-{
-protected:
-    timerfd_manager_t *tfd = NULL;
-    std::string selected_etcd_address;
-    std::vector<std::string> addresses_to_try;
-    int ws_keepalive_timer = -1;
-    int ws_alive = 0;
-    bool rand_initialized = false;
-    int etcd_watches_initialised = 0;
-    timespec etcd_last_reload = {};
-    int load_pgs_timer_id = -1;
-    http_co_t *keepalive_client = NULL;
-
-    void pick_next_etcd();
-    void start_etcd_watcher();
-    void stop_ws_keepalive();
-    void start_ws_keepalive();
-public:
-    http_co_t *etcd_watch_ws = NULL;
-    etcd_state_client_http_t(timerfd_manager_t *tfd);
-    void etcd_call_oneshot(std::string etcd_address, std::string api, json11::Json payload, int timeout, std::function<void(std::string, json11::Json)> callback) override;
-    void etcd_call(std::string api, json11::Json payload, int timeout, int retries, int interval, std::function<void(std::string, json11::Json)> callback) override;
-    void etcd_add_watch(json11::Json watch) override;
-    void load_global_config() override;
-    void load_pgs() override;
-    void parse_config(const json11::Json & config) override;
-    ~etcd_state_client_http_t();
-};

src/client/etcd_state_client_mock.cpp

@@ -1,205 +0,0 @@
-// Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
-
-#include <assert.h>
-#include "etcd_state_client_mock.h"
-#include "str_util.h"
-
-etcd_state_client_mock_t::etcd_state_client_mock_t()
-{
-    timespec tv;
-    clock_gettime(CLOCK_REALTIME, &tv);
-    srand48(tv.tv_sec*1000000000 + tv.tv_nsec);
-}
-
-void etcd_state_client_mock_t::etcd_add_watch(json11::Json watch)
-{
-}
-
-void etcd_state_client_mock_t::etcd_call_oneshot(std::string etcd_address, std::string api, json11::Json payload,
-    int timeout, std::function<void(std::string, json11::Json)> callback)
-{
-}
-
-void etcd_state_client_mock_t::pause()
-{
-    paused = true;
-}
-
-void etcd_state_client_mock_t::resume()
-{
-    paused = false;
-    auto queue = std::move(this->queue);
-    for (auto& req: queue)
-    {
-        etcd_call(req.api, req.payload, req.timeout, req.retries, req.interval, req.callback);
-    }
-}
-
-void etcd_state_client_mock_t::set(const std::string& key, json11::Json data, uint64_t mod_revision, uint64_t lease_id)
-{
-    if (!mod_revision)
-        mod_revision = ++this->mod_revision;
-    this->data[key] = (etcd_mock_key_data_t){ .value = data.dump(), .mod_revision = mod_revision, .lease_id = lease_id };
-}
-
-void etcd_state_client_mock_t::etcd_call(std::string api, json11::Json payload, int timeout,
-    int retries, int interval, std::function<void(std::string, json11::Json)> callback)
-{
-    if (paused)
-    {
-        queue.push_back({ api, payload, timeout, retries, interval, callback });
-        return;
-    }
-    printf("+ etcd: %s\n", api.c_str());
-    if (api == "/kv/txn")
-    {
-        bool ok = true;
-        for (auto& check: payload["compare"].array_items())
-        {
-            auto key = base64_decode(check["key"].string_value());
-            etcd_mock_key_data_t *key_data = data.find(key) != data.end() ? &data.at(key) : NULL;
-            auto target = check["target"].string_value();
-            auto res = check["result"].string_value();
-            assert(res == "LESS" || res == "");
-            bool less = res == "LESS";
-            if (target == "MOD")
-            {
-                uint64_t rev = check["mod_revision"].uint64_value();
-                assert(!less || rev);
-                ok = ok && (less ? (!key_data || key_data->mod_revision < rev) : (key_data && key_data->mod_revision == rev));
-            }
-            else if (target == "CREATE")
-            {
-                uint64_t rev = check["create_revision"].uint64_value();
-                assert(rev == 0 && !less);
-                ok = ok && !key_data;
-            }
-            else if (target == "VERSION")
-            {
-                uint64_t rev = check["version"].uint64_value();
-                assert(rev == 0 && !less);
-                ok = ok && !key_data;
-            }
-            else if (target == "LEASE")
-            {
-                assert(!less);
-                uint64_t lease_id = check["lease"].uint64_value();
-                ok = ok && key_data && key_data->lease_id == lease_id;
-            }
-            else
-                assert(0);
-        }
-        std::map<std::string, etcd_kv_t> changes;
-        bool has_mod = false;
-        for (auto& op: payload[ok ? "success" : "failure"].array_items())
-        {
-            auto& obj = op.object_items();
-            has_mod = has_mod || obj.find("request_put") != obj.end() ||
-                obj.find("request_delete_range") != obj.end();
-        }
-        if (has_mod)
-        {
-            mod_revision++;
-        }
-        json11::Json::array responses;
-        for (auto& op_ptr: payload[ok ? "success" : "failure"].array_items())
-        {
-            auto& op = op_ptr.object_items();
-            if (op.find("request_range") != op.end())
-            {
-                json11::Json::array kvs;
-                auto req = op.at("request_range");
-                auto key = base64_decode(req["key"].string_value());
-                auto range_end = base64_decode(req["range_end"].string_value());
-                auto begin_it = range_end.empty() ? data.find(key) : data.lower_bound(key);
-                auto end_it = range_end.empty() ? (begin_it == data.end() ? begin_it : std::next(begin_it)) : data.lower_bound(range_end);
-                for (auto it = begin_it; it != end_it; it++)
-                {
-                    printf("\\- get: %s = %s, rev %ju\n", it->first.c_str(), it->second.value.c_str(), it->second.mod_revision);
-                    kvs.push_back(json11::Json::object {
-                        { "key", base64_encode(it->first) },
-                        { "value", base64_encode(it->second.value) },
-                        { "mod_revision", it->second.mod_revision },
-                    });
-                }
-                responses.push_back(json11::Json::object {
-                    { "response_range", json11::Json::object{ { "header", json11::Json::object{ { "revision", mod_revision } } }, { "kvs", kvs } } },
-                });
-            }
-            else if (op.find("request_put") != op.end())
-            {
-                auto req = op.at("request_put");
-                auto key = base64_decode(req["key"].string_value());
-                auto value = base64_decode(req["value"].string_value());
-                auto lease_id = req["lease"].uint64_value();
-                printf("\\- put: %s = %s, rev %ju, lease %ju\n", key.c_str(), value.c_str(), mod_revision, lease_id);
-                data[key] = {
-                    .value = value,
-                    .mod_revision = mod_revision,
-                    .lease_id = lease_id,
-                };
-                std::string err;
-                json11::Json json_value = json11::Json::parse(value, err);
-                if (err != "")
-                {
-                    fprintf(stderr, "Invalid JSON in etcd key %s during test: %s\n", key.c_str(), value.c_str());
-                    exit(1);
-                }
-                changes[key] = { .key = key, .value = json_value, .mod_revision = mod_revision };
-                responses.push_back(json11::Json::object {
-                    { "response_put", json11::Json::object{ { "header", json11::Json::object{ { "revision", mod_revision } } } } },
-                });
-            }
-            else if (op.find("request_delete_range") != op.end())
-            {
-                auto req = op.at("request_delete_range");
-                auto key = base64_decode(req["key"].string_value());
-                auto range_end = base64_decode(req["range_end"].string_value());
-                uint64_t n_del = 0;
-                for (auto it = data.lower_bound(key); it != data.end() && (range_end == "" || it->first < range_end); )
-                {
-                    auto & key = it->first;
-                    printf("\\- del: %s\n", key.c_str());
-                    changes[key] = { .key = key, .mod_revision = mod_revision };
-                    n_del++;
-                    data.erase(it++);
-                }
-                responses.push_back(json11::Json::object {
-                    { "response_delete_range", json11::Json::object{ { "header", json11::Json::object{ { "revision", mod_revision } } }, { "deleted", n_del } } },
-                });
-            }
-        }
-        callback("", json11::Json::object{
-            { "header", json11::Json::object{ { "revision", mod_revision } } },
-            { "succeeded", ok },
-            { "responses", responses }
-        });
-        // Push changes to watcher
-        if (changes.size())
-        {
-            for (auto & kv: changes)
-                parse_state(kv.second);
-            if (on_change_hook != NULL)
-                on_change_hook(changes);
-        }
-    }
-    else if (api == "/lease/grant")
-    {
-        uint64_t lease_id = (((uint64_t)lrand48()) << 32) | lrand48();
-        leases[lease_id] = payload["TTL"].uint64_value();
-        callback("", json11::Json::object{ { "ID", std::to_string(lease_id) } });
-    }
-    else
-        callback("Unsupported", json11::Json());
-}
-
-void etcd_state_client_mock_t::load_global_config()
-{
-    etcd_state_client_t::load_global_config([this](const std::string & err) {});
-}
-
-void etcd_state_client_mock_t::load_pgs()
-{
-    etcd_state_client_t::load_pgs([this](const std::string & err) {});
-}

src/client/etcd_state_client_mock.h

@@ -1,42 +0,0 @@
-// Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
-
-#pragma once
-
-#include "etcd_state_client.h"
-
-struct etcd_mock_key_data_t
-{
-    std::string value;
-    uint64_t mod_revision;
-    uint64_t lease_id;
-};
-
-struct etcd_mock_request_t
-{
-    std::string api;
-    json11::Json payload;
-    int timeout;
-    int retries;
-    int interval;
-    std::function<void(std::string, json11::Json)> callback;
-};
-
-struct etcd_state_client_mock_t: public etcd_state_client_t
-{
-    uint64_t mod_revision = 0;
-    bool paused = false;
-    std::vector<etcd_mock_request_t> queue;
-public:
-    std::map<uint64_t, uint64_t> leases;
-    std::map<std::string, etcd_mock_key_data_t> data;
-    etcd_state_client_mock_t();
-    void set(const std::string& key, json11::Json data, uint64_t mod_revision = 0, uint64_t lease_id = 0);
-    void pause();
-    void resume();
-    void etcd_call_oneshot(std::string etcd_address, std::string api, json11::Json payload, int timeout, std::function<void(std::string, json11::Json)> callback) override;
-    void etcd_call(std::string api, json11::Json payload, int timeout, int retries, int interval, std::function<void(std::string, json11::Json)> callback) override;
-    void etcd_add_watch(json11::Json watch) override;
-    void load_global_config() override;
-    void load_pgs() override;
-};

src/client/messenger.cpp

@@ -15,6 +15,106 @@
 #include "msgr_rdma.h"
 #endif
 
+#include <sys/poll.h>
+
+msgr_iothread_t::msgr_iothread_t():
+    ring(RINGLOOP_DEFAULT_SIZE, true),
+    thread(&msgr_iothread_t::run, this)
+{
+    eventfd = ring.register_eventfd();
+    if (eventfd < 0)
+    {
+        throw std::runtime_error(std::string("failed to register eventfd: ") + strerror(-eventfd));
+    }
+}
+
+msgr_iothread_t::~msgr_iothread_t()
+{
+    stop();
+}
+
+void msgr_iothread_t::add_sqe(io_uring_sqe & sqe)
+{
+    mu.lock();
+    queue.push_back((iothread_sqe_t){ .sqe = sqe, .data = std::move(*(ring_data_t*)sqe.user_data) });
+    if (queue.size() == 1)
+    {
+        cond.notify_all();
+    }
+    mu.unlock();
+}
+
+void msgr_iothread_t::stop()
+{
+    mu.lock();
+    if (stopped)
+    {
+        mu.unlock();
+        return;
+    }
+    stopped = true;
+    if (outer_loop_data)
+    {
+        outer_loop_data->callback = [](ring_data_t*){};
+    }
+    cond.notify_all();
+    close(eventfd);
+    mu.unlock();
+    thread.join();
+}
+
+void msgr_iothread_t::add_to_ringloop(ring_loop_t *outer_loop)
+{
+    assert(!this->outer_loop || this->outer_loop == outer_loop);
+    io_uring_sqe *sqe = outer_loop->get_sqe();
+    assert(sqe != NULL);
+    this->outer_loop = outer_loop;
+    this->outer_loop_data = ((ring_data_t*)sqe->user_data);
+    io_uring_prep_poll_add(sqe, eventfd, POLLIN);
+    outer_loop_data->callback = [this](ring_data_t *data)
+    {
+        if (data->res < 0)
+        {
+            throw std::runtime_error(std::string("eventfd poll failed: ") + strerror(-data->res));
+        }
+        outer_loop_data = NULL;
+        if (stopped)
+        {
+            return;
+        }
+        add_to_ringloop(this->outer_loop);
+        ring.loop();
+    };
+}
+
+void msgr_iothread_t::run()
+{
+    while (true)
+    {
+        {
+            std::unique_lock<std::mutex> lk(mu);
+            while (!stopped && !queue.size())
+                cond.wait(lk);
+            if (stopped)
+                return;
+            int i = 0;
+            for (; i < queue.size(); i++)
+            {
+                io_uring_sqe *sqe = ring.get_sqe();
+                if (!sqe)
+                    break;
+                ring_data_t *data = ((ring_data_t*)sqe->user_data);
+                *data = std::move(queue[i].data);
+                *sqe = queue[i].sqe;
+                sqe->user_data = (uint64_t)data;
+            }
+            queue.erase(queue.begin(), queue.begin()+i);
+        }
+        // We only want to offload sendmsg/recvmsg. Callbacks will be called in main thread
+        ring.submit();
+    }
+}
+
 void osd_messenger_t::init()
 {
 #ifdef WITH_RDMACM
@@ -45,7 +145,7 @@ void osd_messenger_t::init()
         rdma_contexts = msgr_rdma_context_t::create_all(
             osd_num && osd_cluster_network_masks.size() ? osd_cluster_network_masks : osd_network_masks,
             rdma_device != "" ? rdma_device.c_str() : NULL,
-            rdma_port_num, rdma_gid_index, rdma_mtu, log_level
+            rdma_port_num, rdma_gid_index, rdma_mtu, rdma_odp, log_level
         );
         if (!rdma_contexts.size())
         {
@@ -73,17 +173,21 @@ void osd_messenger_t::init()
     }
     if (ringloop && iothread_count > 0)
     {
-        init_iothreads();
+        for (int i = 0; i < iothread_count; i++)
+        {
+            auto iot = new msgr_iothread_t();
+            iothreads.push_back(iot);
+            iot->add_to_ringloop(ringloop);
+        }
     }
     keepalive_timer_id = tfd->set_timer(1000, true, [this](int)
     {
-        std::vector<uint64_t> clients_to_stop;
-        std::vector<osd_op_t*> ops_to_send;
         auto cl_it = clients.begin();
         while (cl_it != clients.end())
         {
             auto cl = cl_it->second;
             cl_it++;
+            auto peer_fd = cl->peer_fd;
             if (!cl->osd_num && !cl->in_osd_num || cl->peer_state != PEER_CONNECTED && cl->peer_state != PEER_RDMA)
             {
                 // Do not run keepalive on regular clients
@@ -95,9 +199,10 @@ void osd_messenger_t::init()
                 if (!cl->ping_time_remaining)
                 {
                     // Ping timed out, stop the client
-                    fprintf(stderr, "Ping timed out for OSD %ju (client %ju), disconnecting peer\n",
+                    fprintf(stderr, "Ping timed out for OSD %ju (client %d), disconnecting peer\n", cl->in_osd_num ? cl->in_osd_num : cl->osd_num, cl->peer_fd);
-                        cl->in_osd_num ? cl->in_osd_num : cl->osd_num, cl->client_id);
+                    stop_client(peer_fd, true);
-                    clients_to_stop.push_back(cl->client_id);
+                    // Restart iterator because it may be invalidated
+                    cl_it = clients.upper_bound(peer_fd);
                 }
             }
             else if (cl->idle_time_remaining > 0)
@@ -108,36 +213,37 @@ void osd_messenger_t::init()
                     // Connection is idle for <osd_idle_time>, send ping
                     osd_op_t *op = new osd_op_t();
                     op->op_type = OSD_OP_OUT;
-                    op->client_id = cl->client_id;
+                    op->peer_fd = cl->peer_fd;
                     op->req = (osd_any_op_t){
                         .hdr = {
                             .magic = SECONDARY_OSD_OP_MAGIC,
                             .opcode = OSD_OP_PING,
                         },
                     };
-                    op->callback = [this](osd_op_t *op)
+                    op->callback = [this, cl](osd_op_t *op)
                     {
-                        auto cl_it = clients.find(op->client_id);
+                        auto cl_it = clients.find(op->peer_fd);
-                        if (cl_it == clients.end())
+                        if (cl_it == clients.end() || cl_it->second != cl)
                         {
                             // client is already dropped
                             delete op;
                             return;
                         }
-                        auto cl = cl_it->second;
+                        int fail_fd = (op->reply.hdr.retval != 0 ? op->peer_fd : -1);
-                        uint64_t fail_client_id = (op->reply.hdr.retval != 0 ? op->client_id : 0);
                         auto fail_osd_num = cl->in_osd_num ? cl->in_osd_num : cl->osd_num;
                         cl->ping_time_remaining = 0;
                         delete op;
-                        if (fail_client_id)
+                        if (fail_fd >= 0)
                         {
-                            fprintf(stderr, "Ping failed for OSD %ju (client %ju), disconnecting peer\n", fail_osd_num, fail_client_id);
+                            fprintf(stderr, "Ping failed for OSD %ju (client %d), disconnecting peer\n", fail_osd_num, fail_fd);
-                            stop_client(fail_client_id);
+                            stop_client(fail_fd, true);
                         }
                     };
                     cl->ping_time_remaining = osd_ping_timeout;
                     cl->idle_time_remaining = osd_idle_timeout;
-                    ops_to_send.push_back(op);
+                    outbox_push(op);
+                    // Restart iterator because it may be invalidated
+                    cl_it = clients.upper_bound(peer_fd);
                 }
             }
             else
@@ -145,14 +251,6 @@ void osd_messenger_t::init()
                 cl->idle_time_remaining = osd_idle_timeout;
             }
         }
-        for (uint64_t client_id: clients_to_stop)
-        {
-            stop_client(client_id);
-        }
-        for (osd_op_t *op: ops_to_send)
-        {
-            outbox_push(op);
-        }
     });
 }
 
@@ -165,9 +263,16 @@ osd_messenger_t::~osd_messenger_t()
     }
     while (clients.size() > 0)
     {
-        stop_client(clients.begin()->first, true);
+        stop_client(clients.begin()->first, true, true);
+    }
+    if (iothreads.size())
+    {
+        for (auto iot: iothreads)
+        {
+            delete iot;
+        }
+        iothreads.clear();
     }
-    destroy_iothreads();
 #ifdef WITH_RDMA
     for (auto rdma_context: rdma_contexts)
     {
@@ -217,6 +322,7 @@ void osd_messenger_t::parse_config(const json11::Json & config)
     this->rdma_max_msg = config["rdma_max_msg"].uint64_value();
     if (!this->rdma_max_msg || this->rdma_max_msg > 128*1024*1024)
         this->rdma_max_msg = 129*1024;
+    this->rdma_odp = config["rdma_odp"].bool_value();
 #endif
     if (!osd_num)
         this->iothread_count = (uint32_t)config["client_iothread_count"].uint64_value();
@@ -335,7 +441,7 @@ void osd_messenger_t::try_connect_peer(uint64_t peer_osd)
     {
         return;
     }
-    if (osd_peers.find(peer_osd) != osd_peers.end())
+    if (osd_peer_fds.find(peer_osd) != osd_peer_fds.end())
     {
         wanted_peers.erase(peer_osd);
         return;
@@ -362,20 +468,20 @@ void osd_messenger_t::try_connect_peer_tcp(osd_num_t peer_osd, const char *peer_
 #ifdef WITH_RDMACM
     if (disable_tcp)
     {
-        on_connect_peer(peer_osd, -EINVAL, 0);
+        on_connect_peer(peer_osd, -EINVAL);
         return;
     }
 #endif
     struct sockaddr_storage addr;
     if (!string_to_addr(peer_host, 0, peer_port, &addr))
     {
-        on_connect_peer(peer_osd, -EINVAL, 0);
+        on_connect_peer(peer_osd, -EINVAL);
         return;
     }
     int peer_fd = socket(addr.ss_family, SOCK_STREAM, 0);
     if (peer_fd < 0)
     {
-        on_connect_peer(peer_osd, -errno, 0);
+        on_connect_peer(peer_osd, -errno);
         return;
     }
     fcntl(peer_fd, F_SETFL, fcntl(peer_fd, F_GETFL, 0) | O_NONBLOCK);
@@ -383,25 +489,21 @@ void osd_messenger_t::try_connect_peer_tcp(osd_num_t peer_osd, const char *peer_
     if (r < 0 && errno != EINPROGRESS)
     {
         close(peer_fd);
-        on_connect_peer(peer_osd, -errno, 0);
+        on_connect_peer(peer_osd, -errno);
         return;
     }
-    const uint64_t client_id = next_client_id++;
+    clients[peer_fd] = new osd_client_t();
-    osd_client_t *cl = new osd_client_t();
     if (log_level > 0)
     {
-        fprintf(stderr, "Connecting to OSD %ju at %s:%d (client %ju, FD %d)\n", peer_osd, peer_host, peer_port, client_id, peer_fd);
+        fprintf(stderr, "Connecting to OSD %ju at %s:%d (client %d)\n", peer_osd, peer_host, peer_port, peer_fd);
     }
-    cl->client_id = client_id;
+    clients[peer_fd]->peer_addr = addr;
-    cl->peer_addr = addr;
+    clients[peer_fd]->peer_port = peer_port;
-    cl->peer_port = peer_port;
+    clients[peer_fd]->peer_fd = peer_fd;
-    cl->peer_fd = peer_fd;
+    clients[peer_fd]->peer_state = PEER_CONNECTING;
-    cl->peer_state = PEER_CONNECTING;
+    clients[peer_fd]->connect_timeout_id = -1;
-    cl->connect_timeout_id = -1;
+    clients[peer_fd]->osd_num = peer_osd;
-    cl->osd_num = peer_osd;
+    clients[peer_fd]->in_buf = malloc_or_die(receive_buffer_size);
-    cl->in_buf = malloc_or_die(receive_buffer_size);
-    clients[client_id] = cl;
-    clients_by_fd[peer_fd] = cl;
     tfd->set_fd_handler(peer_fd, true, [this](int peer_fd, int epoll_events)
     {
         // Either OUT (connected) or HUP
@@ -409,11 +511,11 @@ void osd_messenger_t::try_connect_peer_tcp(osd_num_t peer_osd, const char *peer_
     });
     if (peer_connect_timeout > 0)
     {
-        cl->connect_timeout_id = tfd->set_timer(1000*peer_connect_timeout, false, [this, client_id](int timer_id)
+        clients[peer_fd]->connect_timeout_id = tfd->set_timer(1000*peer_connect_timeout, false, [this, peer_fd](int timer_id)
         {
-            osd_num_t peer_osd = clients.at(client_id)->osd_num;
+            osd_num_t peer_osd = clients.at(peer_fd)->osd_num;
-            stop_client(client_id);
+            stop_client(peer_fd, true);
-            on_connect_peer(peer_osd, -EPIPE, 0);
+            on_connect_peer(peer_osd, -EPIPE);
             return;
         });
     }
@@ -421,7 +523,7 @@ void osd_messenger_t::try_connect_peer_tcp(osd_num_t peer_osd, const char *peer_
 
 void osd_messenger_t::handle_connect_epoll(int peer_fd)
 {
-    auto cl = clients_by_fd.at(peer_fd);
+    auto cl = clients[peer_fd];
     if (cl->connect_timeout_id >= 0)
     {
         tfd->clear_timer(cl->connect_timeout_id);
@@ -436,8 +538,8 @@ void osd_messenger_t::handle_connect_epoll(int peer_fd)
     }
     if (result != 0)
     {
-        stop_client(cl->client_id);
+        stop_client(peer_fd, true);
-        on_connect_peer(peer_osd, -result, 0);
+        on_connect_peer(peer_osd, -result);
         return;
     }
     int one = 1;
@@ -454,23 +556,23 @@ void osd_messenger_t::handle_connect_epoll(int peer_fd)
 void osd_messenger_t::handle_peer_epoll(int peer_fd, int epoll_events)
 {
     // Mark client as ready (i.e. some data is available)
-    auto cl = clients_by_fd.at(peer_fd);
     if (epoll_events & EPOLLRDHUP)
     {
         // Stop client
         if (log_level > 0)
         {
-            fprintf(stderr, "[OSD %ju] client %ju disconnected\n", this->osd_num, cl->client_id);
+            fprintf(stderr, "[OSD %ju] client %d disconnected\n", this->osd_num, peer_fd);
         }
-        stop_client(cl->client_id);
+        stop_client(peer_fd, true);
     }
     else if (epoll_events & EPOLLIN)
     {
         // Mark client as ready (i.e. some data is available)
+        auto cl = clients[peer_fd];
         cl->read_ready++;
         if (cl->read_ready == 1)
         {
-            read_ready_clients.push_back(cl->client_id);
+            read_ready_clients.push_back(cl->peer_fd);
             if (ringloop)
                 ringloop->wakeup();
             else
@@ -479,13 +581,13 @@ void osd_messenger_t::handle_peer_epoll(int peer_fd, int epoll_events)
     }
 }
 
-void osd_messenger_t::on_connect_peer(osd_num_t peer_osd, int errcode, uint64_t client_id)
+void osd_messenger_t::on_connect_peer(osd_num_t peer_osd, int peer_fd)
 {
     auto & wp = wanted_peers.at(peer_osd);
     wp.connecting = false;
-    if (errcode < 0)
+    if (peer_fd < 0)
     {
-        fprintf(stderr, "Failed to connect to peer OSD %ju address %s port %d: %s\n", peer_osd, wp.cur_addr.c_str(), wp.cur_port, strerror(-errcode));
+        fprintf(stderr, "Failed to connect to peer OSD %ju address %s port %d: %s\n", peer_osd, wp.cur_addr.c_str(), wp.cur_port, strerror(-peer_fd));
         if (wp.address_changed)
         {
             wp.address_changed = false;
@@ -512,7 +614,7 @@ void osd_messenger_t::on_connect_peer(osd_num_t peer_osd, int errcode, uint64_t
     }
     if (log_level > 0)
     {
-        fprintf(stderr, "[OSD %ju] Connected with peer OSD %ju (client %ju)\n", osd_num, peer_osd, client_id);
+        fprintf(stderr, "[OSD %ju] Connected with peer OSD %ju (client %d)\n", osd_num, peer_osd, peer_fd);
     }
     wanted_peers.erase(peer_osd);
     repeer_pgs(peer_osd);
@@ -522,7 +624,7 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl)
 {
     osd_op_t *op = new osd_op_t();
     op->op_type = OSD_OP_OUT;
-    op->client_id = cl->client_id;
+    op->peer_fd = cl->peer_fd;
     op->req = (osd_any_op_t){
         .show_conf = {
             .header = {
@@ -546,7 +648,7 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl)
         if (!selected_ctx)
         {
             if (log_level > 0)
-                fprintf(stderr, "No RDMA context for OSD %ju connection (client %ju), using only TCP\n", cl->osd_num, cl->client_id);
+                fprintf(stderr, "No RDMA context for OSD %ju connection (peer %d), using only TCP\n", cl->osd_num, cl->peer_fd);
         }
         else
         {
@@ -607,8 +709,8 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl)
         if (err)
         {
             osd_num_t peer_osd = cl->osd_num;
-            stop_client(op->client_id);
+            stop_client(op->peer_fd);
-            on_connect_peer(peer_osd, -EINVAL, 0);
+            on_connect_peer(peer_osd, -EINVAL);
             delete op;
             return;
         }
@@ -638,13 +740,21 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl)
                     fprintf(stderr, "Connected to OSD %ju using RDMA\n", cl->osd_num);
                 }
                 cl->peer_state = PEER_RDMA;
+                tfd->set_fd_handler(cl->peer_fd, false, [this](int peer_fd, int epoll_events)
+                {
+                    // Do not miss the disconnection!
+                    if (epoll_events & EPOLLRDHUP)
+                    {
+                        handle_peer_epoll(peer_fd, epoll_events);
+                    }
+                });
                 // Add the initial receive request
                 init_recv_rdma(cl);
             }
         }
 #endif
-        osd_peers[cl->osd_num] = cl;
+        osd_peer_fds[cl->osd_num] = cl->peer_fd;
-        on_connect_peer(cl->osd_num, 0, cl->client_id);
+        on_connect_peer(cl->osd_num, cl->peer_fd);
         delete op;
     };
     outbox_push(op);
@@ -659,16 +769,13 @@ void osd_messenger_t::accept_connections(int listen_fd)
     while ((peer_fd = accept(listen_fd, (sockaddr*)&addr, &peer_addr_size)) >= 0)
     {
         assert(peer_fd != 0);
-        const uint64_t client_id = next_client_id++;
+        fprintf(stderr, "[OSD %ju] new client %d: connection from %s\n", this->osd_num, peer_fd,
-        fprintf(stderr, "[OSD %ju] new client %ju (FD %d): connection from %s\n", this->osd_num, client_id, peer_fd,
             addr_to_string(addr).c_str());
         fcntl(peer_fd, F_SETFL, fcntl(peer_fd, F_GETFL, 0) | O_NONBLOCK);
         int one = 1;
         setsockopt(peer_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
         auto cl = new osd_client_t();
-        cl->client_id = client_id;
+        clients[peer_fd] = cl;
-        clients[cl->client_id] = cl;
-        clients_by_fd[peer_fd] = cl;
         cl->is_incoming = true;
         cl->peer_addr = addr;
         cl->peer_addr = addr;

src/client/messenger.h

@@ -12,7 +12,6 @@
 #include <deque>
 #include <vector>
 
-#include "../util/robin_hood.h"
 #include "malloc_or_die.h"
 #include "json11/json11.hpp"
 #include "msgr_op.h"
@@ -38,8 +37,6 @@
 #define MSGR_SENDP_HDR 1
 #define MSGR_SENDP_FREE 2
 
-#define MAX_SIMPLE_PAYLOAD_SIZE 1048576
-
 struct msgr_sendp_t
 {
     osd_op_t *op;
@@ -53,7 +50,6 @@ struct msgr_rdma_context_t;
 
 struct osd_client_t
 {
-    uint64_t client_id = 0;
     int refs = 0;
 
     sockaddr_storage peer_addr = {};
@@ -89,7 +85,7 @@ struct osd_client_t
     std::vector<osd_op_t*> received_ops;
 
     // Outbound operations
-    robin_hood::unordered_flat_map<uint64_t, osd_op_t*> sent_ops;
+    std::map<uint64_t, osd_op_t*> sent_ops;
     uint64_t send_op_id = 0;
 
     // PGs dirtied by this client's primary-writes
@@ -131,7 +127,43 @@ struct osd_op_stats_t
     uint64_t subop_stat_count[OSD_OP_MAX+1] = { 0 };
 };
 
+#include <mutex>
+#include <condition_variable>
+#include <thread>
+
+#ifdef __MOCK__
 class msgr_iothread_t;
+#else
+struct iothread_sqe_t
+{
+    io_uring_sqe sqe;
+    ring_data_t data;
+};
+
+class msgr_iothread_t
+{
+protected:
+    ring_loop_t ring;
+    ring_loop_t *outer_loop = NULL;
+    ring_data_t *outer_loop_data = NULL;
+    int eventfd = -1;
+    bool stopped = false;
+    std::mutex mu;
+    std::condition_variable cond;
+    std::vector<iothread_sqe_t> queue;
+    std::thread thread;
+
+    void run();
+public:
+
+    msgr_iothread_t();
+    ~msgr_iothread_t();
+
+    void add_sqe(io_uring_sqe & sqe);
+    void stop();
+    void add_to_ringloop(ring_loop_t *outer_loop);
+};
+#endif
 
 #ifdef WITH_RDMA
 struct rdma_event_channel;
@@ -168,31 +200,27 @@ protected:
     std::vector<msgr_rdma_context_t *> rdma_contexts;
     uint64_t rdma_max_sge = 0, rdma_max_send = 0, rdma_max_recv = 0;
     uint64_t rdma_max_msg = 0;
+    bool rdma_odp = false;
     rdma_event_channel *rdmacm_evch = NULL;
-    robin_hood::unordered_flat_map<rdma_cm_id*, osd_client_t*> rdmacm_connections;
+    std::map<rdma_cm_id*, osd_client_t*> rdmacm_connections;
-    robin_hood::unordered_flat_map<rdma_cm_id*, rdmacm_connecting_t*> rdmacm_connecting;
+    std::map<rdma_cm_id*, rdmacm_connecting_t*> rdmacm_connecting;
 #endif
 
     std::vector<msgr_iothread_t*> iothreads;
-    std::vector<uint64_t> read_ready_clients;
+    std::vector<int> read_ready_clients;
-    std::vector<uint64_t> write_ready_clients;
+    std::vector<int> write_ready_clients;
     // We don't use ringloop->set_immediate here because we may have no ringloop in client :)
-    std::deque<osd_op_t*> set_immediate_ops;
+    std::vector<osd_op_t*> set_immediate_ops;
 
 public:
     timerfd_manager_t *tfd = NULL;
-    ring_loop_i *ringloop = NULL;
+    ring_loop_t *ringloop = NULL;
     bool has_sendmsg_zc = false;
-    uint64_t next_client_id = 1;
+    // osd_num_t is only for logging and asserts
-    // osd_num = 0 for client messenger, osd_num > 0 for OSD messenger
+    osd_num_t osd_num;
-    osd_num_t osd_num = 0;
+    std::map<int, osd_client_t*> clients;
-    uint32_t clean_entry_bitmap_size = 0;
+    std::map<osd_num_t, osd_wanted_peer_t> wanted_peers;
-    uint32_t bs_block_size = 0;
+    std::map<uint64_t, int> osd_peer_fds;
-    uint32_t max_write_request_size = 0;
-    robin_hood::unordered_flat_map<uint64_t, osd_client_t*> clients;
-    robin_hood::unordered_flat_map<uint64_t, osd_client_t*> osd_peers;
-    robin_hood::unordered_flat_map<int, osd_client_t*> clients_by_fd;
-    robin_hood::unordered_flat_map<osd_num_t, osd_wanted_peer_t> wanted_peers;
     std::vector<std::string> osd_networks;
     std::vector<addr_mask_t> osd_network_masks;
     std::vector<std::string> osd_cluster_networks;
@@ -203,11 +231,9 @@ public:
     osd_op_stats_t stats, recovery_stats;
 
     void init();
-    void init_iothreads();
     void parse_config(const json11::Json & config);
     void connect_peer(uint64_t osd_num, json11::Json peer_state);
-    void stop_client(uint64_t client_id, bool force_delete = false);
+    void stop_client(int peer_fd, bool force = false, bool force_delete = false);
-    void destroy_client(osd_client_t *cl);
     void outbox_push(osd_op_t *cur_op);
     std::function<void(osd_op_t*)> exec_op;
     std::function<void(osd_num_t)> repeer_pgs;
@@ -216,7 +242,6 @@ public:
     void read_requests();
     void send_replies();
     void accept_connections(int listen_fd);
-    void destroy_iothreads();
     ~osd_messenger_t();
 
     static json11::Json::object read_config(const json11::Json & config);
@@ -227,7 +252,7 @@ public:
 
 #ifdef WITH_RDMA
     bool is_rdma_enabled();
-    json11::Json connect_rdma(uint64_t client_id, std::string rdma_address, uint64_t client_max_msg);
+    bool connect_rdma(int peer_fd, std::string rdma_address, uint64_t client_max_msg);
 #endif
 #ifdef WITH_RDMACM
     bool is_use_rdmacm();
@@ -243,7 +268,7 @@ protected:
     void try_connect_peer_tcp(osd_num_t peer_osd, const char *peer_host, int peer_port);
     void handle_peer_epoll(int peer_fd, int epoll_events);
     void handle_connect_epoll(int peer_fd);
-    void on_connect_peer(osd_num_t peer_osd, int errcode, uint64_t client_id);
+    void on_connect_peer(osd_num_t peer_osd, int peer_fd);
     void check_peer_config(osd_client_t *cl);
     void cancel_osd_ops(osd_client_t *cl);
     void cancel_op(osd_op_t *op);
@@ -254,17 +279,19 @@ protected:
     bool handle_read(int result, osd_client_t *cl);
     bool handle_read_buffer(osd_client_t *cl, void *curbuf, int remain);
     bool handle_finished_read(osd_client_t *cl);
-    bool handle_op_hdr(osd_client_t *cl);
+    void handle_op_hdr(osd_client_t *cl);
     bool handle_reply_hdr(osd_client_t *cl);
     void handle_reply_ready(osd_op_t *op);
     void handle_immediate_ops();
+    void clear_immediate_ops(int peer_fd);
 
 #ifdef WITH_RDMA
     void try_send_rdma(osd_client_t *cl);
+    void try_send_rdma_odp(osd_client_t *cl);
+    void try_send_rdma_nodp(osd_client_t *cl);
     bool init_recv_rdma(osd_client_t *cl);
     void handle_rdma_events(msgr_rdma_context_t *rdma_context);
     msgr_rdma_context_t* choose_rdma_context(osd_client_t *cl);
-    void destroy_rdma_conn(msgr_rdma_connection_t *rdma_conn);
 #endif
 #ifdef WITH_RDMACM
     void handle_rdmacm_events();