Release 3.0.10

Important bug fixes (new store):
- Fix OSDs possibly refusing to start with "write metadata failed at offset xxx: Invalid argument"
  (fix buffer alignment during initial garbage collection)
- Rollback change from 3.0.4 - on-disk garbage entries are not skipped on start again. This change
  doesn't have any impact normally, but OSDs originally running 3.0.0-3.0.2 and then upgraded
  to 3.0.9 may hit a bug where 3.0.9 refuses to start due to entries marked as garbage too
  early and flushed to disk in 3.0.0-3.0.2.

Other changes:
- Auto-select the only RDMA device/port if there is only one
- Rollback one 3.0.9 change - there was no actual use-after-free :)
  (the problem was only relevant to an unstable development version)
- Fix `vitastor-nfs --trace` option
- Fix an unintended 1 second sleep in vitastor-cli rm-data
- Fix inode statistics not being cleared for a deleted pool
- Fix print to stdout in client

.gitea/workflows/buildenv.Dockerfile

@@ -1,28 +1,29 @@
-FROM node:16-bullseye
+FROM node:16-bookworm
 
 WORKDIR /root
 
-ADD ./docker/vitastor.gpg /etc/apt/trusted.gpg.d
+ADD ./docker/etc/apt/trusted.gpg.d /etc/apt/trusted.gpg.d
 
-RUN echo 'deb http://deb.debian.org/debian bullseye-backports main' >> /etc/apt/sources.list; \
-    echo 'deb http://vitastor.io/debian bullseye main' >> /etc/apt/sources.list; \
+RUN echo 'deb http://deb.debian.org/debian bookworm-backports main' >> /etc/apt/sources.list; \
+    echo 'deb http://vitastor.io/debian bookworm main' >> /etc/apt/sources.list; \
     echo >> /etc/apt/preferences; \
     echo 'Package: *' >> /etc/apt/preferences; \
-    echo 'Pin: release a=bullseye-backports' >> /etc/apt/preferences; \
+    echo 'Pin: release n=bookworm-backports' >> /etc/apt/preferences; \
     echo 'Pin-Priority: 500' >> /etc/apt/preferences; \
     echo >> /etc/apt/preferences; \
     echo 'Package: *' >> /etc/apt/preferences; \
     echo 'Pin: origin "vitastor.io"' >> /etc/apt/preferences; \
     echo 'Pin-Priority: 1000' >> /etc/apt/preferences; \
+    perl -i -pe 's/Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/debian.sources; \
     grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb/deb-src/' >> /etc/apt/sources.list; \
     echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf; \
     echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
 
 RUN apt-get update
-RUN apt-get -y install etcd qemu-system-x86 qemu-block-extra qemu-utils fio libasan5 \
+RUN apt-get -y install etcd qemu-system-x86 qemu-block-extra qemu-utils fio libasan8 \
     libgoogle-perftools-dev devscripts libjerasure-dev cmake libibverbs-dev libisal-dev
 RUN apt-get -y build-dep fio qemu=`dpkg -s qemu-system-x86|grep ^Version:|awk '{print $2}'`
-RUN apt-get update && apt-get -y install jq lp-solve sudo nfs-common fdisk parted
+RUN apt-get update && apt-get -y install jq lp-solve sudo nfs-common fdisk parted libc-ares-dev udev
 RUN apt-get --download-only source fio qemu=`dpkg -s qemu-system-x86|grep ^Version:|awk '{print $2}'`
 
 RUN set -ex; \

.gitea/workflows/test.yml

@@ -63,7 +63,7 @@ jobs:
     container: ${{env.TEST_IMAGE}}:${{github.sha}}
     steps:
     # leak sanitizer sometimes crashes
-    - run: cd /root/vitastor/build && ASAN_OPTIONS=detect_leaks=0 make -j16 test
+    - run: cd /root/vitastor/build && ASAN_OPTIONS=detect_leaks=0 make -j16 build_tests test
 
   npm_lint:
     runs-on: ubuntu-latest
@@ -1206,6 +1206,78 @@ jobs:
           echo ""
         done
 
+  test_checksum:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_checksum.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_checksum:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_checksum.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_corrupt_all:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_corrupt_all.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_corrupt_all:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_corrupt_all.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_reweight_half:
     runs-on: ubuntu-latest
     needs: build

CMakeLists.txt

@@ -1,20 +1,20 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 2.8...3.30)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "3.0.2")
+set(VITASTOR_VERSION "3.0.10")
 
 include(CTest)
 
 add_custom_target(build_tests)
-add_custom_target(test
-	COMMAND
-	echo leak:tcmalloc > ${CMAKE_CURRENT_BINARY_DIR}/lsan-suppress.txt &&
-	env LSAN_OPTIONS=suppressions=${CMAKE_CURRENT_BINARY_DIR}/lsan-suppress.txt ${CMAKE_CTEST_COMMAND}
+set_property(TEST PROPERTY ENVIRONMENT LSAN_OPTIONS=suppressions=${CMAKE_CURRENT_BINARY_DIR}/lsan-suppress.txt)
+add_test(gen_lsan_suppress
+    ${CMAKE_COMMAND} -E echo leak:tcmalloc > "${CMAKE_CURRENT_BINARY_DIR}/lsan-suppress.txt"
 )
+set_tests_properties(gen_lsan_suppress PROPERTIES FIXTURES_SETUP f_lsan_suppress)
+set_property(TEST PROPERTY FIXTURES_REQUIRED f_lsan_suppress)
 # make -j16 -C ../../build test_heap && ../../build/src/test/test_heap
 # make -j16 -C ../../build test_heap && rm -f $(find ../../build -name '*.gcda') && ctest -V -T test -T coverage -R heap --test-dir ../../build && (cd ../../build; gcovr -f ../src --html --html-nested -o coverage/index.html; cd ../src/test)
 # make -j16 -C ../../build test_blockstore && rm -f $(find ../../build -name '*.gcda') && ctest -V -T test -T coverage -R blockstore --test-dir ../../build && (cd ../../build; gcovr -f ../src --html --html-nested -o coverage/index.html; cd ../src/test)
 # kcov --include-path=../../../src ../../kcov ./test_blockstore
-add_dependencies(test build_tests)
 add_subdirectory(src)

csi/Dockerfile

@@ -1,5 +1,5 @@
 # Compile stage
-FROM golang:bookworm AS build
+FROM golang:trixie AS build
 
 ADD go.sum go.mod /app/
 RUN cd /app; CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go mod download -x
@@ -9,7 +9,7 @@ RUN perl -i -e '$/ = undef; while(<>) { s/\n\s*(\{\s*\n)/$1\n/g; s/\}(\s*\n\s*)e
     CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o vitastor-csi
 
 # Final stage
-FROM debian:bookworm
+FROM debian:trixie
 
 LABEL maintainers="Vitaliy Filippov <vitalif@yourcmc.ru>"
 LABEL description="Vitastor CSI Driver"
@@ -25,20 +25,20 @@ RUN apt-get update && \
         # NFS mount dependencies
         nfs-common netbase \
         # dependencies of qemu-storage-daemon
-        libnuma1 liburing2 libglib2.0-0 libfuse3-3 libaio1 libzstd1 libnettle8 \
-        libgmp10 libhogweed6 libp11-kit0 libidn2-0 libunistring2 libtasn1-6 libpcre2-8-0 libffi8 && \
+        libaio1t64 libc6 libfuse3-4 libglib2.0-0t64 libgmp10 libgnutls30t64 \
+        libhogweed6t64 libnettle8t64 libnuma1 libselinux1 liburing2 libzstd1 zlib1g && \
     apt-get clean && \
     (echo options nbd nbds_max=128 > /etc/modprobe.d/nbd.conf)
 
 COPY --from=build /app/vitastor-csi /bin/
 
-RUN (echo deb http://vitastor.io/debian bookworm main > /etc/apt/sources.list.d/vitastor.list) && \
+RUN (echo deb http://vitastor.io/debian trixie main > /etc/apt/sources.list.d/vitastor.list) && \
     ((echo 'Package: *'; echo 'Pin: origin "vitastor.io"'; echo 'Pin-Priority: 1000') > /etc/apt/preferences.d/vitastor.pref) && \
     wget -q -O /etc/apt/trusted.gpg.d/vitastor.gpg https://vitastor.io/debian/pubkey.gpg && \
     apt-get update && \
     apt-get install -y vitastor-client ibverbs-providers && \
-    wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-utils_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
-    wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-block-extra_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-trixie-10.0.2%2Bds-2%2Bvitastor1/qemu-utils_10.0.2%2Bds-2%2Bvitastor1_amd64.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-trixie-10.0.2%2Bds-2%2Bvitastor1/qemu-block-extra_10.0.2%2Bds-2%2Bvitastor1_amd64.deb && \
     dpkg -x qemu-utils*.deb tmp1 && \
     dpkg -x qemu-block-extra*.deb tmp1 && \
     cp -a tmp1/usr/bin/qemu-storage-daemon /usr/bin/ && \

csi/Dockerfile.local

@@ -1,5 +1,5 @@
 # Compile stage
-FROM golang:bookworm AS build
+FROM golang:trixie AS build
 
 ADD go.sum go.mod /app/
 RUN cd /app; CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go mod download -x
@@ -9,7 +9,7 @@ RUN perl -i -e '$/ = undef; while(<>) { s/\n\s*(\{\s*\n)/$1\n/g; s/\}(\s*\n\s*)e
     CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o vitastor-csi
 
 # Final stage
-FROM debian:bookworm
+FROM debian:trixie
 
 LABEL maintainers="Vitaliy Filippov <vitalif@yourcmc.ru>"
 LABEL description="Vitastor CSI Driver"
@@ -36,8 +36,8 @@ ADD deb /deb
 
 RUN apt-get update && \
     apt-get -y install /deb/vitastor-client_*.deb && \
-    wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-utils_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
-    wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-block-extra_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-trixie-9.2.2%2Bds-1%2Bvitastor4/qemu-utils_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-trixie-9.2.2%2Bds-1%2Bvitastor4/qemu-block-extra_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
     dpkg -x qemu-utils*.deb tmp1 && \
     dpkg -x qemu-block-extra*.deb tmp1 && \
     cp -a tmp1/usr/bin/qemu-storage-daemon /usr/bin/ && \

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.2
+VITASTOR_VERSION ?= v3.0.10
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v3.0.2
+          image: vitalif/vitastor-csi:v3.0.10
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v3.0.2
+          image: vitalif/vitastor-csi:v3.0.10
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "3.0.2"
+    vitastorCSIDriverVersion = "3.0.10"
 )
 
 // Config struct fills the parameters of request or user input

debian/build-vitastor-ubuntu-resolute.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+# 26.04 Resolute Raccoon
+
+docker build --build-arg DISTRO=ubuntu --build-arg REL=resolute -t vitastor-buildenv:resolute -f vitastor-buildenv.Dockerfile .
+docker run -it --rm -e REL=resolute -v `dirname $0`/../:/root/vitastor vitastor-buildenv:resolute /root/vitastor/debian/vitastor-build.sh

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (3.0.2-1) unstable; urgency=medium
+vitastor (3.0.10-1) unstable; urgency=medium
 
   * Bugfixes
 

debian/vitastor-build.sh

@@ -44,7 +44,7 @@ curl -s https://git.yourcmc.ru/vitalif/antietcd/archive/master.tar.gz | tar -zx
 curl -s https://git.yourcmc.ru/vitalif/tinyraft/archive/master.tar.gz | tar -zx
 
 cd /root/vitastor/packages/vitastor-$REL
-if [[ "$REL" = "trixie" && -e ../vitastor-bookworm/vitastor_$VER.orig.tar.xz ]]; then
+if [[ ( "$REL" = "trixie" || "$REL" = "resolute" ) && -e ../vitastor-bookworm/vitastor_$VER.orig.tar.xz ]]; then
     # Fucking shit, archives differ between bookworm (xz 5.4.1) and trixie (xz 5.8.1)
     cp ../vitastor-bookworm/vitastor_$VER.orig.tar.xz .
 else

docker/Dockerfile

@@ -1,6 +1,6 @@
 # Build Docker image with Vitastor packages
 
-FROM debian:bookworm
+FROM debian:trixie
 
 ADD etc/apt /etc/apt/
 RUN apt-get update && apt-get -y install vitastor ibverbs-providers udev systemd qemu-system-x86 qemu-system-common qemu-block-extra qemu-utils jq nfs-common && apt-get clean

docker/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.2
+VITASTOR_VERSION ?= v3.0.10
 
 all: build push
 

docker/etc/apt/preferences

@@ -1,3 +1,3 @@
 Package: *
-Pin: release n=bookworm-backports
+Pin: release n=trixie-backports
 Pin-Priority: 500

docker/etc/apt/sources.list.d/vitastor.list

@@ -1,2 +1,2 @@
-deb http://vitastor.io/debian bookworm main
-deb http://http.debian.net/debian/ bookworm-backports main
+deb http://vitastor.io/debian trixie main
+#deb http://http.debian.net/debian/ trixie-backports main

docker/etc/systemd/system/vitastor-host.service

@@ -7,7 +7,7 @@ PartOf=vitastor.target
 [Service]
 Restart=always
 EnvironmentFile=/etc/vitastor/docker.conf
-ExecStart=bash -c 'docker run --rm -i -v /etc/vitastor:/etc/vitastor -v /dev:/dev -v /run:/run \
+ExecStart=bash -c 'docker run --rm -i -v /etc/vitastor:/etc/vitastor -v /dev:/dev -v /run:/run -e SYSTEMD_IN_CHROOT=0 \
     --security-opt seccomp=unconfined --privileged --pid=host --log-driver none --network host --name vitastor vitastor:$VITASTOR_VERSION \
     sleep.sh'
 ExecStartPost=udevadm trigger

docker/etc/vitastor/docker.conf

@@ -4,7 +4,7 @@
 #
 
 # Desired Vitastor version
-VITASTOR_VERSION=v3.0.2
+VITASTOR_VERSION=v3.0.10
 
 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here

docker/install.sh

@@ -2,8 +2,7 @@
 
 set -e
 
-cp -urv /etc/default /host-etc/
-cp -urv /etc/systemd /host-etc/
-cp -urv /etc/udev /host-etc/
+cp -urv /etc/systemd/system/vitastor* /host-etc/systemd/system/
+cp -urv /etc/udev/rules.d /host-etc/udev/
 cp -urnv /etc/vitastor /host-etc/
 cp -urnv /opt/scripts/* /host-bin/

docs/config/network.en.md

@@ -22,7 +22,6 @@ between clients, OSDs and etcd.
 - [rdma_max_msg](#rdma_max_msg)
 - [rdma_max_recv](#rdma_max_recv)
 - [rdma_max_send](#rdma_max_send)
-- [rdma_odp](#rdma_odp)
 - [peer_connect_interval](#peer_connect_interval)
 - [peer_connect_timeout](#peer_connect_timeout)
 - [osd_idle_timeout](#osd_idle_timeout)
@@ -102,11 +101,6 @@ found or if `osd_network` is not specified. Auto-selection is also
 unsupported with old libibverbs < v32, like in Debian 10 Buster or
 CentOS 7.
 
-Vitastor supports all adapters, even ones without ODP support, like
-Mellanox ConnectX-3 and non-Mellanox cards. Versions up to Vitastor
-1.2.0 required ODP which is only present in Mellanox ConnectX >= 4.
-See also [rdma_odp](#rdma_odp).
-
 Run `ibv_devinfo -v` as root to list available RDMA devices and their
 features.
 
@@ -116,6 +110,23 @@ the manual of your network vendor for details about setting up the switch
 for RoCEv2 correctly. Usually it means setting up Lossless Ethernet with
 PFC (Priority Flow Control) and ECN (Explicit Congestion Notification).
 
+Vitastor supports all adapters, even ones without ODP (On-Demand Paging)
+support, like Mellanox ConnectX-3 and non-Mellanox cards. ODP is only present
+in Mellanox ConnectX >= 4 adapters and allows to skip memory registration
+for RDMA and thus, in theory, avoid memory copying.
+
+Versions up to Vitastor 1.2.0 required ODP, then it was disabled by default,
+but it was still supported up to 3.0.3. Now ODP support is removed because it
+actually only hurts performance: an example 3-node cluster with 8 NVMe in each
+node and 2*25 GBit/s ConnectX-6 RDMA network pushed 3950000 read iops without
+ODP, but only 239000 iops with ODP.
+
+This happens because Mellanox ODP implementation seems to be based on
+message retransmissions when the adapter doesn't know about the buffer yet -
+it likely uses standard "RNR retransmissions" (RNR = receiver not ready)
+which is generally slow in RDMA/RoCE networks. Here's a presentation about
+it from ISPASS-2021 conference: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
+
 ## rdma_port_num
 
 - Type: integer
@@ -187,28 +198,6 @@ less than `rdma_max_recv` so the receiving side doesn't run out of buffers.
 Doesn't affect memory usage - additional memory isn't allocated for send
 operations.
 
-## rdma_odp
-
-- Type: boolean
-- Default: false
-
-Use RDMA with On-Demand Paging. ODP is currently only available on Mellanox
-ConnectX-4 and newer adapters. ODP allows to not register memory explicitly
-for RDMA adapter to be able to use it. This, in turn, allows to skip memory
-copying during sending. One would think this should improve performance, but
-**in reality** RDMA performance with ODP is **drastically** worse. Example
-3-node cluster with 8 NVMe in each node and 2*25 GBit/s ConnectX-6 RDMA network
-without ODP pushes 3950000 read iops, but only 239000 iops with ODP...
-
-This happens because Mellanox ODP implementation seems to be based on
-message retransmissions when the adapter doesn't know about the buffer yet -
-it likely uses standard "RNR retransmissions" (RNR = receiver not ready)
-which is generally slow in RDMA/RoCE networks. Here's a presentation about
-it from ISPASS-2021 conference: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
-
-ODP support is retained in the code just in case a good ODP implementation
-appears one day.
-
 ## peer_connect_interval
 
 - Type: seconds

docs/config/network.ru.md

@@ -22,7 +22,6 @@
 - [rdma_max_msg](#rdma_max_msg)
 - [rdma_max_recv](#rdma_max_recv)
 - [rdma_max_send](#rdma_max_send)
-- [rdma_odp](#rdma_odp)
 - [peer_connect_interval](#peer_connect_interval)
 - [peer_connect_timeout](#peer_connect_timeout)
 - [osd_idle_timeout](#osd_idle_timeout)
@@ -101,12 +100,6 @@ RoCEv1/RoCEv2, и даже позволяет полностью отключи
 не задана. Также автовыбор не поддерживается со старыми версиями библиотеки
 libibverbs < v32, например в Debian 10 Buster или CentOS 7.
 
-Vitastor поддерживает все модели адаптеров, включая те, у которых
-нет поддержки ODP, то есть вы можете использовать RDMA с ConnectX-3 и
-картами производства не Mellanox. Версии Vitastor до 1.2.0 включительно
-требовали ODP, который есть только на Mellanox ConnectX 4 и более новых.
-См. также [rdma_odp](#rdma_odp).
-
 Запустите `ibv_devinfo -v` от имени суперпользователя, чтобы посмотреть
 список доступных RDMA-устройств, их параметры и возможности.
 
@@ -117,6 +110,24 @@ Vitastor поддерживает все модели адаптеров, вкл
 подразумевает настройку сети без потерь на основе PFC (Priority Flow
 Control) и ECN (Explicit Congestion Notification).
 
+Vitastor поддерживает все модели адаптеров, включая те, у которых нет
+поддержки ODP (On-Demand Paging), например, ConnectX-3 и карты производства
+не Mellanox. Функция ODP доступна только на адаптерах Mellanox ConnectX-4 и
+более новых и позволяет не регистрировать память для её использования RDMA-картой,
+благодаря чему в теории можно избежать лишних копирований памяти.
+
+Версии Vitastor до 1.2.0 включительно требовали ODP, потом функция был отключена
+по умолчанию, но поддерживалась вплоть до версии 3.0.3. Сейчас поддержка ODP
+полностью удалена, так как на самом деле она только портит производительность:
+например, на 3-узловом кластере с 8 NVMe в каждом узле и сетью 2*25 Гбит/с на
+чтение с RDMA без ODP удаётся снять 3950000 iops, а с ODP - всего 239000 iops.
+
+Это происходит из-за того, что реализация ODP у Mellanox неоптимальная и
+основана на повторной передаче сообщений, когда карте не известен буфер -
+вероятно, на стандартных "RNR retransmission" (RNR = receiver not ready).
+А данные повторные передачи в RDMA/RoCE - всегда очень медленная штука.
+Презентация на эту тему с конференции ISPASS-2021: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
+
 ## rdma_port_num
 
 - Тип: целое число
@@ -192,29 +203,6 @@ OSD в любом случае согласовывают реальное зн
 Не влияет на потребление памяти - дополнительная память на операции отправки
 не выделяется.
 
-## rdma_odp
-
-- Тип: булево (да/нет)
-- Значение по умолчанию: false
-
-Использовать RDMA с On-Demand Paging. ODP - функция, доступная пока что
-исключительно на адаптерах Mellanox ConnectX-4 и более новых. ODP позволяет
-не регистрировать память для её использования RDMA-картой. Благодаря этому
-можно не копировать данные при отправке их в сеть и, казалось бы, это должно
-улучшать производительность - но **по факту** получается так, что
-производительность только ухудшается, причём сильно. Пример - на 3-узловом
-кластере с 8 NVMe в каждом узле и сетью 2*25 Гбит/с на чтение с RDMA без ODP
-удаётся снять 3950000 iops, а с ODP - всего 239000 iops...
-
-Это происходит из-за того, что реализация ODP у Mellanox неоптимальная и
-основана на повторной передаче сообщений, когда карте не известен буфер -
-вероятно, на стандартных "RNR retransmission" (RNR = receiver not ready).
-А данные повторные передачи в RDMA/RoCE - всегда очень медленная штука.
-Презентация на эту тему с конференции ISPASS-2021: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
-
-Возможность использования ODP сохранена в коде на случай, если вдруг в один
-прекрасный день появится хорошая реализация ODP.
-
 ## peer_connect_interval
 
 - Тип: секунды

docs/config/osd.en.md

@@ -38,6 +38,7 @@ with an OSD restart or, for some of them, even without restarting by updating co
 - [journal_io](#journal_io)
 - [journal_sector_buffer_count](#journal_sector_buffer_count)
 - [journal_no_same_sector_overwrites](#journal_no_same_sector_overwrites)
+- [skip_corrupted_meta_entries](#skip_corrupted_meta_entries)
 - [throttle_small_writes](#throttle_small_writes)
 - [throttle_target_iops](#throttle_target_iops)
 - [throttle_target_mbs](#throttle_target_mbs)
@@ -69,6 +70,7 @@ with an OSD restart or, for some of them, even without restarting by updating co
 - [use_atomic_flag](#use_atomic_flag)
 - [pg_reshard_chunk_size](#pg_reshard_chunk_size)
 - [pg_reshard_chunk_pause_ms](#pg_reshard_chunk_pause_ms)
+- [gc_on_start](#gc_on_start)
 
 ## bind_address
 
@@ -279,13 +281,19 @@ Maximum number of journal flushers (see above min_flusher_count).
 - Type: boolean
 - Default: true
 
-This parameter makes Vitastor always keep metadata area of the block device
-in memory. It's required for good performance because it allows to avoid
-additional read-modify-write cycles during metadata modifications. Metadata
-area size is currently roughly 224 MB per 1 TB of data. You can turn it off
-to reduce memory usage by this value, but it will hurt performance. This
-restriction is likely to be removed in the future along with the upgrade
-of the metadata storage scheme.
+Only for the old store ([meta_format](layout-osd.en.md#meta_format) 2).
+
+This parameter makes Vitastor keep a copy of metadata area in memory as it is
+on disk, in addition to the metadata database. When the option is enabled, every
+metadata entry is effectively stored in RAM twice. It's required for good performance
+because it allows to avoid additional read-modify-write cycles during metadata
+modifications. Metadata area size with the old store is roughly 224 MB per 1 TB
+of data. You can turn the option off to reduce memory usage by this value, but
+it will reduce performance.
+
+For the new store ([meta_format](layout-osd.en.md#meta_format) 3), the option
+may be changed in the future to support operation without loading full metadata
+database in memory.
 
 ## inmemory_journal
 
@@ -364,6 +372,8 @@ blocks. The only situation when you should increase it to a larger value
 is when you enable journal_no_same_sector_overwrites. In this case set
 it to, for example, 1024.
 
+Not applicable to the new store ([meta_format](layout-osd.en.md#meta_format) 3).
+
 ## journal_no_same_sector_overwrites
 
 - Type: boolean
@@ -377,6 +387,17 @@ journal after writing it instead of possibly overwriting it the second time.
 
 Most (99%) other SSDs don't need this option.
 
+Not applicable to the new store ([meta_format](layout-osd.en.md#meta_format) 3).
+
+## skip_corrupted_meta_entries
+
+- Type: boolean
+- Default: false
+
+Only for the new store ([meta_format](layout-osd.en.md#meta_format) 3).
+Allow OSD to start when some metadata entries or blocks are corrupted by
+skipping them. Should be only used as an emergency measure.
+
 ## throttle_small_writes
 
 - Type: boolean
@@ -733,3 +754,9 @@ This option sets the maximum number of object is a chunk. Moving 100k objects us
 - Default: 100
 
 This option sets the interval between handling two PG count change chunks.
+
+## gc_on_start
+
+- Type: boolean
+
+Forcibly clean all garbage entries in the new store on every OSD restart.

docs/config/osd.ru.md

@@ -39,6 +39,7 @@
 - [journal_io](#journal_io)
 - [journal_sector_buffer_count](#journal_sector_buffer_count)
 - [journal_no_same_sector_overwrites](#journal_no_same_sector_overwrites)
+- [skip_corrupted_meta_entries](#skip_corrupted_meta_entries)
 - [throttle_small_writes](#throttle_small_writes)
 - [throttle_target_iops](#throttle_target_iops)
 - [throttle_target_mbs](#throttle_target_mbs)
@@ -70,6 +71,7 @@
 - [use_atomic_flag](#use_atomic_flag)
 - [pg_reshard_chunk_size](#pg_reshard_chunk_size)
 - [pg_reshard_chunk_pause_ms](#pg_reshard_chunk_pause_ms)
+- [gc_on_start](#gc_on_start)
 
 ## bind_address
 
@@ -287,13 +289,19 @@ Flusher - это микро-поток (корутина), которая коп
 - Тип: булево (да/нет)
 - Значение по умолчанию: true
 
-Данный параметр заставляет Vitastor всегда держать область метаданных диска
-в памяти. Это нужно, чтобы избегать дополнительных операций чтения с диска
-при записи. Размер области метаданных на данный момент составляет примерно
-224 МБ на 1 ТБ данных. При включении потребление памяти снизится примерно
-на эту величину, но при этом также снизится и производительность. В будущем,
-после обновления схемы хранения метаданных, это ограничение, скорее всего,
-будет ликвидировано.
+Только для старого хранилища ([meta_format](layout-osd.en.md#meta_format) 2).
+
+Данный параметр заставляет Vitastor всегда держать копию области метаданных
+в памяти в том же виде, как она лежит на диске, в дополнение к БД метаданных.
+То есть, с включённой опцией каждая запись метаданных хранится в памяти дважды.
+Это нужно, чтобы избегать дополнительных операций чтения с диска при записи.
+Размер области метаданных в старом хранилище составляет примерно 224 МБ на
+1 ТБ данных. Вы можете отключить опцию, чтобы снизить потребление памяти
+примерно на эту величину, но при этом также снизится и производительность.
+
+Для нового хранилища ([meta_format](layout-osd.en.md#meta_format) 3) опция,
+возможно, будет переработана в будущем для поддержки работы без полной
+загрузки метаданных в памяти.
 
 ## inmemory_journal
 
@@ -376,6 +384,8 @@ fsync небезопасным даже с режимом "directsync".
 нужно менять - это если вы включаете journal_no_same_sector_overwrites. В
 этом случае установите данный параметр, например, в 1024.
 
+Неприменимо к новому хранилищу ([meta_format](layout-osd.en.md#meta_format) 3).
+
 ## journal_no_same_sector_overwrites
 
 - Тип: булево (да/нет)
@@ -391,6 +401,18 @@ fsync небезопасным даже с режимом "directsync".
 
 Почти все другие SSD (99% моделей) не требуют данной опции.
 
+Неприменимо к новому хранилищу ([meta_format](layout-osd.en.md#meta_format) 3).
+
+## skip_corrupted_meta_entries
+
+- Тип: булево (да/нет)
+- Значение по умолчанию: false
+
+Только для нового хранилища ([meta_format](layout-osd.en.md#meta_format) 3).
+Разрешить OSD запускаться, даже если часть блоков или записей метаданных
+повреждена, пропуская их. Опция предназначена для использования только в
+целях аварийного восстановления.
+
 ## throttle_small_writes
 
 - Тип: булево (да/нет)
@@ -772,3 +794,9 @@ pg_minsize OSD во время переключений, что может по
 - Значение по умолчанию: 100
 
 Данная опция задаёт интервал между обработкой двух порций изменения числа PG пулов.
+
+## gc_on_start
+
+- Тип: булево (да/нет)
+
+Принудительно очищать все мусорные записи в новом хранилище при каждом запуске OSD.

docs/config/src/network.yml

@@ -84,11 +84,6 @@
     unsupported with old libibverbs < v32, like in Debian 10 Buster or
     CentOS 7.
 
-    Vitastor supports all adapters, even ones without ODP support, like
-    Mellanox ConnectX-3 and non-Mellanox cards. Versions up to Vitastor
-    1.2.0 required ODP which is only present in Mellanox ConnectX >= 4.
-    See also [rdma_odp](#rdma_odp).
-
     Run `ibv_devinfo -v` as root to list available RDMA devices and their
     features.
 
@@ -97,6 +92,23 @@
     the manual of your network vendor for details about setting up the switch
     for RoCEv2 correctly. Usually it means setting up Lossless Ethernet with
     PFC (Priority Flow Control) and ECN (Explicit Congestion Notification).
+
+    Vitastor supports all adapters, even ones without ODP (On-Demand Paging)
+    support, like Mellanox ConnectX-3 and non-Mellanox cards. ODP is only present
+    in Mellanox ConnectX >= 4 adapters and allows to skip memory registration
+    for RDMA and thus, in theory, avoid memory copying.
+
+    Versions up to Vitastor 1.2.0 required ODP, then it was disabled by default,
+    but it was still supported up to 3.0.3. Now ODP support is removed because it
+    actually only hurts performance: an example 3-node cluster with 8 NVMe in each
+    node and 2*25 GBit/s ConnectX-6 RDMA network pushed 3950000 read iops without
+    ODP, but only 239000 iops with ODP.
+
+    This happens because Mellanox ODP implementation seems to be based on
+    message retransmissions when the adapter doesn't know about the buffer yet -
+    it likely uses standard "RNR retransmissions" (RNR = receiver not ready)
+    which is generally slow in RDMA/RoCE networks. Here's a presentation about
+    it from ISPASS-2021 conference: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
   info_ru: |
     Название RDMA-устройства для связи с Vitastor OSD (например, "rocep5s0f0").
     Если не указано, Vitastor попробует найти RoCE-устройство, соответствующее
@@ -105,12 +117,6 @@
     не задана. Также автовыбор не поддерживается со старыми версиями библиотеки
     libibverbs < v32, например в Debian 10 Buster или CentOS 7.
 
-    Vitastor поддерживает все модели адаптеров, включая те, у которых
-    нет поддержки ODP, то есть вы можете использовать RDMA с ConnectX-3 и
-    картами производства не Mellanox. Версии Vitastor до 1.2.0 включительно
-    требовали ODP, который есть только на Mellanox ConnectX 4 и более новых.
-    См. также [rdma_odp](#rdma_odp).
-
     Запустите `ibv_devinfo -v` от имени суперпользователя, чтобы посмотреть
     список доступных RDMA-устройств, их параметры и возможности.
 
@@ -120,6 +126,24 @@
     коммутатора для RoCEv2 ищите в документации производителя. Обычно это
     подразумевает настройку сети без потерь на основе PFC (Priority Flow
     Control) и ECN (Explicit Congestion Notification).
+
+    Vitastor поддерживает все модели адаптеров, включая те, у которых нет
+    поддержки ODP (On-Demand Paging), например, ConnectX-3 и карты производства
+    не Mellanox. Функция ODP доступна только на адаптерах Mellanox ConnectX-4 и
+    более новых и позволяет не регистрировать память для её использования RDMA-картой,
+    благодаря чему в теории можно избежать лишних копирований памяти.
+
+    Версии Vitastor до 1.2.0 включительно требовали ODP, потом функция был отключена
+    по умолчанию, но поддерживалась вплоть до версии 3.0.3. Сейчас поддержка ODP
+    полностью удалена, так как на самом деле она только портит производительность:
+    например, на 3-узловом кластере с 8 NVMe в каждом узле и сетью 2*25 Гбит/с на
+    чтение с RDMA без ODP удаётся снять 3950000 iops, а с ODP - всего 239000 iops.
+
+    Это происходит из-за того, что реализация ODP у Mellanox неоптимальная и
+    основана на повторной передаче сообщений, когда карте не известен буфер -
+    вероятно, на стандартных "RNR retransmission" (RNR = receiver not ready).
+    А данные повторные передачи в RDMA/RoCE - всегда очень медленная штука.
+    Презентация на эту тему с конференции ISPASS-2021: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
 - name: rdma_port_num
   type: int
   info: |
@@ -218,45 +242,6 @@
     у принимающей стороны в процессе работы не заканчивались буферы на приём.
     Не влияет на потребление памяти - дополнительная память на операции отправки
     не выделяется.
-- name: rdma_odp
-  type: bool
-  default: false
-  online: false
-  info: |
-    Use RDMA with On-Demand Paging. ODP is currently only available on Mellanox
-    ConnectX-4 and newer adapters. ODP allows to not register memory explicitly
-    for RDMA adapter to be able to use it. This, in turn, allows to skip memory
-    copying during sending. One would think this should improve performance, but
-    **in reality** RDMA performance with ODP is **drastically** worse. Example
-    3-node cluster with 8 NVMe in each node and 2*25 GBit/s ConnectX-6 RDMA network
-    without ODP pushes 3950000 read iops, but only 239000 iops with ODP...
-
-    This happens because Mellanox ODP implementation seems to be based on
-    message retransmissions when the adapter doesn't know about the buffer yet -
-    it likely uses standard "RNR retransmissions" (RNR = receiver not ready)
-    which is generally slow in RDMA/RoCE networks. Here's a presentation about
-    it from ISPASS-2021 conference: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
-
-    ODP support is retained in the code just in case a good ODP implementation
-    appears one day.
-  info_ru: |
-    Использовать RDMA с On-Demand Paging. ODP - функция, доступная пока что
-    исключительно на адаптерах Mellanox ConnectX-4 и более новых. ODP позволяет
-    не регистрировать память для её использования RDMA-картой. Благодаря этому
-    можно не копировать данные при отправке их в сеть и, казалось бы, это должно
-    улучшать производительность - но **по факту** получается так, что
-    производительность только ухудшается, причём сильно. Пример - на 3-узловом
-    кластере с 8 NVMe в каждом узле и сетью 2*25 Гбит/с на чтение с RDMA без ODP
-    удаётся снять 3950000 iops, а с ODP - всего 239000 iops...
-
-    Это происходит из-за того, что реализация ODP у Mellanox неоптимальная и
-    основана на повторной передаче сообщений, когда карте не известен буфер -
-    вероятно, на стандартных "RNR retransmission" (RNR = receiver not ready).
-    А данные повторные передачи в RDMA/RoCE - всегда очень медленная штука.
-    Презентация на эту тему с конференции ISPASS-2021: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
-
-    Возможность использования ODP сохранена в коде на случай, если вдруг в один
-    прекрасный день появится хорошая реализация ODP.
 - name: peer_connect_interval
   type: sec
   min: 1

docs/config/src/osd.yml

@@ -253,21 +253,33 @@
   type: bool
   default: true
   info: |
-    This parameter makes Vitastor always keep metadata area of the block device
-    in memory. It's required for good performance because it allows to avoid
-    additional read-modify-write cycles during metadata modifications. Metadata
-    area size is currently roughly 224 MB per 1 TB of data. You can turn it off
-    to reduce memory usage by this value, but it will hurt performance. This
-    restriction is likely to be removed in the future along with the upgrade
-    of the metadata storage scheme.
+    Only for the old store ([meta_format](layout-osd.en.md#meta_format) 2).
+
+    This parameter makes Vitastor keep a copy of metadata area in memory as it is
+    on disk, in addition to the metadata database. When the option is enabled, every
+    metadata entry is effectively stored in RAM twice. It's required for good performance
+    because it allows to avoid additional read-modify-write cycles during metadata
+    modifications. Metadata area size with the old store is roughly 224 MB per 1 TB
+    of data. You can turn the option off to reduce memory usage by this value, but
+    it will reduce performance.
+
+    For the new store ([meta_format](layout-osd.en.md#meta_format) 3), the option
+    may be changed in the future to support operation without loading full metadata
+    database in memory.
   info_ru: |
-    Данный параметр заставляет Vitastor всегда держать область метаданных диска
-    в памяти. Это нужно, чтобы избегать дополнительных операций чтения с диска
-    при записи. Размер области метаданных на данный момент составляет примерно
-    224 МБ на 1 ТБ данных. При включении потребление памяти снизится примерно
-    на эту величину, но при этом также снизится и производительность. В будущем,
-    после обновления схемы хранения метаданных, это ограничение, скорее всего,
-    будет ликвидировано.
+    Только для старого хранилища ([meta_format](layout-osd.en.md#meta_format) 2).
+
+    Данный параметр заставляет Vitastor всегда держать копию области метаданных
+    в памяти в том же виде, как она лежит на диске, в дополнение к БД метаданных.
+    То есть, с включённой опцией каждая запись метаданных хранится в памяти дважды.
+    Это нужно, чтобы избегать дополнительных операций чтения с диска при записи.
+    Размер области метаданных в старом хранилище составляет примерно 224 МБ на
+    1 ТБ данных. Вы можете отключить опцию, чтобы снизить потребление памяти
+    примерно на эту величину, но при этом также снизится и производительность.
+
+    Для нового хранилища ([meta_format](layout-osd.en.md#meta_format) 3) опция,
+    возможно, будет переработана в будущем для поддержки работы без полной
+    загрузки метаданных в памяти.
 - name: inmemory_journal
   type: bool
   default: true
@@ -386,11 +398,15 @@
     blocks. The only situation when you should increase it to a larger value
     is when you enable journal_no_same_sector_overwrites. In this case set
     it to, for example, 1024.
+
+    Not applicable to the new store ([meta_format](layout-osd.en.md#meta_format) 3).
   info_ru: |
     Максимальное число буферов, разрешённых для использования под записываемые
     в журнал блоки метаданных. Единственная ситуация, в которой этот параметр
     нужно менять - это если вы включаете journal_no_same_sector_overwrites. В
     этом случае установите данный параметр, например, в 1024.
+
+    Неприменимо к новому хранилищу ([meta_format](layout-osd.en.md#meta_format) 3).
 - name: journal_no_same_sector_overwrites
   type: bool
   default: false
@@ -402,6 +418,8 @@
     journal after writing it instead of possibly overwriting it the second time.
 
     Most (99%) other SSDs don't need this option.
+
+    Not applicable to the new store ([meta_format](layout-osd.en.md#meta_format) 3).
   info_ru: |
     Включайте данную опцию для SSD вроде Intel D3-S4510 и D3-S4610, которые
     ОЧЕНЬ не любят, когда ПО перезаписывает один и тот же сектор несколько раз
@@ -412,6 +430,20 @@
     самого сектора.
 
     Почти все другие SSD (99% моделей) не требуют данной опции.
+
+    Неприменимо к новому хранилищу ([meta_format](layout-osd.en.md#meta_format) 3).
+- name: skip_corrupted_meta_entries
+  type: bool
+  default: false
+  info: |
+    Only for the new store ([meta_format](layout-osd.en.md#meta_format) 3).
+    Allow OSD to start when some metadata entries or blocks are corrupted by
+    skipping them. Should be only used as an emergency measure.
+  info_ru: |
+    Только для нового хранилища ([meta_format](layout-osd.en.md#meta_format) 3).
+    Разрешить OSD запускаться, даже если часть блоков или записей метаданных
+    повреждена, пропуская их. Опция предназначена для использования только в
+    целях аварийного восстановления.
 - name: throttle_small_writes
   type: bool
   default: false
@@ -906,3 +938,7 @@
     This option sets the interval between handling two PG count change chunks.
   info_ru: |
     Данная опция задаёт интервал между обработкой двух порций изменения числа PG пулов.
+- name: gc_on_start
+  type: bool
+  info: Forcibly clean all garbage entries in the new store on every OSD restart.
+  info_ru: Принудительно очищать все мусорные записи в новом хранилище при каждом запуске OSD.

docs/installation/docker.en.md

@@ -26,13 +26,37 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.
 
 1. Download a Docker image of the desired version: \
-   `docker pull vitalif/vitastor:v3.0.2`
+   `docker pull vitalif/vitastor:v3.0.10`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.2 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.10 install.sh`
 3. Reload udev rules: \
    `udevadm control --reload-rules`
+4. Enable the vitastor-host service: \
+   `systemctl enable --now vitastor-host`
 
-And you can return to [Quick Start](../intro/quickstart.en.md).
+After these steps, you can return to [Quick Start](../intro/quickstart.en.md).
+
+## Podman
+
+If you use Podman, run the following commands as root before installing Vitastor containers:
+
+```
+ln -s podman /usr/bin/docker
+
+mkdir -p /etc/systemd/system/systemd-udevd.service.d
+
+cat >/etc/systemd/system/systemd-udevd.service.d/override.conf <<EOF
+[Service]
+CapabilityBoundingSet=~
+SystemCallFilter=@mount capset
+EOF
+
+systemctl daemon-reload
+
+systemctl restart systemd-udevd
+```
+
+Without it, udev fails to do calls into a Podman container and Vitastor disk detection doesn't work.
 
 ## Upgrading Containers
 

docs/installation/docker.ru.md

@@ -25,14 +25,39 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.
 
 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitalif/vitastor:v3.0.2`
+   `docker pull vitalif/vitastor:v3.0.10`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.2 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.10 install.sh`
 3. Перезагрузите правила udev: \
    `udevadm control --reload-rules`
+4. Включите сервис vitastor-host: \
+   `systemctl enable --now vitastor-host`
 
 После этого вы можете возвращаться к разделу [Быстрый старт](../intro/quickstart.ru.md).
 
+## Podman
+
+Если вы используете Podman, перед установкой контейнеров Vitastor выполните следующие
+команды от имени суперпользователя:
+
+```
+ln -s podman /usr/bin/docker
+
+mkdir -p /etc/systemd/system/systemd-udevd.service.d
+
+cat >/etc/systemd/system/systemd-udevd.service.d/override.conf <<EOF
+[Service]
+CapabilityBoundingSet=~
+SystemCallFilter=@mount capset
+EOF
+
+systemctl daemon-reload
+
+systemctl restart systemd-udevd
+```
+
+Без этих настроек udev не может делать вызовы внутрь Podman-контейнеров и определение дисков Vitastor не работает.
+
 ## Обновление контейнеров
 
 Сначала обязательно проверьте раздел [Обновление Vitastor](../usage/admin.ru.md#обновление-vitastor),

docs/installation/packages.en.md

@@ -17,6 +17,7 @@
   - Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
   - Ubuntu 22.04 (Jammy): `deb https://vitastor.io/debian jammy main`
   - Ubuntu 24.04 (Noble): `deb https://vitastor.io/debian noble main`
+  - Ubuntu 26.04 (Resolute): `deb https://vitastor.io/debian resolute main`
   - Add `-oldstable` to bookworm/bullseye/buster in this line to install the last
     stable version from 0.9.x branch instead of 1.x
 - To always prefer vitastor-patched QEMU and Libvirt versions, add the following to `/etc/apt/preferences`:
@@ -33,15 +34,17 @@
   - CentOS 7: `yum install https://vitastor.io/rpms/centos/7/vitastor-release.rpm`
   - CentOS 8: `dnf install https://vitastor.io/rpms/centos/8/vitastor-release.rpm`
   - AlmaLinux 9 and other RHEL 9 clones (Rocky, Oracle...): `dnf install https://vitastor.io/rpms/centos/9/vitastor-release.rpm`
+  - AlmaLinux 10 and other RHEL 10 clones: `dnf install https://vitastor.io/rpms/centos/10/vitastor-release.rpm`
 - Enable EPEL: `yum/dnf install epel-release`
 - Enable additional CentOS repositories:
   - CentOS 7: `yum install centos-release-scl`
   - CentOS 8: `dnf install centos-release-advanced-virtualization`
-  - RHEL 9 clones: not required
+  - RHEL 9/10 clones: not required
 - Enable elrepo-kernel:
   - CentOS 7: `yum install https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm`
   - CentOS 8: `dnf install https://www.elrepo.org/elrepo-release-8.el8.elrepo.noarch.rpm`
   - RHEL 9 clones: `dnf install https://www.elrepo.org/elrepo-release-9.el9.elrepo.noarch.rpm`
+  - RHEL 10 clones: not required
 - Install packages: `yum/dnf install vitastor lpsolve etcd kernel-ml qemu-kvm`
 
 ## Installation requirements

docs/installation/packages.ru.md

@@ -17,6 +17,7 @@
   - Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
   - Ubuntu 22.04 (Jammy): `deb https://vitastor.io/debian jammy main`
   - Ubuntu 24.04 (Noble): `deb https://vitastor.io/debian noble main`
+  - Ubuntu 26.04 (Resolute): `deb https://vitastor.io/debian resolute main`
   - Добавьте `-oldstable` к слову bookworm/bullseye/buster в этой строке, чтобы
     установить последнюю стабильную версию из ветки 0.9.x вместо 1.x
 - Чтобы всегда предпочитались версии пакетов QEMU и Libvirt с патчами Vitastor, добавьте в `/etc/apt/preferences`:
@@ -33,15 +34,17 @@
   - CentOS 7: `yum install https://vitastor.io/rpms/centos/7/vitastor-release.rpm`
   - CentOS 8: `dnf install https://vitastor.io/rpms/centos/8/vitastor-release.rpm`
   - AlmaLinux 9 и другие клоны RHEL 9 (Rocky, Oracle...): `dnf install https://vitastor.io/rpms/centos/9/vitastor-release.rpm`
+  - AlmaLinux 10 и другие клоны RHEL 10: `dnf install https://vitastor.io/rpms/centos/10/vitastor-release.rpm`
 - Включите EPEL: `yum/dnf install epel-release`
 - Включите дополнительные репозитории CentOS:
   - CentOS 7: `yum install centos-release-scl`
   - CentOS 8: `dnf install centos-release-advanced-virtualization`
-  - Клоны RHEL 9: не нужно
+  - Клоны RHEL 9/10: не нужно
 - Включите elrepo-kernel:
   - CentOS 7: `yum install https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm`
   - CentOS 8: `dnf install https://www.elrepo.org/elrepo-release-8.el8.elrepo.noarch.rpm`
   - Клоны RHEL 9: `dnf install https://www.elrepo.org/elrepo-release-9.el9.elrepo.noarch.rpm`
+  - Клоны RHEL 10: не нужно
 - Установите пакеты: `yum/dnf install vitastor lpsolve etcd kernel-ml qemu-kvm`
 
 ## Установочные требования

docs/usage/nfs.en.md

@@ -262,3 +262,4 @@ Options:
 | `--logfile <FILE>` | log to the specified file                                |
 | `--enforce 1`      | enforce permissions at the server side (no by default)   |
 | `--foreground 1`   | stay in foreground, do not daemonize                     |
+| `--trace`          | trace all NFS requests                                   |

docs/usage/nfs.ru.md

@@ -274,3 +274,4 @@ VitastorFS из GPUDirect.
 | `--logfile <FILE>` | записывать логи в заданный файл                         |
 | `--enforce 1`      | проверять права доступа на стороне сервера (по умолчанию нет) |
 | `--foreground 1`   | не уходить в фон после запуска                          |
+| `--trace`          | логгировать все запросы NFS                             |

mon/http_server.js

@@ -16,7 +16,7 @@ async function create_http_server(cfg, handler)
         };
         if (cfg.mon_https_ca)
         {
-            tls.mon_https_ca = await fsp.readFile(cfg.mon_https_ca);
+            tls.ca = await fsp.readFile(cfg.mon_https_ca);
         }
         if (cfg.mon_https_client_auth)
         {

mon/lp_optimizer/lp_optimizer.js

@@ -10,16 +10,19 @@ const NO_OSD = 'Z';
 async function lp_solve(text)
 {
     const cp = child_process.spawn('lp_solve');
-    let stdout = '', stderr = '', finish_cb;
+    let stdout = '', stderr = '', finish_cb, finished = 0;
     cp.stdout.on('data', buf => stdout += buf.toString());
     cp.stderr.on('data', buf => stderr += buf.toString());
-    cp.on('exit', () => finish_cb && finish_cb());
+    cp.stdout.on('end', () => finish_cb());
+    cp.stderr.on('end', () => finish_cb());
     cp.stdin.write(text);
     cp.stdin.end();
-    if (cp.exitCode == null)
+    await new Promise(ok => (finish_cb = () =>
     {
-        await new Promise(ok => finish_cb = ok);
-    }
+        finished++;
+        if (finished == 2)
+            ok();
+    }));
     if (!stdout.trim())
     {
         return null;

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "3.0.2",
+  "version": "3.0.10",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {
@@ -9,7 +9,7 @@
   "author": "Vitaliy Filippov",
   "license": "UNLICENSED",
   "dependencies": {
-    "antietcd": "^1.2.2",
+    "antietcd": "^1.2.4",
     "sprintf-js": "^1.1.2",
     "ws": "^7.2.5"
   },

mon/pg_gen.js

@@ -195,7 +195,7 @@ async function generate_pool_pgs(state, global_config, pool_id, osd_tree, levels
     const folded_tree = make_hier_tree(global_config, folded.nodes.reduce((a, c) => { a[c.id] = c; return a; }, {}));
     const old_pg_count = prev_pgs.length;
     const optimize_cfg = {
-        osd_weights: folded.nodes.reduce((a, c) => { if (Number(c.id)) { a[c.id] = c.size; } return a; }, {}),
+        osd_weights: folded.nodes.reduce((a, c) => { if (/^\d+$/.exec(c.id) && c.size != null) { a[c.id] = c.size||0; } return a; }, {}),
         combinator: use_rules
             // new algorithm:
             ? new RuleCombinator(folded_tree, rules, pool_cfg.max_osd_combinations)

mon/stats.js

@@ -37,6 +37,7 @@ function derive_osd_stats(st, prev, prev_diff)
         const n = c.count - BigInt(pr && pr.count||0);
         diff.recovery_stats[op] = { ...c, bps: n > 0 ? b*1000n/timediff : 0n, iops: n > 0 ? n*1000n/timediff : 0n };
     }
+    diff.inode_stats = {};
     for (const pool_id in st.inode_stats||{})
     {
         diff.inode_stats[pool_id] = {};

node-binding/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor",
-  "version": "3.0.2",
+  "version": "3.0.10",
   "description": "Low-level native bindings to Vitastor client library",
   "main": "index.js",
   "keywords": [

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '3.0.2'
+VITASTOR_VERSION = '3.0.10'
 
 LOG = logging.getLogger(__name__)
 
@@ -275,7 +275,7 @@ class VitastorDriver(driver.CloneableImageVD,
             LOG.exception('error getting vitastor pool stats: '+str(e))
 
         self._stats = stats
-        
+
     def get_volume_stats(self, refresh=False):
         """Get volume stats.
         If 'refresh' is True, run update the stats first.
@@ -291,6 +291,14 @@ class VitastorDriver(driver.CloneableImageVD,
         else:
             return (1 + resp['kvs'][0]['value'], resp['kvs'][0]['mod_revision'])
 
+    def _cli(self, descr, *args):
+        args = [ 'vitastor-cli', *args, *(self._vitastor_args()) ]
+        try:
+            self._execute(*args)
+        except processutils.ProcessExecutionError as exc:
+            LOG.error("Failed to "+descr+": "+exc)
+            raise exception.VolumeBackendAPIException(data = exc.stderr)
+
     def create_volume(self, volume):
         """Creates a logical volume."""
 
@@ -302,7 +310,7 @@ class VitastorDriver(driver.CloneableImageVD,
 
         LOG.debug("creating volume '%s'", vol_name)
 
-        self._create_image(vol_name, { 'size': size })
+        self._cli('create volume', 'create', vol_name, '--size', size)
 
         if volume.encryption_key_id:
             self._create_encrypted_volume(volume, volume.obj_context)
@@ -346,7 +354,7 @@ class VitastorDriver(driver.CloneableImageVD,
         snap_name = utils.convert_str(snapshot.name)
         if snap_name.find('@') >= 0 or snap_name.find('/') >= 0:
             raise exception.VolumeBackendAPIException(data = '@ and / are forbidden in volume and snapshot names')
-        self._create_snapshot(vol_name, vol_name+'@'+snap_name)
+        self._cli('create snapshot', 'snap-create', vol_name+'@'+snap_name)
 
     def snapshot_revert_use_temp_snapshot(self):
         """Disable the use of a temporary snapshot on revert."""
@@ -359,21 +367,8 @@ class VitastorDriver(driver.CloneableImageVD,
         snap_name = utils.convert_str(snapshot.name)
 
         # Delete the image and recreate it from the snapshot
-        args = [ 'vitastor-cli', 'rm', vol_name, *(self._vitastor_args()) ]
-        try:
-            self._execute(*args)
-        except processutils.ProcessExecutionError as exc:
-            LOG.error("Failed to delete image "+vol_name+": "+exc)
-            raise exception.VolumeBackendAPIException(data = exc.stderr)
-        args = [
-            'vitastor-cli', 'create', '--parent', vol_name+'@'+snap_name,
-            vol_name, *(self._vitastor_args())
-        ]
-        try:
-            self._execute(*args)
-        except processutils.ProcessExecutionError as exc:
-            LOG.error("Failed to recreate image "+vol_name+" from "+vol_name+"@"+snap_name+": "+exc)
-            raise exception.VolumeBackendAPIException(data = exc.stderr)
+        self._cli('delete image', 'rm', vol_name)
+        self._cli('recreate image', 'create', '--parent', vol_name+'@'+snap_name, vol_name)
 
     def delete_snapshot(self, snapshot):
         """Deletes a snapshot."""
@@ -381,15 +376,7 @@ class VitastorDriver(driver.CloneableImageVD,
         vol_name = utils.convert_str(snapshot.volume_name)
         snap_name = utils.convert_str(snapshot.name)
 
-        args = [
-            'vitastor-cli', 'rm', vol_name+'@'+snap_name,
-            *(self._vitastor_args())
-        ]
-        try:
-            self._execute(*args)
-        except processutils.ProcessExecutionError as exc:
-            LOG.error("Failed to remove snapshot "+vol_name+'@'+snap_name+": "+exc)
-            raise exception.VolumeBackendAPIException(data = exc.stderr)
+        self._cli('remove snapshot', 'rm', vol_name+'@'+snap_name)
 
     def _child_count(self, parents):
         children = 0
@@ -427,13 +414,7 @@ class VitastorDriver(driver.CloneableImageVD,
         if src_vref.admin_metadata.get('readonly') == 'True':
             # source volume is a volume-image cache entry or other readonly volume
             # clone without intermediate snapshot
-            src = self._get_image(src_name)
-            LOG.debug("creating image '%s' from '%s'", dest_name, src_name)
-            new_cfg = self._create_image(dest_name, {
-                'size': size,
-                'parent_id': src['idx']['id'],
-                'parent_pool_id': src['idx']['pool_id'],
-            })
+            self._cli('create clone', 'create', '--parent', src_name, '--size', size, dest_name)
             return {}
 
         clone_snap = "%s@%s.clone_snap" % (src_name, dest_name)
@@ -446,15 +427,12 @@ class VitastorDriver(driver.CloneableImageVD,
             clone_snap = dest_name
             make_img = False
 
-        LOG.debug("creating layer '%s' under '%s'", clone_snap, src_name)
-        new_cfg = self._create_snapshot(src_name, clone_snap, True)
+        LOG.debug("creating snapshot '%s'", clone_snap)
+        self._cli('create base snapshot', 'snap-create', '--allow-existing', '1', clone_snap)
+
         if make_img:
             # Then create a clone from it
-            new_cfg = self._create_image(dest_name, {
-                'size': size,
-                'parent_id': new_cfg['parent_id'],
-                'parent_pool_id': new_cfg['parent_pool_id'],
-            })
+            self._cli('create clone', 'create', '--parent', clone_snap, '--size', size, dest_name)
 
         return {}
 
@@ -464,7 +442,8 @@ class VitastorDriver(driver.CloneableImageVD,
         vol_name = utils.convert_str(volume.name)
         snap_name = utils.convert_str(snapshot.name)
 
-        snap = self._get_image('volume-'+snapshot.volume_id+'@'+snap_name)
+        src_snap = 'volume-'+snapshot.volume_id+'@'+snap_name
+        snap = self._get_image(src_snap)
         if not snap:
             raise exception.SnapshotNotFound(snapshot_id = snap_name)
         snap_inode_id = int(resp['responses'][0]['kvs'][0]['value']['id'])
@@ -473,12 +452,8 @@ class VitastorDriver(driver.CloneableImageVD,
         size = snap['cfg']['size']
         if int(volume.size):
             size = int(volume.size) * units.Gi
-        new_cfg = self._create_image(vol_name, {
-            'size': size,
-            'parent_id': snap['idx']['id'],
-            'parent_pool_id': snap['idx']['pool_id'],
-        })
 
+        self._cli('create clone', 'create', vol_name, '--size', size, '--parent', src_snap)
         return {}
 
     def _vitastor_args(self):
@@ -505,49 +480,7 @@ class VitastorDriver(driver.CloneableImageVD,
         """Deletes a logical volume."""
 
         vol_name = utils.convert_str(volume.name)
-
-        # Find the volume and all its snapshots
-        range_end = b'index/image/' + vol_name.encode('utf-8')
-        range_end = range_end[0 : len(range_end)-1] + six.int2byte(range_end[len(range_end)-1] + 1)
-        resp = self._etcd_txn({ 'success': [
-            { 'request_range': { 'key': 'index/image/'+vol_name, 'range_end': range_end } },
-        ] })
-        if len(resp['responses'][0]['kvs']) == 0:
-            # already deleted
-            LOG.info("volume %s no longer exists in backend", vol_name)
-            return
-        layers = resp['responses'][0]['kvs']
-        layer_ids = {}
-        for kv in layers:
-            inode_id = int(kv['value']['id'])
-            pool_id = int(kv['value']['pool_id'])
-            inode_pool_id = (pool_id << 48) | (inode_id & 0xffffffffffff)
-            layer_ids[inode_pool_id] = True
-
-        # Check if the volume has clones and raise 'busy' if so
-        children = self._child_count(layer_ids)
-        if children > 0:
-            raise exception.VolumeIsBusy(volume_name = vol_name)
-
-        # Clear data
-        for kv in layers:
-            args = [
-                'vitastor-cli', 'rm-data', '--pool', str(kv['value']['pool_id']),
-                '--inode', str(kv['value']['id']), '--progress', '0',
-                *(self._vitastor_args())
-            ]
-            try:
-                self._execute(*args)
-            except processutils.ProcessExecutionError as exc:
-                LOG.error("Failed to remove layer "+kv['key']+": "+exc)
-                raise exception.VolumeBackendAPIException(data = exc.stderr)
-
-        # Delete all layers from etcd
-        requests = []
-        for kv in layers:
-            requests.append({ 'request_delete_range': { 'key': kv['key'] } })
-            requests.append({ 'request_delete_range': { 'key': 'config/inode/'+str(kv['value']['pool_id'])+'/'+str(kv['value']['id']) } })
-        self._etcd_txn({ 'success': requests })
+        self._cli('delete volume', 'rm', '--matching', vol_name, vol_name+'@*', '--progress', '0')
 
     def retype(self, context, volume, new_type, diff, host):
         """Change extra type specifications for a volume."""
@@ -567,98 +500,6 @@ class VitastorDriver(driver.CloneableImageVD,
         """Removes an export for a logical volume."""
         pass
 
-    def _create_image(self, vol_name, cfg):
-        pool_s = str(self.cfg['pool_id'])
-        image_id = 0
-        while image_id == 0:
-            # check if the image already exists and find a free ID
-            resp = self._etcd_txn({ 'success': [
-                { 'request_range': { 'key': 'index/image/'+vol_name } },
-                { 'request_range': { 'key': 'index/maxid/'+pool_s } },
-            ] })
-            if len(resp['responses'][0]['kvs']) > 0:
-                # already exists
-                raise exception.VolumeBackendAPIException(data = 'Volume '+vol_name+' already exists')
-            image_id, id_mod = self._next_id(resp['responses'][1])
-            # try to create the image
-            resp = self._etcd_txn({ 'compare': [
-                { 'target': 'MOD', 'mod_revision': id_mod, 'key': 'index/maxid/'+pool_s },
-                { 'target': 'VERSION', 'version': 0, 'key': 'index/image/'+vol_name },
-                { 'target': 'VERSION', 'version': 0, 'key': 'config/inode/'+pool_s+'/'+str(image_id) },
-            ], 'success': [
-                { 'request_put': { 'key': 'index/maxid/'+pool_s, 'value': image_id } },
-                { 'request_put': { 'key': 'index/image/'+vol_name, 'value': json.dumps({
-                    'id': image_id, 'pool_id': self.cfg['pool_id']
-                }) } },
-                { 'request_put': { 'key': 'config/inode/'+pool_s+'/'+str(image_id), 'value': json.dumps({
-                    **cfg, 'name': vol_name,
-                }) } },
-            ] })
-            if not resp.get('succeeded'):
-                # repeat
-                image_id = 0
-
-    def _create_snapshot(self, vol_name, snap_vol_name, allow_existing = False):
-        while True:
-            # check if the image already exists and snapshot doesn't
-            resp = self._etcd_txn({ 'success': [
-                { 'request_range': { 'key': 'index/image/'+vol_name } },
-                { 'request_range': { 'key': 'index/image/'+snap_vol_name } },
-            ] })
-            if len(resp['responses'][0]['kvs']) == 0:
-                raise exception.VolumeBackendAPIException(data = 'Volume '+vol_name+' does not exist')
-            if len(resp['responses'][1]['kvs']) > 0:
-                if allow_existing:
-                    snap_idx = resp['responses'][1]['kvs'][0]['value']
-                    resp = self._etcd_txn({ 'success': [
-                        { 'request_range': { 'key': 'config/inode/'+str(snap_idx['pool_id'])+'/'+str(snap_idx['id']) } },
-                    ] })
-                    if len(resp['responses'][0]['kvs']) == 0:
-                        raise exception.VolumeBackendAPIException(data =
-                            'Volume '+snap_vol_name+' is already indexed, but does not exist'
-                        )
-                    return resp['responses'][0]['kvs'][0]['value']
-                raise exception.VolumeBackendAPIException(
-                    data = 'Volume '+snap_vol_name+' already exists'
-                )
-            vol_idx = resp['responses'][0]['kvs'][0]['value']
-            vol_idx_mod = resp['responses'][0]['kvs'][0]['mod_revision']
-            # get image inode config and find a new ID
-            resp = self._etcd_txn({ 'success': [
-                { 'request_range': { 'key': 'config/inode/'+str(vol_idx['pool_id'])+'/'+str(vol_idx['id']) } },
-                { 'request_range': { 'key': 'index/maxid/'+str(self.cfg['pool_id']) } },
-            ] })
-            if len(resp['responses'][0]['kvs']) == 0:
-                raise exception.VolumeBackendAPIException(data = 'Volume '+vol_name+' does not exist')
-            vol_cfg = resp['responses'][0]['kvs'][0]['value']
-            vol_mod = resp['responses'][0]['kvs'][0]['mod_revision']
-            new_id, id_mod = self._next_id(resp['responses'][1])
-            # try to redirect image to the new inode
-            new_cfg = {
-                **vol_cfg, 'name': vol_name, 'parent_id': vol_idx['id'], 'parent_pool_id': vol_idx['pool_id']
-            }
-            resp = self._etcd_txn({ 'compare': [
-                { 'target': 'MOD', 'mod_revision': vol_idx_mod, 'key': 'index/image/'+vol_name },
-                { 'target': 'MOD', 'mod_revision': vol_mod, 'key': 'config/inode/'+str(vol_idx['pool_id'])+'/'+str(vol_idx['id']) },
-                { 'target': 'MOD', 'mod_revision': id_mod, 'key': 'index/maxid/'+str(self.cfg['pool_id']) },
-                { 'target': 'VERSION', 'version': 0, 'key': 'index/image/'+snap_vol_name },
-                { 'target': 'VERSION', 'version': 0, 'key': 'config/inode/'+str(self.cfg['pool_id'])+'/'+str(new_id) },
-            ], 'success': [
-                { 'request_put': { 'key': 'index/maxid/'+str(self.cfg['pool_id']), 'value': new_id } },
-                { 'request_put': { 'key': 'index/image/'+vol_name, 'value': json.dumps({
-                    'id': new_id, 'pool_id': self.cfg['pool_id']
-                }) } },
-                { 'request_put': { 'key': 'config/inode/'+str(self.cfg['pool_id'])+'/'+str(new_id), 'value': json.dumps(new_cfg) } },
-                { 'request_put': { 'key': 'index/image/'+snap_vol_name, 'value': json.dumps({
-                    'id': vol_idx['id'], 'pool_id': vol_idx['pool_id']
-                }) } },
-                { 'request_put': { 'key': 'config/inode/'+str(vol_idx['pool_id'])+'/'+str(vol_idx['id']), 'value': json.dumps({
-                    **vol_cfg, 'name': snap_vol_name, 'readonly': True
-                }) } }
-            ] })
-            if resp.get('succeeded'):
-                return new_cfg
-
     def initialize_connection(self, volume, connector):
         data = {
             'driver_volume_type': 'vitastor',
@@ -697,13 +538,9 @@ class VitastorDriver(driver.CloneableImageVD,
                     size = int(volume.size) * units.Gi
                     dest_name = utils.convert_str(volume.name)
                     # Find or create the base snapshot
-                    snap_cfg = self._create_snapshot(base_vol.name, base_vol.name+'@.clone_snap', True)
+                    self._cli('create base snapshot', 'create', '--allow-existing', '1', base_vol.name+'@.clone_snap')
                     # Then create a clone from it
-                    new_cfg = self._create_image(dest_name, {
-                        'size': size,
-                        'parent_id': snap_cfg['parent_id'],
-                        'parent_pool_id': snap_cfg['parent_pool_id'],
-                    })
+                    self._cli('create clone', 'create', dest_name, '--size', size, '--parent', base_vol.name+'@.clone_snap')
                     return ({}, True)
         return ({}, False)
 
@@ -770,26 +607,8 @@ class VitastorDriver(driver.CloneableImageVD,
     def extend_volume(self, volume, new_size):
         """Extend an existing volume."""
         vol_name = utils.convert_str(volume.name)
-        while True:
-            vol = self._get_image(vol_name)
-            if not vol:
-                raise exception.VolumeBackendAPIException(data = 'Volume '+vol_name+' does not exist')
-            # change size
-            size = int(new_size) * units.Gi
-            if size == vol['cfg']['size']:
-                break
-            resp = self._etcd_txn({ 'compare': [ {
-                'target': 'MOD',
-                'mod_revision': vol['cfg_mod'],
-                'key': 'config/inode/'+str(vol['idx']['pool_id'])+'/'+str(vol['idx']['id']),
-            } ], 'success': [
-                { 'request_put': {
-                    'key': 'config/inode/'+str(vol['idx']['pool_id'])+'/'+str(vol['idx']['id']),
-                    'value': json.dumps({ **vol['cfg'], 'size': size }),
-                } },
-            ] })
-            if resp.get('succeeded'):
-                break
+        size = int(new_size) * units.Gi
+        self._cli('extend volume', 'modify', vol_name, '--resize', new_size)
         LOG.debug(
             "Extend volume from %(old_size)s GB to %(new_size)s GB.",
             {'old_size': volume.size, 'new_size': new_size}
@@ -862,28 +681,7 @@ class VitastorDriver(driver.CloneableImageVD,
         """
         from_name = self._get_existing_name(existing_ref)
         to_name = utils.convert_str(volume.name)
-        self._rename(from_name, to_name)
-
-    def _rename(self, from_name, to_name):
-        while True:
-            vol = self._get_image(from_name)
-            if not vol:
-                raise exception.VolumeBackendAPIException(data = 'Volume '+from_name+' does not exist')
-            to = self._get_image(to_name)
-            if to:
-                raise exception.VolumeBackendAPIException(data = 'Volume '+to_name+' already exists')
-            resp = self._etcd_txn({ 'compare': [
-                { 'target': 'MOD', 'mod_revision': vol['idx_mod'], 'key': 'index/image/'+vol['cfg']['name'] },
-                { 'target': 'MOD', 'mod_revision': vol['cfg_mod'], 'key': 'config/inode/'+str(vol['idx']['pool_id'])+'/'+str(vol['idx']['id']) },
-                { 'target': 'VERSION', 'version': 0, 'key': 'index/image/'+to_name },
-            ], 'success': [
-                { 'request_delete_range': { 'key': 'index/image/'+vol['cfg']['name'] } },
-                { 'request_put': { 'key': 'index/image/'+to_name, 'value': json.dumps(vol['idx']) } },
-                { 'request_put': { 'key': 'config/inode/'+str(vol['idx']['pool_id'])+'/'+str(vol['idx']['id']),
-                    'value': json.dumps({ **vol['cfg'], 'name': to_name }) } },
-            ] })
-            if resp.get('succeeded'):
-                break
+        self._cli('rename', 'modify', from_name, '--rename', to_name)
 
     def unmanage(self, volume):
         pass
@@ -956,7 +754,7 @@ class VitastorDriver(driver.CloneableImageVD,
         snap_name = self._get_existing_name(existing_ref)
         from_name = vol_name+'@'+snap_name
         to_name = vol_name+'@'+utils.convert_str(snapshot.name)
-        self._rename(from_name, to_name)
+        self._cli('rename', 'modify', from_name, '--rename', to_name)
 
     def unmanage_snapshot(self, snapshot):
         """Removes the specified snapshot from Cinder management."""

patches/libvirt-12.0-vitastor.diff

@@ -0,0 +1,637 @@
+diff --git a/include/libvirt/libvirt-storage.h b/include/libvirt/libvirt-storage.h
+index aaad4a3da1..5f5daa8341 100644
+--- a/include/libvirt/libvirt-storage.h
++++ b/include/libvirt/libvirt-storage.h
+@@ -326,6 +326,7 @@ typedef enum {
+     VIR_CONNECT_LIST_STORAGE_POOLS_ZFS           = 1 << 17, /* (Since: 1.2.8) */
+     VIR_CONNECT_LIST_STORAGE_POOLS_VSTORAGE      = 1 << 18, /* (Since: 3.1.0) */
+     VIR_CONNECT_LIST_STORAGE_POOLS_ISCSI_DIRECT  = 1 << 19, /* (Since: 5.6.0) */
++    VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR      = 1 << 20, /* (Since: 5.0.0) */
+ } virConnectListAllStoragePoolsFlags;
+ 
+ int                     virConnectListAllStoragePools(virConnectPtr conn,
+diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
+index 9ca5c2450c..cc52f00c0c 100644
+--- a/src/conf/domain_conf.c
++++ b/src/conf/domain_conf.c
+@@ -7453,7 +7453,8 @@ virDomainDiskSourceNetworkParse(xmlNodePtr node,
+     src->configFile = virXPathString("string(./config/@file)", ctxt);
+ 
+     if (src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTP ||
+-        src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTPS)
++        src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTPS ||
++        src->protocol == VIR_STORAGE_NET_PROTOCOL_VITASTOR)
+         src->query = virXMLPropString(node, "query");
+ 
+     if (virDomainStorageNetworkParseHosts(node, ctxt, &src->hosts, &src->nhosts) < 0)
+@@ -32187,6 +32188,7 @@ virDomainStorageSourceTranslateSourcePool(virStorageSource *src,
+ 
+     case VIR_STORAGE_POOL_MPATH:
+     case VIR_STORAGE_POOL_RBD:
++    case VIR_STORAGE_POOL_VITASTOR:
+     case VIR_STORAGE_POOL_SHEEPDOG:
+     case VIR_STORAGE_POOL_GLUSTER:
+     case VIR_STORAGE_POOL_LAST:
+diff --git a/src/conf/domain_validate.c b/src/conf/domain_validate.c
+index 7346a61731..83e94d762e 100644
+--- a/src/conf/domain_validate.c
++++ b/src/conf/domain_validate.c
+@@ -520,6 +520,7 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
+         case VIR_STORAGE_NET_PROTOCOL_RBD:
+             break;
+ 
++        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+         case VIR_STORAGE_NET_PROTOCOL_NBD:
+         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
+         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
+@@ -592,7 +593,7 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
+         }
+     }
+ 
+-    /* internal snapshots and config files are currently supported only with rbd: */
++    /* internal snapshots are currently supported only with rbd: */
+     if (virStorageSourceGetActualType(src) != VIR_STORAGE_TYPE_NETWORK &&
+         src->protocol != VIR_STORAGE_NET_PROTOCOL_RBD) {
+         if (src->snapshot) {
+@@ -600,10 +601,14 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
+                            _("<snapshot> element is currently supported only with 'rbd' disks"));
+             return -1;
+         }
+-
++    }
++    /* config files are currently supported only with rbd and vitastor: */
++    if (virStorageSourceGetActualType(src) != VIR_STORAGE_TYPE_NETWORK &&
++        src->protocol != VIR_STORAGE_NET_PROTOCOL_RBD &&
++        src->protocol != VIR_STORAGE_NET_PROTOCOL_VITASTOR) {
+         if (src->configFile) {
+             virReportError(VIR_ERR_XML_ERROR, "%s",
+-                           _("<config> element is currently supported only with 'rbd' disks"));
++                           _("<config> element is currently supported only with 'rbd' and 'vitastor' disks"));
+             return -1;
+         }
+     }
+diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng
+index 114dd3f96f..c71f9a3277 100644
+--- a/src/conf/schemas/domaincommon.rng
++++ b/src/conf/schemas/domaincommon.rng
+@@ -2093,6 +2093,35 @@
+     </element>
+   </define>
+ 
++  <define name="diskSourceNetworkProtocolVitastor">
++    <element name="source">
++      <interleave>
++        <attribute name="protocol">
++          <value>vitastor</value>
++        </attribute>
++        <ref name="diskSourceCommon"/>
++        <optional>
++          <attribute name="name"/>
++        </optional>
++        <optional>
++          <attribute name="query"/>
++        </optional>
++        <zeroOrMore>
++          <ref name="diskSourceNetworkHost"/>
++        </zeroOrMore>
++        <optional>
++          <element name="config">
++            <attribute name="file">
++              <ref name="absFilePath"/>
++            </attribute>
++            <empty/>
++          </element>
++        </optional>
++        <empty/>
++      </interleave>
++    </element>
++  </define>
++
+   <define name="diskSourceNetworkProtocolISCSI">
+     <element name="source">
+       <attribute name="protocol">
+@@ -2443,6 +2472,7 @@
+       <ref name="diskSourceNetworkProtocolSimple"/>
+       <ref name="diskSourceNetworkProtocolVxHS"/>
+       <ref name="diskSourceNetworkProtocolNFS"/>
++      <ref name="diskSourceNetworkProtocolVitastor"/>
+     </choice>
+   </define>
+ 
+diff --git a/src/conf/storage_conf.c b/src/conf/storage_conf.c
+index 1dc9365bf2..a8a736be81 100644
+--- a/src/conf/storage_conf.c
++++ b/src/conf/storage_conf.c
+@@ -56,7 +56,7 @@ VIR_ENUM_IMPL(virStoragePool,
+               "logical", "disk", "iscsi",
+               "iscsi-direct", "scsi", "mpath",
+               "rbd", "sheepdog", "gluster",
+-              "zfs", "vstorage",
++              "zfs", "vstorage", "vitastor",
+ );
+ 
+ VIR_ENUM_IMPL(virStoragePoolFormatFileSystem,
+@@ -242,6 +242,18 @@ static virStoragePoolTypeInfo poolTypeInfo[] = {
+           .formatToString = virStorageFileFormatTypeToString,
+       }
+     },
++    {.poolType = VIR_STORAGE_POOL_VITASTOR,
++     .poolOptions = {
++         .flags = (VIR_STORAGE_POOL_SOURCE_HOST |
++                   VIR_STORAGE_POOL_SOURCE_NETWORK |
++                   VIR_STORAGE_POOL_SOURCE_NAME),
++      },
++      .volOptions = {
++          .defaultFormat = VIR_STORAGE_FILE_RAW,
++          .formatFromString = virStorageVolumeFormatFromString,
++          .formatToString = virStorageFileFormatTypeToString,
++      }
++    },
+     {.poolType = VIR_STORAGE_POOL_SHEEPDOG,
+      .poolOptions = {
+          .flags = (VIR_STORAGE_POOL_SOURCE_HOST |
+@@ -538,6 +550,11 @@ virStoragePoolDefParseSource(xmlXPathContextPtr ctxt,
+                        _("element 'name' is mandatory for RBD pool"));
+         return -1;
+     }
++    if (pool_type == VIR_STORAGE_POOL_VITASTOR && source->name == NULL) {
++        virReportError(VIR_ERR_XML_ERROR, "%s",
++                       _("element 'name' is mandatory for Vitastor pool"));
++        return -1;
++    }
+ 
+     if (options->formatFromString) {
+         g_autofree char *format = NULL;
+@@ -1127,6 +1144,7 @@ virStoragePoolDefFormatBuf(virBuffer *buf,
+     /* RBD, Sheepdog, Gluster and Iscsi-direct devices are not local block devs nor
+      * files, so they don't have a target */
+     if (def->type != VIR_STORAGE_POOL_RBD &&
++        def->type != VIR_STORAGE_POOL_VITASTOR &&
+         def->type != VIR_STORAGE_POOL_SHEEPDOG &&
+         def->type != VIR_STORAGE_POOL_GLUSTER &&
+         def->type != VIR_STORAGE_POOL_ISCSI_DIRECT) {
+diff --git a/src/conf/storage_conf.h b/src/conf/storage_conf.h
+index fc67957cfe..720c07ef74 100644
+--- a/src/conf/storage_conf.h
++++ b/src/conf/storage_conf.h
+@@ -103,6 +103,7 @@ typedef enum {
+     VIR_STORAGE_POOL_GLUSTER,  /* Gluster device */
+     VIR_STORAGE_POOL_ZFS,      /* ZFS */
+     VIR_STORAGE_POOL_VSTORAGE, /* Virtuozzo Storage */
++    VIR_STORAGE_POOL_VITASTOR, /* Vitastor */
+ 
+     VIR_STORAGE_POOL_LAST,
+ } virStoragePoolType;
+@@ -454,6 +455,7 @@ VIR_ENUM_DECL(virStoragePartedFs);
+                  VIR_CONNECT_LIST_STORAGE_POOLS_SCSI     | \
+                  VIR_CONNECT_LIST_STORAGE_POOLS_MPATH    | \
+                  VIR_CONNECT_LIST_STORAGE_POOLS_RBD      | \
++                 VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR | \
+                  VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG | \
+                  VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER  | \
+                  VIR_CONNECT_LIST_STORAGE_POOLS_ZFS      | \
+diff --git a/src/conf/storage_source_conf.c b/src/conf/storage_source_conf.c
+index d7b9bdfecb..38aefd0dd4 100644
+--- a/src/conf/storage_source_conf.c
++++ b/src/conf/storage_source_conf.c
+@@ -90,6 +90,7 @@ VIR_ENUM_IMPL(virStorageNetProtocol,
+               "ssh",
+               "vxhs",
+               "nfs",
++              "vitastor",
+ );
+ 
+ 
+@@ -1317,6 +1318,7 @@ virStorageSourceNetworkDefaultPort(virStorageNetProtocol protocol)
+         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
+             return 24007;
+ 
++        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+         case VIR_STORAGE_NET_PROTOCOL_RBD:
+             /* we don't provide a default for RBD */
+             return 0;
+diff --git a/src/conf/storage_source_conf.h b/src/conf/storage_source_conf.h
+index 22c35d420d..f1e32ea83d 100644
+--- a/src/conf/storage_source_conf.h
++++ b/src/conf/storage_source_conf.h
+@@ -131,6 +131,7 @@ typedef enum {
+     VIR_STORAGE_NET_PROTOCOL_SSH,
+     VIR_STORAGE_NET_PROTOCOL_VXHS,
+     VIR_STORAGE_NET_PROTOCOL_NFS,
++    VIR_STORAGE_NET_PROTOCOL_VITASTOR,
+ 
+     VIR_STORAGE_NET_PROTOCOL_LAST
+ } virStorageNetProtocol;
+diff --git a/src/conf/virstorageobj.c b/src/conf/virstorageobj.c
+index 59fa5da372..4739167f5f 100644
+--- a/src/conf/virstorageobj.c
++++ b/src/conf/virstorageobj.c
+@@ -1438,6 +1438,7 @@ virStoragePoolObjSourceFindDuplicateCb(const void *payload,
+             return 1;
+         break;
+ 
++    case VIR_STORAGE_POOL_VITASTOR:
+     case VIR_STORAGE_POOL_ISCSI_DIRECT:
+     case VIR_STORAGE_POOL_RBD:
+     case VIR_STORAGE_POOL_LAST:
+@@ -1921,6 +1922,8 @@ virStoragePoolObjMatch(virStoragePoolObj *obj,
+                (obj->def->type == VIR_STORAGE_POOL_MPATH))   ||
+               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_RBD) &&
+                (obj->def->type == VIR_STORAGE_POOL_RBD))     ||
++              (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR) &&
++               (obj->def->type == VIR_STORAGE_POOL_VITASTOR)) ||
+               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG) &&
+                (obj->def->type == VIR_STORAGE_POOL_SHEEPDOG)) ||
+               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER) &&
+diff --git a/src/libvirt-storage.c b/src/libvirt-storage.c
+index db7660aac4..561df34709 100644
+--- a/src/libvirt-storage.c
++++ b/src/libvirt-storage.c
+@@ -94,6 +94,7 @@ virStoragePoolGetConnect(virStoragePoolPtr pool)
+  * VIR_CONNECT_LIST_STORAGE_POOLS_SCSI
+  * VIR_CONNECT_LIST_STORAGE_POOLS_MPATH
+  * VIR_CONNECT_LIST_STORAGE_POOLS_RBD
++ * VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR
+  * VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG
+  * VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER
+  * VIR_CONNECT_LIST_STORAGE_POOLS_ZFS
+diff --git a/src/libxl/libxl_conf.c b/src/libxl/libxl_conf.c
+index 2b988157fa..9d0eb47b25 100644
+--- a/src/libxl/libxl_conf.c
++++ b/src/libxl/libxl_conf.c
+@@ -1069,6 +1069,7 @@ libxlMakeNetworkDiskSrcStr(virStorageSource *src,
+     case VIR_STORAGE_NET_PROTOCOL_SSH:
+     case VIR_STORAGE_NET_PROTOCOL_VXHS:
+     case VIR_STORAGE_NET_PROTOCOL_NFS:
++    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+     case VIR_STORAGE_NET_PROTOCOL_LAST:
+     case VIR_STORAGE_NET_PROTOCOL_NONE:
+         virReportError(VIR_ERR_NO_SUPPORT,
+diff --git a/src/libxl/xen_xl.c b/src/libxl/xen_xl.c
+index e72e7d7f44..8482c21805 100644
+--- a/src/libxl/xen_xl.c
++++ b/src/libxl/xen_xl.c
+@@ -1461,6 +1461,7 @@ xenFormatXLDiskSrcNet(virStorageSource *src)
+     case VIR_STORAGE_NET_PROTOCOL_SSH:
+     case VIR_STORAGE_NET_PROTOCOL_VXHS:
+     case VIR_STORAGE_NET_PROTOCOL_NFS:
++    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+     case VIR_STORAGE_NET_PROTOCOL_LAST:
+     case VIR_STORAGE_NET_PROTOCOL_NONE:
+         virReportError(VIR_ERR_NO_SUPPORT,
+diff --git a/src/qemu/qemu_block.c b/src/qemu/qemu_block.c
+index 9b43279797..459d8e8a65 100644
+--- a/src/qemu/qemu_block.c
++++ b/src/qemu/qemu_block.c
+@@ -743,6 +743,38 @@ qemuBlockStorageSourceGetRBDProps(virStorageSource *src,
+ }
+ 
+ 
++static virJSONValue *
++qemuBlockStorageSourceGetVitastorProps(virStorageSource *src)
++{
++    virJSONValue *ret = NULL;
++    virStorageNetHostDef *host;
++    size_t i;
++    g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;
++    g_autofree char *etcd = NULL;
++
++    for (i = 0; i < src->nhosts; i++) {
++        host = src->hosts + i;
++        if ((virStorageNetHostTransport)host->transport != VIR_STORAGE_NET_HOST_TRANS_TCP) {
++            return NULL;
++        }
++        virBufferAsprintf(&buf, i > 0 ? ",%s:%u" : "%s:%u", host->name, host->port);
++    }
++    if (src->nhosts > 0) {
++        etcd = virBufferContentAndReset(&buf);
++    }
++
++    if (virJSONValueObjectAdd(&ret,
++                              "S:etcd-host", etcd,
++                              "S:etcd-prefix", src->query,
++                              "S:config-path", src->configFile,
++                              "s:image", src->path,
++                              NULL) < 0)
++        return NULL;
++
++    return ret;
++}
++
++
+ static virJSONValue *
+ qemuBlockStorageSourceGetSshProps(virStorageSource *src)
+ {
+@@ -1094,6 +1126,12 @@ qemuBlockStorageSourceGetBackendProps(virStorageSource *src,
+                 return NULL;
+             break;
+ 
++        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
++            driver = "vitastor";
++            if (!(fileprops = qemuBlockStorageSourceGetVitastorProps(src)))
++                return NULL;
++            break;
++
+         case VIR_STORAGE_NET_PROTOCOL_SSH:
+             driver = "ssh";
+             if (!(fileprops = qemuBlockStorageSourceGetSshProps(src)))
+@@ -1997,6 +2035,7 @@ qemuBlockGetBackingStoreString(virStorageSource *src,
+ 
+             case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
+             case VIR_STORAGE_NET_PROTOCOL_RBD:
++            case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+             case VIR_STORAGE_NET_PROTOCOL_VXHS:
+             case VIR_STORAGE_NET_PROTOCOL_NFS:
+             case VIR_STORAGE_NET_PROTOCOL_SSH:
+@@ -2377,6 +2416,12 @@ qemuBlockStorageSourceCreateGetStorageProps(virStorageSource *src,
+                 return -1;
+             break;
+ 
++        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
++            driver = "vitastor";
++            if (!(location = qemuBlockStorageSourceGetVitastorProps(src)))
++                return -1;
++            break;
++
+         case VIR_STORAGE_NET_PROTOCOL_SSH:
+             if (srcPriv->nbdkitProcess) {
+                 /* disk creation not yet supported with nbdkit, and even if it
+diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
+index ac56fc7cb4..9e407b4aab 100644
+--- a/src/qemu/qemu_domain.c
++++ b/src/qemu/qemu_domain.c
+@@ -4677,7 +4677,8 @@ qemuDomainValidateStorageSource(virStorageSource *src,
+     if (src->query &&
+         (actualType != VIR_STORAGE_TYPE_NETWORK ||
+          (src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTPS &&
+-          src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTP))) {
++          src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTP &&
++          src->protocol != VIR_STORAGE_NET_PROTOCOL_VITASTOR))) {
+         virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                        _("query is supported only with HTTP(S) protocols"));
+         return -1;
+@@ -9103,6 +9104,7 @@ qemuDomainPrepareStorageSourceTLS(virStorageSource *src,
+         break;
+ 
+     case VIR_STORAGE_NET_PROTOCOL_RBD:
++    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+     case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
+     case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
+     case VIR_STORAGE_NET_PROTOCOL_ISCSI:
+diff --git a/src/qemu/qemu_snapshot.c b/src/qemu/qemu_snapshot.c
+index e738afffc3..37d64f469b 100644
+--- a/src/qemu/qemu_snapshot.c
++++ b/src/qemu/qemu_snapshot.c
+@@ -665,6 +665,7 @@ qemuSnapshotPrepareDiskExternalInactive(virDomainSnapshotDiskDef *snapdisk,
+         case VIR_STORAGE_NET_PROTOCOL_NONE:
+         case VIR_STORAGE_NET_PROTOCOL_NBD:
+         case VIR_STORAGE_NET_PROTOCOL_RBD:
++        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
+         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
+         case VIR_STORAGE_NET_PROTOCOL_ISCSI:
+@@ -893,6 +894,7 @@ qemuSnapshotPrepareDiskInternal(virDomainDiskDef *disk,
+         case VIR_STORAGE_NET_PROTOCOL_NONE:
+         case VIR_STORAGE_NET_PROTOCOL_NBD:
+         case VIR_STORAGE_NET_PROTOCOL_RBD:
++        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
+         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
+         case VIR_STORAGE_NET_PROTOCOL_ISCSI:
+diff --git a/src/storage/storage_driver.c b/src/storage/storage_driver.c
+index e19e032427..59f91f4710 100644
+--- a/src/storage/storage_driver.c
++++ b/src/storage/storage_driver.c
+@@ -1626,6 +1626,7 @@ storageVolLookupByPathCallback(virStoragePoolObj *obj,
+ 
+         case VIR_STORAGE_POOL_GLUSTER:
+         case VIR_STORAGE_POOL_RBD:
++        case VIR_STORAGE_POOL_VITASTOR:
+         case VIR_STORAGE_POOL_SHEEPDOG:
+         case VIR_STORAGE_POOL_ZFS:
+         case VIR_STORAGE_POOL_LAST:
+diff --git a/src/storage_file/storage_source_backingstore.c b/src/storage_file/storage_source_backingstore.c
+index 821378883c..2211f6891b 100644
+--- a/src/storage_file/storage_source_backingstore.c
++++ b/src/storage_file/storage_source_backingstore.c
+@@ -264,6 +264,75 @@ virStorageSourceParseRBDColonString(const char *rbdstr,
+ }
+ 
+ 
++static int
++virStorageSourceParseVitastorColonString(const char *colonstr,
++                                         virStorageSource *src)
++{
++    char *p, *e, *next;
++    g_autofree char *options = NULL;
++
++    /* optionally skip the "vitastor:" prefix if provided */
++    if (STRPREFIX(colonstr, "vitastor:"))
++        colonstr += strlen("vitastor:");
++
++    options = g_strdup(colonstr);
++
++    p = options;
++    while (*p) {
++        /* find : delimiter or end of string */
++        for (e = p; *e && *e != ':'; ++e) {
++            if (*e == '\\') {
++                e++;
++                if (*e == '\0')
++                    break;
++            }
++        }
++        if (*e == '\0') {
++            next = e;    /* last kv pair */
++        } else {
++            next = e + 1;
++            *e = '\0';
++        }
++
++        if (STRPREFIX(p, "image=")) {
++            src->path = g_strdup(p + strlen("image="));
++        } else if (STRPREFIX(p, "etcd-prefix=")) {
++            src->query = g_strdup(p + strlen("etcd-prefix="));
++        } else if (STRPREFIX(p, "config-path=")) {
++            src->configFile = g_strdup(p + strlen("config-path="));
++        } else if (STRPREFIX(p, "etcd-host=")) {
++            char *h, *sep;
++
++            h = p + strlen("etcd-host=");
++            while (h < e) {
++                for (sep = h; sep < e; ++sep) {
++                    if (*sep == '\\' && (sep[1] == ',' ||
++                                         sep[1] == ';' ||
++                                         sep[1] == ' ')) {
++                        *sep = '\0';
++                        sep += 2;
++                        break;
++                    }
++                }
++
++                if (virStorageSourceRBDAddHost(src, h) < 0)
++                    return -1;
++
++                h = sep;
++            }
++        }
++
++        p = next;
++    }
++
++    if (!src->path) {
++        return -1;
++    }
++
++    return 0;
++}
++
++
+ static int
+ virStorageSourceParseNBDColonString(const char *nbdstr,
+                                     virStorageSource *src)
+@@ -379,6 +448,11 @@ virStorageSourceParseBackingColon(virStorageSource *src,
+             return -1;
+         break;
+ 
++    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
++        if (virStorageSourceParseVitastorColonString(path, src) < 0)
++            return -1;
++        break;
++
+     case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
+     case VIR_STORAGE_NET_PROTOCOL_LAST:
+     case VIR_STORAGE_NET_PROTOCOL_NONE:
+@@ -953,6 +1027,54 @@ virStorageSourceParseBackingJSONRBD(virStorageSource *src,
+     return 0;
+ }
+ 
++static int
++virStorageSourceParseBackingJSONVitastor(virStorageSource *src,
++                                         virJSONValue *json,
++                                         const char *jsonstr G_GNUC_UNUSED,
++                                         int opaque G_GNUC_UNUSED)
++{
++    const char *filename;
++    const char *image = virJSONValueObjectGetString(json, "image");
++    const char *conf = virJSONValueObjectGetString(json, "config-path");
++    const char *etcd_prefix = virJSONValueObjectGetString(json, "etcd-prefix");
++    virJSONValue *servers = virJSONValueObjectGetArray(json, "server");
++    size_t nservers;
++    size_t i;
++
++    src->type = VIR_STORAGE_TYPE_NETWORK;
++    src->protocol = VIR_STORAGE_NET_PROTOCOL_VITASTOR;
++
++    /* legacy syntax passed via 'filename' option */
++    if ((filename = virJSONValueObjectGetString(json, "filename")))
++        return virStorageSourceParseVitastorColonString(filename, src);
++
++    if (!image) {
++        virReportError(VIR_ERR_INVALID_ARG, "%s",
++                       _("missing image name in Vitastor backing volume "
++                         "JSON specification"));
++        return -1;
++    }
++
++    src->path = g_strdup(image);
++    src->configFile = g_strdup(conf);
++    src->query = g_strdup(etcd_prefix);
++
++    if (servers) {
++        nservers = virJSONValueArraySize(servers);
++
++        src->hosts = g_new0(virStorageNetHostDef, nservers);
++        src->nhosts = nservers;
++
++        for (i = 0; i < nservers; i++) {
++            if (virStorageSourceParseBackingJSONInetSocketAddress(src->hosts + i,
++                                                                  virJSONValueArrayGet(servers, i)) < 0)
++                return -1;
++        }
++    }
++
++    return 0;
++}
++
+ static int
+ virStorageSourceParseBackingJSONRaw(virStorageSource *src,
+                                     virJSONValue *json,
+@@ -1130,6 +1252,7 @@ static const struct virStorageSourceJSONDriverParser jsonParsers[] = {
+     {"sheepdog", false, virStorageSourceParseBackingJSONSheepdog, 0},
+     {"ssh", false, virStorageSourceParseBackingJSONSSH, 0},
+     {"rbd", false, virStorageSourceParseBackingJSONRBD, 0},
++    {"vitastor", false, virStorageSourceParseBackingJSONVitastor, 0},
+     {"raw", true, virStorageSourceParseBackingJSONRaw, 0},
+     {"nfs", false, virStorageSourceParseBackingJSONNFS, 0},
+     {"vxhs", false, virStorageSourceParseBackingJSONVxHS, 0},
+diff --git a/src/test/test_driver.c b/src/test/test_driver.c
+index 1165689de7..bba846351c 100644
+--- a/src/test/test_driver.c
++++ b/src/test/test_driver.c
+@@ -7345,6 +7345,7 @@ testStorageVolumeTypeForPool(int pooltype)
+     case VIR_STORAGE_POOL_ISCSI_DIRECT:
+     case VIR_STORAGE_POOL_GLUSTER:
+     case VIR_STORAGE_POOL_RBD:
++    case VIR_STORAGE_POOL_VITASTOR:
+         return VIR_STORAGE_VOL_NETWORK;
+     case VIR_STORAGE_POOL_LOGICAL:
+     case VIR_STORAGE_POOL_DISK:
+diff --git a/tests/storagepoolcapsschemadata/poolcaps-fs.xml b/tests/storagepoolcapsschemadata/poolcaps-fs.xml
+index eee75af746..8bd0a57bdd 100644
+--- a/tests/storagepoolcapsschemadata/poolcaps-fs.xml
++++ b/tests/storagepoolcapsschemadata/poolcaps-fs.xml
+@@ -204,4 +204,11 @@
+       </enum>
+     </volOptions>
+   </pool>
++  <pool type='vitastor' supported='no'>
++    <volOptions>
++      <defaultFormat type='raw'/>
++      <enum name='targetFormatType'>
++      </enum>
++    </volOptions>
++  </pool>
+ </storagepoolCapabilities>
+diff --git a/tests/storagepoolcapsschemadata/poolcaps-full.xml b/tests/storagepoolcapsschemadata/poolcaps-full.xml
+index 805950a937..852df0de16 100644
+--- a/tests/storagepoolcapsschemadata/poolcaps-full.xml
++++ b/tests/storagepoolcapsschemadata/poolcaps-full.xml
+@@ -204,4 +204,11 @@
+       </enum>
+     </volOptions>
+   </pool>
++  <pool type='vitastor' supported='yes'>
++    <volOptions>
++      <defaultFormat type='raw'/>
++      <enum name='targetFormatType'>
++      </enum>
++    </volOptions>
++  </pool>
+ </storagepoolCapabilities>
+diff --git a/tests/storagepoolxml2argvtest.c b/tests/storagepoolxml2argvtest.c
+index d5c2531ab8..b19308ac38 100644
+--- a/tests/storagepoolxml2argvtest.c
++++ b/tests/storagepoolxml2argvtest.c
+@@ -57,6 +57,7 @@ testCompareXMLToArgvFiles(bool shouldFail,
+     case VIR_STORAGE_POOL_GLUSTER:
+     case VIR_STORAGE_POOL_ZFS:
+     case VIR_STORAGE_POOL_VSTORAGE:
++    case VIR_STORAGE_POOL_VITASTOR:
+     case VIR_STORAGE_POOL_LAST:
+     default:
+         VIR_TEST_DEBUG("pool type '%s' has no xml2argv test", defTypeStr);
+diff --git a/tools/virsh-pool.c b/tools/virsh-pool.c
+index 2010ef1356..072e2ff9e8 100644
+--- a/tools/virsh-pool.c
++++ b/tools/virsh-pool.c
+@@ -1187,6 +1187,9 @@ cmdPoolList(vshControl *ctl, const vshCmd *cmd G_GNUC_UNUSED)
+             case VIR_STORAGE_POOL_VSTORAGE:
+                 flags |= VIR_CONNECT_LIST_STORAGE_POOLS_VSTORAGE;
+                 break;
++            case VIR_STORAGE_POOL_VITASTOR:
++                flags |= VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR;
++                break;
+             case VIR_STORAGE_POOL_LAST:
+                 break;
+             }

patches/qemu-10.1-vitastor.patch

@@ -1,29 +1,172 @@
-diff --git a/src/client/qemu_driver.c b/src/client/qemu_driver.c
-index d8356dab..5f4cd50d 100644
---- a/src/client/qemu_driver.c
-+++ b/src/client/qemu_driver.c
-@@ -974,14 +974,21 @@ static void vitastor_co_read_bitmap_cb(void *opaque, long retval, uint8_t *bitma
- #endif
- }
+diff --git a/block/meson.build b/block/meson.build
+index 34b1b2a306..24ca0f1e52 100644
+--- a/block/meson.build
++++ b/block/meson.build
+@@ -114,6 +114,7 @@ foreach m : [
+   [libnfs, 'nfs', files('nfs.c')],
+   [libssh, 'ssh', files('ssh.c')],
+   [rbd, 'rbd', files('rbd.c')],
++  [vitastor, 'vitastor', files('vitastor.c')],
+ ]
+   if m[0].found()
+     module_ss = ss.source_set()
+diff --git a/meson.build b/meson.build
+index 50c774a195..e5c7a3a4b1 100644
+--- a/meson.build
++++ b/meson.build
+@@ -1652,6 +1652,26 @@ if not get_option('rbd').auto() or have_block
+   endif
+ endif
  
--static int coroutine_fn vitastor_co_block_status(
--    BlockDriverState *bs, bool want_zero, int64_t offset, int64_t bytes,
--    int64_t *pnum, int64_t *map, BlockDriverState **file)
-+static int coroutine_fn vitastor_co_block_status(BlockDriverState *bs,
-+#if QEMU_VERSION_MAJOR > 10 || QEMU_VERSION_MAJOR == 10 && QEMU_VERSION_MINOR >= 1
-+    unsigned int mode,
-+#else
-+    bool want_zero,
-+#endif
-+    int64_t offset, int64_t bytes, int64_t *pnum, int64_t *map, BlockDriverState **file)
- {
-     // Allocated => return BDRV_BLOCK_DATA|BDRV_BLOCK_OFFSET_VALID
-     // Not allocated => return 0
-     // Error => return -errno
-     // Set pnum to length of the extent, `*map` = `offset`, `*file` = `bs`
-+#if QEMU_VERSION_MAJOR > 10 || QEMU_VERSION_MAJOR == 10 && QEMU_VERSION_MINOR >= 1
-+    int want_zero = (mode == BDRV_WANT_PRECISE);
-+#endif
-     VitastorRPC task;
-     VitastorClient *client = bs->opaque;
-     uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode;
++vitastor = not_found
++if not get_option('vitastor').auto() or have_block
++  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
++    required: get_option('vitastor'))
++  if libvitastor_client.found()
++    if cc.links('''
++      #include <vitastor_c.h>
++      int main(void) {
++        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
++        return 0;
++      }''', dependencies: libvitastor_client)
++      vitastor = declare_dependency(dependencies: libvitastor_client)
++    elif get_option('vitastor').enabled()
++      error('could not link libvitastor_client')
++    else
++      warning('could not link libvitastor_client, disabling')
++    endif
++  endif
++endif
++
+ glusterfs = not_found
+ glusterfs_ftruncate_has_stat = false
+ glusterfs_iocb_has_stat = false
+@@ -2547,6 +2567,7 @@ endif
+ config_host_data.set('CONFIG_OPENGL', opengl.found())
+ config_host_data.set('CONFIG_PLUGIN', get_option('plugins'))
+ config_host_data.set('CONFIG_RBD', rbd.found())
++config_host_data.set('CONFIG_VITASTOR', vitastor.found())
+ config_host_data.set('CONFIG_RDMA', rdma.found())
+ config_host_data.set('CONFIG_RELOCATABLE', get_option('relocatable'))
+ config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack'))
+@@ -4972,6 +4993,7 @@ summary_info += {'fdt support':       fdt_opt == 'internal' ? 'internal' : fdt}
+ summary_info += {'libcap-ng support': libcap_ng}
+ summary_info += {'bpf support':       libbpf}
+ summary_info += {'rbd support':       rbd}
++summary_info += {'vitastor support':  vitastor}
+ summary_info += {'smartcard support': cacard}
+ summary_info += {'U2F support':       u2f}
+ summary_info += {'libusb':            libusb}
+diff --git a/meson_options.txt b/meson_options.txt
+index fff1521e58..f0844c0e00 100644
+--- a/meson_options.txt
++++ b/meson_options.txt
+@@ -202,6 +202,8 @@ option('pvg', type: 'feature', value: 'auto',
+        description: 'macOS paravirtualized graphics support')
+ option('rbd', type : 'feature', value : 'auto',
+        description: 'Ceph block device driver')
++option('vitastor', type : 'feature', value : 'auto',
++       description: 'Vitastor block device driver')
+ option('opengl', type : 'feature', value : 'auto',
+        description: 'OpenGL support')
+ option('rdma', type : 'feature', value : 'auto',
+diff --git a/qapi/block-core.json b/qapi/block-core.json
+index dc6eb4ae23..d043f4340e 100644
+--- a/qapi/block-core.json
++++ b/qapi/block-core.json
+@@ -3280,7 +3280,7 @@
+             'parallels', 'preallocate', 'qcow', 'qcow2', 'qed', 'quorum',
+             'raw', 'rbd',
+             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
+-            'ssh', 'throttle', 'vdi', 'vhdx',
++            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
+             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
+@@ -4363,6 +4363,28 @@
+             '*key-secret': 'str',
+             '*server': ['InetSocketAddressBase'] } }
+ 
++##
++# @BlockdevOptionsVitastor:
++#
++# Driver specific block device options for vitastor
++#
++# @image:       Image name
++# @inode:       Inode number
++# @pool:        Pool ID
++# @size:        Desired image size in bytes
++# @config-path: Path to Vitastor configuration
++# @etcd-host:   etcd connection address(es)
++# @etcd-prefix: etcd key/value prefix
++##
++{ 'struct': 'BlockdevOptionsVitastor',
++  'data': { '*inode': 'uint64',
++            '*pool': 'uint64',
++            '*size': 'uint64',
++            '*image': 'str',
++            '*config-path': 'str',
++            '*etcd-host': 'str',
++            '*etcd-prefix': 'str' } }
++
+ ##
+ # @ReplicationMode:
+ #
+@@ -4831,6 +4853,7 @@
+       'throttle':   'BlockdevOptionsThrottle',
+       'vdi':        'BlockdevOptionsGenericFormat',
+       'vhdx':       'BlockdevOptionsGenericFormat',
++      'vitastor':   'BlockdevOptionsVitastor',
+       'virtio-blk-vfio-pci':
+                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
+                       'if': 'CONFIG_BLKIO' },
+@@ -5304,6 +5327,20 @@
+             '*cluster-size' :   'size',
+             '*encrypt' :        'RbdEncryptionCreateOptions' } }
+ 
++##
++# @BlockdevCreateOptionsVitastor:
++#
++# Driver specific image creation options for Vitastor.
++#
++# @location: Where to store the new image file.  This location cannot
++#     point to a snapshot.
++#
++# @size: Size of the virtual disk in bytes
++##
++{ 'struct': 'BlockdevCreateOptionsVitastor',
++  'data': { 'location':         'BlockdevOptionsVitastor',
++            'size':             'size' } }
++
+ ##
+ # @BlockdevVmdkSubformat:
+ #
+@@ -5526,6 +5563,7 @@
+       'ssh':            'BlockdevCreateOptionsSsh',
+       'vdi':            'BlockdevCreateOptionsVdi',
+       'vhdx':           'BlockdevCreateOptionsVhdx',
++      'vitastor':       'BlockdevCreateOptionsVitastor',
+       'vmdk':           'BlockdevCreateOptionsVmdk',
+       'vpc':            'BlockdevCreateOptionsVpc'
+   } }
+diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh
+index 0ebe6bc52a..2c37ad3892 100644
+--- a/scripts/meson-buildoptions.sh
++++ b/scripts/meson-buildoptions.sh
+@@ -175,6 +175,7 @@ meson_options_help() {
+   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
+   printf "%s\n" '  qpl             Query Processing Library support'
+   printf "%s\n" '  rbd             Ceph block device driver'
++  printf "%s\n" '  vitastor        Vitastor block device driver'
+   printf "%s\n" '  rdma            Enable RDMA-based migration'
+   printf "%s\n" '  replication     replication support'
+   printf "%s\n" '  rust            Rust support'
+@@ -459,6 +460,8 @@ _meson_option_parse() {
+     --disable-qpl) printf "%s" -Dqpl=disabled ;;
+     --enable-rbd) printf "%s" -Drbd=enabled ;;
+     --disable-rbd) printf "%s" -Drbd=disabled ;;
++    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
++    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
+     --enable-rdma) printf "%s" -Drdma=enabled ;;
+     --disable-rdma) printf "%s" -Drdma=disabled ;;
+     --enable-relocatable) printf "%s" -Drelocatable=true ;;

patches/qemu-10.2-vitastor.patch

@@ -0,0 +1,172 @@
+diff --git a/block/meson.build b/block/meson.build
+index 34b1b2a306..24ca0f1e52 100644
+--- a/block/meson.build
++++ b/block/meson.build
+@@ -114,6 +114,7 @@ foreach m : [
+   [libnfs, 'nfs', files('nfs.c')],
+   [libssh, 'ssh', files('ssh.c')],
+   [rbd, 'rbd', files('rbd.c')],
++  [vitastor, 'vitastor', files('vitastor.c')],
+ ]
+   if m[0].found()
+     module_ss = ss.source_set()
+diff --git a/meson.build b/meson.build
+index d9293294d8..776a5becc6 100644
+--- a/meson.build
++++ b/meson.build
+@@ -1665,6 +1665,26 @@ if not get_option('rbd').auto() or have_block
+   endif
+ endif
+ 
++vitastor = not_found
++if not get_option('vitastor').auto() or have_block
++  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
++    required: get_option('vitastor'))
++  if libvitastor_client.found()
++    if cc.links('''
++      #include <vitastor_c.h>
++      int main(void) {
++        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
++        return 0;
++      }''', dependencies: libvitastor_client)
++      vitastor = declare_dependency(dependencies: libvitastor_client)
++    elif get_option('vitastor').enabled()
++      error('could not link libvitastor_client')
++    else
++      warning('could not link libvitastor_client, disabling')
++    endif
++  endif
++endif
++
+ glusterfs = not_found
+ glusterfs_ftruncate_has_stat = false
+ glusterfs_iocb_has_stat = false
+@@ -2509,6 +2529,7 @@ endif
+ config_host_data.set('CONFIG_OPENGL', opengl.found())
+ config_host_data.set('CONFIG_PLUGIN', get_option('plugins'))
+ config_host_data.set('CONFIG_RBD', rbd.found())
++config_host_data.set('CONFIG_VITASTOR', vitastor.found())
+ config_host_data.set('CONFIG_RDMA', rdma.found())
+ config_host_data.set('CONFIG_RELOCATABLE', get_option('relocatable'))
+ config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack'))
+@@ -4948,6 +4969,7 @@ summary_info += {'fdt support':       fdt_opt == 'internal' ? 'internal' : fdt}
+ summary_info += {'libcap-ng support': libcap_ng}
+ summary_info += {'bpf support':       libbpf}
+ summary_info += {'rbd support':       rbd}
++summary_info += {'vitastor support':  vitastor}
+ summary_info += {'smartcard support': cacard}
+ summary_info += {'U2F support':       u2f}
+ summary_info += {'libusb':            libusb}
+diff --git a/meson_options.txt b/meson_options.txt
+index 2836156257..148086cc6f 100644
+--- a/meson_options.txt
++++ b/meson_options.txt
+@@ -206,6 +206,8 @@ option('pvg', type: 'feature', value: 'auto',
+        description: 'macOS paravirtualized graphics support')
+ option('rbd', type : 'feature', value : 'auto',
+        description: 'Ceph block device driver')
++option('vitastor', type : 'feature', value : 'auto',
++       description: 'Vitastor block device driver')
+ option('opengl', type : 'feature', value : 'auto',
+        description: 'OpenGL support')
+ option('rdma', type : 'feature', value : 'auto',
+diff --git a/qapi/block-core.json b/qapi/block-core.json
+index b82af74256..f25a6f5ce8 100644
+--- a/qapi/block-core.json
++++ b/qapi/block-core.json
+@@ -3351,7 +3351,7 @@
+             'parallels', 'preallocate', 'qcow', 'qcow2', 'qed', 'quorum',
+             'raw', 'rbd',
+             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
+-            'ssh', 'throttle', 'vdi', 'vhdx',
++            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
+             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
+@@ -4434,6 +4434,28 @@
+             '*key-secret': 'str',
+             '*server': ['InetSocketAddressBase'] } }
+ 
++##
++# @BlockdevOptionsVitastor:
++#
++# Driver specific block device options for vitastor
++#
++# @image:       Image name
++# @inode:       Inode number
++# @pool:        Pool ID
++# @size:        Desired image size in bytes
++# @config-path: Path to Vitastor configuration
++# @etcd-host:   etcd connection address(es)
++# @etcd-prefix: etcd key/value prefix
++##
++{ 'struct': 'BlockdevOptionsVitastor',
++  'data': { '*inode': 'uint64',
++            '*pool': 'uint64',
++            '*size': 'uint64',
++            '*image': 'str',
++            '*config-path': 'str',
++            '*etcd-host': 'str',
++            '*etcd-prefix': 'str' } }
++
+ ##
+ # @ReplicationMode:
+ #
+@@ -4902,6 +4924,7 @@
+       'throttle':   'BlockdevOptionsThrottle',
+       'vdi':        'BlockdevOptionsGenericFormat',
+       'vhdx':       'BlockdevOptionsGenericFormat',
++      'vitastor':   'BlockdevOptionsVitastor',
+       'virtio-blk-vfio-pci':
+                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
+                       'if': 'CONFIG_BLKIO' },
+@@ -5376,6 +5399,20 @@
+             '*cluster-size' :   'size',
+             '*encrypt' :        'RbdEncryptionCreateOptions' } }
+ 
++##
++# @BlockdevCreateOptionsVitastor:
++#
++# Driver specific image creation options for Vitastor.
++#
++# @location: Where to store the new image file.  This location cannot
++#     point to a snapshot.
++#
++# @size: Size of the virtual disk in bytes
++##
++{ 'struct': 'BlockdevCreateOptionsVitastor',
++  'data': { 'location':         'BlockdevOptionsVitastor',
++            'size':             'size' } }
++
+ ##
+ # @BlockdevVmdkSubformat:
+ #
+@@ -5598,6 +5635,7 @@
+       'ssh':            'BlockdevCreateOptionsSsh',
+       'vdi':            'BlockdevCreateOptionsVdi',
+       'vhdx':           'BlockdevCreateOptionsVhdx',
++      'vitastor':       'BlockdevCreateOptionsVitastor',
+       'vmdk':           'BlockdevCreateOptionsVmdk',
+       'vpc':            'BlockdevCreateOptionsVpc'
+   } }
+diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh
+index 3d0d132344..65ee8c855e 100644
+--- a/scripts/meson-buildoptions.sh
++++ b/scripts/meson-buildoptions.sh
+@@ -177,6 +177,7 @@ meson_options_help() {
+   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
+   printf "%s\n" '  qpl             Query Processing Library support'
+   printf "%s\n" '  rbd             Ceph block device driver'
++  printf "%s\n" '  vitastor        Vitastor block device driver'
+   printf "%s\n" '  rdma            Enable RDMA-based migration'
+   printf "%s\n" '  replication     replication support'
+   printf "%s\n" '  rust            Rust support'
+@@ -464,6 +465,8 @@ _meson_option_parse() {
+     --disable-qpl) printf "%s" -Dqpl=disabled ;;
+     --enable-rbd) printf "%s" -Drbd=enabled ;;
+     --disable-rbd) printf "%s" -Drbd=disabled ;;
++    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
++    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
+     --enable-rdma) printf "%s" -Drdma=enabled ;;
+     --disable-rdma) printf "%s" -Drdma=disabled ;;
+     --enable-relocatable) printf "%s" -Drelocatable=true ;;

rpm/build-tarball.sh

@@ -21,7 +21,7 @@ rpmbuild -bp fio.spec
 cd $VITASTOR
 VER=$(grep ^Version: rpm/vitastor-$REL.spec | awk '{print $2}')
 rm -rf fio
-ln -s ~/rpmbuild/BUILD/fio*/ fio
+ln -s $(ls -d ~/rpmbuild/BUILD/fio*/ | grep -v SPECPARTS) fio
 sh copy-fio-includes.sh
 rm fio
 mv fio-copy fio

rpm/vitastor-el10.Dockerfile

@@ -0,0 +1,17 @@
+# Build packages for AlmaLinux 10 inside a container
+# cd ..
+# docker pull --platform=linux/amd64/v2 quay.io/almalinuxorg/almalinux:10
+# docker build -t vitastor-buildenv:el10 -f rpm/vitastor-el10.Dockerfile .
+# docker run -i --rm -v ./:/root/vitastor vitastor-buildenv:el10 /root/vitastor/rpm/vitastor-build.sh
+
+FROM quay.io/almalinuxorg/almalinux:10
+
+WORKDIR /root
+
+RUN sed -i 's/enabled=0/enabled=1/' /etc/yum.repos.d/*.repo
+RUN dnf -y install epel-release dnf-plugins-core
+RUN dnf -y install https://vitastor.io/rpms/centos/10/vitastor-release-1.0-1.el10.noarch.rpm
+RUN dnf -y install gcc-c++ gperftools-devel fio nodejs rpm-build jerasure-devel isa-l-devel gf-complete-devel rdma-core-devel cmake libnl3-devel
+RUN dnf download --source fio
+RUN rpm --nomd5 -i fio*.src.rpm
+RUN cd ~/rpmbuild/SPECS && dnf builddep -y --spec fio.spec

rpm/vitastor-el10.spec

@@ -0,0 +1,198 @@
+Name:           vitastor
+Version:        3.0.10
+Release:        1%{?dist}
+Summary:        Vitastor, a fast software-defined clustered block storage
+
+License:        Vitastor Network Public License 1.1
+URL:            https://vitastor.io/
+Source0:        vitastor-3.0.10.el10.tar.gz
+
+BuildRequires:  gperftools-devel
+BuildRequires:  gcc-c++
+BuildRequires:  nodejs >= 10
+BuildRequires:  jerasure-devel
+BuildRequires:  isa-l-devel
+BuildRequires:  gf-complete-devel
+BuildRequires:  rdma-core-devel
+BuildRequires:  cmake
+BuildRequires:  libnl3-devel
+Requires:       vitastor-osd = %{version}-%{release}
+Requires:       vitastor-mon = %{version}-%{release}
+Requires:       vitastor-client = %{version}-%{release}
+Requires:       vitastor-client-devel = %{version}-%{release}
+Requires:       vitastor-fio = %{version}-%{release}
+
+%description
+Vitastor is a small, simple and fast clustered block storage (storage for VM drives),
+architecturally similar to Ceph which means strong consistency, primary-replication,
+symmetric clustering and automatic data distribution over any number of drives of any
+size with configurable redundancy (replication or erasure codes/XOR).
+
+
+%package -n vitastor-osd
+Summary:        Vitastor - OSD
+Requires:       vitastor-client = %{version}-%{release}
+Requires:       util-linux
+Requires:       parted
+
+
+%description -n vitastor-osd
+Vitastor object storage daemon, i.e. server program that stores data.
+
+
+%package -n vitastor-mon
+Summary:        Vitastor - monitor
+Requires:       nodejs >= 10
+Requires:       lpsolve
+
+
+%description -n vitastor-mon
+Vitastor monitor, i.e. server program responsible for watching cluster state and
+scheduling cluster-level operations.
+
+
+%package -n vitastor-client
+Summary:        Vitastor - client
+
+
+%description -n vitastor-client
+Vitastor client library and command-line interface.
+
+
+%package -n vitastor-client-devel
+Summary:        Vitastor - development files
+Group:          Development/Libraries
+Requires:       vitastor-client = %{version}-%{release}
+
+
+%description -n vitastor-client-devel
+Vitastor library headers for development.
+
+
+%package -n vitastor-fio
+Summary:        Vitastor - fio drivers
+Group:          Development/Libraries
+Requires:       vitastor-client = %{version}-%{release}
+Requires:       fio = 3.36-5.el10
+
+
+%description -n vitastor-fio
+Vitastor fio drivers for benchmarking.
+
+
+%package -n vitastor-opennebula
+Summary:        Vitastor for OpenNebula
+Group:          Development/Libraries
+Requires:       vitastor-client
+Requires:       jq
+Requires:       python3-lxml
+Requires:       patch
+Requires:       qemu-kvm-block-vitastor
+
+
+%description -n vitastor-opennebula
+Vitastor storage plugin for OpenNebula.
+
+
+%prep
+%setup -q
+
+
+%build
+%cmake
+%cmake_build
+
+
+%install
+rm -rf $RPM_BUILD_ROOT
+%cmake_install
+cd mon
+npm install --production
+cd ..
+mkdir -p %buildroot/usr/lib/vitastor
+cp -r mon %buildroot/usr/lib/vitastor
+mv %buildroot/usr/lib/vitastor/mon/scripts/make-etcd %buildroot/usr/lib/vitastor/mon/
+mkdir -p %buildroot/lib/systemd/system
+cp mon/scripts/vitastor.target mon/scripts/vitastor-mon.service mon/scripts/vitastor-osd@.service %buildroot/lib/systemd/system
+mkdir -p %buildroot/lib/udev/rules.d
+cp mon/scripts/90-vitastor.rules %buildroot/lib/udev/rules.d
+mkdir -p %buildroot/var/lib/one
+cp -r opennebula/remotes %buildroot/var/lib/one
+cp opennebula/install.sh %buildroot/var/lib/one/remotes/datastore/vitastor/
+mkdir -p %buildroot/etc/
+cp -r opennebula/sudoers.d %buildroot/etc/
+
+
+%files
+%doc GPL-2.0.txt VNPL-1.1.txt README.md README-ru.md
+
+
+%files -n vitastor-osd
+%_bindir/vitastor-osd
+%_bindir/vitastor-disk
+%_bindir/vitastor-dump-journal
+/lib/systemd/system/vitastor-osd@.service
+/lib/systemd/system/vitastor.target
+/lib/udev/rules.d/90-vitastor.rules
+
+
+%pre -n vitastor-osd
+groupadd -r -f vitastor 2>/dev/null ||:
+useradd -r -g vitastor -s /sbin/nologin -c "Vitastor daemons" -M -d /nonexistent vitastor 2>/dev/null ||:
+install -o vitastor -g vitastor -d /var/log/vitastor
+mkdir -p /etc/vitastor
+
+
+%files -n vitastor-mon
+/usr/lib/vitastor/mon
+/lib/systemd/system/vitastor-mon.service
+
+
+%pre -n vitastor-mon
+groupadd -r -f vitastor 2>/dev/null ||:
+useradd -r -g vitastor -s /sbin/nologin -c "Vitastor daemons" -M -d /nonexistent vitastor 2>/dev/null ||:
+mkdir -p /etc/vitastor
+mkdir -p /var/lib/vitastor
+chown vitastor:vitastor /var/lib/vitastor
+
+
+%files -n vitastor-client
+%_bindir/vitastor-nbd
+%_bindir/vitastor-ublk
+%_bindir/vitastor-nfs
+%_bindir/vitastor-cli
+%_bindir/vitastor-rm
+%_bindir/vitastor-kv
+%_bindir/vitastor-kv-stress
+%_bindir/vita
+%_libdir/libvitastor_client.so*
+%_libdir/libvitastor_kv.so*
+
+
+%files -n vitastor-client-devel
+%_includedir/vitastor_c.h
+%_includedir/vitastor_kv.h
+%_libdir/pkgconfig
+
+
+%files -n vitastor-fio
+%_libdir/libfio_vitastor.so
+%_libdir/libfio_vitastor_blk.so
+%_libdir/libfio_vitastor_sec.so
+
+
+%files -n vitastor-opennebula
+/var/lib/one
+/etc/sudoers.d/opennebula-vitastor
+
+
+%triggerin -n vitastor-opennebula -- opennebula
+[ $2 = 0 ] || exit 0
+/var/lib/one/remotes/datastore/vitastor/install.sh
+
+
+# Turn off the brp-python-bytecompile script
+%global __os_install_post %(echo '%{__os_install_post}' | sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g')
+
+
+%changelog

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.2
+Version:        3.0.10
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.2.el7.tar.gz
+Source0:        vitastor-3.0.10.el7.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  devtoolset-9-gcc-c++

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.2
+Version:        3.0.10
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.2.el8.tar.gz
+Source0:        vitastor-3.0.10.el8.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-toolset-9-gcc-c++

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.2
+Version:        3.0.10
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.2.el9.tar.gz
+Source0:        vitastor-3.0.10.el9.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++

src/CMakeLists.txt

@@ -1,9 +1,8 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 2.8...3.30)
 
 project(vitastor)
 
 include(GNUInstallDirs)
-include(CTest)
 include(CheckIncludeFile)
 
 find_package(PkgConfig)
@@ -21,7 +20,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 endif()
 set(ENABLE_COVERAGE false CACHE BOOL "Enable code coverage")
 
-add_definitions(-DVITASTOR_VERSION="3.0.2")
+add_definitions(-DVITASTOR_VERSION="3.0.10")
 add_definitions(-D_GNU_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -fvisibility=hidden -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/blockstore/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 2.8...3.30)
 
 project(vitastor)
 

src/blockstore/blockstore.h

@@ -187,7 +187,6 @@ public:
     // MUST be called only when nobody makes any modifications to the DB for this pool
     virtual void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit) = 0;
     virtual bool reshard_continue(void *reshard_state, uint64_t chunk_limit) = 0;
-    virtual void reshard_abort(void *reshard_state) = 0;
 
     // Event loop
     virtual void loop() = 0;
@@ -229,4 +228,9 @@ public:
     virtual uint64_t get_journal_size() = 0;
 
     virtual uint32_t get_bitmap_granularity() = 0;
+
+    virtual uint64_t get_live_entries() = 0;
+    virtual uint64_t get_live_memory() = 0;
+    virtual uint64_t get_garbage_entries() = 0;
+    virtual uint64_t get_garbage_memory() = 0;
 };

src/blockstore/blockstore_disk.cpp

@@ -94,6 +94,8 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     csum_block_size = parse_size(config["csum_block_size"]);
     discard_on_start = config.find("discard_on_start") != config.end() &&
         (config["discard_on_start"] == "true" || config["discard_on_start"] == "1" || config["discard_on_start"] == "yes");
+    gc_on_start = config.find("gc_on_start") == config.end() ||
+        (config["gc_on_start"] == "true" || config["gc_on_start"] == "1" || config["gc_on_start"] == "yes");
     min_discard_size = parse_size(config["min_discard_size"]);
     if (!min_discard_size)
         min_discard_size = 1024*1024;
@@ -173,9 +175,7 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     }
     if (data_block_size / bitmap_granularity < 8)
     {
-        fprintf(stderr, "Warning: block_size (%u) / bitmap_granularity (%u) = %u bits. "
-            "Consider using larger block_size or bitmap_granularity for better performance.\n",
-            data_block_size, bitmap_granularity, data_block_size / bitmap_granularity);
+        throw std::runtime_error("Data block size must be at least bitmap_granularity*8");
     }
     if (!data_csum_type)
     {

src/blockstore/blockstore_disk.h

@@ -26,14 +26,14 @@ class allocator_t;
 struct blockstore_disk_t
 {
     std::string data_device, meta_device, journal_device;
-    uint32_t data_block_size;
+    uint64_t data_block_size;
     uint64_t cfg_journal_size, cfg_data_size;
     // Required write alignment and journal/metadata/data areas' location alignment
     uint32_t disk_alignment = 4096;
     // Journal block size - minimum_io_size of the journal device is the best choice
-    uint32_t journal_block_size = 4096;
+    uint64_t journal_block_size = 4096;
     // Metadata block size - minimum_io_size of the metadata device is the best choice
-    uint32_t meta_block_size = 4096;
+    uint64_t meta_block_size = 4096;
     // Atomic write size of the data block device
     uint32_t atomic_write_size = 4096;
     // Whether we should set RWF_ATOMIC on atomic writes
@@ -57,6 +57,8 @@ struct blockstore_disk_t
     bool inmemory_journal = true;
     // Data discard granularity and minimum size (for the sake of performance)
     bool discard_on_start = false;
+    // GC on start (new store)
+    bool gc_on_start = true;
     uint64_t min_discard_size = 1024*1024;
     uint64_t discard_granularity = 0;
 

src/blockstore/blockstore_heap.cpp

@@ -29,6 +29,8 @@
 #define IMAP_MALLOC_LOW_BITS ((size_t)0x0F)
 #define IMAP_MAX_LOW 16
 
+#define list_item_overhead(a) (((a) + sizeof(heap_list_item_t) - sizeof(heap_entry_t) + sizeof(void*) + 15) & ~15)
+
 void inode_map_put(void* & inode_idx, heap_list_item_t* li);
 void inode_map_get(void *inode_idx, heap_inode_map_t::iterator & li_it, heap_list_item_t* & li, uint64_t stripe);
 void inode_map_free(void* inode_idx);
@@ -244,7 +246,7 @@ blockstore_heap_t::blockstore_heap_t(blockstore_disk_t *dsk, uint8_t *buffer_are
     assert(dsk->meta_block_size < 32768);
     assert(dsk->meta_area_size > 0);
     assert(dsk->journal_len > 0);
-    meta_alloc = new multilist_index_t(meta_block_count, META_ALLOC_LEVELS+1, 0);
+    meta_alloc = new multilist_index_t(meta_block_count, META_ALLOC_LEVELS+1, 2);
     block_info.resize(meta_block_count);
     assert(dsk->block_count <= 0xFFFF0000);
     data_alloc = new allocator_t(dsk->block_count);
@@ -301,12 +303,13 @@ int blockstore_heap_t::read_blocks(uint64_t disk_offset, uint64_t disk_size, uin
             heap_entry_t *wr = (heap_entry_t*)data;
             if (wr->size > dsk->meta_block_size-block_offset)
             {
-                fprintf(stderr, "Error: entry is too large in metadata block %u at %u (%u > max %u bytes). ",
+                fprintf(stderr, "Error: entry is too large in metadata block %u at %u (%u > max %ju bytes). ",
                     block_num, block_offset, wr->size, dsk->meta_block_size-block_offset);
 corrupted_block:
                 if (allow_corrupted)
                 {
                     fprintf(stderr, "Metadata block is corrupted, skipping\n");
+                    recheck_modified_blocks.insert(block_num);
                     break;
                 }
                 else
@@ -331,6 +334,11 @@ corrupted_block:
                     block_num, block_offset, wr->size, sizeof(heap_entry_t));
                 goto corrupted_block;
             }
+            // Garbage collection is only performed when writing new entries into the block
+            // because it needs a fake LSN and modified blocks require consecutive modified LSNs
+            // At the same time, further modifications _after_ putting new entries into the block,
+            // but _before_ writing it, may mark some entries in it as garbage. That's why garbage
+            // entries may still be present on disk.
             wr->entry_type &= ~BS_HEAP_GARBAGE;
             if ((wr->entry_type & BS_HEAP_TYPE) < BS_HEAP_BIG_WRITE ||
                 (wr->entry_type & BS_HEAP_TYPE) > BS_HEAP_ROLLBACK ||
@@ -345,6 +353,7 @@ corrupted_object:
                 if (allow_corrupted)
                 {
                     fprintf(stderr, "Entry is corrupted, skipping\n");
+                    recheck_modified_blocks.insert(block_num);
                     block_offset += wr->size;
                     continue;
                 }
@@ -360,7 +369,7 @@ corrupted_object:
             {
                 // Small writes require accessing offset & len to calculate correct length,
                 // so require at least sizeof(heap_small_write_t) for them
-                fprintf(stderr, "Error: entry %jx:%jx v%ju has invalid size in metadata block %u at %u (%u < min %zu bytes). Metadata is corrupted, aborting\n",
+                fprintf(stderr, "Error: entry %jx:%jx v%ju has invalid size in metadata block %u at %u (%u < min %zu bytes)\n",
                     wr->inode, wr->stripe, wr->version, block_num, block_offset, wr->size, sizeof(heap_small_write_t));
                 goto corrupted_object;
             }
@@ -374,8 +383,8 @@ corrupted_object:
             uint32_t expected_crc32c = wr->calc_crc32c();
             if (wr->crc32c != expected_crc32c)
             {
-                fprintf(stderr, "Error: entry %jx:%jx v%ju in metadata block %u at %u is corrupt (crc32c mismatch: expected %08x, got %08x). Metadata is corrupted, aborting\n",
-                    wr->inode, wr->stripe, wr->version,
+                fprintf(stderr, "Error: entry %jx:%jx v%ju l%ju in metadata block %u at %u is corrupt (crc32c mismatch: expected %08x, got %08x). ",
+                    wr->inode, wr->stripe, wr->version, wr->lsn,
                     block_num, block_offset, expected_crc32c, wr->crc32c);
                 goto corrupted_object;
             }
@@ -385,7 +394,7 @@ corrupted_object:
                 wr->small().offset % dsk->bitmap_granularity ||
                 wr->small().len % dsk->bitmap_granularity))
             {
-                fprintf(stderr, "Error: %s entry %jx:%jx v%ju has invalid offset/length: %u/%u. Metadata is incompatible with current parameters, aborting\n",
+                fprintf(stderr, "Error: %s entry %jx:%jx v%ju has invalid offset/length: %u/%u. Metadata is incompatible with current parameters. ",
                     wr->type() == BS_HEAP_SMALL_WRITE ? "small_write" : "intent_write",
                     wr->inode, wr->stripe, wr->version, wr->small().offset, wr->small().len);
                 goto corrupted_object;
@@ -395,7 +404,7 @@ corrupted_object:
                 wr->big_intent().offset % dsk->bitmap_granularity ||
                 wr->big_intent().len % dsk->bitmap_granularity))
             {
-                fprintf(stderr, "Error: big_intent entry %jx:%jx v%ju has invalid offset/length: %u/%u. Metadata is incompatible with current parameters, aborting\n",
+                fprintf(stderr, "Error: big_intent entry %jx:%jx v%ju has invalid offset/length: %u/%u. Metadata is incompatible with current parameters. ",
                     wr->inode, wr->stripe, wr->version, wr->big_intent().offset, wr->big_intent().len);
                 goto corrupted_object;
             }
@@ -412,7 +421,10 @@ int blockstore_heap_t::load_blocks(uint64_t disk_offset, uint64_t size, uint8_t
     entries_loaded = 0;
     return read_blocks(disk_offset, size, buf, allow_corrupted, [&](uint32_t block_num, heap_entry_t *wr_orig)
     {
-        heap_list_item_t *li = (heap_list_item_t*)malloc_or_die(wr_orig->size + sizeof(heap_list_item_t) - sizeof(heap_entry_t));
+        auto alloc_size = wr_orig->size + sizeof(heap_list_item_t) - sizeof(heap_entry_t);
+        heap_list_item_t *li = (heap_list_item_t*)malloc_or_die(alloc_size);
+        live_entries++;
+        live_memory += list_item_overhead(wr_orig->size);
         li->block_num = block_num;
         li->prev = li->next = NULL;
         memcpy(&li->entry, wr_orig, wr_orig->size);
@@ -422,7 +434,7 @@ int blockstore_heap_t::load_blocks(uint64_t disk_offset, uint64_t size, uint8_t
             next_lsn = wr->lsn;
         }
         entries_loaded++;
-        insert_list_item(li);
+        loaded_list_items.push_back(li);
         modify_alloc(block_num, [&](heap_block_info_t & inf)
         {
             if (!inf.entries.size())
@@ -458,19 +470,22 @@ bool blockstore_heap_t::validate_object(heap_entry_t *obj)
         next_wr = wr;
         if (wr->type() == BS_HEAP_ROLLBACK)
         {
-            if (commit_wr && wr->version > commit_wr->version)
-            {
-                // rollback may not come before commit with a smaller version
-                fprintf(stderr, "Error: rollback entry %jx:%jx v%ju l%ju comes before a commit entry v%ju l%ju\n",
-                    wr->inode, wr->stripe, wr->version, wr->lsn, commit_wr->version, commit_wr->lsn);
-                return false;
-            }
             rollback_wr = wr;
             continue;
         }
         if (wr->type() == BS_HEAP_COMMIT)
         {
-            commit_wr = wr;
+            if (commit_wr && wr->version > commit_wr->version)
+            {
+                // commit may not come before commit with a smaller version
+                fprintf(stderr, "Error: commit entry %jx:%jx v%ju l%ju comes before a commit entry v%ju l%ju\n",
+                    wr->inode, wr->stripe, wr->version, wr->lsn, commit_wr->version, commit_wr->lsn);
+                return false;
+            }
+            if (!commit_wr)
+            {
+                commit_wr = wr;
+            }
             continue;
         }
         if (wr->entry_type & BS_HEAP_STABLE)
@@ -523,6 +538,23 @@ bool blockstore_heap_t::validate_object(heap_entry_t *obj)
     return true;
 }
 
+void blockstore_heap_t::finish_load()
+{
+    if (loaded_list_items.size())
+    {
+        // Sort everything and load in correct order
+        std::sort(loaded_list_items.begin(), loaded_list_items.end(), [this](const heap_list_item_t* a, const heap_list_item_t* b)
+        {
+            return a->entry.lsn < b->entry.lsn;
+        });
+        for (auto & li: loaded_list_items)
+        {
+            insert_list_item(li);
+        }
+        loaded_list_items.clear();
+    }
+}
+
 void blockstore_heap_t::fill_recheck_queue()
 {
     for (auto & pgp: block_index)
@@ -577,10 +609,11 @@ int blockstore_heap_t::mark_used_blocks()
                 if (wr->entry_type == (BS_HEAP_DELETE|BS_HEAP_STABLE) && !li->prev)
                 {
                     wr->set_garbage();
+                    garbage_entries++;
+                    garbage_memory += list_item_overhead(wr->size);
                     modify_alloc(li->block_num, [&](heap_block_info_t & inf)
                     {
-                        inf.used_space -= wr->size;
-                        inf.has_garbage = true;
+                        inf.garbage_space += wr->size;
                     });
                     li = NULL;
                 }
@@ -590,10 +623,11 @@ int blockstore_heap_t::mark_used_blocks()
                     if (overwritten)
                     {
                         wr->set_garbage();
+                        garbage_entries++;
+                        garbage_memory += list_item_overhead(wr->size);
                         modify_alloc(li->block_num, [&](heap_block_info_t & inf)
                         {
-                            inf.used_space -= wr->size;
-                            inf.has_garbage = true;
+                            inf.garbage_space += wr->size;
                         });
                         continue;
                     }
@@ -632,15 +666,66 @@ int blockstore_heap_t::mark_used_blocks()
             });
         }
     }
+    if (dsk->gc_on_start)
+    {
+        recheck_full_gc();
+    }
     return res;
 }
 
+void blockstore_heap_t::recheck_full_gc()
+{
+    uint32_t block_num = 0;
+    for (auto & inf: block_info)
+    {
+        // Instantly collect all garbage on restart
+        if (inf.garbage_space > 0)
+        {
+            if (log_level > 5)
+            {
+                fprintf(stderr, "Clearing %u out of %u garbage bytes in block %u\n", inf.garbage_space, inf.used_space, block_num);
+            }
+            uint32_t collected_garbage = 0;
+            size_t i = 0, j = 0;
+            for (; i < inf.entries.size(); i++)
+            {
+                if (inf.entries[i]->entry.is_garbage())
+                {
+                    collected_garbage += inf.entries[i]->entry.size;
+                    remove_list_item(inf.entries[i]);
+                }
+                else
+                {
+                    if (j != i)
+                        inf.entries[j] = inf.entries[i];
+                    j++;
+                }
+            }
+            inf.entries.resize(j);
+            modify_alloc(block_num, [&](heap_block_info_t & inf)
+            {
+                inf.used_space -= collected_garbage;
+                inf.garbage_space -= collected_garbage;
+            });
+            recheck_modified_blocks.insert(block_num);
+        }
+        block_num++;
+    }
+}
+
 void blockstore_heap_t::recheck_buffer(heap_entry_t *cwr, uint8_t *buf)
 {
     auto free_entry = [&](heap_list_item_t *li)
     {
         uint32_t block_num = li->block_num;
         auto wr_size = li->entry.size;
+        if (li->entry.is_garbage())
+        {
+            garbage_entries--;
+            garbage_memory -= list_item_overhead(wr_size);
+        }
+        live_entries--;
+        live_memory -= list_item_overhead(wr_size);
         free(li);
         modify_alloc(block_num, [&](heap_block_info_t & inf)
         {
@@ -678,7 +763,12 @@ void blockstore_heap_t::recheck_buffer(heap_entry_t *cwr, uint8_t *buf)
             assert(li->entry.entry_type == cwr->entry_type);
             auto prev = li->prev;
             li->next = li->prev = NULL;
-            li->entry.set_garbage();
+            if (!li->entry.is_garbage())
+            {
+                garbage_entries++;
+                garbage_memory += list_item_overhead(li->entry.size);
+                li->entry.set_garbage();
+            }
             li = prev;
             rolled_back++;
         }
@@ -709,6 +799,7 @@ bool blockstore_heap_t::recheck_small_writes(std::function<void(bool is_data, ui
     }
     if (!recheck_queue_filled)
     {
+        finish_load();
         fill_recheck_queue();
         recheck_queue_filled = true;
     }
@@ -800,7 +891,7 @@ std::vector<uint32_t> blockstore_heap_t::get_recheck_modified_blocks()
     return modified;
 }
 
-int blockstore_heap_t::finish_load(bool allow_corrupted)
+int blockstore_heap_t::finish_recheck()
 {
     if (!marked_used_blocks)
     {
@@ -837,7 +928,6 @@ bool blockstore_heap_t::calc_checksums(heap_entry_t *wr, uint8_t *data, bool set
         {
             return true;
         }
-        uint32_t len = 0;
         if (wr->type() == BS_HEAP_SMALL_WRITE || wr->type() == BS_HEAP_INTENT_WRITE)
             len = wr->small().len;
         else if (wr->type() == BS_HEAP_BIG_INTENT)
@@ -854,13 +944,14 @@ bool blockstore_heap_t::calc_checksums(heap_entry_t *wr, uint8_t *data, bool set
     }
     if (wr->type() == BS_HEAP_BIG_WRITE)
     {
+        assert(offset != UINT32_MAX && len != UINT32_MAX);
         return calc_block_checksums((uint32_t*)(wr->get_checksums(this) + offset/dsk->csum_block_size * (dsk->data_csum_type & 0xFF)),
             data, wr->get_int_bitmap(this), offset, offset+len, set, NULL);
     }
     if (wr->type() == BS_HEAP_BIG_INTENT)
     {
         auto & bi = wr->big_intent();
-        return calc_block_checksums((uint32_t*)(wr->get_checksums(this) + offset/dsk->csum_block_size * (dsk->data_csum_type & 0xFF)),
+        return calc_block_checksums((uint32_t*)(wr->get_checksums(this) + bi.offset/dsk->csum_block_size * (dsk->data_csum_type & 0xFF)),
             data, wr->get_int_bitmap(this), bi.offset, bi.offset+bi.len, set, NULL);
     }
     assert(wr->type() == BS_HEAP_SMALL_WRITE || wr->type() == BS_HEAP_INTENT_WRITE);
@@ -1077,16 +1168,6 @@ bool blockstore_heap_t::reshard_check(pool_id_t pool, uint32_t pg_count, uint32_
         set_it->second.pg_stripe_size == pg_stripe_size);
 }
 
-void blockstore_heap_t::reshard_abort(void* reshard_state)
-{
-    heap_reshard_state_t *st = (heap_reshard_state_t*)reshard_state;
-    for (auto sh_it = st->old_shards.begin(); sh_it != st->old_shards.end(); sh_it++)
-    {
-        block_index[sh_it->first] = std::move(sh_it->second);
-    }
-    delete st;
-}
-
 heap_entry_t *blockstore_heap_t::lock_and_read_entry(object_id oid)
 {
     auto obj = read_entry(oid);
@@ -1099,27 +1180,6 @@ heap_entry_t *blockstore_heap_t::lock_and_read_entry(object_id oid)
     return obj;
 }
 
-heap_entry_t *blockstore_heap_t::read_locked_entry(object_id oid, uint64_t lsn)
-{
-    auto obj = read_entry(oid);
-    assert(obj);
-    for (auto wr = obj; wr; wr = prev(wr))
-    {
-        if (wr->is_overwrite())
-        {
-            if (lsn == wr->lsn)
-            {
-                return obj;
-            }
-            else
-            {
-                obj = prev(wr);
-            }
-        }
-    }
-    return NULL;
-}
-
 bool blockstore_heap_t::unlock_entry(object_id oid)
 {
     auto mvcc_it = object_mvcc.find(oid);
@@ -1156,73 +1216,9 @@ heap_entry_t *blockstore_heap_t::read_entry(object_id oid)
     return &li->entry;
 }
 
-int blockstore_heap_t::allocate_entry(uint32_t entry_size, uint32_t *block_num, bool allow_last_free)
+void blockstore_heap_t::gc_block(heap_block_info_t & inf)
 {
-    if (last_allocated_block != UINT32_MAX)
-    {
-        // First try to write into the same block as the previous time
-        auto & inf = block_info.at(last_allocated_block);
-        auto free_space = dsk->meta_block_size - inf.used_space;
-        if (inf.is_writing || free_space < entry_size ||
-            // Do not allow to make the last non-nearfull block nearfull
-            !allow_last_free && meta_nearfull_blocks >= meta_block_count-1 &&
-            free_space >= max_entry_size && free_space < max_entry_size+entry_size)
-        {
-            last_allocated_block = UINT32_MAX;
-        }
-    }
-    if (last_allocated_block == UINT32_MAX)
-    {
-        int i;
-        for (i = 0; last_allocated_block == UINT32_MAX && i < META_ALLOC_LEVELS-1; i++)
-        {
-            // First try to write into most free blocks
-            last_allocated_block = meta_alloc->find(i);
-        }
-        if (last_allocated_block != UINT32_MAX && i == META_ALLOC_LEVELS-1 && !allow_last_free && meta_nearfull_blocks >= meta_block_count-1)
-        {
-            // Do not allow to make the last non-nearfull block nearfull
-            auto & inf = block_info.at(last_allocated_block);
-            auto free_space = dsk->meta_block_size - inf.used_space;
-            if (free_space >= max_entry_size && free_space < max_entry_size+entry_size)
-            {
-                last_allocated_block = UINT32_MAX;
-            }
-        }
-        if (last_allocated_block == UINT32_MAX)
-        {
-            // Then into nearfull blocks
-            for (uint32_t b = meta_alloc->find(META_ALLOC_LEVELS-1); b != UINT32_MAX; b = meta_alloc->next(b))
-            {
-                auto & inf = block_info.at(b);
-                auto free_space = dsk->meta_block_size - inf.used_space;
-                if (free_space >= entry_size)
-                {
-                    last_allocated_block = b;
-                    break;
-                }
-            }
-        }
-        if (last_allocated_block == UINT32_MAX)
-        {
-            // Then fail :)
-            return ENOSPC;
-        }
-    }
-    if (!allow_last_free && meta_nearfull_blocks >= meta_block_count-1)
-    {
-        // Do not allow to make the last non-nearfull block nearfull
-        auto & inf = block_info.at(last_allocated_block);
-        if (dsk->meta_block_size-inf.used_space >= max_entry_size &&
-            dsk->meta_block_size-inf.used_space+entry_size < max_entry_size)
-        {
-            last_allocated_block = UINT32_MAX;
-            return ENOSPC;
-        }
-    }
-    // Write into the same block
-    auto & inf = block_info.at(last_allocated_block);
-    if (inf.has_garbage)
+    if (inf.garbage_space > 0)
     {
         size_t i = 0, j = 0;
         for (; i < inf.entries.size(); i++)
@@ -1245,12 +1241,89 @@ int blockstore_heap_t::allocate_entry(uint32_t entry_size, uint32_t *block_num,
             }
         }
         inf.entries.resize(j);
-        inf.has_garbage = false;
+        inf.used_space -= inf.garbage_space;
+        inf.garbage_space = 0;
     }
+}
+
+int blockstore_heap_t::allocate_entry(uint32_t entry_size, uint32_t *block_num, bool allow_last_free)
+{
+    if (last_allocated_block != UINT32_MAX)
+    {
+        // First try to write into the same block as the previous time
+        auto & inf = block_info.at(last_allocated_block);
+        if (inf.is_writing ||
+            inf.used_space - inf.garbage_space + entry_size > dsk->meta_block_size ||
+            // Do not allow to make the last non-nearfull block nearfull
+            !allow_last_free && meta_nearfull_blocks >= meta_block_count-1 &&
+            inf.used_space - inf.garbage_space <= dsk->meta_block_size-max_entry_size &&
+            inf.used_space - inf.garbage_space + entry_size > dsk->meta_block_size-max_entry_size)
+        {
+            last_allocated_block = UINT32_MAX;
+        }
+    }
+    if (last_allocated_block == UINT32_MAX)
+    {
+        int i;
+        for (i = 0; last_allocated_block == UINT32_MAX && i < META_ALLOC_LEVELS-1; i++)
+        {
+            // First try to write into most free blocks
+            last_allocated_block = meta_alloc->find(i);
+        }
+        if (last_allocated_block != UINT32_MAX && i == META_ALLOC_LEVELS-1 && !allow_last_free && meta_nearfull_blocks >= meta_block_count-1)
+        {
+            // Do not allow to make the last non-nearfull block nearfull
+            auto & inf = block_info.at(last_allocated_block);
+            if (inf.used_space - inf.garbage_space <= dsk->meta_block_size-max_entry_size &&
+                inf.used_space - inf.garbage_space + entry_size > dsk->meta_block_size-max_entry_size)
+            {
+                last_allocated_block = UINT32_MAX;
+            }
+        }
+        if (last_allocated_block == UINT32_MAX)
+        {
+            // Then into nearfull blocks
+            for (uint32_t b = meta_alloc->find(META_ALLOC_LEVELS-1); b != UINT32_MAX; b = meta_alloc->next(b))
+            {
+                auto & inf = block_info.at(b);
+                if (inf.used_space - inf.garbage_space + entry_size <= dsk->meta_block_size)
+                {
+                    last_allocated_block = b;
+                    break;
+                }
+            }
+        }
+        if (last_allocated_block == UINT32_MAX)
+        {
+            // Then fail :)
+            return ENOSPC;
+        }
+    }
+    if (!allow_last_free && meta_nearfull_blocks >= meta_block_count-1)
+    {
+        // Do not allow to make the last non-nearfull block nearfull
+        auto & inf = block_info.at(last_allocated_block);
+        if (inf.used_space - inf.garbage_space <= dsk->meta_block_size-max_entry_size &&
+            inf.used_space - inf.garbage_space + entry_size > dsk->meta_block_size-max_entry_size)
+        {
+            last_allocated_block = UINT32_MAX;
+            return ENOSPC;
+        }
+    }
+    // Write into the same block
     *block_num = last_allocated_block;
     modify_alloc(last_allocated_block, [&](heap_block_info_t & inf)
     {
+        // Write just 1 entry to the block to collect garbage
+        if (inf.garbage_space > (inf.used_space-inf.garbage_space)/2)
+            last_allocated_block = UINT32_MAX;
+        gc_block(inf);
         inf.used_space += entry_size;
+        assert(inf.used_space - inf.garbage_space <= dsk->meta_block_size);
+        assert(!inf.mod_lsn_to || inf.mod_lsn_to == next_lsn);
+        ++next_lsn;
+        inf.mod_lsn = inf.mod_lsn ? inf.mod_lsn : next_lsn;
+        inf.mod_lsn_to = next_lsn;
     });
     return 0;
 }
@@ -1309,16 +1382,15 @@ int blockstore_heap_t::add_entry(uint32_t wr_size, uint32_t *modified_block,
         *modified_block = block_num;
     }
     auto li = (heap_list_item_t*)malloc_or_die(wr_size + sizeof(heap_list_item_t) - sizeof(heap_entry_t));
+    live_entries++;
+    live_memory += list_item_overhead(wr_size);
     auto new_wr = &li->entry;
     auto & inf = block_info.at(block_num);
     if (!inf.entries.size())
         inf.entries.reserve(dsk->meta_block_size / max_entry_size);
     inf.entries.push_back(li);
-    assert(!inf.mod_lsn_to || inf.mod_lsn_to == next_lsn);
-    new_wr->lsn = ++next_lsn;
+    new_wr->lsn = next_lsn;
     fill_entry(new_wr);
-    inf.mod_lsn = inf.mod_lsn ? inf.mod_lsn : next_lsn;
-    inf.mod_lsn_to = next_lsn;
     // Remember the object as dirty and remove older entries when this block is written and fsynced
     push_inflight_lsn(next_lsn, new_wr,
         (explicit_complete ? HEAP_INFLIGHT_EXPLICIT : 0) |
@@ -1418,7 +1490,7 @@ int blockstore_heap_t::add_redirect_intent(object_id oid, heap_entry_t **obj_ptr
         bitmap_set(wr->get_int_bitmap(this), offset, len, dsk->bitmap_granularity);
         if (dsk->data_csum_type)
             memset(wr->get_checksums(this), 0, get_csum_size(wr));
-        calc_checksums(wr, (uint8_t*)data, true, offset, len);
+        calc_checksums(wr, (uint8_t*)data, true);
         *obj_ptr = wr;
     });
 }
@@ -1457,11 +1529,11 @@ int blockstore_heap_t::add_big_intent(object_id oid, heap_entry_t **obj_ptr, uin
         if (dsk->data_csum_type)
         {
             if (checksums)
-                memcpy(wr->get_checksums(this), checksums, dsk->clean_entry_bitmap_size);
+                memcpy(wr->get_checksums(this), checksums, get_csum_size(wr));
             else
             {
-                memcpy(wr->get_checksums(this), obj->get_checksums(this), dsk->clean_entry_bitmap_size);
-                calc_checksums(wr, (uint8_t*)data, true, offset, len);
+                memcpy(wr->get_checksums(this), obj->get_checksums(this), get_csum_size(wr));
+                calc_checksums(wr, (uint8_t*)data, true);
             }
         }
         else
@@ -1582,7 +1654,7 @@ int blockstore_heap_t::add_commit(heap_entry_t *obj, uint64_t version, uint32_t
     }
     if (!uncommitted)
     {
-        return EBUSY;
+        return 0;
     }
     return add_simple(obj, version, modified_block, BS_HEAP_COMMIT);
 }
@@ -1592,23 +1664,32 @@ int blockstore_heap_t::add_rollback(heap_entry_t *obj, uint64_t version, uint32_
     heap_entry_t *wr = obj;
     bool found_uncommitted = false;
     uint64_t commit_version = 0;
-    while (wr && !wr->is_overwrite())
+    uint64_t rollback_version = UINT64_MAX;
+    while (wr)
     {
         if (wr->type() == BS_HEAP_ROLLBACK)
         {
-            auto rollback_version = wr->version;
-            wr = prev(wr);
-            while (wr->version > rollback_version)
+            if (wr->version <= version)
             {
-                assert(!(wr->entry_type & BS_HEAP_STABLE));
-                wr = prev(wr);
+                // All previous writes are already rolled back, stop
+                break;
             }
+            rollback_version = wr->version;
+            wr = prev(wr);
             continue;
         }
         if (wr->type() == BS_HEAP_COMMIT)
         {
             if (commit_version < wr->version)
+            {
                 commit_version = wr->version;
+            }
+            wr = prev(wr);
+            continue;
+        }
+        if (wr->version > rollback_version)
+        {
+            // Already rolled back, skip
             wr = prev(wr);
             continue;
         }
@@ -1619,14 +1700,10 @@ int blockstore_heap_t::add_rollback(heap_entry_t *obj, uint64_t version, uint32_
             {
                 return EBUSY;
             }
-            else if (wr->version == version)
+            else
             {
                 break;
             }
-            else if (wr->version < version)
-            {
-                return ENOENT;
-            }
         }
         else if (wr->version > version)
         {
@@ -1652,28 +1729,38 @@ int blockstore_heap_t::add_delete(heap_entry_t *obj, uint32_t *modified_block)
 
 uint32_t blockstore_heap_t::meta_alloc_pos(const heap_block_info_t & inf)
 {
-    if (inf.is_writing || inf.used_space > dsk->meta_block_size-sizeof(heap_entry_t))
+    auto real_used = (inf.used_space-inf.garbage_space);
+    if (inf.is_writing || inf.mod_lsn || real_used > dsk->meta_block_size-sizeof(heap_entry_t))
     {
         // 100% full - no entry can be written into this block at all
         return META_ALLOC_LEVELS;
     }
-    if (inf.used_space > dsk->meta_block_size-max_entry_size)
+    if (real_used > dsk->meta_block_size-max_entry_size)
     {
         // nearfull - big_entries won't fit into this block so it can't be used for compaction
         return META_ALLOC_LEVELS-1;
     }
-    // normal block
-    return inf.used_space / ((dsk->meta_block_size-max_entry_size+META_ALLOC_LEVELS-2) / (META_ALLOC_LEVELS-1));
+    // First we want to write to blocks with most garbage:
+    // >= 2*used, >= used/2
+    // (i.e. 66% garbage, 33% garbage)
+    // Then to mostly free blocks:
+    // >= 75% free, >= 50% free, >= 25% free
+    if (inf.garbage_space > real_used*2)
+        return 0;
+    if (inf.garbage_space > real_used/2)
+        return 1;
+    // META_ALLOC_LEVELS-3 levels left
+    return 2 + real_used / ((dsk->meta_block_size-max_entry_size+META_ALLOC_LEVELS-4) / (META_ALLOC_LEVELS-3));
 }
 
 void blockstore_heap_t::modify_alloc(uint32_t block_num, std::function<void(heap_block_info_t &)> change_cb)
 {
     auto & inf = block_info.at(block_num);
     uint32_t old_pos = meta_alloc_pos(inf);
-    uint32_t old_used = inf.used_space;
+    uint32_t old_used = inf.used_space-inf.garbage_space;
     change_cb(inf);
     uint32_t new_pos = meta_alloc_pos(inf);
-    uint32_t new_used = inf.used_space;
+    uint32_t new_used = inf.used_space-inf.garbage_space;
     meta_alloc->change(block_num, old_pos, new_pos);
     meta_used_space -= old_used;
     meta_used_space += new_used;
@@ -1685,11 +1772,19 @@ void blockstore_heap_t::modify_alloc(uint32_t block_num, std::function<void(heap
 
 void blockstore_heap_t::start_block_write(uint32_t block_num)
 {
-    modify_alloc(block_num, [&](heap_block_info_t & inf)
+    auto & inf = block_info.at(block_num);
+    assert(!inf.is_writing);
+    if (!inf.mod_lsn)
+    {
+        modify_alloc(block_num, [&](heap_block_info_t & inf)
+        {
+            inf.is_writing = true;
+        });
+    }
+    else
     {
-        assert(!inf.is_writing);
         inf.is_writing = true;
-    });
+    }
 }
 
 void blockstore_heap_t::complete_block_write(uint32_t block_num)
@@ -1754,6 +1849,8 @@ void blockstore_heap_t::mark_garbage_up_to(heap_entry_t *wr)
 void blockstore_heap_t::mark_garbage(uint32_t block_num, heap_entry_t *prev_wr, uint32_t used_big)
 {
     prev_wr->set_garbage();
+    garbage_entries++;
+    garbage_memory += list_item_overhead(prev_wr->size);
     // And this is the moment when we can free the data reference
     if (prev_wr->type() == BS_HEAP_SMALL_WRITE && prev_wr->small().len > 0)
     {
@@ -1769,8 +1866,7 @@ void blockstore_heap_t::mark_garbage(uint32_t block_num, heap_entry_t *prev_wr,
     }
     modify_alloc(block_num, [&](heap_block_info_t & inf)
     {
-        inf.used_space -= prev_wr->size;
-        inf.has_garbage = true;
+        inf.garbage_space += prev_wr->size;
     });
 }
 
@@ -1807,9 +1903,9 @@ void blockstore_heap_t::iterate_with_stable(heap_entry_t *obj, uint64_t max_lsn,
         }
         else
         {
-            // 1) 1 2 3 ROLLBACK(2) COMMIT(3) -> impossible
+            // 1) 1 2 3 ROLLBACK(2) COMMIT(3) -> 3 is unstable
             // 2) 1 2 3 4 ROLLBACK(3) COMMIT(2) -> OK
-            // 3) 1 2 3 ROLLBACK(2) 3 COMMIT(3) -> first 3 shouldn't be treated as stable
+            // 3) 1 2 3 ROLLBACK(2) 3 COMMIT(3) -> first 3 is unstable
             // 4) 1 2 3 COMMIT(3) ROLLBACK(2) -> impossible
             //    I.e. a rollback always has version >= previous commit
             // 5) 1 2 3 4 5 ROLLBACK(4) 5 ROLLBACK(3)
@@ -1966,24 +2062,21 @@ int blockstore_heap_t::list_objects(uint32_t pg_num, object_id min_oid, object_i
                 return;
             }
             uint64_t stable_version = 0;
-            auto first_wr = obj;
-            for (auto wr = first_wr; wr; wr = prev(wr))
+            iterate_with_stable(obj, UINT64_MAX, [&](heap_entry_t* wr, bool stable)
             {
-                if ((wr->entry_type & BS_HEAP_STABLE) || wr->type() == BS_HEAP_COMMIT || wr->type() == BS_HEAP_ROLLBACK)
+                if (stable)
                 {
                     stable_version = wr->version;
-                    break;
+                    return false;
                 }
-                else
+                if (unstable_size >= unstable_alloc)
                 {
-                    if (unstable_size >= unstable_alloc)
-                    {
-                        unstable_alloc = (!unstable_alloc ? 128 : unstable_alloc*2);
-                        unstable = (obj_ver_id*)realloc_or_die(unstable, sizeof(obj_ver_id) * unstable_alloc);
-                    }
-                    unstable[unstable_size++] = (obj_ver_id){ .oid = oid, .version = wr->version };
+                    unstable_alloc = (!unstable_alloc ? 128 : unstable_alloc*2);
+                    unstable = (obj_ver_id*)realloc_or_die(unstable, sizeof(obj_ver_id) * unstable_alloc);
                 }
-            }
+                unstable[unstable_size++] = (obj_ver_id){ .oid = oid, .version = wr->version };
+                return true;
+            });
             if (stable_version)
             {
                 if (res_size >= res_alloc)
@@ -2045,7 +2138,13 @@ void blockstore_heap_t::free_data(inode_t inode, uint64_t location)
         inode = (INODE_POOL(inode) << POOL_ID_BITS);
     assert(data_alloc->get(location / dsk->data_block_size));
     data_alloc->set(location / dsk->data_block_size, false);
-    inode_space_stats[inode] -= dsk->data_block_size;
+    auto sp_it = inode_space_stats.find(inode);
+    if (sp_it != inode_space_stats.end())
+    {
+        sp_it->second -= dsk->data_block_size;
+        if (sp_it->second == 0)
+            inode_space_stats.erase(sp_it);
+    }
     data_used_space -= dsk->data_block_size;
 }
 
@@ -2137,7 +2236,7 @@ void blockstore_heap_t::fill_block_empty_space(uint8_t *buffer, uint32_t pos)
 uint32_t blockstore_heap_t::get_meta_block_used_space(uint32_t block_num)
 {
     auto & inf = block_info.at(block_num);
-    return inf.used_space;
+    return inf.used_space - inf.garbage_space;
 }
 
 uint64_t blockstore_heap_t::get_data_used_space()
@@ -2180,6 +2279,26 @@ uint64_t blockstore_heap_t::get_compacted_count()
     return compacted_count;
 }
 
+uint64_t blockstore_heap_t::get_live_entries()
+{
+    return live_entries-garbage_entries;
+}
+
+uint64_t blockstore_heap_t::get_live_memory()
+{
+    return live_memory-garbage_memory;
+}
+
+uint64_t blockstore_heap_t::get_garbage_entries()
+{
+    return garbage_entries;
+}
+
+uint64_t blockstore_heap_t::get_garbage_memory()
+{
+    return garbage_memory;
+}
+
 void blockstore_heap_t::push_inflight_lsn(uint64_t lsn, heap_entry_t *wr, uint64_t flags)
 {
     uint64_t next_inf = first_inflight_lsn + inflight_lsn.size();
@@ -2266,35 +2385,50 @@ void blockstore_heap_t::apply_inflight(heap_inflight_lsn_t & inflight)
     {
         // Remove entry
         auto li = list_item(wr);
-        auto prev = li->prev;
-        auto next = li->next;
-        if (prev)
-        {
-            prev->next = next;
-        }
-        if (!next)
-        {
-            assert(!prev);
-            auto & pg_idx = block_index[get_pg_id(wr->inode, wr->stripe)];
-            auto & inode_idx = pg_idx[wr->inode];
-            heap_inode_map_t::iterator li_it;
-            heap_list_item_t *old_li = NULL;
-            inode_map_get(inode_idx, li_it, old_li, wr->stripe);
-            inode_map_erase(pg_idx, inode_idx, li_it, old_li);
-        }
-        else
-        {
-            next->prev = prev;
-            if (!prev && next->entry.entry_type == (BS_HEAP_DELETE|BS_HEAP_STABLE))
-            {
-                // free BS_HEAP_DELETEs when all previous entries are also freed
-                mark_garbage(next->block_num, &next->entry, UINT32_MAX);
-            }
-        }
-        free(li);
+        remove_list_item(li);
     }
 }
 
+void blockstore_heap_t::remove_list_item(heap_list_item_t *li)
+{
+    auto prev = li->prev;
+    auto next = li->next;
+    if (prev)
+    {
+        prev->next = next;
+    }
+    if (!next)
+    {
+        // The last freed entry must be a deletion
+        assert(!prev);
+        auto wr = &li->entry;
+        assert(wr->entry_type == BS_HEAP_DELETE|BS_HEAP_STABLE);
+        auto & pg_idx = block_index[get_pg_id(wr->inode, wr->stripe)];
+        auto & inode_idx = pg_idx[wr->inode];
+        heap_inode_map_t::iterator li_it;
+        heap_list_item_t *old_li = NULL;
+        inode_map_get(inode_idx, li_it, old_li, wr->stripe);
+        inode_map_erase(pg_idx, inode_idx, li_it, old_li);
+    }
+    else
+    {
+        next->prev = prev;
+        if (!prev && next->entry.entry_type == (BS_HEAP_DELETE|BS_HEAP_STABLE))
+        {
+            // free BS_HEAP_DELETEs when all previous entries are also freed
+            mark_garbage(next->block_num, &next->entry, UINT32_MAX);
+        }
+    }
+    if (li->entry.is_garbage())
+    {
+        garbage_entries--;
+        garbage_memory -= list_item_overhead(li->entry.size);
+    }
+    live_entries--;
+    live_memory -= list_item_overhead(li->entry.size);
+    free(li);
+}
+
 bool blockstore_heap_t::is_lsn_completed(uint64_t lsn)
 {
     if (lsn <= completed_lsn)
@@ -2328,7 +2462,7 @@ void blockstore_heap_t::set_no_inode_stats(const std::vector<uint64_t> & pool_id
     {
         // Recalculate if changed
         if (ps.second.no_inode_stats == 2 || ps.second.no_inode_stats == 1)
-            recalc_inode_space_stats(ps.first, ps.second.no_inode_stats == 1);
+            recalc_inode_space_stats(ps.first, ps.second.no_inode_stats == 2);
         ps.second.no_inode_stats &= 1;
     }
 }
@@ -2339,8 +2473,8 @@ void blockstore_heap_t::recalc_inode_space_stats(uint64_t pool_id, bool per_inod
     auto sp_begin = inode_space_stats.lower_bound((pool_id << (64-POOL_ID_BITS)));
     auto sp_end = inode_space_stats.lower_bound(((pool_id+1) << (64-POOL_ID_BITS)));
     inode_space_stats.erase(sp_begin, sp_end);
-    uint32_t pg_count = ps.pg_count ? ps.pg_count : 1;
-    for (uint32_t pg_num = 1; pg_num <= pg_count; pg_num++)
+    uint32_t pg_count = ps.pg_count;
+    for (uint32_t pg_num = pg_count ? 1 : 0; pg_num <= pg_count; pg_num++)
     {
         auto & pg_idx = block_index[(pool_id << (64-POOL_ID_BITS)) | pg_num];
         for (auto & ip: pg_idx)
@@ -2378,7 +2512,10 @@ void inode_map_get(void *inode_idx, heap_inode_map_t::iterator & li_it, heap_lis
     size_t map_n = ((size_t)inode_idx & IMAP_MALLOC_LOW_BITS);
     if (!map_n)
     {
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Warray-bounds"
         li_it = ((heap_inode_map_t*)inode_idx)->find(list_item_key(&stripe));
+#pragma GCC diagnostic pop
         li = li_it != ((heap_inode_map_t*)inode_idx)->end() ? *li_it : NULL;
     }
     else if (map_n == 1)

src/blockstore/blockstore_heap.h

@@ -117,10 +117,13 @@ struct heap_object_mvcc_t
 
 struct heap_block_info_t
 {
-    uint32_t used_space = 0;
+    struct __attribute__((__packed__))
+    {
+        uint32_t used_space = 0;
+        uint32_t garbage_space = 0;
+    };
     uint64_t mod_lsn = 0, mod_lsn_to = 0; // only 1 block write of LSN sequence is allowed at a moment
-    bool is_writing: 1;
-    bool has_garbage: 1;
+    bool is_writing = false;
     std::vector<heap_list_item_t*> entries;
 };
 
@@ -184,6 +187,11 @@ class blockstore_heap_t
     uint64_t buffer_area_used_space = 0;
     uint64_t data_used_space = 0;
 
+    uint64_t live_entries = 0;
+    uint64_t live_memory = 0;
+    uint64_t garbage_entries = 0;
+    uint64_t garbage_memory = 0;
+
     uint64_t next_lsn = 0;
     uint32_t last_allocated_block = UINT32_MAX;
     heap_mvcc_map_t object_mvcc;
@@ -200,6 +208,7 @@ class blockstore_heap_t
 
     bool marked_used_blocks = false;
     bool recheck_queue_filled = false;
+    std::vector<heap_list_item_t*> loaded_list_items;
     std::set<uint32_t> recheck_modified_blocks;
     std::deque<heap_entry_t*> recheck_queue;
     int recheck_in_progress = 0;
@@ -211,12 +220,15 @@ class blockstore_heap_t
     bool validate_object(heap_entry_t *obj);
     void fill_recheck_queue();
     int mark_used_blocks();
+    void recheck_full_gc();
     void recheck_buffer(heap_entry_t *cwr, uint8_t *buf);
     void defragment_block(uint32_t block_num);
     void reshard_add(heap_reshard_state_t *st, heap_list_item_t *li);
 
+    void gc_block(heap_block_info_t & inf);
     int allocate_entry(uint32_t entry_size, uint32_t *block_num, bool allow_last_free);
     void insert_list_item(heap_list_item_t *li);
+    void remove_list_item(heap_list_item_t *li);
     int add_entry(uint32_t wr_size, uint32_t *modified_block, bool allow_last_free,
         bool explicit_complete, std::function<void(heap_entry_t *wr)> fill_entry);
     int add_simple(heap_entry_t *obj, uint64_t version, uint32_t *modified_block, uint32_t entry_type);
@@ -237,31 +249,29 @@ public:
         std::function<void(uint32_t, uint32_t, uint8_t*)> handle_block);
     int load_blocks(uint64_t disk_offset, uint64_t size, uint8_t *buf,
         bool allow_corrupted, uint64_t &entries_loaded);
-    // finish loading
-    int finish_load(bool allow_corrupted = false);
+    // finish loading - should be called after load_blocks
+    void finish_load();
     // get blocks which are modified during loading and should be written to the disk
     // before finishing initialization if not R/O
     std::vector<uint32_t> get_recheck_modified_blocks();
     // recheck small write data after reading the database from disk
     bool recheck_small_writes(std::function<void(bool is_data, uint64_t offset, uint64_t len, uint8_t* buf, std::function<void()>)> read_buffer, int queue_depth);
+    int finish_recheck();
     // reshard database according to the pool's PG count
     void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
     bool reshard_continue(void* reshard_state, uint64_t chunk_limit);
     bool reshard_check(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size);
-    void reshard_abort(void* reshard_state);
     void set_no_inode_stats(const std::vector<uint64_t> & pool_ids);
     void recalc_inode_space_stats(uint64_t pool_id, bool per_inode);
     // read an object entry and lock it against removal
     // in the future, may become asynchronous
     heap_entry_t *lock_and_read_entry(object_id oid);
-    // re-read a locked object entry with the given lsn (pointer may be invalidated)
-    heap_entry_t *read_locked_entry(object_id oid, uint64_t lsn);
     // read an object entry without locking it
     heap_entry_t *read_entry(object_id oid);
     // unlock an entry
     bool unlock_entry(object_id oid);
     // set or verify checksums in a write request
-    bool calc_checksums(heap_entry_t *wr, uint8_t *data, bool set, uint32_t offset = 0, uint32_t len = 0);
+    bool calc_checksums(heap_entry_t *wr, uint8_t *data, bool set, uint32_t offset = UINT32_MAX, uint32_t len = UINT32_MAX);
     // set or verify raw block checksums
     bool calc_block_checksums(uint32_t *block_csums, uint8_t *data, uint8_t *bitmap, uint32_t start, uint32_t end,
         bool set, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);
@@ -345,6 +355,10 @@ public:
     uint32_t get_compact_queue_size();
     uint32_t get_to_compact_count();
     uint64_t get_compacted_count();
+    uint64_t get_live_entries();
+    uint64_t get_live_memory();
+    uint64_t get_garbage_entries();
+    uint64_t get_garbage_memory();
 
     uint64_t entry_pos(uint32_t block_num, uint32_t offset);
     heap_entry_t *entry_from_pos(uint64_t entry_pos, bool allow_unallocated = false);

src/blockstore/blockstore_impl.cpp

@@ -193,12 +193,12 @@ void blockstore_impl_t::loop()
             heap->start_block_write(block_num);
             mb.sent = true;
         }
+        pending_modified_blocks.clear();
         int ret = ringloop->submit();
         if (ret < 0)
         {
             throw std::runtime_error(std::string("io_uring_submit: ") + strerror(-ret));
         }
-        pending_modified_blocks.clear();
         if ((initial_ring_space - ringloop->space_left()) > 0)
         {
             live = true;
@@ -406,8 +406,3 @@ bool blockstore_impl_t::reshard_continue(void *reshard_state, uint64_t chunk_lim
 {
     return heap->reshard_continue(reshard_state, chunk_limit);
 }
-
-void blockstore_impl_t::reshard_abort(void *reshard_state)
-{
-    return heap->reshard_abort(reshard_state);
-}

src/blockstore/blockstore_impl.h

@@ -78,6 +78,7 @@ public:
     // Suitable only for server SSDs with capacitors, requires disabled data and journal fsyncs
     int immediate_commit = IMMEDIATE_NONE;
     bool inmemory_meta = false;
+    bool skip_corrupted_meta_entries = false;
     uint32_t meta_write_recheck_parallelism = 0;
     // Maximum and minimum flusher count
     unsigned max_flusher_count = 0, min_flusher_count = 0;
@@ -191,7 +192,6 @@ public:
 
     void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
     bool reshard_continue(void *reshard_state, uint64_t chunk_limit);
-    void reshard_abort(void *reshard_state);
 
     // Event loop
     void loop();
@@ -229,4 +229,9 @@ public:
     uint64_t get_free_block_count();
     inline uint32_t get_bitmap_granularity() { return dsk.bitmap_granularity; }
     inline uint64_t get_journal_size() { return dsk.journal_len; }
+
+    inline uint64_t get_live_entries() { return heap->get_live_entries(); }
+    inline uint64_t get_live_memory() { return heap->get_live_memory(); }
+    inline uint64_t get_garbage_entries() { return heap->get_garbage_entries(); }
+    inline uint64_t get_garbage_memory() { return heap->get_garbage_memory(); }
 };

src/blockstore/blockstore_init.cpp

@@ -145,7 +145,7 @@ resume_1:
             printf(
                 "Configuration stored in metadata superblock"
                 " (meta_block_size=%u, data_block_size=%u, bitmap_granularity=%u, data_csum_type=%u, csum_block_size=%u, meta_area_size=%ju)"
-                " differs from OSD configuration (%u/%u/%u, %u/%u, %ju).\n",
+                " differs from OSD configuration (%ju/%ju/%u, %u/%u, %ju).\n",
                 hdr->meta_block_size, hdr->data_block_size, hdr->bitmap_granularity,
                 hdr->data_csum_type, hdr->csum_block_size, hdr->meta_area_size,
                 bs->dsk.meta_block_size, bs->dsk.data_block_size, bs->dsk.bitmap_granularity,
@@ -225,7 +225,7 @@ resume_4:
         {
             // Handle result
             uint64_t loaded = 0;
-            int r = bs->heap->load_blocks(bufs[i].offset-bs->dsk.meta_block_size, bufs[i].size, bufs[i].buf, false, loaded);
+            int r = bs->heap->load_blocks(bufs[i].offset-bs->dsk.meta_block_size, bufs[i].size, bufs[i].buf, bs->skip_corrupted_meta_entries, loaded);
             if (r != 0)
                 exit(1);
             entries_loaded += loaded;
@@ -239,7 +239,8 @@ resume_4:
         return 1;
     }
     // metadata read finished
-    printf("Metadata entries loaded: %ju, used blocks: %ju / %ju\n", entries_loaded, bs->heap->get_data_used_space() / bs->dsk.data_block_size, bs->dsk.block_count);
+    bs->heap->finish_load();
+    printf("Metadata entries loaded: %ju, rechecking unfinished writes and garbage entries\n", entries_loaded);
     if (zero_on_init && !bs->dsk.disable_meta_fsync)
     {
         GET_SQE();
@@ -284,7 +285,7 @@ resume_6:
     }, bs->meta_write_recheck_parallelism);
     return 1;
 resume_7:
-    if (bs->heap->finish_load() != 0)
+    if (bs->heap->finish_recheck() != 0)
     {
         exit(1);
     }
@@ -292,6 +293,11 @@ resume_7:
     if (bs->readonly)
     {
         recheck_mod.clear();
+        printf("Actual metadata entries: %ju\n", bs->heap->get_live_entries());
+    }
+    else
+    {
+        printf("Actual metadata entries: %ju, clearing garbage in %zu metadata blocks\n", bs->heap->get_live_entries(), recheck_mod.size());
     }
     for (i = 0; i < recheck_mod.size(); i++)
     {
@@ -305,7 +311,7 @@ resume_8:
         uint32_t block_num = recheck_mod[i];
         uint64_t block_offset = bs->dsk.meta_offset + (uint64_t)(block_num+1) * bs->dsk.meta_block_size;
         data = ((ring_data_t*)sqe->user_data);
-        uint8_t *buf = (uint8_t*)malloc_or_die(bs->dsk.meta_block_size);
+        uint8_t *buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, bs->dsk.meta_block_size);
         bs->heap->get_meta_block(block_num, buf);
         data->iov = { buf, bs->dsk.meta_block_size };
         data->callback = [this, buf, block_offset](ring_data_t *data)
@@ -331,5 +337,9 @@ resume_9:
     }
     free(metadata_buffer);
     metadata_buffer = NULL;
+    printf("Loading finished. Data used: %ju / %ju bytes (%s / %s)\n",
+        bs->heap->get_data_used_space(), bs->dsk.block_count * bs->dsk.data_block_size,
+        format_size(bs->heap->get_data_used_space()).c_str(),
+        format_size(bs->dsk.block_count * bs->dsk.data_block_size).c_str());
     return 0;
 }

src/blockstore/blockstore_open.cpp

@@ -28,6 +28,7 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init)
     throttle_target_parallelism = strtoull(config["throttle_target_parallelism"].c_str(), NULL, 10);
     throttle_threshold_us = strtoull(config["throttle_threshold_us"].c_str(), NULL, 10);
     perfect_csum_update = config["perfect_csum_update"] == "true" || config["perfect_csum_update"] == "1" || config["perfect_csum_update"] == "yes";
+    skip_corrupted_meta_entries = config["skip_corrupted_meta_entries"] == "true" || config["skip_corrupted_meta_entries"] == "1" || config["skip_corrupted_meta_entries"] == "yes";
     if (config["autosync_writes"] != "")
     {
         autosync_writes = strtoull(config["autosync_writes"].c_str(), NULL, 10);

src/blockstore/blockstore_read.cpp

@@ -462,6 +462,10 @@ int blockstore_impl_t::read_bitmap(object_id oid, uint64_t target_version, void
         {
             if (target_version >= wr->version)
             {
+                if (wr->type() == BS_HEAP_DELETE)
+                {
+                    return false;
+                }
                 found = true;
                 if (result_version)
                 {

src/blockstore/blockstore_stable.cpp

@@ -57,9 +57,9 @@ int blockstore_impl_t::dequeue_stable(blockstore_op_t *op)
             }
             assert(res == 0);
         }
+resume_1:
         if (priv->modified_block != UINT32_MAX && priv->modified_block2 != priv->modified_block)
         {
-resume_1:
             BS_SUBMIT_CHECK_SQES(1);
             prepare_meta_block_write(priv->modified_block);
 resume_2:

src/blockstore/blockstore_write.cpp

@@ -22,21 +22,23 @@ void blockstore_impl_t::prepare_meta_block_write(uint32_t modified_block)
     ring_data_t *data = ((ring_data_t*)sqe->user_data);
     uint8_t *buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size);
     data->iov = (struct iovec){ buf, (size_t)dsk.meta_block_size };
-    data->callback = [this, modified_block, buf](ring_data_t *data)
+    data->callback = [this, modified_block](ring_data_t *data)
     {
-        free(buf);
         live = true;
         if (data->res != data->iov.iov_len)
         {
             // FIXME: our state becomes corrupted after a write error. maybe do something better than just die
             disk_error_abort("data write", data->res, data->iov.iov_len);
         }
-        modified_blocks.erase(modified_block);
+        auto it = modified_blocks.find(modified_block);
+        assert(it != modified_blocks.end());
+        free(it->second.buf);
+        modified_blocks.erase(it);
         heap->complete_block_write(modified_block);
         ringloop->wakeup();
     };
     io_uring_prep_writev(
-        sqe, dsk.meta_fd, &data->iov, 1, dsk.meta_offset + (modified_block+1)*dsk.meta_block_size
+        sqe, dsk.meta_fd, &data->iov, 1, dsk.meta_offset + ((uint64_t)modified_block+1)*dsk.meta_block_size
     );
     unsynced_meta_write_count++;
     pending_modified_blocks.push_back(modified_block);
@@ -249,13 +251,12 @@ enospc:
             goto enospc;
         assert(res == 0);
         PRIV(op)->lsn = obj->lsn;
-        if (op->len)
-            heap->use_buffer_area(op->oid.inode, loc, op->len);
         prepare_meta_block_write(PRIV(op)->modified_block);
         PRIV(op)->pending_ops++;
         if (op->len > 0)
         {
             // Prepare buffered data write
+            heap->use_buffer_area(op->oid.inode, loc, op->len);
             if (dsk.inmemory_journal)
             {
                 memcpy((uint8_t*)buffer_area + loc, op->buf, op->len);
@@ -348,6 +349,7 @@ resume_12:
     }
 resume_4:
     {
+        BS_SUBMIT_CHECK_SQES(1);
         auto obj = heap->read_entry(op->oid);
         int res = 0;
         if (PRIV(op)->write_type == _REDIRECT_INTENT)
@@ -404,11 +406,12 @@ resume_6:
         if (ref_us > exec_us + throttle_threshold_us)
         {
             // Pause reply
+            PRIV(op)->pending_ops++;
             PRIV(op)->op_state = 7;
             // Remember that the timer can in theory be called right here
             tfd->set_timer_us(ref_us-exec_us, false, [this, op](int timer_id)
             {
-                PRIV(op)->op_state = 8;
+                PRIV(op)->pending_ops--;
                 ringloop->wakeup();
             });
             return 1;

src/blockstore/v1/impl.cpp

@@ -480,17 +480,6 @@ resume_1:
     return true;
 }
 
-void blockstore_impl_t::reshard_abort(void *reshard_state)
-{
-    bs_reshard_state_t *st = (bs_reshard_state_t*)reshard_state;
-    for (auto sh_it = st->old_shards.begin(); sh_it != st->old_shards.end(); sh_it++)
-    {
-        auto & to = clean_db_shards[sh_it->first];
-        to.swap(sh_it->second);
-    }
-    delete st;
-}
-
 void blockstore_impl_t::process_list(blockstore_op_t *op)
 {
     uint32_t list_pg = op->pg_number+1;
@@ -866,4 +855,29 @@ std::string blockstore_impl_t::get_op_diag(blockstore_op_t *op)
     return std::string(buf);
 }
 
+uint64_t blockstore_impl_t::get_live_entries()
+{
+    return used_blocks;
+}
+
+uint64_t blockstore_impl_t::get_live_memory()
+{
+    uint64_t used = 0;
+    for (auto & kv: clean_db_shards)
+    {
+        used += kv.second.size() * sizeof(blockstore_clean_db_t::value_type);
+    }
+    return used;
+}
+
+uint64_t blockstore_impl_t::get_garbage_entries()
+{
+    return dirty_db.size();
+}
+
+uint64_t blockstore_impl_t::get_garbage_memory()
+{
+    return (sizeof(obj_ver_id) + sizeof(dirty_entry) + 32) * dirty_db.size();
+}
+
 } // namespace v1

src/blockstore/v1/impl.h

@@ -290,7 +290,6 @@ public:
     // Reshard database for a pool
     void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
     bool reshard_continue(void *reshard_state, uint64_t chunk_limit);
-    void reshard_abort(void *reshard_state);
 
     // Event loop
     void loop();
@@ -333,6 +332,10 @@ public:
     inline uint64_t get_free_block_count() { return dsk.block_count - used_blocks; }
     inline uint32_t get_bitmap_granularity() { return dsk.disk_alignment; }
     inline uint64_t get_journal_size() { return dsk.journal_len; }
+    uint64_t get_live_entries();
+    uint64_t get_live_memory();
+    uint64_t get_garbage_entries();
+    uint64_t get_garbage_memory();
 };
 
 } // namespace v1

src/blockstore/v1/init.cpp

@@ -189,7 +189,7 @@ resume_1:
             printf(
                 "Configuration stored in metadata superblock"
                 " (meta_block_size=%u, data_block_size=%u, bitmap_granularity=%u, data_csum_type=%u, csum_block_size=%u)"
-                " differs from OSD configuration (%u/%u/%u, %u/%u).\n",
+                " differs from OSD configuration (%ju/%ju/%u, %u/%u).\n",
                 hdr->meta_block_size, hdr->data_block_size, hdr->bitmap_granularity,
                 hdr->data_csum_type, hdr->csum_block_size,
                 bs->dsk.meta_block_size, bs->dsk.data_block_size, bs->dsk.bitmap_granularity,

src/blockstore/v1/read.cpp

@@ -620,8 +620,7 @@ bool blockstore_impl_t::fulfill_clean_read(blockstore_op_t *read_op, uint64_t &
     else if (from_journal)
     {
         // Don't scan bitmap - journal writes don't have holes (internal bitmap)!
-        uint8_t *csum = !dsk.csum_block_size ? 0 : (clean_entry_bitmap + dsk.clean_entry_bitmap_size +
-            item_start/dsk.csum_block_size*(dsk.data_csum_type & 0xFF));
+        uint8_t *csum = !dsk.csum_block_size ? 0 : (clean_entry_bitmap + dsk.clean_entry_bitmap_size);
         if (!fulfill_read(read_op, fulfilled, item_start, item_end,
             (BS_ST_BIG_WRITE | BS_ST_STABLE), 0, clean_loc + item_start, 0, csum, dyn_data))
         {

src/client/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 2.8...3.30)
 
 project(vitastor)
 

src/client/cluster_client.cpp

@@ -27,7 +27,7 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
     msgr.ringloop = ringloop;
     msgr.repeer_pgs = [this](osd_num_t peer_osd)
     {
-        if (msgr.osd_peer_fds.find(peer_osd) != msgr.osd_peer_fds.end())
+        if (msgr.osd_peers.find(peer_osd) != msgr.osd_peers.end())
         {
             // peer_osd just connected
             continue_ops();
@@ -47,8 +47,8 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
     msgr.exec_op = [this](osd_op_t *op)
     {
         // Garbage in
-        fprintf(stderr, "Incoming garbage from peer %d\n", op->peer_fd);
-        msgr.stop_client(op->peer_fd);
+        fprintf(stderr, "Can't handle incoming operation from client %lu\n", op->client_id);
+        msgr.stop_client(op->client_id);
         delete op;
     };
     msgr.parse_config(config);
@@ -156,7 +156,7 @@ void cluster_client_t::continue_raw_ops(osd_num_t peer_osd)
     {
         auto op = it->second;
         op->op_type = OSD_OP_OUT;
-        op->peer_fd = msgr.osd_peer_fds.at(peer_osd);
+        op->client_id = msgr.osd_peers.at(peer_osd)->client_id;
         msgr.outbox_push(op);
         raw_ops.erase(it++);
     }
@@ -590,7 +590,7 @@ void cluster_client_t::on_change_pool_config_hook()
         {
             if (log_level > 2 && pg_counts[pool_item.first])
             {
-                printf("Pool %u (%s) PG count changed from %lu to %lu\n", pool_item.first, pool_item.second.name.c_str(),
+                fprintf(stderr, "Pool %u (%s) PG count changed from %lu to %lu\n", pool_item.first, pool_item.second.name.c_str(),
                     pg_counts[pool_item.first], pool_item.second.real_pg_count);
             }
             // At this point, all pool operations should have been suspended
@@ -871,13 +871,13 @@ void cluster_client_t::execute_cas(cluster_op_t *op)
         if (op->retval != expected && op->retval >= 0)
             op->retval = -EIO;
         op->retval = op->retval == -EPIPE ? -EINTR : op->retval;
-        auto peer_it = msgr.osd_peer_fds.find(op->parts[0].osd_num);
+        auto peer_it = msgr.osd_peers.find(op->parts[0].osd_num);
         if (op->retval != 0 || (op->flags & OP_IMMEDIATE_COMMIT))
         {
             auto cb = std::move(op->callback);
             cb(op);
         }
-        else if (peer_it == msgr.osd_peer_fds.end())
+        else if (peer_it == msgr.osd_peers.end())
         {
             // Care must be taken to make sure that the client doesn't reconnect to the OSD
             // before executing the previously completed operation callback (!)
@@ -888,10 +888,10 @@ void cluster_client_t::execute_cas(cluster_op_t *op)
         else
         {
             // CAS writes have a built-in sync
-            auto peer_fd = peer_it->second;
+            osd_client_t *cl = peer_it->second;
             *part = (osd_op_t){
                 .op_type = OSD_OP_OUT,
-                .peer_fd = peer_fd,
+                .client_id = cl->client_id,
                 .req = {
                     .hdr = {
                         .magic = SECONDARY_OSD_OP_MAGIC,
@@ -1004,11 +1004,11 @@ bool cluster_client_t::check_rw(cluster_op_t *op)
 
 void cluster_client_t::execute_raw(osd_num_t osd_num, osd_op_t *op)
 {
-    auto fd_it = msgr.osd_peer_fds.find(osd_num);
-    if (fd_it != msgr.osd_peer_fds.end())
+    auto peer_it = msgr.osd_peers.find(osd_num);
+    if (peer_it != msgr.osd_peers.end())
     {
         op->op_type = OSD_OP_OUT;
-        op->peer_fd = fd_it->second;
+        op->client_id = peer_it->second->client_id;
         msgr.outbox_push(op);
     }
     else
@@ -1401,10 +1401,10 @@ int cluster_client_t::try_send(cluster_op_t *op, int i, std::function<void(osd_o
                 primary_osd = nearest_osd;
         }
         part->osd_num = primary_osd;
-        auto peer_it = msgr.osd_peer_fds.find(primary_osd);
-        if (peer_it != msgr.osd_peer_fds.end())
+        auto peer_it = msgr.osd_peers.find(primary_osd);
+        if (peer_it != msgr.osd_peers.end())
         {
-            int peer_fd = peer_it->second;
+            osd_client_t *cl = peer_it->second;
             part->flags |= PART_SENT|PART_VALID;
             op->inflight_count++;
             uint64_t pg_bitmap_size = (pool_cfg.data_block_size / pool_cfg.bitmap_granularity / 8) * (
@@ -1419,7 +1419,7 @@ int cluster_client_t::try_send(cluster_op_t *op, int i, std::function<void(osd_o
             }
             part->op = (osd_op_t){
                 .op_type = OSD_OP_OUT,
-                .peer_fd = peer_fd,
+                .client_id = cl->client_id,
                 .req = { .rw = {
                     .header = {
                         .magic = SECONDARY_OSD_OP_MAGIC,
@@ -1468,8 +1468,8 @@ int cluster_client_t::continue_sync(cluster_op_t *op)
     for (auto do_it = dirty_osds.begin(); do_it != dirty_osds.end(); )
     {
         osd_num_t sync_osd = *do_it;
-        auto peer_it = msgr.osd_peer_fds.find(sync_osd);
-        if (peer_it == msgr.osd_peer_fds.end())
+        auto peer_it = msgr.osd_peers.find(sync_osd);
+        if (peer_it == msgr.osd_peers.end())
             dirty_osds.erase(do_it++);
         else
             do_it++;
@@ -1522,12 +1522,12 @@ resume_1:
 
 void cluster_client_t::send_sync(cluster_op_t *op, cluster_op_part_t *part)
 {
-    auto peer_fd = msgr.osd_peer_fds.at(part->osd_num);
+    osd_client_t *cl = msgr.osd_peers.at(part->osd_num);
     part->flags |= PART_SENT;
     op->inflight_count++;
     part->op = (osd_op_t){
         .op_type = OSD_OP_OUT,
-        .peer_fd = peer_fd,
+        .client_id = cl->client_id,
         .req = {
             .hdr = {
                 .magic = SECONDARY_OSD_OP_MAGIC,
@@ -1567,10 +1567,10 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
             // Error priority: EIO > ENOSPC > ETIMEDOUT > EPIPE
             op->retval = part->op.reply.hdr.retval;
         }
-        int stop_fd = -1;
+        uint64_t stop_client_id = 0;
         if (op->retval != -EINTR && op->retval != -EIO && op->retval != -ENOSPC)
         {
-            stop_fd = part->op.peer_fd;
+            stop_client_id = part->op.client_id;
             if (op->retval != -EPIPE || log_level > 0)
             {
                 fprintf(
@@ -1597,9 +1597,9 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
             op->retry_after = op->retval != -EPIPE ? client_eio_retry_interval : client_retry_interval;
         }
         reset_retry_timer(op->retry_after);
-        if (stop_fd >= 0)
+        if (stop_client_id)
         {
-            msgr.stop_client(stop_fd);
+            msgr.stop_client(stop_client_id);
         }
         op->inflight_count--;
         if (op->inflight_count == 0 && !op->retry_after)
@@ -1634,7 +1634,7 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
             if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
             {
                 for (auto & part: op->parts)
-                    if (part.flags == (PART_SENT|PART_VALID|PART_DONE))
+                    if ((part.flags & (PART_SENT|PART_VALID|PART_DONE)) == (PART_SENT|PART_VALID|PART_DONE))
                         copy_part_bitmap(op, &part);
             }
             if (op->opcode == OSD_OP_SYNC)

src/client/cluster_client_list.cpp

@@ -295,7 +295,7 @@ int cluster_client_t::start_pg_listing(inode_list_pg_t *pg)
     bool conn = true;
     for (osd_num_t peer_osd: all_peers)
     {
-        if (msgr.osd_peer_fds.find(peer_osd) == msgr.osd_peer_fds.end())
+        if (msgr.osd_peers.find(peer_osd) == msgr.osd_peers.end())
         {
             // Initiate connection
             if (st_cli.peer_states[peer_osd].is_null())
@@ -340,7 +340,7 @@ void cluster_client_t::send_list(inode_list_osd_t *cur_list)
     osd_op_t *op = new osd_op_t();
     op->op_type = OSD_OP_OUT;
     // Already checked that it exists above, but anyway
-    op->peer_fd = msgr.osd_peer_fds.at(cur_list->osd_num);
+    op->client_id = msgr.osd_peers.at(cur_list->osd_num)->client_id;
     op->req = (osd_any_op_t){
         .sec_list = {
             .header = {

src/client/etcd_state_client.cpp

@@ -414,7 +414,10 @@ void etcd_state_client_t::start_etcd_watcher()
                 }
                 // Save revision only if it's present in the message - because sometimes etcd sends something without a header, like:
                 // {"error": {"grpc_code": 14, "http_code": 503, "http_status": "Service Unavailable", "message": "error reading from server: EOF"}}
-                if (etcd_watches_initialised == ETCD_TOTAL_WATCHES && !data["result"]["header"]["revision"].is_null())
+                // Also don't save revision from the initial created: true messages because they always contain the latest revision
+                if (etcd_watches_initialised == ETCD_TOTAL_WATCHES &&
+                    !data["result"]["header"]["revision"].is_null() &&
+                    !data["result"]["created"].bool_value())
                 {
                     // Restart watchers from the same revision number as in the last received message,
                     // not from the next one to protect against revision being split into multiple messages,

src/client/etcd_state_client.h

@@ -6,7 +6,7 @@
 #include <set>
 
 #include "json11/json11.hpp"
-#include "osd_id.h"
+#include "object_id.h"
 #include "timerfd_manager.h"
 
 #define ETCD_CONFIG_WATCH_ID 1

src/client/messenger.cpp

@@ -145,7 +145,7 @@ void osd_messenger_t::init()
         rdma_contexts = msgr_rdma_context_t::create_all(
             osd_num && osd_cluster_network_masks.size() ? osd_cluster_network_masks : osd_network_masks,
             rdma_device != "" ? rdma_device.c_str() : NULL,
-            rdma_port_num, rdma_gid_index, rdma_mtu, rdma_odp, log_level
+            rdma_port_num, rdma_gid_index, rdma_mtu, log_level
         );
         if (!rdma_contexts.size())
         {
@@ -182,12 +182,13 @@ void osd_messenger_t::init()
     }
     keepalive_timer_id = tfd->set_timer(1000, true, [this](int)
     {
+        std::vector<uint64_t> clients_to_stop;
+        std::vector<osd_op_t*> ops_to_send;
         auto cl_it = clients.begin();
         while (cl_it != clients.end())
         {
             auto cl = cl_it->second;
             cl_it++;
-            auto peer_fd = cl->peer_fd;
             if (!cl->osd_num && !cl->in_osd_num || cl->peer_state != PEER_CONNECTED && cl->peer_state != PEER_RDMA)
             {
                 // Do not run keepalive on regular clients
@@ -199,10 +200,9 @@ void osd_messenger_t::init()
                 if (!cl->ping_time_remaining)
                 {
                     // Ping timed out, stop the client
-                    fprintf(stderr, "Ping timed out for OSD %ju (client %d), disconnecting peer\n", cl->in_osd_num ? cl->in_osd_num : cl->osd_num, cl->peer_fd);
-                    stop_client(peer_fd, true);
-                    // Restart iterator because it may be invalidated
-                    cl_it = clients.upper_bound(peer_fd);
+                    fprintf(stderr, "Ping timed out for OSD %ju (client %ju), disconnecting peer\n",
+                        cl->in_osd_num ? cl->in_osd_num : cl->osd_num, cl->client_id);
+                    clients_to_stop.push_back(cl->client_id);
                 }
             }
             else if (cl->idle_time_remaining > 0)
@@ -213,7 +213,7 @@ void osd_messenger_t::init()
                     // Connection is idle for <osd_idle_time>, send ping
                     osd_op_t *op = new osd_op_t();
                     op->op_type = OSD_OP_OUT;
-                    op->peer_fd = cl->peer_fd;
+                    op->client_id = cl->client_id;
                     op->req = (osd_any_op_t){
                         .hdr = {
                             .magic = SECONDARY_OSD_OP_MAGIC,
@@ -222,28 +222,26 @@ void osd_messenger_t::init()
                     };
                     op->callback = [this, cl](osd_op_t *op)
                     {
-                        auto cl_it = clients.find(op->peer_fd);
-                        if (cl_it == clients.end() || cl_it->second != cl)
+                        auto cl_it = clients.find(op->client_id);
+                        if (cl_it == clients.end())
                         {
                             // client is already dropped
                             delete op;
                             return;
                         }
-                        int fail_fd = (op->reply.hdr.retval != 0 ? op->peer_fd : -1);
+                        uint64_t fail_client_id = (op->reply.hdr.retval != 0 ? op->client_id : 0);
                         auto fail_osd_num = cl->in_osd_num ? cl->in_osd_num : cl->osd_num;
                         cl->ping_time_remaining = 0;
                         delete op;
-                        if (fail_fd >= 0)
+                        if (fail_client_id)
                         {
-                            fprintf(stderr, "Ping failed for OSD %ju (client %d), disconnecting peer\n", fail_osd_num, fail_fd);
-                            stop_client(fail_fd, true);
+                            fprintf(stderr, "Ping failed for OSD %ju (client %ju), disconnecting peer\n", fail_osd_num, fail_client_id);
+                            stop_client(fail_client_id);
                         }
                     };
                     cl->ping_time_remaining = osd_ping_timeout;
                     cl->idle_time_remaining = osd_idle_timeout;
-                    outbox_push(op);
-                    // Restart iterator because it may be invalidated
-                    cl_it = clients.upper_bound(peer_fd);
+                    ops_to_send.push_back(op);
                 }
             }
             else
@@ -251,6 +249,14 @@ void osd_messenger_t::init()
                 cl->idle_time_remaining = osd_idle_timeout;
             }
         }
+        for (uint64_t client_id: clients_to_stop)
+        {
+            stop_client(client_id);
+        }
+        for (osd_op_t *op: ops_to_send)
+        {
+            outbox_push(op);
+        }
     });
 }
 
@@ -263,7 +269,7 @@ osd_messenger_t::~osd_messenger_t()
     }
     while (clients.size() > 0)
     {
-        stop_client(clients.begin()->first, true, true);
+        stop_client(clients.begin()->first, true);
     }
     if (iothreads.size())
     {
@@ -322,7 +328,6 @@ void osd_messenger_t::parse_config(const json11::Json & config)
     this->rdma_max_msg = config["rdma_max_msg"].uint64_value();
     if (!this->rdma_max_msg || this->rdma_max_msg > 128*1024*1024)
         this->rdma_max_msg = 129*1024;
-    this->rdma_odp = config["rdma_odp"].bool_value();
 #endif
     if (!osd_num)
         this->iothread_count = (uint32_t)config["client_iothread_count"].uint64_value();
@@ -441,7 +446,7 @@ void osd_messenger_t::try_connect_peer(uint64_t peer_osd)
     {
         return;
     }
-    if (osd_peer_fds.find(peer_osd) != osd_peer_fds.end())
+    if (osd_peers.find(peer_osd) != osd_peers.end())
     {
         wanted_peers.erase(peer_osd);
         return;
@@ -468,20 +473,20 @@ void osd_messenger_t::try_connect_peer_tcp(osd_num_t peer_osd, const char *peer_
 #ifdef WITH_RDMACM
     if (disable_tcp)
     {
-        on_connect_peer(peer_osd, -EINVAL);
+        on_connect_peer(peer_osd, -EINVAL, 0);
         return;
     }
 #endif
     struct sockaddr_storage addr;
     if (!string_to_addr(peer_host, 0, peer_port, &addr))
     {
-        on_connect_peer(peer_osd, -EINVAL);
+        on_connect_peer(peer_osd, -EINVAL, 0);
         return;
     }
     int peer_fd = socket(addr.ss_family, SOCK_STREAM, 0);
     if (peer_fd < 0)
     {
-        on_connect_peer(peer_osd, -errno);
+        on_connect_peer(peer_osd, -errno, 0);
         return;
     }
     fcntl(peer_fd, F_SETFL, fcntl(peer_fd, F_GETFL, 0) | O_NONBLOCK);
@@ -489,21 +494,25 @@ void osd_messenger_t::try_connect_peer_tcp(osd_num_t peer_osd, const char *peer_
     if (r < 0 && errno != EINPROGRESS)
     {
         close(peer_fd);
-        on_connect_peer(peer_osd, -errno);
+        on_connect_peer(peer_osd, -errno, 0);
         return;
     }
-    clients[peer_fd] = new osd_client_t();
+    const uint64_t client_id = next_client_id++;
+    osd_client_t *cl = new osd_client_t();
     if (log_level > 0)
     {
-        fprintf(stderr, "Connecting to OSD %ju at %s:%d (client %d)\n", peer_osd, peer_host, peer_port, peer_fd);
+        fprintf(stderr, "Connecting to OSD %ju at %s:%d (client %ju, FD %d)\n", peer_osd, peer_host, peer_port, client_id, peer_fd);
     }
-    clients[peer_fd]->peer_addr = addr;
-    clients[peer_fd]->peer_port = peer_port;
-    clients[peer_fd]->peer_fd = peer_fd;
-    clients[peer_fd]->peer_state = PEER_CONNECTING;
-    clients[peer_fd]->connect_timeout_id = -1;
-    clients[peer_fd]->osd_num = peer_osd;
-    clients[peer_fd]->in_buf = malloc_or_die(receive_buffer_size);
+    cl->client_id = client_id;
+    cl->peer_addr = addr;
+    cl->peer_port = peer_port;
+    cl->peer_fd = peer_fd;
+    cl->peer_state = PEER_CONNECTING;
+    cl->connect_timeout_id = -1;
+    cl->osd_num = peer_osd;
+    cl->in_buf = malloc_or_die(receive_buffer_size);
+    clients[client_id] = cl;
+    clients_by_fd[peer_fd] = cl;
     tfd->set_fd_handler(peer_fd, true, [this](int peer_fd, int epoll_events)
     {
         // Either OUT (connected) or HUP
@@ -511,11 +520,11 @@ void osd_messenger_t::try_connect_peer_tcp(osd_num_t peer_osd, const char *peer_
     });
     if (peer_connect_timeout > 0)
     {
-        clients[peer_fd]->connect_timeout_id = tfd->set_timer(1000*peer_connect_timeout, false, [this, peer_fd](int timer_id)
+        cl->connect_timeout_id = tfd->set_timer(1000*peer_connect_timeout, false, [this, client_id](int timer_id)
         {
-            osd_num_t peer_osd = clients.at(peer_fd)->osd_num;
-            stop_client(peer_fd, true);
-            on_connect_peer(peer_osd, -EPIPE);
+            osd_num_t peer_osd = clients.at(client_id)->osd_num;
+            stop_client(client_id);
+            on_connect_peer(peer_osd, -EPIPE, 0);
             return;
         });
     }
@@ -523,7 +532,7 @@ void osd_messenger_t::try_connect_peer_tcp(osd_num_t peer_osd, const char *peer_
 
 void osd_messenger_t::handle_connect_epoll(int peer_fd)
 {
-    auto cl = clients[peer_fd];
+    auto cl = clients_by_fd.at(peer_fd);
     if (cl->connect_timeout_id >= 0)
     {
         tfd->clear_timer(cl->connect_timeout_id);
@@ -538,8 +547,8 @@ void osd_messenger_t::handle_connect_epoll(int peer_fd)
     }
     if (result != 0)
     {
-        stop_client(peer_fd, true);
-        on_connect_peer(peer_osd, -result);
+        stop_client(cl->client_id);
+        on_connect_peer(peer_osd, -result, 0);
         return;
     }
     int one = 1;
@@ -556,23 +565,23 @@ void osd_messenger_t::handle_connect_epoll(int peer_fd)
 void osd_messenger_t::handle_peer_epoll(int peer_fd, int epoll_events)
 {
     // Mark client as ready (i.e. some data is available)
+    auto cl = clients_by_fd.at(peer_fd);
     if (epoll_events & EPOLLRDHUP)
     {
         // Stop client
         if (log_level > 0)
         {
-            fprintf(stderr, "[OSD %ju] client %d disconnected\n", this->osd_num, peer_fd);
+            fprintf(stderr, "[OSD %ju] client %ju disconnected\n", this->osd_num, cl->client_id);
         }
-        stop_client(peer_fd, true);
+        stop_client(cl->client_id);
     }
     else if (epoll_events & EPOLLIN)
     {
         // Mark client as ready (i.e. some data is available)
-        auto cl = clients[peer_fd];
         cl->read_ready++;
         if (cl->read_ready == 1)
         {
-            read_ready_clients.push_back(cl->peer_fd);
+            read_ready_clients.push_back(cl->client_id);
             if (ringloop)
                 ringloop->wakeup();
             else
@@ -581,13 +590,13 @@ void osd_messenger_t::handle_peer_epoll(int peer_fd, int epoll_events)
     }
 }
 
-void osd_messenger_t::on_connect_peer(osd_num_t peer_osd, int peer_fd)
+void osd_messenger_t::on_connect_peer(osd_num_t peer_osd, int errcode, uint64_t client_id)
 {
     auto & wp = wanted_peers.at(peer_osd);
     wp.connecting = false;
-    if (peer_fd < 0)
+    if (errcode < 0)
     {
-        fprintf(stderr, "Failed to connect to peer OSD %ju address %s port %d: %s\n", peer_osd, wp.cur_addr.c_str(), wp.cur_port, strerror(-peer_fd));
+        fprintf(stderr, "Failed to connect to peer OSD %ju address %s port %d: %s\n", peer_osd, wp.cur_addr.c_str(), wp.cur_port, strerror(-errcode));
         if (wp.address_changed)
         {
             wp.address_changed = false;
@@ -614,7 +623,7 @@ void osd_messenger_t::on_connect_peer(osd_num_t peer_osd, int peer_fd)
     }
     if (log_level > 0)
     {
-        fprintf(stderr, "[OSD %ju] Connected with peer OSD %ju (client %d)\n", osd_num, peer_osd, peer_fd);
+        fprintf(stderr, "[OSD %ju] Connected with peer OSD %ju (client %ju)\n", osd_num, peer_osd, client_id);
     }
     wanted_peers.erase(peer_osd);
     repeer_pgs(peer_osd);
@@ -624,7 +633,7 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl)
 {
     osd_op_t *op = new osd_op_t();
     op->op_type = OSD_OP_OUT;
-    op->peer_fd = cl->peer_fd;
+    op->client_id = cl->client_id;
     op->req = (osd_any_op_t){
         .show_conf = {
             .header = {
@@ -648,7 +657,7 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl)
         if (!selected_ctx)
         {
             if (log_level > 0)
-                fprintf(stderr, "No RDMA context for OSD %ju connection (peer %d), using only TCP\n", cl->osd_num, cl->peer_fd);
+                fprintf(stderr, "No RDMA context for OSD %ju connection (client %ju), using only TCP\n", cl->osd_num, cl->client_id);
         }
         else
         {
@@ -709,8 +718,8 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl)
         if (err)
         {
             osd_num_t peer_osd = cl->osd_num;
-            stop_client(op->peer_fd);
-            on_connect_peer(peer_osd, -EINVAL);
+            stop_client(op->client_id);
+            on_connect_peer(peer_osd, -EINVAL, 0);
             delete op;
             return;
         }
@@ -740,21 +749,13 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl)
                     fprintf(stderr, "Connected to OSD %ju using RDMA\n", cl->osd_num);
                 }
                 cl->peer_state = PEER_RDMA;
-                tfd->set_fd_handler(cl->peer_fd, false, [this](int peer_fd, int epoll_events)
-                {
-                    // Do not miss the disconnection!
-                    if (epoll_events & EPOLLRDHUP)
-                    {
-                        handle_peer_epoll(peer_fd, epoll_events);
-                    }
-                });
                 // Add the initial receive request
                 init_recv_rdma(cl);
             }
         }
 #endif
-        osd_peer_fds[cl->osd_num] = cl->peer_fd;
-        on_connect_peer(cl->osd_num, cl->peer_fd);
+        osd_peers[cl->osd_num] = cl;
+        on_connect_peer(cl->osd_num, 0, cl->client_id);
         delete op;
     };
     outbox_push(op);
@@ -769,13 +770,16 @@ void osd_messenger_t::accept_connections(int listen_fd)
     while ((peer_fd = accept(listen_fd, (sockaddr*)&addr, &peer_addr_size)) >= 0)
     {
         assert(peer_fd != 0);
-        fprintf(stderr, "[OSD %ju] new client %d: connection from %s\n", this->osd_num, peer_fd,
+        const uint64_t client_id = next_client_id++;
+        fprintf(stderr, "[OSD %ju] new client %ju (FD %d): connection from %s\n", this->osd_num, client_id, peer_fd,
             addr_to_string(addr).c_str());
         fcntl(peer_fd, F_SETFL, fcntl(peer_fd, F_GETFL, 0) | O_NONBLOCK);
         int one = 1;
         setsockopt(peer_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
         auto cl = new osd_client_t();
-        clients[peer_fd] = cl;
+        cl->client_id = client_id;
+        clients[cl->client_id] = cl;
+        clients_by_fd[peer_fd] = cl;
         cl->is_incoming = true;
         cl->peer_addr = addr;
         cl->peer_addr = addr;

src/client/messenger.h

@@ -12,6 +12,7 @@
 #include <deque>
 #include <vector>
 
+#include "../util/robin_hood.h"
 #include "malloc_or_die.h"
 #include "json11/json11.hpp"
 #include "msgr_op.h"
@@ -50,6 +51,7 @@ struct msgr_rdma_context_t;
 
 struct osd_client_t
 {
+    uint64_t client_id = 0;
     int refs = 0;
 
     sockaddr_storage peer_addr = {};
@@ -85,7 +87,7 @@ struct osd_client_t
     std::vector<osd_op_t*> received_ops;
 
     // Outbound operations
-    std::map<uint64_t, osd_op_t*> sent_ops;
+    robin_hood::unordered_flat_map<uint64_t, osd_op_t*> sent_ops;
     uint64_t send_op_id = 0;
 
     // PGs dirtied by this client's primary-writes
@@ -200,27 +202,28 @@ protected:
     std::vector<msgr_rdma_context_t *> rdma_contexts;
     uint64_t rdma_max_sge = 0, rdma_max_send = 0, rdma_max_recv = 0;
     uint64_t rdma_max_msg = 0;
-    bool rdma_odp = false;
     rdma_event_channel *rdmacm_evch = NULL;
-    std::map<rdma_cm_id*, osd_client_t*> rdmacm_connections;
-    std::map<rdma_cm_id*, rdmacm_connecting_t*> rdmacm_connecting;
+    robin_hood::unordered_flat_map<rdma_cm_id*, osd_client_t*> rdmacm_connections;
+    robin_hood::unordered_flat_map<rdma_cm_id*, rdmacm_connecting_t*> rdmacm_connecting;
 #endif
 
     std::vector<msgr_iothread_t*> iothreads;
-    std::vector<int> read_ready_clients;
-    std::vector<int> write_ready_clients;
+    std::vector<uint64_t> read_ready_clients;
+    std::vector<uint64_t> write_ready_clients;
     // We don't use ringloop->set_immediate here because we may have no ringloop in client :)
-    std::vector<osd_op_t*> set_immediate_ops;
+    std::deque<osd_op_t*> set_immediate_ops;
 
 public:
     timerfd_manager_t *tfd = NULL;
     ring_loop_t *ringloop = NULL;
     bool has_sendmsg_zc = false;
     // osd_num_t is only for logging and asserts
+    uint64_t next_client_id = 1;
     osd_num_t osd_num;
-    std::map<int, osd_client_t*> clients;
-    std::map<osd_num_t, osd_wanted_peer_t> wanted_peers;
-    std::map<uint64_t, int> osd_peer_fds;
+    robin_hood::unordered_flat_map<uint64_t, osd_client_t*> clients;
+    robin_hood::unordered_flat_map<uint64_t, osd_client_t*> osd_peers;
+    robin_hood::unordered_flat_map<int, osd_client_t*> clients_by_fd;
+    robin_hood::unordered_flat_map<osd_num_t, osd_wanted_peer_t> wanted_peers;
     std::vector<std::string> osd_networks;
     std::vector<addr_mask_t> osd_network_masks;
     std::vector<std::string> osd_cluster_networks;
@@ -233,7 +236,8 @@ public:
     void init();
     void parse_config(const json11::Json & config);
     void connect_peer(uint64_t osd_num, json11::Json peer_state);
-    void stop_client(int peer_fd, bool force = false, bool force_delete = false);
+    void stop_client(uint64_t client_id, bool force_delete = false);
+    void destroy_client(osd_client_t *cl);
     void outbox_push(osd_op_t *cur_op);
     std::function<void(osd_op_t*)> exec_op;
     std::function<void(osd_num_t)> repeer_pgs;
@@ -252,7 +256,7 @@ public:
 
 #ifdef WITH_RDMA
     bool is_rdma_enabled();
-    bool connect_rdma(int peer_fd, std::string rdma_address, uint64_t client_max_msg);
+    bool connect_rdma(uint64_t client_id, std::string rdma_address, uint64_t client_max_msg);
 #endif
 #ifdef WITH_RDMACM
     bool is_use_rdmacm();
@@ -268,7 +272,7 @@ protected:
     void try_connect_peer_tcp(osd_num_t peer_osd, const char *peer_host, int peer_port);
     void handle_peer_epoll(int peer_fd, int epoll_events);
     void handle_connect_epoll(int peer_fd);
-    void on_connect_peer(osd_num_t peer_osd, int peer_fd);
+    void on_connect_peer(osd_num_t peer_osd, int errcode, uint64_t client_id);
     void check_peer_config(osd_client_t *cl);
     void cancel_osd_ops(osd_client_t *cl);
     void cancel_op(osd_op_t *op);
@@ -283,12 +287,9 @@ protected:
     bool handle_reply_hdr(osd_client_t *cl);
     void handle_reply_ready(osd_op_t *op);
     void handle_immediate_ops();
-    void clear_immediate_ops(int peer_fd);
 
 #ifdef WITH_RDMA
     void try_send_rdma(osd_client_t *cl);
-    void try_send_rdma_odp(osd_client_t *cl);
-    void try_send_rdma_nodp(osd_client_t *cl);
     bool init_recv_rdma(osd_client_t *cl);
     void handle_rdma_events(msgr_rdma_context_t *rdma_context);
     msgr_rdma_context_t* choose_rdma_context(osd_client_t *cl);

src/client/msgr_op.h

@@ -156,7 +156,7 @@ struct __attribute__((visibility("default"))) osd_op_t
 {
     timespec tv_begin = { 0 }, tv_end = { 0 };
     uint64_t op_type = OSD_OP_IN;
-    int peer_fd;
+    uint64_t client_id = 0;
     osd_any_op_t req;
     osd_any_reply_t reply;
     blockstore_op_t *bs_op = NULL;

src/client/msgr_rdma.cpp

@@ -59,8 +59,6 @@ msgr_rdma_context_t::~msgr_rdma_context_t()
         ibv_destroy_cq(cq);
     if (channel)
         ibv_destroy_comp_channel(channel);
-    if (mr)
-        ibv_dereg_mr(mr);
     if (pd)
         ibv_dealloc_pd(pd);
     if (context && !is_cm)
@@ -182,13 +180,15 @@ static int match_port_gid(const std::vector<addr_mask_t> & osd_network_masks, ib
 #endif
 
 std::vector<msgr_rdma_context_t*> msgr_rdma_context_t::create_all(const std::vector<addr_mask_t> & osd_network_masks,
-    const char *sel_dev_name, int sel_port_num, int sel_gid_index, uint32_t sel_mtu, bool odp, int log_level)
+    const char *sel_dev_name, int sel_port_num, int sel_gid_index, uint32_t sel_mtu, int log_level)
 {
     int res;
     std::vector<msgr_rdma_context_t*> ret;
     ibv_device **raw_dev_list = NULL;
     ibv_device **dev_list = NULL;
     ibv_device *single_list[2] = {};
+    int up_ports = 0;
+    int single_port_num = 0;
 
     raw_dev_list = dev_list = ibv_get_device_list(NULL);
     if (!dev_list || !*dev_list)
@@ -223,6 +223,7 @@ std::vector<msgr_rdma_context_t*> msgr_rdma_context_t::create_all(const std::vec
         dev_list = single_list;
     }
 
+retry:
     for (int i = 0; dev_list[i]; ++i)
     {
         auto dev = dev_list[i];
@@ -260,6 +261,9 @@ std::vector<msgr_rdma_context_t*> msgr_rdma_context_t::create_all(const std::vec
                 fprintf(stderr, "RDMA device %s port %d GID %d does not exist\n", ibv_get_device_name(dev), port_num, sel_gid_index);
                 continue;
             }
+            up_ports++;
+            single_port_num = port_num;
+            single_list[0] = dev;
             uint32_t port_mtu = sel_mtu ? sel_mtu : ibv_mtu_to_bytes(portinfo.active_mtu);
 #ifdef IBV_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT
             if (sel_gid_index < 0)
@@ -271,7 +275,7 @@ std::vector<msgr_rdma_context_t*> msgr_rdma_context_t::create_all(const std::vec
                 {
                     if (log_level > 0)
                         log_rdma_dev_port_gid(dev, port_num, best_gid_idx, port_mtu, best_gidx);
-                    auto ctx = msgr_rdma_context_t::create(dev, portinfo, port_num, best_gid_idx, port_mtu, odp, log_level);
+                    auto ctx = msgr_rdma_context_t::create(dev, portinfo, port_num, best_gid_idx, port_mtu, log_level);
                     if (ctx)
                     {
                         ctx->net_mask = osd_network_masks[net_num];
@@ -291,7 +295,7 @@ std::vector<msgr_rdma_context_t*> msgr_rdma_context_t::create_all(const std::vec
                     log_rdma_dev_port_gid(dev, port_num, best_gid_idx, port_mtu, gidx);
                 }
 #endif
-                auto ctx = msgr_rdma_context_t::create(dev, portinfo, port_num, best_gid_idx, port_mtu, odp, log_level);
+                auto ctx = msgr_rdma_context_t::create(dev, portinfo, port_num, best_gid_idx, port_mtu, log_level);
                 if (ctx)
                     ret.push_back(ctx);
             }
@@ -300,13 +304,21 @@ cleanup_dev:
         ibv_close_device(context);
     }
 
+    if (!ret.size() && up_ports == 1 && dev_list != single_list)
+    {
+        // Auto-select the only available device/port if there is only one
+        dev_list = single_list;
+        sel_port_num = single_port_num;
+        goto retry;
+    }
+
 cleanup:
     if (raw_dev_list)
         ibv_free_device_list(raw_dev_list);
     return ret;
 }
 
-msgr_rdma_context_t *msgr_rdma_context_t::create(ibv_device *dev, ibv_port_attr & portinfo, int ib_port, int gid_index, uint32_t mtu, bool odp, int log_level)
+msgr_rdma_context_t *msgr_rdma_context_t::create(ibv_device *dev, ibv_port_attr & portinfo, int ib_port, int gid_index, uint32_t mtu, int log_level)
 {
     msgr_rdma_context_t *ctx = new msgr_rdma_context_t();
     ibv_context *context = ibv_open_device(dev);
@@ -346,30 +358,6 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(ibv_device *dev, ibv_port_attr
         goto cleanup;
     }
 
-    ctx->odp = odp;
-    if (ctx->odp)
-    {
-        if (!(ctx->attrx.odp_caps.general_caps & IBV_ODP_SUPPORT) ||
-            !(ctx->attrx.odp_caps.general_caps & IBV_ODP_SUPPORT_IMPLICIT) ||
-            !(ctx->attrx.odp_caps.per_transport_caps.rc_odp_caps & IBV_ODP_SUPPORT_SEND) ||
-            !(ctx->attrx.odp_caps.per_transport_caps.rc_odp_caps & IBV_ODP_SUPPORT_RECV))
-        {
-            ctx->odp = false;
-            if (log_level > 0)
-                fprintf(stderr, "The RDMA device isn't implicit ODP (On-Demand Paging) capable, disabling it\n");
-        }
-    }
-
-    if (ctx->odp)
-    {
-        ctx->mr = ibv_reg_mr(ctx->pd, NULL, SIZE_MAX, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_ON_DEMAND);
-        if (!ctx->mr)
-        {
-            fprintf(stderr, "Couldn't register RDMA memory region\n");
-            goto cleanup;
-        }
-    }
-
     ctx->channel = ibv_create_comp_channel(ctx->context);
     if (!ctx->channel)
     {
@@ -388,8 +376,6 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(ibv_device *dev, ibv_port_attr
     return ctx;
 
 cleanup:
-    if (context)
-        ibv_close_device(context);
     delete ctx;
     return NULL;
 }
@@ -521,7 +507,7 @@ int msgr_rdma_connection_t::connect(msgr_rdma_address_t *dest)
     return 0;
 }
 
-bool osd_messenger_t::connect_rdma(int peer_fd, std::string rdma_address, uint64_t client_max_msg)
+bool osd_messenger_t::connect_rdma(uint64_t client_id, std::string rdma_address, uint64_t client_max_msg)
 {
     // Try to connect to the peer using RDMA
     msgr_rdma_address_t addr;
@@ -531,12 +517,12 @@ bool osd_messenger_t::connect_rdma(int peer_fd, std::string rdma_address, uint64
         {
             client_max_msg = rdma_max_msg;
         }
-        auto cl = clients.at(peer_fd);
+        auto cl = clients.at(client_id);
         msgr_rdma_context_t *selected_ctx = choose_rdma_context(cl);
         if (!selected_ctx)
         {
             if (log_level > 0)
-                fprintf(stderr, "No RDMA context for peer %d, using only TCP\n", cl->peer_fd);
+                fprintf(stderr, "No RDMA context for peer %ju, using only TCP\n", client_id);
             return false;
         }
         msgr_rdma_connection_t *rdma_conn = msgr_rdma_connection_t::create(selected_ctx, rdma_max_send, rdma_max_recv, rdma_max_sge, client_max_msg);
@@ -547,14 +533,13 @@ bool osd_messenger_t::connect_rdma(int peer_fd, std::string rdma_address, uint64
             {
                 delete rdma_conn;
                 fprintf(
-                    stderr, "Failed to connect RDMA queue pair to %s (client %d)\n",
-                    addr.to_string().c_str(), peer_fd
+                    stderr, "Failed to connect RDMA queue pair to %s (client %ju)\n",
+                    addr.to_string().c_str(), client_id
                 );
             }
             else
             {
                 // Remember connection, but switch to RDMA only after sending the configuration response
-                auto cl = clients.at(peer_fd);
                 cl->rdma_conn = rdma_conn;
                 cl->peer_state = PEER_RDMA_CONNECTING;
                 return true;
@@ -568,7 +553,7 @@ static void try_send_rdma_wr(osd_client_t *cl, ibv_sge *sge, int op_sge)
 {
     ibv_send_wr *bad_wr = NULL;
     ibv_send_wr wr = {
-        .wr_id = (uint64_t)(cl->peer_fd*2+1),
+        .wr_id = cl->client_id,
         .sg_list = sge,
         .num_sge = op_sge,
         .opcode = IBV_WR_SEND,
@@ -605,52 +590,7 @@ static int try_send_rdma_copy(osd_client_t *cl, uint8_t *dst, int dst_len)
     return total_dst_len-dst_len;
 }
 
-void osd_messenger_t::try_send_rdma_odp(osd_client_t *cl)
-{
-    auto rc = cl->rdma_conn;
-    if (!cl->send_list.size() || rc->cur_send >= rc->max_send)
-    {
-        return;
-    }
-    uint64_t op_size = 0, op_sge = 0;
-    ibv_sge sge[rc->max_sge];
-    while (rc->send_pos < cl->send_list.size())
-    {
-        iovec & iov = cl->send_list[rc->send_pos];
-        if (op_size >= rc->max_msg || op_sge >= rc->max_sge)
-        {
-            rc->send_sizes.push_back(op_size);
-            try_send_rdma_wr(cl, sge, op_sge);
-            op_sge = 0;
-            op_size = 0;
-            if (rc->cur_send >= rc->max_send)
-            {
-                break;
-            }
-        }
-        uint32_t len = (uint32_t)(op_size+iov.iov_len-rc->send_buf_pos < rc->max_msg
-            ? iov.iov_len-rc->send_buf_pos : rc->max_msg-op_size);
-        sge[op_sge++] = {
-            .addr = (uintptr_t)((uint8_t*)iov.iov_base+rc->send_buf_pos),
-            .length = len,
-            .lkey = rc->ctx->mr->lkey,
-        };
-        op_size += len;
-        rc->send_buf_pos += len;
-        if (rc->send_buf_pos >= iov.iov_len)
-        {
-            rc->send_pos++;
-            rc->send_buf_pos = 0;
-        }
-    }
-    if (op_sge > 0)
-    {
-        rc->send_sizes.push_back(op_size);
-        try_send_rdma_wr(cl, sge, op_sge);
-    }
-}
-
-void osd_messenger_t::try_send_rdma_nodp(osd_client_t *cl)
+void osd_messenger_t::try_send_rdma(osd_client_t *cl)
 {
     auto rc = cl->rdma_conn;
     if (!rc->send_out_size)
@@ -658,14 +598,11 @@ void osd_messenger_t::try_send_rdma_nodp(osd_client_t *cl)
         // Allocate send ring buffer, if not yet
         rc->send_out_size = rc->max_msg*rdma_max_send;
         rc->send_out.buf = (uint8_t*)malloc_or_die(rc->send_out_size);
-        if (!rc->ctx->odp)
+        rc->send_out.mr = ibv_reg_mr(rc->ctx->pd, rc->send_out.buf, rc->send_out_size, 0);
+        if (!rc->send_out.mr)
         {
-            rc->send_out.mr = ibv_reg_mr(rc->ctx->pd, rc->send_out.buf, rc->send_out_size, 0);
-            if (!rc->send_out.mr)
-            {
-                fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
-                exit(1);
-            }
+            fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
+            exit(1);
         }
     }
     // Copy data into the buffer and send it
@@ -690,7 +627,7 @@ void osd_messenger_t::try_send_rdma_nodp(osd_client_t *cl)
             ibv_sge sge = {
                 .addr = (uintptr_t)dst,
                 .length = (uint32_t)copied,
-                .lkey = rc->ctx->odp ? rc->ctx->mr->lkey : rc->send_out.mr->lkey,
+                .lkey = rc->send_out.mr->lkey,
             };
             try_send_rdma_wr(cl, &sge, 1);
             rc->send_sizes.push_back(copied);
@@ -698,24 +635,16 @@ void osd_messenger_t::try_send_rdma_nodp(osd_client_t *cl)
     }
 }
 
-void osd_messenger_t::try_send_rdma(osd_client_t *cl)
-{
-    if (cl->rdma_conn->ctx->odp)
-        try_send_rdma_odp(cl);
-    else
-        try_send_rdma_nodp(cl);
-}
-
 static void try_recv_rdma_wr(osd_client_t *cl, void *buf)
 {
     ibv_sge sge = {
         .addr = (uintptr_t)buf,
         .length = (uint32_t)cl->rdma_conn->max_msg,
-        .lkey = cl->rdma_conn->ctx->odp ? cl->rdma_conn->ctx->mr->lkey : cl->rdma_conn->recv_buf.mr->lkey,
+        .lkey = cl->rdma_conn->recv_buf.mr->lkey,
     };
     ibv_recv_wr *bad_wr = NULL;
     ibv_recv_wr wr = {
-        .wr_id = (uint64_t)(cl->peer_fd*2),
+        .wr_id = cl->client_id,
         .sg_list = &sge,
         .num_sge = 1,
     };
@@ -733,14 +662,11 @@ bool osd_messenger_t::init_recv_rdma(osd_client_t *cl)
     auto rc = cl->rdma_conn;
     assert(!rc->recv_buf.buf);
     rc->recv_buf.buf = (uint8_t*)malloc_or_die(rc->max_msg * rc->max_recv);
-    if (!rc->ctx->odp)
+    rc->recv_buf.mr = ibv_reg_mr(rc->ctx->pd, rc->recv_buf.buf, rc->max_msg * rc->max_recv, IBV_ACCESS_LOCAL_WRITE);
+    if (!rc->recv_buf.mr)
     {
-        rc->recv_buf.mr = ibv_reg_mr(rc->ctx->pd, rc->recv_buf.buf, rc->max_msg * rc->max_recv, IBV_ACCESS_LOCAL_WRITE);
-        if (!rc->recv_buf.mr)
-        {
-            fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
-            exit(1);
-        }
+        fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
+        exit(1);
     }
     for (uint32_t i = 0; i < rc->max_recv; i++)
     {
@@ -775,25 +701,28 @@ void osd_messenger_t::handle_rdma_events(msgr_rdma_context_t *rdma_context)
         event_count = ibv_poll_cq(rdma_context->cq, RDMA_EVENTS_AT_ONCE, wc);
         for (int i = 0; i < event_count; i++)
         {
-            int client_id = wc[i].wr_id >> 1;
-            bool is_send = wc[i].wr_id & 1;
+            uint64_t client_id = wc[i].wr_id;
+            bool is_send = wc[i].opcode == IBV_WC_SEND;
             auto cl_it = clients.find(client_id);
             if (cl_it == clients.end())
             {
                 continue;
             }
             osd_client_t *cl = cl_it->second;
+            if (cl->peer_state == PEER_STOPPED)
+            {
+                continue;
+            }
             auto rc = cl->rdma_conn;
             if (wc[i].status != IBV_WC_SUCCESS)
             {
-                fprintf(stderr, "RDMA work request failed for client %d", client_id);
+                fprintf(stderr, "RDMA work request failed for client %ju", client_id);
                 if (cl->osd_num)
                 {
                     fprintf(stderr, " (OSD %ju)", cl->osd_num);
                 }
                 fprintf(stderr, " with status: %s, stopping client\n", ibv_wc_status_str(wc[i].status));
                 stop_client(client_id);
-                clear_immediate_ops(client_id);
                 continue;
             }
             if (!is_send)
@@ -804,8 +733,6 @@ void osd_messenger_t::handle_rdma_events(msgr_rdma_context_t *rdma_context)
                 rc->cur_recv--;
                 if (!handle_read_buffer(cl, rc->recv_buffers[rc->next_recv_buf], wc[i].byte_len))
                 {
-                    // handle_read_buffer may stop the client
-                    clear_immediate_ops(client_id);
                     continue;
                 }
                 try_recv_rdma_wr(cl, rc->recv_buffers[rc->next_recv_buf]);
@@ -816,14 +743,11 @@ void osd_messenger_t::handle_rdma_events(msgr_rdma_context_t *rdma_context)
                 rc->cur_send--;
                 uint64_t sent_size = rc->send_sizes.at(0);
                 rc->send_sizes.erase(rc->send_sizes.begin(), rc->send_sizes.begin()+1);
-                if (!rdma_context->odp)
-                {
-                    rc->send_done_pos += sent_size;
-                    rc->send_out_full = false;
-                    if (rc->send_done_pos == rc->send_out_size)
-                        rc->send_done_pos = 0;
-                    assert(rc->send_done_pos < rc->send_out_size);
-                }
+                rc->send_done_pos += sent_size;
+                rc->send_out_full = false;
+                if (rc->send_done_pos == rc->send_out_size)
+                    rc->send_done_pos = 0;
+                assert(rc->send_done_pos < rc->send_out_size);
                 int send_pos = 0, send_buf_pos = 0;
                 while (sent_size > 0)
                 {

src/client/msgr_rdma.h

@@ -26,8 +26,6 @@ struct msgr_rdma_context_t
     ibv_context *context = NULL;
     ibv_device_attr_ex attrx;
     ibv_pd *pd = NULL;
-    bool odp = false;
-    ibv_mr *mr = NULL;
     ibv_comp_channel *channel = NULL;
     ibv_cq *cq = NULL;
     ibv_port_attr portinfo;
@@ -43,9 +41,9 @@ struct msgr_rdma_context_t
     int cm_refs = 0;
 
     static std::vector<msgr_rdma_context_t*> create_all(const std::vector<addr_mask_t> & osd_network_masks,
-        const char *sel_dev_name, int sel_port_num, int sel_gid_index, uint32_t sel_mtu, bool odp, int log_level);
+        const char *sel_dev_name, int sel_port_num, int sel_gid_index, uint32_t sel_mtu, int log_level);
     static msgr_rdma_context_t *create(ibv_device *dev, ibv_port_attr & portinfo,
-        int ib_port, int gid_index, uint32_t mtu, bool odp, int log_level);
+        int ib_port, int gid_index, uint32_t mtu, int log_level);
     static msgr_rdma_context_t* create_cm(ibv_context *ctx);
     bool reserve_cqe(int n);
 

src/client/msgr_rdmacm.cpp

@@ -11,7 +11,7 @@
 struct rdmacm_connecting_t
 {
     rdma_cm_id *cmid = NULL;
-    int peer_fd = -1;
+    uint64_t client_id = 0;
     osd_num_t peer_osd = 0;
     std::string addr;
     sockaddr_storage parsed_addr = {};
@@ -117,9 +117,9 @@ void osd_messenger_t::handle_rdmacm_events()
             auto cli_it = rdmacm_connections.find(ev->id);
             if (cli_it != rdmacm_connections.end())
             {
-                fprintf(stderr, "Received %s event for peer %d, closing connection\n",
-                    event_type_name, cli_it->second->peer_fd);
-                stop_client(cli_it->second->peer_fd);
+                fprintf(stderr, "Received %s event for client %ju, closing connection\n",
+                    event_type_name, cli_it->second->client_id);
+                stop_client(cli_it->second->client_id);
             }
             else if (rdmacm_connecting.find(ev->id) != rdmacm_connecting.end())
             {
@@ -178,7 +178,6 @@ msgr_rdma_context_t* msgr_rdma_context_t::create_cm(ibv_context *ctx)
         delete rdma_context;
         return NULL;
     }
-    rdma_context->odp = false;
     rdma_context->channel = ibv_create_comp_channel(rdma_context->context);
     if (!rdma_context->channel)
     {
@@ -266,14 +265,6 @@ msgr_rdma_context_t* osd_messenger_t::rdmacm_create_qp(rdma_cm_id *cmid)
 
 void osd_messenger_t::rdmacm_accept(rdma_cm_event *ev)
 {
-    // Make a fake FD (FIXME: do not use FDs for identifying clients!)
-    int fake_fd = socket(AF_INET, SOCK_STREAM, 0);
-    if (fake_fd < 0)
-    {
-        fprintf(stderr, "Failed to allocate a fake socket for RDMA-CM client: %s (code %d)\n", strerror(errno), errno);
-        rdma_destroy_id(ev->id);
-        return;
-    }
     auto rdma_context = rdmacm_create_qp(ev->id);
     if (!rdma_context)
     {
@@ -298,12 +289,12 @@ void osd_messenger_t::rdmacm_accept(rdma_cm_event *ev)
     // Wait for RDMA_CM_ESTABLISHED, and enable the connection only after it
     auto conn = new rdmacm_connecting_t;
     conn->cmid = ev->id;
-    conn->peer_fd = fake_fd;
+    conn->client_id = next_client_id++;
     conn->parsed_addr = *(sockaddr_storage*)rdma_get_peer_addr(ev->id);
     conn->rdma_context = rdma_context;
     rdmacm_set_conn_timeout(conn);
     rdmacm_connecting[ev->id] = conn;
-    fprintf(stderr, "[OSD %ju] new client %d: connection from %s via RDMA-CM\n", this->osd_num, conn->peer_fd,
+    fprintf(stderr, "[OSD %ju] new client %ju: connection from %s via RDMA-CM\n", this->osd_num, conn->client_id,
         addr_to_string(conn->parsed_addr).c_str());
 }
 
@@ -333,8 +324,6 @@ void osd_messenger_t::rdmacm_on_connect_peer_error(rdma_cm_id *cmid, int res)
     auto peer_osd = conn->peer_osd;
     if (conn->timeout_id >= 0)
         tfd->clear_timer(conn->timeout_id);
-    if (conn->peer_fd >= 0)
-        close(conn->peer_fd);
     if (conn->rdma_context)
         conn->rdma_context->reserve_cqe(-rdma_max_send-rdma_max_recv);
     if (conn->cmid)
@@ -355,7 +344,7 @@ void osd_messenger_t::rdmacm_on_connect_peer_error(rdma_cm_id *cmid, int res)
         else
         {
             // TCP is disabled
-            on_connect_peer(peer_osd, res == 0 ? -EINVAL : (res > 0 ? -res : res));
+            on_connect_peer(peer_osd, res == 0 ? -EINVAL : (res > 0 ? -res : res), 0);
         }
     }
 }
@@ -366,7 +355,7 @@ void osd_messenger_t::rdmacm_try_connect_peer(uint64_t peer_osd, const std::stri
     if (!string_to_addr(addr, false, rdmacm_port, &sa))
     {
         fprintf(stderr, "Address %s is invalid\n", addr.c_str());
-        on_connect_peer(peer_osd, -EINVAL);
+        on_connect_peer(peer_osd, -EINVAL, 0);
         return;
     }
     rdma_cm_id *cmid = NULL;
@@ -377,17 +366,7 @@ void osd_messenger_t::rdmacm_try_connect_peer(uint64_t peer_osd, const std::stri
         if (!disable_tcp)
             try_connect_peer_tcp(peer_osd, addr.c_str(), fallback_tcp_port);
         else
-            on_connect_peer(peer_osd, res);
-        return;
-    }
-    // Make a fake FD (FIXME: do not use FDs for identifying clients!)
-    int fake_fd = socket(AF_INET, SOCK_STREAM, 0);
-    if (fake_fd < 0)
-    {
-        int res = -errno;
-        rdma_destroy_id(cmid);
-        // Can't create socket, pointless to try TCP
-        on_connect_peer(peer_osd, res);
+            on_connect_peer(peer_osd, res, 0);
         return;
     }
     if (log_level > 0)
@@ -395,7 +374,7 @@ void osd_messenger_t::rdmacm_try_connect_peer(uint64_t peer_osd, const std::stri
     auto conn = new rdmacm_connecting_t;
     rdmacm_connecting[cmid] = conn;
     conn->cmid = cmid;
-    conn->peer_fd = fake_fd;
+    conn->client_id = next_client_id++;
     conn->peer_osd = peer_osd;
     conn->addr = addr;
     conn->parsed_addr = sa;
@@ -512,13 +491,13 @@ void osd_messenger_t::rdmacm_established(rdma_cm_event *ev)
     auto cl = new osd_client_t();
     cl->peer_addr = conn->parsed_addr;
     cl->peer_port = conn->rdmacm_port;
-    cl->peer_fd = conn->peer_fd;
+    cl->client_id = conn->client_id;
     cl->peer_state = PEER_RDMA;
     cl->connect_timeout_id = -1;
     cl->osd_num = peer_osd;
     cl->in_buf = malloc_or_die(receive_buffer_size);
     cl->rdma_conn = rc;
-    clients[conn->peer_fd] = cl;
+    clients[conn->client_id] = cl;
     if (conn->timeout_id >= 0)
     {
         tfd->clear_timer(conn->timeout_id);

src/client/msgr_receive.cpp

@@ -7,9 +7,10 @@ void osd_messenger_t::read_requests()
 {
     for (int i = 0; i < read_ready_clients.size(); i++)
     {
-        int peer_fd = read_ready_clients[i];
-        auto cl_it = clients.find(peer_fd);
-        if (cl_it == clients.end() || !cl_it->second || cl_it->second->read_msg.msg_iovlen)
+        uint64_t client_id = read_ready_clients[i];
+        auto cl_it = clients.find(client_id);
+        if (cl_it == clients.end() || !cl_it->second || cl_it->second->read_msg.msg_iovlen ||
+            cl_it->second->peer_state != PEER_CONNECTED)
         {
             continue;
         }
@@ -31,7 +32,7 @@ void osd_messenger_t::read_requests()
         cl->refs++;
         if (ringloop && !use_sync_send_recv)
         {
-            auto iothread = iothreads.size() ? iothreads[peer_fd % iothreads.size()] : NULL;
+            auto iothread = iothreads.size() ? iothreads[cl->peer_fd % iothreads.size()] : NULL;
             io_uring_sqe sqe_local;
             ring_data_t data_local;
             io_uring_sqe* sqe = (iothread ? &sqe_local : ringloop->get_sqe());
@@ -49,7 +50,7 @@ void osd_messenger_t::read_requests()
             }
             ring_data_t* data = ((ring_data_t*)sqe->user_data);
             data->callback = [this, cl](ring_data_t *data) { handle_read(data->res, cl); };
-            io_uring_prep_recvmsg(sqe, peer_fd, &cl->read_msg, 0);
+            io_uring_prep_recvmsg(sqe, cl->peer_fd, &cl->read_msg, 0);
             if (iothread)
             {
                 iothread->add_sqe(sqe_local);
@@ -57,7 +58,7 @@ void osd_messenger_t::read_requests()
         }
         else
         {
-            int result = recvmsg(peer_fd, &cl->read_msg, 0);
+            int result = recvmsg(cl->peer_fd, &cl->read_msg, 0);
             if (result < 0)
             {
                 result = -errno;
@@ -72,14 +73,17 @@ void osd_messenger_t::read_requests()
 bool osd_messenger_t::handle_read(int result, osd_client_t *cl)
 {
     bool ret = false;
-    int peer_fd = cl->peer_fd;
     cl->read_msg.msg_iovlen = 0;
     cl->refs--;
+    if (cl->peer_state == PEER_RDMA)
+    {
+        return true;
+    }
     if (cl->peer_state == PEER_STOPPED)
     {
         if (cl->refs <= 0)
         {
-            delete cl;
+            destroy_client(cl);
         }
         return false;
     }
@@ -88,20 +92,20 @@ bool osd_messenger_t::handle_read(int result, osd_client_t *cl)
         // this is a client socket, so don't panic on error. just disconnect it
         if (result != 0)
         {
-            fprintf(stderr, "Client %d socket read error: %d (%s). Disconnecting client\n", cl->peer_fd, -result, strerror(-result));
+            fprintf(stderr, "Client %ju socket read error: %d (%s). Disconnecting client\n", cl->client_id, -result, strerror(-result));
         }
-        stop_client(cl->peer_fd);
+        stop_client(cl->client_id);
         return false;
     }
     if (result == -EAGAIN || result == -EINTR || result < cl->read_iov.iov_len)
     {
         cl->read_ready--;
         if (cl->read_ready > 0)
-            read_ready_clients.push_back(cl->peer_fd);
+            read_ready_clients.push_back(cl->client_id);
     }
     else
     {
-        read_ready_clients.push_back(cl->peer_fd);
+        read_ready_clients.push_back(cl->client_id);
     }
     if (result > 0)
     {
@@ -109,7 +113,6 @@ bool osd_messenger_t::handle_read(int result, osd_client_t *cl)
         {
             if (!handle_read_buffer(cl, cl->in_buf, result))
             {
-                clear_immediate_ops(peer_fd);
                 handle_immediate_ops();
                 return false;
             }
@@ -123,7 +126,6 @@ bool osd_messenger_t::handle_read(int result, osd_client_t *cl)
             {
                 if (!handle_finished_read(cl))
                 {
-                    clear_immediate_ops(peer_fd);
                     handle_immediate_ops();
                     return false;
                 }
@@ -138,33 +140,19 @@ bool osd_messenger_t::handle_read(int result, osd_client_t *cl)
     return ret;
 }
 
-void osd_messenger_t::clear_immediate_ops(int peer_fd)
-{
-    size_t i = 0, j = 0;
-    while (i < set_immediate_ops.size())
-    {
-        if (set_immediate_ops[i]->peer_fd == peer_fd && set_immediate_ops[i]->op_type == OSD_OP_IN)
-        {
-            delete set_immediate_ops[i];
-        }
-        else
-        {
-            if (i != j)
-                set_immediate_ops[j] = set_immediate_ops[i];
-            j++;
-        }
-        i++;
-    }
-    set_immediate_ops.resize(j);
-}
-
 void osd_messenger_t::handle_immediate_ops()
 {
-    for (auto op: set_immediate_ops)
+    while (set_immediate_ops.size())
     {
+        auto op = set_immediate_ops.front();
+        set_immediate_ops.pop_front();
         if (op->op_type == OSD_OP_IN)
         {
-            exec_op(op);
+            auto cl_it = clients.find(op->client_id);
+            if (cl_it != clients.end() && cl_it->second->peer_state != PEER_STOPPED)
+                exec_op(op);
+            else
+                delete op;
         }
         else
         {
@@ -172,7 +160,6 @@ void osd_messenger_t::handle_immediate_ops()
             std::function<void(osd_op_t*)>(op->callback)(op);
         }
     }
-    set_immediate_ops.clear();
 }
 
 bool osd_messenger_t::handle_read_buffer(osd_client_t *cl, void *curbuf, int remain)
@@ -183,7 +170,7 @@ bool osd_messenger_t::handle_read_buffer(osd_client_t *cl, void *curbuf, int rem
         if (!cl->read_op)
         {
             cl->read_op = new osd_op_t;
-            cl->read_op->peer_fd = cl->peer_fd;
+            cl->read_op->client_id = cl->client_id;
             cl->read_op->op_type = OSD_OP_IN;
             cl->recv_list.push_back(cl->read_op->req.buf, OSD_PACKET_SIZE);
             cl->read_remaining = OSD_PACKET_SIZE;
@@ -237,8 +224,8 @@ bool osd_messenger_t::handle_finished_read(osd_client_t *cl)
             {
                 if (cl->read_op->req.hdr.id != cl->read_op_id)
                 {
-                    fprintf(stderr, "Warning: operation sequencing is broken on client %d: expected num %ju, got %ju, stopping client\n", cl->peer_fd, cl->read_op_id, cl->read_op->req.hdr.id);
-                    stop_client(cl->peer_fd);
+                    fprintf(stderr, "Warning: operation sequencing is broken on client %ju: expected num %ju, got %ju, stopping client\n", cl->client_id, cl->read_op_id, cl->read_op->req.hdr.id);
+                    stop_client(cl->client_id);
                     return false;
                 }
                 cl->read_op_id++;
@@ -247,8 +234,8 @@ bool osd_messenger_t::handle_finished_read(osd_client_t *cl)
         }
         else
         {
-            fprintf(stderr, "Received garbage: magic=%jx id=%ju opcode=%jx from %d\n", cl->read_op->req.hdr.magic, cl->read_op->req.hdr.id, cl->read_op->req.hdr.opcode, cl->peer_fd);
-            stop_client(cl->peer_fd);
+            fprintf(stderr, "Received garbage: magic=%jx id=%ju opcode=%jx from client %ju\n", cl->read_op->req.hdr.magic, cl->read_op->req.hdr.id, cl->read_op->req.hdr.opcode, cl->client_id);
+            stop_client(cl->client_id);
             return false;
         }
     }
@@ -364,8 +351,8 @@ bool osd_messenger_t::handle_reply_hdr(osd_client_t *cl)
     if (req_it == cl->sent_ops.end())
     {
         // Command out of sync. Drop connection
-        fprintf(stderr, "Client %d command out of sync: id %ju\n", cl->peer_fd, cl->read_op->req.hdr.id);
-        stop_client(cl->peer_fd);
+        fprintf(stderr, "Client %ju command out of sync: id %ju\n", cl->client_id, cl->read_op->req.hdr.id);
+        stop_client(cl->client_id);
         return false;
     }
     osd_op_t *op = req_it->second;
@@ -379,10 +366,10 @@ bool osd_messenger_t::handle_reply_hdr(osd_client_t *cl)
         if (op->reply.hdr.retval >= 0 && (op->reply.hdr.retval != expected_size || bmp_len > op->bitmap_len))
         {
             // Check reply length to not overflow the buffer
-            fprintf(stderr, "Client %d read reply of different length: expected %u+%u, got %jd+%u\n",
-                cl->peer_fd, expected_size, op->bitmap_len, op->reply.hdr.retval, bmp_len);
+            fprintf(stderr, "Client %ju read reply of different length: expected %u+%u, got %jd+%u\n",
+                cl->client_id, expected_size, op->bitmap_len, op->reply.hdr.retval, bmp_len);
             cl->sent_ops[op->req.hdr.id] = op;
-            stop_client(cl->peer_fd);
+            stop_client(cl->client_id);
             return false;
         }
         if (bmp_len > 0)

src/client/msgr_send.cpp

@@ -9,8 +9,14 @@
 
 void osd_messenger_t::outbox_push(osd_op_t *cur_op)
 {
-    assert(cur_op->peer_fd);
-    osd_client_t *cl = clients.at(cur_op->peer_fd);
+    assert(cur_op->client_id);
+    auto cl_it = clients.find(cur_op->client_id);
+    if (cl_it == clients.end() || cl_it->second->peer_state == PEER_STOPPED)
+    {
+        delete cur_op;
+        return;
+    }
+    osd_client_t *cl = cl_it->second;
     if (cur_op->op_type == OSD_OP_OUT)
     {
         clock_gettime(CLOCK_REALTIME, &cur_op->tv_begin);
@@ -18,8 +24,7 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
     }
     else
     {
-        // Check that operation actually belongs to this client
-        // FIXME: Review if this is still needed
+        // Remove the operation from received op list
         bool found = false;
         for (auto it = cl->received_ops.begin(); it != cl->received_ops.end(); it++)
         {
@@ -30,11 +35,8 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
                 break;
             }
         }
-        if (!found)
-        {
-            delete cur_op;
-            return;
-        }
+        // Can't be not found because client IDs are unique
+        assert(found);
     }
     auto & to_send_list = cl->write_msg.msg_iovlen ? cl->next_send_list : cl->send_list;
     auto & to_outbox = cl->write_msg.msg_iovlen ? cl->next_outbox : cl->outbox;
@@ -126,7 +128,7 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
         if ((cl->write_msg.msg_iovlen > 0 || !try_send(cl)) && (cl->write_state == 0))
         {
             cl->write_state = CL_WRITE_READY;
-            write_ready_clients.push_back(cur_op->peer_fd);
+            write_ready_clients.push_back(cur_op->client_id);
         }
         ringloop->wakeup();
     }
@@ -183,15 +185,14 @@ void osd_messenger_t::measure_exec(osd_op_t *cur_op)
 
 bool osd_messenger_t::try_send(osd_client_t *cl)
 {
-    int peer_fd = cl->peer_fd;
-    if (!cl->send_list.size() || cl->write_msg.msg_iovlen > 0)
+    if (!cl->send_list.size() || cl->write_msg.msg_iovlen > 0 || cl->peer_state == PEER_STOPPED || cl->peer_fd < 0)
     {
         return true;
     }
     assert(cl->peer_state != PEER_RDMA);
     if (ringloop && !use_sync_send_recv)
     {
-        auto iothread = iothreads.size() ? iothreads[peer_fd % iothreads.size()] : NULL;
+        auto iothread = iothreads.size() ? iothreads[cl->peer_fd % iothreads.size()] : NULL;
         io_uring_sqe sqe_local;
         ring_data_t data_local;
         io_uring_sqe* sqe = (iothread ? &sqe_local : ringloop->get_sqe());
@@ -218,11 +219,11 @@ bool osd_messenger_t::try_send(osd_client_t *cl)
         }
         if (use_zc)
         {
-            io_uring_prep_sendmsg_zc(sqe, peer_fd, &cl->write_msg, MSG_WAITALL);
+            io_uring_prep_sendmsg_zc(sqe, cl->peer_fd, &cl->write_msg, MSG_WAITALL);
         }
         else
         {
-            io_uring_prep_sendmsg(sqe, peer_fd, &cl->write_msg, MSG_WAITALL);
+            io_uring_prep_sendmsg(sqe, cl->peer_fd, &cl->write_msg, MSG_WAITALL);
         }
         if (iothread)
         {
@@ -234,7 +235,7 @@ bool osd_messenger_t::try_send(osd_client_t *cl)
         cl->write_msg.msg_iov = cl->send_list.data();
         cl->write_msg.msg_iovlen = cl->send_list.size() < IOV_MAX ? cl->send_list.size() : IOV_MAX;
         cl->refs++;
-        int result = sendmsg(peer_fd, &cl->write_msg, MSG_NOSIGNAL);
+        int result = sendmsg(cl->peer_fd, &cl->write_msg, MSG_NOSIGNAL);
         if (result < 0)
         {
             result = -errno;
@@ -249,9 +250,9 @@ void osd_messenger_t::send_replies()
 {
     for (int i = 0; i < write_ready_clients.size(); i++)
     {
-        int peer_fd = write_ready_clients[i];
-        auto cl_it = clients.find(peer_fd);
-        if (cl_it != clients.end() && !try_send(cl_it->second))
+        uint64_t client_id = write_ready_clients[i];
+        auto cl_it = clients.find(client_id);
+        if (cl_it != clients.end() && cl_it->second->peer_state != PEER_RDMA && !try_send(cl_it->second))
         {
             write_ready_clients.erase(write_ready_clients.begin(), write_ready_clients.begin() + i);
             return;
@@ -274,15 +275,15 @@ void osd_messenger_t::handle_send(int result, bool prev, bool more, osd_client_t
     {
         if (cl->refs <= 0)
         {
-            delete cl;
+            destroy_client(cl);
         }
         return;
     }
     if (result < 0 && result != -EAGAIN && result != -EINTR)
     {
         // this is a client socket, so don't panic. just disconnect it
-        fprintf(stderr, "Client %d socket write error: %d (%s). Disconnecting client\n", cl->peer_fd, -result, strerror(-result));
-        stop_client(cl->peer_fd);
+        fprintf(stderr, "Client %ju socket write error: %d (%s). Disconnecting client\n", cl->client_id, -result, strerror(-result));
+        stop_client(cl->client_id);
         return;
     }
     if (result >= 0)
@@ -326,9 +327,9 @@ void osd_messenger_t::handle_send(int result, bool prev, bool more, osd_client_t
             int expected = cl->send_list.size() < IOV_MAX ? cl->send_list.size() : IOV_MAX;
             if (done != expected)
             {
-                fprintf(stderr, "Client %d socket write error: expected to send "
-                    "%d iovecs with MSG_WAITALL but sent %d. Disconnecting client\n", cl->peer_fd, expected, done);
-                stop_client(cl->peer_fd);
+                fprintf(stderr, "Client %ju socket write error: expected to send "
+                    "%d iovecs with MSG_WAITALL but sent %d. Disconnecting client\n", cl->client_id, expected, done);
+                stop_client(cl->client_id);
                 return;
             }
             cl->zc_free_list.push_back(NULL); // end marker
@@ -349,21 +350,12 @@ void osd_messenger_t::handle_send(int result, bool prev, bool more, osd_client_t
 #ifdef WITH_RDMA
         if (cl->rdma_conn && !cl->outbox.size() && cl->peer_state == PEER_RDMA_CONNECTING)
         {
-            // FIXME: Do something better than just forgetting the FD
             // FIXME: Ignore pings during RDMA state transition
             if (log_level > 0)
             {
-                fprintf(stderr, "Successfully connected with client %d using RDMA\n", cl->peer_fd);
+                fprintf(stderr, "Successfully connected with client %ju using RDMA\n", cl->client_id);
             }
             cl->peer_state = PEER_RDMA;
-            tfd->set_fd_handler(cl->peer_fd, false, [this](int peer_fd, int epoll_events)
-            {
-                // Do not miss the disconnection!
-                if (epoll_events & EPOLLRDHUP)
-                {
-                    handle_peer_epoll(peer_fd, epoll_events);
-                }
-            });
             // Add the initial receive request
             init_recv_rdma(cl);
         }
@@ -371,6 +363,6 @@ void osd_messenger_t::handle_send(int result, bool prev, bool more, osd_client_t
     }
     if (cl->write_state != 0)
     {
-        write_ready_clients.push_back(cl->peer_fd);
+        write_ready_clients.push_back(cl->client_id);
     }
 }

src/client/msgr_stop.cpp

@@ -43,33 +43,40 @@ void osd_op_t::cancel()
     }
 }
 
-void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
+// force_delete means stop the client anyway, even if there are refs to it in the event loop.
+// the flag should be used in the destructor.
+// why? - because yes, we could close the FD first and let it fail all requests in the event loop,
+// but in that case it can be quickly reopened and we can get old failed responses for the new FD.
+void osd_messenger_t::stop_client(uint64_t client_id, bool force_delete)
 {
-    assert(peer_fd != 0);
-    auto it = clients.find(peer_fd);
-    if (it == clients.end())
+    auto it = clients.find(client_id);
+    if (!client_id || it == clients.end())
     {
         return;
     }
     osd_client_t *cl = it->second;
-    // FIXME: This 'force' flag is probably an ugly reenterability hack - check its logic and maybe remove it
-    if (cl->peer_state == PEER_CONNECTING && !force || cl->peer_state == PEER_STOPPED)
+    if (cl->peer_state == PEER_STOPPED)
     {
+        if (force_delete)
+        {
+            destroy_client(cl);
+        }
         return;
     }
+    cl->received_ops.clear();
     if (log_level > 0)
     {
         if (cl->osd_num)
         {
-            fprintf(stderr, "[OSD %ju] Stopping client %d (OSD peer %ju)\n", osd_num, peer_fd, cl->osd_num);
+            fprintf(stderr, "[OSD %ju] Stopping client %ju (OSD peer %ju)\n", osd_num, client_id, cl->osd_num);
         }
         else if (cl->in_osd_num)
         {
-            fprintf(stderr, "[OSD %ju] Stopping client %d (incoming OSD peer %ju)\n", osd_num, peer_fd, cl->in_osd_num);
+            fprintf(stderr, "[OSD %ju] Stopping client %ju (incoming OSD peer %ju)\n", osd_num, client_id, cl->in_osd_num);
         }
         else
         {
-            fprintf(stderr, "[OSD %ju] Stopping client %d (regular client)\n", osd_num, peer_fd);
+            fprintf(stderr, "[OSD %ju] Stopping client %ju (regular client)\n", osd_num, client_id);
         }
     }
     // First set state to STOPPED so another stop_client() call doesn't try to free it again
@@ -78,11 +85,11 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
     cl->peer_state = PEER_STOPPED;
     if (cl->osd_num)
     {
-        auto osd_it = osd_peer_fds.find(cl->osd_num);
-        if (osd_it != osd_peer_fds.end() && osd_it->second == cl->peer_fd)
+        auto osd_it = osd_peers.find(cl->osd_num);
+        if (osd_it != osd_peers.end() && osd_it->second == cl)
         {
             // ...and forget OSD peer
-            osd_peer_fds.erase(osd_it);
+            osd_peers.erase(osd_it);
         }
     }
 #ifdef WITH_RDMA
@@ -96,29 +103,11 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
     }
 #endif
 #ifndef __MOCK__
-    // Then remove FD from the eventloop so we don't accidentally read something
-    tfd->set_fd_handler(peer_fd, false, NULL);
     if (cl->connect_timeout_id >= 0)
     {
         tfd->clear_timer(cl->connect_timeout_id);
         cl->connect_timeout_id = -1;
     }
-    for (auto rit = read_ready_clients.begin(); rit != read_ready_clients.end(); rit++)
-    {
-        if (*rit == peer_fd)
-        {
-            read_ready_clients.erase(rit);
-            break;
-        }
-    }
-    for (auto wit = write_ready_clients.begin(); wit != write_ready_clients.end(); wit++)
-    {
-        if (*wit == peer_fd)
-        {
-            write_ready_clients.erase(wit);
-            break;
-        }
-    }
 #endif
     if (cl->in_osd_num && break_pg_locks)
     {
@@ -133,19 +122,51 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
         // so do not repeer on it.
         repeer_pgs(cl->osd_num);
     }
-    // Find the item again because it can be invalidated at this point
-    it = clients.find(peer_fd);
-    if (it != clients.end())
+    if (cl->peer_fd >= 0)
     {
-        clients.erase(it);
+        int r = shutdown(cl->peer_fd, SHUT_RDWR);
+        if (r != 0 && errno != ENOTCONN)
+        {
+            fprintf(stderr, "[OSD %ju] failed to shutdown a socket: %s (code %d)\n", osd_num, strerror(errno), errno);
+        }
     }
     cl->refs--;
     if (cl->refs <= 0 || force_delete)
     {
-        delete cl;
+        destroy_client(cl);
     }
 }
 
+void osd_messenger_t::destroy_client(osd_client_t *cl)
+{
+    // Find the item again because it can be invalidated at this point
+    clients.erase(cl->client_id);
+    if (cl->peer_fd >= 0)
+    {
+#ifndef __MOCK__
+        tfd->set_fd_handler(cl->peer_fd, false, NULL);
+#endif
+        for (auto rit = read_ready_clients.begin(); rit != read_ready_clients.end(); rit++)
+        {
+            if (*rit == cl->client_id)
+            {
+                read_ready_clients.erase(rit);
+                break;
+            }
+        }
+        for (auto wit = write_ready_clients.begin(); wit != write_ready_clients.end(); wit++)
+        {
+            if (*wit == cl->client_id)
+            {
+                write_ready_clients.erase(wit);
+                break;
+            }
+        }
+        clients_by_fd.erase(cl->peer_fd);
+    }
+    delete cl;
+}
+
 osd_client_t::~osd_client_t()
 {
     free(in_buf);

src/client/object_id.h

@@ -20,6 +20,15 @@ typedef uint64_t inode_t;
 // Pool ID is 16 bits long
 typedef uint32_t pool_id_t;
 
+typedef uint64_t osd_num_t;
+typedef uint32_t pg_num_t;
+
+struct pool_pg_num_t
+{
+    pool_id_t pool_id;
+    pg_num_t pg_num;
+};
+
 // 16 bytes per object/stripe id
 // stripe = (start of the parity stripe + peer role)
 // i.e. for example (256KB + one of 0,1,2)
@@ -61,6 +70,21 @@ inline bool operator < (const obj_ver_id & a, const obj_ver_id & b)
     return a.oid < b.oid || a.oid == b.oid && a.version < b.version;
 }
 
+inline bool operator < (const pool_pg_num_t & a, const pool_pg_num_t & b)
+{
+    return a.pool_id < b.pool_id || a.pool_id == b.pool_id && a.pg_num < b.pg_num;
+}
+
+inline bool operator == (const pool_pg_num_t & a, const pool_pg_num_t & b)
+{
+    return a.pool_id == b.pool_id && a.pg_num == b.pg_num;
+}
+
+inline bool operator != (const pool_pg_num_t & a, const pool_pg_num_t & b)
+{
+    return a.pool_id != b.pool_id || a.pg_num != b.pg_num;
+}
+
 namespace std
 {
     template<> struct hash<object_id>

src/client/osd_ops.h

@@ -4,7 +4,6 @@
 #pragma once
 
 #include "object_id.h"
-#include "osd_id.h"
 
 // Magic numbers
 #define SECONDARY_OSD_OP_MAGIC      0x2bd7b10325434553l

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 3.0.2
+Version: 3.0.10
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/cmd/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 2.8...3.30)
 
 project(vitastor)
 

src/cmd/cli.h

@@ -7,7 +7,6 @@
 
 #include "json11/json11.hpp"
 #include "object_id.h"
-#include "osd_id.h"
 #include "ringloop.h"
 #include <functional>
 

src/cmd/cli_dd.cpp

@@ -442,7 +442,7 @@ struct cli_dd_t
                 }
                 delete cur_read;
             }
-            else if (!is_zero(read_op->bitmap_buf, read_op->len/iinfo.in_granularity/8))
+            else if (!is_zero(read_op->bitmap_buf, (read_op->len/iinfo.in_granularity+7)/8))
             {
                 vitastor_read(cur_read);
             }

src/cmd/cli_rm_data.cpp

@@ -286,6 +286,8 @@ struct rm_inode_t
                     .data = data,
                 };
             }
+            // Wakeup callers (otherwise they wake up only on next loop())
+            parent->ringloop->wakeup();
         }
         in_continue = false;
     }

src/disk_tool/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 2.8...3.30)
 
 project(vitastor)
 

src/disk_tool/disk_tool_meta.cpp

@@ -3,7 +3,6 @@
 
 #include "disk_tool.h"
 #include "rw_blocking.h"
-#include "osd_id.h"
 #include "json_util.h"
 #include "malloc_or_die.h"
 
@@ -109,6 +108,7 @@ close_error:
             r = heap->load_blocks(meta_pos-dsk.meta_block_size, read_len, data, true, entries_loaded);
             meta_pos += read_len;
         }
+        heap->finish_load();
         heap->iterate_objects([&](heap_entry_t* obj, uint32_t meta_block_num)
         {
             obj_fn(heap, obj, meta_block_num);
@@ -331,7 +331,7 @@ void disk_tool_t::dump_meta_header(blockstore_meta_header_v3_t *hdr)
     }
     else
     {
-        printf("{\"version\":\"0.5\",\"meta_block_size\":%u,\"entries\":[\n", dsk.meta_block_size);
+        printf("{\"version\":\"0.5\",\"meta_block_size\":%ju,\"entries\":[\n", dsk.meta_block_size);
     }
     first_entry = true;
 }

src/disk_tool/disk_tool_prepare.cpp

@@ -4,7 +4,6 @@
 #include "disk_tool.h"
 #include "str_util.h"
 #include "json_util.h"
-#include "osd_id.h"
 
 void disk_tool_t::parse_meta_reserve()
 {
@@ -150,9 +149,12 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
     {
         if (options["block_size"] == "")
             options["block_size"] = "1M";
+        if (is_hybrid && options["atomic_write_size"] == "")
+            options["atomic_write_size"] = "0";
         if (is_hybrid && options["throttle_small_writes"] == "")
             options["throttle_small_writes"] = "1";
-        if (!is_hybrid && options.find("data_csum_type") != options.end() && options.at("data_csum_type") != "")
+        if (!is_hybrid && options.find("data_csum_type") != options.end() && options.at("data_csum_type") != "" &&
+            options["csum_block_size"] == "")
             options["csum_block_size"] = "32k";
     }
     else if (!json_is_true(options["disable_data_fsync"]))

src/disk_tool/disk_tool_resize.cpp

@@ -212,11 +212,11 @@ void disk_tool_t::resize_init(blockstore_meta_header_v3_t *hdr)
     dsk.calc_lengths();
     if (((new_data_offset-dsk.data_offset) % dsk.data_block_size))
     {
-        fprintf(stderr, "Data alignment mismatch: old data offset is 0x%jx, new is 0x%jx, but alignment on %x should be equal\n",
+        fprintf(stderr, "Data alignment mismatch: old data offset is 0x%jx, new is 0x%jx, but alignment on %jx should be equal\n",
             dsk.data_offset, new_data_offset, dsk.data_block_size);
         exit(1);
     }
-    data_idx_diff = ((int64_t)(dsk.data_offset-new_data_offset)) / dsk.data_block_size;
+    data_idx_diff = ((int64_t)(dsk.data_offset-new_data_offset))/((int64_t)dsk.data_block_size);
     free_first = new_data_offset > dsk.data_offset ? (new_data_offset-dsk.data_offset) / dsk.data_block_size : 0;
     free_last = (new_data_offset+new_data_len < dsk.data_offset+dsk.data_len)
         ? (dsk.data_offset+dsk.data_len-new_data_offset-new_data_len) / dsk.data_block_size
@@ -351,7 +351,7 @@ int disk_tool_t::resize_copy_data()
                         if (data->res != dsk.data_block_size)
                         {
                             fprintf(
-                                stderr, "Failed to read %u bytes at %ju from %s: %s\n", dsk.data_block_size,
+                                stderr, "Failed to read %ju bytes at %ju from %s: %s\n", dsk.data_block_size,
                                 dsk.data_offset + moving_blocks[i].old_loc*dsk.data_block_size, dsk.data_device.c_str(),
                                 data->res < 0 ? strerror(-data->res) : "short read"
                             );
@@ -376,7 +376,7 @@ int disk_tool_t::resize_copy_data()
                         if (data->res != dsk.data_block_size)
                         {
                             fprintf(
-                                stderr, "Failed to write %u bytes at %ju to %s: %s\n", dsk.data_block_size,
+                                stderr, "Failed to write %ju bytes at %ju to %s: %s\n", dsk.data_block_size,
                                 dsk.data_offset + moving_blocks[i].new_loc*dsk.data_block_size, dsk.data_device.c_str(),
                                 data->res < 0 ? strerror(-data->res) : "short write"
                             );

src/disk_tool/disk_tool_udev.cpp

@@ -140,7 +140,10 @@ uint32_t disk_tool_t::write_osd_superblock(std::string device, json11::Json para
     }
     close(fd);
     free(buf);
-    shell_exec({ "udevadm", "trigger", "--settle", device }, "", NULL, NULL);
+    if (!test_mode)
+    {
+        shell_exec({ "udevadm", "trigger", "--settle", device }, "", NULL, NULL);
+    }
     return sb_size;
 }
 

src/kv/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 2.8...3.30)
 
 project(vitastor)
 

src/liburing/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 2.8...3.30)
 
 project(vitastor)
 

src/nfs/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 2.8...3.30)
 
 project(vitastor)
 

src/nfs/nfs_kv_setattr.cpp

@@ -61,7 +61,7 @@ resume_1:
     }
     if (st->ientry["type"].string_value() != "file" &&
         st->ientry["type"].string_value() != "" &&
-        !st->set_attrs["size"].is_null())
+        st->set_attrs.find("size") != st->set_attrs.end())
     {
         auto cb = std::move(st->cb);
         cb(-EINVAL);
@@ -96,8 +96,13 @@ resume_1:
         nfs_kv_continue_setattr(st, 2);
     }, [st](int res, const std::string & cas_value)
     {
+        if ((res == 0 || res == -ENOENT && st->ino == KV_ROOT_INODE) && cas_value == st->ientry_text)
+        {
+            st->cas_res = 0;
+            return true;
+        }
         st->cas_res = res;
-        return (res == 0 || res == -ENOENT && st->ino == KV_ROOT_INODE) && cas_value == st->ientry_text;
+        return false;
     });
     return;
 resume_2:

src/nfs/nfs_proxy.cpp

@@ -121,6 +121,7 @@ static const char* help_text =
     "  --logfile <FILE>  log to the specified file\n"
     "  --enforce 1       enforce permissions at the server side (default is disabled)\n"
     "  --foreground 1    stay in foreground, do not daemonize\n"
+    "  --trace           trace all NFS requests\n"
     "\n"
     "NFS proxy is stateless if you use immediate_commit=all in your cluster and if\n"
     "you do not use client_enable_writeback=true, so you can freely use multiple\n"
@@ -158,7 +159,7 @@ json11::Json::object nfs_proxy_t::parse_args(int narg, const char *args[])
         {
             const char *opt = args[i]+2;
             cfg[str_replace(opt, "-", "_")] = !strcmp(opt, "json") || !strcmp(opt, "block") ||
-                !strcmp(opt, "dry-run") || !strcmp(opt, "recalc-stats") ||
+                !strcmp(opt, "dry-run") || !strcmp(opt, "recalc-stats") || !strcmp(opt, "trace") ||
                 !strcmp(opt, "include-empty") || !strcmp(opt, "no-rm") || i == narg-1 ? "1" : args[++i];
         }
         else

src/osd/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 2.8...3.30)
 
 project(vitastor)
 

src/osd/osd.cpp

@@ -529,6 +529,11 @@ void osd_t::exec_op(osd_op_t *cur_op)
 
 void osd_t::print_stats()
 {
+    if (bs && log_level > 1)
+    {
+        printf("[OSD %ju] Live entries: %ju (%ju bytes), garbage entries: %ju (%ju bytes)\n", osd_num,
+            bs->get_live_entries(), bs->get_live_memory(), bs->get_garbage_entries(), bs->get_garbage_memory());
+    }
     for (int i = OSD_OP_MIN; i <= OSD_OP_MAX; i++)
     {
         if (msgr.stats.op_stat_count[i] != prev_stats.op_stat_count[i] && i != OSD_OP_PING)
@@ -616,11 +621,11 @@ void osd_t::print_slow()
                 bufprintf("[OSD %ju] Slow op %jx", osd_num, (uint64_t)op);
                 if (kv.second->osd_num)
                 {
-                    bufprintf(" from peer OSD %ju (client %d)", kv.second->osd_num, kv.second->peer_fd);
+                    bufprintf(" from peer OSD %ju (client %ju)", kv.second->osd_num, kv.second->client_id);
                 }
                 else
                 {
-                    bufprintf(" from client %d", kv.second->peer_fd);
+                    bufprintf(" from client %ju", kv.second->client_id);
                 }
                 bufprintf(": %s id=%ju", osd_op_names[op->req.hdr.opcode], op->req.hdr.id);
                 if (op->req.hdr.opcode == OSD_OP_SEC_READ || op->req.hdr.opcode == OSD_OP_SEC_WRITE ||