WIP refactor dequeue_write/continue_write into a single "coroutine"

.gitea/workflows/buildenv.Dockerfile

@@ -1,29 +1,28 @@
-FROM node:16-bookworm
+FROM node:16-bullseye
 
 WORKDIR /root
 
-ADD ./docker/etc/apt/trusted.gpg.d /etc/apt/trusted.gpg.d
+ADD ./docker/vitastor.gpg /etc/apt/trusted.gpg.d
 
-RUN echo 'deb http://deb.debian.org/debian bookworm-backports main' >> /etc/apt/sources.list; \
+RUN echo 'deb http://deb.debian.org/debian bullseye-backports main' >> /etc/apt/sources.list; \
-    echo 'deb http://vitastor.io/debian bookworm main' >> /etc/apt/sources.list; \
+    echo 'deb http://vitastor.io/debian bullseye main' >> /etc/apt/sources.list; \
     echo >> /etc/apt/preferences; \
     echo 'Package: *' >> /etc/apt/preferences; \
-    echo 'Pin: release n=bookworm-backports' >> /etc/apt/preferences; \
+    echo 'Pin: release a=bullseye-backports' >> /etc/apt/preferences; \
     echo 'Pin-Priority: 500' >> /etc/apt/preferences; \
     echo >> /etc/apt/preferences; \
     echo 'Package: *' >> /etc/apt/preferences; \
     echo 'Pin: origin "vitastor.io"' >> /etc/apt/preferences; \
     echo 'Pin-Priority: 1000' >> /etc/apt/preferences; \
-    perl -i -pe 's/Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/debian.sources; \
     grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb/deb-src/' >> /etc/apt/sources.list; \
     echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf; \
     echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
 
 RUN apt-get update
-RUN apt-get -y install etcd qemu-system-x86 qemu-block-extra qemu-utils fio libasan8 \
+RUN apt-get -y install etcd qemu-system-x86 qemu-block-extra qemu-utils fio libasan5 \
     libgoogle-perftools-dev devscripts libjerasure-dev cmake libibverbs-dev libisal-dev
 RUN apt-get -y build-dep fio qemu=`dpkg -s qemu-system-x86|grep ^Version:|awk '{print $2}'`
-RUN apt-get update && apt-get -y install jq lp-solve sudo nfs-common fdisk parted libc-ares-dev udev
+RUN apt-get update && apt-get -y install jq lp-solve sudo nfs-common fdisk parted
 RUN apt-get --download-only source fio qemu=`dpkg -s qemu-system-x86|grep ^Version:|awk '{print $2}'`
 
 RUN set -ex; \

.gitea/workflows/test.yml

@@ -1206,78 +1206,6 @@ jobs:
           echo ""
         done
 
-  test_checksum:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_checksum.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_old_checksum:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 /root/vitastor/tests/test_checksum.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_corrupt_all:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_corrupt_all.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_old_corrupt_all:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 /root/vitastor/tests/test_corrupt_all.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
   test_reweight_half:
     runs-on: ubuntu-latest
     needs: build

CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "3.0.4")
+set(VITASTOR_VERSION "3.0.2")
 
 include(CTest)
 

csi/Dockerfile

@@ -1,5 +1,5 @@
 # Compile stage
-FROM golang:trixie AS build
+FROM golang:bookworm AS build
 
 ADD go.sum go.mod /app/
 RUN cd /app; CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go mod download -x
@@ -9,7 +9,7 @@ RUN perl -i -e '$/ = undef; while(<>) { s/\n\s*(\{\s*\n)/$1\n/g; s/\}(\s*\n\s*)e
     CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o vitastor-csi
 
 # Final stage
-FROM debian:trixie
+FROM debian:bookworm
 
 LABEL maintainers="Vitaliy Filippov <vitalif@yourcmc.ru>"
 LABEL description="Vitastor CSI Driver"
@@ -25,20 +25,20 @@ RUN apt-get update && \
         # NFS mount dependencies
         nfs-common netbase \
         # dependencies of qemu-storage-daemon
-        libaio1t64 libc6 libfuse3-4 libglib2.0-0t64 libgmp10 libgnutls30t64 \
+        libnuma1 liburing2 libglib2.0-0 libfuse3-3 libaio1 libzstd1 libnettle8 \
-        libhogweed6t64 libnettle8t64 libnuma1 libselinux1 liburing2 libzstd1 zlib1g && \
+        libgmp10 libhogweed6 libp11-kit0 libidn2-0 libunistring2 libtasn1-6 libpcre2-8-0 libffi8 && \
     apt-get clean && \
     (echo options nbd nbds_max=128 > /etc/modprobe.d/nbd.conf)
 
 COPY --from=build /app/vitastor-csi /bin/
 
-RUN (echo deb http://vitastor.io/debian trixie main > /etc/apt/sources.list.d/vitastor.list) && \
+RUN (echo deb http://vitastor.io/debian bookworm main > /etc/apt/sources.list.d/vitastor.list) && \
     ((echo 'Package: *'; echo 'Pin: origin "vitastor.io"'; echo 'Pin-Priority: 1000') > /etc/apt/preferences.d/vitastor.pref) && \
     wget -q -O /etc/apt/trusted.gpg.d/vitastor.gpg https://vitastor.io/debian/pubkey.gpg && \
     apt-get update && \
     apt-get install -y vitastor-client ibverbs-providers && \
-    wget https://vitastor.io/archive/qemu/qemu-trixie-10.0.2%2Bds-2%2Bvitastor1/qemu-utils_10.0.2%2Bds-2%2Bvitastor1_amd64.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-utils_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
-    wget https://vitastor.io/archive/qemu/qemu-trixie-10.0.2%2Bds-2%2Bvitastor1/qemu-block-extra_10.0.2%2Bds-2%2Bvitastor1_amd64.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-block-extra_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
     dpkg -x qemu-utils*.deb tmp1 && \
     dpkg -x qemu-block-extra*.deb tmp1 && \
     cp -a tmp1/usr/bin/qemu-storage-daemon /usr/bin/ && \

csi/Dockerfile.local

@@ -1,5 +1,5 @@
 # Compile stage
-FROM golang:trixie AS build
+FROM golang:bookworm AS build
 
 ADD go.sum go.mod /app/
 RUN cd /app; CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go mod download -x
@@ -9,7 +9,7 @@ RUN perl -i -e '$/ = undef; while(<>) { s/\n\s*(\{\s*\n)/$1\n/g; s/\}(\s*\n\s*)e
     CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o vitastor-csi
 
 # Final stage
-FROM debian:trixie
+FROM debian:bookworm
 
 LABEL maintainers="Vitaliy Filippov <vitalif@yourcmc.ru>"
 LABEL description="Vitastor CSI Driver"
@@ -36,8 +36,8 @@ ADD deb /deb
 
 RUN apt-get update && \
     apt-get -y install /deb/vitastor-client_*.deb && \
-    wget https://vitastor.io/archive/qemu/qemu-trixie-9.2.2%2Bds-1%2Bvitastor4/qemu-utils_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-utils_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
-    wget https://vitastor.io/archive/qemu/qemu-trixie-9.2.2%2Bds-1%2Bvitastor4/qemu-block-extra_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-block-extra_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
     dpkg -x qemu-utils*.deb tmp1 && \
     dpkg -x qemu-block-extra*.deb tmp1 && \
     cp -a tmp1/usr/bin/qemu-storage-daemon /usr/bin/ && \

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.4
+VITASTOR_VERSION ?= v3.0.2
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v3.0.4
+          image: vitalif/vitastor-csi:v3.0.2
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v3.0.4
+          image: vitalif/vitastor-csi:v3.0.2
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "3.0.4"
+    vitastorCSIDriverVersion = "3.0.2"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (3.0.4-1) unstable; urgency=medium
+vitastor (3.0.2-1) unstable; urgency=medium
 
   * Bugfixes
 

docker/Dockerfile

@@ -1,6 +1,6 @@
 # Build Docker image with Vitastor packages
 
-FROM debian:trixie
+FROM debian:bookworm
 
 ADD etc/apt /etc/apt/
 RUN apt-get update && apt-get -y install vitastor ibverbs-providers udev systemd qemu-system-x86 qemu-system-common qemu-block-extra qemu-utils jq nfs-common && apt-get clean

docker/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.4
+VITASTOR_VERSION ?= v3.0.2
 
 all: build push
 

docker/etc/apt/preferences

@@ -1,3 +1,3 @@
 Package: *
-Pin: release n=trixie-backports
+Pin: release n=bookworm-backports
 Pin-Priority: 500

docker/etc/apt/sources.list.d/vitastor.list

@@ -1,2 +1,2 @@
-deb http://vitastor.io/debian trixie main
+deb http://vitastor.io/debian bookworm main
-#deb http://http.debian.net/debian/ trixie-backports main
+deb http://http.debian.net/debian/ bookworm-backports main

docker/etc/vitastor/docker.conf

@@ -4,7 +4,7 @@
 #
 
 # Desired Vitastor version
-VITASTOR_VERSION=v3.0.4
+VITASTOR_VERSION=v3.0.2
 
 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here

docs/config/network.en.md

@@ -22,6 +22,7 @@ between clients, OSDs and etcd.
 - [rdma_max_msg](#rdma_max_msg)
 - [rdma_max_recv](#rdma_max_recv)
 - [rdma_max_send](#rdma_max_send)
+- [rdma_odp](#rdma_odp)
 - [peer_connect_interval](#peer_connect_interval)
 - [peer_connect_timeout](#peer_connect_timeout)
 - [osd_idle_timeout](#osd_idle_timeout)
@@ -101,6 +102,11 @@ found or if `osd_network` is not specified. Auto-selection is also
 unsupported with old libibverbs < v32, like in Debian 10 Buster or
 CentOS 7.
 
+Vitastor supports all adapters, even ones without ODP support, like
+Mellanox ConnectX-3 and non-Mellanox cards. Versions up to Vitastor
+1.2.0 required ODP which is only present in Mellanox ConnectX >= 4.
+See also [rdma_odp](#rdma_odp).
+
 Run `ibv_devinfo -v` as root to list available RDMA devices and their
 features.
 
@@ -110,23 +116,6 @@ the manual of your network vendor for details about setting up the switch
 for RoCEv2 correctly. Usually it means setting up Lossless Ethernet with
 PFC (Priority Flow Control) and ECN (Explicit Congestion Notification).
 
-Vitastor supports all adapters, even ones without ODP (On-Demand Paging)
-support, like Mellanox ConnectX-3 and non-Mellanox cards. ODP is only present
-in Mellanox ConnectX >= 4 adapters and allows to skip memory registration
-for RDMA and thus, in theory, avoid memory copying.
-
-Versions up to Vitastor 1.2.0 required ODP, then it was disabled by default,
-but it was still supported up to 3.0.3. Now ODP support is removed because it
-actually only hurts performance: an example 3-node cluster with 8 NVMe in each
-node and 2*25 GBit/s ConnectX-6 RDMA network pushed 3950000 read iops without
-ODP, but only 239000 iops with ODP.
-
-This happens because Mellanox ODP implementation seems to be based on
-message retransmissions when the adapter doesn't know about the buffer yet -
-it likely uses standard "RNR retransmissions" (RNR = receiver not ready)
-which is generally slow in RDMA/RoCE networks. Here's a presentation about
-it from ISPASS-2021 conference: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
-
 ## rdma_port_num
 
 - Type: integer
@@ -198,6 +187,28 @@ less than `rdma_max_recv` so the receiving side doesn't run out of buffers.
 Doesn't affect memory usage - additional memory isn't allocated for send
 operations.
 
+## rdma_odp
+
+- Type: boolean
+- Default: false
+
+Use RDMA with On-Demand Paging. ODP is currently only available on Mellanox
+ConnectX-4 and newer adapters. ODP allows to not register memory explicitly
+for RDMA adapter to be able to use it. This, in turn, allows to skip memory
+copying during sending. One would think this should improve performance, but
+**in reality** RDMA performance with ODP is **drastically** worse. Example
+3-node cluster with 8 NVMe in each node and 2*25 GBit/s ConnectX-6 RDMA network
+without ODP pushes 3950000 read iops, but only 239000 iops with ODP...
+
+This happens because Mellanox ODP implementation seems to be based on
+message retransmissions when the adapter doesn't know about the buffer yet -
+it likely uses standard "RNR retransmissions" (RNR = receiver not ready)
+which is generally slow in RDMA/RoCE networks. Here's a presentation about
+it from ISPASS-2021 conference: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
+
+ODP support is retained in the code just in case a good ODP implementation
+appears one day.
+
 ## peer_connect_interval
 
 - Type: seconds

docs/config/network.ru.md

@@ -22,6 +22,7 @@
 - [rdma_max_msg](#rdma_max_msg)
 - [rdma_max_recv](#rdma_max_recv)
 - [rdma_max_send](#rdma_max_send)
+- [rdma_odp](#rdma_odp)
 - [peer_connect_interval](#peer_connect_interval)
 - [peer_connect_timeout](#peer_connect_timeout)
 - [osd_idle_timeout](#osd_idle_timeout)
@@ -100,6 +101,12 @@ RoCEv1/RoCEv2, и даже позволяет полностью отключи
 не задана. Также автовыбор не поддерживается со старыми версиями библиотеки
 libibverbs < v32, например в Debian 10 Buster или CentOS 7.
 
+Vitastor поддерживает все модели адаптеров, включая те, у которых
+нет поддержки ODP, то есть вы можете использовать RDMA с ConnectX-3 и
+картами производства не Mellanox. Версии Vitastor до 1.2.0 включительно
+требовали ODP, который есть только на Mellanox ConnectX 4 и более новых.
+См. также [rdma_odp](#rdma_odp).
+
 Запустите `ibv_devinfo -v` от имени суперпользователя, чтобы посмотреть
 список доступных RDMA-устройств, их параметры и возможности.
 
@@ -110,24 +117,6 @@ libibverbs < v32, например в Debian 10 Buster или CentOS 7.
 подразумевает настройку сети без потерь на основе PFC (Priority Flow
 Control) и ECN (Explicit Congestion Notification).
 
-Vitastor поддерживает все модели адаптеров, включая те, у которых нет
-поддержки ODP (On-Demand Paging), например, ConnectX-3 и карты производства
-не Mellanox. Функция ODP доступна только на адаптерах Mellanox ConnectX-4 и
-более новых и позволяет не регистрировать память для её использования RDMA-картой,
-благодаря чему в теории можно избежать лишних копирований памяти.
-
-Версии Vitastor до 1.2.0 включительно требовали ODP, потом функция был отключена
-по умолчанию, но поддерживалась вплоть до версии 3.0.3. Сейчас поддержка ODP
-полностью удалена, так как на самом деле она только портит производительность:
-например, на 3-узловом кластере с 8 NVMe в каждом узле и сетью 2*25 Гбит/с на
-чтение с RDMA без ODP удаётся снять 3950000 iops, а с ODP - всего 239000 iops.
-
-Это происходит из-за того, что реализация ODP у Mellanox неоптимальная и
-основана на повторной передаче сообщений, когда карте не известен буфер -
-вероятно, на стандартных "RNR retransmission" (RNR = receiver not ready).
-А данные повторные передачи в RDMA/RoCE - всегда очень медленная штука.
-Презентация на эту тему с конференции ISPASS-2021: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
-
 ## rdma_port_num
 
 - Тип: целое число
@@ -203,6 +192,29 @@ OSD в любом случае согласовывают реальное зн
 Не влияет на потребление памяти - дополнительная память на операции отправки
 не выделяется.
 
+## rdma_odp
+
+- Тип: булево (да/нет)
+- Значение по умолчанию: false
+
+Использовать RDMA с On-Demand Paging. ODP - функция, доступная пока что
+исключительно на адаптерах Mellanox ConnectX-4 и более новых. ODP позволяет
+не регистрировать память для её использования RDMA-картой. Благодаря этому
+можно не копировать данные при отправке их в сеть и, казалось бы, это должно
+улучшать производительность - но **по факту** получается так, что
+производительность только ухудшается, причём сильно. Пример - на 3-узловом
+кластере с 8 NVMe в каждом узле и сетью 2*25 Гбит/с на чтение с RDMA без ODP
+удаётся снять 3950000 iops, а с ODP - всего 239000 iops...
+
+Это происходит из-за того, что реализация ODP у Mellanox неоптимальная и
+основана на повторной передаче сообщений, когда карте не известен буфер -
+вероятно, на стандартных "RNR retransmission" (RNR = receiver not ready).
+А данные повторные передачи в RDMA/RoCE - всегда очень медленная штука.
+Презентация на эту тему с конференции ISPASS-2021: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
+
+Возможность использования ODP сохранена в коде на случай, если вдруг в один
+прекрасный день появится хорошая реализация ODP.
+
 ## peer_connect_interval
 
 - Тип: секунды

docs/config/osd.en.md

@@ -38,7 +38,6 @@ with an OSD restart or, for some of them, even without restarting by updating co
 - [journal_io](#journal_io)
 - [journal_sector_buffer_count](#journal_sector_buffer_count)
 - [journal_no_same_sector_overwrites](#journal_no_same_sector_overwrites)
-- [skip_corrupted_meta_entries](#skip_corrupted_meta_entries)
 - [throttle_small_writes](#throttle_small_writes)
 - [throttle_target_iops](#throttle_target_iops)
 - [throttle_target_mbs](#throttle_target_mbs)
@@ -280,19 +279,13 @@ Maximum number of journal flushers (see above min_flusher_count).
 - Type: boolean
 - Default: true
 
-Only for the old store ([meta_format](layout-osd.en.md#meta_format) 2).
+This parameter makes Vitastor always keep metadata area of the block device
-
+in memory. It's required for good performance because it allows to avoid
-This parameter makes Vitastor keep a copy of metadata area in memory as it is
+additional read-modify-write cycles during metadata modifications. Metadata
-on disk, in addition to the metadata database. When the option is enabled, every
+area size is currently roughly 224 MB per 1 TB of data. You can turn it off
-metadata entry is effectively stored in RAM twice. It's required for good performance
+to reduce memory usage by this value, but it will hurt performance. This
-because it allows to avoid additional read-modify-write cycles during metadata
+restriction is likely to be removed in the future along with the upgrade
-modifications. Metadata area size with the old store is roughly 224 MB per 1 TB
+of the metadata storage scheme.
-of data. You can turn the option off to reduce memory usage by this value, but
-it will reduce performance.
-
-For the new store ([meta_format](layout-osd.en.md#meta_format) 3), the option
-may be changed in the future to support operation without loading full metadata
-database in memory.
 
 ## inmemory_journal
 
@@ -371,8 +364,6 @@ blocks. The only situation when you should increase it to a larger value
 is when you enable journal_no_same_sector_overwrites. In this case set
 it to, for example, 1024.
 
-Not applicable to the new store ([meta_format](layout-osd.en.md#meta_format) 3).
-
 ## journal_no_same_sector_overwrites
 
 - Type: boolean
@@ -386,17 +377,6 @@ journal after writing it instead of possibly overwriting it the second time.
 
 Most (99%) other SSDs don't need this option.
 
-Not applicable to the new store ([meta_format](layout-osd.en.md#meta_format) 3).
-
-## skip_corrupted_meta_entries
-
-- Type: boolean
-- Default: false
-
-Only for the new store ([meta_format](layout-osd.en.md#meta_format) 3).
-Allow OSD to start when some metadata entries or blocks are corrupted by
-skipping them. Should be only used as an emergency measure.
-
 ## throttle_small_writes
 
 - Type: boolean

docs/config/osd.ru.md

@@ -39,7 +39,6 @@
 - [journal_io](#journal_io)
 - [journal_sector_buffer_count](#journal_sector_buffer_count)
 - [journal_no_same_sector_overwrites](#journal_no_same_sector_overwrites)
-- [skip_corrupted_meta_entries](#skip_corrupted_meta_entries)
 - [throttle_small_writes](#throttle_small_writes)
 - [throttle_target_iops](#throttle_target_iops)
 - [throttle_target_mbs](#throttle_target_mbs)
@@ -288,19 +287,13 @@ Flusher - это микро-поток (корутина), которая коп
 - Тип: булево (да/нет)
 - Значение по умолчанию: true
 
-Только для старого хранилища ([meta_format](layout-osd.en.md#meta_format) 2).
+Данный параметр заставляет Vitastor всегда держать область метаданных диска
-
+в памяти. Это нужно, чтобы избегать дополнительных операций чтения с диска
-Данный параметр заставляет Vitastor всегда держать копию области метаданных
+при записи. Размер области метаданных на данный момент составляет примерно
-в памяти в том же виде, как она лежит на диске, в дополнение к БД метаданных.
+224 МБ на 1 ТБ данных. При включении потребление памяти снизится примерно
-То есть, с включённой опцией каждая запись метаданных хранится в памяти дважды.
+на эту величину, но при этом также снизится и производительность. В будущем,
-Это нужно, чтобы избегать дополнительных операций чтения с диска при записи.
+после обновления схемы хранения метаданных, это ограничение, скорее всего,
-Размер области метаданных в старом хранилище составляет примерно 224 МБ на
+будет ликвидировано.
-1 ТБ данных. Вы можете отключить опцию, чтобы снизить потребление памяти
-примерно на эту величину, но при этом также снизится и производительность.
-
-Для нового хранилища ([meta_format](layout-osd.en.md#meta_format) 3) опция,
-возможно, будет переработана в будущем для поддержки работы без полной
-загрузки метаданных в памяти.
 
 ## inmemory_journal
 
@@ -383,8 +376,6 @@ fsync небезопасным даже с режимом "directsync".
 нужно менять - это если вы включаете journal_no_same_sector_overwrites. В
 этом случае установите данный параметр, например, в 1024.
 
-Неприменимо к новому хранилищу ([meta_format](layout-osd.en.md#meta_format) 3).
-
 ## journal_no_same_sector_overwrites
 
 - Тип: булево (да/нет)
@@ -400,18 +391,6 @@ fsync небезопасным даже с режимом "directsync".
 
 Почти все другие SSD (99% моделей) не требуют данной опции.
 
-Неприменимо к новому хранилищу ([meta_format](layout-osd.en.md#meta_format) 3).
-
-## skip_corrupted_meta_entries
-
-- Тип: булево (да/нет)
-- Значение по умолчанию: false
-
-Только для нового хранилища ([meta_format](layout-osd.en.md#meta_format) 3).
-Разрешить OSD запускаться, даже если часть блоков или записей метаданных
-повреждена, пропуская их. Опция предназначена для использования только в
-целях аварийного восстановления.
-
 ## throttle_small_writes
 
 - Тип: булево (да/нет)

docs/config/src/network.yml

@@ -84,6 +84,11 @@
     unsupported with old libibverbs < v32, like in Debian 10 Buster or
     CentOS 7.
 
+    Vitastor supports all adapters, even ones without ODP support, like
+    Mellanox ConnectX-3 and non-Mellanox cards. Versions up to Vitastor
+    1.2.0 required ODP which is only present in Mellanox ConnectX >= 4.
+    See also [rdma_odp](#rdma_odp).
+
     Run `ibv_devinfo -v` as root to list available RDMA devices and their
     features.
 
@@ -92,23 +97,6 @@
     the manual of your network vendor for details about setting up the switch
     for RoCEv2 correctly. Usually it means setting up Lossless Ethernet with
     PFC (Priority Flow Control) and ECN (Explicit Congestion Notification).
-
-    Vitastor supports all adapters, even ones without ODP (On-Demand Paging)
-    support, like Mellanox ConnectX-3 and non-Mellanox cards. ODP is only present
-    in Mellanox ConnectX >= 4 adapters and allows to skip memory registration
-    for RDMA and thus, in theory, avoid memory copying.
-
-    Versions up to Vitastor 1.2.0 required ODP, then it was disabled by default,
-    but it was still supported up to 3.0.3. Now ODP support is removed because it
-    actually only hurts performance: an example 3-node cluster with 8 NVMe in each
-    node and 2*25 GBit/s ConnectX-6 RDMA network pushed 3950000 read iops without
-    ODP, but only 239000 iops with ODP.
-
-    This happens because Mellanox ODP implementation seems to be based on
-    message retransmissions when the adapter doesn't know about the buffer yet -
-    it likely uses standard "RNR retransmissions" (RNR = receiver not ready)
-    which is generally slow in RDMA/RoCE networks. Here's a presentation about
-    it from ISPASS-2021 conference: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
   info_ru: |
     Название RDMA-устройства для связи с Vitastor OSD (например, "rocep5s0f0").
     Если не указано, Vitastor попробует найти RoCE-устройство, соответствующее
@@ -117,6 +105,12 @@
     не задана. Также автовыбор не поддерживается со старыми версиями библиотеки
     libibverbs < v32, например в Debian 10 Buster или CentOS 7.
 
+    Vitastor поддерживает все модели адаптеров, включая те, у которых
+    нет поддержки ODP, то есть вы можете использовать RDMA с ConnectX-3 и
+    картами производства не Mellanox. Версии Vitastor до 1.2.0 включительно
+    требовали ODP, который есть только на Mellanox ConnectX 4 и более новых.
+    См. также [rdma_odp](#rdma_odp).
+
     Запустите `ibv_devinfo -v` от имени суперпользователя, чтобы посмотреть
     список доступных RDMA-устройств, их параметры и возможности.
 
@@ -126,24 +120,6 @@
     коммутатора для RoCEv2 ищите в документации производителя. Обычно это
     подразумевает настройку сети без потерь на основе PFC (Priority Flow
     Control) и ECN (Explicit Congestion Notification).
-
-    Vitastor поддерживает все модели адаптеров, включая те, у которых нет
-    поддержки ODP (On-Demand Paging), например, ConnectX-3 и карты производства
-    не Mellanox. Функция ODP доступна только на адаптерах Mellanox ConnectX-4 и
-    более новых и позволяет не регистрировать память для её использования RDMA-картой,
-    благодаря чему в теории можно избежать лишних копирований памяти.
-
-    Версии Vitastor до 1.2.0 включительно требовали ODP, потом функция был отключена
-    по умолчанию, но поддерживалась вплоть до версии 3.0.3. Сейчас поддержка ODP
-    полностью удалена, так как на самом деле она только портит производительность:
-    например, на 3-узловом кластере с 8 NVMe в каждом узле и сетью 2*25 Гбит/с на
-    чтение с RDMA без ODP удаётся снять 3950000 iops, а с ODP - всего 239000 iops.
-
-    Это происходит из-за того, что реализация ODP у Mellanox неоптимальная и
-    основана на повторной передаче сообщений, когда карте не известен буфер -
-    вероятно, на стандартных "RNR retransmission" (RNR = receiver not ready).
-    А данные повторные передачи в RDMA/RoCE - всегда очень медленная штука.
-    Презентация на эту тему с конференции ISPASS-2021: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
 - name: rdma_port_num
   type: int
   info: |
@@ -242,6 +218,45 @@
     у принимающей стороны в процессе работы не заканчивались буферы на приём.
     Не влияет на потребление памяти - дополнительная память на операции отправки
     не выделяется.
+- name: rdma_odp
+  type: bool
+  default: false
+  online: false
+  info: |
+    Use RDMA with On-Demand Paging. ODP is currently only available on Mellanox
+    ConnectX-4 and newer adapters. ODP allows to not register memory explicitly
+    for RDMA adapter to be able to use it. This, in turn, allows to skip memory
+    copying during sending. One would think this should improve performance, but
+    **in reality** RDMA performance with ODP is **drastically** worse. Example
+    3-node cluster with 8 NVMe in each node and 2*25 GBit/s ConnectX-6 RDMA network
+    without ODP pushes 3950000 read iops, but only 239000 iops with ODP...
+
+    This happens because Mellanox ODP implementation seems to be based on
+    message retransmissions when the adapter doesn't know about the buffer yet -
+    it likely uses standard "RNR retransmissions" (RNR = receiver not ready)
+    which is generally slow in RDMA/RoCE networks. Here's a presentation about
+    it from ISPASS-2021 conference: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
+
+    ODP support is retained in the code just in case a good ODP implementation
+    appears one day.
+  info_ru: |
+    Использовать RDMA с On-Demand Paging. ODP - функция, доступная пока что
+    исключительно на адаптерах Mellanox ConnectX-4 и более новых. ODP позволяет
+    не регистрировать память для её использования RDMA-картой. Благодаря этому
+    можно не копировать данные при отправке их в сеть и, казалось бы, это должно
+    улучшать производительность - но **по факту** получается так, что
+    производительность только ухудшается, причём сильно. Пример - на 3-узловом
+    кластере с 8 NVMe в каждом узле и сетью 2*25 Гбит/с на чтение с RDMA без ODP
+    удаётся снять 3950000 iops, а с ODP - всего 239000 iops...
+
+    Это происходит из-за того, что реализация ODP у Mellanox неоптимальная и
+    основана на повторной передаче сообщений, когда карте не известен буфер -
+    вероятно, на стандартных "RNR retransmission" (RNR = receiver not ready).
+    А данные повторные передачи в RDMA/RoCE - всегда очень медленная штука.
+    Презентация на эту тему с конференции ISPASS-2021: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
+
+    Возможность использования ODP сохранена в коде на случай, если вдруг в один
+    прекрасный день появится хорошая реализация ODP.
 - name: peer_connect_interval
   type: sec
   min: 1

docs/config/src/osd.yml

@@ -253,33 +253,21 @@
   type: bool
   default: true
   info: |
-    Only for the old store ([meta_format](layout-osd.en.md#meta_format) 2).
+    This parameter makes Vitastor always keep metadata area of the block device
-
+    in memory. It's required for good performance because it allows to avoid
-    This parameter makes Vitastor keep a copy of metadata area in memory as it is
+    additional read-modify-write cycles during metadata modifications. Metadata
-    on disk, in addition to the metadata database. When the option is enabled, every
+    area size is currently roughly 224 MB per 1 TB of data. You can turn it off
-    metadata entry is effectively stored in RAM twice. It's required for good performance
+    to reduce memory usage by this value, but it will hurt performance. This
-    because it allows to avoid additional read-modify-write cycles during metadata
+    restriction is likely to be removed in the future along with the upgrade
-    modifications. Metadata area size with the old store is roughly 224 MB per 1 TB
+    of the metadata storage scheme.
-    of data. You can turn the option off to reduce memory usage by this value, but
-    it will reduce performance.
-
-    For the new store ([meta_format](layout-osd.en.md#meta_format) 3), the option
-    may be changed in the future to support operation without loading full metadata
-    database in memory.
   info_ru: |
-    Только для старого хранилища ([meta_format](layout-osd.en.md#meta_format) 2).
+    Данный параметр заставляет Vitastor всегда держать область метаданных диска
-
+    в памяти. Это нужно, чтобы избегать дополнительных операций чтения с диска
-    Данный параметр заставляет Vitastor всегда держать копию области метаданных
+    при записи. Размер области метаданных на данный момент составляет примерно
-    в памяти в том же виде, как она лежит на диске, в дополнение к БД метаданных.
+    224 МБ на 1 ТБ данных. При включении потребление памяти снизится примерно
-    То есть, с включённой опцией каждая запись метаданных хранится в памяти дважды.
+    на эту величину, но при этом также снизится и производительность. В будущем,
-    Это нужно, чтобы избегать дополнительных операций чтения с диска при записи.
+    после обновления схемы хранения метаданных, это ограничение, скорее всего,
-    Размер области метаданных в старом хранилище составляет примерно 224 МБ на
+    будет ликвидировано.
-    1 ТБ данных. Вы можете отключить опцию, чтобы снизить потребление памяти
-    примерно на эту величину, но при этом также снизится и производительность.
-
-    Для нового хранилища ([meta_format](layout-osd.en.md#meta_format) 3) опция,
-    возможно, будет переработана в будущем для поддержки работы без полной
-    загрузки метаданных в памяти.
 - name: inmemory_journal
   type: bool
   default: true
@@ -398,15 +386,11 @@
     blocks. The only situation when you should increase it to a larger value
     is when you enable journal_no_same_sector_overwrites. In this case set
     it to, for example, 1024.
-
-    Not applicable to the new store ([meta_format](layout-osd.en.md#meta_format) 3).
   info_ru: |
     Максимальное число буферов, разрешённых для использования под записываемые
     в журнал блоки метаданных. Единственная ситуация, в которой этот параметр
     нужно менять - это если вы включаете journal_no_same_sector_overwrites. В
     этом случае установите данный параметр, например, в 1024.
-
-    Неприменимо к новому хранилищу ([meta_format](layout-osd.en.md#meta_format) 3).
 - name: journal_no_same_sector_overwrites
   type: bool
   default: false
@@ -418,8 +402,6 @@
     journal after writing it instead of possibly overwriting it the second time.
 
     Most (99%) other SSDs don't need this option.
-
-    Not applicable to the new store ([meta_format](layout-osd.en.md#meta_format) 3).
   info_ru: |
     Включайте данную опцию для SSD вроде Intel D3-S4510 и D3-S4610, которые
     ОЧЕНЬ не любят, когда ПО перезаписывает один и тот же сектор несколько раз
@@ -430,20 +412,6 @@
     самого сектора.
 
     Почти все другие SSD (99% моделей) не требуют данной опции.
-
-    Неприменимо к новому хранилищу ([meta_format](layout-osd.en.md#meta_format) 3).
-- name: skip_corrupted_meta_entries
-  type: bool
-  default: false
-  info: |
-    Only for the new store ([meta_format](layout-osd.en.md#meta_format) 3).
-    Allow OSD to start when some metadata entries or blocks are corrupted by
-    skipping them. Should be only used as an emergency measure.
-  info_ru: |
-    Только для нового хранилища ([meta_format](layout-osd.en.md#meta_format) 3).
-    Разрешить OSD запускаться, даже если часть блоков или записей метаданных
-    повреждена, пропуская их. Опция предназначена для использования только в
-    целях аварийного восстановления.
 - name: throttle_small_writes
   type: bool
   default: false

docs/installation/docker.en.md

@@ -26,9 +26,9 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.
 
 1. Download a Docker image of the desired version: \
-   `docker pull vitalif/vitastor:v3.0.4`
+   `docker pull vitalif/vitastor:v3.0.2`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.4 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.2 install.sh`
 3. Reload udev rules: \
    `udevadm control --reload-rules`
 

docs/installation/docker.ru.md

@@ -25,9 +25,9 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.
 
 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitalif/vitastor:v3.0.4`
+   `docker pull vitalif/vitastor:v3.0.2`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.4 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.2 install.sh`
 3. Перезагрузите правила udev: \
    `udevadm control --reload-rules`
 

docs/installation/packages.en.md

@@ -33,17 +33,15 @@
   - CentOS 7: `yum install https://vitastor.io/rpms/centos/7/vitastor-release.rpm`
   - CentOS 8: `dnf install https://vitastor.io/rpms/centos/8/vitastor-release.rpm`
   - AlmaLinux 9 and other RHEL 9 clones (Rocky, Oracle...): `dnf install https://vitastor.io/rpms/centos/9/vitastor-release.rpm`
-  - AlmaLinux 10 and other RHEL 10 clones: `dnf install https://vitastor.io/rpms/centos/10/vitastor-release.rpm`
 - Enable EPEL: `yum/dnf install epel-release`
 - Enable additional CentOS repositories:
   - CentOS 7: `yum install centos-release-scl`
   - CentOS 8: `dnf install centos-release-advanced-virtualization`
-  - RHEL 9/10 clones: not required
+  - RHEL 9 clones: not required
 - Enable elrepo-kernel:
   - CentOS 7: `yum install https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm`
   - CentOS 8: `dnf install https://www.elrepo.org/elrepo-release-8.el8.elrepo.noarch.rpm`
   - RHEL 9 clones: `dnf install https://www.elrepo.org/elrepo-release-9.el9.elrepo.noarch.rpm`
-  - RHEL 10 clones: not required
 - Install packages: `yum/dnf install vitastor lpsolve etcd kernel-ml qemu-kvm`
 
 ## Installation requirements

docs/installation/packages.ru.md

@@ -33,17 +33,15 @@
   - CentOS 7: `yum install https://vitastor.io/rpms/centos/7/vitastor-release.rpm`
   - CentOS 8: `dnf install https://vitastor.io/rpms/centos/8/vitastor-release.rpm`
   - AlmaLinux 9 и другие клоны RHEL 9 (Rocky, Oracle...): `dnf install https://vitastor.io/rpms/centos/9/vitastor-release.rpm`
-  - AlmaLinux 10 и другие клоны RHEL 10: `dnf install https://vitastor.io/rpms/centos/10/vitastor-release.rpm`
 - Включите EPEL: `yum/dnf install epel-release`
 - Включите дополнительные репозитории CentOS:
   - CentOS 7: `yum install centos-release-scl`
   - CentOS 8: `dnf install centos-release-advanced-virtualization`
-  - Клоны RHEL 9/10: не нужно
+  - Клоны RHEL 9: не нужно
 - Включите elrepo-kernel:
   - CentOS 7: `yum install https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm`
   - CentOS 8: `dnf install https://www.elrepo.org/elrepo-release-8.el8.elrepo.noarch.rpm`
   - Клоны RHEL 9: `dnf install https://www.elrepo.org/elrepo-release-9.el9.elrepo.noarch.rpm`
-  - Клоны RHEL 10: не нужно
 - Установите пакеты: `yum/dnf install vitastor lpsolve etcd kernel-ml qemu-kvm`
 
 ## Установочные требования

mon/lp_optimizer/lp_optimizer.js

@@ -10,19 +10,16 @@ const NO_OSD = 'Z';
 async function lp_solve(text)
 {
     const cp = child_process.spawn('lp_solve');
-    let stdout = '', stderr = '', finish_cb, finished = 0;
+    let stdout = '', stderr = '', finish_cb;
     cp.stdout.on('data', buf => stdout += buf.toString());
     cp.stderr.on('data', buf => stderr += buf.toString());
-    cp.stdout.on('end', () => finish_cb());
+    cp.on('exit', () => finish_cb && finish_cb());
-    cp.stderr.on('end', () => finish_cb());
     cp.stdin.write(text);
     cp.stdin.end();
-    await new Promise(ok => (finish_cb = () =>
+    if (cp.exitCode == null)
     {
-        finished++;
+        await new Promise(ok => finish_cb = ok);
-        if (finished == 2)
+    }
-            ok();
-    }));
     if (!stdout.trim())
     {
         return null;

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "3.0.4",
+  "version": "3.0.2",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {
@@ -9,7 +9,7 @@
   "author": "Vitaliy Filippov",
   "license": "UNLICENSED",
   "dependencies": {
-    "antietcd": "^1.2.4",
+    "antietcd": "^1.2.2",
     "sprintf-js": "^1.1.2",
     "ws": "^7.2.5"
   },

mon/pg_gen.js

@@ -195,7 +195,7 @@ async function generate_pool_pgs(state, global_config, pool_id, osd_tree, levels
     const folded_tree = make_hier_tree(global_config, folded.nodes.reduce((a, c) => { a[c.id] = c; return a; }, {}));
     const old_pg_count = prev_pgs.length;
     const optimize_cfg = {
-        osd_weights: folded.nodes.reduce((a, c) => { if (/^\d+$/.exec(c.id) && c.size != null) { a[c.id] = c.size||0; } return a; }, {}),
+        osd_weights: folded.nodes.reduce((a, c) => { if (Number(c.id)) { a[c.id] = c.size; } return a; }, {}),
         combinator: use_rules
             // new algorithm:
             ? new RuleCombinator(folded_tree, rules, pool_cfg.max_osd_combinations)

node-binding/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor",
-  "version": "3.0.4",
+  "version": "3.0.2",
   "description": "Low-level native bindings to Vitastor client library",
   "main": "index.js",
   "keywords": [

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '3.0.4'
+VITASTOR_VERSION = '3.0.2'
 
 LOG = logging.getLogger(__name__)
 

rpm/build-tarball.sh

@@ -21,7 +21,7 @@ rpmbuild -bp fio.spec
 cd $VITASTOR
 VER=$(grep ^Version: rpm/vitastor-$REL.spec | awk '{print $2}')
 rm -rf fio
-ln -s $(ls -d ~/rpmbuild/BUILD/fio*/ | grep -v SPECPARTS) fio
+ln -s ~/rpmbuild/BUILD/fio*/ fio
 sh copy-fio-includes.sh
 rm fio
 mv fio-copy fio

rpm/vitastor-el10.Dockerfile

@@ -1,17 +0,0 @@
-# Build packages for AlmaLinux 10 inside a container
-# cd ..
-# docker pull --platform=linux/amd64/v2 quay.io/almalinuxorg/almalinux:10
-# docker build -t vitastor-buildenv:el10 -f rpm/vitastor-el10.Dockerfile .
-# docker run -i --rm -v ./:/root/vitastor vitastor-buildenv:el10 /root/vitastor/rpm/vitastor-build.sh
-
-FROM quay.io/almalinuxorg/almalinux:10
-
-WORKDIR /root
-
-RUN sed -i 's/enabled=0/enabled=1/' /etc/yum.repos.d/*.repo
-RUN dnf -y install epel-release dnf-plugins-core
-RUN dnf -y install https://vitastor.io/rpms/centos/10/vitastor-release-1.0-1.el10.noarch.rpm
-RUN dnf -y install gcc-c++ gperftools-devel fio nodejs rpm-build jerasure-devel isa-l-devel gf-complete-devel rdma-core-devel cmake libnl3-devel
-RUN dnf download --source fio
-RUN rpm --nomd5 -i fio*.src.rpm
-RUN cd ~/rpmbuild/SPECS && dnf builddep -y --spec fio.spec

rpm/vitastor-el10.spec

@@ -1,198 +0,0 @@
-Name:           vitastor
-Version:        3.0.4
-Release:        1%{?dist}
-Summary:        Vitastor, a fast software-defined clustered block storage
-
-License:        Vitastor Network Public License 1.1
-URL:            https://vitastor.io/
-Source0:        vitastor-3.0.4.el10.tar.gz
-
-BuildRequires:  gperftools-devel
-BuildRequires:  gcc-c++
-BuildRequires:  nodejs >= 10
-BuildRequires:  jerasure-devel
-BuildRequires:  isa-l-devel
-BuildRequires:  gf-complete-devel
-BuildRequires:  rdma-core-devel
-BuildRequires:  cmake
-BuildRequires:  libnl3-devel
-Requires:       vitastor-osd = %{version}-%{release}
-Requires:       vitastor-mon = %{version}-%{release}
-Requires:       vitastor-client = %{version}-%{release}
-Requires:       vitastor-client-devel = %{version}-%{release}
-Requires:       vitastor-fio = %{version}-%{release}
-
-%description
-Vitastor is a small, simple and fast clustered block storage (storage for VM drives),
-architecturally similar to Ceph which means strong consistency, primary-replication,
-symmetric clustering and automatic data distribution over any number of drives of any
-size with configurable redundancy (replication or erasure codes/XOR).
-
-
-%package -n vitastor-osd
-Summary:        Vitastor - OSD
-Requires:       vitastor-client = %{version}-%{release}
-Requires:       util-linux
-Requires:       parted
-
-
-%description -n vitastor-osd
-Vitastor object storage daemon, i.e. server program that stores data.
-
-
-%package -n vitastor-mon
-Summary:        Vitastor - monitor
-Requires:       nodejs >= 10
-Requires:       lpsolve
-
-
-%description -n vitastor-mon
-Vitastor monitor, i.e. server program responsible for watching cluster state and
-scheduling cluster-level operations.
-
-
-%package -n vitastor-client
-Summary:        Vitastor - client
-
-
-%description -n vitastor-client
-Vitastor client library and command-line interface.
-
-
-%package -n vitastor-client-devel
-Summary:        Vitastor - development files
-Group:          Development/Libraries
-Requires:       vitastor-client = %{version}-%{release}
-
-
-%description -n vitastor-client-devel
-Vitastor library headers for development.
-
-
-%package -n vitastor-fio
-Summary:        Vitastor - fio drivers
-Group:          Development/Libraries
-Requires:       vitastor-client = %{version}-%{release}
-Requires:       fio = 3.36-5.el10
-
-
-%description -n vitastor-fio
-Vitastor fio drivers for benchmarking.
-
-
-%package -n vitastor-opennebula
-Summary:        Vitastor for OpenNebula
-Group:          Development/Libraries
-Requires:       vitastor-client
-Requires:       jq
-Requires:       python3-lxml
-Requires:       patch
-Requires:       qemu-kvm-block-vitastor
-
-
-%description -n vitastor-opennebula
-Vitastor storage plugin for OpenNebula.
-
-
-%prep
-%setup -q
-
-
-%build
-%cmake
-%cmake_build
-
-
-%install
-rm -rf $RPM_BUILD_ROOT
-%cmake_install
-cd mon
-npm install --production
-cd ..
-mkdir -p %buildroot/usr/lib/vitastor
-cp -r mon %buildroot/usr/lib/vitastor
-mv %buildroot/usr/lib/vitastor/mon/scripts/make-etcd %buildroot/usr/lib/vitastor/mon/
-mkdir -p %buildroot/lib/systemd/system
-cp mon/scripts/vitastor.target mon/scripts/vitastor-mon.service mon/scripts/vitastor-osd@.service %buildroot/lib/systemd/system
-mkdir -p %buildroot/lib/udev/rules.d
-cp mon/scripts/90-vitastor.rules %buildroot/lib/udev/rules.d
-mkdir -p %buildroot/var/lib/one
-cp -r opennebula/remotes %buildroot/var/lib/one
-cp opennebula/install.sh %buildroot/var/lib/one/remotes/datastore/vitastor/
-mkdir -p %buildroot/etc/
-cp -r opennebula/sudoers.d %buildroot/etc/
-
-
-%files
-%doc GPL-2.0.txt VNPL-1.1.txt README.md README-ru.md
-
-
-%files -n vitastor-osd
-%_bindir/vitastor-osd
-%_bindir/vitastor-disk
-%_bindir/vitastor-dump-journal
-/lib/systemd/system/vitastor-osd@.service
-/lib/systemd/system/vitastor.target
-/lib/udev/rules.d/90-vitastor.rules
-
-
-%pre -n vitastor-osd
-groupadd -r -f vitastor 2>/dev/null ||:
-useradd -r -g vitastor -s /sbin/nologin -c "Vitastor daemons" -M -d /nonexistent vitastor 2>/dev/null ||:
-install -o vitastor -g vitastor -d /var/log/vitastor
-mkdir -p /etc/vitastor
-
-
-%files -n vitastor-mon
-/usr/lib/vitastor/mon
-/lib/systemd/system/vitastor-mon.service
-
-
-%pre -n vitastor-mon
-groupadd -r -f vitastor 2>/dev/null ||:
-useradd -r -g vitastor -s /sbin/nologin -c "Vitastor daemons" -M -d /nonexistent vitastor 2>/dev/null ||:
-mkdir -p /etc/vitastor
-mkdir -p /var/lib/vitastor
-chown vitastor:vitastor /var/lib/vitastor
-
-
-%files -n vitastor-client
-%_bindir/vitastor-nbd
-%_bindir/vitastor-ublk
-%_bindir/vitastor-nfs
-%_bindir/vitastor-cli
-%_bindir/vitastor-rm
-%_bindir/vitastor-kv
-%_bindir/vitastor-kv-stress
-%_bindir/vita
-%_libdir/libvitastor_client.so*
-%_libdir/libvitastor_kv.so*
-
-
-%files -n vitastor-client-devel
-%_includedir/vitastor_c.h
-%_includedir/vitastor_kv.h
-%_libdir/pkgconfig
-
-
-%files -n vitastor-fio
-%_libdir/libfio_vitastor.so
-%_libdir/libfio_vitastor_blk.so
-%_libdir/libfio_vitastor_sec.so
-
-
-%files -n vitastor-opennebula
-/var/lib/one
-/etc/sudoers.d/opennebula-vitastor
-
-
-%triggerin -n vitastor-opennebula -- opennebula
-[ $2 = 0 ] || exit 0
-/var/lib/one/remotes/datastore/vitastor/install.sh
-
-
-# Turn off the brp-python-bytecompile script
-%global __os_install_post %(echo '%{__os_install_post}' | sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g')
-
-
-%changelog

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.4
+Version:        3.0.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.4.el7.tar.gz
+Source0:        vitastor-3.0.2.el7.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  devtoolset-9-gcc-c++

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.4
+Version:        3.0.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.4.el8.tar.gz
+Source0:        vitastor-3.0.2.el8.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-toolset-9-gcc-c++

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.4
+Version:        3.0.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.4.el9.tar.gz
+Source0:        vitastor-3.0.2.el9.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++

src/CMakeLists.txt

@@ -21,7 +21,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 endif()
 set(ENABLE_COVERAGE false CACHE BOOL "Enable code coverage")
 
-add_definitions(-DVITASTOR_VERSION="3.0.4")
+add_definitions(-DVITASTOR_VERSION="3.0.2")
 add_definitions(-D_GNU_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -fvisibility=hidden -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/blockstore/blockstore.h

@@ -187,6 +187,7 @@ public:
     // MUST be called only when nobody makes any modifications to the DB for this pool
     virtual void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit) = 0;
     virtual bool reshard_continue(void *reshard_state, uint64_t chunk_limit) = 0;
+    virtual void reshard_abort(void *reshard_state) = 0;
 
     // Event loop
     virtual void loop() = 0;

src/blockstore/blockstore_disk.cpp

@@ -173,7 +173,9 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     }
     if (data_block_size / bitmap_granularity < 8)
     {
-        throw std::runtime_error("Data block size must be at least bitmap_granularity*8");
+        fprintf(stderr, "Warning: block_size (%u) / bitmap_granularity (%u) = %u bits. "
+            "Consider using larger block_size or bitmap_granularity for better performance.\n",
+            data_block_size, bitmap_granularity, data_block_size / bitmap_granularity);
     }
     if (!data_csum_type)
     {

src/blockstore/blockstore_heap.cpp

@@ -307,7 +307,6 @@ corrupted_block:
                 if (allow_corrupted)
                 {
                     fprintf(stderr, "Metadata block is corrupted, skipping\n");
-                    recheck_modified_blocks.insert(block_num);
                     break;
                 }
                 else
@@ -332,19 +331,7 @@ corrupted_block:
                     block_num, block_offset, wr->size, sizeof(heap_entry_t));
                 goto corrupted_block;
             }
-            if (wr->is_garbage())
+            wr->entry_type &= ~BS_HEAP_GARBAGE;
-            {
-                // Garbage collection is only performed when writing new entries into the block
-                // because it needs a fake LSN and modified blocks require consecutive modified LSNs
-                // That's why garbage entries may persist on disk
-                if (log_level > 5)
-                {
-                    fprintf(stderr, "Notice: skipping garbage entry %jx:%jx v%ju l%ju in metadata block %u at %u\n",
-                        wr->inode, wr->stripe, wr->version, wr->lsn, block_num, block_offset);
-                }
-                block_offset += wr->size;
-                continue;
-            }
             if ((wr->entry_type & BS_HEAP_TYPE) < BS_HEAP_BIG_WRITE ||
                 (wr->entry_type & BS_HEAP_TYPE) > BS_HEAP_ROLLBACK ||
                 (wr->entry_type & ~(BS_HEAP_TYPE|BS_HEAP_STABLE)) ||
@@ -358,7 +345,6 @@ corrupted_object:
                 if (allow_corrupted)
                 {
                     fprintf(stderr, "Entry is corrupted, skipping\n");
-                    recheck_modified_blocks.insert(block_num);
                     block_offset += wr->size;
                     continue;
                 }
@@ -374,7 +360,7 @@ corrupted_object:
             {
                 // Small writes require accessing offset & len to calculate correct length,
                 // so require at least sizeof(heap_small_write_t) for them
-                fprintf(stderr, "Error: entry %jx:%jx v%ju has invalid size in metadata block %u at %u (%u < min %zu bytes)\n",
+                fprintf(stderr, "Error: entry %jx:%jx v%ju has invalid size in metadata block %u at %u (%u < min %zu bytes). Metadata is corrupted, aborting\n",
                     wr->inode, wr->stripe, wr->version, block_num, block_offset, wr->size, sizeof(heap_small_write_t));
                 goto corrupted_object;
             }
@@ -388,8 +374,8 @@ corrupted_object:
             uint32_t expected_crc32c = wr->calc_crc32c();
             if (wr->crc32c != expected_crc32c)
             {
-                fprintf(stderr, "Error: entry %jx:%jx v%ju l%ju in metadata block %u at %u is corrupt (crc32c mismatch: expected %08x, got %08x). ",
+                fprintf(stderr, "Error: entry %jx:%jx v%ju in metadata block %u at %u is corrupt (crc32c mismatch: expected %08x, got %08x). Metadata is corrupted, aborting\n",
-                    wr->inode, wr->stripe, wr->version, wr->lsn,
+                    wr->inode, wr->stripe, wr->version,
                     block_num, block_offset, expected_crc32c, wr->crc32c);
                 goto corrupted_object;
             }
@@ -399,7 +385,7 @@ corrupted_object:
                 wr->small().offset % dsk->bitmap_granularity ||
                 wr->small().len % dsk->bitmap_granularity))
             {
-                fprintf(stderr, "Error: %s entry %jx:%jx v%ju has invalid offset/length: %u/%u. Metadata is incompatible with current parameters. ",
+                fprintf(stderr, "Error: %s entry %jx:%jx v%ju has invalid offset/length: %u/%u. Metadata is incompatible with current parameters, aborting\n",
                     wr->type() == BS_HEAP_SMALL_WRITE ? "small_write" : "intent_write",
                     wr->inode, wr->stripe, wr->version, wr->small().offset, wr->small().len);
                 goto corrupted_object;
@@ -409,7 +395,7 @@ corrupted_object:
                 wr->big_intent().offset % dsk->bitmap_granularity ||
                 wr->big_intent().len % dsk->bitmap_granularity))
             {
-                fprintf(stderr, "Error: big_intent entry %jx:%jx v%ju has invalid offset/length: %u/%u. Metadata is incompatible with current parameters. ",
+                fprintf(stderr, "Error: big_intent entry %jx:%jx v%ju has invalid offset/length: %u/%u. Metadata is incompatible with current parameters, aborting\n",
                     wr->inode, wr->stripe, wr->version, wr->big_intent().offset, wr->big_intent().len);
                 goto corrupted_object;
             }
@@ -436,7 +422,7 @@ int blockstore_heap_t::load_blocks(uint64_t disk_offset, uint64_t size, uint8_t
             next_lsn = wr->lsn;
         }
         entries_loaded++;
-        loaded_list_items.push_back(li);
+        insert_list_item(li);
         modify_alloc(block_num, [&](heap_block_info_t & inf)
         {
             if (!inf.entries.size())
@@ -472,22 +458,19 @@ bool blockstore_heap_t::validate_object(heap_entry_t *obj)
         next_wr = wr;
         if (wr->type() == BS_HEAP_ROLLBACK)
         {
+            if (commit_wr && wr->version > commit_wr->version)
+            {
+                // rollback may not come before commit with a smaller version
+                fprintf(stderr, "Error: rollback entry %jx:%jx v%ju l%ju comes before a commit entry v%ju l%ju\n",
+                    wr->inode, wr->stripe, wr->version, wr->lsn, commit_wr->version, commit_wr->lsn);
+                return false;
+            }
             rollback_wr = wr;
             continue;
         }
         if (wr->type() == BS_HEAP_COMMIT)
         {
-            if (commit_wr && wr->version > commit_wr->version)
+            commit_wr = wr;
-            {
-                // commit may not come before commit with a smaller version
-                fprintf(stderr, "Error: commit entry %jx:%jx v%ju l%ju comes before a commit entry v%ju l%ju\n",
-                    wr->inode, wr->stripe, wr->version, wr->lsn, commit_wr->version, commit_wr->lsn);
-                return false;
-            }
-            if (!commit_wr)
-            {
-                commit_wr = wr;
-            }
             continue;
         }
         if (wr->entry_type & BS_HEAP_STABLE)
@@ -540,23 +523,6 @@ bool blockstore_heap_t::validate_object(heap_entry_t *obj)
     return true;
 }
 
-void blockstore_heap_t::finish_load()
-{
-    if (loaded_list_items.size())
-    {
-        // Sort everything and load in correct order
-        std::sort(loaded_list_items.begin(), loaded_list_items.end(), [this](const heap_list_item_t* a, const heap_list_item_t* b)
-        {
-            return a->entry.lsn < b->entry.lsn;
-        });
-        for (auto & li: loaded_list_items)
-        {
-            insert_list_item(li);
-        }
-        loaded_list_items.clear();
-    }
-}
-
 void blockstore_heap_t::fill_recheck_queue()
 {
     for (auto & pgp: block_index)
@@ -743,7 +709,6 @@ bool blockstore_heap_t::recheck_small_writes(std::function<void(bool is_data, ui
     }
     if (!recheck_queue_filled)
     {
-        finish_load();
         fill_recheck_queue();
         recheck_queue_filled = true;
     }
@@ -835,7 +800,7 @@ std::vector<uint32_t> blockstore_heap_t::get_recheck_modified_blocks()
     return modified;
 }
 
-int blockstore_heap_t::finish_recheck()
+int blockstore_heap_t::finish_load(bool allow_corrupted)
 {
     if (!marked_used_blocks)
     {
@@ -872,6 +837,7 @@ bool blockstore_heap_t::calc_checksums(heap_entry_t *wr, uint8_t *data, bool set
         {
             return true;
         }
+        uint32_t len = 0;
         if (wr->type() == BS_HEAP_SMALL_WRITE || wr->type() == BS_HEAP_INTENT_WRITE)
             len = wr->small().len;
         else if (wr->type() == BS_HEAP_BIG_INTENT)
@@ -888,14 +854,13 @@ bool blockstore_heap_t::calc_checksums(heap_entry_t *wr, uint8_t *data, bool set
     }
     if (wr->type() == BS_HEAP_BIG_WRITE)
     {
-        assert(offset != UINT32_MAX && len != UINT32_MAX);
         return calc_block_checksums((uint32_t*)(wr->get_checksums(this) + offset/dsk->csum_block_size * (dsk->data_csum_type & 0xFF)),
             data, wr->get_int_bitmap(this), offset, offset+len, set, NULL);
     }
     if (wr->type() == BS_HEAP_BIG_INTENT)
     {
         auto & bi = wr->big_intent();
-        return calc_block_checksums((uint32_t*)(wr->get_checksums(this) + bi.offset/dsk->csum_block_size * (dsk->data_csum_type & 0xFF)),
+        return calc_block_checksums((uint32_t*)(wr->get_checksums(this) + offset/dsk->csum_block_size * (dsk->data_csum_type & 0xFF)),
             data, wr->get_int_bitmap(this), bi.offset, bi.offset+bi.len, set, NULL);
     }
     assert(wr->type() == BS_HEAP_SMALL_WRITE || wr->type() == BS_HEAP_INTENT_WRITE);
@@ -1112,6 +1077,16 @@ bool blockstore_heap_t::reshard_check(pool_id_t pool, uint32_t pg_count, uint32_
         set_it->second.pg_stripe_size == pg_stripe_size);
 }
 
+void blockstore_heap_t::reshard_abort(void* reshard_state)
+{
+    heap_reshard_state_t *st = (heap_reshard_state_t*)reshard_state;
+    for (auto sh_it = st->old_shards.begin(); sh_it != st->old_shards.end(); sh_it++)
+    {
+        block_index[sh_it->first] = std::move(sh_it->second);
+    }
+    delete st;
+}
+
 heap_entry_t *blockstore_heap_t::lock_and_read_entry(object_id oid)
 {
     auto obj = read_entry(oid);
@@ -1124,6 +1099,27 @@ heap_entry_t *blockstore_heap_t::lock_and_read_entry(object_id oid)
     return obj;
 }
 
+heap_entry_t *blockstore_heap_t::read_locked_entry(object_id oid, uint64_t lsn)
+{
+    auto obj = read_entry(oid);
+    assert(obj);
+    for (auto wr = obj; wr; wr = prev(wr))
+    {
+        if (wr->is_overwrite())
+        {
+            if (lsn == wr->lsn)
+            {
+                return obj;
+            }
+            else
+            {
+                obj = prev(wr);
+            }
+        }
+    }
+    return NULL;
+}
+
 bool blockstore_heap_t::unlock_entry(object_id oid)
 {
     auto mvcc_it = object_mvcc.find(oid);
@@ -1160,35 +1156,6 @@ heap_entry_t *blockstore_heap_t::read_entry(object_id oid)
     return &li->entry;
 }
 
-void blockstore_heap_t::gc_block(heap_block_info_t & inf)
-{
-    if (inf.has_garbage)
-    {
-        size_t i = 0, j = 0;
-        for (; i < inf.entries.size(); i++)
-        {
-            if (inf.entries[i]->entry.is_garbage())
-            {
-                // old entry invalidated by a newer one, mark it as freeable on block write
-                // assign a 'virtual' LSN to track GC completion
-                assert(!inf.mod_lsn_to || inf.mod_lsn_to == next_lsn);
-                uint64_t gc_lsn = ++next_lsn;
-                inf.mod_lsn = inf.mod_lsn ? inf.mod_lsn : gc_lsn;
-                inf.mod_lsn_to = gc_lsn;
-                push_inflight_lsn(gc_lsn, &inf.entries[i]->entry, HEAP_INFLIGHT_GC);
-            }
-            else
-            {
-                if (j != i)
-                    inf.entries[j] = inf.entries[i];
-                j++;
-            }
-        }
-        inf.entries.resize(j);
-        inf.has_garbage = false;
-    }
-}
-
 int blockstore_heap_t::allocate_entry(uint32_t entry_size, uint32_t *block_num, bool allow_last_free)
 {
     if (last_allocated_block != UINT32_MAX)
@@ -1255,7 +1222,31 @@ int blockstore_heap_t::allocate_entry(uint32_t entry_size, uint32_t *block_num,
     }
     // Write into the same block
     auto & inf = block_info.at(last_allocated_block);
-    gc_block(inf);
+    if (inf.has_garbage)
+    {
+        size_t i = 0, j = 0;
+        for (; i < inf.entries.size(); i++)
+        {
+            if (inf.entries[i]->entry.is_garbage())
+            {
+                // old entry invalidated by a newer one, mark it as freeable on block write
+                // assign a 'virtual' LSN to track GC completion
+                assert(!inf.mod_lsn_to || inf.mod_lsn_to == next_lsn);
+                uint64_t gc_lsn = ++next_lsn;
+                inf.mod_lsn = inf.mod_lsn ? inf.mod_lsn : gc_lsn;
+                inf.mod_lsn_to = gc_lsn;
+                push_inflight_lsn(gc_lsn, &inf.entries[i]->entry, HEAP_INFLIGHT_GC);
+            }
+            else
+            {
+                if (j != i)
+                    inf.entries[j] = inf.entries[i];
+                j++;
+            }
+        }
+        inf.entries.resize(j);
+        inf.has_garbage = false;
+    }
     *block_num = last_allocated_block;
     modify_alloc(last_allocated_block, [&](heap_block_info_t & inf)
     {
@@ -1356,6 +1347,7 @@ int blockstore_heap_t::add_small_write(object_id oid, heap_entry_t **obj_ptr, ui
     // Small writes are written in parallel with buffered data so they require explicit_complete
     return add_entry(wr_size, modified_block, false, true, [&](heap_entry_t *wr)
     {
+        printf("add_small_write t%u %lx:%lx l%lu v%lu %u +%u loc:%lx\n", type, oid.inode, oid.stripe, wr->lsn, version, offset, len, location);
         wr->entry_type = type;
         wr->inode = oid.inode;
         wr->stripe = oid.stripe;
@@ -1385,6 +1377,7 @@ int blockstore_heap_t::add_big_write(object_id oid, heap_entry_t *old_head, bool
     // Big writes are written after writing data so they don't require explicit_complete
     return add_entry(wr_size, modified_block, false, false, [&](heap_entry_t *wr)
     {
+        printf("add_big_write %lx:%lx l%lu v%lu loc:%lx\n", oid.inode, oid.stripe, wr->lsn, version, location);
         wr->entry_type = BS_HEAP_BIG_WRITE | (stable ? BS_HEAP_STABLE : 0);
         wr->inode = oid.inode;
         wr->stripe = oid.stripe;
@@ -1411,6 +1404,7 @@ int blockstore_heap_t::add_redirect_intent(object_id oid, heap_entry_t **obj_ptr
     // Big-redirect intents, just like regular big writes, are written after writing data so they don't require explicit_complete
     return add_entry(wr_size, modified_block, false, false, [&](heap_entry_t *wr)
     {
+        printf("add_redir_intent %lx:%lx l%lu v%lu %u +%u loc:%lx\n", oid.inode, oid.stripe, wr->lsn, version, offset, len, location);
         wr->entry_type = BS_HEAP_BIG_INTENT|BS_HEAP_STABLE;
         wr->inode = oid.inode;
         wr->stripe = oid.stripe;
@@ -1427,7 +1421,7 @@ int blockstore_heap_t::add_redirect_intent(object_id oid, heap_entry_t **obj_ptr
         bitmap_set(wr->get_int_bitmap(this), offset, len, dsk->bitmap_granularity);
         if (dsk->data_csum_type)
             memset(wr->get_checksums(this), 0, get_csum_size(wr));
-        calc_checksums(wr, (uint8_t*)data, true);
+        calc_checksums(wr, (uint8_t*)data, true, offset, len);
         *obj_ptr = wr;
     });
 }
@@ -1447,6 +1441,7 @@ int blockstore_heap_t::add_big_intent(object_id oid, heap_entry_t **obj_ptr, uin
     // Big intents are written before writing data so they require explicit_complete
     return add_entry(wr_size, modified_block, false, true, [&](heap_entry_t *wr)
     {
+        printf("add_big_intent %lx:%lx l%lu v%lu %u +%u loc:%lx\n", oid.inode, oid.stripe, wr->lsn, version, offset, len, obj->big_location(this));
         wr->entry_type = BS_HEAP_BIG_INTENT | BS_HEAP_STABLE;
         wr->inode = oid.inode;
         wr->stripe = oid.stripe;
@@ -1466,11 +1461,11 @@ int blockstore_heap_t::add_big_intent(object_id oid, heap_entry_t **obj_ptr, uin
         if (dsk->data_csum_type)
         {
             if (checksums)
-                memcpy(wr->get_checksums(this), checksums, get_csum_size(wr));
+                memcpy(wr->get_checksums(this), checksums, dsk->clean_entry_bitmap_size);
             else
             {
-                memcpy(wr->get_checksums(this), obj->get_checksums(this), get_csum_size(wr));
+                memcpy(wr->get_checksums(this), obj->get_checksums(this), dsk->clean_entry_bitmap_size);
-                calc_checksums(wr, (uint8_t*)data, true);
+                calc_checksums(wr, (uint8_t*)data, true, offset, len);
             }
         }
         else
@@ -1502,6 +1497,7 @@ int blockstore_heap_t::add_compact(heap_entry_t *obj, uint64_t compact_version,
     // Compaction entry is added after copying data so it doesn't require explicit_complete
     return add_entry(wr_size, modified_block, true, false, [&](heap_entry_t *new_wr)
     {
+        printf("add_compact %lx:%lx l%lu v%lu loc:%lx\n", obj->inode, obj->stripe, compact_lsn, compact_version, compact_location);
         new_wr->entry_type = BS_HEAP_BIG_WRITE|BS_HEAP_STABLE;
         new_wr->inode = obj->inode;
         new_wr->stripe = obj->stripe;
@@ -1591,7 +1587,7 @@ int blockstore_heap_t::add_commit(heap_entry_t *obj, uint64_t version, uint32_t
     }
     if (!uncommitted)
     {
-        return 0;
+        return EBUSY;
     }
     return add_simple(obj, version, modified_block, BS_HEAP_COMMIT);
 }
@@ -1601,32 +1597,23 @@ int blockstore_heap_t::add_rollback(heap_entry_t *obj, uint64_t version, uint32_
     heap_entry_t *wr = obj;
     bool found_uncommitted = false;
     uint64_t commit_version = 0;
-    uint64_t rollback_version = UINT64_MAX;
+    while (wr && !wr->is_overwrite())
-    while (wr)
     {
         if (wr->type() == BS_HEAP_ROLLBACK)
         {
-            if (wr->version <= version)
+            auto rollback_version = wr->version;
-            {
-                // All previous writes are already rolled back, stop
-                break;
-            }
-            rollback_version = wr->version;
             wr = prev(wr);
+            while (wr->version > rollback_version)
+            {
+                assert(!(wr->entry_type & BS_HEAP_STABLE));
+                wr = prev(wr);
+            }
             continue;
         }
         if (wr->type() == BS_HEAP_COMMIT)
         {
             if (commit_version < wr->version)
-            {
                 commit_version = wr->version;
-            }
-            wr = prev(wr);
-            continue;
-        }
-        if (wr->version > rollback_version)
-        {
-            // Already rolled back, skip
             wr = prev(wr);
             continue;
         }
@@ -1637,10 +1624,14 @@ int blockstore_heap_t::add_rollback(heap_entry_t *obj, uint64_t version, uint32_
             {
                 return EBUSY;
             }
-            else
+            else if (wr->version == version)
             {
                 break;
             }
+            else if (wr->version < version)
+            {
+                return ENOENT;
+            }
         }
         else if (wr->version > version)
         {
@@ -1821,9 +1812,9 @@ void blockstore_heap_t::iterate_with_stable(heap_entry_t *obj, uint64_t max_lsn,
         }
         else
         {
-            // 1) 1 2 3 ROLLBACK(2) COMMIT(3) -> 3 is unstable
+            // 1) 1 2 3 ROLLBACK(2) COMMIT(3) -> impossible
             // 2) 1 2 3 4 ROLLBACK(3) COMMIT(2) -> OK
-            // 3) 1 2 3 ROLLBACK(2) 3 COMMIT(3) -> first 3 is unstable
+            // 3) 1 2 3 ROLLBACK(2) 3 COMMIT(3) -> first 3 shouldn't be treated as stable
             // 4) 1 2 3 COMMIT(3) ROLLBACK(2) -> impossible
             //    I.e. a rollback always has version >= previous commit
             // 5) 1 2 3 4 5 ROLLBACK(4) 5 ROLLBACK(3)
@@ -1980,21 +1971,24 @@ int blockstore_heap_t::list_objects(uint32_t pg_num, object_id min_oid, object_i
                 return;
             }
             uint64_t stable_version = 0;
-            iterate_with_stable(obj, UINT64_MAX, [&](heap_entry_t* wr, bool stable)
+            auto first_wr = obj;
+            for (auto wr = first_wr; wr; wr = prev(wr))
             {
-                if (stable)
+                if ((wr->entry_type & BS_HEAP_STABLE) || wr->type() == BS_HEAP_COMMIT || wr->type() == BS_HEAP_ROLLBACK)
                 {
                     stable_version = wr->version;
-                    return false;
+                    break;
                 }
-                if (unstable_size >= unstable_alloc)
+                else
                 {
-                    unstable_alloc = (!unstable_alloc ? 128 : unstable_alloc*2);
+                    if (unstable_size >= unstable_alloc)
-                    unstable = (obj_ver_id*)realloc_or_die(unstable, sizeof(obj_ver_id) * unstable_alloc);
+                    {
+                        unstable_alloc = (!unstable_alloc ? 128 : unstable_alloc*2);
+                        unstable = (obj_ver_id*)realloc_or_die(unstable, sizeof(obj_ver_id) * unstable_alloc);
+                    }
+                    unstable[unstable_size++] = (obj_ver_id){ .oid = oid, .version = wr->version };
                 }
-                unstable[unstable_size++] = (obj_ver_id){ .oid = oid, .version = wr->version };
+            }
-                return true;
-            });
             if (stable_version)
             {
                 if (res_size >= res_alloc)
@@ -2056,13 +2050,7 @@ void blockstore_heap_t::free_data(inode_t inode, uint64_t location)
         inode = (INODE_POOL(inode) << POOL_ID_BITS);
     assert(data_alloc->get(location / dsk->data_block_size));
     data_alloc->set(location / dsk->data_block_size, false);
-    auto sp_it = inode_space_stats.find(inode);
+    inode_space_stats[inode] -= dsk->data_block_size;
-    if (sp_it != inode_space_stats.end())
-    {
-        sp_it->second -= dsk->data_block_size;
-        if (sp_it->second == 0)
-            inode_space_stats.erase(sp_it);
-    }
     data_used_space -= dsk->data_block_size;
 }
 
@@ -2345,7 +2333,7 @@ void blockstore_heap_t::set_no_inode_stats(const std::vector<uint64_t> & pool_id
     {
         // Recalculate if changed
         if (ps.second.no_inode_stats == 2 || ps.second.no_inode_stats == 1)
-            recalc_inode_space_stats(ps.first, ps.second.no_inode_stats == 2);
+            recalc_inode_space_stats(ps.first, ps.second.no_inode_stats == 1);
         ps.second.no_inode_stats &= 1;
     }
 }
@@ -2356,8 +2344,8 @@ void blockstore_heap_t::recalc_inode_space_stats(uint64_t pool_id, bool per_inod
     auto sp_begin = inode_space_stats.lower_bound((pool_id << (64-POOL_ID_BITS)));
     auto sp_end = inode_space_stats.lower_bound(((pool_id+1) << (64-POOL_ID_BITS)));
     inode_space_stats.erase(sp_begin, sp_end);
-    uint32_t pg_count = ps.pg_count;
+    uint32_t pg_count = ps.pg_count ? ps.pg_count : 1;
-    for (uint32_t pg_num = pg_count ? 1 : 0; pg_num <= pg_count; pg_num++)
+    for (uint32_t pg_num = 1; pg_num <= pg_count; pg_num++)
     {
         auto & pg_idx = block_index[(pool_id << (64-POOL_ID_BITS)) | pg_num];
         for (auto & ip: pg_idx)
@@ -2395,10 +2383,7 @@ void inode_map_get(void *inode_idx, heap_inode_map_t::iterator & li_it, heap_lis
     size_t map_n = ((size_t)inode_idx & IMAP_MALLOC_LOW_BITS);
     if (!map_n)
     {
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Warray-bounds"
         li_it = ((heap_inode_map_t*)inode_idx)->find(list_item_key(&stripe));
-#pragma GCC diagnostic pop
         li = li_it != ((heap_inode_map_t*)inode_idx)->end() ? *li_it : NULL;
     }
     else if (map_n == 1)

src/blockstore/blockstore_heap.h

@@ -200,7 +200,6 @@ class blockstore_heap_t
 
     bool marked_used_blocks = false;
     bool recheck_queue_filled = false;
-    std::vector<heap_list_item_t*> loaded_list_items;
     std::set<uint32_t> recheck_modified_blocks;
     std::deque<heap_entry_t*> recheck_queue;
     int recheck_in_progress = 0;
@@ -216,7 +215,6 @@ class blockstore_heap_t
     void defragment_block(uint32_t block_num);
     void reshard_add(heap_reshard_state_t *st, heap_list_item_t *li);
 
-    void gc_block(heap_block_info_t & inf);
     int allocate_entry(uint32_t entry_size, uint32_t *block_num, bool allow_last_free);
     void insert_list_item(heap_list_item_t *li);
     int add_entry(uint32_t wr_size, uint32_t *modified_block, bool allow_last_free,
@@ -239,29 +237,31 @@ public:
         std::function<void(uint32_t, uint32_t, uint8_t*)> handle_block);
     int load_blocks(uint64_t disk_offset, uint64_t size, uint8_t *buf,
         bool allow_corrupted, uint64_t &entries_loaded);
-    // finish loading - should be called after load_blocks
+    // finish loading
-    void finish_load();
+    int finish_load(bool allow_corrupted = false);
     // get blocks which are modified during loading and should be written to the disk
     // before finishing initialization if not R/O
     std::vector<uint32_t> get_recheck_modified_blocks();
     // recheck small write data after reading the database from disk
     bool recheck_small_writes(std::function<void(bool is_data, uint64_t offset, uint64_t len, uint8_t* buf, std::function<void()>)> read_buffer, int queue_depth);
-    int finish_recheck();
     // reshard database according to the pool's PG count
     void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
     bool reshard_continue(void* reshard_state, uint64_t chunk_limit);
     bool reshard_check(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size);
+    void reshard_abort(void* reshard_state);
     void set_no_inode_stats(const std::vector<uint64_t> & pool_ids);
     void recalc_inode_space_stats(uint64_t pool_id, bool per_inode);
     // read an object entry and lock it against removal
     // in the future, may become asynchronous
     heap_entry_t *lock_and_read_entry(object_id oid);
+    // re-read a locked object entry with the given lsn (pointer may be invalidated)
+    heap_entry_t *read_locked_entry(object_id oid, uint64_t lsn);
     // read an object entry without locking it
     heap_entry_t *read_entry(object_id oid);
     // unlock an entry
     bool unlock_entry(object_id oid);
     // set or verify checksums in a write request
-    bool calc_checksums(heap_entry_t *wr, uint8_t *data, bool set, uint32_t offset = UINT32_MAX, uint32_t len = UINT32_MAX);
+    bool calc_checksums(heap_entry_t *wr, uint8_t *data, bool set, uint32_t offset = 0, uint32_t len = 0);
     // set or verify raw block checksums
     bool calc_block_checksums(uint32_t *block_csums, uint8_t *data, uint8_t *bitmap, uint32_t start, uint32_t end,
         bool set, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);

src/blockstore/blockstore_impl.cpp

@@ -406,3 +406,8 @@ bool blockstore_impl_t::reshard_continue(void *reshard_state, uint64_t chunk_lim
 {
     return heap->reshard_continue(reshard_state, chunk_limit);
 }
+
+void blockstore_impl_t::reshard_abort(void *reshard_state)
+{
+    return heap->reshard_abort(reshard_state);
+}

src/blockstore/blockstore_impl.h

@@ -78,7 +78,6 @@ public:
     // Suitable only for server SSDs with capacitors, requires disabled data and journal fsyncs
     int immediate_commit = IMMEDIATE_NONE;
     bool inmemory_meta = false;
-    bool skip_corrupted_meta_entries = false;
     uint32_t meta_write_recheck_parallelism = 0;
     // Maximum and minimum flusher count
     unsigned max_flusher_count = 0, min_flusher_count = 0;
@@ -192,6 +191,7 @@ public:
 
     void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
     bool reshard_continue(void *reshard_state, uint64_t chunk_limit);
+    void reshard_abort(void *reshard_state);
 
     // Event loop
     void loop();

src/blockstore/blockstore_init.cpp

@@ -225,7 +225,7 @@ resume_4:
         {
             // Handle result
             uint64_t loaded = 0;
-            int r = bs->heap->load_blocks(bufs[i].offset-bs->dsk.meta_block_size, bufs[i].size, bufs[i].buf, bs->skip_corrupted_meta_entries, loaded);
+            int r = bs->heap->load_blocks(bufs[i].offset-bs->dsk.meta_block_size, bufs[i].size, bufs[i].buf, false, loaded);
             if (r != 0)
                 exit(1);
             entries_loaded += loaded;
@@ -239,7 +239,6 @@ resume_4:
         return 1;
     }
     // metadata read finished
-    bs->heap->finish_load();
     printf("Metadata entries loaded: %ju, used blocks: %ju / %ju\n", entries_loaded, bs->heap->get_data_used_space() / bs->dsk.data_block_size, bs->dsk.block_count);
     if (zero_on_init && !bs->dsk.disable_meta_fsync)
     {
@@ -285,7 +284,7 @@ resume_6:
     }, bs->meta_write_recheck_parallelism);
     return 1;
 resume_7:
-    if (bs->heap->finish_recheck() != 0)
+    if (bs->heap->finish_load() != 0)
     {
         exit(1);
     }

src/blockstore/blockstore_open.cpp

@@ -28,7 +28,6 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init)
     throttle_target_parallelism = strtoull(config["throttle_target_parallelism"].c_str(), NULL, 10);
     throttle_threshold_us = strtoull(config["throttle_threshold_us"].c_str(), NULL, 10);
     perfect_csum_update = config["perfect_csum_update"] == "true" || config["perfect_csum_update"] == "1" || config["perfect_csum_update"] == "yes";
-    skip_corrupted_meta_entries = config["skip_corrupted_meta_entries"] == "true" || config["skip_corrupted_meta_entries"] == "1" || config["skip_corrupted_meta_entries"] == "yes";
     if (config["autosync_writes"] != "")
     {
         autosync_writes = strtoull(config["autosync_writes"].c_str(), NULL, 10);

src/blockstore/blockstore_stable.cpp

@@ -57,9 +57,9 @@ int blockstore_impl_t::dequeue_stable(blockstore_op_t *op)
             }
             assert(res == 0);
         }
-resume_1:
         if (priv->modified_block != UINT32_MAX && priv->modified_block2 != priv->modified_block)
         {
+resume_1:
             BS_SUBMIT_CHECK_SQES(1);
             prepare_meta_block_write(priv->modified_block);
 resume_2:

src/blockstore/blockstore_write.cpp

@@ -13,10 +13,13 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
     return true;
 }
 
-void blockstore_impl_t::prepare_meta_block_write(uint32_t modified_block)
+bool blockstore_impl_t::prepare_meta_block_write(uint32_t modified_block)
 {
-    if (modified_blocks.find(modified_block) != modified_blocks.end())
+    auto mod_it = modified_blocks.find(modified_block);
-        return;
+    if (mod_it != modified_blocks.end())
+    {
+        return !mod_it->second.sent;
+    }
     io_uring_sqe *sqe = get_sqe();
     assert(sqe != NULL);
     ring_data_t *data = ((ring_data_t*)sqe->user_data);
@@ -41,6 +44,7 @@ void blockstore_impl_t::prepare_meta_block_write(uint32_t modified_block)
     unsynced_meta_write_count++;
     pending_modified_blocks.push_back(modified_block);
     modified_blocks[modified_block] = { .sent = false, .buf = buf };
+    return true;
 }
 
 bool blockstore_impl_t::meta_block_is_pending(uint32_t modified_block)
@@ -121,6 +125,7 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
     heap_entry_t *obj = heap->read_entry(op->oid);
     if (op->opcode == BS_OP_DELETE)
     {
+        return continue_delete(op, 0);
         // Delete
         if (!obj || obj->type() == BS_HEAP_DELETE)
         {
@@ -133,17 +138,38 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
         BS_SUBMIT_CHECK_SQES(1);
         int res = heap->add_delete(obj, &PRIV(op)->modified_block);
         if (res == ENOSPC)
+        {
             goto enospc;
+        }
         assert(res == 0);
-        prepare_meta_block_write(PRIV(op)->modified_block);
-        PRIV(op)->pending_ops++;
-        PRIV(op)->op_state = 5;
         write_iodepth++;
+resume_1:
+        while (!prepare_meta_block_write(PRIV(op)->modified_block))
+        {
+            PRIV(op)->op_state = 1;
+            return 1;
+        }
+rseume_2:
+        while (meta_block_is_pending(PRIV(op)->modified_block))
+        {
+            PRIV(op)->op_state = 2;
+            return 1;
+        }
+resume_3:
+resume_4:
+        if (!throttle_write(op, 3))
+        {
+            return 1;
+        }
+        write_iodepth--;
+        ack_write(op);
+        return 2;
     }
     // FIXME: Allow to do initial writes as buffered, not redirected
     // FIXME: Allow to do direct writes over holes
     else if (!obj || obj->type() == BS_HEAP_DELETE || op->offset == 0 && op->len == dsk.data_block_size)
     {
+        return continue_big_write(op, 10);
         // Big (redirect) write
         PRIV(op)->write_type = dsk.disable_data_fsync || op->opcode != BS_OP_WRITE_STABLE ? BS_HEAP_BIG_WRITE : _REDIRECT_INTENT;
         BS_SUBMIT_CHECK_SQES(1);
@@ -163,6 +189,7 @@ enospc:
             flusher->request_trim();
             return 0;
         }
+        write_iodepth++;
         uint64_t loc = PRIV(op)->location;
 #ifdef BLOCKSTORE_DEBUG
         printf(
@@ -176,18 +203,72 @@ enospc:
         data->iov = (struct iovec){ op->buf, op->len };
         data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
         io_uring_prep_writev(sqe, dsk.data_fd, &data->iov, 1, dsk.data_offset + loc + op->offset);
+        if (PRIV(op)->write_type == BS_HEAP_BIG_WRITE)
+            inflight_big++;
         PRIV(op)->pending_ops++;
-        write_iodepth++;
+resume_10:
+        if (PRIV(op)->pending_ops > 0)
+        {
+            PRIV(op)->op_state = 10;
+            return 1;
+        }
         if (PRIV(op)->write_type == BS_HEAP_BIG_WRITE)
         {
-            PRIV(op)->op_state = 1;
+            inflight_big--;
-            inflight_big++;
+resume_11:
+resume_12:
+resume_13:
+            if (!fsync_big_write(op, 11))
+                return 1;
+        }
+        heap_entry_t *obj = heap->read_entry(op->oid);
+        int res = 0;
+        if (PRIV(op)->write_type == _REDIRECT_INTENT)
+        {
+            res = heap->add_redirect_intent(op->oid, &obj, op->version, op->offset, op->len,
+                PRIV(op)->location, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
         }
         else
-            PRIV(op)->op_state = 3;
+        {
+            res = heap->add_big_write(op->oid, obj, op->opcode == BS_OP_WRITE_STABLE,
+                op->version, op->offset, op->len, PRIV(op)->location, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
+        }
+        if (res == ENOSPC)
+        {
+            if (!heap->get_to_compact_count())
+            {
+                // no space
+                heap->free_data(op->oid.inode, PRIV(op)->location);
+                write_iodepth--;
+                op->retval = -ENOSPC;
+                FINISH_OP(op);
+                return 2;
+            }
+            PRIV(op)->wait_for = WAIT_COMPACTION;
+            PRIV(op)->wait_detail = heap->get_compacted_count();
+            flusher->request_trim();
+            return 0;
+        }
+        assert(res == 0);
+resume_14:
+        while (!prepare_meta_block_write(PRIV(op)->modified_block))
+        {
+            PRIV(op)->op_state = 14;
+            return 1;
+        }
+resume_15:
+        while (meta_block_is_pending(PRIV(op)->modified_block))
+        {
+            PRIV(op)->op_state = 15;
+            return 1;
+        }
+        write_iodepth--;
+        ack_write(op);
+        return 2;
     }
     else if (intent_write_allowed(op, obj))
     {
+        return continue_intent_write(op, 20);
         // Direct intent-write
         BS_SUBMIT_CHECK_SQES(1);
         int res = 0;
@@ -223,13 +304,41 @@ enospc:
             assert(res == 0);
             PRIV(op)->lsn = obj->lsn;
         }
-        prepare_meta_block_write(PRIV(op)->modified_block);
-        PRIV(op)->pending_ops++;
-        PRIV(op)->op_state = 9;
         write_iodepth++;
+resume_20:
+        while (!prepare_meta_block_write(PRIV(op)->modified_block))
+        {
+            PRIV(op)->op_state = 20;
+            return 1;
+        }
+resume_21:
+        while (meta_block_is_pending(PRIV(op)->modified_block))
+        {
+            PRIV(op)->op_state = 21;
+            return 1;
+        }
+        // Direct intent-write
+        // LSN is not marked as completed so big_write won't be freed
+        BS_SUBMIT_GET_SQE(sqe, data);
+        data->iov = (struct iovec){ op->buf, op->len };
+        data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
+        io_uring_prep_writev(sqe, dsk.data_fd, &data->iov, 1, dsk.data_offset + PRIV(op)->location + op->offset);
+        if (dsk.use_atomic_flag)
+            sqe->rw_flags = RWF_ATOMIC;
+        PRIV(op)->pending_ops++;
+resume_22:
+        if (PRIV(op)->pending_ops > 0)
+        {
+            PRIV(op)->op_state = 22;
+            return 1;
+        }
+        write_iodepth--;
+        ack_write(op);
+        return 2;
     }
     else
     {
+        return continue_small_write(op, 30);
         // Small (buffered) overwrite
         // First check if there is free buffer space
         PRIV(op)->write_type = BS_HEAP_SMALL_WRITE;
@@ -242,7 +351,7 @@ enospc:
             return 0;
         }
         // There is sufficient space. Check SQE(s)
-        BS_SUBMIT_CHECK_SQES(1 + (op->len > 0 ? 1 : 0));
+        BS_SUBMIT_CHECK_SQES(1 + (op->len > 0 ? 1 : 0)); ---> refactor too
         int res = heap->add_small_write(op->oid, &obj, (BS_HEAP_SMALL_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0)),
             op->version, op->offset, op->len, loc, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
         if (res == ENOSPC)
@@ -311,27 +420,46 @@ again:
         goto again;
     }
 resume_2:
+
+
+resume_4:
+resume_6:
+
+resume_8:
+    ack
+    return 2;
+resume_10:
+    return 1;
+}
+
+bool blockstore_impl_t::fsync_big_write(blockstore_op_t *op, int base_state)
+{
+    if (PRIV(op)->state == base_state)
+        goto resume_0;
+    else if (PRIV(op)->state == base_state+1)
+        goto resume_1;
+    else if (PRIV(op)->state == base_state+2)
+        goto resume_2;
     // We must fsync all big writes to avoid complex write workflows
     // It's OK for all HDDs and for server SSDs, but slightly worse for desktop SSDs
-    inflight_big--;
     if (!dsk.disable_data_fsync)
     {
         // fsync data in a batch
-resume_11:
+resume_0:
         if (inflight_big > 0)
         {
-            PRIV(op)->op_state = 11;
+            PRIV(op)->op_state = base_state;
-            return 1;
+            return false;
         }
         if (fsyncing_data)
         {
-resume_12:
+resume_1:
             if (fsyncing_data)
             {
-                PRIV(op)->op_state = 12;
+                PRIV(op)->op_state = base_state+1;
-                return 1;
+                return false;
             }
-            goto resume_4;
+            return true;
         }
         fsyncing_data = true;
         BS_SUBMIT_GET_SQE(sqe, data);
@@ -343,47 +471,23 @@ resume_12:
             handle_write_event(data, op);
         };
         PRIV(op)->pending_ops++;
-        PRIV(op)->op_state = 3;
+resume_2:
-        return 1;
+        if (PRIV(op)->pending_ops > 0)
+        {
+            PRIV(op)->op_state = base_state+2;
+            return false;
+        }
     }
-resume_4:
+    return true;
-    {
+}
-        auto obj = heap->read_entry(op->oid);
+
-        int res = 0;
+bool blockstore_impl_t::throttle_write(blockstore_op_t *op, int base_state)
-        if (PRIV(op)->write_type == _REDIRECT_INTENT)
+{
-        {
-            res = heap->add_redirect_intent(op->oid, &obj, op->version, op->offset, op->len,
-                PRIV(op)->location, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
-        }
-        else
-        {
-            res = heap->add_big_write(op->oid, obj, op->opcode == BS_OP_WRITE_STABLE,
-                op->version, op->offset, op->len, PRIV(op)->location, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
-        }
-        if (res == ENOSPC)
-        {
-            if (!heap->get_to_compact_count())
-            {
-                // no space
-                heap->free_data(op->oid.inode, PRIV(op)->location);
-                write_iodepth--;
-                op->retval = -ENOSPC;
-                FINISH_OP(op);
-                return 2;
-            }
-            PRIV(op)->wait_for = WAIT_COMPACTION;
-            PRIV(op)->wait_detail = heap->get_compacted_count();
-            flusher->request_trim();
-            return 0;
-        }
-        assert(res == 0);
-        prepare_meta_block_write(PRIV(op)->modified_block);
-        PRIV(op)->pending_ops++;
-        PRIV(op)->op_state = 5;
-        return 1;
-    }
-resume_6:
     // Apply throttling to not fill the journal too quickly for the SSD+HDD case
+    if (PRIV(op)->op_state >= base_state+1)
+    {
+        return true;
+    }
     if (PRIV(op)->write_type == BS_HEAP_SMALL_WRITE && throttle_small_writes)
     {
         // Apply throttling
@@ -404,18 +508,21 @@ resume_6:
         if (ref_us > exec_us + throttle_threshold_us)
         {
             // Pause reply
-            PRIV(op)->pending_ops++;
+            PRIV(op)->op_state = base_state;
-            PRIV(op)->op_state = 7;
             // Remember that the timer can in theory be called right here
             tfd->set_timer_us(ref_us-exec_us, false, [this, op](int timer_id)
             {
-                PRIV(op)->pending_ops--;
+                PRIV(op)->op_state++;
                 ringloop->wakeup();
             });
-            return 1;
+            return false;
         }
     }
-resume_8:
+    return true;
+}
+
+void blockstore_impl_t::ack_write(blockstore_op_t *op)
+{
     // Acknowledge write
 #ifdef BLOCKSTORE_DEBUG
     printf("Ack write %jx:%jx v%ju\n", op->oid.inode, op->oid.stripe, op->version);
@@ -442,21 +549,7 @@ resume_8:
         unsynced_data_write_count++;
         intent_write_counter++;
     }
-    write_iodepth--;
     FINISH_OP(op);
-    return 2;
-resume_10:
-    // Direct intent-write
-    // LSN is not marked as completed so big_write won't be freed
-    BS_SUBMIT_GET_SQE(sqe, data);
-    data->iov = (struct iovec){ op->buf, op->len };
-    data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
-    io_uring_prep_writev(sqe, dsk.data_fd, &data->iov, 1, dsk.data_offset + PRIV(op)->location + op->offset);
-    if (dsk.use_atomic_flag)
-        sqe->rw_flags = RWF_ATOMIC;
-    PRIV(op)->pending_ops++;
-    PRIV(op)->op_state = 7;
-    return 1;
 }
 
 void blockstore_impl_t::handle_write_event(ring_data_t *data, blockstore_op_t *op)

src/blockstore/v1/impl.cpp

@@ -480,6 +480,17 @@ resume_1:
     return true;
 }
 
+void blockstore_impl_t::reshard_abort(void *reshard_state)
+{
+    bs_reshard_state_t *st = (bs_reshard_state_t*)reshard_state;
+    for (auto sh_it = st->old_shards.begin(); sh_it != st->old_shards.end(); sh_it++)
+    {
+        auto & to = clean_db_shards[sh_it->first];
+        to.swap(sh_it->second);
+    }
+    delete st;
+}
+
 void blockstore_impl_t::process_list(blockstore_op_t *op)
 {
     uint32_t list_pg = op->pg_number+1;

src/blockstore/v1/impl.h

@@ -290,6 +290,7 @@ public:
     // Reshard database for a pool
     void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
     bool reshard_continue(void *reshard_state, uint64_t chunk_limit);
+    void reshard_abort(void *reshard_state);
 
     // Event loop
     void loop();

src/client/etcd_state_client.cpp

@@ -414,10 +414,7 @@ void etcd_state_client_t::start_etcd_watcher()
                 }
                 // Save revision only if it's present in the message - because sometimes etcd sends something without a header, like:
                 // {"error": {"grpc_code": 14, "http_code": 503, "http_status": "Service Unavailable", "message": "error reading from server: EOF"}}
-                // Also don't save revision from the initial created: true messages because they always contain the latest revision
+                if (etcd_watches_initialised == ETCD_TOTAL_WATCHES && !data["result"]["header"]["revision"].is_null())
-                if (etcd_watches_initialised == ETCD_TOTAL_WATCHES &&
-                    !data["result"]["header"]["revision"].is_null() &&
-                    !data["result"]["created"].bool_value())
                 {
                     // Restart watchers from the same revision number as in the last received message,
                     // not from the next one to protect against revision being split into multiple messages,

src/client/etcd_state_client.h

@@ -6,7 +6,7 @@
 #include <set>
 
 #include "json11/json11.hpp"
-#include "object_id.h"
+#include "osd_id.h"
 #include "timerfd_manager.h"
 
 #define ETCD_CONFIG_WATCH_ID 1

src/client/messenger.cpp

@@ -145,7 +145,7 @@ void osd_messenger_t::init()
         rdma_contexts = msgr_rdma_context_t::create_all(
             osd_num && osd_cluster_network_masks.size() ? osd_cluster_network_masks : osd_network_masks,
             rdma_device != "" ? rdma_device.c_str() : NULL,
-            rdma_port_num, rdma_gid_index, rdma_mtu, log_level
+            rdma_port_num, rdma_gid_index, rdma_mtu, rdma_odp, log_level
         );
         if (!rdma_contexts.size())
         {
@@ -322,6 +322,7 @@ void osd_messenger_t::parse_config(const json11::Json & config)
     this->rdma_max_msg = config["rdma_max_msg"].uint64_value();
     if (!this->rdma_max_msg || this->rdma_max_msg > 128*1024*1024)
         this->rdma_max_msg = 129*1024;
+    this->rdma_odp = config["rdma_odp"].bool_value();
 #endif
     if (!osd_num)
         this->iothread_count = (uint32_t)config["client_iothread_count"].uint64_value();

src/client/messenger.h

@@ -200,6 +200,7 @@ protected:
     std::vector<msgr_rdma_context_t *> rdma_contexts;
     uint64_t rdma_max_sge = 0, rdma_max_send = 0, rdma_max_recv = 0;
     uint64_t rdma_max_msg = 0;
+    bool rdma_odp = false;
     rdma_event_channel *rdmacm_evch = NULL;
     std::map<rdma_cm_id*, osd_client_t*> rdmacm_connections;
     std::map<rdma_cm_id*, rdmacm_connecting_t*> rdmacm_connecting;
@@ -209,7 +210,7 @@ protected:
     std::vector<int> read_ready_clients;
     std::vector<int> write_ready_clients;
     // We don't use ringloop->set_immediate here because we may have no ringloop in client :)
-    std::deque<osd_op_t*> set_immediate_ops;
+    std::vector<osd_op_t*> set_immediate_ops;
 
 public:
     timerfd_manager_t *tfd = NULL;
@@ -286,6 +287,8 @@ protected:
 
 #ifdef WITH_RDMA
     void try_send_rdma(osd_client_t *cl);
+    void try_send_rdma_odp(osd_client_t *cl);
+    void try_send_rdma_nodp(osd_client_t *cl);
     bool init_recv_rdma(osd_client_t *cl);
     void handle_rdma_events(msgr_rdma_context_t *rdma_context);
     msgr_rdma_context_t* choose_rdma_context(osd_client_t *cl);

src/client/msgr_rdma.cpp

@@ -59,6 +59,8 @@ msgr_rdma_context_t::~msgr_rdma_context_t()
         ibv_destroy_cq(cq);
     if (channel)
         ibv_destroy_comp_channel(channel);
+    if (mr)
+        ibv_dereg_mr(mr);
     if (pd)
         ibv_dealloc_pd(pd);
     if (context && !is_cm)
@@ -180,7 +182,7 @@ static int match_port_gid(const std::vector<addr_mask_t> & osd_network_masks, ib
 #endif
 
 std::vector<msgr_rdma_context_t*> msgr_rdma_context_t::create_all(const std::vector<addr_mask_t> & osd_network_masks,
-    const char *sel_dev_name, int sel_port_num, int sel_gid_index, uint32_t sel_mtu, int log_level)
+    const char *sel_dev_name, int sel_port_num, int sel_gid_index, uint32_t sel_mtu, bool odp, int log_level)
 {
     int res;
     std::vector<msgr_rdma_context_t*> ret;
@@ -269,7 +271,7 @@ std::vector<msgr_rdma_context_t*> msgr_rdma_context_t::create_all(const std::vec
                 {
                     if (log_level > 0)
                         log_rdma_dev_port_gid(dev, port_num, best_gid_idx, port_mtu, best_gidx);
-                    auto ctx = msgr_rdma_context_t::create(dev, portinfo, port_num, best_gid_idx, port_mtu, log_level);
+                    auto ctx = msgr_rdma_context_t::create(dev, portinfo, port_num, best_gid_idx, port_mtu, odp, log_level);
                     if (ctx)
                     {
                         ctx->net_mask = osd_network_masks[net_num];
@@ -289,7 +291,7 @@ std::vector<msgr_rdma_context_t*> msgr_rdma_context_t::create_all(const std::vec
                     log_rdma_dev_port_gid(dev, port_num, best_gid_idx, port_mtu, gidx);
                 }
 #endif
-                auto ctx = msgr_rdma_context_t::create(dev, portinfo, port_num, best_gid_idx, port_mtu, log_level);
+                auto ctx = msgr_rdma_context_t::create(dev, portinfo, port_num, best_gid_idx, port_mtu, odp, log_level);
                 if (ctx)
                     ret.push_back(ctx);
             }
@@ -304,7 +306,7 @@ cleanup:
     return ret;
 }
 
-msgr_rdma_context_t *msgr_rdma_context_t::create(ibv_device *dev, ibv_port_attr & portinfo, int ib_port, int gid_index, uint32_t mtu, int log_level)
+msgr_rdma_context_t *msgr_rdma_context_t::create(ibv_device *dev, ibv_port_attr & portinfo, int ib_port, int gid_index, uint32_t mtu, bool odp, int log_level)
 {
     msgr_rdma_context_t *ctx = new msgr_rdma_context_t();
     ibv_context *context = ibv_open_device(dev);
@@ -344,6 +346,30 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(ibv_device *dev, ibv_port_attr
         goto cleanup;
     }
 
+    ctx->odp = odp;
+    if (ctx->odp)
+    {
+        if (!(ctx->attrx.odp_caps.general_caps & IBV_ODP_SUPPORT) ||
+            !(ctx->attrx.odp_caps.general_caps & IBV_ODP_SUPPORT_IMPLICIT) ||
+            !(ctx->attrx.odp_caps.per_transport_caps.rc_odp_caps & IBV_ODP_SUPPORT_SEND) ||
+            !(ctx->attrx.odp_caps.per_transport_caps.rc_odp_caps & IBV_ODP_SUPPORT_RECV))
+        {
+            ctx->odp = false;
+            if (log_level > 0)
+                fprintf(stderr, "The RDMA device isn't implicit ODP (On-Demand Paging) capable, disabling it\n");
+        }
+    }
+
+    if (ctx->odp)
+    {
+        ctx->mr = ibv_reg_mr(ctx->pd, NULL, SIZE_MAX, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_ON_DEMAND);
+        if (!ctx->mr)
+        {
+            fprintf(stderr, "Couldn't register RDMA memory region\n");
+            goto cleanup;
+        }
+    }
+
     ctx->channel = ibv_create_comp_channel(ctx->context);
     if (!ctx->channel)
     {
@@ -362,6 +388,8 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(ibv_device *dev, ibv_port_attr
     return ctx;
 
 cleanup:
+    if (context)
+        ibv_close_device(context);
     delete ctx;
     return NULL;
 }
@@ -577,7 +605,52 @@ static int try_send_rdma_copy(osd_client_t *cl, uint8_t *dst, int dst_len)
     return total_dst_len-dst_len;
 }
 
-void osd_messenger_t::try_send_rdma(osd_client_t *cl)
+void osd_messenger_t::try_send_rdma_odp(osd_client_t *cl)
+{
+    auto rc = cl->rdma_conn;
+    if (!cl->send_list.size() || rc->cur_send >= rc->max_send)
+    {
+        return;
+    }
+    uint64_t op_size = 0, op_sge = 0;
+    ibv_sge sge[rc->max_sge];
+    while (rc->send_pos < cl->send_list.size())
+    {
+        iovec & iov = cl->send_list[rc->send_pos];
+        if (op_size >= rc->max_msg || op_sge >= rc->max_sge)
+        {
+            rc->send_sizes.push_back(op_size);
+            try_send_rdma_wr(cl, sge, op_sge);
+            op_sge = 0;
+            op_size = 0;
+            if (rc->cur_send >= rc->max_send)
+            {
+                break;
+            }
+        }
+        uint32_t len = (uint32_t)(op_size+iov.iov_len-rc->send_buf_pos < rc->max_msg
+            ? iov.iov_len-rc->send_buf_pos : rc->max_msg-op_size);
+        sge[op_sge++] = {
+            .addr = (uintptr_t)((uint8_t*)iov.iov_base+rc->send_buf_pos),
+            .length = len,
+            .lkey = rc->ctx->mr->lkey,
+        };
+        op_size += len;
+        rc->send_buf_pos += len;
+        if (rc->send_buf_pos >= iov.iov_len)
+        {
+            rc->send_pos++;
+            rc->send_buf_pos = 0;
+        }
+    }
+    if (op_sge > 0)
+    {
+        rc->send_sizes.push_back(op_size);
+        try_send_rdma_wr(cl, sge, op_sge);
+    }
+}
+
+void osd_messenger_t::try_send_rdma_nodp(osd_client_t *cl)
 {
     auto rc = cl->rdma_conn;
     if (!rc->send_out_size)
@@ -585,11 +658,14 @@ void osd_messenger_t::try_send_rdma(osd_client_t *cl)
         // Allocate send ring buffer, if not yet
         rc->send_out_size = rc->max_msg*rdma_max_send;
         rc->send_out.buf = (uint8_t*)malloc_or_die(rc->send_out_size);
-        rc->send_out.mr = ibv_reg_mr(rc->ctx->pd, rc->send_out.buf, rc->send_out_size, 0);
+        if (!rc->ctx->odp)
-        if (!rc->send_out.mr)
         {
-            fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
+            rc->send_out.mr = ibv_reg_mr(rc->ctx->pd, rc->send_out.buf, rc->send_out_size, 0);
-            exit(1);
+            if (!rc->send_out.mr)
+            {
+                fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
+                exit(1);
+            }
         }
     }
     // Copy data into the buffer and send it
@@ -614,7 +690,7 @@ void osd_messenger_t::try_send_rdma(osd_client_t *cl)
             ibv_sge sge = {
                 .addr = (uintptr_t)dst,
                 .length = (uint32_t)copied,
-                .lkey = rc->send_out.mr->lkey,
+                .lkey = rc->ctx->odp ? rc->ctx->mr->lkey : rc->send_out.mr->lkey,
             };
             try_send_rdma_wr(cl, &sge, 1);
             rc->send_sizes.push_back(copied);
@@ -622,12 +698,20 @@ void osd_messenger_t::try_send_rdma(osd_client_t *cl)
     }
 }
 
+void osd_messenger_t::try_send_rdma(osd_client_t *cl)
+{
+    if (cl->rdma_conn->ctx->odp)
+        try_send_rdma_odp(cl);
+    else
+        try_send_rdma_nodp(cl);
+}
+
 static void try_recv_rdma_wr(osd_client_t *cl, void *buf)
 {
     ibv_sge sge = {
         .addr = (uintptr_t)buf,
         .length = (uint32_t)cl->rdma_conn->max_msg,
-        .lkey = cl->rdma_conn->recv_buf.mr->lkey,
+        .lkey = cl->rdma_conn->ctx->odp ? cl->rdma_conn->ctx->mr->lkey : cl->rdma_conn->recv_buf.mr->lkey,
     };
     ibv_recv_wr *bad_wr = NULL;
     ibv_recv_wr wr = {
@@ -649,11 +733,14 @@ bool osd_messenger_t::init_recv_rdma(osd_client_t *cl)
     auto rc = cl->rdma_conn;
     assert(!rc->recv_buf.buf);
     rc->recv_buf.buf = (uint8_t*)malloc_or_die(rc->max_msg * rc->max_recv);
-    rc->recv_buf.mr = ibv_reg_mr(rc->ctx->pd, rc->recv_buf.buf, rc->max_msg * rc->max_recv, IBV_ACCESS_LOCAL_WRITE);
+    if (!rc->ctx->odp)
-    if (!rc->recv_buf.mr)
     {
-        fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
+        rc->recv_buf.mr = ibv_reg_mr(rc->ctx->pd, rc->recv_buf.buf, rc->max_msg * rc->max_recv, IBV_ACCESS_LOCAL_WRITE);
-        exit(1);
+        if (!rc->recv_buf.mr)
+        {
+            fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
+            exit(1);
+        }
     }
     for (uint32_t i = 0; i < rc->max_recv; i++)
     {
@@ -729,11 +816,14 @@ void osd_messenger_t::handle_rdma_events(msgr_rdma_context_t *rdma_context)
                 rc->cur_send--;
                 uint64_t sent_size = rc->send_sizes.at(0);
                 rc->send_sizes.erase(rc->send_sizes.begin(), rc->send_sizes.begin()+1);
-                rc->send_done_pos += sent_size;
+                if (!rdma_context->odp)
-                rc->send_out_full = false;
+                {
-                if (rc->send_done_pos == rc->send_out_size)
+                    rc->send_done_pos += sent_size;
-                    rc->send_done_pos = 0;
+                    rc->send_out_full = false;
-                assert(rc->send_done_pos < rc->send_out_size);
+                    if (rc->send_done_pos == rc->send_out_size)
+                        rc->send_done_pos = 0;
+                    assert(rc->send_done_pos < rc->send_out_size);
+                }
                 int send_pos = 0, send_buf_pos = 0;
                 while (sent_size > 0)
                 {

src/client/msgr_rdma.h

@@ -26,6 +26,8 @@ struct msgr_rdma_context_t
     ibv_context *context = NULL;
     ibv_device_attr_ex attrx;
     ibv_pd *pd = NULL;
+    bool odp = false;
+    ibv_mr *mr = NULL;
     ibv_comp_channel *channel = NULL;
     ibv_cq *cq = NULL;
     ibv_port_attr portinfo;
@@ -41,9 +43,9 @@ struct msgr_rdma_context_t
     int cm_refs = 0;
 
     static std::vector<msgr_rdma_context_t*> create_all(const std::vector<addr_mask_t> & osd_network_masks,
-        const char *sel_dev_name, int sel_port_num, int sel_gid_index, uint32_t sel_mtu, int log_level);
+        const char *sel_dev_name, int sel_port_num, int sel_gid_index, uint32_t sel_mtu, bool odp, int log_level);
     static msgr_rdma_context_t *create(ibv_device *dev, ibv_port_attr & portinfo,
-        int ib_port, int gid_index, uint32_t mtu, int log_level);
+        int ib_port, int gid_index, uint32_t mtu, bool odp, int log_level);
     static msgr_rdma_context_t* create_cm(ibv_context *ctx);
     bool reserve_cqe(int n);
 

src/client/msgr_rdmacm.cpp

@@ -178,6 +178,7 @@ msgr_rdma_context_t* msgr_rdma_context_t::create_cm(ibv_context *ctx)
         delete rdma_context;
         return NULL;
     }
+    rdma_context->odp = false;
     rdma_context->channel = ibv_create_comp_channel(rdma_context->context);
     if (!rdma_context->channel)
     {

src/client/msgr_receive.cpp

@@ -160,10 +160,8 @@ void osd_messenger_t::clear_immediate_ops(int peer_fd)
 
 void osd_messenger_t::handle_immediate_ops()
 {
-    while (set_immediate_ops.size())
+    for (auto op: set_immediate_ops)
     {
-        auto op = set_immediate_ops.front();
-        set_immediate_ops.pop_front();
         if (op->op_type == OSD_OP_IN)
         {
             exec_op(op);
@@ -174,6 +172,7 @@ void osd_messenger_t::handle_immediate_ops()
             std::function<void(osd_op_t*)>(op->callback)(op);
         }
     }
+    set_immediate_ops.clear();
 }
 
 bool osd_messenger_t::handle_read_buffer(osd_client_t *cl, void *curbuf, int remain)

src/client/msgr_stop.cpp

@@ -57,7 +57,6 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
     {
         return;
     }
-    clear_immediate_ops(peer_fd);
     if (log_level > 0)
     {
         if (cl->osd_num)

src/client/object_id.h

@@ -20,15 +20,6 @@ typedef uint64_t inode_t;
 // Pool ID is 16 bits long
 typedef uint32_t pool_id_t;
 
-typedef uint64_t osd_num_t;
-typedef uint32_t pg_num_t;
-
-struct pool_pg_num_t
-{
-    pool_id_t pool_id;
-    pg_num_t pg_num;
-};
-
 // 16 bytes per object/stripe id
 // stripe = (start of the parity stripe + peer role)
 // i.e. for example (256KB + one of 0,1,2)
@@ -70,21 +61,6 @@ inline bool operator < (const obj_ver_id & a, const obj_ver_id & b)
     return a.oid < b.oid || a.oid == b.oid && a.version < b.version;
 }
 
-inline bool operator < (const pool_pg_num_t & a, const pool_pg_num_t & b)
-{
-    return a.pool_id < b.pool_id || a.pool_id == b.pool_id && a.pg_num < b.pg_num;
-}
-
-inline bool operator == (const pool_pg_num_t & a, const pool_pg_num_t & b)
-{
-    return a.pool_id == b.pool_id && a.pg_num == b.pg_num;
-}
-
-inline bool operator != (const pool_pg_num_t & a, const pool_pg_num_t & b)
-{
-    return a.pool_id != b.pool_id || a.pg_num != b.pg_num;
-}
-
 namespace std
 {
     template<> struct hash<object_id>

src/client/osd_ops.h

@@ -4,6 +4,7 @@
 #pragma once
 
 #include "object_id.h"
+#include "osd_id.h"
 
 // Magic numbers
 #define SECONDARY_OSD_OP_MAGIC      0x2bd7b10325434553l

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 3.0.4
+Version: 3.0.2
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/cmd/cli.h

@@ -7,6 +7,7 @@
 
 #include "json11/json11.hpp"
 #include "object_id.h"
+#include "osd_id.h"
 #include "ringloop.h"
 #include <functional>
 

src/cmd/cli_dd.cpp

@@ -442,7 +442,7 @@ struct cli_dd_t
                 }
                 delete cur_read;
             }
-            else if (!is_zero(read_op->bitmap_buf, (read_op->len/iinfo.in_granularity+7)/8))
+            else if (!is_zero(read_op->bitmap_buf, read_op->len/iinfo.in_granularity/8))
             {
                 vitastor_read(cur_read);
             }

src/disk_tool/disk_tool_meta.cpp

@@ -3,6 +3,7 @@
 
 #include "disk_tool.h"
 #include "rw_blocking.h"
+#include "osd_id.h"
 #include "json_util.h"
 #include "malloc_or_die.h"
 
@@ -108,7 +109,6 @@ close_error:
             r = heap->load_blocks(meta_pos-dsk.meta_block_size, read_len, data, true, entries_loaded);
             meta_pos += read_len;
         }
-        heap->finish_load();
         heap->iterate_objects([&](heap_entry_t* obj, uint32_t meta_block_num)
         {
             obj_fn(heap, obj, meta_block_num);

src/disk_tool/disk_tool_prepare.cpp

@@ -4,6 +4,7 @@
 #include "disk_tool.h"
 #include "str_util.h"
 #include "json_util.h"
+#include "osd_id.h"
 
 void disk_tool_t::parse_meta_reserve()
 {
@@ -149,12 +150,9 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
     {
         if (options["block_size"] == "")
             options["block_size"] = "1M";
-        if (is_hybrid && options["atomic_write_size"] == "")
-            options["atomic_write_size"] = "0";
         if (is_hybrid && options["throttle_small_writes"] == "")
             options["throttle_small_writes"] = "1";
-        if (!is_hybrid && options.find("data_csum_type") != options.end() && options.at("data_csum_type") != "" &&
+        if (!is_hybrid && options.find("data_csum_type") != options.end() && options.at("data_csum_type") != "")
-            options["csum_block_size"] == "")
             options["csum_block_size"] = "32k";
     }
     else if (!json_is_true(options["disable_data_fsync"]))

src/disk_tool/disk_tool_udev.cpp

@@ -140,10 +140,7 @@ uint32_t disk_tool_t::write_osd_superblock(std::string device, json11::Json para
     }
     close(fd);
     free(buf);
-    if (!test_mode)
+    shell_exec({ "udevadm", "trigger", "--settle", device }, "", NULL, NULL);
-    {
-        shell_exec({ "udevadm", "trigger", "--settle", device }, "", NULL, NULL);
-    }
     return sb_size;
 }
 

src/nfs/nfs_kv_setattr.cpp

@@ -61,7 +61,7 @@ resume_1:
     }
     if (st->ientry["type"].string_value() != "file" &&
         st->ientry["type"].string_value() != "" &&
-        st->set_attrs.find("size") != st->set_attrs.end())
+        !st->set_attrs["size"].is_null())
     {
         auto cb = std::move(st->cb);
         cb(-EINVAL);
@@ -96,13 +96,8 @@ resume_1:
         nfs_kv_continue_setattr(st, 2);
     }, [st](int res, const std::string & cas_value)
     {
-        if ((res == 0 || res == -ENOENT && st->ino == KV_ROOT_INODE) && cas_value == st->ientry_text)
-        {
-            st->cas_res = 0;
-            return true;
-        }
         st->cas_res = res;
-        return false;
+        return (res == 0 || res == -ENOENT && st->ino == KV_ROOT_INODE) && cas_value == st->ientry_text;
     });
     return;
 resume_2:

src/osd/osd_cluster.cpp

@@ -809,17 +809,21 @@ again:
         goto again;
     }
     auto & pool_cfg = pool_it->second;
-    bool done = bs->reshard_continue(pool_cfg.reshard_state, pg_reshard_chunk_size);
+    bool done = false;
-    if (done &&
+    if (pool_cfg.real_pg_count != pool_cfg.applied_pg_count ||
-        (pool_cfg.real_pg_count != pool_cfg.applied_pg_count ||
+        pool_cfg.pg_stripe_size != pool_cfg.applied_pg_stripe_size)
-        pool_cfg.pg_stripe_size != pool_cfg.applied_pg_stripe_size))
     {
         // PG count changed again, reshard again
+        bs->reshard_abort(pool_cfg.reshard_state);
         pool_cfg.applied_pg_count = pool_cfg.real_pg_count;
         pool_cfg.applied_pg_stripe_size = pool_cfg.pg_stripe_size;
         pool_cfg.reshard_state = bs->reshard_start(pool_id, pool_cfg.real_pg_count, pool_cfg.pg_stripe_size, pg_reshard_chunk_size);
         done = !pool_cfg.reshard_state;
     }
+    else
+    {
+        done = bs->reshard_continue(pool_cfg.reshard_state, pg_reshard_chunk_size);
+    }
     if (done)
     {
         // Pool is resharded

src/osd/osd_id.h

@@ -0,0 +1,30 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+#pragma once
+
+#include "object_id.h"
+
+typedef uint64_t osd_num_t;
+typedef uint32_t pg_num_t;
+
+struct pool_pg_num_t
+{
+    pool_id_t pool_id;
+    pg_num_t pg_num;
+};
+
+inline bool operator < (const pool_pg_num_t & a, const pool_pg_num_t & b)
+{
+    return a.pool_id < b.pool_id || a.pool_id == b.pool_id && a.pg_num < b.pg_num;
+}
+
+inline bool operator == (const pool_pg_num_t & a, const pool_pg_num_t & b)
+{
+    return a.pool_id == b.pool_id && a.pg_num == b.pg_num;
+}
+
+inline bool operator != (const pool_pg_num_t & a, const pool_pg_num_t & b)
+{
+    return a.pool_id != b.pool_id || a.pg_num != b.pg_num;
+}

src/osd/osd_peering_pg.cpp

@@ -334,44 +334,36 @@ pg_osd_set_state_t* pg_t::add_object_to_state(const object_id oid, const uint64_
     if (it == state_dict.end())
     {
         std::vector<osd_num_t> read_target;
-        bool found = false;
-        uint32_t bad_mask = (LOC_OUTDATED | LOC_CORRUPTED);
-retry:
         if (scheme == POOL_SCHEME_REPLICATED)
         {
             for (auto & o: osd_set)
             {
-                if (!(o.loc_bad & bad_mask))
+                if (!(o.loc_bad & (LOC_OUTDATED | LOC_CORRUPTED)))
                 {
                     read_target.push_back(o.osd_num);
-                    found = true;
                 }
             }
-            if (read_target.size() < pg_size)
+            while (read_target.size() < pg_size)
             {
                 // FIXME: This is because we then use .data() and assume it's at least <pg_size> long
-                read_target.resize(pg_size);
+                read_target.push_back(0);
             }
         }
         else
         {
             read_target.resize(pg_size);
+            for (int i = 0; i < pg_size; i++)
+            {
+                read_target[i] = 0;
+            }
             for (auto & o: osd_set)
             {
-                if (!(o.loc_bad & bad_mask))
+                if (!(o.loc_bad & (LOC_OUTDATED | LOC_CORRUPTED)))
                 {
                     read_target[o.role] = o.osd_num;
-                    found = true;
                 }
             }
         }
-        if (!found && (bad_mask & LOC_CORRUPTED))
-        {
-            // Allow to try reading corrupted copies in rare cases when the object is corrupted on all OSDs
-            bad_mask = LOC_OUTDATED;
-            read_target.clear();
-            goto retry;
-        }
         state_dict[osd_set] = {
             .read_target = read_target,
             .osd_set = osd_set,

src/osd/osd_primary.cpp

@@ -271,6 +271,7 @@ resume_0:
             }
             cur_op->buf = alloc_read_buffer(op_data->stripes, pg ? pg->pg_data_size : 1, 0);
             submit_primary_subops(SUBMIT_RMW_READ, op_data->target_ver, op_data->prev_set, cur_op);
+            op_data->st = 1;
         }
         else
         {
@@ -283,14 +284,11 @@ resume_0:
             op_data->degraded = 1;
             cur_op->buf = alloc_read_buffer(op_data->stripes, pg->pg_size, 0);
             submit_primary_subops(SUBMIT_RMW_READ, op_data->target_ver, op_data->prev_set, cur_op);
+            op_data->st = 1;
         }
     }
 resume_1:
-    if (op_data->n_subops > 0)
+    return;
-    {
-        op_data->st = 1;
-        return;
-    }
 resume_2:
     if (op_data->errors > 0)
     {
@@ -298,9 +296,8 @@ resume_2:
         {
             // I/O or checksum error
             // FIXME: ref = true ideally... because new_state != state is not necessarily true if it's freed and recreated
-            auto new_object_state = mark_object_corrupted(*pg, op_data->oid, op_data->object_state, op_data->stripes, false);
+            op_data->object_state = mark_object_corrupted(*pg, op_data->oid, op_data->object_state, op_data->stripes, false);
-            if (new_object_state != op_data->object_state)
+            goto resume_0;
-                goto resume_0;
         }
         finish_op(cur_op, op_data->errcode);
         return;
@@ -739,11 +736,8 @@ resume_1:
     submit_primary_subops(SUBMIT_RMW_READ, UINT64_MAX, op_data->prev_set, cur_op);
     op_data->prev_set = NULL;
 resume_2:
-    if (op_data->n_subops > 0)
+    op_data->st = 2;
-    {
+    return;
-        op_data->st = 2;
-        return;
-    }
 resume_3:
     if (op_data->errors > 0)
     {

src/osd/osd_primary_subops.cpp

@@ -134,17 +134,11 @@ void osd_t::submit_primary_subops(int submit_type, uint64_t op_version, const ui
         n_subops = 1;
     else
         zero_read = -1;
+    osd_op_t *subops = new osd_op_t[n_subops];
     op_data->fact_ver = 0;
     op_data->done = op_data->errors = op_data->drops = op_data->errcode = 0;
     op_data->n_subops = n_subops;
-    if (!n_subops)
+    op_data->subops = subops;
-    {
-        op_data->errcode = -EIO;
-        op_data->subops = NULL;
-        op_data->errors = 1;
-        return;
-    }
-    op_data->subops = new osd_op_t[n_subops];
     int sent = submit_primary_subop_batch(submit_type, op_data->oid.inode, op_version, op_data->stripes, osd_set, cur_op, 0, zero_read);
     assert(sent == n_subops);
 }
@@ -410,9 +404,8 @@ void osd_t::handle_primary_subop(osd_op_t *subop, osd_op_t *cur_op)
             if (op_data->fact_ver != 0 && op_data->fact_ver != version)
             {
                 fprintf(
-                    stderr, "different fact_versions returned from %s %jx:%jx subops for a %s op: %ju vs %ju\n",
+                    stderr, "different fact_versions returned from %s %jx:%jx subops: %ju vs %ju\n",
-                    osd_op_names[opcode], subop->req.sec_rw.oid.inode, subop->req.sec_rw.oid.stripe,
+                    osd_op_names[opcode], subop->req.sec_rw.oid.inode, subop->req.sec_rw.oid.stripe, version, op_data->fact_ver
-                    osd_op_names[cur_op->req.hdr.opcode], version, op_data->fact_ver
                 );
                 retval = -ERANGE;
             }

src/osd/osd_primary_write.cpp

@@ -108,31 +108,57 @@ retry_1:
         }
     }
     // Read required blocks
-    submit_primary_subops(SUBMIT_RMW_READ, UINT64_MAX, op_data->prev_set, cur_op);
-resume_2:
-    if (op_data->n_subops > 0)
     {
-        op_data->st = 2;
+        if (op_data->object_state && (op_data->object_state->state & OBJ_INCOMPLETE))
-        return;
+        {
+            // Allow to read version number (just version number!) from corrupted chunks
+            // to allow full overwrite of a corrupted object
+            bool found = false;
+            for (int role = 0; role < pg.pg_size; role++)
+            {
+                if (op_data->prev_set[role] != 0 || op_data->stripes[role].read_end > op_data->stripes[role].read_start)
+                {
+                    found = true;
+                    break;
+                }
+            }
+            if (!found)
+            {
+                osd_num_t corrupted_target[pg.pg_size];
+                for (int role = 0; role < pg.pg_size; role++)
+                {
+                    corrupted_target[role] = 0;
+                }
+                for (auto & loc: op_data->object_state->osd_set)
+                {
+                    if (!(loc.loc_bad & LOC_OUTDATED) && !corrupted_target[loc.role])
+                    {
+                        corrupted_target[loc.role] = loc.osd_num;
+                    }
+                }
+                submit_primary_subops(SUBMIT_RMW_READ, UINT64_MAX, corrupted_target, cur_op);
+                goto resume_2;
+            }
+        }
+        submit_primary_subops(SUBMIT_RMW_READ, UINT64_MAX, op_data->prev_set, cur_op);
     }
+resume_2:
+    op_data->st = 2;
+    return;
 resume_3:
     if (op_data->errors > 0)
     {
         if (op_data->errcode == -EIO || op_data->errcode == -EDOM)
         {
             // Mark object corrupted and retry
-            pg_osd_set_state_t *new_object_state = mark_object_corrupted(pg, op_data->oid, op_data->object_state, op_data->stripes, true);
+            op_data->object_state = mark_object_corrupted(pg, op_data->oid, op_data->object_state, op_data->stripes, true);
-            if (new_object_state != op_data->object_state)
+            op_data->prev_set = op_data->object_state ? op_data->object_state->read_target.data() : pg.cur_set.data();
+            if (cur_op->rmw_buf)
             {
-                op_data->object_state = new_object_state;
+                free(cur_op->rmw_buf);
-                op_data->prev_set = op_data->object_state ? op_data->object_state->read_target.data() : pg.cur_set.data();
+                cur_op->rmw_buf = NULL;
-                if (cur_op->rmw_buf)
-                {
-                    free(cur_op->rmw_buf);
-                    cur_op->rmw_buf = NULL;
-                }
-                goto retry_1;
             }
+            goto retry_1;
         }
         deref_object_state(pg, &op_data->object_state, true);
         pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->errcode);

src/osd/osd_rmw.h

@@ -6,6 +6,7 @@
 #include <stdint.h>
 #include <vector>
 #include "object_id.h"
+#include "osd_id.h"
 
 struct buf_len_t
 {

src/test/mock/messenger.cpp

@@ -57,7 +57,3 @@ json11::Json::object osd_messenger_t::merge_configs(const json11::Json::object &
 {
     return cli_config;
 }
-
-void osd_messenger_t::clear_immediate_ops(int peer_fd)
-{
-}

tests/run_tests.sh

@@ -96,11 +96,6 @@ TEST_NAME=local_read POOLCFG='"local_reads":"random",' ./test_heal.sh
 SCHEME=ec ./test_heal.sh
 ANTIETCD=1 ./test_heal.sh
 
-./test_checksum.sh
-OLD=1 ./test_checksum.sh
-./test_corrupt_all.sh
-OLD=1 ./test_corrupt_all.sh
-
 ./test_reweight_half.sh
 ./test_snapshot_pool2.sh
 ./test_snapshot_read_bitmap.sh

tests/test_corrupt_all.sh

@@ -1,46 +0,0 @@
-#!/bin/bash -ex
-
-SCHEME=replicated
-PG_SIZE=2
-OSD_COUNT=2
-IMG_SIZE=128
-OSD_ARGS="--data_csum_type crc32c --csum_block_size 4k --inmemory_journal false --journal_trim_interval $((IMG_SIZE*8)) $OSD_ARGS"
-OFFSET_ARGS="--data_csum_type crc32c --csum_block_size 4k $OFFSET_ARGS"
-GLOBAL_CONFIG=',"client_eio_retry_interval":0'
-. `dirname $0`/run_3osds.sh
-check_qemu
-
-$ETCDCTL put /vitastor/config/inode/1/1 '{"name":"testimg","size":'$((IMG_SIZE*1024*1024))'}'
-
-# Write
-$VITASTOR_FIO -bs=1M -direct=1 -iodepth=4 -end_fsync=1 -rw=write -image=testimg -runtime=10
-#$VITASTOR_FIO -bs=4k -direct=1 -iodepth=16 -end_fsync=1 -rw=randwrite -image=testimg -number_ios=10000
-
-# Intentionally corrupt OSD data and restart both of them
-kill $OSD1_PID $OSD2_PID
-data_offset=$(build/src/disk_tool/vitastor-disk simple-offsets ./testdata/bin/test_osd1.bin $OFFSET_ARGS | grep data_offset | awk '{print $2}')
-truncate -s $data_offset ./testdata/bin/test_osd1.bin
-dd if=/dev/zero of=./testdata/bin/test_osd1.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
-truncate -s $data_offset ./testdata/bin/test_osd2.bin
-dd if=/dev/zero of=./testdata/bin/test_osd2.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
-start_osd 1
-start_osd 2
-
-# Wait until start
-wait_up 10
-
-# Trigger scrub
-$ETCDCTL put /vitastor/pg/history/1/1 `$ETCDCTL get --print-value-only /vitastor/pg/history/1/1 | jq -s -c '(.[0] // {}) + {"next_scrub":1}'`
-
-# Wait for scrub to finish
-wait_condition 300 "$ETCDCTL get --prefix /vitastor/pg/history/ --print-value-only | jq -s -e '([ .[] | select(.next_scrub == 0 or .next_scrub == null) ] | length) == $PG_COUNT'" Scrubbing
-
-# Verify that ALL objects are now corrupted+incomplete
-$VITASTOR_CLI describe --json &>./testdata/describe.json
-$VITASTOR_CLI describe --json | jq -e '[ .[] | select(.corrupted) ] | length == '$((IMG_SIZE * 8 * PG_SIZE))
-
-# Check that we can remove or overwrite them
-$VITASTOR_FIO -bs=4M -direct=1 -iodepth=1 -end_fsync=1 -rw=write -offset=$((IMG_SIZE/2))M -image=testimg -runtime=10
-$VITASTOR_CLI rm-data --pool 1 --inode 1
-
-format_green OK

tests/test_nfs.sh

@@ -15,8 +15,6 @@ sudo mount localhost:/ ./testdata/nfs -o port=2050,mountport=2050,nfsvers=3,soft
 MNT=$(pwd)/testdata/nfs
 trap "sudo umount -f $MNT"' || true; kill -9 $(jobs -p)' EXIT
 
-chown 1000:1000 ./testdata/nfs
-
 touch ./testdata/nfs/f1
 chown 1000:1000 ./testdata/nfs/f1
 chmod 600 ./testdata/nfs/f1

tests/test_resize_auto.sh

@@ -15,7 +15,7 @@ trap "kill -9 $(jobs -p) || true; sudo losetup -d $LOOP1 $LOOP2"' || true' EXIT
 # also test prepare --hybrid :)
 # non-vitastor random type UUID to prevent udev activation
 mount | grep '/dev type devtmpfs' || sudo mount udev /dev/ -t devtmpfs
-sudo -E build/src/disk_tool/vitastor-disk-test prepare $OFFSET_ARGS --no_init 1 --meta_reserve 1x,1M \
+sudo build/src/disk_tool/vitastor-disk-test prepare $OFFSET_ARGS --no_init 1 --meta_reserve 1x,1M \
     --block_size 131072 --osd_num 987654 --part_type_uuid 0df42ae0-3695-4395-a957-7d5ff3645c56 \
     --hybrid --fast-devices $LOOP2 $LOOP1
 
@@ -27,8 +27,8 @@ console.log(JSON.stringify([
     {"type":"big_write_instant","inode":"0x1000000000001","stripe":"0xc60000","ver":"10","offset":0,"len":131072,"loc":"0x18ffdc0000","bitmap":"ffffffff"}
 ]));
 EOF
-    sudo -E build/src/disk_tool/vitastor-disk write-journal ${LOOP1}p1 < ./testdata/journal.json
+    sudo build/src/disk_tool/vitastor-disk write-journal ${LOOP1}p1 < ./testdata/journal.json
-    sudo -E build/src/disk_tool/vitastor-disk dump-journal --json --format data ${LOOP1}p1 | jq -S '[ .[] | del(.crc32, .crc32_prev) ]' > ./testdata/j2.json
+    sudo build/src/disk_tool/vitastor-disk dump-journal --json --format data ${LOOP1}p1 | jq -S '[ .[] | del(.crc32, .crc32_prev) ]' > ./testdata/j2.json
     jq -S '[ .[] + {"valid":true} ]' < ./testdata/journal.json > ./testdata/j1.json
     diff ./testdata/j1.json ./testdata/j2.json
 fi
@@ -84,15 +84,15 @@ EOF
 fi
 
 # also test write & dump
-sudo -E build/src/disk_tool/vitastor-disk write-meta ${LOOP1}p1 < ./testdata/meta.json
+sudo build/src/disk_tool/vitastor-disk write-meta ${LOOP1}p1 < ./testdata/meta.json
-sudo -E build/src/disk_tool/vitastor-disk dump-meta ${LOOP1}p1 > ./testdata/compare.json
+sudo build/src/disk_tool/vitastor-disk dump-meta ${LOOP1}p1 > ./testdata/compare.json
 jq -S '. + {"entries": (.entries | sort_by(.stripe)) }' < ./testdata/meta.json > ./testdata/1.json
 jq -S '. + {"entries": (.entries | sort_by(.stripe)) }' < ./testdata/compare.json > ./testdata/2.json
 diff ./testdata/1.json ./testdata/2.json
 
 # move journal & meta back, data will become smaller; end indexes should be shifted by -1251
-sudo -E build/src/disk_tool/vitastor-disk-test resize --move-journal '' --move-meta '' ${LOOP1}p1
+sudo build/src/disk_tool/vitastor-disk-test resize --move-journal '' --move-meta '' ${LOOP1}p1
-sudo -E build/src/disk_tool/vitastor-disk dump-meta ${LOOP1}p1 | jq -S '. + {"entries": (.entries | sort_by(.stripe)) }' > ./testdata/2.json
+sudo build/src/disk_tool/vitastor-disk dump-meta ${LOOP1}p1 | jq -S '. + {"entries": (.entries | sort_by(.stripe)) }' > ./testdata/2.json
 if [[ -n "$OLD" ]]; then
     jq -S '. + {"entries": ([ .entries[] | (. + { "block": (.block-1251) }) ] | sort_by(.stripe))}' < ./testdata/meta.json > ./testdata/1.json
 else
@@ -100,24 +100,24 @@ else
 fi
 diff ./testdata/1.json ./testdata/2.json
 if [[ -n "$OLD" ]]; then
-    sudo -E build/src/disk_tool/vitastor-disk dump-journal --json --format data ${LOOP1}p1 | jq -S '[ .[] | del(.crc32, .crc32_prev) ]' > ./testdata/j2.json
+    sudo build/src/disk_tool/vitastor-disk dump-journal --json --format data ${LOOP1}p1 | jq -S '[ .[] | del(.crc32, .crc32_prev) ]' > ./testdata/j2.json
     jq -S '[ (.[] + {"valid":true}) | (if .type == "big_write_instant" then . + {"loc":"0x18f6160000"} else . end) ]' < ./testdata/journal.json > ./testdata/j1.json
     diff ./testdata/j1.json ./testdata/j2.json
 fi
 
 # move journal & meta out, data will become larger; end indexes should be shifted back by +1251
-sudo -E build/src/disk_tool/vitastor-disk-test resize --move-journal ${LOOP2}p1 --move-meta ${LOOP2}p2 ${LOOP1}p1
+sudo build/src/disk_tool/vitastor-disk-test resize --move-journal ${LOOP2}p1 --move-meta ${LOOP2}p2 ${LOOP1}p1
-sudo -E build/src/disk_tool/vitastor-disk dump-meta ${LOOP1}p1 | jq -S '. + {"entries": (.entries | sort_by(.stripe)) }' > ./testdata/2.json
+sudo build/src/disk_tool/vitastor-disk dump-meta ${LOOP1}p1 | jq -S '. + {"entries": (.entries | sort_by(.stripe)) }' > ./testdata/2.json
 jq -S '. + {"entries": (.entries | sort_by(.stripe)) }' < ./testdata/meta.json > ./testdata/1.json
 diff ./testdata/1.json ./testdata/2.json
 if [[ -n "$OLD" ]]; then
     jq -S '[ .[] + {"valid":true} ]' < ./testdata/journal.json > ./testdata/j1.json
-    sudo -E build/src/disk_tool/vitastor-disk dump-journal --json --format data ${LOOP1}p1 | jq -S '[ .[] | del(.crc32, .crc32_prev) ]' > ./testdata/j2.json
+    sudo build/src/disk_tool/vitastor-disk dump-journal --json --format data ${LOOP1}p1 | jq -S '[ .[] | del(.crc32, .crc32_prev) ]' > ./testdata/j2.json
 fi
 
 # reduce data device size by exactly 128k * 99 (occupied blocks); exactly 1 should be left in place :)
-sudo -E build/src/disk_tool/vitastor-disk-test resize --data-size $((DATA_DEV_SIZE-128*1024*99)) ${LOOP1}p1
+sudo build/src/disk_tool/vitastor-disk-test resize --data-size $((DATA_DEV_SIZE-128*1024*99)) ${LOOP1}p1
-sudo -E build/src/disk_tool/vitastor-disk dump-meta ${LOOP1}p1 | jq -S '. + {"entries": (.entries | sort_by(.stripe)) }' > ./testdata/2.json
+sudo build/src/disk_tool/vitastor-disk dump-meta ${LOOP1}p1 | jq -S '. + {"entries": (.entries | sort_by(.stripe)) }' > ./testdata/2.json
 if [[ -n "$OLD" ]]; then
     jq -S '. + {"entries": ([ .entries[] | (. + { "block": (.block | if . > '$BLOCK_COUNT'-100 then .-('$BLOCK_COUNT'-100+1) else '$BLOCK_COUNT'-100 end) }) ]
         | .[1:] + [ .[0] ]) | sort_by(.stripe)}' < ./testdata/meta.json > ./testdata/1.json
@@ -128,12 +128,12 @@ fi
 diff ./testdata/1.json ./testdata/2.json
 if [[ -n "$OLD" ]]; then
     jq -S '[ .[] + {"valid":true} ]' < ./testdata/journal.json > ./testdata/j1.json
-    sudo -E build/src/disk_tool/vitastor-disk dump-journal --json --format data ${LOOP1}p1 | jq -S '[ .[] | del(.crc32, .crc32_prev) ]' > ./testdata/j2.json
+    sudo build/src/disk_tool/vitastor-disk dump-journal --json --format data ${LOOP1}p1 | jq -S '[ .[] | del(.crc32, .crc32_prev) ]' > ./testdata/j2.json
 fi
 
 # extend data device size to maximum
-sudo -E build/src/disk_tool/vitastor-disk-test resize --data-size max ${LOOP1}p1
+sudo build/src/disk_tool/vitastor-disk-test resize --data-size max ${LOOP1}p1
-sudo -E build/src/disk_tool/vitastor-disk dump-meta ${LOOP1}p1 | jq -S '. + {"entries": (.entries | sort_by(.stripe)) }' > ./testdata/2.json
+sudo build/src/disk_tool/vitastor-disk dump-meta ${LOOP1}p1 | jq -S '. + {"entries": (.entries | sort_by(.stripe)) }' > ./testdata/2.json
 diff ./testdata/1.json ./testdata/2.json
 
 format_green OK