Use 32-bit big write location (OK for up to 512 TB OSDs)

Do not store offset & len in big_writes
Fix object crc32c calculation
2025-11-23 19:08:24 +03:00 · 2025-11-23 19:08:24 +03:00 · 2025-11-23 19:08:24 +03:00 · 2025-11-23 19:08:24 +03:00 · 2025-11-23 19:08:24 +03:00 · 2025-11-23 19:08:24 +03:00
281 changed files with 6750 additions and 27303 deletions
@@ -1,29 +1,28 @@
-FROM node:16-bookworm
+FROM node:16-bullseye

 WORKDIR /root

-ADD ./docker/etc/apt/trusted.gpg.d /etc/apt/trusted.gpg.d
+ADD ./docker/vitastor.gpg /etc/apt/trusted.gpg.d

-RUN echo 'deb http://deb.debian.org/debian bookworm-backports main' >> /etc/apt/sources.list; \
-    echo 'deb http://vitastor.io/debian bookworm main' >> /etc/apt/sources.list; \
+RUN echo 'deb http://deb.debian.org/debian bullseye-backports main' >> /etc/apt/sources.list; \
+    echo 'deb http://vitastor.io/debian bullseye main' >> /etc/apt/sources.list; \
    echo >> /etc/apt/preferences; \
    echo 'Package: *' >> /etc/apt/preferences; \
-    echo 'Pin: release n=bookworm-backports' >> /etc/apt/preferences; \
+    echo 'Pin: release a=bullseye-backports' >> /etc/apt/preferences; \
    echo 'Pin-Priority: 500' >> /etc/apt/preferences; \
    echo >> /etc/apt/preferences; \
    echo 'Package: *' >> /etc/apt/preferences; \
    echo 'Pin: origin "vitastor.io"' >> /etc/apt/preferences; \
    echo 'Pin-Priority: 1000' >> /etc/apt/preferences; \
-    perl -i -pe 's/Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/debian.sources; \
    grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb/deb-src/' >> /etc/apt/sources.list; \
    echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf; \
    echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf

 RUN apt-get update
-RUN apt-get -y install etcd qemu-system-x86 qemu-block-extra qemu-utils fio libasan8 \
+RUN apt-get -y install etcd qemu-system-x86 qemu-block-extra qemu-utils fio libasan5 \
    libgoogle-perftools-dev devscripts libjerasure-dev cmake libibverbs-dev libisal-dev
 RUN apt-get -y build-dep fio qemu=`dpkg -s qemu-system-x86|grep ^Version:|awk '{print $2}'`
-RUN apt-get update && apt-get -y install jq lp-solve sudo nfs-common fdisk parted libc-ares-dev udev
+RUN apt-get update && apt-get -y install jq lp-solve sudo nfs-common fdisk parted
 RUN apt-get --download-only source fio qemu=`dpkg -s qemu-system-x86|grep ^Version:|awk '{print $2}'`

 RUN set -ex; \
@@ -38,18 +38,6 @@ for my $line (<>)
            {
                $test_name .= '_antietcd';
            }
-            elsif ($1 eq 'ETCD_SCHEME' && $2 eq 'https')
-            {
-                $test_name .= '_https';
-            }
-            elsif ($1 eq 'ENCRYPTED')
-            {
-                $test_name .= '_encrypted';
-            }
-            elsif ($1 eq 'OLD')
-            {
-                $test_name =~ s/^test_/test_old_/s;
-            }
            else
            {
                $test_name .= '_'.lc($1).'_'.$2;
@@ -3,4 +3,3 @@
 package-lock.json
 fio
 qemu
-node_modules
@@ -1,20 +1,20 @@
-cmake_minimum_required(VERSION 2.8...3.30)
+cmake_minimum_required(VERSION 2.8.12)

 project(vitastor)

-set(VITASTOR_VERSION "3.0.9")
+set(VITASTOR_VERSION "2.4.3")

 include(CTest)

 add_custom_target(build_tests)
-set_property(TEST PROPERTY ENVIRONMENT LSAN_OPTIONS=suppressions=${CMAKE_CURRENT_BINARY_DIR}/lsan-suppress.txt)
-add_test(gen_lsan_suppress
-    ${CMAKE_COMMAND} -E echo leak:tcmalloc > "${CMAKE_CURRENT_BINARY_DIR}/lsan-suppress.txt"
+add_custom_target(test
+	COMMAND
+	echo leak:tcmalloc > ${CMAKE_CURRENT_BINARY_DIR}/lsan-suppress.txt &&
+	env LSAN_OPTIONS=suppressions=${CMAKE_CURRENT_BINARY_DIR}/lsan-suppress.txt ${CMAKE_CTEST_COMMAND}
 )
-set_tests_properties(gen_lsan_suppress PROPERTIES FIXTURES_SETUP f_lsan_suppress)
-set_property(TEST PROPERTY FIXTURES_REQUIRED f_lsan_suppress)
 # make -j16 -C ../../build test_heap && ../../build/src/test/test_heap
 # make -j16 -C ../../build test_heap && rm -f $(find ../../build -name '*.gcda') && ctest -V -T test -T coverage -R heap --test-dir ../../build && (cd ../../build; gcovr -f ../src --html --html-nested -o coverage/index.html; cd ../src/test)
 # make -j16 -C ../../build test_blockstore && rm -f $(find ../../build -name '*.gcda') && ctest -V -T test -T coverage -R blockstore --test-dir ../../build && (cd ../../build; gcovr -f ../src --html --html-nested -o coverage/index.html; cd ../src/test)
 # kcov --include-path=../../../src ../../kcov ./test_blockstore
+add_dependencies(test build_tests)
 add_subdirectory(src)
@@ -26,15 +26,11 @@ Vitastor поддерживает QEMU-драйвер, протоколы UBLK,

 ## Презентации и записи докладов

- KuberConf'2025: [видео](https://vitastor.io/presentation/kuberconf.webm)
- Highload'2025: [видео](https://vitastor.io/presentation/hl2025/hl2025.webm),
-  [на youtube](https://www.youtube.com/watch?v=0R8MLjFtz7g), презентация
-  ([на русском](https://vitastor.io/presentation/hl2025/), [на английском](https://vitastor.io/presentation/hl2025/en.html))
- Highload'2022: презентация ([на русском](https://vitastor.io/presentation/highload/highload.html)),
-  [видео](https://vitastor.io/presentation/highload/talk.webm)
 - DevOpsConf'2021: презентация ([на русском](https://vitastor.io/presentation/devopsconf/devopsconf.html),
  [на английском](https://vitastor.io/presentation/devopsconf/devopsconf_en.html)),
  [видео](https://vitastor.io/presentation/devopsconf/talk.webm)
+- Highload'2022: презентация ([на русском](https://vitastor.io/presentation/highload/highload.html)),
+  [видео](https://vitastor.io/presentation/highload/talk.webm)

 ## Документация

@@ -62,7 +58,6 @@ Vitastor поддерживает QEMU-драйвер, протоколы UBLK,
    - [Дисковые параметры OSD](docs/config/layout-osd.ru.md)
    - [Прочие параметры OSD](docs/config/osd.ru.md)
    - [Параметры мониторов](docs/config/monitor.ru.md)
-    - [Безопасность](docs/config/security.ru.md)
  - [Настройки пулов](docs/config/pool.ru.md)
  - [Метаданные образов в etcd](docs/config/inode.ru.md)
 - Использование
@@ -26,15 +26,11 @@ Read more details in the documentation. You can start from here: [Quick Start](d

 ## Talks and presentations

- KuberConf'2025: [video](https://vitastor.io/presentation/kuberconf.webm)
- Highload'2025: [video](https://vitastor.io/presentation/hl2025/hl2025.webm),
-  [youtube](https://www.youtube.com/watch?v=0R8MLjFtz7g), presentation
-  ([in Russian](https://vitastor.io/presentation/hl2025/), [in English](https://vitastor.io/presentation/hl2025/en.html))
- Highload'2022: presentation ([in Russian](https://vitastor.io/presentation/highload/highload.html)),
-  [video](https://vitastor.io/presentation/highload/talk.webm)
 - DevOpsConf'2021: presentation ([in Russian](https://vitastor.io/presentation/devopsconf/devopsconf.html),
  [in English](https://vitastor.io/presentation/devopsconf/devopsconf_en.html)),
  [video](https://vitastor.io/presentation/devopsconf/talk.webm)
+- Highload'2022: presentation ([in Russian](https://vitastor.io/presentation/highload/highload.html)),
+  [video](https://vitastor.io/presentation/highload/talk.webm)

 ## Documentation

@@ -62,7 +58,6 @@ Read more details in the documentation. You can start from here: [Quick Start](d
    - [OSD Disk Layout](docs/config/layout-osd.en.md)
    - [OSD Runtime Parameters](docs/config/osd.en.md)
    - [Monitor](docs/config/monitor.en.md)
-    - [Security](docs/config/security.en.md)
  - [Pool configuration](docs/config/pool.en.md)
  - [Image metadata in etcd](docs/config/inode.en.md)
 - Usage
@@ -1,5 +1,5 @@
 # Compile stage
-FROM golang:trixie AS build
+FROM golang:bookworm AS build

 ADD go.sum go.mod /app/
 RUN cd /app; CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go mod download -x
@@ -9,7 +9,7 @@ RUN perl -i -e '$/ = undef; while(<>) { s/\n\s*(\{\s*\n)/$1\n/g; s/\}(\s*\n\s*)e
    CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o vitastor-csi

 # Final stage
-FROM debian:trixie
+FROM debian:bookworm

 LABEL maintainers="Vitaliy Filippov <vitalif@yourcmc.ru>"
 LABEL description="Vitastor CSI Driver"
@@ -25,20 +25,20 @@ RUN apt-get update && \
        # NFS mount dependencies
        nfs-common netbase \
        # dependencies of qemu-storage-daemon
-        libaio1t64 libc6 libfuse3-4 libglib2.0-0t64 libgmp10 libgnutls30t64 \
-        libhogweed6t64 libnettle8t64 libnuma1 libselinux1 liburing2 libzstd1 zlib1g && \
+        libnuma1 liburing2 libglib2.0-0 libfuse3-3 libaio1 libzstd1 libnettle8 \
+        libgmp10 libhogweed6 libp11-kit0 libidn2-0 libunistring2 libtasn1-6 libpcre2-8-0 libffi8 && \
    apt-get clean && \
    (echo options nbd nbds_max=128 > /etc/modprobe.d/nbd.conf)

 COPY --from=build /app/vitastor-csi /bin/

-RUN (echo deb http://vitastor.io/debian trixie main > /etc/apt/sources.list.d/vitastor.list) && \
+RUN (echo deb http://vitastor.io/debian bookworm main > /etc/apt/sources.list.d/vitastor.list) && \
    ((echo 'Package: *'; echo 'Pin: origin "vitastor.io"'; echo 'Pin-Priority: 1000') > /etc/apt/preferences.d/vitastor.pref) && \
    wget -q -O /etc/apt/trusted.gpg.d/vitastor.gpg https://vitastor.io/debian/pubkey.gpg && \
    apt-get update && \
    apt-get install -y vitastor-client ibverbs-providers && \
-    wget https://vitastor.io/archive/qemu/qemu-trixie-10.0.2%2Bds-2%2Bvitastor1/qemu-utils_10.0.2%2Bds-2%2Bvitastor1_amd64.deb && \
-    wget https://vitastor.io/archive/qemu/qemu-trixie-10.0.2%2Bds-2%2Bvitastor1/qemu-block-extra_10.0.2%2Bds-2%2Bvitastor1_amd64.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-utils_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-block-extra_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
    dpkg -x qemu-utils*.deb tmp1 && \
    dpkg -x qemu-block-extra*.deb tmp1 && \
    cp -a tmp1/usr/bin/qemu-storage-daemon /usr/bin/ && \
@@ -1,5 +1,5 @@
 # Compile stage
-FROM golang:trixie AS build
+FROM golang:bookworm AS build

 ADD go.sum go.mod /app/
 RUN cd /app; CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go mod download -x
@@ -9,7 +9,7 @@ RUN perl -i -e '$/ = undef; while(<>) { s/\n\s*(\{\s*\n)/$1\n/g; s/\}(\s*\n\s*)e
    CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o vitastor-csi

 # Final stage
-FROM debian:trixie
+FROM debian:bookworm

 LABEL maintainers="Vitaliy Filippov <vitalif@yourcmc.ru>"
 LABEL description="Vitastor CSI Driver"
@@ -36,8 +36,8 @@ ADD deb /deb

 RUN apt-get update && \
    apt-get -y install /deb/vitastor-client_*.deb && \
-    wget https://vitastor.io/archive/qemu/qemu-trixie-9.2.2%2Bds-1%2Bvitastor4/qemu-utils_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
-    wget https://vitastor.io/archive/qemu/qemu-trixie-9.2.2%2Bds-1%2Bvitastor4/qemu-block-extra_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-utils_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-block-extra_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
    dpkg -x qemu-utils*.deb tmp1 && \
    dpkg -x qemu-block-extra*.deb tmp1 && \
    cp -a tmp1/usr/bin/qemu-storage-daemon /usr/bin/ && \
@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.9
+VITASTOR_VERSION ?= v2.4.3

 all: build push

@@ -49,7 +49,7 @@ spec:
            capabilities:
              add: ["SYS_ADMIN"]
            allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v3.0.9
+          image: vitalif/vitastor-csi:v2.4.3
          args:
            - "--node=$(NODE_ID)"
            - "--endpoint=$(CSI_ENDPOINT)"
@@ -121,7 +121,7 @@ spec:
            privileged: true
            capabilities:
              add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v3.0.9
+          image: vitalif/vitastor-csi:v2.4.3
          args:
            - "--node=$(NODE_ID)"
            - "--endpoint=$(CSI_ENDPOINT)"
@@ -5,7 +5,7 @@ package vitastor

 const (
    vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "3.0.9"
+    vitastorCSIDriverVersion = "2.4.3"
 )

 // Config struct fills the parameters of request or user input
@@ -1,5 +0,0 @@
-#!/bin/bash
-# 26.04 Resolute Raccoon
-
-docker build --build-arg DISTRO=ubuntu --build-arg REL=resolute -t vitastor-buildenv:resolute -f vitastor-buildenv.Dockerfile .
-docker run -it --rm -e REL=resolute -v `dirname $0`/../:/root/vitastor vitastor-buildenv:resolute /root/vitastor/debian/vitastor-build.sh
@@ -1,4 +1,4 @@
-vitastor (3.0.9-1) unstable; urgency=medium
+vitastor (2.4.3-1) unstable; urgency=medium

  * Bugfixes

@@ -3,7 +3,7 @@ Section: admin
 Priority: optional
 Maintainer: Vitaliy Filippov <vitalif@yourcmc.ru>
 Build-Depends: debhelper, g++ (>= 8), libstdc++6 (>= 8),
-  linux-libc-dev, libgoogle-perftools-dev, libjerasure-dev, libgf-complete-dev, libc-ares-dev,
+  linux-libc-dev, libgoogle-perftools-dev, libjerasure-dev, libgf-complete-dev,
  libibverbs-dev, librdmacm-dev, libisal-dev, cmake, pkg-config, libnl-3-dev, libnl-genl-3-dev,
  node-bindings <!nocheck>, node-gyp, node-nan
 Standards-Version: 4.5.0
@@ -44,7 +44,7 @@ curl -s https://git.yourcmc.ru/vitalif/antietcd/archive/master.tar.gz | tar -zx
 curl -s https://git.yourcmc.ru/vitalif/tinyraft/archive/master.tar.gz | tar -zx

 cd /root/vitastor/packages/vitastor-$REL
-if [[ ( "$REL" = "trixie" || "$REL" = "resolute" ) && -e ../vitastor-bookworm/vitastor_$VER.orig.tar.xz ]]; then
+if [[ "$REL" = "trixie" && -e ../vitastor-bookworm/vitastor_$VER.orig.tar.xz ]]; then
    # Fucking shit, archives differ between bookworm (xz 5.4.1) and trixie (xz 5.8.1)
    cp ../vitastor-bookworm/vitastor_$VER.orig.tar.xz .
 else
@@ -25,7 +25,7 @@ RUN set -e -x; \
    echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf

 RUN apt-get update && \
-    apt-get -y install fio libgoogle-perftools-dev devscripts libjerasure-dev cmake libc-ares-dev \
+    apt-get -y install fio libgoogle-perftools-dev devscripts libjerasure-dev cmake \
        libibverbs-dev librdmacm-dev libisal-dev libnl-3-dev libnl-genl-3-dev curl nodejs npm node-nan node-bindings && \
    apt-get -y build-dep fio && \
    apt-get --download-only source fio
@@ -1,6 +1,6 @@
 # Build Docker image with Vitastor packages

-FROM debian:trixie
+FROM debian:bookworm

 ADD etc/apt /etc/apt/
 RUN apt-get update && apt-get -y install vitastor ibverbs-providers udev systemd qemu-system-x86 qemu-system-common qemu-block-extra qemu-utils jq nfs-common && apt-get clean
@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.9
+VITASTOR_VERSION ?= v2.4.3

 all: build push

@@ -1,3 +1,3 @@
 Package: *
-Pin: release n=trixie-backports
+Pin: release n=bookworm-backports
 Pin-Priority: 500
@@ -1,2 +1,2 @@
-deb http://vitastor.io/debian trixie main
-#deb http://http.debian.net/debian/ trixie-backports main
+deb http://vitastor.io/debian bookworm main
+deb http://http.debian.net/debian/ bookworm-backports main
@@ -7,7 +7,7 @@ PartOf=vitastor.target
 [Service]
 Restart=always
 EnvironmentFile=/etc/vitastor/docker.conf
-ExecStart=bash -c 'docker run --rm -i -v /etc/vitastor:/etc/vitastor -v /dev:/dev -v /run:/run -e SYSTEMD_IN_CHROOT=0 \
+ExecStart=bash -c 'docker run --rm -i -v /etc/vitastor:/etc/vitastor -v /dev:/dev -v /run:/run \
    --security-opt seccomp=unconfined --privileged --pid=host --log-driver none --network host --name vitastor vitastor:$VITASTOR_VERSION \
    sleep.sh'
 ExecStartPost=udevadm trigger
@@ -4,7 +4,7 @@
 #

 # Desired Vitastor version
-VITASTOR_VERSION=v3.0.9
+VITASTOR_VERSION=v2.4.3

 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here
@@ -2,7 +2,8 @@

 set -e

-cp -urv /etc/systemd/system/vitastor* /host-etc/systemd/system/
-cp -urv /etc/udev/rules.d /host-etc/udev/
+cp -urv /etc/default /host-etc/
+cp -urv /etc/systemd /host-etc/
+cp -urv /etc/udev /host-etc/
 cp -urnv /etc/vitastor /host-etc/
 cp -urnv /opt/scripts/* /host-bin/
@@ -38,4 +38,3 @@ In the future, additional configuration methods may be added:
 - [OSD Disk Layout](config/layout-osd.en.md)
 - [OSD Runtime Parameters](config/osd.en.md)
 - [Monitor](config/monitor.en.md)
- [Security Parameters](config/security.en.md)
@@ -41,4 +41,3 @@
 - [Дисковые параметры OSD](config/layout-osd.ru.md)
 - [Прочие параметры OSD](config/osd.ru.md)
 - [Параметры мониторов](config/monitor.ru.md)
- [Параметры безопасности](config/security.ru.md)
@@ -9,7 +9,6 @@
 These parameters apply to OSDs, are fixed at the moment of OSD drive
 initialization and can't be changed after it without losing data.

- [meta_format](#meta_format)
 - [data_device](#data_device)
 - [meta_device](#meta_device)
 - [journal_device](#journal_device)
@@ -28,21 +27,6 @@ initialization and can't be changed after it without losing data.
 - [data_csum_type](#data_csum_type)
 - [csum_block_size](#csum_block_size)

-## meta_format
-
- Type: integer
- Default: 3
-
-OSD store implementation version and on-disk metadata format.
-
-Three versions are currently supported: 3, 2 and 1.
- 3 the new log-structured store, it's overall faster, has lower Write
-  Amplification, which may be even close to 1 (i.e. almost no extra writes)
-  if your SSDs support atomic writes (see [atomic_write_size](osd.en.md#atomic_write_size)).
- 2 is the old stable store from Vitastor 0.9-2.x.
- 1 is the same old store but with a legacy metadata format from Vitastor
-  versions to up 0.8.x, without any support for checksums.
-
 ## data_device

 - Type: string
@@ -198,14 +182,8 @@ put a modified value into etcd key /vitastor/config/global.
 - Type: string
 - Default: none

-Data and metadata checksum type to use. May be "crc32c", "xxh3_32" or "none".
-Select crc32c or xxh3_32 and set csum_block_size to enable data checksums.
-
-Both crc32c and xxh3_32 are almost equally fast, xxh3_32 is safer. xxh3_32 is
-the xxhash3 algorithm truncated from 64 to 32 bits (which is still a good hash).
-
-Note that enabled data checksums either increase memory usage or reduce
-performance. Check details in [csum_block_size](#csum_block_size) description.
+Data checksum type to use. May be "crc32c" or "none". Set to "crc32c" to
+enable data checksums.

 ## csum_block_size

@@ -10,7 +10,6 @@
 дисковые параметры, задаются в момент инициализации дисков OSD и не могут быть
 изменены после этого без потери данных.

- [meta_format](#meta_format)
 - [data_device](#data_device)
 - [meta_device](#meta_device)
 - [journal_device](#journal_device)
@@ -29,23 +28,6 @@
 - [data_csum_type](#data_csum_type)
 - [csum_block_size](#csum_block_size)

-## meta_format
-
- Тип: целое число
- Значение по умолчанию: 3
-
-Версия реализации дискового хранилища OSD и дискового формата метаданных.
-
-Поддерживаются три версии: 3, 2 и 1.
- 3 - новое лог-структурированное хранилище, в целом более быстрое, со
-  сниженным фактором амплификации записи, который может составлять около 1
-  (то есть, практически без лишней служебной записи), если ваши SSD
-  поддерживают атомарную запись (см. [atomic_write_size](osd.ru.md#atomic_write_size)).
- 2 - старое стабильное хранилище из версий Vitastor 0.9-2.x.
- 1 - то же самое стабильное хранилище, но с ещё более старым форматом
-  метаданных из версий Vitastor до 0.8.x, без какой-либо поддержки
-  контрольных сумм.
-
 ## data_device

 - Тип: строка
@@ -209,12 +191,8 @@ journal_block_size и meta_block_size. Однако на данный момен
 - Тип: строка
 - Значение по умолчанию: none

-Тип используемых OSD контрольных сумм данных и метаданных. Может быть "crc32c",
-"xxh3_32" или "none". Выберите crc32c или xxh3_32 и установите csum_block_size,
-чтобы включить контрольные суммы данных.
-
-И crc32c, и xxh3_32 примерно одинаково быстры, xxh3_32 надёжней. xxh3_32 - это
-алгоритм xxhash3, обрезанный с 64 до 32 бит (это всё равно хороший хеш).
+Тип используемых OSD контрольных сумм данных. Может быть "crc32c" или "none".
+Установите в "crc32c", чтобы включить расчёт и проверку контрольных сумм данных.

 Следует понимать, что контрольные суммы в зависимости от размера блока их
 расчёта либо увеличивают потребление памяти, либо снижают производительность.
@@ -22,6 +22,7 @@ between clients, OSDs and etcd.
 - [rdma_max_msg](#rdma_max_msg)
 - [rdma_max_recv](#rdma_max_recv)
 - [rdma_max_send](#rdma_max_send)
+- [rdma_odp](#rdma_odp)
 - [peer_connect_interval](#peer_connect_interval)
 - [peer_connect_timeout](#peer_connect_timeout)
 - [osd_idle_timeout](#osd_idle_timeout)
@@ -101,6 +102,11 @@ found or if `osd_network` is not specified. Auto-selection is also
 unsupported with old libibverbs < v32, like in Debian 10 Buster or
 CentOS 7.

+Vitastor supports all adapters, even ones without ODP support, like
+Mellanox ConnectX-3 and non-Mellanox cards. Versions up to Vitastor
+1.2.0 required ODP which is only present in Mellanox ConnectX >= 4.
+See also [rdma_odp](#rdma_odp).
+
 Run `ibv_devinfo -v` as root to list available RDMA devices and their
 features.

@@ -110,23 +116,6 @@ the manual of your network vendor for details about setting up the switch
 for RoCEv2 correctly. Usually it means setting up Lossless Ethernet with
 PFC (Priority Flow Control) and ECN (Explicit Congestion Notification).

-Vitastor supports all adapters, even ones without ODP (On-Demand Paging)
-support, like Mellanox ConnectX-3 and non-Mellanox cards. ODP is only present
-in Mellanox ConnectX >= 4 adapters and allows to skip memory registration
-for RDMA and thus, in theory, avoid memory copying.
-
-Versions up to Vitastor 1.2.0 required ODP, then it was disabled by default,
-but it was still supported up to 3.0.3. Now ODP support is removed because it
-actually only hurts performance: an example 3-node cluster with 8 NVMe in each
-node and 2*25 GBit/s ConnectX-6 RDMA network pushed 3950000 read iops without
-ODP, but only 239000 iops with ODP.
-
-This happens because Mellanox ODP implementation seems to be based on
-message retransmissions when the adapter doesn't know about the buffer yet -
-it likely uses standard "RNR retransmissions" (RNR = receiver not ready)
-which is generally slow in RDMA/RoCE networks. Here's a presentation about
-it from ISPASS-2021 conference: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
-
 ## rdma_port_num

 - Type: integer
@@ -198,6 +187,28 @@ less than `rdma_max_recv` so the receiving side doesn't run out of buffers.
 Doesn't affect memory usage - additional memory isn't allocated for send
 operations.

+## rdma_odp
+
+- Type: boolean
+- Default: false
+
+Use RDMA with On-Demand Paging. ODP is currently only available on Mellanox
+ConnectX-4 and newer adapters. ODP allows to not register memory explicitly
+for RDMA adapter to be able to use it. This, in turn, allows to skip memory
+copying during sending. One would think this should improve performance, but
+**in reality** RDMA performance with ODP is **drastically** worse. Example
+3-node cluster with 8 NVMe in each node and 2*25 GBit/s ConnectX-6 RDMA network
+without ODP pushes 3950000 read iops, but only 239000 iops with ODP...
+
+This happens because Mellanox ODP implementation seems to be based on
+message retransmissions when the adapter doesn't know about the buffer yet -
+it likely uses standard "RNR retransmissions" (RNR = receiver not ready)
+which is generally slow in RDMA/RoCE networks. Here's a presentation about
+it from ISPASS-2021 conference: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
+
+ODP support is retained in the code just in case a good ODP implementation
+appears one day.
+
 ## peer_connect_interval

 - Type: seconds
@@ -22,6 +22,7 @@
 - [rdma_max_msg](#rdma_max_msg)
 - [rdma_max_recv](#rdma_max_recv)
 - [rdma_max_send](#rdma_max_send)
+- [rdma_odp](#rdma_odp)
 - [peer_connect_interval](#peer_connect_interval)
 - [peer_connect_timeout](#peer_connect_timeout)
 - [osd_idle_timeout](#osd_idle_timeout)
@@ -100,6 +101,12 @@ RoCEv1/RoCEv2, и даже позволяет полностью отключи
 не задана. Также автовыбор не поддерживается со старыми версиями библиотеки
 libibverbs < v32, например в Debian 10 Buster или CentOS 7.

+Vitastor поддерживает все модели адаптеров, включая те, у которых
+нет поддержки ODP, то есть вы можете использовать RDMA с ConnectX-3 и
+картами производства не Mellanox. Версии Vitastor до 1.2.0 включительно
+требовали ODP, который есть только на Mellanox ConnectX 4 и более новых.
+См. также [rdma_odp](#rdma_odp).
+
 Запустите `ibv_devinfo -v` от имени суперпользователя, чтобы посмотреть
 список доступных RDMA-устройств, их параметры и возможности.

@@ -110,24 +117,6 @@ libibverbs < v32, например в Debian 10 Buster или CentOS 7.
 подразумевает настройку сети без потерь на основе PFC (Priority Flow
 Control) и ECN (Explicit Congestion Notification).

-Vitastor поддерживает все модели адаптеров, включая те, у которых нет
-поддержки ODP (On-Demand Paging), например, ConnectX-3 и карты производства
-не Mellanox. Функция ODP доступна только на адаптерах Mellanox ConnectX-4 и
-более новых и позволяет не регистрировать память для её использования RDMA-картой,
-благодаря чему в теории можно избежать лишних копирований памяти.
-
-Версии Vitastor до 1.2.0 включительно требовали ODP, потом функция был отключена
-по умолчанию, но поддерживалась вплоть до версии 3.0.3. Сейчас поддержка ODP
-полностью удалена, так как на самом деле она только портит производительность:
-например, на 3-узловом кластере с 8 NVMe в каждом узле и сетью 2*25 Гбит/с на
-чтение с RDMA без ODP удаётся снять 3950000 iops, а с ODP - всего 239000 iops.
-
-Это происходит из-за того, что реализация ODP у Mellanox неоптимальная и
-основана на повторной передаче сообщений, когда карте не известен буфер -
-вероятно, на стандартных "RNR retransmission" (RNR = receiver not ready).
-А данные повторные передачи в RDMA/RoCE - всегда очень медленная штука.
-Презентация на эту тему с конференции ISPASS-2021: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
-
 ## rdma_port_num

 - Тип: целое число
@@ -203,6 +192,29 @@ OSD в любом случае согласовывают реальное зн
 Не влияет на потребление памяти - дополнительная память на операции отправки
 не выделяется.

+## rdma_odp
+
+- Тип: булево (да/нет)
+- Значение по умолчанию: false
+
+Использовать RDMA с On-Demand Paging. ODP - функция, доступная пока что
+исключительно на адаптерах Mellanox ConnectX-4 и более новых. ODP позволяет
+не регистрировать память для её использования RDMA-картой. Благодаря этому
+можно не копировать данные при отправке их в сеть и, казалось бы, это должно
+улучшать производительность - но **по факту** получается так, что
+производительность только ухудшается, причём сильно. Пример - на 3-узловом
+кластере с 8 NVMe в каждом узле и сетью 2*25 Гбит/с на чтение с RDMA без ODP
+удаётся снять 3950000 iops, а с ODP - всего 239000 iops...
+
+Это происходит из-за того, что реализация ODP у Mellanox неоптимальная и
+основана на повторной передаче сообщений, когда карте не известен буфер -
+вероятно, на стандартных "RNR retransmission" (RNR = receiver not ready).
+А данные повторные передачи в RDMA/RoCE - всегда очень медленная штука.
+Презентация на эту тему с конференции ISPASS-2021: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
+
+Возможность использования ODP сохранена в коде на случай, если вдруг в один
+прекрасный день появится хорошая реализация ODP.
+
 ## peer_connect_interval

 - Тип: секунды
@@ -38,7 +38,6 @@ with an OSD restart or, for some of them, even without restarting by updating co
 - [journal_io](#journal_io)
 - [journal_sector_buffer_count](#journal_sector_buffer_count)
 - [journal_no_same_sector_overwrites](#journal_no_same_sector_overwrites)
- [skip_corrupted_meta_entries](#skip_corrupted_meta_entries)
 - [throttle_small_writes](#throttle_small_writes)
 - [throttle_target_iops](#throttle_target_iops)
 - [throttle_target_mbs](#throttle_target_mbs)
@@ -66,11 +65,6 @@ with an OSD restart or, for some of them, even without restarting by updating co
 - [allow_net_split](#allow_net_split)
 - [enable_pg_locks](#enable_pg_locks)
 - [pg_lock_retry_interval_ms](#pg_lock_retry_interval_ms)
- [atomic_write_size](#atomic_write_size)
- [use_atomic_flag](#use_atomic_flag)
- [pg_reshard_chunk_size](#pg_reshard_chunk_size)
- [pg_reshard_chunk_pause_ms](#pg_reshard_chunk_pause_ms)
- [gc_on_start](#gc_on_start)

 ## bind_address

@@ -281,19 +275,13 @@ Maximum number of journal flushers (see above min_flusher_count).
 - Type: boolean
 - Default: true

-Only for the old store ([meta_format](layout-osd.en.md#meta_format) 2).
-
-This parameter makes Vitastor keep a copy of metadata area in memory as it is
-on disk, in addition to the metadata database. When the option is enabled, every
-metadata entry is effectively stored in RAM twice. It's required for good performance
-because it allows to avoid additional read-modify-write cycles during metadata
-modifications. Metadata area size with the old store is roughly 224 MB per 1 TB
-of data. You can turn the option off to reduce memory usage by this value, but
-it will reduce performance.
-
-For the new store ([meta_format](layout-osd.en.md#meta_format) 3), the option
-may be changed in the future to support operation without loading full metadata
-database in memory.
+This parameter makes Vitastor always keep metadata area of the block device
+in memory. It's required for good performance because it allows to avoid
+additional read-modify-write cycles during metadata modifications. Metadata
+area size is currently roughly 224 MB per 1 TB of data. You can turn it off
+to reduce memory usage by this value, but it will hurt performance. This
+restriction is likely to be removed in the future along with the upgrade
+of the metadata storage scheme.

 ## inmemory_journal

@@ -372,8 +360,6 @@ blocks. The only situation when you should increase it to a larger value
 is when you enable journal_no_same_sector_overwrites. In this case set
 it to, for example, 1024.

-Not applicable to the new store ([meta_format](layout-osd.en.md#meta_format) 3).
-
 ## journal_no_same_sector_overwrites

 - Type: boolean
@@ -387,17 +373,6 @@ journal after writing it instead of possibly overwriting it the second time.

 Most (99%) other SSDs don't need this option.

-Not applicable to the new store ([meta_format](layout-osd.en.md#meta_format) 3).
-
-## skip_corrupted_meta_entries
-
- Type: boolean
- Default: false
-
-Only for the new store ([meta_format](layout-osd.en.md#meta_format) 3).
-Allow OSD to start when some metadata entries or blocks are corrupted by
-skipping them. Should be only used as an emergency measure.
-
 ## throttle_small_writes

 - Type: boolean
@@ -691,72 +666,3 @@ Use this parameter to enable or disable this function for all pools.
 - Default: 100

 Retry interval for failed PG lock attempts.
-
-## atomic_write_size
-
- Type: integer
- Default: 4096
-
-Maximum data device atomic write size allowed for OSD to use.
-
-Atomic writes allow to reduce the Write Amplification factor with the new store
-([meta_format](layout-osd.en.md#meta_format)=3) to almost 1 (i.e. almost no extra writes)
-with replicated pools and reach the best possible write performance.
-
-Default value is auto-detected during OSD initialization from
-`/sys/block/xx/queue/atomic_write_max_bytes` or assumed to be 4096 bytes
-because all known disks support 4 KB atomic writes. Auto-detection is only used for
-NVMe disks because SAS disks require the explicit WRITE ATOMIC command which requires
-RWF_ATOMIC (see below [#use_atomic_flag]) but that flag works incorrectly in current
-Linux versions.
-
-You can also check if your NVMe drives support atomic writes by running
-the command `nvme id-ctrl /dev/nvme0n1 | grep awupf`. If the reported value,
-plus 1, multiplied by the currently selected block size of the NVMe,
-is more than 4 KB, then the new store can utilize it for better performance.
-The only drives known to support it currently are [Micron and Kioxia](../intro/quickstart.en.md).
-
-Atomic writes allow to skip double data writes in replicated pools, thus
-reducing Write Amplification and improving write performance up to 2 times.
-
-## use_atomic_flag
-
- Type: boolean
-
-This option controls whether Vitastor OSDs use RWF_ATOMIC write flag with atomic writes.
-This flag is supported since Linux 6.11 and adds some safety to atomic writes - the kernel
-guarantees to not fragment write requests with it and also to check them against the actual
-device atomic write capabilities.
-
-However, the option is disabled by default because the flag is currently UNUSABLE - Linux
-incorrectly requires writes with that flag to be of power-of-2 length and length-aligned.
-I.e., for example, 12 KB writes and not-8-KB aligned 8 KB writes are forbidden by the kernel,
-even though the NVMe specification allows them.
-
-For NVMe disks with `scheduler=none` writes aren't fragmented anyway so it's not a big deal.
-However, you can rebuild your kernel with [this patch](../../patches/linux-fix-atomic-write-checks.diff)
-and turn this option on. It will make your atomic writes a bit safer.
-
-## pg_reshard_chunk_size
-
- Type: integer
- Default: 100000
-
-Pool PG count change is a CPU-intensive operation because OSDs store the full object database
-in memory and have to move all entries between old and new PGs. Thus it's performed in chunks,
-with pauses between chunks to prevent blocking OSD's event loop and other clients' operations.
-This option sets the maximum number of object is a chunk. Moving 100k objects usually takes
-50-100ms. Chunk size equal to 0 means unlimited.
-
-## pg_reshard_chunk_pause_ms
-
- Type: milliseconds
- Default: 100
-
-This option sets the interval between handling two PG count change chunks.
-
-## gc_on_start
-
- Type: boolean
-
-Forcibly clean all garbage entries in the new store on every OSD restart.
@@ -39,7 +39,6 @@
 - [journal_io](#journal_io)
 - [journal_sector_buffer_count](#journal_sector_buffer_count)
 - [journal_no_same_sector_overwrites](#journal_no_same_sector_overwrites)
- [skip_corrupted_meta_entries](#skip_corrupted_meta_entries)
 - [throttle_small_writes](#throttle_small_writes)
 - [throttle_target_iops](#throttle_target_iops)
 - [throttle_target_mbs](#throttle_target_mbs)
@@ -67,11 +66,6 @@
 - [allow_net_split](#allow_net_split)
 - [enable_pg_locks](#enable_pg_locks)
 - [pg_lock_retry_interval_ms](#pg_lock_retry_interval_ms)
- [atomic_write_size](#atomic_write_size)
- [use_atomic_flag](#use_atomic_flag)
- [pg_reshard_chunk_size](#pg_reshard_chunk_size)
- [pg_reshard_chunk_pause_ms](#pg_reshard_chunk_pause_ms)
- [gc_on_start](#gc_on_start)

 ## bind_address

@@ -289,19 +283,13 @@ Flusher - это микро-поток (корутина), которая коп
 - Тип: булево (да/нет)
 - Значение по умолчанию: true

-Только для старого хранилища ([meta_format](layout-osd.en.md#meta_format) 2).
-
-Данный параметр заставляет Vitastor всегда держать копию области метаданных
-в памяти в том же виде, как она лежит на диске, в дополнение к БД метаданных.
-То есть, с включённой опцией каждая запись метаданных хранится в памяти дважды.
-Это нужно, чтобы избегать дополнительных операций чтения с диска при записи.
-Размер области метаданных в старом хранилище составляет примерно 224 МБ на
-1 ТБ данных. Вы можете отключить опцию, чтобы снизить потребление памяти
-примерно на эту величину, но при этом также снизится и производительность.
-
-Для нового хранилища ([meta_format](layout-osd.en.md#meta_format) 3) опция,
-возможно, будет переработана в будущем для поддержки работы без полной
-загрузки метаданных в памяти.
+Данный параметр заставляет Vitastor всегда держать область метаданных диска
+в памяти. Это нужно, чтобы избегать дополнительных операций чтения с диска
+при записи. Размер области метаданных на данный момент составляет примерно
+224 МБ на 1 ТБ данных. При включении потребление памяти снизится примерно
+на эту величину, но при этом также снизится и производительность. В будущем,
+после обновления схемы хранения метаданных, это ограничение, скорее всего,
+будет ликвидировано.

 ## inmemory_journal

@@ -384,8 +372,6 @@ fsync небезопасным даже с режимом "directsync".
 нужно менять - это если вы включаете journal_no_same_sector_overwrites. В
 этом случае установите данный параметр, например, в 1024.

-Неприменимо к новому хранилищу ([meta_format](layout-osd.en.md#meta_format) 3).
-
 ## journal_no_same_sector_overwrites

 - Тип: булево (да/нет)
@@ -401,18 +387,6 @@ fsync небезопасным даже с режимом "directsync".

 Почти все другие SSD (99% моделей) не требуют данной опции.

-Неприменимо к новому хранилищу ([meta_format](layout-osd.en.md#meta_format) 3).
-
-## skip_corrupted_meta_entries
-
- Тип: булево (да/нет)
- Значение по умолчанию: false
-
-Только для нового хранилища ([meta_format](layout-osd.en.md#meta_format) 3).
-Разрешить OSD запускаться, даже если часть блоков или записей метаданных
-повреждена, пропуская их. Опция предназначена для использования только в
-целях аварийного восстановления.
-
 ## throttle_small_writes

 - Тип: булево (да/нет)
@@ -725,78 +699,3 @@ pg_minsize OSD во время переключений, что может по
 - Значение по умолчанию: 100

 Интервал повтора неудачных попыток блокировки PG.
-
-## atomic_write_size
-
- Тип: целое число
- Значение по умолчанию: 4096
-
-Максимальный размер атомарной записи на диск данных, который OSD разрешено использовать.
-
-Поддержка атомарной записи позволяет снизить мультипликатор записи (Write Amplification)
-на диск с новым хранилищем ([meta_format](layout-osd.ru.md#meta_format)=3)
-практически до 1 (то есть, почти до нулевого объёма лишней записи) в реплицированных
-пулах и достигнуть наилучшей возможной производительности записи.
-
-Значение по умолчанию авто-определяется во время инициализации OSD из
-`/sys/block/xx/queue/atomic_write_max_bytes` либо принимается равным 4096,
-так как все известные диски поддерживают атомарную запись 4 КБ блоков.
-Автоопределение применяется только для NVMe-дисков, так как SAS диски требуют
-использования отдельной команды WRITE ATOMIC, а для неё нужен флаг RWF_ATOMIC
-(см. ниже [#use_atomic_flag]), а он в текущих версиях Linux работает некорректно.
-
-Вы также можете проверить, поддерживают ли ваши NVMe-диски атомарную запись,
-с помощью команды `nvme id-ctrl /dev/nvme0n1 | grep awupf`. Если значение awupf
-плюс 1, умноженное на текущий выбранный размер блока NVMe-диска, больше 4 КБ,
-то новое хранилище может использовать атомарные записи для достижения лучшей
-производительности. Единственные известные диски, которые поддерживают это сейчас -
-[Micron и Kioxia](../intro/quickstart.ru.md).
-
-Атомарная запись позволяет не использовать двойную запись данных (в журнал и на
-устройство данных) в реплицированных пулах и таким образом снижает амплификацию
-записи (объём служебной записи на диск) и улучшает производительность записи
-вплоть до 2-х кратного прироста.
-
-## use_atomic_flag
-
- Тип: булево (да/нет)
-
-Данная опция контролирует использование Vitastor OSD флага RWF_ATOMIC при атомарной записи
-блоков. Этот флаг поддерживается, начиная с версии ядра Linux 6.11 и добавляет немного корректности
-атомарным записям - ядро гарантирует отсутствие фрагментации запросов записи с этим флагом и
-проверяет их на соответствие реальным возможностям устройства.
-
-Однако, данная опция по умолчанию отключена, так как флаг в текущих версиях Linux работает
-абсолютно НЕКОРРЕКТНО - при нём Linux требует, чтобы запросы записи имели длину, равную
-степени двойки и были выровнены на эту длину. То есть, например, 12 КБ запросы записи, а также
-8 КБ запросы записи по не-кратному 8 КБ смещению запрещаются ядром, хотя спецификация NVMe их
-разрешает.
-
-Для NVMe-дисков с `scheduler=none` запросы записи и так не фрагментируются, так что это не так
-уж и важно, однако вы можете пересобрать своё ядро с [этим патчем](../../patches/linux-fix-atomic-write-checks.diff)
-и включить данную опцию. Это сделает вашу атомарную запись капельку безопаснее.
-
-## pg_reshard_chunk_size
-
- Тип: целое число
- Значение по умолчанию: 100000
-
-Изменение числа PG в пуле заметно загружает процессор, так как OSD хранят полную базу данных
-объектов в памяти и им приходится перемещать все записи объектов между старыми и новыми PG.
-Поэтому изменение применяется порциями, с паузами между порциями, чтобы не блокировать обработку
-событий OSD и операции остальных клиентов. Данная опция задаёт максимальное число объектов
-в порции. Перемещение 100 тысяч объектов (значение по умолчанию) обычно занимает порядка
-50-100 миллисекунд. Значение опции 0 отключает лимит размера порции.
-
-## pg_reshard_chunk_pause_ms
-
- Тип: миллисекунды
- Значение по умолчанию: 100
-
-Данная опция задаёт интервал между обработкой двух порций изменения числа PG пулов.
-
-## gc_on_start
-
- Тип: булево (да/нет)
-
-Принудительно очищать все мусорные записи в новом хранилище при каждом запуске OSD.
@@ -1,150 +0,0 @@
-[Documentation](../../README.md#documentation) → [Configuration](../config.en.md) → Security Parameters
-
-----
-
-[Читать на русском](security.ru.md)
-
-# Security Parameters
-
-These parameters affect your Vitastor installation security and apply to OSDs, monitors and clients.
-
-Most of them can be set in /etc/vitastor/vitastor.conf and in etcd, but don't support online modification.
-
- [etcd_client_cert](#etcd_client_cert)
- [etcd_client_key](#etcd_client_key)
- [etcd_ca](#etcd_ca)
- [osd_etcd_client_cert](#osd_etcd_client_cert)
- [osd_etcd_client_key](#osd_etcd_client_key)
- [mon_etcd_client_cert](#mon_etcd_client_cert)
- [mon_etcd_client_key](#mon_etcd_client_key)
- [vault_url](#vault_url)
- [vault_secret_api_path](#vault_secret_api_path)
- [vault_client_cert](#vault_client_cert)
- [vault_client_key](#vault_client_key)
- [vault_ca](#vault_ca)
- [vault_timeout_ms](#vault_timeout_ms)
- [vault_error_timeout_sec](#vault_error_timeout_sec)
- [vault_refresh_leeway_sec](#vault_refresh_leeway_sec)
- [max_aes_xts_pool_size](#max_aes_xts_pool_size)
-
-## etcd_client_cert
-
- Type: string
-
-Client TLS certificate to use for Vitastor client (not OSD and not monitor)
-etcd https connections. May be path to a file or just a PEM string with certificate.
-In the latter case, string must begin with "-----BEGIN CERTIFICATE-----".
-
-## etcd_client_key
-
- Type: string
-
-Private key for etcd_client_cert (also a file or a PEM string).
-
-## etcd_ca
-
- Type: string
-
-Trusted TLS CA to verify etcd server certificate. May be path to a file,
-directory or just a PEM string with certificate.
-
-## osd_etcd_client_cert
-
- Type: string
-
-Same as [etcd_client_cert](#etcd_client_cert), but only for OSDs.
-OSDs, clients and monitors should have different permissions, so they should
-use different certificates.
-
-## osd_etcd_client_key
-
- Type: string
-
-Same as [etcd_client_key](#etcd_client_key), but only for OSDs.
-
-## mon_etcd_client_cert
-
- Type: string
-
-Same as [etcd_client_cert](#etcd_client_cert), but only for Vitastor monitors.
-
-## mon_etcd_client_key
-
- Type: string
-
-Same as [etcd_client_key](#etcd_client_key), but only for Vitastor monitors.
-
-## vault_url
-
- Type: string
-
-Vault base URL.
-
-Vitastor clients support AES-256-XTS image data encryption with different per-image keys.
-Encryption is performed by the client, OSDs don't have access to decrypted data.
-
-Encryption keys may be stored in etcd or, for the increased security level, in an external
-[HashiCorp Vault](https://developer.hashicorp.com/vault/) or [OpenBao](https://openbao.org/)
-instance.
-
-Vitastor clients use [v1 k/v secrets engine](https://openbao.org/api-docs/secret/kv/kv-v1/)
-and [TLS authentication engine](https://openbao.org/api-docs/auth/cert/) in Vault.
-
-In that case, only key IDs are stored in etcd.
-
-## vault_secret_api_path
-
- Type: string
- Default: /v1/secret/
-
-Vault v1 secret API mount path to use.
-
-## vault_client_cert
-
- Type: string
-
-Client TLS certificate to use for Vault connections. Just like [etcd_client_cert](#etcd_client_cert),
-may be path to a file or just a certificate in PEM string.
-
-## vault_client_key
-
- Type: string
-
-Private key for vault_client_cert (also a file or a PEM string).
-
-## vault_ca
-
- Type: string
-
-Trusted TLS CA to verify Vault server certificate. May be path to a file,
-directory or just a PEM string with certificate.
-
-## vault_timeout_ms
-
- Type: integer
- Default: 5000
-
-Timeout for Vault requests in milliseconds.
-
-## vault_error_timeout_sec
-
- Type: integer
- Default: 60
-
-Time (in seconds) to wait before retrying after receiving an error from Vault.
-
-## vault_refresh_leeway_sec
-
- Type: integer
- Default: 60
-
-Extra time (in seconds) before real Vault token lease_timeout to refresh it, just
-in case of system clock drift.
-
-## max_aes_xts_pool_size
-
- Type: integer
- Default: 256
-
-Maximum number of OpenSSL encryption contexts cached in OSD memory. Probably
-doesn't require modification.
@@ -1,154 +0,0 @@
-[Документация](../../README-ru.md#документация) → [Конфигурация](../config.ru.md) → Параметры безопасности
-
-----
-
-[Read in English](security.en.md)
-
-# Параметры безопасности
-
-Данные параметры затрагивают безопасность инсталляций Vitastor и используются
-OSD, мониторами и клиентами.
-
-Большая их часть может задаваться в /etc/vitastor/vitastor.conf и в etcd, но не
-поддерживает онлайн-изменение.
-
- [etcd_client_cert](#etcd_client_cert)
- [etcd_client_key](#etcd_client_key)
- [etcd_ca](#etcd_ca)
- [osd_etcd_client_cert](#osd_etcd_client_cert)
- [osd_etcd_client_key](#osd_etcd_client_key)
- [mon_etcd_client_cert](#mon_etcd_client_cert)
- [mon_etcd_client_key](#mon_etcd_client_key)
- [vault_url](#vault_url)
- [vault_secret_api_path](#vault_secret_api_path)
- [vault_client_cert](#vault_client_cert)
- [vault_client_key](#vault_client_key)
- [vault_ca](#vault_ca)
- [vault_timeout_ms](#vault_timeout_ms)
- [vault_error_timeout_sec](#vault_error_timeout_sec)
- [vault_refresh_leeway_sec](#vault_refresh_leeway_sec)
- [max_aes_xts_pool_size](#max_aes_xts_pool_size)
-
-## etcd_client_cert
-
- Тип: строка
-
-Клиентский TLS сертификат для https-подключений к etcd для клиентов Vitastor
-(не OSD и не мониторов). Может быть путём к файлу или просто строкой с
-сертификатом в формате PEM. В последнем случае строка должна начинаться с
-"-----BEGIN CERTIFICATE-----".
-
-## etcd_client_key
-
- Тип: строка
-
-Закрытый ключ для сертификата etcd_client_cert (также путь к файлу или PEM строка).
-
-## etcd_ca
-
- Тип: строка
-
-Доверенный корневой TLS-сертификат для проверки сертификата сервера etcd.
-Может быть путём к файлу, директории или просто строкой с сертификатом в
-формате PEM.
-
-## osd_etcd_client_cert
-
- Тип: строка
-
-Аналогично [etcd_client_cert](#etcd_client_cert), но только для OSD.
-OSD, клиенты и мониторы должны иметь разные привилегии, поэтому они должны
-использовать разные сертификаты.
-
-## osd_etcd_client_key
-
- Тип: строка
-
-Аналогично [etcd_client_key](#etcd_client_key), но только для OSD.
-
-## mon_etcd_client_cert
-
- Тип: строка
-
-Аналогично [etcd_client_cert](#etcd_client_cert), но только для мониторов Vitastor.
-
-## mon_etcd_client_key
-
- Тип: строка
-
-Аналогично [etcd_client_key](#etcd_client_key), но только для мониторов Vitastor.
-
-## vault_url
-
- Тип: строка
-
-Базовый адрес Vault.
-
-Клиенты Vitastor поддерживают AES-256-XTS шифрование данных образов с отдельными ключами на
-каждый образ. Данные шифруются клиентами, OSD не имеют доступа к незашифрованным данным.
-
-Ключи шифрования могут храниться в etcd или, для повышенного уровня безопасности, во внешнем
-[HashiCorp Vault](https://developer.hashicorp.com/vault/) или [OpenBao](https://openbao.org/).
-
-Клиенты Vitastor используют [движок секретов v1](https://openbao.org/api-docs/secret/kv/kv-v1/)
-и [TLS-аутентификацию](https://openbao.org/api-docs/auth/cert/) в Vault.
-
-В этом случае, только ID ключей хранятся в etcd.
-
-## vault_secret_api_path
-
- Тип: строка
- Значение по умолчанию: /v1/secret/
-
-Путь к API секретов v1 для использования клиентами.
-
-## vault_client_cert
-
- Тип: строка
-
-Клиентский TLS сертификат для подключений к Vault. Как и [etcd_client_cert](#etcd_client_cert),
-может быть путём к файлу или просто PEM-строкой с сертификатом.
-
-## vault_client_key
-
- Тип: строка
-
-Закрытый ключ для сертификата vault_client_cert (также путь к файлу или PEM строка).
-
-## vault_ca
-
- Тип: строка
-
-Доверенный корневой TLS-сертификат для проверки сертификата сервера Vault.
-Может быть путём к файлу, директории или просто строкой с сертификатом в
-формате PEM.
-
-## vault_timeout_ms
-
- Тип: целое число
- Значение по умолчанию: 5000
-
-Максимально время выполнения Vault-запросов в миллисекундах.
-
-## vault_error_timeout_sec
-
- Тип: целое число
- Значение по умолчанию: 60
-
-Время (в секундах) для ожидания перед повторной попыткой при получении ошибки от Vault.
-
-## vault_refresh_leeway_sec
-
- Тип: целое число
- Значение по умолчанию: 60
-
-Зазор времени (в секундах), чтобы обновлять токены Vault чуть раньше их реального
-lease_timeout, на случай "ухода" системных часов.
-
-## max_aes_xts_pool_size
-
- Тип: целое число
- Значение по умолчанию: 256
-
-Максимальное количество кэшируемых в памяти OSD контекстов шифрования OpenSSL.
-Вряд ли требует изменения.
@@ -44,8 +44,6 @@

 {{../../config/monitor.en.md|indent=2}}

-{{../../config/security.en.md|indent=2}}
-
 {{../../config/pool.en.md|indent=2}}

 {{../../config/inode.en.md|indent=2}}
@@ -44,8 +44,6 @@

 {{../../config/monitor.ru.md|indent=2}}

-{{../../config/security.ru.md|indent=2}}
-
 {{../../config/pool.ru.md|indent=2}}

 {{../../config/inode.ru.md|indent=2}}
@@ -1,28 +1,3 @@
- name: meta_format
-  type: int
-  default: 3
-  info: |
-    OSD store implementation version and on-disk metadata format.
-
-    Three versions are currently supported: 3, 2 and 1.
-    - 3 the new log-structured store, it's overall faster, has lower Write
-      Amplification, which may be even close to 1 (i.e. almost no extra writes)
-      if your SSDs support atomic writes (see [atomic_write_size](osd.en.md#atomic_write_size)).
-    - 2 is the old stable store from Vitastor 0.9-2.x.
-    - 1 is the same old store but with a legacy metadata format from Vitastor
-      versions to up 0.8.x, without any support for checksums.
-  info_ru: |
-    Версия реализации дискового хранилища OSD и дискового формата метаданных.
-
-    Поддерживаются три версии: 3, 2 и 1.
-    - 3 - новое лог-структурированное хранилище, в целом более быстрое, со
-      сниженным фактором амплификации записи, который может составлять около 1
-      (то есть, практически без лишней служебной записи), если ваши SSD
-      поддерживают атомарную запись (см. [atomic_write_size](osd.ru.md#atomic_write_size)).
-    - 2 - старое стабильное хранилище из версий Vitastor 0.9-2.x.
-    - 1 - то же самое стабильное хранилище, но с ещё более старым форматом
-      метаданных из версий Vitastor до 0.8.x, без какой-либо поддержки
-      контрольных сумм.
 - name: data_device
  type: string
  info: |
@@ -233,21 +208,11 @@
  type: string
  default: none
  info: |
-    Data and metadata checksum type to use. May be "crc32c", "xxh3_32" or "none".
-    Select crc32c or xxh3_32 and set csum_block_size to enable data checksums.
-
-    Both crc32c and xxh3_32 are almost equally fast, xxh3_32 is safer. xxh3_32 is
-    the xxhash3 algorithm truncated from 64 to 32 bits (which is still a good hash).
-
-    Note that enabled data checksums either increase memory usage or reduce
-    performance. Check details in [csum_block_size](#csum_block_size) description.
+    Data checksum type to use. May be "crc32c" or "none". Set to "crc32c" to
+    enable data checksums.
  info_ru: |
-    Тип используемых OSD контрольных сумм данных и метаданных. Может быть "crc32c",
-    "xxh3_32" или "none". Выберите crc32c или xxh3_32 и установите csum_block_size,
-    чтобы включить контрольные суммы данных.
-
-    И crc32c, и xxh3_32 примерно одинаково быстры, xxh3_32 надёжней. xxh3_32 - это
-    алгоритм xxhash3, обрезанный с 64 до 32 бит (это всё равно хороший хеш).
+    Тип используемых OSD контрольных сумм данных. Может быть "crc32c" или "none".
+    Установите в "crc32c", чтобы включить расчёт и проверку контрольных сумм данных.

    Следует понимать, что контрольные суммы в зависимости от размера блока их
    расчёта либо увеличивают потребление памяти, либо снижают производительность.
@@ -84,6 +84,11 @@
    unsupported with old libibverbs < v32, like in Debian 10 Buster or
    CentOS 7.

+    Vitastor supports all adapters, even ones without ODP support, like
+    Mellanox ConnectX-3 and non-Mellanox cards. Versions up to Vitastor
+    1.2.0 required ODP which is only present in Mellanox ConnectX >= 4.
+    See also [rdma_odp](#rdma_odp).
+
    Run `ibv_devinfo -v` as root to list available RDMA devices and their
    features.

@@ -92,23 +97,6 @@
    the manual of your network vendor for details about setting up the switch
    for RoCEv2 correctly. Usually it means setting up Lossless Ethernet with
    PFC (Priority Flow Control) and ECN (Explicit Congestion Notification).
-
-    Vitastor supports all adapters, even ones without ODP (On-Demand Paging)
-    support, like Mellanox ConnectX-3 and non-Mellanox cards. ODP is only present
-    in Mellanox ConnectX >= 4 adapters and allows to skip memory registration
-    for RDMA and thus, in theory, avoid memory copying.
-
-    Versions up to Vitastor 1.2.0 required ODP, then it was disabled by default,
-    but it was still supported up to 3.0.3. Now ODP support is removed because it
-    actually only hurts performance: an example 3-node cluster with 8 NVMe in each
-    node and 2*25 GBit/s ConnectX-6 RDMA network pushed 3950000 read iops without
-    ODP, but only 239000 iops with ODP.
-
-    This happens because Mellanox ODP implementation seems to be based on
-    message retransmissions when the adapter doesn't know about the buffer yet -
-    it likely uses standard "RNR retransmissions" (RNR = receiver not ready)
-    which is generally slow in RDMA/RoCE networks. Here's a presentation about
-    it from ISPASS-2021 conference: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
  info_ru: |
    Название RDMA-устройства для связи с Vitastor OSD (например, "rocep5s0f0").
    Если не указано, Vitastor попробует найти RoCE-устройство, соответствующее
@@ -117,6 +105,12 @@
    не задана. Также автовыбор не поддерживается со старыми версиями библиотеки
    libibverbs < v32, например в Debian 10 Buster или CentOS 7.

+    Vitastor поддерживает все модели адаптеров, включая те, у которых
+    нет поддержки ODP, то есть вы можете использовать RDMA с ConnectX-3 и
+    картами производства не Mellanox. Версии Vitastor до 1.2.0 включительно
+    требовали ODP, который есть только на Mellanox ConnectX 4 и более новых.
+    См. также [rdma_odp](#rdma_odp).
+
    Запустите `ibv_devinfo -v` от имени суперпользователя, чтобы посмотреть
    список доступных RDMA-устройств, их параметры и возможности.

@@ -126,24 +120,6 @@
    коммутатора для RoCEv2 ищите в документации производителя. Обычно это
    подразумевает настройку сети без потерь на основе PFC (Priority Flow
    Control) и ECN (Explicit Congestion Notification).
-
-    Vitastor поддерживает все модели адаптеров, включая те, у которых нет
-    поддержки ODP (On-Demand Paging), например, ConnectX-3 и карты производства
-    не Mellanox. Функция ODP доступна только на адаптерах Mellanox ConnectX-4 и
-    более новых и позволяет не регистрировать память для её использования RDMA-картой,
-    благодаря чему в теории можно избежать лишних копирований памяти.
-
-    Версии Vitastor до 1.2.0 включительно требовали ODP, потом функция был отключена
-    по умолчанию, но поддерживалась вплоть до версии 3.0.3. Сейчас поддержка ODP
-    полностью удалена, так как на самом деле она только портит производительность:
-    например, на 3-узловом кластере с 8 NVMe в каждом узле и сетью 2*25 Гбит/с на
-    чтение с RDMA без ODP удаётся снять 3950000 iops, а с ODP - всего 239000 iops.
-
-    Это происходит из-за того, что реализация ODP у Mellanox неоптимальная и
-    основана на повторной передаче сообщений, когда карте не известен буфер -
-    вероятно, на стандартных "RNR retransmission" (RNR = receiver not ready).
-    А данные повторные передачи в RDMA/RoCE - всегда очень медленная штука.
-    Презентация на эту тему с конференции ISPASS-2021: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
 - name: rdma_port_num
  type: int
  info: |
@@ -242,6 +218,45 @@
    у принимающей стороны в процессе работы не заканчивались буферы на приём.
    Не влияет на потребление памяти - дополнительная память на операции отправки
    не выделяется.
+- name: rdma_odp
+  type: bool
+  default: false
+  online: false
+  info: |
+    Use RDMA with On-Demand Paging. ODP is currently only available on Mellanox
+    ConnectX-4 and newer adapters. ODP allows to not register memory explicitly
+    for RDMA adapter to be able to use it. This, in turn, allows to skip memory
+    copying during sending. One would think this should improve performance, but
+    **in reality** RDMA performance with ODP is **drastically** worse. Example
+    3-node cluster with 8 NVMe in each node and 2*25 GBit/s ConnectX-6 RDMA network
+    without ODP pushes 3950000 read iops, but only 239000 iops with ODP...
+
+    This happens because Mellanox ODP implementation seems to be based on
+    message retransmissions when the adapter doesn't know about the buffer yet -
+    it likely uses standard "RNR retransmissions" (RNR = receiver not ready)
+    which is generally slow in RDMA/RoCE networks. Here's a presentation about
+    it from ISPASS-2021 conference: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
+
+    ODP support is retained in the code just in case a good ODP implementation
+    appears one day.
+  info_ru: |
+    Использовать RDMA с On-Demand Paging. ODP - функция, доступная пока что
+    исключительно на адаптерах Mellanox ConnectX-4 и более новых. ODP позволяет
+    не регистрировать память для её использования RDMA-картой. Благодаря этому
+    можно не копировать данные при отправке их в сеть и, казалось бы, это должно
+    улучшать производительность - но **по факту** получается так, что
+    производительность только ухудшается, причём сильно. Пример - на 3-узловом
+    кластере с 8 NVMe в каждом узле и сетью 2*25 Гбит/с на чтение с RDMA без ODP
+    удаётся снять 3950000 iops, а с ODP - всего 239000 iops...
+
+    Это происходит из-за того, что реализация ODP у Mellanox неоптимальная и
+    основана на повторной передаче сообщений, когда карте не известен буфер -
+    вероятно, на стандартных "RNR retransmission" (RNR = receiver not ready).
+    А данные повторные передачи в RDMA/RoCE - всегда очень медленная штука.
+    Презентация на эту тему с конференции ISPASS-2021: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
+
+    Возможность использования ODP сохранена в коде на случай, если вдруг в один
+    прекрасный день появится хорошая реализация ODP.
 - name: peer_connect_interval
  type: sec
  min: 1
@@ -253,33 +253,21 @@
  type: bool
  default: true
  info: |
-    Only for the old store ([meta_format](layout-osd.en.md#meta_format) 2).
-
-    This parameter makes Vitastor keep a copy of metadata area in memory as it is
-    on disk, in addition to the metadata database. When the option is enabled, every
-    metadata entry is effectively stored in RAM twice. It's required for good performance
-    because it allows to avoid additional read-modify-write cycles during metadata
-    modifications. Metadata area size with the old store is roughly 224 MB per 1 TB
-    of data. You can turn the option off to reduce memory usage by this value, but
-    it will reduce performance.
-
-    For the new store ([meta_format](layout-osd.en.md#meta_format) 3), the option
-    may be changed in the future to support operation without loading full metadata
-    database in memory.
+    This parameter makes Vitastor always keep metadata area of the block device
+    in memory. It's required for good performance because it allows to avoid
+    additional read-modify-write cycles during metadata modifications. Metadata
+    area size is currently roughly 224 MB per 1 TB of data. You can turn it off
+    to reduce memory usage by this value, but it will hurt performance. This
+    restriction is likely to be removed in the future along with the upgrade
+    of the metadata storage scheme.
  info_ru: |
-    Только для старого хранилища ([meta_format](layout-osd.en.md#meta_format) 2).
-
-    Данный параметр заставляет Vitastor всегда держать копию области метаданных
-    в памяти в том же виде, как она лежит на диске, в дополнение к БД метаданных.
-    То есть, с включённой опцией каждая запись метаданных хранится в памяти дважды.
-    Это нужно, чтобы избегать дополнительных операций чтения с диска при записи.
-    Размер области метаданных в старом хранилище составляет примерно 224 МБ на
-    1 ТБ данных. Вы можете отключить опцию, чтобы снизить потребление памяти
-    примерно на эту величину, но при этом также снизится и производительность.
-
-    Для нового хранилища ([meta_format](layout-osd.en.md#meta_format) 3) опция,
-    возможно, будет переработана в будущем для поддержки работы без полной
-    загрузки метаданных в памяти.
+    Данный параметр заставляет Vitastor всегда держать область метаданных диска
+    в памяти. Это нужно, чтобы избегать дополнительных операций чтения с диска
+    при записи. Размер области метаданных на данный момент составляет примерно
+    224 МБ на 1 ТБ данных. При включении потребление памяти снизится примерно
+    на эту величину, но при этом также снизится и производительность. В будущем,
+    после обновления схемы хранения метаданных, это ограничение, скорее всего,
+    будет ликвидировано.
 - name: inmemory_journal
  type: bool
  default: true
@@ -398,15 +386,11 @@
    blocks. The only situation when you should increase it to a larger value
    is when you enable journal_no_same_sector_overwrites. In this case set
    it to, for example, 1024.
-
-    Not applicable to the new store ([meta_format](layout-osd.en.md#meta_format) 3).
  info_ru: |
    Максимальное число буферов, разрешённых для использования под записываемые
    в журнал блоки метаданных. Единственная ситуация, в которой этот параметр
    нужно менять - это если вы включаете journal_no_same_sector_overwrites. В
    этом случае установите данный параметр, например, в 1024.
-
-    Неприменимо к новому хранилищу ([meta_format](layout-osd.en.md#meta_format) 3).
 - name: journal_no_same_sector_overwrites
  type: bool
  default: false
@@ -418,8 +402,6 @@
    journal after writing it instead of possibly overwriting it the second time.

    Most (99%) other SSDs don't need this option.
-
-    Not applicable to the new store ([meta_format](layout-osd.en.md#meta_format) 3).
  info_ru: |
    Включайте данную опцию для SSD вроде Intel D3-S4510 и D3-S4610, которые
    ОЧЕНЬ не любят, когда ПО перезаписывает один и тот же сектор несколько раз
@@ -430,20 +412,6 @@
    самого сектора.

    Почти все другие SSD (99% моделей) не требуют данной опции.
-
-    Неприменимо к новому хранилищу ([meta_format](layout-osd.en.md#meta_format) 3).
- name: skip_corrupted_meta_entries
-  type: bool
-  default: false
-  info: |
-    Only for the new store ([meta_format](layout-osd.en.md#meta_format) 3).
-    Allow OSD to start when some metadata entries or blocks are corrupted by
-    skipping them. Should be only used as an emergency measure.
-  info_ru: |
-    Только для нового хранилища ([meta_format](layout-osd.en.md#meta_format) 3).
-    Разрешить OSD запускаться, даже если часть блоков или записей метаданных
-    повреждена, пропуская их. Опция предназначена для использования только в
-    целях аварийного восстановления.
 - name: throttle_small_writes
  type: bool
  default: false
@@ -833,112 +801,3 @@
  default: 100
  info: Retry interval for failed PG lock attempts.
  info_ru: Интервал повтора неудачных попыток блокировки PG.
- name: atomic_write_size
-  type: int
-  default: 4096
-  info: |
-    Maximum data device atomic write size allowed for OSD to use.
-
-    Atomic writes allow to reduce the Write Amplification factor with the new store
-    ([meta_format](layout-osd.en.md#meta_format)=3) to almost 1 (i.e. almost no extra writes)
-    with replicated pools and reach the best possible write performance.
-
-    Default value is auto-detected during OSD initialization from
-    `/sys/block/xx/queue/atomic_write_max_bytes` or assumed to be 4096 bytes
-    because all known disks support 4 KB atomic writes. Auto-detection is only used for
-    NVMe disks because SAS disks require the explicit WRITE ATOMIC command which requires
-    RWF_ATOMIC (see below [#use_atomic_flag]) but that flag works incorrectly in current
-    Linux versions.
-
-    You can also check if your NVMe drives support atomic writes by running
-    the command `nvme id-ctrl /dev/nvme0n1 | grep awupf`. If the reported value,
-    plus 1, multiplied by the currently selected block size of the NVMe,
-    is more than 4 KB, then the new store can utilize it for better performance.
-    The only drives known to support it currently are [Micron and Kioxia](../intro/quickstart.en.md).
-
-    Atomic writes allow to skip double data writes in replicated pools, thus
-    reducing Write Amplification and improving write performance up to 2 times.
-  info_ru: |
-    Максимальный размер атомарной записи на диск данных, который OSD разрешено использовать.
-
-    Поддержка атомарной записи позволяет снизить мультипликатор записи (Write Amplification)
-    на диск с новым хранилищем ([meta_format](layout-osd.ru.md#meta_format)=3)
-    практически до 1 (то есть, почти до нулевого объёма лишней записи) в реплицированных
-    пулах и достигнуть наилучшей возможной производительности записи.
-
-    Значение по умолчанию авто-определяется во время инициализации OSD из
-    `/sys/block/xx/queue/atomic_write_max_bytes` либо принимается равным 4096,
-    так как все известные диски поддерживают атомарную запись 4 КБ блоков.
-    Автоопределение применяется только для NVMe-дисков, так как SAS диски требуют
-    использования отдельной команды WRITE ATOMIC, а для неё нужен флаг RWF_ATOMIC
-    (см. ниже [#use_atomic_flag]), а он в текущих версиях Linux работает некорректно.
-
-    Вы также можете проверить, поддерживают ли ваши NVMe-диски атомарную запись,
-    с помощью команды `nvme id-ctrl /dev/nvme0n1 | grep awupf`. Если значение awupf
-    плюс 1, умноженное на текущий выбранный размер блока NVMe-диска, больше 4 КБ,
-    то новое хранилище может использовать атомарные записи для достижения лучшей
-    производительности. Единственные известные диски, которые поддерживают это сейчас -
-    [Micron и Kioxia](../intro/quickstart.ru.md).
-
-    Атомарная запись позволяет не использовать двойную запись данных (в журнал и на
-    устройство данных) в реплицированных пулах и таким образом снижает амплификацию
-    записи (объём служебной записи на диск) и улучшает производительность записи
-    вплоть до 2-х кратного прироста.
- name: use_atomic_flag
-  type: bool
-  info: |
-    This option controls whether Vitastor OSDs use RWF_ATOMIC write flag with atomic writes.
-    This flag is supported since Linux 6.11 and adds some safety to atomic writes - the kernel
-    guarantees to not fragment write requests with it and also to check them against the actual
-    device atomic write capabilities.
-
-    However, the option is disabled by default because the flag is currently UNUSABLE - Linux
-    incorrectly requires writes with that flag to be of power-of-2 length and length-aligned.
-    I.e., for example, 12 KB writes and not-8-KB aligned 8 KB writes are forbidden by the kernel,
-    even though the NVMe specification allows them.
-
-    For NVMe disks with `scheduler=none` writes aren't fragmented anyway so it's not a big deal.
-    However, you can rebuild your kernel with [this patch](../../patches/linux-fix-atomic-write-checks.diff)
-    and turn this option on. It will make your atomic writes a bit safer.
-  info_ru: |
-    Данная опция контролирует использование Vitastor OSD флага RWF_ATOMIC при атомарной записи
-    блоков. Этот флаг поддерживается, начиная с версии ядра Linux 6.11 и добавляет немного корректности
-    атомарным записям - ядро гарантирует отсутствие фрагментации запросов записи с этим флагом и
-    проверяет их на соответствие реальным возможностям устройства.
-
-    Однако, данная опция по умолчанию отключена, так как флаг в текущих версиях Linux работает
-    абсолютно НЕКОРРЕКТНО - при нём Linux требует, чтобы запросы записи имели длину, равную
-    степени двойки и были выровнены на эту длину. То есть, например, 12 КБ запросы записи, а также
-    8 КБ запросы записи по не-кратному 8 КБ смещению запрещаются ядром, хотя спецификация NVMe их
-    разрешает.
-
-    Для NVMe-дисков с `scheduler=none` запросы записи и так не фрагментируются, так что это не так
-    уж и важно, однако вы можете пересобрать своё ядро с [этим патчем](../../patches/linux-fix-atomic-write-checks.diff)
-    и включить данную опцию. Это сделает вашу атомарную запись капельку безопаснее.
- name: pg_reshard_chunk_size
-  type: int
-  default: 100000
-  info: |
-    Pool PG count change is a CPU-intensive operation because OSDs store the full object database
-    in memory and have to move all entries between old and new PGs. Thus it's performed in chunks,
-    with pauses between chunks to prevent blocking OSD's event loop and other clients' operations.
-    This option sets the maximum number of object is a chunk. Moving 100k objects usually takes
-    50-100ms. Chunk size equal to 0 means unlimited.
-  info_ru: |
-    Изменение числа PG в пуле заметно загружает процессор, так как OSD хранят полную базу данных
-    объектов в памяти и им приходится перемещать все записи объектов между старыми и новыми PG.
-    Поэтому изменение применяется порциями, с паузами между порциями, чтобы не блокировать обработку
-    событий OSD и операции остальных клиентов. Данная опция задаёт максимальное число объектов
-    в порции. Перемещение 100 тысяч объектов (значение по умолчанию) обычно занимает порядка
-    50-100 миллисекунд. Значение опции 0 отключает лимит размера порции.
- name: pg_reshard_chunk_pause_ms
-  type: ms
-  default: 100
-  info: |
-    This option sets the interval between handling two PG count change chunks.
-  info_ru: |
-    Данная опция задаёт интервал между обработкой двух порций изменения числа PG пулов.
- name: gc_on_start
-  type: bool
-  info: Forcibly clean all garbage entries in the new store on every OSD restart.
-  info_ru: Принудительно очищать все мусорные записи в новом хранилище при каждом запуске OSD.
@@ -1,5 +0,0 @@
-{
-  "dependencies": {
-    "yaml": "^2.8.2"
-  }
-}
@@ -1,5 +0,0 @@
-# Security Parameters
-
-These parameters affect your Vitastor installation security and apply to OSDs, monitors and clients.
-
-Most of them can be set in /etc/vitastor/vitastor.conf and in etcd, but don't support online modification.
@@ -1,7 +0,0 @@
-# Параметры безопасности
-
-Данные параметры затрагивают безопасность инсталляций Vitastor и используются
-OSD, мониторами и клиентами.
-
-Большая их часть может задаваться в /etc/vitastor/vitastor.conf и в etcd, но не
-поддерживает онлайн-изменение.
@@ -1,131 +0,0 @@
- name: etcd_client_cert
-  type: string
-  info: |
-    Client TLS certificate to use for Vitastor client (not OSD and not monitor)
-    etcd https connections. May be path to a file or just a PEM string with certificate.
-    In the latter case, string must begin with "-----BEGIN CERTIFICATE-----".
-  info_ru: |
-    Клиентский TLS сертификат для https-подключений к etcd для клиентов Vitastor
-    (не OSD и не мониторов). Может быть путём к файлу или просто строкой с
-    сертификатом в формате PEM. В последнем случае строка должна начинаться с
-    "-----BEGIN CERTIFICATE-----".
- name: etcd_client_key
-  type: string
-  info: Private key for etcd_client_cert (also a file or a PEM string).
-  info_ru: Закрытый ключ для сертификата etcd_client_cert (также путь к файлу или PEM строка).
- name: etcd_ca
-  type: string
-  info: |
-    Trusted TLS CA to verify etcd server certificate. May be path to a file,
-    directory or just a PEM string with certificate.
-  info_ru: |
-    Доверенный корневой TLS-сертификат для проверки сертификата сервера etcd.
-    Может быть путём к файлу, директории или просто строкой с сертификатом в
-    формате PEM.
- name: osd_etcd_client_cert
-  type: string
-  info: |
-    Same as [etcd_client_cert](#etcd_client_cert), but only for OSDs.
-    OSDs, clients and monitors should have different permissions, so they should
-    use different certificates.
-  info_ru: |
-    Аналогично [etcd_client_cert](#etcd_client_cert), но только для OSD.
-    OSD, клиенты и мониторы должны иметь разные привилегии, поэтому они должны
-    использовать разные сертификаты.
- name: osd_etcd_client_key
-  type: string
-  info: Same as [etcd_client_key](#etcd_client_key), but only for OSDs.
-  info_ru: Аналогично [etcd_client_key](#etcd_client_key), но только для OSD.
- name: mon_etcd_client_cert
-  type: string
-  info: Same as [etcd_client_cert](#etcd_client_cert), but only for Vitastor monitors.
-  info_ru: Аналогично [etcd_client_cert](#etcd_client_cert), но только для мониторов Vitastor.
- name: mon_etcd_client_key
-  type: string
-  info: Same as [etcd_client_key](#etcd_client_key), but only for Vitastor monitors.
-  info_ru: Аналогично [etcd_client_key](#etcd_client_key), но только для мониторов Vitastor.
- name: vault_url
-  type: string
-  info: |
-    Vault base URL.
-
-    Vitastor clients support AES-256-XTS image data encryption with different per-image keys.
-    Encryption is performed by the client, OSDs don't have access to decrypted data.
-
-    Encryption keys may be stored in etcd or, for the increased security level, in an external
-    [HashiCorp Vault](https://developer.hashicorp.com/vault/) or [OpenBao](https://openbao.org/)
-    instance.
-
-    Vitastor clients use [v1 k/v secrets engine](https://openbao.org/api-docs/secret/kv/kv-v1/)
-    and [TLS authentication engine](https://openbao.org/api-docs/auth/cert/) in Vault.
-
-    In that case, only key IDs are stored in etcd.
-  info_ru: |
-    Базовый адрес Vault.
-
-    Клиенты Vitastor поддерживают AES-256-XTS шифрование данных образов с отдельными ключами на
-    каждый образ. Данные шифруются клиентами, OSD не имеют доступа к незашифрованным данным.
-
-    Ключи шифрования могут храниться в etcd или, для повышенного уровня безопасности, во внешнем
-    [HashiCorp Vault](https://developer.hashicorp.com/vault/) или [OpenBao](https://openbao.org/).
-
-    Клиенты Vitastor используют [движок секретов v1](https://openbao.org/api-docs/secret/kv/kv-v1/)
-    и [TLS-аутентификацию](https://openbao.org/api-docs/auth/cert/) в Vault.
-
-    В этом случае, только ID ключей хранятся в etcd.
- name: vault_secret_api_path
-  type: string
-  default: /v1/secret/
-  info: Vault v1 secret API mount path to use.
-  info_ru: Путь к API секретов v1 для использования клиентами.
- name: vault_client_cert
-  type: string
-  info: |
-    Client TLS certificate to use for Vault connections. Just like [etcd_client_cert](#etcd_client_cert),
-    may be path to a file or just a certificate in PEM string.
-  info_ru: |
-    Клиентский TLS сертификат для подключений к Vault. Как и [etcd_client_cert](#etcd_client_cert),
-    может быть путём к файлу или просто PEM-строкой с сертификатом.
- name: vault_client_key
-  type: string
-  info: Private key for vault_client_cert (also a file or a PEM string).
-  info_ru: Закрытый ключ для сертификата vault_client_cert (также путь к файлу или PEM строка).
- name: vault_ca
-  type: string
-  info: |
-    Trusted TLS CA to verify Vault server certificate. May be path to a file,
-    directory or just a PEM string with certificate.
-  info_ru: |
-    Доверенный корневой TLS-сертификат для проверки сертификата сервера Vault.
-    Может быть путём к файлу, директории или просто строкой с сертификатом в
-    формате PEM.
- name: vault_timeout_ms
-  type: int
-  default: 5000
-  info: Timeout for Vault requests in milliseconds.
-  info_ru: Максимально время выполнения Vault-запросов в миллисекундах.
- name: vault_error_timeout_sec
-  type: int
-  default: 60
-  info: |
-    Time (in seconds) to wait before retrying after receiving an error from Vault.
-  info_ru: |
-    Время (в секундах) для ожидания перед повторной попыткой при получении ошибки от Vault.
- name: vault_refresh_leeway_sec
-  type: int
-  default: 60
-  info: |
-    Extra time (in seconds) before real Vault token lease_timeout to refresh it, just
-    in case of system clock drift.
-  info_ru: |
-    Зазор времени (в секундах), чтобы обновлять токены Vault чуть раньше их реального
-    lease_timeout, на случай "ухода" системных часов.
- name: max_aes_xts_pool_size
-  type: int
-  default: 256
-  info: |
-    Maximum number of OpenSSL encryption contexts cached in OSD memory. Probably
-    doesn't require modification.
-  info_ru: |
-    Максимальное количество кэшируемых в памяти OSD контекстов шифрования OpenSSL.
-    Вряд ли требует изменения.
@@ -26,37 +26,13 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.

 1. Download a Docker image of the desired version: \
-   `docker pull vitalif/vitastor:v3.0.9`
+   `docker pull vitalif/vitastor:v2.4.3`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.9 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.4.3 install.sh`
 3. Reload udev rules: \
   `udevadm control --reload-rules`
-4. Enable the vitastor-host service: \
-   `systemctl enable --now vitastor-host`

-After these steps, you can return to [Quick Start](../intro/quickstart.en.md).
-
-## Podman
-
-If you use Podman, run the following commands as root before installing Vitastor containers:
-
-```
-ln -s podman /usr/bin/docker
-
-mkdir -p /etc/systemd/system/systemd-udevd.service.d
-
-cat >/etc/systemd/system/systemd-udevd.service.d/override.conf <<EOF
-[Service]
-CapabilityBoundingSet=~
-SystemCallFilter=@mount capset
-EOF
-
-systemctl daemon-reload
-
-systemctl restart systemd-udevd
-```
-
-Without it, udev fails to do calls into a Podman container and Vitastor disk detection doesn't work.
+And you can return to [Quick Start](../intro/quickstart.en.md).

 ## Upgrading Containers

@@ -25,39 +25,14 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.

 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitalif/vitastor:v3.0.9`
+   `docker pull vitalif/vitastor:v2.4.3`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.9 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.4.3 install.sh`
 3. Перезагрузите правила udev: \
   `udevadm control --reload-rules`
-4. Включите сервис vitastor-host: \
-   `systemctl enable --now vitastor-host`

 После этого вы можете возвращаться к разделу [Быстрый старт](../intro/quickstart.ru.md).

-## Podman
-
-Если вы используете Podman, перед установкой контейнеров Vitastor выполните следующие
-команды от имени суперпользователя:
-
-```
-ln -s podman /usr/bin/docker
-
-mkdir -p /etc/systemd/system/systemd-udevd.service.d
-
-cat >/etc/systemd/system/systemd-udevd.service.d/override.conf <<EOF
-[Service]
-CapabilityBoundingSet=~
-SystemCallFilter=@mount capset
-EOF
-
-systemctl daemon-reload
-
-systemctl restart systemd-udevd
-```
-
-Без этих настроек udev не может делать вызовы внутрь Podman-контейнеров и определение дисков Vitastor не работает.
-
 ## Обновление контейнеров

 Сначала обязательно проверьте раздел [Обновление Vitastor](../usage/admin.ru.md#обновление-vitastor),
@@ -17,7 +17,6 @@
  - Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
  - Ubuntu 22.04 (Jammy): `deb https://vitastor.io/debian jammy main`
  - Ubuntu 24.04 (Noble): `deb https://vitastor.io/debian noble main`
-  - Ubuntu 26.04 (Resolute): `deb https://vitastor.io/debian resolute main`
  - Add `-oldstable` to bookworm/bullseye/buster in this line to install the last
    stable version from 0.9.x branch instead of 1.x
 - To always prefer vitastor-patched QEMU and Libvirt versions, add the following to `/etc/apt/preferences`:
@@ -34,17 +33,15 @@
  - CentOS 7: `yum install https://vitastor.io/rpms/centos/7/vitastor-release.rpm`
  - CentOS 8: `dnf install https://vitastor.io/rpms/centos/8/vitastor-release.rpm`
  - AlmaLinux 9 and other RHEL 9 clones (Rocky, Oracle...): `dnf install https://vitastor.io/rpms/centos/9/vitastor-release.rpm`
-  - AlmaLinux 10 and other RHEL 10 clones: `dnf install https://vitastor.io/rpms/centos/10/vitastor-release.rpm`
 - Enable EPEL: `yum/dnf install epel-release`
 - Enable additional CentOS repositories:
  - CentOS 7: `yum install centos-release-scl`
  - CentOS 8: `dnf install centos-release-advanced-virtualization`
-  - RHEL 9/10 clones: not required
+  - RHEL 9 clones: not required
 - Enable elrepo-kernel:
  - CentOS 7: `yum install https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm`
  - CentOS 8: `dnf install https://www.elrepo.org/elrepo-release-8.el8.elrepo.noarch.rpm`
  - RHEL 9 clones: `dnf install https://www.elrepo.org/elrepo-release-9.el9.elrepo.noarch.rpm`
-  - RHEL 10 clones: not required
 - Install packages: `yum/dnf install vitastor lpsolve etcd kernel-ml qemu-kvm`

 ## Installation requirements
@@ -17,7 +17,6 @@
  - Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
  - Ubuntu 22.04 (Jammy): `deb https://vitastor.io/debian jammy main`
  - Ubuntu 24.04 (Noble): `deb https://vitastor.io/debian noble main`
-  - Ubuntu 26.04 (Resolute): `deb https://vitastor.io/debian resolute main`
  - Добавьте `-oldstable` к слову bookworm/bullseye/buster в этой строке, чтобы
    установить последнюю стабильную версию из ветки 0.9.x вместо 1.x
 - Чтобы всегда предпочитались версии пакетов QEMU и Libvirt с патчами Vitastor, добавьте в `/etc/apt/preferences`:
@@ -34,17 +33,15 @@
  - CentOS 7: `yum install https://vitastor.io/rpms/centos/7/vitastor-release.rpm`
  - CentOS 8: `dnf install https://vitastor.io/rpms/centos/8/vitastor-release.rpm`
  - AlmaLinux 9 и другие клоны RHEL 9 (Rocky, Oracle...): `dnf install https://vitastor.io/rpms/centos/9/vitastor-release.rpm`
-  - AlmaLinux 10 и другие клоны RHEL 10: `dnf install https://vitastor.io/rpms/centos/10/vitastor-release.rpm`
 - Включите EPEL: `yum/dnf install epel-release`
 - Включите дополнительные репозитории CentOS:
  - CentOS 7: `yum install centos-release-scl`
  - CentOS 8: `dnf install centos-release-advanced-virtualization`
-  - Клоны RHEL 9/10: не нужно
+  - Клоны RHEL 9: не нужно
 - Включите elrepo-kernel:
  - CentOS 7: `yum install https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm`
  - CentOS 8: `dnf install https://www.elrepo.org/elrepo-release-8.el8.elrepo.noarch.rpm`
  - Клоны RHEL 9: `dnf install https://www.elrepo.org/elrepo-release-9.el9.elrepo.noarch.rpm`
-  - Клоны RHEL 10: не нужно
 - Установите пакеты: `yum/dnf install vitastor lpsolve etcd kernel-ml qemu-kvm`

 ## Установочные требования
@@ -6,7 +6,7 @@

 # Proxmox VE

-To enable Vitastor support in Proxmox Virtual Environment (6.4-9.x are supported):
+To enable Vitastor support in Proxmox Virtual Environment (6.4-8.x are supported):

 - Add the corresponding Vitastor Debian repository into sources.list on Proxmox hosts:
  trixie for 9.0+, bookworm for 8.1+, pve8.0 for 8.0, bullseye for 7.4, pve7.3 for 7.3, pve7.2 for 7.2, pve7.1 for 7.1, buster for 6.4
@@ -6,7 +6,7 @@

 # Proxmox VE

-Чтобы подключить Vitastor к Proxmox Virtual Environment (поддерживаются версии 6.4-9.x):
+Чтобы подключить Vitastor к Proxmox Virtual Environment (поддерживаются версии 6.4-8.x):

 - Добавьте соответствующий Debian-репозиторий Vitastor в sources.list на хостах Proxmox:
  trixie для 9.0+, bookworm для 8.1+, pve8.0 для 8.0, bullseye для 7.4, pve7.3 для 7.3, pve7.2 для 7.2, pve7.1 для 7.1, buster для 6.4
@@ -15,8 +15,8 @@
 - gcc and g++ 8 or newer, clang 10 or newer, or other compiler with C++11 plus
  designated initializers support from C++20
 - CMake
- jerasure, c-ares headers and libraries
- ISA-L, libibverbs, librdmacm, libnl3 headers and libraries (optional)
+- jerasure headers and libraries
+- ISA-L, libibverbs and librdmacm headers and libraries (optional)
 - tcmalloc (google-perftools-dev)

 ## Basic instructions
@@ -15,8 +15,8 @@
 - gcc и g++ >= 8, либо clang >= 10, либо другой компилятор с поддержкой C++11 плюс
  назначенных инициализаторов (designated initializers) из C++20
 - CMake
- Заголовки и библиотеки jerasure, c-ares
- Опционально - заголовки и библиотеки ISA-L, libibverbs, librdmacm, libnl3
+- Заголовки и библиотеки jerasure
+- Опционально - заголовки и библиотеки ISA-L, libibverbs, librdmacm
 - tcmalloc (google-perftools-dev)

 ## Базовая инструкция
@@ -14,8 +14,6 @@

 - Basic part: highly-available block storage with symmetric clustering and no SPOF
 - [Performance](../performance/bench2.en.md) ;-D
-  - [NVMe atomic write support](../config/osd.en.md#atomic_write_size) for reducing the amount
-    of "extra" disk writes to almost zero (Write Amplification = 1)
 - [Multiple redundancy schemes](../config/pool.en.md#scheme): Replication, XOR n+1, Reed-Solomon erasure codes
  based on jerasure and ISA-L libraries with any number of data and parity drives in a group
 - Configuration via simple JSON data structures in etcd (parameters, pools and images)
@@ -41,8 +39,6 @@
 - [Built-in Prometheus metric exporter](../config/monitor.en.md#enable_prometheus)
 - [NFS RDMA support](../usage/nfs.en.md#rdma) (probably also usable for GPUDirect)
 - [S3](../installation/s3.en.md)
- [TLS support for etcd connections](../config/security.en.md)
- [AES-256-XTS image encryption](../usage/cli.en.md#create) and [Vault support](../config/security.en.md#vault_url) for key storage

 ## Plugins and tools

@@ -14,8 +14,6 @@

 - Базовая часть - надёжное кластерное блочное хранилище без единой точки отказа
 - [Производительность](../performance/bench2.ru.md) ;-D
-  - [Поддержка атомарной записи NVMe](../config/osd.ru.md#atomic_write_size) для снижения объёма
-    служебной записи практически до нуля (Write Amplification = 1)
 - [Несколько схем отказоустойчивости](../config/pool.ru.md#scheme): репликация, XOR n+1 (1 диск чётности), коды коррекции ошибок
  Рида-Соломона на основе библиотек jerasure и ISA-L с любым числом дисков данных и чётности в группе
 - Конфигурация через простые человекочитаемые JSON-структуры в etcd
@@ -43,8 +41,6 @@
 - [Встроенный Prometheus-экспортер метрик](../config/monitor.ru.md#enable_prometheus)
 - [Поддержка NFS RDMA](../usage/nfs.ru.md#rdma) (вероятно, также подходящая для GPUDirect)
 - [S3](../installation/s3.ru.md)
- [Поддержка TLS-соединений с etcd](../config/security.ru.md)
- [AES-256-XTS шифрование данных](../usage/cli.ru.md#create) и [поддержка Vault](../config/security.ru.md#vault_url) для хранения ключей

 ## Драйверы и инструменты

@@ -18,10 +18,9 @@

 ## Preparation

- Get some SATA or NVMe SSDs with capacitors (server-grade drives). The best performance
-  is achieved with Micron or Kioxia NVMes with atomic write support (see below). You can use desktop
-  SSDs with lazy fsync, but prepare for inferior single-thread latency. Read more about
-  capacitors [here](../config/layout-cluster.en.md#immediate_commit).
+- Get some SATA or NVMe SSDs with capacitors (server-grade drives). You can use desktop SSDs
+  with lazy fsync, but prepare for inferior single-thread latency. Read more about capacitors
+  [here](../config/layout-cluster.en.md#immediate_commit).
 - If you want to use HDDs, get modern HDDs with Media Cache or SSD Cache: HGST Ultrastar,
  Toshiba MG, Seagate EXOS or something similar. If your drives don't have such cache then
  you also need small SSDs for journal and metadata (even 2 GB per 1 TB of HDD space is enough).
@@ -31,11 +30,9 @@

 ## Recommended drives

- NVMe with atomic write support (ideal!): Micron 7450/7500/7550, Kioxia CD6/CD7/CD8/CD9
- Other NVMe: Micron 9100/9200/9300/9400/9550, Micron 7300, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
-  Intel DC-P3700/P4500/P4600, Intel/Solidigm D5-P4320/P5530, Intel/Solidigm D7-P5500/P5600, Solidigm D7-PS1010/PS1030/P5810,
-  Intel Optane, Kingston DC1000B/DC1500M, Kioxia CD6/CD7/CD8/CD9
- SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel/Solidigm D3-S4510/4520/4610/4620, Kingston DC500M
+- SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel D3-S4510/4520/4610/4620, Kingston DC500M
+- NVMe: Micron 9100/9200/9300/9400, Micron 7300/7450, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
+  Intel DC-P3700/P4500/P4600, Intel D5-P4320/P5530, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
 - HDD: HGST Ultrastar, Toshiba MG, Seagate EXOS

 ## Configure monitors
@@ -18,9 +18,8 @@

 ## Подготовка

- Возьмите серверы с SSD (SATA или NVMe), желательно с конденсаторами (серверные SSD). Наилучшая
-  производительность достигается на дисках Micron и Kioxia с поддержкой атомарной записи (см. ниже).
-  Можно использовать и десктопные SSD, включив режим отложенного fsync, но производительность будет хуже.
+- Возьмите серверы с SSD (SATA или NVMe), желательно с конденсаторами (серверные SSD). Можно
+  использовать и десктопные SSD, включив режим отложенного fsync, но производительность будет хуже.
  О конденсаторах читайте [здесь](../config/layout-cluster.ru.md#immediate_commit).
 - Если хотите использовать HDD, берите современные модели с Media или SSD кэшем - HGST Ultrastar,
  Toshiba MG, Seagate EXOS или что-то похожее. Если такого кэша у ваших дисков нет,
@@ -31,11 +30,9 @@

 ## Рекомендуемые диски

- NVMe с поддержкой атомарной записи (идеально!): Micron 7450/7500/7550, Kioxia CD6/CD7/CD8/CD9
- Другие NVMe: Micron 9100/9200/9300/9400/9550, Micron 7300, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
-  Intel DC-P3700/P4500/P4600, Intel/Solidigm D5-P4320/P5530, Intel/Solidigm D7-P5500/P5600, Solidigm D7-PS1010/PS1030/P5810,
-  Intel Optane, Kingston DC1000B/DC1500M, Kioxia CD6/CD7/CD8/CD9
- SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel/Solidigm D3-S4510/4520/4610/4620, Kingston DC500M
+- SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel D3-S4510/4520/4610/4620, Kingston DC500M
+- NVMe: Micron 9100/9200/9300/9400, Micron 7300/7450, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
+  Intel DC-P3700/P4500/P4600, Intel D5-P4320/P5530, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
 - HDD: HGST Ultrastar, Toshiba MG, Seagate EXOS

 ## Настройте мониторы
@@ -14,7 +14,7 @@ Replicated setups:
 - Linear read: `min(total network bandwidth, sum(disk read MB/s))`.
 - Linear write: `min(total network bandwidth, sum(disk write MB/s / number of replicas))`.
 - Saturated parallel read iops: `min(total network bandwidth, sum(disk read iops))`.
- Saturated parallel write iops: `min(total network bandwidth / number of replicas, sum(disk write iops / number of replicas / write amplification))`.
+- Saturated parallel write iops: `min(total network bandwidth / number of replicas, sum(disk write iops / number of replicas / (write amplification = 4)))`.

 EC/XOR setups (EC N+K):
 - Single-threaded (T1Q1) read latency: 1.5 network roundtrips + 1 disk read.
@@ -26,36 +26,28 @@ EC/XOR setups (EC N+K):
 - Linear read: `min(total network bandwidth, sum(disk read MB/s))`.
 - Linear write: `min(total network bandwidth, sum(disk write MB/s * N/(N+K)))`.
 - Saturated parallel read iops: `min(total network bandwidth, sum(disk read iops))`.
- Saturated parallel write iops: roughly `total iops / (N+K) / WA`. More exactly:
-  - With the new store: `min(total network bandwidth * N/(N+K), sum(disk randrw iops / (2 + N-1 + K*2)))`,
-    with random read/write mix corresponding to `(N-1)/(2 + N-1 + K*2)*100 % reads`.
-    - For example, with EC 2+1 it is: `(20% randrw iops) / 5`.
-    - With EC 6+3 it is: `(38% randrw iops) / 13`.
-  - With the old store: `min(total network bandwidth * N/(N+K), sum(disk randrw iops / (3 + N-1 + K*3)))`,
-    with random read/write mix corresponding to `(N-1)/(3 + N-1 + K*3)*100 % reads`.
-    - For example, with EC 2+1 it is: `(14% randrw iops) / 7`.
-    - With EC 6+3 it is: `(30% randrw iops) / 17`.
+- Saturated parallel write iops: roughly `total iops / (N+K) / WA`. More exactly,
+  `min(total network bandwidth * N/(N+K), sum(disk randrw iops / (N*4 + K*5 + 1)))` with
+  random read/write mix corresponding to `(N-1)/(N*4 + K*5 + 1)*100 % reads`.
+  - For example, with EC 2+1 it is: `(7% randrw iops) / 14`.
+  - With EC 6+3 it is: `(12.5% randrw iops) / 40`.

-Write Amplification factor:
- For the new store and for 4 KB writes: WA is always 1 unless you set [atomic_write_size](../config/osd.en.md#atomic_write_size) to 0 manually.
- For the new store and for 8-124 KB writes: WA is 1 if you use NVMe drives with atomic write support, or roughly 2 if you use other drives.
- For the old store, WA is roughly `(2 * write size + 4 KB) / (write size)`. So, for 4 KB writes it's 3, and for 8-124 KB writes it's closer to 2.
- For both the new and the old store and for writes of [block_size](../config/layout-cluster.en.md#block_size): WA is almost 1.
+Write amplification for 4 KB blocks is usually 3-5 in Vitastor:
+1. Journal block write
+2. Journal data write
+3. Metadata block write
+4. Another journal block write for EC/XOR setups
+5. Data block write

-Write Amplification consists of:
- For the new store:
-  - Buffer block write if non-atomic
-  - Data block write
-  - Metadata write(s) (amortized)
- For the old store:
-  - Journal block write (amortized)
-  - Journal data write
-  - Metadata block write
-  - Another journal block write for EC/XOR setups (amortized)
-  - Data block write
+If you manage to get an SSD which handles 512 byte blocks well (Optane?) you may
+lower 1, 3 and 4 to 512 bytes (1/8 of data size) and get WA as low as 2.375.

-Other possibilities to reduce WA would be to use SSDs with internal 512-byte blocks
-or NVDIMM, but both options seem unavailable on the market at the moment.
+Implemented NVDIMM support can basically eliminate WA at all - all extra writes will
+go to DRAM memory. But this requires a test cluster with NVDIMM - please contact me
+if you want to provide me with such cluster for tests.
+
+Lazy fsync also reduces WA for parallel workloads because journal blocks are only
+written when they fill up or fsync is requested.

 ## In Practice

@@ -27,36 +27,29 @@
 - Линейное чтение: сумма МБ/с чтения всех дисков, либо общая производительность сети, если в сеть упрётся раньше.
 - Линейная запись: сумма МБ/с записи всех дисков * N/(N+K), либо производительность сети * N / (N+K), если в сеть упрётся раньше.
 - Параллельное случайное мелкое чтение: сумма IOPS чтения всех дисков либо производительность сети, если в сеть упрётся раньше.
- Параллельная случайная мелкая запись: грубо `(сумма IOPS / (N+K) / WA)`.
-  Либо `производительность сети * N/(N+K)`, если в сеть упрётся раньше. Если точнее, то:
-  - С новым хранилищем: сумма смешанного IOPS всех дисков при `(N-1)/(2 + N-1 + K*2)*100 %` чтения, делённая на `(2 + N-1 + K*2)`.
-    - Например, при EC 2+1 это: `(сумма IOPS при 20% чтения) / 5`.
-    - При EC 6+3 это: `(сумма IOPS при 38% чтения) / 13`.
-  - Со старым хранилищем: сумма смешанного IOPS всех дисков при `(N-1)/(3 + N-1 + K*3)*100 %` чтения, делённая на `(3 + N-1 + K*3)`.
-    - Например, при EC 2+1 это: `(сумма IOPS при 14% чтения) / 7`.
-    - При EC 6+3 это: `(сумма IOPS при 30% чтения) / 17`.
+- Параллельная случайная мелкая запись: грубо `(сумма IOPS / (N+K) / WA)`. Если точнее, то:
+  сумма смешанного IOPS всех дисков при `(N-1)/(N*4 + K*5 + 1)*100 %` чтения, делённая на `(N*4 + K*5 + 1)`.
+  Либо, производительность сети * N/(N+K), если в сеть упрётся раньше.
+  - Например, при EC 2+1 это: `(сумма IOPS при 7% чтения) / 14`.
+  - При EC 6+3 это: `(сумма IOPS при 12.5% чтения) / 40`.

-WA (Write Amplification, мультипликатор записи):
- С новым хранилищем для 4 КБ записи: WA всегда примерно 1, если только вы не установите [atomic_write_size](../config/osd.ru.md#atomic_write_size) вручную в 0.
- С новым хранилищем и большими записями (8-124 КБ): WA примерно 1, если вы используете NVMe-диски с поддержкой атомарной записи,
-  или примерно 2, если вы используете другие диски.
- Со старым хранилищем, WA примерно `(2 * размер записи + 4 КБ) / (размер записи)`. То есть, для 4 КБ записи WA=3, а для 8-124 КБ WA ближе к 2.
- И с новым, и со старым хранилищем и для записи размером [block_size](../config/layout-cluster.ru.md#block_size): WA примерно равен 1.
+WA (мультипликатор записи) для 4 КБ блоков в Vitastor обычно составляет 3-5:
+1. Запись метаданных в журнал
+2. Запись блока данных в журнал
+3. Запись метаданных в БД
+4. Ещё одна запись метаданных в журнал при использовании EC
+5. Запись блока данных на диск данных

-Мультипликатор записи состоит из:
- С новым хранилищем:
-  - Запись блока буфера, если диски без поддержки атомарной записи
-  - Запись блока данных
-  - Запись(-и) блоков метаданных (амортизированные)
- Со старым хранилищем:
-  - Запись блока журнала (амортизированная)
-  - Запись данных в журнал
-  - Запись блока метаданных
-  - Ещё одна запись блока журнала для EC/XOR пулов (амортизированная)
-  - Запись блока данных
+Если вы найдёте SSD, хорошо работающий с 512-байтными блоками данных (Optane?),
+то 1, 3 и 4 можно снизить до 512 байт (1/8 от размера данных) и получить WA всего 2.375.

-Другими потенциальными возможностями снижения WA могли бы быть SSD с внутренним 512-байтным блоком
-либо NVDIMM, но и то, и другое сейчас выглядит недоступным на рынке.
+Если реализовать поддержку NVDIMM, то WA можно, условно говоря, ликвидировать вообще - все
+дополнительные операции записи смогут обслуживаться DRAM памятью. Но для этого необходим
+тестовый кластер с NVDIMM - пишите, если готовы предоставить такой для тестов.
+
+Кроме того, WA снижается при использовании отложенного/ленивого сброса при параллельной
+нагрузке, т.к. блоки журнала записываются на диск только когда они заполняются или явным
+образом запрашивается fsync.

 ## На практике

@@ -231,18 +231,6 @@ Upgrading from <= 0.5.x to >= 0.6.x is not supported.

 Downgrade are also allowed freely, except the following specific instructions:

-### 3.x -> 2.x
-
-Versions 3.0.0 and newer contain two store implementations - an old one and a new
-one, unsupported in 2.x and previous versions. So you should check your OSD store
-versions before downgrading to 2.x with the following command:
-
-`vitastor-disk read-sb /dev/vitastor/osdXX-data | jq -r .meta_format`
-
-If it prints 3 then OSD uses the new store and you can't downgrade it to 2.x.
-
-If it prints 2 or nothing then OSD uses the old store and the downgrade is allowed.
-
 ### 1.8.0 to 1.7.1

 Before downgrading from version >= 1.8.0 to version <= 1.7.1
@@ -228,18 +228,6 @@ done

 Откат (понижение версии) тоже свободно разрешён, кроме указанных ниже случаев:

-### 3.x -> 2.x
-
-Версии 3.0.0 и более новые содержат две реализации хранилища - старую и новую, не
-поддерживаемую в 2.x и предыдущих версиях. Таким образом, перед откатом на 2.x вам
-следует проверить, какая версия хранилища используется вашими OSD - командой:
-
-`vitastor-disk read-sb /dev/vitastor/osdXX-data | jq -r .meta_format`
-
-Если выводится 3, это новое хранилище и откатить такой OSD до 2.x нельзя.
-
-Если выводится 2 или не выводится ничего, это старое хранилище и откат разрешён.
-
 ### 1.8.0 -> 1.7.1

 Перед понижением версии с >= 1.8.0 до <= 1.7.1 вы должны скопировать ключ
@@ -125,31 +125,18 @@ bench-kaveri         kaveri    10 G  10 G    0 B/s  0     0      0 us  0 B/s  0

 ## create

-`vitastor-cli create -s|--size SIZE [OPTIONS] <name>`
+`vitastor-cli create -s|--size <size> [-p|--pool <id|name>] [--parent <parent_name>[@<snapshot>]] <name>`

-Create an image. Options:
-
-* `-s|--size SIZE` - New image size in bytes or with a K/M/G/T unit suffix.
-* `-p|--pool POOL` - Specify pool for the new image (may be omitted if there is only 1 pool).
-* `--parent PARENT` - Create a copy-on-write image clone based on PARENT (or PARENT@SNAPSHOT).
-  If parent is not a snapshot, it must be a read-only image.
-* `--enc-key random` - Generate a new random AES-256-XTS encryption key for the new image.
-* `--enc-key HEX` - Set a specified AES-256-XTS key (64 bytes in hex) for the new image.
-* `--enc-key vault:ID` - Use an encryption key from an external Vault secret with specified ID.
+Create an image. You may use K/M/G/T suffixes for `<size>`. If `--parent` is specified,
+a copy-on-write image clone is created. Parent must be a snapshot (readonly image).
+Pool must be specified if there is more than one pool.

 ```
-vitastor-cli create --snapshot <snapshot> [OPTIONS] <image>
-vitastor-cli snap-create [OPTIONS] <image>@<snapshot>
+vitastor-cli create --snapshot <snapshot> [-p|--pool <id|name>] <image>
+vitastor-cli snap-create [-p|--pool <id|name>] <image>@<snapshot>
 ```

-Create a snapshot of image `<image>`. May be used live if only a single writer is active.
-
-Options:
-
-* `-p|--pool POOL` - Move image to pool POOL, leaving the snapshot in the old pool.
-* `--enc-key random` - Change image encryption key to a new random AES-256-XTS key.
-* `--enc-key KEY` - Change image encryption key to a specified key, Vault key or to an empty key.
-  By default, the image retains its old encryption key when taking a snapshot.
+Create a snapshot of image `<name>` (either form can be used). May be used live if only a single writer is active.

 See also about [how to export snapshots](qemu.en.md#exporting-snapshots).

@@ -164,7 +151,6 @@ You should resize file system in the image, if present, before shrinking it.
 * `--deleted 1|0` - Set/clear 'deleted image' flag (set automatically during unfinished deletes).
 * `-f|--force` - Proceed with shrinking or setting readwrite flag even if the image has children.
 * `--down-ok` - Proceed with shrinking even if some data will be left on unavailable OSDs.
-* `--enc-key HEX` - Change image encryption key (allowed only with `--force`).

 ## dd

@@ -127,32 +127,19 @@ bench-kaveri         kaveri    10 G  10 G    0 B/s  0     0      0 us  0 B/s  0

 ## create

-`vitastor-cli create -s|--size SIZE [ОПЦИИ] <name>`
+`vitastor-cli create -s|--size <size> [-p|--pool <id|name>] [--parent <parent_name>[@<snapshot>]] <name>`

-Создать образ. Опции:
-
-* `-s|--size SIZE` - Размер нового образа в байтах или с суффиксом K/M/G/T (кило/мега/гига/терабайт).
-* `-p|--pool POOL` - Создать образ в заданном пуле (можно не указывать, если пул всего один).
-* `--parent PARENT` - Создать легковесный клон на основе образа `PARENT` или снимка `PARENT@SNAP`.
-  Если `PARENT` - не снимок, он должен быть помечен как образ только для чтения.
-* `--enc-key random` - Сгенерировать случайный ключ шифрования AES-256-XTS для нового образа.
-* `--enc-key HEX` - Установить заданный ключ AES-256-XTS (64 байта в hex) для нового образа.
-* `--enc-key vault:ID` - Использовать ключ из внешнего секрета с заданным ID из Vault.
+Создать образ. Для размера `<size>` можно использовать суффиксы K/M/G/T (килобайт-мегабайт-гигабайт-терабайт).
+Если указана опция `--parent`, создаётся клон образа. Родитель `<parent_name>[@<snapshot>]` должен быть
+снимком (или просто немодифицируемым образом). Пул обязательно указывать, если в кластере больше одного пула.

 ```
-vitastor-cli create --snapshot <snapshot> [ОПЦИИ] <image>
-vitastor-cli snap-create [ОПЦИИ] <image>@<snapshot>
+vitastor-cli create --snapshot <snapshot> [-p|--pool <id|name>] <image>
+vitastor-cli snap-create [-p|--pool <id|name>] <image>@<snapshot>
 ```

-Создать снимок образа `<image>` (можно использовать любую форму команды).
-Снимок можно создавать без остановки клиентов, если пишущих клиентов не больше одного.
-
-Опции:
-
-* `-p|--pool POOL` - Переместить образ в пул POOL, оставив снимок в старом пуле.
-* `--enc-key random` - Изменить ключ шифрования образа на новый случайный ключ AES-256-XTS.
-* `--enc-key KEY` - Изменить ключ шифрования образа на заданный ключ, ключ из Vault или пустой ключ.
-  По умолчанию шифрованные образы сохраняют старый ключ при снятии снимка.
+Создать снимок образа `<name>` (можно использовать любую форму команды). Снимок можно создавать без остановки
+клиентов, если пишущий клиент максимум 1.

 Смотрите также информацию о том, [как экспортировать снимки](qemu.ru.md#экспорт-снимков).

@@ -169,7 +156,6 @@ vitastor-cli snap-create [ОПЦИИ] <image>@<snapshot>
 * `--deleted 1|0` - Установить/снять флаг "образ удалён" (устанавливается при незавершённом удалении).
 * `-f|--force` - Разрешить уменьшение или перевод в чтение-запись образа, у которого есть клоны.
 * `--down-ok` - Разрешить уменьшение, даже если часть данных останется неудалённой на недоступных OSD.
-* `--enc-key HEX` - Изменить ключ шифрования образа (разрешено только с `--force`).

 ## dd

@@ -51,9 +51,6 @@ Options (automatic mode):
 ```
 --osd_per_disk <N>
  Create <N> OSDs on each disk (default 1)
--meta_format 3
-  Metadata store version. 3 is the new log-structured store, 2 is the stable store
-  from Vitastor 0.9-2.x, 1 is the legacy store from Vitastor 0.6-0.8.
 --hybrid
  Prepare hybrid (HDD+SSD, NVMe+SATA or etc) OSDs using provided devices. By default,
  any passed SSDs will be used for journals and metadata, HDDs will be used for data,
@@ -95,8 +92,6 @@ Options (single-device mode):
 Options (both modes):

 ```
--tags tag1,tag2           Set new OSD tag(s)
--weight <number>          Set new OSD weight (between 0 to 1)
 --journal_size 1G/32M      Set journal size (area or partition size)
 --block_size 1M/128k       Set blockstore object size
 --bitmap_granularity 4k    Set bitmap granularity
@@ -50,9 +50,6 @@ vitastor-disk - инструмент командной строки для уп
 ```
 --osd_per_disk <N>
  Создавать по несколько (<N>) OSD на каждом диске (по умолчанию 1)
--meta_format 3
-  Версия хранилища метаданных. 3 - новое лог-структурированное хранилище,
-  2 - стабильное хранилище из Vitastor 0.9-2.x, 1 - старое хранилище из Vitastor 0.6-0.8.
 --hybrid
  Инициализировать гибридные (HDD+SSD, NVMe+SATA и т.п.) OSD на указанных дисках.
  По умолчанию, SSD будут использованы для журналов и метаданных, а HDD - для данных,
@@ -96,8 +93,6 @@ vitastor-disk - инструмент командной строки для уп
 Опции для обоих режимов:

 ```
--tags tag1,tag2           Задать теги для новых OSD
--weight <number>          Задать вес для новых OSD (от 0 до 1)
 --journal_size 1G/32M      Задать размер журнала (области или раздела журнала)
 --block_size 1M/128k       Задать размер объекта хранилища
 --bitmap_granularity 4k    Задать гранулярность битовых карт
@@ -18,7 +18,7 @@ class AntiEtcdAdapter
                cluster = cluster ? (''+(cluster||'')).split(/,+/) : [];
            cluster = Object.keys(cluster.reduce((a, url) =>
            {
-                a[url.toLowerCase().replace(/^(https?:\/\/)?(.*?)(\/.*)?$/, (m, m1, m2) => (m1||'http://')+m2)] = true;
+                a[url.toLowerCase().replace(/^(https?:\/\/)/, '').replace(/\/.*$/, '')] = true;
                return a;
            }, {}));
            const cfg_port = config.antietcd_port;
@@ -26,8 +26,7 @@ class AntiEtcdAdapter
            is_local['0.0.0.0'] = true;
            is_local['::'] = true;
            is_local[''] = true;
-            // split :, 3 -> <schema>:<//ip>:<port>
-            const selected = cluster.map(s => s.split(':', 3)).filter(ip => is_local[ip[1].substr(2)] && (!cfg_port || ip[2] == cfg_port));
+            const selected = cluster.map(s => s.split(':', 2)).filter(ip => is_local[ip[0]] && (!cfg_port || ip[1] == cfg_port));
            if (selected.length > 1)
            {
                console.error('More than 1 etcd_address matches local IPs, please specify port');
@@ -36,30 +35,16 @@ class AntiEtcdAdapter
            else if (selected.length == 1)
            {
                const antietcd_config = {
-                    ip: selected[0][1].substr(2),
-                    port: selected[0][2],
-                    cert: config.antietcd_cert,
-                    key: config.antietcd_key,
-                    ca: config.etcd_ca,
-                    data: config.antietcd_data_file || ((config.antietcd_data_dir || '/var/lib/vitastor') + '/mon_'+selected[0][2]+'.json.gz'),
+                    ip: selected[0][0],
+                    port: selected[0][1],
+                    data: config.antietcd_data_file || ((config.antietcd_data_dir || '/var/lib/vitastor') + '/mon_'+selected[0][1]+'.json.gz'),
                    persist_filter: vitastor_persist_filter({ vitastor_prefix: config.etcd_prefix || '/vitastor' }),
-                    node_id: selected[0][1].substr(2)+':'+selected[0][2], // node_id = ip:port
-                    cluster: (cluster.length == 1 ? null : cluster.reduce((a, c) => { a[c.replace(/^(https?:\/\/)/, '')] = c; return a; }, {})),
+                    node_id: selected[0][0]+':'+selected[0][1], // node_id = ip:port
+                    cluster: (cluster.length == 1 ? null : cluster.reduce((a, c) => { a[c] = "http://"+c; return a; }, {})),
                    cluster_key: (config.etcd_prefix || '/vitastor'),
                    stale_read: 1,
                    log_level: 1,
                };
-                if (config.use_auth)
-                {
-                    antietcd_config.client_cert_auth = true;
-                    antietcd_config.auth_filter = require('./vitastor_auth_filter.js');
-                    antietcd_config.peer_ca = config.antietcd_server_ca;
-                    if (!config.antietcd_server_ca || config.antietcd_server_ca == config.etcd_ca)
-                    {
-                        console.error('Secure setup requires separate antietcd_server_ca (for signing antietcd server certificates) and etcd_ca (for signing client certificates)');
-                        process.exit(1);
-                    }
-                }
                for (const key in config)
                {
                    if (key.substr(0, 9) === 'antietcd_')
@@ -184,7 +169,7 @@ class AntiEtcdAdapter
                    await new Promise(ok => setTimeout(ok, timeout-(Date.now()-prev)));
                }
                prev = Date.now();
-                const res = await this.antietcd.api(path.replace(/^\/+/, '').replace(/\/+$/, '').replace(/\/+/g, '_'), body, { username: 'root' });
+                const res = await this.antietcd.api(path.replace(/^\/+/, '').replace(/\/+$/, '').replace(/\/+/g, '_'), body);
                if (res.error)
                {
                    console.error('Failed to query antietcd '+path+' (retry '+retry+'/'+retries+'): '+res.error);
@@ -1,9 +1,7 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)

-const fs = require('fs');
 const http = require('http');
-const https = require('https');
 const WebSocket = require('ws');
 const { b64, local_ips } = require('./utils.js');

@@ -17,30 +15,11 @@ class EtcdAdapter
        this.ws = null;
        this.ws_alive = false;
        this.ws_keepalive_timer = null;
-        this.opts = {};
    }

    parse_config(config)
    {
        this.parse_etcd_addresses(config.etcd_address||config.etcd_url);
-        if (config.mon_etcd_client_cert || config.etcd_client_cert)
-        {
-            this.opts.cert = config.mon_etcd_client_cert || config.etcd_client_cert;
-            if (this.opts.cert.substr(0, 5) != '-----')
-                this.opts.cert = fs.readFileSync(this.opts.cert, { encoding: 'utf-8' });
-        }
-        if (config.mon_etcd_client_key || config.etcd_client_key)
-        {
-            this.opts.key = config.mon_etcd_client_key || config.etcd_client_key;
-            if (this.opts.key.substr(0, 5) != '-----')
-                this.opts.key = fs.readFileSync(this.opts.key, { encoding: 'utf-8' });
-        }
-        if (config.etcd_ca)
-        {
-            this.opts.ca = config.etcd_ca;
-            if (this.opts.ca.substr(0, 5) != '-----')
-                this.opts.ca = fs.readFileSync(this.opts.ca, { encoding: 'utf-8' });
-        }
    }

    parse_etcd_addresses(addrs)
@@ -60,7 +39,7 @@ class EtcdAdapter
        for (let url of addrs)
        {
            let scheme = 'http';
-            url = url.trim().replace(/^(https?):\/\//i, (m, m1) => { scheme = m1.toLowerCase(); return ''; });
+            url = url.trim().replace(/^(https?):\/\//, (m, m1) => { scheme = m1; return ''; });
            const slash = url.indexOf('/');
            const colon = url.indexOf(':');
            const is_local = is_local_ip[colon >= 0 ? url.substr(0, colon) : (slash >= 0 ? url.substr(0, slash) : url)];
@@ -151,7 +130,7 @@ class EtcdAdapter
                    }
                    ok(false);
                }, this.mon.config.etcd_mon_timeout);
-                this.ws = new WebSocket(base+'/watch', this.opts);
+                this.ws = new WebSocket(base+'/watch');
                this.ws_used_url = cur_addr;
                const fail = () =>
                {
@@ -293,7 +272,7 @@ class EtcdAdapter
            {
                throw new Error(MON_STOPPED);
            }
-            const res = await POST(base+path, body, timeout, this.opts);
+            const res = await POST(base+path, body, timeout);
            if (this.mon.stopped)
            {
                throw new Error(MON_STOPPED);
@@ -319,7 +298,7 @@ class EtcdAdapter
    }
 }

-function POST(url, body, timeout, opts)
+function POST(url, body, timeout)
 {
    return new Promise(ok =>
    {
@@ -331,10 +310,10 @@ function POST(url, body, timeout, opts)
            req = null;
            ok({ error: 'timeout' });
        }, timeout) : null;
-        let req = (url.substr(0, 5) == 'https' ? https : http).request(url, { method: 'POST', headers: {
+        let req = http.request(url, { method: 'POST', headers: {
            'Content-Type': 'application/json',
            'Content-Length': body_text.length,
-        }, ...(opts||{}) }, (res) =>
+        } }, (res) =>
        {
            if (!req)
            {
@@ -16,7 +16,6 @@ const etcd_allow = new RegExp('^'+[
    'config/pools',
    'config/osd/[1-9]\\d*',
    'config/pgs', // old name
-    'config/user/.*',
    'pg/config',
    'config/inode/[1-9]\\d*/[1-9]\\d*',
    'osd/state/[1-9]\\d*',
@@ -46,14 +45,7 @@ const etcd_tree = {
            config_path: "/etc/vitastor/vitastor.conf",
            etcd_prefix: "/vitastor",
            // etcd connection - configurable online
-            etcd_address: "http://10.0.115.10:2379/v3",
-            etcd_client_cert: "",
-            etcd_client_key: "",
-            osd_etcd_client_cert: "",
-            osd_etcd_client_key: "",
-            mon_etcd_client_cert: "",
-            mon_etcd_client_key: "",
-            etcd_ca: "",
+            etcd_address: "10.0.115.10:2379/v3",
            // mon
            etcd_mon_ttl: 5, // min: 1
            etcd_mon_timeout: 1000, // ms. min: 0
@@ -209,8 +201,6 @@ const etcd_tree = {
                primary_affinity_tags?: 'nvme' | [ 'nvme', ... ],
                // scrub interval
                scrub_interval?: '30d',
-                // users allowed to create images in this pool
-                creator_group?: '',
            },
            ...
        }, */
@@ -227,21 +217,10 @@ const etcd_tree = {
                    parent_id?: <inode_t>,
                    readonly?: boolean,
                    deleted?: boolean,
-                    enc_key?: string,
-                    owner?: string,
-                    owner_group?: string,
-                    reader_group?: string,
                }
            }
        }, */
        inode: {},
-        /* user: {
-            <username>: {
-                type: 'osd'|'mon'|'admin'|'client',
-                groups: string[],
-            },
-        }, */
-        user: {},
    },
    osd: {
        state: {
@@ -16,7 +16,7 @@ async function create_http_server(cfg, handler)
        };
        if (cfg.mon_https_ca)
        {
-            tls.ca = await fsp.readFile(cfg.mon_https_ca);
+            tls.mon_https_ca = await fsp.readFile(cfg.mon_https_ca);
        }
        if (cfg.mon_https_client_auth)
        {
@@ -10,19 +10,16 @@ const NO_OSD = 'Z';
 async function lp_solve(text)
 {
    const cp = child_process.spawn('lp_solve');
-    let stdout = '', stderr = '', finish_cb, finished = 0;
+    let stdout = '', stderr = '', finish_cb;
    cp.stdout.on('data', buf => stdout += buf.toString());
    cp.stderr.on('data', buf => stderr += buf.toString());
-    cp.stdout.on('end', () => finish_cb());
-    cp.stderr.on('end', () => finish_cb());
+    cp.on('exit', () => finish_cb && finish_cb());
    cp.stdin.write(text);
    cp.stdin.end();
-    await new Promise(ok => (finish_cb = () =>
+    if (cp.exitCode == null)
    {
-        finished++;
-        if (finished == 2)
-            ok();
-    }));
+        await new Promise(ok => finish_cb = ok);
+    }
    if (!stdout.trim())
    {
        return null;
@@ -87,7 +87,7 @@ function make_hier_tree(global_config, tree)
    tree[''] = { children: [] };
    for (const node_id in tree)
    {
-        if (node_id === '')
+        if (node_id === '' || !(tree[node_id].children||[]).length && (tree[node_id].size||0) <= 0)
        {
            continue;
        }
@@ -1,6 +1,6 @@
 {
  "name": "vitastor-mon",
-  "version": "3.0.9",
+  "version": "2.4.3",
  "description": "Vitastor SDS monitor service",
  "main": "mon-main.js",
  "scripts": {
@@ -9,7 +9,7 @@
  "author": "Vitaliy Filippov",
  "license": "UNLICENSED",
  "dependencies": {
-    "antietcd": "^1.2.4",
+    "antietcd": "^1.1.3",
    "sprintf-js": "^1.1.2",
    "ws": "^7.2.5"
  },
@@ -9,6 +9,7 @@ const LPOptimizer = require('./lp_optimizer/lp_optimizer.js');
 const { scale_pg_count } = require('./pg_utils.js');
 const { make_hier_tree, filter_osds_by_root_node,
    filter_osds_by_tags, filter_osds_by_block_layout, get_affinity_osds } = require('./osd_tree.js');
+const { select_murmur3 } = require('./lp_optimizer/murmur3.js');

 function pick_primary(pool_id, pg_num, pool_config, osd_set, up_osds, aff_osds)
 {
@@ -38,7 +39,7 @@ function pick_primary(pool_id, pg_num, pool_config, osd_set, up_osds, aff_osds)
    {
        return 0;
    }
-    return alive_set[pg_num % alive_set.length];
+    return alive_set[select_murmur3(alive_set.length, osd_num => pool_id+'/'+pg_num+'/'+osd_num)];
 }

 function recheck_primary(state, global_config, up_osds, osd_tree)
@@ -52,7 +53,6 @@ function recheck_primary(state, global_config, up_osds, osd_tree)
            continue;
        }
        const aff_osds = get_affinity_osds(pool_cfg, up_osds, osd_tree);
-        let paused = false;
        for (let pg_num = 1; pg_num <= pool_cfg.pg_count; pg_num++)
        {
            if (!state.pg.config.items[pool_id])
@@ -75,19 +75,6 @@ function recheck_primary(state, global_config, up_osds, osd_tree)
                    );
                    new_pg_config.items[pool_id][pg_num].primary = new_primary;
                }
-                paused = paused || !!pg_cfg.pause;
-            }
-        }
-        if (paused)
-        {
-            if (!new_pg_config)
-            {
-                new_pg_config = JSON.parse(JSON.stringify(state.pg.config));
-            }
-            console.log(`Resuming paused pool ${pool_id}`);
-            for (const pg in new_pg_config.items[pool_id])
-            {
-                delete new_pg_config.items[pool_id][pg].pause;
            }
        }
    }
@@ -192,10 +179,10 @@ async function generate_pool_pgs(state, global_config, pool_id, osd_tree, levels
    const rules = use_rules ? get_pg_rules(pool_id, pool_cfg, global_config.placement_levels) : null;
    const folded = fold_failure_domains(Object.values(pool_tree), use_rules ? rules : [ [ [ pool_cfg.failure_domain ] ] ]);
    // FIXME: Remove/merge make_hier_tree() step somewhere, however it's needed to remove empty nodes
-    const folded_tree = make_hier_tree(global_config, folded.nodes.reduce((a, c) => { a[c.id] = c; return a; }, {}));
+    const folded_tree = make_hier_tree(global_config, folded.nodes);
    const old_pg_count = prev_pgs.length;
    const optimize_cfg = {
-        osd_weights: folded.nodes.reduce((a, c) => { if (/^\d+$/.exec(c.id) && c.size != null) { a[c.id] = c.size||0; } return a; }, {}),
+        osd_weights: folded.nodes.reduce((a, c) => { if (Number(c.id)) { a[c.id] = c.size; } return a; }, {}),
        combinator: use_rules
            // new algorithm:
            ? new RuleCombinator(folded_tree, rules, pool_cfg.max_osd_combinations)
@@ -37,7 +37,6 @@ function derive_osd_stats(st, prev, prev_diff)
        const n = c.count - BigInt(pr && pr.count||0);
        diff.recovery_stats[op] = { ...c, bps: n > 0 ? b*1000n/timediff : 0n, iops: n > 0 ? n*1000n/timediff : 0n };
    }
-    diff.inode_stats = {};
    for (const pool_id in st.inode_stats||{})
    {
        diff.inode_stats[pool_id] = {};
@@ -1,471 +0,0 @@
-// AntiEtcd authentication filter for Vitastor
-// (c) Vitaliy Filippov, 2026
-// License: Mozilla Public License 2.0 or Vitastor Network Public License 1.1
-
-// Permissions are based on:
-// 1. Users.
-//    Stored in /vitastor/config/user/<username>.
-//    Has 2 properties:
-//    - type, one of: osd, mon, admin, client.
-//      osd, mon types should be used by OSDs/monitors.
-//      admin should be used for administrative access from vitastor-cli.
-//      client should be used for regular clients.
-//    - groups, a list of group names the user is included in.
-// 2. Images.
-//    Stored in /vitastor/config/inode/<pool>/<inode>. Has the following properties:
-//    - owner (user name)
-//    - owner_group (group name)
-//    - reader_group
-
-const static_perms = {
-    invalid: {
-        keys: {},
-        prefixes: {},
-    },
-    osd: {
-        keys: { '/pg/config': false },
-        prefixes: { '/osd/': true, '/pg/state/': true, '/pg/history/': true, '/pgstats/': true },
-    },
-    mon: {
-        keys: { '/pg/config': true, '/stats': true, '/history/last_clean_pgs': true },
-        prefixes: {
-            '/config/': false, '/osd/': false, '/mon/': true, '/pg/history/': true,
-            '/pgstats/': false, '/inode/stats/': true, '/pool/stats/': true,
-        },
-    },
-    admin: {
-        keys: { '/stats': false },
-        prefixes: {
-            '/config/': true, '/osd/': true, '/index/': true, '/pg/history/': true,
-            '/mon/': false, '/pg/': false, '/pgstats/': false, '/inode/stats/': false, '/pool/stats/': false,
-        },
-    },
-    client: {
-        keys: { '/config/global': false, '/config/node_placement': false, '/config/pools': false, '/pg/config': false },
-        prefixes: { '/osd/stats/': false, '/pg/state/': false, '/index/maxid/': false },
-    },
-};
-
-const api_perms = {
-    osd: { lease_grant: true, lease_revoke: true, lease_keepalive: true },
-    mon: { lease_grant: true, lease_revoke: true, lease_keepalive: true },
-    admin: { maintenance_status: true },
-    client: {},
-};
-
-class VitastorAuthFilter
-{
-    constructor(antietcd)
-    {
-        this.cfg = antietcd.cfg;
-        this.antietcd = antietcd;
-        this.prefix = this.cfg.vitastor_prefix || '/vitastor';
-        this.prefix_parts = this.prefix.split('/');
-    }
-
-    _get(path, decode)
-    {
-        let cur = this.antietcd.etctree.state;
-        path = path instanceof Array ? path : path.split('/');
-        for (const p of path)
-        {
-            if (!cur.children)
-            {
-                return null;
-            }
-            cur = cur.children[p];
-            if (!cur)
-            {
-                return null;
-            }
-        }
-        if (decode)
-        {
-            return this._decode(path, cur.value);
-        }
-        return cur;
-    }
-
-    _decode(path, cur)
-    {
-        if (!cur)
-        {
-            return null;
-        }
-        if (cur)
-        {
-            try
-            {
-                cur = JSON.parse(cur);
-            }
-            catch (e)
-            {
-                console.warn('Invalid JSON in '+(path instanceof Array ? path.join('/') : path)+': '+e);
-            }
-        }
-        return cur;
-    }
-
-    // userInfo: { name: string, type: string, perms: static_perms[type], groups: { [string]: true } }
-    _check_compare(check, userInfo, checked)
-    {
-        let key = String(check.key);
-        if (key.substr(0, this.prefix.length) !== this.prefix)
-        {
-            return false;
-        }
-        key = key.substr(this.prefix.length);
-        if (key in userInfo.perms.keys)
-        {
-            return true;
-        }
-        for (const pfx in userInfo.perms.prefixes)
-        {
-            if (key.substr(0, pfx.length) == pfx)
-            {
-                return true;
-            }
-        }
-        if (userInfo.type == 'client')
-        {
-            // Image permissions
-            if (key.substr(0, 14) == '/config/inode/')
-            {
-                // Allowed to check that a key does not exist
-                if (check.target == 'VERSION' && check.version == 0)
-                {
-                    checked['M'+key] = true;
-                    return true;
-                }
-                else if (check.target == 'MOD')
-                {
-                    const data = this._get(check.key);
-                    if (!data || data.mod_revision != check.mod_revision)
-                    {
-                        // Break check to trigger CAS failure
-                        check.mod_revision = '18446744073709551615'; // UINT64_MAX
-                        return true;
-                    }
-                    const inode = this._decode(check.key, data.value);
-                    if (inode && (inode.owner_group && userInfo.groups[inode.owner_group] ||
-                        inode.owner === userInfo.name))
-                    {
-                        checked['M'+key] = true;
-                        return true;
-                    }
-                }
-                return false;
-            }
-            if (key.substr(0, 13) == '/index/image/')
-            {
-                // Allowed to check that a key does not exist
-                if (check.target == 'VERSION' && check.version == 0)
-                {
-                    checked['M'+key] = true;
-                    return true;
-                }
-                else if (check.target == 'MOD')
-                {
-                    let data = this._get(check.key);
-                    if (!data || data.mod_revision != check.mod_revision)
-                    {
-                        // Break check to trigger CAS failure
-                        check.mod_revision = '18446744073709551615'; // UINT64_MAX
-                        return true;
-                    }
-                    data = this._decode(check.key, data.value);
-                    if (data)
-                    {
-                        const inode = this._get([ ...this.prefix_parts, 'config', 'inode', data.pool_id, data.id ], true);
-                        if (inode && (inode.owner_group && userInfo.groups[inode.owner_group] ||
-                            inode.owner === userInfo.name))
-                        {
-                            checked['M'+key] = true;
-                            return true;
-                        }
-                    }
-                }
-                return false;
-            }
-            if (key.substr(0, 13) == '/index/maxid/')
-            {
-                const pool_id = key.substr(13);
-                const pool_cfg = this._get([ ...this.prefix_parts, 'config', 'pools' ], true);
-                if (!pool_cfg || !pool_cfg[pool_id] || !pool_cfg[pool_id].creator_group || !userInfo.groups[pool_cfg[pool_id].creator_group])
-                {
-                    return false;
-                }
-                if (check.target == 'VERSION' && check.version == 0)
-                {
-                    checked['I'+parseInt(key.substr(13))+'_0'] = true;
-                    return true;
-                }
-                else if (check.target == 'MOD')
-                {
-                    const data = this._get(check.key);
-                    if (!data || data.mod_revision != check.mod_revision)
-                    {
-                        // Break check to trigger CAS failure
-                        check.mod_revision = '18446744073709551615'; // UINT64_MAX
-                        return true;
-                    }
-                    checked['I'+parseInt(key.substr(13))+'_'+data.value] = true;
-                    return true;
-                }
-                return false;
-            }
-        }
-        return false;
-    }
-
-    _check_read(kv, userInfo)
-    {
-        let key = String(kv.key);
-        if (key.substr(0, this.prefix.length) !== this.prefix)
-        {
-            return false;
-        }
-        key = key.substr(this.prefix.length);
-        if (key in userInfo.perms.keys)
-        {
-            return true;
-        }
-        for (const pfx in userInfo.perms.prefixes)
-        {
-            if (key.substr(0, pfx.length) == pfx)
-            {
-                return true;
-            }
-        }
-        if (userInfo.type == 'client')
-        {
-            // Image permissions
-            if (key.substr(0, 14) == '/config/inode/')
-            {
-                const inode = this._decode(kv.key, kv.value);
-                if (inode && (inode.reader_group && userInfo.groups[inode.reader_group] ||
-                    inode.owner_group && userInfo.groups[inode.owner_group] ||
-                    inode.owner === userInfo.name))
-                {
-                    return true;
-                }
-                return false;
-            }
-            if (key.substr(0, 13) == '/index/image/')
-            {
-                const data = this._decode(kv.key, kv.value);
-                const inode = this._get([ ...this.prefix_parts, 'config', 'inode', data.pool_id, data.id ], true);
-                if (inode && (inode.reader_group && userInfo.groups[inode.reader_group] ||
-                    inode.owner_group && userInfo.groups[inode.owner_group] ||
-                    inode.owner === userInfo.name))
-                {
-                    return true;
-                }
-                return false;
-            }
-        }
-        return false;
-    }
-
-    _check_write(put, userInfo, checked)
-    {
-        let key = String(put.key);
-        if (key.substr(0, this.prefix.length) !== this.prefix)
-        {
-            return false;
-        }
-        key = key.substr(this.prefix.length);
-        if (userInfo.perms.keys[key])
-        {
-            return true;
-        }
-        for (const pfx in userInfo.perms.prefixes)
-        {
-            if (userInfo.perms.prefixes[pfx] && key.substr(0, pfx.length) == pfx)
-            {
-                return true;
-            }
-        }
-        if (checked && userInfo.type == 'client')
-        {
-            if (key.substr(0, 13) == '/index/maxid/' &&
-                checked['I'+parseInt(key.substr(13))+'_'+(put.value-1)])
-            {
-                // Allowed to increment maxid
-                return true;
-            }
-            if (checked['M'+key])
-            {
-                // Allowed to modify known images with CAS checks
-                return true;
-            }
-        }
-        return false;
-    }
-
-    _check_req(req, userInfo, checked)
-    {
-        let r;
-        if ((r = (req.request_range || req.requestRange)))
-        {
-            // All range queries are allowed, but responses are filtered - it's simpler
-        }
-        else if ((r = (req.request_put || req.requestPut)))
-        {
-            if (!this._check_write(r, userInfo, checked))
-                return false;
-        }
-        else if ((r = (req.request_delete_range || req.requestDeleteRange)))
-        {
-            if (!r.range_end || r.range_end === r.key)
-            {
-                if (!this._check_write({ key: r.key }, userInfo))
-                    return false;
-            }
-            else
-            {
-                // All keys in range must satisfy prefix
-                r.range_end = String(r.range_end);
-                if (r.key.length != r.range_end.length ||
-                    r.key[r.key.length-1] != '/' ||
-                    r.range_end[r.range_end.length-1] != '0')
-                {
-                    return false;
-                }
-                let key = r.key.substr(this.prefix.length);
-                let found = false;
-                for (const pfx in userInfo.perms.prefixes)
-                {
-                    if (userInfo.perms.prefixes[pfx] && key.substr(0, pfx.length) == pfx)
-                    {
-                        found = true;
-                        break;
-                    }
-                }
-                if (!found)
-                    return false;
-            }
-        }
-        return true;
-    }
-
-    _get_user(username)
-    {
-        if (!username)
-        {
-            return null;
-        }
-        let userInfo = this._get([ ...this.prefix_parts, 'config', 'user', username ], true);
-        if (!userInfo)
-        {
-            userInfo = { type: 'client' };
-        }
-        userInfo.perms = static_perms[userInfo.type] || static_perms['invalid'];
-        userInfo.name = username;
-        if (userInfo.groups instanceof Array)
-        {
-            userInfo.groups = userInfo.groups.reduce((a, c) => { a[c] = true; return a; }, {});
-        }
-        else
-        {
-            userInfo.groups = {};
-        }
-        return userInfo;
-    }
-
-    filter_api(username, api/*, data*/)
-    {
-        if (username === 'root')
-        {
-            return true;
-        }
-        const userInfo = this._get([ ...this.prefix_parts, 'config', 'user', username ], true);
-        return userInfo && api_perms[userInfo.type] && api_perms[userInfo.type][api];
-    }
-
-    filter_txn(username, txn)
-    {
-        if (username === 'root')
-        {
-            return true;
-        }
-        const userInfo = this._get_user(username);
-        if (!userInfo)
-        {
-            return null;
-        }
-        const checked = {};
-        if (txn.compare)
-        {
-            for (const check of txn.compare)
-            {
-                if (!this._check_compare(check, userInfo, checked))
-                    return null;
-            }
-        }
-        // Special transactions:
-        // 1. create image: create config/inode and index/image, increment index/maxid/<pool> (with CAS)
-        // 2. create snapshot: same as create image but also rename previous to @snap
-        if (txn.success)
-        {
-            for (const req of txn.success)
-            {
-                if (!this._check_req(req, userInfo, checked))
-                    return null;
-            }
-        }
-        if (txn.failure)
-        {
-            for (const req of txn.failure)
-            {
-                if (!this._check_req(req, userInfo, null))
-                    return null;
-            }
-        }
-        return txn;
-    }
-
-    filter_txn_response(username, txn, res)
-    {
-        if (!res.responses || username === 'root')
-        {
-            return;
-        }
-        const userInfo = this._get_user(username);
-        if (!userInfo)
-        {
-            for (const resp of res.responses)
-            {
-                if (resp.response_range && resp.response_range.kvs)
-                {
-                    resp.response_range.kvs = [];
-                }
-            }
-            return;
-        }
-        for (const resp of res.responses)
-        {
-            if (resp.response_range && resp.response_range.kvs)
-            {
-                resp.response_range.kvs = resp.response_range.kvs.filter(kv => this._check_read(kv, userInfo));
-            }
-        }
-    }
-
-    filter_watch_message(username, msg)
-    {
-        if (!msg.result || !msg.result.events || username === 'root')
-        {
-            return;
-        }
-        const userInfo = this._get_user(username);
-        if (!userInfo)
-        {
-            msg.result.events = [];
-            return;
-        }
-        msg.result.events = msg.result.events.filter(ev => this._check_read(ev.kv, userInfo));
-    }
-}
-
-module.exports = VitastorAuthFilter;
@@ -1,6 +1,6 @@
 {
  "name": "vitastor",
-  "version": "3.0.9",
+  "version": "2.4.3",
  "description": "Low-level native bindings to Vitastor client library",
  "main": "index.js",
  "keywords": [
@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils

-VITASTOR_VERSION = '3.0.9'
+VITASTOR_VERSION = '2.4.3'

 LOG = logging.getLogger(__name__)

@@ -275,7 +275,7 @@ class VitastorDriver(driver.CloneableImageVD,
            LOG.exception('error getting vitastor pool stats: '+str(e))

        self._stats = stats
-
+        
    def get_volume_stats(self, refresh=False):
        """Get volume stats.
        If 'refresh' is True, run update the stats first.
@@ -291,14 +291,6 @@ class VitastorDriver(driver.CloneableImageVD,
        else:
            return (1 + resp['kvs'][0]['value'], resp['kvs'][0]['mod_revision'])

-    def _cli(self, descr, *args):
-        args = [ 'vitastor-cli', *args, *(self._vitastor_args()) ]
-        try:
-            self._execute(*args)
-        except processutils.ProcessExecutionError as exc:
-            LOG.error("Failed to "+descr+": "+exc)
-            raise exception.VolumeBackendAPIException(data = exc.stderr)
-
    def create_volume(self, volume):
        """Creates a logical volume."""

@@ -310,7 +302,7 @@ class VitastorDriver(driver.CloneableImageVD,

        LOG.debug("creating volume '%s'", vol_name)

-        self._cli('create volume', 'create', vol_name, '--size', size)
+        self._create_image(vol_name, { 'size': size })

        if volume.encryption_key_id:
            self._create_encrypted_volume(volume, volume.obj_context)
@@ -354,7 +346,7 @@ class VitastorDriver(driver.CloneableImageVD,
        snap_name = utils.convert_str(snapshot.name)
        if snap_name.find('@') >= 0 or snap_name.find('/') >= 0:
            raise exception.VolumeBackendAPIException(data = '@ and / are forbidden in volume and snapshot names')
-        self._cli('create snapshot', 'snap-create', vol_name+'@'+snap_name)
+        self._create_snapshot(vol_name, vol_name+'@'+snap_name)

    def snapshot_revert_use_temp_snapshot(self):
        """Disable the use of a temporary snapshot on revert."""
@@ -367,8 +359,21 @@ class VitastorDriver(driver.CloneableImageVD,
        snap_name = utils.convert_str(snapshot.name)

        # Delete the image and recreate it from the snapshot
-        self._cli('delete image', 'rm', vol_name)
-        self._cli('recreate image', 'create', '--parent', vol_name+'@'+snap_name, vol_name)
+        args = [ 'vitastor-cli', 'rm', vol_name, *(self._vitastor_args()) ]
+        try:
+            self._execute(*args)
+        except processutils.ProcessExecutionError as exc:
+            LOG.error("Failed to delete image "+vol_name+": "+exc)
+            raise exception.VolumeBackendAPIException(data = exc.stderr)
+        args = [
+            'vitastor-cli', 'create', '--parent', vol_name+'@'+snap_name,
+            vol_name, *(self._vitastor_args())
+        ]
+        try:
+            self._execute(*args)
+        except processutils.ProcessExecutionError as exc:
+            LOG.error("Failed to recreate image "+vol_name+" from "+vol_name+"@"+snap_name+": "+exc)
+            raise exception.VolumeBackendAPIException(data = exc.stderr)

    def delete_snapshot(self, snapshot):
        """Deletes a snapshot."""
@@ -376,7 +381,15 @@ class VitastorDriver(driver.CloneableImageVD,
        vol_name = utils.convert_str(snapshot.volume_name)
        snap_name = utils.convert_str(snapshot.name)

-        self._cli('remove snapshot', 'rm', vol_name+'@'+snap_name)
+        args = [
+            'vitastor-cli', 'rm', vol_name+'@'+snap_name,
+            *(self._vitastor_args())
+        ]
+        try:
+            self._execute(*args)
+        except processutils.ProcessExecutionError as exc:
+            LOG.error("Failed to remove snapshot "+vol_name+'@'+snap_name+": "+exc)
+            raise exception.VolumeBackendAPIException(data = exc.stderr)

    def _child_count(self, parents):
        children = 0
@@ -414,7 +427,13 @@ class VitastorDriver(driver.CloneableImageVD,
        if src_vref.admin_metadata.get('readonly') == 'True':
            # source volume is a volume-image cache entry or other readonly volume
            # clone without intermediate snapshot
-            self._cli('create clone', 'create', '--parent', src_name, '--size', size, dest_name)
+            src = self._get_image(src_name)
+            LOG.debug("creating image '%s' from '%s'", dest_name, src_name)
+            new_cfg = self._create_image(dest_name, {
+                'size': size,
+                'parent_id': src['idx']['id'],
+                'parent_pool_id': src['idx']['pool_id'],
+            })
            return {}

        clone_snap = "%s@%s.clone_snap" % (src_name, dest_name)
@@ -427,12 +446,15 @@ class VitastorDriver(driver.CloneableImageVD,
            clone_snap = dest_name
            make_img = False

-        LOG.debug("creating snapshot '%s'", clone_snap)
-        self._cli('create base snapshot', 'snap-create', '--allow-existing', '1', clone_snap)
-
+        LOG.debug("creating layer '%s' under '%s'", clone_snap, src_name)
+        new_cfg = self._create_snapshot(src_name, clone_snap, True)
        if make_img:
            # Then create a clone from it
-            self._cli('create clone', 'create', '--parent', clone_snap, '--size', size, dest_name)
+            new_cfg = self._create_image(dest_name, {
+                'size': size,
+                'parent_id': new_cfg['parent_id'],
+                'parent_pool_id': new_cfg['parent_pool_id'],
+            })

        return {}

@@ -442,8 +464,7 @@ class VitastorDriver(driver.CloneableImageVD,
        vol_name = utils.convert_str(volume.name)
        snap_name = utils.convert_str(snapshot.name)

-        src_snap = 'volume-'+snapshot.volume_id+'@'+snap_name
-        snap = self._get_image(src_snap)
+        snap = self._get_image('volume-'+snapshot.volume_id+'@'+snap_name)
        if not snap:
            raise exception.SnapshotNotFound(snapshot_id = snap_name)
        snap_inode_id = int(resp['responses'][0]['kvs'][0]['value']['id'])
@@ -452,8 +473,12 @@ class VitastorDriver(driver.CloneableImageVD,
        size = snap['cfg']['size']
        if int(volume.size):
            size = int(volume.size) * units.Gi
+        new_cfg = self._create_image(vol_name, {
+            'size': size,
+            'parent_id': snap['idx']['id'],
+            'parent_pool_id': snap['idx']['pool_id'],
+        })

-        self._cli('create clone', 'create', vol_name, '--size', size, '--parent', src_snap)
        return {}

    def _vitastor_args(self):
@@ -480,7 +505,49 @@ class VitastorDriver(driver.CloneableImageVD,
        """Deletes a logical volume."""

        vol_name = utils.convert_str(volume.name)
-        self._cli('delete volume', 'rm', '--matching', vol_name, vol_name+'@*', '--progress', '0')
+
+        # Find the volume and all its snapshots
+        range_end = b'index/image/' + vol_name.encode('utf-8')
+        range_end = range_end[0 : len(range_end)-1] + six.int2byte(range_end[len(range_end)-1] + 1)
+        resp = self._etcd_txn({ 'success': [
+            { 'request_range': { 'key': 'index/image/'+vol_name, 'range_end': range_end } },
+        ] })
+        if len(resp['responses'][0]['kvs']) == 0:
+            # already deleted
+            LOG.info("volume %s no longer exists in backend", vol_name)
+            return
+        layers = resp['responses'][0]['kvs']
+        layer_ids = {}
+        for kv in layers:
+            inode_id = int(kv['value']['id'])
+            pool_id = int(kv['value']['pool_id'])
+            inode_pool_id = (pool_id << 48) | (inode_id & 0xffffffffffff)
+            layer_ids[inode_pool_id] = True
+
+        # Check if the volume has clones and raise 'busy' if so
+        children = self._child_count(layer_ids)
+        if children > 0:
+            raise exception.VolumeIsBusy(volume_name = vol_name)
+
+        # Clear data
+        for kv in layers:
+            args = [
+                'vitastor-cli', 'rm-data', '--pool', str(kv['value']['pool_id']),
+                '--inode', str(kv['value']['id']), '--progress', '0',
+                *(self._vitastor_args())
+            ]
+            try:
+                self._execute(*args)
+            except processutils.ProcessExecutionError as exc:
+                LOG.error("Failed to remove layer "+kv['key']+": "+exc)
+                raise exception.VolumeBackendAPIException(data = exc.stderr)
+
+        # Delete all layers from etcd
+        requests = []
+        for kv in layers:
+            requests.append({ 'request_delete_range': { 'key': kv['key'] } })
+            requests.append({ 'request_delete_range': { 'key': 'config/inode/'+str(kv['value']['pool_id'])+'/'+str(kv['value']['id']) } })
+        self._etcd_txn({ 'success': requests })

    def retype(self, context, volume, new_type, diff, host):
        """Change extra type specifications for a volume."""
@@ -500,6 +567,98 @@ class VitastorDriver(driver.CloneableImageVD,
        """Removes an export for a logical volume."""
        pass

+    def _create_image(self, vol_name, cfg):
+        pool_s = str(self.cfg['pool_id'])
+        image_id = 0
+        while image_id == 0:
+            # check if the image already exists and find a free ID
+            resp = self._etcd_txn({ 'success': [
+                { 'request_range': { 'key': 'index/image/'+vol_name } },
+                { 'request_range': { 'key': 'index/maxid/'+pool_s } },
+            ] })
+            if len(resp['responses'][0]['kvs']) > 0:
+                # already exists
+                raise exception.VolumeBackendAPIException(data = 'Volume '+vol_name+' already exists')
+            image_id, id_mod = self._next_id(resp['responses'][1])
+            # try to create the image
+            resp = self._etcd_txn({ 'compare': [
+                { 'target': 'MOD', 'mod_revision': id_mod, 'key': 'index/maxid/'+pool_s },
+                { 'target': 'VERSION', 'version': 0, 'key': 'index/image/'+vol_name },
+                { 'target': 'VERSION', 'version': 0, 'key': 'config/inode/'+pool_s+'/'+str(image_id) },
+            ], 'success': [
+                { 'request_put': { 'key': 'index/maxid/'+pool_s, 'value': image_id } },
+                { 'request_put': { 'key': 'index/image/'+vol_name, 'value': json.dumps({
+                    'id': image_id, 'pool_id': self.cfg['pool_id']
+                }) } },
+                { 'request_put': { 'key': 'config/inode/'+pool_s+'/'+str(image_id), 'value': json.dumps({
+                    **cfg, 'name': vol_name,
+                }) } },
+            ] })
+            if not resp.get('succeeded'):
+                # repeat
+                image_id = 0
+
+    def _create_snapshot(self, vol_name, snap_vol_name, allow_existing = False):
+        while True:
+            # check if the image already exists and snapshot doesn't
+            resp = self._etcd_txn({ 'success': [
+                { 'request_range': { 'key': 'index/image/'+vol_name } },
+                { 'request_range': { 'key': 'index/image/'+snap_vol_name } },
+            ] })
+            if len(resp['responses'][0]['kvs']) == 0:
+                raise exception.VolumeBackendAPIException(data = 'Volume '+vol_name+' does not exist')
+            if len(resp['responses'][1]['kvs']) > 0:
+                if allow_existing:
+                    snap_idx = resp['responses'][1]['kvs'][0]['value']
+                    resp = self._etcd_txn({ 'success': [
+                        { 'request_range': { 'key': 'config/inode/'+str(snap_idx['pool_id'])+'/'+str(snap_idx['id']) } },
+                    ] })
+                    if len(resp['responses'][0]['kvs']) == 0:
+                        raise exception.VolumeBackendAPIException(data =
+                            'Volume '+snap_vol_name+' is already indexed, but does not exist'
+                        )
+                    return resp['responses'][0]['kvs'][0]['value']
+                raise exception.VolumeBackendAPIException(
+                    data = 'Volume '+snap_vol_name+' already exists'
+                )
+            vol_idx = resp['responses'][0]['kvs'][0]['value']
+            vol_idx_mod = resp['responses'][0]['kvs'][0]['mod_revision']
+            # get image inode config and find a new ID
+            resp = self._etcd_txn({ 'success': [
+                { 'request_range': { 'key': 'config/inode/'+str(vol_idx['pool_id'])+'/'+str(vol_idx['id']) } },
+                { 'request_range': { 'key': 'index/maxid/'+str(self.cfg['pool_id']) } },
+            ] })
+            if len(resp['responses'][0]['kvs']) == 0:
+                raise exception.VolumeBackendAPIException(data = 'Volume '+vol_name+' does not exist')
+            vol_cfg = resp['responses'][0]['kvs'][0]['value']
+            vol_mod = resp['responses'][0]['kvs'][0]['mod_revision']
+            new_id, id_mod = self._next_id(resp['responses'][1])
+            # try to redirect image to the new inode
+            new_cfg = {
+                **vol_cfg, 'name': vol_name, 'parent_id': vol_idx['id'], 'parent_pool_id': vol_idx['pool_id']
+            }
+            resp = self._etcd_txn({ 'compare': [
+                { 'target': 'MOD', 'mod_revision': vol_idx_mod, 'key': 'index/image/'+vol_name },
+                { 'target': 'MOD', 'mod_revision': vol_mod, 'key': 'config/inode/'+str(vol_idx['pool_id'])+'/'+str(vol_idx['id']) },
+                { 'target': 'MOD', 'mod_revision': id_mod, 'key': 'index/maxid/'+str(self.cfg['pool_id']) },
+                { 'target': 'VERSION', 'version': 0, 'key': 'index/image/'+snap_vol_name },
+                { 'target': 'VERSION', 'version': 0, 'key': 'config/inode/'+str(self.cfg['pool_id'])+'/'+str(new_id) },
+            ], 'success': [
+                { 'request_put': { 'key': 'index/maxid/'+str(self.cfg['pool_id']), 'value': new_id } },
+                { 'request_put': { 'key': 'index/image/'+vol_name, 'value': json.dumps({
+                    'id': new_id, 'pool_id': self.cfg['pool_id']
+                }) } },
+                { 'request_put': { 'key': 'config/inode/'+str(self.cfg['pool_id'])+'/'+str(new_id), 'value': json.dumps(new_cfg) } },
+                { 'request_put': { 'key': 'index/image/'+snap_vol_name, 'value': json.dumps({
+                    'id': vol_idx['id'], 'pool_id': vol_idx['pool_id']
+                }) } },
+                { 'request_put': { 'key': 'config/inode/'+str(vol_idx['pool_id'])+'/'+str(vol_idx['id']), 'value': json.dumps({
+                    **vol_cfg, 'name': snap_vol_name, 'readonly': True
+                }) } }
+            ] })
+            if resp.get('succeeded'):
+                return new_cfg
+
    def initialize_connection(self, volume, connector):
        data = {
            'driver_volume_type': 'vitastor',
@@ -538,9 +697,13 @@ class VitastorDriver(driver.CloneableImageVD,
                    size = int(volume.size) * units.Gi
                    dest_name = utils.convert_str(volume.name)
                    # Find or create the base snapshot
-                    self._cli('create base snapshot', 'create', '--allow-existing', '1', base_vol.name+'@.clone_snap')
+                    snap_cfg = self._create_snapshot(base_vol.name, base_vol.name+'@.clone_snap', True)
                    # Then create a clone from it
-                    self._cli('create clone', 'create', dest_name, '--size', size, '--parent', base_vol.name+'@.clone_snap')
+                    new_cfg = self._create_image(dest_name, {
+                        'size': size,
+                        'parent_id': snap_cfg['parent_id'],
+                        'parent_pool_id': snap_cfg['parent_pool_id'],
+                    })
                    return ({}, True)
        return ({}, False)

@@ -607,8 +770,26 @@ class VitastorDriver(driver.CloneableImageVD,
    def extend_volume(self, volume, new_size):
        """Extend an existing volume."""
        vol_name = utils.convert_str(volume.name)
-        size = int(new_size) * units.Gi
-        self._cli('extend volume', 'modify', vol_name, '--resize', new_size)
+        while True:
+            vol = self._get_image(vol_name)
+            if not vol:
+                raise exception.VolumeBackendAPIException(data = 'Volume '+vol_name+' does not exist')
+            # change size
+            size = int(new_size) * units.Gi
+            if size == vol['cfg']['size']:
+                break
+            resp = self._etcd_txn({ 'compare': [ {
+                'target': 'MOD',
+                'mod_revision': vol['cfg_mod'],
+                'key': 'config/inode/'+str(vol['idx']['pool_id'])+'/'+str(vol['idx']['id']),
+            } ], 'success': [
+                { 'request_put': {
+                    'key': 'config/inode/'+str(vol['idx']['pool_id'])+'/'+str(vol['idx']['id']),
+                    'value': json.dumps({ **vol['cfg'], 'size': size }),
+                } },
+            ] })
+            if resp.get('succeeded'):
+                break
        LOG.debug(
            "Extend volume from %(old_size)s GB to %(new_size)s GB.",
            {'old_size': volume.size, 'new_size': new_size}
@@ -681,7 +862,28 @@ class VitastorDriver(driver.CloneableImageVD,
        """
        from_name = self._get_existing_name(existing_ref)
        to_name = utils.convert_str(volume.name)
-        self._cli('rename', 'modify', from_name, '--rename', to_name)
+        self._rename(from_name, to_name)
+
+    def _rename(self, from_name, to_name):
+        while True:
+            vol = self._get_image(from_name)
+            if not vol:
+                raise exception.VolumeBackendAPIException(data = 'Volume '+from_name+' does not exist')
+            to = self._get_image(to_name)
+            if to:
+                raise exception.VolumeBackendAPIException(data = 'Volume '+to_name+' already exists')
+            resp = self._etcd_txn({ 'compare': [
+                { 'target': 'MOD', 'mod_revision': vol['idx_mod'], 'key': 'index/image/'+vol['cfg']['name'] },
+                { 'target': 'MOD', 'mod_revision': vol['cfg_mod'], 'key': 'config/inode/'+str(vol['idx']['pool_id'])+'/'+str(vol['idx']['id']) },
+                { 'target': 'VERSION', 'version': 0, 'key': 'index/image/'+to_name },
+            ], 'success': [
+                { 'request_delete_range': { 'key': 'index/image/'+vol['cfg']['name'] } },
+                { 'request_put': { 'key': 'index/image/'+to_name, 'value': json.dumps(vol['idx']) } },
+                { 'request_put': { 'key': 'config/inode/'+str(vol['idx']['pool_id'])+'/'+str(vol['idx']['id']),
+                    'value': json.dumps({ **vol['cfg'], 'name': to_name }) } },
+            ] })
+            if resp.get('succeeded'):
+                break

    def unmanage(self, volume):
        pass
@@ -754,7 +956,7 @@ class VitastorDriver(driver.CloneableImageVD,
        snap_name = self._get_existing_name(existing_ref)
        from_name = vol_name+'@'+snap_name
        to_name = vol_name+'@'+utils.convert_str(snapshot.name)
-        self._cli('rename', 'modify', from_name, '--rename', to_name)
+        self._rename(from_name, to_name)

    def unmanage_snapshot(self, snapshot):
        """Removes the specified snapshot from Cinder management."""
@@ -1,637 +0,0 @@
-diff --git a/include/libvirt/libvirt-storage.h b/include/libvirt/libvirt-storage.h
-index aaad4a3da1..5f5daa8341 100644
--- a/include/libvirt/libvirt-storage.h
-+++ b/include/libvirt/libvirt-storage.h
-@@ -326,6 +326,7 @@ typedef enum {
-     VIR_CONNECT_LIST_STORAGE_POOLS_ZFS           = 1 << 17, /* (Since: 1.2.8) */
-     VIR_CONNECT_LIST_STORAGE_POOLS_VSTORAGE      = 1 << 18, /* (Since: 3.1.0) */
-     VIR_CONNECT_LIST_STORAGE_POOLS_ISCSI_DIRECT  = 1 << 19, /* (Since: 5.6.0) */
-+    VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR      = 1 << 20, /* (Since: 5.0.0) */
- } virConnectListAllStoragePoolsFlags;
- 
- int                     virConnectListAllStoragePools(virConnectPtr conn,
-diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
-index 9ca5c2450c..cc52f00c0c 100644
--- a/src/conf/domain_conf.c
-+++ b/src/conf/domain_conf.c
-@@ -7453,7 +7453,8 @@ virDomainDiskSourceNetworkParse(xmlNodePtr node,
-     src->configFile = virXPathString("string(./config/@file)", ctxt);
- 
-     if (src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTP ||
-        src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTPS)
-+        src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTPS ||
-+        src->protocol == VIR_STORAGE_NET_PROTOCOL_VITASTOR)
-         src->query = virXMLPropString(node, "query");
- 
-     if (virDomainStorageNetworkParseHosts(node, ctxt, &src->hosts, &src->nhosts) < 0)
-@@ -32187,6 +32188,7 @@ virDomainStorageSourceTranslateSourcePool(virStorageSource *src,
- 
-     case VIR_STORAGE_POOL_MPATH:
-     case VIR_STORAGE_POOL_RBD:
-+    case VIR_STORAGE_POOL_VITASTOR:
-     case VIR_STORAGE_POOL_SHEEPDOG:
-     case VIR_STORAGE_POOL_GLUSTER:
-     case VIR_STORAGE_POOL_LAST:
-diff --git a/src/conf/domain_validate.c b/src/conf/domain_validate.c
-index 7346a61731..83e94d762e 100644
--- a/src/conf/domain_validate.c
-+++ b/src/conf/domain_validate.c
-@@ -520,6 +520,7 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
-         case VIR_STORAGE_NET_PROTOCOL_RBD:
-             break;
- 
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-         case VIR_STORAGE_NET_PROTOCOL_NBD:
-         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-@@ -592,7 +593,7 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
-         }
-     }
- 
-    /* internal snapshots and config files are currently supported only with rbd: */
-+    /* internal snapshots are currently supported only with rbd: */
-     if (virStorageSourceGetActualType(src) != VIR_STORAGE_TYPE_NETWORK &&
-         src->protocol != VIR_STORAGE_NET_PROTOCOL_RBD) {
-         if (src->snapshot) {
-@@ -600,10 +601,14 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
-                            _("<snapshot> element is currently supported only with 'rbd' disks"));
-             return -1;
-         }
-
-+    }
-+    /* config files are currently supported only with rbd and vitastor: */
-+    if (virStorageSourceGetActualType(src) != VIR_STORAGE_TYPE_NETWORK &&
-+        src->protocol != VIR_STORAGE_NET_PROTOCOL_RBD &&
-+        src->protocol != VIR_STORAGE_NET_PROTOCOL_VITASTOR) {
-         if (src->configFile) {
-             virReportError(VIR_ERR_XML_ERROR, "%s",
-                           _("<config> element is currently supported only with 'rbd' disks"));
-+                           _("<config> element is currently supported only with 'rbd' and 'vitastor' disks"));
-             return -1;
-         }
-     }
-diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng
-index 114dd3f96f..c71f9a3277 100644
--- a/src/conf/schemas/domaincommon.rng
-+++ b/src/conf/schemas/domaincommon.rng
-@@ -2093,6 +2093,35 @@
-     </element>
-   </define>
- 
-+  <define name="diskSourceNetworkProtocolVitastor">
-+    <element name="source">
-+      <interleave>
-+        <attribute name="protocol">
-+          <value>vitastor</value>
-+        </attribute>
-+        <ref name="diskSourceCommon"/>
-+        <optional>
-+          <attribute name="name"/>
-+        </optional>
-+        <optional>
-+          <attribute name="query"/>
-+        </optional>
-+        <zeroOrMore>
-+          <ref name="diskSourceNetworkHost"/>
-+        </zeroOrMore>
-+        <optional>
-+          <element name="config">
-+            <attribute name="file">
-+              <ref name="absFilePath"/>
-+            </attribute>
-+            <empty/>
-+          </element>
-+        </optional>
-+        <empty/>
-+      </interleave>
-+    </element>
-+  </define>
-+
-   <define name="diskSourceNetworkProtocolISCSI">
-     <element name="source">
-       <attribute name="protocol">
-@@ -2443,6 +2472,7 @@
-       <ref name="diskSourceNetworkProtocolSimple"/>
-       <ref name="diskSourceNetworkProtocolVxHS"/>
-       <ref name="diskSourceNetworkProtocolNFS"/>
-+      <ref name="diskSourceNetworkProtocolVitastor"/>
-     </choice>
-   </define>
- 
-diff --git a/src/conf/storage_conf.c b/src/conf/storage_conf.c
-index 1dc9365bf2..a8a736be81 100644
--- a/src/conf/storage_conf.c
-+++ b/src/conf/storage_conf.c
-@@ -56,7 +56,7 @@ VIR_ENUM_IMPL(virStoragePool,
-               "logical", "disk", "iscsi",
-               "iscsi-direct", "scsi", "mpath",
-               "rbd", "sheepdog", "gluster",
-              "zfs", "vstorage",
-+              "zfs", "vstorage", "vitastor",
- );
- 
- VIR_ENUM_IMPL(virStoragePoolFormatFileSystem,
-@@ -242,6 +242,18 @@ static virStoragePoolTypeInfo poolTypeInfo[] = {
-           .formatToString = virStorageFileFormatTypeToString,
-       }
-     },
-+    {.poolType = VIR_STORAGE_POOL_VITASTOR,
-+     .poolOptions = {
-+         .flags = (VIR_STORAGE_POOL_SOURCE_HOST |
-+                   VIR_STORAGE_POOL_SOURCE_NETWORK |
-+                   VIR_STORAGE_POOL_SOURCE_NAME),
-+      },
-+      .volOptions = {
-+          .defaultFormat = VIR_STORAGE_FILE_RAW,
-+          .formatFromString = virStorageVolumeFormatFromString,
-+          .formatToString = virStorageFileFormatTypeToString,
-+      }
-+    },
-     {.poolType = VIR_STORAGE_POOL_SHEEPDOG,
-      .poolOptions = {
-          .flags = (VIR_STORAGE_POOL_SOURCE_HOST |
-@@ -538,6 +550,11 @@ virStoragePoolDefParseSource(xmlXPathContextPtr ctxt,
-                        _("element 'name' is mandatory for RBD pool"));
-         return -1;
-     }
-+    if (pool_type == VIR_STORAGE_POOL_VITASTOR && source->name == NULL) {
-+        virReportError(VIR_ERR_XML_ERROR, "%s",
-+                       _("element 'name' is mandatory for Vitastor pool"));
-+        return -1;
-+    }
- 
-     if (options->formatFromString) {
-         g_autofree char *format = NULL;
-@@ -1127,6 +1144,7 @@ virStoragePoolDefFormatBuf(virBuffer *buf,
-     /* RBD, Sheepdog, Gluster and Iscsi-direct devices are not local block devs nor
-      * files, so they don't have a target */
-     if (def->type != VIR_STORAGE_POOL_RBD &&
-+        def->type != VIR_STORAGE_POOL_VITASTOR &&
-         def->type != VIR_STORAGE_POOL_SHEEPDOG &&
-         def->type != VIR_STORAGE_POOL_GLUSTER &&
-         def->type != VIR_STORAGE_POOL_ISCSI_DIRECT) {
-diff --git a/src/conf/storage_conf.h b/src/conf/storage_conf.h
-index fc67957cfe..720c07ef74 100644
--- a/src/conf/storage_conf.h
-+++ b/src/conf/storage_conf.h
-@@ -103,6 +103,7 @@ typedef enum {
-     VIR_STORAGE_POOL_GLUSTER,  /* Gluster device */
-     VIR_STORAGE_POOL_ZFS,      /* ZFS */
-     VIR_STORAGE_POOL_VSTORAGE, /* Virtuozzo Storage */
-+    VIR_STORAGE_POOL_VITASTOR, /* Vitastor */
- 
-     VIR_STORAGE_POOL_LAST,
- } virStoragePoolType;
-@@ -454,6 +455,7 @@ VIR_ENUM_DECL(virStoragePartedFs);
-                  VIR_CONNECT_LIST_STORAGE_POOLS_SCSI     | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_MPATH    | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_RBD      | \
-+                 VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER  | \
-                  VIR_CONNECT_LIST_STORAGE_POOLS_ZFS      | \
-diff --git a/src/conf/storage_source_conf.c b/src/conf/storage_source_conf.c
-index d7b9bdfecb..38aefd0dd4 100644
--- a/src/conf/storage_source_conf.c
-+++ b/src/conf/storage_source_conf.c
-@@ -90,6 +90,7 @@ VIR_ENUM_IMPL(virStorageNetProtocol,
-               "ssh",
-               "vxhs",
-               "nfs",
-+              "vitastor",
- );
- 
- 
-@@ -1317,6 +1318,7 @@ virStorageSourceNetworkDefaultPort(virStorageNetProtocol protocol)
-         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-             return 24007;
- 
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-         case VIR_STORAGE_NET_PROTOCOL_RBD:
-             /* we don't provide a default for RBD */
-             return 0;
-diff --git a/src/conf/storage_source_conf.h b/src/conf/storage_source_conf.h
-index 22c35d420d..f1e32ea83d 100644
--- a/src/conf/storage_source_conf.h
-+++ b/src/conf/storage_source_conf.h
-@@ -131,6 +131,7 @@ typedef enum {
-     VIR_STORAGE_NET_PROTOCOL_SSH,
-     VIR_STORAGE_NET_PROTOCOL_VXHS,
-     VIR_STORAGE_NET_PROTOCOL_NFS,
-+    VIR_STORAGE_NET_PROTOCOL_VITASTOR,
- 
-     VIR_STORAGE_NET_PROTOCOL_LAST
- } virStorageNetProtocol;
-diff --git a/src/conf/virstorageobj.c b/src/conf/virstorageobj.c
-index 59fa5da372..4739167f5f 100644
--- a/src/conf/virstorageobj.c
-+++ b/src/conf/virstorageobj.c
-@@ -1438,6 +1438,7 @@ virStoragePoolObjSourceFindDuplicateCb(const void *payload,
-             return 1;
-         break;
- 
-+    case VIR_STORAGE_POOL_VITASTOR:
-     case VIR_STORAGE_POOL_ISCSI_DIRECT:
-     case VIR_STORAGE_POOL_RBD:
-     case VIR_STORAGE_POOL_LAST:
-@@ -1921,6 +1922,8 @@ virStoragePoolObjMatch(virStoragePoolObj *obj,
-                (obj->def->type == VIR_STORAGE_POOL_MPATH))   ||
-               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_RBD) &&
-                (obj->def->type == VIR_STORAGE_POOL_RBD))     ||
-+              (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR) &&
-+               (obj->def->type == VIR_STORAGE_POOL_VITASTOR)) ||
-               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG) &&
-                (obj->def->type == VIR_STORAGE_POOL_SHEEPDOG)) ||
-               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER) &&
-diff --git a/src/libvirt-storage.c b/src/libvirt-storage.c
-index db7660aac4..561df34709 100644
--- a/src/libvirt-storage.c
-+++ b/src/libvirt-storage.c
-@@ -94,6 +94,7 @@ virStoragePoolGetConnect(virStoragePoolPtr pool)
-  * VIR_CONNECT_LIST_STORAGE_POOLS_SCSI
-  * VIR_CONNECT_LIST_STORAGE_POOLS_MPATH
-  * VIR_CONNECT_LIST_STORAGE_POOLS_RBD
-+ * VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR
-  * VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG
-  * VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER
-  * VIR_CONNECT_LIST_STORAGE_POOLS_ZFS
-diff --git a/src/libxl/libxl_conf.c b/src/libxl/libxl_conf.c
-index 2b988157fa..9d0eb47b25 100644
--- a/src/libxl/libxl_conf.c
-+++ b/src/libxl/libxl_conf.c
-@@ -1069,6 +1069,7 @@ libxlMakeNetworkDiskSrcStr(virStorageSource *src,
-     case VIR_STORAGE_NET_PROTOCOL_SSH:
-     case VIR_STORAGE_NET_PROTOCOL_VXHS:
-     case VIR_STORAGE_NET_PROTOCOL_NFS:
-+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-     case VIR_STORAGE_NET_PROTOCOL_LAST:
-     case VIR_STORAGE_NET_PROTOCOL_NONE:
-         virReportError(VIR_ERR_NO_SUPPORT,
-diff --git a/src/libxl/xen_xl.c b/src/libxl/xen_xl.c
-index e72e7d7f44..8482c21805 100644
--- a/src/libxl/xen_xl.c
-+++ b/src/libxl/xen_xl.c
-@@ -1461,6 +1461,7 @@ xenFormatXLDiskSrcNet(virStorageSource *src)
-     case VIR_STORAGE_NET_PROTOCOL_SSH:
-     case VIR_STORAGE_NET_PROTOCOL_VXHS:
-     case VIR_STORAGE_NET_PROTOCOL_NFS:
-+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-     case VIR_STORAGE_NET_PROTOCOL_LAST:
-     case VIR_STORAGE_NET_PROTOCOL_NONE:
-         virReportError(VIR_ERR_NO_SUPPORT,
-diff --git a/src/qemu/qemu_block.c b/src/qemu/qemu_block.c
-index 9b43279797..459d8e8a65 100644
--- a/src/qemu/qemu_block.c
-+++ b/src/qemu/qemu_block.c
-@@ -743,6 +743,38 @@ qemuBlockStorageSourceGetRBDProps(virStorageSource *src,
- }
- 
- 
-+static virJSONValue *
-+qemuBlockStorageSourceGetVitastorProps(virStorageSource *src)
-+{
-+    virJSONValue *ret = NULL;
-+    virStorageNetHostDef *host;
-+    size_t i;
-+    g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;
-+    g_autofree char *etcd = NULL;
-+
-+    for (i = 0; i < src->nhosts; i++) {
-+        host = src->hosts + i;
-+        if ((virStorageNetHostTransport)host->transport != VIR_STORAGE_NET_HOST_TRANS_TCP) {
-+            return NULL;
-+        }
-+        virBufferAsprintf(&buf, i > 0 ? ",%s:%u" : "%s:%u", host->name, host->port);
-+    }
-+    if (src->nhosts > 0) {
-+        etcd = virBufferContentAndReset(&buf);
-+    }
-+
-+    if (virJSONValueObjectAdd(&ret,
-+                              "S:etcd-host", etcd,
-+                              "S:etcd-prefix", src->query,
-+                              "S:config-path", src->configFile,
-+                              "s:image", src->path,
-+                              NULL) < 0)
-+        return NULL;
-+
-+    return ret;
-+}
-+
-+
- static virJSONValue *
- qemuBlockStorageSourceGetSshProps(virStorageSource *src)
- {
-@@ -1094,6 +1126,12 @@ qemuBlockStorageSourceGetBackendProps(virStorageSource *src,
-                 return NULL;
-             break;
- 
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-+            driver = "vitastor";
-+            if (!(fileprops = qemuBlockStorageSourceGetVitastorProps(src)))
-+                return NULL;
-+            break;
-+
-         case VIR_STORAGE_NET_PROTOCOL_SSH:
-             driver = "ssh";
-             if (!(fileprops = qemuBlockStorageSourceGetSshProps(src)))
-@@ -1997,6 +2035,7 @@ qemuBlockGetBackingStoreString(virStorageSource *src,
- 
-             case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-             case VIR_STORAGE_NET_PROTOCOL_RBD:
-+            case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-             case VIR_STORAGE_NET_PROTOCOL_VXHS:
-             case VIR_STORAGE_NET_PROTOCOL_NFS:
-             case VIR_STORAGE_NET_PROTOCOL_SSH:
-@@ -2377,6 +2416,12 @@ qemuBlockStorageSourceCreateGetStorageProps(virStorageSource *src,
-                 return -1;
-             break;
- 
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-+            driver = "vitastor";
-+            if (!(location = qemuBlockStorageSourceGetVitastorProps(src)))
-+                return -1;
-+            break;
-+
-         case VIR_STORAGE_NET_PROTOCOL_SSH:
-             if (srcPriv->nbdkitProcess) {
-                 /* disk creation not yet supported with nbdkit, and even if it
-diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
-index ac56fc7cb4..9e407b4aab 100644
--- a/src/qemu/qemu_domain.c
-+++ b/src/qemu/qemu_domain.c
-@@ -4677,7 +4677,8 @@ qemuDomainValidateStorageSource(virStorageSource *src,
-     if (src->query &&
-         (actualType != VIR_STORAGE_TYPE_NETWORK ||
-          (src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTPS &&
-          src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTP))) {
-+          src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTP &&
-+          src->protocol != VIR_STORAGE_NET_PROTOCOL_VITASTOR))) {
-         virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
-                        _("query is supported only with HTTP(S) protocols"));
-         return -1;
-@@ -9103,6 +9104,7 @@ qemuDomainPrepareStorageSourceTLS(virStorageSource *src,
-         break;
- 
-     case VIR_STORAGE_NET_PROTOCOL_RBD:
-+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-     case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-     case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-     case VIR_STORAGE_NET_PROTOCOL_ISCSI:
-diff --git a/src/qemu/qemu_snapshot.c b/src/qemu/qemu_snapshot.c
-index e738afffc3..37d64f469b 100644
--- a/src/qemu/qemu_snapshot.c
-+++ b/src/qemu/qemu_snapshot.c
-@@ -665,6 +665,7 @@ qemuSnapshotPrepareDiskExternalInactive(virDomainSnapshotDiskDef *snapdisk,
-         case VIR_STORAGE_NET_PROTOCOL_NONE:
-         case VIR_STORAGE_NET_PROTOCOL_NBD:
-         case VIR_STORAGE_NET_PROTOCOL_RBD:
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-         case VIR_STORAGE_NET_PROTOCOL_ISCSI:
-@@ -893,6 +894,7 @@ qemuSnapshotPrepareDiskInternal(virDomainDiskDef *disk,
-         case VIR_STORAGE_NET_PROTOCOL_NONE:
-         case VIR_STORAGE_NET_PROTOCOL_NBD:
-         case VIR_STORAGE_NET_PROTOCOL_RBD:
-+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
-         case VIR_STORAGE_NET_PROTOCOL_ISCSI:
-diff --git a/src/storage/storage_driver.c b/src/storage/storage_driver.c
-index e19e032427..59f91f4710 100644
--- a/src/storage/storage_driver.c
-+++ b/src/storage/storage_driver.c
-@@ -1626,6 +1626,7 @@ storageVolLookupByPathCallback(virStoragePoolObj *obj,
- 
-         case VIR_STORAGE_POOL_GLUSTER:
-         case VIR_STORAGE_POOL_RBD:
-+        case VIR_STORAGE_POOL_VITASTOR:
-         case VIR_STORAGE_POOL_SHEEPDOG:
-         case VIR_STORAGE_POOL_ZFS:
-         case VIR_STORAGE_POOL_LAST:
-diff --git a/src/storage_file/storage_source_backingstore.c b/src/storage_file/storage_source_backingstore.c
-index 821378883c..2211f6891b 100644
--- a/src/storage_file/storage_source_backingstore.c
-+++ b/src/storage_file/storage_source_backingstore.c
-@@ -264,6 +264,75 @@ virStorageSourceParseRBDColonString(const char *rbdstr,
- }
- 
- 
-+static int
-+virStorageSourceParseVitastorColonString(const char *colonstr,
-+                                         virStorageSource *src)
-+{
-+    char *p, *e, *next;
-+    g_autofree char *options = NULL;
-+
-+    /* optionally skip the "vitastor:" prefix if provided */
-+    if (STRPREFIX(colonstr, "vitastor:"))
-+        colonstr += strlen("vitastor:");
-+
-+    options = g_strdup(colonstr);
-+
-+    p = options;
-+    while (*p) {
-+        /* find : delimiter or end of string */
-+        for (e = p; *e && *e != ':'; ++e) {
-+            if (*e == '\\') {
-+                e++;
-+                if (*e == '\0')
-+                    break;
-+            }
-+        }
-+        if (*e == '\0') {
-+            next = e;    /* last kv pair */
-+        } else {
-+            next = e + 1;
-+            *e = '\0';
-+        }
-+
-+        if (STRPREFIX(p, "image=")) {
-+            src->path = g_strdup(p + strlen("image="));
-+        } else if (STRPREFIX(p, "etcd-prefix=")) {
-+            src->query = g_strdup(p + strlen("etcd-prefix="));
-+        } else if (STRPREFIX(p, "config-path=")) {
-+            src->configFile = g_strdup(p + strlen("config-path="));
-+        } else if (STRPREFIX(p, "etcd-host=")) {
-+            char *h, *sep;
-+
-+            h = p + strlen("etcd-host=");
-+            while (h < e) {
-+                for (sep = h; sep < e; ++sep) {
-+                    if (*sep == '\\' && (sep[1] == ',' ||
-+                                         sep[1] == ';' ||
-+                                         sep[1] == ' ')) {
-+                        *sep = '\0';
-+                        sep += 2;
-+                        break;
-+                    }
-+                }
-+
-+                if (virStorageSourceRBDAddHost(src, h) < 0)
-+                    return -1;
-+
-+                h = sep;
-+            }
-+        }
-+
-+        p = next;
-+    }
-+
-+    if (!src->path) {
-+        return -1;
-+    }
-+
-+    return 0;
-+}
-+
-+
- static int
- virStorageSourceParseNBDColonString(const char *nbdstr,
-                                     virStorageSource *src)
-@@ -379,6 +448,11 @@ virStorageSourceParseBackingColon(virStorageSource *src,
-             return -1;
-         break;
- 
-+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
-+        if (virStorageSourceParseVitastorColonString(path, src) < 0)
-+            return -1;
-+        break;
-+
-     case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
-     case VIR_STORAGE_NET_PROTOCOL_LAST:
-     case VIR_STORAGE_NET_PROTOCOL_NONE:
-@@ -953,6 +1027,54 @@ virStorageSourceParseBackingJSONRBD(virStorageSource *src,
-     return 0;
- }
- 
-+static int
-+virStorageSourceParseBackingJSONVitastor(virStorageSource *src,
-+                                         virJSONValue *json,
-+                                         const char *jsonstr G_GNUC_UNUSED,
-+                                         int opaque G_GNUC_UNUSED)
-+{
-+    const char *filename;
-+    const char *image = virJSONValueObjectGetString(json, "image");
-+    const char *conf = virJSONValueObjectGetString(json, "config-path");
-+    const char *etcd_prefix = virJSONValueObjectGetString(json, "etcd-prefix");
-+    virJSONValue *servers = virJSONValueObjectGetArray(json, "server");
-+    size_t nservers;
-+    size_t i;
-+
-+    src->type = VIR_STORAGE_TYPE_NETWORK;
-+    src->protocol = VIR_STORAGE_NET_PROTOCOL_VITASTOR;
-+
-+    /* legacy syntax passed via 'filename' option */
-+    if ((filename = virJSONValueObjectGetString(json, "filename")))
-+        return virStorageSourceParseVitastorColonString(filename, src);
-+
-+    if (!image) {
-+        virReportError(VIR_ERR_INVALID_ARG, "%s",
-+                       _("missing image name in Vitastor backing volume "
-+                         "JSON specification"));
-+        return -1;
-+    }
-+
-+    src->path = g_strdup(image);
-+    src->configFile = g_strdup(conf);
-+    src->query = g_strdup(etcd_prefix);
-+
-+    if (servers) {
-+        nservers = virJSONValueArraySize(servers);
-+
-+        src->hosts = g_new0(virStorageNetHostDef, nservers);
-+        src->nhosts = nservers;
-+
-+        for (i = 0; i < nservers; i++) {
-+            if (virStorageSourceParseBackingJSONInetSocketAddress(src->hosts + i,
-+                                                                  virJSONValueArrayGet(servers, i)) < 0)
-+                return -1;
-+        }
-+    }
-+
-+    return 0;
-+}
-+
- static int
- virStorageSourceParseBackingJSONRaw(virStorageSource *src,
-                                     virJSONValue *json,
-@@ -1130,6 +1252,7 @@ static const struct virStorageSourceJSONDriverParser jsonParsers[] = {
-     {"sheepdog", false, virStorageSourceParseBackingJSONSheepdog, 0},
-     {"ssh", false, virStorageSourceParseBackingJSONSSH, 0},
-     {"rbd", false, virStorageSourceParseBackingJSONRBD, 0},
-+    {"vitastor", false, virStorageSourceParseBackingJSONVitastor, 0},
-     {"raw", true, virStorageSourceParseBackingJSONRaw, 0},
-     {"nfs", false, virStorageSourceParseBackingJSONNFS, 0},
-     {"vxhs", false, virStorageSourceParseBackingJSONVxHS, 0},
-diff --git a/src/test/test_driver.c b/src/test/test_driver.c
-index 1165689de7..bba846351c 100644
--- a/src/test/test_driver.c
-+++ b/src/test/test_driver.c
-@@ -7345,6 +7345,7 @@ testStorageVolumeTypeForPool(int pooltype)
-     case VIR_STORAGE_POOL_ISCSI_DIRECT:
-     case VIR_STORAGE_POOL_GLUSTER:
-     case VIR_STORAGE_POOL_RBD:
-+    case VIR_STORAGE_POOL_VITASTOR:
-         return VIR_STORAGE_VOL_NETWORK;
-     case VIR_STORAGE_POOL_LOGICAL:
-     case VIR_STORAGE_POOL_DISK:
-diff --git a/tests/storagepoolcapsschemadata/poolcaps-fs.xml b/tests/storagepoolcapsschemadata/poolcaps-fs.xml
-index eee75af746..8bd0a57bdd 100644
--- a/tests/storagepoolcapsschemadata/poolcaps-fs.xml
-+++ b/tests/storagepoolcapsschemadata/poolcaps-fs.xml
-@@ -204,4 +204,11 @@
-       </enum>
-     </volOptions>
-   </pool>
-+  <pool type='vitastor' supported='no'>
-+    <volOptions>
-+      <defaultFormat type='raw'/>
-+      <enum name='targetFormatType'>
-+      </enum>
-+    </volOptions>
-+  </pool>
- </storagepoolCapabilities>
-diff --git a/tests/storagepoolcapsschemadata/poolcaps-full.xml b/tests/storagepoolcapsschemadata/poolcaps-full.xml
-index 805950a937..852df0de16 100644
--- a/tests/storagepoolcapsschemadata/poolcaps-full.xml
-+++ b/tests/storagepoolcapsschemadata/poolcaps-full.xml
-@@ -204,4 +204,11 @@
-       </enum>
-     </volOptions>
-   </pool>
-+  <pool type='vitastor' supported='yes'>
-+    <volOptions>
-+      <defaultFormat type='raw'/>
-+      <enum name='targetFormatType'>
-+      </enum>
-+    </volOptions>
-+  </pool>
- </storagepoolCapabilities>
-diff --git a/tests/storagepoolxml2argvtest.c b/tests/storagepoolxml2argvtest.c
-index d5c2531ab8..b19308ac38 100644
--- a/tests/storagepoolxml2argvtest.c
-+++ b/tests/storagepoolxml2argvtest.c
-@@ -57,6 +57,7 @@ testCompareXMLToArgvFiles(bool shouldFail,
-     case VIR_STORAGE_POOL_GLUSTER:
-     case VIR_STORAGE_POOL_ZFS:
-     case VIR_STORAGE_POOL_VSTORAGE:
-+    case VIR_STORAGE_POOL_VITASTOR:
-     case VIR_STORAGE_POOL_LAST:
-     default:
-         VIR_TEST_DEBUG("pool type '%s' has no xml2argv test", defTypeStr);
-diff --git a/tools/virsh-pool.c b/tools/virsh-pool.c
-index 2010ef1356..072e2ff9e8 100644
--- a/tools/virsh-pool.c
-+++ b/tools/virsh-pool.c
-@@ -1187,6 +1187,9 @@ cmdPoolList(vshControl *ctl, const vshCmd *cmd G_GNUC_UNUSED)
-             case VIR_STORAGE_POOL_VSTORAGE:
-                 flags |= VIR_CONNECT_LIST_STORAGE_POOLS_VSTORAGE;
-                 break;
-+            case VIR_STORAGE_POOL_VITASTOR:
-+                flags |= VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR;
-+                break;
-             case VIR_STORAGE_POOL_LAST:
-                 break;
-             }
@@ -1,39 +0,0 @@
-From 98d3f68a40130c438854f61db6025f9e9b099cb6 Mon Sep 17 00:00:00 2001
-From: Vitaliy Filippov <vitalifster@gmail.com>
-Date: Sat, 20 Dec 2025 14:44:35 +0300
-Subject: [PATCH] Do not require atomic writes to be power of 2 sized and
- aligned on length boundary
-
-It contradicts NVMe specification where alignment is only required when atomic
-write boundary (NABSPF/NABO) is set and highly limits usage of NVMe atomic writes
-
-Signed-off-by: Vitaliy Filippov <vitalifster@gmail.com>
---
- fs/read_write.c | 8 --------
- 1 file changed, 8 deletions(-)
-
-diff --git a/fs/read_write.c b/fs/read_write.c
-index 833bae068770..5467d710108d 100644
--- a/fs/read_write.c
-+++ b/fs/read_write.c
-@@ -1802,17 +1802,9 @@ int generic_file_rw_checks(struct file *file_in, struct file *file_out)
- 
- int generic_atomic_write_valid(struct kiocb *iocb, struct iov_iter *iter)
- {
-	size_t len = iov_iter_count(iter);
-
- 	if (!iter_is_ubuf(iter))
- 		return -EINVAL;
- 
-	if (!is_power_of_2(len))
-		return -EINVAL;
-
-	if (!IS_ALIGNED(iocb->ki_pos, len))
-		return -EINVAL;
-
- 	if (!(iocb->ki_flags & IOCB_DIRECT))
- 		return -EOPNOTSUPP;
- 
-- 
-2.51.0
-
@@ -1,172 +1,29 @@
-diff --git a/block/meson.build b/block/meson.build
-index 34b1b2a306..24ca0f1e52 100644
--- a/block/meson.build
-+++ b/block/meson.build
-@@ -114,6 +114,7 @@ foreach m : [
-   [libnfs, 'nfs', files('nfs.c')],
-   [libssh, 'ssh', files('ssh.c')],
-   [rbd, 'rbd', files('rbd.c')],
-+  [vitastor, 'vitastor', files('vitastor.c')],
- ]
-   if m[0].found()
-     module_ss = ss.source_set()
-diff --git a/meson.build b/meson.build
-index 50c774a195..e5c7a3a4b1 100644
--- a/meson.build
-+++ b/meson.build
-@@ -1652,6 +1652,26 @@ if not get_option('rbd').auto() or have_block
-   endif
- endif
+diff --git a/src/client/qemu_driver.c b/src/client/qemu_driver.c
+index d8356dab..5f4cd50d 100644
+--- a/src/client/qemu_driver.c
+++ b/src/client/qemu_driver.c
+@@ -974,14 +974,21 @@ static void vitastor_co_read_bitmap_cb(void *opaque, long retval, uint8_t *bitma
+ #endif
+ }
 
-+vitastor = not_found
-+if not get_option('vitastor').auto() or have_block
-+  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
-+    required: get_option('vitastor'))
-+  if libvitastor_client.found()
-+    if cc.links('''
-+      #include <vitastor_c.h>
-+      int main(void) {
-+        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
-+        return 0;
-+      }''', dependencies: libvitastor_client)
-+      vitastor = declare_dependency(dependencies: libvitastor_client)
-+    elif get_option('vitastor').enabled()
-+      error('could not link libvitastor_client')
-+    else
-+      warning('could not link libvitastor_client, disabling')
-+    endif
-+  endif
-+endif
-+
- glusterfs = not_found
- glusterfs_ftruncate_has_stat = false
- glusterfs_iocb_has_stat = false
-@@ -2547,6 +2567,7 @@ endif
- config_host_data.set('CONFIG_OPENGL', opengl.found())
- config_host_data.set('CONFIG_PLUGIN', get_option('plugins'))
- config_host_data.set('CONFIG_RBD', rbd.found())
-+config_host_data.set('CONFIG_VITASTOR', vitastor.found())
- config_host_data.set('CONFIG_RDMA', rdma.found())
- config_host_data.set('CONFIG_RELOCATABLE', get_option('relocatable'))
- config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack'))
-@@ -4972,6 +4993,7 @@ summary_info += {'fdt support':       fdt_opt == 'internal' ? 'internal' : fdt}
- summary_info += {'libcap-ng support': libcap_ng}
- summary_info += {'bpf support':       libbpf}
- summary_info += {'rbd support':       rbd}
-+summary_info += {'vitastor support':  vitastor}
- summary_info += {'smartcard support': cacard}
- summary_info += {'U2F support':       u2f}
- summary_info += {'libusb':            libusb}
-diff --git a/meson_options.txt b/meson_options.txt
-index fff1521e58..f0844c0e00 100644
--- a/meson_options.txt
-+++ b/meson_options.txt
-@@ -202,6 +202,8 @@ option('pvg', type: 'feature', value: 'auto',
-        description: 'macOS paravirtualized graphics support')
- option('rbd', type : 'feature', value : 'auto',
-        description: 'Ceph block device driver')
-+option('vitastor', type : 'feature', value : 'auto',
-+       description: 'Vitastor block device driver')
- option('opengl', type : 'feature', value : 'auto',
-        description: 'OpenGL support')
- option('rdma', type : 'feature', value : 'auto',
-diff --git a/qapi/block-core.json b/qapi/block-core.json
-index dc6eb4ae23..d043f4340e 100644
--- a/qapi/block-core.json
-+++ b/qapi/block-core.json
-@@ -3280,7 +3280,7 @@
-             'parallels', 'preallocate', 'qcow', 'qcow2', 'qed', 'quorum',
-             'raw', 'rbd',
-             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
-            'ssh', 'throttle', 'vdi', 'vhdx',
-+            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
-             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
-             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
-             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
-@@ -4363,6 +4363,28 @@
-             '*key-secret': 'str',
-             '*server': ['InetSocketAddressBase'] } }
- 
-+##
-+# @BlockdevOptionsVitastor:
-+#
-+# Driver specific block device options for vitastor
-+#
-+# @image:       Image name
-+# @inode:       Inode number
-+# @pool:        Pool ID
-+# @size:        Desired image size in bytes
-+# @config-path: Path to Vitastor configuration
-+# @etcd-host:   etcd connection address(es)
-+# @etcd-prefix: etcd key/value prefix
-+##
-+{ 'struct': 'BlockdevOptionsVitastor',
-+  'data': { '*inode': 'uint64',
-+            '*pool': 'uint64',
-+            '*size': 'uint64',
-+            '*image': 'str',
-+            '*config-path': 'str',
-+            '*etcd-host': 'str',
-+            '*etcd-prefix': 'str' } }
-+
- ##
- # @ReplicationMode:
- #
-@@ -4831,6 +4853,7 @@
-       'throttle':   'BlockdevOptionsThrottle',
-       'vdi':        'BlockdevOptionsGenericFormat',
-       'vhdx':       'BlockdevOptionsGenericFormat',
-+      'vitastor':   'BlockdevOptionsVitastor',
-       'virtio-blk-vfio-pci':
-                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
-                       'if': 'CONFIG_BLKIO' },
-@@ -5304,6 +5327,20 @@
-             '*cluster-size' :   'size',
-             '*encrypt' :        'RbdEncryptionCreateOptions' } }
- 
-+##
-+# @BlockdevCreateOptionsVitastor:
-+#
-+# Driver specific image creation options for Vitastor.
-+#
-+# @location: Where to store the new image file.  This location cannot
-+#     point to a snapshot.
-+#
-+# @size: Size of the virtual disk in bytes
-+##
-+{ 'struct': 'BlockdevCreateOptionsVitastor',
-+  'data': { 'location':         'BlockdevOptionsVitastor',
-+            'size':             'size' } }
-+
- ##
- # @BlockdevVmdkSubformat:
- #
-@@ -5526,6 +5563,7 @@
-       'ssh':            'BlockdevCreateOptionsSsh',
-       'vdi':            'BlockdevCreateOptionsVdi',
-       'vhdx':           'BlockdevCreateOptionsVhdx',
-+      'vitastor':       'BlockdevCreateOptionsVitastor',
-       'vmdk':           'BlockdevCreateOptionsVmdk',
-       'vpc':            'BlockdevCreateOptionsVpc'
-   } }
-diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh
-index 0ebe6bc52a..2c37ad3892 100644
--- a/scripts/meson-buildoptions.sh
-+++ b/scripts/meson-buildoptions.sh
-@@ -175,6 +175,7 @@ meson_options_help() {
-   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
-   printf "%s\n" '  qpl             Query Processing Library support'
-   printf "%s\n" '  rbd             Ceph block device driver'
-+  printf "%s\n" '  vitastor        Vitastor block device driver'
-   printf "%s\n" '  rdma            Enable RDMA-based migration'
-   printf "%s\n" '  replication     replication support'
-   printf "%s\n" '  rust            Rust support'
-@@ -459,6 +460,8 @@ _meson_option_parse() {
-     --disable-qpl) printf "%s" -Dqpl=disabled ;;
-     --enable-rbd) printf "%s" -Drbd=enabled ;;
-     --disable-rbd) printf "%s" -Drbd=disabled ;;
-+    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
-+    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
-     --enable-rdma) printf "%s" -Drdma=enabled ;;
-     --disable-rdma) printf "%s" -Drdma=disabled ;;
-     --enable-relocatable) printf "%s" -Drelocatable=true ;;
+-static int coroutine_fn vitastor_co_block_status(
+-    BlockDriverState *bs, bool want_zero, int64_t offset, int64_t bytes,
+-    int64_t *pnum, int64_t *map, BlockDriverState **file)
+static int coroutine_fn vitastor_co_block_status(BlockDriverState *bs,
+#if QEMU_VERSION_MAJOR > 10 || QEMU_VERSION_MAJOR == 10 && QEMU_VERSION_MINOR >= 1
+    unsigned int mode,
+#else
+    bool want_zero,
+#endif
+    int64_t offset, int64_t bytes, int64_t *pnum, int64_t *map, BlockDriverState **file)
+ {
+     // Allocated => return BDRV_BLOCK_DATA|BDRV_BLOCK_OFFSET_VALID
+     // Not allocated => return 0
+     // Error => return -errno
+     // Set pnum to length of the extent, `*map` = `offset`, `*file` = `bs`
+#if QEMU_VERSION_MAJOR > 10 || QEMU_VERSION_MAJOR == 10 && QEMU_VERSION_MINOR >= 1
+    int want_zero = (mode == BDRV_WANT_PRECISE);
+#endif
+     VitastorRPC task;
+     VitastorClient *client = bs->opaque;
+     uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode;
@@ -1,172 +0,0 @@
-diff --git a/block/meson.build b/block/meson.build
-index 34b1b2a306..24ca0f1e52 100644
--- a/block/meson.build
-+++ b/block/meson.build
-@@ -114,6 +114,7 @@ foreach m : [
-   [libnfs, 'nfs', files('nfs.c')],
-   [libssh, 'ssh', files('ssh.c')],
-   [rbd, 'rbd', files('rbd.c')],
-+  [vitastor, 'vitastor', files('vitastor.c')],
- ]
-   if m[0].found()
-     module_ss = ss.source_set()
-diff --git a/meson.build b/meson.build
-index d9293294d8..776a5becc6 100644
--- a/meson.build
-+++ b/meson.build
-@@ -1665,6 +1665,26 @@ if not get_option('rbd').auto() or have_block
-   endif
- endif
- 
-+vitastor = not_found
-+if not get_option('vitastor').auto() or have_block
-+  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
-+    required: get_option('vitastor'))
-+  if libvitastor_client.found()
-+    if cc.links('''
-+      #include <vitastor_c.h>
-+      int main(void) {
-+        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
-+        return 0;
-+      }''', dependencies: libvitastor_client)
-+      vitastor = declare_dependency(dependencies: libvitastor_client)
-+    elif get_option('vitastor').enabled()
-+      error('could not link libvitastor_client')
-+    else
-+      warning('could not link libvitastor_client, disabling')
-+    endif
-+  endif
-+endif
-+
- glusterfs = not_found
- glusterfs_ftruncate_has_stat = false
- glusterfs_iocb_has_stat = false
-@@ -2509,6 +2529,7 @@ endif
- config_host_data.set('CONFIG_OPENGL', opengl.found())
- config_host_data.set('CONFIG_PLUGIN', get_option('plugins'))
- config_host_data.set('CONFIG_RBD', rbd.found())
-+config_host_data.set('CONFIG_VITASTOR', vitastor.found())
- config_host_data.set('CONFIG_RDMA', rdma.found())
- config_host_data.set('CONFIG_RELOCATABLE', get_option('relocatable'))
- config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack'))
-@@ -4948,6 +4969,7 @@ summary_info += {'fdt support':       fdt_opt == 'internal' ? 'internal' : fdt}
- summary_info += {'libcap-ng support': libcap_ng}
- summary_info += {'bpf support':       libbpf}
- summary_info += {'rbd support':       rbd}
-+summary_info += {'vitastor support':  vitastor}
- summary_info += {'smartcard support': cacard}
- summary_info += {'U2F support':       u2f}
- summary_info += {'libusb':            libusb}
-diff --git a/meson_options.txt b/meson_options.txt
-index 2836156257..148086cc6f 100644
--- a/meson_options.txt
-+++ b/meson_options.txt
-@@ -206,6 +206,8 @@ option('pvg', type: 'feature', value: 'auto',
-        description: 'macOS paravirtualized graphics support')
- option('rbd', type : 'feature', value : 'auto',
-        description: 'Ceph block device driver')
-+option('vitastor', type : 'feature', value : 'auto',
-+       description: 'Vitastor block device driver')
- option('opengl', type : 'feature', value : 'auto',
-        description: 'OpenGL support')
- option('rdma', type : 'feature', value : 'auto',
-diff --git a/qapi/block-core.json b/qapi/block-core.json
-index b82af74256..f25a6f5ce8 100644
--- a/qapi/block-core.json
-+++ b/qapi/block-core.json
-@@ -3351,7 +3351,7 @@
-             'parallels', 'preallocate', 'qcow', 'qcow2', 'qed', 'quorum',
-             'raw', 'rbd',
-             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
-            'ssh', 'throttle', 'vdi', 'vhdx',
-+            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
-             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
-             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
-             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
-@@ -4434,6 +4434,28 @@
-             '*key-secret': 'str',
-             '*server': ['InetSocketAddressBase'] } }
- 
-+##
-+# @BlockdevOptionsVitastor:
-+#
-+# Driver specific block device options for vitastor
-+#
-+# @image:       Image name
-+# @inode:       Inode number
-+# @pool:        Pool ID
-+# @size:        Desired image size in bytes
-+# @config-path: Path to Vitastor configuration
-+# @etcd-host:   etcd connection address(es)
-+# @etcd-prefix: etcd key/value prefix
-+##
-+{ 'struct': 'BlockdevOptionsVitastor',
-+  'data': { '*inode': 'uint64',
-+            '*pool': 'uint64',
-+            '*size': 'uint64',
-+            '*image': 'str',
-+            '*config-path': 'str',
-+            '*etcd-host': 'str',
-+            '*etcd-prefix': 'str' } }
-+
- ##
- # @ReplicationMode:
- #
-@@ -4902,6 +4924,7 @@
-       'throttle':   'BlockdevOptionsThrottle',
-       'vdi':        'BlockdevOptionsGenericFormat',
-       'vhdx':       'BlockdevOptionsGenericFormat',
-+      'vitastor':   'BlockdevOptionsVitastor',
-       'virtio-blk-vfio-pci':
-                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
-                       'if': 'CONFIG_BLKIO' },
-@@ -5376,6 +5399,20 @@
-             '*cluster-size' :   'size',
-             '*encrypt' :        'RbdEncryptionCreateOptions' } }
- 
-+##
-+# @BlockdevCreateOptionsVitastor:
-+#
-+# Driver specific image creation options for Vitastor.
-+#
-+# @location: Where to store the new image file.  This location cannot
-+#     point to a snapshot.
-+#
-+# @size: Size of the virtual disk in bytes
-+##
-+{ 'struct': 'BlockdevCreateOptionsVitastor',
-+  'data': { 'location':         'BlockdevOptionsVitastor',
-+            'size':             'size' } }
-+
- ##
- # @BlockdevVmdkSubformat:
- #
-@@ -5598,6 +5635,7 @@
-       'ssh':            'BlockdevCreateOptionsSsh',
-       'vdi':            'BlockdevCreateOptionsVdi',
-       'vhdx':           'BlockdevCreateOptionsVhdx',
-+      'vitastor':       'BlockdevCreateOptionsVitastor',
-       'vmdk':           'BlockdevCreateOptionsVmdk',
-       'vpc':            'BlockdevCreateOptionsVpc'
-   } }
-diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh
-index 3d0d132344..65ee8c855e 100644
--- a/scripts/meson-buildoptions.sh
-+++ b/scripts/meson-buildoptions.sh
-@@ -177,6 +177,7 @@ meson_options_help() {
-   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
-   printf "%s\n" '  qpl             Query Processing Library support'
-   printf "%s\n" '  rbd             Ceph block device driver'
-+  printf "%s\n" '  vitastor        Vitastor block device driver'
-   printf "%s\n" '  rdma            Enable RDMA-based migration'
-   printf "%s\n" '  replication     replication support'
-   printf "%s\n" '  rust            Rust support'
-@@ -464,6 +465,8 @@ _meson_option_parse() {
-     --disable-qpl) printf "%s" -Dqpl=disabled ;;
-     --enable-rbd) printf "%s" -Drbd=enabled ;;
-     --disable-rbd) printf "%s" -Drbd=disabled ;;
-+    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
-+    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
-     --enable-rdma) printf "%s" -Drdma=enabled ;;
-     --disable-rdma) printf "%s" -Drdma=disabled ;;
-     --enable-relocatable) printf "%s" -Drelocatable=true ;;
@@ -21,7 +21,7 @@ rpmbuild -bp fio.spec
 cd $VITASTOR
 VER=$(grep ^Version: rpm/vitastor-$REL.spec | awk '{print $2}')
 rm -rf fio
-ln -s $(ls -d ~/rpmbuild/BUILD/fio*/ | grep -v SPECPARTS) fio
+ln -s ~/rpmbuild/BUILD/fio*/ fio
 sh copy-fio-includes.sh
 rm fio
 mv fio-copy fio
@@ -1,17 +0,0 @@
-# Build packages for AlmaLinux 10 inside a container
-# cd ..
-# docker pull --platform=linux/amd64/v2 quay.io/almalinuxorg/almalinux:10
-# docker build -t vitastor-buildenv:el10 -f rpm/vitastor-el10.Dockerfile .
-# docker run -i --rm -v ./:/root/vitastor vitastor-buildenv:el10 /root/vitastor/rpm/vitastor-build.sh
-
-FROM quay.io/almalinuxorg/almalinux:10
-
-WORKDIR /root
-
-RUN sed -i 's/enabled=0/enabled=1/' /etc/yum.repos.d/*.repo
-RUN dnf -y install epel-release dnf-plugins-core
-RUN dnf -y install https://vitastor.io/rpms/centos/10/vitastor-release-1.0-1.el10.noarch.rpm
-RUN dnf -y install gcc-c++ gperftools-devel fio nodejs rpm-build jerasure-devel isa-l-devel gf-complete-devel rdma-core-devel cmake libnl3-devel c-ares-devel
-RUN dnf download --source fio
-RUN rpm --nomd5 -i fio*.src.rpm
-RUN cd ~/rpmbuild/SPECS && dnf builddep -y --spec fio.spec
@@ -1,199 +0,0 @@
-Name:           vitastor
-Version:        3.0.9
-Release:        1%{?dist}
-Summary:        Vitastor, a fast software-defined clustered block storage
-
-License:        Vitastor Network Public License 1.1
-URL:            https://vitastor.io/
-Source0:        vitastor-3.0.9.el10.tar.gz
-
-BuildRequires:  gperftools-devel
-BuildRequires:  gcc-c++
-BuildRequires:  nodejs >= 10
-BuildRequires:  jerasure-devel
-BuildRequires:  isa-l-devel
-BuildRequires:  gf-complete-devel
-BuildRequires:  rdma-core-devel
-BuildRequires:  cmake
-BuildRequires:  libnl3-devel
-BuildRequires:  c-ares-devel
-Requires:       vitastor-osd = %{version}-%{release}
-Requires:       vitastor-mon = %{version}-%{release}
-Requires:       vitastor-client = %{version}-%{release}
-Requires:       vitastor-client-devel = %{version}-%{release}
-Requires:       vitastor-fio = %{version}-%{release}
-
-%description
-Vitastor is a small, simple and fast clustered block storage (storage for VM drives),
-architecturally similar to Ceph which means strong consistency, primary-replication,
-symmetric clustering and automatic data distribution over any number of drives of any
-size with configurable redundancy (replication or erasure codes/XOR).
-
-
-%package -n vitastor-osd
-Summary:        Vitastor - OSD
-Requires:       vitastor-client = %{version}-%{release}
-Requires:       util-linux
-Requires:       parted
-
-
-%description -n vitastor-osd
-Vitastor object storage daemon, i.e. server program that stores data.
-
-
-%package -n vitastor-mon
-Summary:        Vitastor - monitor
-Requires:       nodejs >= 10
-Requires:       lpsolve
-
-
-%description -n vitastor-mon
-Vitastor monitor, i.e. server program responsible for watching cluster state and
-scheduling cluster-level operations.
-
-
-%package -n vitastor-client
-Summary:        Vitastor - client
-
-
-%description -n vitastor-client
-Vitastor client library and command-line interface.
-
-
-%package -n vitastor-client-devel
-Summary:        Vitastor - development files
-Group:          Development/Libraries
-Requires:       vitastor-client = %{version}-%{release}
-
-
-%description -n vitastor-client-devel
-Vitastor library headers for development.
-
-
-%package -n vitastor-fio
-Summary:        Vitastor - fio drivers
-Group:          Development/Libraries
-Requires:       vitastor-client = %{version}-%{release}
-Requires:       fio = 3.36-5.el10
-
-
-%description -n vitastor-fio
-Vitastor fio drivers for benchmarking.
-
-
-%package -n vitastor-opennebula
-Summary:        Vitastor for OpenNebula
-Group:          Development/Libraries
-Requires:       vitastor-client
-Requires:       jq
-Requires:       python3-lxml
-Requires:       patch
-Requires:       qemu-kvm-block-vitastor
-
-
-%description -n vitastor-opennebula
-Vitastor storage plugin for OpenNebula.
-
-
-%prep
-%setup -q
-
-
-%build
-%cmake
-%cmake_build
-
-
-%install
-rm -rf $RPM_BUILD_ROOT
-%cmake_install
-cd mon
-npm install --production
-cd ..
-mkdir -p %buildroot/usr/lib/vitastor
-cp -r mon %buildroot/usr/lib/vitastor
-mv %buildroot/usr/lib/vitastor/mon/scripts/make-etcd %buildroot/usr/lib/vitastor/mon/
-mkdir -p %buildroot/lib/systemd/system
-cp mon/scripts/vitastor.target mon/scripts/vitastor-mon.service mon/scripts/vitastor-osd@.service %buildroot/lib/systemd/system
-mkdir -p %buildroot/lib/udev/rules.d
-cp mon/scripts/90-vitastor.rules %buildroot/lib/udev/rules.d
-mkdir -p %buildroot/var/lib/one
-cp -r opennebula/remotes %buildroot/var/lib/one
-cp opennebula/install.sh %buildroot/var/lib/one/remotes/datastore/vitastor/
-mkdir -p %buildroot/etc/
-cp -r opennebula/sudoers.d %buildroot/etc/
-
-
-%files
-%doc GPL-2.0.txt VNPL-1.1.txt README.md README-ru.md
-
-
-%files -n vitastor-osd
-%_bindir/vitastor-osd
-%_bindir/vitastor-disk
-%_bindir/vitastor-dump-journal
-/lib/systemd/system/vitastor-osd@.service
-/lib/systemd/system/vitastor.target
-/lib/udev/rules.d/90-vitastor.rules
-
-
-%pre -n vitastor-osd
-groupadd -r -f vitastor 2>/dev/null ||:
-useradd -r -g vitastor -s /sbin/nologin -c "Vitastor daemons" -M -d /nonexistent vitastor 2>/dev/null ||:
-install -o vitastor -g vitastor -d /var/log/vitastor
-mkdir -p /etc/vitastor
-
-
-%files -n vitastor-mon
-/usr/lib/vitastor/mon
-/lib/systemd/system/vitastor-mon.service
-
-
-%pre -n vitastor-mon
-groupadd -r -f vitastor 2>/dev/null ||:
-useradd -r -g vitastor -s /sbin/nologin -c "Vitastor daemons" -M -d /nonexistent vitastor 2>/dev/null ||:
-mkdir -p /etc/vitastor
-mkdir -p /var/lib/vitastor
-chown vitastor:vitastor /var/lib/vitastor
-
-
-%files -n vitastor-client
-%_bindir/vitastor-nbd
-%_bindir/vitastor-ublk
-%_bindir/vitastor-nfs
-%_bindir/vitastor-cli
-%_bindir/vitastor-rm
-%_bindir/vitastor-kv
-%_bindir/vitastor-kv-stress
-%_bindir/vita
-%_libdir/libvitastor_client.so*
-%_libdir/libvitastor_kv.so*
-
-
-%files -n vitastor-client-devel
-%_includedir/vitastor_c.h
-%_includedir/vitastor_kv.h
-%_libdir/pkgconfig
-
-
-%files -n vitastor-fio
-%_libdir/libfio_vitastor.so
-%_libdir/libfio_vitastor_blk.so
-%_libdir/libfio_vitastor_sec.so
-
-
-%files -n vitastor-opennebula
-/var/lib/one
-/etc/sudoers.d/opennebula-vitastor
-
-
-%triggerin -n vitastor-opennebula -- opennebula
-[ $2 = 0 ] || exit 0
-/var/lib/one/remotes/datastore/vitastor/install.sh
-
-
-# Turn off the brp-python-bytecompile script
-%global __os_install_post %(echo '%{__os_install_post}' | sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g')
-
-
-%changelog
@@ -15,7 +15,7 @@ RUN yum -y --enablerepo=extras install centos-release-scl epel-release yum-utils
 RUN perl -i -pe 's!mirrorlist=!#mirrorlist=!s; s!#\s*baseurl=http://mirror.centos.org!baseurl=http://vault.centos.org!' /etc/yum.repos.d/CentOS-SCLo-scl*.repo
 RUN yum -y install https://vitastor.io/rpms/centos/7/vitastor-release-1.0-1.el7.noarch.rpm
 RUN yum -y install devtoolset-9-gcc-c++ devtoolset-9-libatomic-devel gcc make cmake gperftools-devel \
-    fio rh-nodejs12 jerasure-devel libisa-l-devel gf-complete-devel rdma-core-devel libnl3-devel c-ares-devel
+    fio rh-nodejs12 jerasure-devel libisa-l-devel gf-complete-devel rdma-core-devel libnl3-devel
 RUN yumdownloader --disablerepo=centos-sclo-rh --source fio
 RUN rpm --nomd5 -i fio*.src.rpm
 RUN rm -f /etc/yum.repos.d/CentOS-Media.repo
@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.9
+Version:        2.4.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage

 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.9.el7.tar.gz
+Source0:        vitastor-2.4.3.el7.tar.gz

 BuildRequires:  gperftools-devel
 BuildRequires:  devtoolset-9-gcc-c++
@@ -17,7 +17,6 @@ BuildRequires:  gf-complete-devel
 BuildRequires:  rdma-core-devel
 BuildRequires:  cmake3
 BuildRequires:  libnl3-devel
-BuildRequires:  c-ares-devel
 Requires:       vitastor-osd = %{version}-%{release}
 Requires:       vitastor-mon = %{version}-%{release}
 Requires:       vitastor-client = %{version}-%{release}
@@ -172,6 +171,7 @@ chown vitastor:vitastor /var/lib/vitastor
 %_bindir/vitastor-kv
 %_bindir/vitastor-kv-stress
 %_bindir/vita
+%_libdir/libvitastor_blk.so*
 %_libdir/libvitastor_client.so*
 %_libdir/libvitastor_kv.so*

@@ -13,7 +13,7 @@ RUN dnf -y install centos-release-advanced-virtualization epel-release dnf-plugi
 RUN sed -i 's/^mirrorlist=/#mirrorlist=/; s!#baseurl=.*!baseurl=http://vault.centos.org/centos/8.4.2105/virt/$basearch/$avdir/!; s!^baseurl=.*Source/.*!baseurl=http://vault.centos.org/centos/8.4.2105/virt/Source/advanced-virtualization/!' /etc/yum.repos.d/CentOS-Advanced-Virtualization.repo
 RUN yum -y install https://vitastor.io/rpms/centos/8/vitastor-release-1.0-1.el8.noarch.rpm
 RUN dnf -y install gcc-toolset-9 gcc-toolset-9-gcc-c++ gperftools-devel \
-    fio nodejs rpm-build jerasure-devel libisa-l-devel gf-complete-devel libibverbs-devel libarchive cmake libnl3-devel c-ares-devel
+    fio nodejs rpm-build jerasure-devel libisa-l-devel gf-complete-devel libibverbs-devel libarchive cmake libnl3-devel
 RUN dnf download --source fio
 RUN rpm --nomd5 -i fio*.src.rpm
 RUN cd ~/rpmbuild/SPECS && dnf builddep -y --enablerepo=powertools --spec fio.spec
@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.9
+Version:        2.4.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage

 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.9.el8.tar.gz
+Source0:        vitastor-2.4.3.el8.tar.gz

 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-toolset-9-gcc-c++
@@ -16,7 +16,6 @@ BuildRequires:  gf-complete-devel
 BuildRequires:  rdma-core-devel
 BuildRequires:  cmake
 BuildRequires:  libnl3-devel
-BuildRequires:  c-ares-devel
 Requires:       vitastor-osd = %{version}-%{release}
 Requires:       vitastor-mon = %{version}-%{release}
 Requires:       vitastor-client = %{version}-%{release}
@@ -169,6 +168,7 @@ chown vitastor:vitastor /var/lib/vitastor
 %_bindir/vitastor-kv
 %_bindir/vitastor-kv-stress
 %_bindir/vita
+%_libdir/libvitastor_blk.so*
 %_libdir/libvitastor_client.so*
 %_libdir/libvitastor_kv.so*

@@ -10,7 +10,7 @@ WORKDIR /root
 RUN sed -i 's/enabled=0/enabled=1/' /etc/yum.repos.d/*.repo
 RUN dnf -y install epel-release dnf-plugins-core
 RUN dnf -y install https://vitastor.io/rpms/centos/9/vitastor-release-1.0-1.el9.noarch.rpm
-RUN dnf -y install gcc-c++ gperftools-devel fio nodejs rpm-build jerasure-devel libisa-l-devel gf-complete-devel rdma-core-devel libarchive cmake libnl3-devel c-ares-devel
+RUN dnf -y install gcc-c++ gperftools-devel fio nodejs rpm-build jerasure-devel libisa-l-devel gf-complete-devel rdma-core-devel libarchive cmake libnl3-devel
 RUN dnf download --source fio
 RUN rpm --nomd5 -i fio*.src.rpm
 RUN cd ~/rpmbuild/SPECS && dnf builddep -y --spec fio.spec
@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.9
+Version:        2.4.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage

 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.9.el9.tar.gz
+Source0:        vitastor-2.4.3.el9.tar.gz

 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++
@@ -16,7 +16,6 @@ BuildRequires:  gf-complete-devel
 BuildRequires:  rdma-core-devel
 BuildRequires:  cmake
 BuildRequires:  libnl3-devel
-BuildRequires:  c-ares-devel
 Requires:       vitastor-osd = %{version}-%{release}
 Requires:       vitastor-mon = %{version}-%{release}
 Requires:       vitastor-client = %{version}-%{release}
@@ -166,6 +165,7 @@ chown vitastor:vitastor /var/lib/vitastor
 %_bindir/vitastor-kv
 %_bindir/vitastor-kv-stress
 %_bindir/vita
+%_libdir/libvitastor_blk.so*
 %_libdir/libvitastor_client.so*
 %_libdir/libvitastor_kv.so*

@@ -1,8 +1,9 @@
-cmake_minimum_required(VERSION 2.8...3.30)
+cmake_minimum_required(VERSION 2.8.12)

 project(vitastor)

 include(GNUInstallDirs)
+include(CTest)
 include(CheckIncludeFile)

 find_package(PkgConfig)
@@ -20,7 +21,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 endif()
 set(ENABLE_COVERAGE false CACHE BOOL "Enable code coverage")

-add_definitions(-DVITASTOR_VERSION="3.0.9")
+add_definitions(-DVITASTOR_VERSION="2.4.3")
 add_definitions(-D_GNU_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -fvisibility=hidden -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})
@@ -74,14 +75,6 @@ if (RDMACM_LIBRARIES)
 	add_definitions(-DWITH_RDMACM)
 endif (RDMACM_LIBRARIES)

-find_package(OpenSSL REQUIRED)
-if (OPENSSL_FOUND)
-	add_definitions(-DWITH_OPENSSL)
-endif (OPENSSL_FOUND)
-
-pkg_check_modules(CARES REQUIRED libcares)
-include_directories(${CARES_INCLUDE_DIRS})
-
 if (${WITH_SYSTEM_LIBURING})
 	pkg_check_modules(LIBURING REQUIRED liburing>=2.10)
 	include_directories(${LIBURING_INCLUDE_DIRS})
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8...3.30)
+cmake_minimum_required(VERSION 2.8.12)

 project(vitastor)

@@ -183,11 +183,6 @@ public:
    // Update configuration
    virtual void parse_config(blockstore_config_t & config) = 0;

-    // Reshard database for a pool in chunks
-    // MUST be called only when nobody makes any modifications to the DB for this pool
-    virtual void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit) = 0;
-    virtual bool reshard_continue(void *reshard_state, uint64_t chunk_limit) = 0;
-
    // Event loop
    virtual void loop() = 0;

@@ -228,9 +223,4 @@ public:
    virtual uint64_t get_journal_size() = 0;

    virtual uint32_t get_bitmap_granularity() = 0;
-
-    virtual uint64_t get_live_entries() = 0;
-    virtual uint64_t get_live_memory() = 0;
-    virtual uint64_t get_garbage_entries() = 0;
-    virtual uint64_t get_garbage_memory() = 0;
 };
@@ -48,12 +48,11 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
    disk_alignment = parse_size(config["disk_alignment"]);
    journal_block_size = parse_size(config["journal_block_size"]);
    meta_block_size = parse_size(config["meta_block_size"]);
+    meta_block_target_free_space = parse_size(config["meta_block_target_free_space"]);
    bitmap_granularity = parse_size(config["bitmap_granularity"]);
    meta_format = stoull_full(config["meta_format"]);
    atomic_write_size = (config.find("atomic_write_size") != config.end()
        ? parse_size(config["atomic_write_size"]) : 4096);
-    use_atomic_flag = config.find("use_atomic_flag") != config.end() &&
-        (config["use_atomic_flag"] == "true" || config["use_atomic_flag"] == "1" || config["use_atomic_flag"] == "yes");
    if (config.find("data_io") == config.end() &&
        config.find("meta_io") == config.end() &&
        config.find("journal_io") == config.end())
@@ -83,23 +82,17 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
    {
        data_csum_type = BLOCKSTORE_CSUM_CRC32C;
    }
-    else if (config["data_csum_type"] == "xxh3_32")
-    {
-        data_csum_type = BLOCKSTORE_CSUM_XXH3_32;
-    }
    else if (config["data_csum_type"] == "" || config["data_csum_type"] == "none")
    {
        data_csum_type = BLOCKSTORE_CSUM_NONE;
    }
    else
    {
-        throw std::runtime_error("data_csum_type="+config["data_csum_type"]+" is unsupported, only \"crc32c\", \"xxh3_32\" and \"none\" are supported");
+        throw std::runtime_error("data_csum_type="+config["data_csum_type"]+" is unsupported, only \"crc32c\" and \"none\" are supported");
    }
    csum_block_size = parse_size(config["csum_block_size"]);
    discard_on_start = config.find("discard_on_start") != config.end() &&
        (config["discard_on_start"] == "true" || config["discard_on_start"] == "1" || config["discard_on_start"] == "yes");
-    gc_on_start = config.find("gc_on_start") == config.end() ||
-        (config["gc_on_start"] == "true" || config["gc_on_start"] == "1" || config["gc_on_start"] == "yes");
    min_discard_size = parse_size(config["min_discard_size"]);
    if (!min_discard_size)
        min_discard_size = 1024*1024;
@@ -161,6 +154,14 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
    {
        throw std::runtime_error("meta_block_size must not exceed "+std::to_string(MAX_DATA_BLOCK_SIZE));
    }
+    if (!meta_block_target_free_space)
+    {
+        meta_block_target_free_space = 800;
+    }
+    if (meta_block_target_free_space >= meta_block_size)
+    {
+        throw std::runtime_error("meta_block_target_free_space must not exceed "+std::to_string(meta_block_size));
+    }
    if (data_offset % disk_alignment)
    {
        throw std::runtime_error("data_offset must be a multiple of disk_alignment = "+std::to_string(disk_alignment));
@@ -177,10 +178,6 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
    {
        throw std::runtime_error("Data block size must be a multiple of sparse write tracking granularity");
    }
-    if (data_block_size / bitmap_granularity < 8)
-    {
-        throw std::runtime_error("Data block size must be at least bitmap_granularity*8");
-    }
    if (!data_csum_type)
    {
        csum_block_size = 0;
@@ -213,6 +210,10 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
    {
        throw std::runtime_error("journal_offset must be a multiple of journal_block_size = "+std::to_string(journal_block_size));
    }
+    if (!meta_format)
+    {
+        meta_format = BLOCKSTORE_META_FORMAT_HEAP;
+    }
    if (meta_device == data_device)
    {
        disable_meta_fsync = disable_data_fsync;
@@ -223,7 +224,7 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
    }
 }

-void blockstore_disk_t::calc_lengths(bool skip_meta_check)
+void blockstore_disk_t::calc_lengths()
 {
    // data
    data_len = data_device_size - data_offset;
@@ -269,13 +270,13 @@ void blockstore_disk_t::calc_lengths(bool skip_meta_check)
    }
    // required metadata size
    block_count = data_len / data_block_size;
-    clean_entry_bitmap_size = (data_block_size / bitmap_granularity + 7) / 8;
+    clean_entry_bitmap_size = data_block_size / bitmap_granularity / 8;
    clean_dyn_size = clean_entry_bitmap_size*2 + (csum_block_size
        ? data_block_size/csum_block_size*(data_csum_type & 0xFF) : 0);
-recalc:
    if (meta_format == BLOCKSTORE_META_FORMAT_HEAP)
    {
-        uint32_t entries_per_block = meta_block_size / (sizeof(heap_big_write_t) + clean_dyn_size);
+        uint32_t entries_per_block = ((meta_block_size-meta_block_target_free_space) /
+            (sizeof(heap_object_t) + sizeof(heap_write_t) + clean_dyn_size));
        min_meta_len = (block_count+entries_per_block-1) / entries_per_block * meta_block_size;
    }
    else if (meta_format == BLOCKSTORE_META_FORMAT_V1)
@@ -283,32 +284,11 @@ recalc:
        clean_entry_size = 24 /*sizeof(clean_disk_entry)*/ + 2*clean_entry_bitmap_size;
        min_meta_len = (1 + (block_count - 1 + meta_block_size / clean_entry_size)
            / (meta_block_size / clean_entry_size)) * meta_block_size;
-        if (!skip_meta_check && meta_area_size < min_meta_len)
-        {
-too_small:
-            throw std::runtime_error("Metadata area is too small, need at least "+std::to_string(min_meta_len)+
-                " bytes, have only "+std::to_string(meta_area_size)+" bytes");
-        }
    }
-    else if (meta_format == BLOCKSTORE_META_FORMAT_V2 || !meta_format)
+    else if (meta_format == BLOCKSTORE_META_FORMAT_V2)
    {
-        meta_format = BLOCKSTORE_META_FORMAT_V2;
        clean_entry_size = 24 /*sizeof(clean_disk_entry)*/ + clean_dyn_size + 4 /*entry_csum*/;
        min_meta_len = (1 + (block_count - 1 + meta_block_size / clean_entry_size) / (meta_block_size / clean_entry_size)) * meta_block_size;
-        if (!skip_meta_check && meta_area_size < min_meta_len)
-        {
-            if (!data_csum_type)
-            {
-                printf("Warning: Using old metadata format without checksums because the new format"
-                    " doesn't fit into provided area (%ju bytes required, %ju bytes available)\n", min_meta_len, meta_area_size);
-                meta_format = BLOCKSTORE_META_FORMAT_V1;
-                goto recalc;
-            }
-            else
-            {
-                goto too_small;
-            }
-        }
    }
    else
    {
@@ -16,7 +16,6 @@
 #define BLOCKSTORE_CSUM_NONE 0
 // Lower byte of checksum type is its length
 #define BLOCKSTORE_CSUM_CRC32C 0x104
-#define BLOCKSTORE_CSUM_XXH3_32 0x204

 #define MOCK_DATA_FD 1000
 #define MOCK_META_FD 1001
@@ -27,18 +26,18 @@ class allocator_t;
 struct blockstore_disk_t
 {
    std::string data_device, meta_device, journal_device;
-    uint64_t data_block_size;
+    uint32_t data_block_size;
    uint64_t cfg_journal_size, cfg_data_size;
    // Required write alignment and journal/metadata/data areas' location alignment
    uint32_t disk_alignment = 4096;
    // Journal block size - minimum_io_size of the journal device is the best choice
-    uint64_t journal_block_size = 4096;
+    uint32_t journal_block_size = 4096;
    // Metadata block size - minimum_io_size of the metadata device is the best choice
-    uint64_t meta_block_size = 4096;
+    uint32_t meta_block_size = 4096;
    // Atomic write size of the data block device
    uint32_t atomic_write_size = 4096;
-    // Whether we should set RWF_ATOMIC on atomic writes
-    bool use_atomic_flag = false;
+    // Target free space in metadata blocks
+    uint32_t meta_block_target_free_space = 800;
    // Sparse write tracking granularity. 4 KB is a good choice. Must be a multiple of disk_alignment
    uint32_t bitmap_granularity = 4096;
    // Data checksum type, BLOCKSTORE_CSUM_NONE or BLOCKSTORE_CSUM_CRC32C
@@ -58,16 +57,13 @@ struct blockstore_disk_t
    bool inmemory_journal = true;
    // Data discard granularity and minimum size (for the sake of performance)
    bool discard_on_start = false;
-    // GC on start (new store)
-    bool gc_on_start = true;
    uint64_t min_discard_size = 1024*1024;
    uint64_t discard_granularity = 0;

    int meta_fd = -1, data_fd = -1, journal_fd = -1;
-    uint64_t meta_offset = 0, meta_device_sect = 0, meta_device_size = 0, meta_area_size = 0, min_meta_len = 0;
-    uint64_t data_offset = 0, data_device_sect = 0, data_device_size = 0, data_len = 0;
-    uint64_t journal_offset = 0, journal_device_sect = 0, journal_device_size = 0, journal_len = 0;
-    uint64_t meta_format = 0;
+    uint64_t meta_offset, meta_device_sect, meta_device_size, meta_area_size, min_meta_len, meta_format = 0;
+    uint64_t data_offset, data_device_sect, data_device_size, data_len;
+    uint64_t journal_offset, journal_device_sect, journal_device_size, journal_len;

    uint64_t block_count = 0;
    uint32_t clean_entry_bitmap_size = 0;
@@ -79,7 +75,7 @@ struct blockstore_disk_t
    void open_data();
    void open_meta();
    void open_journal();
-    void calc_lengths(bool skip_meta_check = false);
+    void calc_lengths();
    void check_lengths();
    void close_all();
    int trim_data(std::function<bool(uint64_t)> is_free);
@@ -18,18 +18,12 @@ journal_flusher_t::journal_flusher_t(blockstore_impl_t *bs)
    this->cur_flusher_count = bs->min_flusher_count;
    this->target_flusher_count = bs->min_flusher_count;
    active_flushers = 0;
+    advance_lsn_counter = 0;
    co = new journal_flusher_co[max_flusher_count];
    for (int i = 0; i < max_flusher_count; i++)
    {
        co[i].co_id = i;
        co[i].bs = bs;
-        co[i].new_bmp = (uint8_t*)malloc_or_die(3*bs->dsk.clean_entry_bitmap_size);
-        co[i].new_ext_bmp = co[i].new_bmp + bs->dsk.clean_entry_bitmap_size;
-        co[i].punch_bmp = co[i].new_bmp + 2*bs->dsk.clean_entry_bitmap_size;
-        if (bs->dsk.csum_block_size > 0)
-        {
-            co[i].new_csums = (uint8_t*)malloc_or_die(bs->dsk.data_block_size / bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF));
-        }
        co[i].flusher = this;
    }
 }
@@ -60,18 +54,6 @@ journal_flusher_t::~journal_flusher_t()

 journal_flusher_co::~journal_flusher_co()
 {
-    if (new_csums)
-    {
-        free(new_csums);
-        new_csums = NULL;
-    }
-    if (new_bmp)
-    {
-        free(new_bmp);
-        new_bmp = NULL;
-    }
-    new_ext_bmp = NULL;
-    punch_bmp = NULL;
    free_buffers();
 }

@@ -80,6 +62,11 @@ int journal_flusher_t::get_syncing_buffer()
    return syncing_buffer;
 }

+uint64_t journal_flusher_t::get_compact_counter()
+{
+    return compact_counter;
+}
+
 bool journal_flusher_t::is_active()
 {
    return active_flushers > 0;
@@ -99,10 +86,9 @@ void journal_flusher_t::release_trim()
 void journal_flusher_t::dump_diagnostics()
 {
    printf(
-        "Compaction queue: %u items + %u future, data: %ju/%ju blocks used, buffer: %ju/%ju bytes used, meta: %ju/%ju bytes used, %u/%ju blocks nearfull\n",
-        bs->heap->get_compact_queue_size(), bs->heap->get_to_compact_count(),
+        "Compaction queue: %u items, data: %ju/%ju blocks used, meta: %ju/%ju bytes used, %u/%ju blocks nearfull\n",
+        bs->heap->get_to_compact_count(),
        bs->heap->get_data_used_space()/bs->dsk.data_block_size, bs->dsk.block_count,
-        bs->heap->get_buffer_area_used_space(), bs->dsk.journal_len,
        bs->heap->get_meta_used_space(), bs->heap->get_meta_total_space(),
        bs->heap->get_meta_nearfull_blocks(), bs->dsk.meta_area_size/bs->dsk.meta_block_size-1
    );
@@ -127,12 +113,8 @@ void journal_flusher_t::loop()
        }
    }
    int prev_active = active_flushers;
-    for (int i = 0; (active_flushers > 0 || force_start > 0 ||
-        bs->heap->get_compact_queue_size() > bs->flusher_start_threshold ||
-        i == 0 && bs->intent_write_counter >= bs->journal_trim_interval) && i < cur_flusher_count; i++)
-    {
+    for (int i = 0; (active_flushers > 0 || force_start > 0 || bs->heap->get_to_compact_count() > bs->flusher_start_threshold) && i < cur_flusher_count; i++)
        co[i].loop();
-    }
    if (prev_active && !active_flushers && force_start > 0)
        bs->ringloop->wakeup();
 }
@@ -174,33 +156,22 @@ bool journal_flusher_co::loop()
    else if (wait_state == 19) goto resume_19;
    else if (wait_state == 20) goto resume_20;
    else if (wait_state == 21) goto resume_21;
+    else if (wait_state == 22) goto resume_22;
+    else if (wait_state == 23) goto resume_23;
+    else if (wait_state == 24) goto resume_24;
 resume_0:
    wait_state = 0;
    wait_count = 0;
    cur_oid = {};
    res = bs->heap->get_next_compact(cur_oid);
-    // Advance fsynced_lsn every <journal_trim_interval> intent writes
-    if ((bs->intent_write_counter >= bs->journal_trim_interval) && co_id == 0)
-    {
-        bs->intent_write_counter = 0;
-resume_17:
-resume_18:
-resume_19:
-        if (!fsync_buffer(17))
-            return false;
-resume_20:
-resume_21:
-        if (!trim_lsn(20))
-            return false;
-    }
    if (res == ENOENT && flusher->force_start > 0 && co_id == 0 &&
-        (!bs->dsk.disable_journal_fsync || !bs->dsk.disable_meta_fsync || !bs->dsk.disable_data_fsync))
+        (!bs->dsk.disable_journal_fsync || !bs->dsk.disable_meta_fsync))
    {
        flusher->active_flushers++;
-resume_14:
-resume_15:
-resume_16:
-        if (!fsync_buffer(14))
+resume_21:
+resume_22:
+        res = fsync_buffer(21);
+        if (!res)
        {
            return false;
        }
@@ -209,94 +180,61 @@ resume_16:
    }
    if (res == ENOENT)
    {
+        if (co_id == 0 && flusher->force_start > 0)
+        {
+            flusher->active_flushers++;
+resume_16:
+resume_17:
+resume_18:
+resume_19:
+resume_20:
+            if (!trim_lsn(16))
+                return false;
+            flusher->active_flushers--;
+        }
        cur_oid = {};
        wait_state = 0;
        return true;
    }
-    if (flusher->flushing.find(cur_oid) != flusher->flushing.end())
+    for (int i = 0; i < flusher->cur_flusher_count; i++)
    {
-        for (int i = 0; i < flusher->cur_flusher_count; i++)
+        if (i != co_id && flusher->co[i].cur_oid == cur_oid)
        {
-            if (i != co_id && flusher->co[i].cur_oid == cur_oid)
-            {
-                // Already flushing it
-                flusher->co[i].should_repeat = true;
-                goto resume_0;
-            }
+            // Already flushing it
+            flusher->co[i].should_repeat = true;
+            goto resume_0;
        }
-        assert(false);
    }
-    flusher->flushing.insert(cur_oid);
 resume_1:
-    wait_state = 1;
    should_repeat = false;
-    cur_obj = bs->heap->lock_and_read_entry(cur_oid);
+    cur_obj = bs->heap->lock_and_read_entry(cur_oid, copy_id);
    if (!cur_obj)
    {
        // Object does not exist
-        flusher->flushing.erase(cur_oid);
        goto resume_0;
    }
+    cur_version = cur_obj->get_writes()->version;
+    // Find the range to compact
+    compact_lsn = bs->heap->get_fsynced_lsn();
+    bs->heap->get_compact_range(cur_obj, compact_lsn, &begin_wr, &end_wr);
+    if (!begin_wr)
+    {
+        // Nothing to flush
+        bs->heap->unlock_entry(cur_oid, copy_id);
+        goto resume_0;
+    }
+    assert(!end_wr->next() && end_wr->entry_type == (BS_HEAP_BIG_WRITE|BS_HEAP_STABLE));
+    clean_loc = end_wr->big_location(bs->heap);
+    if (bs->log_level > 10)
+        printf("Compacting %jx:%jx l%ju .. l%ju (last l%ju)\n", cur_oid.inode, cur_oid.stripe, end_wr->lsn, begin_wr->lsn, compact_lsn);
+    flusher->active_flushers++;
    // Scan versions to flush
    free_buffers();
    copy_count = 0;
-    fsynced_lsn = bs->heap->get_fsynced_lsn();
-    bitmap_copied = false;
-    memset(new_bmp, 0, bs->dsk.clean_entry_bitmap_size);
-    csum_copy.clear();
-    compact_info = bs->heap->iterate_compaction(cur_obj, fsynced_lsn, flusher->force_start, [&](heap_entry_t *wr)
+    for (auto wr = begin_wr; wr != end_wr; wr = wr->next())
    {
-        if (!bitmap_copied)
-        {
-            memcpy(new_ext_bmp, wr->get_ext_bitmap(bs->heap), bs->dsk.clean_entry_bitmap_size);
-            bitmap_copied = true;
-        }
-        bitmap_set(new_bmp, wr->small().offset, wr->small().len, bs->dsk.bitmap_granularity);
-        if (bs->dsk.csum_block_size && bs->dsk.csum_block_size <= bs->dsk.bitmap_granularity)
-        {
-            csum_copy.push_back(wr);
-        }
-        if (wr->type() == BS_HEAP_SMALL_WRITE ||
-            wr->type() == BS_HEAP_INTENT_WRITE && bs->dsk.csum_block_size > bs->dsk.bitmap_granularity)
-        {
-            auto res = bs->prepare_read(read_vec, cur_obj, wr, 0, bs->dsk.data_block_size,
-                wr->type() == BS_HEAP_INTENT_WRITE && bs->dsk.csum_block_size > bs->dsk.bitmap_granularity && !bs->perfect_csum_update
-                ? COPY_BUF_SKIP_CSUM : 0);
-            if (res > 0)
-                copy_count++;
-        }
-    });
-    if (!compact_info.compact_lsn)
-    {
-        // Flushing is aborted
-        flusher->flushing.erase(cur_oid);
-        bs->heap->unlock_entry(cur_oid);
-        goto resume_0;
-    }
-    mem_or(new_bmp, compact_info.clean_wr->get_int_bitmap(bs->heap), bs->dsk.clean_entry_bitmap_size);
-    if (!bitmap_copied)
-    {
-        memcpy(new_ext_bmp, compact_info.clean_wr->get_ext_bitmap(bs->heap), bs->dsk.clean_entry_bitmap_size);
-        bitmap_copied = true;
-    }
-    if (bs->dsk.csum_block_size && bs->dsk.csum_block_size <= bs->dsk.bitmap_granularity)
-    {
-        memcpy(new_csums, compact_info.clean_wr->get_checksums(bs->heap), bs->dsk.data_block_size/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF));
-        for (size_t i = csum_copy.size(); i > 0; i--)
-        {
-            auto wr = csum_copy[i-1];
-            memcpy(new_csums + wr->small().offset/bs->dsk.csum_block_size*(bs->dsk.data_csum_type & 0xFF),
-                wr->get_checksums(bs->heap), wr->small().len/bs->dsk.csum_block_size*(bs->dsk.data_csum_type & 0xFF));
-        }
-        csum_copy.clear();
-    }
-    clean_loc = compact_info.clean_wr->big_location(bs->heap);
-    flusher->active_flushers++;
-    if (bs->log_level > 10)
-    {
-        printf("Compacting %jx:%jx v%ju..v%ju / l%ju..l%ju (%d writes)\n", cur_oid.inode, cur_oid.stripe,
-            compact_info.clean_wr->version, compact_info.compact_version,
-            compact_info.clean_wr->lsn, compact_info.compact_lsn, copy_count);
+        bs->prepare_read(read_vec, cur_obj, wr, 0, bs->dsk.data_block_size);
+        copy_count++;
    }
    overwrite_start = overwrite_end = 0;
    if (read_vec.size() > 0)
@@ -316,6 +254,7 @@ resume_1:
    }
    // Read buffered data
    cur_obj = NULL;
+    begin_wr = end_wr = NULL;
 resume_2:
 resume_3:
    if (!read_buffered(2))
@@ -333,49 +272,29 @@ resume_3:
        flusher->wanting_meta_fsync--;
    }
    res = check_and_punch_checksums();
-    if (res == ENOENT || res == EDOM)
-    {
-        // Abort compaction
-        flusher->flushing.erase(cur_oid);
-        bs->heap->unlock_entry(cur_oid);
-        flusher->active_flushers--;
-        goto resume_0;
-    }
    if (res == EBUSY)
    {
 resume_4:
-        modified_block = UINT32_MAX;
-        res = bs->heap->punch_holes(compact_info.clean_wr, punch_bmp, new_csums, &modified_block);
-        if (res == ENOENT)
-        {
-            // Abort compaction
-            flusher->flushing.erase(cur_oid);
-            bs->heap->unlock_entry(cur_oid);
-            flusher->active_flushers--;
-            goto resume_0;
-        }
-        if (res == EAGAIN)
-        {
-            // Retry, block is busy
-            wait_state = 4;
-            return false;
-        }
-        assert(res == 0);
 resume_5:
-resume_6:
-        if (!write_meta_block(5))
+        if (!write_meta_block(4))
        {
            return false;
        }
+resume_6:
 resume_7:
 resume_8:
-resume_9:
-        if (!fsync_meta(7))
+        if (!fsync_meta(6))
        {
            return false;
        }
        res = 0;
    }
+    else if (res == ENOENT || res == EDOM)
+    {
+        // Abort compaction
+        flusher->active_flushers--;
+        goto resume_0;
+    }
    assert(res == 0);
    // Submit data writes
    for (i = 0; i < read_vec.size(); i++)
@@ -385,7 +304,7 @@ resume_9:
            (read_vec[i].copy_flags & COPY_BUF_PADDED)) // FIXME Shit, simplify these flags
        {
            assert(read_vec[i].buf);
-            await_sqe(10);
+            await_sqe(9);
            data->iov = (struct iovec){ read_vec[i].buf + (read_vec[i].copy_flags & COPY_BUF_PADDED
                ? read_vec[i].offset - read_vec[i].disk_offset : 0), (size_t)read_vec[i].len };
            data->callback = simple_callback_w;
@@ -393,57 +312,67 @@ resume_9:
            wait_count++;
        }
    }
-resume_11:
+resume_10:
    if (wait_count > 0)
    {
-        wait_state = 11;
+        wait_state = 10;
        return false;
    }
    // Lock is only needed to prevent freeing the big_write because we overwrite it...
-    bs->heap->unlock_entry(cur_oid);
+    bs->heap->unlock_entry(cur_oid, copy_id);
    // Mark the object compacted, but don't free and remove small_writes
    // We'll free and remove them only when trimming
    // The only thing we modify here are big_write block checksums if >4k block is used
-    cur_obj = bs->heap->read_entry(cur_oid);
+    cur_obj = bs->heap->read_entry(cur_oid, &modified_block);
    if (!cur_obj)
    {
        // Abort compaction
-        flusher->flushing.erase(cur_oid);
        goto resume_0;
    }
    if (!calc_block_checksums())
    {
        // Abort compaction
-        flusher->flushing.erase(cur_oid);
        goto resume_0;
    }
-    res = bs->heap->add_compact(cur_obj, compact_info.compact_version, compact_info.compact_lsn, clean_loc,
-        compact_info.do_delete, &modified_block, new_bmp, new_ext_bmp, new_csums);
-    if (res == EBUSY)
+    if (read_to_fill_incomplete)
    {
-        // Abort compaction, object is already overwritten by something else
-        flusher->flushing.erase(cur_oid);
-        goto resume_0;
-    }
-    assert(res == 0);
-resume_12:
-resume_13:
-    if (!write_meta_block(12))
-    {
-        return false;
+resume_23:
+resume_24:
+        if (!write_meta_block(23))
+        {
+            return false;
+        }
+        // Recheck the object because it could be invalidated again
+        cur_obj = bs->heap->read_entry(cur_oid, &modified_block);
+        if (!cur_obj)
+        {
+            // Abort compaction
+            goto resume_0;
+        }
    }
+    bs->heap->mark_object_compacted(cur_obj, compact_lsn);
    // Done
    if (bs->log_level > 10)
-    {
-        printf("Compacted %jx:%jx l%ju (%d writes)\n", cur_oid.inode, cur_oid.stripe, compact_info.compact_lsn, copy_count);
-    }
+        printf("Compacted %jx:%jx l%ju (%d writes)\n", cur_oid.inode, cur_oid.stripe, compact_lsn, copy_count);
+    flusher->compact_counter++;
    flusher->active_flushers--;
+    // Advance compacted_lsn every <journal_trim_interval> objects
+    if (co_id == 0 && !((++flusher->advance_lsn_counter) % bs->journal_trim_interval))
+    {
+        flusher->advance_lsn_counter = 0;
+resume_11:
+resume_12:
+resume_13:
+resume_14:
+resume_15:
+        if (!trim_lsn(11))
+            return false;
+    }
    if (should_repeat)
    {
        // Flush the same object again
        goto resume_1;
    }
-    flusher->flushing.erase(cur_oid);
    // All done
    goto resume_0;
 }
@@ -480,11 +409,11 @@ void journal_flusher_co::fill_partial_checksum_blocks()
                .copy_flags = COPY_BUF_DATA | copy_flags,
                .offset = blk_begin,
                .len = blk_end - blk_begin,
-                .disk_loc = clean_loc,
+                .disk_loc = end_wr->big_location(bs->heap),
                .disk_offset = blk_begin,
                .disk_len = blk_end - blk_begin,
                .buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, blk_end - blk_begin),
-                .wr = compact_info.clean_wr,
+                .wr_lsn = end_wr->lsn,
            });
        }
        auto & vec = read_vec[read_vec.size()-1];
@@ -522,28 +451,26 @@ int journal_flusher_co::check_and_punch_checksums()
        return 0;
    }
    // Verify data checksums
-    cur_obj = bs->heap->read_entry(cur_oid);
-    if (!cur_obj)
-    {
-        // Object is deleted, abort compaction
-        return ENOENT;
-    }
+    cur_obj = bs->heap->read_locked_entry(cur_oid, copy_id);
    bool csum_ok = true;
    for (int i = 0; i < read_vec.size(); i++)
    {
        auto & vec = read_vec[i];
        if (!(vec.copy_flags & (COPY_BUF_COALESCED|COPY_BUF_ZERO|COPY_BUF_SKIP_CSUM)))
        {
-            uint32_t *csums = (uint32_t*)(vec.wr->get_checksums(bs->heap)
+            heap_write_t *wr = cur_obj->get_writes();
+            while (wr && wr->lsn != vec.wr_lsn)
+                wr = wr->next();
+            assert(wr);
+            uint32_t *csums = (uint32_t*)(wr->get_checksums(bs->heap)
                + (vec.disk_offset/bs->dsk.csum_block_size)*(bs->dsk.data_csum_type & 0xFF)
-                - ((vec.wr->type() == BS_HEAP_BIG_WRITE || vec.wr->type() == BS_HEAP_BIG_INTENT)
-                    ? 0 : (vec.wr->small().offset/bs->dsk.csum_block_size)*(bs->dsk.data_csum_type & 0xFF)));
+                - ((wr->type() == BS_HEAP_BIG_WRITE) ? 0 : (wr->small().offset/bs->dsk.csum_block_size)*(bs->dsk.data_csum_type & 0xFF)));
            bs->heap->calc_block_checksums(
-                csums, vec.buf, vec.wr->get_int_bitmap(bs->heap), vec.disk_offset, vec.disk_offset+vec.disk_len, false,
+                csums, vec.buf, wr->get_int_bitmap(bs->heap), vec.disk_offset, vec.disk_offset+vec.disk_len, false,
                [&](uint32_t mismatch_pos, uint32_t expected_csum, uint32_t real_csum)
                {
                    printf("Checksum mismatch during compaction in object %jx:%jx v%ju, offset 0x%x in %s area at offset 0x%jx: got %08x, expected %08x\n",
-                        cur_oid.inode, cur_oid.stripe, vec.wr->version, mismatch_pos,
+                        cur_oid.inode, cur_oid.stripe, wr->version, mismatch_pos,
                        (vec.copy_flags & COPY_BUF_JOURNAL ? "buffer" : "data"),
                        vec.disk_loc+vec.disk_offset, real_csum, expected_csum);
                    csum_ok = false;
@@ -562,8 +489,20 @@ int journal_flusher_co::check_and_punch_checksums()
        // Nothing to do
        return 0;
    }
-    memcpy(punch_bmp, compact_info.clean_wr->get_int_bitmap(bs->heap), bs->dsk.clean_entry_bitmap_size);
-    memcpy(new_csums, compact_info.clean_wr->get_checksums(bs->heap), bs->dsk.data_block_size/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF));
+    cur_obj = bs->heap->read_entry(cur_oid, &modified_block);
+    if (!cur_obj)
+    {
+        // Object is deleted, abort compaction
+        return ENOENT;
+    }
+    bs->heap->get_compact_range(cur_obj, compact_lsn, &begin_wr, &end_wr);
+    if (!begin_wr)
+    {
+        // Object is overwritten, abort compaction
+        return ENOENT;
+    }
+    uint8_t *bmp = end_wr->get_int_bitmap(bs->heap);
+    uint8_t *csums = end_wr->get_checksums(bs->heap);
    // Clear bits
    for (auto & vec: read_vec)
    {
@@ -574,7 +513,7 @@ int journal_flusher_co::check_and_punch_checksums()
        if (!(vec.copy_flags & COPY_BUF_COALESCED) &&
            ((vec.offset % bs->dsk.csum_block_size) || (vec.len % bs->dsk.csum_block_size)))
        {
-            bitmap_clear(punch_bmp, vec.offset, vec.len, bs->dsk.bitmap_granularity);
+            bitmap_clear(bmp, vec.offset, vec.len, bs->dsk.bitmap_granularity);
        }
    }
    // Update partial block checksums
@@ -583,20 +522,42 @@ int journal_flusher_co::check_and_punch_checksums()
        if (vec.copy_flags & COPY_BUF_CSUM_FILL)
        {
            uint32_t csum_off = vec.offset/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF);
-            bs->heap->calc_block_checksums((uint32_t*)(new_csums+csum_off), vec.buf, punch_bmp, vec.offset, vec.offset+vec.len, true, NULL);
+            bs->heap->calc_block_checksums((uint32_t*)(csums+csum_off), vec.buf, bmp, vec.offset, vec.offset+vec.len, true, NULL);
        }
    }
-    // Modified, we should add_punch_holes and then write the block to disk
+    cur_obj->crc32c = cur_obj->calc_crc32c();
+    if (res == ENOENT)
+    {
+        // Object is deleted, abort compaction
+        return ENOENT;
+    }
+    // Modified, we should write the block to disk
+    assert(!res);
    return EBUSY;
 }

 bool journal_flusher_co::calc_block_checksums()
 {
-    if (bs->dsk.csum_block_size <= bs->dsk.bitmap_granularity)
+    if (bs->dsk.csum_block_size <= bs->dsk.bitmap_granularity || !read_vec.size())
    {
        return true;
    }
-    memcpy(new_csums, compact_info.clean_wr->get_checksums(bs->heap), bs->dsk.data_block_size/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF));
+    bs->heap->get_compact_range(cur_obj, compact_lsn, &begin_wr, &end_wr);
+    if (!begin_wr)
+    {
+        // Object is overwritten, abort compaction
+        return false;
+    }
+    uint8_t *bmp = end_wr->get_int_bitmap(bs->heap);
+    uint8_t *csums = end_wr->get_checksums(bs->heap);
+    // Set bits
+    for (auto & vec: read_vec)
+    {
+        if (!(vec.copy_flags & (COPY_BUF_COALESCED|COPY_BUF_CSUM_FILL)))
+        {
+            bitmap_set(bmp, vec.offset, vec.len, bs->dsk.bitmap_granularity);
+        }
+    }
    // Update block checksums
    size_t i = 0;
    while (i < read_vec.size() && !(read_vec[i].copy_flags & COPY_BUF_CSUM_FILL))
@@ -615,7 +576,7 @@ bool journal_flusher_co::calc_block_checksums()
        assert(!(end % bs->dsk.csum_block_size));
        uint32_t csum_off = start/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF);
        bs->heap->calc_block_checksums(
-            (uint32_t*)(new_csums+csum_off), new_bmp, start, end,
+            (uint32_t*)(csums+csum_off), bmp, start, end,
            [&](uint32_t start, uint32_t & len)
            {
                // O(n^2) search, may be fixed later :-p
@@ -632,6 +593,7 @@ bool journal_flusher_co::calc_block_checksums()
            }, true, NULL
        );
    }
+    cur_obj->crc32c = cur_obj->calc_crc32c();
    return true;
 }

@@ -641,15 +603,13 @@ bool journal_flusher_co::write_meta_block(int wait_base)
        goto resume_0;
    else if (wait_state == wait_base+1)
        goto resume_1;
-resume_0:
-    if (bs->ringloop->space_left() < 1)
-    {
-        wait_state = wait_base+0;
-        return 0;
-    }
-    bs->prepare_meta_block_write(modified_block);
+    await_sqe(0);
+    data->iov = (struct iovec){ bs->heap->get_meta_block(modified_block), (size_t)bs->dsk.meta_block_size };
+    data->callback = simple_callback_w;
+    io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + (modified_block+1)*bs->dsk.meta_block_size);
+    wait_count++;
 resume_1:
-    if (bs->meta_block_is_pending(modified_block))
+    if (wait_count > 0)
    {
        wait_state = wait_base+1;
        return false;
@@ -704,11 +664,11 @@ bool journal_flusher_co::fsync_meta(int wait_base)
    if (wait_state == wait_base)        goto resume_0;
    else if (wait_state == wait_base+1) goto resume_1;
    else if (wait_state == wait_base+2) goto resume_2;
+resume_0:
    if (bs->dsk.disable_meta_fsync)
    {
        return true;
    }
-resume_0:
    if (flusher->wanting_meta_fsync || flusher->fsyncing_meta > 0)
    {
        wait_state = wait_base;
@@ -733,30 +693,69 @@ resume_2:
    return true;
 }

-bool journal_flusher_co::fsync_buffer(int wait_base)
+int journal_flusher_co::fsync_buffer(int wait_base)
+{
+    if (wait_state == wait_base)        goto resume_0;
+    else if (wait_state == wait_base+1) goto resume_1;
+    if (bs->dsk.disable_journal_fsync && bs->dsk.disable_meta_fsync && bs->dsk.disable_data_fsync || !bs->unsynced_big_write_count && !bs->unsynced_small_write_count)
+    {
+        return 1;
+    }
+    if (flusher->syncing_buffer)
+    {
+        return 0;
+    }
+    flusher->active_flushers++;
+    flusher->syncing_buffer++;
+resume_0:
+    assert(!wait_count);
+    compact_lsn = bs->heap->get_completed_lsn();
+    if (!bs->submit_fsyncs(wait_count))
+    {
+        wait_state = wait_base+0;
+        return 0;
+    }
+resume_1:
+    if (wait_count > 0)
+    {
+        wait_state = wait_base+1;
+        return 0;
+    }
+    bs->heap->mark_lsn_fsynced(compact_lsn);
+    flusher->active_flushers--;
+    flusher->syncing_buffer--;
+    return 2;
+}
+
+bool journal_flusher_co::trim_lsn(int wait_base)
 {
    if (wait_state == wait_base)        goto resume_0;
    else if (wait_state == wait_base+1) goto resume_1;
    else if (wait_state == wait_base+2) goto resume_2;
-    if (!bs->has_unsynced())
+    else if (wait_state == wait_base+3) goto resume_3;
+    else if (wait_state == wait_base+4) goto resume_4;
+    compact_lsn = bs->heap->get_compacted_lsn();
+    if (((blockstore_meta_header_v3_t*)bs->meta_superblock)->compacted_lsn == compact_lsn)
    {
        return true;
    }
-resume_0:
-    if (flusher->syncing_buffer)
-    {
-        wait_state = wait_base+0;
-        return false;
-    }
    flusher->active_flushers++;
-    flusher->syncing_buffer++;
-resume_1:
    assert(!wait_count);
-    fsynced_lsn = bs->heap->get_completed_lsn();
-    if (!bs->submit_fsyncs(wait_count))
+    if (!bs->dsk.disable_meta_fsync)
    {
-        wait_state = wait_base+1;
-        return false;
+        await_sqe(0);
+        data->iov = { 0 };
+        data->callback = simple_callback_w;
+        io_uring_prep_fsync(sqe, bs->dsk.meta_fd, IORING_FSYNC_DATASYNC);
+        wait_count++;
+    }
+    if (!bs->dsk.disable_data_fsync && bs->dsk.data_fd != bs->dsk.meta_fd)
+    {
+        await_sqe(1);
+        data->iov = { 0 };
+        data->callback = simple_callback_w;
+        io_uring_prep_fsync(sqe, bs->dsk.data_fd, IORING_FSYNC_DATASYNC);
+        wait_count++;
    }
 resume_2:
    if (wait_count > 0)
@@ -764,37 +763,23 @@ resume_2:
        wait_state = wait_base+2;
        return false;
    }
-    bs->heap->mark_lsn_fsynced(fsynced_lsn);
-    flusher->active_flushers--;
-    flusher->syncing_buffer--;
-    return true;
-}
-
-bool journal_flusher_co::trim_lsn(int wait_base)
-{
-    if (wait_state == wait_base)        goto resume_0;
-    else if (wait_state == wait_base+1) goto resume_1;
-    fsynced_lsn = bs->heap->get_fsynced_lsn();
-    if (((blockstore_meta_header_v3_t*)bs->meta_superblock)->completed_lsn == fsynced_lsn)
-    {
-        return true;
-    }
-    flusher->active_flushers++;
-    ((blockstore_meta_header_v3_t*)bs->meta_superblock)->completed_lsn = fsynced_lsn;
+    ((blockstore_meta_header_v3_t*)bs->meta_superblock)->compacted_lsn = compact_lsn;
    ((blockstore_meta_header_v3_t*)bs->meta_superblock)->set_crc32c();
-    await_sqe(0);
+    await_sqe(3);
    data->iov = (struct iovec){ bs->meta_superblock, (size_t)bs->dsk.meta_block_size };
    data->callback = simple_callback_w;
    io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset);
    // Update superblock with datasync
    sqe->rw_flags = RWF_DSYNC;
    wait_count++;
-resume_1:
+resume_4:
    if (wait_count > 0)
    {
-        wait_state = wait_base+1;
+        wait_state = wait_base+4;
        return false;
    }
+    bs->heap->mark_lsn_trimmed(compact_lsn);
+    flusher->compact_counter++;
    flusher->active_flushers--;
    return true;
 }
@@ -6,7 +6,7 @@ struct copy_buffer_t
    uint32_t copy_flags;
    uint64_t offset, len, disk_loc, disk_offset, disk_len;
    uint8_t *buf;
-    heap_entry_t *wr;
+    uint64_t wr_lsn;
 };

 struct meta_sector_t
@@ -36,28 +36,27 @@ class journal_flusher_co
    int wait_state, wait_count;
    struct io_uring_sqe *sqe;
    struct ring_data_t *data;
-    uint8_t *new_csums = NULL;
-    uint8_t *new_bmp = NULL;
-    uint8_t *punch_bmp = NULL;
-    uint8_t *new_ext_bmp = NULL;

    std::function<void(ring_data_t*)> simple_callback_r, simple_callback_w;

    object_id cur_oid;
-    heap_entry_t *cur_obj;
-    uint64_t fsynced_lsn;
-    heap_compact_t compact_info;
-    uint64_t clean_loc;
+    uint64_t copy_id;
+    uint64_t compact_lsn;
+    uint64_t cur_version;
+    heap_object_t *cur_obj;
+    heap_write_t *begin_wr, *end_wr;
    uint32_t modified_block;
-    bool bitmap_copied;
    bool should_repeat;

    std::vector<copy_buffer_t> read_vec;
-    std::vector<heap_entry_t*> csum_copy;
    uint32_t overwrite_start, overwrite_end;
+    uint32_t big_start, big_end;
    int i, res;
    bool read_to_fill_incomplete;
    int copy_count;
+    uint64_t clean_loc;
+    flusher_meta_write_t meta_old, meta_new;
+    bool do_repeat = false;

    friend class journal_flusher_t;

@@ -69,7 +68,7 @@ class journal_flusher_co
    bool write_meta_block(int wait_base);
    bool read_buffered(int wait_base);
    bool fsync_meta(int wait_base);
-    bool fsync_buffer(int wait_base);
+    int fsync_buffer(int wait_base);
    bool trim_lsn(int wait_base);
 public:
    journal_flusher_co();
@@ -86,7 +85,9 @@ class journal_flusher_t
    blockstore_impl_t *bs;
    friend class journal_flusher_co;

-    robin_hood::unordered_flat_set<object_id> flushing;
+    int advance_lsn_counter = 0;
+    uint64_t compact_counter = 0;
+
    int active_flushers = 0;
    int wanting_meta_fsync = 0;
    bool fsyncing_meta = false;
@@ -97,6 +98,7 @@ public:
    ~journal_flusher_t();
    void loop();
    int get_syncing_buffer();
+    uint64_t get_compact_counter();
    bool is_active();
    void request_trim();
    void release_trim();
@@ -1,4 +1,4 @@
-// Metadata storage version 3 ("lsm heap")
+// Metadata storage version 3 ("heap")
 // Copyright (c) Vitaliy Filippov, 2025+
 // License: VNPL-1.1 (see README.md for details)

@@ -19,315 +19,326 @@ struct pool_shard_settings_t
 {
    uint32_t pg_count;
    uint32_t pg_stripe_size;
-    uint32_t no_inode_stats;
 };

-#define BS_HEAP_TYPE 0x07
-#define BS_HEAP_BIG_WRITE 1
+#define BS_HEAP_TYPE 7
+#define BS_HEAP_OBJECT 1
 #define BS_HEAP_SMALL_WRITE 2
-#define BS_HEAP_INTENT_WRITE 3
-#define BS_HEAP_BIG_INTENT 4
-#define BS_HEAP_DELETE 5
-#define BS_HEAP_COMMIT 6
-#define BS_HEAP_ROLLBACK 7
-#define BS_HEAP_STABLE 0x40
-#define BS_HEAP_GARBAGE 0x80
+#define BS_HEAP_BIG_WRITE 3
+#define BS_HEAP_TOMBSTONE 4
+#define BS_HEAP_INTENT_WRITE 5
+#define BS_HEAP_STABLE 8

 class blockstore_heap_t;

-struct heap_small_write_t;
-struct heap_big_write_t;
-struct heap_big_intent_t;
-
-struct __attribute__((__packed__)) heap_entry_t
+struct __attribute__((__packed__)) heap_small_write_t
 {
    uint16_t size;
-    uint16_t entry_type;
-    uint32_t checksum;
+    int16_t next_pos;
+    uint8_t flags;
    uint64_t lsn;
-    uint64_t inode;
-    uint64_t stripe;
    uint64_t version;
+    uint64_t location;
+    uint32_t offset;
+    uint32_t len;
+};
+
+struct __attribute__((__packed__)) heap_big_write_t
+{
+    uint16_t size;
+    int16_t next_pos;
+    uint8_t flags;
+    uint64_t lsn;
+    uint64_t version;
+    uint32_t block_num;
+};
+
+struct __attribute__((__packed__)) heap_tombstone_t
+{
+    uint16_t size;
+    int16_t next_pos;
+    uint8_t flags;
+    uint64_t lsn;
+    uint64_t version;
+};
+
+struct __attribute__((__packed__)) heap_write_t
+{
+    // size should have top bit cleared
+    uint16_t size = 0;
+    int16_t next_pos = 0;
+    uint8_t entry_type = 0; // BS_HEAP_*
+    uint64_t lsn = 0;
+    uint64_t version = 0;

    // uint8_t[] external_bitmap
    // uint8_t[] internal_bitmap
    // uint32_t[] checksums

+    heap_write_t *next();
    inline uint8_t type() const { return (entry_type & BS_HEAP_TYPE); }
    inline heap_small_write_t& small() { return *(heap_small_write_t*)this; }
    inline heap_big_write_t& big() { return *(heap_big_write_t*)this; }
-    inline heap_big_intent_t& big_intent() { return *(heap_big_intent_t*)this; }
-    bool is_garbage();
-    void set_garbage();
-    bool is_overwrite();
-    bool is_compactable();
-    bool is_before(heap_entry_t *other);
    uint32_t get_size(blockstore_heap_t *heap);
+    uint32_t get_csum_size(blockstore_heap_t *heap);
+    bool needs_recheck(blockstore_heap_t *heap);
+    bool needs_compact(blockstore_heap_t *heap);
+    bool is_compacted(uint64_t compacted_lsn);
+    bool can_be_collapsed(blockstore_heap_t *heap);
+    bool is_allowed_before_compacted(uint64_t compacted_lsn, bool is_last_entry);
    uint8_t *get_ext_bitmap(blockstore_heap_t *heap);
    uint8_t *get_int_bitmap(blockstore_heap_t *heap);
    uint8_t *get_checksums(blockstore_heap_t *heap);
    uint32_t *get_checksum(blockstore_heap_t *heap);
    uint64_t big_location(blockstore_heap_t *heap);
    void set_big_location(blockstore_heap_t *heap, uint64_t location);
-    uint32_t calc_checksum(blockstore_heap_t *heap);
-    uint32_t calc_checksum(blockstore_disk_t *dsk);
 };

-struct __attribute__((__packed__)) heap_small_write_t
+struct __attribute__((__packed__)) heap_object_t
 {
-    heap_entry_t hdr;
+    // size should have top bit cleared
+    uint16_t size = 0;
+    // linked list of write entries...
+    // newest entries are stored first to simplify scanning
+    int16_t write_pos = 0;
+    uint8_t entry_type = 0; // BS_HEAP_*
+    uint32_t crc32c = 0;
+    uint64_t inode = 0;
+    uint64_t stripe = 0;

-    uint64_t location;
-    uint32_t offset;
-    uint32_t len;
-
-    // Also includes 1 bitmap and 1 checksum after the bitmap if block checksums are disabled
+    heap_write_t *get_writes();
+    uint32_t calc_crc32c();
 };

-struct __attribute__((__packed__)) heap_big_write_t
+struct heap_object_lsn_t
 {
-    heap_entry_t hdr;
-
-    uint32_t block_num;
+    object_id oid;
+    uint64_t lsn;
 };

-struct __attribute__((__packed__)) heap_big_intent_t
+inline bool operator < (const heap_object_lsn_t & a, const heap_object_lsn_t & b)
 {
-    heap_entry_t hdr;
+    return a.oid < b.oid || a.oid == b.oid && a.lsn < b.lsn;
+}

-    uint32_t block_num;
-    uint32_t offset;
-    uint32_t len;
-
-    // Also includes 2 bitmaps and 1 checksums if block checksums are disabled
+struct tmp_compact_item_t
+{
+    object_id oid;
+    uint64_t lsn;
+    bool compact;
 };

-struct __attribute__((__packed__)) heap_list_item_t
+struct heap_mvcc_copy_id_t
 {
-    heap_list_item_t *prev;
-    heap_list_item_t *next;
-    uint32_t block_num;
-    heap_entry_t entry;
+    object_id oid;
+    uint64_t copy_id;
+};
+
+inline bool operator == (const heap_mvcc_copy_id_t & a, const heap_mvcc_copy_id_t & b)
+{
+    return a.oid.inode == b.oid.inode && a.oid.stripe == b.oid.stripe && a.copy_id == b.copy_id;
+}
+
+namespace std
+{
+    template<> struct hash<heap_mvcc_copy_id_t>
+    {
+        inline size_t operator()(const heap_mvcc_copy_id_t &s) const
+        {
+            size_t seed = std::hash<object_id>()(s.oid);
+            // Copy-pasted from spp::hash_combine()
+            seed ^= (s.copy_id + 0xc6a4a7935bd1e995 + (seed << 6) + (seed >> 2));
+            return seed;
+        }
+    };
 };

 struct heap_object_mvcc_t
 {
    uint32_t readers = 0;
-    heap_entry_t *garbage_entry = NULL;
+    heap_object_t *entry_copy = NULL;
 };

-struct heap_block_info_t
+struct __attribute__((__packed__)) heap_block_info_t
 {
-    struct __attribute__((__packed__))
-    {
-        uint32_t used_space = 0;
-        uint32_t garbage_space = 0;
-    };
-    uint64_t mod_lsn = 0, mod_lsn_to = 0; // only 1 block write of LSN sequence is allowed at a moment
-    bool is_writing = false;
-    std::vector<heap_list_item_t*> entries;
+    uint32_t used_space = 0;
+    uint32_t free_pos = 0;
+    uint8_t *data = NULL;
 };

 struct heap_inflight_lsn_t
 {
+    object_id oid;
    uint64_t flags;
-    heap_entry_t *wr;
 };

-struct heap_compact_t
+struct heap_refqi_t
 {
-    uint64_t compact_lsn, compact_version;
-    heap_entry_t *clean_wr;
-    bool do_delete;
-};
-
-struct heap_reshard_state_t;
-
-struct heap_li_hash
-{
-    size_t operator()(const heap_list_item_t* li) const noexcept
-    {
-        return robin_hood::hash_int(li->entry.stripe);
-    }
-};
-
-struct heap_li_equal
-{
-    constexpr bool operator()(const heap_list_item_t* a, const heap_list_item_t* b) const noexcept
-    {
-        return a->entry.stripe == b->entry.stripe;
-    }
+    uint64_t lsn;
+    uint64_t inode;
+    uint64_t location;
+    uint32_t len;
+    bool is_data;
 };

 using i64hash_t = robin_hood::hash<uint64_t>;
-using heap_inode_map_t = robin_hood::unordered_flat_set<heap_list_item_t*, heap_li_hash, heap_li_equal, 88>;
 using heap_block_index_t = robin_hood::unordered_flat_map<uint64_t,
-    robin_hood::unordered_flat_map<inode_t, void*, i64hash_t>, i64hash_t>;
-using heap_mvcc_map_t = robin_hood::unordered_flat_map<object_id, heap_object_mvcc_t>;
+    robin_hood::unordered_flat_map<inode_t, robin_hood::unordered_flat_map<uint64_t, uint64_t, i64hash_t, std::equal_to<uint64_t>, 88>, i64hash_t>, i64hash_t>;
+using heap_mvcc_map_t = robin_hood::unordered_flat_map<heap_mvcc_copy_id_t, heap_object_mvcc_t>;

 class blockstore_heap_t
 {
-    friend struct heap_entry_t;
+    friend class heap_write_t;
+    friend class heap_object_t;

    blockstore_disk_t *dsk = NULL;
    uint8_t* buffer_area = NULL;
+    bool abort_on_corruption = false;
+    bool abort_on_overlap = true;
    int log_level = 0;
-    const uint32_t meta_block_count = 0;
-    const uint32_t max_entry_size = 0;

+    const uint32_t meta_block_count = 0;
+    uint32_t target_block_free_space = 800;
+
+    uint64_t next_lsn = 0;
    robin_hood::unordered_flat_map<pool_id_t, pool_shard_settings_t> pool_shard_settings;
    // PG => inode => stripe => block number
    heap_block_index_t block_index;
    std::vector<heap_block_info_t> block_info;
    allocator_t *data_alloc = NULL;
    multilist_index_t *meta_alloc = NULL;
-    uint32_t meta_nearfull_blocks = 0;
+    uint32_t meta_alloc_count = 0;
    uint64_t meta_used_space = 0;
    multilist_alloc_t *buffer_alloc = NULL;
+    heap_mvcc_map_t object_mvcc;
+    std::unordered_map<uint64_t, uint32_t> mvcc_data_refs;
+    std::unordered_map<uint64_t, uint32_t> mvcc_buffer_refs;
    std::map<uint64_t, uint64_t> inode_space_stats;
    uint64_t buffer_area_used_space = 0;
    uint64_t data_used_space = 0;

-    uint64_t live_entries = 0;
-    uint64_t live_memory = 0;
-    uint64_t garbage_entries = 0;
-    uint64_t garbage_memory = 0;
-
-    uint64_t next_lsn = 0;
-    uint32_t last_allocated_block = UINT32_MAX;
-    heap_mvcc_map_t object_mvcc;
-
-    // LSN queue: inflight (writing) -> completed [-> fsynced]
+    // LSN queue: inflight (writing) -> completed [-> fsynced] -> compactable -> compacted [-> fsynced] -> trimmed and removed
    std::deque<heap_inflight_lsn_t> inflight_lsn;
    uint32_t to_compact_count = 0;
-    uint64_t compacted_count = 0;
-    uint32_t inflight_overwrite_count = 0;
    uint64_t first_inflight_lsn = 0;
    uint64_t completed_lsn = 0;
    uint64_t fsynced_lsn = 0;
-    std::deque<object_id> compact_queue;
+    uint64_t compacted_lsn = 0;
+    uint64_t next_compact_lsn = 0;
+    std::deque<heap_refqi_t> overwrite_ref_queue;

-    bool marked_used_blocks = false;
-    bool recheck_queue_filled = false;
-    std::vector<heap_list_item_t*> loaded_list_items;
-    std::set<uint32_t> recheck_modified_blocks;
-    std::deque<heap_entry_t*> recheck_queue;
+    std::vector<tmp_compact_item_t> tmp_compact_queue;
+    std::deque<object_id> recheck_queue;
    int recheck_in_progress = 0;
    bool in_recheck = false;
    std::function<void(bool is_data, uint64_t offset, uint64_t len, uint8_t* buf, std::function<void()>)> recheck_cb;
    int recheck_queue_depth = 0;

-    uint64_t get_pg_id(inode_t inode, uint64_t stripe);
-    bool validate_object(heap_entry_t *obj);
-    void fill_recheck_queue();
-    int mark_used_blocks();
-    void recheck_full_gc();
-    void recheck_buffer(heap_entry_t *cwr, uint8_t *buf);
-    void defragment_block(uint32_t block_num);
-    void reshard_add(heap_reshard_state_t *st, heap_list_item_t *li);
+    const uint32_t max_write_entry_size;
+
+    uint64_t get_pg_id(inode_t inode, uint64_t stripe);
+    void defragment_block(uint32_t block_num);
+    uint32_t find_block_run(heap_block_info_t & block, uint32_t space);
+    uint32_t find_block_space(uint32_t block_num, uint32_t space);
+    uint32_t block_has_compactable(uint8_t *data);
+    uint32_t compact_object_to(heap_object_t *obj, uint64_t lsn, uint8_t *new_csums, bool do_free);
+    void copy_full_object(uint8_t *dst, heap_object_t *obj);
+    bool mvcc_save_copy(heap_object_t *obj);
+    bool mvcc_check_tracking(object_id oid);
+    void free_mvcc(heap_mvcc_map_t::iterator mvcc_it);
+    void allocate_block(heap_block_info_t & inf);
+    int allocate_new_object(object_id oid, uint32_t full_object_size, uint32_t *modified_block, heap_object_t **new_obj);
+    int add_object(object_id oid, heap_write_t *wr, uint32_t *modified_block);
+    void mark_overwritten(uint64_t over_lsn, uint64_t inode, heap_write_t *wr, heap_write_t *end_wr, bool tracking_active);
+    int update_object(uint32_t block_num, heap_object_t *obj, heap_write_t *wr, uint32_t *modified_block, uint32_t *moved_from_block);
+    void init_erase(uint32_t block_num, heap_object_t *obj);
+    void erase_object(uint32_t block_num, heap_object_t *obj, uint64_t lsn, bool tracking_active);
+    void reindex_block(uint32_t block_num, heap_object_t *from_obj);
+    void erase_block_index(inode_t inode, uint64_t stripe);
+    void deref_data(uint64_t inode, uint64_t location, bool free_at_0);
+    void deref_buffer(uint64_t inode, uint64_t location, uint32_t len, bool free_at_0);
+    void deref_overwrites(uint64_t lsn);
+    void free_object_space(inode_t inode, heap_write_t *from, heap_write_t *to, int mode = 0);
+    void add_used_space(uint32_t block_num, int32_t used_delta);
+    void push_inflight_lsn(object_id oid, uint64_t lsn, uint64_t flags);

-    void gc_block(heap_block_info_t & inf);
-    int allocate_entry(uint32_t entry_size, uint32_t *block_num, bool allow_last_free);
-    void insert_list_item(heap_list_item_t *li);
-    void remove_list_item(heap_list_item_t *li);
-    int add_entry(uint32_t wr_size, uint32_t *modified_block, bool allow_last_free,
-        bool explicit_complete, std::function<void(heap_entry_t *wr)> fill_entry);
-    int add_simple(heap_entry_t *obj, uint64_t version, uint32_t *modified_block, uint32_t entry_type);
-    uint32_t meta_alloc_pos(const heap_block_info_t & inf);
-    void modify_alloc(uint32_t block_num, std::function<void(heap_block_info_t &)> change_cb);
-    void mark_garbage_up_to(heap_entry_t *wr);
-    void mark_garbage(uint32_t block_num, heap_entry_t *prev_wr, uint32_t used_big);
-    void push_inflight_lsn(uint64_t lsn, heap_entry_t *wr, uint64_t flags);
-    void mark_completed_lsns(uint64_t mod_lsn);
-    void apply_inflight(heap_inflight_lsn_t & inflight);
 public:
    blockstore_heap_t(blockstore_disk_t *dsk, uint8_t *buffer_area, int log_level = 0);
    ~blockstore_heap_t();
-    void start_load(uint64_t completed_lsn);
-    // load data from the disk, returns EDOM on corruption
-    int read_blocks(uint64_t disk_offset, uint64_t size, uint8_t *buf, bool allow_corrupted,
-        std::function<void(uint32_t block_num, heap_entry_t* wr)> handle_write,
-        std::function<void(uint32_t, uint32_t, uint8_t*)> handle_block);
-    int load_blocks(uint64_t disk_offset, uint64_t size, uint8_t *buf,
-        bool allow_corrupted, uint64_t &entries_loaded);
-    // finish loading - should be called after load_blocks
+    // set initially compacted lsn - should be done before loading
+    void set_compacted_lsn(uint64_t compacted_lsn);
+    uint64_t get_compacted_lsn();
+    // load data from the disk, returns count of loaded write entries
+    void read_blocks(uint64_t disk_offset, uint64_t size, uint8_t *buf,
+        std::function<void(heap_object_t*)> handle_object, std::function<void(uint32_t, uint32_t, uint8_t*)> handle_block);
+    uint64_t load_blocks(uint64_t disk_offset, uint64_t size, uint8_t *buf);
+    // finish loading
    void finish_load();
-    // get blocks which are modified during loading and should be written to the disk
-    // before finishing initialization if not R/O
-    std::vector<uint32_t> get_recheck_modified_blocks();
    // recheck small write data after reading the database from disk
    bool recheck_small_writes(std::function<void(bool is_data, uint64_t offset, uint64_t len, uint8_t* buf, std::function<void()>)> read_buffer, int queue_depth);
-    int finish_recheck();
+    // initialize metadata area (fill it with empty data)
+    // returns 0 when done, EAGAIN when the caller has to wait more
+    int initialize();
+    // read from the metadata area
+    // returns 0 when done, EAGAIN when the caller has to wait more
+    int read();
    // reshard database according to the pool's PG count
-    void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
-    bool reshard_continue(void* reshard_state, uint64_t chunk_limit);
-    bool reshard_check(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size);
-    void set_no_inode_stats(const std::vector<uint64_t> & pool_ids);
-    void recalc_inode_space_stats(uint64_t pool_id, bool per_inode);
+    void reshard(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size);
    // read an object entry and lock it against removal
    // in the future, may become asynchronous
-    heap_entry_t *lock_and_read_entry(object_id oid);
+    heap_object_t *lock_and_read_entry(object_id oid, uint64_t & copy_id);
+    // re-read a locked object entry with the given lsn (pointer may be invalidated)
+    heap_object_t *read_locked_entry(object_id oid, uint64_t copy_id);
    // read an object entry without locking it
-    heap_entry_t *read_entry(object_id oid);
+    heap_object_t *read_entry(object_id oid, uint32_t *block_num_ptr, bool for_update = false);
    // unlock an entry
-    bool unlock_entry(object_id oid);
+    bool unlock_entry(object_id oid, uint64_t copy_id);
    // set or verify checksums in a write request
-    bool calc_checksums(heap_entry_t *wr, uint8_t *data, bool set, uint32_t offset = UINT32_MAX, uint32_t len = UINT32_MAX);
+    bool calc_checksums(heap_write_t *wr, uint8_t *data, bool set, uint32_t offset = 0, uint32_t len = 0);
    // set or verify raw block checksums
    bool calc_block_checksums(uint32_t *block_csums, uint8_t *data, uint8_t *bitmap, uint32_t start, uint32_t end,
        bool set, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);
    bool calc_block_checksums(uint32_t *block_csums, uint8_t *bitmap,
        uint32_t start, uint32_t end, std::function<uint8_t*(uint32_t start, uint32_t & len)> next,
        bool set, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);
-    // adds a small_write or intent_write entry to an object
+    // copy an object as is
+    int copy_object(heap_object_t *obj, uint32_t *modified_block);
+    // auto-compacts the object, then adds a write entry to it and to the compaction queue
    // return 0 if OK, or maybe ENOSPC
-    int add_small_write(object_id oid, heap_entry_t **obj_ptr, uint16_t type, uint64_t version,
-        uint32_t offset, uint32_t len, uint64_t location, uint8_t *bitmap, uint8_t *data, uint32_t *modified_block);
-    // adds a big_write (overwrite) entry to an object
-    int add_big_write(object_id oid, heap_entry_t *old_head, bool stable, uint64_t version,
-        uint32_t offset, uint32_t len, uint64_t location, uint8_t *bitmap, uint8_t *data, uint32_t *modified_block);
-    // adds a "redirecting" big_intent entry to an object (same as big_write, used to avoid fsync on desktop SSDs)
-    int add_redirect_intent(object_id oid, heap_entry_t **obj_ptr, uint64_t version,
-        uint32_t offset, uint32_t len, uint64_t location, uint8_t *bitmap, uint8_t *data, uint32_t *modified_block);
-    // adds a big_intent (atomic partial modification) entry to an object
-    int add_big_intent(object_id oid, heap_entry_t **obj_ptr, uint64_t version,
-        uint32_t offset, uint32_t len, uint8_t *bitmap, uint8_t *data, uint8_t *checksums, uint32_t *modified_block);
-    // adds a compacted up to <version> entry to an object
-    int add_compact(heap_entry_t *obj, uint64_t compact_version, uint64_t compact_lsn, uint64_t compact_location,
-        bool do_delete, uint32_t *modified_block, uint8_t *new_int_bitmap, uint8_t *new_ext_bitmap, uint8_t *new_csums);
-    // "punch holes" in a big_entry
-    int punch_holes(heap_entry_t *wr, uint8_t *new_bitmap, uint8_t *new_csums, uint32_t *modified_block);
+    int post_write(object_id oid, heap_write_t *wr, uint32_t *modified_block, uint32_t *moved_from_block);
+    int post_write(uint32_t & block_num, object_id oid, heap_object_t *obj, heap_write_t *wr, uint32_t *moved_from_block);
    // stabilize an unstable object version
    // return 0 if OK, ENOENT if not exists
-    int add_commit(heap_entry_t *obj, uint64_t version, uint32_t *modified_block);
+    int post_stabilize(object_id oid, uint64_t version, uint32_t *modified_block, uint64_t *new_lsn, uint64_t *new_to_lsn);
    // rollback an unstable object version
    // return 0 if OK, ENOENT if not exists, EBUSY if already stable
-    int add_rollback(heap_entry_t *obj, uint64_t version, uint32_t *modified_block);
+    int post_rollback(object_id oid, uint64_t version, uint64_t *new_lsn, uint32_t *modified_block);
    // forget an object
    // return error code
-    int add_delete(heap_entry_t *obj, uint32_t *modified_block);
+    int post_delete(object_id oid, uint64_t *new_lsn, uint32_t *modified_block);
+    int post_delete(uint32_t block_num, heap_object_t *obj, uint64_t *new_lsn);
    // get the next object to compact
    // guaranteed to return objects in min lsn order
    // returns 0 if OK, ENOENT if nothing to compact
    int get_next_compact(object_id & oid);
-    void iterate_with_stable(heap_entry_t *obj, uint64_t max_lsn, std::function<bool(heap_entry_t*, bool stable)> cb);
-    // iterate compactable entries
-    heap_compact_t iterate_compaction(heap_entry_t *obj, uint64_t fsynced_lsn, bool under_pressure,
-        std::function<void(heap_entry_t*)> small_wr_cb);
-    // iterate all objects
-    void iterate_objects(std::function<void(heap_entry_t*, uint32_t block_num)> cb);
+    // get the range of an object eligible for compaction
+    void get_compact_range(heap_object_t *obj, uint64_t max_lsn, heap_write_t **begin_wr, heap_write_t **end_wr);
+    // mark an object as compacted up to the given lsn
+    int compact_object(object_id oid, uint64_t lsn, uint8_t *new_csums);
    // retrieve object listing from a PG
    int list_objects(uint32_t pg_num, object_id min_oid, object_id max_oid,
        obj_ver_id **result_list, size_t *stable_count, size_t *unstable_count);
+    // set a block number for a new object and returns error status: 0, EAGAIN or ENOSPC
+    int get_block_for_new_object(uint32_t & out_block_num, uint32_t size = 0);

    // inflight write tracking
-    void start_block_write(uint32_t block_num);
-    void complete_block_write(uint32_t block_num);
-    void complete_lsn_write(uint64_t lsn);
-    bool is_lsn_completed(uint64_t lsn);
+    void mark_lsn_completed(uint64_t lsn);
+    void mark_lsn_fsynced(uint64_t lsn);
+    void mark_lsn_compacted(uint64_t lsn, bool allow_undone = false);
+    void mark_object_compacted(heap_object_t *obj, uint64_t max_lsn);
+    void mark_lsn_trimmed(uint64_t lsn);
    uint64_t get_completed_lsn();
    uint64_t get_fsynced_lsn();
-    void mark_lsn_fsynced(uint64_t lsn);

    // data device block allocator functions
    uint64_t find_free_data();
@@ -343,8 +354,7 @@ public:
    uint64_t get_buffer_area_used_space();

    // get metadata block data buffer and used space
-    void get_meta_block(uint32_t block_num, uint8_t *buffer);
-    void fill_block_empty_space(uint8_t *buffer, uint32_t pos);
+    uint8_t *get_meta_block(uint32_t block_num);
    uint32_t get_meta_block_used_space(uint32_t block_num);

    // get space usage statistics
@@ -353,21 +363,14 @@ public:
    uint64_t get_meta_total_space();
    uint64_t get_meta_used_space();
    uint32_t get_meta_nearfull_blocks();
+    uint32_t get_inflight_queue_size();
    uint32_t get_compact_queue_size();
    uint32_t get_to_compact_count();
-    uint64_t get_compacted_count();
-    uint64_t get_live_entries();
-    uint64_t get_live_memory();
-    uint64_t get_garbage_entries();
-    uint64_t get_garbage_memory();

-    uint64_t entry_pos(uint32_t block_num, uint32_t offset);
-    heap_entry_t *entry_from_pos(uint64_t entry_pos, bool allow_unallocated = false);
-    heap_entry_t *prev(heap_entry_t *wr);
-    uint32_t get_simple_entry_size();
-    uint32_t get_big_entry_size();
-    uint32_t get_big_intent_entry_size();
-    uint32_t get_small_entry_size(uint32_t offset, uint32_t len);
-    uint32_t get_csum_size(heap_entry_t *wr);
-    uint32_t get_csum_size(uint32_t entry_type, uint32_t offset = 0, uint32_t len = 0);
+    // get maximum size for a temporary heap_write_t buffer
+    uint32_t get_max_write_entry_size();
+
+    // only for tests
+    void set_abort_on_corruption(bool fail);
+    void set_abort_on_overlap(bool fail);
 };
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Vitaliy Filippov	ec10a25905	Use 32-bit big write location (OK for up to 512 TB OSDs)	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	923c41b928	Do not store offset & len in big_writes	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	8f179eb9a6	Fix object crc32c calculation	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	172b106d99	Add entry_type to heap_object_t too This is required to: 1) later inline the last "big_write" entry into the object to slightly reduce memory usage 2) eliminate an ugly hack where entry type is determined by its size 3) make the storage scheme extensible i.e. when adding new entry types	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	59a51363d0	Rename flags to entry_type	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	65d5d69ae3	Use robin_hood::unordered_flat_map - it has 1 byte overhead instead of 8 byte	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	5d100b4e35	Revert try_get	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	7a894efcb7	WIP dump/load heap	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	3bad0f6fa8	Add missing list_stable_limit support	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	f215c791c8	Fix checksum validation in !inmemory_journal mode	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	0a3260672b	Fix checksum padding during read	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	171b339eb3	Remove unused bs->buffer_area + buffer calc	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	ac331fa77b	Add missing memset zero_object to 0	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	915be747d6	Mark in-memory data reads as SKIP_CSUM	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	b14fbbf5cc	Add missing calc_crc32c after updating block checksums	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	e379fc6df4	Fix checksum padding during flush	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	eec2f2b799	Fix free_read_buffers	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	d85e1dc204	Fix assert(region_marker) at the end of the block in find_block_run()	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	c2edcf63e1	Fix assert(is_buffer_area_free) with size=0	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	3a8f0f53de	Add a copy of wyhash	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	9b70bfc2e4	Fix test build with isa-l	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	d1678c67d9	Remove assert !region_marker & FREE	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	ad308ed92a	Use wyhash	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	b0bf45c26f	Change emhashes	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	2a550d4d13	unordered_map mvcc	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	d4d3d34a84	Fix bad resharding due to the lack of iteration order in a hashmap	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	b8bec78879	Use emhash::try_get	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	fa77a321ab	Use emhash hashmap (2x faster)	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	cfce09be03	Use unordered_maps for object-block index	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	ed3332ea83	B-tree is slower...	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	0a94c437e2	Unordered_map for pool settings	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	3bb514321f	Fix read fio bench	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	1a4c6746c4	Fix vitastor-disk prepare and param validation	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	3040bcfbfa	Fix skipping of corrupted objects, fix use_buffer_area with zero size	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	08e16ba985	Support heap format in simple-offsets	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	9afa3f9a13	Return all unstable versions in listing	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	bbf5063b25	Actually fsync after stabilize	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	f45c88d38c	Move test & build_test to top-level cmakelists	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	8fa0638347	Add missing request_trim	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	0c2e0b8a49	Use multilist_index_t instead of multiple bitmap allocators	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	36ccf63922	Implement another multilist-style allocator for metadata blocks	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	4bc1d09fbd	Fix a bug with unstable_big over unstable_small	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	dc31650110	Limit the number of unstable versions per object	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	598e1ed1db	Integrate moving objects	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	a49c176423	Support moving objects between blocks	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	ecd26ebac3	Fix zero-length writes	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	d0690091c1	Fix op_stable slowdowns	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	9369b643a1	Fix a bs_read bug	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	ed2a886ab4	Fix test dependencies	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	98bdcfbff9	Add read_blocks() API	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	3cce10d9c7	Fix block checksum calculation in write_journal for the old blockstore version	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	8762ae07a6	Remove block_order parameter	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	b2b30201c4	Extract (flags & BS_HEAP_TYPE) into a function	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	a912420424	Make dump-journal --format data default	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	27c250e4da	Add fio options	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	0b28ec8fc9	Fix loading for out-of-order lsns	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	b9b2f8cec1	Add include for older gcc	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	4581084540	Fix vitastor-disk build (with old metadata and journal formats)	2025-11-23 19:08:24 +03:00
Vitaliy Filippov	4dfb80e6a1	Collapse intent_writes on other write types too	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	1aff129f99	Fix buffer overflow in test_heap	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	2282ed0345	Add a test for parallel reads with block checksums	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	4b95fe4919	Remove cancel_all_writes Not needed because a) parallel writes to the same object are forbidden b) subsequent writes don't depend on previous ones anyway.	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	ad6354a537	Add a test for "perfect_csum_update" mode	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	346d20ce29	Add 2 tests for intent writes	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	eb9633f21f	Add a test with fsync	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	f86bed106c	Implement buffered disk_mock_t mode, extract ringloop_mock.cpp	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	cc2975efe6	unaligned_intent does not need special handling anymore	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	27ef64cef4	Add an option for global coverage	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	7f7d79fdc9	Fix some blockstore bugs discovered by the mocked test!	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	7b94969869	Add a basic mocked blockstore test	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	92a0942cd6	Add mocks for blockstore integration tests: timerfd, ring_loop_mock_t and disk_mock_t	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	c5c332cb9a	Test calc_checksums	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	cedbedc1e5	Check 2/3 blocks in test_recheck	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	b28c67ba5f	Fix intent writes with padded checksums	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	26fb08cc02	Fix reads from intent writes	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	186f37e286	Disable punching block checksums and allow to enable it with a parameter	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	8c3b83ff74	Process big_writes as intents to avoid fsyncs	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	5fe474573e	Fix collapsing intent-over-intent checksums	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	89644692ac	Allow 1 intent_write over big_write in fsync mode	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	8581988c2c	Block lists by previous writes	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	fb83ebbfff	Allow multiple writes with the same version	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	abf741048a	Add tests for intent writes	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	cb056b4413	Call finish_load after async recheck	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	43c4510a29	More tests for incorrect data cases	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	2ba34bfd75	Do not block writes on previous writes	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	481005b062	Add atomic_write_size parameter	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	6464dae129	Use ui32 for block sizes	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	e0d60dd49f	Remove extra unneeded read_entry-s	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	abeb8b73fa	Do not recheck data location on intent-write	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	15d47a2695	Remove FIXMEs	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	661c45cbb5	Fsync after stabilizing	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	bff3d35c85	Fsync & update metadata when block checksums are enabled	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	18a2858c62	Return new_lsn from erase and rollback	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	999d0e2961	Allow to cancel compaction for unfinished writes	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	df1ab722dd	Fix space allocation & compaction on start	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	40a4952592	Use inflight_lsn iterators	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	298a5ec499	Free space on overwrites correctly	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	dd28f2f963	Use fsynced_lsn in flusher	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	41256ea460	Correctly track fsynced_lsn when fsyncs are enabled	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	229c19dee4	Extract multilist_alloc_t	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	1ccb241eef	Check for overlaps during blockstore loading	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	b8245f86eb	Fix multilist_alloc_t bug, move verify and print to lib	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	31f7cbc19a	Fix pending_ops	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	20226200a4	Fix repeating cur_oid	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	48f8ae8e30	Mark overwritten heap_writes as immediately compacted	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	38faa21966	Move "ack write" debug message	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	2afb16cd19	Fsync data on trim_lsn, not when writing compacted data	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	908ce74500	Batch big_write data fsyncs	2025-11-23 19:08:23 +03:00
Vitaliy Filippov	98da2cb6e0	Use the same "inflight" queue to track compaction	2025-11-23 19:08:22 +03:00
Vitaliy Filippov	0fd557fe95	Use new LSNs on stabilize	2025-11-23 19:07:43 +03:00
Vitaliy Filippov	a52dd0dfb5	Assign new LSN on stabilize	2025-11-23 19:07:43 +03:00
Vitaliy Filippov	66847653aa	WIP Only save MVCC copy when overwriting an object	2025-11-23 19:07:43 +03:00
Vitaliy Filippov	393f9d6b6f	Do not use wr_offset	2025-11-23 19:07:43 +03:00
Vitaliy Filippov	9a87452d23	Prevent compaction of incomplete object writes	2025-11-23 19:07:43 +03:00
Vitaliy Filippov	385973f777	Experimental INTENT_WRITE write mode with WA=2 instead of 3	2025-11-23 19:07:43 +03:00
Vitaliy Filippov	b958f7f119	Add test_compact_block	2025-11-23 19:07:43 +03:00
Vitaliy Filippov	4d649b1726	Remove alloc_buffer_area	2025-11-23 19:07:43 +03:00
Vitaliy Filippov	bf180f5292	Two more unordered_maps	2025-11-23 19:07:43 +03:00
Vitaliy Filippov	6ee1093837	Remove sync_to_repeat map and use simpler repeating	2025-11-23 19:07:43 +03:00
Vitaliy Filippov	1747cd6a95	Use a sequence of bitmap_allocs for metadata instead of std::sets...	2025-11-23 19:07:43 +03:00
Vitaliy Filippov	ec9eba63af	It seems tcmalloc is actually slower, disable it	2025-11-23 19:07:43 +03:00
Vitaliy Filippov	86dbd99268	Use single add_used_space call instead of unmark+mark allocated_block	2025-11-23 19:07:43 +03:00
Vitaliy Filippov	7ad3001afd	Remove compact_queue_lsn map	2025-11-23 19:07:43 +03:00
Vitaliy Filippov	5222c2724f	Implement a really crazy "multi-linked-list" allocator for buffered data	2025-11-23 19:07:43 +03:00
Vitaliy Filippov	622cc55924	Use linked list heap in blockstore code WIP, still slower than the old version :-E	2025-11-23 19:07:43 +03:00
Vitaliy Filippov	e2fa675ad0	Use linked lists in heap to avoid excessive memory copying	2025-11-23 19:07:43 +03:00
Vitaliy Filippov	388b5f19a1	v1 (old) store fixes for merged version	2025-11-23 19:07:43 +03:00
Vitaliy Filippov	19a386e4b3	Integrate "heap" metadata storage into blockstore	2025-11-23 19:07:42 +03:00
Vitaliy Filippov	9264ca96a4	"Heap" metadata storage scheme	2025-11-23 15:18:01 +03:00