Trigger event loop more intelligently and automatically in libvitastor_c

Build node.js addon as a Debian package
Trigger event loop once after each vitastor_c_* call
2025-03-02 18:04:56 +03:00 · 2025-03-02 18:04:56 +03:00 · 2025-03-02 01:23:41 +03:00 · 2025-03-01 13:39:42 +03:00 · 2025-03-01 13:39:42 +03:00 · 2025-03-01 13:29:48 +03:00
94 changed files with 1690 additions and 673 deletions
@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)

 project(vitastor)

-set(VITASTOR_VERSION "1.10.1")
+set(VITASTOR_VERSION "1.11.0")

 add_subdirectory(src)
@@ -41,6 +41,7 @@ Vitastor поддерживает QEMU-драйвер, протоколы NBD и
  - [Автор и лицензия](docs/intro/author.ru.md)
 - Установка
  - [Пакеты](docs/installation/packages.ru.md)
+  - [Docker](docs/installation/docker.ru.md)
  - [Proxmox](docs/installation/proxmox.ru.md)
  - [OpenNebula](docs/installation/opennebula.ru.md)
  - [OpenStack](docs/installation/openstack.ru.md)
@@ -41,6 +41,7 @@ Read more details in the documentation. You can start from here: [Quick Start](d
  - [Author and license](docs/intro/author.en.md)
 - Installation
  - [Packages](docs/installation/packages.en.md)
+  - [Docker](docs/installation/docker.en.md)
  - [Proxmox](docs/installation/proxmox.en.md)
  - [OpenNebula](docs/installation/opennebula.en.md)
  - [OpenStack](docs/installation/openstack.en.md)
@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v1.10.1
+VITASTOR_VERSION ?= v1.11.0

 all: build push

@@ -49,7 +49,7 @@ spec:
            capabilities:
              add: ["SYS_ADMIN"]
            allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v1.10.1
+          image: vitalif/vitastor-csi:v1.11.0
          args:
            - "--node=$(NODE_ID)"
            - "--endpoint=$(CSI_ENDPOINT)"
@@ -121,7 +121,7 @@ spec:
            privileged: true
            capabilities:
              add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v1.10.1
+          image: vitalif/vitastor-csi:v1.11.0
          args:
            - "--node=$(NODE_ID)"
            - "--endpoint=$(CSI_ENDPOINT)"
@@ -5,7 +5,7 @@ package vitastor

 const (
    vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "1.10.1"
+    vitastorCSIDriverVersion = "1.11.0"
 )

 // Config struct fills the parameters of request or user input
@@ -1,4 +1,4 @@
-vitastor (1.10.1-1) unstable; urgency=medium
+vitastor (1.11.0-1) unstable; urgency=medium

  * Bugfixes

@@ -2,7 +2,10 @@ Source: vitastor
 Section: admin
 Priority: optional
 Maintainer: Vitaliy Filippov <vitalif@yourcmc.ru>
-Build-Depends: debhelper, liburing-dev (>= 0.6), g++ (>= 8), libstdc++6 (>= 8), linux-libc-dev, libgoogle-perftools-dev, libjerasure-dev, libgf-complete-dev, libibverbs-dev, libisal-dev, cmake, pkg-config, libnl-3-dev, libnl-genl-3-dev
+Build-Depends: debhelper, liburing-dev (>= 0.6), g++ (>= 8), libstdc++6 (>= 8),
+  linux-libc-dev, libgoogle-perftools-dev, libjerasure-dev, libgf-complete-dev,
+  libibverbs-dev, libisal-dev, cmake, pkg-config, libnl-3-dev, libnl-genl-3-dev,
+  node-bindings <!nocheck>, node-gyp, node-nan
 Standards-Version: 4.5.0
 Homepage: https://vitastor.io/
 Rules-Requires-Root: no
@@ -59,3 +62,9 @@ Architecture: amd64
 Depends: ${shlibs:Depends}, ${misc:Depends}, vitastor-client, patch, python3, jq
 Description: Vitastor OpenNebula storage plugin
 Vitastor storage plugin for OpenNebula.
+
+Package: node-vitastor
+Architecture: amd64
+Depends: ${shlibs:Depends}, ${misc:Depends}, node-bindings
+Description: Node.js bindings for Vitastor client
+ Node.js native bindings for the Vitastor client library (vitastor-client).
@@ -0,0 +1 @@
+usr/lib/x86_64-linux-gnu/nodejs/vitastor
@@ -1,8 +1,10 @@
 # Build patched QEMU for Debian inside a container
 # cd ..; podman build --build-arg REL=bullseye -v `pwd`/packages:/root/packages -f debian/patched-qemu.Dockerfile .

+ARG DISTRO=debian
 ARG REL=
-FROM debian:$REL
+FROM $DISTRO:$REL
+ARG DISTRO=debian
 ARG REL=

 WORKDIR /root
@@ -20,8 +22,8 @@ RUN if [ "$REL" = "buster" -o "$REL" = "bullseye" -o "$REL" = "bookworm" ]; then
    echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf

 RUN apt-get update
-RUN apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts
-RUN apt-get -y build-dep qemu
+RUN DEBIAN_FRONTEND=noninteractive TZ=Europe/Moscow apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts
+RUN DEBIAN_FRONTEND=noninteractive TZ=Europe/Moscow apt-get -y build-dep qemu
 # To build a custom version
 #RUN cp /root/packages/qemu-orig/* /root
 RUN apt-get --download-only source qemu
@@ -38,9 +40,9 @@ ADD src/client/qemu_driver.c /root/qemu_driver.c
 #    apt-get install -y vitastor-client vitastor-client-dev quilt

 RUN set -e; \
-    dpkg -i /root/packages/vitastor-$REL/vitastor-client_*.deb /root/packages/vitastor-$REL/vitastor-client-dev_*.deb; \
+    DEBIAN_FRONTEND=noninteractive TZ=Europe/Moscow apt-get -y install /root/packages/vitastor-$REL/vitastor-client_*.deb /root/packages/vitastor-$REL/vitastor-client-dev_*.deb; \
    apt-get update; \
-    apt-get install -y quilt; \
+    DEBIAN_FRONTEND=noninteractive TZ=Europe/Moscow apt-get -y install quilt; \
    mkdir -p /root/packages/qemu-$REL; \
    rm -rf /root/packages/qemu-$REL/*; \
    cd /root/packages/qemu-$REL; \
@@ -4,6 +4,14 @@ export DH_VERBOSE = 1
 %:
 	dh $@

+override_dh_install:
+	perl -pe 's!prefix=/usr!prefix='`pwd`'/debian/tmp/usr!' < obj-x86_64-linux-gnu/src/client/vitastor.pc > node-binding/vitastor.pc
+	cd node-binding && PKG_CONFIG_PATH=./ PKG_CONFIG_ALLOW_SYSTEM_CFLAGS=1 npm install --unsafe-perm || exit 1
+	mkdir -p debian/tmp/usr/lib/x86_64-linux-gnu/nodejs/vitastor/build/Release
+	cp -v node-binding/package.json node-binding/index.js node-binding/addon.cc node-binding/addon.h node-binding/client.cc node-binding/client.h debian/tmp/usr/lib/x86_64-linux-gnu/nodejs/vitastor
+	cp -v node-binding/build/Release/addon.node debian/tmp/usr/lib/x86_64-linux-gnu/nodejs/vitastor/build/Release
+	dh_install
+
 override_dh_installdeb:
 	cat debian/fio_version >> debian/vitastor-fio.substvars
 	[ -f debian/qemu_version ] && (cat debian/qemu_version >> debian/vitastor-qemu.substvars) || true
@@ -22,7 +22,8 @@ RUN set -e -x; \
    echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf

 RUN apt-get update && \
-    apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts libjerasure-dev cmake libibverbs-dev librdmacm-dev libisal-dev libnl-3-dev libnl-genl-3-dev curl && \
+    apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts libjerasure-dev cmake \
+        libibverbs-dev librdmacm-dev libisal-dev libnl-3-dev libnl-genl-3-dev curl nodejs npm node-nan node-bindings && \
    apt-get -y build-dep fio && \
    apt-get --download-only source fio

@@ -1,9 +1,11 @@
 # Build Docker image with Vitastor packages

-FROM debian:bullseye
+FROM debian:bookworm

-ADD vitastor.list /etc/apt/sources.list.d
-ADD vitastor.gpg /etc/apt/trusted.gpg.d
-ADD vitastor.pref /etc/apt/preferences.d
-ADD apt.conf /etc/apt/
-RUN apt-get update && apt-get -y install vitastor qemu-system-x86 qemu-system-common && apt-get clean
+ADD etc/apt /etc/apt/
+RUN apt-get update && apt-get -y install vitastor qemu-system-x86 qemu-system-common qemu-block-extra qemu-utils jq nfs-common && apt-get clean
+ADD sleep.sh /usr/bin/
+ADD install.sh /usr/bin/
+ADD scripts /opt/scripts/
+ADD etc /etc/
+RUN ln -s /usr/lib/vitastor/mon/make-etcd /usr/bin/make-etcd
@@ -0,0 +1,9 @@
+VITASTOR_VERSION ?= v1.11.0
+
+all: build push
+
+build:
+	@docker build --rm -t vitalif/vitastor:$(VITASTOR_VERSION) .
+
+push:
+	@docker push vitalif/vitastor:$(VITASTOR_VERSION)
@@ -0,0 +1 @@
+deb http://vitastor.io/debian bookworm main
@@ -0,0 +1,27 @@
+[Unit]
+Description=Containerized etcd for Vitastor
+After=network-online.target local-fs.target time-sync.target docker.service vitastor-host.service
+Wants=network-online.target local-fs.target time-sync.target docker.service vitastor-host.service
+PartOf=vitastor.target
+
+[Service]
+Restart=always
+Environment=GOGC=50
+EnvironmentFile=/etc/vitastor/docker.conf
+EnvironmentFile=/etc/vitastor/etcd.conf
+SyslogIdentifier=etcd
+ExecStart=bash -c 'docker run --rm -i -v /var/lib/vitastor/etcd:/data \
+    --log-driver none --network host $CONTAINER_OPTIONS --name vitastor-etcd \
+    $ETCD_IMAGE /usr/local/bin/etcd --name "$ETCD_NAME" --data-dir /data \
+    --snapshot-count 10000 --advertise-client-urls http://$ETCD_IP:2379 --listen-client-urls http://$ETCD_IP:2379 \
+    --initial-advertise-peer-urls http://$ETCD_IP:2380 --listen-peer-urls http://$ETCD_IP:2380 \
+    --initial-cluster-token vitastor-etcd-1 --initial-cluster "$ETCD_INITIAL_CLUSTER" \
+    --initial-cluster-state new --max-txn-ops=100000 --max-request-bytes=104857600 \
+    --auto-compaction-retention=10 --auto-compaction-mode=revision'
+ExecStop=docker stop vitastor-etcd
+Restart=always
+StartLimitInterval=0
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,23 @@
+[Unit]
+Description=Empty container for running Vitastor commands
+After=network-online.target local-fs.target time-sync.target docker.service
+Wants=network-online.target local-fs.target time-sync.target docker.service
+PartOf=vitastor.target
+
+[Service]
+Restart=always
+EnvironmentFile=/etc/vitastor/docker.conf
+ExecStart=bash -c 'docker run --rm -i -v /etc/vitastor:/etc/vitastor -v /dev:/dev \
+    --privileged --log-driver none --network host --name vitastor vitastor:$VITASTOR_VERSION \
+    sleep.sh'
+ExecStartPost=udevadm trigger
+ExecStop=docker stop vitastor
+WorkingDirectory=/
+PrivateTmp=false
+TasksMax=infinity
+Restart=always
+StartLimitInterval=0
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,23 @@
+[Unit]
+Description=Containerized Vitastor monitor
+After=network-online.target local-fs.target time-sync.target docker.service
+Wants=network-online.target local-fs.target time-sync.target docker.service
+PartOf=vitastor.target
+
+[Service]
+Restart=always
+EnvironmentFile=/etc/vitastor/docker.conf
+SyslogIdentifier=vitastor-mon
+ExecStart=bash -c 'docker run --rm -i -v /etc/vitastor:/etc/vitastor -v /var/lib/vitastor:/var/lib/vitastor -v /dev:/dev \
+    --log-driver none --network host $CONTAINER_OPTIONS --name vitastor-mon vitastor:$VITASTOR_VERSION \
+    node /usr/lib/vitastor/mon/mon-main.js'
+ExecStop=docker stop vitastor-mon
+WorkingDirectory=/
+PrivateTmp=false
+TasksMax=infinity
+Restart=always
+StartLimitInterval=0
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,27 @@
+[Unit]
+Description=Containerized Vitastor object storage daemon osd.%i
+After=network-online.target local-fs.target time-sync.target docker.service vitastor-host.service
+Wants=network-online.target local-fs.target time-sync.target docker.service vitastor-host.service
+PartOf=vitastor.target
+
+[Service]
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitMEMLOCK=infinity
+EnvironmentFile=/etc/vitastor/docker.conf
+SyslogIdentifier=vitastor-osd%i
+ExecStart=bash -c 'docker run --rm -i -v /etc/vitastor:/etc/vitastor -v /dev:/dev \
+    $(for i in $(ls /dev/vitastor/osd%i-*); do echo --device $i:$i; done) \
+    --log-driver none --network host --ulimit nofile=1048576 --ulimit memlock=-1 $CONTAINER_OPTIONS --name vitastor-osd%i \
+    vitastor:$VITASTOR_VERSION vitastor-disk exec-osd /dev/vitastor/osd%i-data'
+ExecStartPre=+docker exec vitastor vitastor-disk pre-exec /dev/vitastor/osd%i-data
+ExecStop=docker stop vitastor-etcd%i
+WorkingDirectory=/
+PrivateTmp=false
+TasksMax=infinity
+Restart=always
+StartLimitInterval=0
+RestartSec=10
+
+[Install]
+WantedBy=vitastor.target
@@ -0,0 +1,4 @@
+[Unit]
+Description=vitastor target
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,7 @@
+SUBSYSTEM=="block", ENV{ID_PART_ENTRY_TYPE}=="e7009fac-a5a1-4d72-af72-53de13059903", \
+    OWNER="vitastor", GROUP="vitastor", \
+    IMPORT{program}="/usr/bin/docker exec vitastor vitastor-disk udev $devnode", \
+    SYMLINK+="vitastor/$env{VITASTOR_ALIAS}"
+
+ENV{VITASTOR_OSD_NUM}!="", ACTION=="add", RUN{program}+="/usr/bin/systemctl enable --now --no-block vitastor-osd@$env{VITASTOR_OSD_NUM}"
+ENV{VITASTOR_OSD_NUM}!="", ACTION=="remove", RUN{program}+="/usr/bin/systemctl disable --now --no-block vitastor-osd@$env{VITASTOR_OSD_NUM}"
@@ -0,0 +1,11 @@
+#
+# Configuration file for containerized Vitastor installation
+# (non-Kubernetes, with systemd and udev-based orchestration)
+#
+
+# Desired Vitastor version
+VITASTOR_VERSION=1.11.0
+
+# Additional arguments for all containers
+# For example, you may want to specify a custom logging driver here
+CONTAINER_OPTIONS=""
@@ -0,0 +1,4 @@
+ETCD_IMAGE=quay.io/coreos/etcd:v3.5.18
+ETCD_NAME=""
+ETCD_IP=""
+ETCD_INITIAL_CLUSTER=""
@@ -0,0 +1,2 @@
+{
+}
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+cp -urv /etc/default /host-etc/
+cp -urv /etc/systemd /host-etc/
+cp -urv /etc/udev /host-etc/
+cp -urnv /etc/vitastor /host-etc/
+cp -urnv /opt/scripts/* /host-bin/
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker exec -it vitastor vitastor-cli "$@"
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker exec -it vitastor vitastor-disk "$@"
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker exec -it vitastor fio "$@"
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker exec -it vitastor vitastor-nbd "$@"
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+while :; do sleep infinity; done
@@ -1 +0,0 @@
-deb http://vitastor.io/debian bullseye main
@@ -13,7 +13,7 @@ Vitastor configuration consists of:
 - [Separate OSD settings](config/pool.en.md#osd-settings)
 - [Inode configuration](config/inode.en.md) i.e. image metadata like name, size and parent reference

-Configuration parameters can be set in 3 places:
+Configuration parameters can be set in 4 places:
 - Configuration file (`/etc/vitastor/vitastor.conf` or other path)
 - etcd key `/vitastor/config/global`. Most variables can be set there, but etcd
  connection parameters should obviously be set in the configuration file.
@@ -14,7 +14,7 @@
 - [Настроек инодов](config/inode.ru.md), т.е. метаданных образов, таких, как имя, размер и ссылки на
  родительский образ

-Параметры конфигурации могут задаваться в 3 местах:
+Параметры конфигурации могут задаваться в 4 местах:
 - Файле конфигурации (`/etc/vitastor/vitastor.conf` или по другому пути)
 - Ключе в etcd `/vitastor/config/global`. Большая часть параметров может
  задаваться там, кроме, естественно, самих параметров соединения с etcd,
@@ -14,8 +14,12 @@

 {{../../installation/packages.en.md}}

+{{../../installation/docker.en.md}}
+
 {{../../installation/proxmox.en.md}}

+{{../../installation/opennebula.en.md}}
+
 {{../../installation/openstack.en.md}}

 {{../../installation/kubernetes.en.md}}
@@ -14,8 +14,12 @@

 {{../../installation/packages.ru.md}}

+{{../../installation/docker.ru.md}}
+
 {{../../installation/proxmox.ru.md}}

+{{../../installation/opennebula.ru.md}}
+
 {{../../installation/openstack.ru.md}}

 {{../../installation/kubernetes.ru.md}}
@@ -0,0 +1,60 @@
+[Documentation](../../README.md#documentation) → Installation → Dockerized Installation
+
+-----
+
+[Читать на русском](docker.ru.md)
+
+# Dockerized Installation
+
+Vitastor may be installed in Docker/Podman. In such setups etcd, monitors and OSD
+all run in containers, but everything else looks as close as possible to a usual
+setup with packages:
+- host network is used
+- auto-start is implemented through udev and systemd
+- logs are written to journald (not docker json log files)
+- command-line wrapper scripts are installed to the host system to call vitastor-disk,
+  vitastor-cli and others through the container
+
+Such installations may be useful when it's impossible or inconvenient to install
+Vitastor from packages, for example, in exotic Linux distributions.
+
+If you don't want just a simple containerized installation, you can also take a look
+at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operator
+
+## Installing Containers
+
+The instruction is very simple.
+
+1. Download a Docker image of the desired version: \
+   `docker pull vitastor:1.10.2`
+2. Install scripts to the host system: \
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:1.10.2 install.sh`
+3. Reload udev rules: \
+   `udevadm control --reload-rules`
+
+And you can return to [Quick Start](../intro/quickstart.en.md).
+
+## Upgrading Containers
+
+First make sure to check the topic [Upgrading Vitastor](../usage/admin.en.md#upgrading-vitastor)
+to figure out if you need any additional steps.
+
+Then, to upgrade a containerized installation, you just need to change the `VITASTOR_VERSION`
+option in `/etc/vitastor/docker.conf` and restart all Vitastor services:
+
+`systemctl restart vitastor.target`
+
+## QEMU
+
+Vitastor Docker image also contains QEMU, qemu-img and qemu-storage-daemon built with Vitastor support.
+
+However, running QEMU in Docker is harder to setup and it depends on the used virtualization UI
+(OpenNebula, Proxmox and so on). Some of them also required patched Libvirt.
+
+That's why containerized installation of Vitastor doesn't contain a ready-made QEMU setup and it's
+recommended to install QEMU from packages or build it manually.
+
+## fio
+
+Vitastor Docker image also contains fio and installs a wrapper called `vitastor-fio` to use it from
+the host system.
@@ -0,0 +1,60 @@
+[Документация](../../README-ru.md#документация) → Установка → Установка в Docker
+
+-----
+
+[Read in English](docker.en.md)
+
+# Установка в Docker
+
+Vitastor можно установить в Docker/Podman. При этом etcd, мониторы и OSD запускаются
+в контейнерах, но всё остальное выглядит максимально приближенно к установке из пакетов:
+- используется сеть хост-системы
+- для автозапуска используются udev и systemd
+- журналы записываются в journald (не в json-файлы журналов docker)
+- в хост-систему устанавливаются обёртки для вызова консольных инструментов vitastor-disk,
+  vitastor-cli и других через контейнер
+
+Такая установка полезна тогда, когда установка из пакетов невозможна или неудобна,
+например, в нестандартных Linux-дистрибутивах.
+
+Если вам нужна не просто контейнеризованная инсталляция, вы также можете обратить внимание
+на Vitastor Kubernetes-оператор: https://github.com/Antilles7227/vitastor-operator
+
+## Установка контейнеров
+
+Инструкция по установке максимально простая.
+
+1. Скачайте Docker-образ желаемой версии: \
+   `docker pull vitastor:1.10.2`
+2. Установите скрипты в хост-систему командой: \
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:1.10.2 install.sh`
+3. Перезагрузите правила udev: \
+   `udevadm control --reload-rules`
+
+После этого вы можете возвращаться к разделу [Быстрый старт](../intro/quickstart.ru.md).
+
+## Обновление контейнеров
+
+Сначала обязательно проверьте раздел [Обновление Vitastor](../usage/admin.ru.md#обновление-vitastor),
+чтобы понять, не требуются ли вам какие-то дополнительные действия.
+
+После этого для обновления Docker-инсталляции вам нужно просто поменять опцию `VITASTOR_VERSION`
+в файле `/etc/vitastor/docker.conf` и перезапустить все сервисы Vitastor командой:
+
+`systemctl restart vitastor.target`
+
+## QEMU
+
+В Docker-образ также входят QEMU, qemu-img и qemu-storage-daemon, собранные с поддержкой Vitastor.
+
+Однако настроить запуск QEMU в Docker сложнее и способ запуска зависит от используемого интерфейса
+виртуализации (OpenNebula, Proxmox и т.п.). Также для OpenNebula, например, требуется патченый
+Libvirt.
+
+Поэтому по умолчанию Docker-сборка пока что не включает в себя готового способа запуска QEMU
+и QEMU рекомендуется устанавливать из пакетов или собирать самостоятельно.
+
+## fio
+
+fio также входит в Docker-контейнер vitastor, и в хост-систему устанавливается обёртка `vitastor-fio`
+для запуска fio в контейнер.
@@ -14,6 +14,7 @@
  - Debian 12 (Bookworm/Sid): `deb https://vitastor.io/debian bookworm main`
  - Debian 11 (Bullseye): `deb https://vitastor.io/debian bullseye main`
  - Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
+  - Ubuntu 22.04 (Jammy): `deb https://vitastor.io/debian jammy main`
  - Add `-oldstable` to bookworm/bullseye/buster in this line to install the last
    stable version from 0.9.x branch instead of 1.x
 - Install packages: `apt update; apt install vitastor lp-solve etcd linux-image-amd64 qemu-system-x86`
@@ -14,6 +14,7 @@
  - Debian 12 (Bookworm/Sid): `deb https://vitastor.io/debian bookworm main`
  - Debian 11 (Bullseye): `deb https://vitastor.io/debian bullseye main`
  - Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
+  - Ubuntu 22.04 (Jammy): `deb https://vitastor.io/debian jammy main`
  - Добавьте `-oldstable` к слову bookworm/bullseye/buster в этой строке, чтобы
    установить последнюю стабильную версию из ветки 0.9.x вместо 1.x
 - Установите пакеты: `apt update; apt install vitastor lp-solve etcd linux-image-amd64 qemu-system-x86`
@@ -26,13 +26,13 @@
  you also need small SSDs for journal and metadata (even 2 GB per 1 TB of HDD space is enough).
 - Get a fast network (at least 10 Gbit/s). Something like Mellanox ConnectX-4 with RoCEv2 is ideal.
 - Disable CPU powersaving: `cpupower idle-set -D 0 && cpupower frequency-set -g performance`.
- [Install Vitastor packages](../installation/packages.en.md).
+- Either [install Vitastor packages](../installation/packages.en.md) or [install Vitastor in Docker](../installation/docker.en.md).

 ## Recommended drives

 - SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel D3-S4510/4520/4610/4620, Kingston DC500M
 - NVMe: Micron 9100/9200/9300/9400, Micron 7300/7450, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
-  Intel DC-P3700/P4500/P4600, Intel D5-P4320, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
+  Intel DC-P3700/P4500/P4600, Intel D5-P4320/P5530, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
 - HDD: HGST Ultrastar, Toshiba MG, Seagate EXOS

 ## Configure monitors
@@ -45,7 +45,8 @@ On the monitor hosts:
  }
  ```
 - Create systemd units for etcd by running: `/usr/lib/vitastor/mon/make-etcd`
- Start etcd and monitors: `systemctl enable --now etcd vitastor-mon`
+  Or, if you installed Vitastor in Docker, run `systemctl start vitastor-host; docker exec vitastor make-etcd`.
+- Start etcd and monitors: `systemctl enable --now vitastor-etcd vitastor-mon`

 ## Configure OSDs

@@ -26,13 +26,13 @@
  обязательно возьмите SSD под метаданные и журнал (маленькие, буквально 2 ГБ на 1 ТБ HDD-места).
 - Возьмите быструю сеть, минимум 10 гбит/с. Идеал - что-то вроде Mellanox ConnectX-4 с RoCEv2.
 - Для лучшей производительности отключите энергосбережение CPU: `cpupower idle-set -D 0 && cpupower frequency-set -g performance`.
- [Установите пакеты Vitastor](../installation/packages.ru.md).
+- Либо [установите пакеты Vitastor](../installation/packages.ru.md), либо [установите Vitastor в Docker](../installation/docker.ru.md).

 ## Рекомендуемые диски

 - SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel D3-S4510/4520/4610/4620, Kingston DC500M
 - NVMe: Micron 9100/9200/9300/9400, Micron 7300/7450, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
-  Intel DC-P3700/P4500/P4600, Intel D5-P4320, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
+  Intel DC-P3700/P4500/P4600, Intel D5-P4320/P5530, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
 - HDD: HGST Ultrastar, Toshiba MG, Seagate EXOS

 ## Настройте мониторы
@@ -44,8 +44,9 @@
    "etcd_address": ["10.200.1.10:2379","10.200.1.11:2379","10.200.1.12:2379"]
  }
  ```
- Инициализируйте сервисы etcd, запустив `/usr/lib/vitastor/mon/make-etcd`
- Запустите etcd и мониторы: `systemctl enable --now etcd vitastor-mon`
+- Инициализируйте сервисы etcd, запустив `/usr/lib/vitastor/mon/make-etcd`.\
+  Либо, если вы установили Vitastor в Docker, запустите `systemctl start vitastor-host; docker exec vitastor make-etcd`.
+- Запустите etcd и мониторы: `systemctl enable --now vitastor-etcd vitastor-mon`

 ## Настройте OSD

@@ -37,7 +37,7 @@ It supports the following commands:
 Global options:

 ```
--config_file FILE   Path to Vitastor configuration file
+--config_path FILE   Path to Vitastor configuration file
 --etcd_address URL   Etcd connection address
 --iodepth N          Send N operations in parallel to each OSD when possible (default 32)
 --parallel_osds M    Work with M osds in parallel when possible (default 4)
@@ -36,7 +36,7 @@ vitastor-cli - интерфейс командной строки для адм
 Глобальные опции:

 ```
--config_file FILE   Путь к файлу конфигурации Vitastor
+--config_path FILE   Путь к файлу конфигурации Vitastor
 --etcd_address URL   Адрес соединения с etcd
 --iodepth N          Отправлять параллельно N операций на каждый OSD (по умолчанию 32)
 --parallel_osds M    Работать параллельно с M OSD (по умолчанию 4)
@@ -36,7 +36,7 @@ It will output a block device name like /dev/nbd0 which you can then use as a no

 You can also use `--pool <POOL> --inode <INODE> --size <SIZE>` instead of `--image <IMAGE>` if you want.

-vitastor-nbd supports all usual Vitastor configuration options like `--config_file <path_to_config>` plus NBD-specific:
+vitastor-nbd supports all usual Vitastor configuration options like `--config_path <path_to_config>` plus NBD-specific:

 * `--nbd_timeout 0` \
  Timeout for I/O operations in seconds after exceeding which the kernel stops the device.
@@ -54,16 +54,18 @@ vitastor-nbd supports all usual Vitastor configuration options like `--config_fi
  Stay in foreground, do not daemonize.

 Note that `nbd_timeout`, `nbd_max_devices` and `nbd_max_part` options may also be specified
-in `/etc/vitastor/vitastor.conf` or in other configuration file specified with `--config_file`.
+in `/etc/vitastor/vitastor.conf` or in other configuration file specified with `--config_path`.

 ## unmap

 To unmap the device run:

 ```
-vitastor-nbd unmap /dev/nbd0
+vitastor-nbd unmap [--force] /dev/nbd0
 ```

+If `--force` is specified, `vitastor-nbd` doesn't check if the device is actually mapped.
+
 ## ls

 ```
@@ -41,7 +41,7 @@ vitastor-nbd map [/dev/nbdN] --image testimg
 Для обращения по номеру инода, аналогично другим командам, можно использовать опции
 `--pool <POOL> --inode <INODE> --size <SIZE>` вместо `--image testimg`.

-vitastor-nbd поддерживает все обычные опции Vitastor, например, `--config_file <path_to_config>`,
+vitastor-nbd поддерживает все обычные опции Vitastor, например, `--config_path <path_to_config>`,
 плюс специфичные для NBD:

 * `--nbd_timeout 0` \
@@ -62,16 +62,19 @@ vitastor-nbd поддерживает все обычные опции Vitastor,

 Обратите внимание, что опции `nbd_timeout`, `nbd_max_devices` и `nbd_max_part` можно
 также задавать в `/etc/vitastor/vitastor.conf` или в другом файле конфигурации,
-заданном опцией `--config_file`.
+заданном опцией `--config_path`.

 ## unmap

 Для отключения устройства выполните:

 ```
-vitastor-nbd unmap /dev/nbd0
+vitastor-nbd unmap [--force] /dev/nbd0
 ```

+Если задана опция `--force`, `vitastor-nbd` не проверяет, подключено ли устройство,
+перед попыткой его отключить.
+
 ## ls

 ```
@@ -773,23 +773,27 @@ class Mon
                }
            }
        }
-        for (const pool_id in this.state.pool.stats)
+        if (!this.recheck_pgs_active)
        {
-            if (!seen_pools[pool_id])
+            // PG recheck also modifies /pool/stats, so don't touch it here if it's active
+            for (const pool_id in this.state.pool.stats)
            {
-                txn.push({ requestDeleteRange: {
-                    key: b64(this.config.etcd_prefix+'/pool/stats/'+pool_id),
-                } });
-                delete this.state.pool.stats[pool_id];
-            }
-            else
-            {
-                const pool_stats = { ...this.state.pool.stats[pool_id] };
-                serialize_bigints(pool_stats);
-                txn.push({ requestPut: {
-                    key: b64(this.config.etcd_prefix+'/pool/stats/'+pool_id),
-                    value: b64(JSON.stringify(pool_stats)),
-                } });
+                if (!seen_pools[pool_id])
+                {
+                    txn.push({ requestDeleteRange: {
+                        key: b64(this.config.etcd_prefix+'/pool/stats/'+pool_id),
+                    } });
+                    delete this.state.pool.stats[pool_id];
+                }
+                else
+                {
+                    const pool_stats = { ...this.state.pool.stats[pool_id] };
+                    serialize_bigints(pool_stats);
+                    txn.push({ requestPut: {
+                        key: b64(this.config.etcd_prefix+'/pool/stats/'+pool_id),
+                        value: b64(JSON.stringify(pool_stats)),
+                    } });
+                }
            }
        }
        if (txn.length)
@@ -1,6 +1,6 @@
 {
  "name": "vitastor-mon",
-  "version": "1.10.1",
+  "version": "1.11.0",
  "description": "Vitastor SDS monitor service",
  "main": "mon-main.js",
  "scripts": {
@@ -9,7 +9,7 @@
  "author": "Vitaliy Filippov",
  "license": "UNLICENSED",
  "dependencies": {
-    "antietcd": "^1.1.0",
+    "antietcd": "^1.1.2",
    "sprintf-js": "^1.1.2",
    "ws": "^7.2.5"
  },
@@ -8,23 +8,9 @@ const LPOptimizer = require('./lp_optimizer/lp_optimizer.js');
 const { scale_pg_count } = require('./pg_utils.js');
 const { make_hier_tree, filter_osds_by_root_node,
    filter_osds_by_tags, filter_osds_by_block_layout, get_affinity_osds } = require('./osd_tree.js');
+const { select_murmur3 } = require('./lp_optimizer/murmur3.js');

-let seed;
-
-function reset_rng()
-{
-    seed = 0x5f020e43;
-}
-
-function rng()
-{
-    seed ^= seed << 13;
-    seed ^= seed >> 17;
-    seed ^= seed << 5;
-    return seed + 2147483648;
-}
-
-function pick_primary(pool_config, osd_set, up_osds, aff_osds)
+function pick_primary(pool_id, pg_num, pool_config, osd_set, up_osds, aff_osds)
 {
    let alive_set;
    if (pool_config.scheme === 'replicated')
@@ -52,7 +38,7 @@ function pick_primary(pool_config, osd_set, up_osds, aff_osds)
    {
        return 0;
    }
-    return alive_set[rng() % alive_set.length];
+    return alive_set[select_murmur3(alive_set.length, osd_num => pool_id+'/'+pg_num+'/'+osd_num)];
 }

 function recheck_primary(state, global_config, up_osds, osd_tree)
@@ -66,7 +52,6 @@ function recheck_primary(state, global_config, up_osds, osd_tree)
            continue;
        }
        const aff_osds = get_affinity_osds(pool_cfg, up_osds, osd_tree);
-        reset_rng();
        for (let pg_num = 1; pg_num <= pool_cfg.pg_count; pg_num++)
        {
            if (!state.pg.config.items[pool_id])
@@ -76,7 +61,7 @@ function recheck_primary(state, global_config, up_osds, osd_tree)
            const pg_cfg = state.pg.config.items[pool_id][pg_num];
            if (pg_cfg)
            {
-                const new_primary = pick_primary(state.config.pools[pool_id], pg_cfg.osd_set, up_osds, aff_osds);
+                const new_primary = pick_primary(pool_id, pg_num, state.config.pools[pool_id], pg_cfg.osd_set, up_osds, aff_osds);
                if (pg_cfg.primary != new_primary)
                {
                    if (!new_pg_config)
@@ -99,13 +84,12 @@ function save_new_pgs_txn(save_to, request, state, etcd_prefix, etcd_watch_revis
 {
    const aff_osds = get_affinity_osds(state.config.pools[pool_id] || {}, up_osds, osd_tree);
    const pg_items = {};
-    reset_rng();
    new_pgs.map((osd_set, i) =>
    {
        osd_set = osd_set.map(osd_num => osd_num === LPOptimizer.NO_OSD ? 0 : osd_num);
        pg_items[i+1] = {
            osd_set,
-            primary: pick_primary(state.config.pools[pool_id], osd_set, up_osds, aff_osds),
+            primary: pick_primary(pool_id, i+1, state.config.pools[pool_id], osd_set, up_osds, aff_osds),
        };
        if (prev_pgs[i] && prev_pgs[i].join(' ') != osd_set.join(' ') &&
            prev_pgs[i].filter(osd_num => osd_num).length > 0)
@@ -33,9 +33,11 @@ async function run()
        console.log(config_path+' is missing');
        process.exit(1);
    }
-    if (fs.existsSync("/etc/systemd/system/etcd.service"))
+    const in_docker = fs.existsSync("/etc/vitastor/etcd.conf") &&
+        fs.existsSync("/etc/vitastor/docker.conf");
+    if (!in_docker && fs.existsSync("/etc/systemd/system/vitastor-etcd.service"))
    {
-        console.log("/etc/systemd/system/etcd.service already exists");
+        console.log("/etc/systemd/system/vitastor-etcd.service already exists");
        process.exit(1);
    }
    const config = JSON.parse(fs.readFileSync(config_path, { encoding: 'utf-8' }));
@@ -52,10 +54,21 @@ async function run()
        console.log('No matching IPs in etcd_address from '+config_path);
        process.exit(0);
    }
-    const etcd_cluster = etcds.map((e, i) => `etcd${i}=http://${e}:2380`).join(',');
-    await system(`mkdir -p /var/lib/etcd${num}.etcd`);
+    const etcd_name = 'etcd'+etcds[num].replace(/[^0-9a-z_]/ig, '_');
+    const etcd_cluster = etcds.map(e => `etcd${e.replace(/[^0-9a-z_]/ig, '_')}=http://${e}:2380`).join(',');
+    if (in_docker)
+    {
+        let etcd_conf = fs.readFileSync("/etc/vitastor/etcd.conf", { encoding: 'utf-8' });
+        etcd_conf = replace_env(etcd_conf, 'ETCD_NAME', etcd_name);
+        etcd_conf = replace_env(etcd_conf, 'ETCD_IP', etcds[num]);
+        etcd_conf = replace_env(etcd_conf, 'ETCD_INITIAL_CLUSTER', etcd_cluster);
+        fs.writeFileSync("/etc/vitastor/etcd.conf", etcd_conf);
+        console.log('etcd for Vitastor configured. Run `systemctl enable --now vitastor-etcd` to start etcd');
+        process.exit(0);
+    }
+    await system(`mkdir -p /var/lib/etcd`);
    fs.writeFileSync(
-        "/etc/systemd/system/etcd.service",
+        "/etc/systemd/system/vitastor-etcd.service",
 `[Unit]
 Description=etcd for vitastor
 After=network-online.target local-fs.target time-sync.target
@@ -64,14 +77,14 @@ Wants=network-online.target local-fs.target time-sync.target
 [Service]
 Restart=always
 Environment=GOGC=50
-ExecStart=etcd -name etcd${num} --data-dir /var/lib/etcd${num}.etcd \\
+ExecStart=etcd --name ${etcd_name} --data-dir /var/lib/etcd \\
    --snapshot-count 10000 --advertise-client-urls http://${etcds[num]}:2379 --listen-client-urls http://${etcds[num]}:2379 \\
    --initial-advertise-peer-urls http://${etcds[num]}:2380 --listen-peer-urls http://${etcds[num]}:2380 \\
    --initial-cluster-token vitastor-etcd-1 --initial-cluster ${etcd_cluster} \\
    --initial-cluster-state new --max-txn-ops=100000 --max-request-bytes=104857600 \\
    --auto-compaction-retention=10 --auto-compaction-mode=revision
-WorkingDirectory=/var/lib/etcd${num}.etcd
-ExecStartPre=+chown -R etcd /var/lib/etcd${num}.etcd
+WorkingDirectory=/var/lib/etcd
+ExecStartPre=+chown -R etcd /var/lib/etcd
 User=etcd
 PrivateTmp=false
 TasksMax=infinity
@@ -89,6 +102,13 @@ WantedBy=multi-user.target
    process.exit(0);
 }

+function replace_env(text, key, value)
+{
+    let found = false;
+    text = text.replace(new RegExp('^'+key+'\\s*=.*', 'm'), () => { found = true; return key+'='+value; });
+    return found ? text : text.replace(/\s*$/, '\n')+key+'='+value+'\n';
+}
+
 function select_local_etcd(etcds)
 {
    const ifaces = os.networkInterfaces();
@@ -5,6 +5,7 @@ Wants=network-online.target local-fs.target time-sync.target

 [Service]
 Restart=always
+SyslogIdentifier=vitastor-mon
 ExecStart=node /usr/lib/vitastor/mon/mon-main.js
 WorkingDirectory=/
 User=vitastor
@@ -8,6 +8,7 @@ PartOf=vitastor.target
 LimitNOFILE=1048576
 LimitNPROC=1048576
 LimitMEMLOCK=infinity
+SyslogIdentifier=vitastor-osd%i
 # Use the following for direct logs to files
 #ExecStart=bash -c 'exec vitastor-disk exec-osd /dev/vitastor/osd%i-data >>/var/log/vitastor/osd%i.log 2>&1'
 ExecStart=vitastor-disk exec-osd /dev/vitastor/osd%i-data
@@ -14,8 +14,13 @@ NAN_MODULE_INIT(InitAddon)

    Nan::SetPrototypeMethod(tpl, "read", NodeVitastor::Read);
    Nan::SetPrototypeMethod(tpl, "write", NodeVitastor::Write);
+    Nan::SetPrototypeMethod(tpl, "delete", NodeVitastor::Delete);
    Nan::SetPrototypeMethod(tpl, "sync", NodeVitastor::Sync);
    Nan::SetPrototypeMethod(tpl, "read_bitmap", NodeVitastor::ReadBitmap);
+    Nan::SetPrototypeMethod(tpl, "on_ready", NodeVitastor::OnReady);
+    Nan::SetPrototypeMethod(tpl, "get_min_io_size", NodeVitastor::GetMinIoSize);
+    Nan::SetPrototypeMethod(tpl, "get_max_atomic_write_size", NodeVitastor::GetMaxAtomicWriteSize);
+    Nan::SetPrototypeMethod(tpl, "get_immediate_commit", NodeVitastor::GetImmediateCommit);
    //Nan::SetPrototypeMethod(tpl, "destroy", NodeVitastor::Destroy);

    Nan::Set(target, Nan::New("Client").ToLocalChecked(), Nan::GetFunction(tpl).ToLocalChecked());
@@ -63,6 +68,10 @@ NAN_MODULE_INIT(InitAddon)
    Nan::Set(target, Nan::New("ENOSYS").ToLocalChecked(), Nan::New<v8::Int32>(-ENOSYS));
    Nan::Set(target, Nan::New("EAGAIN").ToLocalChecked(), Nan::New<v8::Int32>(-EAGAIN));

+    Nan::Set(target, Nan::New("IMMEDIATE_NONE").ToLocalChecked(), Nan::New<v8::Int32>(IMMEDIATE_NONE));
+    Nan::Set(target, Nan::New("IMMEDIATE_SMALL").ToLocalChecked(), Nan::New<v8::Int32>(IMMEDIATE_SMALL));
+    Nan::Set(target, Nan::New("IMMEDIATE_ALL").ToLocalChecked(), Nan::New<v8::Int32>(IMMEDIATE_ALL));
+
    // Listing handle

    tpl = Nan::New<v8::FunctionTemplate>(NodeVitastorKVListing::Create);
@@ -5,9 +5,10 @@

 #define NODE_VITASTOR_READ 1
 #define NODE_VITASTOR_WRITE 2
-#define NODE_VITASTOR_SYNC 3
-#define NODE_VITASTOR_READ_BITMAP 4
-#define NODE_VITASTOR_GET_INFO 5
+#define NODE_VITASTOR_DELETE 3
+#define NODE_VITASTOR_SYNC 4
+#define NODE_VITASTOR_READ_BITMAP 5
+#define NODE_VITASTOR_GET_INFO 6

 #ifndef INODE_POOL
 #define INODE_POOL(inode) (uint32_t)((inode) >> (64 - POOL_ID_BITS))
@@ -80,6 +81,11 @@ NAN_METHOD(NodeVitastor::Create)
    NodeVitastor* cli = new NodeVitastor();
    cli->c = vitastor_c_create_uring_json(c_cfg, cfg.size());
    delete[] c_cfg;
+    if (!cli->c)
+    {
+        Nan::ThrowError("failed to initialize io_uring (old kernel or insufficient ulimit -l?)");
+        return;
+    }

    int res = vitastor_c_uring_register_eventfd(cli->c);
    if (res >= 0)
@@ -133,12 +139,12 @@ NodeVitastorRequest* NodeVitastor::get_read_request(const Nan::FunctionCallbackI
    return req;
 }

-// read(pool, inode, offset, len, callback(err, buffer, version))
+// read(pool, inode, offset, length, callback(err, buffer, version))
 NAN_METHOD(NodeVitastor::Read)
 {
    TRACE("NodeVitastor::Read");
    if (info.Length() < 5)
-        Nan::ThrowError("Not enough arguments to read(pool, inode, offset, len, callback(err, buffer, version))");
+        Nan::ThrowError("Not enough arguments to read(pool, inode, offset, length, callback(err, buffer, version))");

    NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());

@@ -149,6 +155,9 @@ NAN_METHOD(NodeVitastor::Read)

    self->Ref();
    vitastor_c_read(self->c, ((pool << (64-POOL_ID_BITS)) | inode), req->offset, req->len, &req->iov, 1, on_read_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(self->c);
+#endif
 }

 NodeVitastorRequest* NodeVitastor::get_write_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos)
@@ -217,6 +226,58 @@ NAN_METHOD(NodeVitastor::Write)
        req->iov_list.size() ? req->iov_list.data() : &req->iov,
        req->iov_list.size() ? req->iov_list.size() : 1,
        on_write_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(self->c);
+#endif
+}
+
+NodeVitastorRequest* NodeVitastor::get_delete_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos)
+{
+    uint64_t offset = get_ui64(info[argpos+0]);
+    uint64_t len = get_ui64(info[argpos+1]);
+    uint64_t version = 0;
+    if (!info[argpos+2].IsEmpty() &&
+        !info[argpos+2]->IsFunction() &&
+        info[argpos+2]->IsObject())
+    {
+        auto key = Nan::New<v8::String>("version").ToLocalChecked();
+        auto params = info[argpos+2].As<v8::Object>();
+        auto versionObj = Nan::Get(params, key).ToLocalChecked();
+        if (!versionObj.IsEmpty())
+            version = get_ui64(versionObj);
+        argpos++;
+    }
+
+    v8::Local<v8::Function> callback = info[argpos+2].As<v8::Function>();
+    auto req = new NodeVitastorRequest(this, callback);
+
+    req->offset = offset;
+    req->len = len;
+    req->version = version;
+
+    return req;
+}
+
+// delete(pool, inode, offset, length, { version }?, callback(err))
+NAN_METHOD(NodeVitastor::Delete)
+{
+    TRACE("NodeVitastor::Delete");
+    if (info.Length() < 5)
+        Nan::ThrowError("Not enough arguments to delete(pool, inode, offset, length, { version }?, callback(err))");
+
+    NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
+
+    uint64_t pool = get_ui64(info[0]);
+    uint64_t inode = get_ui64(info[1]);
+
+    auto req = self->get_delete_request(info, 2);
+
+    self->Ref();
+    vitastor_c_delete(self->c, ((pool << (64-POOL_ID_BITS)) | inode), req->offset, req->len, req->version,
+        on_write_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(self->c);
+#endif
 }

 // sync(callback(err))
@@ -233,14 +294,17 @@ NAN_METHOD(NodeVitastor::Sync)

    self->Ref();
    vitastor_c_sync(self->c, on_write_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(self->c);
+#endif
 }

-// read_bitmap(pool, inode, offset, len, with_parents, callback(err, bitmap_buffer))
+// read_bitmap(pool, inode, offset, length, with_parents, callback(err, bitmap_buffer))
 NAN_METHOD(NodeVitastor::ReadBitmap)
 {
    TRACE("NodeVitastor::ReadBitmap");
    if (info.Length() < 6)
-        Nan::ThrowError("Not enough arguments to read_bitmap(pool, inode, offset, len, with_parents, callback(err, bitmap_buffer))");
+        Nan::ThrowError("Not enough arguments to read_bitmap(pool, inode, offset, length, with_parents, callback(err, bitmap_buffer))");

    NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());

@@ -254,6 +318,9 @@ NAN_METHOD(NodeVitastor::ReadBitmap)
    auto req = new NodeVitastorRequest(self, callback);
    self->Ref();
    vitastor_c_read_bitmap(self->c, ((pool << (64-POOL_ID_BITS)) | inode), offset, len, with_parents, on_read_bitmap_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(self->c);
+#endif
 }

 static void on_error(NodeVitastorRequest *req, Nan::Callback & nanCallback, long retval)
@@ -267,6 +334,67 @@ static void on_error(NodeVitastorRequest *req, Nan::Callback & nanCallback, long
    nanCallback.Call(1, args, req);
 }

+// on_ready(callback(err))
+NAN_METHOD(NodeVitastor::OnReady)
+{
+    TRACE("NodeVitastor::OnReady");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to on_ready(callback(err))");
+    NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
+    v8::Local<v8::Function> callback = info[0].As<v8::Function>();
+    auto req = new NodeVitastorRequest(self, callback);
+    self->Ref();
+    vitastor_c_on_ready(self->c, on_ready_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(self->c);
+#endif
+}
+
+void NodeVitastor::on_ready_finish(void *opaque, long retval)
+{
+    TRACE("NodeVitastor::on_ready_finish");
+    auto req = (NodeVitastorRequest*)opaque;
+    auto self = req->cli;
+    Nan::HandleScope scope;
+    Nan::Callback nanCallback(Nan::New(req->callback));
+    nanCallback.Call(0, NULL, req);
+    self->Unref();
+    delete req;
+}
+
+// get_min_io_size(pool_id)
+NAN_METHOD(NodeVitastor::GetMinIoSize)
+{
+    TRACE("NodeVitastor::GetMinIoSize");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to get_min_io_size(pool_id)");
+    NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
+    uint64_t pool = get_ui64(info[0]);
+    info.GetReturnValue().Set(Nan::New<v8::Number>(vitastor_c_inode_get_bitmap_granularity(self->c, INODE_WITH_POOL(pool, 1))));
+}
+
+// get_max_atomic_write_size(pool_id)
+NAN_METHOD(NodeVitastor::GetMaxAtomicWriteSize)
+{
+    TRACE("NodeVitastor::GetMaxAtomicWriteSize");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to get_max_atomic_write_size(pool_id)");
+    NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
+    uint64_t pool = get_ui64(info[0]);
+    info.GetReturnValue().Set(Nan::New<v8::Number>(vitastor_c_inode_get_block_size(self->c, INODE_WITH_POOL(pool, 1))));
+}
+
+// get_immediate_commit(pool_id)
+NAN_METHOD(NodeVitastor::GetImmediateCommit)
+{
+    TRACE("NodeVitastor::GetImmediateCommit");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to get_immediate_commit(pool_id)");
+    NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
+    uint64_t pool = get_ui64(info[0]);
+    info.GetReturnValue().Set(Nan::New<v8::Number>(vitastor_c_inode_get_immediate_commit(self->c, INODE_WITH_POOL(pool, 1))));
+}
+
 void NodeVitastor::on_read_finish(void *opaque, long retval, uint64_t version)
 {
    TRACE("NodeVitastor::on_read_finish");
@@ -364,6 +492,9 @@ NAN_METHOD(NodeVitastorImage::Create)
    img->Ref();
    cli->Ref();
    vitastor_c_watch_inode(cli->c, (char*)img->name.c_str(), on_watch_start, img);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(cli->c);
+#endif

    info.GetReturnValue().Set(info.This());
 }
@@ -378,12 +509,12 @@ NodeVitastorImage::~NodeVitastorImage()
    cli->Unref();
 }

-// read(offset, len, callback(err, buffer, version))
+// read(offset, length, callback(err, buffer, version))
 NAN_METHOD(NodeVitastorImage::Read)
 {
    TRACE("NodeVitastorImage::Read");
    if (info.Length() < 3)
-        Nan::ThrowError("Not enough arguments to read(offset, len, callback(err, buffer, version))");
+        Nan::ThrowError("Not enough arguments to read(offset, length, callback(err, buffer, version))");

    NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());

@@ -394,12 +525,12 @@ NAN_METHOD(NodeVitastorImage::Read)
    img->exec_or_wait(req);
 }

-// write(offset, buffer, { version }?, callback(err))
+// write(offset, buf: Buffer | Buffer[], { version }?, callback(err))
 NAN_METHOD(NodeVitastorImage::Write)
 {
    TRACE("NodeVitastorImage::Write");
    if (info.Length() < 3)
-        Nan::ThrowError("Not enough arguments to write(offset, buffer, { version }?, callback(err))");
+        Nan::ThrowError("Not enough arguments to write(offset, buf: Buffer | Buffer[], { version }?, callback(err))");

    NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());

@@ -410,6 +541,22 @@ NAN_METHOD(NodeVitastorImage::Write)
    img->exec_or_wait(req);
 }

+// delete(offset, length, { version }?, callback(err))
+NAN_METHOD(NodeVitastorImage::Delete)
+{
+    TRACE("NodeVitastorImage::Delete");
+    if (info.Length() < 3)
+        Nan::ThrowError("Not enough arguments to delete(offset, length, { version }?, callback(err))");
+
+    NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
+
+    auto req = img->cli->get_delete_request(info, 0);
+    req->img = img;
+    req->op = NODE_VITASTOR_DELETE;
+
+    img->exec_or_wait(req);
+}
+
 // sync(callback(err))
 NAN_METHOD(NodeVitastorImage::Sync)
 {
@@ -427,12 +574,12 @@ NAN_METHOD(NodeVitastorImage::Sync)
    img->exec_or_wait(req);
 }

-// read_bitmap(offset, len, with_parents, callback(err, bitmap_buffer))
+// read_bitmap(offset, length, with_parents, callback(err, bitmap_buffer))
 NAN_METHOD(NodeVitastorImage::ReadBitmap)
 {
    TRACE("NodeVitastorImage::ReadBitmap");
    if (info.Length() < 4)
-        Nan::ThrowError("Not enough arguments to read_bitmap(offset, len, with_parents, callback(err, bitmap_buffer))");
+        Nan::ThrowError("Not enough arguments to read_bitmap(offset, length, with_parents, callback(err, bitmap_buffer))");

    NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());

@@ -488,6 +635,9 @@ void NodeVitastorImage::exec_request(NodeVitastorRequest *req)
        uint64_t ino = vitastor_c_inode_get_num(watch);
        cli->Ref();
        vitastor_c_read(cli->c, ino, req->offset, req->len, &req->iov, 1, NodeVitastor::on_read_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+        vitastor_c_uring_handle_events(cli->c);
+#endif
    }
    else if (req->op == NODE_VITASTOR_WRITE)
    {
@@ -497,6 +647,19 @@ void NodeVitastorImage::exec_request(NodeVitastorRequest *req)
            req->iov_list.size() ? req->iov_list.data() : &req->iov,
            req->iov_list.size() ? req->iov_list.size() : 1,
            NodeVitastor::on_write_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+        vitastor_c_uring_handle_events(cli->c);
+#endif
+    }
+    else if (req->op == NODE_VITASTOR_DELETE)
+    {
+        uint64_t ino = vitastor_c_inode_get_num(watch);
+        cli->Ref();
+        vitastor_c_delete(cli->c, ino, req->offset, req->len, req->version,
+            NodeVitastor::on_write_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+        vitastor_c_uring_handle_events(cli->c);
+#endif
    }
    else if (req->op == NODE_VITASTOR_SYNC)
    {
@@ -506,6 +669,9 @@ void NodeVitastorImage::exec_request(NodeVitastorRequest *req)
        if (imm != IMMEDIATE_ALL)
        {
            vitastor_c_sync(cli->c, NodeVitastor::on_write_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+            vitastor_c_uring_handle_events(cli->c);
+#endif
        }
        else
        {
@@ -517,6 +683,9 @@ void NodeVitastorImage::exec_request(NodeVitastorRequest *req)
        uint64_t ino = vitastor_c_inode_get_num(watch);
        cli->Ref();
        vitastor_c_read_bitmap(cli->c, ino, req->offset, req->len, req->with_parents, NodeVitastor::on_read_bitmap_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+        vitastor_c_uring_handle_events(cli->c);
+#endif
    }
    else if (req->op == NODE_VITASTOR_GET_INFO)
    {
@@ -648,6 +817,9 @@ NAN_METHOD(NodeVitastorKV::Open)
        delete req;
        kv->Unref();
    });
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(kv->cli->c);
+#endif
 }

 // close(callback(err))
@@ -671,6 +843,9 @@ NAN_METHOD(NodeVitastorKV::Close)
        delete req;
        kv->Unref();
    });
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(kv->cli->c);
+#endif
 }

 // set_config({ ...config })
@@ -729,6 +904,9 @@ void NodeVitastorKV::get_impl(const Nan::FunctionCallbackInfo<v8::Value> & info,
        delete req;
        kv->Unref();
    }, allow_cache);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(kv->cli->c);
+#endif
 }

 // get(key, callback(err, value))
@@ -801,6 +979,9 @@ NAN_METHOD(NodeVitastorKV::Set)
            delete cas_req;
        kv->Unref();
    }, cas_cb);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(kv->cli->c);
+#endif
 }

 // del(key, callback(err), cas_compare(old_value)?)
@@ -839,6 +1020,9 @@ NAN_METHOD(NodeVitastorKV::Del)
            delete cas_req;
        kv->Unref();
    }, cas_cb);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(kv->cli->c);
+#endif
 }

 // list(start_key?)
@@ -909,7 +1093,7 @@ NodeVitastorKVListing::~NodeVitastorKVListing()
    kv->Unref();
 }

-// next(callback(err, value)?)
+// next(callback(err, key, value)?)
 NAN_METHOD(NodeVitastorKVListing::Next)
 {
    TRACE("NodeVitastorKVListing::Next");
@@ -959,6 +1143,9 @@ NAN_METHOD(NodeVitastorKVListing::Next)
            list->iter = req;
        list->kv->Unref();
    });
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(list->kv->cli->c);
+#endif
 }

 // close()
@@ -15,14 +15,24 @@ class NodeVitastor: public Nan::ObjectWrap
 public:
    // constructor({ ...config })
    static NAN_METHOD(Create);
-    // read(pool, inode, offset, len, callback(err, buffer, version))
+    // read(pool_id, inode, offset, length, callback(err, buffer, version))
    static NAN_METHOD(Read);
-    // write(pool, inode, offset, buf: Buffer | Buffer[], { version }?, callback(err))
+    // write(pool_id, inode, offset, buf: Buffer | Buffer[], { version }?, callback(err))
    static NAN_METHOD(Write);
+    // delete(pool_id, inode, offset, length, { version }?, callback(err))
+    static NAN_METHOD(Delete);
    // sync(callback(err))
    static NAN_METHOD(Sync);
-    // read_bitmap(pool, inode, offset, len, with_parents, callback(err, bitmap_buffer))
+    // read_bitmap(pool_id, inode, offset, length, with_parents, callback(err, bitmap_buffer))
    static NAN_METHOD(ReadBitmap);
+    // on_ready(callback(err))
+    static NAN_METHOD(OnReady);
+    // get_min_io_size(pool_id)
+    static NAN_METHOD(GetMinIoSize);
+    // get_max_atomic_write_size(pool_id)
+    static NAN_METHOD(GetMaxAtomicWriteSize);
+    // get_immediate_commit(pool_id)
+    static NAN_METHOD(GetImmediateCommit);
 //    // destroy()
 //    static NAN_METHOD(Destroy);

@@ -37,11 +47,13 @@ private:

    static void on_io_readable(uv_poll_t* handle, int status, int revents);
    static void on_read_finish(void *opaque, long retval, uint64_t version);
+    static void on_ready_finish(void *opaque, long retval);
    static void on_write_finish(void *opaque, long retval);
    static void on_read_bitmap_finish(void *opaque, long retval, uint8_t *bitmap);

    NodeVitastorRequest* get_read_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos);
    NodeVitastorRequest* get_write_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos);
+    NodeVitastorRequest* get_delete_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos);

    friend class NodeVitastorImage;
    friend class NodeVitastorKV;
@@ -53,13 +65,15 @@ class NodeVitastorImage: public Nan::ObjectWrap
 public:
    // constructor(node_vitastor, name)
    static NAN_METHOD(Create);
-    // read(offset, len, callback(err, buffer, version))
+    // read(offset, length, callback(err, buffer, version))
    static NAN_METHOD(Read);
    // write(offset, buf: Buffer | Buffer[], { version }?, callback(err))
    static NAN_METHOD(Write);
+    // delete(offset, length, { version }?, callback(err))
+    static NAN_METHOD(Delete);
    // sync(callback(err))
    static NAN_METHOD(Sync);
-    // read_bitmap(offset, len, with_parents, callback(err, bitmap_buffer))
+    // read_bitmap(offset, length, with_parents, callback(err, bitmap_buffer))
    static NAN_METHOD(ReadBitmap);
    // get_info(callback({ num, name, size, parent_id?, readonly?, meta?, mod_revision, block_size, bitmap_granularity, immediate_commit }))
    static NAN_METHOD(GetInfo);
@@ -120,7 +134,7 @@ class NodeVitastorKVListing: public Nan::ObjectWrap
 public:
    // constructor(node_vitastor_kv, start_key?)
    static NAN_METHOD(Create);
-    // next(callback(err, value)?)
+    // next(callback(err, key, value)?)
    static NAN_METHOD(Next);
    // close()
    static NAN_METHOD(Close);
@@ -1,6 +1,6 @@
 {
  "name": "vitastor",
-  "version": "1.7.0",
+  "version": "1.11.0",
  "description": "Low-level native bindings to Vitastor client library",
  "main": "index.js",
  "keywords": [
@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils

-VITASTOR_VERSION = '1.10.1'
+VITASTOR_VERSION = '1.11.0'

 LOG = logging.getLogger(__name__)

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.10.1
+Version:        1.11.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage

 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.10.1.el7.tar.gz
+Source0:        vitastor-1.11.0.el7.tar.gz

 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel
@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.10.1
+Version:        1.11.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage

 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.10.1.el8.tar.gz
+Source0:        vitastor-1.11.0.el8.tar.gz

 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel
@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.10.1
+Version:        1.11.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage

 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.10.1.el9.tar.gz
+Source0:        vitastor-1.11.0.el9.tar.gz

 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel
@@ -19,7 +19,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()

-add_definitions(-DVITASTOR_VERSION="1.10.1")
+add_definitions(-DVITASTOR_VERSION="1.11.0")
 add_definitions(-D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})
@@ -256,6 +256,7 @@ resume_2:
    }
    if (entries_to_zero.size() && !bs->inmemory_meta && !bs->readonly)
    {
+        std::sort(entries_to_zero.begin(), entries_to_zero.end());
        // we have to zero out additional entries
        for (i = 0; i < entries_to_zero.size(); )
        {
@@ -338,6 +339,15 @@ bool blockstore_init_meta::handle_meta_block(uint8_t *buf, uint64_t entries_per_
                if (*entry_csum != crc32c(0, entry, bs->dsk.clean_entry_size - 4))
                {
                    printf("Metadata entry %ju is corrupt (checksum mismatch), skipping\n", done_cnt+i);
+                    // zero out the invalid entry, otherwise we'll hit "tried to overwrite non-zero metadata entry" later
+                    if (bs->inmemory_meta)
+                    {
+                        memset(entry, 0, bs->dsk.clean_entry_size);
+                    }
+                    else
+                    {
+                        entries_to_zero.push_back(done_cnt+i);
+                    }
                    continue;
                }
            }
@@ -63,6 +63,10 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd

    st_cli.parse_config(config);
    st_cli.infinite_start = false;
+    if (!config["client_infinite_start"].is_null())
+    {
+        st_cli.infinite_start = config["client_infinite_start"].bool_value();
+    }
    st_cli.load_global_config();

    scrap_buffer_size = SCRAP_BUFFER_SIZE;
@@ -309,16 +313,17 @@ void cluster_client_t::erase_op(cluster_op_t *op)
    }
    if (flags & OP_FLUSH_BUFFER)
    {
+        auto overflow = std::move(wb->writeback_overflow);
        int i = 0;
-        while (i < wb->writeback_overflow.size() && wb->writebacks_active < client_max_writeback_iodepth)
+        while (i < overflow.size() && wb->writebacks_active < client_max_writeback_iodepth)
        {
-            execute_internal(wb->writeback_overflow[i]);
+            execute_internal(overflow[i]);
            i++;
        }
        if (i > 0)
-        {
-            wb->writeback_overflow.erase(wb->writeback_overflow.begin(), wb->writeback_overflow.begin()+i);
-        }
+            overflow.erase(overflow.begin(), overflow.begin()+i);
+        assert(!wb->writeback_overflow.size());
+        wb->writeback_overflow.swap(overflow);
    }
 }

@@ -673,7 +678,7 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
        }
        // Just copy and acknowledge the operation
        wb->copy_write(op, CACHE_DIRTY);
-        while (wb->writeback_bytes + op->len > client_max_buffered_bytes || wb->writeback_queue_size > client_max_buffered_ops)
+        while (wb->writeback_bytes > client_max_buffered_bytes || wb->writeback_queue_size > client_max_buffered_ops)
        {
            // Initiate some writeback (asynchronously)
            wb->start_writebacks(this, 1);
@@ -791,6 +796,36 @@ bool cluster_client_t::check_rw(cluster_op_t *op)
            return false;
        }
    }
+    op->deoptimise_snapshot = false;
+    if (enable_writeback && (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP))
+    {
+        auto ino_it = st_cli.inode_config.find(op->inode);
+        if (ino_it != st_cli.inode_config.end())
+        {
+            int chain_size = 0;
+            while (ino_it != st_cli.inode_config.end() && ino_it->second.parent_id)
+            {
+                // Check for loops - FIXME check it in etcd_state_client
+                if (ino_it->second.parent_id == op->inode ||
+                    chain_size > st_cli.inode_config.size())
+                {
+                    op->retval = -EINVAL;
+                    auto cb = std::move(op->callback);
+                    cb(op);
+                    return false;
+                }
+                if (INODE_POOL(ino_it->second.parent_id) == INODE_POOL(ino_it->first) &&
+                    wb->has_inode(ino_it->second.parent_id))
+                {
+                    // Deoptimise reads - we have dirty data for one of the parent layer(s).
+                    op->deoptimise_snapshot = true;
+                    break;
+                }
+                chain_size++;
+                ino_it = st_cli.inode_config.find(ino_it->second.parent_id);
+            }
+        }
+    }
    return true;
 }

@@ -922,12 +957,21 @@ resume_2:
        {
            // Check parent inode
            auto ino_it = st_cli.inode_config.find(op->cur_inode);
-            while (ino_it != st_cli.inode_config.end() && ino_it->second.parent_id &&
-                INODE_POOL(ino_it->second.parent_id) == INODE_POOL(op->cur_inode) &&
-                // Check for loops
-                ino_it->second.parent_id != op->inode)
+            // Skip parents from the same pool
+            int skipped = 0;
+            while (!op->deoptimise_snapshot &&
+                ino_it != st_cli.inode_config.end() && ino_it->second.parent_id &&
+                INODE_POOL(ino_it->second.parent_id) == INODE_POOL(op->cur_inode))
            {
-                // Skip parents from the same pool
+                // Check for loops - FIXME check it in etcd_state_client
+                if (ino_it->second.parent_id == op->inode ||
+                    skipped > st_cli.inode_config.size())
+                {
+                    op->retval = -EINVAL;
+                    erase_op(op);
+                    return 1;
+                }
+                skipped++;
                ino_it = st_cli.inode_config.find(ino_it->second.parent_id);
            }
            if (ino_it != st_cli.inode_config.end() &&
@@ -1106,7 +1150,7 @@ void cluster_client_t::slice_rw(cluster_op_t *op)
            if (end == begin)
            {
                op->done_count++;
-                op->parts[i].flags = PART_DONE;
+                op->parts[i].flags = PART_SENT|PART_DONE;
            }
        }
        else if (op->opcode != OSD_OP_READ_BITMAP && op->opcode != OSD_OP_READ_CHAIN_BITMAP && op->opcode != OSD_OP_DELETE)
@@ -1189,7 +1233,7 @@ int cluster_client_t::try_send(cluster_op_t *op, int i)
                pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks
            );
            uint64_t meta_rev = 0;
-            if (op->opcode != OSD_OP_READ_BITMAP && op->opcode != OSD_OP_DELETE)
+            if (op->opcode != OSD_OP_READ_BITMAP && op->opcode != OSD_OP_DELETE && !op->deoptimise_snapshot)
            {
                auto ino_it = st_cli.inode_config.find(op->cur_inode);
                if (ino_it != st_cli.inode_config.end())
@@ -60,7 +60,8 @@ struct cluster_op_t
 protected:
    int state = 0;
    uint64_t cur_inode; // for snapshot reads
-    bool needs_reslice = false;
+    bool needs_reslice: 1;
+    bool deoptimise_snapshot: 1;
    int retry_after = 0;
    int inflight_count = 0, done_count = 0;
    timespec wait_up_until = {};
@@ -42,10 +42,10 @@ public:
    std::multimap<uint64_t, uint64_t*> flushed_buffers; // flush_id => refcnt

    ~writeback_cache_t();
+    bool has_inode(uint64_t inode);
    dirty_buf_it_t find_dirty(uint64_t inode, uint64_t offset);
    bool is_left_merged(dirty_buf_it_t dirty_it);
    bool is_right_merged(dirty_buf_it_t dirty_it);
-    bool is_merged(const dirty_buf_it_t & dirty_it);
    void copy_write(cluster_op_t *op, int state, uint64_t new_flush_id = 0);
    int repeat_ops_for(cluster_client_t *cli, osd_num_t peer_osd, pool_id_t pool_id, pg_num_t pg_num);
    void start_writebacks(cluster_client_t *cli, int count);
@@ -17,6 +17,15 @@ writeback_cache_t::~writeback_cache_t()
    dirty_buffers.clear();
 }

+bool writeback_cache_t::has_inode(uint64_t inode)
+{
+    auto dirty_it = dirty_buffers.lower_bound((object_id){
+        .inode = inode,
+        .stripe = 0,
+    });
+    return dirty_it != dirty_buffers.end() && dirty_it->first.inode == inode;
+}
+
 dirty_buf_it_t writeback_cache_t::find_dirty(uint64_t inode, uint64_t offset)
 {
    auto dirty_it = dirty_buffers.lower_bound((object_id){
@@ -33,7 +42,11 @@ dirty_buf_it_t writeback_cache_t::find_dirty(uint64_t inode, uint64_t offset)
            break;
        }
    }
-    return dirty_it;
+    if (dirty_it != dirty_buffers.end() && dirty_it->first.inode == inode)
+    {
+        return dirty_it;
+    }
+    return dirty_buffers.end();
 }

 bool writeback_cache_t::is_left_merged(dirty_buf_it_t dirty_it)
@@ -68,11 +81,6 @@ bool writeback_cache_t::is_right_merged(dirty_buf_it_t dirty_it)
    return false;
 }

-bool writeback_cache_t::is_merged(const dirty_buf_it_t & dirty_it)
-{
-    return is_left_merged(dirty_it) || is_right_merged(dirty_it);
-}
-
 void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flush_id)
 {
    // Save operation for replay when one of PGs goes out of sync
@@ -110,7 +118,10 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
                (*dirty_it->second.refcnt)++;
                if (dirty_it->second.state == CACHE_DIRTY)
                {
-                    writeback_bytes -= op->len;
+                    if (dirty_it->second.buf)
+                    {
+                        writeback_bytes -= op->len;
+                    }
                    writeback_queue_size++;
                }
                break;
@@ -120,8 +131,11 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
                // Only leave the beginning
                if (dirty_it->second.state == CACHE_DIRTY)
                {
-                    writeback_bytes -= old_end - op->offset;
-                    if (is_left_merged(dirty_it) && !is_right_merged(dirty_it))
+                    if (dirty_it->second.buf)
+                    {
+                        writeback_bytes -= old_end - op->offset;
+                    }
+                    if (is_right_merged(dirty_it))
                    {
                        writeback_queue_size++;
                    }
@@ -135,8 +149,11 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
            // Only leave the end
            if (dirty_it->second.state == CACHE_DIRTY)
            {
-                writeback_bytes -= new_end - dirty_it->first.stripe;
-                if (!is_left_merged(dirty_it) && is_right_merged(dirty_it))
+                if (dirty_it->second.buf)
+                {
+                    writeback_bytes -= new_end - dirty_it->first.stripe;
+                }
+                if (is_left_merged(dirty_it))
                {
                    writeback_queue_size++;
                }
@@ -158,11 +175,23 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
        else
        {
            // Remove the whole buffer
-            if (dirty_it->second.state == CACHE_DIRTY && !is_merged(dirty_it))
+            if (dirty_it->second.state == CACHE_DIRTY)
            {
-                writeback_bytes -= dirty_it->second.len;
-                assert(writeback_queue_size > 0);
-                writeback_queue_size--;
+                if (dirty_it->second.buf)
+                {
+                    writeback_bytes -= dirty_it->second.len;
+                }
+                bool lm = is_left_merged(dirty_it);
+                bool rm = is_right_merged(dirty_it);
+                if (!lm && !rm)
+                {
+                    assert(writeback_queue_size > 0);
+                    writeback_queue_size--;
+                }
+                else if (lm && rm)
+                {
+                    writeback_queue_size++;
+                }
            }
            if (!--(*dirty_it->second.refcnt))
            {
@@ -190,7 +219,9 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
    {
        writeback_bytes += is_del ? 0 : op->len;
        // Track consecutive write-back operations
-        if (!is_merged(dirty_it))
+        bool lm = is_left_merged(dirty_it);
+        bool rm = is_right_merged(dirty_it);
+        if (!lm && !rm)
        {
            // <writeback_queue> is OK to contain more than actual number of consecutive
            // requests as long as it doesn't miss anything. But <writeback_queue_size>
@@ -201,6 +232,11 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
                .stripe = op->offset,
            });
        }
+        else if (lm && rm)
+        {
+            assert(writeback_queue_size > 0);
+            writeback_queue_size--;
+        }
    }
    if (!is_del)
    {
@@ -225,7 +261,7 @@ int writeback_cache_t::repeat_ops_for(cluster_client_t *cli, osd_num_t peer_osd,
        for (auto wr_it = dirty_buffers.begin(), flush_it = wr_it, last_it = wr_it; ; )
        {
            bool end = wr_it == dirty_buffers.end();
-            bool flush_this = !end && wr_it->second.state != CACHE_REPEATING;
+            bool flush_this = !end && wr_it->second.state != CACHE_REPEATING && wr_it->second.state != CACHE_DIRTY;
            if (peer_osd)
                flush_this = flush_this && cli->affects_osd(wr_it->first.inode, wr_it->first.stripe, wr_it->second.len, peer_osd);
            if (pool_id && pg_num)
@@ -376,6 +412,7 @@ void writeback_cache_t::start_writebacks(cluster_client_t *cli, int count)
        assert(writeback_queue_size > 0);
        writeback_queue_size--;
        writeback_bytes -= off - from_it->first.stripe;
+        assert(writeback_queue_size > 0 || !writeback_bytes);
        flush_buffers(cli, from_it, to_it);
    }
    queue_copy.erase(queue_copy.begin(), queue_copy.begin()+i);
@@ -275,8 +275,8 @@ const char *help_text =
    "  --foreground 1\n"
    "    Stay in foreground, do not daemonize.\n"
    "\n"
-    "vitastor-nbd unmap /dev/nbdN\n"
-    "  Unmap an ioctl-mapped NBD device.\n"
+    "vitastor-nbd unmap [--force] /dev/nbdN\n"
+    "  Unmap an ioctl-mapped NBD device. Do not check if it's actually mapped if --force is specified.\n"
    "\n"
    "vitastor-nbd ls [--json]\n"
    "  List ioctl-mapped Vitastor NBD devices, optionally in JSON format.\n"
@@ -313,7 +313,7 @@ const char *help_text =
 #endif
    "Use vitastor-nbd --help <command> for command details or vitastor-nbd --help --all for all details.\n"
    "\n"
-    "All usual Vitastor config options like --config_file <path_to_config> may also be specified in CLI.\n"
+    "All usual Vitastor config options like --config_path <path_to_config> may also be specified in CLI.\n"
 ;

 class nbd_proxy
@@ -372,7 +372,8 @@ public:
            else if (args[i][0] == '-' && args[i][1] == '-')
            {
                const char *opt = args[i]+2;
-                cfg[opt] = !strcmp(opt, "json") || !strcmp(opt, "all") || i == narg-1 ? "1" : args[++i];
+                cfg[opt] = !strcmp(opt, "json") || !strcmp(opt, "all") ||
+                    !strcmp(opt, "force") || i == narg-1 ? "1" : args[++i];
            }
            else if (pos == 0)
            {
@@ -381,8 +382,9 @@ public:
            }
            else if (pos == 1)
            {
+                char c = 0;
                int n = 0;
-                if (sscanf(args[i], "/dev/nbd%d", &n) > 0)
+                if (sscanf(args[i], "/dev/nbd%d%c", &n, &c) == 1)
                    cfg["dev_num"] = n;
                else
                    cfg["dev_num"] = args[i];
@@ -404,18 +406,14 @@ public:
        }
        else if (cfg["command"] == "unmap")
        {
-            if (cfg["dev_num"].is_null())
+            if (!cfg["dev_num"].is_number() &&
+                cfg["dev_num"].string_value() != "0" &&
+                !cfg["dev_num"].uint64_value())
            {
                fprintf(stderr, "device name or number is missing\n");
                exit(1);
            }
-            if (cfg["netlink"].is_null())
-            {
-                ioctl_unmap(cfg["dev_num"].uint64_value());
-            }
-            else
-            {
-            }
+            ioctl_unmap(cfg["dev_num"].uint64_value(), cfg["force"].bool_value());
        }
 #ifdef HAVE_NBD_NETLINK_H
        else if (cfg["command"] == "netlink-map")
@@ -444,9 +442,18 @@ help:
        }
    }

-    void ioctl_unmap(int dev_num)
+    void ioctl_unmap(int dev_num, bool force)
    {
        char path[64] = { 0 };
+        // Check if mapped
+        sprintf(path, "/sys/block/nbd%d/pid", dev_num);
+        if (access(path, F_OK) != 0)
+        {
+            fprintf(stderr, "/dev/nbd%d is not mapped: /sys/block/nbd%d/pid does not exist\n", dev_num, dev_num);
+            if (!force)
+                exit(1);
+        }
+        // Run unmap
        sprintf(path, "/dev/nbd%d", dev_num);
        int r, nbd = open(path, O_RDWR);
        if (nbd < 0)
@@ -610,36 +617,43 @@ help:
        {
            if (!cfg["dev_num"].is_null())
            {
-                if (run_nbd(sockfd, cfg["dev_num"].int64_value(), device_size, NBD_FLAG_SEND_FLUSH, nbd_timeout, bg) < 0)
+                int r;
+                if ((r = run_nbd(sockfd, cfg["dev_num"].int64_value(), device_size, NBD_FLAG_SEND_FLUSH, nbd_timeout, bg)) != 0)
                {
-                    perror("run_nbd");
+                    fprintf(stderr, "run_nbd: %s\n", strerror(-r));
                    exit(1);
                }
            }
            else
            {
                // Find an unused device
+                auto mapped = list_mapped();
                int i = 0;
                while (true)
                {
+                    if (mapped.find("/dev/nbd"+std::to_string(i)) != mapped.end())
+                    {
+                        i++;
+                        continue;
+                    }
                    int r = run_nbd(sockfd, i, device_size, NBD_FLAG_SEND_FLUSH, nbd_timeout, bg);
                    if (r == 0)
                    {
                        printf("/dev/nbd%d\n", i);
                        break;
                    }
-                    else if (r == -1 && errno == ENOENT)
+                    else if (r == -ENOENT)
                    {
                        fprintf(stderr, "No free NBD devices found\n");
                        exit(1);
                    }
-                    else if (r == -2 && errno == EBUSY)
+                    else if (r == -EBUSY)
                    {
                        i++;
                    }
                    else
                    {
-                        perror("run_nbd");
+                        fprintf(stderr, "run_nbd: %s\n", strerror(-r));
                        exit(1);
                    }
                }
@@ -869,81 +883,114 @@ protected:
        // Check handle size
        assert(sizeof(cur_req.handle) == 8);
        char path[64] = { 0 };
-        sprintf(path, "/dev/nbd%d", dev_num);
-        int r, nbd = open(path, O_RDWR), qd_fd;
-        if (nbd < 0)
+        int notifyfd[2] = { 0 };
+        if (socketpair(AF_UNIX, SOCK_STREAM, 0, notifyfd) < 0)
        {
-            return -1;
+            return -errno;
        }
-        r = ioctl(nbd, NBD_SET_SOCK, sockfd[1]);
-        if (r < 0)
+        if (!fork())
        {
-            goto end_close;
-        }
-        r = ioctl(nbd, NBD_SET_BLKSIZE, 4096);
-        if (r < 0)
-        {
-            goto end_unmap;
-        }
-        r = ioctl(nbd, NBD_SET_SIZE, size);
-        if (r < 0)
-        {
-            goto end_unmap;
-        }
-        ioctl(nbd, NBD_SET_FLAGS, flags);
-        if (timeout > 0)
-        {
-            r = ioctl(nbd, NBD_SET_TIMEOUT, (unsigned long)timeout);
+            // Do all NBD configuration in the child process, after the last fork.
+            // Why? It's needed because there is a race condition in the Linux kernel nbd driver
+            // in nbd_add_socket() - it saves `current` task pointer as `nbd->task_setup` and
+            // then rechecks if the new `current` is the same. Problem is that if that process
+            // is already dead, `current` may be freed and then replaced by another process
+            // with the same pointer value. So the check passes and NBD allows a different process
+            // to set up a device which is already set up. Proper fix would have to be done in the
+            // kernel code, but the workaround is obviously to perform NBD setup from the process
+            // which will then actually call NBD_DO_IT. That process stays alive during the whole
+            // time of NBD device execution and the (nbd->task_setup != current) check always
+            // works correctly, and we don't accidentally break previous NBD devices while setting
+            // up a new device. Forking to check every device is of course rather slow, so we also
+            // do an additional check by calling list_mapped() before searching for a free NBD device.
+            if (bg)
+            {
+                daemonize_fork();
+            }
+            close(notifyfd[0]);
+            sprintf(path, "/dev/nbd%d", dev_num);
+            int r, nbd = open(path, O_RDWR), qd_fd;
+            if (nbd < 0)
+            {
+                write(notifyfd[1], &errno, sizeof(errno));
+                exit(1);
+            }
+            r = ioctl(nbd, NBD_SET_SOCK, sockfd[1]);
+            if (r < 0)
+            {
+                goto end_close;
+            }
+            r = ioctl(nbd, NBD_SET_BLKSIZE, 4096);
            if (r < 0)
            {
                goto end_unmap;
            }
-        }
-        // Configure request size
-        sprintf(path, "/sys/block/nbd%d/queue/max_sectors_kb", dev_num);
-        qd_fd = open(path, O_WRONLY);
-        if (qd_fd < 0)
-        {
-            goto end_unmap;
-        }
-        r = write(qd_fd, "32768", 5);
-        if (r != 5)
-        {
-            fprintf(stderr, "Warning: Failed to configure max_sectors_kb\n");
-        }
-        close(qd_fd);
-        if (!fork())
-        {
-            // Run in child
+            r = ioctl(nbd, NBD_SET_SIZE, size);
+            if (r < 0)
+            {
+                goto end_unmap;
+            }
+            ioctl(nbd, NBD_SET_FLAGS, flags);
+            if (timeout > 0)
+            {
+                r = ioctl(nbd, NBD_SET_TIMEOUT, (unsigned long)timeout);
+                if (r < 0)
+                {
+                    goto end_unmap;
+                }
+            }
+            // Configure request size
+            sprintf(path, "/sys/block/nbd%d/queue/max_sectors_kb", dev_num);
+            qd_fd = open(path, O_WRONLY);
+            if (qd_fd < 0)
+            {
+                goto end_unmap;
+            }
+            r = write(qd_fd, "32768", 5);
+            if (r != 5)
+            {
+                fprintf(stderr, "Warning: Failed to configure max_sectors_kb\n");
+            }
+            close(qd_fd);
+            // Notify parent
+            errno = 0;
+            write(notifyfd[1], &errno, sizeof(errno));
+            close(notifyfd[1]);
            close(sockfd[0]);
            if (bg)
            {
-                daemonize();
+                daemonize_reopen_stdio();
            }
            r = ioctl(nbd, NBD_DO_IT);
            if (r < 0)
            {
-                fprintf(stderr, "NBD device terminated with error: %s\n", strerror(errno));
+                fprintf(stderr, "NBD device /dev/nbd%d terminated with error: %s\n", dev_num, strerror(errno));
            }
            close(sockfd[1]);
            ioctl(nbd, NBD_CLEAR_QUE);
            ioctl(nbd, NBD_CLEAR_SOCK);
            exit(0);
-        }
-        close(sockfd[1]);
-        close(nbd);
-        return 0;
    end_close:
-        r = errno;
-        close(nbd);
-        errno = r;
-        return -2;
+            write(notifyfd[1], &errno, sizeof(errno));
+            close(nbd);
+            exit(2);
    end_unmap:
-        r = errno;
-        ioctl(nbd, NBD_CLEAR_SOCK);
-        close(nbd);
-        errno = r;
-        return -3;
+            write(notifyfd[1], &errno, sizeof(errno));
+            ioctl(nbd, NBD_CLEAR_SOCK);
+            close(nbd);
+            exit(3);
+        }
+        // Parent - check status
+        close(notifyfd[1]);
+        int child_errno = 0;
+        int ok = read(notifyfd[0], &child_errno, sizeof(child_errno));
+        close(notifyfd[0]);
+        if (ok && !child_errno)
+        {
+            close(sockfd[1]);
+            return 0;
+        }
+        return -child_errno;
    }

    void submit_send()
@@ -294,7 +294,9 @@ static void coroutine_fn vitastor_co_get_metadata(VitastorRPC *task)

    qemu_mutex_lock(&client->mutex);
    vitastor_c_watch_inode(client->proxy, client->image, vitastor_co_generic_cb, task);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
    vitastor_schedule_uring_handler(client);
+#endif
    qemu_mutex_unlock(&client->mutex);

    while (!task->complete)
@@ -749,7 +751,9 @@ static int coroutine_fn vitastor_co_preadv(BlockDriverState *bs,
    uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode;
    qemu_mutex_lock(&client->mutex);
    vitastor_c_read(client->proxy, inode, offset, bytes, iov->iov, iov->niov, vitastor_co_read_cb, &task);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
    vitastor_schedule_uring_handler(client);
+#endif
    qemu_mutex_unlock(&client->mutex);

    while (!task.complete)
@@ -783,7 +787,9 @@ static int coroutine_fn vitastor_co_pwritev(BlockDriverState *bs,
    uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode;
    qemu_mutex_lock(&client->mutex);
    vitastor_c_write(client->proxy, inode, offset, bytes, 0, iov->iov, iov->niov, vitastor_co_generic_cb, &task);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
    vitastor_schedule_uring_handler(client);
+#endif
    qemu_mutex_unlock(&client->mutex);

    while (!task.complete)
@@ -863,7 +869,9 @@ static int coroutine_fn vitastor_co_block_status(
        task.bitmap = client->last_bitmap = NULL;
        qemu_mutex_lock(&client->mutex);
        vitastor_c_read_bitmap(client->proxy, task.inode, task.offset, task.len, !client->skip_parents, vitastor_co_read_bitmap_cb, &task);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
        vitastor_schedule_uring_handler(client);
+#endif
        qemu_mutex_unlock(&client->mutex);
        while (!task.complete)
        {
@@ -950,7 +958,9 @@ static int coroutine_fn vitastor_co_flush(BlockDriverState *bs)

    qemu_mutex_lock(&client->mutex);
    vitastor_c_sync(client->proxy, vitastor_co_generic_cb, &task);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
    vitastor_schedule_uring_handler(client);
+#endif
    qemu_mutex_unlock(&client->mutex);

    while (!task.complete)
@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@

 Name: Vitastor
 Description: Vitastor client library
-Version: 1.10.1
+Version: 1.11.0
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}

@@ -127,6 +127,7 @@ vitastor_c *vitastor_c_create_qemu_uring(QEMUSetFDHandler *aio_set_fd_handler, v
    auto self = vitastor_c_create_qemu_common(aio_set_fd_handler, aio_context);
    self->ringloop = ringloop;
    self->cli = new cluster_client_t(self->ringloop, self->tfd, cfg_json);
+    ringloop->loop_continue();
    return self;
 }

@@ -150,6 +151,7 @@ vitastor_c *vitastor_c_create_uring(const char *config_path, const char *etcd_ho
    self->ringloop = ringloop;
    self->epmgr = new epoll_manager_t(self->ringloop);
    self->cli = new cluster_client_t(self->ringloop, self->epmgr->tfd, cfg_json);
+    ringloop->loop_continue();
    return self;
 }

@@ -183,6 +185,7 @@ vitastor_c *vitastor_c_create_uring_json(const char **options, int options_len)
    self->ringloop = ringloop;
    self->epmgr = new epoll_manager_t(self->ringloop);
    self->cli = new cluster_client_t(self->ringloop, self->epmgr->tfd, cfg_json);
+    ringloop->loop_continue();
    return self;
 }

@@ -222,6 +225,18 @@ int vitastor_c_is_ready(vitastor_c *client)
    return client->cli->is_ready();
 }

+void vitastor_c_on_ready(vitastor_c *client, VitastorIOHandler cb, void *opaque)
+{
+    client->cli->on_ready([=]()
+    {
+        cb(opaque, 0);
+    });
+    if (client->ringloop)
+    {
+        client->ringloop->loop_continue();
+    }
+}
+
 void vitastor_c_uring_wait_ready(vitastor_c *client)
 {
    while (!client->cli->is_ready())
@@ -276,6 +291,10 @@ void vitastor_c_read(vitastor_c *client, uint64_t inode, uint64_t offset, uint64
        delete op;
    };
    client->cli->execute(op);
+    if (client->ringloop)
+    {
+        client->ringloop->loop_continue();
+    }
 }

 void vitastor_c_write(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, uint64_t check_version,
@@ -297,6 +316,31 @@ void vitastor_c_write(vitastor_c *client, uint64_t inode, uint64_t offset, uint6
        delete op;
    };
    client->cli->execute(op);
+    if (client->ringloop)
+    {
+        client->ringloop->loop_continue();
+    }
+}
+
+void vitastor_c_delete(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, uint64_t check_version,
+    VitastorIOHandler cb, void *opaque)
+{
+    cluster_op_t *op = new cluster_op_t;
+    op->opcode = OSD_OP_DELETE;
+    op->inode = inode;
+    op->offset = offset;
+    op->len = len;
+    op->version = check_version;
+    op->callback = [cb, opaque](cluster_op_t *op)
+    {
+        cb(opaque, op->retval);
+        delete op;
+    };
+    client->cli->execute(op);
+    if (client->ringloop)
+    {
+        client->ringloop->loop_continue();
+    }
 }

 void vitastor_c_read_bitmap(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len,
@@ -319,6 +363,10 @@ void vitastor_c_read_bitmap(vitastor_c *client, uint64_t inode, uint64_t offset,
        delete op;
    };
    client->cli->execute(op);
+    if (client->ringloop)
+    {
+        client->ringloop->loop_continue();
+    }
 }

 void vitastor_c_sync(vitastor_c *client, VitastorIOHandler cb, void *opaque)
@@ -331,6 +379,10 @@ void vitastor_c_sync(vitastor_c *client, VitastorIOHandler cb, void *opaque)
        delete op;
    };
    client->cli->execute(op);
+    if (client->ringloop)
+    {
+        client->ringloop->loop_continue();
+    }
 }

 void vitastor_c_watch_inode(vitastor_c *client, char *image, VitastorIOHandler cb, void *opaque)
@@ -340,6 +392,10 @@ void vitastor_c_watch_inode(vitastor_c *client, char *image, VitastorIOHandler c
        auto watch = client->cli->st_cli.watch_inode(std::string(image));
        cb(opaque, (long)watch);
    });
+    if (client->ringloop)
+    {
+        client->ringloop->loop_continue();
+    }
 }

 void vitastor_c_close_watch(vitastor_c *client, void *handle)
@@ -7,7 +7,7 @@
 #define VITASTOR_QEMU_PROXY_H

 // C API wrapper version
-#define VITASTOR_C_API_VERSION 4
+#define VITASTOR_C_API_VERSION 5

 #ifndef POOL_ID_BITS
 #define POOL_ID_BITS 16
@@ -51,6 +51,7 @@ vitastor_c *vitastor_c_create_epoll_json(const char **options, int options_len);
 void* vitastor_c_get_internal_client(vitastor_c *client);
 void vitastor_c_destroy(vitastor_c *client);
 int vitastor_c_is_ready(vitastor_c *client);
+void vitastor_c_on_ready(vitastor_c *client, VitastorIOHandler cb, void *opaque);
 int vitastor_c_uring_register_eventfd(vitastor_c *client);
 void vitastor_c_uring_wait_ready(vitastor_c *client);
 void vitastor_c_uring_handle_events(vitastor_c *client);
@@ -62,6 +63,8 @@ void vitastor_c_read(vitastor_c *client, uint64_t inode, uint64_t offset, uint64
    struct iovec *iov, int iovcnt, VitastorReadHandler cb, void *opaque);
 void vitastor_c_write(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, uint64_t check_version,
    struct iovec *iov, int iovcnt, VitastorIOHandler cb, void *opaque);
+void vitastor_c_delete(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, uint64_t check_version,
+    VitastorIOHandler cb, void *opaque);
 void vitastor_c_read_bitmap(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len,
    int with_parents, VitastorReadBitmapHandler cb, void *opaque);
 void vitastor_c_sync(vitastor_c *client, VitastorIOHandler cb, void *opaque);
@@ -224,7 +224,7 @@ static const char* help_text =
    "Use vitastor-cli --help <command> for command details or vitastor-cli --help --all for all details.\n"
    "\n"
    "GLOBAL OPTIONS:\n"
-    "  --config_file FILE  Path to Vitastor configuration file\n"
+    "  --config_path FILE  Path to Vitastor configuration file\n"
    "  --etcd_address URL  Etcd connection address\n"
    "  --iodepth N         Send N operations in parallel to each OSD when possible (default 32)\n"
    "  --parallel_osds M   Work with M osds in parallel when possible (default 4)\n"
@@ -215,6 +215,7 @@ resume_3:
            goto resume_3;
        else if (state == 4)
            goto resume_4;
+        // FIXME: take all info from etcd requests, not mixed with st_cli.inode_config
        for (auto & ic: parent->cli->st_cli.inode_config)
        {
            if (ic.second.name == image_name+"@"+new_snap)
@@ -286,6 +287,7 @@ resume_4:

    json11::Json::object get_next_id()
    {
+        assert(new_pool_id);
        return json11::Json::object {
            { "request_range", json11::Json::object {
                { "key", base64_encode(
@@ -321,6 +323,17 @@ resume_4:
            goto resume_2;
        else if (state == 3)
            goto resume_3;
+        if (!new_pool_id)
+        {
+            for (auto & ic: parent->cli->st_cli.inode_config)
+            {
+                if (ic.second.name == image_name)
+                {
+                    new_pool_id = INODE_POOL(ic.first);
+                    break;
+                }
+            }
+        }
        parent->etcd_txn(json11::Json::object { { "success", json11::Json::array {
            get_next_id(),
            json11::Json::object {
@@ -546,7 +546,7 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_ls(json11::Json cfg)
    lister->list_pool_name = lister->list_pool_id ? "" : cfg["pool"].as_string();
    lister->show_stats = cfg["long"].bool_value();
    lister->show_delete = cfg["del"].bool_value();
-    lister->sort_field = cfg["sort"].string_value();
+    lister->sort_field = cfg["sort"].string_value() != "" ? cfg["sort"].string_value() : "name";
    lister->reverse = cfg["reverse"].bool_value();
    lister->max_count = cfg["count"].uint64_value();
    for (auto & item: cfg["names"].array_items())
@@ -281,6 +281,8 @@ class osd_t
    int pick_next_scrub(object_id & next_oid);
    void submit_scrub_op(object_id oid);
    bool continue_scrub();
+    void submit_scrub_subops(osd_op_t *cur_op);
+    void scrub_check_results(osd_op_t *cur_op);
    void plan_scrub(pg_t & pg, bool report_state = true);
    void schedule_scrub(pg_t & pg);

@@ -313,7 +315,7 @@ class osd_t
    pg_osd_set_state_t *mark_object(pg_t & pg, object_id oid, pg_osd_set_state_t *prev_object_state, bool ref,
        std::function<int(pg_osd_set_t & new_set)> calc_set);
    pg_osd_set_state_t *mark_object_corrupted(pg_t & pg, object_id oid, pg_osd_set_state_t *prev_object_state,
-        osd_rmw_stripe_t *stripes, bool ref, bool inconsistent);
+        osd_rmw_stripe_t *stripes, bool ref);
    pg_osd_set_state_t *mark_partial_write(pg_t & pg, object_id oid, pg_osd_set_state_t *prev_object_state,
        osd_rmw_stripe_t *stripes, bool ref);
    void deref_object_state(pg_t & pg, pg_osd_set_state_t **object_state, bool deref);
@@ -326,6 +328,8 @@ class osd_t
    void submit_primary_subops(int submit_type, uint64_t op_version, const uint64_t* osd_set, osd_op_t *cur_op);
    int submit_primary_subop_batch(int submit_type, inode_t inode, uint64_t op_version,
        osd_rmw_stripe_t *stripes, const uint64_t* osd_set, osd_op_t *cur_op, int subop_idx, int zero_read);
+    void submit_primary_subop(osd_op_t *cur_op, osd_op_t *subop,
+        osd_rmw_stripe_t *si, bool wr, inode_t inode, uint64_t op_version);
    void submit_primary_del_subops(osd_op_t *cur_op, uint64_t *cur_set, uint64_t set_size, pg_osd_set_t & loc_set);
    void submit_primary_del_batch(osd_op_t *cur_op, obj_ver_osd_t *chunks_to_delete, int chunks_to_delete_count);
    int submit_primary_sync_subops(osd_op_t *cur_op);
@@ -3,6 +3,7 @@

 #include "osd.h"

+#include <sys/prctl.h>
 #include <signal.h>

 static osd_t *osd = NULL;
@@ -56,6 +57,9 @@ int main(int narg, char *args[])
        printf("%s", help_text);
        return 1;
    }
+    char osdname[16] = { 0 };
+    snprintf(osdname, 16, "osd%lu", config["osd_num"].uint64_value());
+    prctl(PR_SET_NAME, (unsigned long)osdname, 0, 0, 0);
    signal(SIGINT, handle_sigint);
    signal(SIGTERM, handle_sigint);
    ring_loop_t *ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
@@ -28,7 +28,6 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
        return false;
    }
    auto & pool_cfg = pool_cfg_it->second;
-    // FIXME: op_data->pg_data_size can probably be removed (there's pg.pg_data_size)
    uint64_t pg_data_size = (pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks);
    uint64_t pg_block_size = bs_block_size * pg_data_size;
    object_id oid = {
@@ -52,13 +51,13 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
        finish_op(cur_op, -EINVAL);
        return false;
    }
-    // Scrub is similar to r/w, so it's also handled here
-    int stripe_count = (pool_cfg.scheme == POOL_SCHEME_REPLICATED
-        && cur_op->req.hdr.opcode != OSD_OP_SCRUB ? 1 : pg_it->second.pg_size);
+    int stripe_count = (cur_op->req.hdr.opcode == OSD_OP_SCRUB ? 0 :
+        (pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pg_it->second.pg_size));
    int chain_size = 0;
    if (cur_op->req.hdr.opcode == OSD_OP_READ && cur_op->req.rw.meta_revision > 0)
    {
        // Chained read
+        // FIXME: Introduce an explicit opcode for chained reads
        auto inode_it = st_cli.inode_config.find(cur_op->req.rw.inode);
        if (inode_it->second.mod_revision != cur_op->req.rw.meta_revision)
        {
@@ -72,7 +71,7 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
            inode_it->second.parent_id &&
            INODE_POOL(inode_it->second.parent_id) == pg_it->second.pool_id)
        {
-            // Check for loops
+            // Check for loops - FIXME check it in etcd_state_client
            if (inode_it->second.parent_id == cur_op->req.rw.inode ||
                inode_it->second.parent_id == inode_it->second.num ||
                chain_size > st_cli.inode_config.size())
@@ -110,20 +109,22 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
    );
    void *data_buf = (uint8_t*)op_data + sizeof(osd_primary_op_data_t);
    op_data->pg_num = pg_num;
+    op_data->pg = &pg_it->second;
    op_data->oid = oid;
    op_data->stripes = (osd_rmw_stripe_t*)data_buf;
+    op_data->stripe_count = stripe_count;
    data_buf = (uint8_t*)data_buf + sizeof(osd_rmw_stripe_t) * stripe_count;
-    op_data->scheme = pool_cfg.scheme;
-    op_data->pg_data_size = pg_data_size;
-    op_data->pg_size = pg_it->second.pg_size;
    cur_op->op_data = op_data;
-    split_stripes(pg_data_size, bs_block_size, (uint32_t)(cur_op->req.rw.offset - oid.stripe), cur_op->req.rw.len, op_data->stripes);
-    // Resulting bitmaps have to survive op_data and be freed with the op itself
-    assert(!cur_op->bitmap_buf);
-    cur_op->bitmap_buf = calloc_or_die(1, clean_entry_bitmap_size * stripe_count);
-    for (int i = 0; i < stripe_count; i++)
+    if (cur_op->req.hdr.opcode != OSD_OP_SCRUB)
    {
-        op_data->stripes[i].bmp_buf = (uint8_t*)cur_op->bitmap_buf + clean_entry_bitmap_size * i;
+        split_stripes(pg_data_size, bs_block_size, (uint32_t)(cur_op->req.rw.offset - oid.stripe), cur_op->req.rw.len, op_data->stripes);
+        // Resulting bitmaps have to survive op_data and be freed with the op itself
+        assert(!cur_op->bitmap_buf);
+        cur_op->bitmap_buf = calloc_or_die(1, clean_entry_bitmap_size * stripe_count);
+        for (int i = 0; i < stripe_count; i++)
+        {
+            op_data->stripes[i].bmp_buf = (uint8_t*)cur_op->bitmap_buf + clean_entry_bitmap_size * i;
+        }
    }
    op_data->chain_size = chain_size;
    if (chain_size > 0)
@@ -205,11 +206,11 @@ void osd_t::continue_primary_read(osd_op_t *cur_op)
 resume_0:
    cur_op->reply.rw.bitmap_len = 0;
    {
-        auto & pg = pgs.at({ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num });
+        auto & pg = *op_data->pg;
        if (cur_op->req.rw.len == 0)
        {
            // len=0 => bitmap read
-            for (int role = 0; role < op_data->pg_data_size; role++)
+            for (int role = 0; role < pg.pg_data_size; role++)
            {
                op_data->stripes[role].read_start = 0;
                op_data->stripes[role].read_end = UINT32_MAX;
@@ -217,7 +218,7 @@ resume_0:
        }
        else
        {
-            for (int role = 0; role < op_data->pg_data_size; role++)
+            for (int role = 0; role < pg.pg_data_size; role++)
            {
                op_data->stripes[role].read_start = op_data->stripes[role].req_start;
                op_data->stripes[role].read_end = op_data->stripes[role].req_end;
@@ -228,29 +229,27 @@ resume_0:
        op_data->target_ver = vo_it != pg.ver_override.end() ? vo_it->second : UINT64_MAX;
        // PG may have degraded or misplaced objects
        op_data->prev_set = get_object_osd_set(pg, op_data->oid, &op_data->object_state);
-        if (pg.state == PG_ACTIVE || op_data->scheme == POOL_SCHEME_REPLICATED)
+        if (pg.state == PG_ACTIVE || pg.scheme == POOL_SCHEME_REPLICATED)
        {
            // Fast happy-path
-            if (op_data->scheme == POOL_SCHEME_REPLICATED &&
+            if (pg.scheme == POOL_SCHEME_REPLICATED &&
                op_data->object_state && (op_data->object_state->state & OBJ_INCOMPLETE))
            {
                finish_op(cur_op, -EIO);
                return;
            }
-            cur_op->buf = alloc_read_buffer(op_data->stripes, op_data->pg_data_size, 0);
+            cur_op->buf = alloc_read_buffer(op_data->stripes, pg.pg_data_size, 0);
            submit_primary_subops(SUBMIT_RMW_READ, op_data->target_ver, op_data->prev_set, cur_op);
            op_data->st = 1;
        }
        else
        {
-            if (extend_missing_stripes(op_data->stripes, op_data->prev_set, op_data->pg_data_size, pg.pg_size) < 0)
+            if (extend_missing_stripes(op_data->stripes, op_data->prev_set, pg.pg_data_size, pg.pg_size) < 0)
            {
                finish_op(cur_op, -EIO);
                return;
            }
            // Submit reads
-            op_data->pg_size = pg.pg_size;
-            op_data->scheme = pg.scheme;
            op_data->degraded = 1;
            cur_op->buf = alloc_read_buffer(op_data->stripes, pg.pg_size, 0);
            submit_primary_subops(SUBMIT_RMW_READ, op_data->target_ver, op_data->prev_set, cur_op);
@@ -265,30 +264,29 @@ resume_2:
        if (op_data->errcode == -EIO || op_data->errcode == -EDOM)
        {
            // I/O or checksum error
-            auto & pg = pgs.at({ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num });
            // FIXME: ref = true ideally... because new_state != state is not necessarily true if it's freed and recreated
-            op_data->object_state = mark_object_corrupted(pg, op_data->oid, op_data->object_state, op_data->stripes, false, false);
+            op_data->object_state = mark_object_corrupted(*op_data->pg, op_data->oid, op_data->object_state, op_data->stripes, false);
            goto resume_0;
        }
        finish_op(cur_op, op_data->errcode);
        return;
    }
    cur_op->reply.rw.version = op_data->fact_ver;
-    cur_op->reply.rw.bitmap_len = op_data->pg_data_size * clean_entry_bitmap_size;
+    cur_op->reply.rw.bitmap_len = op_data->pg->pg_data_size * clean_entry_bitmap_size;
    if (op_data->degraded)
    {
        // Reconstruct missing stripes
        osd_rmw_stripe_t *stripes = op_data->stripes;
-        if (op_data->scheme == POOL_SCHEME_XOR)
+        if (op_data->pg->scheme == POOL_SCHEME_XOR)
        {
-            reconstruct_stripes_xor(stripes, op_data->pg_size, clean_entry_bitmap_size);
+            reconstruct_stripes_xor(stripes, op_data->pg->pg_size, clean_entry_bitmap_size);
        }
-        else if (op_data->scheme == POOL_SCHEME_EC)
+        else if (op_data->pg->scheme == POOL_SCHEME_EC)
        {
-            reconstruct_stripes_ec(stripes, op_data->pg_size, op_data->pg_data_size, clean_entry_bitmap_size);
+            reconstruct_stripes_ec(stripes, op_data->pg->pg_size, op_data->pg->pg_data_size, clean_entry_bitmap_size);
        }
        cur_op->iov.push_back(op_data->stripes[0].bmp_buf, cur_op->reply.rw.bitmap_len);
-        for (int role = 0; role < op_data->pg_size; role++)
+        for (int role = 0; role < op_data->pg->pg_size; role++)
        {
            if (stripes[role].req_end != 0)
            {
@@ -360,10 +358,10 @@ pg_osd_set_state_t *osd_t::mark_object(pg_t & pg, object_id oid, pg_osd_set_stat
    return object_state;
 }

-pg_osd_set_state_t *osd_t::mark_object_corrupted(pg_t & pg, object_id oid, pg_osd_set_state_t *prev_object_state,
-    osd_rmw_stripe_t *stripes, bool ref, bool inconsistent)
+pg_osd_set_state_t *osd_t::mark_object_corrupted(pg_t & pg, object_id oid,
+    pg_osd_set_state_t *prev_object_state, osd_rmw_stripe_t *stripes, bool ref)
 {
-    return mark_object(pg, oid, prev_object_state, ref, [stripes, inconsistent](pg_osd_set_t & new_set)
+    return mark_object(pg, oid, prev_object_state, ref, [stripes](pg_osd_set_t & new_set)
    {
        // Mark object chunk(s) as corrupted
        int changes = 0;
@@ -390,16 +388,6 @@ pg_osd_set_state_t *osd_t::mark_object_corrupted(pg_t & pg, object_id oid, pg_os
                    chunk.loc_bad &= ~LOC_CORRUPTED;
                }
            }
-            if (inconsistent && !chunk.loc_bad)
-            {
-                changes++;
-                chunk.loc_bad |= LOC_INCONSISTENT;
-            }
-            else if (!inconsistent && (chunk.loc_bad & LOC_INCONSISTENT))
-            {
-                changes++;
-                chunk.loc_bad &= ~LOC_INCONSISTENT;
-            }
            chunk_it++;
        }
        return changes;
@@ -695,7 +683,7 @@ void osd_t::continue_primary_del(osd_op_t *cur_op)
        return;
    }
    osd_primary_op_data_t *op_data = cur_op->op_data;
-    auto & pg = pgs.at({ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num });
+    auto & pg = *op_data->pg;
    if (op_data->st == 1)      goto resume_1;
    else if (op_data->st == 2) goto resume_2;
    else if (op_data->st == 3) goto resume_3;
@@ -20,14 +20,15 @@ struct unstable_osd_num_t
 struct osd_primary_op_data_t
 {
    int st = 0;
-    pg_num_t pg_num;
-    object_id oid;
-    uint64_t target_ver;
+    pg_num_t pg_num = 0;
+    object_id oid = {};
+    uint64_t target_ver = 0;
    uint64_t orig_ver = 0, fact_ver = 0;
-    uint64_t scheme = 0;
    int n_subops = 0, done = 0, errors = 0, drops = 0, errcode = 0;
-    int degraded = 0, pg_size, pg_data_size;
-    osd_rmw_stripe_t *stripes;
+    int degraded = 0;
+    int stripe_count = 0;
+    osd_rmw_stripe_t *stripes = NULL;
+    pg_t *pg = NULL;
    osd_op_t *subops = NULL;
    uint64_t *prev_set = NULL;
    pg_osd_set_state_t *object_state = NULL;
@@ -7,7 +7,7 @@
 void osd_t::continue_chained_read(osd_op_t *cur_op)
 {
    osd_primary_op_data_t *op_data = cur_op->op_data;
-    auto & pg = pgs.at({ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num });
+    auto & pg = *op_data->pg;
    if (op_data->st == 1)
        goto resume_1;
    else if (op_data->st == 2)
@@ -17,7 +17,7 @@ void osd_t::continue_chained_read(osd_op_t *cur_op)
    else if (op_data->st == 4)
        goto resume_4;
    cur_op->reply.rw.bitmap_len = 0;
-    for (int role = 0; role < op_data->pg_data_size; role++)
+    for (int role = 0; role < pg.pg_data_size; role++)
    {
        op_data->stripes[role].read_start = op_data->stripes[role].req_start;
        op_data->stripes[role].read_end = op_data->stripes[role].req_end;
@@ -329,7 +329,7 @@ std::vector<osd_chain_read_t> osd_t::collect_chained_read_requests(osd_op_t *cur
 {
    osd_primary_op_data_t *op_data = cur_op->op_data;
    std::vector<osd_chain_read_t> chain_reads;
-    int stripe_count = (op_data->scheme == POOL_SCHEME_REPLICATED ? 1 : op_data->pg_size);
+    int stripe_count = (op_data->pg->scheme == POOL_SCHEME_REPLICATED ? 1 : op_data->pg->pg_size);
    memset(op_data->stripes[0].bmp_buf, 0, stripe_count * clean_entry_bitmap_size);
    uint8_t *global_bitmap = (uint8_t*)op_data->stripes[0].bmp_buf;
    // We always use at most 1 read request per layer
@@ -337,7 +337,7 @@ std::vector<osd_chain_read_t> osd_t::collect_chained_read_requests(osd_op_t *cur
    {
        uint8_t *part_bitmap = ((uint8_t*)op_data->snapshot_bitmaps) + chain_pos*stripe_count*clean_entry_bitmap_size;
        int start = !cur_op->req.rw.len ? 0 : (cur_op->req.rw.offset - op_data->oid.stripe)/bs_bitmap_granularity;
-        int end = !cur_op->req.rw.len ? op_data->pg_data_size*clean_entry_bitmap_size*8 : start + cur_op->req.rw.len/bs_bitmap_granularity;
+        int end = !cur_op->req.rw.len ? op_data->pg->pg_data_size*clean_entry_bitmap_size*8 : start + cur_op->req.rw.len/bs_bitmap_granularity;
        // Skip unneeded part in the beginning
        while (start < end && (
            ((global_bitmap[start>>3] >> (start&7)) & 1) ||
@@ -410,11 +410,11 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
        // FIXME: maybe introduce split_read_stripes to shorten these lines and to remove read_start=req_start
        osd_rmw_stripe_t *stripes = chain_stripes + chain_reads[cri].chain_pos*stripe_count;
        split_stripes(pg.pg_data_size, bs_block_size, chain_reads[cri].offset, chain_reads[cri].len, stripes);
-        if (op_data->scheme == POOL_SCHEME_REPLICATED && !stripes[0].req_end)
+        if (pg.scheme == POOL_SCHEME_REPLICATED && !stripes[0].req_end)
        {
            continue;
        }
-        for (int role = 0; role < op_data->pg_data_size; role++)
+        for (int role = 0; role < pg.pg_data_size; role++)
        {
            stripes[role].read_start = stripes[role].req_start;
            stripes[role].read_end = stripes[role].req_end;
@@ -423,7 +423,7 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
        if (pg.state != PG_ACTIVE)
        {
            cur_set = get_object_osd_set(pg, cur_oid, &op_data->chain_states[chain_reads[cri].chain_pos]);
-            if (op_data->scheme != POOL_SCHEME_REPLICATED)
+            if (pg.scheme != POOL_SCHEME_REPLICATED)
            {
                if (extend_missing_stripes(stripes, cur_set, pg.pg_data_size, pg.pg_size) < 0)
                {
@@ -446,7 +446,7 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
                }
            }
        }
-        if (op_data->scheme == POOL_SCHEME_REPLICATED)
+        if (pg.scheme == POOL_SCHEME_REPLICATED)
        {
            n_subops++;
            read_buffer_size += stripes[0].read_end - stripes[0].read_start;
@@ -491,7 +491,7 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
    for (int cri = 0; cri < chain_reads.size(); cri++)
    {
        osd_rmw_stripe_t *stripes = chain_stripes + chain_reads[cri].chain_pos*stripe_count;
-        if (op_data->scheme == POOL_SCHEME_REPLICATED && !stripes[0].req_end)
+        if (pg.scheme == POOL_SCHEME_REPLICATED && !stripes[0].req_end)
        {
            continue;
        }
@@ -501,9 +501,9 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
        auto cur_state = op_data->chain_states[chain_reads[cri].chain_pos];
        uint64_t *cur_set = (pg.state != PG_ACTIVE && cur_state ? cur_state->read_target.data() : pg.cur_set.data());
        int zero_read = -1;
-        if (op_data->scheme == POOL_SCHEME_REPLICATED)
+        if (pg.scheme == POOL_SCHEME_REPLICATED)
        {
-            for (int role = 0; role < op_data->pg_size; role++)
+            for (int role = 0; role < pg.pg_size; role++)
                if (cur_set[role] == this->osd_num || zero_read == -1)
                    zero_read = role;
        }
@@ -535,7 +535,7 @@ void osd_t::check_corrupted_chained(pg_t & pg, osd_op_t *cur_op)
        }
        if (corrupted)
        {
-            mark_object_corrupted(pg, cur_oid, op_data->chain_states[op_data->chain_reads[cri].chain_pos], stripes, false, false);
+            mark_object_corrupted(pg, cur_oid, op_data->chain_states[op_data->chain_reads[cri].chain_pos], stripes, false);
        }
    }
 }
@@ -555,18 +555,18 @@ void osd_t::send_chained_read_results(pg_t & pg, osd_op_t *cur_op)
        {
            // Reconstruct missing stripes
            osd_rmw_stripe_t *stripes = chain_stripes + op_data->chain_reads[cri].chain_pos*stripe_count;
-            if (op_data->scheme == POOL_SCHEME_XOR)
+            if (pg.scheme == POOL_SCHEME_XOR)
            {
                reconstruct_stripes_xor(stripes, pg.pg_size, clean_entry_bitmap_size);
            }
-            else if (op_data->scheme == POOL_SCHEME_EC)
+            else if (pg.scheme == POOL_SCHEME_EC)
            {
                reconstruct_stripes_ec(stripes, pg.pg_size, pg.pg_data_size, clean_entry_bitmap_size);
            }
        }
    }
    // Send bitmap
-    cur_op->reply.rw.bitmap_len = op_data->pg_data_size * clean_entry_bitmap_size;
+    cur_op->reply.rw.bitmap_len = pg.pg_data_size * clean_entry_bitmap_size;
    cur_op->iov.push_back(op_data->stripes[0].bmp_buf, cur_op->reply.rw.bitmap_len);
    // And finally compose the result
    uint64_t sent = 0;
@@ -67,7 +67,7 @@ void osd_t::finish_op(osd_op_t *cur_op, int retval)
        if (cur_op->req.hdr.opcode == OSD_OP_DELETE)
        {
            if (cur_op->op_data)
-                inode_stats[cur_op->req.rw.inode].op_bytes[inode_st_op] += cur_op->op_data->pg_data_size * bs_block_size;
+                inode_stats[cur_op->req.rw.inode].op_bytes[inode_st_op] += cur_op->op_data->pg->pg_data_size * bs_block_size;
        }
        else
            inode_stats[cur_op->req.rw.inode].op_bytes[inode_st_op] += cur_op->req.rw.len;
@@ -76,7 +76,7 @@ void osd_t::finish_op(osd_op_t *cur_op, int retval)
    {
        if (cur_op->op_data->pg_num > 0)
        {
-            auto & pg = pgs.at({ .pool_id = INODE_POOL(cur_op->op_data->oid.inode), .pg_num = cur_op->op_data->pg_num });
+            auto & pg = *cur_op->op_data->pg;
            pg.inflight--;
            assert(pg.inflight >= 0);
            if ((pg.state & PG_STOPPING) && pg.inflight == 0 && !pg.flush_batch)
@@ -126,10 +126,10 @@ void osd_t::submit_primary_subops(int submit_type, uint64_t op_version, const ui
    bool wr = submit_type == SUBMIT_WRITE;
    osd_primary_op_data_t *op_data = cur_op->op_data;
    osd_rmw_stripe_t *stripes = op_data->stripes;
-    bool rep = op_data->scheme == POOL_SCHEME_REPLICATED;
+    bool rep = op_data->pg->scheme == POOL_SCHEME_REPLICATED;
    // Allocate subops
    int n_subops = 0, zero_read = -1;
-    for (int role = 0; role < op_data->pg_size; role++)
+    for (int role = 0; role < op_data->pg->pg_size; role++)
    {
        if (osd_set[role] == this->osd_num || osd_set[role] != 0 && zero_read == -1)
            zero_read = role;
@@ -152,11 +152,11 @@ void osd_t::submit_primary_subops(int submit_type, uint64_t op_version, const ui
 int osd_t::submit_primary_subop_batch(int submit_type, inode_t inode, uint64_t op_version,
    osd_rmw_stripe_t *stripes, const uint64_t* osd_set, osd_op_t *cur_op, int subop_idx, int zero_read)
 {
+    bool rep = cur_op->op_data->pg->scheme == POOL_SCHEME_REPLICATED;
    bool wr = submit_type == SUBMIT_WRITE;
    osd_primary_op_data_t *op_data = cur_op->op_data;
-    bool rep = op_data->scheme == POOL_SCHEME_REPLICATED;
    int i = subop_idx;
-    for (int role = 0; role < op_data->pg_size; role++)
+    for (int role = 0; role < op_data->pg->pg_size; role++)
    {
        // We always submit zero-length writes to all replicas, even if the stripe is not modified
        if (!(wr || !rep && stripes[role].read_end != 0 || zero_read == role || submit_type == SUBMIT_SCRUB_READ))
@@ -168,109 +168,9 @@ int osd_t::submit_primary_subop_batch(int submit_type, inode_t inode, uint64_t o
        osd_rmw_stripe_t *si = stripes + (submit_type == SUBMIT_SCRUB_READ ? role : stripe_num);
        if (role_osd_num != 0)
        {
-            osd_op_t *subop = op_data->subops + i;
-            uint32_t subop_len = wr
-                ? si->write_end - si->write_start
-                : si->read_end - si->read_start;
-            if (!wr && si->read_end == UINT32_MAX)
-            {
-                subop_len = 0;
-            }
            si->osd_num = role_osd_num;
-            si->read_error = false;
-            subop->bitmap = si->bmp_buf;
-            subop->bitmap_len = clean_entry_bitmap_size;
-            // Using rmw_buf to pass pointer to stripes. Dirty but should work
-            subop->rmw_buf = si;
-            if (role_osd_num == this->osd_num)
-            {
-                clock_gettime(CLOCK_REALTIME, &subop->tv_begin);
-                subop->op_type = (uint64_t)cur_op;
-                subop->bs_op = new blockstore_op_t((blockstore_op_t){
-                    .opcode = (uint64_t)(wr ? (rep ? BS_OP_WRITE_STABLE : BS_OP_WRITE) : BS_OP_READ),
-                    .callback = [subop, this](blockstore_op_t *bs_subop)
-                    {
-                        handle_primary_bs_subop(subop);
-                    },
-                    { {
-                        .oid = (object_id){
-                            .inode = inode,
-                            .stripe = op_data->oid.stripe | stripe_num,
-                        },
-                        .version = op_version,
-                        .offset = wr ? si->write_start : si->read_start,
-                        .len = subop_len,
-                    } },
-                    .buf = wr ? si->write_buf : si->read_buf,
-                    .bitmap = si->bmp_buf,
-                });
-#ifdef OSD_DEBUG
-                printf(
-                    "Submit %s to local: %jx:%jx v%ju %u-%u\n", wr ? "write" : "read",
-                    inode, op_data->oid.stripe | stripe_num, op_version,
-                    subop->bs_op->offset, subop->bs_op->len
-                );
-#endif
-                bs->enqueue_op(subop->bs_op);
-            }
-            else
-            {
-                subop->op_type = OSD_OP_OUT;
-                subop->req.sec_rw = {
-                    .header = {
-                        .magic = SECONDARY_OSD_OP_MAGIC,
-                        .id = msgr.next_subop_id++,
-                        .opcode = (uint64_t)(wr ? (rep ? OSD_OP_SEC_WRITE_STABLE : OSD_OP_SEC_WRITE) : OSD_OP_SEC_READ),
-                    },
-                    .oid = {
-                        .inode = inode,
-                        .stripe = op_data->oid.stripe | stripe_num,
-                    },
-                    .version = op_version,
-                    .offset = wr ? si->write_start : si->read_start,
-                    .len = subop_len,
-                    .attr_len = wr ? clean_entry_bitmap_size : 0,
-                    .flags = cur_op->peer_fd == SELF_FD && cur_op->req.hdr.opcode != OSD_OP_SCRUB ? OSD_OP_RECOVERY_RELATED : 0,
-                };
-#ifdef OSD_DEBUG
-                printf(
-                    "Submit %s to osd %ju: %jx:%jx v%ju %u-%u\n", wr ? "write" : "read", role_osd_num,
-                    inode, op_data->oid.stripe | stripe_num, op_version,
-                    subop->req.sec_rw.offset, subop->req.sec_rw.len
-                );
-#endif
-                if (wr)
-                {
-                    if (si->write_end > si->write_start)
-                    {
-                        subop->iov.push_back(si->write_buf, si->write_end - si->write_start);
-                    }
-                }
-                else
-                {
-                    if (subop_len > 0)
-                    {
-                        subop->iov.push_back(si->read_buf, subop_len);
-                    }
-                }
-                subop->callback = [cur_op, this](osd_op_t *subop)
-                {
-                    handle_primary_subop(subop, cur_op);
-                };
-                auto peer_fd_it = msgr.osd_peer_fds.find(role_osd_num);
-                if (peer_fd_it != msgr.osd_peer_fds.end())
-                {
-                    subop->peer_fd = peer_fd_it->second;
-                    msgr.outbox_push(subop);
-                }
-                else
-                {
-                    // Fail it immediately
-                    subop->peer_fd = -1;
-                    subop->reply.hdr.retval = -EPIPE;
-                    ringloop->set_immediate([subop]() { std::function<void(osd_op_t*)>(subop->callback)(subop); });
-                }
-            }
+            si->role = stripe_num;
+            submit_primary_subop(cur_op, &op_data->subops[i], si, wr, inode, op_version);
            i++;
        }
        else
@@ -281,6 +181,112 @@ int osd_t::submit_primary_subop_batch(int submit_type, inode_t inode, uint64_t o
    return i-subop_idx;
 }

+void osd_t::submit_primary_subop(osd_op_t *cur_op, osd_op_t *subop,
+    osd_rmw_stripe_t *si, bool wr, inode_t inode, uint64_t op_version)
+{
+    uint32_t subop_len = wr
+        ? si->write_end - si->write_start
+        : si->read_end - si->read_start;
+    if (!wr && si->read_end == UINT32_MAX)
+    {
+        subop_len = 0;
+    }
+    si->read_error = false;
+    subop->bitmap = si->bmp_buf;
+    subop->bitmap_len = clean_entry_bitmap_size;
+    // Using rmw_buf to pass pointer to stripes. Dirty but works
+    subop->rmw_buf = si;
+    if (si->osd_num == this->osd_num)
+    {
+        clock_gettime(CLOCK_REALTIME, &subop->tv_begin);
+        subop->op_type = (uint64_t)cur_op; // also dirty
+        subop->bs_op = new blockstore_op_t((blockstore_op_t){
+            .opcode = (uint64_t)(wr ? (cur_op->op_data->pg->scheme == POOL_SCHEME_REPLICATED ? BS_OP_WRITE_STABLE : BS_OP_WRITE) : BS_OP_READ),
+            .callback = [subop, this](blockstore_op_t *bs_subop)
+            {
+                handle_primary_bs_subop(subop);
+            },
+            { {
+                .oid = (object_id){
+                    .inode = inode,
+                    .stripe = cur_op->op_data->oid.stripe | si->role,
+                },
+                .version = op_version,
+                .offset = wr ? si->write_start : si->read_start,
+                .len = subop_len,
+            } },
+            .buf = wr ? si->write_buf : si->read_buf,
+            .bitmap = si->bmp_buf,
+        });
+#ifdef OSD_DEBUG
+         printf(
+             "Submit %s to local: %jx:%jx v%ju %u-%u\n", wr ? "write" : "read",
+             inode, op_data->oid.stripe | si->role, op_version,
+             subop->bs_op->offset, subop->bs_op->len
+         );
+#endif
+        bs->enqueue_op(subop->bs_op);
+    }
+    else
+    {
+        subop->op_type = OSD_OP_OUT;
+        subop->req.sec_rw = (osd_op_sec_rw_t){
+            .header = {
+                .magic = SECONDARY_OSD_OP_MAGIC,
+                .id = msgr.next_subop_id++,
+                .opcode = (uint64_t)(wr ? (cur_op->op_data->pg->scheme == POOL_SCHEME_REPLICATED ? OSD_OP_SEC_WRITE_STABLE : OSD_OP_SEC_WRITE) : OSD_OP_SEC_READ),
+            },
+            .oid = {
+                .inode = inode,
+                .stripe = cur_op->op_data->oid.stripe | si->role,
+            },
+            .version = op_version,
+            .offset = wr ? si->write_start : si->read_start,
+            .len = subop_len,
+            .attr_len = wr ? clean_entry_bitmap_size : 0,
+            .flags = cur_op->peer_fd == SELF_FD && cur_op->req.hdr.opcode != OSD_OP_SCRUB ? OSD_OP_RECOVERY_RELATED : 0,
+        };
+#ifdef OSD_DEBUG
+        printf(
+            "Submit %s to osd %ju: %jx:%jx v%ju %u-%u\n", wr ? "write" : "read", si->osd_num,
+            inode, op_data->oid.stripe | si->role, op_version,
+            subop->req.sec_rw.offset, subop->req.sec_rw.len
+        );
+#endif
+        if (wr)
+        {
+            if (si->write_end > si->write_start)
+            {
+                subop->iov.push_back(si->write_buf, si->write_end - si->write_start);
+            }
+        }
+        else
+        {
+            if (subop_len > 0)
+            {
+                subop->iov.push_back(si->read_buf, subop_len);
+            }
+        }
+        subop->callback = [cur_op, this](osd_op_t *subop)
+        {
+            handle_primary_subop(subop, cur_op);
+        };
+        auto peer_fd_it = msgr.osd_peer_fds.find(si->osd_num);
+        if (peer_fd_it != msgr.osd_peer_fds.end())
+        {
+            subop->peer_fd = peer_fd_it->second;
+            msgr.outbox_push(subop);
+        }
+        else
+        {
+            // Fail it immediately
+            subop->peer_fd = -1;
+            subop->reply.hdr.retval = -EPIPE;
+            ringloop->set_immediate([subop]() { std::function<void(osd_op_t*)>(subop->callback)(subop); });
+        }
+    }
+}
+
 static uint64_t bs_op_to_osd_op[] = {
    0,
    OSD_OP_SEC_READ,            // BS_OP_READ = 1
@@ -401,7 +407,7 @@ void osd_t::handle_primary_subop(osd_op_t *subop, osd_op_t *cur_op)
        printf("subop %s %jx:%jx from osd %jd: version = %ju\n", osd_op_names[opcode],
            subop->req.sec_rw.oid.inode, subop->req.sec_rw.oid.stripe, peer_osd, version);
 #endif
-        if (op_data->fact_ver != UINT64_MAX)
+        if (version != 0 && op_data->fact_ver != UINT64_MAX)
        {
            if (op_data->fact_ver != 0 && op_data->fact_ver != version)
            {
@@ -526,7 +532,7 @@ bool contains_osd(osd_num_t *osd_set, uint64_t size, osd_num_t osd_num)
 void osd_t::submit_primary_del_subops(osd_op_t *cur_op, osd_num_t *cur_set, uint64_t set_size, pg_osd_set_t & loc_set)
 {
    osd_primary_op_data_t *op_data = cur_op->op_data;
-    bool rep = op_data->scheme == POOL_SCHEME_REPLICATED;
+    bool rep = op_data->pg->scheme == POOL_SCHEME_REPLICATED;
    obj_ver_osd_t extra_chunks[loc_set.size()];
    int chunks_to_del = 0;
    for (auto & chunk: loc_set)
@@ -738,10 +744,10 @@ void osd_t::submit_primary_rollback_subops(osd_op_t *cur_op, const uint64_t* osd
 {
    osd_primary_op_data_t *op_data = cur_op->op_data;
    osd_rmw_stripe_t *stripes = op_data->stripes;
-    assert(op_data->scheme != POOL_SCHEME_REPLICATED);
+    assert(op_data->pg->scheme != POOL_SCHEME_REPLICATED);
    // Allocate subops
    int n_subops = 0;
-    for (int role = 0; role < op_data->pg_size; role++)
+    for (int role = 0; role < op_data->pg->pg_size; role++)
    {
        if (osd_set[role] != 0 && !stripes[role].read_error &&
            (osd_set[role] == this->osd_num || msgr.osd_peer_fds.find(osd_set[role]) != msgr.osd_peer_fds.end()))
@@ -758,7 +764,7 @@ void osd_t::submit_primary_rollback_subops(osd_op_t *cur_op, const uint64_t* osd
    op_data->subops = new osd_op_t[n_subops];
    op_data->unstable_writes = new obj_ver_id[n_subops];
    int i = 0;
-    for (int role = 0; role < op_data->pg_size; role++)
+    for (int role = 0; role < op_data->pg->pg_size; role++)
    {
        if (osd_set[role] != 0 && !stripes[role].read_error &&
            (osd_set[role] == this->osd_num || msgr.osd_peer_fds.find(osd_set[role]) != msgr.osd_peer_fds.end()))
@@ -44,7 +44,7 @@ void osd_t::continue_primary_write(osd_op_t *cur_op)
        return;
    }
    osd_primary_op_data_t *op_data = cur_op->op_data;
-    auto & pg = pgs.at({ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num });
+    auto & pg = *cur_op->op_data->pg;
    if (op_data->st == 1)      goto resume_1;
    else if (op_data->st == 2) goto resume_2;
    else if (op_data->st == 3) goto resume_3;
@@ -73,7 +73,7 @@ resume_1:
        op_data->object_state->ref_count++;
    }
 retry_1:
-    if (op_data->scheme == POOL_SCHEME_REPLICATED)
+    if (pg.scheme == POOL_SCHEME_REPLICATED)
    {
        // Simplified algorithm
        op_data->stripes[0].write_start = op_data->stripes[0].req_start;
@@ -99,7 +99,7 @@ retry_1:
    {
        assert(!cur_op->rmw_buf);
        cur_op->rmw_buf = calc_rmw(cur_op->buf, op_data->stripes, op_data->prev_set,
-            pg.pg_size, op_data->pg_data_size, pg.pg_cursize, pg.cur_set.data(), bs_block_size, clean_entry_bitmap_size);
+            pg.pg_size, pg.pg_data_size, pg.pg_cursize, pg.cur_set.data(), bs_block_size, clean_entry_bitmap_size);
        if (!cur_op->rmw_buf)
        {
            // Refuse partial overwrite of an incomplete object
@@ -114,7 +114,7 @@ retry_1:
            // Allow to read version number (just version number!) from corrupted chunks
            // to allow full overwrite of a corrupted object
            bool found = false;
-            for (int role = 0; role < op_data->pg_size; role++)
+            for (int role = 0; role < pg.pg_size; role++)
            {
                if (op_data->prev_set[role] != 0 || op_data->stripes[role].read_end > op_data->stripes[role].read_start)
                {
@@ -124,8 +124,8 @@ retry_1:
            }
            if (!found)
            {
-                osd_num_t corrupted_target[op_data->pg_size];
-                for (int role = 0; role < op_data->pg_size; role++)
+                osd_num_t corrupted_target[pg.pg_size];
+                for (int role = 0; role < pg.pg_size; role++)
                {
                    corrupted_target[role] = 0;
                }
@@ -151,7 +151,7 @@ resume_3:
        if (op_data->errcode == -EIO || op_data->errcode == -EDOM)
        {
            // Mark object corrupted and retry
-            op_data->object_state = mark_object_corrupted(pg, op_data->oid, op_data->object_state, op_data->stripes, true, false);
+            op_data->object_state = mark_object_corrupted(pg, op_data->oid, op_data->object_state, op_data->stripes, true);
            op_data->prev_set = op_data->object_state ? op_data->object_state->read_target.data() : pg.cur_set.data();
            if (cur_op->rmw_buf)
            {
@@ -165,6 +165,7 @@ resume_3:
        return;
    }
    // Check CAS version
+    // FIXME: Handle CAS writes as "immediate" in non-immediate_commit pools, otherwise CAS doesn't make sense
    if (cur_op->req.rw.version && op_data->fact_ver != (cur_op->req.rw.version-1))
    {
        deref_object_state(pg, &op_data->object_state, true);
@@ -172,7 +173,7 @@ resume_3:
        cur_op->reply.rw.version = op_data->fact_ver;
        goto continue_others;
    }
-    if (op_data->scheme == POOL_SCHEME_REPLICATED)
+    if (pg.scheme == POOL_SCHEME_REPLICATED)
    {
        // Set bitmap bits
        bitmap_set(op_data->stripes[0].bmp_buf, op_data->stripes[0].write_start,
@@ -203,7 +204,7 @@ resume_3:
        }
        else if (pg.scheme == POOL_SCHEME_EC)
        {
-            calc_rmw_parity_ec(op_data->stripes, pg.pg_size, op_data->pg_data_size, op_data->prev_set, pg.cur_set.data(), bs_block_size, clean_entry_bitmap_size);
+            calc_rmw_parity_ec(op_data->stripes, pg.pg_size, pg.pg_data_size, op_data->prev_set, pg.cur_set.data(), bs_block_size, clean_entry_bitmap_size);
        }
    }
    // Send writes
@@ -269,7 +270,7 @@ resume_5:
        // and rollback successful part updates in case of EC.
        if (op_data->done > 0 && !op_data->drops)
        {
-            if (op_data->scheme != POOL_SCHEME_REPLICATED)
+            if (pg.scheme != POOL_SCHEME_REPLICATED)
            {
                submit_primary_rollback_subops(cur_op, pg.cur_set.data());
 resume_11:
@@ -293,7 +294,7 @@ resume_12:
        pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->errcode);
        return;
    }
-    if (op_data->scheme != POOL_SCHEME_REPLICATED)
+    if (pg.scheme != POOL_SCHEME_REPLICATED)
    {
        // Remove version override just after the write, but before stabilizing
        pg.ver_override.erase(op_data->oid);
@@ -329,7 +330,7 @@ resume_7:
                memset(&recovery_stat[recovery_type], 0, sizeof(recovery_stat[recovery_type]));
                recovery_stat[recovery_type].count++;
            }
-            for (int role = 0; role < (op_data->scheme == POOL_SCHEME_REPLICATED ? 1 : pg.pg_size); role++)
+            for (int role = 0; role < (pg.scheme == POOL_SCHEME_REPLICATED ? 1 : pg.pg_size); role++)
            {
                recovery_stat[recovery_type].bytes += op_data->stripes[role].write_end - op_data->stripes[role].write_start;
            }
@@ -353,7 +354,7 @@ resume_7:
            for (auto & chunk: op_data->object_state->osd_set)
            {
                // Check is the same as in submit_primary_del_subops()
-                if (op_data->scheme == POOL_SCHEME_REPLICATED
+                if (pg.scheme == POOL_SCHEME_REPLICATED
                    ? !contains_osd(pg.cur_set.data(), pg.pg_size, chunk.osd_num)
                    : (chunk.osd_num != pg.cur_set[chunk.role]))
                {
@@ -361,7 +362,7 @@ resume_7:
                        .osd_num = chunk.osd_num,
                        .oid = {
                            .inode = op_data->oid.inode,
-                            .stripe = op_data->oid.stripe | (op_data->scheme == POOL_SCHEME_REPLICATED ? 0 : chunk.role),
+                            .stripe = op_data->oid.stripe | (pg.scheme == POOL_SCHEME_REPLICATED ? 0 : chunk.role),
                        },
                        .version = op_data->fact_ver,
                    });
@@ -472,7 +473,7 @@ bool osd_t::remember_unstable_write(osd_op_t *cur_op, pg_t & pg, pg_osd_set_t &
    if (immediate_commit == IMMEDIATE_ALL)
    {
 immediate:
-        if (op_data->scheme != POOL_SCHEME_REPLICATED)
+        if (pg.scheme != POOL_SCHEME_REPLICATED)
        {
            // Send STABILIZE ops immediately
            op_data->unstable_write_osds = new std::vector<unstable_osd_num_t>();
@@ -515,7 +516,7 @@ resume_7:
    }
    else if (immediate_commit == IMMEDIATE_SMALL)
    {
-        int stripe_count = (op_data->scheme == POOL_SCHEME_REPLICATED ? 1 : op_data->pg_size);
+        int stripe_count = (pg.scheme == POOL_SCHEME_REPLICATED ? 1 : pg.pg_size);
        for (int role = 0; role < stripe_count; role++)
        {
            if (op_data->stripes[role].write_start == 0 &&
@@ -531,7 +532,7 @@ resume_7:
    {
 lazy:
        unstable_write_count++;
-        if (op_data->scheme != POOL_SCHEME_REPLICATED)
+        if (pg.scheme != POOL_SCHEME_REPLICATED)
        {
            // Remember version as unstable for EC/XOR
            for (auto & chunk: loc_set)
@@ -1118,139 +1118,230 @@ static bool next_combination(int *subset, int k, int n)
    return true;
 }

-static int c_n_k(int n, int k)
+static uint64_t c_n_k(uint64_t n, uint64_t k)
 {
-    int c = 1;
-    for (int i = n; i > k; i--)
+    uint64_t c = 1;
+    for (uint64_t i = n; i > k; i--)
+    {
+        if ((c*i) < i)
+            return UINT64_MAX;
        c *= i;
-    for (int i = 2; i <= (n-k); i++)
+    }
+    for (uint64_t i = 2; i <= (n-k); i++)
        c /= i;
    return c;
 }

-std::vector<int> ec_find_good(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsize, bool is_xor,
-    uint32_t chunk_size, uint32_t bitmap_size, int max_bruteforce)
+static std::vector<int> ec_check_combination(osd_rmw_stripe_t *stripes, int stripe_count,
+    int *subset, int pg_size, int pg_minsize, bool is_xor,
+    uint32_t chunk_size, uint32_t bitmap_size, uint8_t *tmp_buf)
+{
+    osd_num_t fake_osd_set[pg_size];
+    for (int i = 0; i < pg_size; i++)
+    {
+        fake_osd_set[i] = i+1;
+    }
+    osd_rmw_stripe_t brute_stripes[pg_size];
+    memset(brute_stripes, 0, sizeof(osd_rmw_stripe_t)*pg_size);
+    for (int i = 0; i < pg_size; i++)
+    {
+        auto & bs = brute_stripes[i];
+        bs.req_end = bs.read_end = chunk_size;
+    }
+    for (int i = 0; i < pg_minsize; i++)
+    {
+        auto & src = stripes[subset[i]];
+        auto & bs = brute_stripes[src.role];
+        bs.bmp_buf = src.bmp_buf;
+        bs.write_buf = bs.read_buf = src.read_buf;
+    }
+    for (int i = 0; i < pg_size; i++)
+    {
+        auto & bs = brute_stripes[i];
+        if (!bs.read_buf)
+        {
+            // missing chunks are recovered in read_bufs and write_bufs are used as source for parity
+            bs.missing = true;
+            bs.read_buf = bs.write_buf = tmp_buf+i*chunk_size;
+            bs.bmp_buf = tmp_buf + stripe_count*chunk_size + i*bitmap_size;
+        }
+        else if (i >= pg_minsize)
+        {
+            // parity chunks are regenerated in their write_bufs, so use a temporary buffer
+            bs.write_buf = tmp_buf+i*chunk_size;
+        }
+    }
+    if (is_xor)
+    {
+        assert(pg_size == pg_minsize+1);
+        reconstruct_stripes_xor(brute_stripes, pg_size, bitmap_size);
+    }
+    else
+    {
+        reconstruct_stripes_ec(brute_stripes, pg_size, pg_minsize, bitmap_size);
+        calc_rmw_parity_ec(brute_stripes, pg_size, pg_minsize, fake_osd_set, fake_osd_set, chunk_size, bitmap_size);
+    }
+    bool matched_other = false;
+    std::vector<int> good_set;
+    for (int i = 0; i < stripe_count; i++)
+    {
+        if (stripes[i].read_error || stripes[i].not_exists)
+        {
+            continue;
+        }
+        auto & bs = brute_stripes[stripes[i].role];
+        if (!bs.missing && bs.read_buf == stripes[i].read_buf)
+        {
+            // source chunk, mark OK
+            good_set.push_back(i);
+        }
+        else if (memcmp(stripes[i].role < pg_minsize ? bs.read_buf : bs.write_buf, stripes[i].read_buf, chunk_size) == 0)
+        {
+            // matching chunk, mark OK
+            good_set.push_back(i);
+            matched_other = true;
+        }
+    }
+    if (!matched_other)
+    {
+        good_set.clear();
+    }
+    return good_set;
+}
+
+static int count_roles(osd_rmw_stripe_t *stripes, std::vector<int> & valid_chunks, int pg_size)
+{
+    bool role_ok[pg_size];
+    for (int i = 0; i < pg_size; i++)
+    {
+        role_ok[i] = false;
+    }
+    for (int idx: valid_chunks)
+    {
+        role_ok[stripes[idx].role] = true;
+    }
+    int ok_count = 0;
+    for (int i = 0; i < pg_size; i++)
+    {
+        if (role_ok[i])
+            ok_count++;
+    }
+    return ok_count;
+}
+
+std::vector<int> ec_find_good(osd_rmw_stripe_t *stripes, int stripe_count, int pg_size, int pg_minsize, bool is_xor,
+    uint32_t chunk_size, uint32_t bitmap_size, uint64_t max_bruteforce, bool find_best)
 {
    std::vector<int> found_valid;
-    int cur_live[pg_size], live_count = 0, exists_count = 0;
-    osd_num_t fake_osd_set[pg_size];
-    for (int role = 0; role < pg_size; role++)
+    std::vector<std::vector<int>> live_variants(pg_size);
+    int eq_to[stripe_count];
+    int live_roles = 0, live_total = 0;
+    for (int i = 0; i < pg_size; i++)
    {
-        if (!stripes[role].missing)
-        {
-            if (!stripes[role].not_exists)
-                exists_count++;
-            cur_live[live_count++] = role;
-            fake_osd_set[role] = role+1;
-        }
+        eq_to[i] = i;
    }
-    if (live_count <= pg_minsize)
+    for (int i = 0; i < stripe_count; i++)
    {
-        return std::vector<int>();
-    }
-    if (exists_count <= pg_minsize)
-    {
-        // Special case: user manually deleted some chunks
-        for (int role = 0; role < pg_size; role++)
-            if (!stripes[role].missing && !stripes[role].not_exists)
-                found_valid.push_back(role);
-        return found_valid;
-    }
-    // Try to locate errors using brute force if there isn't too many combinations
-    osd_rmw_stripe_t brute_stripes[pg_size];
-    int out_count = live_count-pg_minsize;
-    bool brute_force = out_count > 1 && c_n_k(live_count-1, out_count-1) <= max_bruteforce;
-    int subset[pg_minsize], outset[out_count];
-    // Select all combinations with items except the last one (== anything to compare)
-    first_combination(subset, pg_minsize, live_count-1);
-    uint8_t *tmp_buf = (uint8_t*)malloc_or_die(pg_size*chunk_size);
-    do
-    {
-        memcpy(brute_stripes, stripes, sizeof(osd_rmw_stripe_t)*pg_size);
-        int i = 0, j = 0, k = 0;
-        for (; i < pg_minsize; i++, j++)
-            while (j < subset[i])
-                outset[k++] = j++;
-        while (j < pg_size)
-            outset[k++] = j++;
-        for (int i = 0; i < out_count; i++)
+        if (!stripes[i].read_error && !stripes[i].not_exists)
        {
-            brute_stripes[cur_live[outset[i]]].missing = true;
-            brute_stripes[cur_live[outset[i]]].read_buf = tmp_buf+cur_live[outset[i]]*chunk_size;
-        }
-        for (int i = 0; i < pg_minsize; i++)
-        {
-            brute_stripes[i].write_buf = brute_stripes[i].read_buf;
-            brute_stripes[i].req_start = 0;
-            brute_stripes[i].req_end = chunk_size;
-        }
-        for (int i = pg_minsize; i < pg_size; i++)
-        {
-            brute_stripes[i].write_buf = tmp_buf+i*chunk_size;
-        }
-        if (is_xor)
-        {
-            assert(pg_size == pg_minsize+1);
-            reconstruct_stripes_xor(brute_stripes, pg_size, bitmap_size);
-        }
-        else
-        {
-            reconstruct_stripes_ec(brute_stripes, pg_size, pg_minsize, bitmap_size);
-            calc_rmw_parity_ec(brute_stripes, pg_size, pg_minsize, fake_osd_set, fake_osd_set, chunk_size, bitmap_size);
-        }
-        for (int i = pg_minsize; i < pg_size; i++)
-        {
-            brute_stripes[i].read_buf = brute_stripes[i].write_buf;
-        }
-        int valid_count = 0;
-        for (int i = 0; i < out_count; i++)
-        {
-            if (memcmp(brute_stripes[cur_live[outset[i]]].read_buf,
-                    stripes[cur_live[outset[i]]].read_buf, chunk_size) == 0)
+            if (live_variants[stripes[i].role].size() > 0)
            {
-                brute_stripes[cur_live[outset[i]]].missing = false;
-                valid_count++;
-            }
-        }
-        if (valid_count > 0)
-        {
-            if (found_valid.size())
-            {
-                // Check if we found the same set from the different point of view,
-                // like 1 2 3 -> valid 4 5 and 1 3 4 -> valid 2 5
-                for (int i = 0, j = 0; i < pg_size; i++)
+                for (int j = 0; j < i; j++)
                {
-                    if (!brute_stripes[i].missing)
+                    if (stripes[j].role == stripes[i].role &&
+                        memcmp(stripes[i].read_buf, stripes[j].read_buf, chunk_size) == 0)
                    {
-                        if (j >= found_valid.size() || found_valid[j] != i)
-                        {
-                            // Ambiguity: we found multiple valid sets and don't know which one is correct
-                            found_valid.clear();
-                            break;
-                        }
-                        j++;
+                        eq_to[i] = eq_to[j];
+                        break;
                    }
                }
-                if (!found_valid.size())
-                {
-                    break;
-                }
            }
            else
            {
-                for (int i = 0; i < pg_size; i++)
+                live_roles++;
+            }
+            if (eq_to[i] == i)
+            {
+                live_variants[stripes[i].role].push_back(i);
+                live_total++;
+            }
+        }
+    }
+    if (live_roles == pg_minsize && live_total > pg_minsize)
+    {
+        // Nothing to validate and there are chunks with different data => object is inconsistent
+        return std::vector<int>();
+    }
+    if (live_roles <= pg_minsize)
+    {
+        // Nothing to validate, just return all live chunks
+        for (int i = 0; i < stripe_count; i++)
+            if (!stripes[i].read_error)
+                found_valid.push_back(i);
+        return found_valid;
+    }
+    // Try to locate errors using brute force if there isn't too many combinations
+    bool brute_force = c_n_k(live_roles, pg_minsize) <= max_bruteforce;
+    int combination[pg_minsize], subset[pg_minsize], subvar[pg_minsize];
+    // To translate 0..live_roles into 0..pg_size
+    int comb_to_subset[live_roles];
+    for (int i = 0, r = 0; i < pg_size; i++)
+    {
+        if (live_variants[i].size() > 0)
+            comb_to_subset[r++] = i;
+    }
+    // Select all combinations with items except the last one (== anything to compare)
+    first_combination(combination, pg_minsize, live_roles);
+    uint8_t *tmp_buf = (uint8_t*)malloc_or_die(stripe_count*(chunk_size+bitmap_size));
+    do
+    {
+        // Then loop over all subvariants (if some roles have multiple diverged variants of data)
+        for (int i = 0; i < pg_minsize; i++)
+        {
+            subvar[i] = 0;
+        }
+        while (true)
+        {
+            // Transform combination[] + subvar[] into subset[]
+            for (int i = 0; i < pg_minsize; i++)
+            {
+                subset[i] = live_variants[comb_to_subset[combination[i]]][subvar[i]];
+            }
+            // Check the combination
+            auto valid_chunks = ec_check_combination(stripes, stripe_count, subset, pg_size, pg_minsize, is_xor, chunk_size, bitmap_size, tmp_buf);
+            // The same set may be found from different points of view,
+            // like 1 2 3 -> valid 4 5 and 1 3 4 -> valid 2 5
+            if (valid_chunks.size() > 0)
+            {
+                if (found_valid.size() >= valid_chunks.size() && found_valid != valid_chunks)
                {
-                    if (!brute_stripes[i].missing)
-                    {
-                        found_valid.push_back(i);
-                    }
+                    // Ambiguity: we found multiple valid sets and don't know which one is correct
+                    printf("Scrub found 2 different correct chunk subsets: OSD ");
+                    for (int i = 0; i < found_valid.size(); i++)
+                        printf(i > 0 ? ", %ju" : "%ju", stripes[found_valid[i]].osd_num);
+                    printf(" and OSD ");
+                    for (int i = 0; i < valid_chunks.size(); i++)
+                        printf(i > 0 ? ", %ju" : "%ju", stripes[valid_chunks[i]].osd_num);
+                    printf("\n");
+                    found_valid.clear();
+                    goto out;
+                }
+                else if (!found_valid.size() && (find_best || count_roles(stripes, valid_chunks, pg_size) >= pg_size))
+                {
+                    found_valid = valid_chunks;
                }
            }
-            if (valid_count == out_count)
+            // Select next subvariant
+            int i = 0;
+            for (i = 0; i < pg_minsize; i++)
            {
-                // All chunks are good
-                break;
+                subvar[i]++;
+                if (subvar[i] < live_variants[combination[i]].size())
+                    break;
+                subvar[i] = 0;
            }
+            if (i >= pg_minsize)
+                break;
        }
        if (!brute_force)
        {
@@ -1258,7 +1349,8 @@ std::vector<int> ec_find_good(osd_rmw_stripe_t *stripes, int pg_size, int pg_min
            // if we find it we won't be able to check that it's the only good one
            break;
        }
-    } while (out_count > 1 && next_combination(subset, pg_minsize, live_count-1));
+    } while (next_combination(combination, pg_minsize, live_roles));
+out:
    free(tmp_buf);
    return found_valid;
 }
@@ -28,6 +28,7 @@ struct osd_rmw_stripe_t
    uint32_t read_start, read_end;
    uint32_t write_start, write_end;
    osd_num_t osd_num;
+    int role;
    bool missing: 1;
    bool read_error: 1;
    bool not_exists: 1;
@@ -57,5 +58,5 @@ void reconstruct_stripes_ec(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsi
 void calc_rmw_parity_ec(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsize,
    uint64_t *read_osd_set, uint64_t *write_osd_set, uint32_t chunk_size, uint32_t bitmap_size);

-std::vector<int> ec_find_good(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsize, bool is_xor,
-    uint32_t chunk_size, uint32_t bitmap_size, int max_bruteforce);
+std::vector<int> ec_find_good(osd_rmw_stripe_t *stripes, int stripe_count, int pg_size, int pg_minsize, bool is_xor,
+    uint32_t chunk_size, uint32_t bitmap_size, uint64_t max_bruteforce, bool find_best);
@@ -1160,24 +1160,26 @@ void test_ec43_error_bruteforce()
        stripes[i].read_end = 4096;
        stripes[i].read_buf = write_buf+i*4096;
        stripes[i].write_buf = NULL;
+        stripes[i].role = i;
+        stripes[i].osd_num = i+1;
    }
    // All good chunks
-    auto res = ec_find_good(stripes, 7, 4, false, 4096, 0, 100);
+    auto res = ec_find_good(stripes, 7, 7, 4, false, 4096, 0, 100, true);
    assert_eq_vec(res, std::vector<int>({0, 1, 2, 3, 4, 5, 6}));
    // 1 missing chunk
    set_pattern(write_buf+1*4096, 4096, 0);
-    res = ec_find_good(stripes, 7, 4, false, 4096, 0, 100);
+    res = ec_find_good(stripes, 7, 7, 4, false, 4096, 0, 100, true);
    assert_eq_vec(res, std::vector<int>({0, 2, 3, 4, 5, 6}));
    // 2 missing chunks
    set_pattern(write_buf+1*4096, 4096, 0);
    set_pattern(write_buf+5*4096, 4096, 0);
-    res = ec_find_good(stripes, 7, 4, false, 4096, 0, 100);
+    res = ec_find_good(stripes, 7, 7, 4, false, 4096, 0, 100, true);
    assert_eq_vec(res, std::vector<int>({0, 2, 3, 4, 6}));
    // 3 missing chunks
    set_pattern(write_buf+1*4096, 4096, 0);
    set_pattern(write_buf+5*4096, 4096, 0);
    set_pattern(write_buf+6*4096, 4096, 0);
-    res = ec_find_good(stripes, 7, 4, false, 4096, 0, 100);
+    res = ec_find_good(stripes, 7, 7, 4, false, 4096, 0, 100, true);
    assert_eq_vec(res, std::vector<int>());
    // Done
    free(rmw_buf);
@@ -369,114 +369,102 @@ void osd_t::schedule_scrub(pg_t & pg)
    }
 }

-void osd_t::continue_primary_scrub(osd_op_t *cur_op)
+void osd_t::submit_scrub_subops(osd_op_t *cur_op)
 {
-    if (!cur_op->op_data && !prepare_primary_rw(cur_op))
-        return;
    osd_primary_op_data_t *op_data = cur_op->op_data;
-    if (op_data->st == 1)
-        goto resume_1;
-    else if (op_data->st == 2)
-        goto resume_2;
+    assert(!op_data->stripe_count);
+    cur_op->req.rw.len = bs_block_size * op_data->pg->pg_data_size;
+    // Determine version
+    auto vo_it = op_data->pg->ver_override.find(op_data->oid);
+    op_data->target_ver = vo_it != op_data->pg->ver_override.end() ? vo_it->second : UINT64_MAX;
+    // Find object state
+    op_data->prev_set = get_object_osd_set(*op_data->pg, op_data->oid, &op_data->object_state);
+    if (!op_data->object_state)
    {
-        auto & pg = pgs.at({ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num });
-        cur_op->req.rw.len = bs_block_size * pg.pg_data_size;
-        // Determine version
-        auto vo_it = pg.ver_override.find(op_data->oid);
-        op_data->target_ver = vo_it != pg.ver_override.end() ? vo_it->second : UINT64_MAX;
-        // PG may have degraded or misplaced objects
-        op_data->prev_set = get_object_osd_set(pg, op_data->oid, &op_data->object_state);
-        // Read all available chunks
-        int n_copies = 0;
-        op_data->degraded = false;
-        for (int role = 0; role < op_data->pg_size; role++)
+        op_data->stripe_count = op_data->pg->pg_size;
+        op_data->stripes = (osd_rmw_stripe_t*)calloc_or_die(op_data->stripe_count, sizeof(osd_rmw_stripe_t));
+        for (int i = 0; i < op_data->pg->pg_size; i++)
        {
-            op_data->stripes[role].write_buf = NULL;
-            op_data->stripes[role].read_start = 0;
-            op_data->stripes[role].read_end = bs_block_size;
-            if (op_data->prev_set[role] != 0)
-            {
-                n_copies++;
-            }
-            else
-            {
-                op_data->stripes[role].missing = true;
-                if (op_data->scheme != POOL_SCHEME_REPLICATED && role < op_data->pg_data_size)
-                {
-                    op_data->degraded = true;
-                }
-            }
-        }
-        if (n_copies <= op_data->pg_data_size)
-        {
-            // Nothing to compare, even if we'd like to
-            finish_op(cur_op, 0);
-            return;
-        }
-        cur_op->buf = alloc_read_buffer(op_data->stripes, op_data->pg_size, 0);
-        // Submit reads
-        osd_op_t *subops = new osd_op_t[n_copies];
-        op_data->fact_ver = 0;
-        op_data->done = op_data->errors = op_data->errcode = 0;
-        op_data->n_subops = n_copies;
-        op_data->subops = subops;
-        int sent = submit_primary_subop_batch(SUBMIT_SCRUB_READ, op_data->oid.inode, op_data->target_ver,
-            op_data->stripes, op_data->prev_set, cur_op, 0, -1);
-        assert(sent == n_copies);
-        op_data->st = 1;
-    }
-resume_1:
-    return;
-resume_2:
-    if (op_data->errors > 0)
-    {
-        if (op_data->errcode == -EIO || op_data->errcode == -EDOM)
-        {
-            // I/O or checksum error
-            int n_copies = 0;
-            for (int role = 0; role < op_data->pg_size; role++)
-            {
-                if (op_data->stripes[role].read_error)
-                {
-                    op_data->stripes[role].missing = true;
-                    if (op_data->scheme != POOL_SCHEME_REPLICATED && role < op_data->pg_data_size)
-                    {
-                        op_data->degraded = true;
-                    }
-                }
-                else if (!op_data->stripes[role].missing)
-                {
-                    n_copies++;
-                }
-            }
-            if (n_copies <= op_data->pg_data_size)
-            {
-                // Nothing to compare, just mark the object as corrupted
-                auto & pg = pgs.at({ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num });
-                // FIXME: ref = true ideally... because new_state != state is not necessarily true if it's freed and recreated
-                op_data->object_state = mark_object_corrupted(pg, op_data->oid, op_data->object_state, op_data->stripes, false, false);
-                // Operation is treated as unsuccessful only if the object becomes unreadable
-                finish_op(cur_op, n_copies < op_data->pg_data_size ? op_data->errcode : 0);
-                return;
-            }
-            // Proceed, we can still compare chunks that were successfully read
-        }
-        else
-        {
-            finish_op(cur_op, op_data->errcode);
-            return;
+            op_data->stripes[i].osd_num = op_data->prev_set[i];
+            op_data->stripes[i].role = (op_data->pg->scheme == POOL_SCHEME_REPLICATED ? 0 : i);
+            op_data->stripes[i].read_end = bs_block_size;
        }
    }
+    else
+    {
+        op_data->stripe_count = 0;
+        for (auto & chunk: op_data->object_state->osd_set)
+        {
+            // Read all chunks except outdated
+            if (!(chunk.loc_bad & LOC_OUTDATED))
+                op_data->stripe_count++;
+        }
+        op_data->stripes = (osd_rmw_stripe_t*)calloc_or_die(op_data->stripe_count, sizeof(osd_rmw_stripe_t));
+        int i = 0;
+        for (auto & chunk: op_data->object_state->osd_set)
+        {
+            if (!(chunk.loc_bad & LOC_OUTDATED))
+            {
+                op_data->stripes[i].osd_num = chunk.osd_num;
+                op_data->stripes[i].role = chunk.role;
+                op_data->stripes[i].read_end = bs_block_size;
+                i++;
+            }
+        }
+    }
+    assert(!cur_op->bitmap_buf);
+    cur_op->bitmap_buf = calloc_or_die(1, clean_entry_bitmap_size * op_data->stripe_count);
+    for (int i = 0; i < op_data->stripe_count; i++)
+    {
+        op_data->stripes[i].bmp_buf = (uint8_t*)cur_op->bitmap_buf + clean_entry_bitmap_size * i;
+    }
+    cur_op->buf = alloc_read_buffer(op_data->stripes, op_data->stripe_count, 0);
+    op_data->fact_ver = 0;
+    op_data->done = op_data->errors = op_data->errcode = 0;
+    op_data->n_subops = op_data->stripe_count;
+    op_data->subops = new osd_op_t[op_data->stripe_count];
+    op_data->st = 1;
+    for (int i = 0; i < op_data->stripe_count; i++)
+    {
+        submit_primary_subop(cur_op, &op_data->subops[i], &op_data->stripes[i],
+            false, op_data->oid.inode, op_data->target_ver);
+    }
+}
+
+// The idea is that scrub should not only find out if the object
+// is corrupted, but it should also verify availability of all copies
+void osd_t::scrub_check_results(osd_op_t *cur_op)
+{
+    osd_primary_op_data_t *op_data = cur_op->op_data;
    bool inconsistent = false;
-    if (op_data->scheme == POOL_SCHEME_REPLICATED)
+    int total = 0;
+    for (int role = 0; role < op_data->stripe_count; role++)
+    {
+        if (!op_data->stripes[role].not_exists)
+            total++;
+    }
+    if (!total)
+    {
+        // Object is deleted manually from all OSDs, forget it
+        printf(
+            "[PG %u/%u] Scrub detected a deleted object %jx:%jx\n",
+            INODE_POOL(op_data->oid.inode), op_data->pg_num,
+            op_data->oid.inode, op_data->oid.stripe
+        );
+        remove_object_from_state(op_data->oid, &op_data->object_state, *op_data->pg, false);
+        deref_object_state(*op_data->pg, &op_data->object_state, true);
+        return;
+    }
+    if (op_data->pg->scheme == POOL_SCHEME_REPLICATED)
    {
        // Check that all chunks have returned the same data
        int total = 0;
-        int eq_to[op_data->pg_size];
-        for (int role = 0; role < op_data->pg_size; role++)
+        int eq_to[op_data->stripe_count];
+        for (int role = 0; role < op_data->stripe_count; role++)
        {
            eq_to[role] = -1;
-            if (op_data->stripes[role].read_end != 0 && !op_data->stripes[role].missing &&
+            if (op_data->stripes[role].read_end != 0 &&
+                !op_data->stripes[role].read_error &&
                !op_data->stripes[role].not_exists)
            {
                total++;
@@ -492,16 +480,16 @@ resume_2:
                }
            }
        }
-        int votes[op_data->pg_size];
-        for (int role = 0; role < op_data->pg_size; role++)
+        int votes[op_data->stripe_count];
+        for (int role = 0; role < op_data->stripe_count; role++)
            votes[role] = 0;
-        for (int role = 0; role < op_data->pg_size; role++)
+        for (int role = 0; role < op_data->stripe_count; role++)
        {
            if (eq_to[role] != -1)
                votes[eq_to[role]]++;
        }
        int best = -1;
-        for (int role = 0; role < op_data->pg_size; role++)
+        for (int role = 0; role < op_data->stripe_count; role++)
        {
            if (votes[role] > (best >= 0 ? votes[best] : 0))
                best = role;
@@ -509,7 +497,7 @@ resume_2:
        if (best >= 0 && votes[best] < total)
        {
            bool unknown = false;
-            for (int role = 0; role < op_data->pg_size; role++)
+            for (int role = 0; role < op_data->stripe_count; role++)
            {
                if (role != best && votes[role] == votes[best])
                {
@@ -550,10 +538,11 @@ resume_2:
    }
    else
    {
-        assert(op_data->scheme == POOL_SCHEME_EC || op_data->scheme == POOL_SCHEME_XOR);
+        assert(op_data->pg->scheme == POOL_SCHEME_EC || op_data->pg->scheme == POOL_SCHEME_XOR);
        auto good_subset = ec_find_good(
-            op_data->stripes, op_data->pg_size, op_data->pg_data_size, op_data->scheme == POOL_SCHEME_XOR,
-            bs_block_size, clean_entry_bitmap_size, scrub_ec_max_bruteforce
+            op_data->stripes, op_data->stripe_count,
+            op_data->pg->pg_size, op_data->pg->pg_data_size, op_data->pg->scheme == POOL_SCHEME_XOR,
+            bs_block_size, clean_entry_bitmap_size, scrub_ec_max_bruteforce, scrub_find_best
        );
        if (!good_subset.size())
        {
@@ -567,62 +556,115 @@ resume_2:
        else
        {
            int total = 0;
-            for (int role = 0; role < op_data->pg_size; role++)
+            for (int i = 0; i < op_data->stripe_count; i++)
            {
-                if (!op_data->stripes[role].missing)
+                if (!op_data->stripes[i].not_exists)
                {
+                    // use "missing" flag to distinguish actual read errors and inconsistent chunks
                    total++;
-                    op_data->stripes[role].read_error = true;
+                    op_data->stripes[i].missing = true;
                }
            }
-            for (int role: good_subset)
+            for (int i: good_subset)
            {
-                op_data->stripes[role].read_error = false;
+                op_data->stripes[i].missing = false;
            }
-            for (int role = 0; role < op_data->pg_size; role++)
+            for (int i = 0; i < op_data->stripe_count; i++)
            {
-                if (!op_data->stripes[role].missing && op_data->stripes[role].read_error)
+                if (op_data->stripes[i].missing)
                {
+                    op_data->stripes[i].read_error = true;
                    printf(
                        "[PG %u/%u] Object %jx:%jx v%ju chunk %d on OSD %ju doesn't match other chunks%s\n",
                        INODE_POOL(op_data->oid.inode), op_data->pg_num,
                        op_data->oid.inode, op_data->oid.stripe, op_data->fact_ver,
-                        role, op_data->stripes[role].osd_num,
+                        op_data->stripes[i].role, op_data->stripes[i].osd_num,
                        scrub_find_best ? ", marking it as corrupted" : ""
                    );
                }
            }
-            if (!scrub_find_best && good_subset.size() < total)
+        }
+    }
+    bool mark = inconsistent;
+    for (int role = 0; !mark && role < op_data->stripe_count; role++)
+    {
+        if (op_data->stripes[role].read_error || op_data->stripes[role].not_exists)
+            mark = true;
+    }
+    if (!mark)
+    {
+        return;
+    }
+    // FIXME: ref = true ideally... because new_state != state is not necessarily true if it's freed and recreated
+    op_data->object_state = mark_object(*op_data->pg, op_data->oid, op_data->object_state, false /*ref*/, [op_data, inconsistent](pg_osd_set_t & new_set)
+    {
+        // Mark object chunk(s) as corrupted and/or missing and/or inconsistent
+        int changes = 0;
+        for (int i = 0; i < op_data->stripe_count; i++)
+        {
+            // Find the same stripe in new_set
+            int set_pos = 0;
+            while (set_pos < new_set.size() && (op_data->stripes[i].osd_num != new_set[set_pos].osd_num ||
+                op_data->stripes[i].role != new_set[set_pos].role))
            {
-                inconsistent = true;
-                printf(
-                    "[PG %u/%u] Object %jx:%jx v%ju is marked as inconsistent because scrub_find_best is turned off. Use vitastor-cli fix to fix it\n",
-                    INODE_POOL(op_data->oid.inode), op_data->pg_num,
-                    op_data->oid.inode, op_data->oid.stripe, op_data->fact_ver
-                );
-                for (int role = 0; role < op_data->pg_size; role++)
-                {
-                    if (!op_data->stripes[role].missing && op_data->stripes[role].read_error)
-                    {
-                        // Undo error locator marking chunk as bad
-                        op_data->stripes[role].read_error = false;
-                    }
-                }
+                set_pos++;
+            }
+            if (set_pos >= new_set.size())
+            {
+                continue;
+            }
+            if (op_data->stripes[i].not_exists)
+            {
+                changes++;
+                new_set.erase(new_set.begin()+set_pos, new_set.begin()+set_pos+1);
+                continue;
+            }
+            auto & chunk = new_set[set_pos];
+            if (op_data->stripes[i].read_error && chunk.loc_bad != LOC_CORRUPTED)
+            {
+                changes++;
+                chunk.loc_bad = LOC_CORRUPTED;
+            }
+            else if (op_data->stripes[i].read_end > 0 && !op_data->stripes[chunk.role].missing &&
+                (chunk.loc_bad & LOC_CORRUPTED))
+            {
+                changes++;
+                chunk.loc_bad &= ~LOC_CORRUPTED;
+            }
+            if (inconsistent && !(chunk.loc_bad & LOC_INCONSISTENT))
+            {
+                changes++;
+                chunk.loc_bad |= LOC_INCONSISTENT;
+            }
+            else if (!inconsistent && (chunk.loc_bad & LOC_INCONSISTENT))
+            {
+                changes++;
+                chunk.loc_bad &= ~LOC_INCONSISTENT;
            }
        }
-    }
-    for (int role = 0; role < op_data->pg_size; role++)
+        return changes;
+    });
+}
+
+void osd_t::continue_primary_scrub(osd_op_t *cur_op)
+{
+    if (!cur_op->op_data && !prepare_primary_rw(cur_op))
+        return;
+    if (cur_op->op_data->st == 1)
+        goto resume_1;
+    else if (cur_op->op_data->st == 2)
+        goto resume_2;
+    submit_scrub_subops(cur_op);
+resume_1:
+    return;
+resume_2:
+    if (cur_op->op_data->errors > 0 &&
+        // I/O and checksum errors (represented by stripes[i].read_error) are OK
+        (cur_op->op_data->errcode != -EIO && cur_op->op_data->errcode != -EDOM))
    {
-        if (op_data->stripes[role].osd_num != 0 &&
-            (op_data->stripes[role].read_error || op_data->stripes[role].not_exists) ||
-            inconsistent)
-        {
-            // Got at least 1 read error or mismatch, mark the object as corrupted
-            auto & pg = pgs.at({ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num });
-            // FIXME: ref = true ideally... because new_state != state is not necessarily true if it's freed and recreated
-            op_data->object_state = mark_object_corrupted(pg, op_data->oid, op_data->object_state, op_data->stripes, false, inconsistent);
-            break;
-        }
+        finish_op(cur_op, cur_op->op_data->errcode);
+        return;
    }
+    scrub_check_results(cur_op);
    finish_op(cur_op, 0);
 }
@@ -173,5 +173,7 @@ int main(int narg, char *args[])
    {
        fprintf(stderr, "Files are equal\n");
    }
+    free(buf1);
+    free(buf2);
    return 0;
 }
@@ -480,10 +480,76 @@ void test_writeback()
    printf("[ok] writeback test\n");
 }

+static void copy_write_for_test(writeback_cache_t *wb, uint64_t offset, uint64_t len, int state, uint64_t new_flush_id)
+{
+    void *buf = malloc_or_die(len);
+    cluster_op_t *op = new cluster_op_t();
+    op->opcode = OSD_OP_WRITE;
+    op->inode = 0x1000000000001;
+    op->offset = offset;
+    op->len = len;
+    op->iov.push_back(buf, len);
+    wb->copy_write(op, state, new_flush_id);
+    delete op;
+    free(buf);
+}
+
+void test_writeback_merge()
+{
+    writeback_cache_t *wb = new writeback_cache_t;
+    // [1000..3000]
+    copy_write_for_test(wb, 1000, 2000, CACHE_DIRTY, 0);
+    assert(wb->writeback_bytes == 2000);
+    assert(wb->writeback_queue_size == 1);
+    // [1000..3000][3000..4000]
+    copy_write_for_test(wb, 3000, 1000, CACHE_DIRTY, 0);
+    assert(wb->writeback_bytes == 3000);
+    assert(wb->writeback_queue_size == 1);
+    // [1000..3000][3000..4000], [5000..6000]
+    copy_write_for_test(wb, 5000, 1000, CACHE_DIRTY, 0);
+    assert(wb->writeback_bytes == 4000);
+    assert(wb->writeback_queue_size == 2);
+    // [1000..2500], [3500..4000], [5000..6000]
+    copy_write_for_test(wb, 2500, 1000, CACHE_WRITTEN, 0);
+    assert(wb->writeback_bytes == 3000);
+    assert(wb->writeback_queue_size == 3);
+    // [1000..2500], [3500..4000][4000...5000][5000..6000]
+    copy_write_for_test(wb, 4000, 1000, CACHE_DIRTY, 0);
+    assert(wb->writeback_bytes == 4000);
+    assert(wb->writeback_queue_size == 2);
+    // [1000..2500], [3500..4500][4500...5000][5000..6000]
+    copy_write_for_test(wb, 3500, 1000, CACHE_DIRTY, 0);
+    assert(wb->writeback_bytes == 4000);
+    assert(wb->writeback_queue_size == 2);
+    // [1000..2500], [3500..4500], [5000..6000]
+    copy_write_for_test(wb, 4500, 500, CACHE_WRITTEN, 0);
+    assert(wb->writeback_bytes == 3500);
+    assert(wb->writeback_queue_size == 3);
+    // [1000..2500][2500..3500][3500..4500], [5000..6000]
+    copy_write_for_test(wb, 2500, 1000, CACHE_DIRTY, 0);
+    assert(wb->writeback_bytes == 4500);
+    assert(wb->writeback_queue_size == 2);
+    // [1000..2500][2500..3500][3500..4500], [5500..6000]
+    copy_write_for_test(wb, 5000, 500, CACHE_WRITTEN, 0);
+    assert(wb->writeback_bytes == 4000);
+    assert(wb->writeback_queue_size == 2);
+    // [1000..2500][2500..3500][3500..4000], [5500..6000]
+    copy_write_for_test(wb, 4000, 1000, CACHE_WRITTEN, 0);
+    assert(wb->writeback_bytes == 3500);
+    assert(wb->writeback_queue_size == 2);
+    // [1000..2500][2500..3500][3500..4000][4000..5500][5500..6000]
+    copy_write_for_test(wb, 4000, 1500, CACHE_DIRTY, 0);
+    assert(wb->writeback_bytes == 5000);
+    assert(wb->writeback_queue_size == 1);
+    delete wb;
+    printf("[ok] writeback merge test\n");
+}
+
 int main(int narg, char *args[])
 {
    test1();
    test2();
    test_writeback();
+    test_writeback_merge();
    return 0;
 }
@@ -3,6 +3,7 @@

 #include <sys/epoll.h>
 #include <sys/poll.h>
+#include <poll.h>
 #include <unistd.h>
 #include <stdexcept>

@@ -74,6 +75,27 @@ void epoll_manager_t::set_fd_handler(int fd, bool wr, std::function<void(int, in
            throw std::runtime_error(std::string("epoll_ctl: ") + strerror(errno));
        }
        epoll_handlers[fd] = handler;
+        // We use edge-triggered epoll so it may miss events which already happened
+        // on the FD at the moment of adding it to epoll. So check for these with poll()
+        struct pollfd initpoll = { .fd = fd, .events = (short)((wr ? POLLOUT : 0) | POLLIN | POLLRDHUP) };
+        int r = poll(&initpoll, 1, 0);
+        if (r < 0)
+            throw std::runtime_error(std::string("poll: ") + strerror(errno));
+        if (r > 0)
+        {
+            auto events = ((initpoll.revents & POLLOUT) ? EPOLLOUT : 0) |
+                ((initpoll.revents & POLLIN) ? EPOLLIN : 0) |
+                ((initpoll.revents & POLLRDHUP) ? EPOLLRDHUP : 0);
+            tfd->set_timer_us(1, false, [this, fd, events](int)
+            {
+                auto cb_it = epoll_handlers.find(fd);
+                if (cb_it != epoll_handlers.end())
+                {
+                    auto & cb = cb_it->second;
+                    cb(fd, events);
+                }
+            });
+        }
    }
    else
    {
@@ -29,6 +29,7 @@ ring_loop_t::ring_loop_t(int qd, bool multithreaded)
    {
        free_ring_data[i] = i;
    }
+    in_loop = false;
 }

 ring_loop_t::~ring_loop_t()
@@ -86,6 +87,11 @@ io_uring_sqe* ring_loop_t::get_sqe()

 void ring_loop_t::loop()
 {
+    if (in_loop)
+    {
+        return;
+    }
+    in_loop = true;
    if (ring_eventfd >= 0)
    {
        // Reset eventfd counter
@@ -125,6 +131,17 @@ void ring_loop_t::loop()
        }
        io_uring_cqe_seen(&ring, cqe);
    }
+    in_loop = false;
+    loop_continue();
+}
+
+void ring_loop_t::loop_continue()
+{
+    if (in_loop)
+    {
+        return;
+    }
+    in_loop = true;
    do
    {
        loop_again = false;
@@ -140,6 +157,7 @@ void ring_loop_t::loop()
            }
        }
    } while (loop_again);
+    in_loop = false;
 }

 unsigned ring_loop_t::save()
@@ -189,5 +207,7 @@ int ring_loop_t::register_eventfd()
        ring_eventfd = -1;
        return r;
    }
+    // Loop once to prevent skipping events happened before eventfd was registered
+    loop();
    return ring_eventfd;
 }
@@ -129,6 +129,7 @@ class ring_loop_t
    bool mt;
    int *free_ring_data;
    unsigned free_ring_data_ptr;
+    bool in_loop;
    bool loop_again;
    struct io_uring ring;
    int ring_eventfd = -1;
@@ -164,6 +165,7 @@ public:
    }

    void loop();
+    void loop_continue();
    void wakeup();

    unsigned save();
@@ -12,7 +12,7 @@ $ETCDCTL put /vitastor/config/inode/1/1 '{"name":"testimg","size":'$((IMG_SIZE*1

 NBD_DEV=$(sudo build/src/client/vitastor-nbd map --nbd_timeout 180 --etcd_address $ETCD_URL --image testimg --logfile ./testdata/nbd.log &)

-trap "sudo build/src/client/vitastor-nbd unmap $NBD_DEV"'; kill -9 $(jobs -p)' EXIT
+trap "sudo build/src/client/vitastor-nbd unmap $NBD_DEV"' || true; kill -9 $(jobs -p)' EXIT

 sudo chown $(id -u) $NBD_DEV
Author	SHA1	Message	Date
Vitaliy Filippov	194f7e0187	Trigger event loop more intelligently and automatically in libvitastor_c	2025-03-02 18:04:56 +03:00
Vitaliy Filippov	10216a5fb5	Build node.js addon as a Debian package	2025-03-02 18:04:56 +03:00
Vitaliy Filippov	3932eb7ff6	Trigger event loop once after each vitastor_c_* call	2025-03-02 01:23:41 +03:00
Vitaliy Filippov	69cbe7bbb2	Release 1.11.0 New features: - Support containerized Vitastor installations: http://vitastor.io/docs/installation/docker.html - Add new functions to the node.js binding: delete(), get_immediate_commit(), on_ready(), get_min_io_size(), get_max_atomic_write_size() - S3 (Zenko Cloudserver with Vitastor support) is coming shortly and will be released separately Bug fixes: - Use IP-derived etcd node names in make-etcd - Set short name of the OSD process to display in `top` - Fix snap-create without pool_id failing when there are multiple pools - Several bugs are fixed in the write-back cache, it should now be stable: - Fix incorrect snapshot reads from dirty write-back cache - Do not try to repeat pending writebacks on OSD reconnections - Fix client hangs with multiple SYNCs in the writeback queue - Fix client hangs do to incorrect calculation of the writeback queue size - Several improvements for NBD mapping/unmapping: - Add a workaround for race condition in the Linux kernel NBD driver leading to vitastor-nbd sometimes breaking a previously mapped device instead of setting up a new one - Check if the device is actually mapped in vitastor-nbd unmap - Fix device name/number validation in vitastor-nbd - Fix OSD crashes after starting with corrupted metadata - from now it will skip corrupted metadata entries and heal itself - Fix scrubbing of misplaced objects and object state recalculation after vitastor-cli fix - previously, an OSD restart could be required to fix object states - Make primary OSD distribution more stable by using murmur3 hash instead of the old pseudo-rng - Fix monitor sometimes racing with itself - do not touch /pool/stats from stats aggregation if PG recheck is active - Sort vitastor-cli ls output by name by default - Update antietcd to 1.1.2	2025-03-01 13:39:42 +03:00
Vitaliy Filippov	4950a1636c	Allow "infinite" startup for clients if explicitly requested	2025-03-01 13:39:42 +03:00
Vitaliy Filippov	2eb20dff28	Do not crash on io_uring initialization failure in node-vitastor	2025-03-01 13:29:48 +03:00
Vitaliy Filippov	59f0b0427c	Support containerized Vitastor installations	2025-02-27 20:06:15 +03:00
Vitaliy Filippov	124162ad38	Use IP-derived etcd node names in make-etcd	2025-02-26 11:54:37 +03:00
Vitaliy Filippov	391c92af1a	Set OSD process name	2025-02-26 11:54:37 +03:00
Vitaliy Filippov	c3d8fdd855	Fix snap-create without pool_id ID generation with multiple pools	2025-02-26 11:54:28 +03:00
Vitaliy Filippov	9ccf3af97b	Add qemu-block-extra and qemu-utils	2025-02-23 15:08:16 +03:00
Vitaliy Filippov	568a209f0d	Update docker image to debian bookworm	2025-02-23 13:27:32 +03:00
Vitaliy Filippov	b151013201	Fix snapshot reads from a dirty write-back cache	2025-02-23 02:31:19 +03:00
Vitaliy Filippov	4a763725fe	Add free() to bindiff.c	2025-02-22 16:52:19 +03:00
Vitaliy Filippov	b8d83cd7f4	No, it's not a good idea to destroy client in the child nbd process Should probably have been an obvious side effect :-) Child process gets open file descriptors to parent's epoll/timerfd, and it's totally OK to just close() all of them, but it's absolutely NOT OK to run destructors - they modify the kernel state of epoll/timerfd before destroying. So, basically, when we destroy the client in the child process, we break it in the parent too. This also means that cluster_client_t doesn't support fork(). :-)	2025-02-22 15:10:27 +03:00
Vitaliy Filippov	2e9ee2fe20	Do not try to repeat pending writebacks	2025-02-22 14:16:44 +03:00
Vitaliy Filippov	508ae852e4	Fix trap in test_rebalance_verify	2025-02-22 02:18:41 +03:00
Vitaliy Filippov	97ee400505	Add a workaround for race condition in the Linux kernel NBD driver Do all NBD configuration in the child process, after the last fork. Why? It's needed because there is a race condition in the Linux kernel nbd driver in nbd_add_socket() - it saves `current` task pointer as `nbd->task_setup` and then rechecks if the new `current` is the same. Problem is that if that process is already dead, `current` may be freed and then replaced by another process with the same pointer value. So the check passes and NBD allows a different process to set up a device which is already set up. Proper fix would have to be done in the kernel code, but the workaround is obviously to perform NBD setup from the process which will then actually call NBD_DO_IT. That process stays alive during the whole time of NBD device execution and the (nbd->task_setup != current) check always works correctly, and we don't accidentally break previous NBD devices while setting up a new device. Forking to check every device is of course rather slow, so we also do an additional check by calling list_mapped() before searching for a free NBD device.	2025-02-21 13:17:37 +03:00
Vitaliy Filippov	5ee4894fab	Check if mapped in vitastor-nbd unmap	2025-02-21 01:28:06 +03:00
Vitaliy Filippov	125dcafb11	Prevent OSD crashes when metadata is corrupted	2025-02-20 02:19:32 +03:00
Vitaliy Filippov	9f44cf71df	Fix device name/number validation in vitastor-nbd	2025-02-20 01:33:11 +03:00
Vitaliy Filippov	df3c63ca7f	Sort vitastor-cli ls by name by default	2025-02-20 01:32:49 +03:00
Vitaliy Filippov	be66edd09f	Prevent infinite loops on syncs in writeback_overflow	2025-02-19 01:44:12 +03:00
Vitaliy Filippov	ccbc0c5928	Add assert !writeback_bytes	2025-02-19 01:15:46 +03:00
Vitaliy Filippov	78ca4538bf	Fix qemu docker build for ubuntu	2025-02-18 23:44:16 +03:00
Vitaliy Filippov	86b5760ec1	Fix writeback incorrectly calculating queue size which was leading to client hangs	2025-02-18 23:42:55 +03:00
Vitaliy Filippov	27f3803d2f	Add vitastor_c_delete() and delete() to the node.js binding	2025-02-15 18:27:17 +03:00
Vitaliy Filippov	2ead06e126	Add ubuntu jammy to docs	2025-02-12 15:32:35 +03:00
Vitaliy Filippov	a5d5559f8e	Add get_immediate_commit() to the node.js binding	2025-02-06 01:35:48 +03:00
Vitaliy Filippov	e8e7ba8fde	Add FIXME for CAS in non-immediate_commit mode	2025-02-06 01:35:48 +03:00
Vitaliy Filippov	6fd831a299	Add on_ready(), get_min_io_size(), get_max_atomic_write_size() to the node.js binding	2025-02-06 01:35:48 +03:00
Vitaliy Filippov	069808dfce	Fix --config_path option in docs	2025-01-24 17:21:11 +03:00
Vitaliy Filippov	bcefa42bc0	Scrub all chunks, not just 1 chunk per position	2025-01-23 02:02:55 +03:00
Vitaliy Filippov	4636e02d43	Remove scheme, pg_size, pg_data_size from op_data	2025-01-23 01:20:31 +03:00
Vitaliy Filippov	e4c7d1c147	s/3/4/	2025-01-23 01:20:31 +03:00
Vitaliy Filippov	a4677f3e69	Mention P5530	2025-01-23 01:20:31 +03:00
Vitaliy Filippov	7cbf207d65	Use murmur3 to select primary OSD instead of old pseudo-rng	2025-01-18 12:28:54 +03:00
Vitaliy Filippov	7c9711af20	Do not touch /pool/stats from stats aggregation if PG recheck is active	2025-01-16 20:41:16 +03:00
Vitaliy Filippov	33ef701464	Update antietcd to 1.1.2	2025-01-04 02:13:36 +03:00
				`@@ -0,0 +1 @@`
				`deb http://vitastor.io/debian bookworm main`
				`@@ -1 +0,0 @@`
				`deb http://vitastor.io/debian bullseye main`