Destroy qemu driver under lock (may fix crashes after attach/detach?)

Support used_for_app instead of used_for_fs
Fix PG object count statistics on deletion of non-existing objects
2025-03-07 13:13:27 +03:00 · 2025-03-07 01:03:43 +03:00 · 2025-03-04 00:40:56 +03:00 · 2025-03-04 00:40:35 +03:00 · 2025-03-03 00:57:09 +03:00 · 2025-03-03 00:57:00 +03:00
110 changed files with 1861 additions and 773 deletions
@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)

 project(vitastor)

-set(VITASTOR_VERSION "1.10.1")
+set(VITASTOR_VERSION "1.11.0")

 add_subdirectory(src)
@@ -41,6 +41,7 @@ Vitastor поддерживает QEMU-драйвер, протоколы NBD и
  - [Автор и лицензия](docs/intro/author.ru.md)
 - Установка
  - [Пакеты](docs/installation/packages.ru.md)
+  - [Docker](docs/installation/docker.ru.md)
  - [Proxmox](docs/installation/proxmox.ru.md)
  - [OpenNebula](docs/installation/opennebula.ru.md)
  - [OpenStack](docs/installation/openstack.ru.md)
@@ -41,6 +41,7 @@ Read more details in the documentation. You can start from here: [Quick Start](d
  - [Author and license](docs/intro/author.en.md)
 - Installation
  - [Packages](docs/installation/packages.en.md)
+  - [Docker](docs/installation/docker.en.md)
  - [Proxmox](docs/installation/proxmox.en.md)
  - [OpenNebula](docs/installation/opennebula.en.md)
  - [OpenStack](docs/installation/openstack.en.md)
@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v1.10.1
+VITASTOR_VERSION ?= v1.11.0

 all: build push

@@ -49,7 +49,7 @@ spec:
            capabilities:
              add: ["SYS_ADMIN"]
            allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v1.10.1
+          image: vitalif/vitastor-csi:v1.11.0
          args:
            - "--node=$(NODE_ID)"
            - "--endpoint=$(CSI_ENDPOINT)"
@@ -121,7 +121,7 @@ spec:
            privileged: true
            capabilities:
              add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v1.10.1
+          image: vitalif/vitastor-csi:v1.11.0
          args:
            - "--node=$(NODE_ID)"
            - "--endpoint=$(CSI_ENDPOINT)"
@@ -5,7 +5,7 @@ package vitastor

 const (
    vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "1.10.1"
+    vitastorCSIDriverVersion = "1.11.0"
 )

 // Config struct fills the parameters of request or user input
@@ -1,4 +1,4 @@
-vitastor (1.10.1-1) unstable; urgency=medium
+vitastor (1.11.0-1) unstable; urgency=medium

  * Bugfixes

@@ -2,7 +2,10 @@ Source: vitastor
 Section: admin
 Priority: optional
 Maintainer: Vitaliy Filippov <vitalif@yourcmc.ru>
-Build-Depends: debhelper, liburing-dev (>= 0.6), g++ (>= 8), libstdc++6 (>= 8), linux-libc-dev, libgoogle-perftools-dev, libjerasure-dev, libgf-complete-dev, libibverbs-dev, libisal-dev, cmake, pkg-config, libnl-3-dev, libnl-genl-3-dev
+Build-Depends: debhelper, liburing-dev (>= 0.6), g++ (>= 8), libstdc++6 (>= 8),
+  linux-libc-dev, libgoogle-perftools-dev, libjerasure-dev, libgf-complete-dev,
+  libibverbs-dev, libisal-dev, cmake, pkg-config, libnl-3-dev, libnl-genl-3-dev,
+  node-bindings <!nocheck>, node-gyp, node-nan
 Standards-Version: 4.5.0
 Homepage: https://vitastor.io/
 Rules-Requires-Root: no
@@ -59,3 +62,9 @@ Architecture: amd64
 Depends: ${shlibs:Depends}, ${misc:Depends}, vitastor-client, patch, python3, jq
 Description: Vitastor OpenNebula storage plugin
 Vitastor storage plugin for OpenNebula.
+
+Package: node-vitastor
+Architecture: amd64
+Depends: ${shlibs:Depends}, ${misc:Depends}, node-bindings
+Description: Node.js bindings for Vitastor client
+ Node.js native bindings for the Vitastor client library (vitastor-client).
@@ -0,0 +1 @@
+usr/lib/x86_64-linux-gnu/nodejs/vitastor
@@ -1,8 +1,10 @@
 # Build patched QEMU for Debian inside a container
 # cd ..; podman build --build-arg REL=bullseye -v `pwd`/packages:/root/packages -f debian/patched-qemu.Dockerfile .

+ARG DISTRO=debian
 ARG REL=
-FROM debian:$REL
+FROM $DISTRO:$REL
+ARG DISTRO=debian
 ARG REL=

 WORKDIR /root
@@ -20,8 +22,8 @@ RUN if [ "$REL" = "buster" -o "$REL" = "bullseye" -o "$REL" = "bookworm" ]; then
    echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf

 RUN apt-get update
-RUN apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts
-RUN apt-get -y build-dep qemu
+RUN DEBIAN_FRONTEND=noninteractive TZ=Europe/Moscow apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts
+RUN DEBIAN_FRONTEND=noninteractive TZ=Europe/Moscow apt-get -y build-dep qemu
 # To build a custom version
 #RUN cp /root/packages/qemu-orig/* /root
 RUN apt-get --download-only source qemu
@@ -38,9 +40,9 @@ ADD src/client/qemu_driver.c /root/qemu_driver.c
 #    apt-get install -y vitastor-client vitastor-client-dev quilt

 RUN set -e; \
-    dpkg -i /root/packages/vitastor-$REL/vitastor-client_*.deb /root/packages/vitastor-$REL/vitastor-client-dev_*.deb; \
+    DEBIAN_FRONTEND=noninteractive TZ=Europe/Moscow apt-get -y install /root/packages/vitastor-$REL/vitastor-client_*.deb /root/packages/vitastor-$REL/vitastor-client-dev_*.deb; \
    apt-get update; \
-    apt-get install -y quilt; \
+    DEBIAN_FRONTEND=noninteractive TZ=Europe/Moscow apt-get -y install quilt; \
    mkdir -p /root/packages/qemu-$REL; \
    rm -rf /root/packages/qemu-$REL/*; \
    cd /root/packages/qemu-$REL; \
@@ -4,6 +4,14 @@ export DH_VERBOSE = 1
 %:
 	dh $@

+override_dh_install:
+	perl -pe 's!prefix=/usr!prefix='`pwd`'/debian/tmp/usr!' < obj-x86_64-linux-gnu/src/client/vitastor.pc > node-binding/vitastor.pc
+	cd node-binding && PKG_CONFIG_PATH=./ PKG_CONFIG_ALLOW_SYSTEM_CFLAGS=1 npm install --unsafe-perm || exit 1
+	mkdir -p debian/tmp/usr/lib/x86_64-linux-gnu/nodejs/vitastor/build/Release
+	cp -v node-binding/package.json node-binding/index.js node-binding/addon.cc node-binding/addon.h node-binding/client.cc node-binding/client.h debian/tmp/usr/lib/x86_64-linux-gnu/nodejs/vitastor
+	cp -v node-binding/build/Release/addon.node debian/tmp/usr/lib/x86_64-linux-gnu/nodejs/vitastor/build/Release
+	dh_install
+
 override_dh_installdeb:
 	cat debian/fio_version >> debian/vitastor-fio.substvars
 	[ -f debian/qemu_version ] && (cat debian/qemu_version >> debian/vitastor-qemu.substvars) || true
@@ -22,7 +22,8 @@ RUN set -e -x; \
    echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf

 RUN apt-get update && \
-    apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts libjerasure-dev cmake libibverbs-dev librdmacm-dev libisal-dev libnl-3-dev libnl-genl-3-dev curl && \
+    apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts libjerasure-dev cmake \
+        libibverbs-dev librdmacm-dev libisal-dev libnl-3-dev libnl-genl-3-dev curl nodejs npm node-nan node-bindings && \
    apt-get -y build-dep fio && \
    apt-get --download-only source fio

@@ -1,9 +1,11 @@
 # Build Docker image with Vitastor packages

-FROM debian:bullseye
+FROM debian:bookworm

-ADD vitastor.list /etc/apt/sources.list.d
-ADD vitastor.gpg /etc/apt/trusted.gpg.d
-ADD vitastor.pref /etc/apt/preferences.d
-ADD apt.conf /etc/apt/
-RUN apt-get update && apt-get -y install vitastor qemu-system-x86 qemu-system-common && apt-get clean
+ADD etc/apt /etc/apt/
+RUN apt-get update && apt-get -y install vitastor qemu-system-x86 qemu-system-common qemu-block-extra qemu-utils jq nfs-common && apt-get clean
+ADD sleep.sh /usr/bin/
+ADD install.sh /usr/bin/
+ADD scripts /opt/scripts/
+ADD etc /etc/
+RUN ln -s /usr/lib/vitastor/mon/make-etcd /usr/bin/make-etcd
@@ -0,0 +1,9 @@
+VITASTOR_VERSION ?= v1.11.0
+
+all: build push
+
+build:
+	@docker build --rm -t vitalif/vitastor:$(VITASTOR_VERSION) .
+
+push:
+	@docker push vitalif/vitastor:$(VITASTOR_VERSION)
@@ -0,0 +1 @@
+deb http://vitastor.io/debian bookworm main
@@ -0,0 +1,27 @@
+[Unit]
+Description=Containerized etcd for Vitastor
+After=network-online.target local-fs.target time-sync.target docker.service vitastor-host.service
+Wants=network-online.target local-fs.target time-sync.target docker.service vitastor-host.service
+PartOf=vitastor.target
+
+[Service]
+Restart=always
+Environment=GOGC=50
+EnvironmentFile=/etc/vitastor/docker.conf
+EnvironmentFile=/etc/vitastor/etcd.conf
+SyslogIdentifier=etcd
+ExecStart=bash -c 'docker run --rm -i -v /var/lib/vitastor/etcd:/data \
+    --log-driver none --network host $CONTAINER_OPTIONS --name vitastor-etcd \
+    $ETCD_IMAGE /usr/local/bin/etcd --name "$ETCD_NAME" --data-dir /data \
+    --snapshot-count 10000 --advertise-client-urls http://$ETCD_IP:2379 --listen-client-urls http://$ETCD_IP:2379 \
+    --initial-advertise-peer-urls http://$ETCD_IP:2380 --listen-peer-urls http://$ETCD_IP:2380 \
+    --initial-cluster-token vitastor-etcd-1 --initial-cluster "$ETCD_INITIAL_CLUSTER" \
+    --initial-cluster-state new --max-txn-ops=100000 --max-request-bytes=104857600 \
+    --auto-compaction-retention=10 --auto-compaction-mode=revision'
+ExecStop=docker stop vitastor-etcd
+Restart=always
+StartLimitInterval=0
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,23 @@
+[Unit]
+Description=Empty container for running Vitastor commands
+After=network-online.target local-fs.target time-sync.target docker.service
+Wants=network-online.target local-fs.target time-sync.target docker.service
+PartOf=vitastor.target
+
+[Service]
+Restart=always
+EnvironmentFile=/etc/vitastor/docker.conf
+ExecStart=bash -c 'docker run --rm -i -v /etc/vitastor:/etc/vitastor -v /dev:/dev \
+    --privileged --log-driver none --network host --name vitastor vitastor:$VITASTOR_VERSION \
+    sleep.sh'
+ExecStartPost=udevadm trigger
+ExecStop=docker stop vitastor
+WorkingDirectory=/
+PrivateTmp=false
+TasksMax=infinity
+Restart=always
+StartLimitInterval=0
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,23 @@
+[Unit]
+Description=Containerized Vitastor monitor
+After=network-online.target local-fs.target time-sync.target docker.service
+Wants=network-online.target local-fs.target time-sync.target docker.service
+PartOf=vitastor.target
+
+[Service]
+Restart=always
+EnvironmentFile=/etc/vitastor/docker.conf
+SyslogIdentifier=vitastor-mon
+ExecStart=bash -c 'docker run --rm -i -v /etc/vitastor:/etc/vitastor -v /var/lib/vitastor:/var/lib/vitastor -v /dev:/dev \
+    --log-driver none --network host $CONTAINER_OPTIONS --name vitastor-mon vitastor:$VITASTOR_VERSION \
+    node /usr/lib/vitastor/mon/mon-main.js'
+ExecStop=docker stop vitastor-mon
+WorkingDirectory=/
+PrivateTmp=false
+TasksMax=infinity
+Restart=always
+StartLimitInterval=0
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,27 @@
+[Unit]
+Description=Containerized Vitastor object storage daemon osd.%i
+After=network-online.target local-fs.target time-sync.target docker.service vitastor-host.service
+Wants=network-online.target local-fs.target time-sync.target docker.service vitastor-host.service
+PartOf=vitastor.target
+
+[Service]
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitMEMLOCK=infinity
+EnvironmentFile=/etc/vitastor/docker.conf
+SyslogIdentifier=vitastor-osd%i
+ExecStart=bash -c 'docker run --rm -i -v /etc/vitastor:/etc/vitastor -v /dev:/dev \
+    $(for i in $(ls /dev/vitastor/osd%i-*); do echo --device $i:$i; done) \
+    --log-driver none --network host --ulimit nofile=1048576 --ulimit memlock=-1 $CONTAINER_OPTIONS --name vitastor-osd%i \
+    vitastor:$VITASTOR_VERSION vitastor-disk exec-osd /dev/vitastor/osd%i-data'
+ExecStartPre=+docker exec vitastor vitastor-disk pre-exec /dev/vitastor/osd%i-data
+ExecStop=docker stop vitastor-etcd%i
+WorkingDirectory=/
+PrivateTmp=false
+TasksMax=infinity
+Restart=always
+StartLimitInterval=0
+RestartSec=10
+
+[Install]
+WantedBy=vitastor.target
@@ -0,0 +1,4 @@
+[Unit]
+Description=vitastor target
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,7 @@
+SUBSYSTEM=="block", ENV{ID_PART_ENTRY_TYPE}=="e7009fac-a5a1-4d72-af72-53de13059903", \
+    OWNER="vitastor", GROUP="vitastor", \
+    IMPORT{program}="/usr/bin/docker exec vitastor vitastor-disk udev $devnode", \
+    SYMLINK+="vitastor/$env{VITASTOR_ALIAS}"
+
+ENV{VITASTOR_OSD_NUM}!="", ACTION=="add", RUN{program}+="/usr/bin/systemctl enable --now --no-block vitastor-osd@$env{VITASTOR_OSD_NUM}"
+ENV{VITASTOR_OSD_NUM}!="", ACTION=="remove", RUN{program}+="/usr/bin/systemctl disable --now --no-block vitastor-osd@$env{VITASTOR_OSD_NUM}"
@@ -0,0 +1,11 @@
+#
+# Configuration file for containerized Vitastor installation
+# (non-Kubernetes, with systemd and udev-based orchestration)
+#
+
+# Desired Vitastor version
+VITASTOR_VERSION=1.11.0
+
+# Additional arguments for all containers
+# For example, you may want to specify a custom logging driver here
+CONTAINER_OPTIONS=""
@@ -0,0 +1,4 @@
+ETCD_IMAGE=quay.io/coreos/etcd:v3.5.18
+ETCD_NAME=""
+ETCD_IP=""
+ETCD_INITIAL_CLUSTER=""
@@ -0,0 +1,2 @@
+{
+}
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+cp -urv /etc/default /host-etc/
+cp -urv /etc/systemd /host-etc/
+cp -urv /etc/udev /host-etc/
+cp -urnv /etc/vitastor /host-etc/
+cp -urnv /opt/scripts/* /host-bin/
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker exec -it vitastor vitastor-cli "$@"
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker exec -it vitastor vitastor-disk "$@"
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker exec -it vitastor fio "$@"
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker exec -it vitastor vitastor-nbd "$@"
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+while :; do sleep infinity; done
@@ -1 +0,0 @@
-deb http://vitastor.io/debian bullseye main
@@ -13,7 +13,7 @@ Vitastor configuration consists of:
 - [Separate OSD settings](config/pool.en.md#osd-settings)
 - [Inode configuration](config/inode.en.md) i.e. image metadata like name, size and parent reference

-Configuration parameters can be set in 3 places:
+Configuration parameters can be set in 4 places:
 - Configuration file (`/etc/vitastor/vitastor.conf` or other path)
 - etcd key `/vitastor/config/global`. Most variables can be set there, but etcd
  connection parameters should obviously be set in the configuration file.
@@ -14,7 +14,7 @@
 - [Настроек инодов](config/inode.ru.md), т.е. метаданных образов, таких, как имя, размер и ссылки на
  родительский образ

-Параметры конфигурации могут задаваться в 3 местах:
+Параметры конфигурации могут задаваться в 4 местах:
 - Файле конфигурации (`/etc/vitastor/vitastor.conf` или по другому пути)
 - Ключе в etcd `/vitastor/config/global`. Большая часть параметров может
  задаваться там, кроме, естественно, самих параметров соединения с etcd,
@@ -13,6 +13,7 @@ affect their interaction with the cluster.
 - [client_retry_interval](#client_retry_interval)
 - [client_eio_retry_interval](#client_eio_retry_interval)
 - [client_retry_enospc](#client_retry_enospc)
+- [client_wait_up_timeout](#client_wait_up_timeout)
 - [client_max_dirty_bytes](#client_max_dirty_bytes)
 - [client_max_dirty_ops](#client_max_dirty_ops)
 - [client_enable_writeback](#client_enable_writeback)
@@ -70,6 +71,19 @@ and clients are not blocked and just get EIO error code instead.
 Retry writes on out of space errors to wait until some space is freed on
 OSDs.

+## client_wait_up_timeout
+
+- Type: seconds
+- Default: 16
+- Can be changed online: yes
+
+Wait for this number of seconds until PGs are up when doing operations
+which require all PGs to be up. Currently only used by object listings
+in delete and merge-based commands ([vitastor-cli rm](../usage/cli.en.md#rm), merge and so on).
+
+The default value is calculated as `1 + OSD lease timeout`, which is
+`1 + etcd_report_interval + max_etcd_attempts*2*etcd_quick_timeout`.
+
 ## client_max_dirty_bytes

 - Type: integer
@@ -13,6 +13,7 @@
 - [client_retry_interval](#client_retry_interval)
 - [client_eio_retry_interval](#client_eio_retry_interval)
 - [client_retry_enospc](#client_retry_enospc)
+- [client_wait_up_timeout](#client_wait_up_timeout)
 - [client_max_dirty_bytes](#client_max_dirty_bytes)
 - [client_max_dirty_ops](#client_max_dirty_ops)
 - [client_enable_writeback](#client_enable_writeback)
@@ -72,6 +73,19 @@ RDMA и хотите повысить пиковую производитель
 Повторять запросы записи, завершившиеся с ошибками нехватки места, т.е.
 ожидать, пока на OSD не освободится место.

+## client_wait_up_timeout
+
+- Тип: секунды
+- Значение по умолчанию: 16
+- Можно менять на лету: да
+
+Время ожидания поднятия PG при операциях, требующих активности всех PG.
+В данный момент используется листингами объектов в командах, использующих
+удаление и слияние ([vitastor-cli rm](../usage/cli.ru.md#rm), merge и подобные).
+
+Значение по умолчанию вычисляется как `1 + время lease OSD`, равное
+`1 + etcd_report_interval + max_etcd_attempts*2*etcd_quick_timeout`.
+
 ## client_max_dirty_bytes

 - Тип: целое число
@@ -316,7 +316,7 @@ for hot data and slower disks - HDDs and maybe SATA SSDs - but will slightly
 decrease write performance for fast disks because page cache is an overhead
 itself.

-Choose "directsync" to use [immediate_commit](layout-cluster.ru.md#immediate_commit)
+Choose "directsync" to use [immediate_commit](layout-cluster.en.md#immediate_commit)
 (which requires disable_data_fsync) with drives having write-back cache
 which can't be turned off, for example, Intel Optane. Also note that *some*
 desktop SSDs (for example, HP EX950) may ignore O_SYNC thus making
@@ -43,7 +43,7 @@ Parameters:
 - [osd_tags](#osd_tags)
 - [primary_affinity_tags](#primary_affinity_tags)
 - [scrub_interval](#scrub_interval)
- [used_for_fs](#used_for_fs)
+- [used_for_app](#used_for_app)

 Examples:

@@ -377,24 +377,37 @@ of the OSDs containing a data chunk for a PG.
 Automatic scrubbing interval for this pool. Overrides
 [global scrub_interval setting](osd.en.md#scrub_interval).

-## used_for_fs
+## used_for_app

 - Type: string

-If non-empty, the pool is marked as used for VitastorFS with metadata stored
-in block image (regular Vitastor volume) named as the value of this pool parameter.
+If non-empty, the pool is marked as used for a separate application, for example,
+VitastorFS or S3, which allocates Vitastor volume IDs by itself and does not use
+image/inode metadata in etcd.

-When a pool is marked as used for VitastorFS, regular block volume creation in it
+When a pool is marked as used for such app, regular block volume creation in it
 is disabled (vitastor-cli refuses to create images without --force) to protect
-the user from block volume and FS file ID collisions and data loss.
+the user from block volume and FS/S3 volume ID collisions and data loss.

-[vitastor-nfs](../usage/nfs.ru.md), in its turn, refuses to use pools not marked
+Also such pools do not calculate per-inode space usage statistics in etcd because
+using it for an external application implies that it may contain a very large
+number of volumes and their statistics may take too much space in etcd.
+
+Setting used_for_app to `fs:<name>` tells Vitastor that the pool is used for VitastorFS
+with VitastorKV metadata base stored in a block image (regular Vitastor volume) named
+`<name>`.
+
+[vitastor-nfs](../usage/nfs.en.md), in its turn, refuses to use pools not marked
 for the corresponding FS when starting. This also implies that you can use one
 pool only for one VitastorFS.

-The second thing that is disabled for VitastorFS pools is reporting per-inode space
-usage statistics in etcd because a FS pool may store a very large number of files
-and statistics for them all would take a lot of space in etcd.
+If you plan to use the pool for S3, set its used_for_app to `s3:<name>`. `<name>` may
+be basically anything you want (for example, `s3:standard`) - it's not validated
+by Vitastor S3 components in any way.
+
+All other values except prefixed with `fs:` or `s3:` may be used freely and don't
+mean anything special for Vitastor core components. For now, you can use them as
+you wish.

 # Examples

@@ -42,7 +42,7 @@
 - [osd_tags](#osd_tags)
 - [primary_affinity_tags](#primary_affinity_tags)
 - [scrub_interval](#scrub_interval)
- [used_for_fs](#used_for_fs)
+- [used_for_app](#used_for_app)

 Примеры:

@@ -256,7 +256,7 @@ PG в Vitastor эферемерны, то есть вы можете менят

 ## raw_placement

- Type: string
+- Тип: строка

 Низкоуровневые правила генерации PG в форме DSL (доменно-специфичного языка).
 Используйте, только если действительно знаете, зачем вам это надо :)
@@ -383,26 +383,39 @@ OSD с "all".
 Интервал скраба, то есть, автоматической фоновой проверки данных для данного пула.
 Переопределяет [глобальную настройку scrub_interval](osd.ru.md#scrub_interval).

-## used_for_fs
+## used_for_app

- Type: string
+- Тип: строка

-Если непусто, пул помечается как используемый для файловой системы VitastorFS с
-метаданными, хранимыми в блочном образе Vitastor с именем, равным значению
-этого параметра.
+Если непусто, пул помечается как используемый для отдельного приложения, например,
+для VitastorFS или S3, которое распределяет ID образов в пуле само и не использует
+метаданные образов/инодов в etcd.

-Когда пул помечается как используемый для VitastorFS, создание обычных блочных
-образов в нём отключается (vitastor-cli отказывается создавать образы без --force),
-чтобы защитить пользователя от коллизий ID файлов и блочных образов и, таким
-образом, от потери данных.
+Когда пул помечается используемым для такого приложения, создание обычных блочных
+образов в нём запрещается (vitastor-cli отказывается создавать образы без --force),
+чтобы защитить пользователя от коллизий ID блочных образов и томов ФС/S3, и,
+таким образом, от потери данных.
+
+Также для таких пулов отключается передача статистики в etcd по отдельным инодам,
+так как использование для внешнего приложения подразумевает, что пул может содержать
+очень много томов и их статистика может занять слишком много места в etcd.
+
+Установка used_for_app в значение `fs:<name>` сообщает о том, что пул используется
+для VitastorFS с базой метаданных VitastorKV, хранимой в блочном образе с именем
+`<name>`.

 [vitastor-nfs](../usage/nfs.ru.md), в свою очередь, при запуске отказывается
-использовать для ФС пулы, не выделенные для неё. Это также означает, что один
-пул может использоваться только для одной VitastorFS.
+использовать для ФС пулы, не помеченные, как используемые для неё. Это также
+означает, что один пул может использоваться только для одной VitastorFS.

-Также для ФС-пулов отключается передача статистики в etcd по отдельным инодам,
-так как ФС-пул может содержать очень много файлов и статистика по ним всем
-заняла бы очень много места в etcd.
+Если же вы планируете использовать пул для данных S3, установите его used_for_app
+в значение `s3:<name>`, где `<name>` - любое название по вашему усмотрению
+(например, `s3:standard`) - конкретное содержимое `<name>` пока никак не проверяется
+компонентами Vitastor S3.
+
+Все остальные значения used_for_app, кроме начинающихся на `fs:` или `s3:`, не
+означают ничего особенного для основных компонентов Vitastor. Поэтому сейчас вы
+можете использовать их свободно любым желаемым способом.

 # Примеры

@@ -14,8 +14,12 @@

 {{../../installation/packages.en.md}}

+{{../../installation/docker.en.md}}
+
 {{../../installation/proxmox.en.md}}

+{{../../installation/opennebula.en.md}}
+
 {{../../installation/openstack.en.md}}

 {{../../installation/kubernetes.en.md}}
@@ -14,8 +14,12 @@

 {{../../installation/packages.ru.md}}

+{{../../installation/docker.ru.md}}
+
 {{../../installation/proxmox.ru.md}}

+{{../../installation/opennebula.ru.md}}
+
 {{../../installation/openstack.ru.md}}

 {{../../installation/kubernetes.ru.md}}
@@ -315,7 +315,7 @@
    decrease write performance for fast disks because page cache is an overhead
    itself.

-    Choose "directsync" to use [immediate_commit](layout-cluster.ru.md#immediate_commit)
+    Choose "directsync" to use [immediate_commit](layout-cluster.en.md#immediate_commit)
    (which requires disable_data_fsync) with drives having write-back cache
    which can't be turned off, for example, Intel Optane. Also note that *some*
    desktop SSDs (for example, HP EX950) may ignore O_SYNC thus making
@@ -0,0 +1,60 @@
+[Documentation](../../README.md#documentation) → Installation → Dockerized Installation
+
+-----
+
+[Читать на русском](docker.ru.md)
+
+# Dockerized Installation
+
+Vitastor may be installed in Docker/Podman. In such setups etcd, monitors and OSD
+all run in containers, but everything else looks as close as possible to a usual
+setup with packages:
+- host network is used
+- auto-start is implemented through udev and systemd
+- logs are written to journald (not docker json log files)
+- command-line wrapper scripts are installed to the host system to call vitastor-disk,
+  vitastor-cli and others through the container
+
+Such installations may be useful when it's impossible or inconvenient to install
+Vitastor from packages, for example, in exotic Linux distributions.
+
+If you don't want just a simple containerized installation, you can also take a look
+at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operator
+
+## Installing Containers
+
+The instruction is very simple.
+
+1. Download a Docker image of the desired version: \
+   `docker pull vitastor:1.10.2`
+2. Install scripts to the host system: \
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:1.10.2 install.sh`
+3. Reload udev rules: \
+   `udevadm control --reload-rules`
+
+And you can return to [Quick Start](../intro/quickstart.en.md).
+
+## Upgrading Containers
+
+First make sure to check the topic [Upgrading Vitastor](../usage/admin.en.md#upgrading-vitastor)
+to figure out if you need any additional steps.
+
+Then, to upgrade a containerized installation, you just need to change the `VITASTOR_VERSION`
+option in `/etc/vitastor/docker.conf` and restart all Vitastor services:
+
+`systemctl restart vitastor.target`
+
+## QEMU
+
+Vitastor Docker image also contains QEMU, qemu-img and qemu-storage-daemon built with Vitastor support.
+
+However, running QEMU in Docker is harder to setup and it depends on the used virtualization UI
+(OpenNebula, Proxmox and so on). Some of them also required patched Libvirt.
+
+That's why containerized installation of Vitastor doesn't contain a ready-made QEMU setup and it's
+recommended to install QEMU from packages or build it manually.
+
+## fio
+
+Vitastor Docker image also contains fio and installs a wrapper called `vitastor-fio` to use it from
+the host system.
@@ -0,0 +1,60 @@
+[Документация](../../README-ru.md#документация) → Установка → Установка в Docker
+
+-----
+
+[Read in English](docker.en.md)
+
+# Установка в Docker
+
+Vitastor можно установить в Docker/Podman. При этом etcd, мониторы и OSD запускаются
+в контейнерах, но всё остальное выглядит максимально приближенно к установке из пакетов:
+- используется сеть хост-системы
+- для автозапуска используются udev и systemd
+- журналы записываются в journald (не в json-файлы журналов docker)
+- в хост-систему устанавливаются обёртки для вызова консольных инструментов vitastor-disk,
+  vitastor-cli и других через контейнер
+
+Такая установка полезна тогда, когда установка из пакетов невозможна или неудобна,
+например, в нестандартных Linux-дистрибутивах.
+
+Если вам нужна не просто контейнеризованная инсталляция, вы также можете обратить внимание
+на Vitastor Kubernetes-оператор: https://github.com/Antilles7227/vitastor-operator
+
+## Установка контейнеров
+
+Инструкция по установке максимально простая.
+
+1. Скачайте Docker-образ желаемой версии: \
+   `docker pull vitastor:1.10.2`
+2. Установите скрипты в хост-систему командой: \
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:1.10.2 install.sh`
+3. Перезагрузите правила udev: \
+   `udevadm control --reload-rules`
+
+После этого вы можете возвращаться к разделу [Быстрый старт](../intro/quickstart.ru.md).
+
+## Обновление контейнеров
+
+Сначала обязательно проверьте раздел [Обновление Vitastor](../usage/admin.ru.md#обновление-vitastor),
+чтобы понять, не требуются ли вам какие-то дополнительные действия.
+
+После этого для обновления Docker-инсталляции вам нужно просто поменять опцию `VITASTOR_VERSION`
+в файле `/etc/vitastor/docker.conf` и перезапустить все сервисы Vitastor командой:
+
+`systemctl restart vitastor.target`
+
+## QEMU
+
+В Docker-образ также входят QEMU, qemu-img и qemu-storage-daemon, собранные с поддержкой Vitastor.
+
+Однако настроить запуск QEMU в Docker сложнее и способ запуска зависит от используемого интерфейса
+виртуализации (OpenNebula, Proxmox и т.п.). Также для OpenNebula, например, требуется патченый
+Libvirt.
+
+Поэтому по умолчанию Docker-сборка пока что не включает в себя готового способа запуска QEMU
+и QEMU рекомендуется устанавливать из пакетов или собирать самостоятельно.
+
+## fio
+
+fio также входит в Docker-контейнер vitastor, и в хост-систему устанавливается обёртка `vitastor-fio`
+для запуска fio в контейнер.
@@ -14,6 +14,7 @@
  - Debian 12 (Bookworm/Sid): `deb https://vitastor.io/debian bookworm main`
  - Debian 11 (Bullseye): `deb https://vitastor.io/debian bullseye main`
  - Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
+  - Ubuntu 22.04 (Jammy): `deb https://vitastor.io/debian jammy main`
  - Add `-oldstable` to bookworm/bullseye/buster in this line to install the last
    stable version from 0.9.x branch instead of 1.x
 - Install packages: `apt update; apt install vitastor lp-solve etcd linux-image-amd64 qemu-system-x86`
@@ -14,6 +14,7 @@
  - Debian 12 (Bookworm/Sid): `deb https://vitastor.io/debian bookworm main`
  - Debian 11 (Bullseye): `deb https://vitastor.io/debian bullseye main`
  - Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
+  - Ubuntu 22.04 (Jammy): `deb https://vitastor.io/debian jammy main`
  - Добавьте `-oldstable` к слову bookworm/bullseye/buster в этой строке, чтобы
    установить последнюю стабильную версию из ветки 0.9.x вместо 1.x
 - Установите пакеты: `apt update; apt install vitastor lp-solve etcd linux-image-amd64 qemu-system-x86`
@@ -26,13 +26,13 @@
  you also need small SSDs for journal and metadata (even 2 GB per 1 TB of HDD space is enough).
 - Get a fast network (at least 10 Gbit/s). Something like Mellanox ConnectX-4 with RoCEv2 is ideal.
 - Disable CPU powersaving: `cpupower idle-set -D 0 && cpupower frequency-set -g performance`.
- [Install Vitastor packages](../installation/packages.en.md).
+- Either [install Vitastor packages](../installation/packages.en.md) or [install Vitastor in Docker](../installation/docker.en.md).

 ## Recommended drives

 - SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel D3-S4510/4520/4610/4620, Kingston DC500M
 - NVMe: Micron 9100/9200/9300/9400, Micron 7300/7450, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
-  Intel DC-P3700/P4500/P4600, Intel D5-P4320, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
+  Intel DC-P3700/P4500/P4600, Intel D5-P4320/P5530, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
 - HDD: HGST Ultrastar, Toshiba MG, Seagate EXOS

 ## Configure monitors
@@ -45,7 +45,8 @@ On the monitor hosts:
  }
  ```
 - Create systemd units for etcd by running: `/usr/lib/vitastor/mon/make-etcd`
- Start etcd and monitors: `systemctl enable --now etcd vitastor-mon`
+  Or, if you installed Vitastor in Docker, run `systemctl start vitastor-host; docker exec vitastor make-etcd`.
+- Start etcd and monitors: `systemctl enable --now vitastor-etcd vitastor-mon`

 ## Configure OSDs

@@ -26,13 +26,13 @@
  обязательно возьмите SSD под метаданные и журнал (маленькие, буквально 2 ГБ на 1 ТБ HDD-места).
 - Возьмите быструю сеть, минимум 10 гбит/с. Идеал - что-то вроде Mellanox ConnectX-4 с RoCEv2.
 - Для лучшей производительности отключите энергосбережение CPU: `cpupower idle-set -D 0 && cpupower frequency-set -g performance`.
- [Установите пакеты Vitastor](../installation/packages.ru.md).
+- Либо [установите пакеты Vitastor](../installation/packages.ru.md), либо [установите Vitastor в Docker](../installation/docker.ru.md).

 ## Рекомендуемые диски

 - SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel D3-S4510/4520/4610/4620, Kingston DC500M
 - NVMe: Micron 9100/9200/9300/9400, Micron 7300/7450, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
-  Intel DC-P3700/P4500/P4600, Intel D5-P4320, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
+  Intel DC-P3700/P4500/P4600, Intel D5-P4320/P5530, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
 - HDD: HGST Ultrastar, Toshiba MG, Seagate EXOS

 ## Настройте мониторы
@@ -44,8 +44,9 @@
    "etcd_address": ["10.200.1.10:2379","10.200.1.11:2379","10.200.1.12:2379"]
  }
  ```
- Инициализируйте сервисы etcd, запустив `/usr/lib/vitastor/mon/make-etcd`
- Запустите etcd и мониторы: `systemctl enable --now etcd vitastor-mon`
+- Инициализируйте сервисы etcd, запустив `/usr/lib/vitastor/mon/make-etcd`.\
+  Либо, если вы установили Vitastor в Docker, запустите `systemctl start vitastor-host; docker exec vitastor make-etcd`.
+- Запустите etcd и мониторы: `systemctl enable --now vitastor-etcd vitastor-mon`

 ## Настройте OSD

@@ -37,7 +37,7 @@ It supports the following commands:
 Global options:

 ```
--config_file FILE   Path to Vitastor configuration file
+--config_path FILE   Path to Vitastor configuration file
 --etcd_address URL   Etcd connection address
 --iodepth N          Send N operations in parallel to each OSD when possible (default 32)
 --parallel_osds M    Work with M osds in parallel when possible (default 4)
@@ -398,7 +398,8 @@ Optional parameters:
 | `--raw_placement <rules>`      | Specify raw PG generation rules ([details](../config/pool.en.md#raw_placement)) |
 | `--primary_affinity_tags tags` | Prefer to put primary copies on OSDs with all specified tags               |
 | `--scrub_interval <time>`      | Enable regular scrubbing for this pool. Format: number + unit s/m/h/d/M/y  |
-| `--used_for_fs <name>`         | Mark pool as used for VitastorFS with metadata in image <name>             |
+| `--used_for_app fs:<name>`     | Mark pool as used for VitastorFS with metadata in image `<name>`           |
+| `--used_for_app s3:<name>`     | Mark pool as used for S3 location with name `<name>`                       |
 | `--pg_stripe_size <number>`    | Increase object grouping stripe                                            |
 | `--max_osd_combinations 10000` | Maximum number of random combinations for LP solver input                  |
 | `--wait`                       | Wait for the new pool to come online                                       |
@@ -36,7 +36,7 @@ vitastor-cli - интерфейс командной строки для адм
 Глобальные опции:

 ```
--config_file FILE   Путь к файлу конфигурации Vitastor
+--config_path FILE   Путь к файлу конфигурации Vitastor
 --etcd_address URL   Адрес соединения с etcd
 --iodepth N          Отправлять параллельно N операций на каждый OSD (по умолчанию 32)
 --parallel_osds M    Работать параллельно с M OSD (по умолчанию 4)
@@ -36,7 +36,7 @@ It will output a block device name like /dev/nbd0 which you can then use as a no

 You can also use `--pool <POOL> --inode <INODE> --size <SIZE>` instead of `--image <IMAGE>` if you want.

-vitastor-nbd supports all usual Vitastor configuration options like `--config_file <path_to_config>` plus NBD-specific:
+vitastor-nbd supports all usual Vitastor configuration options like `--config_path <path_to_config>` plus NBD-specific:

 * `--nbd_timeout 0` \
  Timeout for I/O operations in seconds after exceeding which the kernel stops the device.
@@ -54,16 +54,18 @@ vitastor-nbd supports all usual Vitastor configuration options like `--config_fi
  Stay in foreground, do not daemonize.

 Note that `nbd_timeout`, `nbd_max_devices` and `nbd_max_part` options may also be specified
-in `/etc/vitastor/vitastor.conf` or in other configuration file specified with `--config_file`.
+in `/etc/vitastor/vitastor.conf` or in other configuration file specified with `--config_path`.

 ## unmap

 To unmap the device run:

 ```
-vitastor-nbd unmap /dev/nbd0
+vitastor-nbd unmap [--force] /dev/nbd0
 ```

+If `--force` is specified, `vitastor-nbd` doesn't check if the device is actually mapped.
+
 ## ls

 ```
@@ -41,7 +41,7 @@ vitastor-nbd map [/dev/nbdN] --image testimg
 Для обращения по номеру инода, аналогично другим командам, можно использовать опции
 `--pool <POOL> --inode <INODE> --size <SIZE>` вместо `--image testimg`.

-vitastor-nbd поддерживает все обычные опции Vitastor, например, `--config_file <path_to_config>`,
+vitastor-nbd поддерживает все обычные опции Vitastor, например, `--config_path <path_to_config>`,
 плюс специфичные для NBD:

 * `--nbd_timeout 0` \
@@ -62,16 +62,19 @@ vitastor-nbd поддерживает все обычные опции Vitastor,

 Обратите внимание, что опции `nbd_timeout`, `nbd_max_devices` и `nbd_max_part` можно
 также задавать в `/etc/vitastor/vitastor.conf` или в другом файле конфигурации,
-заданном опцией `--config_file`.
+заданном опцией `--config_path`.

 ## unmap

 Для отключения устройства выполните:

 ```
-vitastor-nbd unmap /dev/nbd0
+vitastor-nbd unmap [--force] /dev/nbd0
 ```

+Если задана опция `--force`, `vitastor-nbd` не проверяет, подключено ли устройство,
+перед попыткой его отключить.
+
 ## ls

 ```
@@ -58,7 +58,7 @@ To use VitastorFS:
 2. Create an image for FS metadata, preferably in a faster (SSD or replica-HDD) pool,
   but you can create it in the data pool too if you want (image size doesn't matter):
   `vitastor-cli create -s 10G -p fastpool testfs`
-3. Mark data pool as an FS pool: `vitastor-cli modify-pool --used-for-fs testfs data-pool`
+3. Mark data pool as an FS pool: `vitastor-cli modify-pool --used-for-app fs:testfs data-pool`
 4. Either mount the FS: `vitastor-nfs mount --fs testfs --pool data-pool /mnt/vita`
 5. Or start the NFS server: `vitastor-nfs start --fs testfs --pool data-pool`

@@ -60,7 +60,7 @@ JSON-формате :-). Для инспекции содержимого БД
   или по крайней мере на HDD, но без EC), но можно и в том же пуле, что данные
   (размер образа значения не имеет):
   `vitastor-cli create -s 10G -p fastpool testfs`
-3. Пометьте пул данных как ФС-пул: `vitastor-cli modify-pool --used-for-fs testfs data-pool`
+3. Пометьте пул данных как ФС-пул: `vitastor-cli modify-pool --used-for-app fs:testfs data-pool`
 4. Либо примонтируйте ФС: `vitastor-nfs mount --fs testfs --pool data-pool /mnt/vita`
 5. Либо запустите сетевой NFS-сервер: `vitastor-nfs start --fs testfs --pool data-pool`

@@ -773,23 +773,27 @@ class Mon
                }
            }
        }
-        for (const pool_id in this.state.pool.stats)
+        if (!this.recheck_pgs_active)
        {
-            if (!seen_pools[pool_id])
+            // PG recheck also modifies /pool/stats, so don't touch it here if it's active
+            for (const pool_id in this.state.pool.stats)
            {
-                txn.push({ requestDeleteRange: {
-                    key: b64(this.config.etcd_prefix+'/pool/stats/'+pool_id),
-                } });
-                delete this.state.pool.stats[pool_id];
-            }
-            else
-            {
-                const pool_stats = { ...this.state.pool.stats[pool_id] };
-                serialize_bigints(pool_stats);
-                txn.push({ requestPut: {
-                    key: b64(this.config.etcd_prefix+'/pool/stats/'+pool_id),
-                    value: b64(JSON.stringify(pool_stats)),
-                } });
+                if (!seen_pools[pool_id])
+                {
+                    txn.push({ requestDeleteRange: {
+                        key: b64(this.config.etcd_prefix+'/pool/stats/'+pool_id),
+                    } });
+                    delete this.state.pool.stats[pool_id];
+                }
+                else
+                {
+                    const pool_stats = { ...this.state.pool.stats[pool_id] };
+                    serialize_bigints(pool_stats);
+                    txn.push({ requestPut: {
+                        key: b64(this.config.etcd_prefix+'/pool/stats/'+pool_id),
+                        value: b64(JSON.stringify(pool_stats)),
+                    } });
+                }
            }
        }
        if (txn.length)
@@ -1,6 +1,6 @@
 {
  "name": "vitastor-mon",
-  "version": "1.10.1",
+  "version": "1.11.0",
  "description": "Vitastor SDS monitor service",
  "main": "mon-main.js",
  "scripts": {
@@ -9,7 +9,7 @@
  "author": "Vitaliy Filippov",
  "license": "UNLICENSED",
  "dependencies": {
-    "antietcd": "^1.1.0",
+    "antietcd": "^1.1.2",
    "sprintf-js": "^1.1.2",
    "ws": "^7.2.5"
  },
@@ -8,23 +8,9 @@ const LPOptimizer = require('./lp_optimizer/lp_optimizer.js');
 const { scale_pg_count } = require('./pg_utils.js');
 const { make_hier_tree, filter_osds_by_root_node,
    filter_osds_by_tags, filter_osds_by_block_layout, get_affinity_osds } = require('./osd_tree.js');
+const { select_murmur3 } = require('./lp_optimizer/murmur3.js');

-let seed;
-
-function reset_rng()
-{
-    seed = 0x5f020e43;
-}
-
-function rng()
-{
-    seed ^= seed << 13;
-    seed ^= seed >> 17;
-    seed ^= seed << 5;
-    return seed + 2147483648;
-}
-
-function pick_primary(pool_config, osd_set, up_osds, aff_osds)
+function pick_primary(pool_id, pg_num, pool_config, osd_set, up_osds, aff_osds)
 {
    let alive_set;
    if (pool_config.scheme === 'replicated')
@@ -52,7 +38,7 @@ function pick_primary(pool_config, osd_set, up_osds, aff_osds)
    {
        return 0;
    }
-    return alive_set[rng() % alive_set.length];
+    return alive_set[select_murmur3(alive_set.length, osd_num => pool_id+'/'+pg_num+'/'+osd_num)];
 }

 function recheck_primary(state, global_config, up_osds, osd_tree)
@@ -66,7 +52,6 @@ function recheck_primary(state, global_config, up_osds, osd_tree)
            continue;
        }
        const aff_osds = get_affinity_osds(pool_cfg, up_osds, osd_tree);
-        reset_rng();
        for (let pg_num = 1; pg_num <= pool_cfg.pg_count; pg_num++)
        {
            if (!state.pg.config.items[pool_id])
@@ -76,7 +61,7 @@ function recheck_primary(state, global_config, up_osds, osd_tree)
            const pg_cfg = state.pg.config.items[pool_id][pg_num];
            if (pg_cfg)
            {
-                const new_primary = pick_primary(state.config.pools[pool_id], pg_cfg.osd_set, up_osds, aff_osds);
+                const new_primary = pick_primary(pool_id, pg_num, state.config.pools[pool_id], pg_cfg.osd_set, up_osds, aff_osds);
                if (pg_cfg.primary != new_primary)
                {
                    if (!new_pg_config)
@@ -99,13 +84,12 @@ function save_new_pgs_txn(save_to, request, state, etcd_prefix, etcd_watch_revis
 {
    const aff_osds = get_affinity_osds(state.config.pools[pool_id] || {}, up_osds, osd_tree);
    const pg_items = {};
-    reset_rng();
    new_pgs.map((osd_set, i) =>
    {
        osd_set = osd_set.map(osd_num => osd_num === LPOptimizer.NO_OSD ? 0 : osd_num);
        pg_items[i+1] = {
            osd_set,
-            primary: pick_primary(state.config.pools[pool_id], osd_set, up_osds, aff_osds),
+            primary: pick_primary(pool_id, i+1, state.config.pools[pool_id], osd_set, up_osds, aff_osds),
        };
        if (prev_pgs[i] && prev_pgs[i].join(' ') != osd_set.join(' ') &&
            prev_pgs[i].filter(osd_num => osd_num).length > 0)
@@ -33,9 +33,11 @@ async function run()
        console.log(config_path+' is missing');
        process.exit(1);
    }
-    if (fs.existsSync("/etc/systemd/system/etcd.service"))
+    const in_docker = fs.existsSync("/etc/vitastor/etcd.conf") &&
+        fs.existsSync("/etc/vitastor/docker.conf");
+    if (!in_docker && fs.existsSync("/etc/systemd/system/vitastor-etcd.service"))
    {
-        console.log("/etc/systemd/system/etcd.service already exists");
+        console.log("/etc/systemd/system/vitastor-etcd.service already exists");
        process.exit(1);
    }
    const config = JSON.parse(fs.readFileSync(config_path, { encoding: 'utf-8' }));
@@ -52,10 +54,21 @@ async function run()
        console.log('No matching IPs in etcd_address from '+config_path);
        process.exit(0);
    }
-    const etcd_cluster = etcds.map((e, i) => `etcd${i}=http://${e}:2380`).join(',');
-    await system(`mkdir -p /var/lib/etcd${num}.etcd`);
+    const etcd_name = 'etcd'+etcds[num].replace(/[^0-9a-z_]/ig, '_');
+    const etcd_cluster = etcds.map(e => `etcd${e.replace(/[^0-9a-z_]/ig, '_')}=http://${e}:2380`).join(',');
+    if (in_docker)
+    {
+        let etcd_conf = fs.readFileSync("/etc/vitastor/etcd.conf", { encoding: 'utf-8' });
+        etcd_conf = replace_env(etcd_conf, 'ETCD_NAME', etcd_name);
+        etcd_conf = replace_env(etcd_conf, 'ETCD_IP', etcds[num]);
+        etcd_conf = replace_env(etcd_conf, 'ETCD_INITIAL_CLUSTER', etcd_cluster);
+        fs.writeFileSync("/etc/vitastor/etcd.conf", etcd_conf);
+        console.log('etcd for Vitastor configured. Run `systemctl enable --now vitastor-etcd` to start etcd');
+        process.exit(0);
+    }
+    await system(`mkdir -p /var/lib/etcd`);
    fs.writeFileSync(
-        "/etc/systemd/system/etcd.service",
+        "/etc/systemd/system/vitastor-etcd.service",
 `[Unit]
 Description=etcd for vitastor
 After=network-online.target local-fs.target time-sync.target
@@ -64,14 +77,14 @@ Wants=network-online.target local-fs.target time-sync.target
 [Service]
 Restart=always
 Environment=GOGC=50
-ExecStart=etcd -name etcd${num} --data-dir /var/lib/etcd${num}.etcd \\
+ExecStart=etcd --name ${etcd_name} --data-dir /var/lib/etcd \\
    --snapshot-count 10000 --advertise-client-urls http://${etcds[num]}:2379 --listen-client-urls http://${etcds[num]}:2379 \\
    --initial-advertise-peer-urls http://${etcds[num]}:2380 --listen-peer-urls http://${etcds[num]}:2380 \\
    --initial-cluster-token vitastor-etcd-1 --initial-cluster ${etcd_cluster} \\
    --initial-cluster-state new --max-txn-ops=100000 --max-request-bytes=104857600 \\
    --auto-compaction-retention=10 --auto-compaction-mode=revision
-WorkingDirectory=/var/lib/etcd${num}.etcd
-ExecStartPre=+chown -R etcd /var/lib/etcd${num}.etcd
+WorkingDirectory=/var/lib/etcd
+ExecStartPre=+chown -R etcd /var/lib/etcd
 User=etcd
 PrivateTmp=false
 TasksMax=infinity
@@ -89,6 +102,13 @@ WantedBy=multi-user.target
    process.exit(0);
 }

+function replace_env(text, key, value)
+{
+    let found = false;
+    text = text.replace(new RegExp('^'+key+'\\s*=.*', 'm'), () => { found = true; return key+'='+value; });
+    return found ? text : text.replace(/\s*$/, '\n')+key+'='+value+'\n';
+}
+
 function select_local_etcd(etcds)
 {
    const ifaces = os.networkInterfaces();
@@ -5,6 +5,7 @@ Wants=network-online.target local-fs.target time-sync.target

 [Service]
 Restart=always
+SyslogIdentifier=vitastor-mon
 ExecStart=node /usr/lib/vitastor/mon/mon-main.js
 WorkingDirectory=/
 User=vitastor
@@ -8,6 +8,7 @@ PartOf=vitastor.target
 LimitNOFILE=1048576
 LimitNPROC=1048576
 LimitMEMLOCK=infinity
+SyslogIdentifier=vitastor-osd%i
 # Use the following for direct logs to files
 #ExecStart=bash -c 'exec vitastor-disk exec-osd /dev/vitastor/osd%i-data >>/var/log/vitastor/osd%i.log 2>&1'
 ExecStart=vitastor-disk exec-osd /dev/vitastor/osd%i-data
@@ -14,8 +14,13 @@ NAN_MODULE_INIT(InitAddon)

    Nan::SetPrototypeMethod(tpl, "read", NodeVitastor::Read);
    Nan::SetPrototypeMethod(tpl, "write", NodeVitastor::Write);
+    Nan::SetPrototypeMethod(tpl, "delete", NodeVitastor::Delete);
    Nan::SetPrototypeMethod(tpl, "sync", NodeVitastor::Sync);
    Nan::SetPrototypeMethod(tpl, "read_bitmap", NodeVitastor::ReadBitmap);
+    Nan::SetPrototypeMethod(tpl, "on_ready", NodeVitastor::OnReady);
+    Nan::SetPrototypeMethod(tpl, "get_min_io_size", NodeVitastor::GetMinIoSize);
+    Nan::SetPrototypeMethod(tpl, "get_max_atomic_write_size", NodeVitastor::GetMaxAtomicWriteSize);
+    Nan::SetPrototypeMethod(tpl, "get_immediate_commit", NodeVitastor::GetImmediateCommit);
    //Nan::SetPrototypeMethod(tpl, "destroy", NodeVitastor::Destroy);

    Nan::Set(target, Nan::New("Client").ToLocalChecked(), Nan::GetFunction(tpl).ToLocalChecked());
@@ -63,6 +68,10 @@ NAN_MODULE_INIT(InitAddon)
    Nan::Set(target, Nan::New("ENOSYS").ToLocalChecked(), Nan::New<v8::Int32>(-ENOSYS));
    Nan::Set(target, Nan::New("EAGAIN").ToLocalChecked(), Nan::New<v8::Int32>(-EAGAIN));

+    Nan::Set(target, Nan::New("IMMEDIATE_NONE").ToLocalChecked(), Nan::New<v8::Int32>(IMMEDIATE_NONE));
+    Nan::Set(target, Nan::New("IMMEDIATE_SMALL").ToLocalChecked(), Nan::New<v8::Int32>(IMMEDIATE_SMALL));
+    Nan::Set(target, Nan::New("IMMEDIATE_ALL").ToLocalChecked(), Nan::New<v8::Int32>(IMMEDIATE_ALL));
+
    // Listing handle

    tpl = Nan::New<v8::FunctionTemplate>(NodeVitastorKVListing::Create);
@@ -5,9 +5,10 @@

 #define NODE_VITASTOR_READ 1
 #define NODE_VITASTOR_WRITE 2
-#define NODE_VITASTOR_SYNC 3
-#define NODE_VITASTOR_READ_BITMAP 4
-#define NODE_VITASTOR_GET_INFO 5
+#define NODE_VITASTOR_DELETE 3
+#define NODE_VITASTOR_SYNC 4
+#define NODE_VITASTOR_READ_BITMAP 5
+#define NODE_VITASTOR_GET_INFO 6

 #ifndef INODE_POOL
 #define INODE_POOL(inode) (uint32_t)((inode) >> (64 - POOL_ID_BITS))
@@ -80,6 +81,11 @@ NAN_METHOD(NodeVitastor::Create)
    NodeVitastor* cli = new NodeVitastor();
    cli->c = vitastor_c_create_uring_json(c_cfg, cfg.size());
    delete[] c_cfg;
+    if (!cli->c)
+    {
+        Nan::ThrowError("failed to initialize io_uring (old kernel or insufficient ulimit -l?)");
+        return;
+    }

    int res = vitastor_c_uring_register_eventfd(cli->c);
    if (res >= 0)
@@ -133,12 +139,12 @@ NodeVitastorRequest* NodeVitastor::get_read_request(const Nan::FunctionCallbackI
    return req;
 }

-// read(pool, inode, offset, len, callback(err, buffer, version))
+// read(pool, inode, offset, length, callback(err, buffer, version))
 NAN_METHOD(NodeVitastor::Read)
 {
    TRACE("NodeVitastor::Read");
    if (info.Length() < 5)
-        Nan::ThrowError("Not enough arguments to read(pool, inode, offset, len, callback(err, buffer, version))");
+        Nan::ThrowError("Not enough arguments to read(pool, inode, offset, length, callback(err, buffer, version))");

    NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());

@@ -149,6 +155,9 @@ NAN_METHOD(NodeVitastor::Read)

    self->Ref();
    vitastor_c_read(self->c, ((pool << (64-POOL_ID_BITS)) | inode), req->offset, req->len, &req->iov, 1, on_read_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(self->c);
+#endif
 }

 NodeVitastorRequest* NodeVitastor::get_write_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos)
@@ -217,6 +226,58 @@ NAN_METHOD(NodeVitastor::Write)
        req->iov_list.size() ? req->iov_list.data() : &req->iov,
        req->iov_list.size() ? req->iov_list.size() : 1,
        on_write_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(self->c);
+#endif
+}
+
+NodeVitastorRequest* NodeVitastor::get_delete_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos)
+{
+    uint64_t offset = get_ui64(info[argpos+0]);
+    uint64_t len = get_ui64(info[argpos+1]);
+    uint64_t version = 0;
+    if (!info[argpos+2].IsEmpty() &&
+        !info[argpos+2]->IsFunction() &&
+        info[argpos+2]->IsObject())
+    {
+        auto key = Nan::New<v8::String>("version").ToLocalChecked();
+        auto params = info[argpos+2].As<v8::Object>();
+        auto versionObj = Nan::Get(params, key).ToLocalChecked();
+        if (!versionObj.IsEmpty())
+            version = get_ui64(versionObj);
+        argpos++;
+    }
+
+    v8::Local<v8::Function> callback = info[argpos+2].As<v8::Function>();
+    auto req = new NodeVitastorRequest(this, callback);
+
+    req->offset = offset;
+    req->len = len;
+    req->version = version;
+
+    return req;
+}
+
+// delete(pool, inode, offset, length, { version }?, callback(err))
+NAN_METHOD(NodeVitastor::Delete)
+{
+    TRACE("NodeVitastor::Delete");
+    if (info.Length() < 5)
+        Nan::ThrowError("Not enough arguments to delete(pool, inode, offset, length, { version }?, callback(err))");
+
+    NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
+
+    uint64_t pool = get_ui64(info[0]);
+    uint64_t inode = get_ui64(info[1]);
+
+    auto req = self->get_delete_request(info, 2);
+
+    self->Ref();
+    vitastor_c_delete(self->c, ((pool << (64-POOL_ID_BITS)) | inode), req->offset, req->len, req->version,
+        on_write_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(self->c);
+#endif
 }

 // sync(callback(err))
@@ -233,14 +294,17 @@ NAN_METHOD(NodeVitastor::Sync)

    self->Ref();
    vitastor_c_sync(self->c, on_write_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(self->c);
+#endif
 }

-// read_bitmap(pool, inode, offset, len, with_parents, callback(err, bitmap_buffer))
+// read_bitmap(pool, inode, offset, length, with_parents, callback(err, bitmap_buffer))
 NAN_METHOD(NodeVitastor::ReadBitmap)
 {
    TRACE("NodeVitastor::ReadBitmap");
    if (info.Length() < 6)
-        Nan::ThrowError("Not enough arguments to read_bitmap(pool, inode, offset, len, with_parents, callback(err, bitmap_buffer))");
+        Nan::ThrowError("Not enough arguments to read_bitmap(pool, inode, offset, length, with_parents, callback(err, bitmap_buffer))");

    NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());

@@ -254,6 +318,9 @@ NAN_METHOD(NodeVitastor::ReadBitmap)
    auto req = new NodeVitastorRequest(self, callback);
    self->Ref();
    vitastor_c_read_bitmap(self->c, ((pool << (64-POOL_ID_BITS)) | inode), offset, len, with_parents, on_read_bitmap_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(self->c);
+#endif
 }

 static void on_error(NodeVitastorRequest *req, Nan::Callback & nanCallback, long retval)
@@ -267,6 +334,67 @@ static void on_error(NodeVitastorRequest *req, Nan::Callback & nanCallback, long
    nanCallback.Call(1, args, req);
 }

+// on_ready(callback(err))
+NAN_METHOD(NodeVitastor::OnReady)
+{
+    TRACE("NodeVitastor::OnReady");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to on_ready(callback(err))");
+    NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
+    v8::Local<v8::Function> callback = info[0].As<v8::Function>();
+    auto req = new NodeVitastorRequest(self, callback);
+    self->Ref();
+    vitastor_c_on_ready(self->c, on_ready_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(self->c);
+#endif
+}
+
+void NodeVitastor::on_ready_finish(void *opaque, long retval)
+{
+    TRACE("NodeVitastor::on_ready_finish");
+    auto req = (NodeVitastorRequest*)opaque;
+    auto self = req->cli;
+    Nan::HandleScope scope;
+    Nan::Callback nanCallback(Nan::New(req->callback));
+    nanCallback.Call(0, NULL, req);
+    self->Unref();
+    delete req;
+}
+
+// get_min_io_size(pool_id)
+NAN_METHOD(NodeVitastor::GetMinIoSize)
+{
+    TRACE("NodeVitastor::GetMinIoSize");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to get_min_io_size(pool_id)");
+    NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
+    uint64_t pool = get_ui64(info[0]);
+    info.GetReturnValue().Set(Nan::New<v8::Number>(vitastor_c_inode_get_bitmap_granularity(self->c, INODE_WITH_POOL(pool, 1))));
+}
+
+// get_max_atomic_write_size(pool_id)
+NAN_METHOD(NodeVitastor::GetMaxAtomicWriteSize)
+{
+    TRACE("NodeVitastor::GetMaxAtomicWriteSize");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to get_max_atomic_write_size(pool_id)");
+    NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
+    uint64_t pool = get_ui64(info[0]);
+    info.GetReturnValue().Set(Nan::New<v8::Number>(vitastor_c_inode_get_block_size(self->c, INODE_WITH_POOL(pool, 1))));
+}
+
+// get_immediate_commit(pool_id)
+NAN_METHOD(NodeVitastor::GetImmediateCommit)
+{
+    TRACE("NodeVitastor::GetImmediateCommit");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to get_immediate_commit(pool_id)");
+    NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
+    uint64_t pool = get_ui64(info[0]);
+    info.GetReturnValue().Set(Nan::New<v8::Number>(vitastor_c_inode_get_immediate_commit(self->c, INODE_WITH_POOL(pool, 1))));
+}
+
 void NodeVitastor::on_read_finish(void *opaque, long retval, uint64_t version)
 {
    TRACE("NodeVitastor::on_read_finish");
@@ -364,6 +492,9 @@ NAN_METHOD(NodeVitastorImage::Create)
    img->Ref();
    cli->Ref();
    vitastor_c_watch_inode(cli->c, (char*)img->name.c_str(), on_watch_start, img);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(cli->c);
+#endif

    info.GetReturnValue().Set(info.This());
 }
@@ -378,12 +509,12 @@ NodeVitastorImage::~NodeVitastorImage()
    cli->Unref();
 }

-// read(offset, len, callback(err, buffer, version))
+// read(offset, length, callback(err, buffer, version))
 NAN_METHOD(NodeVitastorImage::Read)
 {
    TRACE("NodeVitastorImage::Read");
    if (info.Length() < 3)
-        Nan::ThrowError("Not enough arguments to read(offset, len, callback(err, buffer, version))");
+        Nan::ThrowError("Not enough arguments to read(offset, length, callback(err, buffer, version))");

    NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());

@@ -394,12 +525,12 @@ NAN_METHOD(NodeVitastorImage::Read)
    img->exec_or_wait(req);
 }

-// write(offset, buffer, { version }?, callback(err))
+// write(offset, buf: Buffer | Buffer[], { version }?, callback(err))
 NAN_METHOD(NodeVitastorImage::Write)
 {
    TRACE("NodeVitastorImage::Write");
    if (info.Length() < 3)
-        Nan::ThrowError("Not enough arguments to write(offset, buffer, { version }?, callback(err))");
+        Nan::ThrowError("Not enough arguments to write(offset, buf: Buffer | Buffer[], { version }?, callback(err))");

    NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());

@@ -410,6 +541,22 @@ NAN_METHOD(NodeVitastorImage::Write)
    img->exec_or_wait(req);
 }

+// delete(offset, length, { version }?, callback(err))
+NAN_METHOD(NodeVitastorImage::Delete)
+{
+    TRACE("NodeVitastorImage::Delete");
+    if (info.Length() < 3)
+        Nan::ThrowError("Not enough arguments to delete(offset, length, { version }?, callback(err))");
+
+    NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
+
+    auto req = img->cli->get_delete_request(info, 0);
+    req->img = img;
+    req->op = NODE_VITASTOR_DELETE;
+
+    img->exec_or_wait(req);
+}
+
 // sync(callback(err))
 NAN_METHOD(NodeVitastorImage::Sync)
 {
@@ -427,12 +574,12 @@ NAN_METHOD(NodeVitastorImage::Sync)
    img->exec_or_wait(req);
 }

-// read_bitmap(offset, len, with_parents, callback(err, bitmap_buffer))
+// read_bitmap(offset, length, with_parents, callback(err, bitmap_buffer))
 NAN_METHOD(NodeVitastorImage::ReadBitmap)
 {
    TRACE("NodeVitastorImage::ReadBitmap");
    if (info.Length() < 4)
-        Nan::ThrowError("Not enough arguments to read_bitmap(offset, len, with_parents, callback(err, bitmap_buffer))");
+        Nan::ThrowError("Not enough arguments to read_bitmap(offset, length, with_parents, callback(err, bitmap_buffer))");

    NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());

@@ -488,6 +635,9 @@ void NodeVitastorImage::exec_request(NodeVitastorRequest *req)
        uint64_t ino = vitastor_c_inode_get_num(watch);
        cli->Ref();
        vitastor_c_read(cli->c, ino, req->offset, req->len, &req->iov, 1, NodeVitastor::on_read_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+        vitastor_c_uring_handle_events(cli->c);
+#endif
    }
    else if (req->op == NODE_VITASTOR_WRITE)
    {
@@ -497,6 +647,19 @@ void NodeVitastorImage::exec_request(NodeVitastorRequest *req)
            req->iov_list.size() ? req->iov_list.data() : &req->iov,
            req->iov_list.size() ? req->iov_list.size() : 1,
            NodeVitastor::on_write_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+        vitastor_c_uring_handle_events(cli->c);
+#endif
+    }
+    else if (req->op == NODE_VITASTOR_DELETE)
+    {
+        uint64_t ino = vitastor_c_inode_get_num(watch);
+        cli->Ref();
+        vitastor_c_delete(cli->c, ino, req->offset, req->len, req->version,
+            NodeVitastor::on_write_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+        vitastor_c_uring_handle_events(cli->c);
+#endif
    }
    else if (req->op == NODE_VITASTOR_SYNC)
    {
@@ -506,6 +669,9 @@ void NodeVitastorImage::exec_request(NodeVitastorRequest *req)
        if (imm != IMMEDIATE_ALL)
        {
            vitastor_c_sync(cli->c, NodeVitastor::on_write_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+            vitastor_c_uring_handle_events(cli->c);
+#endif
        }
        else
        {
@@ -517,6 +683,9 @@ void NodeVitastorImage::exec_request(NodeVitastorRequest *req)
        uint64_t ino = vitastor_c_inode_get_num(watch);
        cli->Ref();
        vitastor_c_read_bitmap(cli->c, ino, req->offset, req->len, req->with_parents, NodeVitastor::on_read_bitmap_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+        vitastor_c_uring_handle_events(cli->c);
+#endif
    }
    else if (req->op == NODE_VITASTOR_GET_INFO)
    {
@@ -648,6 +817,9 @@ NAN_METHOD(NodeVitastorKV::Open)
        delete req;
        kv->Unref();
    });
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(kv->cli->c);
+#endif
 }

 // close(callback(err))
@@ -671,6 +843,9 @@ NAN_METHOD(NodeVitastorKV::Close)
        delete req;
        kv->Unref();
    });
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(kv->cli->c);
+#endif
 }

 // set_config({ ...config })
@@ -729,6 +904,9 @@ void NodeVitastorKV::get_impl(const Nan::FunctionCallbackInfo<v8::Value> & info,
        delete req;
        kv->Unref();
    }, allow_cache);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(kv->cli->c);
+#endif
 }

 // get(key, callback(err, value))
@@ -801,6 +979,9 @@ NAN_METHOD(NodeVitastorKV::Set)
            delete cas_req;
        kv->Unref();
    }, cas_cb);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(kv->cli->c);
+#endif
 }

 // del(key, callback(err), cas_compare(old_value)?)
@@ -839,6 +1020,9 @@ NAN_METHOD(NodeVitastorKV::Del)
            delete cas_req;
        kv->Unref();
    }, cas_cb);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(kv->cli->c);
+#endif
 }

 // list(start_key?)
@@ -909,7 +1093,7 @@ NodeVitastorKVListing::~NodeVitastorKVListing()
    kv->Unref();
 }

-// next(callback(err, value)?)
+// next(callback(err, key, value)?)
 NAN_METHOD(NodeVitastorKVListing::Next)
 {
    TRACE("NodeVitastorKVListing::Next");
@@ -959,6 +1143,9 @@ NAN_METHOD(NodeVitastorKVListing::Next)
            list->iter = req;
        list->kv->Unref();
    });
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(list->kv->cli->c);
+#endif
 }

 // close()
@@ -15,14 +15,24 @@ class NodeVitastor: public Nan::ObjectWrap
 public:
    // constructor({ ...config })
    static NAN_METHOD(Create);
-    // read(pool, inode, offset, len, callback(err, buffer, version))
+    // read(pool_id, inode, offset, length, callback(err, buffer, version))
    static NAN_METHOD(Read);
-    // write(pool, inode, offset, buf: Buffer | Buffer[], { version }?, callback(err))
+    // write(pool_id, inode, offset, buf: Buffer | Buffer[], { version }?, callback(err))
    static NAN_METHOD(Write);
+    // delete(pool_id, inode, offset, length, { version }?, callback(err))
+    static NAN_METHOD(Delete);
    // sync(callback(err))
    static NAN_METHOD(Sync);
-    // read_bitmap(pool, inode, offset, len, with_parents, callback(err, bitmap_buffer))
+    // read_bitmap(pool_id, inode, offset, length, with_parents, callback(err, bitmap_buffer))
    static NAN_METHOD(ReadBitmap);
+    // on_ready(callback(err))
+    static NAN_METHOD(OnReady);
+    // get_min_io_size(pool_id)
+    static NAN_METHOD(GetMinIoSize);
+    // get_max_atomic_write_size(pool_id)
+    static NAN_METHOD(GetMaxAtomicWriteSize);
+    // get_immediate_commit(pool_id)
+    static NAN_METHOD(GetImmediateCommit);
 //    // destroy()
 //    static NAN_METHOD(Destroy);

@@ -37,11 +47,13 @@ private:

    static void on_io_readable(uv_poll_t* handle, int status, int revents);
    static void on_read_finish(void *opaque, long retval, uint64_t version);
+    static void on_ready_finish(void *opaque, long retval);
    static void on_write_finish(void *opaque, long retval);
    static void on_read_bitmap_finish(void *opaque, long retval, uint8_t *bitmap);

    NodeVitastorRequest* get_read_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos);
    NodeVitastorRequest* get_write_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos);
+    NodeVitastorRequest* get_delete_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos);

    friend class NodeVitastorImage;
    friend class NodeVitastorKV;
@@ -53,13 +65,15 @@ class NodeVitastorImage: public Nan::ObjectWrap
 public:
    // constructor(node_vitastor, name)
    static NAN_METHOD(Create);
-    // read(offset, len, callback(err, buffer, version))
+    // read(offset, length, callback(err, buffer, version))
    static NAN_METHOD(Read);
    // write(offset, buf: Buffer | Buffer[], { version }?, callback(err))
    static NAN_METHOD(Write);
+    // delete(offset, length, { version }?, callback(err))
+    static NAN_METHOD(Delete);
    // sync(callback(err))
    static NAN_METHOD(Sync);
-    // read_bitmap(offset, len, with_parents, callback(err, bitmap_buffer))
+    // read_bitmap(offset, length, with_parents, callback(err, bitmap_buffer))
    static NAN_METHOD(ReadBitmap);
    // get_info(callback({ num, name, size, parent_id?, readonly?, meta?, mod_revision, block_size, bitmap_granularity, immediate_commit }))
    static NAN_METHOD(GetInfo);
@@ -120,7 +134,7 @@ class NodeVitastorKVListing: public Nan::ObjectWrap
 public:
    // constructor(node_vitastor_kv, start_key?)
    static NAN_METHOD(Create);
-    // next(callback(err, value)?)
+    // next(callback(err, key, value)?)
    static NAN_METHOD(Next);
    // close()
    static NAN_METHOD(Close);
@@ -1,6 +1,6 @@
 {
  "name": "vitastor",
-  "version": "1.7.0",
+  "version": "1.11.0",
  "description": "Low-level native bindings to Vitastor client library",
  "main": "index.js",
  "keywords": [
@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils

-VITASTOR_VERSION = '1.10.1'
+VITASTOR_VERSION = '1.11.0'

 LOG = logging.getLogger(__name__)

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.10.1
+Version:        1.11.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage

 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.10.1.el7.tar.gz
+Source0:        vitastor-1.11.0.el7.tar.gz

 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel
@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.10.1
+Version:        1.11.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage

 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.10.1.el8.tar.gz
+Source0:        vitastor-1.11.0.el8.tar.gz

 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel
@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.10.1
+Version:        1.11.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage

 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.10.1.el9.tar.gz
+Source0:        vitastor-1.11.0.el9.tar.gz

 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel
@@ -19,7 +19,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()

-add_definitions(-DVITASTOR_VERSION="1.10.1")
+add_definitions(-DVITASTOR_VERSION="1.11.0")
 add_definitions(-D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})
@@ -256,6 +256,7 @@ resume_2:
    }
    if (entries_to_zero.size() && !bs->inmemory_meta && !bs->readonly)
    {
+        std::sort(entries_to_zero.begin(), entries_to_zero.end());
        // we have to zero out additional entries
        for (i = 0; i < entries_to_zero.size(); )
        {
@@ -338,6 +339,15 @@ bool blockstore_init_meta::handle_meta_block(uint8_t *buf, uint64_t entries_per_
                if (*entry_csum != crc32c(0, entry, bs->dsk.clean_entry_size - 4))
                {
                    printf("Metadata entry %ju is corrupt (checksum mismatch), skipping\n", done_cnt+i);
+                    // zero out the invalid entry, otherwise we'll hit "tried to overwrite non-zero metadata entry" later
+                    if (bs->inmemory_meta)
+                    {
+                        memset(entry, 0, bs->dsk.clean_entry_size);
+                    }
+                    else
+                    {
+                        entries_to_zero.push_back(done_cnt+i);
+                    }
                    continue;
                }
            }
@@ -63,6 +63,10 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd

    st_cli.parse_config(config);
    st_cli.infinite_start = false;
+    if (!config["client_infinite_start"].is_null())
+    {
+        st_cli.infinite_start = config["client_infinite_start"].bool_value();
+    }
    st_cli.load_global_config();

    scrap_buffer_size = SCRAP_BUFFER_SIZE;
@@ -309,16 +313,17 @@ void cluster_client_t::erase_op(cluster_op_t *op)
    }
    if (flags & OP_FLUSH_BUFFER)
    {
+        auto overflow = std::move(wb->writeback_overflow);
        int i = 0;
-        while (i < wb->writeback_overflow.size() && wb->writebacks_active < client_max_writeback_iodepth)
+        while (i < overflow.size() && wb->writebacks_active < client_max_writeback_iodepth)
        {
-            execute_internal(wb->writeback_overflow[i]);
+            execute_internal(overflow[i]);
            i++;
        }
        if (i > 0)
-        {
-            wb->writeback_overflow.erase(wb->writeback_overflow.begin(), wb->writeback_overflow.begin()+i);
-        }
+            overflow.erase(overflow.begin(), overflow.begin()+i);
+        assert(!wb->writeback_overflow.size());
+        wb->writeback_overflow.swap(overflow);
    }
 }

@@ -673,7 +678,7 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
        }
        // Just copy and acknowledge the operation
        wb->copy_write(op, CACHE_DIRTY);
-        while (wb->writeback_bytes + op->len > client_max_buffered_bytes || wb->writeback_queue_size > client_max_buffered_ops)
+        while (wb->writeback_bytes > client_max_buffered_bytes || wb->writeback_queue_size > client_max_buffered_ops)
        {
            // Initiate some writeback (asynchronously)
            wb->start_writebacks(this, 1);
@@ -791,6 +796,36 @@ bool cluster_client_t::check_rw(cluster_op_t *op)
            return false;
        }
    }
+    op->deoptimise_snapshot = false;
+    if (enable_writeback && (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP))
+    {
+        auto ino_it = st_cli.inode_config.find(op->inode);
+        if (ino_it != st_cli.inode_config.end())
+        {
+            int chain_size = 0;
+            while (ino_it != st_cli.inode_config.end() && ino_it->second.parent_id)
+            {
+                // Check for loops - FIXME check it in etcd_state_client
+                if (ino_it->second.parent_id == op->inode ||
+                    chain_size > st_cli.inode_config.size())
+                {
+                    op->retval = -EINVAL;
+                    auto cb = std::move(op->callback);
+                    cb(op);
+                    return false;
+                }
+                if (INODE_POOL(ino_it->second.parent_id) == INODE_POOL(ino_it->first) &&
+                    wb->has_inode(ino_it->second.parent_id))
+                {
+                    // Deoptimise reads - we have dirty data for one of the parent layer(s).
+                    op->deoptimise_snapshot = true;
+                    break;
+                }
+                chain_size++;
+                ino_it = st_cli.inode_config.find(ino_it->second.parent_id);
+            }
+        }
+    }
    return true;
 }

@@ -922,12 +957,21 @@ resume_2:
        {
            // Check parent inode
            auto ino_it = st_cli.inode_config.find(op->cur_inode);
-            while (ino_it != st_cli.inode_config.end() && ino_it->second.parent_id &&
-                INODE_POOL(ino_it->second.parent_id) == INODE_POOL(op->cur_inode) &&
-                // Check for loops
-                ino_it->second.parent_id != op->inode)
+            // Skip parents from the same pool
+            int skipped = 0;
+            while (!op->deoptimise_snapshot &&
+                ino_it != st_cli.inode_config.end() && ino_it->second.parent_id &&
+                INODE_POOL(ino_it->second.parent_id) == INODE_POOL(op->cur_inode))
            {
-                // Skip parents from the same pool
+                // Check for loops - FIXME check it in etcd_state_client
+                if (ino_it->second.parent_id == op->inode ||
+                    skipped > st_cli.inode_config.size())
+                {
+                    op->retval = -EINVAL;
+                    erase_op(op);
+                    return 1;
+                }
+                skipped++;
                ino_it = st_cli.inode_config.find(ino_it->second.parent_id);
            }
            if (ino_it != st_cli.inode_config.end() &&
@@ -1106,7 +1150,7 @@ void cluster_client_t::slice_rw(cluster_op_t *op)
            if (end == begin)
            {
                op->done_count++;
-                op->parts[i].flags = PART_DONE;
+                op->parts[i].flags = PART_SENT|PART_DONE;
            }
        }
        else if (op->opcode != OSD_OP_READ_BITMAP && op->opcode != OSD_OP_READ_CHAIN_BITMAP && op->opcode != OSD_OP_DELETE)
@@ -1189,7 +1233,7 @@ int cluster_client_t::try_send(cluster_op_t *op, int i)
                pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks
            );
            uint64_t meta_rev = 0;
-            if (op->opcode != OSD_OP_READ_BITMAP && op->opcode != OSD_OP_DELETE)
+            if (op->opcode != OSD_OP_READ_BITMAP && op->opcode != OSD_OP_DELETE && !op->deoptimise_snapshot)
            {
                auto ino_it = st_cli.inode_config.find(op->cur_inode);
                if (ino_it != st_cli.inode_config.end())
@@ -60,7 +60,8 @@ struct cluster_op_t
 protected:
    int state = 0;
    uint64_t cur_inode; // for snapshot reads
-    bool needs_reslice = false;
+    bool needs_reslice: 1;
+    bool deoptimise_snapshot: 1;
    int retry_after = 0;
    int inflight_count = 0, done_count = 0;
    timespec wait_up_until = {};
@@ -42,10 +42,10 @@ public:
    std::multimap<uint64_t, uint64_t*> flushed_buffers; // flush_id => refcnt

    ~writeback_cache_t();
+    bool has_inode(uint64_t inode);
    dirty_buf_it_t find_dirty(uint64_t inode, uint64_t offset);
    bool is_left_merged(dirty_buf_it_t dirty_it);
    bool is_right_merged(dirty_buf_it_t dirty_it);
-    bool is_merged(const dirty_buf_it_t & dirty_it);
    void copy_write(cluster_op_t *op, int state, uint64_t new_flush_id = 0);
    int repeat_ops_for(cluster_client_t *cli, osd_num_t peer_osd, pool_id_t pool_id, pg_num_t pg_num);
    void start_writebacks(cluster_client_t *cli, int count);
@@ -9,7 +9,7 @@ writeback_cache_t::~writeback_cache_t()
 {
    for (auto & bp: dirty_buffers)
    {
-        if (!--(*bp.second.refcnt))
+        if (bp.second.buf && !--(*bp.second.refcnt))
        {
            free(bp.second.refcnt); // refcnt is allocated with the buffer
        }
@@ -17,6 +17,15 @@ writeback_cache_t::~writeback_cache_t()
    dirty_buffers.clear();
 }

+bool writeback_cache_t::has_inode(uint64_t inode)
+{
+    auto dirty_it = dirty_buffers.lower_bound((object_id){
+        .inode = inode,
+        .stripe = 0,
+    });
+    return dirty_it != dirty_buffers.end() && dirty_it->first.inode == inode;
+}
+
 dirty_buf_it_t writeback_cache_t::find_dirty(uint64_t inode, uint64_t offset)
 {
    auto dirty_it = dirty_buffers.lower_bound((object_id){
@@ -33,7 +42,11 @@ dirty_buf_it_t writeback_cache_t::find_dirty(uint64_t inode, uint64_t offset)
            break;
        }
    }
-    return dirty_it;
+    if (dirty_it != dirty_buffers.end() && dirty_it->first.inode == inode)
+    {
+        return dirty_it;
+    }
+    return dirty_buffers.end();
 }

 bool writeback_cache_t::is_left_merged(dirty_buf_it_t dirty_it)
@@ -68,11 +81,6 @@ bool writeback_cache_t::is_right_merged(dirty_buf_it_t dirty_it)
    return false;
 }

-bool writeback_cache_t::is_merged(const dirty_buf_it_t & dirty_it)
-{
-    return is_left_merged(dirty_it) || is_right_merged(dirty_it);
-}
-
 void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flush_id)
 {
    // Save operation for replay when one of PGs goes out of sync
@@ -107,10 +115,16 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
                    .flush_id = dirty_it->second.flush_id,
                    .refcnt = dirty_it->second.refcnt,
                });
-                (*dirty_it->second.refcnt)++;
+                if (dirty_it->second.buf)
+                {
+                    (*dirty_it->second.refcnt)++;
+                }
                if (dirty_it->second.state == CACHE_DIRTY)
                {
-                    writeback_bytes -= op->len;
+                    if (dirty_it->second.buf)
+                    {
+                        writeback_bytes -= op->len;
+                    }
                    writeback_queue_size++;
                }
                break;
@@ -120,8 +134,11 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
                // Only leave the beginning
                if (dirty_it->second.state == CACHE_DIRTY)
                {
-                    writeback_bytes -= old_end - op->offset;
-                    if (is_left_merged(dirty_it) && !is_right_merged(dirty_it))
+                    if (dirty_it->second.buf)
+                    {
+                        writeback_bytes -= old_end - op->offset;
+                    }
+                    if (is_right_merged(dirty_it))
                    {
                        writeback_queue_size++;
                    }
@@ -135,8 +152,11 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
            // Only leave the end
            if (dirty_it->second.state == CACHE_DIRTY)
            {
-                writeback_bytes -= new_end - dirty_it->first.stripe;
-                if (!is_left_merged(dirty_it) && is_right_merged(dirty_it))
+                if (dirty_it->second.buf)
+                {
+                    writeback_bytes -= new_end - dirty_it->first.stripe;
+                }
+                if (is_left_merged(dirty_it))
                {
                    writeback_queue_size++;
                }
@@ -158,13 +178,25 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
        else
        {
            // Remove the whole buffer
-            if (dirty_it->second.state == CACHE_DIRTY && !is_merged(dirty_it))
+            if (dirty_it->second.state == CACHE_DIRTY)
            {
-                writeback_bytes -= dirty_it->second.len;
-                assert(writeback_queue_size > 0);
-                writeback_queue_size--;
+                if (dirty_it->second.buf)
+                {
+                    writeback_bytes -= dirty_it->second.len;
+                }
+                bool lm = is_left_merged(dirty_it);
+                bool rm = is_right_merged(dirty_it);
+                if (!lm && !rm)
+                {
+                    assert(writeback_queue_size > 0);
+                    writeback_queue_size--;
+                }
+                else if (lm && rm)
+                {
+                    writeback_queue_size++;
+                }
            }
-            if (!--(*dirty_it->second.refcnt))
+            if (dirty_it->second.buf && !--(*dirty_it->second.refcnt))
            {
                free(dirty_it->second.refcnt);
            }
@@ -175,7 +207,10 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
    bool is_del = op->opcode == OSD_OP_DELETE;
    uint64_t *refcnt = is_del ? NULL : (uint64_t*)malloc_or_die(sizeof(uint64_t) + op->len);
    uint8_t *buf = is_del ? NULL : ((uint8_t*)refcnt + sizeof(uint64_t));
-    *refcnt = 1;
+    if (!is_del)
+    {
+        *refcnt = 1;
+    }
    dirty_it = dirty_buffers.emplace_hint(dirty_it, (object_id){
        .inode = op->inode,
        .stripe = op->offset,
@@ -190,7 +225,9 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
    {
        writeback_bytes += is_del ? 0 : op->len;
        // Track consecutive write-back operations
-        if (!is_merged(dirty_it))
+        bool lm = is_left_merged(dirty_it);
+        bool rm = is_right_merged(dirty_it);
+        if (!lm && !rm)
        {
            // <writeback_queue> is OK to contain more than actual number of consecutive
            // requests as long as it doesn't miss anything. But <writeback_queue_size>
@@ -201,6 +238,11 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
                .stripe = op->offset,
            });
        }
+        else if (lm && rm)
+        {
+            assert(writeback_queue_size > 0);
+            writeback_queue_size--;
+        }
    }
    if (!is_del)
    {
@@ -225,7 +267,7 @@ int writeback_cache_t::repeat_ops_for(cluster_client_t *cli, osd_num_t peer_osd,
        for (auto wr_it = dirty_buffers.begin(), flush_it = wr_it, last_it = wr_it; ; )
        {
            bool end = wr_it == dirty_buffers.end();
-            bool flush_this = !end && wr_it->second.state != CACHE_REPEATING;
+            bool flush_this = !end && wr_it->second.state != CACHE_REPEATING && wr_it->second.state != CACHE_DIRTY;
            if (peer_osd)
                flush_this = flush_this && cli->affects_osd(wr_it->first.inode, wr_it->first.stripe, wr_it->second.len, peer_osd);
            if (pool_id && pg_num)
@@ -376,6 +418,7 @@ void writeback_cache_t::start_writebacks(cluster_client_t *cli, int count)
        assert(writeback_queue_size > 0);
        writeback_queue_size--;
        writeback_bytes -= off - from_it->first.stripe;
+        assert(writeback_queue_size > 0 || !writeback_bytes);
        flush_buffers(cli, from_it, to_it);
    }
    queue_copy.erase(queue_copy.begin(), queue_copy.begin()+i);
@@ -523,8 +566,10 @@ void writeback_cache_t::fsync_ok()
    {
        if (uw_it->second.state == CACHE_FLUSHING)
        {
-            if (!--(*uw_it->second.refcnt))
+            if (uw_it->second.buf && !--(*uw_it->second.refcnt))
+            {
                free(uw_it->second.refcnt);
+            }
            dirty_buffers.erase(uw_it++);
        }
        else
@@ -889,7 +889,11 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
            if (!pc.scrub_interval)
                pc.scrub_interval = 0;
            // Mark pool as VitastorFS pool (disable per-inode stats and block volume creation)
-            pc.used_for_fs = pool_item.second["used_for_fs"].as_string();
+            pc.used_for_app = pool_item.second["used_for_fs"].as_string();
+            if (pc.used_for_app != "")
+                pc.used_for_app = "fs:"+pc.used_for_app;
+            else
+                pc.used_for_app = pool_item.second["used_for_app"].as_string();
            // Immediate Commit Mode
            pc.immediate_commit = pool_item.second["immediate_commit"].is_string()
                ? parse_immediate_commit(pool_item.second["immediate_commit"].string_value(), IMMEDIATE_ALL)
@@ -61,7 +61,7 @@ struct pool_config_t
    uint64_t pg_stripe_size;
    std::map<pg_num_t, pg_config_t> pg_config;
    uint64_t scrub_interval;
-    std::string used_for_fs;
+    std::string used_for_app;
    int backfillfull;
 };

@@ -275,8 +275,8 @@ const char *help_text =
    "  --foreground 1\n"
    "    Stay in foreground, do not daemonize.\n"
    "\n"
-    "vitastor-nbd unmap /dev/nbdN\n"
-    "  Unmap an ioctl-mapped NBD device.\n"
+    "vitastor-nbd unmap [--force] /dev/nbdN\n"
+    "  Unmap an ioctl-mapped NBD device. Do not check if it's actually mapped if --force is specified.\n"
    "\n"
    "vitastor-nbd ls [--json]\n"
    "  List ioctl-mapped Vitastor NBD devices, optionally in JSON format.\n"
@@ -313,7 +313,7 @@ const char *help_text =
 #endif
    "Use vitastor-nbd --help <command> for command details or vitastor-nbd --help --all for all details.\n"
    "\n"
-    "All usual Vitastor config options like --config_file <path_to_config> may also be specified in CLI.\n"
+    "All usual Vitastor config options like --config_path <path_to_config> may also be specified in CLI.\n"
 ;

 class nbd_proxy
@@ -372,7 +372,8 @@ public:
            else if (args[i][0] == '-' && args[i][1] == '-')
            {
                const char *opt = args[i]+2;
-                cfg[opt] = !strcmp(opt, "json") || !strcmp(opt, "all") || i == narg-1 ? "1" : args[++i];
+                cfg[opt] = !strcmp(opt, "json") || !strcmp(opt, "all") ||
+                    !strcmp(opt, "force") || i == narg-1 ? "1" : args[++i];
            }
            else if (pos == 0)
            {
@@ -381,8 +382,9 @@ public:
            }
            else if (pos == 1)
            {
+                char c = 0;
                int n = 0;
-                if (sscanf(args[i], "/dev/nbd%d", &n) > 0)
+                if (sscanf(args[i], "/dev/nbd%d%c", &n, &c) == 1)
                    cfg["dev_num"] = n;
                else
                    cfg["dev_num"] = args[i];
@@ -404,18 +406,14 @@ public:
        }
        else if (cfg["command"] == "unmap")
        {
-            if (cfg["dev_num"].is_null())
+            if (!cfg["dev_num"].is_number() &&
+                cfg["dev_num"].string_value() != "0" &&
+                !cfg["dev_num"].uint64_value())
            {
                fprintf(stderr, "device name or number is missing\n");
                exit(1);
            }
-            if (cfg["netlink"].is_null())
-            {
-                ioctl_unmap(cfg["dev_num"].uint64_value());
-            }
-            else
-            {
-            }
+            ioctl_unmap(cfg["dev_num"].uint64_value(), cfg["force"].bool_value());
        }
 #ifdef HAVE_NBD_NETLINK_H
        else if (cfg["command"] == "netlink-map")
@@ -444,9 +442,18 @@ help:
        }
    }

-    void ioctl_unmap(int dev_num)
+    void ioctl_unmap(int dev_num, bool force)
    {
        char path[64] = { 0 };
+        // Check if mapped
+        sprintf(path, "/sys/block/nbd%d/pid", dev_num);
+        if (access(path, F_OK) != 0)
+        {
+            fprintf(stderr, "/dev/nbd%d is not mapped: /sys/block/nbd%d/pid does not exist\n", dev_num, dev_num);
+            if (!force)
+                exit(1);
+        }
+        // Run unmap
        sprintf(path, "/dev/nbd%d", dev_num);
        int r, nbd = open(path, O_RDWR);
        if (nbd < 0)
@@ -610,36 +617,43 @@ help:
        {
            if (!cfg["dev_num"].is_null())
            {
-                if (run_nbd(sockfd, cfg["dev_num"].int64_value(), device_size, NBD_FLAG_SEND_FLUSH, nbd_timeout, bg) < 0)
+                int r;
+                if ((r = run_nbd(sockfd, cfg["dev_num"].int64_value(), device_size, NBD_FLAG_SEND_FLUSH, nbd_timeout, bg)) != 0)
                {
-                    perror("run_nbd");
+                    fprintf(stderr, "run_nbd: %s\n", strerror(-r));
                    exit(1);
                }
            }
            else
            {
                // Find an unused device
+                auto mapped = list_mapped();
                int i = 0;
                while (true)
                {
+                    if (mapped.find("/dev/nbd"+std::to_string(i)) != mapped.end())
+                    {
+                        i++;
+                        continue;
+                    }
                    int r = run_nbd(sockfd, i, device_size, NBD_FLAG_SEND_FLUSH, nbd_timeout, bg);
                    if (r == 0)
                    {
                        printf("/dev/nbd%d\n", i);
                        break;
                    }
-                    else if (r == -1 && errno == ENOENT)
+                    else if (r == -ENOENT)
                    {
                        fprintf(stderr, "No free NBD devices found\n");
                        exit(1);
                    }
-                    else if (r == -2 && errno == EBUSY)
+                    else if (r == -EBUSY)
                    {
                        i++;
                    }
                    else
                    {
-                        perror("run_nbd");
+                        fprintf(stderr, "run_nbd: %s\n", strerror(-r));
                        exit(1);
                    }
                }
@@ -869,81 +883,114 @@ protected:
        // Check handle size
        assert(sizeof(cur_req.handle) == 8);
        char path[64] = { 0 };
-        sprintf(path, "/dev/nbd%d", dev_num);
-        int r, nbd = open(path, O_RDWR), qd_fd;
-        if (nbd < 0)
+        int notifyfd[2] = { 0 };
+        if (socketpair(AF_UNIX, SOCK_STREAM, 0, notifyfd) < 0)
        {
-            return -1;
+            return -errno;
        }
-        r = ioctl(nbd, NBD_SET_SOCK, sockfd[1]);
-        if (r < 0)
+        if (!fork())
        {
-            goto end_close;
-        }
-        r = ioctl(nbd, NBD_SET_BLKSIZE, 4096);
-        if (r < 0)
-        {
-            goto end_unmap;
-        }
-        r = ioctl(nbd, NBD_SET_SIZE, size);
-        if (r < 0)
-        {
-            goto end_unmap;
-        }
-        ioctl(nbd, NBD_SET_FLAGS, flags);
-        if (timeout > 0)
-        {
-            r = ioctl(nbd, NBD_SET_TIMEOUT, (unsigned long)timeout);
+            // Do all NBD configuration in the child process, after the last fork.
+            // Why? It's needed because there is a race condition in the Linux kernel nbd driver
+            // in nbd_add_socket() - it saves `current` task pointer as `nbd->task_setup` and
+            // then rechecks if the new `current` is the same. Problem is that if that process
+            // is already dead, `current` may be freed and then replaced by another process
+            // with the same pointer value. So the check passes and NBD allows a different process
+            // to set up a device which is already set up. Proper fix would have to be done in the
+            // kernel code, but the workaround is obviously to perform NBD setup from the process
+            // which will then actually call NBD_DO_IT. That process stays alive during the whole
+            // time of NBD device execution and the (nbd->task_setup != current) check always
+            // works correctly, and we don't accidentally break previous NBD devices while setting
+            // up a new device. Forking to check every device is of course rather slow, so we also
+            // do an additional check by calling list_mapped() before searching for a free NBD device.
+            if (bg)
+            {
+                daemonize_fork();
+            }
+            close(notifyfd[0]);
+            sprintf(path, "/dev/nbd%d", dev_num);
+            int r, nbd = open(path, O_RDWR), qd_fd;
+            if (nbd < 0)
+            {
+                write(notifyfd[1], &errno, sizeof(errno));
+                exit(1);
+            }
+            r = ioctl(nbd, NBD_SET_SOCK, sockfd[1]);
+            if (r < 0)
+            {
+                goto end_close;
+            }
+            r = ioctl(nbd, NBD_SET_BLKSIZE, 4096);
            if (r < 0)
            {
                goto end_unmap;
            }
-        }
-        // Configure request size
-        sprintf(path, "/sys/block/nbd%d/queue/max_sectors_kb", dev_num);
-        qd_fd = open(path, O_WRONLY);
-        if (qd_fd < 0)
-        {
-            goto end_unmap;
-        }
-        r = write(qd_fd, "32768", 5);
-        if (r != 5)
-        {
-            fprintf(stderr, "Warning: Failed to configure max_sectors_kb\n");
-        }
-        close(qd_fd);
-        if (!fork())
-        {
-            // Run in child
+            r = ioctl(nbd, NBD_SET_SIZE, size);
+            if (r < 0)
+            {
+                goto end_unmap;
+            }
+            ioctl(nbd, NBD_SET_FLAGS, flags);
+            if (timeout > 0)
+            {
+                r = ioctl(nbd, NBD_SET_TIMEOUT, (unsigned long)timeout);
+                if (r < 0)
+                {
+                    goto end_unmap;
+                }
+            }
+            // Configure request size
+            sprintf(path, "/sys/block/nbd%d/queue/max_sectors_kb", dev_num);
+            qd_fd = open(path, O_WRONLY);
+            if (qd_fd < 0)
+            {
+                goto end_unmap;
+            }
+            r = write(qd_fd, "32768", 5);
+            if (r != 5)
+            {
+                fprintf(stderr, "Warning: Failed to configure max_sectors_kb\n");
+            }
+            close(qd_fd);
+            // Notify parent
+            errno = 0;
+            write(notifyfd[1], &errno, sizeof(errno));
+            close(notifyfd[1]);
            close(sockfd[0]);
            if (bg)
            {
-                daemonize();
+                daemonize_reopen_stdio();
            }
            r = ioctl(nbd, NBD_DO_IT);
            if (r < 0)
            {
-                fprintf(stderr, "NBD device terminated with error: %s\n", strerror(errno));
+                fprintf(stderr, "NBD device /dev/nbd%d terminated with error: %s\n", dev_num, strerror(errno));
            }
            close(sockfd[1]);
            ioctl(nbd, NBD_CLEAR_QUE);
            ioctl(nbd, NBD_CLEAR_SOCK);
            exit(0);
-        }
-        close(sockfd[1]);
-        close(nbd);
-        return 0;
    end_close:
-        r = errno;
-        close(nbd);
-        errno = r;
-        return -2;
+            write(notifyfd[1], &errno, sizeof(errno));
+            close(nbd);
+            exit(2);
    end_unmap:
-        r = errno;
-        ioctl(nbd, NBD_CLEAR_SOCK);
-        close(nbd);
-        errno = r;
-        return -3;
+            write(notifyfd[1], &errno, sizeof(errno));
+            ioctl(nbd, NBD_CLEAR_SOCK);
+            close(nbd);
+            exit(3);
+        }
+        // Parent - check status
+        close(notifyfd[1]);
+        int child_errno = 0;
+        int ok = read(notifyfd[0], &child_errno, sizeof(child_errno));
+        close(notifyfd[0]);
+        if (ok && !child_errno)
+        {
+            close(sockfd[1]);
+            return 0;
+        }
+        return -child_errno;
    }

    void submit_send()
@@ -294,7 +294,9 @@ static void coroutine_fn vitastor_co_get_metadata(VitastorRPC *task)

    qemu_mutex_lock(&client->mutex);
    vitastor_c_watch_inode(client->proxy, client->image, vitastor_co_generic_cb, task);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
    vitastor_schedule_uring_handler(client);
+#endif
    qemu_mutex_unlock(&client->mutex);

    while (!task->complete)
@@ -566,6 +568,7 @@ static int vitastor_file_open(BlockDriverState *bs, QDict *options, int flags, E
 static void vitastor_close(BlockDriverState *bs)
 {
    VitastorClient *client = bs->opaque;
+    qemu_mutex_lock(&client->mutex);
    vitastor_c_destroy(client->proxy);
    if (client->fds)
    {
@@ -573,7 +576,6 @@ static void vitastor_close(BlockDriverState *bs)
        client->fds = NULL;
        client->fd_alloc = client->fd_count = 0;
    }
-    qemu_mutex_destroy(&client->mutex);
    if (client->config_path)
        g_free(client->config_path);
    if (client->etcd_host)
@@ -584,6 +586,8 @@ static void vitastor_close(BlockDriverState *bs)
        g_free(client->image);
    free(client->last_bitmap);
    client->last_bitmap = NULL;
+    qemu_mutex_unlock(&client->mutex);
+    qemu_mutex_destroy(&client->mutex);
 }

 #if QEMU_VERSION_MAJOR >= 3 || QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR > 2
@@ -749,7 +753,9 @@ static int coroutine_fn vitastor_co_preadv(BlockDriverState *bs,
    uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode;
    qemu_mutex_lock(&client->mutex);
    vitastor_c_read(client->proxy, inode, offset, bytes, iov->iov, iov->niov, vitastor_co_read_cb, &task);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
    vitastor_schedule_uring_handler(client);
+#endif
    qemu_mutex_unlock(&client->mutex);

    while (!task.complete)
@@ -783,7 +789,9 @@ static int coroutine_fn vitastor_co_pwritev(BlockDriverState *bs,
    uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode;
    qemu_mutex_lock(&client->mutex);
    vitastor_c_write(client->proxy, inode, offset, bytes, 0, iov->iov, iov->niov, vitastor_co_generic_cb, &task);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
    vitastor_schedule_uring_handler(client);
+#endif
    qemu_mutex_unlock(&client->mutex);

    while (!task.complete)
@@ -863,7 +871,9 @@ static int coroutine_fn vitastor_co_block_status(
        task.bitmap = client->last_bitmap = NULL;
        qemu_mutex_lock(&client->mutex);
        vitastor_c_read_bitmap(client->proxy, task.inode, task.offset, task.len, !client->skip_parents, vitastor_co_read_bitmap_cb, &task);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
        vitastor_schedule_uring_handler(client);
+#endif
        qemu_mutex_unlock(&client->mutex);
        while (!task.complete)
        {
@@ -950,7 +960,9 @@ static int coroutine_fn vitastor_co_flush(BlockDriverState *bs)

    qemu_mutex_lock(&client->mutex);
    vitastor_c_sync(client->proxy, vitastor_co_generic_cb, &task);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
    vitastor_schedule_uring_handler(client);
+#endif
    qemu_mutex_unlock(&client->mutex);

    while (!task.complete)
@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@

 Name: Vitastor
 Description: Vitastor client library
-Version: 1.10.1
+Version: 1.11.0
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}

@@ -127,6 +127,7 @@ vitastor_c *vitastor_c_create_qemu_uring(QEMUSetFDHandler *aio_set_fd_handler, v
    auto self = vitastor_c_create_qemu_common(aio_set_fd_handler, aio_context);
    self->ringloop = ringloop;
    self->cli = new cluster_client_t(self->ringloop, self->tfd, cfg_json);
+    ringloop->loop();
    return self;
 }

@@ -150,6 +151,7 @@ vitastor_c *vitastor_c_create_uring(const char *config_path, const char *etcd_ho
    self->ringloop = ringloop;
    self->epmgr = new epoll_manager_t(self->ringloop);
    self->cli = new cluster_client_t(self->ringloop, self->epmgr->tfd, cfg_json);
+    ringloop->loop();
    return self;
 }

@@ -183,6 +185,7 @@ vitastor_c *vitastor_c_create_uring_json(const char **options, int options_len)
    self->ringloop = ringloop;
    self->epmgr = new epoll_manager_t(self->ringloop);
    self->cli = new cluster_client_t(self->ringloop, self->epmgr->tfd, cfg_json);
+    ringloop->loop();
    return self;
 }

@@ -222,6 +225,18 @@ int vitastor_c_is_ready(vitastor_c *client)
    return client->cli->is_ready();
 }

+void vitastor_c_on_ready(vitastor_c *client, VitastorIOHandler cb, void *opaque)
+{
+    client->cli->on_ready([=]()
+    {
+        cb(opaque, 0);
+    });
+    if (client->ringloop)
+    {
+        client->ringloop->loop();
+    }
+}
+
 void vitastor_c_uring_wait_ready(vitastor_c *client)
 {
    while (!client->cli->is_ready())
@@ -276,6 +291,10 @@ void vitastor_c_read(vitastor_c *client, uint64_t inode, uint64_t offset, uint64
        delete op;
    };
    client->cli->execute(op);
+    if (client->ringloop)
+    {
+        client->ringloop->loop();
+    }
 }

 void vitastor_c_write(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, uint64_t check_version,
@@ -297,6 +316,31 @@ void vitastor_c_write(vitastor_c *client, uint64_t inode, uint64_t offset, uint6
        delete op;
    };
    client->cli->execute(op);
+    if (client->ringloop)
+    {
+        client->ringloop->loop();
+    }
+}
+
+void vitastor_c_delete(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, uint64_t check_version,
+    VitastorIOHandler cb, void *opaque)
+{
+    cluster_op_t *op = new cluster_op_t;
+    op->opcode = OSD_OP_DELETE;
+    op->inode = inode;
+    op->offset = offset;
+    op->len = len;
+    op->version = check_version;
+    op->callback = [cb, opaque](cluster_op_t *op)
+    {
+        cb(opaque, op->retval);
+        delete op;
+    };
+    client->cli->execute(op);
+    if (client->ringloop)
+    {
+        client->ringloop->loop();
+    }
 }

 void vitastor_c_read_bitmap(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len,
@@ -319,6 +363,10 @@ void vitastor_c_read_bitmap(vitastor_c *client, uint64_t inode, uint64_t offset,
        delete op;
    };
    client->cli->execute(op);
+    if (client->ringloop)
+    {
+        client->ringloop->loop();
+    }
 }

 void vitastor_c_sync(vitastor_c *client, VitastorIOHandler cb, void *opaque)
@@ -331,6 +379,10 @@ void vitastor_c_sync(vitastor_c *client, VitastorIOHandler cb, void *opaque)
        delete op;
    };
    client->cli->execute(op);
+    if (client->ringloop)
+    {
+        client->ringloop->loop();
+    }
 }

 void vitastor_c_watch_inode(vitastor_c *client, char *image, VitastorIOHandler cb, void *opaque)
@@ -340,6 +392,10 @@ void vitastor_c_watch_inode(vitastor_c *client, char *image, VitastorIOHandler c
        auto watch = client->cli->st_cli.watch_inode(std::string(image));
        cb(opaque, (long)watch);
    });
+    if (client->ringloop)
+    {
+        client->ringloop->loop();
+    }
 }

 void vitastor_c_close_watch(vitastor_c *client, void *handle)
@@ -7,7 +7,7 @@
 #define VITASTOR_QEMU_PROXY_H

 // C API wrapper version
-#define VITASTOR_C_API_VERSION 4
+#define VITASTOR_C_API_VERSION 5

 #ifndef POOL_ID_BITS
 #define POOL_ID_BITS 16
@@ -51,6 +51,7 @@ vitastor_c *vitastor_c_create_epoll_json(const char **options, int options_len);
 void* vitastor_c_get_internal_client(vitastor_c *client);
 void vitastor_c_destroy(vitastor_c *client);
 int vitastor_c_is_ready(vitastor_c *client);
+void vitastor_c_on_ready(vitastor_c *client, VitastorIOHandler cb, void *opaque);
 int vitastor_c_uring_register_eventfd(vitastor_c *client);
 void vitastor_c_uring_wait_ready(vitastor_c *client);
 void vitastor_c_uring_handle_events(vitastor_c *client);
@@ -62,6 +63,8 @@ void vitastor_c_read(vitastor_c *client, uint64_t inode, uint64_t offset, uint64
    struct iovec *iov, int iovcnt, VitastorReadHandler cb, void *opaque);
 void vitastor_c_write(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, uint64_t check_version,
    struct iovec *iov, int iovcnt, VitastorIOHandler cb, void *opaque);
+void vitastor_c_delete(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, uint64_t check_version,
+    VitastorIOHandler cb, void *opaque);
 void vitastor_c_read_bitmap(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len,
    int with_parents, VitastorReadBitmapHandler cb, void *opaque);
 void vitastor_c_sync(vitastor_c *client, VitastorIOHandler cb, void *opaque);
@@ -186,7 +186,8 @@ static const char* help_text =
    "    --raw_placement <rules>       Specify raw PG generation rules (see documentation for details)\n"
    "    --primary_affinity_tags tags  Prefer to put primary copies on OSDs with all specified tags\n"
    "    --scrub_interval <time>       Enable regular scrubbing for this pool. Format: number + unit s/m/h/d/M/y\n"
-    "    --used_for_fs <name>          Mark pool as used for VitastorFS with metadata in image <name>\n"
+    "    --used_for_app fs:<name>      Mark pool as used for VitastorFS with metadata in image <name>\n"
+    "    --used_for_app s3:<name>      Mark pool as used for S3 location with name <name>\n"
    "    --pg_stripe_size <number>     Increase object grouping stripe\n"
    "    --max_osd_combinations 10000  Maximum number of random combinations for LP solver input\n"
    "    --wait                        Wait for the new pool to come online\n"
@@ -198,7 +199,7 @@ static const char* help_text =
    "vitastor-cli modify-pool|pool-modify <id|name> [--name <new_name>] [PARAMETERS...]\n"
    "  Modify an existing pool. Modifiable parameters:\n"
    "    [-s|--pg_size <number>] [--pg_minsize <number>] [-n|--pg_count <count>]\n"
-    "    [--failure_domain <level>] [--root_node <node>] [--osd_tags <tags>] [--used_for_fs <name>]\n"
+    "    [--failure_domain <level>] [--root_node <node>] [--osd_tags <tags>] [--used_for_app <type>:<name>]\n"
    "    [--max_osd_combinations <number>] [--primary_affinity_tags <tags>] [--scrub_interval <time>]\n"
    "    [--level_placement <rules>] [--raw_placement <rules>]\n"
    "  Non-modifiable parameters (changing them WILL lead to data loss):\n"
@@ -224,7 +225,7 @@ static const char* help_text =
    "Use vitastor-cli --help <command> for command details or vitastor-cli --help --all for all details.\n"
    "\n"
    "GLOBAL OPTIONS:\n"
-    "  --config_file FILE  Path to Vitastor configuration file\n"
+    "  --config_path FILE  Path to Vitastor configuration file\n"
    "  --etcd_address URL  Etcd connection address\n"
    "  --iodepth N         Send N operations in parallel to each OSD when possible (default 32)\n"
    "  --parallel_osds M   Work with M osds in parallel when possible (default 4)\n"
@@ -92,12 +92,12 @@ struct image_creator_t
        {
            new_pool_id = pools.begin()->first;
        }
-        if (new_pool_id && !pools.at(new_pool_id).used_for_fs.empty() && !force)
+        if (new_pool_id && !pools.at(new_pool_id).used_for_app.empty() && !force)
        {
            result = (cli_result_t){
                .err = EINVAL,
                .text = "Pool "+pools.at(new_pool_id).name+
-                    " is used for VitastorFS "+pools.at(new_pool_id).used_for_fs+
+                    " is used for application "+pools.at(new_pool_id).used_for_app+
                    ". Use --force if you really know what you are doing",
            };
            state = 100;
@@ -215,6 +215,7 @@ resume_3:
            goto resume_3;
        else if (state == 4)
            goto resume_4;
+        // FIXME: take all info from etcd requests, not mixed with st_cli.inode_config
        for (auto & ic: parent->cli->st_cli.inode_config)
        {
            if (ic.second.name == image_name+"@"+new_snap)
@@ -286,6 +287,7 @@ resume_4:

    json11::Json::object get_next_id()
    {
+        assert(new_pool_id);
        return json11::Json::object {
            { "request_range", json11::Json::object {
                { "key", base64_encode(
@@ -321,6 +323,17 @@ resume_4:
            goto resume_2;
        else if (state == 3)
            goto resume_3;
+        if (!new_pool_id)
+        {
+            for (auto & ic: parent->cli->st_cli.inode_config)
+            {
+                if (ic.second.name == image_name)
+                {
+                    new_pool_id = INODE_POOL(ic.first);
+                    break;
+                }
+            }
+        }
        parent->etcd_txn(json11::Json::object { { "success", json11::Json::array {
            get_next_id(),
            json11::Json::object {
@@ -546,7 +546,7 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_ls(json11::Json cfg)
    lister->list_pool_name = lister->list_pool_id ? "" : cfg["pool"].as_string();
    lister->show_stats = cfg["long"].bool_value();
    lister->show_delete = cfg["del"].bool_value();
-    lister->sort_field = cfg["sort"].string_value();
+    lister->sort_field = cfg["sort"].string_value() != "" ? cfg["sort"].string_value() : "name";
    lister->reverse = cfg["reverse"].bool_value();
    lister->max_count = cfg["count"].uint64_value();
    for (auto & item: cfg["names"].array_items())
@@ -90,8 +90,8 @@ std::string validate_pool_config(json11::Json::object & new_cfg, json11::Json ol
            value = sz;
        }
        else if (key == "name" || key == "scheme" || key == "immediate_commit" ||
-            key == "failure_domain" || key == "root_node" || key == "scrub_interval" || key == "used_for_fs" ||
-            key == "raw_placement")
+            key == "failure_domain" || key == "root_node" || key == "scrub_interval" || key == "used_for_app" ||
+            key == "used_for_fs" || key == "raw_placement")
        {
            if (!value.is_string())
            {
@@ -156,8 +156,13 @@ std::string validate_pool_config(json11::Json::object & new_cfg, json11::Json ol
    {
        new_cfg.erase("parity_chunks");
    }
-    if (new_cfg.find("used_for_fs") != new_cfg.end() && new_cfg["used_for_fs"].string_value() == "")
+    if (new_cfg.find("used_for_app") != new_cfg.end() && new_cfg["used_for_app"].string_value() == "")
    {
+        new_cfg.erase("used_for_app");
+    }
+    if (new_cfg.find("used_for_app") == new_cfg.end() && new_cfg.find("used_for_fs") != new_cfg.end())
+    {
+        new_cfg["used_for_app"] = "fs:"+new_cfg["used_for_fs"].string_value();
        new_cfg.erase("used_for_fs");
    }

@@ -199,7 +199,9 @@ resume_1:
            auto & st = pool_stats[pool_id];
            for (auto & kv: pp.second.object_items())
            {
-                if (st.find(kv.first) == st.end())
+                if (kv.first == "used_for_fs" && st.find("used_for_app") == st.end())
+                    st["used_for_app"] = "fs:"+kv.second.string_value();
+                else if (st.find(kv.first) == st.end())
                    st[kv.first] = kv.second;
            }
        }
@@ -493,7 +495,7 @@ resume_3:
                { "name", "Name" },
                { "id", "ID" },
                { "scheme_name", "Scheme" },
-                { "used_for_fs", "Used for VitastorFS" },
+                { "used_for_app", "Used for app" },
                { "status", "Status" },
                { "backfillfull_fmt", "Backfillfull" },
                { "pg_count_fmt", "PGs" },
@@ -112,19 +112,21 @@ resume_1:
                return;
            }

-            if (new_cfg.find("used_for_fs") != new_cfg.end() && !force)
+            if (new_cfg.find("used_for_app") != new_cfg.end() && !force)
            {
                // Check that pool doesn't have images
                auto img_it = parent->cli->st_cli.inode_config.lower_bound(INODE_WITH_POOL(pool_id, 0));
-                if (img_it != parent->cli->st_cli.inode_config.end() && INODE_POOL(img_it->first) == pool_id &&
-                    img_it->second.name == new_cfg["used_for_fs"].string_value())
+                if (img_it != parent->cli->st_cli.inode_config.end() &&
+                    INODE_POOL(img_it->first) == pool_id &&
+                    new_cfg["used_for_app"].string_value().substr(0, 3) == "fs:" &&
+                    img_it->second.name == new_cfg["used_for_app"].string_value().substr(3))
                {
                    // Only allow metadata image to exist in the FS pool
                    img_it++;
                }
                if (img_it != parent->cli->st_cli.inode_config.end() && INODE_POOL(img_it->first) == pool_id)
                {
-                    result = (cli_result_t){ .err = ENOENT, .text = "Pool "+pool_name+" has block images, delete them before using it for VitastorFS" };
+                    result = (cli_result_t){ .err = ENOENT, .text = "Pool "+pool_name+" has block images, delete them before using it for VitastorFS, S3 or another app" };
                    state = 100;
                    return;
                }
@@ -245,10 +245,10 @@ void kv_fs_state_t::init(nfs_proxy_t *proxy, json11::Json cfg)
    if (proxy->cli->st_cli.inode_config.find(fs_kv_inode) != proxy->cli->st_cli.inode_config.end())
    {
        auto & name = proxy->cli->st_cli.inode_config.at(fs_kv_inode).name;
-        if (pool_cfg.used_for_fs != name)
+        if (pool_cfg.used_for_app != "fs:"+name)
        {
-            fprintf(stderr, "Please mark pool as used for this file system with `vitastor-cli modify-pool --used-for-fs %s %s`\n",
-                name.c_str(), cfg["fs"].string_value().c_str());
+            fprintf(stderr, "Please mark pool as used for this file system with `vitastor-cli modify-pool --used-for-app fs:%s %s`\n",
+                name.c_str(), pool_cfg.name.c_str());
            exit(1);
        }
    }
@@ -281,6 +281,8 @@ class osd_t
    int pick_next_scrub(object_id & next_oid);
    void submit_scrub_op(object_id oid);
    bool continue_scrub();
+    void submit_scrub_subops(osd_op_t *cur_op);
+    void scrub_check_results(osd_op_t *cur_op);
    void plan_scrub(pg_t & pg, bool report_state = true);
    void schedule_scrub(pg_t & pg);

@@ -313,7 +315,7 @@ class osd_t
    pg_osd_set_state_t *mark_object(pg_t & pg, object_id oid, pg_osd_set_state_t *prev_object_state, bool ref,
        std::function<int(pg_osd_set_t & new_set)> calc_set);
    pg_osd_set_state_t *mark_object_corrupted(pg_t & pg, object_id oid, pg_osd_set_state_t *prev_object_state,
-        osd_rmw_stripe_t *stripes, bool ref, bool inconsistent);
+        osd_rmw_stripe_t *stripes, bool ref);
    pg_osd_set_state_t *mark_partial_write(pg_t & pg, object_id oid, pg_osd_set_state_t *prev_object_state,
        osd_rmw_stripe_t *stripes, bool ref);
    void deref_object_state(pg_t & pg, pg_osd_set_state_t **object_state, bool deref);
@@ -326,6 +328,8 @@ class osd_t
    void submit_primary_subops(int submit_type, uint64_t op_version, const uint64_t* osd_set, osd_op_t *cur_op);
    int submit_primary_subop_batch(int submit_type, inode_t inode, uint64_t op_version,
        osd_rmw_stripe_t *stripes, const uint64_t* osd_set, osd_op_t *cur_op, int subop_idx, int zero_read);
+    void submit_primary_subop(osd_op_t *cur_op, osd_op_t *subop,
+        osd_rmw_stripe_t *si, bool wr, inode_t inode, uint64_t op_version);
    void submit_primary_del_subops(osd_op_t *cur_op, uint64_t *cur_set, uint64_t set_size, pg_osd_set_t & loc_set);
    void submit_primary_del_batch(osd_op_t *cur_op, obj_ver_osd_t *chunks_to_delete, int chunks_to_delete_count);
    int submit_primary_sync_subops(osd_op_t *cur_op);
@@ -673,7 +673,7 @@ void osd_t::apply_no_inode_stats()
    std::vector<uint64_t> no_inode_stats;
    for (auto & pool_item: st_cli.pool_config)
    {
-        if (!pool_item.second.used_for_fs.empty())
+        if (!pool_item.second.used_for_app.empty())
        {
            no_inode_stats.push_back(pool_item.first);
        }
@@ -3,6 +3,7 @@

 #include "osd.h"

+#include <sys/prctl.h>
 #include <signal.h>

 static osd_t *osd = NULL;
@@ -56,6 +57,9 @@ int main(int narg, char *args[])
        printf("%s", help_text);
        return 1;
    }
+    char osdname[16] = { 0 };
+    snprintf(osdname, 16, "osd%lu", config["osd_num"].uint64_value());
+    prctl(PR_SET_NAME, (unsigned long)osdname, 0, 0, 0);
    signal(SIGINT, handle_sigint);
    signal(SIGTERM, handle_sigint);
    ring_loop_t *ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
@@ -28,7 +28,6 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
        return false;
    }
    auto & pool_cfg = pool_cfg_it->second;
-    // FIXME: op_data->pg_data_size can probably be removed (there's pg.pg_data_size)
    uint64_t pg_data_size = (pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks);
    uint64_t pg_block_size = bs_block_size * pg_data_size;
    object_id oid = {
@@ -52,13 +51,13 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
        finish_op(cur_op, -EINVAL);
        return false;
    }
-    // Scrub is similar to r/w, so it's also handled here
-    int stripe_count = (pool_cfg.scheme == POOL_SCHEME_REPLICATED
-        && cur_op->req.hdr.opcode != OSD_OP_SCRUB ? 1 : pg_it->second.pg_size);
+    int stripe_count = (cur_op->req.hdr.opcode == OSD_OP_SCRUB ? 0 :
+        (pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pg_it->second.pg_size));
    int chain_size = 0;
    if (cur_op->req.hdr.opcode == OSD_OP_READ && cur_op->req.rw.meta_revision > 0)
    {
        // Chained read
+        // FIXME: Introduce an explicit opcode for chained reads
        auto inode_it = st_cli.inode_config.find(cur_op->req.rw.inode);
        if (inode_it->second.mod_revision != cur_op->req.rw.meta_revision)
        {
@@ -72,7 +71,7 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
            inode_it->second.parent_id &&
            INODE_POOL(inode_it->second.parent_id) == pg_it->second.pool_id)
        {
-            // Check for loops
+            // Check for loops - FIXME check it in etcd_state_client
            if (inode_it->second.parent_id == cur_op->req.rw.inode ||
                inode_it->second.parent_id == inode_it->second.num ||
                chain_size > st_cli.inode_config.size())
@@ -110,20 +109,22 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
    );
    void *data_buf = (uint8_t*)op_data + sizeof(osd_primary_op_data_t);
    op_data->pg_num = pg_num;
+    op_data->pg = &pg_it->second;
    op_data->oid = oid;
    op_data->stripes = (osd_rmw_stripe_t*)data_buf;
+    op_data->stripe_count = stripe_count;
    data_buf = (uint8_t*)data_buf + sizeof(osd_rmw_stripe_t) * stripe_count;
-    op_data->scheme = pool_cfg.scheme;
-    op_data->pg_data_size = pg_data_size;
-    op_data->pg_size = pg_it->second.pg_size;
    cur_op->op_data = op_data;
-    split_stripes(pg_data_size, bs_block_size, (uint32_t)(cur_op->req.rw.offset - oid.stripe), cur_op->req.rw.len, op_data->stripes);
-    // Resulting bitmaps have to survive op_data and be freed with the op itself
-    assert(!cur_op->bitmap_buf);
-    cur_op->bitmap_buf = calloc_or_die(1, clean_entry_bitmap_size * stripe_count);
-    for (int i = 0; i < stripe_count; i++)
+    if (cur_op->req.hdr.opcode != OSD_OP_SCRUB)
    {
-        op_data->stripes[i].bmp_buf = (uint8_t*)cur_op->bitmap_buf + clean_entry_bitmap_size * i;
+        split_stripes(pg_data_size, bs_block_size, (uint32_t)(cur_op->req.rw.offset - oid.stripe), cur_op->req.rw.len, op_data->stripes);
+        // Resulting bitmaps have to survive op_data and be freed with the op itself
+        assert(!cur_op->bitmap_buf);
+        cur_op->bitmap_buf = calloc_or_die(1, clean_entry_bitmap_size * stripe_count);
+        for (int i = 0; i < stripe_count; i++)
+        {
+            op_data->stripes[i].bmp_buf = (uint8_t*)cur_op->bitmap_buf + clean_entry_bitmap_size * i;
+        }
    }
    op_data->chain_size = chain_size;
    if (chain_size > 0)
@@ -205,11 +206,11 @@ void osd_t::continue_primary_read(osd_op_t *cur_op)
 resume_0:
    cur_op->reply.rw.bitmap_len = 0;
    {
-        auto & pg = pgs.at({ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num });
+        auto & pg = *op_data->pg;
        if (cur_op->req.rw.len == 0)
        {
            // len=0 => bitmap read
-            for (int role = 0; role < op_data->pg_data_size; role++)
+            for (int role = 0; role < pg.pg_data_size; role++)
            {
                op_data->stripes[role].read_start = 0;
                op_data->stripes[role].read_end = UINT32_MAX;
@@ -217,7 +218,7 @@ resume_0:
        }
        else
        {
-            for (int role = 0; role < op_data->pg_data_size; role++)
+            for (int role = 0; role < pg.pg_data_size; role++)
            {
                op_data->stripes[role].read_start = op_data->stripes[role].req_start;
                op_data->stripes[role].read_end = op_data->stripes[role].req_end;
@@ -228,29 +229,27 @@ resume_0:
        op_data->target_ver = vo_it != pg.ver_override.end() ? vo_it->second : UINT64_MAX;
        // PG may have degraded or misplaced objects
        op_data->prev_set = get_object_osd_set(pg, op_data->oid, &op_data->object_state);
-        if (pg.state == PG_ACTIVE || op_data->scheme == POOL_SCHEME_REPLICATED)
+        if (pg.state == PG_ACTIVE || pg.scheme == POOL_SCHEME_REPLICATED)
        {
            // Fast happy-path
-            if (op_data->scheme == POOL_SCHEME_REPLICATED &&
+            if (pg.scheme == POOL_SCHEME_REPLICATED &&
                op_data->object_state && (op_data->object_state->state & OBJ_INCOMPLETE))
            {
                finish_op(cur_op, -EIO);
                return;
            }
-            cur_op->buf = alloc_read_buffer(op_data->stripes, op_data->pg_data_size, 0);
+            cur_op->buf = alloc_read_buffer(op_data->stripes, pg.pg_data_size, 0);
            submit_primary_subops(SUBMIT_RMW_READ, op_data->target_ver, op_data->prev_set, cur_op);
            op_data->st = 1;
        }
        else
        {
-            if (extend_missing_stripes(op_data->stripes, op_data->prev_set, op_data->pg_data_size, pg.pg_size) < 0)
+            if (extend_missing_stripes(op_data->stripes, op_data->prev_set, pg.pg_data_size, pg.pg_size) < 0)
            {
                finish_op(cur_op, -EIO);
                return;
            }
            // Submit reads
-            op_data->pg_size = pg.pg_size;
-            op_data->scheme = pg.scheme;
            op_data->degraded = 1;
            cur_op->buf = alloc_read_buffer(op_data->stripes, pg.pg_size, 0);
            submit_primary_subops(SUBMIT_RMW_READ, op_data->target_ver, op_data->prev_set, cur_op);
@@ -265,30 +264,29 @@ resume_2:
        if (op_data->errcode == -EIO || op_data->errcode == -EDOM)
        {
            // I/O or checksum error
-            auto & pg = pgs.at({ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num });
            // FIXME: ref = true ideally... because new_state != state is not necessarily true if it's freed and recreated
-            op_data->object_state = mark_object_corrupted(pg, op_data->oid, op_data->object_state, op_data->stripes, false, false);
+            op_data->object_state = mark_object_corrupted(*op_data->pg, op_data->oid, op_data->object_state, op_data->stripes, false);
            goto resume_0;
        }
        finish_op(cur_op, op_data->errcode);
        return;
    }
    cur_op->reply.rw.version = op_data->fact_ver;
-    cur_op->reply.rw.bitmap_len = op_data->pg_data_size * clean_entry_bitmap_size;
+    cur_op->reply.rw.bitmap_len = op_data->pg->pg_data_size * clean_entry_bitmap_size;
    if (op_data->degraded)
    {
        // Reconstruct missing stripes
        osd_rmw_stripe_t *stripes = op_data->stripes;
-        if (op_data->scheme == POOL_SCHEME_XOR)
+        if (op_data->pg->scheme == POOL_SCHEME_XOR)
        {
-            reconstruct_stripes_xor(stripes, op_data->pg_size, clean_entry_bitmap_size);
+            reconstruct_stripes_xor(stripes, op_data->pg->pg_size, clean_entry_bitmap_size);
        }
-        else if (op_data->scheme == POOL_SCHEME_EC)
+        else if (op_data->pg->scheme == POOL_SCHEME_EC)
        {
-            reconstruct_stripes_ec(stripes, op_data->pg_size, op_data->pg_data_size, clean_entry_bitmap_size);
+            reconstruct_stripes_ec(stripes, op_data->pg->pg_size, op_data->pg->pg_data_size, clean_entry_bitmap_size);
        }
        cur_op->iov.push_back(op_data->stripes[0].bmp_buf, cur_op->reply.rw.bitmap_len);
-        for (int role = 0; role < op_data->pg_size; role++)
+        for (int role = 0; role < op_data->pg->pg_size; role++)
        {
            if (stripes[role].req_end != 0)
            {
@@ -360,10 +358,10 @@ pg_osd_set_state_t *osd_t::mark_object(pg_t & pg, object_id oid, pg_osd_set_stat
    return object_state;
 }

-pg_osd_set_state_t *osd_t::mark_object_corrupted(pg_t & pg, object_id oid, pg_osd_set_state_t *prev_object_state,
-    osd_rmw_stripe_t *stripes, bool ref, bool inconsistent)
+pg_osd_set_state_t *osd_t::mark_object_corrupted(pg_t & pg, object_id oid,
+    pg_osd_set_state_t *prev_object_state, osd_rmw_stripe_t *stripes, bool ref)
 {
-    return mark_object(pg, oid, prev_object_state, ref, [stripes, inconsistent](pg_osd_set_t & new_set)
+    return mark_object(pg, oid, prev_object_state, ref, [stripes](pg_osd_set_t & new_set)
    {
        // Mark object chunk(s) as corrupted
        int changes = 0;
@@ -390,16 +388,6 @@ pg_osd_set_state_t *osd_t::mark_object_corrupted(pg_t & pg, object_id oid, pg_os
                    chunk.loc_bad &= ~LOC_CORRUPTED;
                }
            }
-            if (inconsistent && !chunk.loc_bad)
-            {
-                changes++;
-                chunk.loc_bad |= LOC_INCONSISTENT;
-            }
-            else if (!inconsistent && (chunk.loc_bad & LOC_INCONSISTENT))
-            {
-                changes++;
-                chunk.loc_bad &= ~LOC_INCONSISTENT;
-            }
            chunk_it++;
        }
        return changes;
@@ -695,7 +683,7 @@ void osd_t::continue_primary_del(osd_op_t *cur_op)
        return;
    }
    osd_primary_op_data_t *op_data = cur_op->op_data;
-    auto & pg = pgs.at({ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num });
+    auto & pg = *op_data->pg;
    if (op_data->st == 1)      goto resume_1;
    else if (op_data->st == 2) goto resume_2;
    else if (op_data->st == 3) goto resume_3;
@@ -734,54 +722,58 @@ resume_3:
        cur_op->reply.rw.version = op_data->fact_ver;
        goto continue_others;
    }
-    // Save version override for parallel reads
-    pg.ver_override[op_data->oid] = op_data->fact_ver;
-    // Submit deletes
-    op_data->fact_ver++;
-    submit_primary_del_subops(cur_op, NULL, 0, op_data->object_state ? op_data->object_state->osd_set : pg.cur_loc_set);
+    // If not already deleted:
+    if (op_data->fact_ver)
+    {
+        // Save version override for parallel reads
+        pg.ver_override[op_data->oid] = op_data->fact_ver;
+        // Submit deletes
+        op_data->fact_ver++;
+        submit_primary_del_subops(cur_op, NULL, 0, op_data->object_state ? op_data->object_state->osd_set : pg.cur_loc_set);
 resume_4:
-    op_data->st = 4;
-    return;
-resume_5:
-    if (op_data->errors > 0)
-    {
-        deref_object_state(pg, &op_data->object_state, true);
-        pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->errcode);
+        op_data->st = 4;
        return;
-    }
-    // Remove version override
-    pg.ver_override.erase(op_data->oid);
-    // Adjust PG stats after "instant stabilize", because we need object_state above
-    if (!op_data->object_state)
-    {
-        pg.clean_count--;
-    }
-    else
-    {
-        remove_object_from_state(op_data->oid, &op_data->object_state, pg);
-        deref_object_state(pg, &op_data->object_state, true);
-    }
-    // Mark PG and OSDs as dirty
-    for (auto & chunk: (op_data->object_state ? op_data->object_state->osd_set : pg.cur_loc_set))
-    {
-        this->dirty_osds.insert(chunk.osd_num);
-    }
-    for (auto cl_it = msgr.clients.find(cur_op->peer_fd); cl_it != msgr.clients.end(); )
-    {
-        cl_it->second->dirty_pgs.insert({ .pool_id = pg.pool_id, .pg_num = pg.pg_num });
-        break;
-    }
-    dirty_pgs.insert({ .pool_id = pg.pool_id, .pg_num = pg.pg_num });
-    if (immediate_commit == IMMEDIATE_NONE)
-    {
-        unstable_write_count++;
-        if (unstable_write_count >= autosync_writes)
+resume_5:
+        if (op_data->errors > 0)
        {
-            unstable_write_count = 0;
-            autosync();
+            deref_object_state(pg, &op_data->object_state, true);
+            pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->errcode);
+            return;
        }
+        // Remove version override
+        pg.ver_override.erase(op_data->oid);
+        // Adjust PG stats after "instant stabilize", because we need object_state above
+        if (!op_data->object_state)
+        {
+            pg.clean_count--;
+        }
+        else
+        {
+            remove_object_from_state(op_data->oid, &op_data->object_state, pg);
+            deref_object_state(pg, &op_data->object_state, true);
+        }
+        // Mark PG and OSDs as dirty
+        for (auto & chunk: (op_data->object_state ? op_data->object_state->osd_set : pg.cur_loc_set))
+        {
+            this->dirty_osds.insert(chunk.osd_num);
+        }
+        for (auto cl_it = msgr.clients.find(cur_op->peer_fd); cl_it != msgr.clients.end(); )
+        {
+            cl_it->second->dirty_pgs.insert({ .pool_id = pg.pool_id, .pg_num = pg.pg_num });
+            break;
+        }
+        dirty_pgs.insert({ .pool_id = pg.pool_id, .pg_num = pg.pg_num });
+        if (immediate_commit == IMMEDIATE_NONE)
+        {
+            unstable_write_count++;
+            if (unstable_write_count >= autosync_writes)
+            {
+                unstable_write_count = 0;
+                autosync();
+            }
+        }
+        pg.total_count--;
    }
-    pg.total_count--;
    cur_op->reply.hdr.retval = 0;
    // indicate possibly unfinished (left_on_dead) deletions
    cur_op->reply.del.flags = OSD_DEL_SUPPORT_LEFT_ON_DEAD;
@@ -20,14 +20,15 @@ struct unstable_osd_num_t
 struct osd_primary_op_data_t
 {
    int st = 0;
-    pg_num_t pg_num;
-    object_id oid;
-    uint64_t target_ver;
+    pg_num_t pg_num = 0;
+    object_id oid = {};
+    uint64_t target_ver = 0;
    uint64_t orig_ver = 0, fact_ver = 0;
-    uint64_t scheme = 0;
    int n_subops = 0, done = 0, errors = 0, drops = 0, errcode = 0;
-    int degraded = 0, pg_size, pg_data_size;
-    osd_rmw_stripe_t *stripes;
+    int degraded = 0;
+    int stripe_count = 0;
+    osd_rmw_stripe_t *stripes = NULL;
+    pg_t *pg = NULL;
    osd_op_t *subops = NULL;
    uint64_t *prev_set = NULL;
    pg_osd_set_state_t *object_state = NULL;
@@ -7,7 +7,7 @@
 void osd_t::continue_chained_read(osd_op_t *cur_op)
 {
    osd_primary_op_data_t *op_data = cur_op->op_data;
-    auto & pg = pgs.at({ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num });
+    auto & pg = *op_data->pg;
    if (op_data->st == 1)
        goto resume_1;
    else if (op_data->st == 2)
@@ -17,7 +17,7 @@ void osd_t::continue_chained_read(osd_op_t *cur_op)
    else if (op_data->st == 4)
        goto resume_4;
    cur_op->reply.rw.bitmap_len = 0;
-    for (int role = 0; role < op_data->pg_data_size; role++)
+    for (int role = 0; role < pg.pg_data_size; role++)
    {
        op_data->stripes[role].read_start = op_data->stripes[role].req_start;
        op_data->stripes[role].read_end = op_data->stripes[role].req_end;
@@ -329,7 +329,7 @@ std::vector<osd_chain_read_t> osd_t::collect_chained_read_requests(osd_op_t *cur
 {
    osd_primary_op_data_t *op_data = cur_op->op_data;
    std::vector<osd_chain_read_t> chain_reads;
-    int stripe_count = (op_data->scheme == POOL_SCHEME_REPLICATED ? 1 : op_data->pg_size);
+    int stripe_count = (op_data->pg->scheme == POOL_SCHEME_REPLICATED ? 1 : op_data->pg->pg_size);
    memset(op_data->stripes[0].bmp_buf, 0, stripe_count * clean_entry_bitmap_size);
    uint8_t *global_bitmap = (uint8_t*)op_data->stripes[0].bmp_buf;
    // We always use at most 1 read request per layer
@@ -337,7 +337,7 @@ std::vector<osd_chain_read_t> osd_t::collect_chained_read_requests(osd_op_t *cur
    {
        uint8_t *part_bitmap = ((uint8_t*)op_data->snapshot_bitmaps) + chain_pos*stripe_count*clean_entry_bitmap_size;
        int start = !cur_op->req.rw.len ? 0 : (cur_op->req.rw.offset - op_data->oid.stripe)/bs_bitmap_granularity;
-        int end = !cur_op->req.rw.len ? op_data->pg_data_size*clean_entry_bitmap_size*8 : start + cur_op->req.rw.len/bs_bitmap_granularity;
+        int end = !cur_op->req.rw.len ? op_data->pg->pg_data_size*clean_entry_bitmap_size*8 : start + cur_op->req.rw.len/bs_bitmap_granularity;
        // Skip unneeded part in the beginning
        while (start < end && (
            ((global_bitmap[start>>3] >> (start&7)) & 1) ||
@@ -410,11 +410,11 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
        // FIXME: maybe introduce split_read_stripes to shorten these lines and to remove read_start=req_start
        osd_rmw_stripe_t *stripes = chain_stripes + chain_reads[cri].chain_pos*stripe_count;
        split_stripes(pg.pg_data_size, bs_block_size, chain_reads[cri].offset, chain_reads[cri].len, stripes);
-        if (op_data->scheme == POOL_SCHEME_REPLICATED && !stripes[0].req_end)
+        if (pg.scheme == POOL_SCHEME_REPLICATED && !stripes[0].req_end)
        {
            continue;
        }
-        for (int role = 0; role < op_data->pg_data_size; role++)
+        for (int role = 0; role < pg.pg_data_size; role++)
        {
            stripes[role].read_start = stripes[role].req_start;
            stripes[role].read_end = stripes[role].req_end;
@@ -423,7 +423,7 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
        if (pg.state != PG_ACTIVE)
        {
            cur_set = get_object_osd_set(pg, cur_oid, &op_data->chain_states[chain_reads[cri].chain_pos]);
-            if (op_data->scheme != POOL_SCHEME_REPLICATED)
+            if (pg.scheme != POOL_SCHEME_REPLICATED)
            {
                if (extend_missing_stripes(stripes, cur_set, pg.pg_data_size, pg.pg_size) < 0)
                {
@@ -446,7 +446,7 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
                }
            }
        }
-        if (op_data->scheme == POOL_SCHEME_REPLICATED)
+        if (pg.scheme == POOL_SCHEME_REPLICATED)
        {
            n_subops++;
            read_buffer_size += stripes[0].read_end - stripes[0].read_start;
@@ -491,7 +491,7 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
    for (int cri = 0; cri < chain_reads.size(); cri++)
    {
        osd_rmw_stripe_t *stripes = chain_stripes + chain_reads[cri].chain_pos*stripe_count;
-        if (op_data->scheme == POOL_SCHEME_REPLICATED && !stripes[0].req_end)
+        if (pg.scheme == POOL_SCHEME_REPLICATED && !stripes[0].req_end)
        {
            continue;
        }
@@ -501,9 +501,9 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
        auto cur_state = op_data->chain_states[chain_reads[cri].chain_pos];
        uint64_t *cur_set = (pg.state != PG_ACTIVE && cur_state ? cur_state->read_target.data() : pg.cur_set.data());
        int zero_read = -1;
-        if (op_data->scheme == POOL_SCHEME_REPLICATED)
+        if (pg.scheme == POOL_SCHEME_REPLICATED)
        {
-            for (int role = 0; role < op_data->pg_size; role++)
+            for (int role = 0; role < pg.pg_size; role++)
                if (cur_set[role] == this->osd_num || zero_read == -1)
                    zero_read = role;
        }
@@ -535,7 +535,7 @@ void osd_t::check_corrupted_chained(pg_t & pg, osd_op_t *cur_op)
        }
        if (corrupted)
        {
-            mark_object_corrupted(pg, cur_oid, op_data->chain_states[op_data->chain_reads[cri].chain_pos], stripes, false, false);
+            mark_object_corrupted(pg, cur_oid, op_data->chain_states[op_data->chain_reads[cri].chain_pos], stripes, false);
        }
    }
 }
@@ -555,18 +555,18 @@ void osd_t::send_chained_read_results(pg_t & pg, osd_op_t *cur_op)
        {
            // Reconstruct missing stripes
            osd_rmw_stripe_t *stripes = chain_stripes + op_data->chain_reads[cri].chain_pos*stripe_count;
-            if (op_data->scheme == POOL_SCHEME_XOR)
+            if (pg.scheme == POOL_SCHEME_XOR)
            {
                reconstruct_stripes_xor(stripes, pg.pg_size, clean_entry_bitmap_size);
            }
-            else if (op_data->scheme == POOL_SCHEME_EC)
+            else if (pg.scheme == POOL_SCHEME_EC)
            {
                reconstruct_stripes_ec(stripes, pg.pg_size, pg.pg_data_size, clean_entry_bitmap_size);
            }
        }
    }
    // Send bitmap
-    cur_op->reply.rw.bitmap_len = op_data->pg_data_size * clean_entry_bitmap_size;
+    cur_op->reply.rw.bitmap_len = pg.pg_data_size * clean_entry_bitmap_size;
    cur_op->iov.push_back(op_data->stripes[0].bmp_buf, cur_op->reply.rw.bitmap_len);
    // And finally compose the result
    uint64_t sent = 0;
@@ -67,7 +67,7 @@ void osd_t::finish_op(osd_op_t *cur_op, int retval)
        if (cur_op->req.hdr.opcode == OSD_OP_DELETE)
        {
            if (cur_op->op_data)
-                inode_stats[cur_op->req.rw.inode].op_bytes[inode_st_op] += cur_op->op_data->pg_data_size * bs_block_size;
+                inode_stats[cur_op->req.rw.inode].op_bytes[inode_st_op] += cur_op->op_data->pg->pg_data_size * bs_block_size;
        }
        else
            inode_stats[cur_op->req.rw.inode].op_bytes[inode_st_op] += cur_op->req.rw.len;
@@ -76,7 +76,7 @@ void osd_t::finish_op(osd_op_t *cur_op, int retval)
    {
        if (cur_op->op_data->pg_num > 0)
        {
-            auto & pg = pgs.at({ .pool_id = INODE_POOL(cur_op->op_data->oid.inode), .pg_num = cur_op->op_data->pg_num });
+            auto & pg = *cur_op->op_data->pg;
            pg.inflight--;
            assert(pg.inflight >= 0);
            if ((pg.state & PG_STOPPING) && pg.inflight == 0 && !pg.flush_batch)
@@ -126,10 +126,10 @@ void osd_t::submit_primary_subops(int submit_type, uint64_t op_version, const ui
    bool wr = submit_type == SUBMIT_WRITE;
    osd_primary_op_data_t *op_data = cur_op->op_data;
    osd_rmw_stripe_t *stripes = op_data->stripes;
-    bool rep = op_data->scheme == POOL_SCHEME_REPLICATED;
+    bool rep = op_data->pg->scheme == POOL_SCHEME_REPLICATED;
    // Allocate subops
    int n_subops = 0, zero_read = -1;
-    for (int role = 0; role < op_data->pg_size; role++)
+    for (int role = 0; role < op_data->pg->pg_size; role++)
    {
        if (osd_set[role] == this->osd_num || osd_set[role] != 0 && zero_read == -1)
            zero_read = role;
@@ -152,11 +152,11 @@ void osd_t::submit_primary_subops(int submit_type, uint64_t op_version, const ui
 int osd_t::submit_primary_subop_batch(int submit_type, inode_t inode, uint64_t op_version,
    osd_rmw_stripe_t *stripes, const uint64_t* osd_set, osd_op_t *cur_op, int subop_idx, int zero_read)
 {
+    bool rep = cur_op->op_data->pg->scheme == POOL_SCHEME_REPLICATED;
    bool wr = submit_type == SUBMIT_WRITE;
    osd_primary_op_data_t *op_data = cur_op->op_data;
-    bool rep = op_data->scheme == POOL_SCHEME_REPLICATED;
    int i = subop_idx;
-    for (int role = 0; role < op_data->pg_size; role++)
+    for (int role = 0; role < op_data->pg->pg_size; role++)
    {
        // We always submit zero-length writes to all replicas, even if the stripe is not modified
        if (!(wr || !rep && stripes[role].read_end != 0 || zero_read == role || submit_type == SUBMIT_SCRUB_READ))
@@ -168,109 +168,9 @@ int osd_t::submit_primary_subop_batch(int submit_type, inode_t inode, uint64_t o
        osd_rmw_stripe_t *si = stripes + (submit_type == SUBMIT_SCRUB_READ ? role : stripe_num);
        if (role_osd_num != 0)
        {
-            osd_op_t *subop = op_data->subops + i;
-            uint32_t subop_len = wr
-                ? si->write_end - si->write_start
-                : si->read_end - si->read_start;
-            if (!wr && si->read_end == UINT32_MAX)
-            {
-                subop_len = 0;
-            }
            si->osd_num = role_osd_num;
-            si->read_error = false;
-            subop->bitmap = si->bmp_buf;
-            subop->bitmap_len = clean_entry_bitmap_size;
-            // Using rmw_buf to pass pointer to stripes. Dirty but should work
-            subop->rmw_buf = si;
-            if (role_osd_num == this->osd_num)
-            {
-                clock_gettime(CLOCK_REALTIME, &subop->tv_begin);
-                subop->op_type = (uint64_t)cur_op;
-                subop->bs_op = new blockstore_op_t((blockstore_op_t){
-                    .opcode = (uint64_t)(wr ? (rep ? BS_OP_WRITE_STABLE : BS_OP_WRITE) : BS_OP_READ),
-                    .callback = [subop, this](blockstore_op_t *bs_subop)
-                    {
-                        handle_primary_bs_subop(subop);
-                    },
-                    { {
-                        .oid = (object_id){
-                            .inode = inode,
-                            .stripe = op_data->oid.stripe | stripe_num,
-                        },
-                        .version = op_version,
-                        .offset = wr ? si->write_start : si->read_start,
-                        .len = subop_len,
-                    } },
-                    .buf = wr ? si->write_buf : si->read_buf,
-                    .bitmap = si->bmp_buf,
-                });
-#ifdef OSD_DEBUG
-                printf(
-                    "Submit %s to local: %jx:%jx v%ju %u-%u\n", wr ? "write" : "read",
-                    inode, op_data->oid.stripe | stripe_num, op_version,
-                    subop->bs_op->offset, subop->bs_op->len
-                );
-#endif
-                bs->enqueue_op(subop->bs_op);
-            }
-            else
-            {
-                subop->op_type = OSD_OP_OUT;
-                subop->req.sec_rw = {
-                    .header = {
-                        .magic = SECONDARY_OSD_OP_MAGIC,
-                        .id = msgr.next_subop_id++,
-                        .opcode = (uint64_t)(wr ? (rep ? OSD_OP_SEC_WRITE_STABLE : OSD_OP_SEC_WRITE) : OSD_OP_SEC_READ),
-                    },
-                    .oid = {
-                        .inode = inode,
-                        .stripe = op_data->oid.stripe | stripe_num,
-                    },
-                    .version = op_version,
-                    .offset = wr ? si->write_start : si->read_start,
-                    .len = subop_len,
-                    .attr_len = wr ? clean_entry_bitmap_size : 0,
-                    .flags = cur_op->peer_fd == SELF_FD && cur_op->req.hdr.opcode != OSD_OP_SCRUB ? OSD_OP_RECOVERY_RELATED : 0,
-                };
-#ifdef OSD_DEBUG
-                printf(
-                    "Submit %s to osd %ju: %jx:%jx v%ju %u-%u\n", wr ? "write" : "read", role_osd_num,
-                    inode, op_data->oid.stripe | stripe_num, op_version,
-                    subop->req.sec_rw.offset, subop->req.sec_rw.len
-                );
-#endif
-                if (wr)
-                {
-                    if (si->write_end > si->write_start)
-                    {
-                        subop->iov.push_back(si->write_buf, si->write_end - si->write_start);
-                    }
-                }
-                else
-                {
-                    if (subop_len > 0)
-                    {
-                        subop->iov.push_back(si->read_buf, subop_len);
-                    }
-                }
-                subop->callback = [cur_op, this](osd_op_t *subop)
-                {
-                    handle_primary_subop(subop, cur_op);
-                };
-                auto peer_fd_it = msgr.osd_peer_fds.find(role_osd_num);
-                if (peer_fd_it != msgr.osd_peer_fds.end())
-                {
-                    subop->peer_fd = peer_fd_it->second;
-                    msgr.outbox_push(subop);
-                }
-                else
-                {
-                    // Fail it immediately
-                    subop->peer_fd = -1;
-                    subop->reply.hdr.retval = -EPIPE;
-                    ringloop->set_immediate([subop]() { std::function<void(osd_op_t*)>(subop->callback)(subop); });
-                }
-            }
+            si->role = stripe_num;
+            submit_primary_subop(cur_op, &op_data->subops[i], si, wr, inode, op_version);
            i++;
        }
        else
@@ -281,6 +181,112 @@ int osd_t::submit_primary_subop_batch(int submit_type, inode_t inode, uint64_t o
    return i-subop_idx;
 }

+void osd_t::submit_primary_subop(osd_op_t *cur_op, osd_op_t *subop,
+    osd_rmw_stripe_t *si, bool wr, inode_t inode, uint64_t op_version)
+{
+    uint32_t subop_len = wr
+        ? si->write_end - si->write_start
+        : si->read_end - si->read_start;
+    if (!wr && si->read_end == UINT32_MAX)
+    {
+        subop_len = 0;
+    }
+    si->read_error = false;
+    subop->bitmap = si->bmp_buf;
+    subop->bitmap_len = clean_entry_bitmap_size;
+    // Using rmw_buf to pass pointer to stripes. Dirty but works
+    subop->rmw_buf = si;
+    if (si->osd_num == this->osd_num)
+    {
+        clock_gettime(CLOCK_REALTIME, &subop->tv_begin);
+        subop->op_type = (uint64_t)cur_op; // also dirty
+        subop->bs_op = new blockstore_op_t((blockstore_op_t){
+            .opcode = (uint64_t)(wr ? (cur_op->op_data->pg->scheme == POOL_SCHEME_REPLICATED ? BS_OP_WRITE_STABLE : BS_OP_WRITE) : BS_OP_READ),
+            .callback = [subop, this](blockstore_op_t *bs_subop)
+            {
+                handle_primary_bs_subop(subop);
+            },
+            { {
+                .oid = (object_id){
+                    .inode = inode,
+                    .stripe = cur_op->op_data->oid.stripe | si->role,
+                },
+                .version = op_version,
+                .offset = wr ? si->write_start : si->read_start,
+                .len = subop_len,
+            } },
+            .buf = wr ? si->write_buf : si->read_buf,
+            .bitmap = si->bmp_buf,
+        });
+#ifdef OSD_DEBUG
+         printf(
+             "Submit %s to local: %jx:%jx v%ju %u-%u\n", wr ? "write" : "read",
+             inode, op_data->oid.stripe | si->role, op_version,
+             subop->bs_op->offset, subop->bs_op->len
+         );
+#endif
+        bs->enqueue_op(subop->bs_op);
+    }
+    else
+    {
+        subop->op_type = OSD_OP_OUT;
+        subop->req.sec_rw = (osd_op_sec_rw_t){
+            .header = {
+                .magic = SECONDARY_OSD_OP_MAGIC,
+                .id = msgr.next_subop_id++,
+                .opcode = (uint64_t)(wr ? (cur_op->op_data->pg->scheme == POOL_SCHEME_REPLICATED ? OSD_OP_SEC_WRITE_STABLE : OSD_OP_SEC_WRITE) : OSD_OP_SEC_READ),
+            },
+            .oid = {
+                .inode = inode,
+                .stripe = cur_op->op_data->oid.stripe | si->role,
+            },
+            .version = op_version,
+            .offset = wr ? si->write_start : si->read_start,
+            .len = subop_len,
+            .attr_len = wr ? clean_entry_bitmap_size : 0,
+            .flags = cur_op->peer_fd == SELF_FD && cur_op->req.hdr.opcode != OSD_OP_SCRUB ? OSD_OP_RECOVERY_RELATED : 0,
+        };
+#ifdef OSD_DEBUG
+        printf(
+            "Submit %s to osd %ju: %jx:%jx v%ju %u-%u\n", wr ? "write" : "read", si->osd_num,
+            inode, op_data->oid.stripe | si->role, op_version,
+            subop->req.sec_rw.offset, subop->req.sec_rw.len
+        );
+#endif
+        if (wr)
+        {
+            if (si->write_end > si->write_start)
+            {
+                subop->iov.push_back(si->write_buf, si->write_end - si->write_start);
+            }
+        }
+        else
+        {
+            if (subop_len > 0)
+            {
+                subop->iov.push_back(si->read_buf, subop_len);
+            }
+        }
+        subop->callback = [cur_op, this](osd_op_t *subop)
+        {
+            handle_primary_subop(subop, cur_op);
+        };
+        auto peer_fd_it = msgr.osd_peer_fds.find(si->osd_num);
+        if (peer_fd_it != msgr.osd_peer_fds.end())
+        {
+            subop->peer_fd = peer_fd_it->second;
+            msgr.outbox_push(subop);
+        }
+        else
+        {
+            // Fail it immediately
+            subop->peer_fd = -1;
+            subop->reply.hdr.retval = -EPIPE;
+            ringloop->set_immediate([subop]() { std::function<void(osd_op_t*)>(subop->callback)(subop); });
+        }
+    }
+}
+
 static uint64_t bs_op_to_osd_op[] = {
    0,
    OSD_OP_SEC_READ,            // BS_OP_READ = 1
@@ -401,7 +407,7 @@ void osd_t::handle_primary_subop(osd_op_t *subop, osd_op_t *cur_op)
        printf("subop %s %jx:%jx from osd %jd: version = %ju\n", osd_op_names[opcode],
            subop->req.sec_rw.oid.inode, subop->req.sec_rw.oid.stripe, peer_osd, version);
 #endif
-        if (op_data->fact_ver != UINT64_MAX)
+        if (version != 0 && op_data->fact_ver != UINT64_MAX)
        {
            if (op_data->fact_ver != 0 && op_data->fact_ver != version)
            {
@@ -526,7 +532,7 @@ bool contains_osd(osd_num_t *osd_set, uint64_t size, osd_num_t osd_num)
 void osd_t::submit_primary_del_subops(osd_op_t *cur_op, osd_num_t *cur_set, uint64_t set_size, pg_osd_set_t & loc_set)
 {
    osd_primary_op_data_t *op_data = cur_op->op_data;
-    bool rep = op_data->scheme == POOL_SCHEME_REPLICATED;
+    bool rep = op_data->pg->scheme == POOL_SCHEME_REPLICATED;
    obj_ver_osd_t extra_chunks[loc_set.size()];
    int chunks_to_del = 0;
    for (auto & chunk: loc_set)
@@ -738,10 +744,10 @@ void osd_t::submit_primary_rollback_subops(osd_op_t *cur_op, const uint64_t* osd
 {
    osd_primary_op_data_t *op_data = cur_op->op_data;
    osd_rmw_stripe_t *stripes = op_data->stripes;
-    assert(op_data->scheme != POOL_SCHEME_REPLICATED);
+    assert(op_data->pg->scheme != POOL_SCHEME_REPLICATED);
    // Allocate subops
    int n_subops = 0;
-    for (int role = 0; role < op_data->pg_size; role++)
+    for (int role = 0; role < op_data->pg->pg_size; role++)
    {
        if (osd_set[role] != 0 && !stripes[role].read_error &&
            (osd_set[role] == this->osd_num || msgr.osd_peer_fds.find(osd_set[role]) != msgr.osd_peer_fds.end()))
@@ -758,7 +764,7 @@ void osd_t::submit_primary_rollback_subops(osd_op_t *cur_op, const uint64_t* osd
    op_data->subops = new osd_op_t[n_subops];
    op_data->unstable_writes = new obj_ver_id[n_subops];
    int i = 0;
-    for (int role = 0; role < op_data->pg_size; role++)
+    for (int role = 0; role < op_data->pg->pg_size; role++)
    {
        if (osd_set[role] != 0 && !stripes[role].read_error &&
            (osd_set[role] == this->osd_num || msgr.osd_peer_fds.find(osd_set[role]) != msgr.osd_peer_fds.end()))
@@ -44,7 +44,7 @@ void osd_t::continue_primary_write(osd_op_t *cur_op)
        return;
    }
    osd_primary_op_data_t *op_data = cur_op->op_data;
-    auto & pg = pgs.at({ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num });
+    auto & pg = *cur_op->op_data->pg;
    if (op_data->st == 1)      goto resume_1;
    else if (op_data->st == 2) goto resume_2;
    else if (op_data->st == 3) goto resume_3;
@@ -73,7 +73,7 @@ resume_1:
        op_data->object_state->ref_count++;
    }
 retry_1:
-    if (op_data->scheme == POOL_SCHEME_REPLICATED)
+    if (pg.scheme == POOL_SCHEME_REPLICATED)
    {
        // Simplified algorithm
        op_data->stripes[0].write_start = op_data->stripes[0].req_start;
@@ -99,7 +99,7 @@ retry_1:
    {
        assert(!cur_op->rmw_buf);
        cur_op->rmw_buf = calc_rmw(cur_op->buf, op_data->stripes, op_data->prev_set,
-            pg.pg_size, op_data->pg_data_size, pg.pg_cursize, pg.cur_set.data(), bs_block_size, clean_entry_bitmap_size);
+            pg.pg_size, pg.pg_data_size, pg.pg_cursize, pg.cur_set.data(), bs_block_size, clean_entry_bitmap_size);
        if (!cur_op->rmw_buf)
        {
            // Refuse partial overwrite of an incomplete object
@@ -114,7 +114,7 @@ retry_1:
            // Allow to read version number (just version number!) from corrupted chunks
            // to allow full overwrite of a corrupted object
            bool found = false;
-            for (int role = 0; role < op_data->pg_size; role++)
+            for (int role = 0; role < pg.pg_size; role++)
            {
                if (op_data->prev_set[role] != 0 || op_data->stripes[role].read_end > op_data->stripes[role].read_start)
                {
@@ -124,8 +124,8 @@ retry_1:
            }
            if (!found)
            {
-                osd_num_t corrupted_target[op_data->pg_size];
-                for (int role = 0; role < op_data->pg_size; role++)
+                osd_num_t corrupted_target[pg.pg_size];
+                for (int role = 0; role < pg.pg_size; role++)
                {
                    corrupted_target[role] = 0;
                }
@@ -151,7 +151,7 @@ resume_3:
        if (op_data->errcode == -EIO || op_data->errcode == -EDOM)
        {
            // Mark object corrupted and retry
-            op_data->object_state = mark_object_corrupted(pg, op_data->oid, op_data->object_state, op_data->stripes, true, false);
+            op_data->object_state = mark_object_corrupted(pg, op_data->oid, op_data->object_state, op_data->stripes, true);
            op_data->prev_set = op_data->object_state ? op_data->object_state->read_target.data() : pg.cur_set.data();
            if (cur_op->rmw_buf)
            {
@@ -165,6 +165,7 @@ resume_3:
        return;
    }
    // Check CAS version
+    // FIXME: Handle CAS writes as "immediate" in non-immediate_commit pools, otherwise CAS doesn't make sense
    if (cur_op->req.rw.version && op_data->fact_ver != (cur_op->req.rw.version-1))
    {
        deref_object_state(pg, &op_data->object_state, true);
@@ -172,7 +173,7 @@ resume_3:
        cur_op->reply.rw.version = op_data->fact_ver;
        goto continue_others;
    }
-    if (op_data->scheme == POOL_SCHEME_REPLICATED)
+    if (pg.scheme == POOL_SCHEME_REPLICATED)
    {
        // Set bitmap bits
        bitmap_set(op_data->stripes[0].bmp_buf, op_data->stripes[0].write_start,
@@ -203,7 +204,7 @@ resume_3:
        }
        else if (pg.scheme == POOL_SCHEME_EC)
        {
-            calc_rmw_parity_ec(op_data->stripes, pg.pg_size, op_data->pg_data_size, op_data->prev_set, pg.cur_set.data(), bs_block_size, clean_entry_bitmap_size);
+            calc_rmw_parity_ec(op_data->stripes, pg.pg_size, pg.pg_data_size, op_data->prev_set, pg.cur_set.data(), bs_block_size, clean_entry_bitmap_size);
        }
    }
    // Send writes
@@ -269,7 +270,7 @@ resume_5:
        // and rollback successful part updates in case of EC.
        if (op_data->done > 0 && !op_data->drops)
        {
-            if (op_data->scheme != POOL_SCHEME_REPLICATED)
+            if (pg.scheme != POOL_SCHEME_REPLICATED)
            {
                submit_primary_rollback_subops(cur_op, pg.cur_set.data());
 resume_11:
@@ -293,7 +294,7 @@ resume_12:
        pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->errcode);
        return;
    }
-    if (op_data->scheme != POOL_SCHEME_REPLICATED)
+    if (pg.scheme != POOL_SCHEME_REPLICATED)
    {
        // Remove version override just after the write, but before stabilizing
        pg.ver_override.erase(op_data->oid);
@@ -329,7 +330,7 @@ resume_7:
                memset(&recovery_stat[recovery_type], 0, sizeof(recovery_stat[recovery_type]));
                recovery_stat[recovery_type].count++;
            }
-            for (int role = 0; role < (op_data->scheme == POOL_SCHEME_REPLICATED ? 1 : pg.pg_size); role++)
+            for (int role = 0; role < (pg.scheme == POOL_SCHEME_REPLICATED ? 1 : pg.pg_size); role++)
            {
                recovery_stat[recovery_type].bytes += op_data->stripes[role].write_end - op_data->stripes[role].write_start;
            }
@@ -353,7 +354,7 @@ resume_7:
            for (auto & chunk: op_data->object_state->osd_set)
            {
                // Check is the same as in submit_primary_del_subops()
-                if (op_data->scheme == POOL_SCHEME_REPLICATED
+                if (pg.scheme == POOL_SCHEME_REPLICATED
                    ? !contains_osd(pg.cur_set.data(), pg.pg_size, chunk.osd_num)
                    : (chunk.osd_num != pg.cur_set[chunk.role]))
                {
@@ -361,7 +362,7 @@ resume_7:
                        .osd_num = chunk.osd_num,
                        .oid = {
                            .inode = op_data->oid.inode,
-                            .stripe = op_data->oid.stripe | (op_data->scheme == POOL_SCHEME_REPLICATED ? 0 : chunk.role),
+                            .stripe = op_data->oid.stripe | (pg.scheme == POOL_SCHEME_REPLICATED ? 0 : chunk.role),
                        },
                        .version = op_data->fact_ver,
                    });
@@ -472,7 +473,7 @@ bool osd_t::remember_unstable_write(osd_op_t *cur_op, pg_t & pg, pg_osd_set_t &
    if (immediate_commit == IMMEDIATE_ALL)
    {
 immediate:
-        if (op_data->scheme != POOL_SCHEME_REPLICATED)
+        if (pg.scheme != POOL_SCHEME_REPLICATED)
        {
            // Send STABILIZE ops immediately
            op_data->unstable_write_osds = new std::vector<unstable_osd_num_t>();
@@ -515,7 +516,7 @@ resume_7:
    }
    else if (immediate_commit == IMMEDIATE_SMALL)
    {
-        int stripe_count = (op_data->scheme == POOL_SCHEME_REPLICATED ? 1 : op_data->pg_size);
+        int stripe_count = (pg.scheme == POOL_SCHEME_REPLICATED ? 1 : pg.pg_size);
        for (int role = 0; role < stripe_count; role++)
        {
            if (op_data->stripes[role].write_start == 0 &&
@@ -531,7 +532,7 @@ resume_7:
    {
 lazy:
        unstable_write_count++;
-        if (op_data->scheme != POOL_SCHEME_REPLICATED)
+        if (pg.scheme != POOL_SCHEME_REPLICATED)
        {
            // Remember version as unstable for EC/XOR
            for (auto & chunk: loc_set)
@@ -1118,139 +1118,230 @@ static bool next_combination(int *subset, int k, int n)
    return true;
 }

-static int c_n_k(int n, int k)
+static uint64_t c_n_k(uint64_t n, uint64_t k)
 {
-    int c = 1;
-    for (int i = n; i > k; i--)
+    uint64_t c = 1;
+    for (uint64_t i = n; i > k; i--)
+    {
+        if ((c*i) < i)
+            return UINT64_MAX;
        c *= i;
-    for (int i = 2; i <= (n-k); i++)
+    }
+    for (uint64_t i = 2; i <= (n-k); i++)
        c /= i;
    return c;
 }

-std::vector<int> ec_find_good(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsize, bool is_xor,
-    uint32_t chunk_size, uint32_t bitmap_size, int max_bruteforce)
+static std::vector<int> ec_check_combination(osd_rmw_stripe_t *stripes, int stripe_count,
+    int *subset, int pg_size, int pg_minsize, bool is_xor,
+    uint32_t chunk_size, uint32_t bitmap_size, uint8_t *tmp_buf)
+{
+    osd_num_t fake_osd_set[pg_size];
+    for (int i = 0; i < pg_size; i++)
+    {
+        fake_osd_set[i] = i+1;
+    }
+    osd_rmw_stripe_t brute_stripes[pg_size];
+    memset(brute_stripes, 0, sizeof(osd_rmw_stripe_t)*pg_size);
+    for (int i = 0; i < pg_size; i++)
+    {
+        auto & bs = brute_stripes[i];
+        bs.req_end = bs.read_end = chunk_size;
+    }
+    for (int i = 0; i < pg_minsize; i++)
+    {
+        auto & src = stripes[subset[i]];
+        auto & bs = brute_stripes[src.role];
+        bs.bmp_buf = src.bmp_buf;
+        bs.write_buf = bs.read_buf = src.read_buf;
+    }
+    for (int i = 0; i < pg_size; i++)
+    {
+        auto & bs = brute_stripes[i];
+        if (!bs.read_buf)
+        {
+            // missing chunks are recovered in read_bufs and write_bufs are used as source for parity
+            bs.missing = true;
+            bs.read_buf = bs.write_buf = tmp_buf+i*chunk_size;
+            bs.bmp_buf = tmp_buf + stripe_count*chunk_size + i*bitmap_size;
+        }
+        else if (i >= pg_minsize)
+        {
+            // parity chunks are regenerated in their write_bufs, so use a temporary buffer
+            bs.write_buf = tmp_buf+i*chunk_size;
+        }
+    }
+    if (is_xor)
+    {
+        assert(pg_size == pg_minsize+1);
+        reconstruct_stripes_xor(brute_stripes, pg_size, bitmap_size);
+    }
+    else
+    {
+        reconstruct_stripes_ec(brute_stripes, pg_size, pg_minsize, bitmap_size);
+        calc_rmw_parity_ec(brute_stripes, pg_size, pg_minsize, fake_osd_set, fake_osd_set, chunk_size, bitmap_size);
+    }
+    bool matched_other = false;
+    std::vector<int> good_set;
+    for (int i = 0; i < stripe_count; i++)
+    {
+        if (stripes[i].read_error || stripes[i].not_exists)
+        {
+            continue;
+        }
+        auto & bs = brute_stripes[stripes[i].role];
+        if (!bs.missing && bs.read_buf == stripes[i].read_buf)
+        {
+            // source chunk, mark OK
+            good_set.push_back(i);
+        }
+        else if (memcmp(stripes[i].role < pg_minsize ? bs.read_buf : bs.write_buf, stripes[i].read_buf, chunk_size) == 0)
+        {
+            // matching chunk, mark OK
+            good_set.push_back(i);
+            matched_other = true;
+        }
+    }
+    if (!matched_other)
+    {
+        good_set.clear();
+    }
+    return good_set;
+}
+
+static int count_roles(osd_rmw_stripe_t *stripes, std::vector<int> & valid_chunks, int pg_size)
+{
+    bool role_ok[pg_size];
+    for (int i = 0; i < pg_size; i++)
+    {
+        role_ok[i] = false;
+    }
+    for (int idx: valid_chunks)
+    {
+        role_ok[stripes[idx].role] = true;
+    }
+    int ok_count = 0;
+    for (int i = 0; i < pg_size; i++)
+    {
+        if (role_ok[i])
+            ok_count++;
+    }
+    return ok_count;
+}
+
+std::vector<int> ec_find_good(osd_rmw_stripe_t *stripes, int stripe_count, int pg_size, int pg_minsize, bool is_xor,
+    uint32_t chunk_size, uint32_t bitmap_size, uint64_t max_bruteforce, bool find_best)
 {
    std::vector<int> found_valid;
-    int cur_live[pg_size], live_count = 0, exists_count = 0;
-    osd_num_t fake_osd_set[pg_size];
-    for (int role = 0; role < pg_size; role++)
+    std::vector<std::vector<int>> live_variants(pg_size);
+    int eq_to[stripe_count];
+    int live_roles = 0, live_total = 0;
+    for (int i = 0; i < pg_size; i++)
    {
-        if (!stripes[role].missing)
-        {
-            if (!stripes[role].not_exists)
-                exists_count++;
-            cur_live[live_count++] = role;
-            fake_osd_set[role] = role+1;
-        }
+        eq_to[i] = i;
    }
-    if (live_count <= pg_minsize)
+    for (int i = 0; i < stripe_count; i++)
    {
-        return std::vector<int>();
-    }
-    if (exists_count <= pg_minsize)
-    {
-        // Special case: user manually deleted some chunks
-        for (int role = 0; role < pg_size; role++)
-            if (!stripes[role].missing && !stripes[role].not_exists)
-                found_valid.push_back(role);
-        return found_valid;
-    }
-    // Try to locate errors using brute force if there isn't too many combinations
-    osd_rmw_stripe_t brute_stripes[pg_size];
-    int out_count = live_count-pg_minsize;
-    bool brute_force = out_count > 1 && c_n_k(live_count-1, out_count-1) <= max_bruteforce;
-    int subset[pg_minsize], outset[out_count];
-    // Select all combinations with items except the last one (== anything to compare)
-    first_combination(subset, pg_minsize, live_count-1);
-    uint8_t *tmp_buf = (uint8_t*)malloc_or_die(pg_size*chunk_size);
-    do
-    {
-        memcpy(brute_stripes, stripes, sizeof(osd_rmw_stripe_t)*pg_size);
-        int i = 0, j = 0, k = 0;
-        for (; i < pg_minsize; i++, j++)
-            while (j < subset[i])
-                outset[k++] = j++;
-        while (j < pg_size)
-            outset[k++] = j++;
-        for (int i = 0; i < out_count; i++)
+        if (!stripes[i].read_error && !stripes[i].not_exists)
        {
-            brute_stripes[cur_live[outset[i]]].missing = true;
-            brute_stripes[cur_live[outset[i]]].read_buf = tmp_buf+cur_live[outset[i]]*chunk_size;
-        }
-        for (int i = 0; i < pg_minsize; i++)
-        {
-            brute_stripes[i].write_buf = brute_stripes[i].read_buf;
-            brute_stripes[i].req_start = 0;
-            brute_stripes[i].req_end = chunk_size;
-        }
-        for (int i = pg_minsize; i < pg_size; i++)
-        {
-            brute_stripes[i].write_buf = tmp_buf+i*chunk_size;
-        }
-        if (is_xor)
-        {
-            assert(pg_size == pg_minsize+1);
-            reconstruct_stripes_xor(brute_stripes, pg_size, bitmap_size);
-        }
-        else
-        {
-            reconstruct_stripes_ec(brute_stripes, pg_size, pg_minsize, bitmap_size);
-            calc_rmw_parity_ec(brute_stripes, pg_size, pg_minsize, fake_osd_set, fake_osd_set, chunk_size, bitmap_size);
-        }
-        for (int i = pg_minsize; i < pg_size; i++)
-        {
-            brute_stripes[i].read_buf = brute_stripes[i].write_buf;
-        }
-        int valid_count = 0;
-        for (int i = 0; i < out_count; i++)
-        {
-            if (memcmp(brute_stripes[cur_live[outset[i]]].read_buf,
-                    stripes[cur_live[outset[i]]].read_buf, chunk_size) == 0)
+            if (live_variants[stripes[i].role].size() > 0)
            {
-                brute_stripes[cur_live[outset[i]]].missing = false;
-                valid_count++;
-            }
-        }
-        if (valid_count > 0)
-        {
-            if (found_valid.size())
-            {
-                // Check if we found the same set from the different point of view,
-                // like 1 2 3 -> valid 4 5 and 1 3 4 -> valid 2 5
-                for (int i = 0, j = 0; i < pg_size; i++)
+                for (int j = 0; j < i; j++)
                {
-                    if (!brute_stripes[i].missing)
+                    if (stripes[j].role == stripes[i].role &&
+                        memcmp(stripes[i].read_buf, stripes[j].read_buf, chunk_size) == 0)
                    {
-                        if (j >= found_valid.size() || found_valid[j] != i)
-                        {
-                            // Ambiguity: we found multiple valid sets and don't know which one is correct
-                            found_valid.clear();
-                            break;
-                        }
-                        j++;
+                        eq_to[i] = eq_to[j];
+                        break;
                    }
                }
-                if (!found_valid.size())
-                {
-                    break;
-                }
            }
            else
            {
-                for (int i = 0; i < pg_size; i++)
+                live_roles++;
+            }
+            if (eq_to[i] == i)
+            {
+                live_variants[stripes[i].role].push_back(i);
+                live_total++;
+            }
+        }
+    }
+    if (live_roles == pg_minsize && live_total > pg_minsize)
+    {
+        // Nothing to validate and there are chunks with different data => object is inconsistent
+        return std::vector<int>();
+    }
+    if (live_roles <= pg_minsize)
+    {
+        // Nothing to validate, just return all live chunks
+        for (int i = 0; i < stripe_count; i++)
+            if (!stripes[i].read_error)
+                found_valid.push_back(i);
+        return found_valid;
+    }
+    // Try to locate errors using brute force if there isn't too many combinations
+    bool brute_force = c_n_k(live_roles, pg_minsize) <= max_bruteforce;
+    int combination[pg_minsize], subset[pg_minsize], subvar[pg_minsize];
+    // To translate 0..live_roles into 0..pg_size
+    int comb_to_subset[live_roles];
+    for (int i = 0, r = 0; i < pg_size; i++)
+    {
+        if (live_variants[i].size() > 0)
+            comb_to_subset[r++] = i;
+    }
+    // Select all combinations with items except the last one (== anything to compare)
+    first_combination(combination, pg_minsize, live_roles);
+    uint8_t *tmp_buf = (uint8_t*)malloc_or_die(stripe_count*(chunk_size+bitmap_size));
+    do
+    {
+        // Then loop over all subvariants (if some roles have multiple diverged variants of data)
+        for (int i = 0; i < pg_minsize; i++)
+        {
+            subvar[i] = 0;
+        }
+        while (true)
+        {
+            // Transform combination[] + subvar[] into subset[]
+            for (int i = 0; i < pg_minsize; i++)
+            {
+                subset[i] = live_variants[comb_to_subset[combination[i]]][subvar[i]];
+            }
+            // Check the combination
+            auto valid_chunks = ec_check_combination(stripes, stripe_count, subset, pg_size, pg_minsize, is_xor, chunk_size, bitmap_size, tmp_buf);
+            // The same set may be found from different points of view,
+            // like 1 2 3 -> valid 4 5 and 1 3 4 -> valid 2 5
+            if (valid_chunks.size() > 0)
+            {
+                if (found_valid.size() >= valid_chunks.size() && found_valid != valid_chunks)
                {
-                    if (!brute_stripes[i].missing)
-                    {
-                        found_valid.push_back(i);
-                    }
+                    // Ambiguity: we found multiple valid sets and don't know which one is correct
+                    printf("Scrub found 2 different correct chunk subsets: OSD ");
+                    for (int i = 0; i < found_valid.size(); i++)
+                        printf(i > 0 ? ", %ju" : "%ju", stripes[found_valid[i]].osd_num);
+                    printf(" and OSD ");
+                    for (int i = 0; i < valid_chunks.size(); i++)
+                        printf(i > 0 ? ", %ju" : "%ju", stripes[valid_chunks[i]].osd_num);
+                    printf("\n");
+                    found_valid.clear();
+                    goto out;
+                }
+                else if (!found_valid.size() && (find_best || count_roles(stripes, valid_chunks, pg_size) >= pg_size))
+                {
+                    found_valid = valid_chunks;
                }
            }
-            if (valid_count == out_count)
+            // Select next subvariant
+            int i = 0;
+            for (i = 0; i < pg_minsize; i++)
            {
-                // All chunks are good
-                break;
+                subvar[i]++;
+                if (subvar[i] < live_variants[combination[i]].size())
+                    break;
+                subvar[i] = 0;
            }
+            if (i >= pg_minsize)
+                break;
        }
        if (!brute_force)
        {
@@ -1258,7 +1349,8 @@ std::vector<int> ec_find_good(osd_rmw_stripe_t *stripes, int pg_size, int pg_min
            // if we find it we won't be able to check that it's the only good one
            break;
        }
-    } while (out_count > 1 && next_combination(subset, pg_minsize, live_count-1));
+    } while (next_combination(combination, pg_minsize, live_roles));
+out:
    free(tmp_buf);
    return found_valid;
 }
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Vitaliy Filippov	afcab324e1	Destroy qemu driver under lock (may fix crashes after attach/detach?)	2025-03-07 13:13:27 +03:00
Vitaliy Filippov	819f1125ae	Support used_for_app instead of used_for_fs	2025-03-07 01:03:43 +03:00
Vitaliy Filippov	108df7329f	Fix PG object count statistics on deletion of non-existing objects	2025-03-04 00:40:56 +03:00
Vitaliy Filippov	d32edf6cdf	Fix deletion writeback	2025-03-04 00:40:35 +03:00
Vitaliy Filippov	dca436d7e6	Trigger event loop automatically in libvitastor_c	2025-03-03 00:57:09 +03:00
Vitaliy Filippov	8129a0b4e3	Loop once after registering eventfd to prevent skipping previous events	2025-03-03 00:57:00 +03:00
Vitaliy Filippov	704c87d512	Trigger initial epoll when adding an FD	2025-03-03 00:56:17 +03:00
Vitaliy Filippov	10216a5fb5	Build node.js addon as a Debian package	2025-03-02 18:04:56 +03:00
Vitaliy Filippov	3932eb7ff6	Trigger event loop once after each vitastor_c_* call	2025-03-02 01:23:41 +03:00
Vitaliy Filippov	69cbe7bbb2	Release 1.11.0 New features: - Support containerized Vitastor installations: http://vitastor.io/docs/installation/docker.html - Add new functions to the node.js binding: delete(), get_immediate_commit(), on_ready(), get_min_io_size(), get_max_atomic_write_size() - S3 (Zenko Cloudserver with Vitastor support) is coming shortly and will be released separately Bug fixes: - Use IP-derived etcd node names in make-etcd - Set short name of the OSD process to display in `top` - Fix snap-create without pool_id failing when there are multiple pools - Several bugs are fixed in the write-back cache, it should now be stable: - Fix incorrect snapshot reads from dirty write-back cache - Do not try to repeat pending writebacks on OSD reconnections - Fix client hangs with multiple SYNCs in the writeback queue - Fix client hangs do to incorrect calculation of the writeback queue size - Several improvements for NBD mapping/unmapping: - Add a workaround for race condition in the Linux kernel NBD driver leading to vitastor-nbd sometimes breaking a previously mapped device instead of setting up a new one - Check if the device is actually mapped in vitastor-nbd unmap - Fix device name/number validation in vitastor-nbd - Fix OSD crashes after starting with corrupted metadata - from now it will skip corrupted metadata entries and heal itself - Fix scrubbing of misplaced objects and object state recalculation after vitastor-cli fix - previously, an OSD restart could be required to fix object states - Make primary OSD distribution more stable by using murmur3 hash instead of the old pseudo-rng - Fix monitor sometimes racing with itself - do not touch /pool/stats from stats aggregation if PG recheck is active - Sort vitastor-cli ls output by name by default - Update antietcd to 1.1.2	2025-03-01 13:39:42 +03:00
Vitaliy Filippov	4950a1636c	Allow "infinite" startup for clients if explicitly requested	2025-03-01 13:39:42 +03:00
Vitaliy Filippov	2eb20dff28	Do not crash on io_uring initialization failure in node-vitastor	2025-03-01 13:29:48 +03:00
Vitaliy Filippov	59f0b0427c	Support containerized Vitastor installations	2025-02-27 20:06:15 +03:00
Vitaliy Filippov	124162ad38	Use IP-derived etcd node names in make-etcd	2025-02-26 11:54:37 +03:00
Vitaliy Filippov	391c92af1a	Set OSD process name	2025-02-26 11:54:37 +03:00
Vitaliy Filippov	c3d8fdd855	Fix snap-create without pool_id ID generation with multiple pools	2025-02-26 11:54:28 +03:00
Vitaliy Filippov	9ccf3af97b	Add qemu-block-extra and qemu-utils	2025-02-23 15:08:16 +03:00
Vitaliy Filippov	568a209f0d	Update docker image to debian bookworm	2025-02-23 13:27:32 +03:00
Vitaliy Filippov	b151013201	Fix snapshot reads from a dirty write-back cache	2025-02-23 02:31:19 +03:00
Vitaliy Filippov	4a763725fe	Add free() to bindiff.c	2025-02-22 16:52:19 +03:00
Vitaliy Filippov	b8d83cd7f4	No, it's not a good idea to destroy client in the child nbd process Should probably have been an obvious side effect :-) Child process gets open file descriptors to parent's epoll/timerfd, and it's totally OK to just close() all of them, but it's absolutely NOT OK to run destructors - they modify the kernel state of epoll/timerfd before destroying. So, basically, when we destroy the client in the child process, we break it in the parent too. This also means that cluster_client_t doesn't support fork(). :-)	2025-02-22 15:10:27 +03:00
Vitaliy Filippov	2e9ee2fe20	Do not try to repeat pending writebacks	2025-02-22 14:16:44 +03:00
Vitaliy Filippov	508ae852e4	Fix trap in test_rebalance_verify	2025-02-22 02:18:41 +03:00
Vitaliy Filippov	97ee400505	Add a workaround for race condition in the Linux kernel NBD driver Do all NBD configuration in the child process, after the last fork. Why? It's needed because there is a race condition in the Linux kernel nbd driver in nbd_add_socket() - it saves `current` task pointer as `nbd->task_setup` and then rechecks if the new `current` is the same. Problem is that if that process is already dead, `current` may be freed and then replaced by another process with the same pointer value. So the check passes and NBD allows a different process to set up a device which is already set up. Proper fix would have to be done in the kernel code, but the workaround is obviously to perform NBD setup from the process which will then actually call NBD_DO_IT. That process stays alive during the whole time of NBD device execution and the (nbd->task_setup != current) check always works correctly, and we don't accidentally break previous NBD devices while setting up a new device. Forking to check every device is of course rather slow, so we also do an additional check by calling list_mapped() before searching for a free NBD device.	2025-02-21 13:17:37 +03:00
Vitaliy Filippov	5ee4894fab	Check if mapped in vitastor-nbd unmap	2025-02-21 01:28:06 +03:00
Vitaliy Filippov	125dcafb11	Prevent OSD crashes when metadata is corrupted	2025-02-20 02:19:32 +03:00
Vitaliy Filippov	9f44cf71df	Fix device name/number validation in vitastor-nbd	2025-02-20 01:33:11 +03:00
Vitaliy Filippov	df3c63ca7f	Sort vitastor-cli ls by name by default	2025-02-20 01:32:49 +03:00
Vitaliy Filippov	be66edd09f	Prevent infinite loops on syncs in writeback_overflow	2025-02-19 01:44:12 +03:00
Vitaliy Filippov	ccbc0c5928	Add assert !writeback_bytes	2025-02-19 01:15:46 +03:00
Vitaliy Filippov	78ca4538bf	Fix qemu docker build for ubuntu	2025-02-18 23:44:16 +03:00
Vitaliy Filippov	86b5760ec1	Fix writeback incorrectly calculating queue size which was leading to client hangs	2025-02-18 23:42:55 +03:00
Vitaliy Filippov	27f3803d2f	Add vitastor_c_delete() and delete() to the node.js binding	2025-02-15 18:27:17 +03:00
Vitaliy Filippov	2ead06e126	Add ubuntu jammy to docs	2025-02-12 15:32:35 +03:00
Vitaliy Filippov	a5d5559f8e	Add get_immediate_commit() to the node.js binding	2025-02-06 01:35:48 +03:00
Vitaliy Filippov	e8e7ba8fde	Add FIXME for CAS in non-immediate_commit mode	2025-02-06 01:35:48 +03:00
Vitaliy Filippov	6fd831a299	Add on_ready(), get_min_io_size(), get_max_atomic_write_size() to the node.js binding	2025-02-06 01:35:48 +03:00
Vitaliy Filippov	069808dfce	Fix --config_path option in docs	2025-01-24 17:21:11 +03:00
Vitaliy Filippov	bcefa42bc0	Scrub all chunks, not just 1 chunk per position	2025-01-23 02:02:55 +03:00
Vitaliy Filippov	4636e02d43	Remove scheme, pg_size, pg_data_size from op_data	2025-01-23 01:20:31 +03:00
Vitaliy Filippov	e4c7d1c147	s/3/4/	2025-01-23 01:20:31 +03:00
Vitaliy Filippov	a4677f3e69	Mention P5530	2025-01-23 01:20:31 +03:00
Vitaliy Filippov	7cbf207d65	Use murmur3 to select primary OSD instead of old pseudo-rng	2025-01-18 12:28:54 +03:00
Vitaliy Filippov	7c9711af20	Do not touch /pool/stats from stats aggregation if PG recheck is active	2025-01-16 20:41:16 +03:00
Vitaliy Filippov	33ef701464	Update antietcd to 1.1.2	2025-01-04 02:13:36 +03:00
				`@@ -0,0 +1 @@`
				`deb http://vitastor.io/debian bookworm main`
				`@@ -1 +0,0 @@`
				`deb http://vitastor.io/debian bullseye main`