Trigger event loop more intelligently and automatically in libvitastor_c

New features:

- Support containerized Vitastor installations: http://vitastor.io/docs/installation/docker.html
- Add new functions to the node.js binding: delete(), get_immediate_commit(), on_ready(),
  get_min_io_size(), get_max_atomic_write_size()
- S3 (Zenko Cloudserver with Vitastor support) is coming shortly and will be released separately

Bug fixes:

- Use IP-derived etcd node names in make-etcd
- Set short name of the OSD process to display in `top`
- Fix snap-create without pool_id failing when there are multiple pools
- Several bugs are fixed in the write-back cache, it should now be stable:
  - Fix incorrect snapshot reads from dirty write-back cache
  - Do not try to repeat pending writebacks on OSD reconnections
  - Fix client hangs with multiple SYNCs in the writeback queue
  - Fix client hangs do to incorrect calculation of the writeback queue size
- Several improvements for NBD mapping/unmapping:
  - Add a workaround for race condition in the Linux kernel NBD driver leading
    to vitastor-nbd sometimes breaking a previously mapped device instead of
    setting up a new one
  - Check if the device is actually mapped in vitastor-nbd unmap
  - Fix device name/number validation in vitastor-nbd
- Fix OSD crashes after starting with corrupted metadata - from now it will skip
  corrupted metadata entries and heal itself
- Fix scrubbing of misplaced objects and object state recalculation after
  vitastor-cli fix - previously, an OSD restart could be required to fix object states
- Make primary OSD distribution more stable by using murmur3 hash instead of the old pseudo-rng
- Fix monitor sometimes racing with itself - do not touch /pool/stats from stats
  aggregation if PG recheck is active
- Sort vitastor-cli ls output by name by default
- Update antietcd to 1.1.2

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "1.10.1")
+set(VITASTOR_VERSION "1.11.0")
 
 add_subdirectory(src)

README-ru.md

@@ -41,6 +41,7 @@ Vitastor поддерживает QEMU-драйвер, протоколы NBD и
   - [Автор и лицензия](docs/intro/author.ru.md)
 - Установка
   - [Пакеты](docs/installation/packages.ru.md)
+  - [Docker](docs/installation/docker.ru.md)
   - [Proxmox](docs/installation/proxmox.ru.md)
   - [OpenNebula](docs/installation/opennebula.ru.md)
   - [OpenStack](docs/installation/openstack.ru.md)

README.md

@@ -41,6 +41,7 @@ Read more details in the documentation. You can start from here: [Quick Start](d
   - [Author and license](docs/intro/author.en.md)
 - Installation
   - [Packages](docs/installation/packages.en.md)
+  - [Docker](docs/installation/docker.en.md)
   - [Proxmox](docs/installation/proxmox.en.md)
   - [OpenNebula](docs/installation/opennebula.en.md)
   - [OpenStack](docs/installation/openstack.en.md)

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v1.10.1
+VITASTOR_VERSION ?= v1.11.0
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v1.10.1
+          image: vitalif/vitastor-csi:v1.11.0
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v1.10.1
+          image: vitalif/vitastor-csi:v1.11.0
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "1.10.1"
+    vitastorCSIDriverVersion = "1.11.0"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (1.10.1-1) unstable; urgency=medium
+vitastor (1.11.0-1) unstable; urgency=medium
 
   * Bugfixes
 

debian/control

@@ -2,7 +2,10 @@ Source: vitastor
 Section: admin
 Priority: optional
 Maintainer: Vitaliy Filippov <vitalif@yourcmc.ru>
-Build-Depends: debhelper, liburing-dev (>= 0.6), g++ (>= 8), libstdc++6 (>= 8), linux-libc-dev, libgoogle-perftools-dev, libjerasure-dev, libgf-complete-dev, libibverbs-dev, libisal-dev, cmake, pkg-config, libnl-3-dev, libnl-genl-3-dev
+Build-Depends: debhelper, liburing-dev (>= 0.6), g++ (>= 8), libstdc++6 (>= 8),
+  linux-libc-dev, libgoogle-perftools-dev, libjerasure-dev, libgf-complete-dev,
+  libibverbs-dev, libisal-dev, cmake, pkg-config, libnl-3-dev, libnl-genl-3-dev,
+  node-bindings <!nocheck>, node-gyp, node-nan
 Standards-Version: 4.5.0
 Homepage: https://vitastor.io/
 Rules-Requires-Root: no
@@ -59,3 +62,9 @@ Architecture: amd64
 Depends: ${shlibs:Depends}, ${misc:Depends}, vitastor-client, patch, python3, jq
 Description: Vitastor OpenNebula storage plugin
  Vitastor storage plugin for OpenNebula.
+
+Package: node-vitastor
+Architecture: amd64
+Depends: ${shlibs:Depends}, ${misc:Depends}, node-bindings
+Description: Node.js bindings for Vitastor client
+ Node.js native bindings for the Vitastor client library (vitastor-client).

debian/node-vitastor.install

@@ -0,0 +1 @@
+usr/lib/x86_64-linux-gnu/nodejs/vitastor

debian/patched-qemu.Dockerfile

@@ -1,8 +1,10 @@
 # Build patched QEMU for Debian inside a container
 # cd ..; podman build --build-arg REL=bullseye -v `pwd`/packages:/root/packages -f debian/patched-qemu.Dockerfile .
 
+ARG DISTRO=debian
 ARG REL=
-FROM debian:$REL
+FROM $DISTRO:$REL
+ARG DISTRO=debian
 ARG REL=
 
 WORKDIR /root
@@ -20,8 +22,8 @@ RUN if [ "$REL" = "buster" -o "$REL" = "bullseye" -o "$REL" = "bookworm" ]; then
     echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
 
 RUN apt-get update
-RUN apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts
+RUN DEBIAN_FRONTEND=noninteractive TZ=Europe/Moscow apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts
-RUN apt-get -y build-dep qemu
+RUN DEBIAN_FRONTEND=noninteractive TZ=Europe/Moscow apt-get -y build-dep qemu
 # To build a custom version
 #RUN cp /root/packages/qemu-orig/* /root
 RUN apt-get --download-only source qemu
@@ -38,9 +40,9 @@ ADD src/client/qemu_driver.c /root/qemu_driver.c
 #    apt-get install -y vitastor-client vitastor-client-dev quilt
 
 RUN set -e; \
-    dpkg -i /root/packages/vitastor-$REL/vitastor-client_*.deb /root/packages/vitastor-$REL/vitastor-client-dev_*.deb; \
+    DEBIAN_FRONTEND=noninteractive TZ=Europe/Moscow apt-get -y install /root/packages/vitastor-$REL/vitastor-client_*.deb /root/packages/vitastor-$REL/vitastor-client-dev_*.deb; \
     apt-get update; \
-    apt-get install -y quilt; \
+    DEBIAN_FRONTEND=noninteractive TZ=Europe/Moscow apt-get -y install quilt; \
     mkdir -p /root/packages/qemu-$REL; \
     rm -rf /root/packages/qemu-$REL/*; \
     cd /root/packages/qemu-$REL; \

debian/rules

@@ -4,6 +4,14 @@ export DH_VERBOSE = 1
 %:
 	dh $@
 
+override_dh_install:
+	perl -pe 's!prefix=/usr!prefix='`pwd`'/debian/tmp/usr!' < obj-x86_64-linux-gnu/src/client/vitastor.pc > node-binding/vitastor.pc
+	cd node-binding && PKG_CONFIG_PATH=./ PKG_CONFIG_ALLOW_SYSTEM_CFLAGS=1 npm install --unsafe-perm || exit 1
+	mkdir -p debian/tmp/usr/lib/x86_64-linux-gnu/nodejs/vitastor/build/Release
+	cp -v node-binding/package.json node-binding/index.js node-binding/addon.cc node-binding/addon.h node-binding/client.cc node-binding/client.h debian/tmp/usr/lib/x86_64-linux-gnu/nodejs/vitastor
+	cp -v node-binding/build/Release/addon.node debian/tmp/usr/lib/x86_64-linux-gnu/nodejs/vitastor/build/Release
+	dh_install
+
 override_dh_installdeb:
 	cat debian/fio_version >> debian/vitastor-fio.substvars
 	[ -f debian/qemu_version ] && (cat debian/qemu_version >> debian/vitastor-qemu.substvars) || true

debian/vitastor.Dockerfile

@@ -22,7 +22,8 @@ RUN set -e -x; \
     echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
 
 RUN apt-get update && \
-    apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts libjerasure-dev cmake libibverbs-dev librdmacm-dev libisal-dev libnl-3-dev libnl-genl-3-dev curl && \
+    apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts libjerasure-dev cmake \
+        libibverbs-dev librdmacm-dev libisal-dev libnl-3-dev libnl-genl-3-dev curl nodejs npm node-nan node-bindings && \
     apt-get -y build-dep fio && \
     apt-get --download-only source fio
 

docker/Dockerfile

@@ -1,9 +1,11 @@
 # Build Docker image with Vitastor packages
 
-FROM debian:bullseye
+FROM debian:bookworm
 
-ADD vitastor.list /etc/apt/sources.list.d
+ADD etc/apt /etc/apt/
-ADD vitastor.gpg /etc/apt/trusted.gpg.d
+RUN apt-get update && apt-get -y install vitastor qemu-system-x86 qemu-system-common qemu-block-extra qemu-utils jq nfs-common && apt-get clean
-ADD vitastor.pref /etc/apt/preferences.d
+ADD sleep.sh /usr/bin/
-ADD apt.conf /etc/apt/
+ADD install.sh /usr/bin/
-RUN apt-get update && apt-get -y install vitastor qemu-system-x86 qemu-system-common && apt-get clean
+ADD scripts /opt/scripts/
+ADD etc /etc/
+RUN ln -s /usr/lib/vitastor/mon/make-etcd /usr/bin/make-etcd

docker/Makefile

@@ -0,0 +1,9 @@
+VITASTOR_VERSION ?= v1.11.0
+
+all: build push
+
+build:
+	@docker build --rm -t vitalif/vitastor:$(VITASTOR_VERSION) .
+
+push:
+	@docker push vitalif/vitastor:$(VITASTOR_VERSION)

docker/etc/apt/sources.list.d/vitastor.list

@@ -0,0 +1 @@
+deb http://vitastor.io/debian bookworm main

docker/etc/systemd/system/vitastor-etcd.service

@@ -0,0 +1,27 @@
+[Unit]
+Description=Containerized etcd for Vitastor
+After=network-online.target local-fs.target time-sync.target docker.service vitastor-host.service
+Wants=network-online.target local-fs.target time-sync.target docker.service vitastor-host.service
+PartOf=vitastor.target
+
+[Service]
+Restart=always
+Environment=GOGC=50
+EnvironmentFile=/etc/vitastor/docker.conf
+EnvironmentFile=/etc/vitastor/etcd.conf
+SyslogIdentifier=etcd
+ExecStart=bash -c 'docker run --rm -i -v /var/lib/vitastor/etcd:/data \
+    --log-driver none --network host $CONTAINER_OPTIONS --name vitastor-etcd \
+    $ETCD_IMAGE /usr/local/bin/etcd --name "$ETCD_NAME" --data-dir /data \
+    --snapshot-count 10000 --advertise-client-urls http://$ETCD_IP:2379 --listen-client-urls http://$ETCD_IP:2379 \
+    --initial-advertise-peer-urls http://$ETCD_IP:2380 --listen-peer-urls http://$ETCD_IP:2380 \
+    --initial-cluster-token vitastor-etcd-1 --initial-cluster "$ETCD_INITIAL_CLUSTER" \
+    --initial-cluster-state new --max-txn-ops=100000 --max-request-bytes=104857600 \
+    --auto-compaction-retention=10 --auto-compaction-mode=revision'
+ExecStop=docker stop vitastor-etcd
+Restart=always
+StartLimitInterval=0
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target

docker/etc/systemd/system/vitastor-host.service

@@ -0,0 +1,23 @@
+[Unit]
+Description=Empty container for running Vitastor commands
+After=network-online.target local-fs.target time-sync.target docker.service
+Wants=network-online.target local-fs.target time-sync.target docker.service
+PartOf=vitastor.target
+
+[Service]
+Restart=always
+EnvironmentFile=/etc/vitastor/docker.conf
+ExecStart=bash -c 'docker run --rm -i -v /etc/vitastor:/etc/vitastor -v /dev:/dev \
+    --privileged --log-driver none --network host --name vitastor vitastor:$VITASTOR_VERSION \
+    sleep.sh'
+ExecStartPost=udevadm trigger
+ExecStop=docker stop vitastor
+WorkingDirectory=/
+PrivateTmp=false
+TasksMax=infinity
+Restart=always
+StartLimitInterval=0
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target

docker/etc/systemd/system/vitastor-mon.service

@@ -0,0 +1,23 @@
+[Unit]
+Description=Containerized Vitastor monitor
+After=network-online.target local-fs.target time-sync.target docker.service
+Wants=network-online.target local-fs.target time-sync.target docker.service
+PartOf=vitastor.target
+
+[Service]
+Restart=always
+EnvironmentFile=/etc/vitastor/docker.conf
+SyslogIdentifier=vitastor-mon
+ExecStart=bash -c 'docker run --rm -i -v /etc/vitastor:/etc/vitastor -v /var/lib/vitastor:/var/lib/vitastor -v /dev:/dev \
+    --log-driver none --network host $CONTAINER_OPTIONS --name vitastor-mon vitastor:$VITASTOR_VERSION \
+    node /usr/lib/vitastor/mon/mon-main.js'
+ExecStop=docker stop vitastor-mon
+WorkingDirectory=/
+PrivateTmp=false
+TasksMax=infinity
+Restart=always
+StartLimitInterval=0
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target

docker/etc/systemd/system/vitastor-osd@.service

@@ -0,0 +1,27 @@
+[Unit]
+Description=Containerized Vitastor object storage daemon osd.%i
+After=network-online.target local-fs.target time-sync.target docker.service vitastor-host.service
+Wants=network-online.target local-fs.target time-sync.target docker.service vitastor-host.service
+PartOf=vitastor.target
+
+[Service]
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitMEMLOCK=infinity
+EnvironmentFile=/etc/vitastor/docker.conf
+SyslogIdentifier=vitastor-osd%i
+ExecStart=bash -c 'docker run --rm -i -v /etc/vitastor:/etc/vitastor -v /dev:/dev \
+    $(for i in $(ls /dev/vitastor/osd%i-*); do echo --device $i:$i; done) \
+    --log-driver none --network host --ulimit nofile=1048576 --ulimit memlock=-1 $CONTAINER_OPTIONS --name vitastor-osd%i \
+    vitastor:$VITASTOR_VERSION vitastor-disk exec-osd /dev/vitastor/osd%i-data'
+ExecStartPre=+docker exec vitastor vitastor-disk pre-exec /dev/vitastor/osd%i-data
+ExecStop=docker stop vitastor-etcd%i
+WorkingDirectory=/
+PrivateTmp=false
+TasksMax=infinity
+Restart=always
+StartLimitInterval=0
+RestartSec=10
+
+[Install]
+WantedBy=vitastor.target

docker/etc/systemd/system/vitastor.target

@@ -0,0 +1,4 @@
+[Unit]
+Description=vitastor target
+[Install]
+WantedBy=multi-user.target

docker/etc/udev/rules.d/90-vitastor.rules

@@ -0,0 +1,7 @@
+SUBSYSTEM=="block", ENV{ID_PART_ENTRY_TYPE}=="e7009fac-a5a1-4d72-af72-53de13059903", \
+    OWNER="vitastor", GROUP="vitastor", \
+    IMPORT{program}="/usr/bin/docker exec vitastor vitastor-disk udev $devnode", \
+    SYMLINK+="vitastor/$env{VITASTOR_ALIAS}"
+
+ENV{VITASTOR_OSD_NUM}!="", ACTION=="add", RUN{program}+="/usr/bin/systemctl enable --now --no-block vitastor-osd@$env{VITASTOR_OSD_NUM}"
+ENV{VITASTOR_OSD_NUM}!="", ACTION=="remove", RUN{program}+="/usr/bin/systemctl disable --now --no-block vitastor-osd@$env{VITASTOR_OSD_NUM}"

docker/etc/vitastor/docker.conf

@@ -0,0 +1,11 @@
+#
+# Configuration file for containerized Vitastor installation
+# (non-Kubernetes, with systemd and udev-based orchestration)
+#
+
+# Desired Vitastor version
+VITASTOR_VERSION=1.11.0
+
+# Additional arguments for all containers
+# For example, you may want to specify a custom logging driver here
+CONTAINER_OPTIONS=""

docker/etc/vitastor/etcd.conf

@@ -0,0 +1,4 @@
+ETCD_IMAGE=quay.io/coreos/etcd:v3.5.18
+ETCD_NAME=""
+ETCD_IP=""
+ETCD_INITIAL_CLUSTER=""

docker/etc/vitastor/vitastor.conf

@@ -0,0 +1,2 @@
+{
+}

docker/install.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+cp -urv /etc/default /host-etc/
+cp -urv /etc/systemd /host-etc/
+cp -urv /etc/udev /host-etc/
+cp -urnv /etc/vitastor /host-etc/
+cp -urnv /opt/scripts/* /host-bin/

docker/scripts/vitastor-cli

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker exec -it vitastor vitastor-cli "$@"

docker/scripts/vitastor-disk

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker exec -it vitastor vitastor-disk "$@"

docker/scripts/vitastor-fio

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker exec -it vitastor fio "$@"

docker/scripts/vitastor-nbd

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker exec -it vitastor vitastor-nbd "$@"

docker/sleep.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+while :; do sleep infinity; done

docker/vitastor.list

@@ -1 +0,0 @@
-deb http://vitastor.io/debian bullseye main

docs/config.en.md

@@ -13,7 +13,7 @@ Vitastor configuration consists of:
 - [Separate OSD settings](config/pool.en.md#osd-settings)
 - [Inode configuration](config/inode.en.md) i.e. image metadata like name, size and parent reference
 
-Configuration parameters can be set in 3 places:
+Configuration parameters can be set in 4 places:
 - Configuration file (`/etc/vitastor/vitastor.conf` or other path)
 - etcd key `/vitastor/config/global`. Most variables can be set there, but etcd
   connection parameters should obviously be set in the configuration file.

docs/config.ru.md

@@ -14,7 +14,7 @@
 - [Настроек инодов](config/inode.ru.md), т.е. метаданных образов, таких, как имя, размер и ссылки на
   родительский образ
 
-Параметры конфигурации могут задаваться в 3 местах:
+Параметры конфигурации могут задаваться в 4 местах:
 - Файле конфигурации (`/etc/vitastor/vitastor.conf` или по другому пути)
 - Ключе в etcd `/vitastor/config/global`. Большая часть параметров может
   задаваться там, кроме, естественно, самих параметров соединения с etcd,

docs/config/src/included.en.md

@@ -14,8 +14,12 @@
 
 {{../../installation/packages.en.md}}
 
+{{../../installation/docker.en.md}}
+
 {{../../installation/proxmox.en.md}}
 
+{{../../installation/opennebula.en.md}}
+
 {{../../installation/openstack.en.md}}
 
 {{../../installation/kubernetes.en.md}}

docs/config/src/included.ru.md

@@ -14,8 +14,12 @@
 
 {{../../installation/packages.ru.md}}
 
+{{../../installation/docker.ru.md}}
+
 {{../../installation/proxmox.ru.md}}
 
+{{../../installation/opennebula.ru.md}}
+
 {{../../installation/openstack.ru.md}}
 
 {{../../installation/kubernetes.ru.md}}

docs/installation/docker.en.md

@@ -0,0 +1,60 @@
+[Documentation](../../README.md#documentation) → Installation → Dockerized Installation
+
+-----
+
+[Читать на русском](docker.ru.md)
+
+# Dockerized Installation
+
+Vitastor may be installed in Docker/Podman. In such setups etcd, monitors and OSD
+all run in containers, but everything else looks as close as possible to a usual
+setup with packages:
+- host network is used
+- auto-start is implemented through udev and systemd
+- logs are written to journald (not docker json log files)
+- command-line wrapper scripts are installed to the host system to call vitastor-disk,
+  vitastor-cli and others through the container
+
+Such installations may be useful when it's impossible or inconvenient to install
+Vitastor from packages, for example, in exotic Linux distributions.
+
+If you don't want just a simple containerized installation, you can also take a look
+at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operator
+
+## Installing Containers
+
+The instruction is very simple.
+
+1. Download a Docker image of the desired version: \
+   `docker pull vitastor:1.10.2`
+2. Install scripts to the host system: \
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:1.10.2 install.sh`
+3. Reload udev rules: \
+   `udevadm control --reload-rules`
+
+And you can return to [Quick Start](../intro/quickstart.en.md).
+
+## Upgrading Containers
+
+First make sure to check the topic [Upgrading Vitastor](../usage/admin.en.md#upgrading-vitastor)
+to figure out if you need any additional steps.
+
+Then, to upgrade a containerized installation, you just need to change the `VITASTOR_VERSION`
+option in `/etc/vitastor/docker.conf` and restart all Vitastor services:
+
+`systemctl restart vitastor.target`
+
+## QEMU
+
+Vitastor Docker image also contains QEMU, qemu-img and qemu-storage-daemon built with Vitastor support.
+
+However, running QEMU in Docker is harder to setup and it depends on the used virtualization UI
+(OpenNebula, Proxmox and so on). Some of them also required patched Libvirt.
+
+That's why containerized installation of Vitastor doesn't contain a ready-made QEMU setup and it's
+recommended to install QEMU from packages or build it manually.
+
+## fio
+
+Vitastor Docker image also contains fio and installs a wrapper called `vitastor-fio` to use it from
+the host system.

docs/installation/docker.ru.md

@@ -0,0 +1,60 @@
+[Документация](../../README-ru.md#документация) → Установка → Установка в Docker
+
+-----
+
+[Read in English](docker.en.md)
+
+# Установка в Docker
+
+Vitastor можно установить в Docker/Podman. При этом etcd, мониторы и OSD запускаются
+в контейнерах, но всё остальное выглядит максимально приближенно к установке из пакетов:
+- используется сеть хост-системы
+- для автозапуска используются udev и systemd
+- журналы записываются в journald (не в json-файлы журналов docker)
+- в хост-систему устанавливаются обёртки для вызова консольных инструментов vitastor-disk,
+  vitastor-cli и других через контейнер
+
+Такая установка полезна тогда, когда установка из пакетов невозможна или неудобна,
+например, в нестандартных Linux-дистрибутивах.
+
+Если вам нужна не просто контейнеризованная инсталляция, вы также можете обратить внимание
+на Vitastor Kubernetes-оператор: https://github.com/Antilles7227/vitastor-operator
+
+## Установка контейнеров
+
+Инструкция по установке максимально простая.
+
+1. Скачайте Docker-образ желаемой версии: \
+   `docker pull vitastor:1.10.2`
+2. Установите скрипты в хост-систему командой: \
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:1.10.2 install.sh`
+3. Перезагрузите правила udev: \
+   `udevadm control --reload-rules`
+
+После этого вы можете возвращаться к разделу [Быстрый старт](../intro/quickstart.ru.md).
+
+## Обновление контейнеров
+
+Сначала обязательно проверьте раздел [Обновление Vitastor](../usage/admin.ru.md#обновление-vitastor),
+чтобы понять, не требуются ли вам какие-то дополнительные действия.
+
+После этого для обновления Docker-инсталляции вам нужно просто поменять опцию `VITASTOR_VERSION`
+в файле `/etc/vitastor/docker.conf` и перезапустить все сервисы Vitastor командой:
+
+`systemctl restart vitastor.target`
+
+## QEMU
+
+В Docker-образ также входят QEMU, qemu-img и qemu-storage-daemon, собранные с поддержкой Vitastor.
+
+Однако настроить запуск QEMU в Docker сложнее и способ запуска зависит от используемого интерфейса
+виртуализации (OpenNebula, Proxmox и т.п.). Также для OpenNebula, например, требуется патченый
+Libvirt.
+
+Поэтому по умолчанию Docker-сборка пока что не включает в себя готового способа запуска QEMU
+и QEMU рекомендуется устанавливать из пакетов или собирать самостоятельно.
+
+## fio
+
+fio также входит в Docker-контейнер vitastor, и в хост-систему устанавливается обёртка `vitastor-fio`
+для запуска fio в контейнер.

docs/installation/packages.en.md

@@ -14,6 +14,7 @@
   - Debian 12 (Bookworm/Sid): `deb https://vitastor.io/debian bookworm main`
   - Debian 11 (Bullseye): `deb https://vitastor.io/debian bullseye main`
   - Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
+  - Ubuntu 22.04 (Jammy): `deb https://vitastor.io/debian jammy main`
   - Add `-oldstable` to bookworm/bullseye/buster in this line to install the last
     stable version from 0.9.x branch instead of 1.x
 - Install packages: `apt update; apt install vitastor lp-solve etcd linux-image-amd64 qemu-system-x86`

docs/installation/packages.ru.md

@@ -14,6 +14,7 @@
   - Debian 12 (Bookworm/Sid): `deb https://vitastor.io/debian bookworm main`
   - Debian 11 (Bullseye): `deb https://vitastor.io/debian bullseye main`
   - Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
+  - Ubuntu 22.04 (Jammy): `deb https://vitastor.io/debian jammy main`
   - Добавьте `-oldstable` к слову bookworm/bullseye/buster в этой строке, чтобы
     установить последнюю стабильную версию из ветки 0.9.x вместо 1.x
 - Установите пакеты: `apt update; apt install vitastor lp-solve etcd linux-image-amd64 qemu-system-x86`

docs/intro/quickstart.en.md

@@ -26,13 +26,13 @@
   you also need small SSDs for journal and metadata (even 2 GB per 1 TB of HDD space is enough).
 - Get a fast network (at least 10 Gbit/s). Something like Mellanox ConnectX-4 with RoCEv2 is ideal.
 - Disable CPU powersaving: `cpupower idle-set -D 0 && cpupower frequency-set -g performance`.
-- [Install Vitastor packages](../installation/packages.en.md).
+- Either [install Vitastor packages](../installation/packages.en.md) or [install Vitastor in Docker](../installation/docker.en.md).
 
 ## Recommended drives
 
 - SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel D3-S4510/4520/4610/4620, Kingston DC500M
 - NVMe: Micron 9100/9200/9300/9400, Micron 7300/7450, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
-  Intel DC-P3700/P4500/P4600, Intel D5-P4320, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
+  Intel DC-P3700/P4500/P4600, Intel D5-P4320/P5530, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
 - HDD: HGST Ultrastar, Toshiba MG, Seagate EXOS
 
 ## Configure monitors
@@ -45,7 +45,8 @@ On the monitor hosts:
   }
   ```
 - Create systemd units for etcd by running: `/usr/lib/vitastor/mon/make-etcd`
-- Start etcd and monitors: `systemctl enable --now etcd vitastor-mon`
+  Or, if you installed Vitastor in Docker, run `systemctl start vitastor-host; docker exec vitastor make-etcd`.
+- Start etcd and monitors: `systemctl enable --now vitastor-etcd vitastor-mon`
 
 ## Configure OSDs
 

docs/intro/quickstart.ru.md

@@ -26,13 +26,13 @@
   обязательно возьмите SSD под метаданные и журнал (маленькие, буквально 2 ГБ на 1 ТБ HDD-места).
 - Возьмите быструю сеть, минимум 10 гбит/с. Идеал - что-то вроде Mellanox ConnectX-4 с RoCEv2.
 - Для лучшей производительности отключите энергосбережение CPU: `cpupower idle-set -D 0 && cpupower frequency-set -g performance`.
-- [Установите пакеты Vitastor](../installation/packages.ru.md).
+- Либо [установите пакеты Vitastor](../installation/packages.ru.md), либо [установите Vitastor в Docker](../installation/docker.ru.md).
 
 ## Рекомендуемые диски
 
 - SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel D3-S4510/4520/4610/4620, Kingston DC500M
 - NVMe: Micron 9100/9200/9300/9400, Micron 7300/7450, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
-  Intel DC-P3700/P4500/P4600, Intel D5-P4320, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
+  Intel DC-P3700/P4500/P4600, Intel D5-P4320/P5530, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
 - HDD: HGST Ultrastar, Toshiba MG, Seagate EXOS
 
 ## Настройте мониторы
@@ -44,8 +44,9 @@
     "etcd_address": ["10.200.1.10:2379","10.200.1.11:2379","10.200.1.12:2379"]
   }
   ```
-- Инициализируйте сервисы etcd, запустив `/usr/lib/vitastor/mon/make-etcd`
+- Инициализируйте сервисы etcd, запустив `/usr/lib/vitastor/mon/make-etcd`.\
-- Запустите etcd и мониторы: `systemctl enable --now etcd vitastor-mon`
+  Либо, если вы установили Vitastor в Docker, запустите `systemctl start vitastor-host; docker exec vitastor make-etcd`.
+- Запустите etcd и мониторы: `systemctl enable --now vitastor-etcd vitastor-mon`
 
 ## Настройте OSD
 

docs/usage/cli.en.md

@@ -37,7 +37,7 @@ It supports the following commands:
 Global options:
 
 ```
---config_file FILE   Path to Vitastor configuration file
+--config_path FILE   Path to Vitastor configuration file
 --etcd_address URL   Etcd connection address
 --iodepth N          Send N operations in parallel to each OSD when possible (default 32)
 --parallel_osds M    Work with M osds in parallel when possible (default 4)

docs/usage/cli.ru.md

@@ -36,7 +36,7 @@ vitastor-cli - интерфейс командной строки для адм
 Глобальные опции:
 
 ```
---config_file FILE   Путь к файлу конфигурации Vitastor
+--config_path FILE   Путь к файлу конфигурации Vitastor
 --etcd_address URL   Адрес соединения с etcd
 --iodepth N          Отправлять параллельно N операций на каждый OSD (по умолчанию 32)
 --parallel_osds M    Работать параллельно с M OSD (по умолчанию 4)

docs/usage/nbd.en.md

@@ -36,7 +36,7 @@ It will output a block device name like /dev/nbd0 which you can then use as a no
 
 You can also use `--pool <POOL> --inode <INODE> --size <SIZE>` instead of `--image <IMAGE>` if you want.
 
-vitastor-nbd supports all usual Vitastor configuration options like `--config_file <path_to_config>` plus NBD-specific:
+vitastor-nbd supports all usual Vitastor configuration options like `--config_path <path_to_config>` plus NBD-specific:
 
 * `--nbd_timeout 0` \
   Timeout for I/O operations in seconds after exceeding which the kernel stops the device.
@@ -54,16 +54,18 @@ vitastor-nbd supports all usual Vitastor configuration options like `--config_fi
   Stay in foreground, do not daemonize.
 
 Note that `nbd_timeout`, `nbd_max_devices` and `nbd_max_part` options may also be specified
-in `/etc/vitastor/vitastor.conf` or in other configuration file specified with `--config_file`.
+in `/etc/vitastor/vitastor.conf` or in other configuration file specified with `--config_path`.
 
 ## unmap
 
 To unmap the device run:
 
 ```
-vitastor-nbd unmap /dev/nbd0
+vitastor-nbd unmap [--force] /dev/nbd0
 ```
 
+If `--force` is specified, `vitastor-nbd` doesn't check if the device is actually mapped.
+
 ## ls
 
 ```

docs/usage/nbd.ru.md

@@ -41,7 +41,7 @@ vitastor-nbd map [/dev/nbdN] --image testimg
 Для обращения по номеру инода, аналогично другим командам, можно использовать опции
 `--pool <POOL> --inode <INODE> --size <SIZE>` вместо `--image testimg`.
 
-vitastor-nbd поддерживает все обычные опции Vitastor, например, `--config_file <path_to_config>`,
+vitastor-nbd поддерживает все обычные опции Vitastor, например, `--config_path <path_to_config>`,
 плюс специфичные для NBD:
 
 * `--nbd_timeout 0` \
@@ -62,16 +62,19 @@ vitastor-nbd поддерживает все обычные опции Vitastor,
 
 Обратите внимание, что опции `nbd_timeout`, `nbd_max_devices` и `nbd_max_part` можно
 также задавать в `/etc/vitastor/vitastor.conf` или в другом файле конфигурации,
-заданном опцией `--config_file`.
+заданном опцией `--config_path`.
 
 ## unmap
 
 Для отключения устройства выполните:
 
 ```
-vitastor-nbd unmap /dev/nbd0
+vitastor-nbd unmap [--force] /dev/nbd0
 ```
 
+Если задана опция `--force`, `vitastor-nbd` не проверяет, подключено ли устройство,
+перед попыткой его отключить.
+
 ## ls
 
 ```

mon/mon.js

@@ -773,6 +773,9 @@ class Mon
                 }
             }
         }
+        if (!this.recheck_pgs_active)
+        {
+            // PG recheck also modifies /pool/stats, so don't touch it here if it's active
             for (const pool_id in this.state.pool.stats)
             {
                 if (!seen_pools[pool_id])
@@ -792,6 +795,7 @@ class Mon
                     } });
                 }
             }
+        }
         if (txn.length)
         {
             await this.etcd.etcd_call('/kv/txn', { success: txn }, this.config.etcd_mon_timeout, 0);

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "1.10.1",
+  "version": "1.11.0",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {
@@ -9,7 +9,7 @@
   "author": "Vitaliy Filippov",
   "license": "UNLICENSED",
   "dependencies": {
-    "antietcd": "^1.1.0",
+    "antietcd": "^1.1.2",
     "sprintf-js": "^1.1.2",
     "ws": "^7.2.5"
   },

mon/pg_gen.js

@@ -8,23 +8,9 @@ const LPOptimizer = require('./lp_optimizer/lp_optimizer.js');
 const { scale_pg_count } = require('./pg_utils.js');
 const { make_hier_tree, filter_osds_by_root_node,
     filter_osds_by_tags, filter_osds_by_block_layout, get_affinity_osds } = require('./osd_tree.js');
+const { select_murmur3 } = require('./lp_optimizer/murmur3.js');
 
-let seed;
+function pick_primary(pool_id, pg_num, pool_config, osd_set, up_osds, aff_osds)
-
-function reset_rng()
-{
-    seed = 0x5f020e43;
-}
-
-function rng()
-{
-    seed ^= seed << 13;
-    seed ^= seed >> 17;
-    seed ^= seed << 5;
-    return seed + 2147483648;
-}
-
-function pick_primary(pool_config, osd_set, up_osds, aff_osds)
 {
     let alive_set;
     if (pool_config.scheme === 'replicated')
@@ -52,7 +38,7 @@ function pick_primary(pool_config, osd_set, up_osds, aff_osds)
     {
         return 0;
     }
-    return alive_set[rng() % alive_set.length];
+    return alive_set[select_murmur3(alive_set.length, osd_num => pool_id+'/'+pg_num+'/'+osd_num)];
 }
 
 function recheck_primary(state, global_config, up_osds, osd_tree)
@@ -66,7 +52,6 @@ function recheck_primary(state, global_config, up_osds, osd_tree)
             continue;
         }
         const aff_osds = get_affinity_osds(pool_cfg, up_osds, osd_tree);
-        reset_rng();
         for (let pg_num = 1; pg_num <= pool_cfg.pg_count; pg_num++)
         {
             if (!state.pg.config.items[pool_id])
@@ -76,7 +61,7 @@ function recheck_primary(state, global_config, up_osds, osd_tree)
             const pg_cfg = state.pg.config.items[pool_id][pg_num];
             if (pg_cfg)
             {
-                const new_primary = pick_primary(state.config.pools[pool_id], pg_cfg.osd_set, up_osds, aff_osds);
+                const new_primary = pick_primary(pool_id, pg_num, state.config.pools[pool_id], pg_cfg.osd_set, up_osds, aff_osds);
                 if (pg_cfg.primary != new_primary)
                 {
                     if (!new_pg_config)
@@ -99,13 +84,12 @@ function save_new_pgs_txn(save_to, request, state, etcd_prefix, etcd_watch_revis
 {
     const aff_osds = get_affinity_osds(state.config.pools[pool_id] || {}, up_osds, osd_tree);
     const pg_items = {};
-    reset_rng();
     new_pgs.map((osd_set, i) =>
     {
         osd_set = osd_set.map(osd_num => osd_num === LPOptimizer.NO_OSD ? 0 : osd_num);
         pg_items[i+1] = {
             osd_set,
-            primary: pick_primary(state.config.pools[pool_id], osd_set, up_osds, aff_osds),
+            primary: pick_primary(pool_id, i+1, state.config.pools[pool_id], osd_set, up_osds, aff_osds),
         };
         if (prev_pgs[i] && prev_pgs[i].join(' ') != osd_set.join(' ') &&
             prev_pgs[i].filter(osd_num => osd_num).length > 0)

mon/scripts/make-etcd

@@ -33,9 +33,11 @@ async function run()
         console.log(config_path+' is missing');
         process.exit(1);
     }
-    if (fs.existsSync("/etc/systemd/system/etcd.service"))
+    const in_docker = fs.existsSync("/etc/vitastor/etcd.conf") &&
+        fs.existsSync("/etc/vitastor/docker.conf");
+    if (!in_docker && fs.existsSync("/etc/systemd/system/vitastor-etcd.service"))
     {
-        console.log("/etc/systemd/system/etcd.service already exists");
+        console.log("/etc/systemd/system/vitastor-etcd.service already exists");
         process.exit(1);
     }
     const config = JSON.parse(fs.readFileSync(config_path, { encoding: 'utf-8' }));
@@ -52,10 +54,21 @@ async function run()
         console.log('No matching IPs in etcd_address from '+config_path);
         process.exit(0);
     }
-    const etcd_cluster = etcds.map((e, i) => `etcd${i}=http://${e}:2380`).join(',');
+    const etcd_name = 'etcd'+etcds[num].replace(/[^0-9a-z_]/ig, '_');
-    await system(`mkdir -p /var/lib/etcd${num}.etcd`);
+    const etcd_cluster = etcds.map(e => `etcd${e.replace(/[^0-9a-z_]/ig, '_')}=http://${e}:2380`).join(',');
+    if (in_docker)
+    {
+        let etcd_conf = fs.readFileSync("/etc/vitastor/etcd.conf", { encoding: 'utf-8' });
+        etcd_conf = replace_env(etcd_conf, 'ETCD_NAME', etcd_name);
+        etcd_conf = replace_env(etcd_conf, 'ETCD_IP', etcds[num]);
+        etcd_conf = replace_env(etcd_conf, 'ETCD_INITIAL_CLUSTER', etcd_cluster);
+        fs.writeFileSync("/etc/vitastor/etcd.conf", etcd_conf);
+        console.log('etcd for Vitastor configured. Run `systemctl enable --now vitastor-etcd` to start etcd');
+        process.exit(0);
+    }
+    await system(`mkdir -p /var/lib/etcd`);
     fs.writeFileSync(
-        "/etc/systemd/system/etcd.service",
+        "/etc/systemd/system/vitastor-etcd.service",
 `[Unit]
 Description=etcd for vitastor
 After=network-online.target local-fs.target time-sync.target
@@ -64,14 +77,14 @@ Wants=network-online.target local-fs.target time-sync.target
 [Service]
 Restart=always
 Environment=GOGC=50
-ExecStart=etcd -name etcd${num} --data-dir /var/lib/etcd${num}.etcd \\
+ExecStart=etcd --name ${etcd_name} --data-dir /var/lib/etcd \\
     --snapshot-count 10000 --advertise-client-urls http://${etcds[num]}:2379 --listen-client-urls http://${etcds[num]}:2379 \\
     --initial-advertise-peer-urls http://${etcds[num]}:2380 --listen-peer-urls http://${etcds[num]}:2380 \\
     --initial-cluster-token vitastor-etcd-1 --initial-cluster ${etcd_cluster} \\
     --initial-cluster-state new --max-txn-ops=100000 --max-request-bytes=104857600 \\
     --auto-compaction-retention=10 --auto-compaction-mode=revision
-WorkingDirectory=/var/lib/etcd${num}.etcd
+WorkingDirectory=/var/lib/etcd
-ExecStartPre=+chown -R etcd /var/lib/etcd${num}.etcd
+ExecStartPre=+chown -R etcd /var/lib/etcd
 User=etcd
 PrivateTmp=false
 TasksMax=infinity
@@ -89,6 +102,13 @@ WantedBy=multi-user.target
     process.exit(0);
 }
 
+function replace_env(text, key, value)
+{
+    let found = false;
+    text = text.replace(new RegExp('^'+key+'\\s*=.*', 'm'), () => { found = true; return key+'='+value; });
+    return found ? text : text.replace(/\s*$/, '\n')+key+'='+value+'\n';
+}
+
 function select_local_etcd(etcds)
 {
     const ifaces = os.networkInterfaces();

mon/scripts/vitastor-mon.service

@@ -5,6 +5,7 @@ Wants=network-online.target local-fs.target time-sync.target
 
 [Service]
 Restart=always
+SyslogIdentifier=vitastor-mon
 ExecStart=node /usr/lib/vitastor/mon/mon-main.js
 WorkingDirectory=/
 User=vitastor

mon/scripts/vitastor-osd@.service

@@ -8,6 +8,7 @@ PartOf=vitastor.target
 LimitNOFILE=1048576
 LimitNPROC=1048576
 LimitMEMLOCK=infinity
+SyslogIdentifier=vitastor-osd%i
 # Use the following for direct logs to files
 #ExecStart=bash -c 'exec vitastor-disk exec-osd /dev/vitastor/osd%i-data >>/var/log/vitastor/osd%i.log 2>&1'
 ExecStart=vitastor-disk exec-osd /dev/vitastor/osd%i-data

node-binding/addon.cc

@@ -14,8 +14,13 @@ NAN_MODULE_INIT(InitAddon)
 
     Nan::SetPrototypeMethod(tpl, "read", NodeVitastor::Read);
     Nan::SetPrototypeMethod(tpl, "write", NodeVitastor::Write);
+    Nan::SetPrototypeMethod(tpl, "delete", NodeVitastor::Delete);
     Nan::SetPrototypeMethod(tpl, "sync", NodeVitastor::Sync);
     Nan::SetPrototypeMethod(tpl, "read_bitmap", NodeVitastor::ReadBitmap);
+    Nan::SetPrototypeMethod(tpl, "on_ready", NodeVitastor::OnReady);
+    Nan::SetPrototypeMethod(tpl, "get_min_io_size", NodeVitastor::GetMinIoSize);
+    Nan::SetPrototypeMethod(tpl, "get_max_atomic_write_size", NodeVitastor::GetMaxAtomicWriteSize);
+    Nan::SetPrototypeMethod(tpl, "get_immediate_commit", NodeVitastor::GetImmediateCommit);
     //Nan::SetPrototypeMethod(tpl, "destroy", NodeVitastor::Destroy);
 
     Nan::Set(target, Nan::New("Client").ToLocalChecked(), Nan::GetFunction(tpl).ToLocalChecked());
@@ -63,6 +68,10 @@ NAN_MODULE_INIT(InitAddon)
     Nan::Set(target, Nan::New("ENOSYS").ToLocalChecked(), Nan::New<v8::Int32>(-ENOSYS));
     Nan::Set(target, Nan::New("EAGAIN").ToLocalChecked(), Nan::New<v8::Int32>(-EAGAIN));
 
+    Nan::Set(target, Nan::New("IMMEDIATE_NONE").ToLocalChecked(), Nan::New<v8::Int32>(IMMEDIATE_NONE));
+    Nan::Set(target, Nan::New("IMMEDIATE_SMALL").ToLocalChecked(), Nan::New<v8::Int32>(IMMEDIATE_SMALL));
+    Nan::Set(target, Nan::New("IMMEDIATE_ALL").ToLocalChecked(), Nan::New<v8::Int32>(IMMEDIATE_ALL));
+
     // Listing handle
 
     tpl = Nan::New<v8::FunctionTemplate>(NodeVitastorKVListing::Create);

node-binding/client.cc

@@ -5,9 +5,10 @@
 
 #define NODE_VITASTOR_READ 1
 #define NODE_VITASTOR_WRITE 2
-#define NODE_VITASTOR_SYNC 3
+#define NODE_VITASTOR_DELETE 3
-#define NODE_VITASTOR_READ_BITMAP 4
+#define NODE_VITASTOR_SYNC 4
-#define NODE_VITASTOR_GET_INFO 5
+#define NODE_VITASTOR_READ_BITMAP 5
+#define NODE_VITASTOR_GET_INFO 6
 
 #ifndef INODE_POOL
 #define INODE_POOL(inode) (uint32_t)((inode) >> (64 - POOL_ID_BITS))
@@ -80,6 +81,11 @@ NAN_METHOD(NodeVitastor::Create)
     NodeVitastor* cli = new NodeVitastor();
     cli->c = vitastor_c_create_uring_json(c_cfg, cfg.size());
     delete[] c_cfg;
+    if (!cli->c)
+    {
+        Nan::ThrowError("failed to initialize io_uring (old kernel or insufficient ulimit -l?)");
+        return;
+    }
 
     int res = vitastor_c_uring_register_eventfd(cli->c);
     if (res >= 0)
@@ -133,12 +139,12 @@ NodeVitastorRequest* NodeVitastor::get_read_request(const Nan::FunctionCallbackI
     return req;
 }
 
-// read(pool, inode, offset, len, callback(err, buffer, version))
+// read(pool, inode, offset, length, callback(err, buffer, version))
 NAN_METHOD(NodeVitastor::Read)
 {
     TRACE("NodeVitastor::Read");
     if (info.Length() < 5)
-        Nan::ThrowError("Not enough arguments to read(pool, inode, offset, len, callback(err, buffer, version))");
+        Nan::ThrowError("Not enough arguments to read(pool, inode, offset, length, callback(err, buffer, version))");
 
     NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
 
@@ -149,6 +155,9 @@ NAN_METHOD(NodeVitastor::Read)
 
     self->Ref();
     vitastor_c_read(self->c, ((pool << (64-POOL_ID_BITS)) | inode), req->offset, req->len, &req->iov, 1, on_read_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(self->c);
+#endif
 }
 
 NodeVitastorRequest* NodeVitastor::get_write_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos)
@@ -217,6 +226,58 @@ NAN_METHOD(NodeVitastor::Write)
         req->iov_list.size() ? req->iov_list.data() : &req->iov,
         req->iov_list.size() ? req->iov_list.size() : 1,
         on_write_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(self->c);
+#endif
+}
+
+NodeVitastorRequest* NodeVitastor::get_delete_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos)
+{
+    uint64_t offset = get_ui64(info[argpos+0]);
+    uint64_t len = get_ui64(info[argpos+1]);
+    uint64_t version = 0;
+    if (!info[argpos+2].IsEmpty() &&
+        !info[argpos+2]->IsFunction() &&
+        info[argpos+2]->IsObject())
+    {
+        auto key = Nan::New<v8::String>("version").ToLocalChecked();
+        auto params = info[argpos+2].As<v8::Object>();
+        auto versionObj = Nan::Get(params, key).ToLocalChecked();
+        if (!versionObj.IsEmpty())
+            version = get_ui64(versionObj);
+        argpos++;
+    }
+
+    v8::Local<v8::Function> callback = info[argpos+2].As<v8::Function>();
+    auto req = new NodeVitastorRequest(this, callback);
+
+    req->offset = offset;
+    req->len = len;
+    req->version = version;
+
+    return req;
+}
+
+// delete(pool, inode, offset, length, { version }?, callback(err))
+NAN_METHOD(NodeVitastor::Delete)
+{
+    TRACE("NodeVitastor::Delete");
+    if (info.Length() < 5)
+        Nan::ThrowError("Not enough arguments to delete(pool, inode, offset, length, { version }?, callback(err))");
+
+    NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
+
+    uint64_t pool = get_ui64(info[0]);
+    uint64_t inode = get_ui64(info[1]);
+
+    auto req = self->get_delete_request(info, 2);
+
+    self->Ref();
+    vitastor_c_delete(self->c, ((pool << (64-POOL_ID_BITS)) | inode), req->offset, req->len, req->version,
+        on_write_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(self->c);
+#endif
 }
 
 // sync(callback(err))
@@ -233,14 +294,17 @@ NAN_METHOD(NodeVitastor::Sync)
 
     self->Ref();
     vitastor_c_sync(self->c, on_write_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(self->c);
+#endif
 }
 
-// read_bitmap(pool, inode, offset, len, with_parents, callback(err, bitmap_buffer))
+// read_bitmap(pool, inode, offset, length, with_parents, callback(err, bitmap_buffer))
 NAN_METHOD(NodeVitastor::ReadBitmap)
 {
     TRACE("NodeVitastor::ReadBitmap");
     if (info.Length() < 6)
-        Nan::ThrowError("Not enough arguments to read_bitmap(pool, inode, offset, len, with_parents, callback(err, bitmap_buffer))");
+        Nan::ThrowError("Not enough arguments to read_bitmap(pool, inode, offset, length, with_parents, callback(err, bitmap_buffer))");
 
     NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
 
@@ -254,6 +318,9 @@ NAN_METHOD(NodeVitastor::ReadBitmap)
     auto req = new NodeVitastorRequest(self, callback);
     self->Ref();
     vitastor_c_read_bitmap(self->c, ((pool << (64-POOL_ID_BITS)) | inode), offset, len, with_parents, on_read_bitmap_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(self->c);
+#endif
 }
 
 static void on_error(NodeVitastorRequest *req, Nan::Callback & nanCallback, long retval)
@@ -267,6 +334,67 @@ static void on_error(NodeVitastorRequest *req, Nan::Callback & nanCallback, long
     nanCallback.Call(1, args, req);
 }
 
+// on_ready(callback(err))
+NAN_METHOD(NodeVitastor::OnReady)
+{
+    TRACE("NodeVitastor::OnReady");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to on_ready(callback(err))");
+    NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
+    v8::Local<v8::Function> callback = info[0].As<v8::Function>();
+    auto req = new NodeVitastorRequest(self, callback);
+    self->Ref();
+    vitastor_c_on_ready(self->c, on_ready_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(self->c);
+#endif
+}
+
+void NodeVitastor::on_ready_finish(void *opaque, long retval)
+{
+    TRACE("NodeVitastor::on_ready_finish");
+    auto req = (NodeVitastorRequest*)opaque;
+    auto self = req->cli;
+    Nan::HandleScope scope;
+    Nan::Callback nanCallback(Nan::New(req->callback));
+    nanCallback.Call(0, NULL, req);
+    self->Unref();
+    delete req;
+}
+
+// get_min_io_size(pool_id)
+NAN_METHOD(NodeVitastor::GetMinIoSize)
+{
+    TRACE("NodeVitastor::GetMinIoSize");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to get_min_io_size(pool_id)");
+    NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
+    uint64_t pool = get_ui64(info[0]);
+    info.GetReturnValue().Set(Nan::New<v8::Number>(vitastor_c_inode_get_bitmap_granularity(self->c, INODE_WITH_POOL(pool, 1))));
+}
+
+// get_max_atomic_write_size(pool_id)
+NAN_METHOD(NodeVitastor::GetMaxAtomicWriteSize)
+{
+    TRACE("NodeVitastor::GetMaxAtomicWriteSize");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to get_max_atomic_write_size(pool_id)");
+    NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
+    uint64_t pool = get_ui64(info[0]);
+    info.GetReturnValue().Set(Nan::New<v8::Number>(vitastor_c_inode_get_block_size(self->c, INODE_WITH_POOL(pool, 1))));
+}
+
+// get_immediate_commit(pool_id)
+NAN_METHOD(NodeVitastor::GetImmediateCommit)
+{
+    TRACE("NodeVitastor::GetImmediateCommit");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to get_immediate_commit(pool_id)");
+    NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
+    uint64_t pool = get_ui64(info[0]);
+    info.GetReturnValue().Set(Nan::New<v8::Number>(vitastor_c_inode_get_immediate_commit(self->c, INODE_WITH_POOL(pool, 1))));
+}
+
 void NodeVitastor::on_read_finish(void *opaque, long retval, uint64_t version)
 {
     TRACE("NodeVitastor::on_read_finish");
@@ -364,6 +492,9 @@ NAN_METHOD(NodeVitastorImage::Create)
     img->Ref();
     cli->Ref();
     vitastor_c_watch_inode(cli->c, (char*)img->name.c_str(), on_watch_start, img);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(cli->c);
+#endif
 
     info.GetReturnValue().Set(info.This());
 }
@@ -378,12 +509,12 @@ NodeVitastorImage::~NodeVitastorImage()
     cli->Unref();
 }
 
-// read(offset, len, callback(err, buffer, version))
+// read(offset, length, callback(err, buffer, version))
 NAN_METHOD(NodeVitastorImage::Read)
 {
     TRACE("NodeVitastorImage::Read");
     if (info.Length() < 3)
-        Nan::ThrowError("Not enough arguments to read(offset, len, callback(err, buffer, version))");
+        Nan::ThrowError("Not enough arguments to read(offset, length, callback(err, buffer, version))");
 
     NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
 
@@ -394,12 +525,12 @@ NAN_METHOD(NodeVitastorImage::Read)
     img->exec_or_wait(req);
 }
 
-// write(offset, buffer, { version }?, callback(err))
+// write(offset, buf: Buffer | Buffer[], { version }?, callback(err))
 NAN_METHOD(NodeVitastorImage::Write)
 {
     TRACE("NodeVitastorImage::Write");
     if (info.Length() < 3)
-        Nan::ThrowError("Not enough arguments to write(offset, buffer, { version }?, callback(err))");
+        Nan::ThrowError("Not enough arguments to write(offset, buf: Buffer | Buffer[], { version }?, callback(err))");
 
     NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
 
@@ -410,6 +541,22 @@ NAN_METHOD(NodeVitastorImage::Write)
     img->exec_or_wait(req);
 }
 
+// delete(offset, length, { version }?, callback(err))
+NAN_METHOD(NodeVitastorImage::Delete)
+{
+    TRACE("NodeVitastorImage::Delete");
+    if (info.Length() < 3)
+        Nan::ThrowError("Not enough arguments to delete(offset, length, { version }?, callback(err))");
+
+    NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
+
+    auto req = img->cli->get_delete_request(info, 0);
+    req->img = img;
+    req->op = NODE_VITASTOR_DELETE;
+
+    img->exec_or_wait(req);
+}
+
 // sync(callback(err))
 NAN_METHOD(NodeVitastorImage::Sync)
 {
@@ -427,12 +574,12 @@ NAN_METHOD(NodeVitastorImage::Sync)
     img->exec_or_wait(req);
 }
 
-// read_bitmap(offset, len, with_parents, callback(err, bitmap_buffer))
+// read_bitmap(offset, length, with_parents, callback(err, bitmap_buffer))
 NAN_METHOD(NodeVitastorImage::ReadBitmap)
 {
     TRACE("NodeVitastorImage::ReadBitmap");
     if (info.Length() < 4)
-        Nan::ThrowError("Not enough arguments to read_bitmap(offset, len, with_parents, callback(err, bitmap_buffer))");
+        Nan::ThrowError("Not enough arguments to read_bitmap(offset, length, with_parents, callback(err, bitmap_buffer))");
 
     NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
 
@@ -488,6 +635,9 @@ void NodeVitastorImage::exec_request(NodeVitastorRequest *req)
         uint64_t ino = vitastor_c_inode_get_num(watch);
         cli->Ref();
         vitastor_c_read(cli->c, ino, req->offset, req->len, &req->iov, 1, NodeVitastor::on_read_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+        vitastor_c_uring_handle_events(cli->c);
+#endif
     }
     else if (req->op == NODE_VITASTOR_WRITE)
     {
@@ -497,6 +647,19 @@ void NodeVitastorImage::exec_request(NodeVitastorRequest *req)
             req->iov_list.size() ? req->iov_list.data() : &req->iov,
             req->iov_list.size() ? req->iov_list.size() : 1,
             NodeVitastor::on_write_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+        vitastor_c_uring_handle_events(cli->c);
+#endif
+    }
+    else if (req->op == NODE_VITASTOR_DELETE)
+    {
+        uint64_t ino = vitastor_c_inode_get_num(watch);
+        cli->Ref();
+        vitastor_c_delete(cli->c, ino, req->offset, req->len, req->version,
+            NodeVitastor::on_write_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+        vitastor_c_uring_handle_events(cli->c);
+#endif
     }
     else if (req->op == NODE_VITASTOR_SYNC)
     {
@@ -506,6 +669,9 @@ void NodeVitastorImage::exec_request(NodeVitastorRequest *req)
         if (imm != IMMEDIATE_ALL)
         {
             vitastor_c_sync(cli->c, NodeVitastor::on_write_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+            vitastor_c_uring_handle_events(cli->c);
+#endif
         }
         else
         {
@@ -517,6 +683,9 @@ void NodeVitastorImage::exec_request(NodeVitastorRequest *req)
         uint64_t ino = vitastor_c_inode_get_num(watch);
         cli->Ref();
         vitastor_c_read_bitmap(cli->c, ino, req->offset, req->len, req->with_parents, NodeVitastor::on_read_bitmap_finish, req);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+        vitastor_c_uring_handle_events(cli->c);
+#endif
     }
     else if (req->op == NODE_VITASTOR_GET_INFO)
     {
@@ -648,6 +817,9 @@ NAN_METHOD(NodeVitastorKV::Open)
         delete req;
         kv->Unref();
     });
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(kv->cli->c);
+#endif
 }
 
 // close(callback(err))
@@ -671,6 +843,9 @@ NAN_METHOD(NodeVitastorKV::Close)
         delete req;
         kv->Unref();
     });
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(kv->cli->c);
+#endif
 }
 
 // set_config({ ...config })
@@ -729,6 +904,9 @@ void NodeVitastorKV::get_impl(const Nan::FunctionCallbackInfo<v8::Value> & info,
         delete req;
         kv->Unref();
     }, allow_cache);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(kv->cli->c);
+#endif
 }
 
 // get(key, callback(err, value))
@@ -801,6 +979,9 @@ NAN_METHOD(NodeVitastorKV::Set)
             delete cas_req;
         kv->Unref();
     }, cas_cb);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(kv->cli->c);
+#endif
 }
 
 // del(key, callback(err), cas_compare(old_value)?)
@@ -839,6 +1020,9 @@ NAN_METHOD(NodeVitastorKV::Del)
             delete cas_req;
         kv->Unref();
     }, cas_cb);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(kv->cli->c);
+#endif
 }
 
 // list(start_key?)
@@ -909,7 +1093,7 @@ NodeVitastorKVListing::~NodeVitastorKVListing()
     kv->Unref();
 }
 
-// next(callback(err, value)?)
+// next(callback(err, key, value)?)
 NAN_METHOD(NodeVitastorKVListing::Next)
 {
     TRACE("NodeVitastorKVListing::Next");
@@ -959,6 +1143,9 @@ NAN_METHOD(NodeVitastorKVListing::Next)
             list->iter = req;
         list->kv->Unref();
     });
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
+    vitastor_c_uring_handle_events(list->kv->cli->c);
+#endif
 }
 
 // close()

node-binding/client.h

@@ -15,14 +15,24 @@ class NodeVitastor: public Nan::ObjectWrap
 public:
     // constructor({ ...config })
     static NAN_METHOD(Create);
-    // read(pool, inode, offset, len, callback(err, buffer, version))
+    // read(pool_id, inode, offset, length, callback(err, buffer, version))
     static NAN_METHOD(Read);
-    // write(pool, inode, offset, buf: Buffer | Buffer[], { version }?, callback(err))
+    // write(pool_id, inode, offset, buf: Buffer | Buffer[], { version }?, callback(err))
     static NAN_METHOD(Write);
+    // delete(pool_id, inode, offset, length, { version }?, callback(err))
+    static NAN_METHOD(Delete);
     // sync(callback(err))
     static NAN_METHOD(Sync);
-    // read_bitmap(pool, inode, offset, len, with_parents, callback(err, bitmap_buffer))
+    // read_bitmap(pool_id, inode, offset, length, with_parents, callback(err, bitmap_buffer))
     static NAN_METHOD(ReadBitmap);
+    // on_ready(callback(err))
+    static NAN_METHOD(OnReady);
+    // get_min_io_size(pool_id)
+    static NAN_METHOD(GetMinIoSize);
+    // get_max_atomic_write_size(pool_id)
+    static NAN_METHOD(GetMaxAtomicWriteSize);
+    // get_immediate_commit(pool_id)
+    static NAN_METHOD(GetImmediateCommit);
 //    // destroy()
 //    static NAN_METHOD(Destroy);
 
@@ -37,11 +47,13 @@ private:
 
     static void on_io_readable(uv_poll_t* handle, int status, int revents);
     static void on_read_finish(void *opaque, long retval, uint64_t version);
+    static void on_ready_finish(void *opaque, long retval);
     static void on_write_finish(void *opaque, long retval);
     static void on_read_bitmap_finish(void *opaque, long retval, uint8_t *bitmap);
 
     NodeVitastorRequest* get_read_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos);
     NodeVitastorRequest* get_write_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos);
+    NodeVitastorRequest* get_delete_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos);
 
     friend class NodeVitastorImage;
     friend class NodeVitastorKV;
@@ -53,13 +65,15 @@ class NodeVitastorImage: public Nan::ObjectWrap
 public:
     // constructor(node_vitastor, name)
     static NAN_METHOD(Create);
-    // read(offset, len, callback(err, buffer, version))
+    // read(offset, length, callback(err, buffer, version))
     static NAN_METHOD(Read);
     // write(offset, buf: Buffer | Buffer[], { version }?, callback(err))
     static NAN_METHOD(Write);
+    // delete(offset, length, { version }?, callback(err))
+    static NAN_METHOD(Delete);
     // sync(callback(err))
     static NAN_METHOD(Sync);
-    // read_bitmap(offset, len, with_parents, callback(err, bitmap_buffer))
+    // read_bitmap(offset, length, with_parents, callback(err, bitmap_buffer))
     static NAN_METHOD(ReadBitmap);
     // get_info(callback({ num, name, size, parent_id?, readonly?, meta?, mod_revision, block_size, bitmap_granularity, immediate_commit }))
     static NAN_METHOD(GetInfo);
@@ -120,7 +134,7 @@ class NodeVitastorKVListing: public Nan::ObjectWrap
 public:
     // constructor(node_vitastor_kv, start_key?)
     static NAN_METHOD(Create);
-    // next(callback(err, value)?)
+    // next(callback(err, key, value)?)
     static NAN_METHOD(Next);
     // close()
     static NAN_METHOD(Close);

node-binding/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor",
-  "version": "1.7.0",
+  "version": "1.11.0",
   "description": "Low-level native bindings to Vitastor client library",
   "main": "index.js",
   "keywords": [

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '1.10.1'
+VITASTOR_VERSION = '1.11.0'
 
 LOG = logging.getLogger(__name__)
 

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.10.1
+Version:        1.11.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.10.1.el7.tar.gz
+Source0:        vitastor-1.11.0.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.10.1
+Version:        1.11.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.10.1.el8.tar.gz
+Source0:        vitastor-1.11.0.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.10.1
+Version:        1.11.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.10.1.el9.tar.gz
+Source0:        vitastor-1.11.0.el9.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -19,7 +19,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVITASTOR_VERSION="1.10.1")
+add_definitions(-DVITASTOR_VERSION="1.11.0")
 add_definitions(-D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/blockstore/blockstore_init.cpp

@@ -256,6 +256,7 @@ resume_2:
     }
     if (entries_to_zero.size() && !bs->inmemory_meta && !bs->readonly)
     {
+        std::sort(entries_to_zero.begin(), entries_to_zero.end());
         // we have to zero out additional entries
         for (i = 0; i < entries_to_zero.size(); )
         {
@@ -338,6 +339,15 @@ bool blockstore_init_meta::handle_meta_block(uint8_t *buf, uint64_t entries_per_
                 if (*entry_csum != crc32c(0, entry, bs->dsk.clean_entry_size - 4))
                 {
                     printf("Metadata entry %ju is corrupt (checksum mismatch), skipping\n", done_cnt+i);
+                    // zero out the invalid entry, otherwise we'll hit "tried to overwrite non-zero metadata entry" later
+                    if (bs->inmemory_meta)
+                    {
+                        memset(entry, 0, bs->dsk.clean_entry_size);
+                    }
+                    else
+                    {
+                        entries_to_zero.push_back(done_cnt+i);
+                    }
                     continue;
                 }
             }

src/client/cluster_client.cpp

@@ -63,6 +63,10 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
 
     st_cli.parse_config(config);
     st_cli.infinite_start = false;
+    if (!config["client_infinite_start"].is_null())
+    {
+        st_cli.infinite_start = config["client_infinite_start"].bool_value();
+    }
     st_cli.load_global_config();
 
     scrap_buffer_size = SCRAP_BUFFER_SIZE;
@@ -309,16 +313,17 @@ void cluster_client_t::erase_op(cluster_op_t *op)
     }
     if (flags & OP_FLUSH_BUFFER)
     {
+        auto overflow = std::move(wb->writeback_overflow);
         int i = 0;
-        while (i < wb->writeback_overflow.size() && wb->writebacks_active < client_max_writeback_iodepth)
+        while (i < overflow.size() && wb->writebacks_active < client_max_writeback_iodepth)
         {
-            execute_internal(wb->writeback_overflow[i]);
+            execute_internal(overflow[i]);
             i++;
         }
         if (i > 0)
-        {
+            overflow.erase(overflow.begin(), overflow.begin()+i);
-            wb->writeback_overflow.erase(wb->writeback_overflow.begin(), wb->writeback_overflow.begin()+i);
+        assert(!wb->writeback_overflow.size());
-        }
+        wb->writeback_overflow.swap(overflow);
     }
 }
 
@@ -673,7 +678,7 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
         }
         // Just copy and acknowledge the operation
         wb->copy_write(op, CACHE_DIRTY);
-        while (wb->writeback_bytes + op->len > client_max_buffered_bytes || wb->writeback_queue_size > client_max_buffered_ops)
+        while (wb->writeback_bytes > client_max_buffered_bytes || wb->writeback_queue_size > client_max_buffered_ops)
         {
             // Initiate some writeback (asynchronously)
             wb->start_writebacks(this, 1);
@@ -791,6 +796,36 @@ bool cluster_client_t::check_rw(cluster_op_t *op)
             return false;
         }
     }
+    op->deoptimise_snapshot = false;
+    if (enable_writeback && (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP))
+    {
+        auto ino_it = st_cli.inode_config.find(op->inode);
+        if (ino_it != st_cli.inode_config.end())
+        {
+            int chain_size = 0;
+            while (ino_it != st_cli.inode_config.end() && ino_it->second.parent_id)
+            {
+                // Check for loops - FIXME check it in etcd_state_client
+                if (ino_it->second.parent_id == op->inode ||
+                    chain_size > st_cli.inode_config.size())
+                {
+                    op->retval = -EINVAL;
+                    auto cb = std::move(op->callback);
+                    cb(op);
+                    return false;
+                }
+                if (INODE_POOL(ino_it->second.parent_id) == INODE_POOL(ino_it->first) &&
+                    wb->has_inode(ino_it->second.parent_id))
+                {
+                    // Deoptimise reads - we have dirty data for one of the parent layer(s).
+                    op->deoptimise_snapshot = true;
+                    break;
+                }
+                chain_size++;
+                ino_it = st_cli.inode_config.find(ino_it->second.parent_id);
+            }
+        }
+    }
     return true;
 }
 
@@ -922,12 +957,21 @@ resume_2:
         {
             // Check parent inode
             auto ino_it = st_cli.inode_config.find(op->cur_inode);
-            while (ino_it != st_cli.inode_config.end() && ino_it->second.parent_id &&
-                INODE_POOL(ino_it->second.parent_id) == INODE_POOL(op->cur_inode) &&
-                // Check for loops
-                ino_it->second.parent_id != op->inode)
-            {
             // Skip parents from the same pool
+            int skipped = 0;
+            while (!op->deoptimise_snapshot &&
+                ino_it != st_cli.inode_config.end() && ino_it->second.parent_id &&
+                INODE_POOL(ino_it->second.parent_id) == INODE_POOL(op->cur_inode))
+            {
+                // Check for loops - FIXME check it in etcd_state_client
+                if (ino_it->second.parent_id == op->inode ||
+                    skipped > st_cli.inode_config.size())
+                {
+                    op->retval = -EINVAL;
+                    erase_op(op);
+                    return 1;
+                }
+                skipped++;
                 ino_it = st_cli.inode_config.find(ino_it->second.parent_id);
             }
             if (ino_it != st_cli.inode_config.end() &&
@@ -1106,7 +1150,7 @@ void cluster_client_t::slice_rw(cluster_op_t *op)
             if (end == begin)
             {
                 op->done_count++;
-                op->parts[i].flags = PART_DONE;
+                op->parts[i].flags = PART_SENT|PART_DONE;
             }
         }
         else if (op->opcode != OSD_OP_READ_BITMAP && op->opcode != OSD_OP_READ_CHAIN_BITMAP && op->opcode != OSD_OP_DELETE)
@@ -1189,7 +1233,7 @@ int cluster_client_t::try_send(cluster_op_t *op, int i)
                 pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks
             );
             uint64_t meta_rev = 0;
-            if (op->opcode != OSD_OP_READ_BITMAP && op->opcode != OSD_OP_DELETE)
+            if (op->opcode != OSD_OP_READ_BITMAP && op->opcode != OSD_OP_DELETE && !op->deoptimise_snapshot)
             {
                 auto ino_it = st_cli.inode_config.find(op->cur_inode);
                 if (ino_it != st_cli.inode_config.end())

src/client/cluster_client.h

@@ -60,7 +60,8 @@ struct cluster_op_t
 protected:
     int state = 0;
     uint64_t cur_inode; // for snapshot reads
-    bool needs_reslice = false;
+    bool needs_reslice: 1;
+    bool deoptimise_snapshot: 1;
     int retry_after = 0;
     int inflight_count = 0, done_count = 0;
     timespec wait_up_until = {};

src/client/cluster_client_impl.h

@@ -42,10 +42,10 @@ public:
     std::multimap<uint64_t, uint64_t*> flushed_buffers; // flush_id => refcnt
 
     ~writeback_cache_t();
+    bool has_inode(uint64_t inode);
     dirty_buf_it_t find_dirty(uint64_t inode, uint64_t offset);
     bool is_left_merged(dirty_buf_it_t dirty_it);
     bool is_right_merged(dirty_buf_it_t dirty_it);
-    bool is_merged(const dirty_buf_it_t & dirty_it);
     void copy_write(cluster_op_t *op, int state, uint64_t new_flush_id = 0);
     int repeat_ops_for(cluster_client_t *cli, osd_num_t peer_osd, pool_id_t pool_id, pg_num_t pg_num);
     void start_writebacks(cluster_client_t *cli, int count);

src/client/cluster_client_wb.cpp

@@ -17,6 +17,15 @@ writeback_cache_t::~writeback_cache_t()
     dirty_buffers.clear();
 }
 
+bool writeback_cache_t::has_inode(uint64_t inode)
+{
+    auto dirty_it = dirty_buffers.lower_bound((object_id){
+        .inode = inode,
+        .stripe = 0,
+    });
+    return dirty_it != dirty_buffers.end() && dirty_it->first.inode == inode;
+}
+
 dirty_buf_it_t writeback_cache_t::find_dirty(uint64_t inode, uint64_t offset)
 {
     auto dirty_it = dirty_buffers.lower_bound((object_id){
@@ -33,8 +42,12 @@ dirty_buf_it_t writeback_cache_t::find_dirty(uint64_t inode, uint64_t offset)
             break;
         }
     }
+    if (dirty_it != dirty_buffers.end() && dirty_it->first.inode == inode)
+    {
         return dirty_it;
     }
+    return dirty_buffers.end();
+}
 
 bool writeback_cache_t::is_left_merged(dirty_buf_it_t dirty_it)
 {
@@ -68,11 +81,6 @@ bool writeback_cache_t::is_right_merged(dirty_buf_it_t dirty_it)
     return false;
 }
 
-bool writeback_cache_t::is_merged(const dirty_buf_it_t & dirty_it)
-{
-    return is_left_merged(dirty_it) || is_right_merged(dirty_it);
-}
-
 void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flush_id)
 {
     // Save operation for replay when one of PGs goes out of sync
@@ -109,8 +117,11 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
                 });
                 (*dirty_it->second.refcnt)++;
                 if (dirty_it->second.state == CACHE_DIRTY)
+                {
+                    if (dirty_it->second.buf)
                     {
                         writeback_bytes -= op->len;
+                    }
                     writeback_queue_size++;
                 }
                 break;
@@ -119,9 +130,12 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
             {
                 // Only leave the beginning
                 if (dirty_it->second.state == CACHE_DIRTY)
+                {
+                    if (dirty_it->second.buf)
                     {
                         writeback_bytes -= old_end - op->offset;
-                    if (is_left_merged(dirty_it) && !is_right_merged(dirty_it))
+                    }
+                    if (is_right_merged(dirty_it))
                     {
                         writeback_queue_size++;
                     }
@@ -134,9 +148,12 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
         {
             // Only leave the end
             if (dirty_it->second.state == CACHE_DIRTY)
+            {
+                if (dirty_it->second.buf)
                 {
                     writeback_bytes -= new_end - dirty_it->first.stripe;
-                if (!is_left_merged(dirty_it) && is_right_merged(dirty_it))
+                }
+                if (is_left_merged(dirty_it))
                 {
                     writeback_queue_size++;
                 }
@@ -158,12 +175,24 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
         else
         {
             // Remove the whole buffer
-            if (dirty_it->second.state == CACHE_DIRTY && !is_merged(dirty_it))
+            if (dirty_it->second.state == CACHE_DIRTY)
+            {
+                if (dirty_it->second.buf)
                 {
                     writeback_bytes -= dirty_it->second.len;
+                }
+                bool lm = is_left_merged(dirty_it);
+                bool rm = is_right_merged(dirty_it);
+                if (!lm && !rm)
+                {
                     assert(writeback_queue_size > 0);
                     writeback_queue_size--;
                 }
+                else if (lm && rm)
+                {
+                    writeback_queue_size++;
+                }
+            }
             if (!--(*dirty_it->second.refcnt))
             {
                 free(dirty_it->second.refcnt);
@@ -190,7 +219,9 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
     {
         writeback_bytes += is_del ? 0 : op->len;
         // Track consecutive write-back operations
-        if (!is_merged(dirty_it))
+        bool lm = is_left_merged(dirty_it);
+        bool rm = is_right_merged(dirty_it);
+        if (!lm && !rm)
         {
             // <writeback_queue> is OK to contain more than actual number of consecutive
             // requests as long as it doesn't miss anything. But <writeback_queue_size>
@@ -201,6 +232,11 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
                 .stripe = op->offset,
             });
         }
+        else if (lm && rm)
+        {
+            assert(writeback_queue_size > 0);
+            writeback_queue_size--;
+        }
     }
     if (!is_del)
     {
@@ -225,7 +261,7 @@ int writeback_cache_t::repeat_ops_for(cluster_client_t *cli, osd_num_t peer_osd,
         for (auto wr_it = dirty_buffers.begin(), flush_it = wr_it, last_it = wr_it; ; )
         {
             bool end = wr_it == dirty_buffers.end();
-            bool flush_this = !end && wr_it->second.state != CACHE_REPEATING;
+            bool flush_this = !end && wr_it->second.state != CACHE_REPEATING && wr_it->second.state != CACHE_DIRTY;
             if (peer_osd)
                 flush_this = flush_this && cli->affects_osd(wr_it->first.inode, wr_it->first.stripe, wr_it->second.len, peer_osd);
             if (pool_id && pg_num)
@@ -376,6 +412,7 @@ void writeback_cache_t::start_writebacks(cluster_client_t *cli, int count)
         assert(writeback_queue_size > 0);
         writeback_queue_size--;
         writeback_bytes -= off - from_it->first.stripe;
+        assert(writeback_queue_size > 0 || !writeback_bytes);
         flush_buffers(cli, from_it, to_it);
     }
     queue_copy.erase(queue_copy.begin(), queue_copy.begin()+i);

src/client/nbd_proxy.cpp

@@ -275,8 +275,8 @@ const char *help_text =
     "  --foreground 1\n"
     "    Stay in foreground, do not daemonize.\n"
     "\n"
-    "vitastor-nbd unmap /dev/nbdN\n"
+    "vitastor-nbd unmap [--force] /dev/nbdN\n"
-    "  Unmap an ioctl-mapped NBD device.\n"
+    "  Unmap an ioctl-mapped NBD device. Do not check if it's actually mapped if --force is specified.\n"
     "\n"
     "vitastor-nbd ls [--json]\n"
     "  List ioctl-mapped Vitastor NBD devices, optionally in JSON format.\n"
@@ -313,7 +313,7 @@ const char *help_text =
 #endif
     "Use vitastor-nbd --help <command> for command details or vitastor-nbd --help --all for all details.\n"
     "\n"
-    "All usual Vitastor config options like --config_file <path_to_config> may also be specified in CLI.\n"
+    "All usual Vitastor config options like --config_path <path_to_config> may also be specified in CLI.\n"
 ;
 
 class nbd_proxy
@@ -372,7 +372,8 @@ public:
             else if (args[i][0] == '-' && args[i][1] == '-')
             {
                 const char *opt = args[i]+2;
-                cfg[opt] = !strcmp(opt, "json") || !strcmp(opt, "all") || i == narg-1 ? "1" : args[++i];
+                cfg[opt] = !strcmp(opt, "json") || !strcmp(opt, "all") ||
+                    !strcmp(opt, "force") || i == narg-1 ? "1" : args[++i];
             }
             else if (pos == 0)
             {
@@ -381,8 +382,9 @@ public:
             }
             else if (pos == 1)
             {
+                char c = 0;
                 int n = 0;
-                if (sscanf(args[i], "/dev/nbd%d", &n) > 0)
+                if (sscanf(args[i], "/dev/nbd%d%c", &n, &c) == 1)
                     cfg["dev_num"] = n;
                 else
                     cfg["dev_num"] = args[i];
@@ -404,18 +406,14 @@ public:
         }
         else if (cfg["command"] == "unmap")
         {
-            if (cfg["dev_num"].is_null())
+            if (!cfg["dev_num"].is_number() &&
+                cfg["dev_num"].string_value() != "0" &&
+                !cfg["dev_num"].uint64_value())
             {
                 fprintf(stderr, "device name or number is missing\n");
                 exit(1);
             }
-            if (cfg["netlink"].is_null())
+            ioctl_unmap(cfg["dev_num"].uint64_value(), cfg["force"].bool_value());
-            {
-                ioctl_unmap(cfg["dev_num"].uint64_value());
-            }
-            else
-            {
-            }
         }
 #ifdef HAVE_NBD_NETLINK_H
         else if (cfg["command"] == "netlink-map")
@@ -444,9 +442,18 @@ help:
         }
     }
 
-    void ioctl_unmap(int dev_num)
+    void ioctl_unmap(int dev_num, bool force)
     {
         char path[64] = { 0 };
+        // Check if mapped
+        sprintf(path, "/sys/block/nbd%d/pid", dev_num);
+        if (access(path, F_OK) != 0)
+        {
+            fprintf(stderr, "/dev/nbd%d is not mapped: /sys/block/nbd%d/pid does not exist\n", dev_num, dev_num);
+            if (!force)
+                exit(1);
+        }
+        // Run unmap
         sprintf(path, "/dev/nbd%d", dev_num);
         int r, nbd = open(path, O_RDWR);
         if (nbd < 0)
@@ -610,36 +617,43 @@ help:
         {
             if (!cfg["dev_num"].is_null())
             {
-                if (run_nbd(sockfd, cfg["dev_num"].int64_value(), device_size, NBD_FLAG_SEND_FLUSH, nbd_timeout, bg) < 0)
+                int r;
+                if ((r = run_nbd(sockfd, cfg["dev_num"].int64_value(), device_size, NBD_FLAG_SEND_FLUSH, nbd_timeout, bg)) != 0)
                 {
-                    perror("run_nbd");
+                    fprintf(stderr, "run_nbd: %s\n", strerror(-r));
                     exit(1);
                 }
             }
             else
             {
                 // Find an unused device
+                auto mapped = list_mapped();
                 int i = 0;
                 while (true)
                 {
+                    if (mapped.find("/dev/nbd"+std::to_string(i)) != mapped.end())
+                    {
+                        i++;
+                        continue;
+                    }
                     int r = run_nbd(sockfd, i, device_size, NBD_FLAG_SEND_FLUSH, nbd_timeout, bg);
                     if (r == 0)
                     {
                         printf("/dev/nbd%d\n", i);
                         break;
                     }
-                    else if (r == -1 && errno == ENOENT)
+                    else if (r == -ENOENT)
                     {
                         fprintf(stderr, "No free NBD devices found\n");
                         exit(1);
                     }
-                    else if (r == -2 && errno == EBUSY)
+                    else if (r == -EBUSY)
                     {
                         i++;
                     }
                     else
                     {
-                        perror("run_nbd");
+                        fprintf(stderr, "run_nbd: %s\n", strerror(-r));
                         exit(1);
                     }
                 }
@@ -869,11 +883,37 @@ protected:
         // Check handle size
         assert(sizeof(cur_req.handle) == 8);
         char path[64] = { 0 };
+        int notifyfd[2] = { 0 };
+        if (socketpair(AF_UNIX, SOCK_STREAM, 0, notifyfd) < 0)
+        {
+            return -errno;
+        }
+        if (!fork())
+        {
+            // Do all NBD configuration in the child process, after the last fork.
+            // Why? It's needed because there is a race condition in the Linux kernel nbd driver
+            // in nbd_add_socket() - it saves `current` task pointer as `nbd->task_setup` and
+            // then rechecks if the new `current` is the same. Problem is that if that process
+            // is already dead, `current` may be freed and then replaced by another process
+            // with the same pointer value. So the check passes and NBD allows a different process
+            // to set up a device which is already set up. Proper fix would have to be done in the
+            // kernel code, but the workaround is obviously to perform NBD setup from the process
+            // which will then actually call NBD_DO_IT. That process stays alive during the whole
+            // time of NBD device execution and the (nbd->task_setup != current) check always
+            // works correctly, and we don't accidentally break previous NBD devices while setting
+            // up a new device. Forking to check every device is of course rather slow, so we also
+            // do an additional check by calling list_mapped() before searching for a free NBD device.
+            if (bg)
+            {
+                daemonize_fork();
+            }
+            close(notifyfd[0]);
             sprintf(path, "/dev/nbd%d", dev_num);
             int r, nbd = open(path, O_RDWR), qd_fd;
             if (nbd < 0)
             {
-            return -1;
+                write(notifyfd[1], &errno, sizeof(errno));
+                exit(1);
             }
             r = ioctl(nbd, NBD_SET_SOCK, sockfd[1]);
             if (r < 0)
@@ -912,38 +952,45 @@ protected:
                 fprintf(stderr, "Warning: Failed to configure max_sectors_kb\n");
             }
             close(qd_fd);
-        if (!fork())
+            // Notify parent
-        {
+            errno = 0;
-            // Run in child
+            write(notifyfd[1], &errno, sizeof(errno));
+            close(notifyfd[1]);
             close(sockfd[0]);
             if (bg)
             {
-                daemonize();
+                daemonize_reopen_stdio();
             }
             r = ioctl(nbd, NBD_DO_IT);
             if (r < 0)
             {
-                fprintf(stderr, "NBD device terminated with error: %s\n", strerror(errno));
+                fprintf(stderr, "NBD device /dev/nbd%d terminated with error: %s\n", dev_num, strerror(errno));
             }
             close(sockfd[1]);
             ioctl(nbd, NBD_CLEAR_QUE);
             ioctl(nbd, NBD_CLEAR_SOCK);
             exit(0);
-        }
-        close(sockfd[1]);
-        close(nbd);
-        return 0;
     end_close:
-        r = errno;
+            write(notifyfd[1], &errno, sizeof(errno));
             close(nbd);
-        errno = r;
+            exit(2);
-        return -2;
     end_unmap:
-        r = errno;
+            write(notifyfd[1], &errno, sizeof(errno));
             ioctl(nbd, NBD_CLEAR_SOCK);
             close(nbd);
-        errno = r;
+            exit(3);
-        return -3;
+        }
+        // Parent - check status
+        close(notifyfd[1]);
+        int child_errno = 0;
+        int ok = read(notifyfd[0], &child_errno, sizeof(child_errno));
+        close(notifyfd[0]);
+        if (ok && !child_errno)
+        {
+            close(sockfd[1]);
+            return 0;
+        }
+        return -child_errno;
     }
 
     void submit_send()

src/client/qemu_driver.c

@@ -294,7 +294,9 @@ static void coroutine_fn vitastor_co_get_metadata(VitastorRPC *task)
 
     qemu_mutex_lock(&client->mutex);
     vitastor_c_watch_inode(client->proxy, client->image, vitastor_co_generic_cb, task);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
     vitastor_schedule_uring_handler(client);
+#endif
     qemu_mutex_unlock(&client->mutex);
 
     while (!task->complete)
@@ -749,7 +751,9 @@ static int coroutine_fn vitastor_co_preadv(BlockDriverState *bs,
     uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode;
     qemu_mutex_lock(&client->mutex);
     vitastor_c_read(client->proxy, inode, offset, bytes, iov->iov, iov->niov, vitastor_co_read_cb, &task);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
     vitastor_schedule_uring_handler(client);
+#endif
     qemu_mutex_unlock(&client->mutex);
 
     while (!task.complete)
@@ -783,7 +787,9 @@ static int coroutine_fn vitastor_co_pwritev(BlockDriverState *bs,
     uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode;
     qemu_mutex_lock(&client->mutex);
     vitastor_c_write(client->proxy, inode, offset, bytes, 0, iov->iov, iov->niov, vitastor_co_generic_cb, &task);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
     vitastor_schedule_uring_handler(client);
+#endif
     qemu_mutex_unlock(&client->mutex);
 
     while (!task.complete)
@@ -863,7 +869,9 @@ static int coroutine_fn vitastor_co_block_status(
         task.bitmap = client->last_bitmap = NULL;
         qemu_mutex_lock(&client->mutex);
         vitastor_c_read_bitmap(client->proxy, task.inode, task.offset, task.len, !client->skip_parents, vitastor_co_read_bitmap_cb, &task);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
         vitastor_schedule_uring_handler(client);
+#endif
         qemu_mutex_unlock(&client->mutex);
         while (!task.complete)
         {
@@ -950,7 +958,9 @@ static int coroutine_fn vitastor_co_flush(BlockDriverState *bs)
 
     qemu_mutex_lock(&client->mutex);
     vitastor_c_sync(client->proxy, vitastor_co_generic_cb, &task);
+#if !defined VITASTOR_C_API_VERSION || VITASTOR_C_API_VERSION < 5
     vitastor_schedule_uring_handler(client);
+#endif
     qemu_mutex_unlock(&client->mutex);
 
     while (!task.complete)

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 1.10.1
+Version: 1.11.0
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/client/vitastor_c.cpp

@@ -127,6 +127,7 @@ vitastor_c *vitastor_c_create_qemu_uring(QEMUSetFDHandler *aio_set_fd_handler, v
     auto self = vitastor_c_create_qemu_common(aio_set_fd_handler, aio_context);
     self->ringloop = ringloop;
     self->cli = new cluster_client_t(self->ringloop, self->tfd, cfg_json);
+    ringloop->loop_continue();
     return self;
 }
 
@@ -150,6 +151,7 @@ vitastor_c *vitastor_c_create_uring(const char *config_path, const char *etcd_ho
     self->ringloop = ringloop;
     self->epmgr = new epoll_manager_t(self->ringloop);
     self->cli = new cluster_client_t(self->ringloop, self->epmgr->tfd, cfg_json);
+    ringloop->loop_continue();
     return self;
 }
 
@@ -183,6 +185,7 @@ vitastor_c *vitastor_c_create_uring_json(const char **options, int options_len)
     self->ringloop = ringloop;
     self->epmgr = new epoll_manager_t(self->ringloop);
     self->cli = new cluster_client_t(self->ringloop, self->epmgr->tfd, cfg_json);
+    ringloop->loop_continue();
     return self;
 }
 
@@ -222,6 +225,18 @@ int vitastor_c_is_ready(vitastor_c *client)
     return client->cli->is_ready();
 }
 
+void vitastor_c_on_ready(vitastor_c *client, VitastorIOHandler cb, void *opaque)
+{
+    client->cli->on_ready([=]()
+    {
+        cb(opaque, 0);
+    });
+    if (client->ringloop)
+    {
+        client->ringloop->loop_continue();
+    }
+}
+
 void vitastor_c_uring_wait_ready(vitastor_c *client)
 {
     while (!client->cli->is_ready())
@@ -276,6 +291,10 @@ void vitastor_c_read(vitastor_c *client, uint64_t inode, uint64_t offset, uint64
         delete op;
     };
     client->cli->execute(op);
+    if (client->ringloop)
+    {
+        client->ringloop->loop_continue();
+    }
 }
 
 void vitastor_c_write(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, uint64_t check_version,
@@ -297,6 +316,31 @@ void vitastor_c_write(vitastor_c *client, uint64_t inode, uint64_t offset, uint6
         delete op;
     };
     client->cli->execute(op);
+    if (client->ringloop)
+    {
+        client->ringloop->loop_continue();
+    }
+}
+
+void vitastor_c_delete(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, uint64_t check_version,
+    VitastorIOHandler cb, void *opaque)
+{
+    cluster_op_t *op = new cluster_op_t;
+    op->opcode = OSD_OP_DELETE;
+    op->inode = inode;
+    op->offset = offset;
+    op->len = len;
+    op->version = check_version;
+    op->callback = [cb, opaque](cluster_op_t *op)
+    {
+        cb(opaque, op->retval);
+        delete op;
+    };
+    client->cli->execute(op);
+    if (client->ringloop)
+    {
+        client->ringloop->loop_continue();
+    }
 }
 
 void vitastor_c_read_bitmap(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len,
@@ -319,6 +363,10 @@ void vitastor_c_read_bitmap(vitastor_c *client, uint64_t inode, uint64_t offset,
         delete op;
     };
     client->cli->execute(op);
+    if (client->ringloop)
+    {
+        client->ringloop->loop_continue();
+    }
 }
 
 void vitastor_c_sync(vitastor_c *client, VitastorIOHandler cb, void *opaque)
@@ -331,6 +379,10 @@ void vitastor_c_sync(vitastor_c *client, VitastorIOHandler cb, void *opaque)
         delete op;
     };
     client->cli->execute(op);
+    if (client->ringloop)
+    {
+        client->ringloop->loop_continue();
+    }
 }
 
 void vitastor_c_watch_inode(vitastor_c *client, char *image, VitastorIOHandler cb, void *opaque)
@@ -340,6 +392,10 @@ void vitastor_c_watch_inode(vitastor_c *client, char *image, VitastorIOHandler c
         auto watch = client->cli->st_cli.watch_inode(std::string(image));
         cb(opaque, (long)watch);
     });
+    if (client->ringloop)
+    {
+        client->ringloop->loop_continue();
+    }
 }
 
 void vitastor_c_close_watch(vitastor_c *client, void *handle)

src/client/vitastor_c.h

@@ -7,7 +7,7 @@
 #define VITASTOR_QEMU_PROXY_H
 
 // C API wrapper version
-#define VITASTOR_C_API_VERSION 4
+#define VITASTOR_C_API_VERSION 5
 
 #ifndef POOL_ID_BITS
 #define POOL_ID_BITS 16
@@ -51,6 +51,7 @@ vitastor_c *vitastor_c_create_epoll_json(const char **options, int options_len);
 void* vitastor_c_get_internal_client(vitastor_c *client);
 void vitastor_c_destroy(vitastor_c *client);
 int vitastor_c_is_ready(vitastor_c *client);
+void vitastor_c_on_ready(vitastor_c *client, VitastorIOHandler cb, void *opaque);
 int vitastor_c_uring_register_eventfd(vitastor_c *client);
 void vitastor_c_uring_wait_ready(vitastor_c *client);
 void vitastor_c_uring_handle_events(vitastor_c *client);
@@ -62,6 +63,8 @@ void vitastor_c_read(vitastor_c *client, uint64_t inode, uint64_t offset, uint64
     struct iovec *iov, int iovcnt, VitastorReadHandler cb, void *opaque);
 void vitastor_c_write(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, uint64_t check_version,
     struct iovec *iov, int iovcnt, VitastorIOHandler cb, void *opaque);
+void vitastor_c_delete(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, uint64_t check_version,
+    VitastorIOHandler cb, void *opaque);
 void vitastor_c_read_bitmap(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len,
     int with_parents, VitastorReadBitmapHandler cb, void *opaque);
 void vitastor_c_sync(vitastor_c *client, VitastorIOHandler cb, void *opaque);

src/cmd/cli.cpp

@@ -224,7 +224,7 @@ static const char* help_text =
     "Use vitastor-cli --help <command> for command details or vitastor-cli --help --all for all details.\n"
     "\n"
     "GLOBAL OPTIONS:\n"
-    "  --config_file FILE  Path to Vitastor configuration file\n"
+    "  --config_path FILE  Path to Vitastor configuration file\n"
     "  --etcd_address URL  Etcd connection address\n"
     "  --iodepth N         Send N operations in parallel to each OSD when possible (default 32)\n"
     "  --parallel_osds M   Work with M osds in parallel when possible (default 4)\n"

src/cmd/cli_create.cpp

@@ -215,6 +215,7 @@ resume_3:
             goto resume_3;
         else if (state == 4)
             goto resume_4;
+        // FIXME: take all info from etcd requests, not mixed with st_cli.inode_config
         for (auto & ic: parent->cli->st_cli.inode_config)
         {
             if (ic.second.name == image_name+"@"+new_snap)
@@ -286,6 +287,7 @@ resume_4:
 
     json11::Json::object get_next_id()
     {
+        assert(new_pool_id);
         return json11::Json::object {
             { "request_range", json11::Json::object {
                 { "key", base64_encode(
@@ -321,6 +323,17 @@ resume_4:
             goto resume_2;
         else if (state == 3)
             goto resume_3;
+        if (!new_pool_id)
+        {
+            for (auto & ic: parent->cli->st_cli.inode_config)
+            {
+                if (ic.second.name == image_name)
+                {
+                    new_pool_id = INODE_POOL(ic.first);
+                    break;
+                }
+            }
+        }
         parent->etcd_txn(json11::Json::object { { "success", json11::Json::array {
             get_next_id(),
             json11::Json::object {

src/cmd/cli_ls.cpp

@@ -546,7 +546,7 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_ls(json11::Json cfg)
     lister->list_pool_name = lister->list_pool_id ? "" : cfg["pool"].as_string();
     lister->show_stats = cfg["long"].bool_value();
     lister->show_delete = cfg["del"].bool_value();
-    lister->sort_field = cfg["sort"].string_value();
+    lister->sort_field = cfg["sort"].string_value() != "" ? cfg["sort"].string_value() : "name";
     lister->reverse = cfg["reverse"].bool_value();
     lister->max_count = cfg["count"].uint64_value();
     for (auto & item: cfg["names"].array_items())

src/osd/osd.h

@@ -281,6 +281,8 @@ class osd_t
     int pick_next_scrub(object_id & next_oid);
     void submit_scrub_op(object_id oid);
     bool continue_scrub();
+    void submit_scrub_subops(osd_op_t *cur_op);
+    void scrub_check_results(osd_op_t *cur_op);
     void plan_scrub(pg_t & pg, bool report_state = true);
     void schedule_scrub(pg_t & pg);
 
@@ -313,7 +315,7 @@ class osd_t
     pg_osd_set_state_t *mark_object(pg_t & pg, object_id oid, pg_osd_set_state_t *prev_object_state, bool ref,
         std::function<int(pg_osd_set_t & new_set)> calc_set);
     pg_osd_set_state_t *mark_object_corrupted(pg_t & pg, object_id oid, pg_osd_set_state_t *prev_object_state,
-        osd_rmw_stripe_t *stripes, bool ref, bool inconsistent);
+        osd_rmw_stripe_t *stripes, bool ref);
     pg_osd_set_state_t *mark_partial_write(pg_t & pg, object_id oid, pg_osd_set_state_t *prev_object_state,
         osd_rmw_stripe_t *stripes, bool ref);
     void deref_object_state(pg_t & pg, pg_osd_set_state_t **object_state, bool deref);
@@ -326,6 +328,8 @@ class osd_t
     void submit_primary_subops(int submit_type, uint64_t op_version, const uint64_t* osd_set, osd_op_t *cur_op);
     int submit_primary_subop_batch(int submit_type, inode_t inode, uint64_t op_version,
         osd_rmw_stripe_t *stripes, const uint64_t* osd_set, osd_op_t *cur_op, int subop_idx, int zero_read);
+    void submit_primary_subop(osd_op_t *cur_op, osd_op_t *subop,
+        osd_rmw_stripe_t *si, bool wr, inode_t inode, uint64_t op_version);
     void submit_primary_del_subops(osd_op_t *cur_op, uint64_t *cur_set, uint64_t set_size, pg_osd_set_t & loc_set);
     void submit_primary_del_batch(osd_op_t *cur_op, obj_ver_osd_t *chunks_to_delete, int chunks_to_delete_count);
     int submit_primary_sync_subops(osd_op_t *cur_op);

src/osd/osd_main.cpp

@@ -3,6 +3,7 @@
 
 #include "osd.h"
 
+#include <sys/prctl.h>
 #include <signal.h>
 
 static osd_t *osd = NULL;
@@ -56,6 +57,9 @@ int main(int narg, char *args[])
         printf("%s", help_text);
         return 1;
     }
+    char osdname[16] = { 0 };
+    snprintf(osdname, 16, "osd%lu", config["osd_num"].uint64_value());
+    prctl(PR_SET_NAME, (unsigned long)osdname, 0, 0, 0);
     signal(SIGINT, handle_sigint);
     signal(SIGTERM, handle_sigint);
     ring_loop_t *ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);

src/osd/osd_primary.cpp

@@ -28,7 +28,6 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
         return false;
     }
     auto & pool_cfg = pool_cfg_it->second;
-    // FIXME: op_data->pg_data_size can probably be removed (there's pg.pg_data_size)
     uint64_t pg_data_size = (pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks);
     uint64_t pg_block_size = bs_block_size * pg_data_size;
     object_id oid = {
@@ -52,13 +51,13 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
         finish_op(cur_op, -EINVAL);
         return false;
     }
-    // Scrub is similar to r/w, so it's also handled here
+    int stripe_count = (cur_op->req.hdr.opcode == OSD_OP_SCRUB ? 0 :
-    int stripe_count = (pool_cfg.scheme == POOL_SCHEME_REPLICATED
+        (pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pg_it->second.pg_size));
-        && cur_op->req.hdr.opcode != OSD_OP_SCRUB ? 1 : pg_it->second.pg_size);
     int chain_size = 0;
     if (cur_op->req.hdr.opcode == OSD_OP_READ && cur_op->req.rw.meta_revision > 0)
     {
         // Chained read
+        // FIXME: Introduce an explicit opcode for chained reads
         auto inode_it = st_cli.inode_config.find(cur_op->req.rw.inode);
         if (inode_it->second.mod_revision != cur_op->req.rw.meta_revision)
         {
@@ -72,7 +71,7 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
             inode_it->second.parent_id &&
             INODE_POOL(inode_it->second.parent_id) == pg_it->second.pool_id)
         {
-            // Check for loops
+            // Check for loops - FIXME check it in etcd_state_client
             if (inode_it->second.parent_id == cur_op->req.rw.inode ||
                 inode_it->second.parent_id == inode_it->second.num ||
                 chain_size > st_cli.inode_config.size())
@@ -110,13 +109,14 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
     );
     void *data_buf = (uint8_t*)op_data + sizeof(osd_primary_op_data_t);
     op_data->pg_num = pg_num;
+    op_data->pg = &pg_it->second;
     op_data->oid = oid;
     op_data->stripes = (osd_rmw_stripe_t*)data_buf;
+    op_data->stripe_count = stripe_count;
     data_buf = (uint8_t*)data_buf + sizeof(osd_rmw_stripe_t) * stripe_count;
-    op_data->scheme = pool_cfg.scheme;
-    op_data->pg_data_size = pg_data_size;
-    op_data->pg_size = pg_it->second.pg_size;
     cur_op->op_data = op_data;
+    if (cur_op->req.hdr.opcode != OSD_OP_SCRUB)
+    {
         split_stripes(pg_data_size, bs_block_size, (uint32_t)(cur_op->req.rw.offset - oid.stripe), cur_op->req.rw.len, op_data->stripes);
         // Resulting bitmaps have to survive op_data and be freed with the op itself
         assert(!cur_op->bitmap_buf);
@@ -125,6 +125,7 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
         {
             op_data->stripes[i].bmp_buf = (uint8_t*)cur_op->bitmap_buf + clean_entry_bitmap_size * i;
         }
+    }
     op_data->chain_size = chain_size;
     if (chain_size > 0)
     {
@@ -205,11 +206,11 @@ void osd_t::continue_primary_read(osd_op_t *cur_op)
 resume_0:
     cur_op->reply.rw.bitmap_len = 0;
     {
-        auto & pg = pgs.at({ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num });
+        auto & pg = *op_data->pg;
         if (cur_op->req.rw.len == 0)
         {
             // len=0 => bitmap read
-            for (int role = 0; role < op_data->pg_data_size; role++)
+            for (int role = 0; role < pg.pg_data_size; role++)
             {
                 op_data->stripes[role].read_start = 0;
                 op_data->stripes[role].read_end = UINT32_MAX;
@@ -217,7 +218,7 @@ resume_0:
         }
         else
         {
-            for (int role = 0; role < op_data->pg_data_size; role++)
+            for (int role = 0; role < pg.pg_data_size; role++)
             {
                 op_data->stripes[role].read_start = op_data->stripes[role].req_start;
                 op_data->stripes[role].read_end = op_data->stripes[role].req_end;
@@ -228,29 +229,27 @@ resume_0:
         op_data->target_ver = vo_it != pg.ver_override.end() ? vo_it->second : UINT64_MAX;
         // PG may have degraded or misplaced objects
         op_data->prev_set = get_object_osd_set(pg, op_data->oid, &op_data->object_state);
-        if (pg.state == PG_ACTIVE || op_data->scheme == POOL_SCHEME_REPLICATED)
+        if (pg.state == PG_ACTIVE || pg.scheme == POOL_SCHEME_REPLICATED)
         {
             // Fast happy-path
-            if (op_data->scheme == POOL_SCHEME_REPLICATED &&
+            if (pg.scheme == POOL_SCHEME_REPLICATED &&
                 op_data->object_state && (op_data->object_state->state & OBJ_INCOMPLETE))
             {
                 finish_op(cur_op, -EIO);
                 return;
             }
-            cur_op->buf = alloc_read_buffer(op_data->stripes, op_data->pg_data_size, 0);
+            cur_op->buf = alloc_read_buffer(op_data->stripes, pg.pg_data_size, 0);
             submit_primary_subops(SUBMIT_RMW_READ, op_data->target_ver, op_data->prev_set, cur_op);
             op_data->st = 1;
         }
         else
         {
-            if (extend_missing_stripes(op_data->stripes, op_data->prev_set, op_data->pg_data_size, pg.pg_size) < 0)
+            if (extend_missing_stripes(op_data->stripes, op_data->prev_set, pg.pg_data_size, pg.pg_size) < 0)
             {
                 finish_op(cur_op, -EIO);
                 return;
             }
             // Submit reads
-            op_data->pg_size = pg.pg_size;
-            op_data->scheme = pg.scheme;
             op_data->degraded = 1;
             cur_op->buf = alloc_read_buffer(op_data->stripes, pg.pg_size, 0);
             submit_primary_subops(SUBMIT_RMW_READ, op_data->target_ver, op_data->prev_set, cur_op);
@@ -265,30 +264,29 @@ resume_2:
         if (op_data->errcode == -EIO || op_data->errcode == -EDOM)
         {
             // I/O or checksum error
-            auto & pg = pgs.at({ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num });
             // FIXME: ref = true ideally... because new_state != state is not necessarily true if it's freed and recreated
-            op_data->object_state = mark_object_corrupted(pg, op_data->oid, op_data->object_state, op_data->stripes, false, false);
+            op_data->object_state = mark_object_corrupted(*op_data->pg, op_data->oid, op_data->object_state, op_data->stripes, false);
             goto resume_0;
         }
         finish_op(cur_op, op_data->errcode);
         return;
     }
     cur_op->reply.rw.version = op_data->fact_ver;
-    cur_op->reply.rw.bitmap_len = op_data->pg_data_size * clean_entry_bitmap_size;
+    cur_op->reply.rw.bitmap_len = op_data->pg->pg_data_size * clean_entry_bitmap_size;
     if (op_data->degraded)
     {
         // Reconstruct missing stripes
         osd_rmw_stripe_t *stripes = op_data->stripes;
-        if (op_data->scheme == POOL_SCHEME_XOR)
+        if (op_data->pg->scheme == POOL_SCHEME_XOR)
         {
-            reconstruct_stripes_xor(stripes, op_data->pg_size, clean_entry_bitmap_size);
+            reconstruct_stripes_xor(stripes, op_data->pg->pg_size, clean_entry_bitmap_size);
         }
-        else if (op_data->scheme == POOL_SCHEME_EC)
+        else if (op_data->pg->scheme == POOL_SCHEME_EC)
         {
-            reconstruct_stripes_ec(stripes, op_data->pg_size, op_data->pg_data_size, clean_entry_bitmap_size);
+            reconstruct_stripes_ec(stripes, op_data->pg->pg_size, op_data->pg->pg_data_size, clean_entry_bitmap_size);
         }
         cur_op->iov.push_back(op_data->stripes[0].bmp_buf, cur_op->reply.rw.bitmap_len);
-        for (int role = 0; role < op_data->pg_size; role++)
+        for (int role = 0; role < op_data->pg->pg_size; role++)
         {
             if (stripes[role].req_end != 0)
             {
@@ -360,10 +358,10 @@ pg_osd_set_state_t *osd_t::mark_object(pg_t & pg, object_id oid, pg_osd_set_stat
     return object_state;
 }
 
-pg_osd_set_state_t *osd_t::mark_object_corrupted(pg_t & pg, object_id oid, pg_osd_set_state_t *prev_object_state,
+pg_osd_set_state_t *osd_t::mark_object_corrupted(pg_t & pg, object_id oid,
-    osd_rmw_stripe_t *stripes, bool ref, bool inconsistent)
+    pg_osd_set_state_t *prev_object_state, osd_rmw_stripe_t *stripes, bool ref)
 {
-    return mark_object(pg, oid, prev_object_state, ref, [stripes, inconsistent](pg_osd_set_t & new_set)
+    return mark_object(pg, oid, prev_object_state, ref, [stripes](pg_osd_set_t & new_set)
     {
         // Mark object chunk(s) as corrupted
         int changes = 0;
@@ -390,16 +388,6 @@ pg_osd_set_state_t *osd_t::mark_object_corrupted(pg_t & pg, object_id oid, pg_os
                     chunk.loc_bad &= ~LOC_CORRUPTED;
                 }
             }
-            if (inconsistent && !chunk.loc_bad)
-            {
-                changes++;
-                chunk.loc_bad |= LOC_INCONSISTENT;
-            }
-            else if (!inconsistent && (chunk.loc_bad & LOC_INCONSISTENT))
-            {
-                changes++;
-                chunk.loc_bad &= ~LOC_INCONSISTENT;
-            }
             chunk_it++;
         }
         return changes;
@@ -695,7 +683,7 @@ void osd_t::continue_primary_del(osd_op_t *cur_op)
         return;
     }
     osd_primary_op_data_t *op_data = cur_op->op_data;
-    auto & pg = pgs.at({ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num });
+    auto & pg = *op_data->pg;
     if (op_data->st == 1)      goto resume_1;
     else if (op_data->st == 2) goto resume_2;
     else if (op_data->st == 3) goto resume_3;

src/osd/osd_primary.h

@@ -20,14 +20,15 @@ struct unstable_osd_num_t
 struct osd_primary_op_data_t
 {
     int st = 0;
-    pg_num_t pg_num;
+    pg_num_t pg_num = 0;
-    object_id oid;
+    object_id oid = {};
-    uint64_t target_ver;
+    uint64_t target_ver = 0;
     uint64_t orig_ver = 0, fact_ver = 0;
-    uint64_t scheme = 0;
     int n_subops = 0, done = 0, errors = 0, drops = 0, errcode = 0;
-    int degraded = 0, pg_size, pg_data_size;
+    int degraded = 0;
-    osd_rmw_stripe_t *stripes;
+    int stripe_count = 0;
+    osd_rmw_stripe_t *stripes = NULL;
+    pg_t *pg = NULL;
     osd_op_t *subops = NULL;
     uint64_t *prev_set = NULL;
     pg_osd_set_state_t *object_state = NULL;

src/osd/osd_primary_chain.cpp

@@ -7,7 +7,7 @@
 void osd_t::continue_chained_read(osd_op_t *cur_op)
 {
     osd_primary_op_data_t *op_data = cur_op->op_data;
-    auto & pg = pgs.at({ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num });
+    auto & pg = *op_data->pg;
     if (op_data->st == 1)
         goto resume_1;
     else if (op_data->st == 2)
@@ -17,7 +17,7 @@ void osd_t::continue_chained_read(osd_op_t *cur_op)
     else if (op_data->st == 4)
         goto resume_4;
     cur_op->reply.rw.bitmap_len = 0;
-    for (int role = 0; role < op_data->pg_data_size; role++)
+    for (int role = 0; role < pg.pg_data_size; role++)
     {
         op_data->stripes[role].read_start = op_data->stripes[role].req_start;
         op_data->stripes[role].read_end = op_data->stripes[role].req_end;
@@ -329,7 +329,7 @@ std::vector<osd_chain_read_t> osd_t::collect_chained_read_requests(osd_op_t *cur
 {
     osd_primary_op_data_t *op_data = cur_op->op_data;
     std::vector<osd_chain_read_t> chain_reads;
-    int stripe_count = (op_data->scheme == POOL_SCHEME_REPLICATED ? 1 : op_data->pg_size);
+    int stripe_count = (op_data->pg->scheme == POOL_SCHEME_REPLICATED ? 1 : op_data->pg->pg_size);
     memset(op_data->stripes[0].bmp_buf, 0, stripe_count * clean_entry_bitmap_size);
     uint8_t *global_bitmap = (uint8_t*)op_data->stripes[0].bmp_buf;
     // We always use at most 1 read request per layer
@@ -337,7 +337,7 @@ std::vector<osd_chain_read_t> osd_t::collect_chained_read_requests(osd_op_t *cur
     {
         uint8_t *part_bitmap = ((uint8_t*)op_data->snapshot_bitmaps) + chain_pos*stripe_count*clean_entry_bitmap_size;
         int start = !cur_op->req.rw.len ? 0 : (cur_op->req.rw.offset - op_data->oid.stripe)/bs_bitmap_granularity;
-        int end = !cur_op->req.rw.len ? op_data->pg_data_size*clean_entry_bitmap_size*8 : start + cur_op->req.rw.len/bs_bitmap_granularity;
+        int end = !cur_op->req.rw.len ? op_data->pg->pg_data_size*clean_entry_bitmap_size*8 : start + cur_op->req.rw.len/bs_bitmap_granularity;
         // Skip unneeded part in the beginning
         while (start < end && (
             ((global_bitmap[start>>3] >> (start&7)) & 1) ||
@@ -410,11 +410,11 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
         // FIXME: maybe introduce split_read_stripes to shorten these lines and to remove read_start=req_start
         osd_rmw_stripe_t *stripes = chain_stripes + chain_reads[cri].chain_pos*stripe_count;
         split_stripes(pg.pg_data_size, bs_block_size, chain_reads[cri].offset, chain_reads[cri].len, stripes);
-        if (op_data->scheme == POOL_SCHEME_REPLICATED && !stripes[0].req_end)
+        if (pg.scheme == POOL_SCHEME_REPLICATED && !stripes[0].req_end)
         {
             continue;
         }
-        for (int role = 0; role < op_data->pg_data_size; role++)
+        for (int role = 0; role < pg.pg_data_size; role++)
         {
             stripes[role].read_start = stripes[role].req_start;
             stripes[role].read_end = stripes[role].req_end;
@@ -423,7 +423,7 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
         if (pg.state != PG_ACTIVE)
         {
             cur_set = get_object_osd_set(pg, cur_oid, &op_data->chain_states[chain_reads[cri].chain_pos]);
-            if (op_data->scheme != POOL_SCHEME_REPLICATED)
+            if (pg.scheme != POOL_SCHEME_REPLICATED)
             {
                 if (extend_missing_stripes(stripes, cur_set, pg.pg_data_size, pg.pg_size) < 0)
                 {
@@ -446,7 +446,7 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
                 }
             }
         }
-        if (op_data->scheme == POOL_SCHEME_REPLICATED)
+        if (pg.scheme == POOL_SCHEME_REPLICATED)
         {
             n_subops++;
             read_buffer_size += stripes[0].read_end - stripes[0].read_start;
@@ -491,7 +491,7 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
     for (int cri = 0; cri < chain_reads.size(); cri++)
     {
         osd_rmw_stripe_t *stripes = chain_stripes + chain_reads[cri].chain_pos*stripe_count;
-        if (op_data->scheme == POOL_SCHEME_REPLICATED && !stripes[0].req_end)
+        if (pg.scheme == POOL_SCHEME_REPLICATED && !stripes[0].req_end)
         {
             continue;
         }
@@ -501,9 +501,9 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
         auto cur_state = op_data->chain_states[chain_reads[cri].chain_pos];
         uint64_t *cur_set = (pg.state != PG_ACTIVE && cur_state ? cur_state->read_target.data() : pg.cur_set.data());
         int zero_read = -1;
-        if (op_data->scheme == POOL_SCHEME_REPLICATED)
+        if (pg.scheme == POOL_SCHEME_REPLICATED)
         {
-            for (int role = 0; role < op_data->pg_size; role++)
+            for (int role = 0; role < pg.pg_size; role++)
                 if (cur_set[role] == this->osd_num || zero_read == -1)
                     zero_read = role;
         }
@@ -535,7 +535,7 @@ void osd_t::check_corrupted_chained(pg_t & pg, osd_op_t *cur_op)
         }
         if (corrupted)
         {
-            mark_object_corrupted(pg, cur_oid, op_data->chain_states[op_data->chain_reads[cri].chain_pos], stripes, false, false);
+            mark_object_corrupted(pg, cur_oid, op_data->chain_states[op_data->chain_reads[cri].chain_pos], stripes, false);
         }
     }
 }
@@ -555,18 +555,18 @@ void osd_t::send_chained_read_results(pg_t & pg, osd_op_t *cur_op)
         {
             // Reconstruct missing stripes
             osd_rmw_stripe_t *stripes = chain_stripes + op_data->chain_reads[cri].chain_pos*stripe_count;
-            if (op_data->scheme == POOL_SCHEME_XOR)
+            if (pg.scheme == POOL_SCHEME_XOR)
             {
                 reconstruct_stripes_xor(stripes, pg.pg_size, clean_entry_bitmap_size);
             }
-            else if (op_data->scheme == POOL_SCHEME_EC)
+            else if (pg.scheme == POOL_SCHEME_EC)
             {
                 reconstruct_stripes_ec(stripes, pg.pg_size, pg.pg_data_size, clean_entry_bitmap_size);
             }
         }
     }
     // Send bitmap
-    cur_op->reply.rw.bitmap_len = op_data->pg_data_size * clean_entry_bitmap_size;
+    cur_op->reply.rw.bitmap_len = pg.pg_data_size * clean_entry_bitmap_size;
     cur_op->iov.push_back(op_data->stripes[0].bmp_buf, cur_op->reply.rw.bitmap_len);
     // And finally compose the result
     uint64_t sent = 0;

src/osd/osd_primary_subops.cpp

@@ -67,7 +67,7 @@ void osd_t::finish_op(osd_op_t *cur_op, int retval)
         if (cur_op->req.hdr.opcode == OSD_OP_DELETE)
         {
             if (cur_op->op_data)
-                inode_stats[cur_op->req.rw.inode].op_bytes[inode_st_op] += cur_op->op_data->pg_data_size * bs_block_size;
+                inode_stats[cur_op->req.rw.inode].op_bytes[inode_st_op] += cur_op->op_data->pg->pg_data_size * bs_block_size;
         }
         else
             inode_stats[cur_op->req.rw.inode].op_bytes[inode_st_op] += cur_op->req.rw.len;
@@ -76,7 +76,7 @@ void osd_t::finish_op(osd_op_t *cur_op, int retval)
     {
         if (cur_op->op_data->pg_num > 0)
         {
-            auto & pg = pgs.at({ .pool_id = INODE_POOL(cur_op->op_data->oid.inode), .pg_num = cur_op->op_data->pg_num });
+            auto & pg = *cur_op->op_data->pg;
             pg.inflight--;
             assert(pg.inflight >= 0);
             if ((pg.state & PG_STOPPING) && pg.inflight == 0 && !pg.flush_batch)
@@ -126,10 +126,10 @@ void osd_t::submit_primary_subops(int submit_type, uint64_t op_version, const ui
     bool wr = submit_type == SUBMIT_WRITE;
     osd_primary_op_data_t *op_data = cur_op->op_data;
     osd_rmw_stripe_t *stripes = op_data->stripes;
-    bool rep = op_data->scheme == POOL_SCHEME_REPLICATED;
+    bool rep = op_data->pg->scheme == POOL_SCHEME_REPLICATED;
     // Allocate subops
     int n_subops = 0, zero_read = -1;
-    for (int role = 0; role < op_data->pg_size; role++)
+    for (int role = 0; role < op_data->pg->pg_size; role++)
     {
         if (osd_set[role] == this->osd_num || osd_set[role] != 0 && zero_read == -1)
             zero_read = role;
@@ -152,11 +152,11 @@ void osd_t::submit_primary_subops(int submit_type, uint64_t op_version, const ui
 int osd_t::submit_primary_subop_batch(int submit_type, inode_t inode, uint64_t op_version,
     osd_rmw_stripe_t *stripes, const uint64_t* osd_set, osd_op_t *cur_op, int subop_idx, int zero_read)
 {
+    bool rep = cur_op->op_data->pg->scheme == POOL_SCHEME_REPLICATED;
     bool wr = submit_type == SUBMIT_WRITE;
     osd_primary_op_data_t *op_data = cur_op->op_data;
-    bool rep = op_data->scheme == POOL_SCHEME_REPLICATED;
     int i = subop_idx;
-    for (int role = 0; role < op_data->pg_size; role++)
+    for (int role = 0; role < op_data->pg->pg_size; role++)
     {
         // We always submit zero-length writes to all replicas, even if the stripe is not modified
         if (!(wr || !rep && stripes[role].read_end != 0 || zero_read == role || submit_type == SUBMIT_SCRUB_READ))
@@ -168,7 +168,22 @@ int osd_t::submit_primary_subop_batch(int submit_type, inode_t inode, uint64_t o
         osd_rmw_stripe_t *si = stripes + (submit_type == SUBMIT_SCRUB_READ ? role : stripe_num);
         if (role_osd_num != 0)
         {
-            osd_op_t *subop = op_data->subops + i;
+            si->osd_num = role_osd_num;
+            si->role = stripe_num;
+            submit_primary_subop(cur_op, &op_data->subops[i], si, wr, inode, op_version);
+            i++;
+        }
+        else
+        {
+            si->osd_num = 0;
+        }
+    }
+    return i-subop_idx;
+}
+
+void osd_t::submit_primary_subop(osd_op_t *cur_op, osd_op_t *subop,
+    osd_rmw_stripe_t *si, bool wr, inode_t inode, uint64_t op_version)
+{
     uint32_t subop_len = wr
         ? si->write_end - si->write_start
         : si->read_end - si->read_start;
@@ -176,18 +191,17 @@ int osd_t::submit_primary_subop_batch(int submit_type, inode_t inode, uint64_t o
     {
         subop_len = 0;
     }
-            si->osd_num = role_osd_num;
     si->read_error = false;
     subop->bitmap = si->bmp_buf;
     subop->bitmap_len = clean_entry_bitmap_size;
-            // Using rmw_buf to pass pointer to stripes. Dirty but should work
+    // Using rmw_buf to pass pointer to stripes. Dirty but works
     subop->rmw_buf = si;
-            if (role_osd_num == this->osd_num)
+    if (si->osd_num == this->osd_num)
     {
         clock_gettime(CLOCK_REALTIME, &subop->tv_begin);
-                subop->op_type = (uint64_t)cur_op;
+        subop->op_type = (uint64_t)cur_op; // also dirty
         subop->bs_op = new blockstore_op_t((blockstore_op_t){
-                    .opcode = (uint64_t)(wr ? (rep ? BS_OP_WRITE_STABLE : BS_OP_WRITE) : BS_OP_READ),
+            .opcode = (uint64_t)(wr ? (cur_op->op_data->pg->scheme == POOL_SCHEME_REPLICATED ? BS_OP_WRITE_STABLE : BS_OP_WRITE) : BS_OP_READ),
             .callback = [subop, this](blockstore_op_t *bs_subop)
             {
                 handle_primary_bs_subop(subop);
@@ -195,7 +209,7 @@ int osd_t::submit_primary_subop_batch(int submit_type, inode_t inode, uint64_t o
             { {
                 .oid = (object_id){
                     .inode = inode,
-                            .stripe = op_data->oid.stripe | stripe_num,
+                    .stripe = cur_op->op_data->oid.stripe | si->role,
                 },
                 .version = op_version,
                 .offset = wr ? si->write_start : si->read_start,
@@ -207,7 +221,7 @@ int osd_t::submit_primary_subop_batch(int submit_type, inode_t inode, uint64_t o
 #ifdef OSD_DEBUG
          printf(
              "Submit %s to local: %jx:%jx v%ju %u-%u\n", wr ? "write" : "read",
-                    inode, op_data->oid.stripe | stripe_num, op_version,
+             inode, op_data->oid.stripe | si->role, op_version,
              subop->bs_op->offset, subop->bs_op->len
          );
 #endif
@@ -216,15 +230,15 @@ int osd_t::submit_primary_subop_batch(int submit_type, inode_t inode, uint64_t o
     else
     {
         subop->op_type = OSD_OP_OUT;
-                subop->req.sec_rw = {
+        subop->req.sec_rw = (osd_op_sec_rw_t){
             .header = {
                 .magic = SECONDARY_OSD_OP_MAGIC,
                 .id = msgr.next_subop_id++,
-                        .opcode = (uint64_t)(wr ? (rep ? OSD_OP_SEC_WRITE_STABLE : OSD_OP_SEC_WRITE) : OSD_OP_SEC_READ),
+                .opcode = (uint64_t)(wr ? (cur_op->op_data->pg->scheme == POOL_SCHEME_REPLICATED ? OSD_OP_SEC_WRITE_STABLE : OSD_OP_SEC_WRITE) : OSD_OP_SEC_READ),
             },
             .oid = {
                 .inode = inode,
-                        .stripe = op_data->oid.stripe | stripe_num,
+                .stripe = cur_op->op_data->oid.stripe | si->role,
             },
             .version = op_version,
             .offset = wr ? si->write_start : si->read_start,
@@ -234,8 +248,8 @@ int osd_t::submit_primary_subop_batch(int submit_type, inode_t inode, uint64_t o
         };
 #ifdef OSD_DEBUG
         printf(
-                    "Submit %s to osd %ju: %jx:%jx v%ju %u-%u\n", wr ? "write" : "read", role_osd_num,
+            "Submit %s to osd %ju: %jx:%jx v%ju %u-%u\n", wr ? "write" : "read", si->osd_num,
-                    inode, op_data->oid.stripe | stripe_num, op_version,
+            inode, op_data->oid.stripe | si->role, op_version,
             subop->req.sec_rw.offset, subop->req.sec_rw.len
         );
 #endif
@@ -257,7 +271,7 @@ int osd_t::submit_primary_subop_batch(int submit_type, inode_t inode, uint64_t o
         {
             handle_primary_subop(subop, cur_op);
         };
-                auto peer_fd_it = msgr.osd_peer_fds.find(role_osd_num);
+        auto peer_fd_it = msgr.osd_peer_fds.find(si->osd_num);
         if (peer_fd_it != msgr.osd_peer_fds.end())
         {
             subop->peer_fd = peer_fd_it->second;
@@ -271,14 +285,6 @@ int osd_t::submit_primary_subop_batch(int submit_type, inode_t inode, uint64_t o
             ringloop->set_immediate([subop]() { std::function<void(osd_op_t*)>(subop->callback)(subop); });
         }
     }
-            i++;
-        }
-        else
-        {
-            si->osd_num = 0;
-        }
-    }
-    return i-subop_idx;
 }
 
 static uint64_t bs_op_to_osd_op[] = {
@@ -401,7 +407,7 @@ void osd_t::handle_primary_subop(osd_op_t *subop, osd_op_t *cur_op)
         printf("subop %s %jx:%jx from osd %jd: version = %ju\n", osd_op_names[opcode],
             subop->req.sec_rw.oid.inode, subop->req.sec_rw.oid.stripe, peer_osd, version);
 #endif
-        if (op_data->fact_ver != UINT64_MAX)
+        if (version != 0 && op_data->fact_ver != UINT64_MAX)
         {
             if (op_data->fact_ver != 0 && op_data->fact_ver != version)
             {
@@ -526,7 +532,7 @@ bool contains_osd(osd_num_t *osd_set, uint64_t size, osd_num_t osd_num)
 void osd_t::submit_primary_del_subops(osd_op_t *cur_op, osd_num_t *cur_set, uint64_t set_size, pg_osd_set_t & loc_set)
 {
     osd_primary_op_data_t *op_data = cur_op->op_data;
-    bool rep = op_data->scheme == POOL_SCHEME_REPLICATED;
+    bool rep = op_data->pg->scheme == POOL_SCHEME_REPLICATED;
     obj_ver_osd_t extra_chunks[loc_set.size()];
     int chunks_to_del = 0;
     for (auto & chunk: loc_set)
@@ -738,10 +744,10 @@ void osd_t::submit_primary_rollback_subops(osd_op_t *cur_op, const uint64_t* osd
 {
     osd_primary_op_data_t *op_data = cur_op->op_data;
     osd_rmw_stripe_t *stripes = op_data->stripes;
-    assert(op_data->scheme != POOL_SCHEME_REPLICATED);
+    assert(op_data->pg->scheme != POOL_SCHEME_REPLICATED);
     // Allocate subops
     int n_subops = 0;
-    for (int role = 0; role < op_data->pg_size; role++)
+    for (int role = 0; role < op_data->pg->pg_size; role++)
     {
         if (osd_set[role] != 0 && !stripes[role].read_error &&
             (osd_set[role] == this->osd_num || msgr.osd_peer_fds.find(osd_set[role]) != msgr.osd_peer_fds.end()))
@@ -758,7 +764,7 @@ void osd_t::submit_primary_rollback_subops(osd_op_t *cur_op, const uint64_t* osd
     op_data->subops = new osd_op_t[n_subops];
     op_data->unstable_writes = new obj_ver_id[n_subops];
     int i = 0;
-    for (int role = 0; role < op_data->pg_size; role++)
+    for (int role = 0; role < op_data->pg->pg_size; role++)
     {
         if (osd_set[role] != 0 && !stripes[role].read_error &&
             (osd_set[role] == this->osd_num || msgr.osd_peer_fds.find(osd_set[role]) != msgr.osd_peer_fds.end()))

src/osd/osd_primary_write.cpp

@@ -44,7 +44,7 @@ void osd_t::continue_primary_write(osd_op_t *cur_op)
         return;
     }
     osd_primary_op_data_t *op_data = cur_op->op_data;
-    auto & pg = pgs.at({ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num });
+    auto & pg = *cur_op->op_data->pg;
     if (op_data->st == 1)      goto resume_1;
     else if (op_data->st == 2) goto resume_2;
     else if (op_data->st == 3) goto resume_3;
@@ -73,7 +73,7 @@ resume_1:
         op_data->object_state->ref_count++;
     }
 retry_1:
-    if (op_data->scheme == POOL_SCHEME_REPLICATED)
+    if (pg.scheme == POOL_SCHEME_REPLICATED)
     {
         // Simplified algorithm
         op_data->stripes[0].write_start = op_data->stripes[0].req_start;
@@ -99,7 +99,7 @@ retry_1:
     {
         assert(!cur_op->rmw_buf);
         cur_op->rmw_buf = calc_rmw(cur_op->buf, op_data->stripes, op_data->prev_set,
-            pg.pg_size, op_data->pg_data_size, pg.pg_cursize, pg.cur_set.data(), bs_block_size, clean_entry_bitmap_size);
+            pg.pg_size, pg.pg_data_size, pg.pg_cursize, pg.cur_set.data(), bs_block_size, clean_entry_bitmap_size);
         if (!cur_op->rmw_buf)
         {
             // Refuse partial overwrite of an incomplete object
@@ -114,7 +114,7 @@ retry_1:
             // Allow to read version number (just version number!) from corrupted chunks
             // to allow full overwrite of a corrupted object
             bool found = false;
-            for (int role = 0; role < op_data->pg_size; role++)
+            for (int role = 0; role < pg.pg_size; role++)
             {
                 if (op_data->prev_set[role] != 0 || op_data->stripes[role].read_end > op_data->stripes[role].read_start)
                 {
@@ -124,8 +124,8 @@ retry_1:
             }
             if (!found)
             {
-                osd_num_t corrupted_target[op_data->pg_size];
+                osd_num_t corrupted_target[pg.pg_size];
-                for (int role = 0; role < op_data->pg_size; role++)
+                for (int role = 0; role < pg.pg_size; role++)
                 {
                     corrupted_target[role] = 0;
                 }
@@ -151,7 +151,7 @@ resume_3:
         if (op_data->errcode == -EIO || op_data->errcode == -EDOM)
         {
             // Mark object corrupted and retry
-            op_data->object_state = mark_object_corrupted(pg, op_data->oid, op_data->object_state, op_data->stripes, true, false);
+            op_data->object_state = mark_object_corrupted(pg, op_data->oid, op_data->object_state, op_data->stripes, true);
             op_data->prev_set = op_data->object_state ? op_data->object_state->read_target.data() : pg.cur_set.data();
             if (cur_op->rmw_buf)
             {
@@ -165,6 +165,7 @@ resume_3:
         return;
     }
     // Check CAS version
+    // FIXME: Handle CAS writes as "immediate" in non-immediate_commit pools, otherwise CAS doesn't make sense
     if (cur_op->req.rw.version && op_data->fact_ver != (cur_op->req.rw.version-1))
     {
         deref_object_state(pg, &op_data->object_state, true);
@@ -172,7 +173,7 @@ resume_3:
         cur_op->reply.rw.version = op_data->fact_ver;
         goto continue_others;
     }
-    if (op_data->scheme == POOL_SCHEME_REPLICATED)
+    if (pg.scheme == POOL_SCHEME_REPLICATED)
     {
         // Set bitmap bits
         bitmap_set(op_data->stripes[0].bmp_buf, op_data->stripes[0].write_start,
@@ -203,7 +204,7 @@ resume_3:
         }
         else if (pg.scheme == POOL_SCHEME_EC)
         {
-            calc_rmw_parity_ec(op_data->stripes, pg.pg_size, op_data->pg_data_size, op_data->prev_set, pg.cur_set.data(), bs_block_size, clean_entry_bitmap_size);
+            calc_rmw_parity_ec(op_data->stripes, pg.pg_size, pg.pg_data_size, op_data->prev_set, pg.cur_set.data(), bs_block_size, clean_entry_bitmap_size);
         }
     }
     // Send writes
@@ -269,7 +270,7 @@ resume_5:
         // and rollback successful part updates in case of EC.
         if (op_data->done > 0 && !op_data->drops)
         {
-            if (op_data->scheme != POOL_SCHEME_REPLICATED)
+            if (pg.scheme != POOL_SCHEME_REPLICATED)
             {
                 submit_primary_rollback_subops(cur_op, pg.cur_set.data());
 resume_11:
@@ -293,7 +294,7 @@ resume_12:
         pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->errcode);
         return;
     }
-    if (op_data->scheme != POOL_SCHEME_REPLICATED)
+    if (pg.scheme != POOL_SCHEME_REPLICATED)
     {
         // Remove version override just after the write, but before stabilizing
         pg.ver_override.erase(op_data->oid);
@@ -329,7 +330,7 @@ resume_7:
                 memset(&recovery_stat[recovery_type], 0, sizeof(recovery_stat[recovery_type]));
                 recovery_stat[recovery_type].count++;
             }
-            for (int role = 0; role < (op_data->scheme == POOL_SCHEME_REPLICATED ? 1 : pg.pg_size); role++)
+            for (int role = 0; role < (pg.scheme == POOL_SCHEME_REPLICATED ? 1 : pg.pg_size); role++)
             {
                 recovery_stat[recovery_type].bytes += op_data->stripes[role].write_end - op_data->stripes[role].write_start;
             }
@@ -353,7 +354,7 @@ resume_7:
             for (auto & chunk: op_data->object_state->osd_set)
             {
                 // Check is the same as in submit_primary_del_subops()
-                if (op_data->scheme == POOL_SCHEME_REPLICATED
+                if (pg.scheme == POOL_SCHEME_REPLICATED
                     ? !contains_osd(pg.cur_set.data(), pg.pg_size, chunk.osd_num)
                     : (chunk.osd_num != pg.cur_set[chunk.role]))
                 {
@@ -361,7 +362,7 @@ resume_7:
                         .osd_num = chunk.osd_num,
                         .oid = {
                             .inode = op_data->oid.inode,
-                            .stripe = op_data->oid.stripe | (op_data->scheme == POOL_SCHEME_REPLICATED ? 0 : chunk.role),
+                            .stripe = op_data->oid.stripe | (pg.scheme == POOL_SCHEME_REPLICATED ? 0 : chunk.role),
                         },
                         .version = op_data->fact_ver,
                     });
@@ -472,7 +473,7 @@ bool osd_t::remember_unstable_write(osd_op_t *cur_op, pg_t & pg, pg_osd_set_t &
     if (immediate_commit == IMMEDIATE_ALL)
     {
 immediate:
-        if (op_data->scheme != POOL_SCHEME_REPLICATED)
+        if (pg.scheme != POOL_SCHEME_REPLICATED)
         {
             // Send STABILIZE ops immediately
             op_data->unstable_write_osds = new std::vector<unstable_osd_num_t>();
@@ -515,7 +516,7 @@ resume_7:
     }
     else if (immediate_commit == IMMEDIATE_SMALL)
     {
-        int stripe_count = (op_data->scheme == POOL_SCHEME_REPLICATED ? 1 : op_data->pg_size);
+        int stripe_count = (pg.scheme == POOL_SCHEME_REPLICATED ? 1 : pg.pg_size);
         for (int role = 0; role < stripe_count; role++)
         {
             if (op_data->stripes[role].write_start == 0 &&
@@ -531,7 +532,7 @@ resume_7:
     {
 lazy:
         unstable_write_count++;
-        if (op_data->scheme != POOL_SCHEME_REPLICATED)
+        if (pg.scheme != POOL_SCHEME_REPLICATED)
         {
             // Remember version as unstable for EC/XOR
             for (auto & chunk: loc_set)

src/osd/osd_rmw.cpp

@@ -1118,75 +1118,58 @@ static bool next_combination(int *subset, int k, int n)
     return true;
 }
 
-static int c_n_k(int n, int k)
+static uint64_t c_n_k(uint64_t n, uint64_t k)
 {
-    int c = 1;
+    uint64_t c = 1;
-    for (int i = n; i > k; i--)
+    for (uint64_t i = n; i > k; i--)
+    {
+        if ((c*i) < i)
+            return UINT64_MAX;
         c *= i;
-    for (int i = 2; i <= (n-k); i++)
+    }
+    for (uint64_t i = 2; i <= (n-k); i++)
         c /= i;
     return c;
 }
 
-std::vector<int> ec_find_good(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsize, bool is_xor,
+static std::vector<int> ec_check_combination(osd_rmw_stripe_t *stripes, int stripe_count,
-    uint32_t chunk_size, uint32_t bitmap_size, int max_bruteforce)
+    int *subset, int pg_size, int pg_minsize, bool is_xor,
+    uint32_t chunk_size, uint32_t bitmap_size, uint8_t *tmp_buf)
 {
-    std::vector<int> found_valid;
-    int cur_live[pg_size], live_count = 0, exists_count = 0;
     osd_num_t fake_osd_set[pg_size];
-    for (int role = 0; role < pg_size; role++)
+    for (int i = 0; i < pg_size; i++)
     {
-        if (!stripes[role].missing)
+        fake_osd_set[i] = i+1;
-        {
-            if (!stripes[role].not_exists)
-                exists_count++;
-            cur_live[live_count++] = role;
-            fake_osd_set[role] = role+1;
     }
-    }
-    if (live_count <= pg_minsize)
-    {
-        return std::vector<int>();
-    }
-    if (exists_count <= pg_minsize)
-    {
-        // Special case: user manually deleted some chunks
-        for (int role = 0; role < pg_size; role++)
-            if (!stripes[role].missing && !stripes[role].not_exists)
-                found_valid.push_back(role);
-        return found_valid;
-    }
-    // Try to locate errors using brute force if there isn't too many combinations
     osd_rmw_stripe_t brute_stripes[pg_size];
-    int out_count = live_count-pg_minsize;
+    memset(brute_stripes, 0, sizeof(osd_rmw_stripe_t)*pg_size);
-    bool brute_force = out_count > 1 && c_n_k(live_count-1, out_count-1) <= max_bruteforce;
+    for (int i = 0; i < pg_size; i++)
-    int subset[pg_minsize], outset[out_count];
-    // Select all combinations with items except the last one (== anything to compare)
-    first_combination(subset, pg_minsize, live_count-1);
-    uint8_t *tmp_buf = (uint8_t*)malloc_or_die(pg_size*chunk_size);
-    do
     {
-        memcpy(brute_stripes, stripes, sizeof(osd_rmw_stripe_t)*pg_size);
+        auto & bs = brute_stripes[i];
-        int i = 0, j = 0, k = 0;
+        bs.req_end = bs.read_end = chunk_size;
-        for (; i < pg_minsize; i++, j++)
-            while (j < subset[i])
-                outset[k++] = j++;
-        while (j < pg_size)
-            outset[k++] = j++;
-        for (int i = 0; i < out_count; i++)
-        {
-            brute_stripes[cur_live[outset[i]]].missing = true;
-            brute_stripes[cur_live[outset[i]]].read_buf = tmp_buf+cur_live[outset[i]]*chunk_size;
     }
     for (int i = 0; i < pg_minsize; i++)
     {
-            brute_stripes[i].write_buf = brute_stripes[i].read_buf;
+        auto & src = stripes[subset[i]];
-            brute_stripes[i].req_start = 0;
+        auto & bs = brute_stripes[src.role];
-            brute_stripes[i].req_end = chunk_size;
+        bs.bmp_buf = src.bmp_buf;
+        bs.write_buf = bs.read_buf = src.read_buf;
     }
-        for (int i = pg_minsize; i < pg_size; i++)
+    for (int i = 0; i < pg_size; i++)
     {
-            brute_stripes[i].write_buf = tmp_buf+i*chunk_size;
+        auto & bs = brute_stripes[i];
+        if (!bs.read_buf)
+        {
+            // missing chunks are recovered in read_bufs and write_bufs are used as source for parity
+            bs.missing = true;
+            bs.read_buf = bs.write_buf = tmp_buf+i*chunk_size;
+            bs.bmp_buf = tmp_buf + stripe_count*chunk_size + i*bitmap_size;
+        }
+        else if (i >= pg_minsize)
+        {
+            // parity chunks are regenerated in their write_bufs, so use a temporary buffer
+            bs.write_buf = tmp_buf+i*chunk_size;
+        }
     }
     if (is_xor)
     {
@@ -1198,59 +1181,167 @@ std::vector<int> ec_find_good(osd_rmw_stripe_t *stripes, int pg_size, int pg_min
         reconstruct_stripes_ec(brute_stripes, pg_size, pg_minsize, bitmap_size);
         calc_rmw_parity_ec(brute_stripes, pg_size, pg_minsize, fake_osd_set, fake_osd_set, chunk_size, bitmap_size);
     }
-        for (int i = pg_minsize; i < pg_size; i++)
+    bool matched_other = false;
+    std::vector<int> good_set;
+    for (int i = 0; i < stripe_count; i++)
     {
-            brute_stripes[i].read_buf = brute_stripes[i].write_buf;
+        if (stripes[i].read_error || stripes[i].not_exists)
+        {
+            continue;
         }
-        int valid_count = 0;
+        auto & bs = brute_stripes[stripes[i].role];
-        for (int i = 0; i < out_count; i++)
+        if (!bs.missing && bs.read_buf == stripes[i].read_buf)
         {
-            if (memcmp(brute_stripes[cur_live[outset[i]]].read_buf,
+            // source chunk, mark OK
-                    stripes[cur_live[outset[i]]].read_buf, chunk_size) == 0)
+            good_set.push_back(i);
+        }
+        else if (memcmp(stripes[i].role < pg_minsize ? bs.read_buf : bs.write_buf, stripes[i].read_buf, chunk_size) == 0)
         {
-                brute_stripes[cur_live[outset[i]]].missing = false;
+            // matching chunk, mark OK
-                valid_count++;
+            good_set.push_back(i);
+            matched_other = true;
         }
     }
-        if (valid_count > 0)
+    if (!matched_other)
     {
-            if (found_valid.size())
+        good_set.clear();
+    }
+    return good_set;
+}
+
+static int count_roles(osd_rmw_stripe_t *stripes, std::vector<int> & valid_chunks, int pg_size)
 {
-                // Check if we found the same set from the different point of view,
+    bool role_ok[pg_size];
-                // like 1 2 3 -> valid 4 5 and 1 3 4 -> valid 2 5
+    for (int i = 0; i < pg_size; i++)
-                for (int i = 0, j = 0; i < pg_size; i++)
     {
-                    if (!brute_stripes[i].missing)
+        role_ok[i] = false;
+    }
+    for (int idx: valid_chunks)
     {
-                        if (j >= found_valid.size() || found_valid[j] != i)
+        role_ok[stripes[idx].role] = true;
+    }
+    int ok_count = 0;
+    for (int i = 0; i < pg_size; i++)
     {
-                            // Ambiguity: we found multiple valid sets and don't know which one is correct
+        if (role_ok[i])
-                            found_valid.clear();
+            ok_count++;
+    }
+    return ok_count;
+}
+
+std::vector<int> ec_find_good(osd_rmw_stripe_t *stripes, int stripe_count, int pg_size, int pg_minsize, bool is_xor,
+    uint32_t chunk_size, uint32_t bitmap_size, uint64_t max_bruteforce, bool find_best)
+{
+    std::vector<int> found_valid;
+    std::vector<std::vector<int>> live_variants(pg_size);
+    int eq_to[stripe_count];
+    int live_roles = 0, live_total = 0;
+    for (int i = 0; i < pg_size; i++)
+    {
+        eq_to[i] = i;
+    }
+    for (int i = 0; i < stripe_count; i++)
+    {
+        if (!stripes[i].read_error && !stripes[i].not_exists)
+        {
+            if (live_variants[stripes[i].role].size() > 0)
+            {
+                for (int j = 0; j < i; j++)
+                {
+                    if (stripes[j].role == stripes[i].role &&
+                        memcmp(stripes[i].read_buf, stripes[j].read_buf, chunk_size) == 0)
+                    {
+                        eq_to[i] = eq_to[j];
                         break;
                     }
-                        j++;
-                    }
-                }
-                if (!found_valid.size())
-                {
-                    break;
                 }
             }
             else
             {
-                for (int i = 0; i < pg_size; i++)
+                live_roles++;
+            }
+            if (eq_to[i] == i)
             {
-                    if (!brute_stripes[i].missing)
+                live_variants[stripes[i].role].push_back(i);
+                live_total++;
+            }
+        }
+    }
+    if (live_roles == pg_minsize && live_total > pg_minsize)
     {
+        // Nothing to validate and there are chunks with different data => object is inconsistent
+        return std::vector<int>();
+    }
+    if (live_roles <= pg_minsize)
+    {
+        // Nothing to validate, just return all live chunks
+        for (int i = 0; i < stripe_count; i++)
+            if (!stripes[i].read_error)
                 found_valid.push_back(i);
+        return found_valid;
     }
-                }
+    // Try to locate errors using brute force if there isn't too many combinations
-            }
+    bool brute_force = c_n_k(live_roles, pg_minsize) <= max_bruteforce;
-            if (valid_count == out_count)
+    int combination[pg_minsize], subset[pg_minsize], subvar[pg_minsize];
+    // To translate 0..live_roles into 0..pg_size
+    int comb_to_subset[live_roles];
+    for (int i = 0, r = 0; i < pg_size; i++)
     {
-                // All chunks are good
+        if (live_variants[i].size() > 0)
-                break;
+            comb_to_subset[r++] = i;
     }
+    // Select all combinations with items except the last one (== anything to compare)
+    first_combination(combination, pg_minsize, live_roles);
+    uint8_t *tmp_buf = (uint8_t*)malloc_or_die(stripe_count*(chunk_size+bitmap_size));
+    do
+    {
+        // Then loop over all subvariants (if some roles have multiple diverged variants of data)
+        for (int i = 0; i < pg_minsize; i++)
+        {
+            subvar[i] = 0;
+        }
+        while (true)
+        {
+            // Transform combination[] + subvar[] into subset[]
+            for (int i = 0; i < pg_minsize; i++)
+            {
+                subset[i] = live_variants[comb_to_subset[combination[i]]][subvar[i]];
+            }
+            // Check the combination
+            auto valid_chunks = ec_check_combination(stripes, stripe_count, subset, pg_size, pg_minsize, is_xor, chunk_size, bitmap_size, tmp_buf);
+            // The same set may be found from different points of view,
+            // like 1 2 3 -> valid 4 5 and 1 3 4 -> valid 2 5
+            if (valid_chunks.size() > 0)
+            {
+                if (found_valid.size() >= valid_chunks.size() && found_valid != valid_chunks)
+                {
+                    // Ambiguity: we found multiple valid sets and don't know which one is correct
+                    printf("Scrub found 2 different correct chunk subsets: OSD ");
+                    for (int i = 0; i < found_valid.size(); i++)
+                        printf(i > 0 ? ", %ju" : "%ju", stripes[found_valid[i]].osd_num);
+                    printf(" and OSD ");
+                    for (int i = 0; i < valid_chunks.size(); i++)
+                        printf(i > 0 ? ", %ju" : "%ju", stripes[valid_chunks[i]].osd_num);
+                    printf("\n");
+                    found_valid.clear();
+                    goto out;
+                }
+                else if (!found_valid.size() && (find_best || count_roles(stripes, valid_chunks, pg_size) >= pg_size))
+                {
+                    found_valid = valid_chunks;
+                }
+            }
+            // Select next subvariant
+            int i = 0;
+            for (i = 0; i < pg_minsize; i++)
+            {
+                subvar[i]++;
+                if (subvar[i] < live_variants[combination[i]].size())
+                    break;
+                subvar[i] = 0;
+            }
+            if (i >= pg_minsize)
+                break;
         }
         if (!brute_force)
         {
@@ -1258,7 +1349,8 @@ std::vector<int> ec_find_good(osd_rmw_stripe_t *stripes, int pg_size, int pg_min
             // if we find it we won't be able to check that it's the only good one
             break;
         }
-    } while (out_count > 1 && next_combination(subset, pg_minsize, live_count-1));
+    } while (next_combination(combination, pg_minsize, live_roles));
+out:
     free(tmp_buf);
     return found_valid;
 }

src/osd/osd_rmw.h

@@ -28,6 +28,7 @@ struct osd_rmw_stripe_t
     uint32_t read_start, read_end;
     uint32_t write_start, write_end;
     osd_num_t osd_num;
+    int role;
     bool missing: 1;
     bool read_error: 1;
     bool not_exists: 1;
@@ -57,5 +58,5 @@ void reconstruct_stripes_ec(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsi
 void calc_rmw_parity_ec(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsize,
     uint64_t *read_osd_set, uint64_t *write_osd_set, uint32_t chunk_size, uint32_t bitmap_size);
 
-std::vector<int> ec_find_good(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsize, bool is_xor,
+std::vector<int> ec_find_good(osd_rmw_stripe_t *stripes, int stripe_count, int pg_size, int pg_minsize, bool is_xor,
-    uint32_t chunk_size, uint32_t bitmap_size, int max_bruteforce);
+    uint32_t chunk_size, uint32_t bitmap_size, uint64_t max_bruteforce, bool find_best);

src/osd/osd_rmw_test.cpp

@@ -1160,24 +1160,26 @@ void test_ec43_error_bruteforce()
         stripes[i].read_end = 4096;
         stripes[i].read_buf = write_buf+i*4096;
         stripes[i].write_buf = NULL;
+        stripes[i].role = i;
+        stripes[i].osd_num = i+1;
     }
     // All good chunks
-    auto res = ec_find_good(stripes, 7, 4, false, 4096, 0, 100);
+    auto res = ec_find_good(stripes, 7, 7, 4, false, 4096, 0, 100, true);
     assert_eq_vec(res, std::vector<int>({0, 1, 2, 3, 4, 5, 6}));
     // 1 missing chunk
     set_pattern(write_buf+1*4096, 4096, 0);
-    res = ec_find_good(stripes, 7, 4, false, 4096, 0, 100);
+    res = ec_find_good(stripes, 7, 7, 4, false, 4096, 0, 100, true);
     assert_eq_vec(res, std::vector<int>({0, 2, 3, 4, 5, 6}));
     // 2 missing chunks
     set_pattern(write_buf+1*4096, 4096, 0);
     set_pattern(write_buf+5*4096, 4096, 0);
-    res = ec_find_good(stripes, 7, 4, false, 4096, 0, 100);
+    res = ec_find_good(stripes, 7, 7, 4, false, 4096, 0, 100, true);
     assert_eq_vec(res, std::vector<int>({0, 2, 3, 4, 6}));
     // 3 missing chunks
     set_pattern(write_buf+1*4096, 4096, 0);
     set_pattern(write_buf+5*4096, 4096, 0);
     set_pattern(write_buf+6*4096, 4096, 0);
-    res = ec_find_good(stripes, 7, 4, false, 4096, 0, 100);
+    res = ec_find_good(stripes, 7, 7, 4, false, 4096, 0, 100, true);
     assert_eq_vec(res, std::vector<int>());
     // Done
     free(rmw_buf);

src/osd/osd_scrub.cpp

@@ -369,114 +369,102 @@ void osd_t::schedule_scrub(pg_t & pg)
     }
 }
 
-void osd_t::continue_primary_scrub(osd_op_t *cur_op)
+void osd_t::submit_scrub_subops(osd_op_t *cur_op)
 {
-    if (!cur_op->op_data && !prepare_primary_rw(cur_op))
-        return;
     osd_primary_op_data_t *op_data = cur_op->op_data;
-    if (op_data->st == 1)
+    assert(!op_data->stripe_count);
-        goto resume_1;
+    cur_op->req.rw.len = bs_block_size * op_data->pg->pg_data_size;
-    else if (op_data->st == 2)
-        goto resume_2;
-    {
-        auto & pg = pgs.at({ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num });
-        cur_op->req.rw.len = bs_block_size * pg.pg_data_size;
     // Determine version
-        auto vo_it = pg.ver_override.find(op_data->oid);
+    auto vo_it = op_data->pg->ver_override.find(op_data->oid);
-        op_data->target_ver = vo_it != pg.ver_override.end() ? vo_it->second : UINT64_MAX;
+    op_data->target_ver = vo_it != op_data->pg->ver_override.end() ? vo_it->second : UINT64_MAX;
-        // PG may have degraded or misplaced objects
+    // Find object state
-        op_data->prev_set = get_object_osd_set(pg, op_data->oid, &op_data->object_state);
+    op_data->prev_set = get_object_osd_set(*op_data->pg, op_data->oid, &op_data->object_state);
-        // Read all available chunks
+    if (!op_data->object_state)
-        int n_copies = 0;
-        op_data->degraded = false;
-        for (int role = 0; role < op_data->pg_size; role++)
     {
-            op_data->stripes[role].write_buf = NULL;
+        op_data->stripe_count = op_data->pg->pg_size;
-            op_data->stripes[role].read_start = 0;
+        op_data->stripes = (osd_rmw_stripe_t*)calloc_or_die(op_data->stripe_count, sizeof(osd_rmw_stripe_t));
-            op_data->stripes[role].read_end = bs_block_size;
+        for (int i = 0; i < op_data->pg->pg_size; i++)
-            if (op_data->prev_set[role] != 0)
         {
-                n_copies++;
+            op_data->stripes[i].osd_num = op_data->prev_set[i];
+            op_data->stripes[i].role = (op_data->pg->scheme == POOL_SCHEME_REPLICATED ? 0 : i);
+            op_data->stripes[i].read_end = bs_block_size;
+        }
     }
     else
     {
-                op_data->stripes[role].missing = true;
+        op_data->stripe_count = 0;
-                if (op_data->scheme != POOL_SCHEME_REPLICATED && role < op_data->pg_data_size)
+        for (auto & chunk: op_data->object_state->osd_set)
         {
-                    op_data->degraded = true;
+            // Read all chunks except outdated
+            if (!(chunk.loc_bad & LOC_OUTDATED))
+                op_data->stripe_count++;
         }
-            }
+        op_data->stripes = (osd_rmw_stripe_t*)calloc_or_die(op_data->stripe_count, sizeof(osd_rmw_stripe_t));
-        }
+        int i = 0;
-        if (n_copies <= op_data->pg_data_size)
+        for (auto & chunk: op_data->object_state->osd_set)
         {
-            // Nothing to compare, even if we'd like to
+            if (!(chunk.loc_bad & LOC_OUTDATED))
-            finish_op(cur_op, 0);
+            {
-            return;
+                op_data->stripes[i].osd_num = chunk.osd_num;
+                op_data->stripes[i].role = chunk.role;
+                op_data->stripes[i].read_end = bs_block_size;
+                i++;
             }
-        cur_op->buf = alloc_read_buffer(op_data->stripes, op_data->pg_size, 0);
+        }
-        // Submit reads
+    }
-        osd_op_t *subops = new osd_op_t[n_copies];
+    assert(!cur_op->bitmap_buf);
+    cur_op->bitmap_buf = calloc_or_die(1, clean_entry_bitmap_size * op_data->stripe_count);
+    for (int i = 0; i < op_data->stripe_count; i++)
+    {
+        op_data->stripes[i].bmp_buf = (uint8_t*)cur_op->bitmap_buf + clean_entry_bitmap_size * i;
+    }
+    cur_op->buf = alloc_read_buffer(op_data->stripes, op_data->stripe_count, 0);
     op_data->fact_ver = 0;
     op_data->done = op_data->errors = op_data->errcode = 0;
-        op_data->n_subops = n_copies;
+    op_data->n_subops = op_data->stripe_count;
-        op_data->subops = subops;
+    op_data->subops = new osd_op_t[op_data->stripe_count];
-        int sent = submit_primary_subop_batch(SUBMIT_SCRUB_READ, op_data->oid.inode, op_data->target_ver,
-            op_data->stripes, op_data->prev_set, cur_op, 0, -1);
-        assert(sent == n_copies);
     op_data->st = 1;
-    }
+    for (int i = 0; i < op_data->stripe_count; i++)
-resume_1:
-    return;
-resume_2:
-    if (op_data->errors > 0)
     {
-        if (op_data->errcode == -EIO || op_data->errcode == -EDOM)
+        submit_primary_subop(cur_op, &op_data->subops[i], &op_data->stripes[i],
+            false, op_data->oid.inode, op_data->target_ver);
+    }
+}
+
+// The idea is that scrub should not only find out if the object
+// is corrupted, but it should also verify availability of all copies
+void osd_t::scrub_check_results(osd_op_t *cur_op)
 {
-            // I/O or checksum error
+    osd_primary_op_data_t *op_data = cur_op->op_data;
-            int n_copies = 0;
-            for (int role = 0; role < op_data->pg_size; role++)
-            {
-                if (op_data->stripes[role].read_error)
-                {
-                    op_data->stripes[role].missing = true;
-                    if (op_data->scheme != POOL_SCHEME_REPLICATED && role < op_data->pg_data_size)
-                    {
-                        op_data->degraded = true;
-                    }
-                }
-                else if (!op_data->stripes[role].missing)
-                {
-                    n_copies++;
-                }
-            }
-            if (n_copies <= op_data->pg_data_size)
-            {
-                // Nothing to compare, just mark the object as corrupted
-                auto & pg = pgs.at({ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num });
-                // FIXME: ref = true ideally... because new_state != state is not necessarily true if it's freed and recreated
-                op_data->object_state = mark_object_corrupted(pg, op_data->oid, op_data->object_state, op_data->stripes, false, false);
-                // Operation is treated as unsuccessful only if the object becomes unreadable
-                finish_op(cur_op, n_copies < op_data->pg_data_size ? op_data->errcode : 0);
-                return;
-            }
-            // Proceed, we can still compare chunks that were successfully read
-        }
-        else
-        {
-            finish_op(cur_op, op_data->errcode);
-            return;
-        }
-    }
     bool inconsistent = false;
-    if (op_data->scheme == POOL_SCHEME_REPLICATED)
+    int total = 0;
+    for (int role = 0; role < op_data->stripe_count; role++)
+    {
+        if (!op_data->stripes[role].not_exists)
+            total++;
+    }
+    if (!total)
+    {
+        // Object is deleted manually from all OSDs, forget it
+        printf(
+            "[PG %u/%u] Scrub detected a deleted object %jx:%jx\n",
+            INODE_POOL(op_data->oid.inode), op_data->pg_num,
+            op_data->oid.inode, op_data->oid.stripe
+        );
+        remove_object_from_state(op_data->oid, &op_data->object_state, *op_data->pg, false);
+        deref_object_state(*op_data->pg, &op_data->object_state, true);
+        return;
+    }
+    if (op_data->pg->scheme == POOL_SCHEME_REPLICATED)
     {
         // Check that all chunks have returned the same data
         int total = 0;
-        int eq_to[op_data->pg_size];
+        int eq_to[op_data->stripe_count];
-        for (int role = 0; role < op_data->pg_size; role++)
+        for (int role = 0; role < op_data->stripe_count; role++)
         {
             eq_to[role] = -1;
-            if (op_data->stripes[role].read_end != 0 && !op_data->stripes[role].missing &&
+            if (op_data->stripes[role].read_end != 0 &&
+                !op_data->stripes[role].read_error &&
                 !op_data->stripes[role].not_exists)
             {
                 total++;
@@ -492,16 +480,16 @@ resume_2:
                 }
             }
         }
-        int votes[op_data->pg_size];
+        int votes[op_data->stripe_count];
-        for (int role = 0; role < op_data->pg_size; role++)
+        for (int role = 0; role < op_data->stripe_count; role++)
             votes[role] = 0;
-        for (int role = 0; role < op_data->pg_size; role++)
+        for (int role = 0; role < op_data->stripe_count; role++)
         {
             if (eq_to[role] != -1)
                 votes[eq_to[role]]++;
         }
         int best = -1;
-        for (int role = 0; role < op_data->pg_size; role++)
+        for (int role = 0; role < op_data->stripe_count; role++)
         {
             if (votes[role] > (best >= 0 ? votes[best] : 0))
                 best = role;
@@ -509,7 +497,7 @@ resume_2:
         if (best >= 0 && votes[best] < total)
         {
             bool unknown = false;
-            for (int role = 0; role < op_data->pg_size; role++)
+            for (int role = 0; role < op_data->stripe_count; role++)
             {
                 if (role != best && votes[role] == votes[best])
                 {
@@ -550,10 +538,11 @@ resume_2:
     }
     else
     {
-        assert(op_data->scheme == POOL_SCHEME_EC || op_data->scheme == POOL_SCHEME_XOR);
+        assert(op_data->pg->scheme == POOL_SCHEME_EC || op_data->pg->scheme == POOL_SCHEME_XOR);
         auto good_subset = ec_find_good(
-            op_data->stripes, op_data->pg_size, op_data->pg_data_size, op_data->scheme == POOL_SCHEME_XOR,
+            op_data->stripes, op_data->stripe_count,
-            bs_block_size, clean_entry_bitmap_size, scrub_ec_max_bruteforce
+            op_data->pg->pg_size, op_data->pg->pg_data_size, op_data->pg->scheme == POOL_SCHEME_XOR,
+            bs_block_size, clean_entry_bitmap_size, scrub_ec_max_bruteforce, scrub_find_best
         );
         if (!good_subset.size())
         {
@@ -567,62 +556,115 @@ resume_2:
         else
         {
             int total = 0;
-            for (int role = 0; role < op_data->pg_size; role++)
+            for (int i = 0; i < op_data->stripe_count; i++)
             {
-                if (!op_data->stripes[role].missing)
+                if (!op_data->stripes[i].not_exists)
                 {
+                    // use "missing" flag to distinguish actual read errors and inconsistent chunks
                     total++;
-                    op_data->stripes[role].read_error = true;
+                    op_data->stripes[i].missing = true;
                 }
             }
-            for (int role: good_subset)
+            for (int i: good_subset)
             {
-                op_data->stripes[role].read_error = false;
+                op_data->stripes[i].missing = false;
             }
-            for (int role = 0; role < op_data->pg_size; role++)
+            for (int i = 0; i < op_data->stripe_count; i++)
             {
-                if (!op_data->stripes[role].missing && op_data->stripes[role].read_error)
+                if (op_data->stripes[i].missing)
                 {
+                    op_data->stripes[i].read_error = true;
                     printf(
                         "[PG %u/%u] Object %jx:%jx v%ju chunk %d on OSD %ju doesn't match other chunks%s\n",
                         INODE_POOL(op_data->oid.inode), op_data->pg_num,
                         op_data->oid.inode, op_data->oid.stripe, op_data->fact_ver,
-                        role, op_data->stripes[role].osd_num,
+                        op_data->stripes[i].role, op_data->stripes[i].osd_num,
                         scrub_find_best ? ", marking it as corrupted" : ""
                     );
                 }
             }
-            if (!scrub_find_best && good_subset.size() < total)
-            {
-                inconsistent = true;
-                printf(
-                    "[PG %u/%u] Object %jx:%jx v%ju is marked as inconsistent because scrub_find_best is turned off. Use vitastor-cli fix to fix it\n",
-                    INODE_POOL(op_data->oid.inode), op_data->pg_num,
-                    op_data->oid.inode, op_data->oid.stripe, op_data->fact_ver
-                );
-                for (int role = 0; role < op_data->pg_size; role++)
-                {
-                    if (!op_data->stripes[role].missing && op_data->stripes[role].read_error)
-                    {
-                        // Undo error locator marking chunk as bad
-                        op_data->stripes[role].read_error = false;
         }
     }
-            }
+    bool mark = inconsistent;
-        }
+    for (int role = 0; !mark && role < op_data->stripe_count; role++)
-    }
-    for (int role = 0; role < op_data->pg_size; role++)
     {
-        if (op_data->stripes[role].osd_num != 0 &&
+        if (op_data->stripes[role].read_error || op_data->stripes[role].not_exists)
-            (op_data->stripes[role].read_error || op_data->stripes[role].not_exists) ||
+            mark = true;
-            inconsistent)
+    }
+    if (!mark)
     {
-            // Got at least 1 read error or mismatch, mark the object as corrupted
+        return;
-            auto & pg = pgs.at({ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num });
+    }
     // FIXME: ref = true ideally... because new_state != state is not necessarily true if it's freed and recreated
-            op_data->object_state = mark_object_corrupted(pg, op_data->oid, op_data->object_state, op_data->stripes, false, inconsistent);
+    op_data->object_state = mark_object(*op_data->pg, op_data->oid, op_data->object_state, false /*ref*/, [op_data, inconsistent](pg_osd_set_t & new_set)
-            break;
+    {
+        // Mark object chunk(s) as corrupted and/or missing and/or inconsistent
+        int changes = 0;
+        for (int i = 0; i < op_data->stripe_count; i++)
+        {
+            // Find the same stripe in new_set
+            int set_pos = 0;
+            while (set_pos < new_set.size() && (op_data->stripes[i].osd_num != new_set[set_pos].osd_num ||
+                op_data->stripes[i].role != new_set[set_pos].role))
+            {
+                set_pos++;
+            }
+            if (set_pos >= new_set.size())
+            {
+                continue;
+            }
+            if (op_data->stripes[i].not_exists)
+            {
+                changes++;
+                new_set.erase(new_set.begin()+set_pos, new_set.begin()+set_pos+1);
+                continue;
+            }
+            auto & chunk = new_set[set_pos];
+            if (op_data->stripes[i].read_error && chunk.loc_bad != LOC_CORRUPTED)
+            {
+                changes++;
+                chunk.loc_bad = LOC_CORRUPTED;
+            }
+            else if (op_data->stripes[i].read_end > 0 && !op_data->stripes[chunk.role].missing &&
+                (chunk.loc_bad & LOC_CORRUPTED))
+            {
+                changes++;
+                chunk.loc_bad &= ~LOC_CORRUPTED;
+            }
+            if (inconsistent && !(chunk.loc_bad & LOC_INCONSISTENT))
+            {
+                changes++;
+                chunk.loc_bad |= LOC_INCONSISTENT;
+            }
+            else if (!inconsistent && (chunk.loc_bad & LOC_INCONSISTENT))
+            {
+                changes++;
+                chunk.loc_bad &= ~LOC_INCONSISTENT;
             }
         }
+        return changes;
+    });
+}
+
+void osd_t::continue_primary_scrub(osd_op_t *cur_op)
+{
+    if (!cur_op->op_data && !prepare_primary_rw(cur_op))
+        return;
+    if (cur_op->op_data->st == 1)
+        goto resume_1;
+    else if (cur_op->op_data->st == 2)
+        goto resume_2;
+    submit_scrub_subops(cur_op);
+resume_1:
+    return;
+resume_2:
+    if (cur_op->op_data->errors > 0 &&
+        // I/O and checksum errors (represented by stripes[i].read_error) are OK
+        (cur_op->op_data->errcode != -EIO && cur_op->op_data->errcode != -EDOM))
+    {
+        finish_op(cur_op, cur_op->op_data->errcode);
+        return;
+    }
+    scrub_check_results(cur_op);
     finish_op(cur_op, 0);
 }

src/test/bindiff.c

@@ -173,5 +173,7 @@ int main(int narg, char *args[])
     {
         fprintf(stderr, "Files are equal\n");
     }
+    free(buf1);
+    free(buf2);
     return 0;
 }

src/test/test_cluster_client.cpp

@@ -480,10 +480,76 @@ void test_writeback()
     printf("[ok] writeback test\n");
 }
 
+static void copy_write_for_test(writeback_cache_t *wb, uint64_t offset, uint64_t len, int state, uint64_t new_flush_id)
+{
+    void *buf = malloc_or_die(len);
+    cluster_op_t *op = new cluster_op_t();
+    op->opcode = OSD_OP_WRITE;
+    op->inode = 0x1000000000001;
+    op->offset = offset;
+    op->len = len;
+    op->iov.push_back(buf, len);
+    wb->copy_write(op, state, new_flush_id);
+    delete op;
+    free(buf);
+}
+
+void test_writeback_merge()
+{
+    writeback_cache_t *wb = new writeback_cache_t;
+    // [1000..3000]
+    copy_write_for_test(wb, 1000, 2000, CACHE_DIRTY, 0);
+    assert(wb->writeback_bytes == 2000);
+    assert(wb->writeback_queue_size == 1);
+    // [1000..3000][3000..4000]
+    copy_write_for_test(wb, 3000, 1000, CACHE_DIRTY, 0);
+    assert(wb->writeback_bytes == 3000);
+    assert(wb->writeback_queue_size == 1);
+    // [1000..3000][3000..4000], [5000..6000]
+    copy_write_for_test(wb, 5000, 1000, CACHE_DIRTY, 0);
+    assert(wb->writeback_bytes == 4000);
+    assert(wb->writeback_queue_size == 2);
+    // [1000..2500], [3500..4000], [5000..6000]
+    copy_write_for_test(wb, 2500, 1000, CACHE_WRITTEN, 0);
+    assert(wb->writeback_bytes == 3000);
+    assert(wb->writeback_queue_size == 3);
+    // [1000..2500], [3500..4000][4000...5000][5000..6000]
+    copy_write_for_test(wb, 4000, 1000, CACHE_DIRTY, 0);
+    assert(wb->writeback_bytes == 4000);
+    assert(wb->writeback_queue_size == 2);
+    // [1000..2500], [3500..4500][4500...5000][5000..6000]
+    copy_write_for_test(wb, 3500, 1000, CACHE_DIRTY, 0);
+    assert(wb->writeback_bytes == 4000);
+    assert(wb->writeback_queue_size == 2);
+    // [1000..2500], [3500..4500], [5000..6000]
+    copy_write_for_test(wb, 4500, 500, CACHE_WRITTEN, 0);
+    assert(wb->writeback_bytes == 3500);
+    assert(wb->writeback_queue_size == 3);
+    // [1000..2500][2500..3500][3500..4500], [5000..6000]
+    copy_write_for_test(wb, 2500, 1000, CACHE_DIRTY, 0);
+    assert(wb->writeback_bytes == 4500);
+    assert(wb->writeback_queue_size == 2);
+    // [1000..2500][2500..3500][3500..4500], [5500..6000]
+    copy_write_for_test(wb, 5000, 500, CACHE_WRITTEN, 0);
+    assert(wb->writeback_bytes == 4000);
+    assert(wb->writeback_queue_size == 2);
+    // [1000..2500][2500..3500][3500..4000], [5500..6000]
+    copy_write_for_test(wb, 4000, 1000, CACHE_WRITTEN, 0);
+    assert(wb->writeback_bytes == 3500);
+    assert(wb->writeback_queue_size == 2);
+    // [1000..2500][2500..3500][3500..4000][4000..5500][5500..6000]
+    copy_write_for_test(wb, 4000, 1500, CACHE_DIRTY, 0);
+    assert(wb->writeback_bytes == 5000);
+    assert(wb->writeback_queue_size == 1);
+    delete wb;
+    printf("[ok] writeback merge test\n");
+}
+
 int main(int narg, char *args[])
 {
     test1();
     test2();
     test_writeback();
+    test_writeback_merge();
     return 0;
 }

src/util/epoll_manager.cpp

@@ -3,6 +3,7 @@
 
 #include <sys/epoll.h>
 #include <sys/poll.h>
+#include <poll.h>
 #include <unistd.h>
 #include <stdexcept>
 
@@ -74,6 +75,27 @@ void epoll_manager_t::set_fd_handler(int fd, bool wr, std::function<void(int, in
             throw std::runtime_error(std::string("epoll_ctl: ") + strerror(errno));
         }
         epoll_handlers[fd] = handler;
+        // We use edge-triggered epoll so it may miss events which already happened
+        // on the FD at the moment of adding it to epoll. So check for these with poll()
+        struct pollfd initpoll = { .fd = fd, .events = (short)((wr ? POLLOUT : 0) | POLLIN | POLLRDHUP) };
+        int r = poll(&initpoll, 1, 0);
+        if (r < 0)
+            throw std::runtime_error(std::string("poll: ") + strerror(errno));
+        if (r > 0)
+        {
+            auto events = ((initpoll.revents & POLLOUT) ? EPOLLOUT : 0) |
+                ((initpoll.revents & POLLIN) ? EPOLLIN : 0) |
+                ((initpoll.revents & POLLRDHUP) ? EPOLLRDHUP : 0);
+            tfd->set_timer_us(1, false, [this, fd, events](int)
+            {
+                auto cb_it = epoll_handlers.find(fd);
+                if (cb_it != epoll_handlers.end())
+                {
+                    auto & cb = cb_it->second;
+                    cb(fd, events);
+                }
+            });
+        }
     }
     else
     {

src/util/ringloop.cpp

@@ -29,6 +29,7 @@ ring_loop_t::ring_loop_t(int qd, bool multithreaded)
     {
         free_ring_data[i] = i;
     }
+    in_loop = false;
 }
 
 ring_loop_t::~ring_loop_t()
@@ -86,6 +87,11 @@ io_uring_sqe* ring_loop_t::get_sqe()
 
 void ring_loop_t::loop()
 {
+    if (in_loop)
+    {
+        return;
+    }
+    in_loop = true;
     if (ring_eventfd >= 0)
     {
         // Reset eventfd counter
@@ -125,6 +131,17 @@ void ring_loop_t::loop()
         }
         io_uring_cqe_seen(&ring, cqe);
     }
+    in_loop = false;
+    loop_continue();
+}
+
+void ring_loop_t::loop_continue()
+{
+    if (in_loop)
+    {
+        return;
+    }
+    in_loop = true;
     do
     {
         loop_again = false;
@@ -140,6 +157,7 @@ void ring_loop_t::loop()
             }
         }
     } while (loop_again);
+    in_loop = false;
 }
 
 unsigned ring_loop_t::save()
@@ -189,5 +207,7 @@ int ring_loop_t::register_eventfd()
         ring_eventfd = -1;
         return r;
     }
+    // Loop once to prevent skipping events happened before eventfd was registered
+    loop();
     return ring_eventfd;
 }

src/util/ringloop.h

@@ -129,6 +129,7 @@ class ring_loop_t
     bool mt;
     int *free_ring_data;
     unsigned free_ring_data_ptr;
+    bool in_loop;
     bool loop_again;
     struct io_uring ring;
     int ring_eventfd = -1;
@@ -164,6 +165,7 @@ public:
     }
 
     void loop();
+    void loop_continue();
     void wakeup();
 
     unsigned save();

tests/test_rebalance_verify.sh

@@ -12,7 +12,7 @@ $ETCDCTL put /vitastor/config/inode/1/1 '{"name":"testimg","size":'$((IMG_SIZE*1
 
 NBD_DEV=$(sudo build/src/client/vitastor-nbd map --nbd_timeout 180 --etcd_address $ETCD_URL --image testimg --logfile ./testdata/nbd.log &)
 
-trap "sudo build/src/client/vitastor-nbd unmap $NBD_DEV"'; kill -9 $(jobs -p)' EXIT
+trap "sudo build/src/client/vitastor-nbd unmap $NBD_DEV"' || true; kill -9 $(jobs -p)' EXIT
 
 sudo chown $(id -u) $NBD_DEV