Release 2.4.4

- Implement `vitastor-kv rescue` command which dumps all readable blocks from the
  database regardless of the tree structure and allows to recover a corrupted database.
- Simplify primary OSD selection - use just PG number instead of the hash. This allows
  to distribute primary OSDs more evenly and reach better read speeds.
- Fix vitastor-kv-stress hang on parallel list & reopen.
- Fix CAS writes without immediate_commit. Fix is sufficient for vitastor-kv to
  operate correctly in non-immediate_commit pools.
- Fix NFS ACCESS calls for broken inodes (allow root access to them).
- Fix broken vitastor-kv-stress --runtime parameter.
- Fix bad vitastor-kv dump output buffering leading to incorrect JSON in dumps.
- Fix writing garbage from uninitialized memory to the empty space in VitastorKV DB.
- Refactor some code in preparation to the release of the new log-structured metadata store.
- Replace assert(done != expected) error message, probably caused by kernel issues with
  zero-copy TCP send, with a more concise error description.

.gitea/workflows/test.yml

@@ -414,6 +414,24 @@ jobs:
           echo ""
         done
 
+  test_degraded:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_degraded.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_rm:
     runs-on: ubuntu-latest
     needs: build
@@ -522,6 +540,42 @@ jobs:
           echo ""
         done
 
+  test_kv_stress:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_kv_stress.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_kv_stress_imm:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_kv_stress.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_splitbrain:
     runs-on: ubuntu-latest
     needs: build
@@ -810,6 +864,60 @@ jobs:
           echo ""
         done
 
+  test_reweight_half:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_reweight_half.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_snapshot_pool2:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_snapshot_pool2.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_snapshot_read_bitmap:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_snapshot_read_bitmap.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_heal_csum_32k_dmj:
     runs-on: ubuntu-latest
     needs: build
@@ -954,24 +1062,6 @@ jobs:
           echo ""
         done
 
-  test_snapshot_pool2:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_snapshot_pool2.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
   test_osd_tags:
     runs-on: ubuntu-latest
     needs: build

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "2.3.0")
+set(VITASTOR_VERSION "2.4.4")
 
 add_subdirectory(src)

csi/Dockerfile

@@ -36,7 +36,7 @@ RUN (echo deb http://vitastor.io/debian bookworm main > /etc/apt/sources.list.d/
     ((echo 'Package: *'; echo 'Pin: origin "vitastor.io"'; echo 'Pin-Priority: 1000') > /etc/apt/preferences.d/vitastor.pref) && \
     wget -q -O /etc/apt/trusted.gpg.d/vitastor.gpg https://vitastor.io/debian/pubkey.gpg && \
     apt-get update && \
-    apt-get install -y vitastor-client && \
+    apt-get install -y vitastor-client ibverbs-providers && \
     wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-utils_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
     wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-block-extra_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
     dpkg -x qemu-utils*.deb tmp1 && \

csi/Dockerfile.local

@@ -0,0 +1,49 @@
+# Compile stage
+FROM golang:bookworm AS build
+
+ADD go.sum go.mod /app/
+RUN cd /app; CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go mod download -x
+ADD . /app
+RUN perl -i -e '$/ = undef; while(<>) { s/\n\s*(\{\s*\n)/$1\n/g; s/\}(\s*\n\s*)else\b/$1} else/g; print; }' `find /app -name '*.go'` && \
+    cd /app && \
+    CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o vitastor-csi
+
+# Final stage
+FROM debian:bookworm
+
+LABEL maintainers="Vitaliy Filippov <vitalif@yourcmc.ru>"
+LABEL description="Vitastor CSI Driver"
+
+ENV NODE_ID=""
+ENV CSI_ENDPOINT=""
+
+RUN apt-get update && \
+    apt-get install -y wget && \
+    (echo "APT::Install-Recommends false;" > /etc/apt/apt.conf) && \
+    apt-get update && \
+    apt-get install -y e2fsprogs xfsprogs kmod iproute2 \
+        # NFS mount dependencies
+        nfs-common netbase \
+        # dependencies of qemu-storage-daemon
+        libnuma1 liburing2 libglib2.0-0 libfuse3-3 libaio1 libzstd1 libnettle8 \
+        libgmp10 libhogweed6 libp11-kit0 libidn2-0 libunistring2 libtasn1-6 libpcre2-8-0 libffi8 && \
+    apt-get clean && \
+    (echo options nbd nbds_max=128 > /etc/modprobe.d/nbd.conf)
+
+COPY --from=build /app/vitastor-csi /bin/
+
+ADD deb /deb
+
+RUN apt-get update && \
+    apt-get -y install /deb/vitastor-client_*.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-utils_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-block-extra_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
+    dpkg -x qemu-utils*.deb tmp1 && \
+    dpkg -x qemu-block-extra*.deb tmp1 && \
+    cp -a tmp1/usr/bin/qemu-storage-daemon /usr/bin/ && \
+    mkdir -p /usr/lib/x86_64-linux-gnu/qemu && \
+    cp -a tmp1/usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so /usr/lib/x86_64-linux-gnu/qemu/ && \
+    rm -rf tmp1 *.deb && \
+    apt-get clean
+
+ENTRYPOINT ["/bin/vitastor-csi"]

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v2.3.0
+VITASTOR_VERSION ?= v2.4.4
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v2.3.0
+          image: vitalif/vitastor-csi:v2.4.4
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v2.3.0
+          image: vitalif/vitastor-csi:v2.4.4
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "2.3.0"
+    vitastorCSIDriverVersion = "2.4.4"
 )
 
 // Config struct fills the parameters of request or user input

csi/src/nodeserver.go

@@ -33,7 +33,7 @@ import (
 type NodeServer struct
 {
     *Driver
-    useVduse        bool
+    method          MountMethod
     stateDir        string
     nfsStageDir     string
     mounter         mount.Interface
@@ -81,16 +81,23 @@ func NewNodeServer(driver *Driver) *NodeServer
     }
     ns := &NodeServer{
         Driver:      driver,
-        useVduse:    checkVduseSupport(),
+        method:      selectMountMethod(),
         stateDir:    stateDir,
         nfsStageDir: nfsStageDir,
         mounter:     mount.New(""),
         volumeLocks: make(map[string]bool),
     }
     ns.cond = sync.NewCond(&ns.mu)
-    if (ns.useVduse)
+    if (ns.method == MOUNT_VDUSE)
     {
         ns.restoreVduseDaemons()
+    }
+    else if (ns.method == MOUNT_UBLK)
+    {
+        ns.restoreUblkDaemons()
+    }
+    if (ns.method == MOUNT_VDUSE || ns.method == MOUNT_UBLK)
+    {
         dur, err := time.ParseDuration(os.Getenv("RESTART_INTERVAL"))
         if (err != nil)
         {
@@ -136,7 +143,14 @@ func (ns *NodeServer) restarter()
     for
     {
         <-ticker.C
-        ns.restoreVduseDaemons()
+        if (ns.method == MOUNT_VDUSE)
+        {
+            ns.restoreVduseDaemons()
+        }
+        else if (ns.method == MOUNT_UBLK)
+        {
+            ns.restoreUblkDaemons()
+        }
     }
 }
 
@@ -231,6 +245,78 @@ func (ns *NodeServer) checkVduseState(stateFile string, devs map[string]interfac
     }
 }
 
+func (ns *NodeServer) restoreUblkDaemons()
+{
+    pattern := ns.stateDir+"vitastor-ublk-*.json"
+    stateFiles, err := filepath.Glob(pattern)
+    if (err != nil)
+    {
+        klog.Errorf("failed to list %s: %v", pattern, err)
+    }
+    if (len(stateFiles) == 0)
+    {
+        return
+    }
+    for _, stateFile := range stateFiles
+    {
+        deviceNum := stateFile[len(ns.stateDir) + len("vitastor-ublk-") :]
+        deviceNum = deviceNum[0:len(deviceNum)-5]
+        ns.checkUblkState(deviceNum)
+    }
+}
+
+func (ns *NodeServer) checkUblkState(deviceNum string)
+{
+    // Check if the ublk daemon is still active
+
+    // Read state file
+    stateFile := ns.stateDir + "vitastor-ublk-" + deviceNum + ".json"
+    stateJSON, err := os.ReadFile(stateFile)
+    if (err != nil)
+    {
+        klog.Warningf("error reading state file %v: %v", stateFile, err)
+        return
+    }
+    var state DeviceState
+    err = json.Unmarshal(stateJSON, &state)
+    if (err != nil)
+    {
+        klog.Warningf("state file %v contains invalid JSON (error %v): %v", stateFile, err, string(stateJSON))
+        return
+    }
+
+    // Lock volume
+    ns.lockVolume(state.ConfigPath+":block:"+state.Image)
+    defer ns.unlockVolume(state.ConfigPath+":block:"+state.Image)
+
+    // Recheck state file after locking
+    _, err = os.ReadFile(stateFile)
+    if (err != nil)
+    {
+        klog.Warningf("state file %v disappeared, skipping volume", stateFile)
+        return
+    }
+
+    // Check if the vitastor-ublk process is still active
+    pidFile := ns.stateDir + "vitastor-ublk-" + deviceNum + ".pid"
+    exists := false
+    proc, err := findByPidFile(pidFile)
+    if (err == nil)
+    {
+        exists = proc.Signal(syscall.Signal(0)) == nil
+    }
+    if (!exists)
+    {
+        // Restart daemon
+        klog.Warningf("recovering UBLK device /dev/ublkb%v for volume %v", deviceNum, state.Image)
+        _, err = mapUblk(ns.stateDir, state.Image, state.ConfigPath, state.Readonly, "/dev/ublkb"+deviceNum)
+        if (err != nil)
+        {
+            klog.Warningf("failed to recover ublk device for volume %v: %v", state.Image, err)
+        }
+    }
+}
+
 func (ns *NodeServer) restoreNfsDaemons()
 {
     pattern := ns.stateDir+"vitastor-nfs-*.json"
@@ -417,14 +503,18 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
     }
 
     var devicePath, vdpaId string
-    if (!ns.useVduse)
+    if (ns.method == MOUNT_UBLK)
     {
-        devicePath, err = mapNbd(volName, ctxVars, false)
+        devicePath, err = mapUblk(ns.stateDir, volName, ctxVars["configPath"], false, "")
     }
-    else
+    else if (ns.method == MOUNT_VDUSE)
     {
         devicePath, vdpaId, err = mapVduse(ns.stateDir, volName, ctxVars, false)
     }
+    else /* if (ns.method == MOUNT_NBD) */
+    {
+        devicePath, err = mapNbd(volName, ctxVars, false)
+    }
     if (err != nil)
     {
         return nil, err
@@ -439,7 +529,8 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
     else
     {
         // Check existing format
-        existingFormat, err := diskMounter.GetDiskFormat(devicePath)
+        var existingFormat string
+        existingFormat, err = diskMounter.GetDiskFormat(devicePath)
         if (err != nil)
         {
             klog.Errorf("failed to get disk format for path %s, error: %v", err)
@@ -495,10 +586,6 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
                 case "xfs":
                     _, err = systemCombined("xfs_growfs", devicePath)
             }
-            if (err != nil)
-            {
-                goto unmap
-            }
         }
     }
     if (err != nil)
@@ -512,14 +599,18 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
     return &csi.NodeStageVolumeResponse{}, nil
 
 unmap:
-    if (!ns.useVduse || len(devicePath) >= 8 && devicePath[0:8] == "/dev/nbd")
+    if (ns.method == MOUNT_UBLK)
     {
-        unmapNbd(devicePath)
+        unmapUblk(ns.stateDir, devicePath)
     }
-    else
+    else if (ns.method == MOUNT_VDUSE)
     {
         unmapVduseById(ns.stateDir, vdpaId)
     }
+    else /* if (ns.method == MOUNT_NBD) */
+    {
+        unmapNbd(devicePath)
+    }
     return nil, err
 }
 
@@ -545,7 +636,7 @@ func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag
     defer ns.unlockVolume(ctxVars["configPath"]+":block:"+volName)
 
     targetPath := req.GetStagingTargetPath()
-    devicePath, _, err := mount.GetDeviceNameFromMount(ns.mounter, targetPath)
+    devicePath, err := GetDeviceNameFromMount(targetPath)
     if (err != nil)
     {
         if (os.IsNotExist(err))
@@ -582,14 +673,18 @@ func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag
     // unmap device
     if (len(refList) == 0)
     {
-        if (!ns.useVduse)
+        if (ns.method == MOUNT_UBLK)
         {
-            unmapNbd(devicePath)
+            unmapUblk(ns.stateDir, devicePath)
         }
-        else
+        else if (ns.method == MOUNT_VDUSE)
         {
             unmapVduse(ns.stateDir, devicePath)
         }
+        else /* if (ns.method == MOUNT_NBD) */
+        {
+            unmapNbd(devicePath)
+        }
     }
 
     return &csi.NodeUnstageVolumeResponse{}, nil
@@ -897,7 +992,7 @@ func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
     }
 
     targetPath := req.GetTargetPath()
-    devicePath, _, err := mount.GetDeviceNameFromMount(ns.mounter, targetPath)
+    devicePath, err := GetDeviceNameFromMount(targetPath)
     if (err != nil)
     {
         if (os.IsNotExist(err))

csi/src/utils.go

@@ -16,10 +16,20 @@ import (
     "syscall"
 
     "k8s.io/klog"
+    "k8s.io/utils/mount"
+
     "google.golang.org/grpc/codes"
     "google.golang.org/grpc/status"
 )
 
+type MountMethod int
+
+const (
+    MOUNT_NBD   MountMethod = 0
+    MOUNT_VDUSE MountMethod = 1
+    MOUNT_UBLK  MountMethod = 2
+)
+
 func Contains(list []string, s string) bool
 {
     for i := 0; i < len(list); i++
@@ -32,29 +42,26 @@ func Contains(list []string, s string) bool
     return false
 }
 
-func checkVduseSupport() bool
+func selectMountMethod() MountMethod
 {
+    // Check UBLK support (ublk_drv kernel module)
+    if (checkModule("ublk_drv"))
+    {
+        klog.Infof("UBLK support enabled successfully")
+        return MOUNT_UBLK
+    }
+    klog.Errorf(
+        "Your host apparently has no UBLK support. UBLK support disabled."+
+        " For UBLK you need at least Linux 6.0 and the ublk_drv kernel module.",
+    )
     // Check VDUSE support (vdpa, vduse, virtio-vdpa kernel modules)
     vduse := true
     for _, mod := range []string{"vdpa", "vduse", "virtio-vdpa"}
     {
-        _, err := os.Stat("/sys/module/"+mod)
-        if (err != nil)
+        if (!checkModule(mod))
         {
-            if (!errors.Is(err, os.ErrNotExist))
-            {
-                klog.Errorf("failed to check /sys/module/%s: %v", mod, err)
-            }
-            c := exec.Command("/sbin/modprobe", mod)
-            c.Stdout = os.Stderr
-            c.Stderr = os.Stderr
-            err := c.Run()
-            if (err != nil)
-            {
-                klog.Errorf("/sbin/modprobe %s failed: %v", mod, err)
-                vduse = false
-                break
-            }
+            vduse = false
+            break
         }
     }
     // Check that vdpa tool functions
@@ -69,18 +76,38 @@ func checkVduseSupport() bool
             vduse = false
         }
     }
-    if (!vduse)
-    {
-        klog.Errorf(
-            "Your host apparently has no VDUSE support. VDUSE support disabled, NBD will be used to map devices."+
-            " For VDUSE you need at least Linux 5.15 and the following kernel modules: vdpa, virtio-vdpa, vduse.",
-        )
-    }
-    else
+    if (vduse)
     {
         klog.Infof("VDUSE support enabled successfully")
+        return MOUNT_VDUSE
     }
-    return vduse
+    klog.Errorf(
+        "Your host apparently has no VDUSE support. VDUSE support disabled, NBD will be used to map devices."+
+        " For VDUSE you need at least Linux 5.15 and the following kernel modules: vdpa, virtio-vdpa, vduse.",
+    )
+    return MOUNT_NBD
+}
+
+func checkModule(mod string) bool
+{
+    _, err := os.Stat("/sys/module/"+mod)
+    if (err != nil)
+    {
+        if (!errors.Is(err, os.ErrNotExist))
+        {
+            klog.Errorf("failed to check /sys/module/%s: %v", mod, err)
+        }
+        c := exec.Command("/sbin/modprobe", mod)
+        c.Stdout = os.Stderr
+        c.Stderr = os.Stderr
+        err := c.Run()
+        if (err != nil)
+        {
+            klog.Errorf("/sbin/modprobe %s failed: %v", mod, err)
+            return false
+        }
+    }
+    return true
 }
 
 func mapNbd(volName string, ctxVars map[string]string, readonly bool) (string, error)
@@ -217,6 +244,7 @@ func mapVduse(stateDir string, volName string, ctxVars map[string]string, readon
                     stateJSON, _ := json.Marshal(&DeviceState{
                         ConfigPath: ctxVars["configPath"],
                         VdpaId:     vdpaId,
+
                         Image:      volName,
                         Blockdev:   blockdev,
                         Readonly:   readonly,
@@ -309,6 +337,117 @@ func unmapVduseById(stateDir, vdpaId string)
     }
 }
 
+func mapUblk(stateDir string, volName string, configPath string, readonly bool, recoverDev string) (string, error)
+{
+    pidFile := ""
+    if (recoverDev != "")
+    {
+        if (len(recoverDev) < 10 || recoverDev[0:10] != "/dev/ublkb")
+        {
+            return "", fmt.Errorf("recover: %s does not start with /dev/ublkb", recoverDev)
+        }
+        pidFile = stateDir + "vitastor-ublk-" + recoverDev[10:] + ".pid"
+    }
+    else
+    {
+        pidFd, err := os.CreateTemp(stateDir, "vitastor-tmp-*.pid")
+        if (err != nil)
+        {
+            return "", err
+        }
+        pidFile = pidFd.Name()
+        pidFd.Close()
+    }
+    // Map device via vitastor-ublk
+    args := []string{
+        "map", "--image", volName, "--pidfile", pidFile,
+    }
+    if (configPath != "")
+    {
+        args = append(args, "--config_path", configPath)
+    }
+    if (readonly)
+    {
+        args = append(args, "--readonly")
+    }
+    if (recoverDev != "")
+    {
+        args = append(args, "--recover", recoverDev)
+    }
+    stdout, stderr, err := system("/usr/bin/vitastor-ublk", args...)
+    if (err != nil)
+    {
+        return "", err
+    }
+    devicePath := strings.TrimSpace(string(stdout))
+    if (devicePath == "")
+    {
+        return "", fmt.Errorf("vitastor-ublk did not return the name of the device. output: %s", stderr)
+    }
+    if (len(devicePath) >= 10 && devicePath[0:10] == "/dev/ublkb")
+    {
+        // Generate state file
+        devNum := devicePath[10:]
+        pidNew := stateDir + "vitastor-ublk-" + devNum + ".pid"
+        if (pidFile != pidNew)
+        {
+            err := os.Rename(pidFile, pidNew)
+            if (err != nil)
+            {
+                klog.Errorf("Failed to rename PID file %s to %s: %v", pidFile, pidNew, err)
+            }
+            else
+            {
+                pidFile = pidNew
+            }
+        }
+        stateFile := stateDir + "vitastor-ublk-" + devNum + ".json"
+        stateJSON, _ := json.Marshal(&DeviceState{
+            ConfigPath: configPath,
+            Image:      volName,
+            Readonly:   readonly,
+            PidFile:    pidFile,
+        })
+        err = os.WriteFile(stateFile, stateJSON, 0600)
+        if (err == nil)
+        {
+            klog.Infof("Attached volume %s via UBLK as %s", volName, devicePath)
+            return devicePath, nil
+        }
+        os.Remove(stateFile)
+    }
+    killErr := killByPidFile(pidFile)
+    if (killErr != nil)
+    {
+        klog.Errorf("Failed to kill started vitastor-ublk: %v", killErr)
+    }
+    os.Remove(pidFile)
+    return "", err
+}
+
+func unmapUblk(stateDir, devicePath string)
+{
+    if (len(devicePath) < 10 || devicePath[0:10] != "/dev/ublkb")
+    {
+        klog.Errorf("%s does not start with /dev/ublkb", devicePath)
+        return
+    }
+    unmapOut, unmapErr := exec.Command("/usr/bin/vitastor-ublk", "unmap", devicePath).CombinedOutput()
+    if (unmapErr != nil)
+    {
+        klog.Errorf("failed to unmap UBLK device %s: %s, error: %v", devicePath, unmapOut, unmapErr)
+    }
+    for _, ext := range []string{"json", "pid"}
+    {
+        fn := stateDir + "vitastor-ublk-" + devicePath[10:] + "." + ext
+        err := os.Remove(fn)
+        if (err != nil)
+        {
+            klog.Errorf("failed to remove %s: %v", fn, err)
+        }
+    }
+}
+
 func system(program string, args ...string) ([]byte, []byte, error)
 {
     klog.Infof("Running "+program+" "+strings.Join(args, " "))
@@ -340,3 +479,43 @@ func systemCombined(program string, args ...string) ([]byte, error)
     }
     return out.Bytes(), nil
 }
+
+func GetDeviceNameFromMount(mountPath string) (string, error)
+{
+    // Use /proc/self/mountinfo to correctly parse bind mounts for block device files
+    mps, err := mount.ParseMountInfo("/proc/self/mountinfo")
+    if (err != nil)
+    {
+        return "", err
+    }
+
+    slTarget, err := filepath.EvalSymlinks(mountPath)
+    if (err != nil)
+    {
+        slTarget = mountPath
+    }
+
+    device := ""
+    for _, mp := range mps
+    {
+        if (mp.MountPoint == slTarget)
+        {
+            device = mp.Source
+            if (device[0] != '/' && mp.Root != "/")
+            {
+                // Handle {Source=udev Root=/vdb MountPoint=/var/lib/kubelet/tralaleylo/tralala}
+                for _, other := range mps
+                {
+                    if (other.Root == "/" && other.Source == mp.Source)
+                    {
+                        device = other.MountPoint + mp.Root
+                        break
+                    }
+                }
+            }
+            break
+        }
+    }
+
+    return device, nil
+}

debian/build-vitastor-bookworm.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 docker build --build-arg DISTRO=debian --build-arg REL=bookworm -t vitastor-buildenv:bookworm -f vitastor-buildenv.Dockerfile .
-docker run -i --rm -e REL=bookworm -v `dirname $0`/../:/root/vitastor vitastor-buildenv:bookworm /root/vitastor/debian/vitastor-build.sh
+docker run -it --rm -e REL=bookworm -v `dirname $0`/../:/root/vitastor vitastor-buildenv:bookworm /root/vitastor/debian/vitastor-build.sh

debian/build-vitastor-bullseye.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 docker build --build-arg DISTRO=debian --build-arg REL=bullseye -t vitastor-buildenv:bullseye -f vitastor-buildenv.Dockerfile .
-docker run -i --rm -e REL=bullseye -v `dirname $0`/../:/root/vitastor vitastor-buildenv:bullseye /root/vitastor/debian/vitastor-build.sh
+docker run -it --rm -e REL=bullseye -v `dirname $0`/../:/root/vitastor vitastor-buildenv:bullseye /root/vitastor/debian/vitastor-build.sh

debian/build-vitastor-buster.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 docker build --build-arg DISTRO=debian --build-arg REL=buster -t vitastor-buildenv:buster -f vitastor-buildenv.Dockerfile .
-docker run -i --rm -e REL=buster -v `dirname $0`/../:/root/vitastor vitastor-buildenv:buster /root/vitastor/debian/vitastor-build.sh
+docker run -it --rm -e REL=buster -v `dirname $0`/../:/root/vitastor vitastor-buildenv:buster /root/vitastor/debian/vitastor-build.sh

debian/build-vitastor-trixie.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 docker build --build-arg DISTRO=debian --build-arg REL=trixie -t vitastor-buildenv:trixie -f vitastor-buildenv.Dockerfile .
-docker run -i --rm -e REL=trixie -v `dirname $0`/../:/root/vitastor vitastor-buildenv:trixie /root/vitastor/debian/vitastor-build.sh
+docker run -it --rm -e REL=trixie -v `dirname $0`/../:/root/vitastor vitastor-buildenv:trixie /root/vitastor/debian/vitastor-build.sh

debian/build-vitastor-ubuntu-jammy.sh

@@ -2,4 +2,4 @@
 # Ubuntu 22.04 Jammy Jellyfish
 
 docker build --build-arg DISTRO=ubuntu --build-arg REL=jammy -t vitastor-buildenv:jammy -f vitastor-buildenv.Dockerfile .
-docker run -i --rm -e REL=jammy -v `dirname $0`/../:/root/vitastor vitastor-buildenv:jammy /root/vitastor/debian/vitastor-build.sh
+docker run -it --rm -e REL=jammy -v `dirname $0`/../:/root/vitastor vitastor-buildenv:jammy /root/vitastor/debian/vitastor-build.sh

debian/build-vitastor-ubuntu-noble.sh

@@ -2,4 +2,4 @@
 # 24.04 Noble Numbat
 
 docker build --build-arg DISTRO=ubuntu --build-arg REL=noble -t vitastor-buildenv:noble -f vitastor-buildenv.Dockerfile .
-docker run -i --rm -e REL=noble -v `dirname $0`/../:/root/vitastor vitastor-buildenv:noble /root/vitastor/debian/vitastor-build.sh
+docker run -it --rm -e REL=noble -v `dirname $0`/../:/root/vitastor vitastor-buildenv:noble /root/vitastor/debian/vitastor-build.sh

debian/build-vitastor-ubuntu-questing.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+# 25.10 Questing quokka
+
+docker build --build-arg DISTRO=ubuntu --build-arg REL=questing -t vitastor-buildenv:questing -f vitastor-buildenv.Dockerfile .
+docker run -it --rm -e REL=questing -v `dirname $0`/../:/root/vitastor vitastor-buildenv:questing /root/vitastor/debian/vitastor-build.sh

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (2.3.0-1) unstable; urgency=medium
+vitastor (2.4.4-1) unstable; urgency=medium
 
   * Bugfixes
 

debian/control

@@ -4,7 +4,7 @@ Priority: optional
 Maintainer: Vitaliy Filippov <vitalif@yourcmc.ru>
 Build-Depends: debhelper, g++ (>= 8), libstdc++6 (>= 8),
   linux-libc-dev, libgoogle-perftools-dev, libjerasure-dev, libgf-complete-dev,
-  libibverbs-dev, libisal-dev, cmake, pkg-config, libnl-3-dev, libnl-genl-3-dev,
+  libibverbs-dev, librdmacm-dev, libisal-dev, cmake, pkg-config, libnl-3-dev, libnl-genl-3-dev,
   node-bindings <!nocheck>, node-gyp, node-nan
 Standards-Version: 4.5.0
 Homepage: https://vitastor.io/

debian/vitastor-build.sh

@@ -44,7 +44,12 @@ curl -s https://git.yourcmc.ru/vitalif/antietcd/archive/master.tar.gz | tar -zx
 curl -s https://git.yourcmc.ru/vitalif/tinyraft/archive/master.tar.gz | tar -zx
 
 cd /root/vitastor/packages/vitastor-$REL
-tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_$VER.orig.tar.xz vitastor-$VER
+if [[ "$REL" = "trixie" && -e ../vitastor-bookworm/vitastor_$VER.orig.tar.xz ]]; then
+    # Fucking shit, archives differ between bookworm (xz 5.4.1) and trixie (xz 5.8.1)
+    cp ../vitastor-bookworm/vitastor_$VER.orig.tar.xz .
+else
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_$VER.orig.tar.xz vitastor-$VER
+fi
 cd vitastor-$VER
 DEBEMAIL="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$FULLVER""$REL" "Rebuild for $REL"
 DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa

docker/Dockerfile

@@ -3,7 +3,7 @@
 FROM debian:bookworm
 
 ADD etc/apt /etc/apt/
-RUN apt-get update && apt-get -y install vitastor udev systemd qemu-system-x86 qemu-system-common qemu-block-extra qemu-utils jq nfs-common && apt-get clean
+RUN apt-get update && apt-get -y install vitastor ibverbs-providers udev systemd qemu-system-x86 qemu-system-common qemu-block-extra qemu-utils jq nfs-common && apt-get clean
 ADD sleep.sh /usr/bin/
 ADD install.sh /usr/bin/
 ADD scripts /opt/scripts/

docker/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v2.3.0
+VITASTOR_VERSION ?= v2.4.4
 
 all: build push
 

docker/etc/vitastor/docker.conf

@@ -4,7 +4,7 @@
 #
 
 # Desired Vitastor version
-VITASTOR_VERSION=v2.3.0
+VITASTOR_VERSION=v2.4.4
 
 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here

docs/installation/docker.en.md

@@ -26,9 +26,9 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.
 
 1. Download a Docker image of the desired version: \
-   `docker pull vitalif/vitastor:v2.3.0`
+   `docker pull vitalif/vitastor:v2.4.4`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.3.0 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.4.4 install.sh`
 3. Reload udev rules: \
    `udevadm control --reload-rules`
 

docs/installation/docker.ru.md

@@ -25,9 +25,9 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.
 
 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitalif/vitastor:v2.3.0`
+   `docker pull vitalif/vitastor:v2.4.4`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.3.0 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.4.4 install.sh`
 3. Перезагрузите правила udev: \
    `udevadm control --reload-rules`
 

docs/installation/openstack.en.md

@@ -30,7 +30,7 @@ volume_backend_name = vitastor-testcluster
 image_volume_cache_enabled = True
 volume_clear = none
 vitastor_etcd_address = 192.168.7.2:2379
-vitastor_etcd_prefix =
+vitastor_etcd_prefix = /vitastor
 vitastor_config_path = /etc/vitastor/vitastor.conf
 vitastor_pool_id = 1
 image_upload_use_cinder_backend = True

docs/installation/openstack.ru.md

@@ -29,7 +29,7 @@ volume_backend_name = vitastor-testcluster
 image_volume_cache_enabled = True
 volume_clear = none
 vitastor_etcd_address = 192.168.7.2:2379
-vitastor_etcd_prefix =
+vitastor_etcd_prefix = /vitastor
 vitastor_config_path = /etc/vitastor/vitastor.conf
 vitastor_pool_id = 1
 image_upload_use_cinder_backend = True

docs/usage/cli.en.md

@@ -100,12 +100,14 @@ List images (only matching `<glob>` pattern(s) if passed).
 Options:
 
 ```
+--exact         Do not match glob patterns as names, select only exact name matches.
 -p|--pool POOL  Filter images by pool ID or name
 -l|--long       Also report allocated size and I/O statistics
 --del           Also include delete operation statistics
 --sort FIELD    Sort by specified field (name, size, used_size, <read|write|delete>_<iops|bps|lat|queue>)
 -r|--reverse    Sort in descending order
 -n|--count N    Only list first N items
+--tree          Show image snapshot/clone tree
 ```
 
 Example output:

docs/usage/cli.ru.md

@@ -102,12 +102,14 @@ kaveri    2/1     32   0 B      10 G    0 B        100%    0%
 Опции:
 
 ```
+--exact         Не применять ФС-шаблоны к именам, выводить только точные совпадения
 -p|--pool POOL  Фильтровать образы по пулу (ID или имени)
 -l|--long       Также выводить статистику занятого места и ввода-вывода
 --del           Также выводить статистику операций удаления
 --sort FIELD    Сортировать по заданному полю (name, size, used_size, <read|write|delete>_<iops|bps|lat|queue>)
 -r|--reverse    Сортировать в обратном порядке
 -n|--count N    Показывать только первые N записей
+--tree          Вывести снапшоты и клоны в виде дерева
 ```
 
 Пример вывода:

docs/usage/disk.en.md

@@ -73,6 +73,8 @@ Options (automatic mode):
 --max_other 10%
   Use disks for OSD data even if they already have non-Vitastor partitions,
   but only if these take up no more than this percent of disk space.
+--dry-run
+  Check and print new OSD count for each disk but do not actually create them.
 ```
 
 Options (single-device mode):

docs/usage/disk.ru.md

@@ -74,6 +74,8 @@ vitastor-disk - инструмент командной строки для уп
 --max_other 10%
   Использовать диски под данные OSD, даже если на них уже есть не-Vitastor-овые
   разделы, но только в случае, если они занимают не более данного процента диска.
+--dry-run
+  Проверить и вывести число новых OSD для каждого диска, но не создавать их.
 ```
 
 Опции для режима одного OSD:

mon/osd_tree.js

@@ -15,7 +15,7 @@ function get_osd_tree(global_config, state)
         const stat = state.osd.stats[osd_num];
         const osd_cfg = state.config.osd[osd_num];
         let reweight = osd_cfg == null ? 1 : Number(osd_cfg.reweight);
-        if (isNaN(reweight) || reweight < 0 || reweight > 0)
+        if (isNaN(reweight) || reweight < 0 || reweight > 1)
             reweight = 1;
         if (stat && stat.size && reweight && (state.osd.state[osd_num] || Number(stat.time) >= down_time ||
             osd_cfg && osd_cfg.noout))

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "2.3.0",
+  "version": "2.4.4",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {

mon/pg_gen.js

@@ -9,7 +9,6 @@ const LPOptimizer = require('./lp_optimizer/lp_optimizer.js');
 const { scale_pg_count } = require('./pg_utils.js');
 const { make_hier_tree, filter_osds_by_root_node,
     filter_osds_by_tags, filter_osds_by_block_layout, get_affinity_osds } = require('./osd_tree.js');
-const { select_murmur3 } = require('./lp_optimizer/murmur3.js');
 
 function pick_primary(pool_id, pg_num, pool_config, osd_set, up_osds, aff_osds)
 {
@@ -39,7 +38,7 @@ function pick_primary(pool_id, pg_num, pool_config, osd_set, up_osds, aff_osds)
     {
         return 0;
     }
-    return alive_set[select_murmur3(alive_set.length, osd_num => pool_id+'/'+pg_num+'/'+osd_num)];
+    return alive_set[pg_num % alive_set.length];
 }
 
 function recheck_primary(state, global_config, up_osds, osd_tree)

mon/scripts/make-etcd

@@ -52,15 +52,16 @@ async function run()
         process.exit(1);
     }
     const etcds = (config.etcd_address instanceof Array ? config.etcd_address : (''+config.etcd_address).split(/,/))
-        .map(s => (''+s).replace(/^https?:\/\/\[?|\]?(:\d+)?(\/.*)?$/g, '').toLowerCase());
+        .map(s => (''+s).replace(/^https?:\/\/|(:\d+)?(\/.*)?$/g, '').replace(/^\[(.*)\]$/, '$1').toLowerCase());
     const num = select_local_etcd(etcds);
     if (num < 0)
     {
         console.log('No matching IPs in etcd_address from '+config_path);
         process.exit(0);
     }
+    const etcd_url = 'http://' + (etcds[num].indexOf(':') >= 0 ? '['+etcds[num]+']' : etcds[num]);
     const etcd_name = 'etcd'+etcds[num].replace(/[^0-9a-z_]/ig, '_');
-    const etcd_cluster = etcds.map(e => `etcd${e.replace(/[^0-9a-z_]/ig, '_')}=http://${e}:2380`).join(',');
+    const etcd_cluster = etcds.map(e => `etcd${e.replace(/[^0-9a-z_]/ig, '_')}=http://${e.indexOf(':') >= 0 ? '['+e+']' : e}:2380`).join(',');
     if (in_docker)
     {
         let etcd_conf = fs.readFileSync("/etc/vitastor/etcd.conf", { encoding: 'utf-8' });
@@ -83,8 +84,8 @@ Wants=network-online.target local-fs.target time-sync.target
 Restart=always
 Environment=GOGC=50
 ExecStart=etcd --name ${etcd_name} --data-dir /var/lib/etcd/vitastor \\
-    --snapshot-count 10000 --advertise-client-urls http://${etcds[num]}:2379 --listen-client-urls http://${etcds[num]}:2379 \\
-    --initial-advertise-peer-urls http://${etcds[num]}:2380 --listen-peer-urls http://${etcds[num]}:2380 \\
+    --snapshot-count 10000 --advertise-client-urls ${etcd_url}:2379 --listen-client-urls ${etcd_url}:2379 \\
+    --initial-advertise-peer-urls ${etcd_url}:2380 --listen-peer-urls ${etcd_url}:2380 \\
     --initial-cluster-token vitastor-etcd-1 --initial-cluster ${etcd_cluster} \\
     --initial-cluster-state new --max-txn-ops=100000 --max-request-bytes=104857600 \\
     --auto-compaction-retention=10 --auto-compaction-mode=revision

mon/stats.js

@@ -276,6 +276,10 @@ function sum_inode_stats(state, prev_stats)
         }
         for (const pool_id in osd_diff.inode_stats)
         {
+            if (!inode_stats[pool_id])
+            {
+                continue;
+            }
             for (const inode_num in prev_stats.osd_diff[osd].inode_stats[pool_id])
             {
                 inode_stats[pool_id][inode_num] = inode_stats[pool_id][inode_num] || inode_stub();

node-binding/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor",
-  "version": "2.3.0",
+  "version": "2.4.4",
   "description": "Low-level native bindings to Vitastor client library",
   "main": "index.js",
   "keywords": [

patches/VitastorPlugin.pm

@@ -499,4 +499,55 @@ sub rename_volume
     return "${storeid}:${base_name}${target_volname}";
 }
 
+sub _monkey_patch_qemu_blockdev_options
+{
+    my ($cfg, $volid, $machine_version, $options) = @_;
+    my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
+
+    my $scfg = PVE::Storage::storage_config($cfg, $storeid);
+
+    my $plugin = PVE::Storage::Plugin->lookup($scfg->{type});
+
+    my ($vtype) = $plugin->parse_volname($volname);
+    die "cannot use volume of type '$vtype' as a QEMU blockdevice\n"
+        if $vtype ne 'images' && $vtype ne 'iso' && $vtype ne 'import';
+
+    return $plugin->qemu_blockdev_options($scfg, $storeid, $volname, $machine_version, $options);
+}
+
+sub qemu_blockdev_options
+{
+    my ($class, $scfg, $storeid, $volname, $machine_version, $options) = @_;
+    my $prefix = defined $scfg->{vitastor_prefix} ? $scfg->{vitastor_prefix} : 'pve/';
+    my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+    $name .= '@'.$options->{'snapshot-name'} if $options->{'snapshot-name'};
+    if ($scfg->{vitastor_nbd})
+    {
+        my $mapped = run_cli($scfg, [ 'ls' ], binary => '/usr/bin/vitastor-nbd');
+        my ($kerneldev) = grep { $mapped->{$_}->{image} eq $prefix.$name } keys %$mapped;
+        die "Image not mapped via NBD" if !$kerneldev;
+        return { driver => 'host_device', filename => $kerneldev };
+    }
+    my $blockdev = {
+        driver => 'vitastor',
+        image => $prefix.$name,
+    };
+    if ($scfg->{vitastor_config_path})
+    {
+        $blockdev->{'config-path'} = $scfg->{vitastor_config_path};
+    }
+    if ($scfg->{vitastor_etcd_address})
+    {
+        # FIXME This is the only exception: etcd_address -> etcd_host for qemu
+        $blockdev->{'etcd-host'} = $scfg->{vitastor_etcd_address};
+    }
+    if ($scfg->{vitastor_etcd_prefix})
+    {
+        $blockdev->{'etcd-prefix'} = $scfg->{vitastor_etcd_prefix};
+    }
+    return $blockdev;
+}
+
+*PVE::Storage::qemu_blockdev_options = *_monkey_patch_qemu_blockdev_options;
+
 1;

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '2.3.0'
+VITASTOR_VERSION = '2.4.4'
 
 LOG = logging.getLogger(__name__)
 

patches/pve-qemu-10.1-vitastor.patch

@@ -0,0 +1,172 @@
+Index: pve-qemu-kvm-10.1.2/block/meson.build
+===================================================================
+--- pve-qemu-kvm-10.1.2.orig/block/meson.build
++++ pve-qemu-kvm-10.1.2/block/meson.build
+@@ -126,6 +126,7 @@ foreach m : [
+   [libnfs, 'nfs', files('nfs.c')],
+   [libssh, 'ssh', files('ssh.c')],
+   [rbd, 'rbd', files('rbd.c')],
++  [vitastor, 'vitastor', files('vitastor.c')],
+ ]
+   if m[0].found()
+     module_ss = ss.source_set()
+Index: pve-qemu-kvm-10.1.2/meson.build
+===================================================================
+--- pve-qemu-kvm-10.1.2.orig/meson.build
++++ pve-qemu-kvm-10.1.2/meson.build
+@@ -1653,6 +1653,26 @@ if not get_option('rbd').auto() or have_
+   endif
+ endif
+ 
++vitastor = not_found
++if not get_option('vitastor').auto() or have_block
++  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
++    required: get_option('vitastor'))
++  if libvitastor_client.found()
++    if cc.links('''
++      #include <vitastor_c.h>
++      int main(void) {
++        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
++        return 0;
++      }''', dependencies: libvitastor_client)
++      vitastor = declare_dependency(dependencies: libvitastor_client)
++    elif get_option('vitastor').enabled()
++      error('could not link libvitastor_client')
++    else
++      warning('could not link libvitastor_client, disabling')
++    endif
++  endif
++endif
++
+ glusterfs = not_found
+ glusterfs_ftruncate_has_stat = false
+ glusterfs_iocb_has_stat = false
+@@ -2552,6 +2572,7 @@ endif
+ config_host_data.set('CONFIG_OPENGL', opengl.found())
+ config_host_data.set('CONFIG_PLUGIN', get_option('plugins'))
+ config_host_data.set('CONFIG_RBD', rbd.found())
++config_host_data.set('CONFIG_VITASTOR', vitastor.found())
+ config_host_data.set('CONFIG_RDMA', rdma.found())
+ config_host_data.set('CONFIG_RELOCATABLE', get_option('relocatable'))
+ config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack'))
+@@ -4984,6 +5005,7 @@ summary_info += {'fdt support':       fd
+ summary_info += {'libcap-ng support': libcap_ng}
+ summary_info += {'bpf support':       libbpf}
+ summary_info += {'rbd support':       rbd}
++summary_info += {'vitastor support':  vitastor}
+ summary_info += {'smartcard support': cacard}
+ summary_info += {'U2F support':       u2f}
+ summary_info += {'libusb':            libusb}
+Index: pve-qemu-kvm-10.1.2/meson_options.txt
+===================================================================
+--- pve-qemu-kvm-10.1.2.orig/meson_options.txt
++++ pve-qemu-kvm-10.1.2/meson_options.txt
+@@ -202,6 +202,8 @@ option('pvg', type: 'feature', value: 'a
+        description: 'macOS paravirtualized graphics support')
+ option('rbd', type : 'feature', value : 'auto',
+        description: 'Ceph block device driver')
++option('vitastor', type : 'feature', value : 'auto',
++       description: 'Vitastor block device driver')
+ option('opengl', type : 'feature', value : 'auto',
+        description: 'OpenGL support')
+ option('rdma', type : 'feature', value : 'auto',
+Index: pve-qemu-kvm-10.1.2/qapi/block-core.json
+===================================================================
+--- pve-qemu-kvm-10.1.2.orig/qapi/block-core.json
++++ pve-qemu-kvm-10.1.2/qapi/block-core.json
+@@ -3647,7 +3647,7 @@
+             'raw', 'rbd',
+             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
+             'pbs',
+-            'ssh', 'throttle', 'vdi', 'vhdx',
++            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
+             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
+@@ -4773,6 +4773,28 @@
+             '*server': ['InetSocketAddressBase'] } }
+ 
+ ##
++# @BlockdevOptionsVitastor:
++#
++# Driver specific block device options for vitastor
++#
++# @image:       Image name
++# @inode:       Inode number
++# @pool:        Pool ID
++# @size:        Desired image size in bytes
++# @config-path: Path to Vitastor configuration
++# @etcd-host:   etcd connection address(es)
++# @etcd-prefix: etcd key/value prefix
++##
++{ 'struct': 'BlockdevOptionsVitastor',
++  'data': { '*inode': 'uint64',
++            '*pool': 'uint64',
++            '*size': 'uint64',
++            '*image': 'str',
++            '*config-path': 'str',
++            '*etcd-host': 'str',
++            '*etcd-prefix': 'str' } }
++
++##
+ # @ReplicationMode:
+ #
+ # An enumeration of replication modes.
+@@ -5242,6 +5264,7 @@
+       'throttle':   'BlockdevOptionsThrottle',
+       'vdi':        'BlockdevOptionsGenericFormat',
+       'vhdx':       'BlockdevOptionsGenericFormat',
++      'vitastor':   'BlockdevOptionsVitastor',
+       'virtio-blk-vfio-pci':
+                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
+                       'if': 'CONFIG_BLKIO' },
+@@ -5722,6 +5745,20 @@
+             '*encrypt' :        'RbdEncryptionCreateOptions' } }
+ 
+ ##
++# @BlockdevCreateOptionsVitastor:
++#
++# Driver specific image creation options for Vitastor.
++#
++# @location: Where to store the new image file.  This location cannot
++#     point to a snapshot.
++#
++# @size: Size of the virtual disk in bytes
++##
++{ 'struct': 'BlockdevCreateOptionsVitastor',
++  'data': { 'location':         'BlockdevOptionsVitastor',
++            'size':             'size' } }
++
++##
+ # @BlockdevVmdkSubformat:
+ #
+ # Subformat options for VMDK images
+@@ -5943,6 +5980,7 @@
+       'ssh':            'BlockdevCreateOptionsSsh',
+       'vdi':            'BlockdevCreateOptionsVdi',
+       'vhdx':           'BlockdevCreateOptionsVhdx',
++      'vitastor':       'BlockdevCreateOptionsVitastor',
+       'vmdk':           'BlockdevCreateOptionsVmdk',
+       'vpc':            'BlockdevCreateOptionsVpc'
+   } }
+Index: pve-qemu-kvm-10.1.2/scripts/meson-buildoptions.sh
+===================================================================
+--- pve-qemu-kvm-10.1.2.orig/scripts/meson-buildoptions.sh
++++ pve-qemu-kvm-10.1.2/scripts/meson-buildoptions.sh
+@@ -175,6 +175,7 @@ meson_options_help() {
+   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
+   printf "%s\n" '  qpl             Query Processing Library support'
+   printf "%s\n" '  rbd             Ceph block device driver'
++  printf "%s\n" '  vitastor        Vitastor block device driver'
+   printf "%s\n" '  rdma            Enable RDMA-based migration'
+   printf "%s\n" '  replication     replication support'
+   printf "%s\n" '  rust            Rust support'
+@@ -459,6 +460,8 @@ _meson_option_parse() {
+     --disable-qpl) printf "%s" -Dqpl=disabled ;;
+     --enable-rbd) printf "%s" -Drbd=enabled ;;
+     --disable-rbd) printf "%s" -Drbd=disabled ;;
++    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
++    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
+     --enable-rdma) printf "%s" -Drdma=enabled ;;
+     --disable-rdma) printf "%s" -Drdma=disabled ;;
+     --enable-relocatable) printf "%s" -Drelocatable=true ;;

patches/qemu-10.1-vitastor.patch

@@ -0,0 +1,29 @@
+diff --git a/src/client/qemu_driver.c b/src/client/qemu_driver.c
+index d8356dab..5f4cd50d 100644
+--- a/src/client/qemu_driver.c
++++ b/src/client/qemu_driver.c
+@@ -974,14 +974,21 @@ static void vitastor_co_read_bitmap_cb(void *opaque, long retval, uint8_t *bitma
+ #endif
+ }
+ 
+-static int coroutine_fn vitastor_co_block_status(
+-    BlockDriverState *bs, bool want_zero, int64_t offset, int64_t bytes,
+-    int64_t *pnum, int64_t *map, BlockDriverState **file)
++static int coroutine_fn vitastor_co_block_status(BlockDriverState *bs,
++#if QEMU_VERSION_MAJOR > 10 || QEMU_VERSION_MAJOR == 10 && QEMU_VERSION_MINOR >= 1
++    unsigned int mode,
++#else
++    bool want_zero,
++#endif
++    int64_t offset, int64_t bytes, int64_t *pnum, int64_t *map, BlockDriverState **file)
+ {
+     // Allocated => return BDRV_BLOCK_DATA|BDRV_BLOCK_OFFSET_VALID
+     // Not allocated => return 0
+     // Error => return -errno
+     // Set pnum to length of the extent, `*map` = `offset`, `*file` = `bs`
++#if QEMU_VERSION_MAJOR > 10 || QEMU_VERSION_MAJOR == 10 && QEMU_VERSION_MINOR >= 1
++    int want_zero = (mode == BDRV_WANT_PRECISE);
++#endif
+     VitastorRPC task;
+     VitastorClient *client = bs->opaque;
+     uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode;

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.3.0
+Version:        2.4.4
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.3.0.el7.tar.gz
+Source0:        vitastor-2.4.4.el7.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  devtoolset-9-gcc-c++

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.3.0
+Version:        2.4.4
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.3.0.el8.tar.gz
+Source0:        vitastor-2.4.4.el8.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-toolset-9-gcc-c++

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.3.0
+Version:        2.4.4
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.3.0.el9.tar.gz
+Source0:        vitastor-2.4.4.el9.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++

src/CMakeLists.txt

@@ -20,7 +20,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVITASTOR_VERSION="2.3.0")
+add_definitions(-DVITASTOR_VERSION="2.4.4")
 add_definitions(-D_GNU_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -fvisibility=hidden -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/blockstore/blockstore.cpp

@@ -3,87 +3,7 @@
 
 #include "blockstore_impl.h"
 
-blockstore_t::blockstore_t(blockstore_config_t & config, ring_loop_t *ringloop, timerfd_manager_t *tfd)
+blockstore_i* blockstore_i::create(blockstore_config_t & config, ring_loop_t *ringloop, timerfd_manager_t *tfd)
 {
-    impl = new blockstore_impl_t(config, ringloop, tfd);
-}
-
-blockstore_t::~blockstore_t()
-{
-    delete impl;
-}
-
-void blockstore_t::parse_config(blockstore_config_t & config)
-{
-    impl->parse_config(config, false);
-}
-
-void blockstore_t::loop()
-{
-    impl->loop();
-}
-
-bool blockstore_t::is_started()
-{
-    return impl->is_started();
-}
-
-bool blockstore_t::is_stalled()
-{
-    return impl->is_stalled();
-}
-
-bool blockstore_t::is_safe_to_stop()
-{
-    return impl->is_safe_to_stop();
-}
-
-void blockstore_t::enqueue_op(blockstore_op_t *op)
-{
-    impl->enqueue_op(op);
-}
-
-int blockstore_t::read_bitmap(object_id oid, uint64_t target_version, void *bitmap, uint64_t *result_version)
-{
-    return impl->read_bitmap(oid, target_version, bitmap, result_version);
-}
-
-std::map<uint64_t, uint64_t> & blockstore_t::get_inode_space_stats()
-{
-    return impl->inode_space_stats;
-}
-
-void blockstore_t::dump_diagnostics()
-{
-    return impl->dump_diagnostics();
-}
-
-uint32_t blockstore_t::get_block_size()
-{
-    return impl->get_block_size();
-}
-
-uint64_t blockstore_t::get_block_count()
-{
-    return impl->get_block_count();
-}
-
-uint64_t blockstore_t::get_free_block_count()
-{
-    return impl->get_free_block_count();
-}
-
-uint64_t blockstore_t::get_journal_size()
-{
-    return impl->get_journal_size();
-}
-
-uint32_t blockstore_t::get_bitmap_granularity()
-{
-    return impl->get_bitmap_granularity();
-}
-
-void blockstore_t::set_no_inode_stats(const std::vector<uint64_t> & pool_ids)
-{
-    impl->set_no_inode_stats(pool_ids);
+    return new blockstore_impl_t(config, ringloop, tfd);
 }

src/blockstore/blockstore.h

@@ -17,22 +17,14 @@
 #include "ringloop.h"
 #include "timerfd_manager.h"
 
-// Memory alignment for direct I/O (usually 512 bytes)
-#ifndef DIRECT_IO_ALIGNMENT
-#define DIRECT_IO_ALIGNMENT 512
-#endif
-
-// Memory allocation alignment (page size is usually optimal)
-#ifndef MEM_ALIGNMENT
-#define MEM_ALIGNMENT 4096
-#endif
-
 // Default block size is 128 KB, current allowed range is 4K - 128M
 #define DEFAULT_DATA_BLOCK_ORDER 17
 #define MIN_DATA_BLOCK_SIZE 4*1024
 #define MAX_DATA_BLOCK_SIZE 128*1024*1024
 #define DEFAULT_BITMAP_GRANULARITY 4096
 
+#define MIN_JOURNAL_SIZE 1024*1024
+
 #define BS_OP_MIN 1
 #define BS_OP_READ 1
 #define BS_OP_WRITE 2
@@ -46,8 +38,18 @@
 
 #define BS_OP_PRIVATE_DATA_SIZE 256
 
+#define IMMEDIATE_NONE 0
+#define IMMEDIATE_SMALL 1
+#define IMMEDIATE_ALL 2
+
 /*
 
+All operations may be submitted in any order, because reads only see completed writes,
+syncs only sync completed writes and writes don't depend on each other.
+
+The only restriction is that the external code MUST NOT submit multiple writes for one
+object in parallel. This is a natural restriction because `version` numbers are used though.
+
 Blockstore opcode documentation:
 
 ## BS_OP_READ / BS_OP_WRITE / BS_OP_WRITE_STABLE
@@ -162,8 +164,8 @@ struct __attribute__ ((visibility("default"))) blockstore_op_t
             uint32_t list_stable_limit;
         };
     };
-    void *buf = NULL;
-    void *bitmap = NULL;
+    uint8_t *buf = NULL;
+    uint8_t *bitmap = NULL;
     int retval = 0;
 
     uint8_t private_data[BS_OP_PRIVATE_DATA_SIZE];
@@ -171,53 +173,54 @@ struct __attribute__ ((visibility("default"))) blockstore_op_t
 
 typedef std::map<std::string, std::string> blockstore_config_t;
 
-class blockstore_impl_t;
-
-class __attribute__((visibility("default"))) blockstore_t
+class __attribute__((visibility("default"))) blockstore_i
 {
-    blockstore_impl_t *impl;
 public:
-    blockstore_t(blockstore_config_t & config, ring_loop_t *ringloop, timerfd_manager_t *tfd);
-    ~blockstore_t();
+    static blockstore_i* create(blockstore_config_t & config, ring_loop_t *ringloop, timerfd_manager_t *tfd);
+
+    virtual ~blockstore_i() = default;
 
     // Update configuration
-    void parse_config(blockstore_config_t & config);
+    virtual void parse_config(blockstore_config_t & config) = 0;
 
     // Event loop
-    void loop();
+    virtual void loop() = 0;
 
     // Returns true when blockstore is ready to process operations
     // (Although you're free to enqueue them before that)
-    bool is_started();
+    virtual bool is_started() = 0;
 
     // Returns true when blockstore is stalled
-    bool is_stalled();
+    virtual bool is_stalled() = 0;
 
     // Returns true when it's safe to destroy the instance. If destroying the instance
     // requires to purge some queues, starts that process. Should be called in the event
     // loop until it returns true.
-    bool is_safe_to_stop();
+    virtual bool is_safe_to_stop() = 0;
 
     // Submission
-    void enqueue_op(blockstore_op_t *op);
+    virtual void enqueue_op(blockstore_op_t *op) = 0;
 
     // Simplified synchronous operation: get object bitmap & current version
-    int read_bitmap(object_id oid, uint64_t target_version, void *bitmap, uint64_t *result_version = NULL);
+    virtual int read_bitmap(object_id oid, uint64_t target_version, void *bitmap, uint64_t *result_version = NULL) = 0;
 
     // Get per-inode space usage statistics
-    std::map<uint64_t, uint64_t> & get_inode_space_stats();
+    virtual const std::map<uint64_t, uint64_t> & get_inode_space_stats() = 0;
 
     // Set per-pool no_inode_stats
-    void set_no_inode_stats(const std::vector<uint64_t> & pool_ids);
+    virtual void set_no_inode_stats(const std::vector<uint64_t> & pool_ids) = 0;
 
     // Print diagnostics to stdout
-    void dump_diagnostics();
+    virtual void dump_diagnostics() = 0;
 
-    uint32_t get_block_size();
-    uint64_t get_block_count();
-    uint64_t get_free_block_count();
+    // Get diagnostic string for an operation
+    virtual std::string get_op_diag(blockstore_op_t *op) = 0;
 
-    uint64_t get_journal_size();
+    virtual uint32_t get_block_size() = 0;
+    virtual uint64_t get_block_count() = 0;
+    virtual uint64_t get_free_block_count() = 0;
 
-    uint32_t get_bitmap_granularity();
+    virtual uint64_t get_journal_size() = 0;
+
+    virtual uint32_t get_bitmap_granularity() = 0;
 };

src/blockstore/blockstore_disk.cpp

@@ -95,7 +95,7 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     {
         data_block_size = (1 << DEFAULT_DATA_BLOCK_ORDER);
     }
-    if ((block_order = is_power_of_two(data_block_size)) >= 64 || data_block_size < MIN_DATA_BLOCK_SIZE || data_block_size >= MAX_DATA_BLOCK_SIZE)
+    if (is_power_of_two(data_block_size) >= 64 || data_block_size < MIN_DATA_BLOCK_SIZE || data_block_size >= MAX_DATA_BLOCK_SIZE)
     {
         throw std::runtime_error("Bad block size");
     }

src/blockstore/blockstore_disk.h

@@ -8,6 +8,11 @@
 #include <string>
 #include <map>
 
+// Memory alignment for direct I/O (usually 512 bytes)
+#ifndef DIRECT_IO_ALIGNMENT
+#define DIRECT_IO_ALIGNMENT 512
+#endif
+
 #define BLOCKSTORE_CSUM_NONE 0
 // Lower byte of checksum type is its length
 #define BLOCKSTORE_CSUM_CRC32C 0x104
@@ -46,7 +51,6 @@ struct blockstore_disk_t
     uint64_t data_offset, data_device_sect, data_device_size, data_len;
     uint64_t journal_offset, journal_device_sect, journal_device_size, journal_len;
 
-    uint32_t block_order = 0;
     uint64_t block_count = 0;
     uint32_t clean_entry_bitmap_size = 0, clean_entry_size = 0, clean_dyn_size = 0;
 

src/blockstore/blockstore_flush.cpp

@@ -2,6 +2,7 @@
 // License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
+#include "blockstore_internal.h"
 
 #define META_BLOCK_UNREAD 0
 #define META_BLOCK_READ 1
@@ -542,7 +543,7 @@ resume_2:
                 if (old_entry->oid.inode != 0 && old_entry->oid != cur.oid)
                 {
                     printf("Fatal error (metadata corruption or bug): tried to wipe metadata entry %ju (%jx:%jx v%ju) as old location of %jx:%jx\n",
-                        old_clean_loc >> bs->dsk.block_order, old_entry->oid.inode, old_entry->oid.stripe,
+                        old_clean_loc / bs->dsk.data_block_size, old_entry->oid.inode, old_entry->oid.stripe,
                         old_entry->version, cur.oid.inode, cur.oid.stripe);
                     exit(1);
                 }
@@ -645,7 +646,7 @@ void journal_flusher_co::update_metadata_entry()
             has_delete
                 ? "Fatal error (metadata corruption or bug): tried to delete metadata entry %ju (%jx:%jx v%ju) while deleting %jx:%jx v%ju\n"
                 : "Fatal error (metadata corruption or bug): tried to overwrite non-zero metadata entry %ju (%jx:%jx v%ju) with %jx:%jx v%ju\n",
-            clean_loc >> bs->dsk.block_order, new_entry->oid.inode, new_entry->oid.stripe,
+            clean_loc / bs->dsk.data_block_size, new_entry->oid.inode, new_entry->oid.stripe,
             new_entry->version, cur.oid.inode, cur.oid.stripe, cur.version
         );
         exit(1);
@@ -695,7 +696,7 @@ void journal_flusher_co::update_metadata_entry()
         new_entry->version = cur.version;
         if (!bs->inmemory_meta)
         {
-            auto inmem_bmp = (uint8_t*)bs->clean_bitmaps + (clean_loc >> bs->dsk.block_order)*2*bs->dsk.clean_entry_bitmap_size;
+            auto inmem_bmp = (uint8_t*)bs->clean_bitmaps + (clean_loc / bs->dsk.data_block_size)*2*bs->dsk.clean_entry_bitmap_size;
             memcpy(inmem_bmp, new_clean_bitmap, 2*bs->dsk.clean_entry_bitmap_size);
         }
         if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_V2)
@@ -848,7 +849,7 @@ bool journal_flusher_co::clear_incomplete_csum_block_bits(int wait_base)
             {
                 printf(
                     "Fatal error (metadata corruption or bug): tried to make holes in %ju (%jx:%jx v%ju) with %jx:%jx v%ju\n",
-                    clean_loc >> bs->dsk.block_order, new_entry->oid.inode, new_entry->oid.stripe,
+                    clean_loc / bs->dsk.data_block_size, new_entry->oid.inode, new_entry->oid.stripe,
                     new_entry->version, cur.oid.inode, cur.oid.stripe, cur.version
                 );
             }
@@ -864,7 +865,7 @@ bool journal_flusher_co::clear_incomplete_csum_block_bits(int wait_base)
             calc_block_checksums(new_data_csums, true);
             if (!bs->inmemory_meta)
             {
-                auto inmem_bmp = (uint8_t*)bs->clean_bitmaps + (clean_loc >> bs->dsk.block_order)*2*bs->dsk.clean_entry_bitmap_size;
+                auto inmem_bmp = (uint8_t*)bs->clean_bitmaps + (clean_loc / bs->dsk.data_block_size)*2*bs->dsk.clean_entry_bitmap_size;
                 memcpy(inmem_bmp, new_clean_bitmap, 2*bs->dsk.clean_entry_bitmap_size);
             }
             if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_V2)
@@ -1231,8 +1232,8 @@ bool journal_flusher_co::modify_meta_read(uint64_t meta_loc, flusher_meta_write_
     // And yet another option is to use LSM trees for metadata, but it sophisticates everything a lot,
     // so I'll avoid it as long as I can.
     wr.submitted = false;
-    wr.sector = ((meta_loc >> bs->dsk.block_order) / (bs->dsk.meta_block_size / bs->dsk.clean_entry_size)) * bs->dsk.meta_block_size;
-    wr.pos = ((meta_loc >> bs->dsk.block_order) % (bs->dsk.meta_block_size / bs->dsk.clean_entry_size));
+    wr.sector = ((meta_loc / bs->dsk.data_block_size) / (bs->dsk.meta_block_size / bs->dsk.clean_entry_size)) * bs->dsk.meta_block_size;
+    wr.pos = ((meta_loc / bs->dsk.data_block_size) % (bs->dsk.meta_block_size / bs->dsk.clean_entry_size));
     if (bs->inmemory_meta)
     {
         wr.buf = (uint8_t*)bs->metadata_buffer + wr.sector;
@@ -1292,14 +1293,14 @@ void journal_flusher_co::free_data_blocks()
 #ifdef BLOCKSTORE_DEBUG
         printf("%s block %ju from %jx:%jx v%ju (new location is %ju)\n",
             used ? "Postpone free" : "Free",
-            old_clean_loc >> bs->dsk.block_order,
+            old_clean_loc / bs->dsk.data_block_size,
             cur.oid.inode, cur.oid.stripe, cur.version,
-            clean_loc >> bs->dsk.block_order);
+            clean_loc / bs->dsk.data_block_size);
 #endif
         if (used)
             uo_it->second.was_freed = true;
         else
-            bs->data_alloc->set(old_clean_loc >> bs->dsk.block_order, false);
+            bs->data_alloc->set(old_clean_loc / bs->dsk.data_block_size, false);
     }
     if (has_delete)
     {
@@ -1309,13 +1310,13 @@ void journal_flusher_co::free_data_blocks()
 #ifdef BLOCKSTORE_DEBUG
         printf("%s block %ju from %jx:%jx v%ju (delete)\n",
             used ? "Postpone free" : "Free",
-            old_clean_loc >> bs->dsk.block_order,
+            old_clean_loc / bs->dsk.data_block_size,
             cur.oid.inode, cur.oid.stripe, cur.version);
 #endif
         if (used)
             uo_it->second.was_freed = true;
         else
-            bs->data_alloc->set(old_clean_loc >> bs->dsk.block_order, false);
+            bs->data_alloc->set(old_clean_loc / bs->dsk.data_block_size, false);
     }
 }
 

src/blockstore/blockstore_flush.h

@@ -1,14 +1,6 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 
-#define COPY_BUF_JOURNAL 1
-#define COPY_BUF_DATA 2
-#define COPY_BUF_ZERO 4
-#define COPY_BUF_CSUM_FILL 8
-#define COPY_BUF_COALESCED 16
-#define COPY_BUF_META_BLOCK 32
-#define COPY_BUF_JOURNALED_BIG 64
-
 struct copy_buffer_t
 {
     int copy_flags;

src/blockstore/blockstore_impl.cpp

@@ -2,6 +2,7 @@
 // License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
+#include "blockstore_internal.h"
 
 blockstore_impl_t::blockstore_impl_t(blockstore_config_t & config, ring_loop_t *ringloop, timerfd_manager_t *tfd)
 {
@@ -228,6 +229,8 @@ void blockstore_impl_t::loop()
         for (auto s: journal.submitting_sectors)
         {
             // Mark journal sector writes as submitted
+            if (journal.sector_info[s].submit_id)
+                journal.sector_info[s].written = true;
             journal.sector_info[s].submit_id = 0;
         }
         journal.submitting_sectors.clear();
@@ -272,7 +275,7 @@ void blockstore_impl_t::check_wait(blockstore_op_t *op)
 {
     if (PRIV(op)->wait_for == WAIT_SQE)
     {
-        if (ringloop->sqes_left() < PRIV(op)->wait_detail)
+        if (ringloop->space_left() < PRIV(op)->wait_detail)
         {
             // stop submission if there's still no free space
 #ifdef BLOCKSTORE_DEBUG
@@ -680,7 +683,7 @@ void blockstore_impl_t::process_list(blockstore_op_t *op)
     free(unstable);
     op->version = stable_count;
     op->retval = stable_count+unstable_count;
-    op->buf = stable;
+    op->buf = (uint8_t*)stable;
     FINISH_OP(op);
 }
 
@@ -703,6 +706,11 @@ void blockstore_impl_t::disk_error_abort(const char *op, int retval, int expecte
     exit(1);
 }
 
+const std::map<uint64_t, uint64_t> & blockstore_impl_t::get_inode_space_stats()
+{
+    return inode_space_stats;
+}
+
 void blockstore_impl_t::set_no_inode_stats(const std::vector<uint64_t> & pool_ids)
 {
     for (auto & np: no_inode_stats)
@@ -785,3 +793,14 @@ void blockstore_impl_t::recalc_inode_space_stats(uint64_t pool_id, bool per_inod
         dirty_it++;
     }
 }
+
+std::string blockstore_impl_t::get_op_diag(blockstore_op_t *op)
+{
+    char buf[256];
+    auto priv = PRIV(op);
+    if (priv->wait_for)
+        snprintf(buf, sizeof(buf), "state=%d wait=%d (detail=%ju)", priv->op_state, priv->wait_for, priv->wait_detail);
+    else
+        snprintf(buf, sizeof(buf), "state=%d", priv->op_state);
+    return std::string(buf);
+}

src/blockstore/blockstore_impl.h

@@ -5,6 +5,7 @@
 
 #include "blockstore.h"
 #include "blockstore_disk.h"
+#include "ondisk_formats.h"
 
 #include <sys/types.h>
 #include <sys/ioctl.h>
@@ -28,112 +29,8 @@
 
 //#define BLOCKSTORE_DEBUG
 
-// States are not stored on disk. Instead, they're deduced from the journal
-
-#define BS_ST_SMALL_WRITE 0x01
-#define BS_ST_BIG_WRITE 0x02
-#define BS_ST_DELETE 0x03
-
-#define BS_ST_WAIT_DEL 0x10
-#define BS_ST_WAIT_BIG 0x20
-#define BS_ST_IN_FLIGHT 0x30
-#define BS_ST_SUBMITTED 0x40
-#define BS_ST_WRITTEN 0x50
-#define BS_ST_SYNCED 0x60
-#define BS_ST_STABLE 0x70
-
-#define BS_ST_INSTANT 0x100
-
-#define IMMEDIATE_NONE 0
-#define IMMEDIATE_SMALL 1
-#define IMMEDIATE_ALL 2
-
-#define BS_ST_TYPE_MASK 0x0F
-#define BS_ST_WORKFLOW_MASK 0xF0
-#define IS_IN_FLIGHT(st) (((st) & 0xF0) <= BS_ST_SUBMITTED)
-#define IS_STABLE(st) (((st) & 0xF0) == BS_ST_STABLE)
-#define IS_SYNCED(st) (((st) & 0xF0) >= BS_ST_SYNCED)
-#define IS_JOURNAL(st) (((st) & 0x0F) == BS_ST_SMALL_WRITE)
-#define IS_BIG_WRITE(st) (((st) & 0x0F) == BS_ST_BIG_WRITE)
-#define IS_DELETE(st) (((st) & 0x0F) == BS_ST_DELETE)
-#define IS_INSTANT(st) (((st) & BS_ST_TYPE_MASK) == BS_ST_DELETE || ((st) & BS_ST_INSTANT))
-
-#define BS_SUBMIT_CHECK_SQES(n) \
-    if (ringloop->sqes_left() < (n))\
-    {\
-        /* Pause until there are more requests available */\
-        PRIV(op)->wait_detail = (n);\
-        PRIV(op)->wait_for = WAIT_SQE;\
-        return 0;\
-    }
-
-#define BS_SUBMIT_GET_SQE(sqe, data) \
-    BS_SUBMIT_GET_ONLY_SQE(sqe); \
-    struct ring_data_t *data = ((ring_data_t*)sqe->user_data)
-
-#define BS_SUBMIT_GET_ONLY_SQE(sqe) \
-    struct io_uring_sqe *sqe = get_sqe();\
-    if (!sqe)\
-    {\
-        /* Pause until there are more requests available */\
-        PRIV(op)->wait_detail = 1;\
-        PRIV(op)->wait_for = WAIT_SQE;\
-        return 0;\
-    }
-
-#define BS_SUBMIT_GET_SQE_DECL(sqe) \
-    sqe = get_sqe();\
-    if (!sqe)\
-    {\
-        /* Pause until there are more requests available */\
-        PRIV(op)->wait_detail = 1;\
-        PRIV(op)->wait_for = WAIT_SQE;\
-        return 0;\
-    }
-
 #include "blockstore_journal.h"
 
-// "VITAstor"
-#define BLOCKSTORE_META_MAGIC_V1 0x726F747341544956l
-#define BLOCKSTORE_META_FORMAT_V1 1
-#define BLOCKSTORE_META_FORMAT_V2 2
-
-// metadata header (superblock)
-struct __attribute__((__packed__)) blockstore_meta_header_v1_t
-{
-    uint64_t zero;
-    uint64_t magic;
-    uint64_t version;
-    uint32_t meta_block_size;
-    uint32_t data_block_size;
-    uint32_t bitmap_granularity;
-};
-
-struct __attribute__((__packed__)) blockstore_meta_header_v2_t
-{
-    uint64_t zero;
-    uint64_t magic;
-    uint64_t version;
-    uint32_t meta_block_size;
-    uint32_t data_block_size;
-    uint32_t bitmap_granularity;
-    uint32_t data_csum_type;
-    uint32_t csum_block_size;
-    uint32_t header_csum;
-};
-
-// 32 bytes = 24 bytes + block bitmap (4 bytes by default) + external attributes (also bitmap, 4 bytes by default)
-// per "clean" entry on disk with fixed metadata tables
-struct __attribute__((__packed__)) clean_disk_entry
-{
-    object_id oid;
-    uint64_t version;
-    uint8_t bitmap[];
-    // Two more fields come after bitmap in metadata version 2:
-    // uint32_t data_csum[];
-    // uint32_t entry_csum;
-};
-
 // 32 = 16 + 16 bytes per "clean" entry in memory (object_id => clean_entry)
 struct __attribute__((__packed__)) clean_entry
 {
@@ -170,15 +67,6 @@ struct __attribute__((__packed__)) dirty_entry
 // Otherwise, the submit order is free, that is all operations may be submitted immediately
 // In fact, adding a write operation must immediately result in dirty_db being populated
 
-// Suspend operation until there are more free SQEs
-#define WAIT_SQE 1
-// Suspend operation until there are <wait_detail> bytes of free space in the journal on disk
-#define WAIT_JOURNAL 3
-// Suspend operation until the next journal sector buffer is free
-#define WAIT_JOURNAL_BUFFER 4
-// Suspend operation until there is some free space on the data device
-#define WAIT_FREE 5
-
 struct used_clean_obj_t
 {
     int refs;
@@ -196,9 +84,6 @@ typedef std::map<obj_ver_id, dirty_entry> blockstore_dirty_db_t;
 
 #include "blockstore_flush.h"
 
-#define PRIV(op) ((blockstore_op_private_t*)(op)->private_data)
-#define FINISH_OP(op) PRIV(op)->~blockstore_op_private_t(); std::function<void (blockstore_op_t*)>(op->callback)(op)
-
 struct blockstore_op_private_t
 {
     // Wait status
@@ -224,23 +109,15 @@ struct blockstore_op_private_t
     std::vector<obj_ver_id> sync_big_writes, sync_small_writes;
 };
 
-typedef uint32_t pool_id_t;
-typedef uint64_t pool_pg_id_t;
-
-#define POOL_ID_BITS 16
-
 struct pool_shard_settings_t
 {
     uint32_t pg_count;
     uint32_t pg_stripe_size;
 };
 
-#define STAB_SPLIT_DONE 1
-#define STAB_SPLIT_WAIT 2
-#define STAB_SPLIT_SYNC 3
-#define STAB_SPLIT_TODO 4
+typedef uint64_t pool_pg_id_t;
 
-class blockstore_impl_t
+class blockstore_impl_t: public blockstore_i
 {
     blockstore_disk_t dsk;
 
@@ -276,6 +153,7 @@ class blockstore_impl_t
     std::map<pool_id_t, pool_shard_settings_t> clean_db_settings;
     std::map<pool_pg_id_t, blockstore_clean_db_t> clean_db_shards;
     std::map<uint64_t, int> no_inode_stats;
+    std::map<uint64_t, uint64_t> inode_space_stats;
     uint8_t *clean_bitmaps = NULL;
     blockstore_dirty_db_t dirty_db;
     std::vector<blockstore_op_t*> submit_queue;
@@ -404,6 +282,7 @@ public:
     blockstore_impl_t(blockstore_config_t & config, ring_loop_t *ringloop, timerfd_manager_t *tfd);
     ~blockstore_impl_t();
 
+    void parse_config(blockstore_config_t & config);
     void parse_config(blockstore_config_t & config, bool init);
 
     // Event loop
@@ -430,8 +309,8 @@ public:
     // Unstable writes are added here (map of object_id -> version)
     std::unordered_map<object_id, uint64_t> unstable_writes;
 
-    // Space usage statistics
-    std::map<uint64_t, uint64_t> inode_space_stats;
+    // Get space usage statistics
+    const std::map<uint64_t, uint64_t> & get_inode_space_stats();
 
     // Set per-pool no_inode_stats
     void set_no_inode_stats(const std::vector<uint64_t> & pool_ids);
@@ -439,6 +318,9 @@ public:
     // Print diagnostics to stdout
     void dump_diagnostics();
 
+    // Get diagnostic string for an operation
+    std::string get_op_diag(blockstore_op_t *op);
+
     inline uint32_t get_block_size() { return dsk.data_block_size; }
     inline uint64_t get_block_count() { return dsk.block_count; }
     inline uint64_t get_free_block_count() { return dsk.block_count - used_blocks; }

src/blockstore/blockstore_init.cpp

@@ -2,6 +2,7 @@
 // License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
+#include "blockstore_internal.h"
 
 #define INIT_META_EMPTY 0
 #define INIT_META_READING 1
@@ -372,7 +373,7 @@ bool blockstore_init_meta::handle_meta_block(uint8_t *buf, uint64_t entries_per_
                     // free the previous block
                     // here we have to zero out the previous entry because otherwise we'll hit
                     // "tried to overwrite non-zero metadata entry" later
-                    uint64_t old_clean_loc = clean_it->second.location >> bs->dsk.block_order;
+                    uint64_t old_clean_loc = clean_it->second.location / bs->dsk.data_block_size;
                     if (bs->inmemory_meta)
                     {
                         uint64_t sector = (old_clean_loc / entries_per_block) * bs->dsk.meta_block_size;
@@ -390,7 +391,7 @@ bool blockstore_init_meta::handle_meta_block(uint8_t *buf, uint64_t entries_per_
                     }
                     else
                     {
-                        entries_to_zero.push_back(clean_it->second.location >> bs->dsk.block_order);
+                        entries_to_zero.push_back(clean_it->second.location / bs->dsk.data_block_size);
                     }
 #ifdef BLOCKSTORE_DEBUG
                     printf("Free block %ju from %jx:%jx v%ju (new location is %ju)\n",
@@ -412,7 +413,7 @@ bool blockstore_init_meta::handle_meta_block(uint8_t *buf, uint64_t entries_per_
                 bs->data_alloc->set(done_cnt+i, true);
                 clean_db[entry->oid] = (struct clean_entry){
                     .version = entry->version,
-                    .location = (done_cnt+i) << bs->dsk.block_order,
+                    .location = (done_cnt+i) * bs->dsk.data_block_size,
                 };
             }
             else
@@ -1001,7 +1002,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                 printf(
                     "je_big_write%s oid=%jx:%jx ver=%ju loc=%ju\n",
                     je->type == JE_BIG_WRITE_INSTANT ? "_instant" : "",
-                    je->big_write.oid.inode, je->big_write.oid.stripe, je->big_write.version, je->big_write.location >> bs->dsk.block_order
+                    je->big_write.oid.inode, je->big_write.oid.stripe, je->big_write.version, je->big_write.location / bs->dsk.data_block_size
                 );
 #endif
                 auto dirty_it = bs->dirty_db.upper_bound((obj_ver_id){
@@ -1064,7 +1065,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                         .journal_sector = proc_pos,
                         .dyn_data = dyn,
                     }).first;
-                    if (bs->data_alloc->get(je->big_write.location >> bs->dsk.block_order))
+                    if (bs->data_alloc->get(je->big_write.location / bs->dsk.data_block_size))
                     {
                         // This is probably a big_write that's already flushed and freed, but it may
                         // also indicate a bug. So we remember such entries and recheck them afterwards.
@@ -1077,11 +1078,11 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
 #ifdef BLOCKSTORE_DEBUG
                         printf(
                             "Allocate block (journal) %ju: %jx:%jx v%ju\n",
-                            je->big_write.location >> bs->dsk.block_order,
+                            je->big_write.location / bs->dsk.data_block_size,
                             ov.oid.inode, ov.oid.stripe, ov.version
                         );
 #endif
-                        bs->data_alloc->set(je->big_write.location >> bs->dsk.block_order, true);
+                        bs->data_alloc->set(je->big_write.location / bs->dsk.data_block_size, true);
                     }
                     bs->journal.used_sectors[proc_pos]++;
 #ifdef BLOCKSTORE_DEBUG

src/blockstore/blockstore_internal.h

@@ -0,0 +1,85 @@
+#pragma once
+
+// States are not stored on disk. Instead, they're deduced from the journal
+
+#define BS_ST_SMALL_WRITE 0x01
+#define BS_ST_BIG_WRITE 0x02
+#define BS_ST_DELETE 0x03
+
+#define BS_ST_WAIT_DEL 0x10
+#define BS_ST_WAIT_BIG 0x20
+#define BS_ST_IN_FLIGHT 0x30
+#define BS_ST_SUBMITTED 0x40
+#define BS_ST_WRITTEN 0x50
+#define BS_ST_SYNCED 0x60
+#define BS_ST_STABLE 0x70
+
+#define BS_ST_INSTANT 0x100
+
+#define BS_ST_TYPE_MASK 0x0F
+#define BS_ST_WORKFLOW_MASK 0xF0
+#define IS_IN_FLIGHT(st) (((st) & 0xF0) <= BS_ST_SUBMITTED)
+#define IS_STABLE(st) (((st) & 0xF0) == BS_ST_STABLE)
+#define IS_SYNCED(st) (((st) & 0xF0) >= BS_ST_SYNCED)
+#define IS_JOURNAL(st) (((st) & 0x0F) == BS_ST_SMALL_WRITE)
+#define IS_BIG_WRITE(st) (((st) & 0x0F) == BS_ST_BIG_WRITE)
+#define IS_DELETE(st) (((st) & 0x0F) == BS_ST_DELETE)
+#define IS_INSTANT(st) (((st) & BS_ST_TYPE_MASK) == BS_ST_DELETE || ((st) & BS_ST_INSTANT))
+
+#define BS_SUBMIT_CHECK_SQES(n) \
+    if (ringloop->space_left() < (n))\
+    {\
+        /* Pause until there are more requests available */\
+        PRIV(op)->wait_detail = (n);\
+        PRIV(op)->wait_for = WAIT_SQE;\
+        return 0;\
+    }
+
+#define BS_SUBMIT_GET_SQE(sqe, data) \
+    BS_SUBMIT_GET_ONLY_SQE(sqe); \
+    struct ring_data_t *data = ((ring_data_t*)sqe->user_data)
+
+#define BS_SUBMIT_GET_ONLY_SQE(sqe) \
+    struct io_uring_sqe *sqe = get_sqe();\
+    if (!sqe)\
+    {\
+        /* Pause until there are more requests available */\
+        PRIV(op)->wait_detail = 1;\
+        PRIV(op)->wait_for = WAIT_SQE;\
+        return 0;\
+    }
+
+#define BS_SUBMIT_GET_SQE_DECL(sqe) \
+    sqe = get_sqe();\
+    if (!sqe)\
+    {\
+        /* Pause until there are more requests available */\
+        PRIV(op)->wait_detail = 1;\
+        PRIV(op)->wait_for = WAIT_SQE;\
+        return 0;\
+    }
+
+#define PRIV(op) ((blockstore_op_private_t*)(op)->private_data)
+#define FINISH_OP(op) PRIV(op)->~blockstore_op_private_t(); std::function<void (blockstore_op_t*)>(op->callback)(op)
+
+// Suspend operation until there are more free SQEs
+#define WAIT_SQE 1
+// Suspend operation until there are <wait_detail> bytes of free space in the journal on disk
+#define WAIT_JOURNAL 3
+// Suspend operation until the next journal sector buffer is free
+#define WAIT_JOURNAL_BUFFER 4
+// Suspend operation until there is some free space on the data device
+#define WAIT_FREE 5
+
+#define COPY_BUF_JOURNAL 1
+#define COPY_BUF_DATA 2
+#define COPY_BUF_ZERO 4
+#define COPY_BUF_CSUM_FILL 8
+#define COPY_BUF_COALESCED 16
+#define COPY_BUF_META_BLOCK 32
+#define COPY_BUF_JOURNALED_BIG 64
+
+#define STAB_SPLIT_DONE 1
+#define STAB_SPLIT_WAIT 2
+#define STAB_SPLIT_SYNC 3
+#define STAB_SPLIT_TODO 4

src/blockstore/blockstore_journal.cpp

@@ -2,6 +2,7 @@
 // License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
+#include "blockstore_internal.h"
 
 blockstore_journal_check_t::blockstore_journal_check_t(blockstore_impl_t *bs)
 {
@@ -178,7 +179,7 @@ void blockstore_impl_t::prepare_journal_sector_write(int cur_sector, blockstore_
         // Caller must ensure availability of an SQE
         assert(sqe != NULL);
         ring_data_t *data = ((ring_data_t*)sqe->user_data);
-        journal.sector_info[cur_sector].written = true;
+        // <written> flag will be set at the moment of actual submission
         journal.sector_info[cur_sector].submit_id = ++journal.submit_id;
         assert(journal.submit_id != 0); // check overflow
         journal.submitting_sectors.push_back(cur_sector);

src/blockstore/blockstore_journal.h

@@ -3,147 +3,7 @@
 
 #pragma once
 
-#include "crc32c.h"
-#include <set>
-
-#define MIN_JOURNAL_SIZE 4*1024*1024
-#define JOURNAL_MAGIC 0x4A33
-#define JOURNAL_VERSION_V1 1
-#define JOURNAL_VERSION_V2 2
-#define JOURNAL_BUFFER_SIZE 4*1024*1024
-#define JOURNAL_ENTRY_HEADER_SIZE 16
-
-// Journal entries
-// Journal entries are linked to each other by their crc32 value
-// The journal is almost a blockchain, because object versions constantly increase
-#define JE_MIN         0x01
-#define JE_START       0x01
-#define JE_SMALL_WRITE 0x02
-#define JE_BIG_WRITE   0x03
-#define JE_STABLE      0x04
-#define JE_DELETE      0x05
-#define JE_ROLLBACK    0x06
-#define JE_SMALL_WRITE_INSTANT 0x07
-#define JE_BIG_WRITE_INSTANT   0x08
-#define JE_MAX         0x08
-
-// crc32c comes first to ease calculation
-struct __attribute__((__packed__)) journal_entry_start
-{
-    uint32_t crc32;
-    uint16_t magic;
-    uint16_t type;
-    uint32_t size;
-    uint32_t reserved;
-    uint64_t journal_start;
-    uint64_t version;
-    uint32_t data_csum_type;
-    uint32_t csum_block_size;
-};
-#define JE_START_V0_SIZE 24
-#define JE_START_V1_SIZE 32
-#define JE_START_V2_SIZE 40
-
-struct __attribute__((__packed__)) journal_entry_small_write
-{
-    uint32_t crc32;
-    uint16_t magic;
-    uint16_t type;
-    uint32_t size;
-    uint32_t crc32_prev;
-    object_id oid;
-    uint64_t version;
-    uint32_t offset;
-    uint32_t len;
-    // small_write entries contain <len> bytes of data which is stored in next sectors
-    // data_offset is its offset within journal
-    uint64_t data_offset;
-    uint32_t crc32_data; // zero when data_csum_type != 0
-    // small_write and big_write entries are followed by the "external" bitmap
-    // its size is dynamic and included in journal entry's <size> field
-    uint8_t bitmap[];
-    // and then data checksums if data_csum_type != 0
-    // uint32_t data_crc32c[];
-};
-
-struct __attribute__((__packed__)) journal_entry_big_write
-{
-    uint32_t crc32;
-    uint16_t magic;
-    uint16_t type;
-    uint32_t size;
-    uint32_t crc32_prev;
-    object_id oid;
-    uint64_t version;
-    uint32_t offset;
-    uint32_t len;
-    uint64_t location;
-    // small_write and big_write entries are followed by the "external" bitmap
-    // its size is dynamic and included in journal entry's <size> field
-    uint8_t bitmap[];
-    // and then data checksums if data_csum_type != 0
-    // uint32_t data_crc32c[];
-};
-
-struct __attribute__((__packed__)) journal_entry_stable
-{
-    uint32_t crc32;
-    uint16_t magic;
-    uint16_t type;
-    uint32_t size;
-    uint32_t crc32_prev;
-    object_id oid;
-    uint64_t version;
-};
-
-struct __attribute__((__packed__)) journal_entry_rollback
-{
-    uint32_t crc32;
-    uint16_t magic;
-    uint16_t type;
-    uint32_t size;
-    uint32_t crc32_prev;
-    object_id oid;
-    uint64_t version;
-};
-
-struct __attribute__((__packed__)) journal_entry_del
-{
-    uint32_t crc32;
-    uint16_t magic;
-    uint16_t type;
-    uint32_t size;
-    uint32_t crc32_prev;
-    object_id oid;
-    uint64_t version;
-};
-
-struct __attribute__((__packed__)) journal_entry
-{
-    union
-    {
-        struct __attribute__((__packed__))
-        {
-            uint32_t crc32;
-            uint16_t magic;
-            uint16_t type;
-            uint32_t size;
-            uint32_t crc32_prev;
-        };
-        journal_entry_start start;
-        journal_entry_small_write small_write;
-        journal_entry_big_write big_write;
-        journal_entry_stable stable;
-        journal_entry_rollback rollback;
-        journal_entry_del del;
-    };
-};
-
-inline uint32_t je_crc32(journal_entry *je)
-{
-    // 0x48674bc7 = crc32(4 zero bytes)
-    return crc32c(0x48674bc7, ((uint8_t*)je)+4, je->size-4);
-}
+class blockstore_impl_t;
 
 struct journal_sector_info_t
 {

src/blockstore/blockstore_open.cpp

@@ -4,6 +4,11 @@
 #include <sys/file.h>
 #include "blockstore_impl.h"
 
+void blockstore_impl_t::parse_config(blockstore_config_t & config)
+{
+    return parse_config(config, false);
+}
+
 void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init)
 {
     // Online-configurable options:

src/blockstore/blockstore_read.cpp

@@ -3,6 +3,7 @@
 
 #include <limits.h>
 #include "blockstore_impl.h"
+#include "blockstore_internal.h"
 
 int blockstore_impl_t::fulfill_read_push(blockstore_op_t *op, void *buf, uint64_t offset, uint64_t len,
     uint32_t item_state, uint64_t item_version)
@@ -151,7 +152,7 @@ int blockstore_impl_t::fulfill_read(blockstore_op_t *read_op,
 uint8_t* blockstore_impl_t::get_clean_entry_bitmap(uint64_t block_loc, int offset)
 {
     uint8_t *clean_entry_bitmap;
-    uint64_t meta_loc = block_loc >> dsk.block_order;
+    uint64_t meta_loc = block_loc / dsk.data_block_size;
     if (inmemory_meta)
     {
         uint64_t sector = (meta_loc / (dsk.meta_block_size / dsk.clean_entry_size)) * dsk.meta_block_size;
@@ -691,8 +692,8 @@ bool blockstore_impl_t::fulfill_clean_read(blockstore_op_t *read_op, uint64_t &
 uint8_t* blockstore_impl_t::read_clean_meta_block(blockstore_op_t *op, uint64_t clean_loc, int rv_pos)
 {
     auto & rv = PRIV(op)->read_vec;
-    auto sector = ((clean_loc >> dsk.block_order) / (dsk.meta_block_size / dsk.clean_entry_size)) * dsk.meta_block_size;
-    auto pos = ((clean_loc >> dsk.block_order) % (dsk.meta_block_size / dsk.clean_entry_size)) * dsk.clean_entry_size;
+    auto sector = ((clean_loc / dsk.data_block_size) / (dsk.meta_block_size / dsk.clean_entry_size)) * dsk.meta_block_size;
+    auto pos = ((clean_loc / dsk.data_block_size) % (dsk.meta_block_size / dsk.clean_entry_size)) * dsk.clean_entry_size;
     uint8_t *buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size);
     rv.insert(rv.begin()+rv_pos, (copy_buffer_t){
         .copy_flags = COPY_BUF_META_BLOCK|COPY_BUF_CSUM_FILL,
@@ -806,7 +807,7 @@ bool blockstore_impl_t::verify_clean_padded_checksums(blockstore_op_t *op, uint6
     uint32_t offset = clean_loc % dsk.data_block_size;
     if (from_journal)
         return verify_padded_checksums(dyn_data, dyn_data + dsk.clean_entry_bitmap_size, offset, iov, n_iov, bad_block_cb);
-    clean_loc = (clean_loc >> dsk.block_order) << dsk.block_order;
+    clean_loc = (clean_loc / dsk.data_block_size) * dsk.data_block_size;
     if (!dyn_data)
     {
         assert(inmemory_meta);
@@ -866,7 +867,7 @@ void blockstore_impl_t::handle_read_event(ring_data_t *data, blockstore_op_t *op
                     {
                         // BIG_WRITE from journal or clean data
                         // Do not verify checksums if the data location is/was mutated by flushers
-                        auto & uo = used_clean_objects.at((rv[i].disk_offset >> dsk.block_order) << dsk.block_order);
+                        auto & uo = used_clean_objects.at((rv[i].disk_offset / dsk.data_block_size) * dsk.data_block_size);
                         if (!uo.was_changed)
                         {
                             verify_clean_padded_checksums(

src/blockstore/blockstore_rollback.cpp

@@ -2,6 +2,7 @@
 // License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
+#include "blockstore_internal.h"
 
 int blockstore_impl_t::dequeue_rollback(blockstore_op_t *op)
 {
@@ -210,10 +211,10 @@ void blockstore_impl_t::erase_dirty(blockstore_dirty_db_t::iterator dirty_start,
             dirty_it->second.location != UINT64_MAX)
         {
 #ifdef BLOCKSTORE_DEBUG
-            printf("Free block %ju from %jx:%jx v%ju\n", dirty_it->second.location >> dsk.block_order,
+            printf("Free block %ju from %jx:%jx v%ju\n", dirty_it->second.location / dsk.data_block_size,
                 dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version);
 #endif
-            data_alloc->set(dirty_it->second.location >> dsk.block_order, false);
+            data_alloc->set(dirty_it->second.location / dsk.data_block_size, false);
         }
         auto used = --journal.used_sectors.at(dirty_it->second.journal_sector);
 #ifdef BLOCKSTORE_DEBUG

src/blockstore/blockstore_stable.cpp

@@ -2,6 +2,7 @@
 // License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
+#include "blockstore_internal.h"
 
 // Stabilize small write:
 // 1) Copy data from the journal to the data device
@@ -228,17 +229,17 @@ int blockstore_impl_t::split_stab_op(blockstore_op_t *op, std::function<int(obj_
         // Split part of the request into a separate operation
         split_stab_op = new blockstore_op_t;
         split_stab_op->opcode = op->opcode;
-        split_stab_op->buf = bad_vers.items;
+        split_stab_op->buf = (uint8_t*)bad_vers.items;
         split_stab_op->len = bad_vers.size;
         init_op(split_stab_op);
         submit_queue.push_back(split_stab_op);
     }
     if (sync_op || split_stab_op || good_vers.items)
     {
-        void *orig_buf = op->buf;
+        uint8_t *orig_buf = op->buf;
         if (good_vers.items)
         {
-            op->buf = good_vers.items;
+            op->buf = (uint8_t*)good_vers.items;
             op->len = good_vers.size;
         }
         // Make a wrapped callback

src/blockstore/blockstore_sync.cpp

@@ -2,6 +2,7 @@
 // License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
+#include "blockstore_internal.h"
 
 #define SYNC_HAS_SMALL 1
 #define SYNC_HAS_BIG 2

src/blockstore/blockstore_write.cpp

@@ -2,6 +2,7 @@
 // License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
+#include "blockstore_internal.h"
 
 bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
 {
@@ -340,7 +341,7 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
         }
         BS_SUBMIT_GET_SQE(sqe, data);
         write_iodepth++;
-        dirty_it->second.location = loc << dsk.block_order;
+        dirty_it->second.location = loc * dsk.data_block_size;
         dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | BS_ST_SUBMITTED;
 #ifdef BLOCKSTORE_DEBUG
         printf(
@@ -366,7 +367,7 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
         data->iov.iov_len = op->len + stripe_offset + stripe_end; // to check it in the callback
         data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
         io_uring_prep_writev(
-            sqe, dsk.data_fd, PRIV(op)->iov_zerofill, vcnt, dsk.data_offset + (loc << dsk.block_order) + op->offset - stripe_offset
+            sqe, dsk.data_fd, PRIV(op)->iov_zerofill, vcnt, dsk.data_offset + (loc * dsk.data_block_size) + op->offset - stripe_offset
         );
         PRIV(op)->pending_ops = 1;
         if (!(dirty_it->second.state & BS_ST_INSTANT))

src/blockstore/fio_engine.cpp

@@ -26,17 +26,20 @@
 
 #include "blockstore.h"
 #include "epoll_manager.h"
+#include "malloc_or_die.h"
 #include "json11/json11.hpp"
 #include "fio_headers.h"
 
 struct bs_data
 {
-    blockstore_t *bs;
+    blockstore_i *bs;
     epoll_manager_t *epmgr;
     ring_loop_t *ringloop;
     /* The list of completed io_u structs. */
     std::vector<io_u*> completed;
     int op_n = 0, inflight = 0;
+    bool ec = false;
+    bool imm = true;
     bool last_sync = false;
     bool trace = false;
 };
@@ -45,6 +48,7 @@ struct bs_options
 {
     int __pad;
     char *json_config = NULL;
+    int ec = 0;
     int trace = 0;
 };
 
@@ -58,6 +62,16 @@ static struct fio_option options[] = {
         .category = FIO_OPT_C_ENGINE,
         .group  = FIO_OPT_G_FILENAME,
     },
+    {
+        .name   = "ec",
+        .lname  = "Use EC write method",
+        .type   = FIO_OPT_BOOL,
+        .off1   = offsetof(struct bs_options, ec),
+        .help   = "Use EC write method",
+        .def    = "0",
+        .category = FIO_OPT_C_ENGINE,
+        .group  = FIO_OPT_G_FILENAME,
+    },
     {
         .name   = "bs_trace",
         .lname  = "trace",
@@ -88,6 +102,7 @@ static int bs_setup(struct thread_data *td)
         return 1;
     }
     td->io_ops_data = bsd;
+    bsd->ec = o->ec;
 
     if (!td->files_index)
     {
@@ -147,7 +162,9 @@ static int bs_init(struct thread_data *td)
     }
     bsd->ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
     bsd->epmgr = new epoll_manager_t(bsd->ringloop);
-    bsd->bs = new blockstore_t(config, bsd->ringloop, bsd->epmgr->tfd);
+    bsd->bs = blockstore_i::create(config, bsd->ringloop, bsd->epmgr->tfd);
+    bsd->imm = config.find("immediate_commit") == config.end() ||
+        config["immediate_commit"] == "all";
     while (1)
     {
         bsd->ringloop->loop();
@@ -183,7 +200,7 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
     {
     case DDIR_READ:
         op->opcode = BS_OP_READ;
-        op->buf = io->xfer_buf;
+        op->buf = (uint8_t*)io->xfer_buf;
         op->oid = {
             .inode = 1,
             .stripe = io->offset / bsd->bs->get_block_size(),
@@ -203,8 +220,8 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
         };
         break;
     case DDIR_WRITE:
-        op->opcode = BS_OP_WRITE_STABLE;
-        op->buf = io->xfer_buf;
+        op->opcode = bsd->ec ? BS_OP_WRITE : BS_OP_WRITE_STABLE;
+        op->buf = (uint8_t*)io->xfer_buf;
         op->oid = {
             .inode = 1,
             .stripe = io->offset / bsd->bs->get_block_size(),
@@ -212,16 +229,57 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
         op->version = 0; // assign automatically
         op->offset = io->offset % bsd->bs->get_block_size();
         op->len = io->xfer_buflen;
-        op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
+        if (bsd->ec)
         {
-            io->error = op->retval < 0 ? -op->retval : 0;
-            bs_data *bsd = (bs_data*)io->engine_data;
-            bsd->inflight--;
-            bsd->completed.push_back(io);
-            if (bsd->trace)
-                printf("--- OP_WRITE %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
-            delete op;
-        };
+            op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
+            {
+                bs_data *bsd = (bs_data*)io->engine_data;
+                if (bsd->trace)
+                    printf("--- OP_WRITE %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
+                if (op->retval < 0)
+                {
+                    io->error = op->retval < 0 ? -op->retval : 0;
+                    bsd->inflight--;
+                    bsd->completed.push_back(io);
+                    delete op;
+                }
+                else
+                {
+                    auto stab_op = new blockstore_op_t;
+                    stab_op->opcode = BS_OP_STABLE;
+                    stab_op->buf = (uint8_t*)malloc_or_die(sizeof(obj_ver_id));
+                    obj_ver_id *ver = (obj_ver_id *)stab_op->buf;
+                    ver[0].oid = op->oid;
+                    ver[0].version = op->version;
+                    stab_op->len = 1;
+                    stab_op->callback = [io, n](blockstore_op_t *op)
+                    {
+                        bs_data *bsd = (bs_data*)io->engine_data;
+                        if (bsd->trace)
+                            printf("--- OP_STABLE %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
+                        io->error = op->retval < 0 ? -op->retval : 0;
+                        bsd->inflight--;
+                        bsd->completed.push_back(io);
+                        delete op;
+                    };
+                    bsd->bs->enqueue_op(stab_op);
+                    delete op;
+                }
+            };
+        }
+        else
+        {
+            op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
+            {
+                bs_data *bsd = (bs_data*)io->engine_data;
+                if (bsd->trace)
+                    printf("--- OP_WRITE_STABLE %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
+                io->error = op->retval < 0 ? -op->retval : 0;
+                bsd->inflight--;
+                bsd->completed.push_back(io);
+                delete op;
+            };
+        }
         bsd->last_sync = false;
         break;
     case DDIR_SYNC:

src/blockstore/ondisk_formats.h

@@ -0,0 +1,186 @@
+// Metadata on-disk structures
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 (see README.md for details)
+
+#pragma once
+
+#include "crc32c.h"
+
+#define JOURNAL_MAGIC 0x4A33
+#define JOURNAL_VERSION_V1 1
+#define JOURNAL_VERSION_V2 2
+#define JOURNAL_BUFFER_SIZE 4*1024*1024
+#define JOURNAL_ENTRY_HEADER_SIZE 16
+
+// Journal entries
+// Journal entries are linked to each other by their crc32 value
+// The journal is almost a blockchain, because object versions constantly increase
+#define JE_MIN         0x01
+#define JE_START       0x01
+#define JE_SMALL_WRITE 0x02
+#define JE_BIG_WRITE   0x03
+#define JE_STABLE      0x04
+#define JE_DELETE      0x05
+#define JE_ROLLBACK    0x06
+#define JE_SMALL_WRITE_INSTANT 0x07
+#define JE_BIG_WRITE_INSTANT   0x08
+#define JE_MAX         0x08
+
+// crc32c comes first to ease calculation
+struct __attribute__((__packed__)) journal_entry_start
+{
+    uint32_t crc32;
+    uint16_t magic;
+    uint16_t type;
+    uint32_t size;
+    uint32_t reserved;
+    uint64_t journal_start;
+    uint64_t version;
+    uint32_t data_csum_type;
+    uint32_t csum_block_size;
+};
+#define JE_START_V0_SIZE 24
+#define JE_START_V1_SIZE 32
+#define JE_START_V2_SIZE 40
+
+struct __attribute__((__packed__)) journal_entry_small_write
+{
+    uint32_t crc32;
+    uint16_t magic;
+    uint16_t type;
+    uint32_t size;
+    uint32_t crc32_prev;
+    object_id oid;
+    uint64_t version;
+    uint32_t offset;
+    uint32_t len;
+    // small_write entries contain <len> bytes of data which is stored in next sectors
+    // data_offset is its offset within journal
+    uint64_t data_offset;
+    uint32_t crc32_data; // zero when data_csum_type != 0
+    // small_write and big_write entries are followed by the "external" bitmap
+    // its size is dynamic and included in journal entry's <size> field
+    uint8_t bitmap[];
+    // and then data checksums if data_csum_type != 0
+    // uint32_t data_crc32c[];
+};
+
+struct __attribute__((__packed__)) journal_entry_big_write
+{
+    uint32_t crc32;
+    uint16_t magic;
+    uint16_t type;
+    uint32_t size;
+    uint32_t crc32_prev;
+    object_id oid;
+    uint64_t version;
+    uint32_t offset;
+    uint32_t len;
+    uint64_t location;
+    // small_write and big_write entries are followed by the "external" bitmap
+    // its size is dynamic and included in journal entry's <size> field
+    uint8_t bitmap[];
+    // and then data checksums if data_csum_type != 0
+    // uint32_t data_crc32c[];
+};
+
+struct __attribute__((__packed__)) journal_entry_stable
+{
+    uint32_t crc32;
+    uint16_t magic;
+    uint16_t type;
+    uint32_t size;
+    uint32_t crc32_prev;
+    object_id oid;
+    uint64_t version;
+};
+
+struct __attribute__((__packed__)) journal_entry_rollback
+{
+    uint32_t crc32;
+    uint16_t magic;
+    uint16_t type;
+    uint32_t size;
+    uint32_t crc32_prev;
+    object_id oid;
+    uint64_t version;
+};
+
+struct __attribute__((__packed__)) journal_entry_del
+{
+    uint32_t crc32;
+    uint16_t magic;
+    uint16_t type;
+    uint32_t size;
+    uint32_t crc32_prev;
+    object_id oid;
+    uint64_t version;
+};
+
+struct __attribute__((__packed__)) journal_entry
+{
+    union
+    {
+        struct __attribute__((__packed__))
+        {
+            uint32_t crc32;
+            uint16_t magic;
+            uint16_t type;
+            uint32_t size;
+            uint32_t crc32_prev;
+        };
+        journal_entry_start start;
+        journal_entry_small_write small_write;
+        journal_entry_big_write big_write;
+        journal_entry_stable stable;
+        journal_entry_rollback rollback;
+        journal_entry_del del;
+    };
+};
+
+inline uint32_t je_crc32(journal_entry *je)
+{
+    // 0x48674bc7 = crc32(4 zero bytes)
+    return crc32c(0x48674bc7, ((uint8_t*)je)+4, je->size-4);
+}
+
+// "VITAstor"
+#define BLOCKSTORE_META_MAGIC_V1 0x726F747341544956l
+#define BLOCKSTORE_META_FORMAT_V1 1
+#define BLOCKSTORE_META_FORMAT_V2 2
+
+// metadata header (superblock)
+struct __attribute__((__packed__)) blockstore_meta_header_v1_t
+{
+    uint64_t zero;
+    uint64_t magic;
+    uint64_t version;
+    uint32_t meta_block_size;
+    uint32_t data_block_size;
+    uint32_t bitmap_granularity;
+};
+
+struct __attribute__((__packed__)) blockstore_meta_header_v2_t
+{
+    uint64_t zero;
+    uint64_t magic;
+    uint64_t version;
+    uint32_t meta_block_size;
+    uint32_t data_block_size;
+    uint32_t bitmap_granularity;
+    uint32_t data_csum_type;
+    uint32_t csum_block_size;
+    uint32_t header_csum;
+};
+
+// 32 bytes = 24 bytes + block bitmap (4 bytes by default) + external attributes (also bitmap, 4 bytes by default)
+// per "clean" entry on disk with fixed metadata tables
+struct __attribute__((__packed__)) clean_disk_entry
+{
+    object_id oid;
+    uint64_t version;
+    uint8_t bitmap[];
+    // Two more fields come after bitmap in metadata version 2:
+    // uint32_t data_csum[];
+    // uint32_t entry_csum;
+};

src/client/cluster_client.cpp

@@ -765,8 +765,13 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
     {
         return;
     }
-    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && enable_writeback && !(op->flags & OP_FLUSH_BUFFER) &&
-        !op->version /* no CAS writeback */)
+    // CAS writes are simplified: they're not cached, not resliced, not retried, and not part of the regular write queue at all
+    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && op->version)
+    {
+        execute_cas(op);
+        return;
+    }
+    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && enable_writeback && !(op->flags & OP_FLUSH_BUFFER))
     {
         if (wb->writebacks_active >= client_max_writeback_iodepth)
         {
@@ -788,7 +793,7 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
     }
     if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && !(op->flags & OP_IMMEDIATE_COMMIT))
     {
-        if (!(op->flags & OP_FLUSH_BUFFER) && !op->version /* no CAS write-repeat */)
+        if (!(op->flags & OP_FLUSH_BUFFER))
         {
             uint64_t flush_id = ++wb->last_flush_id;
             wb->copy_write(op, CACHE_REPEATING, flush_id);
@@ -847,6 +852,72 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
     }
 }
 
+void cluster_client_t::execute_cas(cluster_op_t *op)
+{
+    slice_rw(op);
+    op->needs_reslice = false;
+    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && op->version && op->parts.size() > 1)
+    {
+        // Atomic writes to multiple stripes are unsupported
+        op->retval = -EINVAL;
+        auto cb = std::move(op->callback);
+        cb(op);
+        return;
+    }
+    int res = try_send(op, 0, [this, op](osd_op_t *part)
+    {
+        int expected = part->req.hdr.opcode == OSD_OP_DELETE ? 0 : part->req.rw.len;
+        op->retval = part->reply.hdr.retval;
+        op->retval = op->retval == expected ? 0 : (op->retval >= 0 ? -EIO : op->retval);
+        op->retval = op->retval == -EPIPE ? -EINTR : op->retval;
+        auto peer_it = msgr.osd_peer_fds.find(op->parts[0].osd_num);
+        if (op->retval != 0 || (op->flags & OP_IMMEDIATE_COMMIT))
+        {
+            auto cb = std::move(op->callback);
+            cb(op);
+        }
+        else if (peer_it == msgr.osd_peer_fds.end())
+        {
+            // Care must be taken to make sure that the client doesn't reconnect to the OSD
+            // before executing the previously completed operation callback (!)
+            op->retval = -EINTR;
+            auto cb = std::move(op->callback);
+            cb(op);
+        }
+        else
+        {
+            // CAS writes have a built-in sync
+            auto peer_fd = peer_it->second;
+            *part = (osd_op_t){
+                .op_type = OSD_OP_OUT,
+                .peer_fd = peer_fd,
+                .req = {
+                    .hdr = {
+                        .magic = SECONDARY_OSD_OP_MAGIC,
+                        .opcode = OSD_OP_SYNC,
+                    },
+                },
+                .callback = [this, op](osd_op_t *part)
+                {
+                    op->retval = part->reply.hdr.retval;
+                    op->retval = op->retval == -EPIPE ? -EINTR : op->retval;
+                    auto cb = std::move(op->callback);
+                    cb(op);
+                },
+            };
+            msgr.outbox_push(part);
+        }
+    });
+    if (res == TRY_SEND_CONNECTING || res == TRY_SEND_OFFLINE)
+    {
+        // In theory, CAS writes could wait for the PG to come up, but it's easier to just fail it
+        op->retval = -EINTR;
+        auto cb = std::move(op->callback);
+        cb(op);
+        return;
+    }
+}
+
 bool cluster_client_t::check_rw(cluster_op_t *op)
 {
     if (op->opcode == OSD_OP_SYNC)
@@ -956,13 +1027,6 @@ resume_0:
     // Slice the operation into parts
     slice_rw(op);
     op->needs_reslice = false;
-    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && op->version && op->parts.size() > 1)
-    {
-        // Atomic writes to multiple stripes are unsupported
-        op->retval = -EINVAL;
-        erase_op(op);
-        return 1;
-    }
 resume_1:
     // Send unsent parts, if they're not subject to change
     op->state = 2;
@@ -1307,7 +1371,7 @@ bool cluster_client_t::affects_osd(uint64_t inode, uint64_t offset, uint64_t len
     return false;
 }
 
-int cluster_client_t::try_send(cluster_op_t *op, int i)
+int cluster_client_t::try_send(cluster_op_t *op, int i, std::function<void(osd_op_t *op_part)> cb)
 {
     if (!msgr_initialized)
     {
@@ -1317,7 +1381,8 @@ int cluster_client_t::try_send(cluster_op_t *op, int i)
     auto & pool_cfg = st_cli.pool_config.at(INODE_POOL(op->cur_inode));
     auto pg_it = pool_cfg.pg_config.find(part->pg_num);
     if (pg_it != pool_cfg.pg_config.end() &&
-        !pg_it->second.pause && pg_it->second.cur_primary)
+        !pg_it->second.pause && pg_it->second.cur_primary &&
+        (pg_it->second.cur_state & PG_ACTIVE))
     {
         osd_num_t primary_osd = pg_it->second.cur_primary;
         if (pool_cfg.local_reads != POOL_LOCAL_READ_PRIMARY &&
@@ -1366,7 +1431,7 @@ int cluster_client_t::try_send(cluster_op_t *op, int i)
                     ? (uint8_t*)op->part_bitmaps + pg_bitmap_size*i : NULL),
                 .bitmap_len = (unsigned)(op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP
                     ? pg_bitmap_size : 0),
-                .callback = [this, part](osd_op_t *op_part)
+                .callback = cb ? cb : [this, part](osd_op_t *op_part)
                 {
                     handle_op_part(part);
                 },

src/client/cluster_client.h

@@ -127,7 +127,7 @@ public:
     ring_consumer_t consumer;
     std::vector<std::function<void(void)>> on_ready_hooks;
     int list_retry_timeout_id = -1;
-    timespec list_retry_time;
+    timespec list_retry_time = {};
     std::vector<inode_list_t*> lists;
     std::multimap<osd_num_t, osd_op_t*> raw_ops;
     int continuing_ops = 0;
@@ -175,12 +175,13 @@ protected:
     void on_change_node_placement_hook();
 
     void execute_internal(cluster_op_t *op);
+    void execute_cas(cluster_op_t *op);
     void unshift_op(cluster_op_t *op);
     int continue_rw(cluster_op_t *op);
     bool check_rw(cluster_op_t *op);
     void slice_rw(cluster_op_t *op);
     void reset_retry_timer(int new_duration);
-    int try_send(cluster_op_t *op, int i);
+    int try_send(cluster_op_t *op, int i, std::function<void(osd_op_t *op_part)> cb = nullptr);
     int continue_sync(cluster_op_t *op);
     void send_sync(cluster_op_t *op, cluster_op_part_t *part);
     void handle_op_part(cluster_op_part_t *part);

src/client/cluster_client_list.cpp

@@ -31,7 +31,7 @@ struct inode_list_pg_t
     osd_num_t cur_primary = 0;
     int state = 0;
     int inflight_ops = 0;
-    timespec wait_until;
+    timespec wait_until = {};
     std::vector<inode_list_osd_t> list_osds;
 
     bool has_unstable = false;
@@ -177,6 +177,9 @@ void cluster_client_t::retry_start_pg_listing(inode_list_pg_t *pg)
         if (tv.tv_sec < pg->wait_until.tv_sec ||
             tv.tv_sec == pg->wait_until.tv_sec && tv.tv_nsec < pg->wait_until.tv_nsec)
         {
+            // Ensure that the retry timer is set
+            set_list_retry_timeout(pg->wait_until.tv_sec*1000 - tv.tv_sec*1000 +
+                (pg->wait_until.tv_nsec - tv.tv_nsec)/1000000, pg->wait_until);
             return;
         }
     }

src/client/etcd_state_client.cpp

@@ -1167,7 +1167,7 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
             if (!cur_primary || !value["state"].is_array() || !state ||
                 (state & PG_OFFLINE) && state != PG_OFFLINE ||
                 (state & PG_PEERING) && state != PG_PEERING ||
-                (state & PG_INCOMPLETE) && state != PG_INCOMPLETE)
+                (state & PG_INCOMPLETE) && state != PG_INCOMPLETE && state != (PG_INCOMPLETE|PG_HAS_INVALID))
             {
                 fprintf(stderr, "Unexpected pool %u PG %u state in etcd: primary=%ju, state=%s\n", pool_id, pg_num, cur_primary, value["state"].dump().c_str());
                 return;

src/client/messenger.cpp

@@ -276,6 +276,7 @@ osd_messenger_t::~osd_messenger_t()
 #ifdef WITH_RDMA
     for (auto rdma_context: rdma_contexts)
     {
+        tfd->set_fd_handler(rdma_context->channel->fd, false, NULL);
         delete rdma_context;
     }
     rdma_contexts.clear();

src/client/msgr_receive.cpp

@@ -109,6 +109,7 @@ bool osd_messenger_t::handle_read(int result, osd_client_t *cl)
             if (!handle_read_buffer(cl, cl->in_buf, result))
             {
                 clear_immediate_ops(peer_fd);
+                handle_immediate_ops();
                 return false;
             }
         }
@@ -122,6 +123,7 @@ bool osd_messenger_t::handle_read(int result, osd_client_t *cl)
                 if (!handle_finished_read(cl))
                 {
                     clear_immediate_ops(peer_fd);
+                    handle_immediate_ops();
                     return false;
                 }
             }
@@ -140,7 +142,7 @@ void osd_messenger_t::clear_immediate_ops(int peer_fd)
     size_t i = 0, j = 0;
     while (i < set_immediate_ops.size())
     {
-        if (set_immediate_ops[i]->peer_fd == peer_fd)
+        if (set_immediate_ops[i]->peer_fd == peer_fd && set_immediate_ops[i]->op_type == OSD_OP_IN)
         {
             delete set_immediate_ops[i];
         }

src/client/msgr_send.cpp

@@ -323,8 +323,12 @@ void osd_messenger_t::handle_send(int result, bool prev, bool more, osd_client_t
         }
         if (more)
         {
-            auto expected = cl->send_list.size() < IOV_MAX ? cl->send_list.size() : IOV_MAX;
-            assert(done == expected);
+            int expected = cl->send_list.size() < IOV_MAX ? cl->send_list.size() : IOV_MAX;
+            if (done != expected)
+            {
+                fprintf(stderr, "BUG (maybe kernel): Expected to send %d iovecs with MSG_WAITALL but sent %d\n", expected, done);
+                exit(1);
+            }
             cl->zc_free_list.push_back(NULL); // end marker
         }
         if (done > 0)

src/client/msgr_stop.cpp

@@ -52,6 +52,7 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
         return;
     }
     osd_client_t *cl = it->second;
+    // FIXME: This 'force' flag is probably an ugly reenterability hack - check its logic and maybe remove it
     if (cl->peer_state == PEER_CONNECTING && !force || cl->peer_state == PEER_STOPPED)
     {
         return;
@@ -73,6 +74,7 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
     }
     // First set state to STOPPED so another stop_client() call doesn't try to free it again
     cl->refs++;
+    int prev_state = cl->peer_state;
     cl->peer_state = PEER_STOPPED;
     if (cl->osd_num)
     {
@@ -113,10 +115,12 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
         // Break PG locks
         break_pg_locks(cl->in_osd_num);
     }
-    if (cl->osd_num)
+    if (cl->osd_num && prev_state != PEER_CONNECTING)
     {
         // Then repeer PGs because cancel_op() callbacks can try to perform
         // some actions and we need correct PG states to not do something silly
+        // PEER_CONNECTING has neither 'just dropped the connection' nor 'just connected'
+        // so do not repeer on it.
         repeer_pgs(cl->osd_num);
     }
     // Find the item again because it can be invalidated at this point

src/client/nbd_proxy.cpp

@@ -272,6 +272,8 @@ const char *help_text =
     "  --dev_num N\n"
     "    Use the specified device /dev/nbdN instead of automatic selection (alternative syntax\n"
     "    to /dev/nbdN positional parameter).\n"
+    "  --readonly\n"
+    "    Make the device read-only.\n"
     "  --foreground 1\n"
     "    Stay in foreground, do not daemonize.\n"
     "\n"
@@ -372,7 +374,7 @@ public:
             else if (args[i][0] == '-' && args[i][1] == '-')
             {
                 const char *opt = args[i]+2;
-                cfg[opt] = !strcmp(opt, "json") || !strcmp(opt, "all") ||
+                cfg[opt] = !strcmp(opt, "json") || !strcmp(opt, "all") || !strcmp(opt, "readonly") ||
                     !strcmp(opt, "force") || i == narg-1 ? "1" : args[++i];
             }
             else if (pos == 0)
@@ -695,20 +697,6 @@ help:
             ringloop->loop();
             ringloop->wait();
         }
-        stop = false;
-        cluster_op_t *close_sync = new cluster_op_t;
-        close_sync->opcode = OSD_OP_SYNC;
-        close_sync->callback = [&stop](cluster_op_t *op)
-        {
-            stop = true;
-            delete op;
-        };
-        cli->execute(close_sync);
-        while (!stop)
-        {
-            ringloop->loop();
-            ringloop->wait();
-        }
         cli->flush();
         delete cli;
         delete epmgr;
@@ -912,7 +900,7 @@ protected:
             int r, nbd = open(path, O_RDWR), qd_fd;
             if (nbd < 0)
             {
-                write(notifyfd[1], &errno, sizeof(errno));
+                (void)write(notifyfd[1], &errno, sizeof(errno));
                 exit(1);
             }
             r = ioctl(nbd, NBD_SET_SOCK, sockfd[1]);
@@ -954,7 +942,7 @@ protected:
             close(qd_fd);
             // Notify parent
             errno = 0;
-            write(notifyfd[1], &errno, sizeof(errno));
+            (void)write(notifyfd[1], &errno, sizeof(errno));
             close(notifyfd[1]);
             close(sockfd[0]);
             if (bg)
@@ -971,11 +959,11 @@ protected:
             ioctl(nbd, NBD_CLEAR_SOCK);
             exit(0);
     end_close:
-            write(notifyfd[1], &errno, sizeof(errno));
+            (void)write(notifyfd[1], &errno, sizeof(errno));
             close(nbd);
             exit(2);
     end_unmap:
-            write(notifyfd[1], &errno, sizeof(errno));
+            (void)write(notifyfd[1], &errno, sizeof(errno));
             ioctl(nbd, NBD_CLEAR_SOCK);
             close(nbd);
             exit(3);

src/client/object_id.h

@@ -6,8 +6,20 @@
 #include <stdint.h>
 #include <functional>
 
+#define POOL_SCHEME_REPLICATED 1
+#define POOL_SCHEME_XOR 2
+#define POOL_SCHEME_EC 3
+#define POOL_ID_MAX 0x10000
+#define POOL_ID_BITS 16
+#define INODE_POOL(inode) (pool_id_t)((inode) >> (64 - POOL_ID_BITS))
+#define INODE_NO_POOL(inode) (inode_t)((inode) & (((uint64_t)1 << (64-POOL_ID_BITS)) - 1))
+#define INODE_WITH_POOL(pool_id, inode) (((inode_t)(pool_id) << (64-POOL_ID_BITS)) | INODE_NO_POOL(inode))
+
 typedef uint64_t inode_t;
 
+// Pool ID is 16 bits long
+typedef uint32_t pool_id_t;
+
 // 16 bytes per object/stripe id
 // stripe = (start of the parity stripe + peer role)
 // i.e. for example (256KB + one of 0,1,2)

src/client/osd_ops.h

@@ -44,11 +44,6 @@
 #define DIRECT_IO_ALIGNMENT 512
 #endif
 
-// Memory allocation alignment (page size is usually optimal)
-#ifndef MEM_ALIGNMENT
-#define MEM_ALIGNMENT 4096
-#endif
-
 // Constants for osd_reply_describe_item_t.loc_bad
 #define LOC_OUTDATED 1
 #define LOC_CORRUPTED 2
@@ -267,7 +262,10 @@ struct __attribute__((__packed__)) osd_reply_del_t
     uint32_t left_on_dead_count;
 };
 
-// sync to the primary OSD
+// sync to the primary OSD. semantics:
+// 1) any non-synced write may disappear on OSD restart. even if it's a CAS write.
+// 2) sync only guarantees to commit completed writes, not in-progress ones.
+// 3) sync is a no-op when immediate_commit is active.
 struct __attribute__((__packed__)) osd_op_sync_t
 {
     osd_op_header_t header;

src/client/qemu_driver.c

@@ -974,14 +974,21 @@ static void vitastor_co_read_bitmap_cb(void *opaque, long retval, uint8_t *bitma
 #endif
 }
 
-static int coroutine_fn vitastor_co_block_status(
-    BlockDriverState *bs, bool want_zero, int64_t offset, int64_t bytes,
-    int64_t *pnum, int64_t *map, BlockDriverState **file)
+static int coroutine_fn vitastor_co_block_status(BlockDriverState *bs,
+#if QEMU_VERSION_MAJOR > 10 || QEMU_VERSION_MAJOR == 10 && QEMU_VERSION_MINOR >= 1
+    unsigned int mode,
+#else
+    bool want_zero,
+#endif
+    int64_t offset, int64_t bytes, int64_t *pnum, int64_t *map, BlockDriverState **file)
 {
     // Allocated => return BDRV_BLOCK_DATA|BDRV_BLOCK_OFFSET_VALID
     // Not allocated => return 0
     // Error => return -errno
     // Set pnum to length of the extent, `*map` = `offset`, `*file` = `bs`
+#if QEMU_VERSION_MAJOR > 10 || QEMU_VERSION_MAJOR == 10 && QEMU_VERSION_MINOR >= 1
+    int want_zero = (mode == BDRV_WANT_PRECISE);
+#endif
     VitastorRPC task;
     VitastorClient *client = bs->opaque;
     uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode;

src/client/ublk_server.cpp

@@ -40,12 +40,14 @@ const char *help_text =
     "    Make the device read-only.\n"
     "  --hdd\n"
     "    Mark the device as rotational.\n"
-    "  --logfile /path/to/log/file.txt\n"
-    "    Write log messages to the specified file instead of dropping them (in background mode)\n"
-    "    or printing them to the standard output (in foreground mode).\n"
     "  --dev_num N\n"
     "    Use the specified device /dev/ublkbN instead of automatic selection (alternative syntax\n"
     "    to /dev/ublkbN positional parameter).\n"
+    "  --pidfile /run/ublk_pid_file.pid\n"
+    "    Write process ID to the specified file.\n"
+    "  --logfile /path/to/log/file.txt\n"
+    "    Write log messages to the specified file instead of dropping them (in background mode)\n"
+    "    or printing them to the standard output (in foreground mode).\n"
     "  --foreground 1\n"
     "    Stay in foreground, do not daemonize.\n"
     "\n"
@@ -79,6 +81,7 @@ protected:
     inode_watch_t *watch = NULL;
 
     std::string logfile = "/dev/null";
+    std::string pidfile;
 
 public:
     ublk_server()
@@ -300,10 +303,8 @@ help:
         load_module();
 
         bool bg = cfg["foreground"].is_null();
-        if (cfg["logfile"].string_value() != "")
-        {
-            logfile = cfg["logfile"].string_value();
-        }
+        logfile = cfg["logfile"].string_value();
+        pidfile = cfg["pidfile"].string_value();
 
         open_control();
         if (recover)
@@ -331,11 +332,13 @@ help:
             close(notifyfd[0]);
         }
         start_device(recover);
+        if (pidfile != "")
+            write_pid();
         if (bg)
         {
             daemonize_reopen_stdio();
             int ok = 0;
-            write(notifyfd[1], &ok, sizeof(ok));
+            (void)write(notifyfd[1], &ok, sizeof(ok));
             close(notifyfd[1]);
         }
         else
@@ -346,19 +349,6 @@ help:
             ringloop->loop();
             ringloop->wait();
         }
-        cluster_op_t *close_sync = new cluster_op_t;
-        close_sync->opcode = OSD_OP_SYNC;
-        close_sync->callback = [this](cluster_op_t *op)
-        {
-            stop = true;
-            delete op;
-        };
-        cli->execute(close_sync);
-        while (!stop)
-        {
-            ringloop->loop();
-            ringloop->wait();
-        }
         cli->flush();
         delete cli;
         delete epmgr;
@@ -390,7 +380,7 @@ help:
             // Parent - check status
             close(notifyfd[1]);
             int child_errno = 1;
-            read(notifyfd[0], &child_errno, sizeof(child_errno));
+            (void)read(notifyfd[0], &child_errno, sizeof(child_errno));
             if (!child_errno)
                 printf("/dev/ublkb%d\n", ublk_dev.dev_id);
             exit(child_errno);
@@ -412,6 +402,22 @@ help:
             fprintf(stderr, "Warning: Failed to chdir into /\n");
     }
 
+    void write_pid()
+    {
+        int fd = open(pidfile.c_str(), O_WRONLY|O_CREAT|O_TRUNC, 0666);
+        if (fd < 0)
+        {
+            fprintf(stderr, "Failed to create pid file %s: %s (code %d)\n", pidfile.c_str(), strerror(errno), errno);
+            return;
+        }
+        auto pid = std::to_string(getpid());
+        if (write(fd, pid.c_str(), pid.size()) < 0)
+        {
+            fprintf(stderr, "Failed to write pid to %s: %s (code %d)\n", pidfile.c_str(), strerror(errno), errno);
+        }
+        close(fd);
+    }
+
     json11::Json::object list_mapped()
     {
         int n_in_dev = 0;
@@ -618,13 +624,13 @@ protected:
             .types = UBLK_PARAM_TYPE_BASIC,
             .basic = {
                 .attrs = attrs, // UBLK_ATTR_READ_ONLY | UBLK_ATTR_ROTATIONAL | UBLK_ATTR_VOLATILE_CACHE | UBLK_ATTR_FUA
-                .logical_bs_shift = 9,
+                .logical_bs_shift = phys_shift,
                 .physical_bs_shift = phys_shift,
                 .io_opt_shift = io_opt_shift,
                 .io_min_shift = phys_shift,
-                .max_sectors = max_io_buf_bytes / phys_block_size,
+                .max_sectors = max_io_buf_bytes / 512,
                 .chunk_sectors = 0,
-                .dev_sectors = device_size / phys_block_size,
+                .dev_sectors = device_size / 512,
                 .virt_boundary_mask = 0,
             },
             .discard = {

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 2.3.0
+Version: 2.4.4
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/cmd/cli.cpp

@@ -37,6 +37,7 @@ static const char* help_text =
     "  --sort FIELD    Sort by specified field (name, size, used_size, <read|write|delete>_<iops|bps|lat|queue>)\n"
     "  -r|--reverse    Sort in descending order\n"
     "  -n|--count N    Only list first N items\n"
+    "  --tree          Show image snapshot/clone tree\n"
     "\n"
     "vitastor-cli create -s|--size <size> [-p|--pool <id|name>] [--parent <parent_name>[@<snapshot>]] <name>\n"
     "  Create an image. You may use K/M/G/T suffixes for <size>. If --parent is specified,\n"
@@ -287,7 +288,7 @@ static json11::Json::object parse_args(int narg, const char *args[])
                 !strcmp(opt, "down-ok") || !strcmp(opt, "down_ok") ||
                 !strcmp(opt, "dry-run") || !strcmp(opt, "dry_run") ||
                 !strcmp(opt, "help") || !strcmp(opt, "all") ||
-                !strcmp(opt, "exact") || !strcmp(opt, "matching") ||
+                !strcmp(opt, "exact") || !strcmp(opt, "matching") || !strcmp(opt, "tree") ||
                 !strcmp(opt, "writers-stopped") || !strcmp(opt, "writers_stopped"))
             {
                 cfg[opt] = "1";

src/cmd/cli_ls.cpp

@@ -19,6 +19,7 @@ struct image_lister_t
     std::set<std::string> only_names;
     bool reverse = false;
     bool exact = false;
+    bool tree = false;
     int max_count = 0;
     bool show_stats = false, show_delete = false;
 
@@ -77,6 +78,7 @@ struct image_lister_t
                     ? p_it->second.name : "";
                 item["parent_pool_id"] = (uint64_t)INODE_POOL(ic.second.parent_id);
                 item["parent_inode_num"] = INODE_NO_POOL(ic.second.parent_id);
+                item["parent_inode_id"] = ic.second.parent_id;
             }
             stats[ic.second.num] = item;
         }
@@ -92,16 +94,14 @@ struct image_lister_t
         parent->etcd_txn(json11::Json::object {
             { "success", json11::Json::array {
                 json11::Json::object {
-                    { "request_range", json11::Json::object {
-                        { "key", base64_encode(
-                            parent->cli->st_cli.etcd_prefix+"/pool/stats"+
-                            (list_pool_id ? "/"+std::to_string(list_pool_id) : "")+"/"
-                        ) },
-                        { "range_end", base64_encode(
-                            parent->cli->st_cli.etcd_prefix+"/pool/stats"+
-                            (list_pool_id ? "/"+std::to_string(list_pool_id) : "")+"0"
-                        ) },
-                    } },
+                    { "request_range", (list_pool_id
+                        ? json11::Json::object {
+                            { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/pool/stats/"+std::to_string(list_pool_id)) },
+                        }
+                        : json11::Json::object {
+                            { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/pool/stats/") },
+                            { "range_end", base64_encode(parent->cli->st_cli.etcd_prefix+"/pool/stats0") },
+                        }) },
                 },
                 json11::Json::object {
                     { "request_range", json11::Json::object {
@@ -245,6 +245,33 @@ resume_1:
         return list;
     }
 
+    json11::Json::array to_tree(const json11::Json::array & array)
+    {
+        std::map<uint64_t, json11::Json::array> children;
+        for (const auto & item: array)
+        {
+            uint64_t parent_id = item["parent_inode_id"].uint64_value();
+            children[parent_id].push_back(item);
+        }
+        json11::Json::array tree;
+        std::function<void(uint64_t, const std::string &)> add_children = [&](uint64_t parent_id, const std::string & prefix)
+        {
+            if (children.find(parent_id) != children.end())
+            {
+                for (size_t i = 0; i < children[parent_id].size(); i++)
+                {
+                    bool is_last = (i == children[parent_id].size()-1);
+                    json11::Json::object new_item = children[parent_id][i].object_items();
+                    new_item["name"] = prefix + (parent_id == 0 ? "" : (is_last ? "└─ " : "├─ ")) + new_item["name"].string_value();
+                    tree.push_back(new_item);
+                    add_children(new_item["inode_id"].uint64_value(), prefix + (parent_id == 0 ? "" : (is_last ? "   " : "│  ")));
+                }
+            }
+        };
+        add_children(0, "");
+        return tree;
+    }
+
     void loop()
     {
         if (state == 1)
@@ -375,7 +402,7 @@ resume_1:
             kv.second["ro"] = kv.second["deleted"].bool_value() ? "DEL" :
                 (kv.second["readonly"].bool_value() ? "RO" : "-");
         }
-        result.text = print_table(to_list(), cols, parent->color);
+        result.text = print_table(tree ? to_tree(to_list()) : to_list(), cols, parent->color);
         state = 100;
     }
 };
@@ -542,6 +569,7 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_ls(json11::Json cfg)
     auto lister = new image_lister_t();
     lister->parent = this;
     lister->exact = cfg["exact"].bool_value();
+    lister->tree = cfg["tree"].bool_value();
     lister->list_pool_id = cfg["pool"].uint64_value();
     lister->list_pool_name = lister->list_pool_id ? "" : cfg["pool"].as_string();
     lister->show_stats = cfg["long"].bool_value();

src/cmd/cli_pool_cfg.cpp

@@ -180,7 +180,8 @@ std::string validate_pool_config(json11::Json::object & new_cfg, json11::Json ol
         {
             return "Changing scheme for an existing pool will lead to data loss. Use --force to proceed";
         }
-        if (etcd_state_client_t::parse_scheme(old_cfg["scheme"].string_value()) == POOL_SCHEME_EC)
+        auto old_scheme = etcd_state_client_t::parse_scheme(old_cfg["scheme"].string_value());
+        if (old_scheme == POOL_SCHEME_EC)
         {
             uint64_t old_data_chunks = old_cfg["pg_size"].uint64_value() - old_cfg["parity_chunks"].uint64_value();
             uint64_t new_data_chunks = cfg["pg_size"].uint64_value() - cfg["parity_chunks"].uint64_value();
@@ -189,6 +190,13 @@ std::string validate_pool_config(json11::Json::object & new_cfg, json11::Json ol
                 return "Changing EC data chunk count for an existing pool will lead to data loss. Use --force to proceed";
             }
         }
+        else if (old_scheme != POOL_SCHEME_REPLICATED)
+        {
+            if (old_cfg["pg_size"].uint64_value() != cfg["pg_size"].uint64_value())
+            {
+                return "Changing XOR data chunk count for an existing pool will lead to data loss. Use --force to proceed";
+            }
+        }
         if (old_cfg["block_size"] != cfg["block_size"] ||
             old_cfg["bitmap_granularity"] != cfg["bitmap_granularity"] ||
             old_cfg["immediate_commit"] != cfg["immediate_commit"])

src/cmd/cli_pool_ls.cpp

@@ -87,6 +87,16 @@ struct pool_lister_t
                         ) },
                     } },
                 },
+                json11::Json::object {
+                    { "request_range", json11::Json::object {
+                        { "key", base64_encode(
+                            parent->cli->st_cli.etcd_prefix+"/config/osd/"
+                        ) },
+                        { "range_end", base64_encode(
+                            parent->cli->st_cli.etcd_prefix+"/config/osd0"
+                        ) },
+                    } },
+                },
             } },
         });
         state = base_state+1;
@@ -115,6 +125,14 @@ resume_1:
             // osd/stats/<N>::free
             osd_free[osd_num] = value["free"].uint64_value();
         });
+        std::map<uint64_t, double> osd_reweight;
+        parent->iterate_kvs_1(space_info["responses"][3]["response_range"]["kvs"], "/config/osd/", [&](uint64_t osd_num, json11::Json value)
+        {
+            if (value.object_items().find("reweight") != value.object_items().end())
+            {
+                osd_reweight[osd_num] = value["reweight"].number_value();
+            }
+        });
         // Calculate max_avail for each pool
         for (auto & pp: parent->cli->st_cli.pool_config)
         {
@@ -140,7 +158,9 @@ resume_1:
             }
             for (auto pg_per_pair: pg_per_osd)
             {
-                uint64_t pg_free = osd_free[pg_per_pair.first] * pool_cfg.real_pg_count / pg_per_pair.second;
+                uint64_t pg_free = osd_free[pg_per_pair.first] *
+                    (osd_reweight.find(pg_per_pair.first) != osd_reweight.end() ? osd_reweight[pg_per_pair.first] : 1.0) *
+                    pool_cfg.real_pg_count / pg_per_pair.second;
                 if (pool_avail > pg_free)
                 {
                     pool_avail = pg_free;

src/cmd/cli_rm.cpp

@@ -97,8 +97,6 @@ struct snap_remover_t
             goto resume_8;
         else if (state == 9)
             goto resume_9;
-        else if (state == 10)
-            goto resume_10;
         else if (state == 100)
             goto resume_100;
         assert(!state);
@@ -157,11 +155,6 @@ resume_3:
 resume_4:
             while (!wait_result(4))
                 return;
-            // Mark child as deleted
-            start_mark_deleted(inverse_child);
-resume_9:
-            while (!wait_result(9))
-                return;
             // Delete "inverse" child data
             start_delete_source(inverse_child);
 resume_5:
@@ -190,8 +183,8 @@ resume_6:
             }
             // Mark child as deleted
             start_mark_deleted(chain_list[current_child]);
-resume_10:
-            while (!wait_result(10))
+resume_9:
+            while (!wait_result(9))
                 return;
             start_delete_source(chain_list[current_child]);
 resume_7:
@@ -456,6 +449,7 @@ resume_100:
         );
         // Fill new configuration
         inode_config_t new_cfg = *child_cfg;
+        new_cfg.deleted = false;
         new_cfg.num = target_cfg->num;
         new_cfg.parent_id = new_parent;
         json11::Json::array cmp = json11::Json::array {

src/disk_tool/disk_tool.cpp

@@ -48,6 +48,8 @@ static const char *help_text =
     "    --max_other 10%\n"
     "      Use disks for OSD data even if they already have non-Vitastor partitions,\n"
     "      but only if these take up no more than this percent of disk space.\n"
+    "    --dry-run\n"
+    "      Check and print new OSD count for each disk but do not actually create them.\n"
     "  \n"
     "  Options (single-device mode):\n"
     "    --data_device <DEV>        Use partition <DEV> for data\n"

src/disk_tool/disk_tool.h

@@ -8,14 +8,17 @@
 #endif
 
 #include <map>
+#include <set>
 #include <vector>
 #include <string>
 #include <functional>
 
 #include "json11/json11.hpp"
 #include "blockstore_disk.h"
-#include "blockstore_impl.h"
+#include "blockstore.h"
+#include "ondisk_formats.h"
 #include "crc32c.h"
+#include "allocator.h"
 
 // vITADisk
 #define VITASTOR_DISK_MAGIC 0x6b73694441544976
@@ -134,7 +137,8 @@ struct disk_tool_t
     int prepare(std::vector<std::string> devices);
     std::vector<vitastor_dev_info_t> collect_devices(const std::vector<std::string> & devices);
     json11::Json add_partitions(vitastor_dev_info_t & devinfo, std::vector<std::string> sizes);
-    std::vector<std::string> get_new_data_parts(vitastor_dev_info_t & dev, uint64_t osd_per_disk, uint64_t max_other_percent);
+    std::vector<std::string> get_new_data_parts(vitastor_dev_info_t & dev,
+        uint64_t osd_per_disk, uint64_t max_other_percent, uint64_t *check_new_count);
     int get_meta_partition(std::vector<vitastor_dev_info_t> & ssds, std::map<std::string, std::string> & options);
 
     int upgrade_simple_unit(std::string unit);

src/disk_tool/disk_tool_journal.cpp

@@ -1,7 +1,10 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 
+#include <unistd.h>
+
 #include "disk_tool.h"
+#include "malloc_or_die.h"
 
 int disk_tool_t::dump_journal()
 {

src/disk_tool/disk_tool_meta.cpp

@@ -5,6 +5,7 @@
 #include "rw_blocking.h"
 #include "osd_id.h"
 #include "json_util.h"
+#include "malloc_or_die.h"
 
 int disk_tool_t::process_meta(std::function<void(blockstore_meta_header_v2_t *)> hdr_fn,
     std::function<void(uint64_t, clean_disk_entry*, uint8_t*)> record_fn, bool do_open)

src/disk_tool/disk_tool_prepare.cpp

@@ -435,7 +435,7 @@ json11::Json disk_tool_t::add_partitions(vitastor_dev_info_t & devinfo, std::vec
 }
 
 std::vector<std::string> disk_tool_t::get_new_data_parts(vitastor_dev_info_t & dev,
-    uint64_t osd_per_disk, uint64_t max_other_percent)
+    uint64_t osd_per_disk, uint64_t max_other_percent, uint64_t *check_new_count)
 {
     std::vector<std::string> use_parts;
     uint64_t want_parts = 0;
@@ -457,7 +457,6 @@ std::vector<std::string> disk_tool_t::get_new_data_parts(vitastor_dev_info_t & d
                 {
                     // Use this partition
                     use_parts.push_back(part["uuid"].string_value());
-                    osds_exist++;
                 }
                 else
                 {
@@ -480,9 +479,21 @@ std::vector<std::string> disk_tool_t::get_new_data_parts(vitastor_dev_info_t & d
         }
         // Still create OSD(s) if a disk has no more than (max_other_percent) other data
         if (osds_exist >= osd_per_disk || (dev.free+osds_size) < dev.size*(100-max_other_percent)/100)
+        {
             fprintf(stderr, "%s is already partitioned, skipping\n", dev.path.c_str());
+            use_parts.clear();
+        }
         else
-            want_parts = osd_per_disk-osds_exist;
+        {
+            if (use_parts.size() >= osd_per_disk-osds_exist)
+                use_parts.resize(osd_per_disk-osds_exist);
+            want_parts = osd_per_disk-osds_exist-use_parts.size();
+        }
+    }
+    if (check_new_count)
+    {
+        *check_new_count = want_parts;
+        return use_parts;
     }
     if (want_parts > 0)
     {
@@ -684,10 +695,25 @@ int disk_tool_t::prepare(std::vector<std::string> devices)
     }
     json11::Json::array all_results, errors;
     auto journal_size = options["journal_size"];
+    if (options.find("dry_run") != options.end())
+    {
+        json11::Json::array results;
+        for (auto & dev: devinfo)
+        {
+            uint64_t new_part_count = 0;
+            auto existing_part_count = get_new_data_parts(dev, osd_per_disk, max_other_percent, &new_part_count).size();
+            results.push_back(json11::Json::object{ { "device_path", dev.path }, { "new_osd_count", existing_part_count+new_part_count } });
+            if (!json && new_part_count+existing_part_count > 0)
+                printf("Will initialize %ju OSD(s) on %s\n", existing_part_count+new_part_count, dev.path.c_str());
+        }
+        if (json)
+            printf("%s\n", json11::Json(json11::Json::object{{ "devices", results }}).dump().c_str());
+        return 0;
+    }
     for (auto & dev: devinfo)
     {
         // Select new partitions and create an OSD on each of them
-        for (const auto & uuid: get_new_data_parts(dev, osd_per_disk, max_other_percent))
+        for (const auto & uuid: get_new_data_parts(dev, osd_per_disk, max_other_percent, NULL))
         {
             options["force"] = true;
             options["data_device"] = "/dev/disk/by-partuuid/"+strtolower(uuid);

src/disk_tool/disk_tool_resize.cpp

@@ -4,6 +4,7 @@
 #include "disk_tool.h"
 #include "rw_blocking.h"
 #include "str_util.h"
+#include "malloc_or_die.h"
 
 #define DM_ST_EMPTY 0
 #define DM_ST_TO_READ 1

src/disk_tool/disk_tool_udev.cpp

@@ -7,6 +7,7 @@
 #include "rw_blocking.h"
 #include "str_util.h"
 #include "json_util.h"
+#include "malloc_or_die.h"
 
 struct __attribute__((__packed__)) vitastor_disk_superblock_t
 {

src/disk_tool/disk_tool_utils.cpp

@@ -2,11 +2,13 @@
 // License: VNPL-1.1 (see README.md for details)
 
 #include <sys/wait.h>
+#include <sys/ioctl.h>
 #include <dirent.h>
 
 #include "disk_tool.h"
 #include "rw_blocking.h"
 #include "str_util.h"
+#include "malloc_or_die.h"
 
 uint64_t sscanf_json(const char *fmt, const json11::Json & str)
 {

src/kv/kv_cli.cpp

@@ -97,6 +97,7 @@ void kv_cli_t::parse_args(int narg, const char *args[])
                 "  dump [<start> [end]]\n"
                 "  dumpjson [<start> [end]]\n"
                 "  loadjson\n"
+                "  rescue\n"
                 "\n"
                 "<IMAGE> should be the name of Vitastor image with the DB.\n"
                 "Without <COMMAND>, you get an interactive DB shell.\n"
@@ -298,6 +299,50 @@ struct kv_cli_list_t
     int n = 0;
     std::function<void(int)> cb;
 
+    void handle_key(int res, const std::string & key, const std::string & value)
+    {
+        if (res < 0)
+        {
+            if (res != -ENOENT)
+                fprintf(stderr, "Error: %s (code %d)\n", strerror(-res), res);
+            if (format == 2)
+                write("\n}\n");
+            if (handle)
+                db->list_close(handle);
+            flush();
+            cb(res == -ENOENT ? 0 : res);
+            delete this;
+        }
+        else
+        {
+            if (format == 2)
+            {
+                write(n ? ",\n  " : "{\n  ");
+                write(addslashes(key));
+                write(": ");
+                write(addslashes(value));
+            }
+            else if (format == 1)
+            {
+                write("set ");
+                write(auto_addslashes(key));
+                write(" ");
+                write(value);
+                write("\n");
+            }
+            else
+            {
+                write(key);
+                write(" = ");
+                write(value);
+                write("\n");
+            }
+            n++;
+            if (handle)
+                db->list_next(handle, NULL);
+        }
+    }
+
     void write(const std::string & str)
     {
         if (buf.capacity() < KV_LIST_BUF_SIZE)
@@ -309,8 +354,14 @@ struct kv_cli_list_t
 
     void flush()
     {
-        ::write(1, buf.data(), buf.size());
-        buf.resize(0);
+        size_t done = 0;
+        while (done < buf.size())
+        {
+            ssize_t res = ::write(1, buf.data()+done, buf.size()-done);
+            if (res > 0)
+                done += res;
+        }
+        buf.clear();
     }
 };
 
@@ -619,44 +670,18 @@ void kv_cli_t::handle_cmd(const std::vector<std::string> & cmd, std::function<vo
         lst->cb = std::move(cb);
         db->list_next(lst->handle, [lst](int res, const std::string & key, const std::string & value)
         {
-            if (res < 0)
-            {
-                if (res != -ENOENT)
-                    fprintf(stderr, "Error: %s (code %d)\n", strerror(-res), res);
-                if (lst->format == 2)
-                    lst->write("\n}\n");
-                lst->flush();
-                lst->db->list_close(lst->handle);
-                lst->cb(res == -ENOENT ? 0 : res);
-                delete lst;
-            }
-            else
-            {
-                if (lst->format == 2)
-                {
-                    lst->write(lst->n ? ",\n  " : "{\n  ");
-                    lst->write(addslashes(key));
-                    lst->write(": ");
-                    lst->write(addslashes(value));
-                }
-                else if (lst->format == 1)
-                {
-                    lst->write("set ");
-                    lst->write(auto_addslashes(key));
-                    lst->write(" ");
-                    lst->write(value);
-                    lst->write("\n");
-                }
-                else
-                {
-                    lst->write(key);
-                    lst->write(" = ");
-                    lst->write(value);
-                    lst->write("\n");
-                }
-                lst->n++;
-                lst->db->list_next(lst->handle, NULL);
-            }
+            lst->handle_key(res, key, value);
+        });
+    }
+    else if (opname == "rescue")
+    {
+        kv_cli_list_t *lst = new kv_cli_list_t;
+        lst->db = db;
+        lst->format = 2;
+        lst->cb = std::move(cb);
+        db->rescue([lst](int res, const std::string & key, const std::string & value)
+        {
+            lst->handle_key(res, key, value);
         });
     }
     else if (opname == "loadjson")

src/kv/kv_db.cpp

@@ -96,7 +96,7 @@ struct kv_block_t
 
     void set_data_size();
     static int kv_size(const std::string & key, const std::string & value);
-    int parse(uint64_t offset, uint8_t *data, int size);
+    int parse(uint64_t offset, uint8_t *data, int size, bool allow_empty = false);
     bool serialize(uint8_t *data, int size);
     void apply_change();
     void cancel_change();
@@ -139,7 +139,6 @@ struct kv_db_t
     uint64_t next_free = 0;
     uint32_t kv_block_size = 0;
     uint32_t ino_block_size = 0;
-    bool immediate_commit = false;
     uint64_t memory_limit = 128*1024*1024;
     uint64_t evict_unused_age = 1000;
     uint64_t evict_max_misses = 10;
@@ -173,6 +172,7 @@ struct kv_db_t
     void open(inode_t inode_id, json11::Json cfg, std::function<void(int)> cb);
     void set_config(json11::Json cfg);
     void close(std::function<void()> cb);
+    void rescue(std::function<void(int res, const std::string & key, const std::string & value)> cb);
 
     void find_size(uint64_t min, uint64_t max, int phase, std::function<void(int, uint64_t)> cb);
     void run_continue_update(uint64_t offset);
@@ -243,13 +243,13 @@ static std::string read_string(uint8_t *data, int size, int *pos)
     return key;
 }
 
-int kv_block_t::parse(uint64_t offset, uint8_t *data, int size)
+int kv_block_t::parse(uint64_t offset, uint8_t *data, int size, bool allow_empty)
 {
     kv_stored_block_t *blk = (kv_stored_block_t *)data;
     if (blk->magic == 0 || blk->type == KV_EMPTY)
     {
         // empty block
-        if (offset != 0)
+        if (!allow_empty)
             fprintf(stderr, "K/V: Block %ju is %s\n", offset, blk->magic == 0 ? "empty" : "cleared");
         return -ENOTBLK;
     }
@@ -389,6 +389,10 @@ bool kv_block_t::serialize(uint8_t *buf, int size)
             return false;
         blk->items++;
     }
+    if (pos < size)
+    {
+        memset(buf+pos, 0, size-pos);
+    }
     return true;
 }
 
@@ -524,7 +528,6 @@ void kv_db_t::open(inode_t inode_id, json11::Json cfg, std::function<void(int)>
         return;
     }
     this->inode_id = inode_id;
-    this->immediate_commit = cli->get_immediate_commit(inode_id);
     this->ino_block_size = pool_cfg.data_block_size * pg_data_size;
     this->kv_block_size = kv_block_size;
     this->next_free = 0;
@@ -542,6 +545,127 @@ void kv_db_t::open(inode_t inode_id, json11::Json cfg, std::function<void(int)>
     });
 }
 
+struct kv_rescue_t
+{
+    kv_db_t *db = NULL;
+    uint64_t size = 0;
+    uint64_t pos = 0;
+    uint64_t cur_size = 0;
+    uint64_t cur_offset = 0;
+    int state = 0;
+    kv_block_t blk;
+    std::vector<uint8_t> buf;
+    std::function<void(int res, const std::string & key, const std::string & value)> cb;
+
+    void finish(int retval)
+    {
+        auto cb = std::move(this->cb);
+        cb(retval, "", "");
+        delete this;
+    }
+
+    void send_read()
+    {
+        if (pos >= size)
+        {
+            finish(-ENOENT);
+            return;
+        }
+        if (!buf.size())
+        {
+            buf.resize(1048576);
+        }
+        cluster_op_t *op = new cluster_op_t;
+        op->opcode = OSD_OP_READ;
+        op->inode = db->inode_id;
+        op->offset = pos;
+        cur_size = op->len = pos+buf.size() < size ? buf.size() : size-pos;
+        op->iov.push_back(buf.data(), cur_size);
+        op->callback = [=](cluster_op_t *op)
+        {
+            if (op->retval != op->len)
+            {
+                // error
+                finish(op->retval >= 0 ? -EIO : op->retval);
+                return;
+            }
+            state = 2;
+            cur_offset = 0;
+            run();
+            delete op;
+        };
+        db->cli->execute(op);
+        state = 1;
+    }
+
+    void parse_block()
+    {
+        if (cur_offset < cur_size)
+        {
+            blk = {};
+            int err = blk.parse(pos+cur_offset, buf.data()+cur_offset, db->kv_block_size, true);
+            if (err != 0)
+            {
+            }
+            else if (blk.type == KV_LEAF || blk.type == KV_LEAF_SPLIT)
+            {
+                for (auto it = blk.data.begin(); it != blk.data.end(); it++)
+                {
+                    cb(0, it->first, it->second);
+                }
+            }
+            cur_offset += db->kv_block_size;
+        }
+        else
+        {
+            pos += cur_size;
+            state = 0;
+        }
+    }
+
+    void run()
+    {
+        while (true)
+        {
+            if (state == 0)
+            {
+                send_read();
+                return;
+            }
+            else if (state == 1)
+            {
+                return;
+            }
+            else if (state == 2)
+            {
+                parse_block();
+            }
+        }
+    }
+};
+
+void kv_db_t::rescue(std::function<void(int res, const std::string & key, const std::string & value)> cb)
+{
+    if (!inode_id || closing)
+    {
+        cb(-EINVAL, "", "");
+        return;
+    }
+    find_size(0, 0, 1, [=](int res, uint64_t size)
+    {
+        if (res < 0)
+        {
+            cb(res, "", "");
+            return;
+        }
+        kv_rescue_t *st = new kv_rescue_t();
+        st->db = this;
+        st->size = size;
+        st->cb = cb;
+        st->run();
+    });
+}
+
 void kv_db_t::set_config(json11::Json cfg)
 {
     this->memory_limit = cfg["kv_memory_limit"].is_null() ? 128*1024*1024 : cfg["kv_memory_limit"].uint64_value();
@@ -563,7 +687,6 @@ void kv_db_t::close(std::function<void()> cb)
         next_free = 0;
         kv_block_size = 0;
         ino_block_size = 0;
-        immediate_commit = false;
         block_cache.clear();
         known_versions.clear();
         cb();
@@ -630,7 +753,10 @@ uint64_t kv_db_t::alloc_block()
     {
         // Allow to reconfigure <max_allocate_blocks> online
         if (allocating_blocks.size() > max_allocate_blocks)
+        {
             allocating_blocks.erase(allocating_blocks.begin()+allocating_block_pos, allocating_blocks.begin()+allocating_block_pos+1);
+            allocating_block_pos = allocating_block_pos % allocating_blocks.size();
+        }
         else
             allocating_blocks[allocating_block_pos].offset = UINT64_MAX;
     }
@@ -870,7 +996,7 @@ static void get_block(kv_db_t *db, uint64_t offset, int cur_level, int recheck_p
         }
         // Block already in cache, we can proceed
         blk->usage = db->usage_counter;
-        db->cli->msgr.ringloop->set_immediate([=] { cb(0, BLK_UPDATING); });
+        cb(0, BLK_UPDATING);
         return;
     }
     cluster_op_t *op = new cluster_op_t;
@@ -936,7 +1062,7 @@ static void get_block(kv_db_t *db, uint64_t offset, int cur_level, int recheck_p
                 del_block_level(db, blk);
                 *blk = {};
             }
-            int err = blk->parse(op->offset, (uint8_t*)op->iov.buf[0].iov_base, op->len);
+            int err = blk->parse(op->offset, (uint8_t*)op->iov.buf[0].iov_base, op->len, op->offset == 0);
             if (err == 0)
             {
                 blk->level = cur_level;
@@ -999,6 +1125,8 @@ kv_op_t::~kv_op_t()
     {
         done = true;
         db->active_ops--;
+        if (!db->active_ops && db->closing)
+            db->close(db->on_close);
     }
 }
 
@@ -1269,22 +1397,7 @@ static void write_block(kv_db_t *db, kv_block_t *blk, std::function<void(int)> c
             }
         }
         delete op;
-        if (res < 0 || db->immediate_commit)
-        {
-            cb(res);
-        }
-        else
-        {
-            op = new cluster_op_t;
-            op->opcode = OSD_OP_SYNC;
-            op->callback = [cb](cluster_op_t *op)
-            {
-                auto res = op->retval;
-                delete op;
-                cb(res);
-            };
-            db->cli->execute(op);
-        }
+        cb(res);
     };
     db->cli->execute(op);
 }
@@ -1659,6 +1772,7 @@ void kv_op_t::update_block(int path_pos, bool is_delete, const std::string & key
         {
             if (res < 0)
             {
+                blk->cancel_change();
                 auto blk_offset = blk->offset;
                 del_block_level(db, blk);
                 db->block_cache.erase(blk_offset);
@@ -1791,6 +1905,7 @@ void kv_op_t::update_block(int path_pos, bool is_delete, const std::string & key
             {
                 if (write_res < 0)
                 {
+                    blk->cancel_change();
                     auto blk_offset = blk->offset;
                     del_block_level(db, blk);
                     db->block_cache.erase(blk_offset);
@@ -1978,6 +2093,11 @@ void vitastorkv_dbw_t::close(std::function<void()> cb)
     db->close(cb);
 }
 
+void vitastorkv_dbw_t::rescue(std::function<void(int res, const std::string & key, const std::string & value)> cb)
+{
+    db->rescue(cb);
+}
+
 void vitastorkv_dbw_t::get(const std::string & key, std::function<void(int res, const std::string & value)> cb, bool cached)
 {
     auto *op = new kv_op_t;

src/kv/kv_stress.cpp

@@ -82,6 +82,7 @@ public:
     uint64_t total_prob = 0;
     uint64_t ops_sent = 0, ops_done = 0;
     int stat_timer_id = -1;
+    int run_timer_id = -1;
     int in_progress = 0;
     bool reopening = false;
     std::set<kv_test_listing_t*> listings;
@@ -299,9 +300,11 @@ void kv_test_t::run(json11::Json cfg)
     ringloop->register_consumer(&consumer);
     if (print_stats_interval)
         stat_timer_id = epmgr->tfd->set_timer(print_stats_interval*1000, true, [this](int) { print_stats(prev_stat, prev_stat_time); });
+    if (runtime_sec)
+        run_timer_id = epmgr->tfd->set_timer(runtime_sec*1000, false, [this](int) { run_timer_id = -1; op_count = 0; });
     clock_gettime(CLOCK_REALTIME, &start_stat_time);
     prev_stat_time = start_stat_time;
-    while (!finished)
+    while (!finished || in_progress > 0)
     {
         ringloop->loop();
         if (!finished)
@@ -309,6 +312,8 @@ void kv_test_t::run(json11::Json cfg)
     }
     if (stat_timer_id >= 0)
         epmgr->tfd->clear_timer(stat_timer_id);
+    if (run_timer_id >= 0)
+        epmgr->tfd->clear_timer(run_timer_id);
     ringloop->unregister_consumer(&consumer);
     // Print total stats
     print_total_stats();

src/kv/vitastor_kv.h

@@ -28,6 +28,7 @@ struct __attribute__((visibility("default"))) vitastorkv_dbw_t
     void open(uint64_t inode_id, std::map<std::string, std::string> cfg, std::function<void(int)> cb);
     void set_config(std::map<std::string, std::string> cfg);
     void close(std::function<void()> cb);
+    void rescue(std::function<void(int res, const std::string & key, const std::string & value)> cb);
 
     uint64_t get_size();
 

src/nfs/nfs_kv.cpp

@@ -25,7 +25,10 @@ nfstime3 nfstime_from_str(const std::string & s)
             t.nseconds /= 10;
     }
     else
+    {
         t.seconds = stoull_full(s, 10);
+        t.nseconds = 0;
+    }
     return t;
 }
 

src/nfs/nfs_kv_getattr.cpp

@@ -94,6 +94,10 @@ uint32_t kv_get_access(const authsys_parms & auth_sys, const json11::Json & attr
     uint32_t uid = attrs["uid"].uint64_value();
     uint32_t gid = attrs["gid"].uint64_value();
     uint32_t access = 0;
+    if (!auth_sys.uid)
+    {
+        return (ACCESS3_READ|ACCESS3_LOOKUP|ACCESS3_MODIFY|ACCESS3_EXTEND|ACCESS3_DELETE|ACCESS3_EXECUTE);
+    }
     if (uid == auth_sys.uid)
     {
         access |= ((mode & (1 << 8)) ? ACCESS3_READ|ACCESS3_LOOKUP : 0);
@@ -120,6 +124,8 @@ uint32_t kv_get_access(const authsys_parms & auth_sys, const json11::Json & attr
 bool kv_is_accessible(const authsys_parms & auth_sys, const json11::Json & attrs, uint32_t access)
 {
     uint32_t mode = attrs["mode"].is_null() ? (attrs["type"] == "dir" ? 0755 : 0644) : attrs["mode"].uint64_value();
+    if (!auth_sys.uid)
+        return true;
     uint32_t mask = 1 << (access == ACCESS3_EXECUTE ? 0
         : (access == ACCESS3_MODIFY || access == ACCESS3_EXTEND || access == ACCESS3_DELETE ? 1 : 2));
     if (mode & mask)
@@ -160,7 +166,20 @@ int kv_nfs3_access_proc(void *opaque, rpc_op_t *rop)
             fprintf(stderr, "[%d] ACCESS %ju -> %s\n", self->nfs_fd, ino, value.c_str());
         if (res < 0)
         {
-            *reply = (ACCESS3res){ .status = vitastor_nfs_map_err(-res) };
+            // Broken inode (non-existing), allow only root access
+            if (!rop->auth_sys.uid)
+            {
+                *reply = (ACCESS3res){
+                    .status = NFS3_OK,
+                    .resok = (ACCESS3resok){
+                        .access = args->access,
+                    },
+                };
+            }
+            else
+            {
+                *reply = (ACCESS3res){ .status = vitastor_nfs_map_err(-res) };
+            }
         }
         else
         {

src/nfs/nfs_kv_lookup.cpp

@@ -23,6 +23,81 @@ int kv_nfs3_lookup_proc(void *opaque, rpc_op_t *rop)
         rpc_queue_reply(rop);
         return 0;
     }
+    if (filename == ".")
+    {
+        kv_read_inode(self->parent, dir_ino, [=](int res, const std::string & value, json11::Json ientry)
+        {
+            if (res < 0)
+            {
+                *reply = (LOOKUP3res){ .status = vitastor_nfs_map_err(-res) };
+                rpc_queue_reply(rop);
+                return;
+            }
+            *reply = (LOOKUP3res){
+                .status = NFS3_OK,
+                .resok = (LOOKUP3resok){
+                    .object = xdr_copy_string(rop->xdrs, kv_fh(dir_ino)),
+                    .obj_attributes = {
+                        .attributes_follow = 1,
+                        .attributes = get_kv_attributes(self->parent, dir_ino, ientry),
+                    },
+                },
+            };
+            rpc_queue_reply(rop);
+        });
+        return 1;
+    }
+    if (filename == "..")
+    {
+        kv_read_inode(self->parent, dir_ino, [=](int res, const std::string & value, json11::Json ientry)
+        {
+            if (res < 0)
+            {
+                *reply = (LOOKUP3res){ .status = vitastor_nfs_map_err(-res) };
+                rpc_queue_reply(rop);
+                return;
+            }
+            uint64_t parent_ino = ientry["parent_ino"].uint64_value();
+            if (parent_ino)
+            {
+                kv_read_inode(self->parent, parent_ino, [=](int res, const std::string & value, json11::Json parent_ientry)
+                {
+                    if (res < 0)
+                    {
+                        *reply = (LOOKUP3res){ .status = vitastor_nfs_map_err(-res) };
+                        rpc_queue_reply(rop);
+                        return;
+                    }
+                    *reply = (LOOKUP3res){
+                        .status = NFS3_OK,
+                        .resok = (LOOKUP3resok){
+                            .object = xdr_copy_string(rop->xdrs, kv_fh(parent_ino)),
+                            .obj_attributes = {
+                                .attributes_follow = 1,
+                                .attributes = get_kv_attributes(self->parent, parent_ino, parent_ientry),
+                            },
+                        },
+                    };
+                    rpc_queue_reply(rop);
+                });
+            }
+            else
+            {
+                *reply = (LOOKUP3res){
+                    .status = NFS3_OK,
+                    .resok = (LOOKUP3resok){
+                        .object = xdr_copy_string(rop->xdrs, kv_fh(dir_ino)),
+                        .obj_attributes = {
+                            .attributes_follow = 1,
+                            .attributes = get_kv_attributes(self->parent, dir_ino, ientry),
+                        },
+                    },
+                };
+                rpc_queue_reply(rop);
+            }
+        });
+        return 1;
+    }
     self->parent->db->get(kv_direntry_key(dir_ino, filename), [=](int res, const std::string & value)
     {
         if (res < 0)

src/nfs/nfs_kv_remove.cpp

@@ -199,7 +199,7 @@ resume_4:
         else
         {
             // Not OK, restore direntry
-            st->self->parent->db->del(kv_direntry_key(st->dir_ino, st->filename), [st](int res)
+            st->self->parent->db->set(kv_direntry_key(st->dir_ino, st->filename), st->direntry_text, [st](int res)
             {
                 st->res2 = res;
                 nfs_kv_continue_delete(st, 5);
@@ -282,6 +282,10 @@ resume_6:
             });
             return;
 resume_7:
+            if (!st->res)
+            {
+                st->self->parent->kvfs->touch_queue.insert(st->dir_ino);
+            }
             auto cb = std::move(st->cb);
             cb(st->res);
             return;