Release 1.10.1

New features:

- Add "deleted" image flag which is set when vitastor-cli rm starts to delete an image,
  but can't delete it fully due to inactive PGs or stopped OSDs
- Support JSON output in vitastor-disk prepare and purge
- Show backfillfull pools in vitastor-cli status
- Make object listings consistent (used in vitastor-cli rm/rm-data/merge/etc).
  This means that there is now a guarantee that if a data block is present when you invoke rm,
  rm will attempt to delete it, even if rm is invoked when the PG switches state. Previously in
  such cases rm could skip and leave some objects behind as garbage, and merge probably could
  incorrectly move data between snapshots.
- Make deletions (rm/rm-data) consistent. This means that rm/rm-data will either complete
  successfully and delete all requested image data or complete with an error if some objects
  could not be deleted or if there is a possibility that some data is left on stopped OSDs.
  Previously, when some PGs or OSDs were inactive at the moment of deletion, rm-data was
  behaving incorrectly: it wasn't retrying deletions failed due to dropped OSD connections,
  it could hang waiting for PGs to activate, and it could return with a successful error
  code while some garbage was still possibly left on some OSDs. Deletions are not fully atomic
  cluster-wide yet, which means that you still have to repeat the deletion request after you
  return stopped OSDs back, but now you always know for sure if you have to repeat it.

Bug fixes:

- Fix vitastor-cli rm --exact / --matching command not working
- Finally fix "Unexpected status" in the Proxmox plugin
- Fix vitastor-cli create-snap incorrectly linking multiple snapshots in a different pool
- Fix incomplete image parent_id loop check in OSD
- Fix reads from snapshots in a different pool not working if there are more than 2 snapshots
- Fix append of VITASTOR_CONF to cmdline in the opennebula prebackup script
- Fix OSDs crashing again when the cluster is full with EC (was meant to work since 1.6.0 but didn't)
- Improve logging of subop failures

.gitea/workflows/test.yml

@@ -288,6 +288,24 @@ jobs:
           echo ""
         done
 
+  test_create_halfhost:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_create_halfhost.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_failure_domain:
     runs-on: ubuntu-latest
     needs: build
@@ -396,6 +414,24 @@ jobs:
           echo ""
         done
 
+  test_rm_degraded:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_rm_degraded.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_snapshot_chain:
     runs-on: ubuntu-latest
     needs: build

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "1.9.3")
+set(VITASTOR_VERSION "1.10.1")
 
 add_subdirectory(src)

csi/Dockerfile

@@ -22,6 +22,8 @@ RUN apt-get update && \
     (echo "APT::Install-Recommends false;" > /etc/apt/apt.conf) && \
     apt-get update && \
     apt-get install -y e2fsprogs xfsprogs kmod iproute2 \
+        # NFS mount dependencies
+        nfs-common netbase \
         # dependencies of qemu-storage-daemon
         libnuma1 liburing2 libglib2.0-0 libfuse3-3 libaio1 libzstd1 libnettle8 \
         libgmp10 libhogweed6 libp11-kit0 libidn2-0 libunistring2 libtasn1-6 libpcre2-8-0 libffi8 && \

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v1.9.3
+VITASTOR_VERSION ?= v1.10.1
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v1.9.3
+          image: vitalif/vitastor-csi:v1.10.1
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v1.9.3
+          image: vitalif/vitastor-csi:v1.10.1
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/009-storage-class.yaml

@@ -9,8 +9,16 @@ metadata:
 provisioner: csi.vitastor.io
 volumeBindingMode: Immediate
 parameters:
-  etcdVolumePrefix: ""
-  poolId: "1"
+  # CSI driver can create block-based volumes and VitastorFS-based volumes
+  # only VitastorFS-based volumes and raw block volumes (without FS) support ReadWriteMany mode
+  # set this parameter to VitastorFS metadata volume name to use VitastorFS
+  # if unset, block-based volumes will be created
+  vitastorfs: ""
+  # for block-based storage classes, pool ID may be either a string (name) or a number (ID)
+  # for vitastorFS-based storage classes it must be a string - name of the default pool for FS data
+  poolId: "testpool"
+  # volume name prefix for block-based storage classes or NFS subdirectory (including /) for FS-based volumes
+  volumePrefix: ""
   # you can choose other configuration file if you have it in the config map
   # different etcd URLs and prefixes should also be put in the config
   #configPath: "/etc/vitastor/vitastor.conf"

csi/deploy/example-storage-class-fs.yaml

@@ -0,0 +1,25 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  namespace: vitastor-system
+  name: vitastor
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+provisioner: csi.vitastor.io
+volumeBindingMode: Immediate
+parameters:
+  # CSI driver can create block-based volumes and VitastorFS-based volumes
+  # only VitastorFS-based volumes and raw block volumes (without FS) support ReadWriteMany mode
+  # set this parameter to VitastorFS metadata volume name to use VitastorFS
+  # if unset, block-based volumes will be created
+  vitastorfs: "testfs"
+  # for block-based storage classes, pool ID may be either a string (name) or a number (ID)
+  # for vitastorFS-based storage classes it must be a string - name of the default pool for FS data
+  poolId: "testpool"
+  # volume name prefix for block-based storage classes or NFS subdirectory (including /) for FS-based volumes
+  volumePrefix: "k8s/"
+  # you can choose other configuration file if you have it in the config map
+  # different etcd URLs and prefixes should also be put in the config
+  #configPath: "/etc/vitastor/vitastor.conf"
+allowVolumeExpansion: true

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "1.9.3"
+    vitastorCSIDriverVersion = "1.10.1"
 )
 
 // Config struct fills the parameters of request or user input

csi/src/controllerserver.go

@@ -8,7 +8,6 @@ import (
     "encoding/json"
     "fmt"
     "strings"
-    "strconv"
     "time"
     "os"
     "io/ioutil"
@@ -68,9 +67,10 @@ func GetConnectionParams(params map[string]string) (map[string]string, error)
     {
         configPath = "/etc/vitastor/vitastor.conf"
     }
-    else
+    ctxVars["configPath"] = configPath
+    if (params["vitastorfs"] != "")
     {
-        ctxVars["configPath"] = configPath
+        ctxVars["vitastorfs"] = params["vitastorfs"]
     }
     config := make(map[string]interface{})
     configFD, err := os.Open(configPath)
@@ -140,33 +140,57 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
         return nil, status.Error(codes.InvalidArgument, "volume capabilities is a required field")
     }
 
-    err := cs.checkCaps(volumeCapabilities)
-    if (err != nil)
-    {
-        return nil, err
-    }
-
-    etcdVolumePrefix := req.Parameters["etcdVolumePrefix"]
-    poolId, _ := strconv.ParseUint(req.Parameters["poolId"], 10, 64)
-    if (poolId == 0)
-    {
-        return nil, status.Error(codes.InvalidArgument, "poolId is missing in storage class configuration")
-    }
-
-    volName := etcdVolumePrefix + req.GetName()
-    volSize := 1 * GB
-    if capRange := req.GetCapacityRange(); capRange != nil
-    {
-        volSize = ((capRange.GetRequiredBytes() + MB - 1) / MB) * MB
-    }
-
     ctxVars, err := GetConnectionParams(req.Parameters)
     if (err != nil)
     {
         return nil, err
     }
 
-    args := []string{ "create", volName, "-s", fmt.Sprintf("%v", volSize), "--pool", fmt.Sprintf("%v", poolId) }
+    err = cs.checkCaps(volumeCapabilities, ctxVars["vitastorfs"] != "")
+    if (err != nil)
+    {
+        return nil, err
+    }
+
+    pool := req.Parameters["poolId"]
+    if (pool == "")
+    {
+        return nil, status.Error(codes.InvalidArgument, "poolId is missing in storage class configuration")
+    }
+    volumePrefix := req.Parameters["volumePrefix"]
+    if (volumePrefix == "")
+    {
+        // Old name
+        volumePrefix = req.Parameters["etcdVolumePrefix"]
+    }
+    volName := volumePrefix + req.GetName()
+    volSize := 1 * GB
+    if capRange := req.GetCapacityRange(); capRange != nil
+    {
+        volSize = ((capRange.GetRequiredBytes() + MB - 1) / MB) * MB
+    }
+
+    if (ctxVars["vitastorfs"] != "")
+    {
+        // Nothing to create, subdirectories are created during mounting
+        // FIXME: It would be cool to support quotas some day and set it here
+        if (req.VolumeContentSource.GetSnapshot() != nil)
+        {
+            return nil, status.Error(codes.InvalidArgument, "VitastorFS doesn't support snapshots")
+        }
+        ctxVars["name"] = volName
+        ctxVars["pool"] = pool
+        volumeIdJson, _ := json.Marshal(ctxVars)
+        return &csi.CreateVolumeResponse{
+            Volume: &csi.Volume{
+                // Ugly, but VolumeContext isn't passed to DeleteVolume :-(
+                VolumeId: string(volumeIdJson),
+                CapacityBytes: volSize,
+            },
+        }, nil
+    }
+
+    args := []string{ "create", volName, "-s", fmt.Sprintf("%v", volSize), "--pool", pool }
 
     // Support creation from snapshot
     var src *csi.VolumeContentSource
@@ -249,6 +273,12 @@ func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol
         return nil, err
     }
 
+    if (ctxVars["vitastorfs"] != "")
+    {
+        // FIXME: Delete FS subdirectory
+        return &csi.DeleteVolumeResponse{}, nil
+    }
+
     _, err = invokeCLI(ctxVars, []string{ "rm", volName })
     if (err != nil)
     {
@@ -283,13 +313,25 @@ func (cs *ControllerServer) ValidateVolumeCapabilities(ctx context.Context, req
     {
         return nil, status.Error(codes.InvalidArgument, "volumeId is nil")
     }
+    volVars := make(map[string]string)
+    err := json.Unmarshal([]byte(volumeID), &volVars)
+    if (err != nil)
+    {
+        return nil, status.Error(codes.Internal, "volume ID not in JSON format")
+    }
+    ctxVars, err := GetConnectionParams(volVars)
+    if (err != nil)
+    {
+        return nil, err
+    }
+
     volumeCapabilities := req.GetVolumeCapabilities()
     if (volumeCapabilities == nil)
     {
         return nil, status.Error(codes.InvalidArgument, "volumeCapabilities is nil")
     }
 
-    err := cs.checkCaps(volumeCapabilities)
+    err = cs.checkCaps(volumeCapabilities, ctxVars["vitastorfs"] != "")
     if (err != nil)
     {
         return nil, err
@@ -302,7 +344,7 @@ func (cs *ControllerServer) ValidateVolumeCapabilities(ctx context.Context, req
     }, nil
 }
 
-func (cs *ControllerServer) checkCaps(volumeCapabilities []*csi.VolumeCapability) error
+func (cs *ControllerServer) checkCaps(volumeCapabilities []*csi.VolumeCapability, fs bool) error
 {
     var volumeCapabilityAccessModes []*csi.VolumeCapability_AccessMode
     for _, mode := range []csi.VolumeCapability_AccessMode_Mode{
@@ -318,6 +360,10 @@ func (cs *ControllerServer) checkCaps(volumeCapabilities []*csi.VolumeCapability
     {
         if (capability.GetBlock() != nil)
         {
+            if (fs)
+            {
+                return status.Errorf(codes.InvalidArgument, "%v not supported with FS-based volumes", capability)
+            }
             for _, mode := range []csi.VolumeCapability_AccessMode_Mode{
                 csi.VolumeCapability_AccessMode_MULTI_NODE_SINGLE_WRITER,
                 csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER,
@@ -328,6 +374,12 @@ func (cs *ControllerServer) checkCaps(volumeCapabilities []*csi.VolumeCapability
         }
     }
 
+    if (fs)
+    {
+        // All access modes including RWX are supported with FS-based volumes
+        return nil
+    }
+
     capabilitySupport := false
     for _, capability := range volumeCapabilities
     {
@@ -342,7 +394,7 @@ func (cs *ControllerServer) checkCaps(volumeCapabilities []*csi.VolumeCapability
 
     if (!capabilitySupport)
     {
-        return status.Errorf(codes.NotFound, "%v not supported", volumeCapabilities)
+        return status.Errorf(codes.InvalidArgument, "%v not supported", volumeCapabilities)
     }
 
     return nil
@@ -434,6 +486,12 @@ func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS
     {
         return nil, status.Error(codes.Internal, "volume ID not in JSON format")
     }
+
+    if (ctxVars["vitastorfs"] != "")
+    {
+        return nil, status.Error(codes.InvalidArgument, "VitastorFS doesn't support snapshots")
+    }
+
     volName := ctxVars["name"]
 
     // Create image using vitastor-cli
@@ -492,6 +550,11 @@ func (cs *ControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteS
         return nil, err
     }
 
+    if (ctxVars["vitastorfs"] != "")
+    {
+        return nil, status.Error(codes.InvalidArgument, "VitastorFS doesn't support snapshots")
+    }
+
     _, err = invokeCLI(ctxVars, []string{ "rm", volName+"@"+snapName })
     if (err != nil)
     {
@@ -523,6 +586,11 @@ func (cs *ControllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnap
         return nil, err
     }
 
+    if (ctxVars["vitastorfs"] != "")
+    {
+        return nil, status.Error(codes.InvalidArgument, "VitastorFS doesn't support snapshots")
+    }
+
     inodeCfg, err := invokeList(ctxVars, volName+"@*", false)
     if (err != nil)
     {
@@ -586,6 +654,16 @@ func (cs *ControllerServer) ControllerExpandVolume(ctx context.Context, req *csi
         return nil, err
     }
 
+    if (ctxVars["vitastorfs"] != "")
+    {
+        // Nothing to change
+        // FIXME: Support quotas and change quota here
+        return &csi.ControllerExpandVolumeResponse{
+            CapacityBytes: req.CapacityRange.RequiredBytes,
+            NodeExpansionRequired: false,
+        }, nil
+    }
+
     inodeCfg, err := invokeList(ctxVars, volName, true)
     if (err != nil)
     {

csi/src/nodeserver.go

@@ -5,11 +5,15 @@ package vitastor
 
 import (
     "context"
+    "crypto/sha1"
+    "encoding/hex"
     "encoding/json"
     "fmt"
     "os"
     "os/exec"
     "path/filepath"
+    "regexp"
+    "strconv"
     "strings"
     "sync"
     "syscall"
@@ -29,13 +33,14 @@ import (
 type NodeServer struct
 {
     *Driver
-    useVduse bool
-    stateDir string
-    mounter mount.Interface
+    useVduse        bool
+    stateDir        string
+    nfsStageDir     string
+    mounter         mount.Interface
     restartInterval time.Duration
-    mu sync.Mutex
-    cond *sync.Cond
-    volumeLocks map[string]bool
+    mu              sync.Mutex
+    cond            *sync.Cond
+    volumeLocks     map[string]bool
 }
 
 type DeviceState struct
@@ -48,6 +53,15 @@ type DeviceState struct
     PidFile    string `json:"pidFile"`
 }
 
+type NfsState struct
+{
+    ConfigPath string `json:"configPath"`
+    FsName     string `json:"fsName"`
+    Pool       string `json:"pool"`
+    Path       string `json:"path"`
+    Port       int    `json:"port"`
+}
+
 // NewNodeServer create new instance node
 func NewNodeServer(driver *Driver) *NodeServer
 {
@@ -60,11 +74,17 @@ func NewNodeServer(driver *Driver) *NodeServer
     {
         stateDir += "/"
     }
+    nfsStageDir := os.Getenv("NFS_STAGE_DIR")
+    if (nfsStageDir == "")
+    {
+        nfsStageDir = "/var/lib/kubelet/plugins/csi.vitastor.io/nfs"
+    }
     ns := &NodeServer{
-        Driver: driver,
-        useVduse: checkVduseSupport(),
-        stateDir: stateDir,
-        mounter: mount.New(""),
+        Driver:      driver,
+        useVduse:    checkVduseSupport(),
+        stateDir:    stateDir,
+        nfsStageDir: nfsStageDir,
+        mounter:     mount.New(""),
         volumeLocks: make(map[string]bool),
     }
     ns.cond = sync.NewCond(&ns.mu)
@@ -123,12 +143,12 @@ func (ns *NodeServer) restarter()
 func (ns *NodeServer) restoreVduseDaemons()
 {
     pattern := ns.stateDir+"vitastor-vduse-*.json"
-    matches, err := filepath.Glob(pattern)
+    stateFiles, err := filepath.Glob(pattern)
     if (err != nil)
     {
         klog.Errorf("failed to list %s: %v", pattern, err)
     }
-    if (len(matches) == 0)
+    if (len(stateFiles) == 0)
     {
         return
     }
@@ -146,59 +166,162 @@ func (ns *NodeServer) restoreVduseDaemons()
         klog.Errorf("/sbin/vdpa -j dev list returned bad JSON (error %v): %v", err, string(devListJSON))
         return
     }
-    for _, stateFile := range matches
+    for _, stateFile := range stateFiles
     {
-        vdpaId := filepath.Base(stateFile)
-        vdpaId = vdpaId[0:len(vdpaId)-5]
-        // Check if VDPA device is still added to the bus
-        if (devs[vdpaId] == nil)
-        {
-            // Unused, clean it up
-            unmapVduseById(ns.stateDir, vdpaId)
-            continue
-        }
+        ns.checkVduseState(stateFile, devs)
+    }
+}
 
-        stateJSON, err := os.ReadFile(stateFile)
+func (ns *NodeServer) checkVduseState(stateFile string, devs map[string]interface{})
+{
+    // Check if VDPA device is still added to the bus
+    vdpaId := filepath.Base(stateFile)
+    vdpaId = vdpaId[0:len(vdpaId)-5]
+    if (devs[vdpaId] == nil)
+    {
+        // Unused, clean it up
+        unmapVduseById(ns.stateDir, vdpaId)
+        return
+    }
+
+    // Read state file
+    stateJSON, err := os.ReadFile(stateFile)
+    if (err != nil)
+    {
+        klog.Warningf("error reading state file %v: %v", stateFile, err)
+        return
+    }
+    var state DeviceState
+    err = json.Unmarshal(stateJSON, &state)
+    if (err != nil)
+    {
+        klog.Warningf("state file %v contains invalid JSON (error %v): %v", stateFile, err, string(stateJSON))
+        return
+    }
+
+    // Lock volume
+    ns.lockVolume(state.ConfigPath+":block:"+state.Image)
+    defer ns.unlockVolume(state.ConfigPath+":block:"+state.Image)
+
+    // Recheck state file after locking
+    _, err = os.ReadFile(stateFile)
+    if (err != nil)
+    {
+        klog.Warningf("state file %v disappeared, skipping volume", stateFile)
+        return
+    }
+
+    // Check if the storage daemon is still active
+    pidFile := ns.stateDir + vdpaId + ".pid"
+    exists := false
+    proc, err := findByPidFile(pidFile)
+    if (err == nil)
+    {
+        exists = proc.Signal(syscall.Signal(0)) == nil
+    }
+    if (!exists)
+    {
+        // Restart daemon
+        klog.Warningf("restarting storage daemon for volume %v (VDPA ID %v)", state.Image, vdpaId)
+        err = startStorageDaemon(vdpaId, state.Image, pidFile, state.ConfigPath, state.Readonly)
         if (err != nil)
         {
-            klog.Warningf("error reading state file %v: %v", stateFile, err)
-            continue
+            klog.Warningf("failed to restart storage daemon for volume %v: %v", state.Image, err)
         }
-        var state DeviceState
-        err = json.Unmarshal(stateJSON, &state)
+    }
+}
+
+func (ns *NodeServer) restoreNfsDaemons()
+{
+    pattern := ns.stateDir+"vitastor-nfs-*.json"
+    stateFiles, err := filepath.Glob(pattern)
+    if (err != nil)
+    {
+        klog.Errorf("failed to list %s: %v", pattern, err)
+    }
+    if (len(stateFiles) == 0)
+    {
+        return
+    }
+    activeNFS, err := ns.listActiveNFS()
+    if (err != nil)
+    {
+        return
+    }
+    // Check all state files and try to restore active mounts
+    for _, stateFile := range stateFiles
+    {
+        ns.checkNfsState(stateFile, activeNFS)
+    }
+}
+
+func (ns *NodeServer) readNfsState(stateFile string, allowNotExists bool) (*NfsState, error)
+{
+    stateJSON, err := os.ReadFile(stateFile)
+    if (err != nil)
+    {
+        if (allowNotExists && os.IsNotExist(err))
+        {
+            return nil, nil
+        }
+        klog.Warningf("error reading state file %v: %v", stateFile, err)
+        return nil, err
+    }
+    var state NfsState
+    err = json.Unmarshal(stateJSON, &state)
+    if (err != nil)
+    {
+        klog.Warningf("state file %v contains invalid JSON (error %v): %v", stateFile, err, string(stateJSON))
+        return nil, err
+    }
+    return &state, nil
+}
+
+func (ns *NodeServer) checkNfsState(stateFile string, activeNfs map[int][]string)
+{
+    // Read state file
+    state, err := ns.readNfsState(stateFile, false)
+    if (err != nil)
+    {
+        return
+    }
+    // Lock FS
+    ns.lockVolume(state.ConfigPath+":fs:"+state.FsName)
+    defer ns.unlockVolume(state.ConfigPath+":fs:"+state.FsName)
+    // Check if NFS at this port is still mounted
+    pidFile := ns.stateDir + filepath.Base(stateFile)
+    pidFile = pidFile[0:len(pidFile)-5] + ".pid"
+    if (len(activeNfs[state.Port]) == 0)
+    {
+        // this is a stale state file, remove it
+        klog.Warningf("state file %v contains stale mount at port %d, removing it", stateFile, state.Port)
+        ns.stopNFS(stateFile, pidFile)
+        return
+    }
+    // Check PID file
+    exists := false
+    proc, err := findByPidFile(pidFile)
+    if (err == nil)
+    {
+        exists = proc.Signal(syscall.Signal(0)) == nil
+    }
+    if (!exists)
+    {
+        // Restart vitastor-nfs server
+        klog.Warningf("restarting NFS server for FS %v at port %v", state.FsName, state.Port)
+        _, _, err := system(
+            "/usr/bin/vitastor-nfs", "start",
+            "--pidfile", pidFile,
+            "--bind", "127.0.0.1",
+            "--port", fmt.Sprintf("%d", state.Port),
+            "--fs", state.FsName,
+            "--pool", state.Pool,
+            "--portmap", "0",
+        )
         if (err != nil)
         {
-            klog.Warningf("state file %v contains invalid JSON (error %v): %v", stateFile, err, string(stateJSON))
-            continue
+            klog.Warningf("failed to restart NFS server for FS %v: %v", state.FsName, err)
         }
-
-        ns.lockVolume(state.ConfigPath+":"+state.Image)
-
-        // Recheck state file after locking
-        _, err = os.ReadFile(stateFile)
-        if (err != nil)
-        {
-            klog.Warningf("state file %v disappeared, skipping volume", stateFile)
-            ns.unlockVolume(state.ConfigPath+":"+state.Image)
-            continue
-        }
-
-        // Check if the storage daemon is still active
-        pidFile := ns.stateDir + vdpaId + ".pid"
-        exists := false
-        proc, err := findByPidFile(pidFile)
-        if (err == nil)
-        {
-            exists = proc.Signal(syscall.Signal(0)) == nil
-        }
-        if (!exists)
-        {
-            // Restart daemon
-            klog.Warningf("restarting storage daemon for volume %v (VDPA ID %v)", state.Image, vdpaId)
-            _ = startStorageDaemon(vdpaId, state.Image, pidFile, state.ConfigPath, state.Readonly)
-        }
-
-        ns.unlockVolume(state.ConfigPath+":"+state.Image)
     }
 }
 
@@ -220,8 +343,13 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
     }
     volName := ctxVars["name"]
 
-    ns.lockVolume(ctxVars["configPath"]+":"+volName)
-    defer ns.unlockVolume(ctxVars["configPath"]+":"+volName)
+    if (ctxVars["vitastorfs"] != "")
+    {
+        return &csi.NodeStageVolumeResponse{}, nil
+    }
+
+    ns.lockVolume(ctxVars["configPath"]+":block:"+volName)
+    defer ns.unlockVolume(ctxVars["configPath"]+":block:"+volName)
 
     targetPath := req.GetStagingTargetPath()
     isBlock := req.GetVolumeCapability().GetBlock() != nil
@@ -408,8 +536,13 @@ func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag
     }
     volName := ctxVars["name"]
 
-    ns.lockVolume(ctxVars["configPath"]+":"+volName)
-    defer ns.unlockVolume(ctxVars["configPath"]+":"+volName)
+    if (ctxVars["vitastorfs"] != "")
+    {
+        return &csi.NodeUnstageVolumeResponse{}, nil
+    }
+
+    ns.lockVolume(ctxVars["configPath"]+":block:"+volName)
+    defer ns.unlockVolume(ctxVars["configPath"]+":block:"+volName)
 
     targetPath := req.GetStagingTargetPath()
     devicePath, _, err := mount.GetDeviceNameFromMount(ns.mounter, targetPath)
@@ -462,6 +595,153 @@ func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag
     return &csi.NodeUnstageVolumeResponse{}, nil
 }
 
+// Mount or check if NFS is already mounted
+func (ns *NodeServer) mountNFS(ctxVars map[string]string) (string, error)
+{
+    sum := sha1.Sum([]byte(ctxVars["configPath"]+":fs:"+ctxVars["vitastorfs"]))
+    nfsHash := hex.EncodeToString(sum[:])
+    stateFile := ns.stateDir+"vitastor-nfs-"+nfsHash+".json"
+    pidFile := ns.stateDir+"vitastor-nfs-"+nfsHash+".pid"
+    mountPath := ns.nfsStageDir+"/"+nfsHash
+    state, err := ns.readNfsState(stateFile, true)
+    if (state != nil)
+    {
+        return state.Path, nil
+    }
+    if (err != nil)
+    {
+        return "", err
+    }
+    err = os.MkdirAll(mountPath, 0777)
+    if (err != nil)
+    {
+        return "", err
+    }
+    // Create a new mount
+    state = &NfsState{
+        ConfigPath: ctxVars["configPath"],
+        FsName:     ctxVars["vitastorfs"],
+        Pool:       ctxVars["pool"],
+        Path:       mountPath,
+    }
+    klog.Infof("starting new NFS server for FS %v", state.FsName)
+    stdout, _, err := system(
+        "/usr/bin/vitastor-nfs", "start",
+        "--pidfile", pidFile,
+        "--bind", "127.0.0.1",
+        "--port", "auto",
+        "--fs", state.FsName,
+        "--pool", state.Pool,
+        "--portmap", "0",
+    )
+    if (err != nil)
+    {
+        return "", err
+    }
+    match := regexp.MustCompile("Port: (\\d+)").FindStringSubmatch(string(stdout))
+    if (match == nil)
+    {
+        klog.Errorf("failed to find port in vitastor-nfs output: %v", string(stdout))
+        ns.stopNFS(stateFile, pidFile)
+        return "", fmt.Errorf("failed to find port in vitastor-nfs output (bad vitastor-nfs version?)")
+    }
+    port, _ := strconv.ParseUint(match[1], 0, 16)
+    state.Port = int(port)
+    // Write state file
+    stateJSON, _ := json.Marshal(state)
+    err = os.WriteFile(stateFile, stateJSON, 0600)
+    if (err != nil)
+    {
+        klog.Errorf("failed to write state file %v", stateFile)
+        ns.stopNFS(stateFile, pidFile)
+        return "", err
+    }
+    // Mount NFS
+    _, _, err = system(
+        "mount", "-t", "nfs", "127.0.0.1:/", state.Path,
+        "-o", fmt.Sprintf("port=%d,mountport=%d,nfsvers=3,soft,nolock,tcp", port, port),
+    )
+    if (err != nil)
+    {
+        ns.stopNFS(stateFile, pidFile)
+        return "", err
+    }
+    return state.Path, nil
+}
+
+// Mount or check if NFS is already mounted
+func (ns *NodeServer) checkStopNFS(ctxVars map[string]string)
+{
+    sum := sha1.Sum([]byte(ctxVars["configPath"]+":fs:"+ctxVars["vitastorfs"]))
+    nfsHash := hex.EncodeToString(sum[:])
+    stateFile := ns.stateDir+"vitastor-nfs-"+nfsHash+".json"
+    pidFile := ns.stateDir+"vitastor-nfs-"+nfsHash+".pid"
+    mountPath := ns.nfsStageDir+"/"+nfsHash
+    state, err := ns.readNfsState(stateFile, true)
+    if (state == nil)
+    {
+        return
+    }
+    activeNFS, err := ns.listActiveNFS()
+    if (err != nil)
+    {
+        return
+    }
+    if (len(activeNFS[state.Port]) > 0)
+    {
+        return
+    }
+    // All volume mounts are detached, unmount the root mount and kill the server
+    err = mount.CleanupMountPoint(mountPath, ns.mounter, false)
+    if (err != nil)
+    {
+        klog.Errorf("failed to unmount %v: %v", mountPath, err)
+        return
+    }
+    ns.stopNFS(stateFile, pidFile)
+}
+
+func (ns *NodeServer) stopNFS(stateFile, pidFile string)
+{
+    err := killByPidFile(pidFile)
+    if (err != nil)
+    {
+        klog.Errorf("failed to kill process with pid from %v: %v", pidFile, err)
+    }
+    os.Remove(pidFile)
+    os.Remove(stateFile)
+}
+
+func (ns *NodeServer) listActiveNFS() (map[int][]string, error)
+{
+    mounts, err := mount.ParseMountInfo("/proc/self/mountinfo")
+    if (err != nil)
+    {
+        klog.Errorf("failed to list mounts: %v", err)
+        return nil, err
+    }
+    activeNFS := make(map[int][]string)
+    for _, mount := range mounts
+    {
+        // Volume mounts always refer to subpaths
+        if (mount.FsType == "nfs" && mount.Root != "/")
+        {
+            for _, opt := range mount.MountOptions
+            {
+                if (strings.HasPrefix(opt, "port="))
+                {
+                    port64, err := strconv.ParseUint(opt[5:], 10, 16)
+                    if (err == nil)
+                    {
+                        activeNFS[int(port64)] = append(activeNFS[int(port64)], mount.MountPoint)
+                    }
+                }
+            }
+        }
+    }
+    return activeNFS, nil
+}
+
 // NodePublishVolume mounts the volume mounted to the staging path to the target path
 func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error)
 {
@@ -480,28 +760,39 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
     }
     volName := ctxVars["name"]
 
-    ns.lockVolume(ctxVars["configPath"]+":"+volName)
-    defer ns.unlockVolume(ctxVars["configPath"]+":"+volName)
+    if (ctxVars["vitastorfs"] != "")
+    {
+        ns.lockVolume(ctxVars["configPath"]+":fs:"+ctxVars["vitastorfs"])
+        defer ns.unlockVolume(ctxVars["configPath"]+":fs:"+ctxVars["vitastorfs"])
+    }
+    else
+    {
+        ns.lockVolume(ctxVars["configPath"]+":block:"+volName)
+        defer ns.unlockVolume(ctxVars["configPath"]+":block:"+volName)
+    }
 
     stagingTargetPath := req.GetStagingTargetPath()
     targetPath := req.GetTargetPath()
     isBlock := req.GetVolumeCapability().GetBlock() != nil
 
-    // Check that stagingTargetPath is mounted
-    notmnt, err := mount.IsNotMountPoint(ns.mounter, stagingTargetPath)
-    if (err != nil)
+    if (ctxVars["vitastorfs"] == "")
     {
-        klog.Errorf("staging path %v is not mounted: %w", stagingTargetPath, err)
-        return nil, fmt.Errorf("staging path %v is not mounted: %w", stagingTargetPath, err)
-    }
-    else if (notmnt)
-    {
-        klog.Errorf("staging path %v is not mounted", stagingTargetPath)
-        return nil, fmt.Errorf("staging path %v is not mounted", stagingTargetPath)
+        // Check that stagingTargetPath is mounted
+        notmnt, err := mount.IsNotMountPoint(ns.mounter, stagingTargetPath)
+        if (err != nil)
+        {
+            klog.Errorf("staging path %v is not mounted: %w", stagingTargetPath, err)
+            return nil, fmt.Errorf("staging path %v is not mounted: %w", stagingTargetPath, err)
+        }
+        else if (notmnt)
+        {
+            klog.Errorf("staging path %v is not mounted", stagingTargetPath)
+            return nil, fmt.Errorf("staging path %v is not mounted", stagingTargetPath)
+        }
     }
 
     // Check that targetPath is not already mounted
-    notmnt, err = mount.IsNotMountPoint(ns.mounter, targetPath)
+    notmnt, err := mount.IsNotMountPoint(ns.mounter, targetPath)
     if (err != nil)
     {
         if (os.IsNotExist(err))
@@ -542,6 +833,24 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
         return nil, fmt.Errorf("target path %s is already mounted", targetPath)
     }
 
+    if (ctxVars["vitastorfs"] != "")
+    {
+        nfspath, err := ns.mountNFS(ctxVars)
+        if (err != nil)
+        {
+            ns.checkStopNFS(ctxVars)
+            return nil, err
+        }
+        // volName should include prefix
+        stagingTargetPath = nfspath+"/"+volName
+        err = os.MkdirAll(stagingTargetPath, 0777)
+        if (err != nil && !os.IsExist(err))
+        {
+            ns.checkStopNFS(ctxVars)
+            return nil, err
+        }
+    }
+
     execArgs := []string{"--bind", stagingTargetPath, targetPath}
     if (req.GetReadonly())
     {
@@ -553,6 +862,10 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
     out, err := cmd.Output()
     if (err != nil)
     {
+        if (ctxVars["vitastorfs"] != "")
+        {
+            ns.checkStopNFS(ctxVars)
+        }
         return nil, fmt.Errorf("Error running mount %v: %s", strings.Join(execArgs, " "), out)
     }
 
@@ -572,8 +885,16 @@ func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
     }
     volName := ctxVars["name"]
 
-    ns.lockVolume(ctxVars["configPath"]+":"+volName)
-    defer ns.unlockVolume(ctxVars["configPath"]+":"+volName)
+    if (ctxVars["vitastorfs"] != "")
+    {
+        ns.lockVolume(ctxVars["configPath"]+":fs:"+ctxVars["vitastorfs"])
+        defer ns.unlockVolume(ctxVars["configPath"]+":fs:"+ctxVars["vitastorfs"])
+    }
+    else
+    {
+        ns.lockVolume(ctxVars["configPath"]+":block:"+volName)
+        defer ns.unlockVolume(ctxVars["configPath"]+":block:"+volName)
+    }
 
     targetPath := req.GetTargetPath()
     devicePath, _, err := mount.GetDeviceNameFromMount(ns.mounter, targetPath)
@@ -600,6 +921,11 @@ func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
         return nil, err
     }
 
+    if (ctxVars["vitastorfs"] != "")
+    {
+        ns.checkStopNFS(ctxVars)
+    }
+
     return &csi.NodeUnpublishVolumeResponse{}, nil
 }
 

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (1.9.3-1) unstable; urgency=medium
+vitastor (1.10.1-1) unstable; urgency=medium
 
   * Bugfixes
 

debian/vitastor.Dockerfile

@@ -21,10 +21,10 @@ RUN set -e -x; \
     echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf; \
     echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
 
-RUN apt-get update
-RUN apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts libjerasure-dev cmake libibverbs-dev libisal-dev libnl-3-dev libnl-genl-3-dev curl
-RUN apt-get -y build-dep fio
-RUN apt-get --download-only source fio
+RUN apt-get update && \
+    apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts libjerasure-dev cmake libibverbs-dev librdmacm-dev libisal-dev libnl-3-dev libnl-genl-3-dev curl && \
+    apt-get -y build-dep fio && \
+    apt-get --download-only source fio
 
 ADD . /root/vitastor
 RUN set -e -x; \

docs/config/monitor.en.md

@@ -24,6 +24,7 @@ These parameters only apply to Monitors.
 - [osd_out_time](#osd_out_time)
 - [placement_levels](#placement_levels)
 - [use_old_pg_combinator](#use_old_pg_combinator)
+- [osd_backfillfull_ratio](#osd_backfillfull_ratio)
 
 ## use_antietcd
 
@@ -175,3 +176,18 @@ present in the configuration, then it is defined with the default priority
 
 Use the old PG combination generator which doesn't support [level_placement](pool.en.md#level_placement)
 and [raw_placement](pool.en.md#raw_placement) for pools which don't use this features.
+
+## osd_backfillfull_ratio
+
+- Type: number
+- Default: 0.99
+
+Monitors try to prevent OSDs becoming 100% full during rebalance or recovery by
+calculating how much space will be occupied on every OSD after all rebalance
+and recovery operations finish, and pausing rebalance and recovery if that
+amount of space exceeds OSD capacity multiplied by the value of this
+configuration parameter.
+
+Future used space is calculated by summing space used by all user data blocks
+(objects) in all PGs placed on a specific OSD, even if some of these objects
+currently reside on a different set of OSDs.

docs/config/monitor.ru.md

@@ -24,6 +24,7 @@
 - [osd_out_time](#osd_out_time)
 - [placement_levels](#placement_levels)
 - [use_old_pg_combinator](#use_old_pg_combinator)
+- [osd_backfillfull_ratio](#osd_backfillfull_ratio)
 
 ## use_antietcd
 
@@ -178,3 +179,19 @@ OSD перед обновлением агрегированной статис
 
 Использовать старый генератор комбинаций PG, не поддерживающий [level_placement](pool.ru.md#level_placement)
 и [raw_placement](pool.ru.md#raw_placement) для пулов, которые не используют данные функции.
+
+## osd_backfillfull_ratio
+
+- Тип: число
+- Значение по умолчанию: 0.99
+
+Мониторы стараются предотвратить 100% заполнение OSD в процессе ребаланса
+или восстановления, рассчитывая, сколько места будет занято на каждом OSD после
+завершения всех операций ребаланса и восстановления, и приостанавливая
+ребаланс и восстановление, если рассчитанный объём превышает ёмкость OSD,
+умноженную на значение данного параметра.
+
+Будущее занятое место рассчитывается сложением места, занятого всеми
+пользовательскими блоками данных (объектами) во всех PG, расположенных
+на конкретном OSD, даже если часть этих объектов в данный момент находится
+на другом наборе OSD.

docs/config/network.en.md

@@ -68,11 +68,17 @@ but they are not connected to the cluster.
 - Type: string
 
 RDMA device name to use for Vitastor OSD communications (for example,
-"rocep5s0f0"). Now Vitastor supports all adapters, even ones without
-ODP support, like Mellanox ConnectX-3 and non-Mellanox cards.
+"rocep5s0f0"). If not specified, Vitastor will try to find an RoCE
+device matching [osd_network](osd.en.md#osd_network), preferring RoCEv2,
+or choose the first available RDMA device if no RoCE devices are
+found or if `osd_network` is not specified. Auto-selection is also
+unsupported with old libibverbs < v32, like in Debian 10 Buster or
+CentOS 7.
 
-Versions up to Vitastor 1.2.0 required ODP which is only present in
-Mellanox ConnectX >= 4. See also [rdma_odp](#rdma_odp).
+Vitastor supports all adapters, even ones without ODP support, like
+Mellanox ConnectX-3 and non-Mellanox cards. Versions up to Vitastor
+1.2.0 required ODP which is only present in Mellanox ConnectX >= 4.
+See also [rdma_odp](#rdma_odp).
 
 Run `ibv_devinfo -v` as root to list available RDMA devices and their
 features.
@@ -95,15 +101,17 @@ your device has.
 ## rdma_gid_index
 
 - Type: integer
-- Default: 0
 
 Global address identifier index of the RDMA device to use. Different GID
 indexes may correspond to different protocols like RoCEv1, RoCEv2 and iWARP.
 Search for "GID" in `ibv_devinfo -v` output to determine which GID index
 you need.
 
-**IMPORTANT:** If you want to use RoCEv2 (as recommended) then the correct
-rdma_gid_index is usually 1 (IPv6) or 3 (IPv4).
+If not specified, Vitastor will try to auto-select a RoCEv2 IPv4 GID, then
+RoCEv2 IPv6 GID, then RoCEv1 IPv4 GID, then RoCEv1 IPv6 GID, then IB GID.
+GID auto-selection is unsupported with libibverbs < v32.
+
+A correct rdma_gid_index for RoCEv2 is usually 1 (IPv6) or 3 (IPv4).
 
 ## rdma_mtu
 

docs/config/network.ru.md

@@ -71,12 +71,17 @@ RDMA может быть нужно только если у клиентов е
 - Тип: строка
 
 Название RDMA-устройства для связи с Vitastor OSD (например, "rocep5s0f0").
-Сейчас Vitastor поддерживает все модели адаптеров, включая те, у которых
-нет поддержки ODP, то есть вы можете использовать RDMA с ConnectX-3 и
-картами производства не Mellanox.
+Если не указано, Vitastor попробует найти RoCE-устройство, соответствующее
+[osd_network](osd.en.md#osd_network), предпочитая RoCEv2, или выбрать первое
+попавшееся RDMA-устройство, если RoCE-устройств нет или если сеть `osd_network`
+не задана. Также автовыбор не поддерживается со старыми версиями библиотеки
+libibverbs < v32, например в Debian 10 Buster или CentOS 7.
 
-Версии Vitastor до 1.2.0 включительно требовали ODP, который есть только
-на Mellanox ConnectX 4 и более новых. См. также [rdma_odp](#rdma_odp).
+Vitastor поддерживает все модели адаптеров, включая те, у которых
+нет поддержки ODP, то есть вы можете использовать RDMA с ConnectX-3 и
+картами производства не Mellanox. Версии Vitastor до 1.2.0 включительно
+требовали ODP, который есть только на Mellanox ConnectX 4 и более новых.
+См. также [rdma_odp](#rdma_odp).
 
 Запустите `ibv_devinfo -v` от имени суперпользователя, чтобы посмотреть
 список доступных RDMA-устройств, их параметры и возможности.
@@ -101,15 +106,18 @@ Control) и ECN (Explicit Congestion Notification).
 ## rdma_gid_index
 
 - Тип: целое число
-- Значение по умолчанию: 0
 
 Номер глобального идентификатора адреса RDMA-устройства, который следует
 использовать. Разным gid_index могут соответствовать разные протоколы связи:
 RoCEv1, RoCEv2, iWARP. Чтобы понять, какой нужен вам - смотрите строчки со
 словом "GID" в выводе команды `ibv_devinfo -v`.
 
-**ВАЖНО:** Если вы хотите использовать RoCEv2 (как мы и рекомендуем), то
-правильный rdma_gid_index, как правило, 1 (IPv6) или 3 (IPv4).
+Если не указан, Vitastor попробует автоматически выбрать сначала GID,
+соответствующий RoCEv2 IPv4, потом RoCEv2 IPv6, потом RoCEv1 IPv4, потом
+RoCEv1 IPv6, потом IB. Авто-выбор GID не поддерживается со старыми версиями
+libibverbs < v32.
+
+Правильный rdma_gid_index для RoCEv2, как правило, 1 (IPv6) или 3 (IPv4).
 
 ## rdma_mtu
 

docs/config/src/client.yml

@@ -61,6 +61,24 @@
   info_ru: |
     Повторять запросы записи, завершившиеся с ошибками нехватки места, т.е.
     ожидать, пока на OSD не освободится место.
+- name: client_wait_up_timeout
+  type: sec
+  default: 16
+  online: true
+  info: |
+    Wait for this number of seconds until PGs are up when doing operations
+    which require all PGs to be up. Currently only used by object listings
+    in delete and merge-based commands ([vitastor-cli rm](../usage/cli.en.md#rm), merge and so on).
+
+    The default value is calculated as `1 + OSD lease timeout`, which is
+    `1 + etcd_report_interval + max_etcd_attempts*2*etcd_quick_timeout`.
+  info_ru: |
+    Время ожидания поднятия PG при операциях, требующих активности всех PG.
+    В данный момент используется листингами объектов в командах, использующих
+    удаление и слияние ([vitastor-cli rm](../usage/cli.ru.md#rm), merge и подобные).
+
+    Значение по умолчанию вычисляется как `1 + время lease OSD`, равное
+    `1 + etcd_report_interval + max_etcd_attempts*2*etcd_quick_timeout`.
 - name: client_max_dirty_bytes
   type: int
   default: 33554432

docs/config/src/monitor.yml

@@ -172,3 +172,27 @@
   info_ru: |
     Использовать старый генератор комбинаций PG, не поддерживающий [level_placement](pool.ru.md#level_placement)
     и [raw_placement](pool.ru.md#raw_placement) для пулов, которые не используют данные функции.
+- name: osd_backfillfull_ratio
+  type: float
+  default: 0.99
+  info: |
+    Monitors try to prevent OSDs becoming 100% full during rebalance or recovery by
+    calculating how much space will be occupied on every OSD after all rebalance
+    and recovery operations finish, and pausing rebalance and recovery if that
+    amount of space exceeds OSD capacity multiplied by the value of this
+    configuration parameter.
+
+    Future used space is calculated by summing space used by all user data blocks
+    (objects) in all PGs placed on a specific OSD, even if some of these objects
+    currently reside on a different set of OSDs.
+  info_ru: |
+    Мониторы стараются предотвратить 100% заполнение OSD в процессе ребаланса
+    или восстановления, рассчитывая, сколько места будет занято на каждом OSD после
+    завершения всех операций ребаланса и восстановления, и приостанавливая
+    ребаланс и восстановление, если рассчитанный объём превышает ёмкость OSD,
+    умноженную на значение данного параметра.
+
+    Будущее занятое место рассчитывается сложением места, занятого всеми
+    пользовательскими блоками данных (объектами) во всех PG, расположенных
+    на конкретном OSD, даже если часть этих объектов в данный момент находится
+    на другом наборе OSD.

docs/config/src/network.yml

@@ -48,11 +48,17 @@
   type: string
   info: |
     RDMA device name to use for Vitastor OSD communications (for example,
-    "rocep5s0f0"). Now Vitastor supports all adapters, even ones without
-    ODP support, like Mellanox ConnectX-3 and non-Mellanox cards.
+    "rocep5s0f0"). If not specified, Vitastor will try to find an RoCE
+    device matching [osd_network](osd.en.md#osd_network), preferring RoCEv2,
+    or choose the first available RDMA device if no RoCE devices are
+    found or if `osd_network` is not specified. Auto-selection is also
+    unsupported with old libibverbs < v32, like in Debian 10 Buster or
+    CentOS 7.
 
-    Versions up to Vitastor 1.2.0 required ODP which is only present in
-    Mellanox ConnectX >= 4. See also [rdma_odp](#rdma_odp).
+    Vitastor supports all adapters, even ones without ODP support, like
+    Mellanox ConnectX-3 and non-Mellanox cards. Versions up to Vitastor
+    1.2.0 required ODP which is only present in Mellanox ConnectX >= 4.
+    See also [rdma_odp](#rdma_odp).
 
     Run `ibv_devinfo -v` as root to list available RDMA devices and their
     features.
@@ -64,12 +70,17 @@
     PFC (Priority Flow Control) and ECN (Explicit Congestion Notification).
   info_ru: |
     Название RDMA-устройства для связи с Vitastor OSD (например, "rocep5s0f0").
-    Сейчас Vitastor поддерживает все модели адаптеров, включая те, у которых
-    нет поддержки ODP, то есть вы можете использовать RDMA с ConnectX-3 и
-    картами производства не Mellanox.
+    Если не указано, Vitastor попробует найти RoCE-устройство, соответствующее
+    [osd_network](osd.en.md#osd_network), предпочитая RoCEv2, или выбрать первое
+    попавшееся RDMA-устройство, если RoCE-устройств нет или если сеть `osd_network`
+    не задана. Также автовыбор не поддерживается со старыми версиями библиотеки
+    libibverbs < v32, например в Debian 10 Buster или CentOS 7.
 
-    Версии Vitastor до 1.2.0 включительно требовали ODP, который есть только
-    на Mellanox ConnectX 4 и более новых. См. также [rdma_odp](#rdma_odp).
+    Vitastor поддерживает все модели адаптеров, включая те, у которых
+    нет поддержки ODP, то есть вы можете использовать RDMA с ConnectX-3 и
+    картами производства не Mellanox. Версии Vitastor до 1.2.0 включительно
+    требовали ODP, который есть только на Mellanox ConnectX 4 и более новых.
+    См. также [rdma_odp](#rdma_odp).
 
     Запустите `ibv_devinfo -v` от имени суперпользователя, чтобы посмотреть
     список доступных RDMA-устройств, их параметры и возможности.
@@ -94,23 +105,29 @@
     `ibv_devinfo -v`.
 - name: rdma_gid_index
   type: int
-  default: 0
   info: |
     Global address identifier index of the RDMA device to use. Different GID
     indexes may correspond to different protocols like RoCEv1, RoCEv2 and iWARP.
     Search for "GID" in `ibv_devinfo -v` output to determine which GID index
     you need.
 
-    **IMPORTANT:** If you want to use RoCEv2 (as recommended) then the correct
-    rdma_gid_index is usually 1 (IPv6) or 3 (IPv4).
+    If not specified, Vitastor will try to auto-select a RoCEv2 IPv4 GID, then
+    RoCEv2 IPv6 GID, then RoCEv1 IPv4 GID, then RoCEv1 IPv6 GID, then IB GID.
+    GID auto-selection is unsupported with libibverbs < v32.
+
+    A correct rdma_gid_index for RoCEv2 is usually 1 (IPv6) or 3 (IPv4).
   info_ru: |
     Номер глобального идентификатора адреса RDMA-устройства, который следует
     использовать. Разным gid_index могут соответствовать разные протоколы связи:
     RoCEv1, RoCEv2, iWARP. Чтобы понять, какой нужен вам - смотрите строчки со
     словом "GID" в выводе команды `ibv_devinfo -v`.
 
-    **ВАЖНО:** Если вы хотите использовать RoCEv2 (как мы и рекомендуем), то
-    правильный rdma_gid_index, как правило, 1 (IPv6) или 3 (IPv4).
+    Если не указан, Vitastor попробует автоматически выбрать сначала GID,
+    соответствующий RoCEv2 IPv4, потом RoCEv2 IPv6, потом RoCEv1 IPv4, потом
+    RoCEv1 IPv6, потом IB. Авто-выбор GID не поддерживается со старыми версиями
+    libibverbs < v32.
+
+    Правильный rdma_gid_index для RoCEv2, как правило, 1 (IPv6) или 3 (IPv4).
 - name: rdma_mtu
   type: int
   default: 4096

docs/installation/kubernetes.en.md

@@ -6,9 +6,18 @@
 
 # Kubernetes CSI
 
-Vitastor has a CSI plugin for Kubernetes which supports RWO (and block RWX) volumes.
+Vitastor has a CSI plugin for Kubernetes which supports block-based and VitastorFS-based volumes.
 
-To deploy it, take manifests from [csi/deploy/](../../csi/deploy/) directory, put your
+Block-based volumes may be formatted and mounted with a normal FS (ext4 or xfs). Such volumes
+only support RWO (ReadWriteOnce) mode.
+
+Block-based volumes may also be left without FS and attached into the container as a block
+device. Such volumes also support RWX (ReadWriteMany) mode.
+
+VitastorFS-based volumes use a clustered file system and support FS-based RWX (ReadWriteMany)
+mode. However, such volumes don't support quotas and snapshots.
+
+To deploy the CSI plugin, take manifests from [csi/deploy/](../../csi/deploy/) directory, put your
 Vitastor configuration in [001-csi-config-map.yaml](../../csi/deploy/001-csi-config-map.yaml),
 configure storage class in [009-storage-class.yaml](../../csi/deploy/009-storage-class.yaml)
 and apply all `NNN-*.yaml` manifests to your Kubernetes installation:
@@ -23,16 +32,16 @@ After that you'll be able to create PersistentVolumes.
 kernel modules enabled (vdpa, vduse, virtio-vdpa). If your distribution doesn't
 have them pre-built - build them yourself ([instructions](../usage/qemu.en.md#vduse)),
 I promise it's worth it :-). When VDUSE is unavailable, CSI driver uses [NBD](../usage/nbd.en.md)
-to map Vitastor devices. NBD is slower and prone to timeout issues: if Vitastor
-cluster becomes unresponsible for more than [nbd_timeout](../config/client.en.md#nbd_timeout),
-the NBD device detaches and breaks pods using it.
+to map Vitastor devices. NBD is slower and, with kernels older than 5.19, unmountable
+if the cluster becomes unresponsible.
 
 ## Features
 
 Vitastor CSI supports:
 - Kubernetes starting with 1.20 (or 1.17 for older vitastor-csi <= 1.1.0)
-- Filesystem RWO (ReadWriteOnce) volumes. Example: [PVC](../../csi/deploy/example-pvc.yaml), [pod](../../csi/deploy/example-test-pod.yaml)
+- Block-based FS-formatted RWO (ReadWriteOnce) volumes. Example: [PVC](../../csi/deploy/example-pvc.yaml), [pod](../../csi/deploy/example-test-pod.yaml)
 - Raw block RWX (ReadWriteMany) volumes. Example: [PVC](../../csi/deploy/example-pvc-block.yaml), [pod](../../csi/deploy/example-test-pod-block.yaml)
+- VitastorFS-based volumes RWX (ReadWriteMany) volumes. Example: [storage class](../../csi/deploy/example-storage-class-fs.yaml)
 - Volume expansion
 - Volume snapshots. Example: [snapshot class](../../csi/deploy/example-snapshot-class.yaml), [snapshot](../../csi/deploy/example-snapshot.yaml), [clone](../../csi/deploy/example-snapshot-clone.yaml)
 - [VDUSE](../usage/qemu.en.md#vduse) (preferred) and [NBD](../usage/nbd.en.md) device mapping methods

docs/installation/kubernetes.ru.md

@@ -6,7 +6,17 @@
 
 # Kubernetes CSI
 
-У Vitastor есть CSI-плагин для Kubernetes, поддерживающий RWO, а также блочные RWX, тома.
+У Vitastor есть CSI-плагин для Kubernetes, поддерживающий блочные тома и тома на основе
+кластерной ФС VitastorFS.
+
+Блочные тома могут быть отформатированы и примонтированы со стандартной ФС (ext4 или xfs).
+Такие тома поддерживают только режим RWO (ReadWriteOnce, одновременный доступ с одного узла).
+
+Блочные тома также могут не форматироваться и подключаться в контейнер в виде блочного устройства.
+В таком случае их можно подключать в режиме RWX (ReadWriteMany, одновременный доступ с многих узлов).
+
+Тома на основе VitastorFS используют кластерную ФС и поэтому также поддерживают режим RWX
+(ReadWriteMany). Однако, такие тома не поддерживают ограничение размера и снимки.
 
 Для установки возьмите манифесты из директории [csi/deploy/](../../csi/deploy/), поместите
 вашу конфигурацию подключения к Vitastor в [csi/deploy/001-csi-config-map.yaml](../../csi/deploy/001-csi-config-map.yaml),
@@ -33,6 +43,7 @@ CSI-плагин Vitastor поддерживает:
 - Версии Kubernetes, начиная с 1.20 (или с 1.17 для более старых vitastor-csi <= 1.1.0)
 - Файловые RWO (ReadWriteOnce) тома. Пример: [PVC](../../csi/deploy/example-pvc.yaml), [под](../../csi/deploy/example-test-pod.yaml)
 - Сырые блочные RWX (ReadWriteMany) тома. Пример: [PVC](../../csi/deploy/example-pvc-block.yaml), [под](../../csi/deploy/example-test-pod-block.yaml)
+- Основанные на VitastorFS RWX (ReadWriteMany) тома. Пример: [класс хранения](../../csi/deploy/example-storage-class-fs.yaml)
 - Расширение размера томов
 - Снимки томов. Пример: [класс снимков](../../csi/deploy/example-snapshot-class.yaml), [снимок](../../csi/deploy/example-snapshot.yaml), [клон снимка](../../csi/deploy/example-snapshot-clone.yaml)
 - Способы подключения устройств [VDUSE](../usage/qemu.ru.md#vduse) (предпочитаемый) и [NBD](../usage/nbd.ru.md)

docs/intro/architecture.en.md

@@ -6,7 +6,12 @@
 
 # Architecture
 
+- [Server-side components](#server-side-components)
 - [Basic concepts](#basic-concepts)
+- [Client-side components](#client-side-components)
+- [Additional utilities](#additional-utilities)
+- [Overall read/write process](#overall-read-write-process)
+  - [Nuances of request handling](#nuances-of-request-handling)
 - [Similarities to Ceph](#similarities-to-ceph)
 - [Differences from Ceph](#differences-from-ceph)
 - [Implementation Principles](#implementation-principles)
@@ -39,10 +44,10 @@
 
 ## Client-side components
 
-- **Client library** incapsulates client I/O logic. Client library connects to etcd and to all OSDs,
+- **Client library** encapsulates client I/O logic. Client library connects to etcd and to all OSDs,
   receives cluster state from etcd, sends read and write requests directly to all OSDs. Due
   to the symmetric distributed architecture, all data blocks (each 128 KB by default) are placed
-  to different OSDs, but clients always knows where each data block is stored and connects directly
+  to different OSDs, but clients always know where each data block is stored and connect directly
   to the right OSD.
 
 All other client-side components are based on the client library:
@@ -60,8 +65,9 @@ All other client-side components are based on the client library:
   (at least by now). NBD is an older, non-recommended way to attach disks — you should use
   VDUSE whenever you can.
 - **[CSI driver](../installation/kubernetes.en.md)** — driver for attaching Vitastor images
-  as Kubernetes persistent volumes. Works through VDUSE (when available) or NBD — images are
-  attached as kernel block devices and mounted into containers.
+  and VitastorFS subdirectories as Kubernetes persistent volumes. Block-based CSI uses
+  VDUSE (when available) or NBD — images are attached as kernel block devices and mounted
+  into containers. FS-based CSI uses **[vitastor-nfs](../usage/nfs.en.md)**.
 - **Drivers for Proxmox, OpenStack and so on** — pluggable modules for corresponding systems,
   allowing to use Vitastor as storage in them.
 - **[vitastor-nfs](../usage/nfs.en.md)** — NFS 3.0 server allowing export of two file system variants:
@@ -77,8 +83,8 @@ All other client-side components are based on the client library:
 
 ## Additional utilities
 
-- **vitastor-disk** — утилита для разметки дисков под Vitastor OSD. С её помощью можно
-  создавать, удалять, менять размеры или перемещать разделы OSD.
+- **vitastor-disk** — a Vitastor OSD disk management tool. You can create, remove,
+  resize and move OSD partitions with it.
 
 ## Overall read/write process
 
@@ -213,5 +219,5 @@ All other client-side components are based on the client library:
 - Deleting images in a degraded cluster may currently lead to objects reappearing
   after dead OSDs come back, and in case of erasure-coded pools, they may even
   reappear as incomplete. Just repeat the removal request again in this case.
-  This problem will be fixed in the nearest future, the fix is already implemented
-  in the "epoch-deletions" branch.
+  This problem will be fixed in the future, along with the metadata disk storage
+  format update.

docs/intro/architecture.ru.md

@@ -24,8 +24,8 @@
   Один OSD управляет одним диском (или разделом). OSD общаются с etcd и друг с другом — от etcd они
   получают состояние кластера, а друг другу передают запросы записи и чтения вторичных копий данных.
 - **etcd** — кластерная key/value база данных, используется для хранения настроек и верхнеуровневого
-  состояния кластера, а также предотвращения разделения сознания. Блоки данных в etcd не хранятся,
-  в обработке клиентских запросов чтения и записи etcd не участвует.
+  состояния кластера, а также предотвращения разделения сознания (splitbrain). Блоки данных в etcd не
+  хранятся, в обработке клиентских запросов чтения и записи etcd не участвует.
 - **Монитор** — отдельный демон на node.js, рассчитывающий необходимые изменения в конфигурацию
   кластера, сохраняющий эту информацию в etcd и таким образом командующий OSD применить эти изменения.
   Также агрегирует статистику. Контактирует только с etcd, OSD с монитором не общаются.
@@ -43,8 +43,8 @@
 
 ## Клиентские компоненты
 
-- **Клиентская библиотека** — инкапсулирует логику на стороне клиента. Соединяются с etcd и со всеми OSD,
-  от etcd получают состояние кластера, команды чтения и записи отправляют на все OSD напрямую.
+- **Клиентская библиотека** — инкапсулирует логику на стороне клиента. Соединяется с etcd и со всеми OSD,
+  от etcd получает состояние кластера, команды чтения и записи отправляет на все OSD напрямую.
   В силу архитектуры все отдельные блоки данных (по умолчанию по 128 КБ) располагается на разных
   OSD, но клиент устроен так, что всегда точно знает, к какому OSD обращаться, и подключается
   к нему напрямую.
@@ -65,8 +65,9 @@
   Vitastor нет (по крайней мере, пока). NBD — более старый и нерекомендуемый способ подключения
   дисков — вам следует использовать VDUSE всегда, когда это возможно.
 - **[CSI драйвер](../installation/kubernetes.ru.md)** — драйвер для подключения Vitastor-образов
-  в виде персистентных томов (PV) Kubernetes. Работает через VDUSE (если доступно) или через
-  NBD — образы отражаются в виде блочных устройств и монтируются в контейнеры.
+  и поддиректорий VitastorFS в виде персистентных томов (PV) Kubernetes. Блочный CSI работает через
+  VDUSE (когда это возможно) или через NBD — образы отражаются в виде блочных устройств и монтируются
+  в контейнеры. Файловый CSI использует **[vitastor-nfs](../usage/nfs.ru.md)**.
 - **Драйвера Proxmox, OpenStack и т.п.** — подключаемые модули для соответствующих систем,
   позволяющие использовать Vitastor как хранилище в оных.
 - **[vitastor-nfs](../usage/nfs.ru.md)** — NFS 3.0 сервер, предоставляющий два варианта файловой системы:
@@ -227,5 +228,5 @@
 - Удаление образов в деградированном кластере может в данный момент приводить к повторному
   "появлению" удалённых объектов после поднятия отключённых OSD, причём в случае EC-пулов,
   объекты могут появиться в виде "неполных". Если вы столкнётесь с такой ситуацией, просто
-  повторите запрос удаления. Исправление этой проблемы уже реализовано в ветке "epoch-deletions"
-  и вскоре будет включено в релиз.
+  повторите запрос удаления. Данная проблема будет исправлена в будущем вместе с обновлением
+  дискового формата хранения метаданных.

docs/intro/features.en.md

@@ -36,6 +36,7 @@
 - [Clustered file system](../usage/nfs.en.md#vitastorfs)
 - [Experimental internal etcd replacement - antietcd](../config/monitor.en.md#use_antietcd)
 - [Built-in Prometheus metric exporter](../config/monitor.en.md#enable_prometheus)
+- [NFS RDMA support](../usage/nfs.en.md#rdma) (probably also usable for GPUDirect)
 
 ## Plugins and tools
 

docs/intro/features.ru.md

@@ -38,6 +38,7 @@
 - [Кластерная файловая система](../usage/nfs.ru.md#vitastorfs)
 - [Экспериментальная встроенная замена etcd - antietcd](../config/monitor.ru.md#use_antietcd)
 - [Встроенный Prometheus-экспортер метрик](../config/monitor.ru.md#enable_prometheus)
+- [Поддержка NFS RDMA](../usage/nfs.ru.md#rdma) (вероятно, также подходящая для GPUDirect)
 
 ## Драйверы и инструменты
 

docs/usage/admin.en.md

@@ -58,8 +58,9 @@ and during switching primary OSD of PGs.
 
 **starting**, **repeering**, **stopping** states normally almost aren't visible at all.
 If you notice them for any noticeable time — chances are some operations on some OSDs hung.
-Search for "slow op" in OSD logs to find them — operations hung for more than
-[slow_log_interval](../config/osd.en.md#slow_log_interval) are logged as "slow ops".
+Check `vitastor-cli status` and search for "slow op" in OSD logs to find them — operations
+hung for more than [slow_log_interval](../config/osd.en.md#slow_log_interval) are logged as
+"slow ops" and displayed in `status`.
 
 State transition diagram:
 
@@ -171,7 +172,14 @@ to make them use the new version of the client library.
 
 ### 1.7.x to 1.8.0
 
-After upgrading version <= 1.7.x to version >= 1.8.0, BUT <= 1.9.0: restart all clients
+It's recommended to upgrade from version <= 1.7.x to version >= 1.8.0 with full downtime,
+i.e. you should first stop clients and then the cluster (OSDs and monitor), because 1.8.0
+includes a fix for etcd event stream inconsistency which could lead to "incomplete" objects
+appearing in EC pools, and in rare cases, probably, even to data corruption during mass OSD
+restarts. It doesn't mean that you WILL hit this problem if you upgrade without full downtime,
+but it's better to secure yourself against it.
+
+Also, if you upgrade version from <= 1.7.x to version >= 1.8.0, BUT <= 1.9.0: restart all clients
 (VMs and so on), otherwise they will hang when monitor clears old PG configuration key,
 which happens 24 hours after upgrade.
 

docs/usage/admin.ru.md

@@ -56,9 +56,9 @@ OSD, на протяжении небольшого периода времен
 Состояния **starting**, **repeering**, **stopping** в норме практически не заметны вообще,
 PG должны очень быстро переходить из них в другие. Если эти состояния заметны
 хоть сколько-то значительное время — вероятно, какие-то операции на каких-то OSD зависли.
-Чтобы найти их, ищите "slow op" в журналах OSD — операции, зависшие дольше,
-чем на [slow_log_interval](../config/osd.ru.md#slow_log_interval), записываются в
-журналы OSD как "slow op".
+Чтобы найти их, посморите `vitastor-cli status` и поищите слова "slow op" в журналах OSD —
+операции, зависшие дольше, чем на [slow_log_interval](../config/osd.ru.md#slow_log_interval),
+записываются в журналы OSD как "slow op" и отображаются в `status`.
 
 Диаграмма переходов:
 
@@ -168,7 +168,14 @@ done
 
 ### 1.7.x -> 1.8.0
 
-После обновления с версий <= 1.7.x до версий >= 1.8.0, НО <= 1.9.0: перезапустите всех
+Обновляться с версий <= 1.7.x до версий >= 1.8.0 рекомендуется с полной остановкой
+сначала клиентов, а затем кластера, так как в 1.8.0 исправлена проблема (неконсистентность
+потоков событий от etcd), способная приводить к появлению incomplete объектов в EC-пулах
+и, хоть и редко, но даже к повреждению данных при массовых перезапусках OSD. Если вы
+обновляетесь без полной остановки - это не значит, что вы обязательно столкнётесь с этой
+проблемой, но лучше подстраховаться.
+
+Также, если вы обновляетесь с версии <= 1.7.x до версии >= 1.8.0, НО <= 1.9.0: перезапустите всех
 клиентов (процессы виртуальных машин можно перезапустить путём миграции на другой сервер),
 иначе они зависнут, когда монитор удалит старый ключ конфигурации PG, что происходит через
 24 часа после обновления.

docs/usage/cli.en.md

@@ -146,6 +146,7 @@ Rename, resize image or change its readonly status. Images with children can't b
 If the new size is smaller than the old size, extra data will be purged.
 You should resize file system in the image, if present, before shrinking it.
 
+* `--deleted 1|0` - Set/clear 'deleted image' flag (set automatically during unfinished deletes).
 * `-f|--force` - Proceed with shrinking or setting readwrite flag even if the image has children.
 * `--down-ok` - Proceed with shrinking even if some data will be left on unavailable OSDs.
 
@@ -221,6 +222,7 @@ Remove inode data without changing metadata.
               Requires more memory, but allows to show correct removal progress.
 --min-offset  Purge only data starting with specified offset.
 --max-offset  Purge only data before specified offset.
+--client_wait_up_timeout 16  Timeout for waiting until PGs are up in seconds.
 ```
 
 ## merge-data

docs/usage/cli.ru.md

@@ -149,6 +149,7 @@ vitastor-cli snap-create [-p|--pool <id|name>] <image>@<snapshot>
 Если новый размер меньше старого, "лишние" данные будут удалены, поэтому перед уменьшением
 образа сначала уменьшите файловую систему в нём.
 
+* `--deleted 1|0` - Установить/снять флаг "образ удалён" (устанавливается при незавершённом удалении).
 * `-f|--force` - Разрешить уменьшение или перевод в чтение-запись образа, у которого есть клоны.
 * `--down-ok` - Разрешить уменьшение, даже если часть данных останется неудалённой на недоступных OSD.
 
@@ -226,6 +227,7 @@ vitastor-cli dd [iimg=<image> | if=<file>] [oimg=<image> | of=<file>] [bs=1M] \
               Требует больше памяти, но позволяет правильно печатать прогресс удаления.
 --min-offset  Удалять только данные, начиная с заданного смещения.
 --max-offset  Удалять только данные до (исключительно) заданного смещения.
+--client_wait_up_timeout 16  Время ожидания поднятия PG в секундах.
 ```
 
 ## merge-data

docs/usage/nbd.en.md

@@ -96,7 +96,7 @@ Example output (JSON format):
 vitastor-nbd netlink-map [/dev/nbdN] (--image <image> | --pool <pool> --inode <inode> --size <size in bytes>)
 ```
 
-On recent kernel versions it's also possinle to map NBD devices using netlink interface.
+On recent kernel versions it's also possible to map NBD devices using netlink interface.
 
 This is an experimental feature because it doesn't solve all issues of NBD. Differences from regular ioctl-based 'map':
 

docs/usage/nfs.en.md

@@ -111,6 +111,21 @@ settings, because Vitastor NFS proxy doesn't keep uncommitted data in memory
 with these settings. But it may even work without `immediate_commit=all` because
 the Linux NFS client repeats all uncommitted writes if it loses the connection.
 
+## RDMA
+
+vitastor-nfs supports NFS over RDMA, which, in theory, should also allow to use
+VitastorFS from GPUDirect.
+
+You can test NFS-RDMA even if you don't have an RDMA NIC using SoftROCE:
+
+1. First, add SoftROCE device on both servers: `rdma link add rxe0 type rxe netdev eth0`.
+   Here, `rdma` utility is a part the iproute2 package, and `eth0` should be replaced with
+   the name of your Ethernet NIC.
+
+2. Start vitastor-nfs with RDMA: `vitastor-nfs start (--fs <NAME> | --block) --pool <POOL> --port 20049 --nfs_rdma 20049 --portmap 0`
+
+3. Mount the FS: `mount 192.168.0.10:/mnt/test/ /mnt/vita/ -o port=20049,mountport=20049,nfsvers=3,soft,nolock,rdma`
+
 ## Commands
 
 ### mount
@@ -131,11 +146,16 @@ The server will be automatically stopped when the FS is unmounted.
 
 Start network NFS server. Options:
 
-| <!-- -->        | <!-- -->                                                   |
-|-----------------|------------------------------------------------------------|
-| `--bind <IP>`   | bind service to \<IP> address (default 0.0.0.0)            |
-| `--port <PORT>` | use port \<PORT> for NFS services (default is 2049)        |
-| `--portmap 0`   | do not listen on port 111 (portmap/rpcbind, requires root) |
+| <!-- -->               | <!-- -->                                                                                                                    |
+|------------------------|-----------------------------------------------------------------------------------------------------------------------------|
+| `--bind <IP>`          | bind service to \<IP> address (default 0.0.0.0)                                                                             |
+| `--port <PORT>`        | use port \<PORT> for NFS services (default is 2049). Specify "auto" to auto-select and print port                           |
+| `--portmap 0`          | do not listen on port 111 (portmap/rpcbind, requires root)                                                                  |
+| `--nfs_rdma <PORT>`    | enable NFS-RDMA at RDMA-CM port \<PORT> (you can try 20049). If RDMA is enabled and --port is set to 0, TCP will be disabled |
+| `--nfs_rdma_credit 16` | maximum operation credit for RDMA clients (max iodepth)                                                                     |
+| `--nfs_rdma_send 1024` | maximum RDMA send operation count (should be larger than iodepth)                                                           |
+| `--nfs_rdma_alloc 1M`  | RDMA memory allocation rounding                                                                                             |
+| `--nfs_rdma_gc 64M`    | maximum unused RDMA buffers                                                                                                 |
 
 ### upgrade
 

docs/usage/nfs.ru.md

@@ -116,6 +116,21 @@ JSON-формате :-). Для инспекции содержимого БД
 даже без `immediate_commit=all`, потому что NFS-клиент ядра Linux повторяет все
 незафиксированные запросы при потере соединения.
 
+## RDMA
+
+vitastor-nfs поддерживает NFS через RDMA. В теории это также должно позволять использовать
+VitastorFS из GPUDirect.
+
+Вы можете протестировать NFS-RDMA, даже если у вас нет RDMA-карты, с помощью SoftROCE:
+
+1. Сначала создайте SoftROCE устройства на обоих тестовых серверах: `rdma link add rxe0 type rxe netdev eth0`.
+   Утилита `rdma` входит в состав пакета iproute2, а `eth0` вам нужно заменить на имя своей
+   сетевой карты.
+
+2. Запустите vitastor-nfs с RDMA: `vitastor-nfs start (--fs <NAME> | --block) --pool <POOL> --port 20049 --nfs_rdma 20049 --portmap 0`
+
+3. Смонтируйте ФС: `mount 192.168.0.10:/mnt/test/ /mnt/vita/ -o port=20049,mountport=20049,nfsvers=3,soft,nolock,rdma`
+
 ## Команды
 
 ### mount
@@ -136,11 +151,16 @@ JSON-формате :-). Для инспекции содержимого БД
 
 Запустить сетевой NFS-сервер. Опции:
 
-| <!-- -->        | <!-- -->                                                              |
-|-----------------|-----------------------------------------------------------------------|
-| `--bind <IP>`   | принимать соединения по адресу \<IP> (по умолчанию 0.0.0.0 - на всех) |
-| `--port <PORT>` | использовать порт \<PORT> для NFS-сервисов (по умолчанию 2049)        |
-| `--portmap 0`   | отключить сервис portmap/rpcbind на порту 111 (по умолчанию включён и требует root привилегий) |
+| <!-- -->               | <!-- -->                                                                                                                    |
+|------------------------|-----------------------------------------------------------------------------------------------------------------------------|
+| `--bind <IP>`          | принимать соединения по адресу \<IP> (по умолчанию 0.0.0.0 - на всех)                                                       |
+| `--port <PORT>`        | использовать порт \<PORT> для NFS-сервисов (по умолчанию 2049). Укажите "auto", чтобы выбрать и напечатать случайный порт   |
+| `--portmap 0`          | отключить сервис portmap/rpcbind на порту 111 (по умолчанию включён и требует root привилегий)                              |
+| `--nfs_rdma <PORT>`    | включить NFS-RDMA на порту RDMA-CM \<PORT> (попробуйте 20049). Если RDMA включено и указано `--port 0`, TCP будет отключено |
+| `--nfs_rdma_credit 16` | максимальный "кредит", глубина очереди для NFS-клиентов                                                                     |
+| `--nfs_rdma_send 1024` | максимальное число операций RDMA отправки (должно быть больше nfs_rdma_credit)                                              |
+| `--nfs_rdma_alloc 1M`  | округление выделения памяти для RDMA-клиентов                                                                               |
+| `--nfs_rdma_gc 64M`    | максимальный объём неиспользуемой памяти RDMA-клиентом перед освобождением                                                  |
 
 ### upgrade
 

mon/antietcd_adapter.js

@@ -23,6 +23,9 @@ class AntiEtcdAdapter
             }, {}));
             const cfg_port = config.antietcd_port;
             const is_local = local_ips(true).reduce((a, c) => { a[c] = true; return a; }, {});
+            is_local['0.0.0.0'] = true;
+            is_local['::'] = true;
+            is_local[''] = true;
             const selected = cluster.map(s => s.split(':', 2)).filter(ip => is_local[ip[0]] && (!cfg_port || ip[1] == cfg_port));
             if (selected.length > 1)
             {

mon/etcd_adapter.js

@@ -232,6 +232,7 @@ class EtcdAdapter
     async become_master()
     {
         const state = { ...this.mon.get_mon_state(), id: ''+this.mon.etcd_lease_id };
+        console.log('Waiting to become master');
         // eslint-disable-next-line no-constant-condition
         while (1)
         {
@@ -243,7 +244,6 @@ class EtcdAdapter
             {
                 break;
             }
-            console.log('Waiting to become master');
             await new Promise(ok => setTimeout(ok, this.mon.config.etcd_start_timeout));
         }
         console.log('Became master');

mon/etcd_schema.js

@@ -56,6 +56,7 @@ const etcd_tree = {
             osd_out_time: 600, // seconds. min: 0
             placement_levels: { datacenter: 1, rack: 2, host: 3, osd: 4, ... },
             use_old_pg_combinator: false,
+            osd_backfillfull_ratio: 0.99,
             // client and osd
             tcp_header_buffer_size: 65536,
             use_sync_send_recv: false,
@@ -215,6 +216,7 @@ const etcd_tree = {
                     parent_pool?: <pool_id>,
                     parent_id?: <inode_t>,
                     readonly?: boolean,
+                    deleted?: boolean,
                 }
             }
         }, */

mon/mon.js

@@ -74,6 +74,7 @@ class Mon
         this.state = JSON.parse(JSON.stringify(etcd_tree));
         this.prev_stats = { osd_stats: {}, osd_diff: {} };
         this.recheck_pgs_active = false;
+        this.updating_total_stats = false;
         this.watcher_active = false;
         this.old_pg_config = false;
         this.old_pg_stats_seen = false;
@@ -658,7 +659,19 @@ class Mon
                 this.etcd_watch_revision, pool_id, up_osds, osd_tree, real_prev_pgs, pool_res.pgs, pg_history);
         }
         new_pg_config.hash = tree_hash;
-        return await this.save_pg_config(new_pg_config, etcd_request);
+        const { backfillfull_pools, backfillfull_osds } = sum_object_counts(
+            { ...this.state, pg: { ...this.state.pg, config: new_pg_config } }, this.config
+        );
+        if (backfillfull_pools.join(',') != ((this.state.pg.config||{}).backfillfull_pools||[]).join(','))
+        {
+            this.log_backfillfull(backfillfull_osds, backfillfull_pools);
+        }
+        new_pg_config.backfillfull_pools = backfillfull_pools.length ? backfillfull_pools : undefined;
+        if (!await this.save_pg_config(new_pg_config, etcd_request))
+        {
+            return false;
+        }
+        return true;
     }
 
     async save_pg_config(new_pg_config, etcd_request = { compare: [], success: [] })
@@ -730,7 +743,7 @@ class Mon
     async update_total_stats()
     {
         const txn = [];
-        const { object_counts, object_bytes } = sum_object_counts(this.state, this.config);
+        const { object_counts, object_bytes, backfillfull_pools, backfillfull_osds } = sum_object_counts(this.state, this.config);
         let stats = sum_op_stats(this.state.osd, this.prev_stats);
         let { inode_stats, seen_pools } = sum_inode_stats(this.state, this.prev_stats);
         stats.object_counts = object_counts;
@@ -783,6 +796,27 @@ class Mon
         {
             await this.etcd.etcd_call('/kv/txn', { success: txn }, this.config.etcd_mon_timeout, 0);
         }
+        if (!this.recheck_pgs_active &&
+            backfillfull_pools.join(',') != ((this.state.pg.config||{}).backfillfull_pools||[]).join(','))
+        {
+            this.log_backfillfull(backfillfull_osds, backfillfull_pools);
+            const new_pg_config = { ...this.state.pg.config, backfillfull_pools: backfillfull_pools.length ? backfillfull_pools : undefined };
+            await this.save_pg_config(new_pg_config);
+        }
+    }
+
+    log_backfillfull(osds, pools)
+    {
+        for (const osd in osds)
+        {
+            const bf = osds[osd];
+            console.log('OSD '+osd+' may fill up during rebalance: capacity '+(bf.cap/1024n/1024n)+
+                ' MB, target user data '+(bf.clean/1024n/1024n)+' MB');
+        }
+        console.log(
+            (pools.length ? 'Pool(s) '+pools.join(', ') : 'No pools')+
+            ' are backfillfull now, applying rebalance configuration'
+        );
     }
 
     schedule_update_stats()
@@ -794,7 +828,21 @@ class Mon
         this.stats_timer = setTimeout(() =>
         {
             this.stats_timer = null;
-            this.update_total_stats().catch(console.error);
+            if (this.updating_total_stats)
+            {
+                this.schedule_update_stats();
+                return;
+            }
+            this.updating_total_stats = true;
+            try
+            {
+                this.update_total_stats().catch(console.error);
+            }
+            catch (e)
+            {
+                console.error(e);
+            }
+            this.updating_total_stats = false;
         }, this.config.mon_stats_timeout);
     }
 

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "1.9.3",
+  "version": "1.10.1",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {

mon/stats.js

@@ -109,6 +109,8 @@ function sum_object_counts(state, global_config)
             pgstats[pool_id] = { ...(state.pg.stats[pool_id] || {}), ...(pgstats[pool_id] || {}) };
         }
     }
+    const pool_per_osd = {};
+    const clean_per_osd = {};
     for (const pool_id in pgstats)
     {
         let object_size = 0;
@@ -143,10 +145,45 @@ function sum_object_counts(state, global_config)
                         object_bytes[k] += BigInt(st[k+'_count']) * object_size;
                     }
                 }
+                if (st.object_count)
+                {
+                    for (const pg_osd of (((state.pg.config.items||{})[pool_id]||{})[pg_num]||{}).osd_set||[])
+                    {
+                        if (!(pg_osd in clean_per_osd))
+                        {
+                            clean_per_osd[pg_osd] = 0n;
+                        }
+                        clean_per_osd[pg_osd] += BigInt(st.object_count);
+                        pool_per_osd[pg_osd] = pool_per_osd[pg_osd]||{};
+                        pool_per_osd[pg_osd][pool_id] = true;
+                    }
+                }
             }
         }
     }
-    return { object_counts, object_bytes };
+    // If clean_per_osd[osd] is larger than osd capacity then it will fill up during rebalance
+    let backfillfull_pools = {};
+    let backfillfull_osds = {};
+    for (const osd in clean_per_osd)
+    {
+        const st = state.osd.stats[osd];
+        if (!st || !st.size || !st.data_block_size)
+        {
+            continue;
+        }
+        let cap = BigInt(st.size)/BigInt(st.data_block_size);
+        cap = cap * BigInt((global_config.osd_backfillfull_ratio||0.99)*1000000) / 1000000n;
+        if (cap < clean_per_osd[osd])
+        {
+            backfillfull_osds[osd] = { cap: BigInt(st.size), clean: clean_per_osd[osd]*BigInt(st.data_block_size) };
+            for (const pool_id in pool_per_osd[osd])
+            {
+                backfillfull_pools[pool_id] = true;
+            }
+        }
+    }
+    backfillfull_pools = Object.keys(backfillfull_pools).sort();
+    return { object_counts, object_bytes, backfillfull_pools, backfillfull_osds };
 }
 
 // sum_inode_stats(this.state, this.prev_stats)

opennebula/remotes/tm/vitastor/prebackup

@@ -98,8 +98,8 @@ vm.elements.each 'TEMPLATE/DISK' do |d|
     cmd = 'vitastor-cli'
     qemu_arg = ''
     if d.elements['VITASTOR_CONF']
-        cmd = cmd + ' --config_path ' + d.elements['VITASTOR_CONF']
-        qemu_arg += 'config_path='+d.elements['VITASTOR_CONF']+':'
+        cmd = cmd + ' --config_path ' + d.elements['VITASTOR_CONF'].text
+        qemu_arg += 'config_path=' + d.elements['VITASTOR_CONF'].text + ':'
     end
 
     draw = "#{bck_dir}/disk.#{did}.raw"

patches/VitastorPlugin.pm

@@ -37,16 +37,6 @@ sub run_cli
     $json = 1 if !defined $json;
     my $binary = delete $args{binary};
     $binary = '/usr/bin/vitastor-cli' if !defined $binary;
-    if (!exists($args{errfunc}))
-    {
-        $args{errfunc} = sub
-        {
-            my $line = shift;
-            print STDERR $line;
-            *STDERR->flush();
-            $stderr .= $line;
-        };
-    }
     if (!exists($args{outfunc}))
     {
         $retval = '';

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '1.9.3'
+VITASTOR_VERSION = '1.10.1'
 
 LOG = logging.getLogger(__name__)
 

patches/libvirt-8.0-vitastor.diff

@@ -306,12 +306,12 @@ index e5ff653a60..884ecc79ea 100644
 +        etcd = virBufferContentAndReset(&buf);
 +    }
 +
-+    if (virJSONValueObjectCreate(&ret,
-+                                 "S:etcd-host", etcd,
-+                                 "S:etcd-prefix", src->query,
-+                                 "S:config-path", src->configFile,
-+                                 "s:image", src->path,
-+                                 NULL) < 0)
++    if (virJSONValueObjectAdd(&ret,
++                              "S:etcd-host", etcd,
++                              "S:etcd-prefix", src->query,
++                              "S:config-path", src->configFile,
++                              "s:image", src->path,
++                              NULL) < 0)
 +        return NULL;
 +
 +    return ret;

patches/pve-qemu-9.0-vitastor.patch

@@ -0,0 +1,193 @@
+Index: pve-qemu-kvm-9.0.0/block/meson.build
+===================================================================
+--- pve-qemu-kvm-9.0.0.orig/block/meson.build
++++ pve-qemu-kvm-9.0.0/block/meson.build
+@@ -126,6 +126,7 @@ foreach m : [
+   [libnfs, 'nfs', files('nfs.c')],
+   [libssh, 'ssh', files('ssh.c')],
+   [rbd, 'rbd', files('rbd.c')],
++  [vitastor, 'vitastor', files('vitastor.c')],
+ ]
+   if m[0].found()
+     module_ss = ss.source_set()
+Index: pve-qemu-kvm-9.0.0/meson.build
+===================================================================
+--- pve-qemu-kvm-9.0.0.orig/meson.build
++++ pve-qemu-kvm-9.0.0/meson.build
+@@ -1452,6 +1452,26 @@ if not get_option('rbd').auto() or have_
+   endif
+ endif
+ 
++vitastor = not_found
++if not get_option('vitastor').auto() or have_block
++  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
++    required: get_option('vitastor'))
++  if libvitastor_client.found()
++    if cc.links('''
++      #include <vitastor_c.h>
++      int main(void) {
++        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
++        return 0;
++      }''', dependencies: libvitastor_client)
++      vitastor = declare_dependency(dependencies: libvitastor_client)
++    elif get_option('vitastor').enabled()
++      error('could not link libvitastor_client')
++    else
++      warning('could not link libvitastor_client, disabling')
++    endif
++  endif
++endif
++
+ glusterfs = not_found
+ glusterfs_ftruncate_has_stat = false
+ glusterfs_iocb_has_stat = false
+@@ -2254,6 +2274,7 @@ endif
+ config_host_data.set('CONFIG_OPENGL', opengl.found())
+ config_host_data.set('CONFIG_PLUGIN', get_option('plugins'))
+ config_host_data.set('CONFIG_RBD', rbd.found())
++config_host_data.set('CONFIG_VITASTOR', vitastor.found())
+ config_host_data.set('CONFIG_RDMA', rdma.found())
+ config_host_data.set('CONFIG_RELOCATABLE', get_option('relocatable'))
+ config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack'))
+@@ -4454,6 +4475,7 @@ summary_info += {'fdt support':       fd
+ summary_info += {'libcap-ng support': libcap_ng}
+ summary_info += {'bpf support':       libbpf}
+ summary_info += {'rbd support':       rbd}
++summary_info += {'vitastor support':  vitastor}
+ summary_info += {'smartcard support': cacard}
+ summary_info += {'U2F support':       u2f}
+ summary_info += {'libusb':            libusb}
+Index: pve-qemu-kvm-9.0.0/meson_options.txt
+===================================================================
+--- pve-qemu-kvm-9.0.0.orig/meson_options.txt
++++ pve-qemu-kvm-9.0.0/meson_options.txt
+@@ -194,6 +194,8 @@ option('lzo', type : 'feature', value :
+        description: 'lzo compression support')
+ option('rbd', type : 'feature', value : 'auto',
+        description: 'Ceph block device driver')
++option('vitastor', type : 'feature', value : 'auto',
++       description: 'Vitastor block device driver')
+ option('opengl', type : 'feature', value : 'auto',
+        description: 'OpenGL support')
+ option('rdma', type : 'feature', value : 'auto',
+Index: pve-qemu-kvm-9.0.0/qapi/block-core.json
+===================================================================
+--- pve-qemu-kvm-9.0.0.orig/qapi/block-core.json
++++ pve-qemu-kvm-9.0.0/qapi/block-core.json
+@@ -3481,7 +3481,7 @@
+             'raw', 'rbd',
+             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
+             'pbs',
+-            'ssh', 'throttle', 'vdi', 'vhdx',
++            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
+             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
+@@ -4591,6 +4591,28 @@
+             '*server': ['InetSocketAddressBase'] } }
+ 
+ ##
++# @BlockdevOptionsVitastor:
++#
++# Driver specific block device options for vitastor
++#
++# @image:       Image name
++# @inode:       Inode number
++# @pool:        Pool ID
++# @size:        Desired image size in bytes
++# @config-path: Path to Vitastor configuration
++# @etcd-host:   etcd connection address(es)
++# @etcd-prefix: etcd key/value prefix
++##
++{ 'struct': 'BlockdevOptionsVitastor',
++  'data': { '*inode': 'uint64',
++            '*pool': 'uint64',
++            '*size': 'uint64',
++            '*image': 'str',
++            '*config-path': 'str',
++            '*etcd-host': 'str',
++            '*etcd-prefix': 'str' } }
++
++##
+ # @ReplicationMode:
+ #
+ # An enumeration of replication modes.
+@@ -5053,6 +5075,7 @@
+       'throttle':   'BlockdevOptionsThrottle',
+       'vdi':        'BlockdevOptionsGenericFormat',
+       'vhdx':       'BlockdevOptionsGenericFormat',
++      'vitastor':   'BlockdevOptionsVitastor',
+       'virtio-blk-vfio-pci':
+                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
+                       'if': 'CONFIG_BLKIO' },
+@@ -5498,6 +5521,20 @@
+             '*encrypt' :        'RbdEncryptionCreateOptions' } }
+ 
+ ##
++# @BlockdevCreateOptionsVitastor:
++#
++# Driver specific image creation options for Vitastor.
++#
++# @location: Where to store the new image file.  This location cannot
++#     point to a snapshot.
++#
++# @size: Size of the virtual disk in bytes
++##
++{ 'struct': 'BlockdevCreateOptionsVitastor',
++  'data': { 'location':         'BlockdevOptionsVitastor',
++            'size':             'size' } }
++
++##
+ # @BlockdevVmdkSubformat:
+ #
+ # Subformat options for VMDK images
+@@ -5719,6 +5753,7 @@
+       'ssh':            'BlockdevCreateOptionsSsh',
+       'vdi':            'BlockdevCreateOptionsVdi',
+       'vhdx':           'BlockdevCreateOptionsVhdx',
++      'vitastor':       'BlockdevCreateOptionsVitastor',
+       'vmdk':           'BlockdevCreateOptionsVmdk',
+       'vpc':            'BlockdevCreateOptionsVpc'
+   } }
+Index: pve-qemu-kvm-9.0.0/scripts/ci/org.centos/stream/8/x86_64/configure
+===================================================================
+--- pve-qemu-kvm-9.0.0.orig/scripts/ci/org.centos/stream/8/x86_64/configure
++++ pve-qemu-kvm-9.0.0/scripts/ci/org.centos/stream/8/x86_64/configure
+@@ -30,7 +30,7 @@
+ --with-suffix="qemu-kvm" \
+ --firmwarepath=/usr/share/qemu-firmware \
+ --target-list="x86_64-softmmu" \
+---block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
++--block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,vitastor,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
+ --audio-drv-list="" \
+ --block-drv-ro-whitelist="vmdk,vhdx,vpc,https,ssh" \
+ --with-coroutine=ucontext \
+@@ -176,6 +176,7 @@
+ --enable-opengl \
+ --enable-pie \
+ --enable-rbd \
++--enable-vitastor \
+ --enable-rdma \
+ --enable-seccomp \
+ --enable-snappy \
+Index: pve-qemu-kvm-9.0.0/scripts/meson-buildoptions.sh
+===================================================================
+--- pve-qemu-kvm-9.0.0.orig/scripts/meson-buildoptions.sh
++++ pve-qemu-kvm-9.0.0/scripts/meson-buildoptions.sh
+@@ -168,6 +168,7 @@ meson_options_help() {
+   printf "%s\n" '  qed             qed image format support'
+   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
+   printf "%s\n" '  rbd             Ceph block device driver'
++  printf "%s\n" '  vitastor        Vitastor block device driver'
+   printf "%s\n" '  rdma            Enable RDMA-based migration'
+   printf "%s\n" '  replication     replication support'
+   printf "%s\n" '  rutabaga-gfx    rutabaga_gfx support'
+@@ -445,6 +446,8 @@ _meson_option_parse() {
+     --disable-qom-cast-debug) printf "%s" -Dqom_cast_debug=false ;;
+     --enable-rbd) printf "%s" -Drbd=enabled ;;
+     --disable-rbd) printf "%s" -Drbd=disabled ;;
++    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
++    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
+     --enable-rdma) printf "%s" -Drdma=enabled ;;
+     --disable-rdma) printf "%s" -Drdma=disabled ;;
+     --enable-relocatable) printf "%s" -Drelocatable=true ;;

patches/pve-qemu-9.1-vitastor.patch

@@ -0,0 +1,172 @@
+Index: pve-qemu-kvm-9.1.2/block/meson.build
+===================================================================
+--- pve-qemu-kvm-9.1.2.orig/block/meson.build
++++ pve-qemu-kvm-9.1.2/block/meson.build
+@@ -126,6 +126,7 @@ foreach m : [
+   [libnfs, 'nfs', files('nfs.c')],
+   [libssh, 'ssh', files('ssh.c')],
+   [rbd, 'rbd', files('rbd.c')],
++  [vitastor, 'vitastor', files('vitastor.c')],
+ ]
+   if m[0].found()
+     module_ss = ss.source_set()
+Index: pve-qemu-kvm-9.1.2/meson.build
+===================================================================
+--- pve-qemu-kvm-9.1.2.orig/meson.build
++++ pve-qemu-kvm-9.1.2/meson.build
+@@ -1516,6 +1516,26 @@ if not get_option('rbd').auto() or have_
+   endif
+ endif
+ 
++vitastor = not_found
++if not get_option('vitastor').auto() or have_block
++  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
++    required: get_option('vitastor'))
++  if libvitastor_client.found()
++    if cc.links('''
++      #include <vitastor_c.h>
++      int main(void) {
++        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
++        return 0;
++      }''', dependencies: libvitastor_client)
++      vitastor = declare_dependency(dependencies: libvitastor_client)
++    elif get_option('vitastor').enabled()
++      error('could not link libvitastor_client')
++    else
++      warning('could not link libvitastor_client, disabling')
++    endif
++  endif
++endif
++
+ glusterfs = not_found
+ glusterfs_ftruncate_has_stat = false
+ glusterfs_iocb_has_stat = false
+@@ -2367,6 +2387,7 @@ endif
+ config_host_data.set('CONFIG_OPENGL', opengl.found())
+ config_host_data.set('CONFIG_PLUGIN', get_option('plugins'))
+ config_host_data.set('CONFIG_RBD', rbd.found())
++config_host_data.set('CONFIG_VITASTOR', vitastor.found())
+ config_host_data.set('CONFIG_RDMA', rdma.found())
+ config_host_data.set('CONFIG_RELOCATABLE', get_option('relocatable'))
+ config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack'))
+@@ -4534,6 +4555,7 @@ summary_info += {'fdt support':       fd
+ summary_info += {'libcap-ng support': libcap_ng}
+ summary_info += {'bpf support':       libbpf}
+ summary_info += {'rbd support':       rbd}
++summary_info += {'vitastor support':  vitastor}
+ summary_info += {'smartcard support': cacard}
+ summary_info += {'U2F support':       u2f}
+ summary_info += {'libusb':            libusb}
+Index: pve-qemu-kvm-9.1.2/meson_options.txt
+===================================================================
+--- pve-qemu-kvm-9.1.2.orig/meson_options.txt
++++ pve-qemu-kvm-9.1.2/meson_options.txt
+@@ -194,6 +194,8 @@ option('lzo', type : 'feature', value :
+        description: 'lzo compression support')
+ option('rbd', type : 'feature', value : 'auto',
+        description: 'Ceph block device driver')
++option('vitastor', type : 'feature', value : 'auto',
++       description: 'Vitastor block device driver')
+ option('opengl', type : 'feature', value : 'auto',
+        description: 'OpenGL support')
+ option('rdma', type : 'feature', value : 'auto',
+Index: pve-qemu-kvm-9.1.2/qapi/block-core.json
+===================================================================
+--- pve-qemu-kvm-9.1.2.orig/qapi/block-core.json
++++ pve-qemu-kvm-9.1.2/qapi/block-core.json
+@@ -3477,7 +3477,7 @@
+             'raw', 'rbd',
+             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
+             'pbs',
+-            'ssh', 'throttle', 'vdi', 'vhdx',
++            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
+             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
+@@ -4588,6 +4588,28 @@
+             '*server': ['InetSocketAddressBase'] } }
+ 
+ ##
++# @BlockdevOptionsVitastor:
++#
++# Driver specific block device options for vitastor
++#
++# @image:       Image name
++# @inode:       Inode number
++# @pool:        Pool ID
++# @size:        Desired image size in bytes
++# @config-path: Path to Vitastor configuration
++# @etcd-host:   etcd connection address(es)
++# @etcd-prefix: etcd key/value prefix
++##
++{ 'struct': 'BlockdevOptionsVitastor',
++  'data': { '*inode': 'uint64',
++            '*pool': 'uint64',
++            '*size': 'uint64',
++            '*image': 'str',
++            '*config-path': 'str',
++            '*etcd-host': 'str',
++            '*etcd-prefix': 'str' } }
++
++##
+ # @ReplicationMode:
+ #
+ # An enumeration of replication modes.
+@@ -5050,6 +5072,7 @@
+       'throttle':   'BlockdevOptionsThrottle',
+       'vdi':        'BlockdevOptionsGenericFormat',
+       'vhdx':       'BlockdevOptionsGenericFormat',
++      'vitastor':   'BlockdevOptionsVitastor',
+       'virtio-blk-vfio-pci':
+                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
+                       'if': 'CONFIG_BLKIO' },
+@@ -5497,6 +5520,20 @@
+             '*encrypt' :        'RbdEncryptionCreateOptions' } }
+ 
+ ##
++# @BlockdevCreateOptionsVitastor:
++#
++# Driver specific image creation options for Vitastor.
++#
++# @location: Where to store the new image file.  This location cannot
++#     point to a snapshot.
++#
++# @size: Size of the virtual disk in bytes
++##
++{ 'struct': 'BlockdevCreateOptionsVitastor',
++  'data': { 'location':         'BlockdevOptionsVitastor',
++            'size':             'size' } }
++
++##
+ # @BlockdevVmdkSubformat:
+ #
+ # Subformat options for VMDK images
+@@ -5718,6 +5755,7 @@
+       'ssh':            'BlockdevCreateOptionsSsh',
+       'vdi':            'BlockdevCreateOptionsVdi',
+       'vhdx':           'BlockdevCreateOptionsVhdx',
++      'vitastor':       'BlockdevCreateOptionsVitastor',
+       'vmdk':           'BlockdevCreateOptionsVmdk',
+       'vpc':            'BlockdevCreateOptionsVpc'
+   } }
+Index: pve-qemu-kvm-9.1.2/scripts/meson-buildoptions.sh
+===================================================================
+--- pve-qemu-kvm-9.1.2.orig/scripts/meson-buildoptions.sh
++++ pve-qemu-kvm-9.1.2/scripts/meson-buildoptions.sh
+@@ -168,6 +168,7 @@ meson_options_help() {
+   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
+   printf "%s\n" '  qpl             Query Processing Library support'
+   printf "%s\n" '  rbd             Ceph block device driver'
++  printf "%s\n" '  vitastor        Vitastor block device driver'
+   printf "%s\n" '  rdma            Enable RDMA-based migration'
+   printf "%s\n" '  replication     replication support'
+   printf "%s\n" '  rutabaga-gfx    rutabaga_gfx support'
+@@ -444,6 +445,8 @@ _meson_option_parse() {
+     --disable-qpl) printf "%s" -Dqpl=disabled ;;
+     --enable-rbd) printf "%s" -Drbd=enabled ;;
+     --disable-rbd) printf "%s" -Drbd=disabled ;;
++    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
++    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
+     --enable-rdma) printf "%s" -Drdma=enabled ;;
+     --disable-rdma) printf "%s" -Drdma=disabled ;;
+     --enable-relocatable) printf "%s" -Drelocatable=true ;;

patches/qemu-9.1-vitastor.patch

@@ -0,0 +1,172 @@
+diff --git a/block/meson.build b/block/meson.build
+index f1262ec2ba..3cf3e23f16 100644
+--- a/block/meson.build
++++ b/block/meson.build
+@@ -114,6 +114,7 @@ foreach m : [
+   [libnfs, 'nfs', files('nfs.c')],
+   [libssh, 'ssh', files('ssh.c')],
+   [rbd, 'rbd', files('rbd.c')],
++  [vitastor, 'vitastor', files('vitastor.c')],
+ ]
+   if m[0].found()
+     module_ss = ss.source_set()
+diff --git a/meson.build b/meson.build
+index fbda17c987..3edac22aff 100644
+--- a/meson.build
++++ b/meson.build
+@@ -1510,6 +1510,26 @@ if not get_option('rbd').auto() or have_block
+   endif
+ endif
+ 
++vitastor = not_found
++if not get_option('vitastor').auto() or have_block
++  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
++    required: get_option('vitastor'))
++  if libvitastor_client.found()
++    if cc.links('''
++      #include <vitastor_c.h>
++      int main(void) {
++        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
++        return 0;
++      }''', dependencies: libvitastor_client)
++      vitastor = declare_dependency(dependencies: libvitastor_client)
++    elif get_option('vitastor').enabled()
++      error('could not link libvitastor_client')
++    else
++      warning('could not link libvitastor_client, disabling')
++    endif
++  endif
++endif
++
+ glusterfs = not_found
+ glusterfs_ftruncate_has_stat = false
+ glusterfs_iocb_has_stat = false
+@@ -2351,6 +2371,7 @@ endif
+ config_host_data.set('CONFIG_OPENGL', opengl.found())
+ config_host_data.set('CONFIG_PLUGIN', get_option('plugins'))
+ config_host_data.set('CONFIG_RBD', rbd.found())
++config_host_data.set('CONFIG_VITASTOR', vitastor.found())
+ config_host_data.set('CONFIG_RDMA', rdma.found())
+ config_host_data.set('CONFIG_RELOCATABLE', get_option('relocatable'))
+ config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack'))
+@@ -4510,6 +4531,7 @@ summary_info += {'fdt support':       fdt_opt == 'internal' ? 'internal' : fdt}
+ summary_info += {'libcap-ng support': libcap_ng}
+ summary_info += {'bpf support':       libbpf}
+ summary_info += {'rbd support':       rbd}
++summary_info += {'vitastor support':  vitastor}
+ summary_info += {'smartcard support': cacard}
+ summary_info += {'U2F support':       u2f}
+ summary_info += {'libusb':            libusb}
+diff --git a/meson_options.txt b/meson_options.txt
+index 0269fa0f16..4740ffdc27 100644
+--- a/meson_options.txt
++++ b/meson_options.txt
+@@ -194,6 +194,8 @@ option('lzo', type : 'feature', value : 'auto',
+        description: 'lzo compression support')
+ option('rbd', type : 'feature', value : 'auto',
+        description: 'Ceph block device driver')
++option('vitastor', type : 'feature', value : 'auto',
++       description: 'Vitastor block device driver')
+ option('opengl', type : 'feature', value : 'auto',
+        description: 'OpenGL support')
+ option('rdma', type : 'feature', value : 'auto',
+diff --git a/qapi/block-core.json b/qapi/block-core.json
+index aa40d44f1d..bbee6a0e9c 100644
+--- a/qapi/block-core.json
++++ b/qapi/block-core.json
+@@ -3203,7 +3203,7 @@
+             'parallels', 'preallocate', 'qcow', 'qcow2', 'qed', 'quorum',
+             'raw', 'rbd',
+             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
+-            'ssh', 'throttle', 'vdi', 'vhdx',
++            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
+             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
+@@ -4286,6 +4286,28 @@
+             '*key-secret': 'str',
+             '*server': ['InetSocketAddressBase'] } }
+ 
++##
++# @BlockdevOptionsVitastor:
++#
++# Driver specific block device options for vitastor
++#
++# @image:       Image name
++# @inode:       Inode number
++# @pool:        Pool ID
++# @size:        Desired image size in bytes
++# @config-path: Path to Vitastor configuration
++# @etcd-host:   etcd connection address(es)
++# @etcd-prefix: etcd key/value prefix
++##
++{ 'struct': 'BlockdevOptionsVitastor',
++  'data': { '*inode': 'uint64',
++            '*pool': 'uint64',
++            '*size': 'uint64',
++            '*image': 'str',
++            '*config-path': 'str',
++            '*etcd-host': 'str',
++            '*etcd-prefix': 'str' } }
++
+ ##
+ # @ReplicationMode:
+ #
+@@ -4742,6 +4764,7 @@
+       'throttle':   'BlockdevOptionsThrottle',
+       'vdi':        'BlockdevOptionsGenericFormat',
+       'vhdx':       'BlockdevOptionsGenericFormat',
++      'vitastor':   'BlockdevOptionsVitastor',
+       'virtio-blk-vfio-pci':
+                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
+                       'if': 'CONFIG_BLKIO' },
+@@ -5183,6 +5206,20 @@
+             '*cluster-size' :   'size',
+             '*encrypt' :        'RbdEncryptionCreateOptions' } }
+ 
++##
++# @BlockdevCreateOptionsVitastor:
++#
++# Driver specific image creation options for Vitastor.
++#
++# @location: Where to store the new image file.  This location cannot
++#     point to a snapshot.
++#
++# @size: Size of the virtual disk in bytes
++##
++{ 'struct': 'BlockdevCreateOptionsVitastor',
++  'data': { 'location':         'BlockdevOptionsVitastor',
++            'size':             'size' } }
++
+ ##
+ # @BlockdevVmdkSubformat:
+ #
+@@ -5405,6 +5442,7 @@
+       'ssh':            'BlockdevCreateOptionsSsh',
+       'vdi':            'BlockdevCreateOptionsVdi',
+       'vhdx':           'BlockdevCreateOptionsVhdx',
++      'vitastor':       'BlockdevCreateOptionsVitastor',
+       'vmdk':           'BlockdevCreateOptionsVmdk',
+       'vpc':            'BlockdevCreateOptionsVpc'
+   } }
+diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh
+index c97079a38c..4623f552ec 100644
+--- a/scripts/meson-buildoptions.sh
++++ b/scripts/meson-buildoptions.sh
+@@ -168,6 +168,7 @@ meson_options_help() {
+   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
+   printf "%s\n" '  qpl             Query Processing Library support'
+   printf "%s\n" '  rbd             Ceph block device driver'
++  printf "%s\n" '  vitastor        Vitastor block device driver'
+   printf "%s\n" '  rdma            Enable RDMA-based migration'
+   printf "%s\n" '  replication     replication support'
+   printf "%s\n" '  rutabaga-gfx    rutabaga_gfx support'
+@@ -444,6 +445,8 @@ _meson_option_parse() {
+     --disable-qpl) printf "%s" -Dqpl=disabled ;;
+     --enable-rbd) printf "%s" -Drbd=enabled ;;
+     --disable-rbd) printf "%s" -Drbd=disabled ;;
++    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
++    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
+     --enable-rdma) printf "%s" -Drdma=enabled ;;
+     --disable-rdma) printf "%s" -Drdma=disabled ;;
+     --enable-relocatable) printf "%s" -Drelocatable=true ;;

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.9.3
+Version:        1.10.1
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.9.3.el7.tar.gz
+Source0:        vitastor-1.10.1.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel
@@ -15,7 +15,7 @@ BuildRequires:  rh-nodejs12-npm
 BuildRequires:  jerasure-devel
 BuildRequires:  libisa-l-devel
 BuildRequires:  gf-complete-devel
-BuildRequires:  libibverbs-devel
+BuildRequires:  rdma-core-devel
 BuildRequires:  cmake3
 BuildRequires:  libnl3-devel
 Requires:       vitastor-osd = %{version}-%{release}

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.9.3
+Version:        1.10.1
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.9.3.el8.tar.gz
+Source0:        vitastor-1.10.1.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel
@@ -14,7 +14,7 @@ BuildRequires:  nodejs >= 10
 BuildRequires:  jerasure-devel
 BuildRequires:  libisa-l-devel
 BuildRequires:  gf-complete-devel
-BuildRequires:  libibverbs-devel
+BuildRequires:  rdma-core-devel
 BuildRequires:  cmake
 BuildRequires:  libnl3-devel
 Requires:       vitastor-osd = %{version}-%{release}

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.9.3
+Version:        1.10.1
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.9.3.el9.tar.gz
+Source0:        vitastor-1.10.1.el9.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -19,7 +19,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVITASTOR_VERSION="1.9.3")
+add_definitions(-DVITASTOR_VERSION="1.10.1")
 add_definitions(-D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})
@@ -61,6 +61,10 @@ pkg_check_modules(ISAL libisal)
 if (ISAL_LIBRARIES)
 	add_definitions(-DWITH_ISAL)
 endif (ISAL_LIBRARIES)
+pkg_check_modules(RDMACM librdmacm)
+if (RDMACM_LIBRARIES)
+	add_definitions(-DWITH_RDMACM)
+endif (RDMACM_LIBRARIES)
 
 add_custom_target(build_tests)
 add_custom_target(test

src/client/cluster_client.cpp

@@ -6,6 +6,10 @@
 #include "cluster_client_impl.h"
 #include "json_util.h"
 
+#define TRY_SEND_OFFLINE 0
+#define TRY_SEND_CONNECTING 1
+#define TRY_SEND_OK 2
+
 cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json config)
 {
     wb = new writeback_cache_t();
@@ -67,6 +71,18 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
 
 cluster_client_t::~cluster_client_t()
 {
+    if (retry_timeout_id >= 0)
+    {
+        tfd->clear_timer(retry_timeout_id);
+        retry_timeout_duration = 0;
+        retry_timeout_id = -1;
+    }
+    if (list_retry_timeout_id >= 0)
+    {
+        tfd->clear_timer(list_retry_timeout_id);
+        list_retry_timeout_id = -1;
+        list_retry_time = {};
+    }
     msgr.repeer_pgs = [](osd_num_t){};
     if (ringloop)
     {
@@ -87,6 +103,46 @@ cluster_op_t::~cluster_op_t()
     }
 }
 
+bool cluster_op_t::support_left_on_dead()
+{
+    if (!parts.size())
+    {
+        return false;
+    }
+    for (auto & part: parts)
+    {
+        if (!(part.flags & PART_DONE) ||
+            part.op.reply.hdr.opcode != OSD_OP_DELETE ||
+            part.op.reply.hdr.retval != 0 ||
+            !(part.op.reply.del.flags & OSD_DEL_SUPPORT_LEFT_ON_DEAD))
+        {
+            return false;
+        }
+    }
+    return true;
+}
+
+std::vector<osd_num_t> cluster_op_t::get_left_on_dead()
+{
+    std::set<osd_num_t> osds;
+    for (auto & part: parts)
+    {
+        if ((part.flags & PART_DONE) ||
+            part.op.reply.hdr.opcode == OSD_OP_DELETE &&
+            part.op.reply.hdr.retval == 0 &&
+            (part.op.reply.del.flags & OSD_DEL_LEFT_ON_DEAD) != 0)
+        {
+            int del_count = (OSD_PACKET_SIZE-sizeof(part.op.reply.del)) / sizeof(uint32_t);
+            if (del_count > part.op.reply.del.left_on_dead_count)
+                del_count = part.op.reply.del.left_on_dead_count;
+            uint32_t *left_on_dead = (uint32_t*)((&part.op.reply.del) + 1);
+            for (int i = 0; i < del_count; i++)
+                osds.insert(left_on_dead[i]);
+        }
+    }
+    return std::vector<osd_num_t>(osds.begin(), osds.end());
+}
+
 void cluster_client_t::continue_raw_ops(osd_num_t peer_osd)
 {
     auto it = raw_ops.find(peer_osd);
@@ -134,12 +190,12 @@ void cluster_client_t::unshift_op(cluster_op_t *op)
 void cluster_client_t::calc_wait(cluster_op_t *op)
 {
     op->prev_wait = 0;
-    if (op->opcode == OSD_OP_WRITE)
+    if (op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE)
     {
         for (auto prev = op->prev; prev; prev = prev->prev)
         {
             if (prev->opcode == OSD_OP_SYNC ||
-                prev->opcode == OSD_OP_WRITE && !(op->flags & OP_FLUSH_BUFFER) && (prev->flags & OP_FLUSH_BUFFER))
+                (prev->opcode == OSD_OP_WRITE || prev->opcode == OSD_OP_DELETE) && !(op->flags & OP_FLUSH_BUFFER) && (prev->flags & OP_FLUSH_BUFFER))
             {
                 op->prev_wait++;
             }
@@ -151,7 +207,8 @@ void cluster_client_t::calc_wait(cluster_op_t *op)
     {
         for (auto prev = op->prev; prev; prev = prev->prev)
         {
-            if (prev->opcode == OSD_OP_SYNC || prev->opcode == OSD_OP_WRITE && (!(prev->flags & OP_IMMEDIATE_COMMIT) || enable_writeback))
+            if (prev->opcode == OSD_OP_SYNC || (prev->opcode == OSD_OP_WRITE || prev->opcode == OSD_OP_DELETE) &&
+                (!(prev->flags & OP_IMMEDIATE_COMMIT) || enable_writeback))
             {
                 op->prev_wait++;
             }
@@ -167,7 +224,7 @@ void cluster_client_t::calc_wait(cluster_op_t *op)
 
 void cluster_client_t::inc_wait(uint64_t opcode, uint64_t flags, cluster_op_t *next, int inc)
 {
-    if (opcode != OSD_OP_WRITE && opcode != OSD_OP_SYNC)
+    if (opcode != OSD_OP_WRITE && opcode != OSD_OP_DELETE && opcode != OSD_OP_SYNC)
     {
         return;
     }
@@ -176,10 +233,10 @@ void cluster_client_t::inc_wait(uint64_t opcode, uint64_t flags, cluster_op_t *n
     while (next)
     {
         auto n2 = next->next;
-        if (opcode == OSD_OP_WRITE
+        if ((opcode == OSD_OP_WRITE || opcode == OSD_OP_DELETE)
             ? (next->opcode == OSD_OP_SYNC && (!(flags & OP_IMMEDIATE_COMMIT) || enable_writeback) ||
-                next->opcode == OSD_OP_WRITE && (flags & OP_FLUSH_BUFFER) && !(next->flags & OP_FLUSH_BUFFER))
-            : (next->opcode == OSD_OP_SYNC || next->opcode == OSD_OP_WRITE))
+                (next->opcode == OSD_OP_WRITE || next->opcode == OSD_OP_DELETE) && (flags & OP_FLUSH_BUFFER) && !(next->flags & OP_FLUSH_BUFFER))
+            : (next->opcode == OSD_OP_SYNC || next->opcode == OSD_OP_WRITE || next->opcode == OSD_OP_DELETE))
         {
             next->prev_wait += inc;
             assert(next->prev_wait >= 0);
@@ -314,7 +371,7 @@ void cluster_client_t::reset_retry_timer(int new_duration)
     {
         return;
     }
-    if (retry_timeout_id)
+    if (retry_timeout_id >= 0)
     {
         tfd->clear_timer(retry_timeout_id);
     }
@@ -322,7 +379,7 @@ void cluster_client_t::reset_retry_timer(int new_duration)
     retry_timeout_id = tfd->set_timer(retry_timeout_duration, false, [this](int)
     {
         int time_passed = retry_timeout_duration;
-        retry_timeout_id = 0;
+        retry_timeout_id = -1;
         retry_timeout_duration = 0;
         continue_ops(time_passed);
     });
@@ -397,6 +454,16 @@ void cluster_client_t::on_load_config_hook(json11::Json::object & etcd_global_co
     }
     // client_retry_enospc
     client_retry_enospc = config["client_retry_enospc"].is_null() ? true : config["client_retry_enospc"].bool_value();
+    // client_wait_up_timeout
+    if (!config["client_wait_up_timeout"].is_null())
+        client_wait_up_timeout = config["client_wait_up_timeout"].uint64_value();
+    else
+    {
+        auto etcd_report_interval = config["etcd_report_interval"].uint64_value();
+        if (!etcd_report_interval)
+            etcd_report_interval = 5;
+        client_wait_up_timeout = 1+etcd_report_interval+(st_cli.max_etcd_attempts*(2*st_cli.etcd_quick_timeout)+999)/1000;
+    }
     // log_level
     log_level = config["log_level"].uint64_value();
     msgr.parse_config(config);
@@ -434,7 +501,7 @@ void cluster_client_t::on_change_pool_config_hook()
             // And now they have to be resliced!
             for (auto op = op_queue_head; op; op = op->next)
             {
-                if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_READ ||
+                if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE || op->opcode == OSD_OP_READ ||
                     op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP) &&
                     INODE_POOL(op->cur_inode) == pool_item.first)
                 {
@@ -457,6 +524,7 @@ void cluster_client_t::on_change_pg_state_hook(pool_id_t pool_id, pg_num_t pg_nu
     }
     // Always continue to resume operations hung because of lack of the primary OSD
     continue_ops();
+    continue_lists();
 }
 
 bool cluster_client_t::get_immediate_commit(uint64_t inode)
@@ -477,6 +545,7 @@ void cluster_client_t::on_change_osd_state_hook(uint64_t peer_osd)
     if (msgr.wanted_peers.find(peer_osd) != msgr.wanted_peers.end())
     {
         msgr.connect_peer(peer_osd, st_cli.peer_states[peer_osd]);
+        continue_lists();
     }
 }
 
@@ -559,7 +628,8 @@ bool cluster_client_t::flush()
 void cluster_client_t::execute(cluster_op_t *op)
 {
     if (op->opcode != OSD_OP_SYNC && op->opcode != OSD_OP_READ &&
-        op->opcode != OSD_OP_READ_BITMAP && op->opcode != OSD_OP_READ_CHAIN_BITMAP && op->opcode != OSD_OP_WRITE)
+        op->opcode != OSD_OP_READ_BITMAP && op->opcode != OSD_OP_READ_CHAIN_BITMAP &&
+        op->opcode != OSD_OP_WRITE && op->opcode != OSD_OP_DELETE)
     {
         op->retval = -EINVAL;
         auto cb = std::move(op->callback);
@@ -571,7 +641,7 @@ void cluster_client_t::execute(cluster_op_t *op)
         offline_ops.push_back(op);
         return;
     }
-    op->flags = op->flags & OSD_OP_IGNORE_READONLY; // the only allowed flag
+    op->flags = op->flags & (OSD_OP_IGNORE_READONLY | OSD_OP_WAIT_UP_TIMEOUT); // allowed client flags
     execute_internal(op);
 }
 
@@ -592,7 +662,7 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
     {
         return;
     }
-    if (op->opcode == OSD_OP_WRITE && enable_writeback && !(op->flags & OP_FLUSH_BUFFER) &&
+    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && enable_writeback && !(op->flags & OP_FLUSH_BUFFER) &&
         !op->version /* no CAS writeback */)
     {
         if (wb->writebacks_active >= client_max_writeback_iodepth)
@@ -613,7 +683,7 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
         cb(op);
         return;
     }
-    if (op->opcode == OSD_OP_WRITE && !(op->flags & OP_IMMEDIATE_COMMIT))
+    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && !(op->flags & OP_IMMEDIATE_COMMIT))
     {
         if (!(op->flags & OP_FLUSH_BUFFER) && !op->version /* no CAS write-repeat */)
         {
@@ -633,7 +703,10 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
             };
             execute_internal(sync_op);
         }
-        dirty_bytes += op->len;
+        if (op->opcode != OSD_OP_DELETE)
+        {
+            dirty_bytes += op->len;
+        }
         dirty_ops++;
     }
     else if (op->opcode == OSD_OP_SYNC)
@@ -777,9 +850,48 @@ resume_1:
     {
         if (!(op->parts[i].flags & PART_SENT))
         {
-            if (!try_send(op, i))
+            int is_ok = try_send(op, i);
+            if (is_ok != TRY_SEND_OK)
             {
                 // We'll need to retry again
+                if (op->flags & OSD_OP_WAIT_UP_TIMEOUT)
+                {
+                    if (is_ok != TRY_SEND_OFFLINE)
+                    {
+                        // Reset "wait_up" timer
+                        op->wait_up_until = {};
+                    }
+                    else if (!op->wait_up_until.tv_sec && !client_wait_up_timeout)
+                    {
+                        // Don't wait for the PG to come up at all and fail
+                        op->parts[i].flags |= PART_ERROR;
+                        if (!op->retval)
+                            op->retval = -ETIMEDOUT;
+                        break;
+                    }
+                    else if (!op->wait_up_until.tv_sec)
+                    {
+                        // Set "wait_up" timer
+                        clock_gettime(CLOCK_REALTIME, &op->wait_up_until);
+                        op->wait_up_until.tv_sec += client_wait_up_timeout;
+                    }
+                    else
+                    {
+                        // Check if the timeout expired
+                        timespec tv;
+                        clock_gettime(CLOCK_REALTIME, &tv);
+                        if (tv.tv_sec > op->wait_up_until.tv_sec ||
+                            tv.tv_sec == op->wait_up_until.tv_sec &&
+                            tv.tv_nsec > op->wait_up_until.tv_nsec)
+                        {
+                            // Fail
+                            op->parts[i].flags |= PART_ERROR;
+                            if (!op->retval)
+                                op->retval = -ETIMEDOUT;
+                            break;
+                        }
+                    }
+                }
                 if (op->parts[i].flags & PART_RETRY)
                 {
                     op->retry_after = client_retry_interval;
@@ -1053,7 +1165,7 @@ bool cluster_client_t::affects_osd(uint64_t inode, uint64_t offset, uint64_t len
     return false;
 }
 
-bool cluster_client_t::try_send(cluster_op_t *op, int i)
+int cluster_client_t::try_send(cluster_op_t *op, int i)
 {
     if (!msgr_initialized)
     {
@@ -1079,7 +1191,7 @@ bool cluster_client_t::try_send(cluster_op_t *op, int i)
             uint64_t meta_rev = 0;
             if (op->opcode != OSD_OP_READ_BITMAP && op->opcode != OSD_OP_DELETE)
             {
-                auto ino_it = st_cli.inode_config.find(op->inode);
+                auto ino_it = st_cli.inode_config.find(op->cur_inode);
                 if (ino_it != st_cli.inode_config.end())
                     meta_rev = ino_it->second.mod_revision;
             }
@@ -1109,14 +1221,15 @@ bool cluster_client_t::try_send(cluster_op_t *op, int i)
             };
             part->op.iov = part->iov;
             msgr.outbox_push(&part->op);
-            return true;
+            return TRY_SEND_OK;
         }
         else if (msgr.wanted_peers.find(primary_osd) == msgr.wanted_peers.end())
         {
             msgr.connect_peer(primary_osd, st_cli.peer_states[primary_osd]);
+            return TRY_SEND_CONNECTING;
         }
     }
-    return false;
+    return TRY_SEND_OFFLINE;
 }
 
 int cluster_client_t::continue_sync(cluster_op_t *op)
@@ -1188,13 +1301,12 @@ resume_1:
 
 void cluster_client_t::send_sync(cluster_op_t *op, cluster_op_part_t *part)
 {
-    auto peer_it = msgr.osd_peer_fds.find(part->osd_num);
-    assert(peer_it != msgr.osd_peer_fds.end());
+    auto peer_fd = msgr.osd_peer_fds.at(part->osd_num);
     part->flags |= PART_SENT;
     op->inflight_count++;
     part->op = (osd_op_t){
         .op_type = OSD_OP_OUT,
-        .peer_fd = peer_it->second,
+        .peer_fd = peer_fd,
         .req = {
             .hdr = {
                 .magic = SECONDARY_OSD_OP_MAGIC,
@@ -1228,9 +1340,11 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
     {
         // Operation failed, retry
         part->flags |= PART_ERROR;
-        if (!op->retval || op->retval == -EPIPE || part->op.reply.hdr.retval == -EIO)
+        if (!op->retval || op->retval == -EPIPE ||
+            part->op.reply.hdr.retval == -ENOSPC && op->retval == -ETIMEDOUT ||
+            part->op.reply.hdr.retval == -EIO)
         {
-            // Error priority: EIO > ENOSPC > EPIPE
+            // Error priority: EIO > ENOSPC > ETIMEDOUT > EPIPE
             op->retval = part->op.reply.hdr.retval;
         }
         int stop_fd = -1;
@@ -1293,7 +1407,7 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
                 op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0;
             }
         }
-        else if (op->opcode == OSD_OP_WRITE)
+        else if (op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE)
         {
             op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0;
         }

src/client/cluster_client.h

@@ -11,12 +11,11 @@
 #define DEFAULT_CLIENT_MAX_BUFFERED_BYTES 32*1024*1024
 #define DEFAULT_CLIENT_MAX_BUFFERED_OPS 1024
 #define DEFAULT_CLIENT_MAX_WRITEBACK_IODEPTH 256
-#define INODE_LIST_DONE 1
-#define INODE_LIST_HAS_UNSTABLE 2
 #define OSD_OP_READ_BITMAP OSD_OP_SEC_READ_BMP
 #define OSD_OP_READ_CHAIN_BITMAP 0x102
 
 #define OSD_OP_IGNORE_READONLY 0x08
+#define OSD_OP_WAIT_UP_TIMEOUT 0x10
 
 struct cluster_op_t;
 
@@ -41,7 +40,8 @@ struct cluster_op_t
     // for reads and writes within a single object (stripe),
     // reads can return current version and writes can use "CAS" semantics
     uint64_t version = 0;
-    // now only OSD_OP_IGNORE_READONLY is supported
+    // flags: OSD_OP_IGNORE_READONLY - ignore inode readonly flag
+    // OSD_OP_WAIT_UP_TIMEOUT - do not retry the operation infinitely if PG is inactive, only for for <wait_up_timeout>
     uint64_t flags = 0;
     // negative retval is an error number
     // write and read return len on success
@@ -53,12 +53,17 @@ struct cluster_op_t
     void *bitmap_buf = NULL;
     std::function<void(cluster_op_t*)> callback;
     ~cluster_op_t();
+
+    // for deletions, remove after 'atomic delete':
+    bool support_left_on_dead();
+    std::vector<osd_num_t> get_left_on_dead();
 protected:
     int state = 0;
     uint64_t cur_inode; // for snapshot reads
     bool needs_reslice = false;
     int retry_after = 0;
     int inflight_count = 0, done_count = 0;
+    timespec wait_up_until = {};
     std::vector<cluster_op_part_t> parts;
     void *part_bitmaps = NULL;
     unsigned bitmap_buf_size = 0;
@@ -71,6 +76,7 @@ protected:
 
 struct inode_list_t;
 struct inode_list_osd_t;
+struct inode_list_pg_t;
 class writeback_cache_t;
 
 // FIXME: Split into public and private interfaces
@@ -95,8 +101,9 @@ class cluster_client_t
     int client_retry_interval = 50; // ms
     int client_eio_retry_interval = 1000; // ms
     bool client_retry_enospc = true;
+    int client_wait_up_timeout = 16; // sec (for listings)
 
-    int retry_timeout_id = 0;
+    int retry_timeout_id = -1;
     int retry_timeout_duration = 0;
     std::vector<cluster_op_t*> offline_ops;
     cluster_op_t *op_queue_head = NULL, *op_queue_tail = NULL;
@@ -110,6 +117,8 @@ class cluster_client_t
     bool pgs_loaded = false;
     ring_consumer_t consumer;
     std::vector<std::function<void(void)>> on_ready_hooks;
+    int list_retry_timeout_id = -1;
+    timespec list_retry_time;
     std::vector<inode_list_t*> lists;
     std::multimap<osd_num_t, osd_op_t*> raw_ops;
     int continuing_ops = 0;
@@ -135,11 +144,10 @@ public:
     bool get_immediate_commit(uint64_t inode);
 
     void continue_ops(int time_passed = 0);
-    inode_list_t *list_inode_start(inode_t inode,
-        std::function<void(inode_list_t* lst, std::set<object_id>&& objects, pg_num_t pg_num, osd_num_t primary_osd, int status)> callback);
-    int list_pg_count(inode_list_t *lst);
-    const std::vector<osd_num_t> & list_inode_get_inactive_osds(inode_list_t *lst);
-    void list_inode_next(inode_list_t *lst, int next_pgs);
+
+    void list_inode(inode_t inode, uint64_t min_offset, uint64_t max_offset, int max_parallel_pgs, std::function<void(
+        int status, int pgs_left, pg_num_t pg_num, std::set<object_id>&& objects)> pg_callback);
+
     //inline uint32_t get_bs_bitmap_granularity() { return st_cli.global_bitmap_granularity; }
     //inline uint64_t get_bs_block_size() { return st_cli.global_block_size; }
     uint64_t next_op_id();
@@ -158,7 +166,7 @@ protected:
     bool check_rw(cluster_op_t *op);
     void slice_rw(cluster_op_t *op);
     void reset_retry_timer(int new_duration);
-    bool try_send(cluster_op_t *op, int i);
+    int try_send(cluster_op_t *op, int i);
     int continue_sync(cluster_op_t *op);
     void send_sync(cluster_op_t *op, cluster_op_part_t *part);
     void handle_op_part(cluster_op_part_t *part);
@@ -167,8 +175,14 @@ protected:
     void calc_wait(cluster_op_t *op);
     void inc_wait(uint64_t opcode, uint64_t flags, cluster_op_t *next, int inc);
     void continue_lists();
-    void continue_listing(inode_list_t *lst);
+    bool continue_listing(inode_list_t *lst);
+    bool restart_listing(inode_list_t* lst);
+    void retry_start_pg_listing(inode_list_pg_t *pg);
+    int start_pg_listing(inode_list_pg_t *pg);
     void send_list(inode_list_osd_t *cur_list);
+    void set_list_retry_timeout(int ms, timespec new_time);
+    void finish_list_pg(inode_list_pg_t *pg, bool retry_epipe);
+    bool check_finish_listing(inode_list_t *lst);
     void continue_raw_ops(osd_num_t peer_osd);
 
     friend class writeback_cache_t;

src/client/cluster_client_list.cpp

@@ -2,9 +2,17 @@
 // License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #include <algorithm>
+#include "assert.h"
 #include "pg_states.h"
 #include "cluster_client.h"
 
+#define LIST_PG_INIT 0
+#define LIST_PG_WAIT_ACTIVE 1
+#define LIST_PG_WAIT_CONNECT 2
+#define LIST_PG_WAIT_RETRY 3
+#define LIST_PG_SENT 4
+#define LIST_PG_DONE 5
+
 struct inode_list_t;
 
 struct inode_list_pg_t;
@@ -13,20 +21,22 @@ struct inode_list_osd_t
 {
     inode_list_pg_t *pg = NULL;
     osd_num_t osd_num = 0;
-    bool sent = false;
 };
 
 struct inode_list_pg_t
 {
     inode_list_t *lst = NULL;
-    int pos = 0;
-    pg_num_t pg_num;
-    osd_num_t cur_primary;
-    bool has_unstable = false;
-    int sent = 0;
-    int done = 0;
+    int errcode = 0;
+    pg_num_t pg_num = 0;
+    osd_num_t cur_primary = 0;
+    int state = 0;
+    int inflight_ops = 0;
+    timespec wait_until;
     std::vector<inode_list_osd_t> list_osds;
+
+    bool has_unstable = false;
     std::set<object_id> objects;
+    std::vector<osd_num_t> inactive_osds;
 };
 
 struct inode_list_t
@@ -34,175 +44,295 @@ struct inode_list_t
     cluster_client_t *cli = NULL;
     pool_id_t pool_id = 0;
     inode_t inode = 0;
+    uint64_t min_offset = 0;
+    uint64_t max_offset = 0;
+    int max_parallel_pgs = 16;
+
+    bool fallback = false;
+    int inflight_pgs = 0;
+    std::map<osd_num_t, int> inflight_per_osd;
     int done_pgs = 0;
-    int want = 0;
-    std::vector<osd_num_t> inactive_osds;
+    int onstack = 0;
     std::vector<inode_list_pg_t*> pgs;
-    std::function<void(inode_list_t* lst, std::set<object_id>&& objects, pg_num_t pg_num, osd_num_t primary_osd, int status)> callback;
+    pg_num_t real_pg_count = 0;
+    std::function<void(int status, int pgs_left, pg_num_t pg_num, std::set<object_id>&& objects)> callback;
 };
 
-inode_list_t* cluster_client_t::list_inode_start(inode_t inode,
-    std::function<void(inode_list_t* lst, std::set<object_id>&& objects, pg_num_t pg_num, osd_num_t primary_osd, int status)> callback)
+void cluster_client_t::list_inode(inode_t inode, uint64_t min_offset, uint64_t max_offset, int max_parallel_pgs, std::function<void(
+    int status, int pgs_left, pg_num_t pg_num, std::set<object_id>&& objects)> pg_callback)
 {
     init_msgr();
-    int skipped_pgs = 0;
     pool_id_t pool_id = INODE_POOL(inode);
     if (!pool_id || st_cli.pool_config.find(pool_id) == st_cli.pool_config.end())
     {
         if (log_level > 0)
-        {
             fprintf(stderr, "Pool %u does not exist\n", pool_id);
-        }
-        return NULL;
+        pg_callback(-EINVAL, 0, 0, std::set<object_id>());
+        return;
     }
+    auto pg_stripe_size = st_cli.pool_config.at(pool_id).pg_stripe_size;
+    if (min_offset)
+        min_offset = (min_offset/pg_stripe_size) * pg_stripe_size;
     inode_list_t *lst = new inode_list_t();
     lst->cli = this;
     lst->pool_id = pool_id;
     lst->inode = inode;
-    lst->callback = callback;
-    auto pool_cfg = st_cli.pool_config[pool_id];
-    std::set<osd_num_t> inactive_osd_set;
-    for (auto & pg_item: pool_cfg.pg_config)
-    {
-        auto & pg = pg_item.second;
-        if (pg.pause || !pg.cur_primary || !(pg.cur_state & PG_ACTIVE))
-        {
-            skipped_pgs++;
-            if (log_level > 0)
-            {
-                fprintf(stderr, "PG %u is inactive, skipping\n", pg_item.first);
-            }
-            continue;
-        }
-        inode_list_pg_t *r = new inode_list_pg_t();
-        r->lst = lst;
-        r->pg_num = pg_item.first;
-        r->cur_primary = pg.cur_primary;
-        if (pg.cur_state != PG_ACTIVE)
-        {
-            // Not clean
-            std::set<osd_num_t> all_peers;
-            for (osd_num_t pg_osd: pg.target_set)
-            {
-                if (pg_osd != 0)
-                {
-                    all_peers.insert(pg_osd);
-                }
-            }
-            for (osd_num_t pg_osd: pg.all_peers)
-            {
-                if (pg_osd != 0)
-                {
-                    all_peers.insert(pg_osd);
-                }
-            }
-            for (auto & hist_item: pg.target_history)
-            {
-                for (auto pg_osd: hist_item)
-                {
-                    if (pg_osd != 0)
-                    {
-                        all_peers.insert(pg_osd);
-                    }
-                }
-            }
-            for (osd_num_t peer_osd: all_peers)
-            {
-                if (st_cli.peer_states.find(peer_osd) != st_cli.peer_states.end())
-                {
-                    r->list_osds.push_back((inode_list_osd_t){
-                        .pg = r,
-                        .osd_num = peer_osd,
-                        .sent = false,
-                    });
-                }
-                else
-                {
-                    inactive_osd_set.insert(peer_osd);
-                }
-            }
-        }
-        else
-        {
-            // Clean
-            r->list_osds.push_back((inode_list_osd_t){
-                .pg = r,
-                .osd_num = pg.cur_primary,
-                .sent = false,
-            });
-        }
-        lst->pgs.push_back(r);
-    }
-    std::sort(lst->pgs.begin(), lst->pgs.end(), [](inode_list_pg_t *a, inode_list_pg_t *b)
-    {
-        return a->cur_primary < b->cur_primary ? true : false;
-    });
-    for (int i = 0; i < lst->pgs.size(); i++)
-    {
-        lst->pgs[i]->pos = i;
-    }
-    lst->inactive_osds.insert(lst->inactive_osds.end(), inactive_osd_set.begin(), inactive_osd_set.end());
+    lst->min_offset = min_offset;
+    lst->max_offset = max_offset;
+    lst->callback = pg_callback;
+    lst->max_parallel_pgs = max_parallel_pgs <= 0 ? 16 : max_parallel_pgs;
     lists.push_back(lst);
-    return lst;
-}
-
-int cluster_client_t::list_pg_count(inode_list_t *lst)
-{
-    return lst->pgs.size();
-}
-
-const std::vector<osd_num_t> & cluster_client_t::list_inode_get_inactive_osds(inode_list_t *lst)
-{
-    return lst->inactive_osds;
-}
-
-void cluster_client_t::list_inode_next(inode_list_t *lst, int next_pgs)
-{
-    if (next_pgs >= 0)
-    {
-        lst->want += next_pgs;
-    }
     continue_listing(lst);
 }
 
-void cluster_client_t::continue_listing(inode_list_t *lst)
+bool cluster_client_t::continue_listing(inode_list_t *lst)
 {
-    if (lst->done_pgs >= lst->pgs.size())
+    if (lst->onstack > 0)
     {
-        return;
+        return true;
     }
-    if (lst->want <= 0)
+    lst->onstack++;
+    if (restart_listing(lst))
     {
-        return;
-    }
-    for (int i = 0; i < lst->pgs.size(); i++)
-    {
-        if (lst->pgs[i] && lst->pgs[i]->sent < lst->pgs[i]->list_osds.size())
+        for (int i = 0; i < lst->pgs.size() && lst->inflight_pgs < lst->max_parallel_pgs; i++)
         {
-            for (int j = 0; j < lst->pgs[i]->list_osds.size(); j++)
+            retry_start_pg_listing(lst->pgs[i]);
+        }
+    }
+    if (check_finish_listing(lst))
+    {
+        // Do not change lst->onstack because it's already freed
+        return false;
+    }
+    lst->onstack--;
+    return true;
+}
+
+bool cluster_client_t::restart_listing(inode_list_t* lst)
+{
+    auto pool_it = st_cli.pool_config.find(lst->pool_id);
+    // We want listing to be consistent. To achieve it we should:
+    // 1) retry listing of each PG if its state changes
+    // 2) abort listing if PG count changes during listing
+    // 3) ideally, only talk to the primary OSD - this will be done separately
+    // So first we add all PGs without checking their state
+    if (pool_it == st_cli.pool_config.end() ||
+        lst->real_pg_count != pool_it->second.real_pg_count)
+    {
+        for (auto pg: lst->pgs)
+        {
+            if (pg->inflight_ops > 0)
             {
-                send_list(&lst->pgs[i]->list_osds[j]);
-                if (lst->want <= 0)
-                {
-                    return;
-                }
+                // Wait until all in-progress listings complete or fail
+                return false;
+            }
+        }
+        for (auto pg: lst->pgs)
+        {
+            delete pg;
+        }
+        if (log_level > 0 && lst->real_pg_count)
+        {
+            fprintf(stderr, "PG count in pool %u changed during listing\n", lst->pool_id);
+        }
+        lst->pgs.clear();
+        if (pool_it == st_cli.pool_config.end())
+        {
+            // Unknown pool
+            lst->callback(-EINVAL, 0, 0, std::set<object_id>());
+            return false;
+        }
+        else if (lst->done_pgs)
+        {
+            // PG count changed during listing, it should fail
+            lst->callback(-EAGAIN, 0, 0, std::set<object_id>());
+            return false;
+        }
+        else
+        {
+            lst->real_pg_count = pool_it->second.real_pg_count;
+            for (pg_num_t pg_num = 1; pg_num <= lst->real_pg_count; pg_num++)
+            {
+                inode_list_pg_t *pg = new inode_list_pg_t();
+                pg->lst = lst;
+                pg->pg_num = pg_num;
+                lst->pgs.push_back(pg);
             }
         }
     }
+    return true;
+}
+
+void cluster_client_t::retry_start_pg_listing(inode_list_pg_t *pg)
+{
+    if (pg->state == LIST_PG_SENT || pg->state == LIST_PG_DONE)
+    {
+        return;
+    }
+    if (pg->state == LIST_PG_WAIT_RETRY)
+    {
+        // Check if the timeout expired
+        timespec tv;
+        clock_gettime(CLOCK_REALTIME, &tv);
+        if (tv.tv_sec < pg->wait_until.tv_sec ||
+            tv.tv_sec == pg->wait_until.tv_sec && tv.tv_nsec < pg->wait_until.tv_nsec)
+        {
+            return;
+        }
+    }
+    int new_st = start_pg_listing(pg);
+    if (new_st == LIST_PG_SENT || new_st == LIST_PG_WAIT_CONNECT)
+    {
+        // sent => wait for completion
+        // not connected, but OSD state exists => wait for PG or OSD state change infinitely
+        pg->state = new_st;
+        return;
+    }
+    if (new_st == LIST_PG_WAIT_ACTIVE && pg->state != LIST_PG_WAIT_ACTIVE)
+    {
+        if (!client_wait_up_timeout)
+        {
+            fprintf(stderr, "PG %u/%u is inactive, skipping listing\n", pg->lst->pool_id, pg->pg_num);
+            pg->errcode = -EPIPE;
+            pg->list_osds.clear();
+            pg->objects.clear();
+            finish_list_pg(pg, false);
+            return;
+        }
+        pg->state = new_st;
+        clock_gettime(CLOCK_REALTIME, &pg->wait_until);
+        pg->wait_until.tv_sec += client_wait_up_timeout;
+        if (log_level > 1)
+        {
+            fprintf(stderr, "Waiting for PG %u/%u to become active for %d seconds\n", pg->lst->pool_id, pg->pg_num, client_wait_up_timeout);
+        }
+        set_list_retry_timeout(client_wait_up_timeout*1000, pg->wait_until);
+        return;
+    }
+    assert(pg->state == LIST_PG_WAIT_ACTIVE);
+    // Check if the timeout expired
+    timespec tv;
+    clock_gettime(CLOCK_REALTIME, &tv);
+    if (tv.tv_sec > pg->wait_until.tv_sec ||
+        tv.tv_sec == pg->wait_until.tv_sec && tv.tv_nsec >= pg->wait_until.tv_nsec)
+    {
+        fprintf(stderr, "Failed to wait for PG %u/%u to become active, skipping listing\n", pg->lst->pool_id, pg->pg_num);
+        pg->errcode = -EPIPE;
+        pg->list_osds.clear();
+        pg->objects.clear();
+        finish_list_pg(pg, false);
+    }
+}
+
+void cluster_client_t::set_list_retry_timeout(int ms, timespec new_time)
+{
+    if (!list_retry_time.tv_sec || list_retry_time.tv_sec > new_time.tv_sec ||
+        list_retry_time.tv_sec == new_time.tv_sec && list_retry_time.tv_nsec > new_time.tv_nsec)
+    {
+        list_retry_time = new_time;
+        if (list_retry_timeout_id >= 0)
+        {
+            tfd->clear_timer(list_retry_timeout_id);
+        }
+        list_retry_timeout_id = tfd->set_timer(ms, false, [this](int timer_id)
+        {
+            list_retry_timeout_id = -1;
+            list_retry_time = {};
+            continue_lists();
+        });
+    }
+}
+
+int cluster_client_t::start_pg_listing(inode_list_pg_t *pg)
+{
+    auto & pool_cfg = st_cli.pool_config.at(pg->lst->pool_id);
+    auto pg_it = pool_cfg.pg_config.find(pg->pg_num);
+    assert(pg->lst->real_pg_count == pool_cfg.real_pg_count);
+    if (pg_it == pool_cfg.pg_config.end() ||
+        pg_it->second.pause ||
+        !pg_it->second.cur_primary ||
+        !(pg_it->second.cur_state & PG_ACTIVE))
+    {
+        // PG is (temporarily?) unavailable
+        return LIST_PG_WAIT_ACTIVE;
+    }
+    pg->inactive_osds.clear();
+    std::set<osd_num_t> all_peers;
+    if (pg_it->second.cur_state != PG_ACTIVE && pg->lst->fallback)
+    {
+        // Not clean and OSDs don't support listing from primary
+        for (osd_num_t pg_osd: pg_it->second.target_set)
+            all_peers.insert(pg_osd);
+        for (osd_num_t pg_osd: pg_it->second.all_peers)
+            all_peers.insert(pg_osd);
+        for (auto & hist_item: pg_it->second.target_history)
+            for (auto pg_osd: hist_item)
+                all_peers.insert(pg_osd);
+        // Remove zero OSD number
+        all_peers.erase(0);
+        // Remove unconnectable peers except cur_primary
+        for (auto peer_it = all_peers.begin(); peer_it != all_peers.end(); )
+        {
+            if (*peer_it != pg_it->second.cur_primary &&
+                st_cli.peer_states[*peer_it].is_null())
+            {
+                pg->inactive_osds.push_back(*peer_it);
+                all_peers.erase(peer_it++);
+            }
+            else
+                peer_it++;
+        }
+    }
+    else
+    {
+        // Clean
+        all_peers.insert(pg_it->second.cur_primary);
+    }
+    // Check that we're connected to all PG OSDs
+    bool conn = true;
+    for (osd_num_t peer_osd: all_peers)
+    {
+        if (msgr.osd_peer_fds.find(peer_osd) == msgr.osd_peer_fds.end())
+        {
+            // Initiate connection
+            if (st_cli.peer_states[peer_osd].is_null())
+            {
+                return LIST_PG_WAIT_ACTIVE;
+            }
+            msgr.connect_peer(peer_osd, st_cli.peer_states[peer_osd]);
+            conn = false;
+        }
+    }
+    if (!conn)
+    {
+        return LIST_PG_WAIT_CONNECT;
+    }
+    // Send all listings at once as the simplest way to guarantee that we connect
+    // to the exact same OSDs that are listed in PG state
+    pg->errcode = 0;
+    pg->list_osds.clear();
+    pg->has_unstable = false;
+    pg->objects.clear();
+    pg->cur_primary = pg_it->second.cur_primary;
+    for (osd_num_t peer_osd: all_peers)
+    {
+        pg->list_osds.push_back((inode_list_osd_t){
+            .pg = pg,
+            .osd_num = peer_osd,
+        });
+    }
+    for (auto & list_osd: pg->list_osds)
+    {
+        send_list(&list_osd);
+    }
+    return LIST_PG_SENT;
 }
 
 void cluster_client_t::send_list(inode_list_osd_t *cur_list)
 {
-    if (cur_list->sent)
-    {
-        return;
-    }
-    if (msgr.osd_peer_fds.find(cur_list->osd_num) == msgr.osd_peer_fds.end())
-    {
-        // Initiate connection
-        msgr.connect_peer(cur_list->osd_num, st_cli.peer_states[cur_list->osd_num]);
-        return;
-    }
+    if (!cur_list->pg->inflight_ops)
+        cur_list->pg->lst->inflight_pgs++;
+    cur_list->pg->inflight_ops++;
     auto & pool_cfg = st_cli.pool_config[cur_list->pg->lst->pool_id];
     osd_op_t *op = new osd_op_t();
     op->op_type = OSD_OP_OUT;
@@ -220,6 +350,9 @@ void cluster_client_t::send_list(inode_list_osd_t *cur_list)
             .pg_stripe_size = pool_cfg.pg_stripe_size,
             .min_inode = cur_list->pg->lst->inode,
             .max_inode = cur_list->pg->lst->inode,
+            .min_stripe = cur_list->pg->lst->min_offset,
+            .max_stripe = cur_list->pg->lst->max_offset,
+            .flags = (uint64_t)(cur_list->pg->lst->fallback ? 0 : OSD_LIST_PRIMARY),
         },
     };
     op->callback = [this, cur_list](osd_op_t *op)
@@ -228,6 +361,29 @@ void cluster_client_t::send_list(inode_list_osd_t *cur_list)
         {
             fprintf(stderr, "Failed to get PG %u/%u object list from OSD %ju (retval=%jd), skipping\n",
                 cur_list->pg->lst->pool_id, cur_list->pg->pg_num, cur_list->osd_num, op->reply.hdr.retval);
+            if (!cur_list->pg->errcode ||
+                cur_list->pg->errcode == -EPIPE ||
+                op->reply.hdr.retval != -EPIPE)
+            {
+                cur_list->pg->errcode = op->reply.hdr.retval;
+            }
+        }
+        else if ((op->req.sec_list.flags & OSD_LIST_PRIMARY) &&
+            !(op->reply.sec_list.flags & OSD_LIST_PRIMARY))
+        {
+            // OSD is old and doesn't support listing from primary
+            if (log_level > 0)
+            {
+                fprintf(
+                    stderr, "[PG %u/%u] Primary OSD doesn't support consistent listings, falling back to listings from all peers\n",
+                    cur_list->pg->lst->pool_id, cur_list->pg->pg_num
+                );
+            }
+            cur_list->pg->lst->fallback = true;
+            if (!cur_list->pg->errcode)
+            {
+                cur_list->pg->errcode = -EPIPE;
+            }
         }
         else
         {
@@ -256,55 +412,66 @@ void cluster_client_t::send_list(inode_list_osd_t *cur_list)
             }
         }
         delete op;
-        auto lst = cur_list->pg->lst;
-        auto pg = cur_list->pg;
-        pg->done++;
-        if (pg->done >= pg->list_osds.size())
-        {
-            int status = 0;
-            lst->done_pgs++;
-            if (lst->done_pgs >= lst->pgs.size())
-            {
-                status |= INODE_LIST_DONE;
-            }
-            if (pg->has_unstable)
-            {
-                status |= INODE_LIST_HAS_UNSTABLE;
-            }
-            lst->callback(lst, std::move(pg->objects), pg->pg_num, pg->cur_primary, status);
-            lst->pgs[pg->pos] = NULL;
-            delete pg;
-            if (lst->done_pgs >= lst->pgs.size())
-            {
-                // All done
-                for (int i = 0; i < lists.size(); i++)
-                {
-                    if (lists[i] == lst)
-                    {
-                        lists.erase(lists.begin()+i, lists.begin()+i+1);
-                        break;
-                    }
-                }
-                delete lst;
-                return;
-            }
-        }
-        else
-        {
-            lst->want++;
-        }
-        continue_listing(lst);
+        cur_list->pg->inflight_ops--;
+        if (!cur_list->pg->inflight_ops)
+            cur_list->pg->lst->inflight_pgs--;
+        finish_list_pg(cur_list->pg, true);
+        continue_listing(cur_list->pg->lst);
     };
     msgr.outbox_push(op);
-    cur_list->sent = true;
-    cur_list->pg->sent++;
-    cur_list->pg->lst->want--;
+}
+
+void cluster_client_t::finish_list_pg(inode_list_pg_t *pg, bool retry_epipe)
+{
+    auto lst = pg->lst;
+    if (pg->inflight_ops == 0)
+    {
+        if (pg->errcode == -EPIPE && retry_epipe)
+        {
+            // Retry listing after <client_retry_interval> ms on EPIPE
+            pg->state = LIST_PG_WAIT_RETRY;
+            clock_gettime(CLOCK_REALTIME, &pg->wait_until);
+            pg->wait_until.tv_nsec += client_retry_interval*1000000;
+            pg->wait_until.tv_sec += (pg->wait_until.tv_nsec / 1000000000);
+            pg->wait_until.tv_nsec = (pg->wait_until.tv_nsec % 1000000000);
+            set_list_retry_timeout(client_retry_interval, pg->wait_until);
+            return;
+        }
+        lst->done_pgs++;
+        pg->state = LIST_PG_DONE;
+        lst->callback(pg->errcode, lst->pgs.size()-lst->done_pgs, pg->pg_num, std::move(pg->objects));
+        pg->objects.clear();
+        pg->inactive_osds.clear();
+    }
 }
 
 void cluster_client_t::continue_lists()
 {
-    for (auto lst: lists)
+    for (int i = lists.size()-1; i >= 0; i--)
     {
-        continue_listing(lst);
+        continue_listing(lists[i]);
     }
 }
+
+bool cluster_client_t::check_finish_listing(inode_list_t *lst)
+{
+    if (lst->done_pgs >= lst->pgs.size())
+    {
+        for (auto pg: lst->pgs)
+        {
+            delete pg;
+        }
+        lst->pgs.clear();
+        for (int i = 0; i < lists.size(); i++)
+        {
+            if (lists[i] == lst)
+            {
+                lists.erase(lists.begin()+i, lists.begin()+i+1);
+                break;
+            }
+        }
+        delete lst;
+        return true;
+    }
+    return false;
+}

src/client/cluster_client_wb.cpp

@@ -43,6 +43,7 @@ bool writeback_cache_t::is_left_merged(dirty_buf_it_t dirty_it)
         auto prev_it = dirty_it;
         prev_it--;
         if (prev_it->first.inode == dirty_it->first.inode &&
+            (prev_it->second.buf != NULL) == (dirty_it->second.buf != NULL) &&
             prev_it->first.stripe+prev_it->second.len == dirty_it->first.stripe &&
             prev_it->second.state == CACHE_DIRTY)
         {
@@ -58,6 +59,7 @@ bool writeback_cache_t::is_right_merged(dirty_buf_it_t dirty_it)
     next_it++;
     if (next_it != dirty_buffers.end() &&
         next_it->first.inode == dirty_it->first.inode &&
+        (next_it->second.buf != NULL) == (dirty_it->second.buf != NULL) &&
         next_it->first.stripe == dirty_it->first.stripe+dirty_it->second.len &&
         next_it->second.state == CACHE_DIRTY)
     {
@@ -99,7 +101,7 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
                     .inode = op->inode,
                     .stripe = new_end,
                 }, (cluster_buffer_t){
-                    .buf = dirty_it->second.buf + new_end - dirty_it->first.stripe,
+                    .buf = dirty_it->second.buf ? dirty_it->second.buf + new_end - dirty_it->first.stripe : NULL,
                     .len = old_end - new_end,
                     .state = dirty_it->second.state,
                     .flush_id = dirty_it->second.flush_id,
@@ -143,7 +145,7 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
                 .inode = op->inode,
                 .stripe = new_end,
             }, (cluster_buffer_t){
-                .buf = dirty_it->second.buf + new_end - dirty_it->first.stripe,
+                .buf = dirty_it->second.buf ? dirty_it->second.buf + new_end - dirty_it->first.stripe : NULL,
                 .len = old_end - new_end,
                 .state = dirty_it->second.state,
                 .flush_id = dirty_it->second.flush_id,
@@ -170,8 +172,9 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
         }
     }
     // Overlapping buffers are removed, just insert the new one
-    uint64_t *refcnt = (uint64_t*)malloc_or_die(sizeof(uint64_t) + op->len);
-    uint8_t *buf = (uint8_t*)refcnt + sizeof(uint64_t);
+    bool is_del = op->opcode == OSD_OP_DELETE;
+    uint64_t *refcnt = is_del ? NULL : (uint64_t*)malloc_or_die(sizeof(uint64_t) + op->len);
+    uint8_t *buf = is_del ? NULL : ((uint8_t*)refcnt + sizeof(uint64_t));
     *refcnt = 1;
     dirty_it = dirty_buffers.emplace_hint(dirty_it, (object_id){
         .inode = op->inode,
@@ -185,7 +188,7 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
     });
     if (state == CACHE_DIRTY)
     {
-        writeback_bytes += op->len;
+        writeback_bytes += is_del ? 0 : op->len;
         // Track consecutive write-back operations
         if (!is_merged(dirty_it))
         {
@@ -199,13 +202,16 @@ void writeback_cache_t::copy_write(cluster_op_t *op, int state, uint64_t new_flu
             });
         }
     }
-    uint64_t pos = 0, len = op->len, iov_idx = 0;
-    while (len > 0 && iov_idx < op->iov.count)
+    if (!is_del)
     {
-        auto & iov = op->iov.buf[iov_idx];
-        memcpy(buf + pos, iov.iov_base, iov.iov_len);
-        pos += iov.iov_len;
-        iov_idx++;
+        uint64_t pos = 0, len = op->len, iov_idx = 0;
+        while (len > 0 && iov_idx < op->iov.count)
+        {
+            auto & iov = op->iov.buf[iov_idx];
+            memcpy(buf + pos, iov.iov_base, iov.iov_len);
+            pos += iov.iov_len;
+            iov_idx++;
+        }
     }
 }
 
@@ -250,7 +256,7 @@ void writeback_cache_t::flush_buffers(cluster_client_t *cli, dirty_buf_it_t from
     bool is_writeback = from_it->second.state == CACHE_DIRTY;
     cluster_op_t *op = new cluster_op_t;
     op->flags = OSD_OP_IGNORE_READONLY|OP_FLUSH_BUFFER;
-    op->opcode = OSD_OP_WRITE;
+    op->opcode = from_it->second.buf ? OSD_OP_WRITE : OSD_OP_DELETE;
     op->cur_inode = op->inode = from_it->first.inode;
     op->offset = from_it->first.stripe;
     op->len = prev_it->first.stripe + prev_it->second.len - from_it->first.stripe;
@@ -260,9 +266,12 @@ void writeback_cache_t::flush_buffers(cluster_client_t *cli, dirty_buf_it_t from
     {
         it->second.state = CACHE_REPEATING;
         it->second.flush_id = flush_id;
-        (*it->second.refcnt)++;
-        flushed_buffers.emplace(flush_id, it->second.refcnt);
-        op->iov.push_back(it->second.buf, it->second.len);
+        if (it->second.buf)
+        {
+            (*it->second.refcnt)++;
+            flushed_buffers.emplace(flush_id, it->second.refcnt);
+            op->iov.push_back(it->second.buf, it->second.len);
+        }
         calc_len += it->second.len;
     }
     assert(calc_len == op->len);
@@ -334,10 +343,12 @@ void writeback_cache_t::start_writebacks(cluster_client_t *cli, int count)
         }
         auto from_it = dirty_it;
         uint64_t off = dirty_it->first.stripe;
+        bool is_del = (dirty_it->second.buf == NULL);
         while (from_it != dirty_buffers.begin())
         {
             from_it--;
             if (from_it->second.state != CACHE_DIRTY ||
+                (from_it->second.buf == NULL) != is_del ||
                 from_it->first.inode != req.inode ||
                 from_it->first.stripe+from_it->second.len != off)
             {
@@ -352,6 +363,7 @@ void writeback_cache_t::start_writebacks(cluster_client_t *cli, int count)
         while (to_it != dirty_buffers.end())
         {
             if (to_it->second.state != CACHE_DIRTY ||
+                (to_it->second.buf == NULL) != is_del ||
                 to_it->first.inode != req.inode ||
                 to_it->first.stripe != off)
             {
@@ -391,15 +403,27 @@ static void copy_to_op(cluster_op_t *op, uint64_t offset, uint8_t *buf, uint64_t
             auto & v = op->iov.buf[iov_idx];
             auto begin = (cur_offset < offset ? offset : cur_offset);
             auto end = (cur_offset+v.iov_len > offset+len ? offset+len : cur_offset+v.iov_len);
-            memcpy(
-                (uint8_t*)v.iov_base + begin - cur_offset,
-                buf + (cur_offset <= offset ? 0 : cur_offset-offset),
-                end - begin
-            );
+            if (!buf)
+            {
+                memset((uint8_t*)v.iov_base + begin - cur_offset, 0, end - begin);
+            }
+            else
+            {
+                memcpy(
+                    (uint8_t*)v.iov_base + begin - cur_offset,
+                    buf + (cur_offset <= offset ? 0 : cur_offset-offset),
+                    end - begin
+                );
+            }
             cur_offset += v.iov_len;
             iov_idx++;
         }
     }
+    if (!buf)
+    {
+        // Bitmap is initially zero, don't set it
+        return;
+    }
     // Set bitmap bits
     int start_bit = (offset-op->offset)/bitmap_granularity;
     int end_bit = (offset-op->offset+len)/bitmap_granularity;
@@ -449,7 +473,8 @@ bool writeback_cache_t::read_from_cache(cluster_op_t *op, uint32_t bitmap_granul
                     {
                         // Copy data
                         dirty_copied = true;
-                        copy_to_op(op, prev, dirty_it->second.buf + prev - dirty_it->first.stripe, cur-prev, bitmap_granularity);
+                        copy_to_op(op, prev, dirty_it->second.buf ? (dirty_it->second.buf + prev - dirty_it->first.stripe) : NULL,
+                            cur-prev, bitmap_granularity);
                     }
                     skip_prev = skip;
                     prev = cur;
@@ -461,7 +486,8 @@ bool writeback_cache_t::read_from_cache(cluster_op_t *op, uint32_t bitmap_granul
             {
                 // Copy data
                 dirty_copied = true;
-                copy_to_op(op, prev, dirty_it->second.buf + prev - dirty_it->first.stripe, cur-prev, bitmap_granularity);
+                copy_to_op(op, prev, dirty_it->second.buf ? (dirty_it->second.buf + prev - dirty_it->first.stripe) : NULL,
+                    cur-prev, bitmap_granularity);
             }
             dirty_it++;
         }

src/client/etcd_state_client.cpp

@@ -176,7 +176,7 @@ void etcd_state_client_t::add_etcd_url(std::string addr)
             exit(1);
         }
         if (!local_ips.size())
-            local_ips = getifaddr_list();
+            local_ips = getifaddr_list(std::vector<std::string>(), true);
         std::string check_addr;
         int pos = addr.find('/');
         int pos2 = addr.find(':');
@@ -785,7 +785,7 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
         }
         for (auto & pool_item: value.object_items())
         {
-            pool_config_t pc;
+            pool_config_t pc = {};
             // ID
             pool_id_t pool_id;
             char null_byte = 0;
@@ -931,12 +931,28 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
             // Ignore old key if the new one is present
             return;
         }
+        for (auto & pool_id_json: value["backfillfull_pools"].array_items())
+        {
+            auto pool_id = pool_id_json.uint64_value();
+            auto pool_it = this->pool_config.find(pool_id);
+            if (pool_it != this->pool_config.end())
+            {
+                pool_it->second.backfillfull |= 2;
+            }
+        }
         for (auto & pool_item: this->pool_config)
         {
             for (auto & pg_item: pool_item.second.pg_config)
             {
                 pg_item.second.config_exists = false;
             }
+            // 3 = was 1 and became 1, 0 = was 0 and became 0
+            if (pool_item.second.backfillfull == 2 || pool_item.second.backfillfull == 1)
+            {
+                if (on_change_backfillfull_hook)
+                    on_change_backfillfull_hook(pool_item.first);
+            }
+            pool_item.second.backfillfull = pool_item.second.backfillfull >> 1;
         }
         for (auto & pool_item: value["items"].object_items())
         {
@@ -1201,6 +1217,7 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
                     .size = value["size"].uint64_value(),
                     .parent_id = parent_inode_num,
                     .readonly = value["readonly"].bool_value(),
+                    .deleted = value["deleted"].bool_value(),
                     .meta = value["meta"],
                     .mod_revision = kv.mod_revision,
                 });
@@ -1289,6 +1306,10 @@ json11::Json::object etcd_state_client_t::serialize_inode_cfg(inode_config_t *cf
     {
         new_cfg["readonly"] = true;
     }
+    if (cfg->deleted)
+    {
+        new_cfg["deleted"] = true;
+    }
     if (cfg->meta.is_object())
     {
         new_cfg["meta"] = cfg->meta;

src/client/etcd_state_client.h

@@ -62,6 +62,7 @@ struct pool_config_t
     std::map<pg_num_t, pg_config_t> pg_config;
     uint64_t scrub_interval;
     std::string used_for_fs;
+    int backfillfull;
 };
 
 struct inode_config_t
@@ -71,6 +72,7 @@ struct inode_config_t
     uint64_t size = 0;
     inode_t parent_id = 0;
     bool readonly = false;
+    bool deleted = false;
     // Arbitrary metadata
     json11::Json meta;
     // Change revision of the metadata in etcd
@@ -131,6 +133,7 @@ public:
     std::function<json11::Json()> load_pgs_checks_hook;
     std::function<void(bool)> on_load_pgs_hook;
     std::function<void()> on_change_pool_config_hook;
+    std::function<void(pool_id_t)> on_change_backfillfull_hook;
     std::function<void(pool_id_t, pg_num_t, osd_num_t)> on_change_pg_state_hook;
     std::function<void(pool_id_t, pg_num_t)> on_change_pg_history_hook;
     std::function<void(osd_num_t)> on_change_osd_state_hook;

src/client/http_client.cpp

@@ -62,6 +62,7 @@ struct http_co_t
     inline void end() { ended = true; if (!onstack) { delete this; } }
     void run_cb_and_clear();
     void start_connection();
+    void start_ws_connection();
     void close_connection();
     void next_request();
     void handle_events();
@@ -112,7 +113,7 @@ http_co_t* open_websocket(timerfd_manager_t *tfd, const std::string & host, cons
     handler->keepalive = false;
     handler->request = request;
     handler->response_callback = response_callback;
-    handler->start_connection();
+    handler->start_ws_connection();
     return handler;
 }
 
@@ -282,6 +283,27 @@ void http_co_t::close_connection()
     epoll_events = 0;
 }
 
+void http_co_t::start_ws_connection()
+{
+    stackin();
+    start_connection();
+    if (request_timeout > 0)
+    {
+        timeout_id = tfd->set_timer(request_timeout, false, [this](int timer_id)
+        {
+            stackin();
+            if (state != HTTP_CO_WEBSOCKET)
+            {
+                close_connection();
+                parsed = { .error = "Websocket connection timed out" };
+                run_cb_and_clear();
+            }
+            stackout();
+        });
+    }
+    stackout();
+}
+
 void http_co_t::start_connection()
 {
     stackin();

src/client/messenger.cpp

@@ -121,7 +121,7 @@ void osd_messenger_t::init()
     if (use_rdma)
     {
         rdma_context = msgr_rdma_context_t::create(
-            rdma_device != "" ? rdma_device.c_str() : NULL,
+            osd_networks, rdma_device != "" ? rdma_device.c_str() : NULL,
             rdma_port_num, rdma_gid_index, rdma_mtu, rdma_odp, log_level
         );
         if (!rdma_context)
@@ -266,7 +266,8 @@ void osd_messenger_t::parse_config(const json11::Json & config)
     this->rdma_port_num = (uint8_t)config["rdma_port_num"].uint64_value();
     if (!this->rdma_port_num)
         this->rdma_port_num = 1;
-    this->rdma_gid_index = (uint8_t)config["rdma_gid_index"].uint64_value();
+    if (!config["rdma_gid_index"].is_null())
+        this->rdma_gid_index = (uint8_t)config["rdma_gid_index"].uint64_value();
     this->rdma_mtu = (uint32_t)config["rdma_mtu"].uint64_value();
     this->rdma_max_sge = config["rdma_max_sge"].uint64_value();
     if (!this->rdma_max_sge)
@@ -281,6 +282,15 @@ void osd_messenger_t::parse_config(const json11::Json & config)
     if (!this->rdma_max_msg || this->rdma_max_msg > 128*1024*1024)
         this->rdma_max_msg = 129*1024;
     this->rdma_odp = config["rdma_odp"].bool_value();
+    std::vector<std::string> mask;
+    if (config["bind_address"].is_string())
+        mask.push_back(config["bind_address"].string_value());
+    else if (config["osd_network"].is_string())
+        mask.push_back(config["osd_network"].string_value());
+    else
+        for (auto v: config["osd_network"].array_items())
+            mask.push_back(v.string_value());
+    this->osd_networks = mask;
 #endif
     if (!osd_num)
         this->iothread_count = (uint32_t)config["client_iothread_count"].uint64_value();

src/client/messenger.h

@@ -165,8 +165,10 @@ protected:
 
 #ifdef WITH_RDMA
     bool use_rdma = true;
+    std::vector<std::string> osd_networks;
     std::string rdma_device;
-    uint64_t rdma_port_num = 1, rdma_gid_index = 0, rdma_mtu = 0;
+    uint64_t rdma_port_num = 1, rdma_mtu = 0;
+    int rdma_gid_index = -1;
     msgr_rdma_context_t *rdma_context = NULL;
     uint64_t rdma_max_sge = 0, rdma_max_send = 0, rdma_max_recv = 0;
     uint64_t rdma_max_msg = 0;
@@ -177,7 +179,7 @@ protected:
     std::vector<int> read_ready_clients;
     std::vector<int> write_ready_clients;
     // We don't use ringloop->set_immediate here because we may have no ringloop in client :)
-    std::vector<std::function<void()>> set_immediate;
+    std::vector<osd_op_t*> set_immediate_ops;
 
 public:
     timerfd_manager_t *tfd;
@@ -237,6 +239,8 @@ protected:
     void handle_op_hdr(osd_client_t *cl);
     bool handle_reply_hdr(osd_client_t *cl);
     void handle_reply_ready(osd_op_t *op);
+    void handle_immediate_ops();
+    void clear_immediate_ops(int peer_fd);
 
 #ifdef WITH_RDMA
     void try_send_rdma(osd_client_t *cl);

src/client/msgr_rdma.cpp

@@ -3,6 +3,7 @@
 
 #include <stdio.h>
 #include <stdlib.h>
+#include "addr_util.h"
 #include "msgr_rdma.h"
 #include "messenger.h"
 
@@ -69,7 +70,138 @@ msgr_rdma_connection_t::~msgr_rdma_connection_t()
     send_out_size = 0;
 }
 
-msgr_rdma_context_t *msgr_rdma_context_t::create(const char *ib_devname, uint8_t ib_port, uint8_t gid_index, uint32_t mtu, bool odp, int log_level)
+#ifdef IBV_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT
+static bool is_ipv4_gid(ibv_gid_entry *gidx)
+{
+    return (((uint64_t*)gidx->gid.raw)[0] == 0 &&
+        ((uint32_t*)gidx->gid.raw)[2] == 0xffff0000);
+}
+
+static bool match_gid(ibv_gid_entry *gidx, addr_mask_t *networks, int nnet)
+{
+    if (gidx->gid_type != IBV_GID_TYPE_ROCE_V1 &&
+        gidx->gid_type != IBV_GID_TYPE_ROCE_V2 ||
+        ((uint64_t*)gidx->gid.raw)[0] == 0 &&
+        ((uint64_t*)gidx->gid.raw)[1] == 0)
+    {
+        return false;
+    }
+    if (is_ipv4_gid(gidx))
+    {
+        for (int i = 0; i < nnet; i++)
+        {
+            if (networks[i].family == AF_INET && cidr_match(*(in_addr*)(gidx->gid.raw+12), networks[i].ipv4, networks[i].bits))
+                return true;
+        }
+    }
+    else
+    {
+        for (int i = 0; i < nnet; i++)
+        {
+            if (networks[i].family == AF_INET6 && cidr6_match(*(in6_addr*)gidx->gid.raw, networks[i].ipv6, networks[i].bits))
+                return true;
+        }
+    }
+    return false;
+}
+
+struct matched_dev
+{
+    int dev = -1;
+    int port = -1;
+    int gid = -1;
+    bool rocev2 = false;
+};
+
+static void log_rdma_dev_port_gid(ibv_device *dev, int ib_port, int gid_index, ibv_gid_entry & gidx)
+{
+    bool is4 = ((uint64_t*)gidx.gid.raw)[0] == 0 && ((uint32_t*)gidx.gid.raw)[2] == 0xffff0000;
+    char buf[256];
+    inet_ntop(is4 ? AF_INET : AF_INET6, is4 ? gidx.gid.raw+12 : gidx.gid.raw, buf, sizeof(buf));
+    fprintf(
+        stderr, "Auto-selected RDMA device %s port %d GID %d - ROCEv%d IPv%d %s\n",
+        ibv_get_device_name(dev), ib_port, gid_index,
+        gidx.gid_type == IBV_GID_TYPE_ROCE_V2 ? 2 : 1, is4 ? 4 : 6, buf
+    );
+}
+
+static matched_dev match_device(ibv_device **dev_list, addr_mask_t *networks, int nnet, int log_level)
+{
+    matched_dev best;
+    ibv_device_attr attr;
+    ibv_port_attr portinfo;
+    ibv_gid_entry best_gidx;
+    int res;
+    bool have_non_roce = false, have_roce = false;
+    for (int i = 0; dev_list[i]; ++i)
+    {
+        auto dev = dev_list[i];
+        ibv_context *context = ibv_open_device(dev_list[i]);
+        if ((res = ibv_query_device(context, &attr)) != 0)
+        {
+            fprintf(stderr, "Couldn't query RDMA device %s for its features: %s\n", ibv_get_device_name(dev_list[i]), strerror(res));
+            goto cleanup;
+        }
+        for (int j = 1; j <= attr.phys_port_cnt; j++)
+        {
+            // Try to find a port with matching address
+            if ((res = ibv_query_port(context, j, &portinfo)) != 0)
+            {
+                fprintf(stderr, "Couldn't get RDMA device %s port %d info: %s\n", ibv_get_device_name(dev), j, strerror(res));
+                goto cleanup;
+            }
+            for (int k = 0; k < portinfo.gid_tbl_len; k++)
+            {
+                ibv_gid_entry gidx;
+                if ((res = ibv_query_gid_ex(context, j, k, &gidx, 0)) != 0)
+                {
+                    if (res != ENODATA)
+                    {
+                        fprintf(stderr, "Couldn't read RDMA device %s GID index %d: %s\n", ibv_get_device_name(dev), k, strerror(res));
+                        goto cleanup;
+                    }
+                    else
+                        break;
+                }
+                if (gidx.gid_type != IBV_GID_TYPE_ROCE_V1 &&
+                    gidx.gid_type != IBV_GID_TYPE_ROCE_V2)
+                    have_non_roce = true;
+                else
+                    have_roce = true;
+                if (match_gid(&gidx, networks, nnet))
+                {
+                    // Prefer RoCEv2
+                    if (!best.rocev2)
+                    {
+                        best.dev = i;
+                        best.port = j;
+                        best.gid = k;
+                        best.rocev2 = (gidx.gid_type == IBV_GID_TYPE_ROCE_V2);
+                        best_gidx = gidx;
+                    }
+                }
+            }
+        }
+cleanup:
+        ibv_close_device(context);
+        if (best.rocev2)
+        {
+            break;
+        }
+    }
+    if (best.dev >= 0 && log_level > 0)
+    {
+        log_rdma_dev_port_gid(dev_list[best.dev], best.port, best.gid, best_gidx);
+    }
+    if (best.dev < 0 && have_non_roce && !have_roce)
+    {
+        best.dev = -2;
+    }
+    return best;
+}
+#endif
+
+msgr_rdma_context_t *msgr_rdma_context_t::create(std::vector<std::string> osd_networks, const char *ib_devname, uint8_t ib_port, int gid_index, uint32_t mtu, bool odp, int log_level)
 {
     int res;
     ibv_device **dev_list = NULL;
@@ -80,28 +212,23 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(const char *ib_devname, uint8_t
     clock_gettime(CLOCK_REALTIME, &tv);
     srand48(tv.tv_sec*1000000000 + tv.tv_nsec);
     dev_list = ibv_get_device_list(NULL);
-    if (!dev_list)
+    if (!dev_list || !*dev_list)
     {
         if (errno == -ENOSYS || errno == ENOSYS)
         {
             if (log_level > 0)
                 fprintf(stderr, "No RDMA devices found (RDMA device list returned ENOSYS)\n");
         }
+        else if (!*dev_list)
+        {
+            if (log_level > 0)
+                fprintf(stderr, "No RDMA devices found\n");
+        }
         else
             fprintf(stderr, "Failed to get RDMA device list: %s\n", strerror(errno));
         goto cleanup;
     }
-    if (!ib_devname)
-    {
-        ctx->dev = *dev_list;
-        if (!ctx->dev)
-        {
-            if (log_level > 0)
-                fprintf(stderr, "No RDMA devices found\n");
-            goto cleanup;
-        }
-    }
-    else
+    if (ib_devname)
     {
         int i;
         for (i = 0; dev_list[i]; ++i)
@@ -114,6 +241,39 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(const char *ib_devname, uint8_t
             goto cleanup;
         }
     }
+#ifdef IBV_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT
+    else if (osd_networks.size())
+    {
+        std::vector<addr_mask_t> nets;
+        for (auto & netstr: osd_networks)
+        {
+            nets.push_back(cidr_parse(netstr));
+        }
+        auto best = match_device(dev_list, nets.data(), nets.size(), log_level);
+        if (best.dev == -2)
+        {
+            best.dev = 0;
+            if (log_level > 0)
+                fprintf(stderr, "No RoCE devices found, using first available RDMA device %s\n", ibv_get_device_name(*dev_list));
+        }
+        else if (best.dev < 0)
+        {
+            if (log_level > 0)
+                fprintf(stderr, "RDMA device matching osd_network is not found, disabling RDMA\n");
+            goto cleanup;
+        }
+        else
+        {
+            ib_port = best.port;
+            gid_index = best.gid;
+        }
+        ctx->dev = dev_list[best.dev];
+    }
+#endif
+    else
+    {
+        ctx->dev = *dev_list;
+    }
 
     ctx->context = ibv_open_device(ctx->dev);
     if (!ctx->context)
@@ -123,7 +283,6 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(const char *ib_devname, uint8_t
     }
 
     ctx->ib_port = ib_port;
-    ctx->gid_index = gid_index;
     if ((res = ibv_query_port(ctx->context, ib_port, &ctx->portinfo)) != 0)
     {
         fprintf(stderr, "Couldn't get RDMA device %s port %d info: %s\n", ibv_get_device_name(ctx->dev), ib_port, strerror(res));
@@ -135,11 +294,55 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(const char *ib_devname, uint8_t
         fprintf(stderr, "RDMA device %s must have local LID because it's not Ethernet, but LID is zero\n", ibv_get_device_name(ctx->dev));
         goto cleanup;
     }
-    if (ibv_query_gid(ctx->context, ib_port, gid_index, &ctx->my_gid))
+
+#ifdef IBV_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT
+    if (gid_index != -1)
+#endif
     {
-        fprintf(stderr, "Couldn't read RDMA device %s GID index %d\n", ibv_get_device_name(ctx->dev), gid_index);
-        goto cleanup;
+        ctx->gid_index = gid_index < 0 ? 0 : gid_index;
+        if (ibv_query_gid(ctx->context, ib_port, gid_index, &ctx->my_gid))
+        {
+            fprintf(stderr, "Couldn't read RDMA device %s GID index %d\n", ibv_get_device_name(ctx->dev), gid_index);
+            goto cleanup;
+        }
     }
+#ifdef IBV_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT
+    else
+    {
+        // Auto-guess GID
+        ibv_gid_entry best_gidx;
+        for (int k = 0; k < ctx->portinfo.gid_tbl_len; k++)
+        {
+            ibv_gid_entry gidx;
+            if (ibv_query_gid_ex(ctx->context, ib_port, k, &gidx, 0) != 0)
+            {
+                fprintf(stderr, "Couldn't read RDMA device %s GID index %d\n", ibv_get_device_name(ctx->dev), k);
+                goto cleanup;
+            }
+            // Skip empty GID
+            if (((uint64_t*)gidx.gid.raw)[0] == 0 &&
+                ((uint64_t*)gidx.gid.raw)[1] == 0)
+            {
+                continue;
+            }
+            // Prefer IPv4 RoCEv2 -> IPv6 RoCEv2 -> IPv4 RoCEv1 -> IPv6 RoCEv1 -> IB
+            if (gid_index == -1 ||
+                gidx.gid_type == IBV_GID_TYPE_ROCE_V2 && best_gidx.gid_type != IBV_GID_TYPE_ROCE_V2 ||
+                gidx.gid_type == IBV_GID_TYPE_ROCE_V1 && best_gidx.gid_type == IBV_GID_TYPE_IB ||
+                gidx.gid_type == best_gidx.gid_type && is_ipv4_gid(&gidx))
+            {
+                gid_index = k;
+                best_gidx = gidx;
+            }
+        }
+        ctx->gid_index = gid_index = (gid_index == -1 ? 0 : gid_index);
+        if (log_level > 0)
+        {
+            log_rdma_dev_port_gid(ctx->dev, ctx->ib_port, ctx->gid_index, best_gidx);
+        }
+        ctx->my_gid = best_gidx.gid;
+    }
+#endif
 
     ctx->pd = ibv_alloc_pd(ctx->context);
     if (!ctx->pd)
@@ -398,7 +601,7 @@ static int try_send_rdma_copy(osd_client_t *cl, uint8_t *dst, int dst_len)
         iovec & iov = cl->send_list[rc->send_pos];
         uint32_t len = (uint32_t)(iov.iov_len-rc->send_buf_pos < dst_len
             ? iov.iov_len-rc->send_buf_pos : dst_len);
-        memcpy(dst, iov.iov_base+rc->send_buf_pos, len);
+        memcpy(dst, (uint8_t*)iov.iov_base+rc->send_buf_pos, len);
         dst += len;
         dst_len -= len;
         rc->send_buf_pos += len;
@@ -598,6 +801,7 @@ void osd_messenger_t::handle_rdma_events()
                 }
                 fprintf(stderr, " with status: %s, stopping client\n", ibv_wc_status_str(wc[i].status));
                 stop_client(client_id);
+                clear_immediate_ops(client_id);
                 continue;
             }
             if (!is_send)
@@ -606,6 +810,7 @@ void osd_messenger_t::handle_rdma_events()
                 if (!handle_read_buffer(cl, rc->recv_buffers[rc->next_recv_buf].buf, wc[i].byte_len))
                 {
                     // handle_read_buffer may stop the client
+                    clear_immediate_ops(client_id);
                     continue;
                 }
                 try_recv_rdma_wr(cl, rc->recv_buffers[rc->next_recv_buf]);
@@ -666,9 +871,5 @@ void osd_messenger_t::handle_rdma_events()
             }
         }
     } while (event_count > 0);
-    for (auto cb: set_immediate)
-    {
-        cb();
-    }
-    set_immediate.clear();
+    handle_immediate_ops();
 }

src/client/msgr_rdma.h

@@ -36,7 +36,7 @@ struct msgr_rdma_context_t
     int max_cqe = 0;
     int used_max_cqe = 0;
 
-    static msgr_rdma_context_t *create(const char *ib_devname, uint8_t ib_port, uint8_t gid_index, uint32_t mtu, bool odp, int log_level);
+    static msgr_rdma_context_t *create(std::vector<std::string> osd_networks, const char *ib_devname, uint8_t ib_port, int gid_index, uint32_t mtu, bool odp, int log_level);
     ~msgr_rdma_context_t();
 };
 

src/client/msgr_receive.cpp

@@ -65,6 +65,7 @@ void osd_messenger_t::read_requests()
 bool osd_messenger_t::handle_read(int result, osd_client_t *cl)
 {
     bool ret = false;
+    int peer_fd = cl->peer_fd;
     cl->read_msg.msg_iovlen = 0;
     cl->refs--;
     if (cl->peer_state == PEER_STOPPED)
@@ -101,7 +102,8 @@ bool osd_messenger_t::handle_read(int result, osd_client_t *cl)
         {
             if (!handle_read_buffer(cl, cl->in_buf, result))
             {
-                goto fin;
+                clear_immediate_ops(peer_fd);
+                return false;
             }
         }
         else
@@ -113,7 +115,8 @@ bool osd_messenger_t::handle_read(int result, osd_client_t *cl)
             {
                 if (!handle_finished_read(cl))
                 {
-                    goto fin;
+                    clear_immediate_ops(peer_fd);
+                    return false;
                 }
             }
         }
@@ -122,15 +125,47 @@ bool osd_messenger_t::handle_read(int result, osd_client_t *cl)
             ret = true;
         }
     }
-fin:
-    for (auto cb: set_immediate)
-    {
-        cb();
-    }
-    set_immediate.clear();
+    handle_immediate_ops();
     return ret;
 }
 
+void osd_messenger_t::clear_immediate_ops(int peer_fd)
+{
+    size_t i = 0, j = 0;
+    while (i < set_immediate_ops.size())
+    {
+        if (set_immediate_ops[i]->peer_fd == peer_fd)
+        {
+            delete set_immediate_ops[i];
+        }
+        else
+        {
+            if (i != j)
+                set_immediate_ops[j] = set_immediate_ops[i];
+            j++;
+        }
+        i++;
+    }
+    set_immediate_ops.resize(j);
+}
+
+void osd_messenger_t::handle_immediate_ops()
+{
+    for (auto op: set_immediate_ops)
+    {
+        if (op->op_type == OSD_OP_IN)
+        {
+            exec_op(op);
+        }
+        else
+        {
+            // Copy lambda to be unaffected by `delete op`
+            std::function<void(osd_op_t*)>(op->callback)(op);
+        }
+    }
+    set_immediate_ops.clear();
+}
+
 bool osd_messenger_t::handle_read_buffer(osd_client_t *cl, void *curbuf, int remain)
 {
     // Compose operation(s) from the buffer
@@ -199,7 +234,7 @@ bool osd_messenger_t::handle_finished_read(osd_client_t *cl)
     {
         // Operation is ready
         cl->received_ops.push_back(cl->read_op);
-        set_immediate.push_back([this, op = cl->read_op]() { exec_op(op); });
+        set_immediate_ops.push_back(cl->read_op);
         cl->read_op = NULL;
         cl->read_state = 0;
     }
@@ -295,7 +330,7 @@ void osd_messenger_t::handle_op_hdr(osd_client_t *cl)
     {
         // Operation is ready
         cl->received_ops.push_back(cur_op);
-        set_immediate.push_back([this, cur_op]() { exec_op(cur_op); });
+        set_immediate_ops.push_back(cur_op);
         cl->read_op = NULL;
         cl->read_state = 0;
     }
@@ -416,9 +451,5 @@ void osd_messenger_t::handle_reply_ready(osd_op_t *op)
         (tv_end.tv_sec - op->tv_begin.tv_sec)*1000000 +
         (tv_end.tv_nsec - op->tv_begin.tv_nsec)/1000
     );
-    set_immediate.push_back([op]()
-    {
-        // Copy lambda to be unaffected by `delete op`
-        std::function<void(osd_op_t*)>(op->callback)(op);
-    });
+    set_immediate_ops.push_back(op);
 }

src/client/nbd_proxy.cpp

@@ -64,7 +64,7 @@ static void netlink_sock_alloc(struct netlink_ctx *ctx)
     if (nl_driver_id < 0)
     {
         nl_socket_free(sk);
-        fail("Couldn't resolve the nbd netlink family\n");
+        fail("Couldn't resolve the nbd netlink family: %s (code %d)\n", nl_geterror(nl_driver_id), nl_driver_id);
     }
 
     ctx->driver_id = nl_driver_id;
@@ -555,7 +555,12 @@ help:
         fcntl(sockfd[0], F_SETFL, fcntl(sockfd[0], F_GETFL, 0) | O_NONBLOCK);
         nbd_fd = sockfd[0];
         load_module();
+
         bool bg = cfg["foreground"].is_null();
+        if (cfg["logfile"].string_value() != "")
+        {
+            logfile = cfg["logfile"].string_value();
+        }
 
         if (netlink)
         {
@@ -579,6 +584,10 @@ help:
             if (!cfg["nbd_disconnect_on_close"].is_null())
                 cflags |= NBD_CFLAG_DISCONNECT_ON_CLOSE;
 #endif
+            if (bg)
+            {
+                daemonize_fork();
+            }
             int err = netlink_configure(sockfd + 1, 1, devnum, device_size, 4096, flags, cflags, nbd_timeout, nbd_conn_timeout, NULL, revive);
             if (err < 0)
             {
@@ -588,6 +597,10 @@ help:
             }
             close(sockfd[1]);
             printf("/dev/nbd%d\n", err);
+            if (bg)
+            {
+                daemonize_reopen_stdio();
+            }
 #else
             fprintf(stderr, "netlink support is disabled in this build\n");
             exit(1);
@@ -631,14 +644,10 @@ help:
                     }
                 }
             }
-        }
-        if (cfg["logfile"].string_value() != "")
-        {
-            logfile = cfg["logfile"].string_value();
-        }
-        if (bg)
-        {
-            daemonize();
+            if (bg)
+            {
+                daemonize();
+            }
         }
         // Initialize read state
         read_state = CL_READ_HDR;
@@ -716,13 +725,17 @@ help:
         }
     }
 
-    void daemonize()
+    void daemonize_fork()
     {
         if (fork())
             exit(0);
         setsid();
         if (fork())
             exit(0);
+    }
+
+    void daemonize_reopen_stdio()
+    {
         close(0);
         close(1);
         close(2);
@@ -733,6 +746,12 @@ help:
             fprintf(stderr, "Warning: Failed to chdir into /\n");
     }
 
+    void daemonize()
+    {
+        daemonize_fork();
+        daemonize_reopen_stdio();
+    }
+
     json11::Json::object list_mapped()
     {
         const char *self_filename = exe_name;
@@ -783,8 +802,9 @@ help:
                 if (!strcmp(pid_filename, self_filename))
                 {
                     json11::Json::object cfg = nbd_proxy::parse_args(argv.size(), argv.data());
-                    if (cfg["command"] == "map")
+                    if (cfg["command"] == "map" || cfg["command"] == "netlink-map")
                     {
+                        cfg["interface"] = (cfg["command"] == "netlink-map") ? "netlink" : "nbd";
                         cfg.erase("command");
                         cfg["pid"] = pid;
                         mapped["/dev/nbd"+std::to_string(dev_num)] = cfg;

src/client/osd_ops.h

@@ -51,6 +51,11 @@
 #define LOC_CORRUPTED 2
 #define LOC_INCONSISTENT 4
 
+#define OSD_LIST_PRIMARY 1
+
+#define OSD_DEL_SUPPORT_LEFT_ON_DEAD 1
+#define OSD_DEL_LEFT_ON_DEAD         2
+
 // common request and reply headers
 struct __attribute__((__packed__)) osd_op_header_t
 {
@@ -196,6 +201,9 @@ struct __attribute__((__packed__)) osd_op_sec_list_t
     uint64_t min_stripe, max_stripe;
     // max stable object count
     uint32_t stable_limit;
+    // flags - OSD_LIST_PRIMARY or 0
+    // for OSD_LIST_PRIMARY, only a single-PG listing is allowed
+    uint64_t flags;
 };
 
 struct __attribute__((__packed__)) osd_reply_sec_list_t
@@ -204,9 +212,11 @@ struct __attribute__((__packed__)) osd_reply_sec_list_t
     // stable object version count. header.retval = total object version count
     // FIXME: maybe change to the number of bytes in the reply...
     uint64_t stable_count;
+    // flags - OSD_LIST_PRIMARY or 0
+    uint64_t flags;
 };
 
-// read or write to the primary OSD (must be within individual stripe)
+// read, write or delete command for the primary OSD (must be within individual stripe)
 struct __attribute__((__packed__)) osd_op_rw_t
 {
     osd_op_header_t header;
@@ -235,6 +245,20 @@ struct __attribute__((__packed__)) osd_reply_rw_t
     uint64_t version;
 };
 
+struct __attribute__((__packed__)) osd_reply_del_t
+{
+    osd_reply_header_t header;
+    // OSD_DEL_SUPPORT_LEFT_ON_DEAD and/or OSD_DEL_LEFT_ON_DEAD or 0
+    uint32_t flags;
+    // for deletes, if flags & OSD_DEL_LEFT_ON_DEAD:
+    // count of OSDs from which the object could be not deleted
+    // these come directly after this del_left_on_dead_list_size as uint32_t[]
+    // FIXME it's kind of a hack and will be removed in the future, when Vitastor will
+    // have 'atomic deletions', i.e. when it will be able to remember deleted objects
+    // and complete deletions automatically after extra OSDs are started
+    uint32_t left_on_dead_count;
+};
+
 // sync to the primary OSD
 struct __attribute__((__packed__)) osd_op_sync_t
 {
@@ -307,6 +331,7 @@ union osd_any_reply_t
     osd_reply_sec_list_t sec_list;
     osd_reply_show_config_t show_conf;
     osd_reply_rw_t rw;
+    osd_reply_del_t del;
     osd_reply_sync_t sync;
     osd_reply_describe_t describe;
     uint8_t buf[OSD_PACKET_SIZE];

src/client/qemu_driver.c

@@ -16,7 +16,6 @@
 #include "qapi/error.h"
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qerror.h"
-#include "qemu/uri.h"
 #include "qemu/error-report.h"
 #include "qemu/module.h"
 #include "qemu/option.h"
@@ -1021,7 +1020,11 @@ static BlockDriver bdrv_vitastor = {
     // FIXME: Implement it along with per-inode statistics
     //.bdrv_get_allocated_file_size   = vitastor_get_allocated_file_size,
 
+#if QEMU_VERSION_MAJOR > 9 || QEMU_VERSION_MAJOR == 9 && QEMU_VERSION_MINOR > 0
+    .bdrv_open                      = vitastor_file_open,
+#else
     .bdrv_file_open                 = vitastor_file_open,
+#endif
     .bdrv_close                     = vitastor_close,
 
     // Option list for the create operation

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 1.9.3
+Version: 1.10.1
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/cmd/cli.cpp

@@ -51,8 +51,9 @@ static const char* help_text =
     "  Rename, resize image or change its readonly status. Images with children can't be made read-write.\n"
     "  If the new size is smaller than the old size, extra data will be purged.\n"
     "  You should resize file system in the image, if present, before shrinking it.\n"
-    "  -f|--force  Proceed with shrinking or setting readwrite flag even if the image has children.\n"
-    "  --down-ok   Proceed with shrinking even if some data will be left on unavailable OSDs.\n"
+    "  --deleted 1|0  Set/clear 'deleted image' flag (set automatically during unfinished deletes).\n"
+    "  -f|--force     Proceed with shrinking or setting readwrite flag even if the image has children.\n"
+    "  --down-ok      Proceed with shrinking even if some data will be left on unavailable OSDs.\n"
     "\n"
     "vitastor-cli dd [iimg=<image> | if=<file>] [oimg=<image> | of=<file>] [bs=1M]\n"
     "    [count=N] [seek/oseek=N] [skip/iseek=M] [iodepth=N] [status=progress]\n"
@@ -101,6 +102,7 @@ static const char* help_text =
     "                Requires more memory, but allows to show correct removal progress.\n"
     "  --min-offset  Purge only data starting with specified offset.\n"
     "  --max-offset  Purge only data before specified offset.\n"
+    "  --client_wait_up_timeout 16  Timeout for waiting until PGs are up in seconds.\n"
     "\n"
     "vitastor-cli merge-data <from> <to> [--target <target>]\n"
     "  Merge layer data without changing metadata. Merge <from>..<to> to <target>.\n"
@@ -430,13 +432,22 @@ static int run(cli_tool_t *p, json11::Json::object cfg)
     else if (cmd[0] == "rm")
     {
         // Remove multiple snapshots and rebase their children
-        if (cmd.size() > 1)
+        if (cfg["exact"].bool_value() || cfg["matching"].bool_value())
         {
-            cfg["from"] = cmd[1];
-            if (cmd.size() > 2)
-                cfg["to"] = cmd[2];
+            cmd.erase(cmd.begin(), cmd.begin()+1);
+            cfg["globs"] = cmd;
+            action_cb = p->start_rm_wildcard(cfg);
+        }
+        else
+        {
+            if (cmd.size() > 1)
+            {
+                cfg["from"] = cmd[1];
+                if (cmd.size() > 2)
+                    cfg["to"] = cmd[2];
+            }
+            action_cb = p->start_rm(cfg);
         }
-        action_cb = p->start_rm(cfg);
     }
     else if (cmd[0] == "describe")
     {

src/cmd/cli.h

@@ -30,7 +30,7 @@ struct cli_result_t
 class cli_tool_t
 {
 public:
-    uint64_t iodepth = 4, parallel_osds = 32;
+    uint64_t iodepth = 32, parallel_osds = 4;
     bool progress = false;
     bool list_first = false;
     bool json_output = false;

src/cmd/cli_create.cpp

@@ -192,7 +192,7 @@ resume_3:
             }
         } while (!parent->etcd_result["succeeded"].bool_value());
         // Save into inode_config for library users to be able to take it from there immediately
-        new_cfg.mod_revision = parent->etcd_result["responses"][0]["response_put"]["header"]["revision"].uint64_value();
+        new_cfg.mod_revision = parent->etcd_result["header"]["revision"].uint64_value();
         parent->cli->st_cli.insert_inode_config(new_cfg);
         result = (cli_result_t){
             .err = 0,
@@ -269,7 +269,7 @@ resume_4:
             }
         } while (!parent->etcd_result["succeeded"].bool_value());
         // Save into inode_config for library users to be able to take it from there immediately
-        new_cfg.mod_revision = parent->etcd_result["responses"][0]["response_put"]["header"]["revision"].uint64_value();
+        new_cfg.mod_revision = parent->etcd_result["header"]["revision"].uint64_value();
         parent->cli->st_cli.insert_inode_config(new_cfg);
         result = (cli_result_t){
             .err = 0,
@@ -401,7 +401,7 @@ resume_3:
                 auto kv = parent->cli->st_cli.parse_etcd_kv(parent->etcd_result["responses"][0]["response_range"]["kvs"][0]);
                 size = kv.value["size"].uint64_value();
                 new_parent_id = kv.value["parent_id"].uint64_value();
-                uint64_t parent_pool_id = kv.value["parent_pool_id"].uint64_value();
+                uint64_t parent_pool_id = kv.value["parent_pool"].uint64_value();
                 if (new_parent_id)
                 {
                     new_parent_id = INODE_WITH_POOL(parent_pool_id ? parent_pool_id : old_pool_id, new_parent_id);
@@ -413,7 +413,7 @@ resume_3:
 
     void attempt_create()
     {
-        new_cfg = {
+        new_cfg = (inode_config_t){
             .num = INODE_WITH_POOL(new_pool_id, new_id),
             .name = image_name,
             .size = size,

src/cmd/cli_dd.cpp

@@ -261,6 +261,7 @@ struct dd_out_info_t
             else
             {
                 // ok
+                out_size = owatch->cfg.size;
                 return true;
             }
             // Wait for sub-command
@@ -323,7 +324,7 @@ resume_2:
                 cluster_op_t *sync_op = new cluster_op_t;
                 sync_op->opcode = OSD_OP_SYNC;
                 parent->waiting++;
-                sync_op->callback = [this, parent](cluster_op_t *sync_op)
+                sync_op->callback = [parent](cluster_op_t *sync_op)
                 {
                     parent->waiting--;
                     delete sync_op;
@@ -538,7 +539,7 @@ struct cli_dd_t
                     in_eof = true;
             }
             ring_data_t *data = ((ring_data_t*)sqe->user_data);
-            data->iov = (iovec){ cur_read->buf + cur_read->len, cur_read->max - cur_read->len };
+            data->iov = (iovec){ (uint8_t*)cur_read->buf + cur_read->len, cur_read->max - cur_read->len };
             my_uring_prep_readv(sqe, iinfo.ifd, &data->iov, 1, iinfo.in_seekable ? iseek + cur_read->offset + cur_read->len : -1);
             in_waiting++;
             data->callback = [this, cur_read](ring_data_t *data)
@@ -671,7 +672,7 @@ struct cli_dd_t
                 return false;
             }
             ring_data_t *data = ((ring_data_t*)sqe->user_data);
-            data->iov = (iovec){ .iov_base = cur_read->buf+cur_read->len, .iov_len = cur_read->max-cur_read->len };
+            data->iov = (iovec){ .iov_base = (uint8_t*)cur_read->buf+cur_read->len, .iov_len = cur_read->max-cur_read->len };
             my_uring_prep_writev(sqe, oinfo.ofd, &data->iov, 1, oinfo.out_seekable ? cur_read->offset+cur_read->len+oseek : -1);
             out_waiting++;
             data->callback = [this, cur_read](ring_data_t *data)
@@ -882,6 +883,8 @@ resume_2:
         oinfo.end_fsync = oinfo.end_fsync && oinfo.out_seekable;
         read_offset = 0;
         read_end = iinfo.in_seekable ? iinfo.in_size-iseek : 0;
+        if (oinfo.out_size && (!read_end || read_end > oinfo.out_size-oseek))
+            read_end = oinfo.out_size-oseek;
         if (bytelimit && (!read_end || read_end > bytelimit))
             read_end = bytelimit;
         clock_gettime(CLOCK_REALTIME, &tv_begin);

src/cmd/cli_ls.cpp

@@ -68,6 +68,7 @@ struct image_lister_t
                 { "pool_name", good_pool ? pool_it->second.name : "? (ID:"+std::to_string(INODE_POOL(ic.second.num))+")" },
                 { "inode_num", INODE_NO_POOL(ic.second.num) },
                 { "inode_id", ic.second.num },
+                { "deleted", ic.second.deleted },
             };
             if (ic.second.parent_id)
             {
@@ -371,7 +372,8 @@ resume_1:
                 kv.second["delete_q"] = format_q(kv.second["delete_queue"].number_value());
             }
             kv.second["size_fmt"] = format_size(kv.second["size"].uint64_value());
-            kv.second["ro"] = kv.second["readonly"].bool_value() ? "RO" : "-";
+            kv.second["ro"] = kv.second["deleted"].bool_value() ? "DEL" :
+                (kv.second["readonly"].bool_value() ? "RO" : "-");
         }
         result.text = print_table(to_list(), cols, parent->color);
         state = 100;

src/cmd/cli_merge.cpp

@@ -51,6 +51,7 @@ struct snap_merger_t
     btree::safe_btree_set<uint64_t> merge_offsets;
     btree::safe_btree_set<uint64_t>::iterator oit;
     std::map<inode_t, std::vector<uint64_t>> layer_lists;
+    std::map<inode_t, int> list_errcode;
     std::map<inode_t, uint64_t> layer_block_size;
     std::map<inode_t, uint64_t> layer_list_pos;
     std::vector<snap_rw_op_t*> continue_rwo, continue_rwo2;
@@ -251,6 +252,7 @@ struct snap_merger_t
         // Get parents and so on
         start_merge();
         // First list lower layers
+        list_errcode.clear();
         list_layers(true);
         state = 1;
     resume_1:
@@ -259,6 +261,15 @@ struct snap_merger_t
             // Wait for lists
             return;
         }
+        if (list_errcode.size())
+        {
+            result = (cli_result_t){
+                .err = EIO,
+                .text = "Failed to list lower layer(s) in some PGs, merging would be incorrect",
+            };
+            state = 100;
+            return;
+        }
         if (merge_offsets.size() > 0)
         {
             state = 2;
@@ -295,6 +306,7 @@ struct snap_merger_t
         state = 3;
     resume_3:
         // Then list upper layers
+        list_errcode.clear();
         list_layers(false);
         state = 4;
     resume_4:
@@ -303,6 +315,15 @@ struct snap_merger_t
             // Wait for lists
             return;
         }
+        if (list_errcode.size() > 0)
+        {
+            result = (cli_result_t){
+                .err = EIO,
+                .text = "Failed to list upper layer(s) in some PGs, merging would be incorrect",
+            };
+            state = 100;
+            return;
+        }
         state = 5;
         processed = 0;
         to_process = merge_offsets.size();
@@ -368,9 +389,13 @@ struct snap_merger_t
             if (lower ? (sp.second < target_rank) : (sp.second > target_rank))
             {
                 lists_todo++;
-                inode_list_t* lst = parent->cli->list_inode_start(src, [this, src](
-                    inode_list_t *lst, std::set<object_id>&& objects, pg_num_t pg_num, osd_num_t primary_osd, int status)
+                parent->cli->list_inode(src, 0, 0, parent->parallel_osds, [this, src](
+                    int errcode, int pgs_left, pg_num_t pg_num, std::set<object_id>&& objects)
                 {
+                    if (errcode)
+                    {
+                        list_errcode[src] = errcode;
+                    }
                     uint64_t layer_block = layer_block_size.at(src);
                     for (object_id obj: objects)
                     {
@@ -391,12 +416,18 @@ struct snap_merger_t
                             layer_list[pos++] = obj.stripe;
                         }
                     }
-                    if (status & INODE_LIST_DONE)
+                    if (!pgs_left)
                     {
                         auto & name = parent->cli->st_cli.inode_config.at(src).name;
-                        if (parent->progress)
+                        if (list_errcode.find(src) != list_errcode.end())
                         {
-                            printf("Got listing of layer %s (inode %ju in pool %u)\n", name.c_str(), INODE_NO_POOL(src), INODE_POOL(src));
+                            fprintf(stderr, "Failed to get listing of layer %s (inode %ju in pool %u): %s (code %d)\n",
+                                name.c_str(), INODE_NO_POOL(src), INODE_POOL(src), strerror(-list_errcode[src]), list_errcode[src]);
+                        }
+                        else if (parent->progress)
+                        {
+                            fprintf(stderr, "Got listing of layer %s (inode %ju in pool %u)\n",
+                                name.c_str(), INODE_NO_POOL(src), INODE_POOL(src));
                         }
                         if (delete_source)
                         {
@@ -406,12 +437,7 @@ struct snap_merger_t
                         lists_todo--;
                         continue_merge_reent();
                     }
-                    else
-                    {
-                        parent->cli->list_inode_next(lst, 1);
-                    }
                 });
-                parent->cli->list_inode_next(lst, parent->parallel_osds);
             }
         }
     }
@@ -428,7 +454,7 @@ struct snap_merger_t
         {
             if (op->retval < 0)
             {
-                fprintf(stderr, "error reading target bitmap at offset %jx: %s\n", op->offset, strerror(-op->retval));
+                fprintf(stderr, "Warning: failed to read target bitmap at offset %jx: %s\n", op->offset, strerror(-op->retval));
             }
             else
             {
@@ -585,7 +611,7 @@ struct snap_merger_t
         subop->inode = inode_num;
         subop->offset = offset;
         subop->len = 0;
-        subop->flags = OSD_OP_IGNORE_READONLY;
+        subop->flags = OSD_OP_IGNORE_READONLY | OSD_OP_WAIT_UP_TIMEOUT;
         subop->callback = [](cluster_op_t *subop)
         {
             if (subop->retval != 0)

src/cmd/cli_modify.cpp

@@ -4,6 +4,7 @@
 #include "cli.h"
 #include "cluster_client.h"
 #include "str_util.h"
+#include "json_util.h"
 
 // Rename, resize image (and purge extra data on shrink) or change its readonly status
 struct image_changer_t
@@ -15,6 +16,7 @@ struct image_changer_t
     uint64_t new_size = 0;
     bool force_size = false, inc_size = false;
     bool set_readonly = false, set_readwrite = false, force = false;
+    bool set_deleted = false, new_deleted = false;
     bool down_ok = false;
     // interval between fsyncs
     int fsync_interval = 128;
@@ -82,6 +84,7 @@ struct image_changer_t
         }
         if ((!set_readwrite || !cfg.readonly) &&
             (!set_readonly || cfg.readonly) &&
+            (!set_deleted || cfg.deleted == new_deleted) &&
             (!new_size && !force_size || cfg.size == new_size || cfg.size >= new_size && inc_size) &&
             (new_name == "" || new_name == image_name))
         {
@@ -141,6 +144,10 @@ resume_1:
                 return;
             }
         }
+        if (set_deleted)
+        {
+            cfg.deleted = new_deleted;
+        }
         if (new_name != "")
         {
             cfg.name = new_name;
@@ -219,7 +226,7 @@ resume_2:
             return;
         }
         // Save into inode_config for library users to be able to take it from there immediately
-        cfg.mod_revision = parent->etcd_result["responses"][0]["response_put"]["header"]["revision"].uint64_value();
+        cfg.mod_revision = parent->etcd_result["header"]["revision"].uint64_value();
         if (new_name != "")
         {
             parent->cli->st_cli.inode_by_name.erase(image_name);
@@ -251,6 +258,8 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_modify(json11::Json cfg)
     changer->force = cfg["force"].bool_value();
     changer->set_readonly = cfg["readonly"].bool_value();
     changer->set_readwrite = cfg["readwrite"].bool_value();
+    changer->set_deleted = !cfg["deleted"].is_null();
+    changer->new_deleted = json_is_true(cfg["deleted"]);
     changer->fsync_interval = cfg["fsync_interval"].uint64_value();
     if (!changer->fsync_interval)
         changer->fsync_interval = 128;

src/cmd/cli_osd_tree.cpp

@@ -216,7 +216,7 @@ resume_1:
                 for (uint64_t osd_num: node.child_osds)
                 {
                     auto & osd = placement_tree->osds.at(osd_num);
-                    fmt_items.push_back(json11::Json::object{
+                    auto json_osd = json11::Json::object{
                         { "type", "osd" },
                         { "name", osd.num },
                         { "parent", node.name },
@@ -230,7 +230,16 @@ resume_1:
                         { "bitmap", (uint64_t)osd.bitmap_granularity },
                         { "commit", osd.immediate_commit == IMMEDIATE_NONE ? "none" : (osd.immediate_commit == IMMEDIATE_ALL ? "all" : "small") },
                         { "op_stats", osd_stats[osd_num]["op_stats"] },
-                    });
+                    };
+                    if (osd_stats[osd_num]["slow_ops_primary"].uint64_value() > 0)
+                    {
+                        json_osd["slow_ops_primary"] = osd_stats[osd_num]["slow_ops_primary"];
+                    }
+                    if (osd_stats[osd_num]["slow_ops_secondary"].uint64_value() > 0)
+                    {
+                        json_osd["slow_ops_secondary"] = osd_stats[osd_num]["slow_ops_secondary"];
+                    }
+                    fmt_items.push_back(json_osd);
                 }
             }
             result.data = fmt_items;

src/cmd/cli_pool_create.cpp

@@ -35,6 +35,7 @@ struct pool_creator_t
     uint64_t new_pools_mod_rev;
     json11::Json state_node_tree;
     json11::Json new_pools;
+    std::map<osd_num_t, json11::Json> osd_stats;
 
     bool is_done() { return state == 100; }
 
@@ -46,8 +47,6 @@ struct pool_creator_t
             goto resume_2;
         else if (state == 3)
             goto resume_3;
-        else if (state == 4)
-            goto resume_4;
         else if (state == 5)
             goto resume_5;
         else if (state == 6)
@@ -90,13 +89,19 @@ resume_1:
         // If not forced, check that we have enough osds for pg_size
         if (!force)
         {
-            // Get node_placement configuration from etcd
+            // Get node_placement configuration from etcd and OSD stats
             parent->etcd_txn(json11::Json::object {
                 { "success", json11::Json::array {
                     json11::Json::object {
                         { "request_range", json11::Json::object {
                             { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/node_placement") },
-                        } }
+                        } },
+                    },
+                    json11::Json::object {
+                        { "request_range", json11::Json::object {
+                            { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/osd/stats/") },
+                            { "range_end", base64_encode(parent->cli->st_cli.etcd_prefix+"/osd/stats0") },
+                        } },
                     },
                 } },
             });
@@ -112,10 +117,21 @@ resume_2:
                 return;
             }
 
-            // Get state_node_tree based on node_placement and osd peer states
+            // Get state_node_tree based on node_placement and osd stats
             {
-                auto kv = parent->cli->st_cli.parse_etcd_kv(parent->etcd_result["responses"][0]["response_range"]["kvs"][0]);
-                state_node_tree = get_state_node_tree(kv.value.object_items());
+                auto node_placement_kv = parent->cli->st_cli.parse_etcd_kv(parent->etcd_result["responses"][0]["response_range"]["kvs"][0]);
+                timespec tv_now;
+                clock_gettime(CLOCK_REALTIME, &tv_now);
+                uint64_t osd_out_time = parent->cli->config["osd_out_time"].uint64_value();
+                if (!osd_out_time)
+                    osd_out_time = 600;
+                osd_stats.clear();
+                parent->iterate_kvs_1(parent->etcd_result["responses"][1]["response_range"]["kvs"], "/osd/stats/", [&](uint64_t cur_osd, json11::Json value)
+                {
+                    if ((uint64_t)value["time"].number_value()+osd_out_time >= tv_now.tv_sec)
+                        osd_stats[cur_osd] = value;
+                });
+                state_node_tree = get_state_node_tree(node_placement_kv.value.object_items(), osd_stats);
             }
 
             // Skip tag checks, if pool has none
@@ -158,42 +174,18 @@ resume_3:
                 }
             }
 
-            // Get stats (for block_size, bitmap_granularity, ...) of osds in state_node_tree
-            {
-                json11::Json::array osd_stats;
-
-                for (auto osd_num: state_node_tree["osds"].array_items())
-                {
-                    osd_stats.push_back(json11::Json::object {
-                        { "request_range", json11::Json::object {
-                            { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/osd/stats/"+osd_num.as_string()) },
-                        } }
-                    });
-                }
-
-                parent->etcd_txn(json11::Json::object{ { "success", osd_stats } });
-            }
-
-            state = 4;
-resume_4:
-            if (parent->waiting > 0)
-                return;
-            if (parent->etcd_err.err)
-            {
-                result = parent->etcd_err;
-                state = 100;
-                return;
-            }
-
             // Filter osds from state_node_tree based on pool parameters and osd stats
             {
-                std::vector<json11::Json> osd_stats;
-                for (auto & ocr: parent->etcd_result["responses"].array_items())
+                std::vector<json11::Json> filtered_osd_stats;
+                for (auto & osd_num: state_node_tree["osds"].array_items())
                 {
-                    auto kv = parent->cli->st_cli.parse_etcd_kv(ocr["response_range"]["kvs"][0]);
-                    osd_stats.push_back(kv.value);
+                    auto st_it = osd_stats.find(osd_num.uint64_value());
+                    if (st_it != osd_stats.end())
+                    {
+                        filtered_osd_stats.push_back(st_it->second);
+                    }
                 }
-                guess_block_size(osd_stats);
+                guess_block_size(filtered_osd_stats);
                 state_node_tree = filter_state_node_tree_by_stats(state_node_tree, osd_stats);
             }
 
@@ -201,8 +193,7 @@ resume_4:
             {
                 auto failure_domain = cfg["failure_domain"].string_value() == ""
                     ? "host" : cfg["failure_domain"].string_value();
-                uint64_t max_pg_size = get_max_pg_size(state_node_tree["nodes"].object_items(),
-                    failure_domain, cfg["root_node"].string_value());
+                uint64_t max_pg_size = get_max_pg_size(state_node_tree, failure_domain, cfg["root_node"].string_value());
 
                 if (cfg["pg_size"].uint64_value() > max_pg_size)
                 {
@@ -358,56 +349,50 @@ resume_8:
 
     // Returns a JSON object of form {"nodes": {...}, "osds": [...]} that
     // contains: all nodes (osds, hosts, ...) based on node_placement config
-    // and current peer state, and a list of active peer osds.
-    json11::Json get_state_node_tree(json11::Json::object node_placement)
+    // and current osd stats.
+    json11::Json get_state_node_tree(json11::Json::object node_placement, std::map<osd_num_t, json11::Json> & osd_stats)
     {
-        // Erase non-peer osd nodes from node_placement
+        // Erase non-existing osd nodes from node_placement
         for (auto np_it = node_placement.begin(); np_it != node_placement.end();)
         {
             // Numeric nodes are osds
             osd_num_t osd_num = stoull_full(np_it->first);
 
-            // If node is osd and it is not in peer states, erase it
-            if (osd_num > 0 &&
-                parent->cli->st_cli.peer_states.find(osd_num) == parent->cli->st_cli.peer_states.end())
-            {
+            // If node is osd and its stats do not exist, erase it
+            if (osd_num > 0 && osd_stats.find(osd_num) == osd_stats.end())
                 node_placement.erase(np_it++);
-            }
             else
                 np_it++;
         }
 
-        // List of peer osds
-        std::vector<std::string> peer_osds;
+        // List of osds
+        std::vector<std::string> existing_osds;
 
-        // Record peer osds and add missing osds/hosts to np
-        for (auto & ps: parent->cli->st_cli.peer_states)
+        // Record osds and add missing osds/hosts to np
+        for (auto & ps: osd_stats)
         {
             std::string osd_num = std::to_string(ps.first);
 
-            // Record peer osd
-            peer_osds.push_back(osd_num);
+            // Record osd
+            existing_osds.push_back(osd_num);
 
-            // Add osd, if necessary
-            if (node_placement.find(osd_num) == node_placement.end())
+            // Add host if necessary
+            std::string osd_host = ps.second["host"].as_string();
+            if (node_placement.find(osd_host) == node_placement.end())
             {
-                std::string osd_host = ps.second["host"].as_string();
-
-                // Add host, if necessary
-                if (node_placement.find(osd_host) == node_placement.end())
-                {
-                    node_placement[osd_host] = json11::Json::object {
-                        { "level", "host" }
-                    };
-                }
-
-                node_placement[osd_num] = json11::Json::object {
-                    { "parent", osd_host }
+                node_placement[osd_host] = json11::Json::object {
+                    { "level", "host" }
                 };
             }
+
+            // Add osd
+            node_placement[osd_num] = json11::Json::object {
+                { "parent", node_placement[osd_num]["parent"].is_null() ? osd_host : node_placement[osd_num]["parent"] },
+                { "level", "osd" },
+            };
         }
 
-        return json11::Json::object { { "osds", peer_osds }, { "nodes", node_placement } };
+        return json11::Json::object { { "osds", existing_osds }, { "nodes", node_placement } };
     }
 
     // Returns new state_node_tree based on given state_node_tree with osds
@@ -475,7 +460,7 @@ resume_8:
             }
             if (osd_bs && osd_bs != UINT32_MAX && osd_bs != parent->cli->st_cli.global_block_size)
             {
-                fprintf(stderr, "Auto-selecting block_size=%s because all pool OSDs use it\n", format_size(osd_bs).c_str());
+                fprintf(stderr, "Auto-selecting block_size=%s because all pool OSDs use it\n", format_size(osd_bs, false, true).c_str());
                 upd["block_size"] = osd_bs;
             }
         }
@@ -494,7 +479,7 @@ resume_8:
             }
             if (osd_bg && osd_bg != UINT32_MAX && osd_bg != parent->cli->st_cli.global_bitmap_granularity)
             {
-                fprintf(stderr, "Auto-selecting bitmap_granularity=%s because all pool OSDs use it\n", format_size(osd_bg).c_str());
+                fprintf(stderr, "Auto-selecting bitmap_granularity=%s because all pool OSDs use it\n", format_size(osd_bg, false, true).c_str());
                 upd["bitmap_granularity"] = osd_bg;
             }
         }
@@ -534,15 +519,13 @@ resume_8:
     // filtered out by stats parameters (block_size, bitmap_granularity) in
     // given osd_stats and current pool config.
     // Requires: state_node_tree["osds"] must match osd_stats 1-1
-    json11::Json filter_state_node_tree_by_stats(const json11::Json & state_node_tree, std::vector<json11::Json> & osd_stats)
+    json11::Json filter_state_node_tree_by_stats(const json11::Json & state_node_tree, std::map<osd_num_t, json11::Json> & osd_stats)
     {
-        auto & osds = state_node_tree["osds"].array_items();
-
         // Accepted state_node_tree nodes
         auto accepted_nodes = state_node_tree["nodes"].object_items();
 
         // List of accepted osds
-        std::vector<std::string> accepted_osds;
+        json11::Json::array accepted_osds;
 
         block_size = cfg["block_size"].uint64_value()
             ? cfg["block_size"].uint64_value()
@@ -554,21 +537,25 @@ resume_8:
             ? etcd_state_client_t::parse_immediate_commit(cfg["immediate_commit"].string_value(), IMMEDIATE_ALL)
             : parent->cli->st_cli.global_immediate_commit;
 
-        for (size_t i = 0; i < osd_stats.size(); i++)
+        for (auto osd_num_json: state_node_tree["osds"].array_items())
         {
-            auto & os = osd_stats[i];
-            // Get osd number
-            auto osd_num = osds[i].as_string();
+            auto osd_num = osd_num_json.uint64_value();
+            auto os_it = osd_stats.find(osd_num);
+            if (os_it == osd_stats.end())
+            {
+                continue;
+            }
+            auto & os = os_it->second;
             if (!os["data_block_size"].is_null() && os["data_block_size"] != block_size ||
                 !os["bitmap_granularity"].is_null() && os["bitmap_granularity"] != bitmap_granularity ||
                 !os["immediate_commit"].is_null() &&
                 etcd_state_client_t::parse_immediate_commit(os["immediate_commit"].string_value(), IMMEDIATE_NONE) < immediate_commit)
             {
-                accepted_nodes.erase(osd_num);
+                accepted_nodes.erase(osd_num_json.as_string());
             }
             else
             {
-                accepted_osds.push_back(osd_num);
+                accepted_osds.push_back(osd_num_json);
             }
         }
 
@@ -576,87 +563,28 @@ resume_8:
     }
 
     // Returns maximum pg_size possible for given node_tree and failure_domain, starting at parent_node
-    uint64_t get_max_pg_size(json11::Json::object node_tree, const std::string & level, const std::string & parent_node)
+    uint64_t get_max_pg_size(json11::Json state_node_tree, const std::string & level, const std::string & root_node)
     {
-        uint64_t max_pg_sz = 0;
-
-        std::vector<std::string> nodes;
-
-        // Check if parent node is an osd (numeric)
-        if (parent_node != "" && stoull_full(parent_node))
+        std::set<std::string> level_seen;
+        for (auto & osd: state_node_tree["osds"].array_items())
         {
-            // Add it to node list if osd is in node tree
-            if (node_tree.find(parent_node) != node_tree.end())
-                nodes.push_back(parent_node);
-        }
-        // If parent node given, ...
-        else if (parent_node != "")
-        {
-            // ... look for children nodes of this parent
-            for (auto & sn: node_tree)
+            // find OSD parent at <level>, but stop at <root_node>
+            auto cur_id = osd.string_value();
+            auto cur = state_node_tree["nodes"][cur_id];
+            while (!cur.is_null())
             {
-                auto & props = sn.second.object_items();
-
-                auto parent_prop = props.find("parent");
-                if (parent_prop != props.end() && (parent_prop->second.as_string() == parent_node))
+                if (cur["level"] == level)
                 {
-                    nodes.push_back(sn.first);
-
-                    // If we're not looking for all osds, we only need a single
-                    // child osd node
-                    if (level != "osd" && stoull_full(sn.first))
-                        break;
+                    level_seen.insert(cur_id);
+                    break;
                 }
+                if (cur_id == root_node)
+                    break;
+                cur_id = cur["parent"].string_value();
+                cur = state_node_tree["nodes"][cur_id];
             }
         }
-        // No parent node given, and we're not looking for all osds
-        else if (level != "osd")
-        {
-            // ... look for all level nodes
-            for (auto & sn: node_tree)
-            {
-                auto & props = sn.second.object_items();
-
-                auto level_prop = props.find("level");
-                if (level_prop != props.end() && (level_prop->second.as_string() == level))
-                {
-                    nodes.push_back(sn.first);
-                }
-            }
-        }
-        // Otherwise, ...
-        else
-        {
-            // ... we're looking for osd nodes only
-            for (auto & sn: node_tree)
-            {
-                if (stoull_full(sn.first))
-                {
-                    nodes.push_back(sn.first);
-                }
-            }
-        }
-
-        // Process gathered nodes
-        for (auto & node: nodes)
-        {
-            // Check for osd node, return constant max size
-            if (stoull_full(node))
-            {
-                max_pg_sz += 1;
-            }
-            // Otherwise, ...
-            else
-            {
-                // ... exclude parent node from tree, and ...
-                node_tree.erase(parent_node);
-
-                // ... descend onto the resulting tree
-                max_pg_sz += get_max_pg_size(node_tree, level, node);
-            }
-        }
-
-        return max_pg_sz;
+        return level_seen.size();
     }
 
     json11::Json create_pool(const etcd_kv_t & kv)

src/cmd/cli_pool_ls.cpp

@@ -185,6 +185,7 @@ resume_1:
                 { "space_efficiency", pool_stats[pool_cfg.id]["space_efficiency"].number_value() },
                 { "pg_real_size", pool_stats[pool_cfg.id]["pg_real_size"].uint64_value() },
                 { "osd_count", (uint64_t)pg_per_osd.size() },
+                { "backfillfull", pool_cfg.backfillfull },
             };
         }
         // Include full pool config
@@ -484,6 +485,7 @@ resume_3:
                     st["block_size_fmt"] = format_size(st["block_size"].uint64_value());
                 if (st["bitmap_granularity"].uint64_value())
                     st["bitmap_granularity_fmt"] = format_size(st["bitmap_granularity"].uint64_value());
+                st["backfillfull_fmt"] = st["backfillfull"].bool_value() ? "yes" : "";
             }
             // All pool parameters are only displayed in the "detailed" mode
             // because there's too many of them to show them in table
@@ -493,6 +495,7 @@ resume_3:
                 { "scheme_name", "Scheme" },
                 { "used_for_fs", "Used for VitastorFS" },
                 { "status", "Status" },
+                { "backfillfull_fmt", "Backfillfull" },
                 { "pg_count_fmt", "PGs" },
                 { "pg_minsize", "PG minsize" },
                 { "failure_domain", "Failure domain" },

src/cmd/cli_rm.cpp

@@ -95,6 +95,10 @@ struct snap_remover_t
             goto resume_7;
         else if (state == 8)
             goto resume_8;
+        else if (state == 9)
+            goto resume_9;
+        else if (state == 10)
+            goto resume_10;
         else if (state == 100)
             goto resume_100;
         assert(!state);
@@ -129,23 +133,9 @@ resume_1:
                 continue;
             rebased_images.push_back(parent->cli->st_cli.inode_config.at(merge_children[current_child]).name);
             start_merge_child(merge_children[current_child], merge_children[current_child]);
-            if (state == 100)
-                return;
 resume_2:
-            while (!cb(result))
-            {
-                state = 2;
+            while (!wait_result(2))
                 return;
-            }
-            cb = NULL;
-            if (result.err)
-            {
-                result.data = my_result(result.data);
-                state = 100;
-                return;
-            }
-            else if (parent->progress)
-                printf("%s\n", result.text.c_str());
             parent->change_parent(merge_children[current_child], new_parent, &result);
             state = 3;
 resume_3:
@@ -164,42 +154,19 @@ resume_3:
         if (inverse_child != 0)
         {
             start_merge_child(inverse_child, inverse_parent);
-            if (state == 100)
-                return;
 resume_4:
-            while (!cb(result))
-            {
-                state = 4;
+            while (!wait_result(4))
                 return;
-            }
-            cb = NULL;
-            if (result.err)
-            {
-                result.data = my_result(result.data);
-                state = 100;
+            // Mark child as deleted
+            start_mark_deleted(inverse_child);
+resume_9:
+            while (!wait_result(9))
                 return;
-            }
-            else if (parent->progress)
-                printf("%s\n", result.text.c_str());
             // Delete "inverse" child data
             start_delete_source(inverse_child);
-            if (state == 100)
-                return;
 resume_5:
-            while (!cb(result))
-            {
-                state = 5;
+            while (!wait_result(5))
                 return;
-            }
-            cb = NULL;
-            if (result.err)
-            {
-                result.data = my_result(result.data);
-                state = 100;
-                return;
-            }
-            else if (parent->progress)
-                printf("%s\n", result.text.c_str());
             // Delete "inverse" child metadata, rename parent over it,
             // and also change parent links of the previous "inverse" child
             rename_inverse_parent();
@@ -221,22 +188,15 @@ resume_6:
                     deleted_images.push_back(parent_it->second.name);
                 deleted_ids.push_back(chain_list[current_child]);
             }
+            // Mark child as deleted
+            start_mark_deleted(chain_list[current_child]);
+resume_10:
+            while (!wait_result(10))
+                return;
             start_delete_source(chain_list[current_child]);
 resume_7:
-            while (!cb(result))
-            {
-                state = 7;
+            while (!wait_result(7))
                 return;
-            }
-            cb = NULL;
-            if (result.err)
-            {
-                result.data = my_result(result.data);
-                state = 100;
-                return;
-            }
-            else if (parent->progress)
-                printf("%s\n", result.text.c_str());
             delete_inode_config(chain_list[current_child]);
             if (state == 100)
                 return;
@@ -256,6 +216,27 @@ resume_100:
         return;
     }
 
+    bool wait_result(int base_state)
+    {
+        if (state == 100 || !cb)
+            return true;
+        while (!cb(result))
+        {
+            state = base_state;
+            return false;
+        }
+        cb = NULL;
+        if (result.err)
+        {
+            result.data = my_result(result.data);
+            state = 100;
+            return false;
+        }
+        else if (parent->progress)
+            printf("%s\n", result.text.c_str());
+        return true;
+    }
+
     json11::Json my_result(json11::Json src)
     {
         auto obj = src.object_items();
@@ -667,6 +648,27 @@ resume_100:
         });
     }
 
+    void start_mark_deleted(inode_t inode)
+    {
+        auto ino_it = parent->cli->st_cli.inode_config.find(inode);
+        if (ino_it == parent->cli->st_cli.inode_config.end())
+        {
+            char buf[1024];
+            snprintf(buf, 1024, "Inode 0x%jx disappeared", inode);
+            result = (cli_result_t){ .err = EIO, .text = std::string(buf) };
+            state = 100;
+            return;
+        }
+        if (ino_it->second.deleted)
+        {
+            return;
+        }
+        cb = parent->start_modify(json11::Json::object {
+            { "image", ino_it->second.name },
+            { "deleted", true },
+        });
+    }
+
     void start_delete_source(inode_t inode)
     {
         auto source = parent->cli->st_cli.inode_config.find(inode);
@@ -689,10 +691,6 @@ resume_100:
 
 std::function<bool(cli_result_t &)> cli_tool_t::start_rm(json11::Json cfg)
 {
-    if (cfg["exact"].bool_value() || cfg["matching"].bool_value())
-    {
-        return start_rm_wildcard(cfg);
-    }
     auto snap_remover = new snap_remover_t();
     snap_remover->parent = this;
     snap_remover->from_name = cfg["from"].string_value();

src/cmd/cli_rm_data.cpp

@@ -11,7 +11,6 @@
 struct rm_pg_t
 {
     pg_num_t pg_num;
-    osd_num_t rm_osd_num;
     std::set<object_id> objects;
     std::set<object_id>::iterator obj_pos;
     uint64_t obj_count = 0, obj_done = 0;
@@ -31,160 +30,127 @@ struct rm_inode_t
     cli_tool_t *parent = NULL;
     inode_list_t *lister = NULL;
     std::vector<rm_pg_t*> lists;
-    std::vector<osd_num_t> inactive_osds;
+    std::set<osd_num_t> inactive_osds;
+    std::set<pg_num_t> inactive_pgs;
+    std::set<pg_num_t> fallback_pgs;
     uint64_t total_count = 0, total_done = 0, total_prev_pct = 0;
-    uint64_t pgs_to_list = 0;
     bool lists_done = false;
+    int pgs_to_list = 0;
     int state = 0;
     int error_count = 0;
+    bool in_continue = false;
 
     cli_result_t result;
 
     void start_delete()
     {
-        lister = parent->cli->list_inode_start(inode, [this](inode_list_t *lst,
-            std::set<object_id>&& objects, pg_num_t pg_num, osd_num_t primary_osd, int status)
+        auto pool_it = parent->cli->st_cli.pool_config.find(pool_id);
+        if (pool_it == parent->cli->st_cli.pool_config.end())
         {
-            rm_pg_t *rm = new rm_pg_t((rm_pg_t){
-                .pg_num = pg_num,
-                .rm_osd_num = primary_osd,
-                .objects = objects,
-                .obj_count = objects.size(),
-                .obj_done = 0,
-                .synced = parent->cli->get_immediate_commit(inode),
-            });
-            if (min_offset == 0 && max_offset == 0)
-            {
-                total_count += objects.size();
-            }
-            else
-            {
-                for (object_id oid: objects)
-                {
-                    if (oid.stripe >= min_offset && (!max_offset || oid.stripe < max_offset))
-                    {
-                        total_count++;
-                    }
-                }
-            }
-            rm->obj_pos = rm->objects.begin();
-            lists.push_back(rm);
-            if (parent->list_first && !(status & INODE_LIST_DONE))
-            {
-                // The listing object is dead when DONE => don't call next()
-                parent->cli->list_inode_next(lister, 1);
-            }
-            if (status & INODE_LIST_DONE)
-            {
-                lists_done = true;
-            }
-            pgs_to_list--;
-            continue_delete();
-        });
-        if (!lister)
-        {
-            result = (cli_result_t){
-                .err = EIO,
-                .text = "Failed to list objects of inode "+std::to_string(INODE_NO_POOL(inode))+
-                    " from pool "+std::to_string(INODE_POOL(inode)),
-            };
+            result = (cli_result_t){ .err = EINVAL, .text = "Pool does not exist" };
             state = 100;
             return;
         }
-        inactive_osds = parent->cli->list_inode_get_inactive_osds(lister);
-        if (inactive_osds.size() && !parent->json_output)
+        pgs_to_list = pool_it->second.real_pg_count;
+        parent->cli->list_inode(inode, min_offset, max_offset, parent->parallel_osds, [this](
+            int errcode, int pgs_left, pg_num_t pg_num, std::set<object_id>&& objects)
         {
-            fprintf(stderr, "Some data may remain after delete on OSDs which are currently down: ");
-            for (int i = 0; i < inactive_osds.size(); i++)
+            if (errcode)
             {
-                fprintf(stderr, i > 0 ? ", %ju" : "%ju", inactive_osds[i]);
+                inactive_pgs.insert(pg_num);
             }
-            fprintf(stderr, "\n");
-        }
-        pgs_to_list = parent->cli->list_pg_count(lister);
-        parent->cli->list_inode_next(lister, parent->parallel_osds);
+            else
+            {
+                rm_pg_t *rm = new rm_pg_t((rm_pg_t){
+                    .pg_num = pg_num,
+                    .objects = std::move(objects),
+                    .obj_done = 0,
+                    .synced = !objects.size() || parent->cli->get_immediate_commit(inode),
+                });
+                if (min_offset == 0 && max_offset == 0)
+                {
+                    total_count += objects.size();
+                }
+                else
+                {
+                    for (object_id oid: objects)
+                    {
+                        if (oid.stripe >= min_offset && (!max_offset || oid.stripe < max_offset))
+                        {
+                            total_count++;
+                        }
+                    }
+                }
+                rm->obj_pos = rm->objects.begin();
+                lists.push_back(rm);
+            }
+            pgs_to_list = pgs_left;
+            lists_done = !pgs_to_list;
+            continue_delete();
+        });
     }
 
     void send_ops(rm_pg_t *cur_list)
     {
-        parent->cli->init_msgr();
-        if (parent->cli->msgr.osd_peer_fds.find(cur_list->rm_osd_num) ==
-            parent->cli->msgr.osd_peer_fds.end())
-        {
-            // Initiate connection
-            parent->cli->msgr.connect_peer(cur_list->rm_osd_num, parent->cli->st_cli.peer_states[cur_list->rm_osd_num]);
-            return;
-        }
         while (cur_list->in_flight < parent->iodepth && cur_list->obj_pos != cur_list->objects.end())
         {
             if (cur_list->obj_pos->stripe >= min_offset && (!max_offset || cur_list->obj_pos->stripe < max_offset))
             {
-                osd_op_t *op = new osd_op_t();
-                op->op_type = OSD_OP_OUT;
-                // Already checked that it exists above, but anyway
-                op->peer_fd = parent->cli->msgr.osd_peer_fds.at(cur_list->rm_osd_num);
-                op->req = (osd_any_op_t){
-                    .rw = {
-                        .header = {
-                            .magic = SECONDARY_OSD_OP_MAGIC,
-                            .id = parent->cli->next_op_id(),
-                            .opcode = OSD_OP_DELETE,
-                        },
-                        .inode = cur_list->obj_pos->inode,
-                        .offset = cur_list->obj_pos->stripe,
-                        .len = 0,
-                    },
-                };
-                op->callback = [this, cur_list](osd_op_t *op)
+                cluster_op_t *op = new cluster_op_t;
+                op->opcode = OSD_OP_DELETE;
+                op->inode = cur_list->obj_pos->inode;
+                op->offset = cur_list->obj_pos->stripe;
+                op->len = 0;
+                op->flags = OSD_OP_IGNORE_READONLY | OSD_OP_WAIT_UP_TIMEOUT;
+                op->callback = [this, cur_list](cluster_op_t *op)
                 {
                     cur_list->in_flight--;
-                    if (op->reply.hdr.retval < 0)
+                    if (op->retval < 0)
                     {
-                        fprintf(stderr, "Failed to remove object %jx:%jx from PG %u (OSD %ju) (retval=%jd)\n",
-                            op->req.rw.inode, op->req.rw.offset,
-                            cur_list->pg_num, cur_list->rm_osd_num, op->reply.hdr.retval);
+                        fprintf(stderr, "Failed to remove object %jx:%jx from PG %u (retval=%d)\n",
+                            op->inode, op->offset, cur_list->pg_num, op->retval);
                         error_count++;
                     }
+                    else
+                    {
+                        if (!op->support_left_on_dead())
+                            fallback_pgs.insert(cur_list->pg_num);
+                        for (auto inactive_osd: op->get_left_on_dead())
+                            inactive_osds.insert(inactive_osd);
+                    }
                     delete op;
                     cur_list->obj_done++;
                     total_done++;
                     continue_delete();
                 };
                 cur_list->in_flight++;
-                parent->cli->msgr.outbox_push(op);
+                cur_list->obj_pos++;
+                parent->cli->execute(op);
+            }
+            else
+            {
+                cur_list->obj_pos++;
             }
-            cur_list->obj_pos++;
         }
         if (cur_list->in_flight == 0 && cur_list->obj_pos == cur_list->objects.end() &&
             !cur_list->synced)
         {
-            osd_op_t *op = new osd_op_t();
-            op->op_type = OSD_OP_OUT;
-            op->peer_fd = parent->cli->msgr.osd_peer_fds.at(cur_list->rm_osd_num);
-            op->req = (osd_any_op_t){
-                .sync = {
-                    .header = {
-                        .magic = SECONDARY_OSD_OP_MAGIC,
-                        .id = parent->cli->next_op_id(),
-                        .opcode = OSD_OP_SYNC,
-                    },
-                },
-            };
-            op->callback = [this, cur_list](osd_op_t *op)
+            cluster_op_t *op = new cluster_op_t;
+            op->opcode = OSD_OP_SYNC;
+            op->callback = [this, cur_list](cluster_op_t *op)
             {
                 cur_list->in_flight--;
                 cur_list->synced = true;
-                if (op->reply.hdr.retval < 0)
+                if (op->retval < 0)
                 {
-                    fprintf(stderr, "Failed to sync OSD %ju (retval=%jd)\n",
-                        cur_list->rm_osd_num, op->reply.hdr.retval);
+                    fprintf(stderr, "Failed to sync after deletion (retval=%d)\n", op->retval);
                     error_count++;
                 }
                 delete op;
                 continue_delete();
             };
             cur_list->in_flight++;
-            parent->cli->msgr.outbox_push(op);
+            parent->cli->execute(op);
         }
     }
 
@@ -194,6 +160,11 @@ struct rm_inode_t
         {
             return;
         }
+        if (in_continue)
+        {
+            return;
+        }
+        in_continue = true;
         for (int i = 0; i < lists.size(); i++)
         {
             if (!lists[i]->in_flight && lists[i]->obj_pos == lists[i]->objects.end() &&
@@ -202,10 +173,6 @@ struct rm_inode_t
                 delete lists[i];
                 lists.erase(lists.begin()+i, lists.begin()+i+1);
                 i--;
-                if (!lists_done)
-                {
-                    parent->cli->list_inode_next(lister, 1);
-                }
             }
             else
             {
@@ -215,8 +182,8 @@ struct rm_inode_t
         if (parent->progress && total_count > 0 && total_done*1000/total_count != total_prev_pct)
         {
             fprintf(stderr, parent->color
-                ? "\rRemoved %ju/%ju objects, %ju more PGs to list..."
-                : "Removed %ju/%ju objects, %ju more PGs to list...\n", total_done, total_count, pgs_to_list);
+                ? "\rRemoved %ju/%ju objects, %u more PGs to list..."
+                : "Removed %ju/%ju objects, %u more PGs to list...\n", total_done, total_count, pgs_to_list);
             total_prev_pct = total_done*1000/total_count;
         }
         if (lists_done && !lists.size())
@@ -225,28 +192,102 @@ struct rm_inode_t
             {
                 fprintf(stderr, "\n");
             }
-            bool is_error = (total_done < total_count || inactive_osds.size() > 0 || error_count > 0);
-            if (parent->progress && is_error)
+            if (fallback_pgs.size() && !parent->json_output)
             {
-                fprintf(
-                    stderr, "Warning: Pool:%u,ID:%ju inode data may not have been fully removed.\n"
-                    "Use `vitastor-cli rm-data --pool %u --inode %ju` if you encounter it in listings.\n",
-                    pool_id, INODE_NO_POOL(inode), pool_id, INODE_NO_POOL(inode)
-                );
+                fprintf(stderr, "Warning: some OSDs don't indicate left_on_dead PG OSDs"
+                    " in delete replies, falling back to simpler checks\n");
+                auto pool_it = parent->cli->st_cli.pool_config.find(pool_id);
+                if (pool_it != parent->cli->st_cli.pool_config.end())
+                {
+                    std::set<osd_num_t> all_peers;
+                    for (auto pg_num: fallback_pgs)
+                    {
+                        auto pg_it = pool_it->second.pg_config.find(pg_num);
+                        if (pg_it != pool_it->second.pg_config.end())
+                        {
+                            for (osd_num_t pg_osd: pg_it->second.target_set)
+                                all_peers.insert(pg_osd);
+                            for (osd_num_t pg_osd: pg_it->second.all_peers)
+                                all_peers.insert(pg_osd);
+                            for (auto & hist_item: pg_it->second.target_history)
+                                for (auto pg_osd: hist_item)
+                                    all_peers.insert(pg_osd);
+                        }
+                    }
+                    all_peers.erase(0);
+                    for (auto peer_osd: all_peers)
+                    {
+                        if (parent->cli->st_cli.peer_states[peer_osd].is_null())
+                            inactive_osds.insert(peer_osd);
+                    }
+                }
             }
-            result = (cli_result_t){
-                .err = is_error && !down_ok ? EIO : 0,
-                .text = is_error ? "Some blocks were not removed" : (
-                    "Done, inode "+std::to_string(INODE_NO_POOL(inode))+" from pool "+
-                    std::to_string(pool_id)+" removed"),
-                .data = json11::Json::object {
-                    { "removed_objects", total_done },
-                    { "total_objects", total_count },
-                    { "inactive_osds", inactive_osds },
-                },
+            if (inactive_osds.size() && !parent->json_output)
+            {
+                fprintf(stderr, "Some data may remain after delete on OSDs which are currently down: ");
+                int i = 0;
+                for (auto osd_num: inactive_osds)
+                {
+                    fprintf(stderr, (i++) ? ", %ju" : "%ju", osd_num);
+                }
+                fprintf(stderr, "\n");
+            }
+            if (inactive_pgs.size() && !parent->json_output)
+            {
+                fprintf(stderr, "Failed to list some PGs, deletion is not complete: PG ");
+                int i = 0;
+                for (auto pg_num: inactive_pgs)
+                {
+                    fprintf(stderr, (i++) > 0 ? ", %u" : "%u", pg_num);
+                }
+                fprintf(stderr, "\n");
+            }
+            if (error_count > 0 && !parent->json_output)
+            {
+                fprintf(stderr, "Failed to delete %u objects from active OSD(s).\n", error_count);
+            }
+            json11::Json::array inactive_pgs_json;
+            for (auto pg_num: inactive_pgs)
+            {
+                inactive_pgs_json.push_back((uint64_t)pg_num);
+            }
+            json11::Json data = json11::Json::object {
+                { "removed_objects", total_done },
+                { "total_objects", total_count },
+                { "error_count", error_count },
+                { "inactive_osds", json11::Json::array(inactive_osds.begin(), inactive_osds.end()) },
+                { "inactive_pgs", inactive_pgs_json },
             };
             state = 100;
+            if (total_done < total_count || inactive_pgs.size() > 0 || error_count > 0 ||
+                inactive_osds.size() > 0 && !down_ok)
+            {
+                // Error
+                result = (cli_result_t){
+                    .err = EIO,
+                    .text = "Failed: some blocks were not removed",
+                    .data = data,
+                };
+            }
+            else
+            {
+                if (parent->progress && inactive_osds.size() > 0 && down_ok)
+                {
+                    fprintf(
+                        stderr, "Warning: --down-ok is set and some OSDs are down.\n"
+                        "Pool:%u,ID:%ju inode data may not have been fully removed.\n"
+                        "Use `vitastor-cli rm-data --pool %u --inode %ju` if you encounter it in listings.\n",
+                        pool_id, INODE_NO_POOL(inode), pool_id, INODE_NO_POOL(inode)
+                    );
+                }
+                result = (cli_result_t){
+                    .text = "Done, inode "+std::to_string(INODE_NO_POOL(inode))+" from pool "+
+                        std::to_string(pool_id)+" removed",
+                    .data = data,
+                };
+            }
         }
+        in_continue = false;
     }
 
     bool is_done()

src/cmd/cli_rm_wildcard.cpp

@@ -73,8 +73,11 @@ struct wildcard_remover_t
             {
                 auto & inode_cfg = parent->cli->st_cli.inode_config.at(child_id);
                 ver_chain.push_back((inode_rev_t){ .inode_num = child_id, .meta_rev = inode_cfg.mod_revision });
-                child_id = inode_cfg.parent_id;
-            } while (child_id && child_id != parent_id);
+                if (child_id == parent_id)
+                    break;
+                else
+                    child_id = inode_cfg.parent_id;
+            } while (child_id);
             versioned_chains.push_back(std::move(ver_chain));
         }
         // Sort chains based on parent inode rank to first delete child-most layers
@@ -146,6 +149,7 @@ struct wildcard_remover_t
                 sub_cfg = cfg.object_items();
                 sub_cfg.erase("globs");
                 sub_cfg.erase("exact");
+                sub_cfg.erase("matching");
                 sub_cfg["from"] = from_cfg.name;
                 sub_cfg["to"] = to_cfg.name;
                 sub_cb = parent->start_rm(sub_cfg);

src/cmd/cli_status.cpp

@@ -134,6 +134,7 @@ resume_2:
         }
         int osd_count = 0, osd_up = 0;
         uint64_t total_raw = 0, free_raw = 0, free_down_raw = 0, down_raw = 0;
+        std::vector<uint64_t> slow_op_primary_osds, slow_op_secondary_osds;
         parent->iterate_kvs_1(osd_stats, "/osd/stats/", [&](uint64_t stat_osd_num, json11::Json value)
         {
             osd_count++;
@@ -141,18 +142,29 @@ resume_2:
             auto osd_free = value["free"].uint64_value();
             total_raw += osd_size;
             free_raw += osd_free;
-            if (!osd_free)
+            if (osd_size)
             {
-                osds_full++;
-            }
-            else if (osd_free < (uint64_t)(osd_size*(1-osd_nearfull_ratio)))
-            {
-                osds_nearfull++;
+                if (!osd_free)
+                {
+                    osds_full++;
+                }
+                else if (osd_free < (uint64_t)(osd_size*(1-osd_nearfull_ratio)))
+                {
+                    osds_nearfull++;
+                }
             }
             auto peer_it = parent->cli->st_cli.peer_states.find(stat_osd_num);
             if (peer_it != parent->cli->st_cli.peer_states.end())
             {
                 osd_up++;
+                if (value["slow_ops_primary"].uint64_value() > 0)
+                {
+                    slow_op_primary_osds.push_back(stat_osd_num);
+                }
+                if (value["slow_ops_secondary"].uint64_value() > 0)
+                {
+                    slow_op_secondary_osds.push_back(stat_osd_num);
+                }
             }
             else
             {
@@ -161,6 +173,8 @@ resume_2:
             }
         });
         int pool_count = 0, pools_active = 0;
+        json11::Json::array backfillfull_pool_ids;
+        std::string backfillfull_pool_names;
         std::map<std::string, int> pgs_by_state;
         std::string pgs_by_state_str;
         for (auto & pool_pair: parent->cli->st_cli.pool_config)
@@ -192,6 +206,13 @@ resume_2:
             {
                 pools_active++;
             }
+            if (pool_cfg.backfillfull)
+            {
+                backfillfull_pool_ids.push_back((uint64_t)pool_pair.first);
+                if (backfillfull_pool_names != "")
+                    backfillfull_pool_names += ", ";
+                backfillfull_pool_names += pool_cfg.name;
+            }
         }
         for (auto & kv: pgs_by_state)
         {
@@ -216,6 +237,10 @@ resume_2:
                 { "mon_master", mon_master },
                 { "osd_up", osd_up },
                 { "osd_count", osd_count },
+                { "osds_full", osds_full },
+                { "osds_nearfull", osds_nearfull },
+                { "osds_primary_slow_ops", slow_op_primary_osds },
+                { "osds_secondary_slow_ops", slow_op_secondary_osds },
                 { "total_raw", total_raw },
                 { "free_raw", free_raw },
                 { "down_raw", down_raw },
@@ -226,6 +251,7 @@ resume_2:
                 { "no_scrub", no_scrub },
                 { "pool_count", pool_count },
                 { "active_pool_count", pools_active },
+                { "backfillfull_pools", backfillfull_pool_ids },
                 { "pg_states", pgs_by_state },
                 { "op_stats", agg_stats["op_stats"] },
                 { "recovery_stats", agg_stats["recovery_stats"] },
@@ -300,6 +326,35 @@ resume_2:
             warning_str += "    "+std::to_string(osds_nearfull)+
                 (osds_nearfull > 1 ? " osds are almost full\n" : " osd is almost full\n");
         }
+        if (slow_op_primary_osds.size() > 0)
+        {
+            warning_str += "    "+std::to_string(slow_op_primary_osds.size());
+            warning_str += (slow_op_primary_osds.size() > 1 ? " osds have" : " osd has");
+            warning_str += " slow client ops: ";
+            for (int i = 0; i < slow_op_primary_osds.size(); i++)
+            {
+                warning_str += (i > 0 ? ", " : "")+std::to_string(slow_op_primary_osds[i]);
+            }
+            warning_str += "\n";
+        }
+        if (slow_op_secondary_osds.size() > 0)
+        {
+            warning_str += "    "+std::to_string(slow_op_secondary_osds.size());
+            warning_str += (slow_op_secondary_osds.size() > 1 ? " osds have" : " osd has");
+            warning_str += " slow replication ops: ";
+            for (int i = 0; i < slow_op_secondary_osds.size(); i++)
+            {
+                warning_str += (i > 0 ? ", " : "")+std::to_string(slow_op_secondary_osds[i]);
+            }
+            warning_str += "\n";
+        }
+        if (backfillfull_pool_names != "")
+        {
+            if (backfillfull_pool_ids.size() > 1)
+                warning_str += "    pools "+backfillfull_pool_names+" are backfillfull\n";
+            else
+                warning_str += "    pool "+backfillfull_pool_names+" is backfillfull\n";
+        }
         if (warning_str != "")
         {
             warning_str = "\n  warning:\n"+warning_str;

src/disk_tool/disk_tool.cpp

@@ -68,6 +68,7 @@ static const char *help_text =
     "    --data_device_block 4k     Override data device block size\n"
     "    --meta_device_block 4k     Override metadata device block size\n"
     "    --journal_device_block 4k  Override journal device block size\n"
+    "    --json                     Enable JSON output\n"
     "  \n"
     "  immediate_commit setting is automatically derived from \"disable fsync\" options.\n"
     "  It's set to \"all\" when fsync is disabled on all devices, and to \"small\" if fsync\n"
@@ -102,6 +103,7 @@ static const char *help_text =
     "    --move-meta TARGET       move metadata to TARGET\n"
     "    --journal-size NEW_SIZE  resize journal to NEW_SIZE\n"
     "    --data-size NEW_SIZE     resize data device to NEW_SIZE\n"
+    "    --json                   enable JSON output\n"
     "    --dry-run                only show new layout, do not apply it\n"
     "  \n"
     "  NEW_SIZE may include k/m/g/t suffixes.\n"
@@ -131,10 +133,11 @@ static const char *help_text =
     "  Commands are passed to systemctl with vitastor-osd@<num> units as arguments.\n"
     "  When --now is added to enable/disable, OSDs are also immediately started/stopped.\n"
     "\n"
-    "vitastor-disk purge [--force] [--allow-data-loss] <device> [device2 device3 ...]\n"
+    "vitastor-disk purge [--json] [--force] [--allow-data-loss] <device> [device2 device3 ...]\n"
     "  Purge Vitastor OSD(s) on specified device(s). Uses vitastor-cli rm-osd to check\n"
     "  if deletion is possible without data loss and to actually remove metadata from etcd.\n"
     "  --force and --allow-data-loss options may be used to ignore safety check results.\n"
+    "  --json enables JSON output.\n"
     "  \n"
     "  Requires `vitastor-cli`, `sfdisk` and `partprobe` (from parted) utilities.\n"
     "\n"

src/disk_tool/disk_tool.h

@@ -42,7 +42,7 @@ struct disk_tool_t
 
     std::map<std::string, std::string> options;
     bool test_mode = false;
-    bool all, json, now;
+    bool all = false, json = false, now = false;
     bool dump_with_blocks, dump_with_data;
     blockstore_disk_t dsk;
 
@@ -127,7 +127,7 @@ struct disk_tool_t
     json11::Json read_osd_superblock(std::string device, bool expect_exist = true, bool ignore_nonref = false);
     uint32_t write_osd_superblock(std::string device, json11::Json params);
 
-    int prepare_one(std::map<std::string, std::string> options, int is_hdd = -1);
+    int prepare_one(std::map<std::string, std::string> options, int is_hdd, json11::Json::object & result);
     int check_existing_partition(std::string & dev_by_uuid);
     int fix_partition_type(std::string & dev_by_uuid);
     int prepare(std::vector<std::string> devices);

src/disk_tool/disk_tool_prepare.cpp

@@ -6,7 +6,7 @@
 #include "json_util.h"
 #include "osd_id.h"
 
-int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_hdd)
+int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_hdd, json11::Json::object & result)
 {
     static const char *allow_additional_params[] = {
         "autosync_writes",
@@ -108,6 +108,10 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
     try
     {
         dsk.parse_config(options);
+        // Set all offsets to 4096 to calculate metadata size with excess
+        dsk.journal_offset = 4096;
+        dsk.meta_offset = 4096;
+        dsk.data_offset = 4096;
         dsk.data_io = dsk.meta_io = dsk.journal_io = (options["io"] == "cached" ? "cached" : "direct");
         dsk.open_data();
         dsk.open_meta();
@@ -171,8 +175,8 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
     }
     sb["osd_num"] = osd_num;
     // Zero out metadata and journal
-    if (write_zero(dsk.meta_fd, dsk.meta_offset, dsk.meta_len) != 0 ||
-        write_zero(dsk.journal_fd, dsk.journal_offset, dsk.journal_len) != 0)
+    if (write_zero(dsk.meta_fd, sb["meta_offset"].uint64_value(), dsk.meta_len) != 0 ||
+        write_zero(dsk.journal_fd, sb["journal_offset"].uint64_value(), dsk.journal_len) != 0)
     {
         fprintf(stderr, "Failed to zero out metadata or journal: %s\n", strerror(errno));
         dsk.close_all();
@@ -199,12 +203,14 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
     fprintf(stderr, "Initialized OSD %ju on %s\n", osd_num, desc.c_str());
     if (!test_mode || options.find("no_init") == options.end())
     {
-        if (shell_exec({ "systemctl", "enable", "--now", "vitastor-osd@"+std::to_string(osd_num) }, "", NULL, NULL) != 0)
+        std::string out;
+        if (shell_exec({ "systemctl", "enable", "--now", "vitastor-osd@"+std::to_string(osd_num) }, "", json ? &out : NULL, NULL) != 0)
         {
             fprintf(stderr, "Failed to enable systemd unit vitastor-osd@%ju\n", osd_num);
             return 1;
         }
     }
+    result = sb;
     return 0;
 }
 
@@ -498,6 +504,9 @@ int disk_tool_t::get_meta_partition(std::vector<vitastor_dev_info_t> & ssds, std
     {
         blockstore_disk_t dsk;
         dsk.parse_config(options);
+        dsk.journal_offset = 4096;
+        dsk.meta_offset = 4096;
+        dsk.data_offset = 4096;
         dsk.data_io = dsk.meta_io = dsk.journal_io = "cached";
         dsk.open_data();
         dsk.open_meta();
@@ -571,7 +580,13 @@ int disk_tool_t::prepare(std::vector<std::string> devices)
             fprintf(stderr, "Device list (positional arguments), --osd_per_disk, --hybrid and --fast-devices are incompatible with --data_device\n");
             return 1;
         }
-        return prepare_one(options, options.find("hdd") != options.end() ? 1 : 0);
+        json11::Json::object result;
+        int r = prepare_one(options, options.find("hdd") != options.end() ? 1 : 0, result);
+        if (r)
+            return r;
+        if (json)
+            printf("%s\n", json11::Json(result).dump().c_str());
+        return 0;
     }
     if (!devices.size())
     {
@@ -662,6 +677,7 @@ int disk_tool_t::prepare(std::vector<std::string> devices)
         options.erase("disable_meta_fsync");
         options.erase("disable_journal_fsync");
     }
+    json11::Json::array all_results, errors;
     auto journal_size = options["journal_size"];
     for (auto & dev: devinfo)
     {
@@ -681,7 +697,15 @@ int disk_tool_t::prepare(std::vector<std::string> devices)
                 options.erase("journal_size");
             }
             // Treat all disks as SSDs if not in the hybrid mode
-            prepare_one(options, dev.is_hdd ? 1 : 0);
+            json11::Json::object result;
+            int r = prepare_one(options, dev.is_hdd ? 1 : 0, result);
+            if (json)
+            {
+                if (!r)
+                    all_results.push_back(std::move(result));
+                else
+                    errors.push_back(options);
+            }
             if (hybrid)
             {
                 options["journal_size"] = journal_size;
@@ -690,5 +714,9 @@ int disk_tool_t::prepare(std::vector<std::string> devices)
             }
         }
     }
+    if (json)
+    {
+        printf("%s\n", json11::Json(json11::Json::object{ { "osds", all_results }, { "errors", errors } }).dump().c_str());
+    }
     return 0;
 }

src/disk_tool/disk_tool_udev.cpp

@@ -427,8 +427,12 @@ int disk_tool_t::purge_devices(const std::vector<std::string> & devices)
             }
         }
     }
+    json11::Json::object result;
+    result["removed_osds"] = std::vector<uint64_t>(osd_numbers.begin(), osd_numbers.end());
     if (!osd_numbers.size())
     {
+        if (json)
+            printf("%s\n", json11::Json(result).dump().c_str());
         return 0;
     }
     std::vector<std::string> rm_osd_cli = { "vitastor-cli", "rm-osd" };
@@ -457,17 +461,18 @@ int disk_tool_t::purge_devices(const std::vector<std::string> & devices)
     {
         systemctl_cli.push_back("vitastor-osd@"+std::to_string(osd_num));
     }
-    if (shell_exec(systemctl_cli, "", NULL, NULL) != 0)
+    if (shell_exec(systemctl_cli, "", json ? &dry_run_ignore_stdout : NULL, NULL) != 0)
     {
         return 1;
     }
     // Remove OSD metadata
     rm_osd_cli.pop_back();
-    if (shell_exec(rm_osd_cli, "", NULL, NULL) != 0)
+    if (shell_exec(rm_osd_cli, "", json ? &dry_run_ignore_stdout : NULL, NULL) != 0)
     {
         return 1;
     }
     // Destroy OSD superblocks
+    json11::Json::array removed_devices;
     for (auto & sb: superblocks)
     {
         for (auto dev_type: std::vector<std::string>{ "data", "meta", "journal" })
@@ -521,9 +526,15 @@ int disk_tool_t::purge_devices(const std::vector<std::string> & devices)
                             break;
                         }
                     }
+                    removed_devices.push_back(dev);
                 }
             }
         }
     }
+    if (json)
+    {
+        result["removed_devices"] = removed_devices;
+        printf("%s\n", json11::Json(result).dump().c_str());
+    }
     return 0;
 }

src/kv/kv_cli.cpp

@@ -546,7 +546,7 @@ void kv_cli_t::handle_cmd(const std::vector<std::string> & cmd, std::function<vo
             auto & key = cmd[1];
             if (opname == "get")
             {
-                db->get(key, [this, cb](int res, const std::string & value)
+                db->get(key, [cb](int res, const std::string & value)
                 {
                     if (res < 0)
                         fprintf(stderr, "Error: %s (code %d)\n", strerror(-res), res);

src/kv/kv_db.cpp

@@ -1448,7 +1448,7 @@ void kv_op_t::update()
 
 void kv_op_t::update_find()
 {
-    get_block(db, cur_block, cur_level, recheck_policy, [=, checked_block = cur_block](int res, int refresh)
+    get_block(db, cur_block, cur_level, recheck_policy, [=](int res, int refresh)
     {
         res = handle_block(res, refresh, true);
         if (res == -EAGAIN)

src/nfs/CMakeLists.txt

@@ -5,6 +5,7 @@ project(vitastor)
 # vitastor-nfs
 add_executable(vitastor-nfs
 	nfs_proxy.cpp
+	nfs_proxy_rdma.cpp
 	nfs_block.cpp
 	nfs_kv.cpp
 	nfs_kv_create.cpp
@@ -21,8 +22,10 @@ add_executable(vitastor-nfs
 	nfs_fsstat.cpp
 	nfs_mount.cpp
 	nfs_portmap.cpp
+	rdma_alloc.cpp
 	../util/sha256.c
 	proto/xdr_impl.cpp
+	proto/rpc_rdma_xdr.cpp
 	proto/rpc_xdr.cpp
 	proto/portmap_xdr.cpp
 	proto/nfs_xdr.cpp
@@ -30,4 +33,5 @@ add_executable(vitastor-nfs
 target_link_libraries(vitastor-nfs
 	vitastor_client
 	vitastor_kv
+	${RDMACM_LIBRARIES}
 )

src/nfs/nfs_block.cpp

@@ -315,8 +315,7 @@ static int block_nfs3_read_proc(void *opaque, rpc_op_t *rop)
     if (aligned_count % alignment)
         aligned_count = aligned_count + alignment - (aligned_count % alignment);
     aligned_count -= aligned_offset;
-    void *buf = malloc_or_die(aligned_count);
-    xdr_add_malloc(rop->xdrs, buf);
+    void *buf = self->malloc_or_rdma(rop, aligned_count);
     cluster_op_t *op = new cluster_op_t;
     op->opcode = OSD_OP_READ;
     op->inode = ino_it->second;
@@ -335,10 +334,22 @@ static int block_nfs3_read_proc(void *opaque, rpc_op_t *rop)
         }
         else
         {
+            nfs_client_t *self = (nfs_client_t*)rop->client;
             auto & reply_ok = reply->resok;
             // reply_ok.data.data is already set above
             reply_ok.count = reply_ok.data.size;
             reply_ok.eof = 0;
+            if (self->rdma_conn)
+            {
+                // FIXME Linux NFS RDMA transport has a bug - when the reply
+                // doesn't contain post_op_attr, the data gets offsetted by
+                // 84 bytes (size of attributes)...
+                // So we have to fill it with RDMA. :-(
+                reply_ok.file_attributes = (post_op_attr){
+                    .attributes_follow = 1,
+                    .attributes = get_file_attributes(self, op->inode),
+                };
+            }
         }
         rpc_queue_reply(rop);
         delete op;
@@ -1057,22 +1068,25 @@ static void block_nfs3_readdir_common(void *opaque, rpc_op_t *rop, bool is_plus)
     void *prev = NULL;
     for (auto it = entries.begin(); it != entries.end();)
     {
-        entryplus3 *entry = &it->second;
-        // First fields of entry3 and entryplus3 are the same: fileid, name, cookie
-        entry->name = xdr_copy_string(rop->xdrs, it->first);
-        entry->cookie = idx++;
-        if (prev)
-        {
-            if (is_plus)
-                ((entryplus3*)prev)->nextentry = entry;
-            else
-                ((entry3*)prev)->nextentry = (entry3*)entry;
-        }
-        prev = entry;
-        if (args->cookie > 0 && entry->cookie == args->cookie)
-            entries.erase(entries.begin(), ++it);
+        if (args->cookie > 0 && idx <= args->cookie)
+            entries.erase(it++);
         else
+        {
+            entryplus3 *entry = &it->second;
+            // First fields of entry3 and entryplus3 are the same: fileid, name, cookie
+            entry->name = xdr_copy_string(rop->xdrs, it->first);
+            entry->cookie = idx;
+            if (prev)
+            {
+                if (is_plus)
+                    ((entryplus3*)prev)->nextentry = entry;
+                else
+                    ((entry3*)prev)->nextentry = (entry3*)entry;
+            }
+            prev = entry;
             it++;
+        }
+        idx++;
     }
     // Now limit results based on maximum reply size
     // Sadly we have to calculate reply size by hand

src/nfs/nfs_kv.cpp

@@ -65,7 +65,7 @@ int kv_map_type(const std::string & type)
         (type == "fifo" ? NF3FIFO : -1)))))));
 }
 
-fattr3 get_kv_attributes(nfs_client_t *self, uint64_t ino, json11::Json attrs)
+fattr3 get_kv_attributes(nfs_proxy_t *proxy, uint64_t ino, json11::Json attrs)
 {
     auto type = kv_map_type(attrs["type"].string_value());
     auto mode = attrs["mode"].uint64_value();
@@ -86,7 +86,7 @@ fattr3 get_kv_attributes(nfs_client_t *self, uint64_t ino, json11::Json attrs)
         .rdev = (type == NF3BLK || type == NF3CHR
             ? (specdata3){ (uint32_t)attrs["major"].uint64_value(), (uint32_t)attrs["minor"].uint64_value() }
             : (specdata3){}),
-        .fsid = self->parent->fsid,
+        .fsid = proxy->fsid,
         .fileid = ino,
         .atime = atime,
         .mtime = mtime,
@@ -119,7 +119,7 @@ std::string kv_direntry_filename(const std::string & key)
 std::string kv_inode_prefix_key(uint64_t ino, const char *prefix)
 {
     int max = 32+strlen(prefix);
-    char key[max] = { 0 };
+    char key[max];
     snprintf(key, max, "%s%x", prefix, INODE_POOL(ino));
     int n = strnlen(key, max-1);
     snprintf(key+n+1, max-n-1, "%jx", INODE_NO_POOL(ino));
@@ -349,6 +349,27 @@ kv_fs_state_t::~kv_fs_state_t()
     }
 }
 
+void kv_fs_state_t::write_inode(inode_t ino, json11::Json value, bool hack_cache, std::function<void(int)> cb, std::function<bool(int, const std::string &)> cas_cb)
+{
+    if (!proxy->rdma_context)
+    {
+        proxy->db->set(kv_inode_key(ino), value.dump(), cb, cas_cb);
+        return;
+    }
+    // FIXME Linux NFS RDMA transport has a bug - it corrupts the data (by offsetting it 84 bytes)
+    // when the READ reply doesn't contain post_op_attr. So we have to fill post_op_attr with RDMA. :-(
+    // So we at least cache it to not repeat K/V requests every read.
+    read_hack_cache.erase(ino);
+    proxy->db->set(kv_inode_key(ino), value.dump(), [=](int res)
+    {
+        if (hack_cache || res != 0)
+            read_hack_cache.erase(ino);
+        else
+            read_hack_cache[ino] = value;
+        cb(res);
+    }, cas_cb);
+}
+
 void kv_fs_state_t::update_inode(inode_t ino, bool allow_cache, std::function<void(json11::Json::object &)> change, std::function<void(int)> cb)
 {
     // FIXME: Use "update" query
@@ -356,12 +377,15 @@ void kv_fs_state_t::update_inode(inode_t ino, bool allow_cache, std::function<vo
     {
         if (!res)
         {
+            read_hack_cache.erase(ino);
             auto ientry = attrs.object_items();
             change(ientry);
             bool *found = new bool;
             *found = true;
-            proxy->db->set(kv_inode_key(ino), json11::Json(ientry).dump(), [=](int res)
+            json11::Json ientry_json(ientry);
+            proxy->db->set(kv_inode_key(ino), ientry_json.dump(), [=](int res)
             {
+                read_hack_cache.erase(ino);
                 if (!*found)
                     res = -ENOENT;
                 delete found;
@@ -384,6 +408,8 @@ void kv_fs_state_t::update_inode(inode_t ino, bool allow_cache, std::function<vo
 
 void kv_fs_state_t::touch_inodes()
 {
+    // Clear RDMA read fattr3 "hack" cache every second
+    read_hack_cache.clear();
     std::set<inode_t> q = std::move(touch_queue);
     for (auto ino: q)
     {

src/nfs/nfs_kv.h

@@ -75,6 +75,7 @@ struct kv_fs_state_t
     std::map<inode_t, kv_inode_extend_t> extends;
     std::set<inode_t> touch_queue;
     std::map<inode_t, uint64_t> volume_removed;
+    std::map<inode_t, json11::Json> read_hack_cache;
     uint64_t volume_stats_ctr = 0;
     uint64_t volume_touch_ctr = 0;
 
@@ -87,6 +88,7 @@ struct kv_fs_state_t
     void upgrade_db(std::function<void(int)> cb);
     void defrag_all(json11::Json cfg, std::function<void(int)> cb);
     void defrag_volume(inode_t ino, bool no_rm, bool dry_run, std::function<void(int, uint64_t, uint64_t, uint64_t)> cb);
+    void write_inode(inode_t ino, json11::Json value, bool hack_cache, std::function<void(int)> cb, std::function<bool(int, const std::string &)> cas_cb);
     ~kv_fs_state_t();
 };
 
@@ -116,7 +118,7 @@ nfstime3 nfstime_from_str(const std::string & s);
 std::string nfstime_to_str(nfstime3 t);
 std::string nfstime_now_str();
 int kv_map_type(const std::string & type);
-fattr3 get_kv_attributes(nfs_client_t *self, uint64_t ino, json11::Json attrs);
+fattr3 get_kv_attributes(nfs_proxy_t *proxy, uint64_t ino, json11::Json attrs);
 std::string kv_direntry_key(uint64_t dir_ino, const std::string & filename);
 std::string kv_direntry_filename(const std::string & key);
 std::string kv_inode_prefix_key(uint64_t ino, const char *prefix);

src/nfs/nfs_kv_create.cpp

@@ -143,11 +143,11 @@ resume_2:
         cb(st->res);
         return;
     }
-    st->self->parent->db->set(kv_inode_key(st->new_id), st->attrs.dump().c_str(), [st](int res)
+    st->self->parent->kvfs->write_inode(st->new_id, st->attrs, false, [st](int res)
     {
         st->res = res;
         kv_continue_create(st, 3);
-    }, [st](int res, const std::string & value)
+    }, [](int res, const std::string & value)
     {
         return res == -ENOENT;
     });
@@ -267,7 +267,7 @@ template<class T, class Tok> static void kv_create_reply(kv_create_state *st, in
                 },
                 .obj_attributes = {
                     .attributes_follow = 1,
-                    .attributes = get_kv_attributes(st->self, st->new_id, st->attrs),
+                    .attributes = get_kv_attributes(st->self->parent, st->new_id, st->attrs),
                 },
             },
         };

src/nfs/nfs_kv_defrag.cpp

@@ -81,12 +81,12 @@ void kv_fs_defrag_t::read()
             empty = true;
             for (; bitmap_pos < bitmap_size; bitmap_pos += 8)
             {
-                if (*((uint64_t*)(op->bitmap_buf + bitmap_pos)))
+                if (*((uint64_t*)((uint8_t*)op->bitmap_buf + bitmap_pos)))
                     empty = false;
             }
             for (; bitmap_pos < bitmap_size; bitmap_pos++)
             {
-                if (*((uint8_t*)(op->bitmap_buf + bitmap_pos)))
+                if (*((uint8_t*)((uint8_t*)op->bitmap_buf + bitmap_pos)))
                     empty = false;
             }
             buf_pos = 0;

src/nfs/nfs_kv_getattr.cpp

@@ -68,7 +68,7 @@ int kv_nfs3_getattr_proc(void *opaque, rpc_op_t *rop)
             *reply = (GETATTR3res){
                 .status = NFS3_OK,
                 .resok = (GETATTR3resok){
-                    .obj_attributes = get_kv_attributes(self, ino, attrs),
+                    .obj_attributes = get_kv_attributes(self->parent, ino, attrs),
                 },
             };
         }

src/nfs/nfs_kv_link.cpp

@@ -81,7 +81,7 @@ resume_1:
         {
             st->res = res;
             nfs_kv_continue_link(st, 2);
-        }, [st](int res, const std::string & old_value)
+        }, [](int res, const std::string & old_value)
         {
             return res == -ENOENT;
         });
@@ -102,7 +102,7 @@ resume_2:
         new_ientry["ctime"] = nfstime_now_str();
         st->ientry = new_ientry;
     }
-    st->self->parent->db->set(kv_inode_key(st->ino), st->ientry.dump(), [st](int res)
+    st->self->parent->kvfs->write_inode(st->ino, st->ientry, false, [st](int res)
     {
         st->res = res;
         nfs_kv_continue_link(st, 3);
@@ -180,7 +180,7 @@ int kv_nfs3_link_proc(void *opaque, rpc_op_t *rop)
                 .resok = (LINK3resok){
                     .file_attributes = (post_op_attr){
                         .attributes_follow = 1,
-                        .attributes = get_kv_attributes(st->self, st->ino, st->ientry),
+                        .attributes = get_kv_attributes(st->self->parent, st->ino, st->ientry),
                     },
                 },
             };

src/nfs/nfs_kv_lookup.cpp

@@ -55,7 +55,7 @@ int kv_nfs3_lookup_proc(void *opaque, rpc_op_t *rop)
                     .object = xdr_copy_string(rop->xdrs, kv_fh(ino)),
                     .obj_attributes = {
                         .attributes_follow = 1,
-                        .attributes = get_kv_attributes(self, ino, ientry),
+                        .attributes = get_kv_attributes(self->parent, ino, ientry),
                     },
                 },
             };
@@ -91,10 +91,14 @@ int kv_nfs3_readlink_proc(void *opaque, rpc_op_t *rop)
         }
         else
         {
+            std::string link_target = attrs["symlink"].string_value();
+            char *cp = (char*)self->malloc_or_rdma(rop, link_target.size()+1);
+            memcpy(cp, link_target.data(), link_target.size());
+            cp[link_target.size()] = 0;
             *reply = (READLINK3res){
                 .status = NFS3_OK,
                 .resok = (READLINK3resok){
-                    .data = xdr_copy_string(rop->xdrs, attrs["symlink"].string_value()),
+                    .data = (xdr_string_t){ link_target.size(), cp },
                 },
             };
         }

src/nfs/nfs_kv_read.cpp

@@ -37,6 +37,7 @@ static void nfs_kv_continue_read(nfs_kv_read_state *st, int state)
     else if (state == 1) goto resume_1;
     else if (state == 2) goto resume_2;
     else if (state == 3) goto resume_3;
+    else if (state == 4) goto resume_4;
     else
     {
         fprintf(stderr, "BUG: invalid state in nfs_kv_continue_read()");
@@ -96,7 +97,7 @@ resume_1:
                 }
                 read_size += sizeof(shared_file_header_t);
                 assert(!st->aligned_buf);
-                st->aligned_buf = (uint8_t*)malloc_or_die(read_size);
+                st->aligned_buf = (uint8_t*)st->self->malloc_or_rdma(st->rop, read_size);
                 st->buf = st->aligned_buf + sizeof(shared_file_header_t) + st->offset;
                 st->op->iov.push_back(st->aligned_buf, read_size);
                 st->op->len = align_up(read_offset+read_size) - st->op->offset;
@@ -106,7 +107,7 @@ resume_1:
                         st->op->offset+st->op->len - (read_offset+read_size));
                 }
             }
-            st->op->callback = [st, state](cluster_op_t *op)
+            st->op->callback = [st](cluster_op_t *op)
             {
                 st->res = op->retval == op->len ? 0 : op->retval;
                 delete op;
@@ -117,7 +118,7 @@ resume_1:
 resume_2:
             if (st->res < 0)
             {
-                free(st->aligned_buf);
+                st->self->free_or_rdma(st->rop, st->aligned_buf);
                 st->aligned_buf = NULL;
                 auto cb = std::move(st->cb);
                 cb(st->res);
@@ -131,7 +132,7 @@ resume_2:
                     " 0x%jx offset 0x%jx: probably a read/write conflict, retrying\n",
                     st->ino, st->ientry["shared_ino"].uint64_value(), st->ientry["shared_offset"].uint64_value());
                 st->retry++;
-                free(st->aligned_buf);
+                st->self->free_or_rdma(st->rop, st->aligned_buf);
                 st->aligned_buf = NULL;
                 st->allow_cache = false;
                 goto resume_0;
@@ -141,10 +142,39 @@ resume_2:
             return;
         }
     }
+    else if (st->self->rdma_conn)
+    {
+        // Take ientry from read_hack_cache for RDMA connections
+        {
+            auto rh_it = st->self->parent->kvfs->read_hack_cache.find(st->ino);
+            if (rh_it != st->self->parent->kvfs->read_hack_cache.end())
+            {
+                st->ientry = rh_it->second;
+            }
+        }
+        if (st->ientry.is_null())
+        {
+            kv_read_inode(st->self->parent, st->ino, [st](int res, const std::string & value, json11::Json attrs)
+            {
+                st->res = res;
+                st->ientry = attrs;
+                nfs_kv_continue_read(st, 4);
+            }, st->allow_cache);
+            return;
+resume_4:
+            if (st->res < 0 || kv_map_type(st->ientry["type"].string_value()) != NF3REG)
+            {
+                auto cb = std::move(st->cb);
+                cb(st->res < 0 ? st->res : -EINVAL);
+                return;
+            }
+            st->self->parent->kvfs->read_hack_cache[st->ino] = st->ientry;
+        }
+    }
     st->aligned_offset = align_down(st->offset);
     st->aligned_size = align_up(st->offset+st->size) - st->aligned_offset;
     assert(!st->aligned_buf);
-    st->aligned_buf = (uint8_t*)malloc_or_die(st->aligned_size);
+    st->aligned_buf = (uint8_t*)st->self->malloc_or_rdma(st->rop, st->aligned_size);
     st->buf = st->aligned_buf + st->offset - st->aligned_offset;
     st->op = new cluster_op_t;
     st->op->opcode = OSD_OP_READ;
@@ -163,7 +193,7 @@ resume_2:
 resume_3:
     if (st->res < 0)
     {
-        free(st->aligned_buf);
+        st->self->free_or_rdma(st->rop, st->aligned_buf);
         st->aligned_buf = NULL;
     }
     auto cb = std::move(st->cb);
@@ -194,11 +224,21 @@ int kv_nfs3_read_proc(void *opaque, rpc_op_t *rop)
         *reply = (READ3res){ .status = vitastor_nfs_map_err(res) };
         if (res == 0)
         {
-            xdr_add_malloc(st->rop->xdrs, st->aligned_buf);
             reply->resok.data.data = (char*)st->buf;
             reply->resok.data.size = st->size;
             reply->resok.count = st->size;
             reply->resok.eof = st->eof;
+            if (st->self->rdma_conn)
+            {
+                // FIXME Linux NFS RDMA transport has a bug - when the reply
+                // doesn't contain post_op_attr, the data gets offsetted by
+                // 84 bytes (size of attributes)...
+                // So we have to fill it with RDMA. :-(
+                reply->resok.file_attributes = (post_op_attr){
+                    .attributes_follow = 1,
+                    .attributes = get_kv_attributes(st->self->parent, st->ino, st->ientry),
+                };
+            }
         }
         rpc_queue_reply(st->rop);
         delete st;

src/nfs/nfs_kv_readdir.cpp

@@ -57,7 +57,7 @@ static void kv_getattr_next(nfs_kv_readdir_state *st)
                 st->entries[idx].name_attributes = (post_op_attr){
                     // FIXME: maybe do not read parent attributes and leave them to a GETATTR?
                     .attributes_follow = 1,
-                    .attributes = get_kv_attributes(st->self, st->entries[idx].fileid, ientry),
+                    .attributes = get_kv_attributes(st->self->parent, st->entries[idx].fileid, ientry),
                 };
             }
             st->getattr_running--;
@@ -126,7 +126,7 @@ resume_1:
             dot.fileid = st->dir_ino;
             dot.name_attributes = (post_op_attr){
                 .attributes_follow = 1,
-                .attributes = get_kv_attributes(st->self, st->dir_ino, st->ientry),
+                .attributes = get_kv_attributes(st->self->parent, st->dir_ino, st->ientry),
             };
             dot.name_handle = (post_op_fh3){
                 .handle_follows = 1,
@@ -169,7 +169,7 @@ resume_2:
         dotdot.name_attributes = (post_op_attr){
             // FIXME: maybe do not read parent attributes and leave them to a GETATTR?
             .attributes_follow = 1,
-            .attributes = get_kv_attributes(st->self,
+            .attributes = get_kv_attributes(st->self->parent,
                 st->parent_ino ? st->parent_ino : st->dir_ino,
                 st->parent_ino ? st->parent_ientry : st->ientry),
         };

src/nfs/nfs_kv_remove.cpp

@@ -169,7 +169,7 @@ resume_4:
             {
                 st->res2 = res;
                 nfs_kv_continue_delete(st, 5);
-            }, [st](int res, const std::string & value)
+            }, [](int res, const std::string & value)
             {
                 return res == -ENOENT;
             });
@@ -197,7 +197,7 @@ resume_5:
         auto copy = st->ientry.object_items();
         copy["nlink"] = st->ientry["nlink"].uint64_value()-1;
         copy["ctime"] = nfstime_now_str();
-        st->self->parent->db->set(kv_inode_key(st->ino), json11::Json(copy).dump(), [st](int res)
+        st->self->parent->kvfs->write_inode(st->ino, copy, false, [st](int res)
         {
             st->res = res;
             nfs_kv_continue_delete(st, 6);

src/nfs/nfs_kv_rename.cpp

@@ -240,7 +240,7 @@ resume_7:
                 copy["nlink"] = st->new_ientry["nlink"].uint64_value()-1;
                 copy["ctime"] = nfstime_now_str();
                 copy.erase("verf");
-                st->self->parent->db->set(kv_inode_key(st->new_direntry["ino"].uint64_value()), json11::Json(copy).dump(), [st](int res)
+                st->self->parent->kvfs->write_inode(st->new_direntry["ino"].uint64_value(), copy, false, [st](int res)
                 {
                     st->res = res;
                     nfs_kv_continue_rename(st, 8);
@@ -328,7 +328,7 @@ resume_11:
             ientry_new["parent_ino"] = st->new_dir_ino;
             ientry_new["ctime"] = nfstime_now_str();
             ientry_new.erase("verf");
-            st->self->parent->db->set(kv_inode_key(st->old_direntry["ino"].uint64_value()), json11::Json(ientry_new).dump(), [st](int res)
+            st->self->parent->kvfs->write_inode(st->old_direntry["ino"].uint64_value(), ientry_new, false, [st](int res)
             {
                 st->res = res;
                 nfs_kv_continue_rename(st, 12);

src/nfs/nfs_kv_setattr.cpp

@@ -84,7 +84,7 @@ resume_1:
     }
     st->new_attrs.erase("verf");
     st->new_attrs["ctime"] = nfstime_now_str();
-    st->self->parent->db->set(kv_inode_key(st->ino), json11::Json(st->new_attrs).dump(), [st](int res)
+    st->self->parent->kvfs->write_inode(st->ino, st->new_attrs, false, [st](int res)
     {
         st->res = res;
         nfs_kv_continue_setattr(st, 2);
@@ -190,7 +190,7 @@ int kv_nfs3_setattr_proc(void *opaque, rpc_op_t *rop)
                     .obj_wcc = (wcc_data){
                         .after = (post_op_attr){
                             .attributes_follow = 1,
-                            .attributes = get_kv_attributes(st->self, st->ino, st->new_attrs),
+                            .attributes = get_kv_attributes(st->self->parent, st->ino, st->new_attrs),
                         },
                     },
                 },