Release 1.10.0

New features:

- Implement basic VitastorFS support in [CSI](https://vitastor.io/docs/installation/kubernetes.html)
- Implement [NFS RDMA](https://vitastor.io/docs/usage/nfs.html#rdma) support
- Pause pool rebalance when monitor detects that it can lead to any OSD becoming full ([osd_backfillfull_ratio](https://vitastor.io/docs/config/monitor.html#osd_backfillfull_ratio))
- Auto-select correct [RDMA device and GID](https://vitastor.io/docs/config/network.html#rdma_device) based on osd_network and RoCEv2 priority
- Report slow ops in OSD stats in etcd and show them in vitastor-cli status

Bug fixes:

- Fix possibly incorrect linked list deserialization in NFS
- Fix possible crash in vitastor-nfs --block READDIR operation
- Map netlink after forking to show correct PID in vitastor-nbd ls
- Simplify and fix create-pool OSD count checks for the case of hosts split into sub-nodes
- Make monitor print "Waiting to become master" just once, not every 5s
- Take out_size from oimg if not specified in vitastor-cli dd
- Do not report OSDs with empty statistics as "full" in status
- Trigger double autosync when switching PG state to prevent leaving garbage in non-immediate_commit clusters
- Fix a lack of connection timeout for etcd websockets in OSD leading to slower etcd failover (~70s instead of ~10s)
- Fix a rare OSD crash during client disconnect
- Fix PGs sometimes sticking until OSD restart in the "has_unclean" state with EC pools
- Fix metadata partition zeroing in vitastor-disk prepare
- Add patches for qemu 9.1 and pve-qemu 9.0 and 9.1
- Fix libvirt 8 patch

.gitea/workflows/test.yml

@@ -288,6 +288,24 @@ jobs:
           echo ""
         done
 
+  test_create_halfhost:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_create_halfhost.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_failure_domain:
     runs-on: ubuntu-latest
     needs: build

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "1.9.3")
+set(VITASTOR_VERSION "1.10.0")
 
 add_subdirectory(src)

csi/Dockerfile

@@ -22,6 +22,8 @@ RUN apt-get update && \
     (echo "APT::Install-Recommends false;" > /etc/apt/apt.conf) && \
     apt-get update && \
     apt-get install -y e2fsprogs xfsprogs kmod iproute2 \
+        # NFS mount dependencies
+        nfs-common netbase \
         # dependencies of qemu-storage-daemon
         libnuma1 liburing2 libglib2.0-0 libfuse3-3 libaio1 libzstd1 libnettle8 \
         libgmp10 libhogweed6 libp11-kit0 libidn2-0 libunistring2 libtasn1-6 libpcre2-8-0 libffi8 && \

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v1.9.3
+VITASTOR_VERSION ?= v1.10.0
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v1.9.3
+          image: vitalif/vitastor-csi:v1.10.0
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v1.9.3
+          image: vitalif/vitastor-csi:v1.10.0
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/009-storage-class.yaml

@@ -9,8 +9,16 @@ metadata:
 provisioner: csi.vitastor.io
 volumeBindingMode: Immediate
 parameters:
-  etcdVolumePrefix: ""
-  poolId: "1"
+  # CSI driver can create block-based volumes and VitastorFS-based volumes
+  # only VitastorFS-based volumes and raw block volumes (without FS) support ReadWriteMany mode
+  # set this parameter to VitastorFS metadata volume name to use VitastorFS
+  # if unset, block-based volumes will be created
+  vitastorfs: ""
+  # for block-based storage classes, pool ID may be either a string (name) or a number (ID)
+  # for vitastorFS-based storage classes it must be a string - name of the default pool for FS data
+  poolId: "testpool"
+  # volume name prefix for block-based storage classes or NFS subdirectory (including /) for FS-based volumes
+  volumePrefix: ""
   # you can choose other configuration file if you have it in the config map
   # different etcd URLs and prefixes should also be put in the config
   #configPath: "/etc/vitastor/vitastor.conf"

csi/deploy/example-storage-class-fs.yaml

@@ -0,0 +1,25 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  namespace: vitastor-system
+  name: vitastor
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+provisioner: csi.vitastor.io
+volumeBindingMode: Immediate
+parameters:
+  # CSI driver can create block-based volumes and VitastorFS-based volumes
+  # only VitastorFS-based volumes and raw block volumes (without FS) support ReadWriteMany mode
+  # set this parameter to VitastorFS metadata volume name to use VitastorFS
+  # if unset, block-based volumes will be created
+  vitastorfs: "testfs"
+  # for block-based storage classes, pool ID may be either a string (name) or a number (ID)
+  # for vitastorFS-based storage classes it must be a string - name of the default pool for FS data
+  poolId: "testpool"
+  # volume name prefix for block-based storage classes or NFS subdirectory (including /) for FS-based volumes
+  volumePrefix: "k8s/"
+  # you can choose other configuration file if you have it in the config map
+  # different etcd URLs and prefixes should also be put in the config
+  #configPath: "/etc/vitastor/vitastor.conf"
+allowVolumeExpansion: true

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "1.9.3"
+    vitastorCSIDriverVersion = "1.10.0"
 )
 
 // Config struct fills the parameters of request or user input

csi/src/controllerserver.go

@@ -8,7 +8,6 @@ import (
     "encoding/json"
     "fmt"
     "strings"
-    "strconv"
     "time"
     "os"
     "io/ioutil"
@@ -68,9 +67,10 @@ func GetConnectionParams(params map[string]string) (map[string]string, error)
     {
         configPath = "/etc/vitastor/vitastor.conf"
     }
-    else
+    ctxVars["configPath"] = configPath
+    if (params["vitastorfs"] != "")
     {
-        ctxVars["configPath"] = configPath
+        ctxVars["vitastorfs"] = params["vitastorfs"]
     }
     config := make(map[string]interface{})
     configFD, err := os.Open(configPath)
@@ -140,33 +140,57 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
         return nil, status.Error(codes.InvalidArgument, "volume capabilities is a required field")
     }
 
-    err := cs.checkCaps(volumeCapabilities)
-    if (err != nil)
-    {
-        return nil, err
-    }
-
-    etcdVolumePrefix := req.Parameters["etcdVolumePrefix"]
-    poolId, _ := strconv.ParseUint(req.Parameters["poolId"], 10, 64)
-    if (poolId == 0)
-    {
-        return nil, status.Error(codes.InvalidArgument, "poolId is missing in storage class configuration")
-    }
-
-    volName := etcdVolumePrefix + req.GetName()
-    volSize := 1 * GB
-    if capRange := req.GetCapacityRange(); capRange != nil
-    {
-        volSize = ((capRange.GetRequiredBytes() + MB - 1) / MB) * MB
-    }
-
     ctxVars, err := GetConnectionParams(req.Parameters)
     if (err != nil)
     {
         return nil, err
     }
 
-    args := []string{ "create", volName, "-s", fmt.Sprintf("%v", volSize), "--pool", fmt.Sprintf("%v", poolId) }
+    err = cs.checkCaps(volumeCapabilities, ctxVars["vitastorfs"] != "")
+    if (err != nil)
+    {
+        return nil, err
+    }
+
+    pool := req.Parameters["poolId"]
+    if (pool == "")
+    {
+        return nil, status.Error(codes.InvalidArgument, "poolId is missing in storage class configuration")
+    }
+    volumePrefix := req.Parameters["volumePrefix"]
+    if (volumePrefix == "")
+    {
+        // Old name
+        volumePrefix = req.Parameters["etcdVolumePrefix"]
+    }
+    volName := volumePrefix + req.GetName()
+    volSize := 1 * GB
+    if capRange := req.GetCapacityRange(); capRange != nil
+    {
+        volSize = ((capRange.GetRequiredBytes() + MB - 1) / MB) * MB
+    }
+
+    if (ctxVars["vitastorfs"] != "")
+    {
+        // Nothing to create, subdirectories are created during mounting
+        // FIXME: It would be cool to support quotas some day and set it here
+        if (req.VolumeContentSource.GetSnapshot() != nil)
+        {
+            return nil, status.Error(codes.InvalidArgument, "VitastorFS doesn't support snapshots")
+        }
+        ctxVars["name"] = volName
+        ctxVars["pool"] = pool
+        volumeIdJson, _ := json.Marshal(ctxVars)
+        return &csi.CreateVolumeResponse{
+            Volume: &csi.Volume{
+                // Ugly, but VolumeContext isn't passed to DeleteVolume :-(
+                VolumeId: string(volumeIdJson),
+                CapacityBytes: volSize,
+            },
+        }, nil
+    }
+
+    args := []string{ "create", volName, "-s", fmt.Sprintf("%v", volSize), "--pool", pool }
 
     // Support creation from snapshot
     var src *csi.VolumeContentSource
@@ -249,6 +273,12 @@ func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol
         return nil, err
     }
 
+    if (ctxVars["vitastorfs"] != "")
+    {
+        // FIXME: Delete FS subdirectory
+        return &csi.DeleteVolumeResponse{}, nil
+    }
+
     _, err = invokeCLI(ctxVars, []string{ "rm", volName })
     if (err != nil)
     {
@@ -283,13 +313,25 @@ func (cs *ControllerServer) ValidateVolumeCapabilities(ctx context.Context, req
     {
         return nil, status.Error(codes.InvalidArgument, "volumeId is nil")
     }
+    volVars := make(map[string]string)
+    err := json.Unmarshal([]byte(volumeID), &volVars)
+    if (err != nil)
+    {
+        return nil, status.Error(codes.Internal, "volume ID not in JSON format")
+    }
+    ctxVars, err := GetConnectionParams(volVars)
+    if (err != nil)
+    {
+        return nil, err
+    }
+
     volumeCapabilities := req.GetVolumeCapabilities()
     if (volumeCapabilities == nil)
     {
         return nil, status.Error(codes.InvalidArgument, "volumeCapabilities is nil")
     }
 
-    err := cs.checkCaps(volumeCapabilities)
+    err = cs.checkCaps(volumeCapabilities, ctxVars["vitastorfs"] != "")
     if (err != nil)
     {
         return nil, err
@@ -302,7 +344,7 @@ func (cs *ControllerServer) ValidateVolumeCapabilities(ctx context.Context, req
     }, nil
 }
 
-func (cs *ControllerServer) checkCaps(volumeCapabilities []*csi.VolumeCapability) error
+func (cs *ControllerServer) checkCaps(volumeCapabilities []*csi.VolumeCapability, fs bool) error
 {
     var volumeCapabilityAccessModes []*csi.VolumeCapability_AccessMode
     for _, mode := range []csi.VolumeCapability_AccessMode_Mode{
@@ -318,6 +360,10 @@ func (cs *ControllerServer) checkCaps(volumeCapabilities []*csi.VolumeCapability
     {
         if (capability.GetBlock() != nil)
         {
+            if (fs)
+            {
+                return status.Errorf(codes.InvalidArgument, "%v not supported with FS-based volumes", capability)
+            }
             for _, mode := range []csi.VolumeCapability_AccessMode_Mode{
                 csi.VolumeCapability_AccessMode_MULTI_NODE_SINGLE_WRITER,
                 csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER,
@@ -328,6 +374,12 @@ func (cs *ControllerServer) checkCaps(volumeCapabilities []*csi.VolumeCapability
         }
     }
 
+    if (fs)
+    {
+        // All access modes including RWX are supported with FS-based volumes
+        return nil
+    }
+
     capabilitySupport := false
     for _, capability := range volumeCapabilities
     {
@@ -342,7 +394,7 @@ func (cs *ControllerServer) checkCaps(volumeCapabilities []*csi.VolumeCapability
 
     if (!capabilitySupport)
     {
-        return status.Errorf(codes.NotFound, "%v not supported", volumeCapabilities)
+        return status.Errorf(codes.InvalidArgument, "%v not supported", volumeCapabilities)
     }
 
     return nil
@@ -434,6 +486,12 @@ func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS
     {
         return nil, status.Error(codes.Internal, "volume ID not in JSON format")
     }
+
+    if (ctxVars["vitastorfs"] != "")
+    {
+        return nil, status.Error(codes.InvalidArgument, "VitastorFS doesn't support snapshots")
+    }
+
     volName := ctxVars["name"]
 
     // Create image using vitastor-cli
@@ -492,6 +550,11 @@ func (cs *ControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteS
         return nil, err
     }
 
+    if (ctxVars["vitastorfs"] != "")
+    {
+        return nil, status.Error(codes.InvalidArgument, "VitastorFS doesn't support snapshots")
+    }
+
     _, err = invokeCLI(ctxVars, []string{ "rm", volName+"@"+snapName })
     if (err != nil)
     {
@@ -523,6 +586,11 @@ func (cs *ControllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnap
         return nil, err
     }
 
+    if (ctxVars["vitastorfs"] != "")
+    {
+        return nil, status.Error(codes.InvalidArgument, "VitastorFS doesn't support snapshots")
+    }
+
     inodeCfg, err := invokeList(ctxVars, volName+"@*", false)
     if (err != nil)
     {
@@ -586,6 +654,16 @@ func (cs *ControllerServer) ControllerExpandVolume(ctx context.Context, req *csi
         return nil, err
     }
 
+    if (ctxVars["vitastorfs"] != "")
+    {
+        // Nothing to change
+        // FIXME: Support quotas and change quota here
+        return &csi.ControllerExpandVolumeResponse{
+            CapacityBytes: req.CapacityRange.RequiredBytes,
+            NodeExpansionRequired: false,
+        }, nil
+    }
+
     inodeCfg, err := invokeList(ctxVars, volName, true)
     if (err != nil)
     {

csi/src/nodeserver.go

@@ -5,11 +5,15 @@ package vitastor
 
 import (
     "context"
+    "crypto/sha1"
+    "encoding/hex"
     "encoding/json"
     "fmt"
     "os"
     "os/exec"
     "path/filepath"
+    "regexp"
+    "strconv"
     "strings"
     "sync"
     "syscall"
@@ -29,13 +33,14 @@ import (
 type NodeServer struct
 {
     *Driver
-    useVduse bool
-    stateDir string
-    mounter mount.Interface
+    useVduse        bool
+    stateDir        string
+    nfsStageDir     string
+    mounter         mount.Interface
     restartInterval time.Duration
-    mu sync.Mutex
-    cond *sync.Cond
-    volumeLocks map[string]bool
+    mu              sync.Mutex
+    cond            *sync.Cond
+    volumeLocks     map[string]bool
 }
 
 type DeviceState struct
@@ -48,6 +53,15 @@ type DeviceState struct
     PidFile    string `json:"pidFile"`
 }
 
+type NfsState struct
+{
+    ConfigPath string `json:"configPath"`
+    FsName     string `json:"fsName"`
+    Pool       string `json:"pool"`
+    Path       string `json:"path"`
+    Port       int    `json:"port"`
+}
+
 // NewNodeServer create new instance node
 func NewNodeServer(driver *Driver) *NodeServer
 {
@@ -60,11 +74,17 @@ func NewNodeServer(driver *Driver) *NodeServer
     {
         stateDir += "/"
     }
+    nfsStageDir := os.Getenv("NFS_STAGE_DIR")
+    if (nfsStageDir == "")
+    {
+        nfsStageDir = "/var/lib/kubelet/plugins/csi.vitastor.io/nfs"
+    }
     ns := &NodeServer{
-        Driver: driver,
-        useVduse: checkVduseSupport(),
-        stateDir: stateDir,
-        mounter: mount.New(""),
+        Driver:      driver,
+        useVduse:    checkVduseSupport(),
+        stateDir:    stateDir,
+        nfsStageDir: nfsStageDir,
+        mounter:     mount.New(""),
         volumeLocks: make(map[string]bool),
     }
     ns.cond = sync.NewCond(&ns.mu)
@@ -123,12 +143,12 @@ func (ns *NodeServer) restarter()
 func (ns *NodeServer) restoreVduseDaemons()
 {
     pattern := ns.stateDir+"vitastor-vduse-*.json"
-    matches, err := filepath.Glob(pattern)
+    stateFiles, err := filepath.Glob(pattern)
     if (err != nil)
     {
         klog.Errorf("failed to list %s: %v", pattern, err)
     }
-    if (len(matches) == 0)
+    if (len(stateFiles) == 0)
     {
         return
     }
@@ -146,59 +166,162 @@ func (ns *NodeServer) restoreVduseDaemons()
         klog.Errorf("/sbin/vdpa -j dev list returned bad JSON (error %v): %v", err, string(devListJSON))
         return
     }
-    for _, stateFile := range matches
+    for _, stateFile := range stateFiles
     {
-        vdpaId := filepath.Base(stateFile)
-        vdpaId = vdpaId[0:len(vdpaId)-5]
-        // Check if VDPA device is still added to the bus
-        if (devs[vdpaId] == nil)
-        {
-            // Unused, clean it up
-            unmapVduseById(ns.stateDir, vdpaId)
-            continue
-        }
+        ns.checkVduseState(stateFile, devs)
+    }
+}
 
-        stateJSON, err := os.ReadFile(stateFile)
+func (ns *NodeServer) checkVduseState(stateFile string, devs map[string]interface{})
+{
+    // Check if VDPA device is still added to the bus
+    vdpaId := filepath.Base(stateFile)
+    vdpaId = vdpaId[0:len(vdpaId)-5]
+    if (devs[vdpaId] == nil)
+    {
+        // Unused, clean it up
+        unmapVduseById(ns.stateDir, vdpaId)
+        return
+    }
+
+    // Read state file
+    stateJSON, err := os.ReadFile(stateFile)
+    if (err != nil)
+    {
+        klog.Warningf("error reading state file %v: %v", stateFile, err)
+        return
+    }
+    var state DeviceState
+    err = json.Unmarshal(stateJSON, &state)
+    if (err != nil)
+    {
+        klog.Warningf("state file %v contains invalid JSON (error %v): %v", stateFile, err, string(stateJSON))
+        return
+    }
+
+    // Lock volume
+    ns.lockVolume(state.ConfigPath+":block:"+state.Image)
+    defer ns.unlockVolume(state.ConfigPath+":block:"+state.Image)
+
+    // Recheck state file after locking
+    _, err = os.ReadFile(stateFile)
+    if (err != nil)
+    {
+        klog.Warningf("state file %v disappeared, skipping volume", stateFile)
+        return
+    }
+
+    // Check if the storage daemon is still active
+    pidFile := ns.stateDir + vdpaId + ".pid"
+    exists := false
+    proc, err := findByPidFile(pidFile)
+    if (err == nil)
+    {
+        exists = proc.Signal(syscall.Signal(0)) == nil
+    }
+    if (!exists)
+    {
+        // Restart daemon
+        klog.Warningf("restarting storage daemon for volume %v (VDPA ID %v)", state.Image, vdpaId)
+        err = startStorageDaemon(vdpaId, state.Image, pidFile, state.ConfigPath, state.Readonly)
         if (err != nil)
         {
-            klog.Warningf("error reading state file %v: %v", stateFile, err)
-            continue
+            klog.Warningf("failed to restart storage daemon for volume %v: %v", state.Image, err)
         }
-        var state DeviceState
-        err = json.Unmarshal(stateJSON, &state)
+    }
+}
+
+func (ns *NodeServer) restoreNfsDaemons()
+{
+    pattern := ns.stateDir+"vitastor-nfs-*.json"
+    stateFiles, err := filepath.Glob(pattern)
+    if (err != nil)
+    {
+        klog.Errorf("failed to list %s: %v", pattern, err)
+    }
+    if (len(stateFiles) == 0)
+    {
+        return
+    }
+    activeNFS, err := ns.listActiveNFS()
+    if (err != nil)
+    {
+        return
+    }
+    // Check all state files and try to restore active mounts
+    for _, stateFile := range stateFiles
+    {
+        ns.checkNfsState(stateFile, activeNFS)
+    }
+}
+
+func (ns *NodeServer) readNfsState(stateFile string, allowNotExists bool) (*NfsState, error)
+{
+    stateJSON, err := os.ReadFile(stateFile)
+    if (err != nil)
+    {
+        if (allowNotExists && os.IsNotExist(err))
+        {
+            return nil, nil
+        }
+        klog.Warningf("error reading state file %v: %v", stateFile, err)
+        return nil, err
+    }
+    var state NfsState
+    err = json.Unmarshal(stateJSON, &state)
+    if (err != nil)
+    {
+        klog.Warningf("state file %v contains invalid JSON (error %v): %v", stateFile, err, string(stateJSON))
+        return nil, err
+    }
+    return &state, nil
+}
+
+func (ns *NodeServer) checkNfsState(stateFile string, activeNfs map[int][]string)
+{
+    // Read state file
+    state, err := ns.readNfsState(stateFile, false)
+    if (err != nil)
+    {
+        return
+    }
+    // Lock FS
+    ns.lockVolume(state.ConfigPath+":fs:"+state.FsName)
+    defer ns.unlockVolume(state.ConfigPath+":fs:"+state.FsName)
+    // Check if NFS at this port is still mounted
+    pidFile := ns.stateDir + filepath.Base(stateFile)
+    pidFile = pidFile[0:len(pidFile)-5] + ".pid"
+    if (len(activeNfs[state.Port]) == 0)
+    {
+        // this is a stale state file, remove it
+        klog.Warningf("state file %v contains stale mount at port %d, removing it", stateFile, state.Port)
+        ns.stopNFS(stateFile, pidFile)
+        return
+    }
+    // Check PID file
+    exists := false
+    proc, err := findByPidFile(pidFile)
+    if (err == nil)
+    {
+        exists = proc.Signal(syscall.Signal(0)) == nil
+    }
+    if (!exists)
+    {
+        // Restart vitastor-nfs server
+        klog.Warningf("restarting NFS server for FS %v at port %v", state.FsName, state.Port)
+        _, _, err := system(
+            "/usr/bin/vitastor-nfs", "start",
+            "--pidfile", pidFile,
+            "--bind", "127.0.0.1",
+            "--port", fmt.Sprintf("%d", state.Port),
+            "--fs", state.FsName,
+            "--pool", state.Pool,
+            "--portmap", "0",
+        )
         if (err != nil)
         {
-            klog.Warningf("state file %v contains invalid JSON (error %v): %v", stateFile, err, string(stateJSON))
-            continue
+            klog.Warningf("failed to restart NFS server for FS %v: %v", state.FsName, err)
         }
-
-        ns.lockVolume(state.ConfigPath+":"+state.Image)
-
-        // Recheck state file after locking
-        _, err = os.ReadFile(stateFile)
-        if (err != nil)
-        {
-            klog.Warningf("state file %v disappeared, skipping volume", stateFile)
-            ns.unlockVolume(state.ConfigPath+":"+state.Image)
-            continue
-        }
-
-        // Check if the storage daemon is still active
-        pidFile := ns.stateDir + vdpaId + ".pid"
-        exists := false
-        proc, err := findByPidFile(pidFile)
-        if (err == nil)
-        {
-            exists = proc.Signal(syscall.Signal(0)) == nil
-        }
-        if (!exists)
-        {
-            // Restart daemon
-            klog.Warningf("restarting storage daemon for volume %v (VDPA ID %v)", state.Image, vdpaId)
-            _ = startStorageDaemon(vdpaId, state.Image, pidFile, state.ConfigPath, state.Readonly)
-        }
-
-        ns.unlockVolume(state.ConfigPath+":"+state.Image)
     }
 }
 
@@ -220,8 +343,13 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
     }
     volName := ctxVars["name"]
 
-    ns.lockVolume(ctxVars["configPath"]+":"+volName)
-    defer ns.unlockVolume(ctxVars["configPath"]+":"+volName)
+    if (ctxVars["vitastorfs"] != "")
+    {
+        return &csi.NodeStageVolumeResponse{}, nil
+    }
+
+    ns.lockVolume(ctxVars["configPath"]+":block:"+volName)
+    defer ns.unlockVolume(ctxVars["configPath"]+":block:"+volName)
 
     targetPath := req.GetStagingTargetPath()
     isBlock := req.GetVolumeCapability().GetBlock() != nil
@@ -408,8 +536,13 @@ func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag
     }
     volName := ctxVars["name"]
 
-    ns.lockVolume(ctxVars["configPath"]+":"+volName)
-    defer ns.unlockVolume(ctxVars["configPath"]+":"+volName)
+    if (ctxVars["vitastorfs"] != "")
+    {
+        return &csi.NodeUnstageVolumeResponse{}, nil
+    }
+
+    ns.lockVolume(ctxVars["configPath"]+":block:"+volName)
+    defer ns.unlockVolume(ctxVars["configPath"]+":block:"+volName)
 
     targetPath := req.GetStagingTargetPath()
     devicePath, _, err := mount.GetDeviceNameFromMount(ns.mounter, targetPath)
@@ -462,6 +595,153 @@ func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag
     return &csi.NodeUnstageVolumeResponse{}, nil
 }
 
+// Mount or check if NFS is already mounted
+func (ns *NodeServer) mountNFS(ctxVars map[string]string) (string, error)
+{
+    sum := sha1.Sum([]byte(ctxVars["configPath"]+":fs:"+ctxVars["vitastorfs"]))
+    nfsHash := hex.EncodeToString(sum[:])
+    stateFile := ns.stateDir+"vitastor-nfs-"+nfsHash+".json"
+    pidFile := ns.stateDir+"vitastor-nfs-"+nfsHash+".pid"
+    mountPath := ns.nfsStageDir+"/"+nfsHash
+    state, err := ns.readNfsState(stateFile, true)
+    if (state != nil)
+    {
+        return state.Path, nil
+    }
+    if (err != nil)
+    {
+        return "", err
+    }
+    err = os.MkdirAll(mountPath, 0777)
+    if (err != nil)
+    {
+        return "", err
+    }
+    // Create a new mount
+    state = &NfsState{
+        ConfigPath: ctxVars["configPath"],
+        FsName:     ctxVars["vitastorfs"],
+        Pool:       ctxVars["pool"],
+        Path:       mountPath,
+    }
+    klog.Infof("starting new NFS server for FS %v", state.FsName)
+    stdout, _, err := system(
+        "/usr/bin/vitastor-nfs", "start",
+        "--pidfile", pidFile,
+        "--bind", "127.0.0.1",
+        "--port", "auto",
+        "--fs", state.FsName,
+        "--pool", state.Pool,
+        "--portmap", "0",
+    )
+    if (err != nil)
+    {
+        return "", err
+    }
+    match := regexp.MustCompile("Port: (\\d+)").FindStringSubmatch(string(stdout))
+    if (match == nil)
+    {
+        klog.Errorf("failed to find port in vitastor-nfs output: %v", string(stdout))
+        ns.stopNFS(stateFile, pidFile)
+        return "", fmt.Errorf("failed to find port in vitastor-nfs output (bad vitastor-nfs version?)")
+    }
+    port, _ := strconv.ParseUint(match[1], 0, 16)
+    state.Port = int(port)
+    // Write state file
+    stateJSON, _ := json.Marshal(state)
+    err = os.WriteFile(stateFile, stateJSON, 0600)
+    if (err != nil)
+    {
+        klog.Errorf("failed to write state file %v", stateFile)
+        ns.stopNFS(stateFile, pidFile)
+        return "", err
+    }
+    // Mount NFS
+    _, _, err = system(
+        "mount", "-t", "nfs", "127.0.0.1:/", state.Path,
+        "-o", fmt.Sprintf("port=%d,mountport=%d,nfsvers=3,soft,nolock,tcp", port, port),
+    )
+    if (err != nil)
+    {
+        ns.stopNFS(stateFile, pidFile)
+        return "", err
+    }
+    return state.Path, nil
+}
+
+// Mount or check if NFS is already mounted
+func (ns *NodeServer) checkStopNFS(ctxVars map[string]string)
+{
+    sum := sha1.Sum([]byte(ctxVars["configPath"]+":fs:"+ctxVars["vitastorfs"]))
+    nfsHash := hex.EncodeToString(sum[:])
+    stateFile := ns.stateDir+"vitastor-nfs-"+nfsHash+".json"
+    pidFile := ns.stateDir+"vitastor-nfs-"+nfsHash+".pid"
+    mountPath := ns.nfsStageDir+"/"+nfsHash
+    state, err := ns.readNfsState(stateFile, true)
+    if (state == nil)
+    {
+        return
+    }
+    activeNFS, err := ns.listActiveNFS()
+    if (err != nil)
+    {
+        return
+    }
+    if (len(activeNFS[state.Port]) > 0)
+    {
+        return
+    }
+    // All volume mounts are detached, unmount the root mount and kill the server
+    err = mount.CleanupMountPoint(mountPath, ns.mounter, false)
+    if (err != nil)
+    {
+        klog.Errorf("failed to unmount %v: %v", mountPath, err)
+        return
+    }
+    ns.stopNFS(stateFile, pidFile)
+}
+
+func (ns *NodeServer) stopNFS(stateFile, pidFile string)
+{
+    err := killByPidFile(pidFile)
+    if (err != nil)
+    {
+        klog.Errorf("failed to kill process with pid from %v: %v", pidFile, err)
+    }
+    os.Remove(pidFile)
+    os.Remove(stateFile)
+}
+
+func (ns *NodeServer) listActiveNFS() (map[int][]string, error)
+{
+    mounts, err := mount.ParseMountInfo("/proc/self/mountinfo")
+    if (err != nil)
+    {
+        klog.Errorf("failed to list mounts: %v", err)
+        return nil, err
+    }
+    activeNFS := make(map[int][]string)
+    for _, mount := range mounts
+    {
+        // Volume mounts always refer to subpaths
+        if (mount.FsType == "nfs" && mount.Root != "/")
+        {
+            for _, opt := range mount.MountOptions
+            {
+                if (strings.HasPrefix(opt, "port="))
+                {
+                    port64, err := strconv.ParseUint(opt[5:], 10, 16)
+                    if (err == nil)
+                    {
+                        activeNFS[int(port64)] = append(activeNFS[int(port64)], mount.MountPoint)
+                    }
+                }
+            }
+        }
+    }
+    return activeNFS, nil
+}
+
 // NodePublishVolume mounts the volume mounted to the staging path to the target path
 func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error)
 {
@@ -480,28 +760,39 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
     }
     volName := ctxVars["name"]
 
-    ns.lockVolume(ctxVars["configPath"]+":"+volName)
-    defer ns.unlockVolume(ctxVars["configPath"]+":"+volName)
+    if (ctxVars["vitastorfs"] != "")
+    {
+        ns.lockVolume(ctxVars["configPath"]+":fs:"+ctxVars["vitastorfs"])
+        defer ns.unlockVolume(ctxVars["configPath"]+":fs:"+ctxVars["vitastorfs"])
+    }
+    else
+    {
+        ns.lockVolume(ctxVars["configPath"]+":block:"+volName)
+        defer ns.unlockVolume(ctxVars["configPath"]+":block:"+volName)
+    }
 
     stagingTargetPath := req.GetStagingTargetPath()
     targetPath := req.GetTargetPath()
     isBlock := req.GetVolumeCapability().GetBlock() != nil
 
-    // Check that stagingTargetPath is mounted
-    notmnt, err := mount.IsNotMountPoint(ns.mounter, stagingTargetPath)
-    if (err != nil)
+    if (ctxVars["vitastorfs"] == "")
     {
-        klog.Errorf("staging path %v is not mounted: %w", stagingTargetPath, err)
-        return nil, fmt.Errorf("staging path %v is not mounted: %w", stagingTargetPath, err)
-    }
-    else if (notmnt)
-    {
-        klog.Errorf("staging path %v is not mounted", stagingTargetPath)
-        return nil, fmt.Errorf("staging path %v is not mounted", stagingTargetPath)
+        // Check that stagingTargetPath is mounted
+        notmnt, err := mount.IsNotMountPoint(ns.mounter, stagingTargetPath)
+        if (err != nil)
+        {
+            klog.Errorf("staging path %v is not mounted: %w", stagingTargetPath, err)
+            return nil, fmt.Errorf("staging path %v is not mounted: %w", stagingTargetPath, err)
+        }
+        else if (notmnt)
+        {
+            klog.Errorf("staging path %v is not mounted", stagingTargetPath)
+            return nil, fmt.Errorf("staging path %v is not mounted", stagingTargetPath)
+        }
     }
 
     // Check that targetPath is not already mounted
-    notmnt, err = mount.IsNotMountPoint(ns.mounter, targetPath)
+    notmnt, err := mount.IsNotMountPoint(ns.mounter, targetPath)
     if (err != nil)
     {
         if (os.IsNotExist(err))
@@ -542,6 +833,24 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
         return nil, fmt.Errorf("target path %s is already mounted", targetPath)
     }
 
+    if (ctxVars["vitastorfs"] != "")
+    {
+        nfspath, err := ns.mountNFS(ctxVars)
+        if (err != nil)
+        {
+            ns.checkStopNFS(ctxVars)
+            return nil, err
+        }
+        // volName should include prefix
+        stagingTargetPath = nfspath+"/"+volName
+        err = os.MkdirAll(stagingTargetPath, 0777)
+        if (err != nil && !os.IsExist(err))
+        {
+            ns.checkStopNFS(ctxVars)
+            return nil, err
+        }
+    }
+
     execArgs := []string{"--bind", stagingTargetPath, targetPath}
     if (req.GetReadonly())
     {
@@ -553,6 +862,10 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
     out, err := cmd.Output()
     if (err != nil)
     {
+        if (ctxVars["vitastorfs"] != "")
+        {
+            ns.checkStopNFS(ctxVars)
+        }
         return nil, fmt.Errorf("Error running mount %v: %s", strings.Join(execArgs, " "), out)
     }
 
@@ -572,8 +885,16 @@ func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
     }
     volName := ctxVars["name"]
 
-    ns.lockVolume(ctxVars["configPath"]+":"+volName)
-    defer ns.unlockVolume(ctxVars["configPath"]+":"+volName)
+    if (ctxVars["vitastorfs"] != "")
+    {
+        ns.lockVolume(ctxVars["configPath"]+":fs:"+ctxVars["vitastorfs"])
+        defer ns.unlockVolume(ctxVars["configPath"]+":fs:"+ctxVars["vitastorfs"])
+    }
+    else
+    {
+        ns.lockVolume(ctxVars["configPath"]+":block:"+volName)
+        defer ns.unlockVolume(ctxVars["configPath"]+":block:"+volName)
+    }
 
     targetPath := req.GetTargetPath()
     devicePath, _, err := mount.GetDeviceNameFromMount(ns.mounter, targetPath)
@@ -600,6 +921,11 @@ func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
         return nil, err
     }
 
+    if (ctxVars["vitastorfs"] != "")
+    {
+        ns.checkStopNFS(ctxVars)
+    }
+
     return &csi.NodeUnpublishVolumeResponse{}, nil
 }
 

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (1.9.3-1) unstable; urgency=medium
+vitastor (1.10.0-1) unstable; urgency=medium
 
   * Bugfixes
 

debian/vitastor.Dockerfile

@@ -21,10 +21,10 @@ RUN set -e -x; \
     echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf; \
     echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
 
-RUN apt-get update
-RUN apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts libjerasure-dev cmake libibverbs-dev libisal-dev libnl-3-dev libnl-genl-3-dev curl
-RUN apt-get -y build-dep fio
-RUN apt-get --download-only source fio
+RUN apt-get update && \
+    apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts libjerasure-dev cmake libibverbs-dev librdmacm-dev libisal-dev libnl-3-dev libnl-genl-3-dev curl && \
+    apt-get -y build-dep fio && \
+    apt-get --download-only source fio
 
 ADD . /root/vitastor
 RUN set -e -x; \

docs/config/monitor.en.md

@@ -24,6 +24,7 @@ These parameters only apply to Monitors.
 - [osd_out_time](#osd_out_time)
 - [placement_levels](#placement_levels)
 - [use_old_pg_combinator](#use_old_pg_combinator)
+- [osd_backfillfull_ratio](#osd_backfillfull_ratio)
 
 ## use_antietcd
 
@@ -175,3 +176,18 @@ present in the configuration, then it is defined with the default priority
 
 Use the old PG combination generator which doesn't support [level_placement](pool.en.md#level_placement)
 and [raw_placement](pool.en.md#raw_placement) for pools which don't use this features.
+
+## osd_backfillfull_ratio
+
+- Type: number
+- Default: 0.99
+
+Monitors try to prevent OSDs becoming 100% full during rebalance or recovery by
+calculating how much space will be occupied on every OSD after all rebalance
+and recovery operations finish, and pausing rebalance and recovery if that
+amount of space exceeds OSD capacity multiplied by the value of this
+configuration parameter.
+
+Future used space is calculated by summing space used by all user data blocks
+(objects) in all PGs placed on a specific OSD, even if some of these objects
+currently reside on a different set of OSDs.

docs/config/monitor.ru.md

@@ -24,6 +24,7 @@
 - [osd_out_time](#osd_out_time)
 - [placement_levels](#placement_levels)
 - [use_old_pg_combinator](#use_old_pg_combinator)
+- [osd_backfillfull_ratio](#osd_backfillfull_ratio)
 
 ## use_antietcd
 
@@ -178,3 +179,19 @@ OSD перед обновлением агрегированной статис
 
 Использовать старый генератор комбинаций PG, не поддерживающий [level_placement](pool.ru.md#level_placement)
 и [raw_placement](pool.ru.md#raw_placement) для пулов, которые не используют данные функции.
+
+## osd_backfillfull_ratio
+
+- Тип: число
+- Значение по умолчанию: 0.99
+
+Мониторы стараются предотвратить 100% заполнение OSD в процессе ребаланса
+или восстановления, рассчитывая, сколько места будет занято на каждом OSD после
+завершения всех операций ребаланса и восстановления, и приостанавливая
+ребаланс и восстановление, если рассчитанный объём превышает ёмкость OSD,
+умноженную на значение данного параметра.
+
+Будущее занятое место рассчитывается сложением места, занятого всеми
+пользовательскими блоками данных (объектами) во всех PG, расположенных
+на конкретном OSD, даже если часть этих объектов в данный момент находится
+на другом наборе OSD.

docs/config/network.en.md

@@ -68,11 +68,17 @@ but they are not connected to the cluster.
 - Type: string
 
 RDMA device name to use for Vitastor OSD communications (for example,
-"rocep5s0f0"). Now Vitastor supports all adapters, even ones without
-ODP support, like Mellanox ConnectX-3 and non-Mellanox cards.
+"rocep5s0f0"). If not specified, Vitastor will try to find an RoCE
+device matching [osd_network](osd.en.md#osd_network), preferring RoCEv2,
+or choose the first available RDMA device if no RoCE devices are
+found or if `osd_network` is not specified. Auto-selection is also
+unsupported with old libibverbs < v32, like in Debian 10 Buster or
+CentOS 7.
 
-Versions up to Vitastor 1.2.0 required ODP which is only present in
-Mellanox ConnectX >= 4. See also [rdma_odp](#rdma_odp).
+Vitastor supports all adapters, even ones without ODP support, like
+Mellanox ConnectX-3 and non-Mellanox cards. Versions up to Vitastor
+1.2.0 required ODP which is only present in Mellanox ConnectX >= 4.
+See also [rdma_odp](#rdma_odp).
 
 Run `ibv_devinfo -v` as root to list available RDMA devices and their
 features.
@@ -95,15 +101,17 @@ your device has.
 ## rdma_gid_index
 
 - Type: integer
-- Default: 0
 
 Global address identifier index of the RDMA device to use. Different GID
 indexes may correspond to different protocols like RoCEv1, RoCEv2 and iWARP.
 Search for "GID" in `ibv_devinfo -v` output to determine which GID index
 you need.
 
-**IMPORTANT:** If you want to use RoCEv2 (as recommended) then the correct
-rdma_gid_index is usually 1 (IPv6) or 3 (IPv4).
+If not specified, Vitastor will try to auto-select a RoCEv2 IPv4 GID, then
+RoCEv2 IPv6 GID, then RoCEv1 IPv4 GID, then RoCEv1 IPv6 GID, then IB GID.
+GID auto-selection is unsupported with libibverbs < v32.
+
+A correct rdma_gid_index for RoCEv2 is usually 1 (IPv6) or 3 (IPv4).
 
 ## rdma_mtu
 

docs/config/network.ru.md

@@ -71,12 +71,17 @@ RDMA может быть нужно только если у клиентов е
 - Тип: строка
 
 Название RDMA-устройства для связи с Vitastor OSD (например, "rocep5s0f0").
-Сейчас Vitastor поддерживает все модели адаптеров, включая те, у которых
-нет поддержки ODP, то есть вы можете использовать RDMA с ConnectX-3 и
-картами производства не Mellanox.
+Если не указано, Vitastor попробует найти RoCE-устройство, соответствующее
+[osd_network](osd.en.md#osd_network), предпочитая RoCEv2, или выбрать первое
+попавшееся RDMA-устройство, если RoCE-устройств нет или если сеть `osd_network`
+не задана. Также автовыбор не поддерживается со старыми версиями библиотеки
+libibverbs < v32, например в Debian 10 Buster или CentOS 7.
 
-Версии Vitastor до 1.2.0 включительно требовали ODP, который есть только
-на Mellanox ConnectX 4 и более новых. См. также [rdma_odp](#rdma_odp).
+Vitastor поддерживает все модели адаптеров, включая те, у которых
+нет поддержки ODP, то есть вы можете использовать RDMA с ConnectX-3 и
+картами производства не Mellanox. Версии Vitastor до 1.2.0 включительно
+требовали ODP, который есть только на Mellanox ConnectX 4 и более новых.
+См. также [rdma_odp](#rdma_odp).
 
 Запустите `ibv_devinfo -v` от имени суперпользователя, чтобы посмотреть
 список доступных RDMA-устройств, их параметры и возможности.
@@ -101,15 +106,18 @@ Control) и ECN (Explicit Congestion Notification).
 ## rdma_gid_index
 
 - Тип: целое число
-- Значение по умолчанию: 0
 
 Номер глобального идентификатора адреса RDMA-устройства, который следует
 использовать. Разным gid_index могут соответствовать разные протоколы связи:
 RoCEv1, RoCEv2, iWARP. Чтобы понять, какой нужен вам - смотрите строчки со
 словом "GID" в выводе команды `ibv_devinfo -v`.
 
-**ВАЖНО:** Если вы хотите использовать RoCEv2 (как мы и рекомендуем), то
-правильный rdma_gid_index, как правило, 1 (IPv6) или 3 (IPv4).
+Если не указан, Vitastor попробует автоматически выбрать сначала GID,
+соответствующий RoCEv2 IPv4, потом RoCEv2 IPv6, потом RoCEv1 IPv4, потом
+RoCEv1 IPv6, потом IB. Авто-выбор GID не поддерживается со старыми версиями
+libibverbs < v32.
+
+Правильный rdma_gid_index для RoCEv2, как правило, 1 (IPv6) или 3 (IPv4).
 
 ## rdma_mtu
 

docs/config/src/monitor.yml

@@ -172,3 +172,27 @@
   info_ru: |
     Использовать старый генератор комбинаций PG, не поддерживающий [level_placement](pool.ru.md#level_placement)
     и [raw_placement](pool.ru.md#raw_placement) для пулов, которые не используют данные функции.
+- name: osd_backfillfull_ratio
+  type: float
+  default: 0.99
+  info: |
+    Monitors try to prevent OSDs becoming 100% full during rebalance or recovery by
+    calculating how much space will be occupied on every OSD after all rebalance
+    and recovery operations finish, and pausing rebalance and recovery if that
+    amount of space exceeds OSD capacity multiplied by the value of this
+    configuration parameter.
+
+    Future used space is calculated by summing space used by all user data blocks
+    (objects) in all PGs placed on a specific OSD, even if some of these objects
+    currently reside on a different set of OSDs.
+  info_ru: |
+    Мониторы стараются предотвратить 100% заполнение OSD в процессе ребаланса
+    или восстановления, рассчитывая, сколько места будет занято на каждом OSD после
+    завершения всех операций ребаланса и восстановления, и приостанавливая
+    ребаланс и восстановление, если рассчитанный объём превышает ёмкость OSD,
+    умноженную на значение данного параметра.
+
+    Будущее занятое место рассчитывается сложением места, занятого всеми
+    пользовательскими блоками данных (объектами) во всех PG, расположенных
+    на конкретном OSD, даже если часть этих объектов в данный момент находится
+    на другом наборе OSD.

docs/config/src/network.yml

@@ -48,11 +48,17 @@
   type: string
   info: |
     RDMA device name to use for Vitastor OSD communications (for example,
-    "rocep5s0f0"). Now Vitastor supports all adapters, even ones without
-    ODP support, like Mellanox ConnectX-3 and non-Mellanox cards.
+    "rocep5s0f0"). If not specified, Vitastor will try to find an RoCE
+    device matching [osd_network](osd.en.md#osd_network), preferring RoCEv2,
+    or choose the first available RDMA device if no RoCE devices are
+    found or if `osd_network` is not specified. Auto-selection is also
+    unsupported with old libibverbs < v32, like in Debian 10 Buster or
+    CentOS 7.
 
-    Versions up to Vitastor 1.2.0 required ODP which is only present in
-    Mellanox ConnectX >= 4. See also [rdma_odp](#rdma_odp).
+    Vitastor supports all adapters, even ones without ODP support, like
+    Mellanox ConnectX-3 and non-Mellanox cards. Versions up to Vitastor
+    1.2.0 required ODP which is only present in Mellanox ConnectX >= 4.
+    See also [rdma_odp](#rdma_odp).
 
     Run `ibv_devinfo -v` as root to list available RDMA devices and their
     features.
@@ -64,12 +70,17 @@
     PFC (Priority Flow Control) and ECN (Explicit Congestion Notification).
   info_ru: |
     Название RDMA-устройства для связи с Vitastor OSD (например, "rocep5s0f0").
-    Сейчас Vitastor поддерживает все модели адаптеров, включая те, у которых
-    нет поддержки ODP, то есть вы можете использовать RDMA с ConnectX-3 и
-    картами производства не Mellanox.
+    Если не указано, Vitastor попробует найти RoCE-устройство, соответствующее
+    [osd_network](osd.en.md#osd_network), предпочитая RoCEv2, или выбрать первое
+    попавшееся RDMA-устройство, если RoCE-устройств нет или если сеть `osd_network`
+    не задана. Также автовыбор не поддерживается со старыми версиями библиотеки
+    libibverbs < v32, например в Debian 10 Buster или CentOS 7.
 
-    Версии Vitastor до 1.2.0 включительно требовали ODP, который есть только
-    на Mellanox ConnectX 4 и более новых. См. также [rdma_odp](#rdma_odp).
+    Vitastor поддерживает все модели адаптеров, включая те, у которых
+    нет поддержки ODP, то есть вы можете использовать RDMA с ConnectX-3 и
+    картами производства не Mellanox. Версии Vitastor до 1.2.0 включительно
+    требовали ODP, который есть только на Mellanox ConnectX 4 и более новых.
+    См. также [rdma_odp](#rdma_odp).
 
     Запустите `ibv_devinfo -v` от имени суперпользователя, чтобы посмотреть
     список доступных RDMA-устройств, их параметры и возможности.
@@ -94,23 +105,29 @@
     `ibv_devinfo -v`.
 - name: rdma_gid_index
   type: int
-  default: 0
   info: |
     Global address identifier index of the RDMA device to use. Different GID
     indexes may correspond to different protocols like RoCEv1, RoCEv2 and iWARP.
     Search for "GID" in `ibv_devinfo -v` output to determine which GID index
     you need.
 
-    **IMPORTANT:** If you want to use RoCEv2 (as recommended) then the correct
-    rdma_gid_index is usually 1 (IPv6) or 3 (IPv4).
+    If not specified, Vitastor will try to auto-select a RoCEv2 IPv4 GID, then
+    RoCEv2 IPv6 GID, then RoCEv1 IPv4 GID, then RoCEv1 IPv6 GID, then IB GID.
+    GID auto-selection is unsupported with libibverbs < v32.
+
+    A correct rdma_gid_index for RoCEv2 is usually 1 (IPv6) or 3 (IPv4).
   info_ru: |
     Номер глобального идентификатора адреса RDMA-устройства, который следует
     использовать. Разным gid_index могут соответствовать разные протоколы связи:
     RoCEv1, RoCEv2, iWARP. Чтобы понять, какой нужен вам - смотрите строчки со
     словом "GID" в выводе команды `ibv_devinfo -v`.
 
-    **ВАЖНО:** Если вы хотите использовать RoCEv2 (как мы и рекомендуем), то
-    правильный rdma_gid_index, как правило, 1 (IPv6) или 3 (IPv4).
+    Если не указан, Vitastor попробует автоматически выбрать сначала GID,
+    соответствующий RoCEv2 IPv4, потом RoCEv2 IPv6, потом RoCEv1 IPv4, потом
+    RoCEv1 IPv6, потом IB. Авто-выбор GID не поддерживается со старыми версиями
+    libibverbs < v32.
+
+    Правильный rdma_gid_index для RoCEv2, как правило, 1 (IPv6) или 3 (IPv4).
 - name: rdma_mtu
   type: int
   default: 4096

docs/installation/kubernetes.en.md

@@ -6,9 +6,18 @@
 
 # Kubernetes CSI
 
-Vitastor has a CSI plugin for Kubernetes which supports RWO (and block RWX) volumes.
+Vitastor has a CSI plugin for Kubernetes which supports block-based and VitastorFS-based volumes.
 
-To deploy it, take manifests from [csi/deploy/](../../csi/deploy/) directory, put your
+Block-based volumes may be formatted and mounted with a normal FS (ext4 or xfs). Such volumes
+only support RWO (ReadWriteOnce) mode.
+
+Block-based volumes may also be left without FS and attached into the container as a block
+device. Such volumes also support RWX (ReadWriteMany) mode.
+
+VitastorFS-based volumes use a clustered file system and support FS-based RWX (ReadWriteMany)
+mode. However, such volumes don't support quotas and snapshots.
+
+To deploy the CSI plugin, take manifests from [csi/deploy/](../../csi/deploy/) directory, put your
 Vitastor configuration in [001-csi-config-map.yaml](../../csi/deploy/001-csi-config-map.yaml),
 configure storage class in [009-storage-class.yaml](../../csi/deploy/009-storage-class.yaml)
 and apply all `NNN-*.yaml` manifests to your Kubernetes installation:
@@ -23,16 +32,16 @@ After that you'll be able to create PersistentVolumes.
 kernel modules enabled (vdpa, vduse, virtio-vdpa). If your distribution doesn't
 have them pre-built - build them yourself ([instructions](../usage/qemu.en.md#vduse)),
 I promise it's worth it :-). When VDUSE is unavailable, CSI driver uses [NBD](../usage/nbd.en.md)
-to map Vitastor devices. NBD is slower and prone to timeout issues: if Vitastor
-cluster becomes unresponsible for more than [nbd_timeout](../config/client.en.md#nbd_timeout),
-the NBD device detaches and breaks pods using it.
+to map Vitastor devices. NBD is slower and, with kernels older than 5.19, unmountable
+if the cluster becomes unresponsible.
 
 ## Features
 
 Vitastor CSI supports:
 - Kubernetes starting with 1.20 (or 1.17 for older vitastor-csi <= 1.1.0)
-- Filesystem RWO (ReadWriteOnce) volumes. Example: [PVC](../../csi/deploy/example-pvc.yaml), [pod](../../csi/deploy/example-test-pod.yaml)
+- Block-based FS-formatted RWO (ReadWriteOnce) volumes. Example: [PVC](../../csi/deploy/example-pvc.yaml), [pod](../../csi/deploy/example-test-pod.yaml)
 - Raw block RWX (ReadWriteMany) volumes. Example: [PVC](../../csi/deploy/example-pvc-block.yaml), [pod](../../csi/deploy/example-test-pod-block.yaml)
+- VitastorFS-based volumes RWX (ReadWriteMany) volumes. Example: [storage class](../../csi/deploy/example-storage-class-fs.yaml)
 - Volume expansion
 - Volume snapshots. Example: [snapshot class](../../csi/deploy/example-snapshot-class.yaml), [snapshot](../../csi/deploy/example-snapshot.yaml), [clone](../../csi/deploy/example-snapshot-clone.yaml)
 - [VDUSE](../usage/qemu.en.md#vduse) (preferred) and [NBD](../usage/nbd.en.md) device mapping methods

docs/installation/kubernetes.ru.md

@@ -6,7 +6,17 @@
 
 # Kubernetes CSI
 
-У Vitastor есть CSI-плагин для Kubernetes, поддерживающий RWO, а также блочные RWX, тома.
+У Vitastor есть CSI-плагин для Kubernetes, поддерживающий блочные тома и тома на основе
+кластерной ФС VitastorFS.
+
+Блочные тома могут быть отформатированы и примонтированы со стандартной ФС (ext4 или xfs).
+Такие тома поддерживают только режим RWO (ReadWriteOnce, одновременный доступ с одного узла).
+
+Блочные тома также могут не форматироваться и подключаться в контейнер в виде блочного устройства.
+В таком случае их можно подключать в режиме RWX (ReadWriteMany, одновременный доступ с многих узлов).
+
+Тома на основе VitastorFS используют кластерную ФС и поэтому также поддерживают режим RWX
+(ReadWriteMany). Однако, такие тома не поддерживают ограничение размера и снимки.
 
 Для установки возьмите манифесты из директории [csi/deploy/](../../csi/deploy/), поместите
 вашу конфигурацию подключения к Vitastor в [csi/deploy/001-csi-config-map.yaml](../../csi/deploy/001-csi-config-map.yaml),
@@ -33,6 +43,7 @@ CSI-плагин Vitastor поддерживает:
 - Версии Kubernetes, начиная с 1.20 (или с 1.17 для более старых vitastor-csi <= 1.1.0)
 - Файловые RWO (ReadWriteOnce) тома. Пример: [PVC](../../csi/deploy/example-pvc.yaml), [под](../../csi/deploy/example-test-pod.yaml)
 - Сырые блочные RWX (ReadWriteMany) тома. Пример: [PVC](../../csi/deploy/example-pvc-block.yaml), [под](../../csi/deploy/example-test-pod-block.yaml)
+- Основанные на VitastorFS RWX (ReadWriteMany) тома. Пример: [класс хранения](../../csi/deploy/example-storage-class-fs.yaml)
 - Расширение размера томов
 - Снимки томов. Пример: [класс снимков](../../csi/deploy/example-snapshot-class.yaml), [снимок](../../csi/deploy/example-snapshot.yaml), [клон снимка](../../csi/deploy/example-snapshot-clone.yaml)
 - Способы подключения устройств [VDUSE](../usage/qemu.ru.md#vduse) (предпочитаемый) и [NBD](../usage/nbd.ru.md)

docs/intro/architecture.en.md

@@ -6,7 +6,12 @@
 
 # Architecture
 
+- [Server-side components](#server-side-components)
 - [Basic concepts](#basic-concepts)
+- [Client-side components](#client-side-components)
+- [Additional utilities](#additional-utilities)
+- [Overall read/write process](#overall-read-write-process)
+  - [Nuances of request handling](#nuances-of-request-handling)
 - [Similarities to Ceph](#similarities-to-ceph)
 - [Differences from Ceph](#differences-from-ceph)
 - [Implementation Principles](#implementation-principles)
@@ -39,10 +44,10 @@
 
 ## Client-side components
 
-- **Client library** incapsulates client I/O logic. Client library connects to etcd and to all OSDs,
+- **Client library** encapsulates client I/O logic. Client library connects to etcd and to all OSDs,
   receives cluster state from etcd, sends read and write requests directly to all OSDs. Due
   to the symmetric distributed architecture, all data blocks (each 128 KB by default) are placed
-  to different OSDs, but clients always knows where each data block is stored and connects directly
+  to different OSDs, but clients always know where each data block is stored and connect directly
   to the right OSD.
 
 All other client-side components are based on the client library:
@@ -60,8 +65,9 @@ All other client-side components are based on the client library:
   (at least by now). NBD is an older, non-recommended way to attach disks — you should use
   VDUSE whenever you can.
 - **[CSI driver](../installation/kubernetes.en.md)** — driver for attaching Vitastor images
-  as Kubernetes persistent volumes. Works through VDUSE (when available) or NBD — images are
-  attached as kernel block devices and mounted into containers.
+  and VitastorFS subdirectories as Kubernetes persistent volumes. Block-based CSI uses
+  VDUSE (when available) or NBD — images are attached as kernel block devices and mounted
+  into containers. FS-based CSI uses **[vitastor-nfs](../usage/nfs.en.md)**.
 - **Drivers for Proxmox, OpenStack and so on** — pluggable modules for corresponding systems,
   allowing to use Vitastor as storage in them.
 - **[vitastor-nfs](../usage/nfs.en.md)** — NFS 3.0 server allowing export of two file system variants:
@@ -77,8 +83,8 @@ All other client-side components are based on the client library:
 
 ## Additional utilities
 
-- **vitastor-disk** — утилита для разметки дисков под Vitastor OSD. С её помощью можно
-  создавать, удалять, менять размеры или перемещать разделы OSD.
+- **vitastor-disk** — a Vitastor OSD disk management tool. You can create, remove,
+  resize and move OSD partitions with it.
 
 ## Overall read/write process
 
@@ -213,5 +219,5 @@ All other client-side components are based on the client library:
 - Deleting images in a degraded cluster may currently lead to objects reappearing
   after dead OSDs come back, and in case of erasure-coded pools, they may even
   reappear as incomplete. Just repeat the removal request again in this case.
-  This problem will be fixed in the nearest future, the fix is already implemented
-  in the "epoch-deletions" branch.
+  This problem will be fixed in the future, along with the metadata disk storage
+  format update.

docs/intro/architecture.ru.md

@@ -24,8 +24,8 @@
   Один OSD управляет одним диском (или разделом). OSD общаются с etcd и друг с другом — от etcd они
   получают состояние кластера, а друг другу передают запросы записи и чтения вторичных копий данных.
 - **etcd** — кластерная key/value база данных, используется для хранения настроек и верхнеуровневого
-  состояния кластера, а также предотвращения разделения сознания. Блоки данных в etcd не хранятся,
-  в обработке клиентских запросов чтения и записи etcd не участвует.
+  состояния кластера, а также предотвращения разделения сознания (splitbrain). Блоки данных в etcd не
+  хранятся, в обработке клиентских запросов чтения и записи etcd не участвует.
 - **Монитор** — отдельный демон на node.js, рассчитывающий необходимые изменения в конфигурацию
   кластера, сохраняющий эту информацию в etcd и таким образом командующий OSD применить эти изменения.
   Также агрегирует статистику. Контактирует только с etcd, OSD с монитором не общаются.
@@ -43,8 +43,8 @@
 
 ## Клиентские компоненты
 
-- **Клиентская библиотека** — инкапсулирует логику на стороне клиента. Соединяются с etcd и со всеми OSD,
-  от etcd получают состояние кластера, команды чтения и записи отправляют на все OSD напрямую.
+- **Клиентская библиотека** — инкапсулирует логику на стороне клиента. Соединяется с etcd и со всеми OSD,
+  от etcd получает состояние кластера, команды чтения и записи отправляет на все OSD напрямую.
   В силу архитектуры все отдельные блоки данных (по умолчанию по 128 КБ) располагается на разных
   OSD, но клиент устроен так, что всегда точно знает, к какому OSD обращаться, и подключается
   к нему напрямую.
@@ -65,8 +65,9 @@
   Vitastor нет (по крайней мере, пока). NBD — более старый и нерекомендуемый способ подключения
   дисков — вам следует использовать VDUSE всегда, когда это возможно.
 - **[CSI драйвер](../installation/kubernetes.ru.md)** — драйвер для подключения Vitastor-образов
-  в виде персистентных томов (PV) Kubernetes. Работает через VDUSE (если доступно) или через
-  NBD — образы отражаются в виде блочных устройств и монтируются в контейнеры.
+  и поддиректорий VitastorFS в виде персистентных томов (PV) Kubernetes. Блочный CSI работает через
+  VDUSE (когда это возможно) или через NBD — образы отражаются в виде блочных устройств и монтируются
+  в контейнеры. Файловый CSI использует **[vitastor-nfs](../usage/nfs.ru.md)**.
 - **Драйвера Proxmox, OpenStack и т.п.** — подключаемые модули для соответствующих систем,
   позволяющие использовать Vitastor как хранилище в оных.
 - **[vitastor-nfs](../usage/nfs.ru.md)** — NFS 3.0 сервер, предоставляющий два варианта файловой системы:
@@ -227,5 +228,5 @@
 - Удаление образов в деградированном кластере может в данный момент приводить к повторному
   "появлению" удалённых объектов после поднятия отключённых OSD, причём в случае EC-пулов,
   объекты могут появиться в виде "неполных". Если вы столкнётесь с такой ситуацией, просто
-  повторите запрос удаления. Исправление этой проблемы уже реализовано в ветке "epoch-deletions"
-  и вскоре будет включено в релиз.
+  повторите запрос удаления. Данная проблема будет исправлена в будущем вместе с обновлением
+  дискового формата хранения метаданных.

docs/intro/features.en.md

@@ -36,6 +36,7 @@
 - [Clustered file system](../usage/nfs.en.md#vitastorfs)
 - [Experimental internal etcd replacement - antietcd](../config/monitor.en.md#use_antietcd)
 - [Built-in Prometheus metric exporter](../config/monitor.en.md#enable_prometheus)
+- [NFS RDMA support](../usage/nfs.en.md#rdma) (probably also usable for GPUDirect)
 
 ## Plugins and tools
 

docs/intro/features.ru.md

@@ -38,6 +38,7 @@
 - [Кластерная файловая система](../usage/nfs.ru.md#vitastorfs)
 - [Экспериментальная встроенная замена etcd - antietcd](../config/monitor.ru.md#use_antietcd)
 - [Встроенный Prometheus-экспортер метрик](../config/monitor.ru.md#enable_prometheus)
+- [Поддержка NFS RDMA](../usage/nfs.ru.md#rdma) (вероятно, также подходящая для GPUDirect)
 
 ## Драйверы и инструменты
 

docs/usage/admin.en.md

@@ -58,8 +58,9 @@ and during switching primary OSD of PGs.
 
 **starting**, **repeering**, **stopping** states normally almost aren't visible at all.
 If you notice them for any noticeable time — chances are some operations on some OSDs hung.
-Search for "slow op" in OSD logs to find them — operations hung for more than
-[slow_log_interval](../config/osd.en.md#slow_log_interval) are logged as "slow ops".
+Check `vitastor-cli status` and search for "slow op" in OSD logs to find them — operations
+hung for more than [slow_log_interval](../config/osd.en.md#slow_log_interval) are logged as
+"slow ops" and displayed in `status`.
 
 State transition diagram:
 
@@ -171,7 +172,14 @@ to make them use the new version of the client library.
 
 ### 1.7.x to 1.8.0
 
-After upgrading version <= 1.7.x to version >= 1.8.0, BUT <= 1.9.0: restart all clients
+It's recommended to upgrade from version <= 1.7.x to version >= 1.8.0 with full downtime,
+i.e. you should first stop clients and then the cluster (OSDs and monitor), because 1.8.0
+includes a fix for etcd event stream inconsistency which could lead to "incomplete" objects
+appearing in EC pools, and in rare cases, probably, even to data corruption during mass OSD
+restarts. It doesn't mean that you WILL hit this problem if you upgrade without full downtime,
+but it's better to secure yourself against it.
+
+Also, if you upgrade version from <= 1.7.x to version >= 1.8.0, BUT <= 1.9.0: restart all clients
 (VMs and so on), otherwise they will hang when monitor clears old PG configuration key,
 which happens 24 hours after upgrade.
 

docs/usage/admin.ru.md

@@ -56,9 +56,9 @@ OSD, на протяжении небольшого периода времен
 Состояния **starting**, **repeering**, **stopping** в норме практически не заметны вообще,
 PG должны очень быстро переходить из них в другие. Если эти состояния заметны
 хоть сколько-то значительное время — вероятно, какие-то операции на каких-то OSD зависли.
-Чтобы найти их, ищите "slow op" в журналах OSD — операции, зависшие дольше,
-чем на [slow_log_interval](../config/osd.ru.md#slow_log_interval), записываются в
-журналы OSD как "slow op".
+Чтобы найти их, посморите `vitastor-cli status` и поищите слова "slow op" в журналах OSD —
+операции, зависшие дольше, чем на [slow_log_interval](../config/osd.ru.md#slow_log_interval),
+записываются в журналы OSD как "slow op" и отображаются в `status`.
 
 Диаграмма переходов:
 
@@ -168,7 +168,14 @@ done
 
 ### 1.7.x -> 1.8.0
 
-После обновления с версий <= 1.7.x до версий >= 1.8.0, НО <= 1.9.0: перезапустите всех
+Обновляться с версий <= 1.7.x до версий >= 1.8.0 рекомендуется с полной остановкой
+сначала клиентов, а затем кластера, так как в 1.8.0 исправлена проблема (неконсистентность
+потоков событий от etcd), способная приводить к появлению incomplete объектов в EC-пулах
+и, хоть и редко, но даже к повреждению данных при массовых перезапусках OSD. Если вы
+обновляетесь без полной остановки - это не значит, что вы обязательно столкнётесь с этой
+проблемой, но лучше подстраховаться.
+
+Также, если вы обновляетесь с версии <= 1.7.x до версии >= 1.8.0, НО <= 1.9.0: перезапустите всех
 клиентов (процессы виртуальных машин можно перезапустить путём миграции на другой сервер),
 иначе они зависнут, когда монитор удалит старый ключ конфигурации PG, что происходит через
 24 часа после обновления.

docs/usage/nbd.en.md

@@ -96,7 +96,7 @@ Example output (JSON format):
 vitastor-nbd netlink-map [/dev/nbdN] (--image <image> | --pool <pool> --inode <inode> --size <size in bytes>)
 ```
 
-On recent kernel versions it's also possinle to map NBD devices using netlink interface.
+On recent kernel versions it's also possible to map NBD devices using netlink interface.
 
 This is an experimental feature because it doesn't solve all issues of NBD. Differences from regular ioctl-based 'map':
 

docs/usage/nfs.en.md

@@ -111,6 +111,21 @@ settings, because Vitastor NFS proxy doesn't keep uncommitted data in memory
 with these settings. But it may even work without `immediate_commit=all` because
 the Linux NFS client repeats all uncommitted writes if it loses the connection.
 
+## RDMA
+
+vitastor-nfs supports NFS over RDMA, which, in theory, should also allow to use
+VitastorFS from GPUDirect.
+
+You can test NFS-RDMA even if you don't have an RDMA NIC using SoftROCE:
+
+1. First, add SoftROCE device on both servers: `rdma link add rxe0 type rxe netdev eth0`.
+   Here, `rdma` utility is a part the iproute2 package, and `eth0` should be replaced with
+   the name of your Ethernet NIC.
+
+2. Start vitastor-nfs with RDMA: `vitastor-nfs start (--fs <NAME> | --block) --pool <POOL> --port 20049 --nfs_rdma 20049 --portmap 0`
+
+3. Mount the FS: `mount 192.168.0.10:/mnt/test/ /mnt/vita/ -o port=20049,mountport=20049,nfsvers=3,soft,nolock,rdma`
+
 ## Commands
 
 ### mount
@@ -131,11 +146,16 @@ The server will be automatically stopped when the FS is unmounted.
 
 Start network NFS server. Options:
 
-| <!-- -->        | <!-- -->                                                   |
-|-----------------|------------------------------------------------------------|
-| `--bind <IP>`   | bind service to \<IP> address (default 0.0.0.0)            |
-| `--port <PORT>` | use port \<PORT> for NFS services (default is 2049)        |
-| `--portmap 0`   | do not listen on port 111 (portmap/rpcbind, requires root) |
+| <!-- -->               | <!-- -->                                                                                                                    |
+|------------------------|-----------------------------------------------------------------------------------------------------------------------------|
+| `--bind <IP>`          | bind service to \<IP> address (default 0.0.0.0)                                                                             |
+| `--port <PORT>`        | use port \<PORT> for NFS services (default is 2049). Specify "auto" to auto-select and print port                           |
+| `--portmap 0`          | do not listen on port 111 (portmap/rpcbind, requires root)                                                                  |
+| `--nfs_rdma <PORT>`    | enable NFS-RDMA at RDMA-CM port \<PORT> (you can try 20049). If RDMA is enabled and --port is set to 0, TCP will be disabled |
+| `--nfs_rdma_credit 16` | maximum operation credit for RDMA clients (max iodepth)                                                                     |
+| `--nfs_rdma_send 1024` | maximum RDMA send operation count (should be larger than iodepth)                                                           |
+| `--nfs_rdma_alloc 1M`  | RDMA memory allocation rounding                                                                                             |
+| `--nfs_rdma_gc 64M`    | maximum unused RDMA buffers                                                                                                 |
 
 ### upgrade
 

docs/usage/nfs.ru.md

@@ -116,6 +116,21 @@ JSON-формате :-). Для инспекции содержимого БД
 даже без `immediate_commit=all`, потому что NFS-клиент ядра Linux повторяет все
 незафиксированные запросы при потере соединения.
 
+## RDMA
+
+vitastor-nfs поддерживает NFS через RDMA. В теории это также должно позволять использовать
+VitastorFS из GPUDirect.
+
+Вы можете протестировать NFS-RDMA, даже если у вас нет RDMA-карты, с помощью SoftROCE:
+
+1. Сначала создайте SoftROCE устройства на обоих тестовых серверах: `rdma link add rxe0 type rxe netdev eth0`.
+   Утилита `rdma` входит в состав пакета iproute2, а `eth0` вам нужно заменить на имя своей
+   сетевой карты.
+
+2. Запустите vitastor-nfs с RDMA: `vitastor-nfs start (--fs <NAME> | --block) --pool <POOL> --port 20049 --nfs_rdma 20049 --portmap 0`
+
+3. Смонтируйте ФС: `mount 192.168.0.10:/mnt/test/ /mnt/vita/ -o port=20049,mountport=20049,nfsvers=3,soft,nolock,rdma`
+
 ## Команды
 
 ### mount
@@ -136,11 +151,16 @@ JSON-формате :-). Для инспекции содержимого БД
 
 Запустить сетевой NFS-сервер. Опции:
 
-| <!-- -->        | <!-- -->                                                              |
-|-----------------|-----------------------------------------------------------------------|
-| `--bind <IP>`   | принимать соединения по адресу \<IP> (по умолчанию 0.0.0.0 - на всех) |
-| `--port <PORT>` | использовать порт \<PORT> для NFS-сервисов (по умолчанию 2049)        |
-| `--portmap 0`   | отключить сервис portmap/rpcbind на порту 111 (по умолчанию включён и требует root привилегий) |
+| <!-- -->               | <!-- -->                                                                                                                    |
+|------------------------|-----------------------------------------------------------------------------------------------------------------------------|
+| `--bind <IP>`          | принимать соединения по адресу \<IP> (по умолчанию 0.0.0.0 - на всех)                                                       |
+| `--port <PORT>`        | использовать порт \<PORT> для NFS-сервисов (по умолчанию 2049). Укажите "auto", чтобы выбрать и напечатать случайный порт   |
+| `--portmap 0`          | отключить сервис portmap/rpcbind на порту 111 (по умолчанию включён и требует root привилегий)                              |
+| `--nfs_rdma <PORT>`    | включить NFS-RDMA на порту RDMA-CM \<PORT> (попробуйте 20049). Если RDMA включено и указано `--port 0`, TCP будет отключено |
+| `--nfs_rdma_credit 16` | максимальный "кредит", глубина очереди для NFS-клиентов                                                                     |
+| `--nfs_rdma_send 1024` | максимальное число операций RDMA отправки (должно быть больше nfs_rdma_credit)                                              |
+| `--nfs_rdma_alloc 1M`  | округление выделения памяти для RDMA-клиентов                                                                               |
+| `--nfs_rdma_gc 64M`    | максимальный объём неиспользуемой памяти RDMA-клиентом перед освобождением                                                  |
 
 ### upgrade
 

mon/etcd_adapter.js

@@ -232,6 +232,7 @@ class EtcdAdapter
     async become_master()
     {
         const state = { ...this.mon.get_mon_state(), id: ''+this.mon.etcd_lease_id };
+        console.log('Waiting to become master');
         // eslint-disable-next-line no-constant-condition
         while (1)
         {
@@ -243,7 +244,6 @@ class EtcdAdapter
             {
                 break;
             }
-            console.log('Waiting to become master');
             await new Promise(ok => setTimeout(ok, this.mon.config.etcd_start_timeout));
         }
         console.log('Became master');

mon/etcd_schema.js

@@ -56,6 +56,7 @@ const etcd_tree = {
             osd_out_time: 600, // seconds. min: 0
             placement_levels: { datacenter: 1, rack: 2, host: 3, osd: 4, ... },
             use_old_pg_combinator: false,
+            osd_backfillfull_ratio: 0.99,
             // client and osd
             tcp_header_buffer_size: 65536,
             use_sync_send_recv: false,

mon/mon.js

@@ -74,6 +74,7 @@ class Mon
         this.state = JSON.parse(JSON.stringify(etcd_tree));
         this.prev_stats = { osd_stats: {}, osd_diff: {} };
         this.recheck_pgs_active = false;
+        this.updating_total_stats = false;
         this.watcher_active = false;
         this.old_pg_config = false;
         this.old_pg_stats_seen = false;
@@ -658,7 +659,19 @@ class Mon
                 this.etcd_watch_revision, pool_id, up_osds, osd_tree, real_prev_pgs, pool_res.pgs, pg_history);
         }
         new_pg_config.hash = tree_hash;
-        return await this.save_pg_config(new_pg_config, etcd_request);
+        const { backfillfull_pools, backfillfull_osds } = sum_object_counts(
+            { ...this.state, pg: { ...this.state.pg, config: new_pg_config } }, this.config
+        );
+        if (backfillfull_pools.join(',') != ((this.state.pg.config||{}).backfillfull_pools||[]).join(','))
+        {
+            this.log_backfillfull(backfillfull_osds, backfillfull_pools);
+        }
+        new_pg_config.backfillfull_pools = backfillfull_pools.length ? backfillfull_pools : undefined;
+        if (!await this.save_pg_config(new_pg_config, etcd_request))
+        {
+            return false;
+        }
+        return true;
     }
 
     async save_pg_config(new_pg_config, etcd_request = { compare: [], success: [] })
@@ -730,7 +743,7 @@ class Mon
     async update_total_stats()
     {
         const txn = [];
-        const { object_counts, object_bytes } = sum_object_counts(this.state, this.config);
+        const { object_counts, object_bytes, backfillfull_pools, backfillfull_osds } = sum_object_counts(this.state, this.config);
         let stats = sum_op_stats(this.state.osd, this.prev_stats);
         let { inode_stats, seen_pools } = sum_inode_stats(this.state, this.prev_stats);
         stats.object_counts = object_counts;
@@ -783,6 +796,27 @@ class Mon
         {
             await this.etcd.etcd_call('/kv/txn', { success: txn }, this.config.etcd_mon_timeout, 0);
         }
+        if (!this.recheck_pgs_active &&
+            backfillfull_pools.join(',') != ((this.state.pg.config||{}).backfillfull_pools||[]).join(','))
+        {
+            this.log_backfillfull(backfillfull_osds, backfillfull_pools);
+            const new_pg_config = { ...this.state.pg.config, backfillfull_pools: backfillfull_pools.length ? backfillfull_pools : undefined };
+            await this.save_pg_config(new_pg_config);
+        }
+    }
+
+    log_backfillfull(osds, pools)
+    {
+        for (const osd in osds)
+        {
+            const bf = osds[osd];
+            console.log('OSD '+osd+' may fill up during rebalance: capacity '+(bf.cap/1024n/1024n)+
+                ' MB, target user data '+(bf.clean/1024n/1024n)+' MB');
+        }
+        console.log(
+            (pools.length ? 'Pool(s) '+pools.join(', ') : 'No pools')+
+            ' are backfillfull now, applying rebalance configuration'
+        );
     }
 
     schedule_update_stats()
@@ -794,7 +828,21 @@ class Mon
         this.stats_timer = setTimeout(() =>
         {
             this.stats_timer = null;
-            this.update_total_stats().catch(console.error);
+            if (this.updating_total_stats)
+            {
+                this.schedule_update_stats();
+                return;
+            }
+            this.updating_total_stats = true;
+            try
+            {
+                this.update_total_stats().catch(console.error);
+            }
+            catch (e)
+            {
+                console.error(e);
+            }
+            this.updating_total_stats = false;
         }, this.config.mon_stats_timeout);
     }
 

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "1.9.3",
+  "version": "1.10.0",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {

mon/stats.js

@@ -109,6 +109,8 @@ function sum_object_counts(state, global_config)
             pgstats[pool_id] = { ...(state.pg.stats[pool_id] || {}), ...(pgstats[pool_id] || {}) };
         }
     }
+    const pool_per_osd = {};
+    const clean_per_osd = {};
     for (const pool_id in pgstats)
     {
         let object_size = 0;
@@ -143,10 +145,45 @@ function sum_object_counts(state, global_config)
                         object_bytes[k] += BigInt(st[k+'_count']) * object_size;
                     }
                 }
+                if (st.object_count)
+                {
+                    for (const pg_osd of (((state.pg.config.items||{})[pool_id]||{})[pg_num]||{}).osd_set||[])
+                    {
+                        if (!(pg_osd in clean_per_osd))
+                        {
+                            clean_per_osd[pg_osd] = 0n;
+                        }
+                        clean_per_osd[pg_osd] += BigInt(st.object_count);
+                        pool_per_osd[pg_osd] = pool_per_osd[pg_osd]||{};
+                        pool_per_osd[pg_osd][pool_id] = true;
+                    }
+                }
             }
         }
     }
-    return { object_counts, object_bytes };
+    // If clean_per_osd[osd] is larger than osd capacity then it will fill up during rebalance
+    let backfillfull_pools = {};
+    let backfillfull_osds = {};
+    for (const osd in clean_per_osd)
+    {
+        const st = state.osd.stats[osd];
+        if (!st || !st.size || !st.data_block_size)
+        {
+            continue;
+        }
+        let cap = BigInt(st.size)/BigInt(st.data_block_size);
+        cap = cap * BigInt((global_config.osd_backfillfull_ratio||0.99)*1000000) / 1000000n;
+        if (cap < clean_per_osd[osd])
+        {
+            backfillfull_osds[osd] = { cap: BigInt(st.size), clean: clean_per_osd[osd]*BigInt(st.data_block_size) };
+            for (const pool_id in pool_per_osd[osd])
+            {
+                backfillfull_pools[pool_id] = true;
+            }
+        }
+    }
+    backfillfull_pools = Object.keys(backfillfull_pools).sort();
+    return { object_counts, object_bytes, backfillfull_pools, backfillfull_osds };
 }
 
 // sum_inode_stats(this.state, this.prev_stats)

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '1.9.3'
+VITASTOR_VERSION = '1.10.0'
 
 LOG = logging.getLogger(__name__)
 

patches/libvirt-8.0-vitastor.diff

@@ -306,12 +306,12 @@ index e5ff653a60..884ecc79ea 100644
 +        etcd = virBufferContentAndReset(&buf);
 +    }
 +
-+    if (virJSONValueObjectCreate(&ret,
-+                                 "S:etcd-host", etcd,
-+                                 "S:etcd-prefix", src->query,
-+                                 "S:config-path", src->configFile,
-+                                 "s:image", src->path,
-+                                 NULL) < 0)
++    if (virJSONValueObjectAdd(&ret,
++                              "S:etcd-host", etcd,
++                              "S:etcd-prefix", src->query,
++                              "S:config-path", src->configFile,
++                              "s:image", src->path,
++                              NULL) < 0)
 +        return NULL;
 +
 +    return ret;

patches/pve-qemu-9.0-vitastor.patch

@@ -0,0 +1,193 @@
+Index: pve-qemu-kvm-9.0.0/block/meson.build
+===================================================================
+--- pve-qemu-kvm-9.0.0.orig/block/meson.build
++++ pve-qemu-kvm-9.0.0/block/meson.build
+@@ -126,6 +126,7 @@ foreach m : [
+   [libnfs, 'nfs', files('nfs.c')],
+   [libssh, 'ssh', files('ssh.c')],
+   [rbd, 'rbd', files('rbd.c')],
++  [vitastor, 'vitastor', files('vitastor.c')],
+ ]
+   if m[0].found()
+     module_ss = ss.source_set()
+Index: pve-qemu-kvm-9.0.0/meson.build
+===================================================================
+--- pve-qemu-kvm-9.0.0.orig/meson.build
++++ pve-qemu-kvm-9.0.0/meson.build
+@@ -1452,6 +1452,26 @@ if not get_option('rbd').auto() or have_
+   endif
+ endif
+ 
++vitastor = not_found
++if not get_option('vitastor').auto() or have_block
++  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
++    required: get_option('vitastor'))
++  if libvitastor_client.found()
++    if cc.links('''
++      #include <vitastor_c.h>
++      int main(void) {
++        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
++        return 0;
++      }''', dependencies: libvitastor_client)
++      vitastor = declare_dependency(dependencies: libvitastor_client)
++    elif get_option('vitastor').enabled()
++      error('could not link libvitastor_client')
++    else
++      warning('could not link libvitastor_client, disabling')
++    endif
++  endif
++endif
++
+ glusterfs = not_found
+ glusterfs_ftruncate_has_stat = false
+ glusterfs_iocb_has_stat = false
+@@ -2254,6 +2274,7 @@ endif
+ config_host_data.set('CONFIG_OPENGL', opengl.found())
+ config_host_data.set('CONFIG_PLUGIN', get_option('plugins'))
+ config_host_data.set('CONFIG_RBD', rbd.found())
++config_host_data.set('CONFIG_VITASTOR', vitastor.found())
+ config_host_data.set('CONFIG_RDMA', rdma.found())
+ config_host_data.set('CONFIG_RELOCATABLE', get_option('relocatable'))
+ config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack'))
+@@ -4454,6 +4475,7 @@ summary_info += {'fdt support':       fd
+ summary_info += {'libcap-ng support': libcap_ng}
+ summary_info += {'bpf support':       libbpf}
+ summary_info += {'rbd support':       rbd}
++summary_info += {'vitastor support':  vitastor}
+ summary_info += {'smartcard support': cacard}
+ summary_info += {'U2F support':       u2f}
+ summary_info += {'libusb':            libusb}
+Index: pve-qemu-kvm-9.0.0/meson_options.txt
+===================================================================
+--- pve-qemu-kvm-9.0.0.orig/meson_options.txt
++++ pve-qemu-kvm-9.0.0/meson_options.txt
+@@ -194,6 +194,8 @@ option('lzo', type : 'feature', value :
+        description: 'lzo compression support')
+ option('rbd', type : 'feature', value : 'auto',
+        description: 'Ceph block device driver')
++option('vitastor', type : 'feature', value : 'auto',
++       description: 'Vitastor block device driver')
+ option('opengl', type : 'feature', value : 'auto',
+        description: 'OpenGL support')
+ option('rdma', type : 'feature', value : 'auto',
+Index: pve-qemu-kvm-9.0.0/qapi/block-core.json
+===================================================================
+--- pve-qemu-kvm-9.0.0.orig/qapi/block-core.json
++++ pve-qemu-kvm-9.0.0/qapi/block-core.json
+@@ -3481,7 +3481,7 @@
+             'raw', 'rbd',
+             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
+             'pbs',
+-            'ssh', 'throttle', 'vdi', 'vhdx',
++            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
+             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
+@@ -4591,6 +4591,28 @@
+             '*server': ['InetSocketAddressBase'] } }
+ 
+ ##
++# @BlockdevOptionsVitastor:
++#
++# Driver specific block device options for vitastor
++#
++# @image:       Image name
++# @inode:       Inode number
++# @pool:        Pool ID
++# @size:        Desired image size in bytes
++# @config-path: Path to Vitastor configuration
++# @etcd-host:   etcd connection address(es)
++# @etcd-prefix: etcd key/value prefix
++##
++{ 'struct': 'BlockdevOptionsVitastor',
++  'data': { '*inode': 'uint64',
++            '*pool': 'uint64',
++            '*size': 'uint64',
++            '*image': 'str',
++            '*config-path': 'str',
++            '*etcd-host': 'str',
++            '*etcd-prefix': 'str' } }
++
++##
+ # @ReplicationMode:
+ #
+ # An enumeration of replication modes.
+@@ -5053,6 +5075,7 @@
+       'throttle':   'BlockdevOptionsThrottle',
+       'vdi':        'BlockdevOptionsGenericFormat',
+       'vhdx':       'BlockdevOptionsGenericFormat',
++      'vitastor':   'BlockdevOptionsVitastor',
+       'virtio-blk-vfio-pci':
+                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
+                       'if': 'CONFIG_BLKIO' },
+@@ -5498,6 +5521,20 @@
+             '*encrypt' :        'RbdEncryptionCreateOptions' } }
+ 
+ ##
++# @BlockdevCreateOptionsVitastor:
++#
++# Driver specific image creation options for Vitastor.
++#
++# @location: Where to store the new image file.  This location cannot
++#     point to a snapshot.
++#
++# @size: Size of the virtual disk in bytes
++##
++{ 'struct': 'BlockdevCreateOptionsVitastor',
++  'data': { 'location':         'BlockdevOptionsVitastor',
++            'size':             'size' } }
++
++##
+ # @BlockdevVmdkSubformat:
+ #
+ # Subformat options for VMDK images
+@@ -5719,6 +5753,7 @@
+       'ssh':            'BlockdevCreateOptionsSsh',
+       'vdi':            'BlockdevCreateOptionsVdi',
+       'vhdx':           'BlockdevCreateOptionsVhdx',
++      'vitastor':       'BlockdevCreateOptionsVitastor',
+       'vmdk':           'BlockdevCreateOptionsVmdk',
+       'vpc':            'BlockdevCreateOptionsVpc'
+   } }
+Index: pve-qemu-kvm-9.0.0/scripts/ci/org.centos/stream/8/x86_64/configure
+===================================================================
+--- pve-qemu-kvm-9.0.0.orig/scripts/ci/org.centos/stream/8/x86_64/configure
++++ pve-qemu-kvm-9.0.0/scripts/ci/org.centos/stream/8/x86_64/configure
+@@ -30,7 +30,7 @@
+ --with-suffix="qemu-kvm" \
+ --firmwarepath=/usr/share/qemu-firmware \
+ --target-list="x86_64-softmmu" \
+---block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
++--block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,vitastor,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
+ --audio-drv-list="" \
+ --block-drv-ro-whitelist="vmdk,vhdx,vpc,https,ssh" \
+ --with-coroutine=ucontext \
+@@ -176,6 +176,7 @@
+ --enable-opengl \
+ --enable-pie \
+ --enable-rbd \
++--enable-vitastor \
+ --enable-rdma \
+ --enable-seccomp \
+ --enable-snappy \
+Index: pve-qemu-kvm-9.0.0/scripts/meson-buildoptions.sh
+===================================================================
+--- pve-qemu-kvm-9.0.0.orig/scripts/meson-buildoptions.sh
++++ pve-qemu-kvm-9.0.0/scripts/meson-buildoptions.sh
+@@ -168,6 +168,7 @@ meson_options_help() {
+   printf "%s\n" '  qed             qed image format support'
+   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
+   printf "%s\n" '  rbd             Ceph block device driver'
++  printf "%s\n" '  vitastor        Vitastor block device driver'
+   printf "%s\n" '  rdma            Enable RDMA-based migration'
+   printf "%s\n" '  replication     replication support'
+   printf "%s\n" '  rutabaga-gfx    rutabaga_gfx support'
+@@ -445,6 +446,8 @@ _meson_option_parse() {
+     --disable-qom-cast-debug) printf "%s" -Dqom_cast_debug=false ;;
+     --enable-rbd) printf "%s" -Drbd=enabled ;;
+     --disable-rbd) printf "%s" -Drbd=disabled ;;
++    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
++    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
+     --enable-rdma) printf "%s" -Drdma=enabled ;;
+     --disable-rdma) printf "%s" -Drdma=disabled ;;
+     --enable-relocatable) printf "%s" -Drelocatable=true ;;

patches/pve-qemu-9.1-vitastor.patch

@@ -0,0 +1,172 @@
+Index: pve-qemu-kvm-9.1.2/block/meson.build
+===================================================================
+--- pve-qemu-kvm-9.1.2.orig/block/meson.build
++++ pve-qemu-kvm-9.1.2/block/meson.build
+@@ -126,6 +126,7 @@ foreach m : [
+   [libnfs, 'nfs', files('nfs.c')],
+   [libssh, 'ssh', files('ssh.c')],
+   [rbd, 'rbd', files('rbd.c')],
++  [vitastor, 'vitastor', files('vitastor.c')],
+ ]
+   if m[0].found()
+     module_ss = ss.source_set()
+Index: pve-qemu-kvm-9.1.2/meson.build
+===================================================================
+--- pve-qemu-kvm-9.1.2.orig/meson.build
++++ pve-qemu-kvm-9.1.2/meson.build
+@@ -1516,6 +1516,26 @@ if not get_option('rbd').auto() or have_
+   endif
+ endif
+ 
++vitastor = not_found
++if not get_option('vitastor').auto() or have_block
++  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
++    required: get_option('vitastor'))
++  if libvitastor_client.found()
++    if cc.links('''
++      #include <vitastor_c.h>
++      int main(void) {
++        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
++        return 0;
++      }''', dependencies: libvitastor_client)
++      vitastor = declare_dependency(dependencies: libvitastor_client)
++    elif get_option('vitastor').enabled()
++      error('could not link libvitastor_client')
++    else
++      warning('could not link libvitastor_client, disabling')
++    endif
++  endif
++endif
++
+ glusterfs = not_found
+ glusterfs_ftruncate_has_stat = false
+ glusterfs_iocb_has_stat = false
+@@ -2367,6 +2387,7 @@ endif
+ config_host_data.set('CONFIG_OPENGL', opengl.found())
+ config_host_data.set('CONFIG_PLUGIN', get_option('plugins'))
+ config_host_data.set('CONFIG_RBD', rbd.found())
++config_host_data.set('CONFIG_VITASTOR', vitastor.found())
+ config_host_data.set('CONFIG_RDMA', rdma.found())
+ config_host_data.set('CONFIG_RELOCATABLE', get_option('relocatable'))
+ config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack'))
+@@ -4534,6 +4555,7 @@ summary_info += {'fdt support':       fd
+ summary_info += {'libcap-ng support': libcap_ng}
+ summary_info += {'bpf support':       libbpf}
+ summary_info += {'rbd support':       rbd}
++summary_info += {'vitastor support':  vitastor}
+ summary_info += {'smartcard support': cacard}
+ summary_info += {'U2F support':       u2f}
+ summary_info += {'libusb':            libusb}
+Index: pve-qemu-kvm-9.1.2/meson_options.txt
+===================================================================
+--- pve-qemu-kvm-9.1.2.orig/meson_options.txt
++++ pve-qemu-kvm-9.1.2/meson_options.txt
+@@ -194,6 +194,8 @@ option('lzo', type : 'feature', value :
+        description: 'lzo compression support')
+ option('rbd', type : 'feature', value : 'auto',
+        description: 'Ceph block device driver')
++option('vitastor', type : 'feature', value : 'auto',
++       description: 'Vitastor block device driver')
+ option('opengl', type : 'feature', value : 'auto',
+        description: 'OpenGL support')
+ option('rdma', type : 'feature', value : 'auto',
+Index: pve-qemu-kvm-9.1.2/qapi/block-core.json
+===================================================================
+--- pve-qemu-kvm-9.1.2.orig/qapi/block-core.json
++++ pve-qemu-kvm-9.1.2/qapi/block-core.json
+@@ -3477,7 +3477,7 @@
+             'raw', 'rbd',
+             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
+             'pbs',
+-            'ssh', 'throttle', 'vdi', 'vhdx',
++            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
+             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
+@@ -4588,6 +4588,28 @@
+             '*server': ['InetSocketAddressBase'] } }
+ 
+ ##
++# @BlockdevOptionsVitastor:
++#
++# Driver specific block device options for vitastor
++#
++# @image:       Image name
++# @inode:       Inode number
++# @pool:        Pool ID
++# @size:        Desired image size in bytes
++# @config-path: Path to Vitastor configuration
++# @etcd-host:   etcd connection address(es)
++# @etcd-prefix: etcd key/value prefix
++##
++{ 'struct': 'BlockdevOptionsVitastor',
++  'data': { '*inode': 'uint64',
++            '*pool': 'uint64',
++            '*size': 'uint64',
++            '*image': 'str',
++            '*config-path': 'str',
++            '*etcd-host': 'str',
++            '*etcd-prefix': 'str' } }
++
++##
+ # @ReplicationMode:
+ #
+ # An enumeration of replication modes.
+@@ -5050,6 +5072,7 @@
+       'throttle':   'BlockdevOptionsThrottle',
+       'vdi':        'BlockdevOptionsGenericFormat',
+       'vhdx':       'BlockdevOptionsGenericFormat',
++      'vitastor':   'BlockdevOptionsVitastor',
+       'virtio-blk-vfio-pci':
+                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
+                       'if': 'CONFIG_BLKIO' },
+@@ -5497,6 +5520,20 @@
+             '*encrypt' :        'RbdEncryptionCreateOptions' } }
+ 
+ ##
++# @BlockdevCreateOptionsVitastor:
++#
++# Driver specific image creation options for Vitastor.
++#
++# @location: Where to store the new image file.  This location cannot
++#     point to a snapshot.
++#
++# @size: Size of the virtual disk in bytes
++##
++{ 'struct': 'BlockdevCreateOptionsVitastor',
++  'data': { 'location':         'BlockdevOptionsVitastor',
++            'size':             'size' } }
++
++##
+ # @BlockdevVmdkSubformat:
+ #
+ # Subformat options for VMDK images
+@@ -5718,6 +5755,7 @@
+       'ssh':            'BlockdevCreateOptionsSsh',
+       'vdi':            'BlockdevCreateOptionsVdi',
+       'vhdx':           'BlockdevCreateOptionsVhdx',
++      'vitastor':       'BlockdevCreateOptionsVitastor',
+       'vmdk':           'BlockdevCreateOptionsVmdk',
+       'vpc':            'BlockdevCreateOptionsVpc'
+   } }
+Index: pve-qemu-kvm-9.1.2/scripts/meson-buildoptions.sh
+===================================================================
+--- pve-qemu-kvm-9.1.2.orig/scripts/meson-buildoptions.sh
++++ pve-qemu-kvm-9.1.2/scripts/meson-buildoptions.sh
+@@ -168,6 +168,7 @@ meson_options_help() {
+   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
+   printf "%s\n" '  qpl             Query Processing Library support'
+   printf "%s\n" '  rbd             Ceph block device driver'
++  printf "%s\n" '  vitastor        Vitastor block device driver'
+   printf "%s\n" '  rdma            Enable RDMA-based migration'
+   printf "%s\n" '  replication     replication support'
+   printf "%s\n" '  rutabaga-gfx    rutabaga_gfx support'
+@@ -444,6 +445,8 @@ _meson_option_parse() {
+     --disable-qpl) printf "%s" -Dqpl=disabled ;;
+     --enable-rbd) printf "%s" -Drbd=enabled ;;
+     --disable-rbd) printf "%s" -Drbd=disabled ;;
++    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
++    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
+     --enable-rdma) printf "%s" -Drdma=enabled ;;
+     --disable-rdma) printf "%s" -Drdma=disabled ;;
+     --enable-relocatable) printf "%s" -Drelocatable=true ;;

patches/qemu-9.1-vitastor.patch

@@ -0,0 +1,172 @@
+diff --git a/block/meson.build b/block/meson.build
+index f1262ec2ba..3cf3e23f16 100644
+--- a/block/meson.build
++++ b/block/meson.build
+@@ -114,6 +114,7 @@ foreach m : [
+   [libnfs, 'nfs', files('nfs.c')],
+   [libssh, 'ssh', files('ssh.c')],
+   [rbd, 'rbd', files('rbd.c')],
++  [vitastor, 'vitastor', files('vitastor.c')],
+ ]
+   if m[0].found()
+     module_ss = ss.source_set()
+diff --git a/meson.build b/meson.build
+index fbda17c987..3edac22aff 100644
+--- a/meson.build
++++ b/meson.build
+@@ -1510,6 +1510,26 @@ if not get_option('rbd').auto() or have_block
+   endif
+ endif
+ 
++vitastor = not_found
++if not get_option('vitastor').auto() or have_block
++  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
++    required: get_option('vitastor'))
++  if libvitastor_client.found()
++    if cc.links('''
++      #include <vitastor_c.h>
++      int main(void) {
++        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
++        return 0;
++      }''', dependencies: libvitastor_client)
++      vitastor = declare_dependency(dependencies: libvitastor_client)
++    elif get_option('vitastor').enabled()
++      error('could not link libvitastor_client')
++    else
++      warning('could not link libvitastor_client, disabling')
++    endif
++  endif
++endif
++
+ glusterfs = not_found
+ glusterfs_ftruncate_has_stat = false
+ glusterfs_iocb_has_stat = false
+@@ -2351,6 +2371,7 @@ endif
+ config_host_data.set('CONFIG_OPENGL', opengl.found())
+ config_host_data.set('CONFIG_PLUGIN', get_option('plugins'))
+ config_host_data.set('CONFIG_RBD', rbd.found())
++config_host_data.set('CONFIG_VITASTOR', vitastor.found())
+ config_host_data.set('CONFIG_RDMA', rdma.found())
+ config_host_data.set('CONFIG_RELOCATABLE', get_option('relocatable'))
+ config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack'))
+@@ -4510,6 +4531,7 @@ summary_info += {'fdt support':       fdt_opt == 'internal' ? 'internal' : fdt}
+ summary_info += {'libcap-ng support': libcap_ng}
+ summary_info += {'bpf support':       libbpf}
+ summary_info += {'rbd support':       rbd}
++summary_info += {'vitastor support':  vitastor}
+ summary_info += {'smartcard support': cacard}
+ summary_info += {'U2F support':       u2f}
+ summary_info += {'libusb':            libusb}
+diff --git a/meson_options.txt b/meson_options.txt
+index 0269fa0f16..4740ffdc27 100644
+--- a/meson_options.txt
++++ b/meson_options.txt
+@@ -194,6 +194,8 @@ option('lzo', type : 'feature', value : 'auto',
+        description: 'lzo compression support')
+ option('rbd', type : 'feature', value : 'auto',
+        description: 'Ceph block device driver')
++option('vitastor', type : 'feature', value : 'auto',
++       description: 'Vitastor block device driver')
+ option('opengl', type : 'feature', value : 'auto',
+        description: 'OpenGL support')
+ option('rdma', type : 'feature', value : 'auto',
+diff --git a/qapi/block-core.json b/qapi/block-core.json
+index aa40d44f1d..bbee6a0e9c 100644
+--- a/qapi/block-core.json
++++ b/qapi/block-core.json
+@@ -3203,7 +3203,7 @@
+             'parallels', 'preallocate', 'qcow', 'qcow2', 'qed', 'quorum',
+             'raw', 'rbd',
+             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
+-            'ssh', 'throttle', 'vdi', 'vhdx',
++            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
+             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
+@@ -4286,6 +4286,28 @@
+             '*key-secret': 'str',
+             '*server': ['InetSocketAddressBase'] } }
+ 
++##
++# @BlockdevOptionsVitastor:
++#
++# Driver specific block device options for vitastor
++#
++# @image:       Image name
++# @inode:       Inode number
++# @pool:        Pool ID
++# @size:        Desired image size in bytes
++# @config-path: Path to Vitastor configuration
++# @etcd-host:   etcd connection address(es)
++# @etcd-prefix: etcd key/value prefix
++##
++{ 'struct': 'BlockdevOptionsVitastor',
++  'data': { '*inode': 'uint64',
++            '*pool': 'uint64',
++            '*size': 'uint64',
++            '*image': 'str',
++            '*config-path': 'str',
++            '*etcd-host': 'str',
++            '*etcd-prefix': 'str' } }
++
+ ##
+ # @ReplicationMode:
+ #
+@@ -4742,6 +4764,7 @@
+       'throttle':   'BlockdevOptionsThrottle',
+       'vdi':        'BlockdevOptionsGenericFormat',
+       'vhdx':       'BlockdevOptionsGenericFormat',
++      'vitastor':   'BlockdevOptionsVitastor',
+       'virtio-blk-vfio-pci':
+                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
+                       'if': 'CONFIG_BLKIO' },
+@@ -5183,6 +5206,20 @@
+             '*cluster-size' :   'size',
+             '*encrypt' :        'RbdEncryptionCreateOptions' } }
+ 
++##
++# @BlockdevCreateOptionsVitastor:
++#
++# Driver specific image creation options for Vitastor.
++#
++# @location: Where to store the new image file.  This location cannot
++#     point to a snapshot.
++#
++# @size: Size of the virtual disk in bytes
++##
++{ 'struct': 'BlockdevCreateOptionsVitastor',
++  'data': { 'location':         'BlockdevOptionsVitastor',
++            'size':             'size' } }
++
+ ##
+ # @BlockdevVmdkSubformat:
+ #
+@@ -5405,6 +5442,7 @@
+       'ssh':            'BlockdevCreateOptionsSsh',
+       'vdi':            'BlockdevCreateOptionsVdi',
+       'vhdx':           'BlockdevCreateOptionsVhdx',
++      'vitastor':       'BlockdevCreateOptionsVitastor',
+       'vmdk':           'BlockdevCreateOptionsVmdk',
+       'vpc':            'BlockdevCreateOptionsVpc'
+   } }
+diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh
+index c97079a38c..4623f552ec 100644
+--- a/scripts/meson-buildoptions.sh
++++ b/scripts/meson-buildoptions.sh
+@@ -168,6 +168,7 @@ meson_options_help() {
+   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
+   printf "%s\n" '  qpl             Query Processing Library support'
+   printf "%s\n" '  rbd             Ceph block device driver'
++  printf "%s\n" '  vitastor        Vitastor block device driver'
+   printf "%s\n" '  rdma            Enable RDMA-based migration'
+   printf "%s\n" '  replication     replication support'
+   printf "%s\n" '  rutabaga-gfx    rutabaga_gfx support'
+@@ -444,6 +445,8 @@ _meson_option_parse() {
+     --disable-qpl) printf "%s" -Dqpl=disabled ;;
+     --enable-rbd) printf "%s" -Drbd=enabled ;;
+     --disable-rbd) printf "%s" -Drbd=disabled ;;
++    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
++    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
+     --enable-rdma) printf "%s" -Drdma=enabled ;;
+     --disable-rdma) printf "%s" -Drdma=disabled ;;
+     --enable-relocatable) printf "%s" -Drelocatable=true ;;

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.9.3
+Version:        1.10.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.9.3.el7.tar.gz
+Source0:        vitastor-1.10.0.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel
@@ -15,7 +15,7 @@ BuildRequires:  rh-nodejs12-npm
 BuildRequires:  jerasure-devel
 BuildRequires:  libisa-l-devel
 BuildRequires:  gf-complete-devel
-BuildRequires:  libibverbs-devel
+BuildRequires:  rdma-core-devel
 BuildRequires:  cmake3
 BuildRequires:  libnl3-devel
 Requires:       vitastor-osd = %{version}-%{release}

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.9.3
+Version:        1.10.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.9.3.el8.tar.gz
+Source0:        vitastor-1.10.0.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel
@@ -14,7 +14,7 @@ BuildRequires:  nodejs >= 10
 BuildRequires:  jerasure-devel
 BuildRequires:  libisa-l-devel
 BuildRequires:  gf-complete-devel
-BuildRequires:  libibverbs-devel
+BuildRequires:  rdma-core-devel
 BuildRequires:  cmake
 BuildRequires:  libnl3-devel
 Requires:       vitastor-osd = %{version}-%{release}

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.9.3
+Version:        1.10.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.9.3.el9.tar.gz
+Source0:        vitastor-1.10.0.el9.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -19,7 +19,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVITASTOR_VERSION="1.9.3")
+add_definitions(-DVITASTOR_VERSION="1.10.0")
 add_definitions(-D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})
@@ -61,6 +61,10 @@ pkg_check_modules(ISAL libisal)
 if (ISAL_LIBRARIES)
 	add_definitions(-DWITH_ISAL)
 endif (ISAL_LIBRARIES)
+pkg_check_modules(RDMACM librdmacm)
+if (RDMACM_LIBRARIES)
+	add_definitions(-DWITH_RDMACM)
+endif (RDMACM_LIBRARIES)
 
 add_custom_target(build_tests)
 add_custom_target(test

src/client/etcd_state_client.cpp

@@ -176,7 +176,7 @@ void etcd_state_client_t::add_etcd_url(std::string addr)
             exit(1);
         }
         if (!local_ips.size())
-            local_ips = getifaddr_list();
+            local_ips = getifaddr_list(std::vector<std::string>(), true);
         std::string check_addr;
         int pos = addr.find('/');
         int pos2 = addr.find(':');
@@ -785,7 +785,7 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
         }
         for (auto & pool_item: value.object_items())
         {
-            pool_config_t pc;
+            pool_config_t pc = {};
             // ID
             pool_id_t pool_id;
             char null_byte = 0;
@@ -931,12 +931,28 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
             // Ignore old key if the new one is present
             return;
         }
+        for (auto & pool_id_json: value["backfillfull_pools"].array_items())
+        {
+            auto pool_id = pool_id_json.uint64_value();
+            auto pool_it = this->pool_config.find(pool_id);
+            if (pool_it != this->pool_config.end())
+            {
+                pool_it->second.backfillfull |= 2;
+            }
+        }
         for (auto & pool_item: this->pool_config)
         {
             for (auto & pg_item: pool_item.second.pg_config)
             {
                 pg_item.second.config_exists = false;
             }
+            // 3 = was 1 and became 1, 0 = was 0 and became 0
+            if (pool_item.second.backfillfull == 2 || pool_item.second.backfillfull == 1)
+            {
+                if (on_change_backfillfull_hook)
+                    on_change_backfillfull_hook(pool_item.first);
+            }
+            pool_item.second.backfillfull = pool_item.second.backfillfull >> 1;
         }
         for (auto & pool_item: value["items"].object_items())
         {

src/client/etcd_state_client.h

@@ -62,6 +62,7 @@ struct pool_config_t
     std::map<pg_num_t, pg_config_t> pg_config;
     uint64_t scrub_interval;
     std::string used_for_fs;
+    int backfillfull;
 };
 
 struct inode_config_t
@@ -131,6 +132,7 @@ public:
     std::function<json11::Json()> load_pgs_checks_hook;
     std::function<void(bool)> on_load_pgs_hook;
     std::function<void()> on_change_pool_config_hook;
+    std::function<void(pool_id_t)> on_change_backfillfull_hook;
     std::function<void(pool_id_t, pg_num_t, osd_num_t)> on_change_pg_state_hook;
     std::function<void(pool_id_t, pg_num_t)> on_change_pg_history_hook;
     std::function<void(osd_num_t)> on_change_osd_state_hook;

src/client/http_client.cpp

@@ -62,6 +62,7 @@ struct http_co_t
     inline void end() { ended = true; if (!onstack) { delete this; } }
     void run_cb_and_clear();
     void start_connection();
+    void start_ws_connection();
     void close_connection();
     void next_request();
     void handle_events();
@@ -112,7 +113,7 @@ http_co_t* open_websocket(timerfd_manager_t *tfd, const std::string & host, cons
     handler->keepalive = false;
     handler->request = request;
     handler->response_callback = response_callback;
-    handler->start_connection();
+    handler->start_ws_connection();
     return handler;
 }
 
@@ -282,6 +283,27 @@ void http_co_t::close_connection()
     epoll_events = 0;
 }
 
+void http_co_t::start_ws_connection()
+{
+    stackin();
+    start_connection();
+    if (request_timeout > 0)
+    {
+        timeout_id = tfd->set_timer(request_timeout, false, [this](int timer_id)
+        {
+            stackin();
+            if (state != HTTP_CO_WEBSOCKET)
+            {
+                close_connection();
+                parsed = { .error = "Websocket connection timed out" };
+                run_cb_and_clear();
+            }
+            stackout();
+        });
+    }
+    stackout();
+}
+
 void http_co_t::start_connection()
 {
     stackin();

src/client/messenger.cpp

@@ -121,7 +121,7 @@ void osd_messenger_t::init()
     if (use_rdma)
     {
         rdma_context = msgr_rdma_context_t::create(
-            rdma_device != "" ? rdma_device.c_str() : NULL,
+            osd_networks, rdma_device != "" ? rdma_device.c_str() : NULL,
             rdma_port_num, rdma_gid_index, rdma_mtu, rdma_odp, log_level
         );
         if (!rdma_context)
@@ -266,7 +266,8 @@ void osd_messenger_t::parse_config(const json11::Json & config)
     this->rdma_port_num = (uint8_t)config["rdma_port_num"].uint64_value();
     if (!this->rdma_port_num)
         this->rdma_port_num = 1;
-    this->rdma_gid_index = (uint8_t)config["rdma_gid_index"].uint64_value();
+    if (!config["rdma_gid_index"].is_null())
+        this->rdma_gid_index = (uint8_t)config["rdma_gid_index"].uint64_value();
     this->rdma_mtu = (uint32_t)config["rdma_mtu"].uint64_value();
     this->rdma_max_sge = config["rdma_max_sge"].uint64_value();
     if (!this->rdma_max_sge)
@@ -281,6 +282,15 @@ void osd_messenger_t::parse_config(const json11::Json & config)
     if (!this->rdma_max_msg || this->rdma_max_msg > 128*1024*1024)
         this->rdma_max_msg = 129*1024;
     this->rdma_odp = config["rdma_odp"].bool_value();
+    std::vector<std::string> mask;
+    if (config["bind_address"].is_string())
+        mask.push_back(config["bind_address"].string_value());
+    else if (config["osd_network"].is_string())
+        mask.push_back(config["osd_network"].string_value());
+    else
+        for (auto v: config["osd_network"].array_items())
+            mask.push_back(v.string_value());
+    this->osd_networks = mask;
 #endif
     if (!osd_num)
         this->iothread_count = (uint32_t)config["client_iothread_count"].uint64_value();

src/client/messenger.h

@@ -165,8 +165,10 @@ protected:
 
 #ifdef WITH_RDMA
     bool use_rdma = true;
+    std::vector<std::string> osd_networks;
     std::string rdma_device;
-    uint64_t rdma_port_num = 1, rdma_gid_index = 0, rdma_mtu = 0;
+    uint64_t rdma_port_num = 1, rdma_mtu = 0;
+    int rdma_gid_index = -1;
     msgr_rdma_context_t *rdma_context = NULL;
     uint64_t rdma_max_sge = 0, rdma_max_send = 0, rdma_max_recv = 0;
     uint64_t rdma_max_msg = 0;
@@ -177,7 +179,7 @@ protected:
     std::vector<int> read_ready_clients;
     std::vector<int> write_ready_clients;
     // We don't use ringloop->set_immediate here because we may have no ringloop in client :)
-    std::vector<std::function<void()>> set_immediate;
+    std::vector<osd_op_t*> set_immediate_ops;
 
 public:
     timerfd_manager_t *tfd;
@@ -237,6 +239,8 @@ protected:
     void handle_op_hdr(osd_client_t *cl);
     bool handle_reply_hdr(osd_client_t *cl);
     void handle_reply_ready(osd_op_t *op);
+    void handle_immediate_ops();
+    void clear_immediate_ops(int peer_fd);
 
 #ifdef WITH_RDMA
     void try_send_rdma(osd_client_t *cl);

src/client/msgr_rdma.cpp

@@ -3,6 +3,7 @@
 
 #include <stdio.h>
 #include <stdlib.h>
+#include "addr_util.h"
 #include "msgr_rdma.h"
 #include "messenger.h"
 
@@ -69,7 +70,138 @@ msgr_rdma_connection_t::~msgr_rdma_connection_t()
     send_out_size = 0;
 }
 
-msgr_rdma_context_t *msgr_rdma_context_t::create(const char *ib_devname, uint8_t ib_port, uint8_t gid_index, uint32_t mtu, bool odp, int log_level)
+#ifdef IBV_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT
+static bool is_ipv4_gid(ibv_gid_entry *gidx)
+{
+    return (((uint64_t*)gidx->gid.raw)[0] == 0 &&
+        ((uint32_t*)gidx->gid.raw)[2] == 0xffff0000);
+}
+
+static bool match_gid(ibv_gid_entry *gidx, addr_mask_t *networks, int nnet)
+{
+    if (gidx->gid_type != IBV_GID_TYPE_ROCE_V1 &&
+        gidx->gid_type != IBV_GID_TYPE_ROCE_V2 ||
+        ((uint64_t*)gidx->gid.raw)[0] == 0 &&
+        ((uint64_t*)gidx->gid.raw)[1] == 0)
+    {
+        return false;
+    }
+    if (is_ipv4_gid(gidx))
+    {
+        for (int i = 0; i < nnet; i++)
+        {
+            if (networks[i].family == AF_INET && cidr_match(*(in_addr*)(gidx->gid.raw+12), networks[i].ipv4, networks[i].bits))
+                return true;
+        }
+    }
+    else
+    {
+        for (int i = 0; i < nnet; i++)
+        {
+            if (networks[i].family == AF_INET6 && cidr6_match(*(in6_addr*)gidx->gid.raw, networks[i].ipv6, networks[i].bits))
+                return true;
+        }
+    }
+    return false;
+}
+
+struct matched_dev
+{
+    int dev = -1;
+    int port = -1;
+    int gid = -1;
+    bool rocev2 = false;
+};
+
+static void log_rdma_dev_port_gid(ibv_device *dev, int ib_port, int gid_index, ibv_gid_entry & gidx)
+{
+    bool is4 = ((uint64_t*)gidx.gid.raw)[0] == 0 && ((uint32_t*)gidx.gid.raw)[2] == 0xffff0000;
+    char buf[256];
+    inet_ntop(is4 ? AF_INET : AF_INET6, is4 ? gidx.gid.raw+12 : gidx.gid.raw, buf, sizeof(buf));
+    fprintf(
+        stderr, "Auto-selected RDMA device %s port %d GID %d - ROCEv%d IPv%d %s\n",
+        ibv_get_device_name(dev), ib_port, gid_index,
+        gidx.gid_type == IBV_GID_TYPE_ROCE_V2 ? 2 : 1, is4 ? 4 : 6, buf
+    );
+}
+
+static matched_dev match_device(ibv_device **dev_list, addr_mask_t *networks, int nnet, int log_level)
+{
+    matched_dev best;
+    ibv_device_attr attr;
+    ibv_port_attr portinfo;
+    ibv_gid_entry best_gidx;
+    int res;
+    bool have_non_roce = false, have_roce = false;
+    for (int i = 0; dev_list[i]; ++i)
+    {
+        auto dev = dev_list[i];
+        ibv_context *context = ibv_open_device(dev_list[i]);
+        if ((res = ibv_query_device(context, &attr)) != 0)
+        {
+            fprintf(stderr, "Couldn't query RDMA device %s for its features: %s\n", ibv_get_device_name(dev_list[i]), strerror(res));
+            goto cleanup;
+        }
+        for (int j = 1; j <= attr.phys_port_cnt; j++)
+        {
+            // Try to find a port with matching address
+            if ((res = ibv_query_port(context, j, &portinfo)) != 0)
+            {
+                fprintf(stderr, "Couldn't get RDMA device %s port %d info: %s\n", ibv_get_device_name(dev), j, strerror(res));
+                goto cleanup;
+            }
+            for (int k = 0; k < portinfo.gid_tbl_len; k++)
+            {
+                ibv_gid_entry gidx;
+                if ((res = ibv_query_gid_ex(context, j, k, &gidx, 0)) != 0)
+                {
+                    if (res != ENODATA)
+                    {
+                        fprintf(stderr, "Couldn't read RDMA device %s GID index %d: %s\n", ibv_get_device_name(dev), k, strerror(res));
+                        goto cleanup;
+                    }
+                    else
+                        break;
+                }
+                if (gidx.gid_type != IBV_GID_TYPE_ROCE_V1 &&
+                    gidx.gid_type != IBV_GID_TYPE_ROCE_V2)
+                    have_non_roce = true;
+                else
+                    have_roce = true;
+                if (match_gid(&gidx, networks, nnet))
+                {
+                    // Prefer RoCEv2
+                    if (!best.rocev2)
+                    {
+                        best.dev = i;
+                        best.port = j;
+                        best.gid = k;
+                        best.rocev2 = (gidx.gid_type == IBV_GID_TYPE_ROCE_V2);
+                        best_gidx = gidx;
+                    }
+                }
+            }
+        }
+cleanup:
+        ibv_close_device(context);
+        if (best.rocev2)
+        {
+            break;
+        }
+    }
+    if (best.dev >= 0 && log_level > 0)
+    {
+        log_rdma_dev_port_gid(dev_list[best.dev], best.port, best.gid, best_gidx);
+    }
+    if (best.dev < 0 && have_non_roce && !have_roce)
+    {
+        best.dev = -2;
+    }
+    return best;
+}
+#endif
+
+msgr_rdma_context_t *msgr_rdma_context_t::create(std::vector<std::string> osd_networks, const char *ib_devname, uint8_t ib_port, int gid_index, uint32_t mtu, bool odp, int log_level)
 {
     int res;
     ibv_device **dev_list = NULL;
@@ -80,28 +212,23 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(const char *ib_devname, uint8_t
     clock_gettime(CLOCK_REALTIME, &tv);
     srand48(tv.tv_sec*1000000000 + tv.tv_nsec);
     dev_list = ibv_get_device_list(NULL);
-    if (!dev_list)
+    if (!dev_list || !*dev_list)
     {
         if (errno == -ENOSYS || errno == ENOSYS)
         {
             if (log_level > 0)
                 fprintf(stderr, "No RDMA devices found (RDMA device list returned ENOSYS)\n");
         }
+        else if (!*dev_list)
+        {
+            if (log_level > 0)
+                fprintf(stderr, "No RDMA devices found\n");
+        }
         else
             fprintf(stderr, "Failed to get RDMA device list: %s\n", strerror(errno));
         goto cleanup;
     }
-    if (!ib_devname)
-    {
-        ctx->dev = *dev_list;
-        if (!ctx->dev)
-        {
-            if (log_level > 0)
-                fprintf(stderr, "No RDMA devices found\n");
-            goto cleanup;
-        }
-    }
-    else
+    if (ib_devname)
     {
         int i;
         for (i = 0; dev_list[i]; ++i)
@@ -114,6 +241,39 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(const char *ib_devname, uint8_t
             goto cleanup;
         }
     }
+#ifdef IBV_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT
+    else if (osd_networks.size())
+    {
+        std::vector<addr_mask_t> nets;
+        for (auto & netstr: osd_networks)
+        {
+            nets.push_back(cidr_parse(netstr));
+        }
+        auto best = match_device(dev_list, nets.data(), nets.size(), log_level);
+        if (best.dev == -2)
+        {
+            best.dev = 0;
+            if (log_level > 0)
+                fprintf(stderr, "No RoCE devices found, using first available RDMA device %s\n", ibv_get_device_name(*dev_list));
+        }
+        else if (best.dev < 0)
+        {
+            if (log_level > 0)
+                fprintf(stderr, "RDMA device matching osd_network is not found, disabling RDMA\n");
+            goto cleanup;
+        }
+        else
+        {
+            ib_port = best.port;
+            gid_index = best.gid;
+        }
+        ctx->dev = dev_list[best.dev];
+    }
+#endif
+    else
+    {
+        ctx->dev = *dev_list;
+    }
 
     ctx->context = ibv_open_device(ctx->dev);
     if (!ctx->context)
@@ -123,7 +283,6 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(const char *ib_devname, uint8_t
     }
 
     ctx->ib_port = ib_port;
-    ctx->gid_index = gid_index;
     if ((res = ibv_query_port(ctx->context, ib_port, &ctx->portinfo)) != 0)
     {
         fprintf(stderr, "Couldn't get RDMA device %s port %d info: %s\n", ibv_get_device_name(ctx->dev), ib_port, strerror(res));
@@ -135,11 +294,55 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(const char *ib_devname, uint8_t
         fprintf(stderr, "RDMA device %s must have local LID because it's not Ethernet, but LID is zero\n", ibv_get_device_name(ctx->dev));
         goto cleanup;
     }
-    if (ibv_query_gid(ctx->context, ib_port, gid_index, &ctx->my_gid))
+
+#ifdef IBV_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT
+    if (gid_index != -1)
+#endif
     {
-        fprintf(stderr, "Couldn't read RDMA device %s GID index %d\n", ibv_get_device_name(ctx->dev), gid_index);
-        goto cleanup;
+        ctx->gid_index = gid_index < 0 ? 0 : gid_index;
+        if (ibv_query_gid(ctx->context, ib_port, gid_index, &ctx->my_gid))
+        {
+            fprintf(stderr, "Couldn't read RDMA device %s GID index %d\n", ibv_get_device_name(ctx->dev), gid_index);
+            goto cleanup;
+        }
     }
+#ifdef IBV_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT
+    else
+    {
+        // Auto-guess GID
+        ibv_gid_entry best_gidx;
+        for (int k = 0; k < ctx->portinfo.gid_tbl_len; k++)
+        {
+            ibv_gid_entry gidx;
+            if (ibv_query_gid_ex(ctx->context, ib_port, k, &gidx, 0) != 0)
+            {
+                fprintf(stderr, "Couldn't read RDMA device %s GID index %d\n", ibv_get_device_name(ctx->dev), k);
+                goto cleanup;
+            }
+            // Skip empty GID
+            if (((uint64_t*)gidx.gid.raw)[0] == 0 &&
+                ((uint64_t*)gidx.gid.raw)[1] == 0)
+            {
+                continue;
+            }
+            // Prefer IPv4 RoCEv2 -> IPv6 RoCEv2 -> IPv4 RoCEv1 -> IPv6 RoCEv1 -> IB
+            if (gid_index == -1 ||
+                gidx.gid_type == IBV_GID_TYPE_ROCE_V2 && best_gidx.gid_type != IBV_GID_TYPE_ROCE_V2 ||
+                gidx.gid_type == IBV_GID_TYPE_ROCE_V1 && best_gidx.gid_type == IBV_GID_TYPE_IB ||
+                gidx.gid_type == best_gidx.gid_type && is_ipv4_gid(&gidx))
+            {
+                gid_index = k;
+                best_gidx = gidx;
+            }
+        }
+        ctx->gid_index = gid_index = (gid_index == -1 ? 0 : gid_index);
+        if (log_level > 0)
+        {
+            log_rdma_dev_port_gid(ctx->dev, ctx->ib_port, ctx->gid_index, best_gidx);
+        }
+        ctx->my_gid = best_gidx.gid;
+    }
+#endif
 
     ctx->pd = ibv_alloc_pd(ctx->context);
     if (!ctx->pd)
@@ -398,7 +601,7 @@ static int try_send_rdma_copy(osd_client_t *cl, uint8_t *dst, int dst_len)
         iovec & iov = cl->send_list[rc->send_pos];
         uint32_t len = (uint32_t)(iov.iov_len-rc->send_buf_pos < dst_len
             ? iov.iov_len-rc->send_buf_pos : dst_len);
-        memcpy(dst, iov.iov_base+rc->send_buf_pos, len);
+        memcpy(dst, (uint8_t*)iov.iov_base+rc->send_buf_pos, len);
         dst += len;
         dst_len -= len;
         rc->send_buf_pos += len;
@@ -598,6 +801,7 @@ void osd_messenger_t::handle_rdma_events()
                 }
                 fprintf(stderr, " with status: %s, stopping client\n", ibv_wc_status_str(wc[i].status));
                 stop_client(client_id);
+                clear_immediate_ops(client_id);
                 continue;
             }
             if (!is_send)
@@ -606,6 +810,7 @@ void osd_messenger_t::handle_rdma_events()
                 if (!handle_read_buffer(cl, rc->recv_buffers[rc->next_recv_buf].buf, wc[i].byte_len))
                 {
                     // handle_read_buffer may stop the client
+                    clear_immediate_ops(client_id);
                     continue;
                 }
                 try_recv_rdma_wr(cl, rc->recv_buffers[rc->next_recv_buf]);
@@ -666,9 +871,5 @@ void osd_messenger_t::handle_rdma_events()
             }
         }
     } while (event_count > 0);
-    for (auto cb: set_immediate)
-    {
-        cb();
-    }
-    set_immediate.clear();
+    handle_immediate_ops();
 }

src/client/msgr_rdma.h

@@ -36,7 +36,7 @@ struct msgr_rdma_context_t
     int max_cqe = 0;
     int used_max_cqe = 0;
 
-    static msgr_rdma_context_t *create(const char *ib_devname, uint8_t ib_port, uint8_t gid_index, uint32_t mtu, bool odp, int log_level);
+    static msgr_rdma_context_t *create(std::vector<std::string> osd_networks, const char *ib_devname, uint8_t ib_port, int gid_index, uint32_t mtu, bool odp, int log_level);
     ~msgr_rdma_context_t();
 };
 

src/client/msgr_receive.cpp

@@ -65,6 +65,7 @@ void osd_messenger_t::read_requests()
 bool osd_messenger_t::handle_read(int result, osd_client_t *cl)
 {
     bool ret = false;
+    int peer_fd = cl->peer_fd;
     cl->read_msg.msg_iovlen = 0;
     cl->refs--;
     if (cl->peer_state == PEER_STOPPED)
@@ -101,7 +102,8 @@ bool osd_messenger_t::handle_read(int result, osd_client_t *cl)
         {
             if (!handle_read_buffer(cl, cl->in_buf, result))
             {
-                goto fin;
+                clear_immediate_ops(peer_fd);
+                return false;
             }
         }
         else
@@ -113,7 +115,8 @@ bool osd_messenger_t::handle_read(int result, osd_client_t *cl)
             {
                 if (!handle_finished_read(cl))
                 {
-                    goto fin;
+                    clear_immediate_ops(peer_fd);
+                    return false;
                 }
             }
         }
@@ -122,15 +125,47 @@ bool osd_messenger_t::handle_read(int result, osd_client_t *cl)
             ret = true;
         }
     }
-fin:
-    for (auto cb: set_immediate)
-    {
-        cb();
-    }
-    set_immediate.clear();
+    handle_immediate_ops();
     return ret;
 }
 
+void osd_messenger_t::clear_immediate_ops(int peer_fd)
+{
+    size_t i = 0, j = 0;
+    while (i < set_immediate_ops.size())
+    {
+        if (set_immediate_ops[i]->peer_fd == peer_fd)
+        {
+            delete set_immediate_ops[i];
+        }
+        else
+        {
+            if (i != j)
+                set_immediate_ops[j] = set_immediate_ops[i];
+            j++;
+        }
+        i++;
+    }
+    set_immediate_ops.resize(j);
+}
+
+void osd_messenger_t::handle_immediate_ops()
+{
+    for (auto op: set_immediate_ops)
+    {
+        if (op->op_type == OSD_OP_IN)
+        {
+            exec_op(op);
+        }
+        else
+        {
+            // Copy lambda to be unaffected by `delete op`
+            std::function<void(osd_op_t*)>(op->callback)(op);
+        }
+    }
+    set_immediate_ops.clear();
+}
+
 bool osd_messenger_t::handle_read_buffer(osd_client_t *cl, void *curbuf, int remain)
 {
     // Compose operation(s) from the buffer
@@ -199,7 +234,7 @@ bool osd_messenger_t::handle_finished_read(osd_client_t *cl)
     {
         // Operation is ready
         cl->received_ops.push_back(cl->read_op);
-        set_immediate.push_back([this, op = cl->read_op]() { exec_op(op); });
+        set_immediate_ops.push_back(cl->read_op);
         cl->read_op = NULL;
         cl->read_state = 0;
     }
@@ -295,7 +330,7 @@ void osd_messenger_t::handle_op_hdr(osd_client_t *cl)
     {
         // Operation is ready
         cl->received_ops.push_back(cur_op);
-        set_immediate.push_back([this, cur_op]() { exec_op(cur_op); });
+        set_immediate_ops.push_back(cur_op);
         cl->read_op = NULL;
         cl->read_state = 0;
     }
@@ -416,9 +451,5 @@ void osd_messenger_t::handle_reply_ready(osd_op_t *op)
         (tv_end.tv_sec - op->tv_begin.tv_sec)*1000000 +
         (tv_end.tv_nsec - op->tv_begin.tv_nsec)/1000
     );
-    set_immediate.push_back([op]()
-    {
-        // Copy lambda to be unaffected by `delete op`
-        std::function<void(osd_op_t*)>(op->callback)(op);
-    });
+    set_immediate_ops.push_back(op);
 }

src/client/nbd_proxy.cpp

@@ -64,7 +64,7 @@ static void netlink_sock_alloc(struct netlink_ctx *ctx)
     if (nl_driver_id < 0)
     {
         nl_socket_free(sk);
-        fail("Couldn't resolve the nbd netlink family\n");
+        fail("Couldn't resolve the nbd netlink family: %s (code %d)\n", nl_geterror(nl_driver_id), nl_driver_id);
     }
 
     ctx->driver_id = nl_driver_id;
@@ -555,7 +555,12 @@ help:
         fcntl(sockfd[0], F_SETFL, fcntl(sockfd[0], F_GETFL, 0) | O_NONBLOCK);
         nbd_fd = sockfd[0];
         load_module();
+
         bool bg = cfg["foreground"].is_null();
+        if (cfg["logfile"].string_value() != "")
+        {
+            logfile = cfg["logfile"].string_value();
+        }
 
         if (netlink)
         {
@@ -579,6 +584,10 @@ help:
             if (!cfg["nbd_disconnect_on_close"].is_null())
                 cflags |= NBD_CFLAG_DISCONNECT_ON_CLOSE;
 #endif
+            if (bg)
+            {
+                daemonize_fork();
+            }
             int err = netlink_configure(sockfd + 1, 1, devnum, device_size, 4096, flags, cflags, nbd_timeout, nbd_conn_timeout, NULL, revive);
             if (err < 0)
             {
@@ -588,6 +597,10 @@ help:
             }
             close(sockfd[1]);
             printf("/dev/nbd%d\n", err);
+            if (bg)
+            {
+                daemonize_reopen_stdio();
+            }
 #else
             fprintf(stderr, "netlink support is disabled in this build\n");
             exit(1);
@@ -631,14 +644,10 @@ help:
                     }
                 }
             }
-        }
-        if (cfg["logfile"].string_value() != "")
-        {
-            logfile = cfg["logfile"].string_value();
-        }
-        if (bg)
-        {
-            daemonize();
+            if (bg)
+            {
+                daemonize();
+            }
         }
         // Initialize read state
         read_state = CL_READ_HDR;
@@ -716,13 +725,17 @@ help:
         }
     }
 
-    void daemonize()
+    void daemonize_fork()
     {
         if (fork())
             exit(0);
         setsid();
         if (fork())
             exit(0);
+    }
+
+    void daemonize_reopen_stdio()
+    {
         close(0);
         close(1);
         close(2);
@@ -733,6 +746,12 @@ help:
             fprintf(stderr, "Warning: Failed to chdir into /\n");
     }
 
+    void daemonize()
+    {
+        daemonize_fork();
+        daemonize_reopen_stdio();
+    }
+
     json11::Json::object list_mapped()
     {
         const char *self_filename = exe_name;
@@ -783,8 +802,9 @@ help:
                 if (!strcmp(pid_filename, self_filename))
                 {
                     json11::Json::object cfg = nbd_proxy::parse_args(argv.size(), argv.data());
-                    if (cfg["command"] == "map")
+                    if (cfg["command"] == "map" || cfg["command"] == "netlink-map")
                     {
+                        cfg["interface"] = (cfg["command"] == "netlink-map") ? "netlink" : "nbd";
                         cfg.erase("command");
                         cfg["pid"] = pid;
                         mapped["/dev/nbd"+std::to_string(dev_num)] = cfg;

src/client/qemu_driver.c

@@ -16,7 +16,6 @@
 #include "qapi/error.h"
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qerror.h"
-#include "qemu/uri.h"
 #include "qemu/error-report.h"
 #include "qemu/module.h"
 #include "qemu/option.h"
@@ -1021,7 +1020,11 @@ static BlockDriver bdrv_vitastor = {
     // FIXME: Implement it along with per-inode statistics
     //.bdrv_get_allocated_file_size   = vitastor_get_allocated_file_size,
 
+#if QEMU_VERSION_MAJOR > 9 || QEMU_VERSION_MAJOR == 9 && QEMU_VERSION_MINOR > 0
+    .bdrv_open                      = vitastor_file_open,
+#else
     .bdrv_file_open                 = vitastor_file_open,
+#endif
     .bdrv_close                     = vitastor_close,
 
     // Option list for the create operation

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 1.9.3
+Version: 1.10.0
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/cmd/cli_dd.cpp

@@ -261,6 +261,7 @@ struct dd_out_info_t
             else
             {
                 // ok
+                out_size = owatch->cfg.size;
                 return true;
             }
             // Wait for sub-command
@@ -323,7 +324,7 @@ resume_2:
                 cluster_op_t *sync_op = new cluster_op_t;
                 sync_op->opcode = OSD_OP_SYNC;
                 parent->waiting++;
-                sync_op->callback = [this, parent](cluster_op_t *sync_op)
+                sync_op->callback = [parent](cluster_op_t *sync_op)
                 {
                     parent->waiting--;
                     delete sync_op;
@@ -538,7 +539,7 @@ struct cli_dd_t
                     in_eof = true;
             }
             ring_data_t *data = ((ring_data_t*)sqe->user_data);
-            data->iov = (iovec){ cur_read->buf + cur_read->len, cur_read->max - cur_read->len };
+            data->iov = (iovec){ (uint8_t*)cur_read->buf + cur_read->len, cur_read->max - cur_read->len };
             my_uring_prep_readv(sqe, iinfo.ifd, &data->iov, 1, iinfo.in_seekable ? iseek + cur_read->offset + cur_read->len : -1);
             in_waiting++;
             data->callback = [this, cur_read](ring_data_t *data)
@@ -671,7 +672,7 @@ struct cli_dd_t
                 return false;
             }
             ring_data_t *data = ((ring_data_t*)sqe->user_data);
-            data->iov = (iovec){ .iov_base = cur_read->buf+cur_read->len, .iov_len = cur_read->max-cur_read->len };
+            data->iov = (iovec){ .iov_base = (uint8_t*)cur_read->buf+cur_read->len, .iov_len = cur_read->max-cur_read->len };
             my_uring_prep_writev(sqe, oinfo.ofd, &data->iov, 1, oinfo.out_seekable ? cur_read->offset+cur_read->len+oseek : -1);
             out_waiting++;
             data->callback = [this, cur_read](ring_data_t *data)
@@ -882,6 +883,8 @@ resume_2:
         oinfo.end_fsync = oinfo.end_fsync && oinfo.out_seekable;
         read_offset = 0;
         read_end = iinfo.in_seekable ? iinfo.in_size-iseek : 0;
+        if (oinfo.out_size && (!read_end || read_end > oinfo.out_size-oseek))
+            read_end = oinfo.out_size-oseek;
         if (bytelimit && (!read_end || read_end > bytelimit))
             read_end = bytelimit;
         clock_gettime(CLOCK_REALTIME, &tv_begin);

src/cmd/cli_osd_tree.cpp

@@ -216,7 +216,7 @@ resume_1:
                 for (uint64_t osd_num: node.child_osds)
                 {
                     auto & osd = placement_tree->osds.at(osd_num);
-                    fmt_items.push_back(json11::Json::object{
+                    auto json_osd = json11::Json::object{
                         { "type", "osd" },
                         { "name", osd.num },
                         { "parent", node.name },
@@ -230,7 +230,16 @@ resume_1:
                         { "bitmap", (uint64_t)osd.bitmap_granularity },
                         { "commit", osd.immediate_commit == IMMEDIATE_NONE ? "none" : (osd.immediate_commit == IMMEDIATE_ALL ? "all" : "small") },
                         { "op_stats", osd_stats[osd_num]["op_stats"] },
-                    });
+                    };
+                    if (osd_stats[osd_num]["slow_ops_primary"].uint64_value() > 0)
+                    {
+                        json_osd["slow_ops_primary"] = osd_stats[osd_num]["slow_ops_primary"];
+                    }
+                    if (osd_stats[osd_num]["slow_ops_secondary"].uint64_value() > 0)
+                    {
+                        json_osd["slow_ops_secondary"] = osd_stats[osd_num]["slow_ops_secondary"];
+                    }
+                    fmt_items.push_back(json_osd);
                 }
             }
             result.data = fmt_items;

src/cmd/cli_pool_create.cpp

@@ -35,6 +35,7 @@ struct pool_creator_t
     uint64_t new_pools_mod_rev;
     json11::Json state_node_tree;
     json11::Json new_pools;
+    std::map<osd_num_t, json11::Json> osd_stats;
 
     bool is_done() { return state == 100; }
 
@@ -46,8 +47,6 @@ struct pool_creator_t
             goto resume_2;
         else if (state == 3)
             goto resume_3;
-        else if (state == 4)
-            goto resume_4;
         else if (state == 5)
             goto resume_5;
         else if (state == 6)
@@ -90,13 +89,19 @@ resume_1:
         // If not forced, check that we have enough osds for pg_size
         if (!force)
         {
-            // Get node_placement configuration from etcd
+            // Get node_placement configuration from etcd and OSD stats
             parent->etcd_txn(json11::Json::object {
                 { "success", json11::Json::array {
                     json11::Json::object {
                         { "request_range", json11::Json::object {
                             { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/node_placement") },
-                        } }
+                        } },
+                    },
+                    json11::Json::object {
+                        { "request_range", json11::Json::object {
+                            { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/osd/stats/") },
+                            { "range_end", base64_encode(parent->cli->st_cli.etcd_prefix+"/osd/stats0") },
+                        } },
                     },
                 } },
             });
@@ -112,10 +117,21 @@ resume_2:
                 return;
             }
 
-            // Get state_node_tree based on node_placement and osd peer states
+            // Get state_node_tree based on node_placement and osd stats
             {
-                auto kv = parent->cli->st_cli.parse_etcd_kv(parent->etcd_result["responses"][0]["response_range"]["kvs"][0]);
-                state_node_tree = get_state_node_tree(kv.value.object_items());
+                auto node_placement_kv = parent->cli->st_cli.parse_etcd_kv(parent->etcd_result["responses"][0]["response_range"]["kvs"][0]);
+                timespec tv_now;
+                clock_gettime(CLOCK_REALTIME, &tv_now);
+                uint64_t osd_out_time = parent->cli->config["osd_out_time"].uint64_value();
+                if (!osd_out_time)
+                    osd_out_time = 600;
+                osd_stats.clear();
+                parent->iterate_kvs_1(parent->etcd_result["responses"][1]["response_range"]["kvs"], "/osd/stats/", [&](uint64_t cur_osd, json11::Json value)
+                {
+                    if ((uint64_t)value["time"].number_value()+osd_out_time >= tv_now.tv_sec)
+                        osd_stats[cur_osd] = value;
+                });
+                state_node_tree = get_state_node_tree(node_placement_kv.value.object_items(), osd_stats);
             }
 
             // Skip tag checks, if pool has none
@@ -158,42 +174,18 @@ resume_3:
                 }
             }
 
-            // Get stats (for block_size, bitmap_granularity, ...) of osds in state_node_tree
-            {
-                json11::Json::array osd_stats;
-
-                for (auto osd_num: state_node_tree["osds"].array_items())
-                {
-                    osd_stats.push_back(json11::Json::object {
-                        { "request_range", json11::Json::object {
-                            { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/osd/stats/"+osd_num.as_string()) },
-                        } }
-                    });
-                }
-
-                parent->etcd_txn(json11::Json::object{ { "success", osd_stats } });
-            }
-
-            state = 4;
-resume_4:
-            if (parent->waiting > 0)
-                return;
-            if (parent->etcd_err.err)
-            {
-                result = parent->etcd_err;
-                state = 100;
-                return;
-            }
-
             // Filter osds from state_node_tree based on pool parameters and osd stats
             {
-                std::vector<json11::Json> osd_stats;
-                for (auto & ocr: parent->etcd_result["responses"].array_items())
+                std::vector<json11::Json> filtered_osd_stats;
+                for (auto & osd_num: state_node_tree["osds"].array_items())
                 {
-                    auto kv = parent->cli->st_cli.parse_etcd_kv(ocr["response_range"]["kvs"][0]);
-                    osd_stats.push_back(kv.value);
+                    auto st_it = osd_stats.find(osd_num.uint64_value());
+                    if (st_it != osd_stats.end())
+                    {
+                        filtered_osd_stats.push_back(st_it->second);
+                    }
                 }
-                guess_block_size(osd_stats);
+                guess_block_size(filtered_osd_stats);
                 state_node_tree = filter_state_node_tree_by_stats(state_node_tree, osd_stats);
             }
 
@@ -201,8 +193,7 @@ resume_4:
             {
                 auto failure_domain = cfg["failure_domain"].string_value() == ""
                     ? "host" : cfg["failure_domain"].string_value();
-                uint64_t max_pg_size = get_max_pg_size(state_node_tree["nodes"].object_items(),
-                    failure_domain, cfg["root_node"].string_value());
+                uint64_t max_pg_size = get_max_pg_size(state_node_tree, failure_domain, cfg["root_node"].string_value());
 
                 if (cfg["pg_size"].uint64_value() > max_pg_size)
                 {
@@ -358,56 +349,50 @@ resume_8:
 
     // Returns a JSON object of form {"nodes": {...}, "osds": [...]} that
     // contains: all nodes (osds, hosts, ...) based on node_placement config
-    // and current peer state, and a list of active peer osds.
-    json11::Json get_state_node_tree(json11::Json::object node_placement)
+    // and current osd stats.
+    json11::Json get_state_node_tree(json11::Json::object node_placement, std::map<osd_num_t, json11::Json> & osd_stats)
     {
-        // Erase non-peer osd nodes from node_placement
+        // Erase non-existing osd nodes from node_placement
         for (auto np_it = node_placement.begin(); np_it != node_placement.end();)
         {
             // Numeric nodes are osds
             osd_num_t osd_num = stoull_full(np_it->first);
 
-            // If node is osd and it is not in peer states, erase it
-            if (osd_num > 0 &&
-                parent->cli->st_cli.peer_states.find(osd_num) == parent->cli->st_cli.peer_states.end())
-            {
+            // If node is osd and its stats do not exist, erase it
+            if (osd_num > 0 && osd_stats.find(osd_num) == osd_stats.end())
                 node_placement.erase(np_it++);
-            }
             else
                 np_it++;
         }
 
-        // List of peer osds
-        std::vector<std::string> peer_osds;
+        // List of osds
+        std::vector<std::string> existing_osds;
 
-        // Record peer osds and add missing osds/hosts to np
-        for (auto & ps: parent->cli->st_cli.peer_states)
+        // Record osds and add missing osds/hosts to np
+        for (auto & ps: osd_stats)
         {
             std::string osd_num = std::to_string(ps.first);
 
-            // Record peer osd
-            peer_osds.push_back(osd_num);
+            // Record osd
+            existing_osds.push_back(osd_num);
 
-            // Add osd, if necessary
-            if (node_placement.find(osd_num) == node_placement.end())
+            // Add host if necessary
+            std::string osd_host = ps.second["host"].as_string();
+            if (node_placement.find(osd_host) == node_placement.end())
             {
-                std::string osd_host = ps.second["host"].as_string();
-
-                // Add host, if necessary
-                if (node_placement.find(osd_host) == node_placement.end())
-                {
-                    node_placement[osd_host] = json11::Json::object {
-                        { "level", "host" }
-                    };
-                }
-
-                node_placement[osd_num] = json11::Json::object {
-                    { "parent", osd_host }
+                node_placement[osd_host] = json11::Json::object {
+                    { "level", "host" }
                 };
             }
+
+            // Add osd
+            node_placement[osd_num] = json11::Json::object {
+                { "parent", node_placement[osd_num]["parent"].is_null() ? osd_host : node_placement[osd_num]["parent"] },
+                { "level", "osd" },
+            };
         }
 
-        return json11::Json::object { { "osds", peer_osds }, { "nodes", node_placement } };
+        return json11::Json::object { { "osds", existing_osds }, { "nodes", node_placement } };
     }
 
     // Returns new state_node_tree based on given state_node_tree with osds
@@ -534,15 +519,13 @@ resume_8:
     // filtered out by stats parameters (block_size, bitmap_granularity) in
     // given osd_stats and current pool config.
     // Requires: state_node_tree["osds"] must match osd_stats 1-1
-    json11::Json filter_state_node_tree_by_stats(const json11::Json & state_node_tree, std::vector<json11::Json> & osd_stats)
+    json11::Json filter_state_node_tree_by_stats(const json11::Json & state_node_tree, std::map<osd_num_t, json11::Json> & osd_stats)
     {
-        auto & osds = state_node_tree["osds"].array_items();
-
         // Accepted state_node_tree nodes
         auto accepted_nodes = state_node_tree["nodes"].object_items();
 
         // List of accepted osds
-        std::vector<std::string> accepted_osds;
+        json11::Json::array accepted_osds;
 
         block_size = cfg["block_size"].uint64_value()
             ? cfg["block_size"].uint64_value()
@@ -554,21 +537,25 @@ resume_8:
             ? etcd_state_client_t::parse_immediate_commit(cfg["immediate_commit"].string_value(), IMMEDIATE_ALL)
             : parent->cli->st_cli.global_immediate_commit;
 
-        for (size_t i = 0; i < osd_stats.size(); i++)
+        for (auto osd_num_json: state_node_tree["osds"].array_items())
         {
-            auto & os = osd_stats[i];
-            // Get osd number
-            auto osd_num = osds[i].as_string();
+            auto osd_num = osd_num_json.uint64_value();
+            auto os_it = osd_stats.find(osd_num);
+            if (os_it == osd_stats.end())
+            {
+                continue;
+            }
+            auto & os = os_it->second;
             if (!os["data_block_size"].is_null() && os["data_block_size"] != block_size ||
                 !os["bitmap_granularity"].is_null() && os["bitmap_granularity"] != bitmap_granularity ||
                 !os["immediate_commit"].is_null() &&
                 etcd_state_client_t::parse_immediate_commit(os["immediate_commit"].string_value(), IMMEDIATE_NONE) < immediate_commit)
             {
-                accepted_nodes.erase(osd_num);
+                accepted_nodes.erase(osd_num_json.as_string());
             }
             else
             {
-                accepted_osds.push_back(osd_num);
+                accepted_osds.push_back(osd_num_json);
             }
         }
 
@@ -576,87 +563,28 @@ resume_8:
     }
 
     // Returns maximum pg_size possible for given node_tree and failure_domain, starting at parent_node
-    uint64_t get_max_pg_size(json11::Json::object node_tree, const std::string & level, const std::string & parent_node)
+    uint64_t get_max_pg_size(json11::Json state_node_tree, const std::string & level, const std::string & root_node)
     {
-        uint64_t max_pg_sz = 0;
-
-        std::vector<std::string> nodes;
-
-        // Check if parent node is an osd (numeric)
-        if (parent_node != "" && stoull_full(parent_node))
+        std::set<std::string> level_seen;
+        for (auto & osd: state_node_tree["osds"].array_items())
         {
-            // Add it to node list if osd is in node tree
-            if (node_tree.find(parent_node) != node_tree.end())
-                nodes.push_back(parent_node);
-        }
-        // If parent node given, ...
-        else if (parent_node != "")
-        {
-            // ... look for children nodes of this parent
-            for (auto & sn: node_tree)
+            // find OSD parent at <level>, but stop at <root_node>
+            auto cur_id = osd.string_value();
+            auto cur = state_node_tree["nodes"][cur_id];
+            while (!cur.is_null())
             {
-                auto & props = sn.second.object_items();
-
-                auto parent_prop = props.find("parent");
-                if (parent_prop != props.end() && (parent_prop->second.as_string() == parent_node))
+                if (cur["level"] == level)
                 {
-                    nodes.push_back(sn.first);
-
-                    // If we're not looking for all osds, we only need a single
-                    // child osd node
-                    if (level != "osd" && stoull_full(sn.first))
-                        break;
+                    level_seen.insert(cur_id);
+                    break;
                 }
+                if (cur_id == root_node)
+                    break;
+                cur_id = cur["parent"].string_value();
+                cur = state_node_tree["nodes"][cur_id];
             }
         }
-        // No parent node given, and we're not looking for all osds
-        else if (level != "osd")
-        {
-            // ... look for all level nodes
-            for (auto & sn: node_tree)
-            {
-                auto & props = sn.second.object_items();
-
-                auto level_prop = props.find("level");
-                if (level_prop != props.end() && (level_prop->second.as_string() == level))
-                {
-                    nodes.push_back(sn.first);
-                }
-            }
-        }
-        // Otherwise, ...
-        else
-        {
-            // ... we're looking for osd nodes only
-            for (auto & sn: node_tree)
-            {
-                if (stoull_full(sn.first))
-                {
-                    nodes.push_back(sn.first);
-                }
-            }
-        }
-
-        // Process gathered nodes
-        for (auto & node: nodes)
-        {
-            // Check for osd node, return constant max size
-            if (stoull_full(node))
-            {
-                max_pg_sz += 1;
-            }
-            // Otherwise, ...
-            else
-            {
-                // ... exclude parent node from tree, and ...
-                node_tree.erase(parent_node);
-
-                // ... descend onto the resulting tree
-                max_pg_sz += get_max_pg_size(node_tree, level, node);
-            }
-        }
-
-        return max_pg_sz;
+        return level_seen.size();
     }
 
     json11::Json create_pool(const etcd_kv_t & kv)

src/cmd/cli_status.cpp

@@ -134,6 +134,7 @@ resume_2:
         }
         int osd_count = 0, osd_up = 0;
         uint64_t total_raw = 0, free_raw = 0, free_down_raw = 0, down_raw = 0;
+        std::vector<uint64_t> slow_op_primary_osds, slow_op_secondary_osds;
         parent->iterate_kvs_1(osd_stats, "/osd/stats/", [&](uint64_t stat_osd_num, json11::Json value)
         {
             osd_count++;
@@ -141,18 +142,29 @@ resume_2:
             auto osd_free = value["free"].uint64_value();
             total_raw += osd_size;
             free_raw += osd_free;
-            if (!osd_free)
+            if (osd_size)
             {
-                osds_full++;
-            }
-            else if (osd_free < (uint64_t)(osd_size*(1-osd_nearfull_ratio)))
-            {
-                osds_nearfull++;
+                if (!osd_free)
+                {
+                    osds_full++;
+                }
+                else if (osd_free < (uint64_t)(osd_size*(1-osd_nearfull_ratio)))
+                {
+                    osds_nearfull++;
+                }
             }
             auto peer_it = parent->cli->st_cli.peer_states.find(stat_osd_num);
             if (peer_it != parent->cli->st_cli.peer_states.end())
             {
                 osd_up++;
+                if (value["slow_ops_primary"].uint64_value() > 0)
+                {
+                    slow_op_primary_osds.push_back(stat_osd_num);
+                }
+                if (value["slow_ops_secondary"].uint64_value() > 0)
+                {
+                    slow_op_secondary_osds.push_back(stat_osd_num);
+                }
             }
             else
             {
@@ -216,6 +228,10 @@ resume_2:
                 { "mon_master", mon_master },
                 { "osd_up", osd_up },
                 { "osd_count", osd_count },
+                { "osds_full", osds_full },
+                { "osds_nearfull", osds_nearfull },
+                { "osds_primary_slow_ops", slow_op_primary_osds },
+                { "osds_secondary_slow_ops", slow_op_secondary_osds },
                 { "total_raw", total_raw },
                 { "free_raw", free_raw },
                 { "down_raw", down_raw },
@@ -300,6 +316,28 @@ resume_2:
             warning_str += "    "+std::to_string(osds_nearfull)+
                 (osds_nearfull > 1 ? " osds are almost full\n" : " osd is almost full\n");
         }
+        if (slow_op_primary_osds.size() > 0)
+        {
+            warning_str += "    "+std::to_string(slow_op_primary_osds.size());
+            warning_str += (slow_op_primary_osds.size() > 1 ? " osds have" : " osd has");
+            warning_str += " slow client ops: ";
+            for (int i = 0; i < slow_op_primary_osds.size(); i++)
+            {
+                warning_str += (i > 0 ? ", " : "")+std::to_string(slow_op_primary_osds[i]);
+            }
+            warning_str += "\n";
+        }
+        if (slow_op_secondary_osds.size() > 0)
+        {
+            warning_str += "    "+std::to_string(slow_op_secondary_osds.size());
+            warning_str += (slow_op_secondary_osds.size() > 1 ? " osds have" : " osd has");
+            warning_str += " slow replication ops: ";
+            for (int i = 0; i < slow_op_secondary_osds.size(); i++)
+            {
+                warning_str += (i > 0 ? ", " : "")+std::to_string(slow_op_secondary_osds[i]);
+            }
+            warning_str += "\n";
+        }
         if (warning_str != "")
         {
             warning_str = "\n  warning:\n"+warning_str;

src/disk_tool/disk_tool_prepare.cpp

@@ -108,6 +108,10 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
     try
     {
         dsk.parse_config(options);
+        // Set all offsets to 4096 to calculate metadata size with excess
+        dsk.journal_offset = 4096;
+        dsk.meta_offset = 4096;
+        dsk.data_offset = 4096;
         dsk.data_io = dsk.meta_io = dsk.journal_io = (options["io"] == "cached" ? "cached" : "direct");
         dsk.open_data();
         dsk.open_meta();
@@ -171,8 +175,8 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
     }
     sb["osd_num"] = osd_num;
     // Zero out metadata and journal
-    if (write_zero(dsk.meta_fd, dsk.meta_offset, dsk.meta_len) != 0 ||
-        write_zero(dsk.journal_fd, dsk.journal_offset, dsk.journal_len) != 0)
+    if (write_zero(dsk.meta_fd, sb["meta_offset"].uint64_value(), dsk.meta_len) != 0 ||
+        write_zero(dsk.journal_fd, sb["journal_offset"].uint64_value(), dsk.journal_len) != 0)
     {
         fprintf(stderr, "Failed to zero out metadata or journal: %s\n", strerror(errno));
         dsk.close_all();
@@ -498,6 +502,9 @@ int disk_tool_t::get_meta_partition(std::vector<vitastor_dev_info_t> & ssds, std
     {
         blockstore_disk_t dsk;
         dsk.parse_config(options);
+        dsk.journal_offset = 4096;
+        dsk.meta_offset = 4096;
+        dsk.data_offset = 4096;
         dsk.data_io = dsk.meta_io = dsk.journal_io = "cached";
         dsk.open_data();
         dsk.open_meta();

src/kv/kv_cli.cpp

@@ -546,7 +546,7 @@ void kv_cli_t::handle_cmd(const std::vector<std::string> & cmd, std::function<vo
             auto & key = cmd[1];
             if (opname == "get")
             {
-                db->get(key, [this, cb](int res, const std::string & value)
+                db->get(key, [cb](int res, const std::string & value)
                 {
                     if (res < 0)
                         fprintf(stderr, "Error: %s (code %d)\n", strerror(-res), res);

src/kv/kv_db.cpp

@@ -1448,7 +1448,7 @@ void kv_op_t::update()
 
 void kv_op_t::update_find()
 {
-    get_block(db, cur_block, cur_level, recheck_policy, [=, checked_block = cur_block](int res, int refresh)
+    get_block(db, cur_block, cur_level, recheck_policy, [=](int res, int refresh)
     {
         res = handle_block(res, refresh, true);
         if (res == -EAGAIN)

src/nfs/CMakeLists.txt

@@ -5,6 +5,7 @@ project(vitastor)
 # vitastor-nfs
 add_executable(vitastor-nfs
 	nfs_proxy.cpp
+	nfs_proxy_rdma.cpp
 	nfs_block.cpp
 	nfs_kv.cpp
 	nfs_kv_create.cpp
@@ -21,8 +22,10 @@ add_executable(vitastor-nfs
 	nfs_fsstat.cpp
 	nfs_mount.cpp
 	nfs_portmap.cpp
+	rdma_alloc.cpp
 	../util/sha256.c
 	proto/xdr_impl.cpp
+	proto/rpc_rdma_xdr.cpp
 	proto/rpc_xdr.cpp
 	proto/portmap_xdr.cpp
 	proto/nfs_xdr.cpp
@@ -30,4 +33,5 @@ add_executable(vitastor-nfs
 target_link_libraries(vitastor-nfs
 	vitastor_client
 	vitastor_kv
+	${RDMACM_LIBRARIES}
 )

src/nfs/nfs_block.cpp

@@ -315,8 +315,7 @@ static int block_nfs3_read_proc(void *opaque, rpc_op_t *rop)
     if (aligned_count % alignment)
         aligned_count = aligned_count + alignment - (aligned_count % alignment);
     aligned_count -= aligned_offset;
-    void *buf = malloc_or_die(aligned_count);
-    xdr_add_malloc(rop->xdrs, buf);
+    void *buf = self->malloc_or_rdma(rop, aligned_count);
     cluster_op_t *op = new cluster_op_t;
     op->opcode = OSD_OP_READ;
     op->inode = ino_it->second;
@@ -335,10 +334,22 @@ static int block_nfs3_read_proc(void *opaque, rpc_op_t *rop)
         }
         else
         {
+            nfs_client_t *self = (nfs_client_t*)rop->client;
             auto & reply_ok = reply->resok;
             // reply_ok.data.data is already set above
             reply_ok.count = reply_ok.data.size;
             reply_ok.eof = 0;
+            if (self->rdma_conn)
+            {
+                // FIXME Linux NFS RDMA transport has a bug - when the reply
+                // doesn't contain post_op_attr, the data gets offsetted by
+                // 84 bytes (size of attributes)...
+                // So we have to fill it with RDMA. :-(
+                reply_ok.file_attributes = (post_op_attr){
+                    .attributes_follow = 1,
+                    .attributes = get_file_attributes(self, op->inode),
+                };
+            }
         }
         rpc_queue_reply(rop);
         delete op;
@@ -1057,22 +1068,25 @@ static void block_nfs3_readdir_common(void *opaque, rpc_op_t *rop, bool is_plus)
     void *prev = NULL;
     for (auto it = entries.begin(); it != entries.end();)
     {
-        entryplus3 *entry = &it->second;
-        // First fields of entry3 and entryplus3 are the same: fileid, name, cookie
-        entry->name = xdr_copy_string(rop->xdrs, it->first);
-        entry->cookie = idx++;
-        if (prev)
-        {
-            if (is_plus)
-                ((entryplus3*)prev)->nextentry = entry;
-            else
-                ((entry3*)prev)->nextentry = (entry3*)entry;
-        }
-        prev = entry;
-        if (args->cookie > 0 && entry->cookie == args->cookie)
-            entries.erase(entries.begin(), ++it);
+        if (args->cookie > 0 && idx <= args->cookie)
+            entries.erase(it++);
         else
+        {
+            entryplus3 *entry = &it->second;
+            // First fields of entry3 and entryplus3 are the same: fileid, name, cookie
+            entry->name = xdr_copy_string(rop->xdrs, it->first);
+            entry->cookie = idx;
+            if (prev)
+            {
+                if (is_plus)
+                    ((entryplus3*)prev)->nextentry = entry;
+                else
+                    ((entry3*)prev)->nextentry = (entry3*)entry;
+            }
+            prev = entry;
             it++;
+        }
+        idx++;
     }
     // Now limit results based on maximum reply size
     // Sadly we have to calculate reply size by hand

src/nfs/nfs_kv.cpp

@@ -65,7 +65,7 @@ int kv_map_type(const std::string & type)
         (type == "fifo" ? NF3FIFO : -1)))))));
 }
 
-fattr3 get_kv_attributes(nfs_client_t *self, uint64_t ino, json11::Json attrs)
+fattr3 get_kv_attributes(nfs_proxy_t *proxy, uint64_t ino, json11::Json attrs)
 {
     auto type = kv_map_type(attrs["type"].string_value());
     auto mode = attrs["mode"].uint64_value();
@@ -86,7 +86,7 @@ fattr3 get_kv_attributes(nfs_client_t *self, uint64_t ino, json11::Json attrs)
         .rdev = (type == NF3BLK || type == NF3CHR
             ? (specdata3){ (uint32_t)attrs["major"].uint64_value(), (uint32_t)attrs["minor"].uint64_value() }
             : (specdata3){}),
-        .fsid = self->parent->fsid,
+        .fsid = proxy->fsid,
         .fileid = ino,
         .atime = atime,
         .mtime = mtime,
@@ -119,7 +119,7 @@ std::string kv_direntry_filename(const std::string & key)
 std::string kv_inode_prefix_key(uint64_t ino, const char *prefix)
 {
     int max = 32+strlen(prefix);
-    char key[max] = { 0 };
+    char key[max];
     snprintf(key, max, "%s%x", prefix, INODE_POOL(ino));
     int n = strnlen(key, max-1);
     snprintf(key+n+1, max-n-1, "%jx", INODE_NO_POOL(ino));
@@ -349,6 +349,27 @@ kv_fs_state_t::~kv_fs_state_t()
     }
 }
 
+void kv_fs_state_t::write_inode(inode_t ino, json11::Json value, bool hack_cache, std::function<void(int)> cb, std::function<bool(int, const std::string &)> cas_cb)
+{
+    if (!proxy->rdma_context)
+    {
+        proxy->db->set(kv_inode_key(ino), value.dump(), cb, cas_cb);
+        return;
+    }
+    // FIXME Linux NFS RDMA transport has a bug - it corrupts the data (by offsetting it 84 bytes)
+    // when the READ reply doesn't contain post_op_attr. So we have to fill post_op_attr with RDMA. :-(
+    // So we at least cache it to not repeat K/V requests every read.
+    read_hack_cache.erase(ino);
+    proxy->db->set(kv_inode_key(ino), value.dump(), [=](int res)
+    {
+        if (hack_cache || res != 0)
+            read_hack_cache.erase(ino);
+        else
+            read_hack_cache[ino] = value;
+        cb(res);
+    }, cas_cb);
+}
+
 void kv_fs_state_t::update_inode(inode_t ino, bool allow_cache, std::function<void(json11::Json::object &)> change, std::function<void(int)> cb)
 {
     // FIXME: Use "update" query
@@ -356,12 +377,15 @@ void kv_fs_state_t::update_inode(inode_t ino, bool allow_cache, std::function<vo
     {
         if (!res)
         {
+            read_hack_cache.erase(ino);
             auto ientry = attrs.object_items();
             change(ientry);
             bool *found = new bool;
             *found = true;
-            proxy->db->set(kv_inode_key(ino), json11::Json(ientry).dump(), [=](int res)
+            json11::Json ientry_json(ientry);
+            proxy->db->set(kv_inode_key(ino), ientry_json.dump(), [=](int res)
             {
+                read_hack_cache.erase(ino);
                 if (!*found)
                     res = -ENOENT;
                 delete found;
@@ -384,6 +408,8 @@ void kv_fs_state_t::update_inode(inode_t ino, bool allow_cache, std::function<vo
 
 void kv_fs_state_t::touch_inodes()
 {
+    // Clear RDMA read fattr3 "hack" cache every second
+    read_hack_cache.clear();
     std::set<inode_t> q = std::move(touch_queue);
     for (auto ino: q)
     {

src/nfs/nfs_kv.h

@@ -75,6 +75,7 @@ struct kv_fs_state_t
     std::map<inode_t, kv_inode_extend_t> extends;
     std::set<inode_t> touch_queue;
     std::map<inode_t, uint64_t> volume_removed;
+    std::map<inode_t, json11::Json> read_hack_cache;
     uint64_t volume_stats_ctr = 0;
     uint64_t volume_touch_ctr = 0;
 
@@ -87,6 +88,7 @@ struct kv_fs_state_t
     void upgrade_db(std::function<void(int)> cb);
     void defrag_all(json11::Json cfg, std::function<void(int)> cb);
     void defrag_volume(inode_t ino, bool no_rm, bool dry_run, std::function<void(int, uint64_t, uint64_t, uint64_t)> cb);
+    void write_inode(inode_t ino, json11::Json value, bool hack_cache, std::function<void(int)> cb, std::function<bool(int, const std::string &)> cas_cb);
     ~kv_fs_state_t();
 };
 
@@ -116,7 +118,7 @@ nfstime3 nfstime_from_str(const std::string & s);
 std::string nfstime_to_str(nfstime3 t);
 std::string nfstime_now_str();
 int kv_map_type(const std::string & type);
-fattr3 get_kv_attributes(nfs_client_t *self, uint64_t ino, json11::Json attrs);
+fattr3 get_kv_attributes(nfs_proxy_t *proxy, uint64_t ino, json11::Json attrs);
 std::string kv_direntry_key(uint64_t dir_ino, const std::string & filename);
 std::string kv_direntry_filename(const std::string & key);
 std::string kv_inode_prefix_key(uint64_t ino, const char *prefix);

src/nfs/nfs_kv_create.cpp

@@ -143,11 +143,11 @@ resume_2:
         cb(st->res);
         return;
     }
-    st->self->parent->db->set(kv_inode_key(st->new_id), st->attrs.dump().c_str(), [st](int res)
+    st->self->parent->kvfs->write_inode(st->new_id, st->attrs, false, [st](int res)
     {
         st->res = res;
         kv_continue_create(st, 3);
-    }, [st](int res, const std::string & value)
+    }, [](int res, const std::string & value)
     {
         return res == -ENOENT;
     });
@@ -267,7 +267,7 @@ template<class T, class Tok> static void kv_create_reply(kv_create_state *st, in
                 },
                 .obj_attributes = {
                     .attributes_follow = 1,
-                    .attributes = get_kv_attributes(st->self, st->new_id, st->attrs),
+                    .attributes = get_kv_attributes(st->self->parent, st->new_id, st->attrs),
                 },
             },
         };

src/nfs/nfs_kv_defrag.cpp

@@ -81,12 +81,12 @@ void kv_fs_defrag_t::read()
             empty = true;
             for (; bitmap_pos < bitmap_size; bitmap_pos += 8)
             {
-                if (*((uint64_t*)(op->bitmap_buf + bitmap_pos)))
+                if (*((uint64_t*)((uint8_t*)op->bitmap_buf + bitmap_pos)))
                     empty = false;
             }
             for (; bitmap_pos < bitmap_size; bitmap_pos++)
             {
-                if (*((uint8_t*)(op->bitmap_buf + bitmap_pos)))
+                if (*((uint8_t*)((uint8_t*)op->bitmap_buf + bitmap_pos)))
                     empty = false;
             }
             buf_pos = 0;

src/nfs/nfs_kv_getattr.cpp

@@ -68,7 +68,7 @@ int kv_nfs3_getattr_proc(void *opaque, rpc_op_t *rop)
             *reply = (GETATTR3res){
                 .status = NFS3_OK,
                 .resok = (GETATTR3resok){
-                    .obj_attributes = get_kv_attributes(self, ino, attrs),
+                    .obj_attributes = get_kv_attributes(self->parent, ino, attrs),
                 },
             };
         }

src/nfs/nfs_kv_link.cpp

@@ -81,7 +81,7 @@ resume_1:
         {
             st->res = res;
             nfs_kv_continue_link(st, 2);
-        }, [st](int res, const std::string & old_value)
+        }, [](int res, const std::string & old_value)
         {
             return res == -ENOENT;
         });
@@ -102,7 +102,7 @@ resume_2:
         new_ientry["ctime"] = nfstime_now_str();
         st->ientry = new_ientry;
     }
-    st->self->parent->db->set(kv_inode_key(st->ino), st->ientry.dump(), [st](int res)
+    st->self->parent->kvfs->write_inode(st->ino, st->ientry, false, [st](int res)
     {
         st->res = res;
         nfs_kv_continue_link(st, 3);
@@ -180,7 +180,7 @@ int kv_nfs3_link_proc(void *opaque, rpc_op_t *rop)
                 .resok = (LINK3resok){
                     .file_attributes = (post_op_attr){
                         .attributes_follow = 1,
-                        .attributes = get_kv_attributes(st->self, st->ino, st->ientry),
+                        .attributes = get_kv_attributes(st->self->parent, st->ino, st->ientry),
                     },
                 },
             };

src/nfs/nfs_kv_lookup.cpp

@@ -55,7 +55,7 @@ int kv_nfs3_lookup_proc(void *opaque, rpc_op_t *rop)
                     .object = xdr_copy_string(rop->xdrs, kv_fh(ino)),
                     .obj_attributes = {
                         .attributes_follow = 1,
-                        .attributes = get_kv_attributes(self, ino, ientry),
+                        .attributes = get_kv_attributes(self->parent, ino, ientry),
                     },
                 },
             };
@@ -91,10 +91,14 @@ int kv_nfs3_readlink_proc(void *opaque, rpc_op_t *rop)
         }
         else
         {
+            std::string link_target = attrs["symlink"].string_value();
+            char *cp = (char*)self->malloc_or_rdma(rop, link_target.size()+1);
+            memcpy(cp, link_target.data(), link_target.size());
+            cp[link_target.size()] = 0;
             *reply = (READLINK3res){
                 .status = NFS3_OK,
                 .resok = (READLINK3resok){
-                    .data = xdr_copy_string(rop->xdrs, attrs["symlink"].string_value()),
+                    .data = (xdr_string_t){ link_target.size(), cp },
                 },
             };
         }

src/nfs/nfs_kv_read.cpp

@@ -37,6 +37,7 @@ static void nfs_kv_continue_read(nfs_kv_read_state *st, int state)
     else if (state == 1) goto resume_1;
     else if (state == 2) goto resume_2;
     else if (state == 3) goto resume_3;
+    else if (state == 4) goto resume_4;
     else
     {
         fprintf(stderr, "BUG: invalid state in nfs_kv_continue_read()");
@@ -96,7 +97,7 @@ resume_1:
                 }
                 read_size += sizeof(shared_file_header_t);
                 assert(!st->aligned_buf);
-                st->aligned_buf = (uint8_t*)malloc_or_die(read_size);
+                st->aligned_buf = (uint8_t*)st->self->malloc_or_rdma(st->rop, read_size);
                 st->buf = st->aligned_buf + sizeof(shared_file_header_t) + st->offset;
                 st->op->iov.push_back(st->aligned_buf, read_size);
                 st->op->len = align_up(read_offset+read_size) - st->op->offset;
@@ -106,7 +107,7 @@ resume_1:
                         st->op->offset+st->op->len - (read_offset+read_size));
                 }
             }
-            st->op->callback = [st, state](cluster_op_t *op)
+            st->op->callback = [st](cluster_op_t *op)
             {
                 st->res = op->retval == op->len ? 0 : op->retval;
                 delete op;
@@ -117,7 +118,7 @@ resume_1:
 resume_2:
             if (st->res < 0)
             {
-                free(st->aligned_buf);
+                st->self->free_or_rdma(st->rop, st->aligned_buf);
                 st->aligned_buf = NULL;
                 auto cb = std::move(st->cb);
                 cb(st->res);
@@ -131,7 +132,7 @@ resume_2:
                     " 0x%jx offset 0x%jx: probably a read/write conflict, retrying\n",
                     st->ino, st->ientry["shared_ino"].uint64_value(), st->ientry["shared_offset"].uint64_value());
                 st->retry++;
-                free(st->aligned_buf);
+                st->self->free_or_rdma(st->rop, st->aligned_buf);
                 st->aligned_buf = NULL;
                 st->allow_cache = false;
                 goto resume_0;
@@ -141,10 +142,39 @@ resume_2:
             return;
         }
     }
+    else if (st->self->rdma_conn)
+    {
+        // Take ientry from read_hack_cache for RDMA connections
+        {
+            auto rh_it = st->self->parent->kvfs->read_hack_cache.find(st->ino);
+            if (rh_it != st->self->parent->kvfs->read_hack_cache.end())
+            {
+                st->ientry = rh_it->second;
+            }
+        }
+        if (st->ientry.is_null())
+        {
+            kv_read_inode(st->self->parent, st->ino, [st](int res, const std::string & value, json11::Json attrs)
+            {
+                st->res = res;
+                st->ientry = attrs;
+                nfs_kv_continue_read(st, 4);
+            }, st->allow_cache);
+            return;
+resume_4:
+            if (st->res < 0 || kv_map_type(st->ientry["type"].string_value()) != NF3REG)
+            {
+                auto cb = std::move(st->cb);
+                cb(st->res < 0 ? st->res : -EINVAL);
+                return;
+            }
+            st->self->parent->kvfs->read_hack_cache[st->ino] = st->ientry;
+        }
+    }
     st->aligned_offset = align_down(st->offset);
     st->aligned_size = align_up(st->offset+st->size) - st->aligned_offset;
     assert(!st->aligned_buf);
-    st->aligned_buf = (uint8_t*)malloc_or_die(st->aligned_size);
+    st->aligned_buf = (uint8_t*)st->self->malloc_or_rdma(st->rop, st->aligned_size);
     st->buf = st->aligned_buf + st->offset - st->aligned_offset;
     st->op = new cluster_op_t;
     st->op->opcode = OSD_OP_READ;
@@ -163,7 +193,7 @@ resume_2:
 resume_3:
     if (st->res < 0)
     {
-        free(st->aligned_buf);
+        st->self->free_or_rdma(st->rop, st->aligned_buf);
         st->aligned_buf = NULL;
     }
     auto cb = std::move(st->cb);
@@ -194,11 +224,21 @@ int kv_nfs3_read_proc(void *opaque, rpc_op_t *rop)
         *reply = (READ3res){ .status = vitastor_nfs_map_err(res) };
         if (res == 0)
         {
-            xdr_add_malloc(st->rop->xdrs, st->aligned_buf);
             reply->resok.data.data = (char*)st->buf;
             reply->resok.data.size = st->size;
             reply->resok.count = st->size;
             reply->resok.eof = st->eof;
+            if (st->self->rdma_conn)
+            {
+                // FIXME Linux NFS RDMA transport has a bug - when the reply
+                // doesn't contain post_op_attr, the data gets offsetted by
+                // 84 bytes (size of attributes)...
+                // So we have to fill it with RDMA. :-(
+                reply->resok.file_attributes = (post_op_attr){
+                    .attributes_follow = 1,
+                    .attributes = get_kv_attributes(st->self->parent, st->ino, st->ientry),
+                };
+            }
         }
         rpc_queue_reply(st->rop);
         delete st;

src/nfs/nfs_kv_readdir.cpp

@@ -57,7 +57,7 @@ static void kv_getattr_next(nfs_kv_readdir_state *st)
                 st->entries[idx].name_attributes = (post_op_attr){
                     // FIXME: maybe do not read parent attributes and leave them to a GETATTR?
                     .attributes_follow = 1,
-                    .attributes = get_kv_attributes(st->self, st->entries[idx].fileid, ientry),
+                    .attributes = get_kv_attributes(st->self->parent, st->entries[idx].fileid, ientry),
                 };
             }
             st->getattr_running--;
@@ -126,7 +126,7 @@ resume_1:
             dot.fileid = st->dir_ino;
             dot.name_attributes = (post_op_attr){
                 .attributes_follow = 1,
-                .attributes = get_kv_attributes(st->self, st->dir_ino, st->ientry),
+                .attributes = get_kv_attributes(st->self->parent, st->dir_ino, st->ientry),
             };
             dot.name_handle = (post_op_fh3){
                 .handle_follows = 1,
@@ -169,7 +169,7 @@ resume_2:
         dotdot.name_attributes = (post_op_attr){
             // FIXME: maybe do not read parent attributes and leave them to a GETATTR?
             .attributes_follow = 1,
-            .attributes = get_kv_attributes(st->self,
+            .attributes = get_kv_attributes(st->self->parent,
                 st->parent_ino ? st->parent_ino : st->dir_ino,
                 st->parent_ino ? st->parent_ientry : st->ientry),
         };

src/nfs/nfs_kv_remove.cpp

@@ -169,7 +169,7 @@ resume_4:
             {
                 st->res2 = res;
                 nfs_kv_continue_delete(st, 5);
-            }, [st](int res, const std::string & value)
+            }, [](int res, const std::string & value)
             {
                 return res == -ENOENT;
             });
@@ -197,7 +197,7 @@ resume_5:
         auto copy = st->ientry.object_items();
         copy["nlink"] = st->ientry["nlink"].uint64_value()-1;
         copy["ctime"] = nfstime_now_str();
-        st->self->parent->db->set(kv_inode_key(st->ino), json11::Json(copy).dump(), [st](int res)
+        st->self->parent->kvfs->write_inode(st->ino, copy, false, [st](int res)
         {
             st->res = res;
             nfs_kv_continue_delete(st, 6);

src/nfs/nfs_kv_rename.cpp

@@ -240,7 +240,7 @@ resume_7:
                 copy["nlink"] = st->new_ientry["nlink"].uint64_value()-1;
                 copy["ctime"] = nfstime_now_str();
                 copy.erase("verf");
-                st->self->parent->db->set(kv_inode_key(st->new_direntry["ino"].uint64_value()), json11::Json(copy).dump(), [st](int res)
+                st->self->parent->kvfs->write_inode(st->new_direntry["ino"].uint64_value(), copy, false, [st](int res)
                 {
                     st->res = res;
                     nfs_kv_continue_rename(st, 8);
@@ -328,7 +328,7 @@ resume_11:
             ientry_new["parent_ino"] = st->new_dir_ino;
             ientry_new["ctime"] = nfstime_now_str();
             ientry_new.erase("verf");
-            st->self->parent->db->set(kv_inode_key(st->old_direntry["ino"].uint64_value()), json11::Json(ientry_new).dump(), [st](int res)
+            st->self->parent->kvfs->write_inode(st->old_direntry["ino"].uint64_value(), ientry_new, false, [st](int res)
             {
                 st->res = res;
                 nfs_kv_continue_rename(st, 12);

src/nfs/nfs_kv_setattr.cpp

@@ -84,7 +84,7 @@ resume_1:
     }
     st->new_attrs.erase("verf");
     st->new_attrs["ctime"] = nfstime_now_str();
-    st->self->parent->db->set(kv_inode_key(st->ino), json11::Json(st->new_attrs).dump(), [st](int res)
+    st->self->parent->kvfs->write_inode(st->ino, st->new_attrs, false, [st](int res)
     {
         st->res = res;
         nfs_kv_continue_setattr(st, 2);
@@ -190,7 +190,7 @@ int kv_nfs3_setattr_proc(void *opaque, rpc_op_t *rop)
                     .obj_wcc = (wcc_data){
                         .after = (post_op_attr){
                             .attributes_follow = 1,
-                            .attributes = get_kv_attributes(st->self, st->ino, st->new_attrs),
+                            .attributes = get_kv_attributes(st->self->parent, st->ino, st->new_attrs),
                         },
                     },
                 },

src/nfs/nfs_kv_write.cpp

@@ -553,7 +553,7 @@ static void nfs_do_align_write(nfs_kv_write_state *st, uint64_t ino, uint64_t of
     }
 }
 
-static std::string new_normal_ientry(nfs_kv_write_state *st)
+static json11::Json new_normal_ientry(nfs_kv_write_state *st)
 {
     auto ni = st->ientry.object_items();
     ni.erase("empty");
@@ -564,10 +564,10 @@ static std::string new_normal_ientry(nfs_kv_write_state *st)
     ni["size"] = st->ext->cur_extend;
     ni["ctime"] = ni["mtime"] = nfstime_now_str();
     ni.erase("verf");
-    return json11::Json(ni).dump();
+    return ni;
 }
 
-static std::string new_moved_ientry(nfs_kv_write_state *st)
+static json11::Json new_moved_ientry(nfs_kv_write_state *st)
 {
     auto ni = st->ientry.object_items();
     ni.erase("empty");
@@ -578,10 +578,10 @@ static std::string new_moved_ientry(nfs_kv_write_state *st)
     ni["size"] = st->new_size;
     ni["ctime"] = ni["mtime"] = nfstime_now_str();
     ni.erase("verf");
-    return json11::Json(ni).dump();
+    return ni;
 }
 
-static std::string new_shared_ientry(nfs_kv_write_state *st)
+static json11::Json new_shared_ientry(nfs_kv_write_state *st)
 {
     auto ni = st->ientry.object_items();
     ni.erase("empty");
@@ -589,10 +589,10 @@ static std::string new_shared_ientry(nfs_kv_write_state *st)
     ni["ctime"] = ni["mtime"] = nfstime_now_str();
     ni["shared_ver"] = ni["shared_ver"].uint64_value()+1;
     ni.erase("verf");
-    return json11::Json(ni).dump();
+    return ni;
 }
 
-static std::string new_unshared_ientry(nfs_kv_write_state *st)
+static json11::Json new_unshared_ientry(nfs_kv_write_state *st)
 {
     auto ni = st->ientry.object_items();
     ni.erase("empty");
@@ -602,7 +602,7 @@ static std::string new_unshared_ientry(nfs_kv_write_state *st)
     ni.erase("shared_ver");
     ni["ctime"] = ni["mtime"] = nfstime_now_str();
     ni.erase("verf");
-    return json11::Json(ni).dump();
+    return ni;
 }
 
 static void nfs_kv_extend_inode(nfs_kv_write_state *st, int state, int base_state)
@@ -612,7 +612,7 @@ static void nfs_kv_extend_inode(nfs_kv_write_state *st, int state, int base_stat
     st->ext->cur_extend = st->ext->next_extend;
     st->ext->next_extend = 0;
     st->res2 = -EAGAIN;
-    st->proxy->db->set(kv_inode_key(st->ino), new_normal_ientry(st), [st, base_state](int res)
+    st->proxy->kvfs->write_inode(st->ino, new_normal_ientry(st), true, [st, base_state](int res)
     {
         st->res = res;
         nfs_kv_continue_write(st, base_state+1);
@@ -838,7 +838,7 @@ resume_4:
                 cb(st->res);
                 return;
             }
-            st->proxy->db->set(kv_inode_key(st->ino), new_moved_ientry(st), [st](int res)
+            st->proxy->kvfs->write_inode(st->ino, new_moved_ientry(st), true, [st](int res)
             {
                 st->res = res;
                 nfs_kv_continue_write(st, 5);
@@ -881,7 +881,7 @@ resume_7:
             }
 resume_8:
             // We always have to change inode entry on shared writes
-            st->proxy->db->set(kv_inode_key(st->ino), new_shared_ientry(st), [st](int res)
+            st->proxy->kvfs->write_inode(st->ino, new_shared_ientry(st), true, [st](int res)
             {
                 st->res = res;
                 nfs_kv_continue_write(st, 9);
@@ -930,7 +930,7 @@ resume_11:
                 return;
             }
         }
-        st->proxy->db->set(kv_inode_key(st->ino), new_unshared_ientry(st), [st](int res)
+        st->proxy->kvfs->write_inode(st->ino, new_unshared_ientry(st), true, [st](int res)
         {
             st->res = res;
             nfs_kv_continue_write(st, 12);
@@ -953,7 +953,7 @@ resume_12:
         }
         // Record removed part of the shared inode as obsolete in statistics
         st->proxy->kvfs->volume_removed[st->ientry["shared_ino"].uint64_value()] += st->ientry["shared_alloc"].uint64_value();
-        st->ientry_text = new_unshared_ientry(st);
+        st->ientry_text = new_unshared_ientry(st).dump();
     }
     // Non-shared write
     nfs_do_align_write(st, st->ino, st->offset, 0, 13);

src/nfs/nfs_proxy.cpp

@@ -34,6 +34,9 @@ const char *exe_name = NULL;
 
 nfs_proxy_t::~nfs_proxy_t()
 {
+#ifdef WITH_RDMACM
+    destroy_rdma();
+#endif
     if (kvfs)
         delete kvfs;
     if (blockfs)
@@ -65,9 +68,16 @@ static const char* help_text =
     "\n"
     "vitastor-nfs (--fs <NAME> | --block) start\n"
     "  Start network NFS server. Options:\n"
-    "  --bind <IP>       bind service to <IP> address (default 0.0.0.0)\n"
-    "  --port <PORT>     use port <PORT> for NFS services (default is 2049)\n"
-    "  --portmap 0       do not listen on port 111 (portmap/rpcbind, requires root)\n"
+    "  --bind <IP>           bind service to <IP> address (default 0.0.0.0)\n"
+    "  --port <PORT>         use port <PORT> for NFS services (default is 2049)\n"
+    "                        specify \"auto\" to auto-select and print port\n"
+    "  --portmap 0           do not listen on port 111 (portmap/rpcbind, requires root)\n"
+    "  --nfs_rdma <PORT>     enable NFS-RDMA at RDMA-CM port <PORT> (you can try 20049)\n"
+    "                        if RDMA is enabled and --port is set to 0, TCP will be disabled\n"
+    "  --nfs_rdma_credit 16  maximum operation credit for RDMA clients (max iodepth)\n"
+    "  --nfs_rdma_send 1024  maximum RDMA send operation count (should be larger than iodepth)\n"
+    "  --nfs_rdma_alloc 1M   RDMA memory allocation rounding\n"
+    "  --nfs_rdma_gc 64M     maximum unused RDMA buffers\n"
     "\n"
     "vitastor-nfs --fs <NAME> upgrade\n"
     "  Upgrade FS metadata. Can be run online, but server(s) should be restarted\n"
@@ -184,6 +194,7 @@ void nfs_proxy_t::run(json11::Json cfg)
     srand48(tv.tv_sec*1000000000 + tv.tv_nsec);
     server_id = (uint64_t)lrand48() | ((uint64_t)lrand48() << 31) | ((uint64_t)lrand48() << 62);
     // Parse options
+    mountpoint = cfg["mount"].string_value();
     if (cfg["logfile"].string_value() != "")
         logfile = cfg["logfile"].string_value();
     pidfile = cfg["pidfile"].string_value();
@@ -194,8 +205,24 @@ void nfs_proxy_t::run(json11::Json cfg)
     default_pool = cfg["pool"].as_string();
     portmap_enabled = !json_is_false(cfg["portmap"]);
     nfs_port = cfg["port"].uint64_value() & 0xffff;
+    nfs_rdma_port = cfg["nfs_rdma"].uint64_value() & 0xffff;
+    // Allow RDMA-only mode if port is explicitly set to 0
+    // Allow port auto-selection in server mode if explicitly set to --port auto
+    nfs_port_auto = cfg["port"] == "auto";
     if (!nfs_port)
-        nfs_port = 2049;
+        nfs_port = nfs_port_auto ? 0 : (!cfg["port"].is_null() && nfs_rdma_port ? -1 : 2049);
+    nfs_rdma_credit = cfg["nfs_rdma_credit"].uint64_value();
+    if (!nfs_rdma_credit)
+        nfs_rdma_credit = 16;
+    nfs_rdma_max_send = cfg["nfs_rdma_send"].uint64_value();
+    if (!nfs_rdma_max_send)
+        nfs_rdma_max_send = 1024;
+    nfs_rdma_alloc = cfg["nfs_rdma_alloc"].uint64_value();
+    if (!nfs_rdma_alloc)
+        nfs_rdma_alloc = 1048576;
+    nfs_rdma_gc = cfg["nfs_rdma_gc"].uint64_value();
+    if (!nfs_rdma_gc)
+        nfs_rdma_gc = 64*1048576;
     export_root = cfg["nfspath"].string_value();
     if (!export_root.size())
         export_root = "/";
@@ -207,7 +234,6 @@ void nfs_proxy_t::run(json11::Json cfg)
         obj["client_writeback_allowed"] = true;
         cfg = obj;
     }
-    mountpoint = cfg["mount"].string_value();
     if (mountpoint != "")
     {
         bind_address = "127.0.0.1";
@@ -292,49 +318,60 @@ void nfs_proxy_t::run(json11::Json cfg)
 
 void nfs_proxy_t::run_server(json11::Json cfg)
 {
+    if (nfs_port != -1)
+    {
+        // Create NFS socket and add it to epoll
+        int nfs_socket = create_and_bind_socket(bind_address, nfs_port, 128, &listening_port);
+        fcntl(nfs_socket, F_SETFL, fcntl(nfs_socket, F_GETFL, 0) | O_NONBLOCK);
+        epmgr->tfd->set_fd_handler(nfs_socket, false, [this](int nfs_socket, int epoll_events)
+        {
+            if (epoll_events & EPOLLRDHUP)
+            {
+                fprintf(stderr, "Listening portmap socket disconnected, exiting\n");
+                exit(1);
+            }
+            else
+            {
+                do_accept(nfs_socket);
+            }
+        });
+        if (nfs_port_auto)
+        {
+            printf("Port: %d\n", listening_port);
+        }
+    }
+    else
+    {
+        listening_port = nfs_rdma_port;
+    }
     // Self-register portmap and NFS
     pmap.reg_ports.insert((portmap_id_t){
         .prog = PMAP_PROGRAM,
         .vers = PMAP_V2,
-        .port = portmap_enabled ? 111 : nfs_port,
+        .port = (unsigned)(portmap_enabled ? 111 : listening_port),
         .owner = "portmapper-service",
-        .addr = portmap_enabled ? "0.0.0.0.0.111" : ("0.0.0.0.0."+std::to_string(nfs_port)),
+        .addr = portmap_enabled ? "0.0.0.0.0.111" : ("0.0.0.0.0."+std::to_string(listening_port)),
     });
     pmap.reg_ports.insert((portmap_id_t){
         .prog = PMAP_PROGRAM,
         .vers = PMAP_V3,
-        .port = portmap_enabled ? 111 : nfs_port,
+        .port = (unsigned)(portmap_enabled ? 111 : listening_port),
         .owner = "portmapper-service",
-        .addr = portmap_enabled ? "0.0.0.0.0.111" : ("0.0.0.0.0."+std::to_string(nfs_port)),
+        .addr = portmap_enabled ? "0.0.0.0.0.111" : ("0.0.0.0.0."+std::to_string(listening_port)),
     });
     pmap.reg_ports.insert((portmap_id_t){
         .prog = NFS_PROGRAM,
         .vers = NFS_V3,
-        .port = nfs_port,
+        .port = (unsigned)listening_port,
         .owner = "nfs-server",
-        .addr = "0.0.0.0.0."+std::to_string(nfs_port),
+        .addr = "0.0.0.0.0."+std::to_string(listening_port),
     });
     pmap.reg_ports.insert((portmap_id_t){
         .prog = MOUNT_PROGRAM,
         .vers = MOUNT_V3,
-        .port = nfs_port,
+        .port = (unsigned)listening_port,
         .owner = "rpc.mountd",
-        .addr = "0.0.0.0.0."+std::to_string(nfs_port),
-    });
-    // Create NFS socket and add it to epoll
-    int nfs_socket = create_and_bind_socket(bind_address, nfs_port, 128, &listening_port);
-    fcntl(nfs_socket, F_SETFL, fcntl(nfs_socket, F_GETFL, 0) | O_NONBLOCK);
-    epmgr->tfd->set_fd_handler(nfs_socket, false, [this](int nfs_socket, int epoll_events)
-    {
-        if (epoll_events & EPOLLRDHUP)
-        {
-            fprintf(stderr, "Listening portmap socket disconnected, exiting\n");
-            exit(1);
-        }
-        else
-        {
-            do_accept(nfs_socket);
-        }
+        .addr = "0.0.0.0.0."+std::to_string(listening_port),
     });
     if (portmap_enabled)
     {
@@ -354,6 +391,10 @@ void nfs_proxy_t::run_server(json11::Json cfg)
             }
         });
     }
+    if (nfs_rdma_port)
+    {
+        rdma_context = create_rdma(bind_address, nfs_rdma_port, nfs_rdma_credit, nfs_rdma_max_send, nfs_rdma_alloc, nfs_rdma_gc);
+    }
     if (mountpoint != "")
     {
         mount_fs();
@@ -499,6 +540,20 @@ void nfs_proxy_t::check_default_pool()
     }
 }
 
+nfs_client_t *nfs_proxy_t::create_client()
+{
+    auto cli = new nfs_client_t();
+    cli->parent = this;
+    if (kvfs)
+        nfs_kv_procs(cli);
+    else
+        nfs_block_procs(cli);
+    for (auto & fn: pmap.proc_table)
+        cli->proc_table.insert(fn);
+    rpc_clients.insert(cli);
+    return cli;
+}
+
 void nfs_proxy_t::do_accept(int listen_fd)
 {
     struct sockaddr_storage addr;
@@ -512,18 +567,8 @@ void nfs_proxy_t::do_accept(int listen_fd)
         fcntl(nfs_fd, F_SETFL, fcntl(nfs_fd, F_GETFL, 0) | O_NONBLOCK);
         int one = 1;
         setsockopt(nfs_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
-        auto cli = new nfs_client_t();
-        if (kvfs)
-            nfs_kv_procs(cli);
-        else
-            nfs_block_procs(cli);
-        cli->parent = this;
+        auto cli = this->create_client();
         cli->nfs_fd = nfs_fd;
-        for (auto & fn: pmap.proc_table)
-        {
-            cli->proc_table.insert(fn);
-        }
-        rpc_clients[nfs_fd] = cli;
         epmgr->tfd->set_fd_handler(nfs_fd, true, [cli](int nfs_fd, int epoll_events)
         {
             // Handle incoming event
@@ -780,11 +825,17 @@ void nfs_client_t::stop()
     stopped = true;
     if (refs <= 0)
     {
+#ifdef WITH_RDMACM
+        destroy_rdma_conn();
+#endif
         auto parent = this->parent;
-        parent->rpc_clients.erase(nfs_fd);
+        parent->rpc_clients.erase(this);
         parent->active_connections--;
-        parent->epmgr->tfd->set_fd_handler(nfs_fd, true, NULL);
-        close(nfs_fd);
+        if (nfs_fd >= 0)
+        {
+            parent->epmgr->tfd->set_fd_handler(nfs_fd, true, NULL);
+            close(nfs_fd);
+        }
         delete this;
         parent->check_exit();
     }
@@ -813,8 +864,7 @@ void nfs_client_t::handle_send(int result)
                 if (rop)
                 {
                     // Reply fully sent
-                    xdr_reset(rop->xdrs);
-                    parent->xdr_pool.push_back(rop->xdrs);
+                    parent->free_xdr(rop->xdrs);
                     if (rop->buffer && rop->referenced)
                     {
                         // Dereference the buffer
@@ -831,7 +881,7 @@ void nfs_client_t::handle_send(int result)
                             {
                                 // FIXME Maybe put free_buffers into parent
                                 free_buffers.push_back((rpc_free_buffer_t){
-                                    .buf = rop->buffer,
+                                    .buf = (uint8_t*)rop->buffer,
                                     .size = ub.size,
                                 });
                                 used_buffers.erase(rop->buffer);
@@ -876,8 +926,6 @@ void nfs_client_t::handle_send(int result)
 void rpc_queue_reply(rpc_op_t *rop)
 {
     nfs_client_t *self = (nfs_client_t*)rop->client;
-    iovec *iov_list = NULL;
-    unsigned iov_count = 0;
     int r = xdr_encode(rop->xdrs, (xdrproc_t)xdr_rpc_msg, &rop->out_msg);
     assert(r);
     if (rop->reply_fn != NULL)
@@ -885,55 +933,78 @@ void rpc_queue_reply(rpc_op_t *rop)
         r = xdr_encode(rop->xdrs, rop->reply_fn, rop->reply);
         assert(r);
     }
-    xdr_encode_finish(rop->xdrs, &iov_list, &iov_count);
-    assert(iov_count > 0);
-    rop->reply_marker = 0;
-    for (unsigned i = 0; i < iov_count; i++)
+#ifdef WITH_RDMACM
+    if (!self->rdma_conn)
+#endif
     {
-        rop->reply_marker += iov_list[i].iov_len;
-    }
-    rop->reply_marker = htobe32(rop->reply_marker | 0x80000000);
-    auto & to_send_list = self->write_msg.msg_iovlen ? self->next_send_list : self->send_list;
-    auto & to_outbox = self->write_msg.msg_iovlen ? self->next_outbox : self->outbox;
-    to_send_list.push_back((iovec){ .iov_base = &rop->reply_marker, .iov_len = 4 });
-    to_outbox.push_back(NULL);
-    for (unsigned i = 0; i < iov_count; i++)
-    {
-        to_send_list.push_back(iov_list[i]);
+        iovec *iov_list = NULL;
+        unsigned iov_count = 0;
+        xdr_encode_finish(rop->xdrs, &iov_list, &iov_count);
+        assert(iov_count > 0);
+        rop->reply_marker = 0;
+        for (unsigned i = 0; i < iov_count; i++)
+        {
+            rop->reply_marker += iov_list[i].iov_len;
+        }
+        rop->reply_marker = htobe32(rop->reply_marker | 0x80000000);
+        auto & to_send_list = self->write_msg.msg_iovlen ? self->next_send_list : self->send_list;
+        auto & to_outbox = self->write_msg.msg_iovlen ? self->next_outbox : self->outbox;
+        to_send_list.push_back((iovec){ .iov_base = &rop->reply_marker, .iov_len = 4 });
         to_outbox.push_back(NULL);
+        for (unsigned i = 0; i < iov_count; i++)
+        {
+            to_send_list.push_back(iov_list[i]);
+            to_outbox.push_back(NULL);
+        }
+        to_outbox[to_outbox.size()-1] = rop;
+        self->submit_send();
     }
-    to_outbox[to_outbox.size()-1] = rop;
-    self->submit_send();
+#ifdef WITH_RDMACM
+    else
+    {
+        self->rdma_queue_reply(rop);
+    }
+#endif
 }
 
-int nfs_client_t::handle_rpc_message(void *base_buf, void *msg_buf, uint32_t msg_len)
+XDR *nfs_proxy_t::get_xdr()
 {
     // Take an XDR object from the pool
     XDR *xdrs;
-    if (parent->xdr_pool.size())
+    if (xdr_pool.size())
     {
-        xdrs = parent->xdr_pool.back();
-        parent->xdr_pool.pop_back();
+        xdrs = xdr_pool.back();
+        xdr_pool.pop_back();
     }
     else
     {
         xdrs = xdr_create();
     }
+    return xdrs;
+}
+
+void nfs_proxy_t::free_xdr(XDR *xdrs)
+{
+    xdr_reset(xdrs);
+    xdr_pool.push_back(xdrs);
+}
+
+int nfs_client_t::handle_rpc_message(void *base_buf, void *msg_buf, uint32_t msg_len)
+{
+    XDR *xdrs = parent->get_xdr();
     // Decode the RPC header
     char inmsg_data[sizeof(rpc_msg)];
     rpc_msg *inmsg = (rpc_msg*)&inmsg_data;
     if (!xdr_decode(xdrs, msg_buf, msg_len, (xdrproc_t)xdr_rpc_msg, inmsg))
     {
         // Invalid message, ignore it
-        xdr_reset(xdrs);
-        parent->xdr_pool.push_back(xdrs);
+        parent->free_xdr(xdrs);
         return 0;
     }
     if (inmsg->body.dir != RPC_CALL)
     {
         // Reply sent to the server? Strange thing. Also ignore it
-        xdr_reset(xdrs);
-        parent->xdr_pool.push_back(xdrs);
+        parent->free_xdr(xdrs);
         return 0;
     }
     if (inmsg->body.cbody.rpcvers != RPC_MSG_VERSION)
@@ -968,6 +1039,17 @@ int nfs_client_t::handle_rpc_message(void *base_buf, void *msg_buf, uint32_t msg
         // Incoming buffer isn't needed to handle request, so return 0
         return 0;
     }
+    auto rop = create_rpc_op(xdrs, base_buf, inmsg, NULL);
+    if (!rop)
+    {
+        // No such procedure
+        return 0;
+    }
+    return handle_rpc_op(rop);
+}
+
+rpc_op_t *nfs_client_t::create_rpc_op(XDR *xdrs, void *buffer, rpc_msg *inmsg, rdma_msg *rmsg)
+{
     // Find decoder for the request
     auto proc_it = proc_table.find((rpc_service_proc_t){
         .prog = inmsg->body.cbody.prog,
@@ -995,6 +1077,7 @@ int nfs_client_t::handle_rpc_message(void *base_buf, void *msg_buf, uint32_t msg
         rpc_op_t *rop = (rpc_op_t*)malloc_or_die(sizeof(rpc_op_t));
         *rop = (rpc_op_t){
             .client = this,
+            .buffer = buffer,
             .xdrs = xdrs,
             .out_msg = (rpc_msg){
                 .xid = inmsg->xid,
@@ -1017,9 +1100,15 @@ int nfs_client_t::handle_rpc_message(void *base_buf, void *msg_buf, uint32_t msg
                 },
             },
         };
+        // FIXME: malloc and avoid copy?
+        memcpy(&rop->in_msg, inmsg, sizeof(rpc_msg));
+        if (rmsg)
+        {
+            memcpy(&rop->in_rdma_msg, rmsg, sizeof(rdma_msg));
+        }
         rpc_queue_reply(rop);
         // Incoming buffer isn't needed to handle request, so return 0
-        return 0;
+        return NULL;
     }
     // Allocate memory
     rpc_op_t *rop = (rpc_op_t*)malloc_or_die(
@@ -1028,7 +1117,7 @@ int nfs_client_t::handle_rpc_message(void *base_buf, void *msg_buf, uint32_t msg
     rpc_reply_stat x = RPC_MSG_ACCEPTED;
     *rop = (rpc_op_t){
         .client = this,
-        .buffer = (uint8_t*)base_buf,
+        .buffer = buffer,
         .xdrs = xdrs,
         .out_msg = (rpc_msg){
             .xid = inmsg->xid,
@@ -1045,10 +1134,25 @@ int nfs_client_t::handle_rpc_message(void *base_buf, void *msg_buf, uint32_t msg
         .request = ((uint8_t*)rop) + sizeof(rpc_op_t),
         .reply = ((uint8_t*)rop) + sizeof(rpc_op_t) + proc_it->req_size,
     };
+    // FIXME: malloc and avoid copy?
     memcpy(&rop->in_msg, inmsg, sizeof(rpc_msg));
+    if (rmsg)
+    {
+        memcpy(&rop->in_rdma_msg, rmsg, sizeof(rdma_msg));
+    }
+    return rop;
+}
+
+int nfs_client_t::handle_rpc_op(rpc_op_t *rop)
+{
     // Try to decode the request
     // req_fn may be NULL, that means function has no arguments
-    if (proc_it->req_fn && !proc_it->req_fn(xdrs, rop->request))
+    auto proc_it = proc_table.find((rpc_service_proc_t){
+        .prog = rop->in_msg.body.cbody.prog,
+        .vers = rop->in_msg.body.cbody.vers,
+        .proc = rop->in_msg.body.cbody.proc,
+    });
+    if (proc_it == proc_table.end() || proc_it->req_fn && !proc_it->req_fn(rop->xdrs, rop->request))
     {
         // Invalid request
         rop->out_msg.body.rbody.areply.reply_data.stat = RPC_GARBAGE_ARGS;
@@ -1058,18 +1162,55 @@ int nfs_client_t::handle_rpc_message(void *base_buf, void *msg_buf, uint32_t msg
     }
     rop->out_msg.body.rbody.areply.reply_data.stat = RPC_SUCCESS;
     rop->reply_fn = proc_it->resp_fn;
+    rop->referenced = 0;
     int ref = proc_it->handler_fn(proc_it->opaque, rop);
-    rop->referenced = ref ? 1 : 0;
+    if (ref)
+        rop->referenced = 1;
     return ref;
 }
 
+void *nfs_client_t::malloc_or_rdma(rpc_op_t *rop, size_t size)
+{
+#ifdef WITH_RDMACM
+    if (!rdma_conn)
+    {
+#endif
+        void *buf = malloc_or_die(size);
+        xdr_add_malloc(rop->xdrs, buf);
+        return buf;
+#ifdef WITH_RDMACM
+    }
+    void *buf = rdma_malloc(size);
+    xdr_set_rdma_chunk(rop->xdrs, buf);
+    return buf;
+#endif
+}
+
+void nfs_client_t::free_or_rdma(rpc_op_t *rop, void *buf)
+{
+#ifdef WITH_RDMACM
+    if (!rdma_conn)
+    {
+#endif
+        xdr_del_malloc(rop->xdrs, buf);
+        free(buf);
+#ifdef WITH_RDMACM
+    }
+    else
+    {
+        xdr_set_rdma_chunk(rop->xdrs, NULL);
+        rdma_free(buf);
+    }
+#endif
+}
+
 void nfs_proxy_t::daemonize()
 {
     // Stop all clients because client I/O sometimes breaks during daemonize
     // I.e. the new process stops receiving events on the old FD
     // It doesn't happen if we call sleep(1) here, but we don't want to call sleep(1)...
-    for (auto & clp: rpc_clients)
-        clp.second->stop();
+    for (auto & cli: rpc_clients)
+        cli->stop();
     if (fork())
         exit(0);
     setsid();

src/nfs/nfs_proxy.h

@@ -22,6 +22,7 @@ class cli_tool_t;
 struct kv_fs_state_t;
 struct block_fs_state_t;
 class nfs_client_t;
+struct nfs_rdma_context_t;
 
 class nfs_proxy_t
 {
@@ -33,7 +34,13 @@ public:
     std::string default_pool;
     std::string export_root;
     bool portmap_enabled;
-    unsigned nfs_port;
+    bool nfs_port_auto = false;
+    unsigned nfs_port = 0;
+    unsigned nfs_rdma_port = 0;
+    uint32_t nfs_rdma_credit = 16;
+    uint32_t nfs_rdma_max_send = 1024;
+    uint64_t nfs_rdma_alloc = 1048576;
+    uint64_t nfs_rdma_gc = 500*1048576;
     int trace = 0;
     std::string logfile = "/dev/null";
     std::string pidfile;
@@ -55,7 +62,8 @@ public:
     vitastorkv_dbw_t *db = NULL;
     kv_fs_state_t *kvfs = NULL;
     block_fs_state_t *blockfs = NULL;
-    std::map<int, nfs_client_t*> rpc_clients;
+    nfs_rdma_context_t* rdma_context = NULL;
+    std::set<nfs_client_t*> rpc_clients;
 
     std::vector<XDR*> xdr_pool;
 
@@ -72,12 +80,20 @@ public:
     void watch_stats();
     void parse_stats(etcd_kv_t & kv);
     void check_default_pool();
+    nfs_client_t* create_client();
     void do_accept(int listen_fd);
     void daemonize();
     void write_pid();
     void mount_fs();
     void check_already_mounted();
     void check_exit();
+
+    nfs_rdma_context_t* create_rdma(const std::string & bind_address, int rdmacm_port,
+        uint32_t max_iodepth, uint32_t max_send_wr, uint64_t rdma_malloc_round_to, uint64_t rdma_max_unused_buffers);
+    void destroy_rdma();
+
+    XDR *get_xdr();
+    void free_xdr(XDR *xdrs);
 };
 
 struct rpc_cur_buffer_t
@@ -101,19 +117,24 @@ struct rpc_free_buffer_t
     unsigned size;
 };
 
+struct nfs_rdma_conn_t;
+
 class nfs_client_t
 {
 public:
     nfs_proxy_t *parent = NULL;
-    int nfs_fd;
-    int epoll_events = 0;
     int refs = 0;
     bool stopped = false;
     std::set<rpc_service_proc_t> proc_table;
+    nfs_rdma_conn_t *rdma_conn = NULL;
+
+    // <TCP>
+    int nfs_fd = -1;
+    int epoll_events = 0;
 
     // Read state
     rpc_cur_buffer_t cur_buffer = { 0 };
-    std::map<uint8_t*, rpc_used_buffer_t> used_buffers;
+    std::map<void*, rpc_used_buffer_t> used_buffers;
     std::vector<rpc_free_buffer_t> free_buffers;
 
     iovec read_iov;
@@ -130,7 +151,16 @@ public:
     void submit_send();
     void handle_send(int result);
     int handle_rpc_message(void *base_buf, void *msg_buf, uint32_t msg_len);
+    // </TCP>
 
+    rpc_op_t *create_rpc_op(XDR *xdrs, void *buffer, rpc_msg *inmsg, rdma_msg *rmsg);
+    int handle_rpc_op(rpc_op_t *rop);
     bool deref();
     void stop();
+    void *malloc_or_rdma(rpc_op_t *rop, size_t size);
+    void free_or_rdma(rpc_op_t *rop, void *buf);
+    void *rdma_malloc(size_t size);
+    void rdma_free(void *buf);
+    void rdma_queue_reply(rpc_op_t *rop);
+    void destroy_rdma_conn();
 };

src/nfs/proto/nfs.x

@@ -168,7 +168,7 @@ struct WRITE3args {
 	offset3     offset;
 	count3      count;
 	stable_how  stable;
-	opaque      data<>;
+	opaque      data<>; /* RDMA DDP-eligible */
 };
 
 typedef opaque writeverf3[NFS3_WRITEVERFSIZE];
@@ -409,7 +409,7 @@ struct READ3resok {
 	post_op_attr   file_attributes;
 	count3         count;
 	bool           eof;
-	opaque         data<>;
+	opaque         data<>; /* RDMA DDP-eligible */
 };
 
 struct READ3resfail {
@@ -514,7 +514,7 @@ typedef string nfspath3<>;
 
 struct symlinkdata3 {
 	sattr3    symlink_attributes;
-	nfspath3  symlink_data;
+	nfspath3  symlink_data; /* RDMA DDP-eligible */
 };
 
 struct SYMLINK3args {
@@ -546,7 +546,7 @@ struct READLINK3args {
 
 struct READLINK3resok {
 	post_op_attr   symlink_attributes;
-	nfspath3       data;
+	nfspath3       data; /* RDMA DDP-eligible */
 };
 
 struct READLINK3resfail {

src/nfs/proto/nfs_xdr.cpp

@@ -272,7 +272,7 @@ xdr_WRITE3args (XDR *xdrs, WRITE3args *objp)
 		 return FALSE;
 	 if (!xdr_stable_how (xdrs, &objp->stable))
 		 return FALSE;
-	 if (!xdr_bytes(xdrs, &objp->data, ~0))
+	 if (!xdr_bytes(xdrs, &objp->data, ~0, true))
 		 return FALSE;
 	return TRUE;
 }
@@ -829,7 +829,7 @@ xdr_READ3resok (XDR *xdrs, READ3resok *objp)
 		 return FALSE;
 	 if (!xdr_bool (xdrs, &objp->eof))
 		 return FALSE;
-	 if (!xdr_bytes(xdrs, &objp->data, ~0))
+	 if (!xdr_bytes(xdrs, &objp->data, ~0, true))
 		 return FALSE;
 	return TRUE;
 }
@@ -1173,10 +1173,10 @@ xdr_PATHCONF3res (XDR *xdrs, PATHCONF3res *objp)
 }
 
 bool_t
-xdr_nfspath3 (XDR *xdrs, nfspath3 *objp)
+xdr_nfspath3 (XDR *xdrs, nfspath3 *objp, bool rdma_chunk)
 {
 	
-	 if (!xdr_string (xdrs, objp, ~0))
+	 if (!xdr_string (xdrs, objp, ~0, rdma_chunk))
 		 return FALSE;
 	return TRUE;
 }
@@ -1187,7 +1187,7 @@ xdr_symlinkdata3 (XDR *xdrs, symlinkdata3 *objp)
 	
 	 if (!xdr_sattr3 (xdrs, &objp->symlink_attributes))
 		 return FALSE;
-	 if (!xdr_nfspath3 (xdrs, &objp->symlink_data))
+	 if (!xdr_nfspath3 (xdrs, &objp->symlink_data, true))
 		 return FALSE;
 	return TRUE;
 }
@@ -1259,7 +1259,7 @@ xdr_READLINK3resok (XDR *xdrs, READLINK3resok *objp)
 	
 	 if (!xdr_post_op_attr (xdrs, &objp->symlink_attributes))
 		 return FALSE;
-	 if (!xdr_nfspath3 (xdrs, &objp->data))
+	 if (!xdr_nfspath3 (xdrs, &objp->data, true))
 		 return FALSE;
 	return TRUE;
 }

src/nfs/proto/nfs_xdr.cpp.diff

@@ -0,0 +1,53 @@
+diff --git a/src/nfs/proto/nfs_xdr.cpp b/src/nfs/proto/nfs_xdr.cpp
+index 87451293..5897e6ad 100644
+--- a/src/nfs/proto/nfs_xdr.cpp
++++ b/src/nfs/proto/nfs_xdr.cpp
+@@ -272,7 +272,7 @@ xdr_WRITE3args (XDR *xdrs, WRITE3args *objp)
+ 		 return FALSE;
+ 	 if (!xdr_stable_how (xdrs, &objp->stable))
+ 		 return FALSE;
+-	 if (!xdr_bytes(xdrs, &objp->data, ~0))
++	 if (!xdr_bytes(xdrs, &objp->data, ~0, true))
+ 		 return FALSE;
+ 	return TRUE;
+ }
+@@ -829,7 +829,7 @@ xdr_READ3resok (XDR *xdrs, READ3resok *objp)
+ 		 return FALSE;
+ 	 if (!xdr_bool (xdrs, &objp->eof))
+ 		 return FALSE;
+-	 if (!xdr_bytes(xdrs, &objp->data, ~0))
++	 if (!xdr_bytes(xdrs, &objp->data, ~0, true))
+ 		 return FALSE;
+ 	return TRUE;
+ }
+@@ -1173,10 +1173,10 @@ xdr_PATHCONF3res (XDR *xdrs, PATHCONF3res *objp)
+ }
+ 
+ bool_t
+-xdr_nfspath3 (XDR *xdrs, nfspath3 *objp)
++xdr_nfspath3 (XDR *xdrs, nfspath3 *objp, bool rdma_chunk)
+ {
+ 	
+-	 if (!xdr_string (xdrs, objp, ~0))
++	 if (!xdr_string (xdrs, objp, ~0, rdma_chunk))
+ 		 return FALSE;
+ 	return TRUE;
+ }
+@@ -1187,7 +1187,7 @@ xdr_symlinkdata3 (XDR *xdrs, symlinkdata3 *objp)
+ 	
+ 	 if (!xdr_sattr3 (xdrs, &objp->symlink_attributes))
+ 		 return FALSE;
+-	 if (!xdr_nfspath3 (xdrs, &objp->symlink_data))
++	 if (!xdr_nfspath3 (xdrs, &objp->symlink_data, true))
+ 		 return FALSE;
+ 	return TRUE;
+ }
+@@ -1259,7 +1259,7 @@ xdr_READLINK3resok (XDR *xdrs, READLINK3resok *objp)
+ 	
+ 	 if (!xdr_post_op_attr (xdrs, &objp->symlink_attributes))
+ 		 return FALSE;
+-	 if (!xdr_nfspath3 (xdrs, &objp->data))
++	 if (!xdr_nfspath3 (xdrs, &objp->data, true))
+ 		 return FALSE;
+ 	return TRUE;
+ }

src/nfs/proto/rpc_impl.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "rpc.h"
+#include "rpc_rdma.h"
 
 struct rpc_op_t;
 
@@ -27,12 +28,16 @@ inline bool operator < (const rpc_service_proc_t & a, const rpc_service_proc_t &
     return a.prog < b.prog || a.prog == b.prog && (a.vers < b.vers || a.vers == b.vers && a.proc < b.proc);
 }
 
+struct rdma_msg;
+
 struct rpc_op_t
 {
     void *client;
-    uint8_t *buffer;
+    void *buffer;
     XDR *xdrs;
     rpc_msg in_msg, out_msg;
+    rdma_msg in_rdma_msg;
+    rpc_rdma_errcode rdma_error;
     void *request;
     void *reply;
     xdrproc_t reply_fn;

src/nfs/proto/rpc_rdma.h

@@ -0,0 +1,144 @@
+/*
+ * Please do not edit this file.
+ * It was generated using rpcgen.
+ */
+
+#ifndef _RPC_RDMA_H_RPCGEN
+#define _RPC_RDMA_H_RPCGEN
+
+#include "xdr_impl.h"
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+struct xdr_rdma_segment {
+	uint32_t handle;
+	uint32_t length;
+	uint64_t offset;
+};
+typedef struct xdr_rdma_segment xdr_rdma_segment;
+
+struct xdr_read_chunk {
+	uint32_t position;
+	struct xdr_rdma_segment target;
+};
+typedef struct xdr_read_chunk xdr_read_chunk;
+
+struct xdr_read_list {
+	struct xdr_read_chunk entry;
+	struct xdr_read_list *next;
+};
+typedef struct xdr_read_list xdr_read_list;
+
+struct xdr_write_chunk {
+	struct {
+		u_int target_len;
+		struct xdr_rdma_segment *target_val;
+	} target;
+};
+typedef struct xdr_write_chunk xdr_write_chunk;
+
+struct xdr_write_list {
+	struct xdr_write_chunk entry;
+	struct xdr_write_list *next;
+};
+typedef struct xdr_write_list xdr_write_list;
+
+struct rpc_rdma_header {
+	struct xdr_read_list *rdma_reads;
+	struct xdr_write_list *rdma_writes;
+	struct xdr_write_chunk *rdma_reply;
+};
+typedef struct rpc_rdma_header rpc_rdma_header;
+
+struct rpc_rdma_header_nomsg {
+	struct xdr_read_list *rdma_reads;
+	struct xdr_write_list *rdma_writes;
+	struct xdr_write_chunk *rdma_reply;
+};
+typedef struct rpc_rdma_header_nomsg rpc_rdma_header_nomsg;
+
+struct rpc_rdma_header_padded {
+	uint32_t rdma_align;
+	uint32_t rdma_thresh;
+	struct xdr_read_list *rdma_reads;
+	struct xdr_write_list *rdma_writes;
+	struct xdr_write_chunk *rdma_reply;
+};
+typedef struct rpc_rdma_header_padded rpc_rdma_header_padded;
+
+enum rpc_rdma_errcode {
+	ERR_VERS = 1,
+	ERR_CHUNK = 2,
+};
+typedef enum rpc_rdma_errcode rpc_rdma_errcode;
+
+struct rpc_rdma_errvers {
+	uint32_t rdma_vers_low;
+	uint32_t rdma_vers_high;
+};
+typedef struct rpc_rdma_errvers rpc_rdma_errvers;
+
+struct rpc_rdma_error {
+	rpc_rdma_errcode err;
+	union {
+		rpc_rdma_errvers range;
+	};
+};
+typedef struct rpc_rdma_error rpc_rdma_error;
+
+enum rdma_proc {
+	RDMA_MSG = 0,
+	RDMA_NOMSG = 1,
+	RDMA_MSGP = 2,
+	RDMA_DONE = 3,
+	RDMA_ERROR = 4,
+};
+typedef enum rdma_proc rdma_proc;
+
+struct rdma_body {
+	rdma_proc proc;
+	union {
+		rpc_rdma_header rdma_msg;
+		rpc_rdma_header_nomsg rdma_nomsg;
+		rpc_rdma_header_padded rdma_msgp;
+		rpc_rdma_error rdma_error;
+	};
+};
+typedef struct rdma_body rdma_body;
+
+struct rdma_msg {
+	uint32_t rdma_xid;
+	uint32_t rdma_vers;
+	uint32_t rdma_credit;
+	rdma_body rdma_body;
+};
+typedef struct rdma_msg rdma_msg;
+
+/* the xdr functions */
+
+
+extern  bool_t xdr_xdr_rdma_segment (XDR *, xdr_rdma_segment*);
+extern  bool_t xdr_xdr_read_chunk (XDR *, xdr_read_chunk*);
+extern  bool_t xdr_xdr_read_list (XDR *, xdr_read_list*);
+extern  bool_t xdr_xdr_write_chunk (XDR *, xdr_write_chunk*);
+extern  bool_t xdr_xdr_write_list (XDR *, xdr_write_list*);
+extern  bool_t xdr_rpc_rdma_header (XDR *, rpc_rdma_header*);
+extern  bool_t xdr_rpc_rdma_header_nomsg (XDR *, rpc_rdma_header_nomsg*);
+extern  bool_t xdr_rpc_rdma_header_padded (XDR *, rpc_rdma_header_padded*);
+extern  bool_t xdr_rpc_rdma_errcode (XDR *, rpc_rdma_errcode*);
+extern  bool_t xdr_rpc_rdma_errvers (XDR *, rpc_rdma_errvers*);
+extern  bool_t xdr_rpc_rdma_error (XDR *, rpc_rdma_error*);
+extern  bool_t xdr_rdma_proc (XDR *, rdma_proc*);
+extern  bool_t xdr_rdma_body (XDR *, rdma_body*);
+extern  bool_t xdr_rdma_msg (XDR *, rdma_msg*);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !_RPC_RDMA_H_RPCGEN */

src/nfs/proto/rpc_rdma.x

@@ -0,0 +1,166 @@
+/* RFC 8166 - Remote Direct Memory Access Transport for Remote Procedure Call Version 1 */
+
+/*
+ * Copyright (c) 2010-2017 IETF Trust and the persons
+ * identified as authors of the code.  All rights reserved.
+ *
+ * The authors of the code are:
+ * B. Callaghan, T. Talpey, and C. Lever
+ *
+ * Redistribution and use in source and binary forms, with
+ * or without modification, are permitted provided that the
+ * following conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ *   copyright notice, this list of conditions and the
+ *   following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the
+ *   following disclaimer in the documentation and/or other
+ *   materials provided with the distribution.
+ *
+ * - Neither the name of Internet Society, IETF or IETF
+ *   Trust, nor the names of specific contributors, may be
+ *   used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS
+ *   AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
+ *   WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ *   FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO
+ *   EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ *   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ *   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *   NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ *   SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ *   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ *   OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ *   IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ *   ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Plain RDMA segment (Section 3.4.3)
+ */
+struct xdr_rdma_segment {
+	uint32_t handle;           /* Registered memory handle */
+	uint32_t length;           /* Length of the chunk in bytes */
+	uint64_t offset;           /* Chunk virtual address or offset */
+};
+
+/*
+ * RDMA read segment (Section 3.4.5)
+ */
+struct xdr_read_chunk {
+	uint32_t position;        /* Position in XDR stream */
+	struct xdr_rdma_segment target;
+};
+
+/*
+ * Read list (Section 4.3.1)
+ */
+struct xdr_read_list {
+	struct xdr_read_chunk entry;
+	struct xdr_read_list  *next;
+};
+
+/*
+ * Write chunk (Section 3.4.6)
+ */
+struct xdr_write_chunk {
+	struct xdr_rdma_segment target<>;
+};
+
+/*
+ * Write list (Section 4.3.2)
+ */
+struct xdr_write_list {
+	struct xdr_write_chunk entry;
+	struct xdr_write_list  *next;
+};
+
+/*
+ * Chunk lists (Section 4.3)
+ */
+struct rpc_rdma_header {
+	struct xdr_read_list   *rdma_reads;
+	struct xdr_write_list  *rdma_writes;
+	struct xdr_write_chunk *rdma_reply;
+	/* rpc body follows */
+};
+
+struct rpc_rdma_header_nomsg {
+	struct xdr_read_list   *rdma_reads;
+	struct xdr_write_list  *rdma_writes;
+	struct xdr_write_chunk *rdma_reply;
+};
+
+/* Not to be used */
+struct rpc_rdma_header_padded {
+	uint32_t               rdma_align;
+	uint32_t               rdma_thresh;
+	struct xdr_read_list   *rdma_reads;
+	struct xdr_write_list  *rdma_writes;
+	struct xdr_write_chunk *rdma_reply;
+	/* rpc body follows */
+};
+
+/*
+ * Error handling (Section 4.5)
+ */
+enum rpc_rdma_errcode {
+	ERR_VERS = 1,       /* Value fixed for all versions */
+	ERR_CHUNK = 2
+};
+
+/* Structure fixed for all versions */
+struct rpc_rdma_errvers {
+	uint32_t rdma_vers_low;
+	uint32_t rdma_vers_high;
+};
+
+union rpc_rdma_error switch (rpc_rdma_errcode err) {
+	case ERR_VERS:
+		rpc_rdma_errvers range;
+	case ERR_CHUNK:
+		void;
+};
+
+/*
+ * Procedures (Section 4.2.4)
+ */
+enum rdma_proc {
+	RDMA_MSG = 0,     /* Value fixed for all versions */
+	RDMA_NOMSG = 1,   /* Value fixed for all versions */
+	RDMA_MSGP = 2,    /* Not to be used */
+	RDMA_DONE = 3,    /* Not to be used */
+	RDMA_ERROR = 4    /* Value fixed for all versions */
+};
+
+/* The position of the proc discriminator field is
+ * fixed for all versions */
+union rdma_body switch (rdma_proc proc) {
+	case RDMA_MSG:
+		rpc_rdma_header rdma_msg;
+	case RDMA_NOMSG:
+		rpc_rdma_header_nomsg rdma_nomsg;
+	case RDMA_MSGP:   /* Not to be used */
+		rpc_rdma_header_padded rdma_msgp;
+	case RDMA_DONE:   /* Not to be used */
+		void;
+	case RDMA_ERROR:
+		rpc_rdma_error rdma_error;
+};
+
+/*
+ * Fixed header fields (Section 4.2)
+ */
+struct rdma_msg {
+	uint32_t  rdma_xid;      /* Position fixed for all versions */
+	uint32_t  rdma_vers;     /* Position fixed for all versions */
+	uint32_t  rdma_credit;   /* Position fixed for all versions */
+	rdma_body rdma_body;
+};

src/nfs/proto/rpc_rdma_xdr.cpp

@@ -0,0 +1,200 @@
+/*
+ * Please do not edit this file.
+ * It was generated using rpcgen.
+ */
+
+#include "rpc_rdma.h"
+#include "xdr_impl_inline.h"
+
+bool_t
+xdr_xdr_rdma_segment (XDR *xdrs, xdr_rdma_segment *objp)
+{
+	
+	 if (!xdr_uint32_t (xdrs, &objp->handle))
+		 return FALSE;
+	 if (!xdr_uint32_t (xdrs, &objp->length))
+		 return FALSE;
+	 if (!xdr_uint64_t (xdrs, &objp->offset))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_xdr_read_chunk (XDR *xdrs, xdr_read_chunk *objp)
+{
+	
+	 if (!xdr_uint32_t (xdrs, &objp->position))
+		 return FALSE;
+	 if (!xdr_xdr_rdma_segment (xdrs, &objp->target))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_xdr_read_list (XDR *xdrs, xdr_read_list *objp)
+{
+	
+	 if (!xdr_xdr_read_chunk (xdrs, &objp->entry))
+		 return FALSE;
+	 if (!xdr_pointer (xdrs, (char **)&objp->next, sizeof (xdr_read_list), (xdrproc_t) xdr_xdr_read_list))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_xdr_write_chunk (XDR *xdrs, xdr_write_chunk *objp)
+{
+	
+	 if (!xdr_array (xdrs, (char **)&objp->target.target_val, (u_int *) &objp->target.target_len, ~0,
+		sizeof (xdr_rdma_segment), (xdrproc_t) xdr_xdr_rdma_segment))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_xdr_write_list (XDR *xdrs, xdr_write_list *objp)
+{
+	
+	 if (!xdr_xdr_write_chunk (xdrs, &objp->entry))
+		 return FALSE;
+	 if (!xdr_pointer (xdrs, (char **)&objp->next, sizeof (xdr_write_list), (xdrproc_t) xdr_xdr_write_list))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_rpc_rdma_header (XDR *xdrs, rpc_rdma_header *objp)
+{
+	
+	 if (!xdr_pointer (xdrs, (char **)&objp->rdma_reads, sizeof (xdr_read_list), (xdrproc_t) xdr_xdr_read_list))
+		 return FALSE;
+	 if (!xdr_pointer (xdrs, (char **)&objp->rdma_writes, sizeof (xdr_write_list), (xdrproc_t) xdr_xdr_write_list))
+		 return FALSE;
+	 if (!xdr_pointer (xdrs, (char **)&objp->rdma_reply, sizeof (xdr_write_chunk), (xdrproc_t) xdr_xdr_write_chunk))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_rpc_rdma_header_nomsg (XDR *xdrs, rpc_rdma_header_nomsg *objp)
+{
+	
+	 if (!xdr_pointer (xdrs, (char **)&objp->rdma_reads, sizeof (xdr_read_list), (xdrproc_t) xdr_xdr_read_list))
+		 return FALSE;
+	 if (!xdr_pointer (xdrs, (char **)&objp->rdma_writes, sizeof (xdr_write_list), (xdrproc_t) xdr_xdr_write_list))
+		 return FALSE;
+	 if (!xdr_pointer (xdrs, (char **)&objp->rdma_reply, sizeof (xdr_write_chunk), (xdrproc_t) xdr_xdr_write_chunk))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_rpc_rdma_header_padded (XDR *xdrs, rpc_rdma_header_padded *objp)
+{
+	
+	 if (!xdr_uint32_t (xdrs, &objp->rdma_align))
+		 return FALSE;
+	 if (!xdr_uint32_t (xdrs, &objp->rdma_thresh))
+		 return FALSE;
+	 if (!xdr_pointer (xdrs, (char **)&objp->rdma_reads, sizeof (xdr_read_list), (xdrproc_t) xdr_xdr_read_list))
+		 return FALSE;
+	 if (!xdr_pointer (xdrs, (char **)&objp->rdma_writes, sizeof (xdr_write_list), (xdrproc_t) xdr_xdr_write_list))
+		 return FALSE;
+	 if (!xdr_pointer (xdrs, (char **)&objp->rdma_reply, sizeof (xdr_write_chunk), (xdrproc_t) xdr_xdr_write_chunk))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_rpc_rdma_errcode (XDR *xdrs, rpc_rdma_errcode *objp)
+{
+	
+	 if (!xdr_enum (xdrs, (enum_t *) objp))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_rpc_rdma_errvers (XDR *xdrs, rpc_rdma_errvers *objp)
+{
+	
+	 if (!xdr_uint32_t (xdrs, &objp->rdma_vers_low))
+		 return FALSE;
+	 if (!xdr_uint32_t (xdrs, &objp->rdma_vers_high))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_rpc_rdma_error (XDR *xdrs, rpc_rdma_error *objp)
+{
+	
+	 if (!xdr_rpc_rdma_errcode (xdrs, &objp->err))
+		 return FALSE;
+	switch (objp->err) {
+	case ERR_VERS:
+		 if (!xdr_rpc_rdma_errvers (xdrs, &objp->range))
+			 return FALSE;
+		break;
+	case ERR_CHUNK:
+		break;
+	default:
+		return FALSE;
+	}
+	return TRUE;
+}
+
+bool_t
+xdr_rdma_proc (XDR *xdrs, rdma_proc *objp)
+{
+	
+	 if (!xdr_enum (xdrs, (enum_t *) objp))
+		 return FALSE;
+	return TRUE;
+}
+
+bool_t
+xdr_rdma_body (XDR *xdrs, rdma_body *objp)
+{
+	
+	 if (!xdr_rdma_proc (xdrs, &objp->proc))
+		 return FALSE;
+	switch (objp->proc) {
+	case RDMA_MSG:
+		 if (!xdr_rpc_rdma_header (xdrs, &objp->rdma_msg))
+			 return FALSE;
+		break;
+	case RDMA_NOMSG:
+		 if (!xdr_rpc_rdma_header_nomsg (xdrs, &objp->rdma_nomsg))
+			 return FALSE;
+		break;
+	case RDMA_MSGP:
+		 if (!xdr_rpc_rdma_header_padded (xdrs, &objp->rdma_msgp))
+			 return FALSE;
+		break;
+	case RDMA_DONE:
+		break;
+	case RDMA_ERROR:
+		 if (!xdr_rpc_rdma_error (xdrs, &objp->rdma_error))
+			 return FALSE;
+		break;
+	default:
+		return FALSE;
+	}
+	return TRUE;
+}
+
+bool_t
+xdr_rdma_msg (XDR *xdrs, rdma_msg *objp)
+{
+	
+	 if (!xdr_uint32_t (xdrs, &objp->rdma_xid))
+		 return FALSE;
+	 if (!xdr_uint32_t (xdrs, &objp->rdma_vers))
+		 return FALSE;
+	 if (!xdr_uint32_t (xdrs, &objp->rdma_credit))
+		 return FALSE;
+	 if (!xdr_rdma_body (xdrs, &objp->rdma_body))
+		 return FALSE;
+	return TRUE;
+}

src/nfs/proto/run-rpcgen.sh

@@ -46,3 +46,5 @@ run_rpcgen() {
 run_rpcgen nfs
 run_rpcgen rpc
 run_rpcgen portmap
+run_rpcgen rpc_rdma
+patch nfs_xdr.cpp < nfs_xdr.cpp.diff

src/nfs/proto/xdr_impl.cpp

@@ -16,6 +16,22 @@ void xdr_destroy(XDR* xdrs)
     delete xdrs;
 }
 
+void xdr_set_rdma(XDR *xdrs)
+{
+    xdrs->rdma = true;
+}
+
+void xdr_set_rdma_chunk(XDR *xdrs, void *chunk)
+{
+    assert(!xdrs->rdma_chunk || !chunk);
+    xdrs->rdma_chunk = chunk;
+}
+
+void* xdr_get_rdma_chunk(XDR *xdrs)
+{
+    return xdrs->rdma_chunk;
+}
+
 void xdr_reset(XDR *xdrs)
 {
     for (auto buf: xdrs->allocs)
@@ -23,6 +39,9 @@ void xdr_reset(XDR *xdrs)
         free(buf);
     }
     xdrs->buf = NULL;
+    xdrs->rdma = false;
+    xdrs->rdma_chunk = NULL;
+    xdrs->rdma_chunk_used = false;
     xdrs->avail = 0;
     xdrs->allocs.resize(0);
     xdrs->in_linked_list.resize(0);
@@ -45,6 +64,20 @@ int xdr_encode(XDR *xdrs, xdrproc_t fn, void *data)
     return fn(xdrs, data);
 }
 
+size_t xdr_encode_get_size(XDR *xdrs)
+{
+    size_t len = 0;
+    for (auto & buf: xdrs->buf_list)
+    {
+        len += buf.iov_len;
+    }
+    if (xdrs->last_end < xdrs->cur_out.size())
+    {
+        len += xdrs->cur_out.size() - xdrs->last_end;
+    }
+    return len;
+}
+
 void xdr_encode_finish(XDR *xdrs, iovec **iov_list, unsigned *iov_count)
 {
     if (xdrs->last_end < xdrs->cur_out.size())
@@ -83,6 +116,18 @@ void xdr_add_malloc(XDR *xdrs, void *buf)
     xdrs->allocs.push_back(buf);
 }
 
+void xdr_del_malloc(XDR *xdrs, void *buf)
+{
+    for (int i = 0; i < xdrs->allocs.size(); i++)
+    {
+        if (xdrs->allocs[i] == buf)
+        {
+            xdrs->allocs.erase(xdrs->allocs.begin()+i);
+            break;
+        }
+    }
+}
+
 xdr_string_t xdr_copy_string(XDR *xdrs, const std::string & str)
 {
     char *cp = (char*)malloc_or_die(str.size()+1);

src/nfs/proto/xdr_impl.h

@@ -55,6 +55,15 @@ void xdr_destroy(XDR* xdrs);
 // Free resources from any previous xdr_decode/xdr_encode calls
 void xdr_reset(XDR *xdrs);
 
+// Mark XDR as used for RDMA
+void xdr_set_rdma(XDR *xdrs);
+
+// Set (single) RDMA chunk buffer for this xdr before decoding an RDMA message
+void xdr_set_rdma_chunk(XDR *xdrs, void *chunk);
+
+// Get the current RDMA chunk buffer
+void* xdr_get_rdma_chunk(XDR *xdrs);
+
 // Try to decode <size> bytes from buffer <buf> using <fn>
 // Result may contain memory allocations that will be valid until the next call to xdr_{reset,destroy,decode,encode}
 int xdr_decode(XDR *xdrs, void *buf, unsigned size, xdrproc_t fn, void *data);
@@ -64,6 +73,9 @@ int xdr_decode(XDR *xdrs, void *buf, unsigned size, xdrproc_t fn, void *data);
 // May be called multiple times to encode multiple parts of the same message
 int xdr_encode(XDR *xdrs, xdrproc_t fn, void *data);
 
+// Get current size of encoded data in <xdrs>
+size_t xdr_encode_get_size(XDR *xdrs);
+
 // Get the result of previous xdr_encodes as a list of <struct iovec>'s
 // in <iov_list> (start) and <iov_count> (count).
 // The resulting iov_list is valid until the next call to xdr_{reset,destroy}.
@@ -74,6 +86,9 @@ void xdr_encode_finish(XDR *xdrs, iovec **iov_list, unsigned *iov_count);
 // Remember an allocated buffer to free it later on xdr_reset() or xdr_destroy()
 void xdr_add_malloc(XDR *xdrs, void *buf);
 
+// Remove an allocated buffer from XDR
+void xdr_del_malloc(XDR *xdrs, void *buf);
+
 xdr_string_t xdr_copy_string(XDR *xdrs, const std::string & str);
 
 xdr_string_t xdr_copy_string(XDR *xdrs, const char *str);

src/nfs/proto/xdr_impl_inline.h

@@ -28,6 +28,19 @@
 // RPC over TCP:
 //
 // BE 32bit length, then rpc_msg, then the procedure message itself
+//
+// RPC over RDMA:
+// RFC 8166 - Remote Direct Memory Access Transport for Remote Procedure Call Version 1
+// RFC 8267 - Network File System (NFS) Upper-Layer Binding to RPC-over-RDMA Version 1
+// RFC 8797 - Remote Direct Memory Access - Connection Manager (RDMA-CM) Private Data for RPC-over-RDMA Version 1
+// message is received in an RDMA Receive operation
+// message: list of read chunks, list of write chunks, optional reply write chunk, then actual RPC body if present
+// read chunk: BE 32bit position, BE 32bit registered memory key, BE 32bit length, BE 64bit offset
+// write chunk: BE 32bit registered memory key, BE 32bit length, BE 64bit offset
+// in reality for NFS 3.0: only 1 read chunk in write3 and symlink3, only 1 write chunk in read3 and readlink3
+// read chunk is read by the server using RDMA Read from the client memory after receiving RPC request
+// write chunk is pushed by the server using RDMA Write to the client memory before sending RPC reply
+// connection is established using RDMA-CM at default port 20049
 
 #pragma once
 
@@ -35,6 +48,7 @@
 
 #include <string.h>
 #include <endian.h>
+#include <assert.h>
 #include <vector>
 
 #include "malloc_or_die.h"
@@ -61,6 +75,9 @@ struct xdr_linked_list_t
 struct XDR
 {
     int x_op;
+    bool rdma = false;
+    void *rdma_chunk = NULL;
+    bool rdma_chunk_used = false;
 
     // For decoding:
     uint8_t *buf = NULL;
@@ -106,13 +123,22 @@ inline int xdr_opaque(XDR *xdrs, void *data, uint32_t len)
     return 1;
 }
 
-inline int xdr_bytes(XDR *xdrs, xdr_string_t *data, uint32_t maxlen)
+inline int xdr_bytes(XDR *xdrs, xdr_string_t *data, uint32_t maxlen, bool rdma_chunk = false)
 {
     if (xdrs->x_op == XDR_DECODE)
     {
         if (xdrs->avail < 4)
             return 0;
         uint32_t len = be32toh(*((uint32_t*)xdrs->buf));
+        if (rdma_chunk && xdrs->rdma && xdrs->rdma_chunk)
+        {
+            // Take (only a single) RDMA chunk from xdrs->rdma_chunk while decoding
+            assert(!xdrs->rdma_chunk_used);
+            xdrs->rdma_chunk_used = true;
+            data->data = (char*)xdrs->rdma_chunk;
+            data->size = len;
+            return 1;
+        }
         uint32_t padded = len_pad4(len);
         if (xdrs->avail < 4+padded)
             return 0;
@@ -123,7 +149,8 @@ inline int xdr_bytes(XDR *xdrs, xdr_string_t *data, uint32_t maxlen)
     }
     else
     {
-        if (data->size < XDR_COPY_LENGTH)
+        // Always encode RDMA chunks as separate iovecs
+        if (data->size < XDR_COPY_LENGTH && (!rdma_chunk || !xdrs->rdma))
         {
             unsigned old = xdrs->cur_out.size();
             xdrs->cur_out.resize(old + 4+data->size);
@@ -146,8 +173,9 @@ inline int xdr_bytes(XDR *xdrs, xdr_string_t *data, uint32_t maxlen)
                 .iov_len = data->size,
             });
         }
-        if (data->size & 3)
+        if ((data->size & 3) && (!rdma_chunk || !xdrs->rdma))
         {
+            // No padding for RDMA chunks
             int pad = 4-(data->size & 3);
             unsigned old = xdrs->cur_out.size();
             xdrs->cur_out.resize(old+pad);
@@ -158,9 +186,9 @@ inline int xdr_bytes(XDR *xdrs, xdr_string_t *data, uint32_t maxlen)
     return 1;
 }
 
-inline int xdr_string(XDR *xdrs, xdr_string_t *data, uint32_t maxlen)
+inline int xdr_string(XDR *xdrs, xdr_string_t *data, uint32_t maxlen, bool rdma_chunk = false)
 {
-    return xdr_bytes(xdrs, data, maxlen);
+    return xdr_bytes(xdrs, data, maxlen, rdma_chunk);
 }
 
 inline int xdr_u_int(XDR *xdrs, void *data)
@@ -182,6 +210,11 @@ inline int xdr_u_int(XDR *xdrs, void *data)
     return 1;
 }
 
+inline int xdr_uint32_t(XDR *xdrs, void *data)
+{
+    return xdr_u_int(xdrs, data);
+}
+
 inline int xdr_enum(XDR *xdrs, void *data)
 {
     return xdr_u_int(xdrs, data);
@@ -259,6 +292,7 @@ inline int xdr_pointer(XDR *xdrs, char **data, unsigned entry_size, xdrproc_t en
                     *(void**)((uint8_t*)ll.base + i*ll.entry_size + ll.link_offset) =
                         (uint8_t*)ll.base + (i+1)*ll.entry_size;
                 }
+                *data = (char*)ll.base;
                 xdrs->allocs.push_back(ll.base);
                 xdrs->in_linked_list.pop_back();
             }

src/nfs/rdma_alloc.cpp

@@ -0,0 +1,216 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 (see README.md for details)
+//
+// Simple & stupid RDMA-enabled memory allocator (allocates buffers within ibv_mr's)
+
+#include <stdio.h>
+#include <assert.h>
+#include <map>
+#include <set>
+#include "rdma_alloc.h"
+#include "malloc_or_die.h"
+
+struct rdma_region_t
+{
+    void *buf = NULL;
+    size_t len = 0;
+    ibv_mr *mr = NULL;
+};
+
+struct rdma_frag_t
+{
+    rdma_region_t *rgn = NULL;
+    size_t len = 0;
+    bool is_free = false;
+};
+
+struct rdma_free_t
+{
+    size_t len = 0;
+    void *buf = NULL;
+};
+
+inline bool operator < (const rdma_free_t &a, const rdma_free_t &b)
+{
+    return a.len < b.len || a.len == b.len && a.buf < b.buf;
+}
+
+struct rdma_allocator_t
+{
+    size_t rdma_alloc_size = 1048576;
+    size_t rdma_max_unused = 500*1048576;
+    int rdma_access = IBV_ACCESS_LOCAL_WRITE;
+    ibv_pd *pd = NULL;
+
+    std::set<rdma_region_t*> regions;
+    std::map<void*, rdma_frag_t> frags;
+    std::set<rdma_free_t> freelist;
+    size_t freebuffers = 0;
+};
+
+rdma_allocator_t *rdma_malloc_create(ibv_pd *pd, size_t rdma_alloc_size, size_t rdma_max_unused, int rdma_access)
+{
+    rdma_allocator_t *self = new rdma_allocator_t();
+    self->pd = pd;
+    self->rdma_alloc_size = rdma_alloc_size ? rdma_alloc_size : 1048576;
+    self->rdma_max_unused = rdma_max_unused ? rdma_max_unused : 500*1048576;
+    self->rdma_access = rdma_access;
+    return self;
+}
+
+static void rdma_malloc_free_unused_buffers(rdma_allocator_t *self, size_t max_unused, bool force)
+{
+    auto free_it = self->freelist.end();
+    if (free_it == self->freelist.begin())
+        return;
+    free_it--;
+    do
+    {
+        auto frag_it = self->frags.find(free_it->buf);
+        assert(frag_it != self->frags.end());
+        if (frag_it->second.len != frag_it->second.rgn->len)
+        {
+            if (force)
+            {
+                fprintf(stderr, "BUG: Attempt to destroy RDMA allocator while buffers are not freed yet\n");
+                abort();
+            }
+            break;
+        }
+        self->freebuffers -= frag_it->second.rgn->len;
+        ibv_dereg_mr(frag_it->second.rgn->mr);
+        free(frag_it->second.rgn);
+        self->regions.erase(frag_it->second.rgn);
+        self->frags.erase(frag_it);
+        if (free_it == self->freelist.begin())
+        {
+            self->freelist.erase(free_it);
+            break;
+        }
+        self->freelist.erase(free_it--);
+    } while (self->freebuffers > max_unused);
+}
+
+void rdma_malloc_destroy(rdma_allocator_t *self)
+{
+    rdma_malloc_free_unused_buffers(self, 0, true);
+    assert(!self->freebuffers);
+    assert(!self->regions.size());
+    assert(!self->frags.size());
+    assert(!self->freelist.size());
+    delete self;
+}
+
+void *rdma_malloc_alloc(rdma_allocator_t *self, size_t size)
+{
+    auto it = self->freelist.lower_bound((rdma_free_t){ .len = size });
+    if (it == self->freelist.end())
+    {
+        // round size up to rdma_malloc_size (1 MB)
+        size_t alloc_size = ((size + self->rdma_alloc_size - 1) / self->rdma_alloc_size) * self->rdma_alloc_size;
+        rdma_region_t *r = (rdma_region_t*)malloc_or_die(alloc_size + sizeof(rdma_region_t));
+        r->buf = r+1;
+        r->len = alloc_size;
+        r->mr = ibv_reg_mr(self->pd, r->buf, r->len, self->rdma_access);
+        if (!r->mr)
+        {
+            fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
+            exit(1);
+        }
+        self->regions.insert(r);
+        self->frags[r->buf] = (rdma_frag_t){ .rgn = r, .len = alloc_size, .is_free = true };
+        it = self->freelist.insert((rdma_free_t){ .len = alloc_size, .buf = r->buf }).first;
+        self->freebuffers += alloc_size;
+    }
+    void *ptr = it->buf;
+    auto & frag = self->frags.at(ptr);
+    self->freelist.erase(it);
+    assert(frag.len >= size && frag.is_free);
+    if (frag.len == frag.rgn->len)
+    {
+        self->freebuffers -= frag.rgn->len;
+    }
+    if (frag.len == size)
+    {
+        frag.is_free = false;
+    }
+    else
+    {
+        frag.len -= size;
+        ptr = (uint8_t*)ptr + frag.len;
+        self->freelist.insert((rdma_free_t){ .len = frag.len, .buf = frag.rgn->buf });
+        self->frags[ptr] = (rdma_frag_t){ .rgn = frag.rgn, .len = size, .is_free = false };
+    }
+    return ptr;
+}
+
+void rdma_malloc_free(rdma_allocator_t *self, void *buf)
+{
+    auto frag_it = self->frags.find(buf);
+    if (frag_it == self->frags.end())
+    {
+        fprintf(stderr, "BUG: Attempt to double-free RDMA buffer fragment 0x%jx\n", (size_t)buf);
+        return;
+    }
+    auto prev_it = frag_it, next_it = frag_it;
+    if (frag_it != self->frags.begin())
+        prev_it--;
+    next_it++;
+    bool merge_back = prev_it != frag_it &&
+        prev_it->second.is_free &&
+        prev_it->second.rgn == frag_it->second.rgn &&
+        (uint8_t*)prev_it->first+prev_it->second.len == frag_it->first;
+    bool merge_next = next_it != self->frags.end() &&
+        next_it->second.is_free &&
+        next_it->second.rgn == frag_it->second.rgn &&
+        next_it->first == (uint8_t*)frag_it->first+frag_it->second.len;
+    if (merge_back && merge_next)
+    {
+        prev_it->second.len += frag_it->second.len + next_it->second.len;
+        self->freelist.erase((rdma_free_t){ .len = next_it->second.len, .buf = next_it->first });
+        self->frags.erase(next_it);
+        self->frags.erase(frag_it);
+        frag_it = prev_it;
+    }
+    else if (merge_back)
+    {
+        prev_it->second.len += frag_it->second.len;
+        self->frags.erase(frag_it);
+        frag_it = prev_it;
+    }
+    else if (merge_next)
+    {
+        frag_it->second.is_free = true;
+        frag_it->second.len += next_it->second.len;
+        self->freelist.erase((rdma_free_t){ .len = next_it->second.len, .buf = next_it->first });
+        self->frags.erase(next_it);
+    }
+    else
+    {
+        frag_it->second.is_free = true;
+        self->freelist.insert((rdma_free_t){ .len = frag_it->second.len, .buf = frag_it->first });
+    }
+    assert(frag_it->second.len <= frag_it->second.rgn->len);
+    if (frag_it->second.len == frag_it->second.rgn->len)
+    {
+        // The whole buffer is freed
+        self->freebuffers += frag_it->second.rgn->len;
+        if (self->freebuffers > self->rdma_max_unused)
+        {
+            rdma_malloc_free_unused_buffers(self, self->rdma_max_unused, false);
+        }
+    }
+}
+
+uint32_t rdma_malloc_get_lkey(rdma_allocator_t *self, void *buf)
+{
+    auto frag_it = self->frags.upper_bound(buf);
+    if (frag_it != self->frags.begin())
+    {
+        frag_it--;
+        if ((uint8_t*)frag_it->first + frag_it->second.len > buf)
+            return frag_it->second.rgn->mr->lkey;
+    }
+    fprintf(stderr, "BUG: Attempt to use an unknown RDMA buffer fragment 0x%zx\n", (size_t)buf);
+    abort();
+}

src/nfs/rdma_alloc.h

@@ -0,0 +1,17 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 (see README.md for details)
+//
+// Simple & stupid RDMA-enabled memory allocator (allocates buffers within ibv_mr's)
+
+#pragma once
+
+#include <infiniband/verbs.h>
+#include <stdint.h>
+
+struct rdma_allocator_t;
+
+rdma_allocator_t *rdma_malloc_create(ibv_pd *pd, size_t rdma_alloc_size, size_t rdma_max_unused, int rdma_access);
+void rdma_malloc_destroy(rdma_allocator_t *self);
+void *rdma_malloc_alloc(rdma_allocator_t *self, size_t size);
+void rdma_malloc_free(rdma_allocator_t *self, void *buf);
+uint32_t rdma_malloc_get_lkey(rdma_allocator_t *self, void *buf);

src/osd/osd.cpp

@@ -535,10 +535,12 @@ void osd_t::print_stats()
 
 void osd_t::print_slow()
 {
-    bool has_slow = false;
+    cur_slow_op_primary = 0;
+    cur_slow_op_secondary = 0;
     char alloc[1024];
     timespec now;
     clock_gettime(CLOCK_REALTIME, &now);
+    // FIXME: Also track slow local blockstore ops and recovery/flush/scrub ops
     for (auto & kv: msgr.clients)
     {
         for (auto op: kv.second->received_ops)
@@ -608,6 +610,7 @@ void osd_t::print_slow()
                     op->req.hdr.opcode == OSD_OP_SEC_STABILIZE || op->req.hdr.opcode == OSD_OP_SEC_ROLLBACK ||
                     op->req.hdr.opcode == OSD_OP_SEC_READ_BMP)
                 {
+                    cur_slow_op_secondary++;
                     bufprintf(" state=%d", op->bs_op ? PRIV(op->bs_op)->op_state : -1);
                     int wait_for = op->bs_op ? PRIV(op->bs_op)->wait_for : 0;
                     if (wait_for)
@@ -618,15 +621,19 @@ void osd_t::print_slow()
                 else if (op->req.hdr.opcode == OSD_OP_READ || op->req.hdr.opcode == OSD_OP_WRITE ||
                     op->req.hdr.opcode == OSD_OP_SYNC || op->req.hdr.opcode == OSD_OP_DELETE)
                 {
+                    cur_slow_op_primary++;
                     bufprintf(" state=%d", !op->op_data ? -1 : op->op_data->st);
                 }
+                else
+                {
+                    cur_slow_op_primary++;
+                }
 #undef bufprintf
                 printf("%s\n", alloc);
-                has_slow = true;
             }
         }
     }
-    if (has_slow && bs)
+    if ((cur_slow_op_primary+cur_slow_op_secondary) > 0 && bs)
     {
         bs->dump_diagnostics();
     }

src/osd/osd.h

@@ -150,7 +150,9 @@ class osd_t
     bool pg_config_applied = false;
     bool etcd_reporting_pg_state = false;
     bool etcd_reporting_stats = false;
-    int autosync_timer_id = -1, print_stats_timer_id = -1, slow_log_timer_id = -1;
+    int print_stats_timer_id = -1, slow_log_timer_id = -1;
+    uint64_t cur_slow_op_primary = 0;
+    uint64_t cur_slow_op_secondary = 0;
 
     // peers and PGs
 
@@ -168,6 +170,8 @@ class osd_t
     object_id recovery_last_oid;
     int recovery_pg_done = 0, recovery_done = 0;
     osd_op_t *autosync_op = NULL;
+    int autosync_copies_to_delete = 0;
+    int autosync_timer_id = -1;
 
     // Scrubbing
     uint64_t scrub_nearest_ts = 0;
@@ -222,6 +226,7 @@ class osd_t
     void parse_config(bool init);
     void init_cluster();
     void on_change_osd_state_hook(osd_num_t peer_osd);
+    void on_change_backfillfull_hook(pool_id_t pool_id);
     void on_change_pg_history_hook(pool_id_t pool_id, pg_num_t pg_num);
     void on_change_etcd_state_hook(std::map<std::string, etcd_kv_t> & changes);
     void on_load_config_hook(json11::Json::object & changes);

src/osd/osd_cluster.cpp

@@ -65,6 +65,7 @@ void osd_t::init_cluster()
         st_cli.tfd = tfd;
         st_cli.log_level = log_level;
         st_cli.on_change_osd_state_hook = [this](osd_num_t peer_osd) { on_change_osd_state_hook(peer_osd); };
+        st_cli.on_change_backfillfull_hook = [this](pool_id_t pool_id) { on_change_backfillfull_hook(pool_id); };
         st_cli.on_change_pg_history_hook = [this](pool_id_t pool_id, pg_num_t pg_num) { on_change_pg_history_hook(pool_id, pg_num); };
         st_cli.on_change_hook = [this](std::map<std::string, etcd_kv_t> & changes) { on_change_etcd_state_hook(changes); };
         st_cli.on_load_config_hook = [this](json11::Json::object & cfg) { on_load_config_hook(cfg); };
@@ -201,6 +202,14 @@ json11::Json osd_t::get_statistics()
     st["immediate_commit"] = immediate_commit == IMMEDIATE_ALL ? "all" : (immediate_commit == IMMEDIATE_SMALL ? "small" : "none");
     st["host"] = self_state["host"];
     st["version"] = VITASTOR_VERSION;
+    if (cur_slow_op_primary > 0)
+    {
+        st["slow_ops_primary"] = cur_slow_op_primary;
+    }
+    if (cur_slow_op_secondary > 0)
+    {
+        st["slow_ops_secondary"] = cur_slow_op_secondary;
+    }
     json11::Json::object op_stats, subop_stats;
     for (int i = OSD_OP_MIN; i <= OSD_OP_MAX; i++)
     {
@@ -406,6 +415,14 @@ void osd_t::on_change_osd_state_hook(osd_num_t peer_osd)
     }
 }
 
+void osd_t::on_change_backfillfull_hook(pool_id_t pool_id)
+{
+    if (!(peering_state & (OSD_RECOVERING | OSD_FLUSHING_PGS)))
+    {
+        peering_state = peering_state | OSD_RECOVERING;
+    }
+}
+
 void osd_t::on_change_etcd_state_hook(std::map<std::string, etcd_kv_t> & changes)
 {
     if (changes.find(st_cli.etcd_prefix+"/config/global") != changes.end())
@@ -1007,9 +1024,6 @@ void osd_t::report_pg_states()
                 rpgnames += (rpgnames.size() ? ", " : "")+std::to_string(pp.pool_pg_num.pool_id)+"/"+std::to_string(pp.pool_pg_num.pg_num);
             }
             printf("Error reporting PG %s states, will repeat the attempt: %s\n", rpgnames.c_str(), err.c_str());
-        }
-        if (!data["succeeded"].bool_value())
-        {
             // One of PG state updates failed, put dirty flags back
             for (auto pp: reporting_pgs)
             {

src/osd/osd_flush.cpp

@@ -13,10 +13,11 @@ void osd_t::submit_pg_flush_ops(pg_t & pg)
     bool first = true;
     while (it != pg.flush_actions.end())
     {
-        if (!first && (it->first.oid.inode != prev_it->first.oid.inode ||
-            (it->first.oid.stripe & ~STRIPE_MASK) != (prev_it->first.oid.stripe & ~STRIPE_MASK)) &&
-            fb->rollback_lists[it->first.osd_num].size() >= FLUSH_BATCH ||
-            fb->stable_lists[it->first.osd_num].size() >= FLUSH_BATCH)
+        if (!first &&
+            (it->first.oid.inode != prev_it->first.oid.inode ||
+                (it->first.oid.stripe & ~STRIPE_MASK) != (prev_it->first.oid.stripe & ~STRIPE_MASK)) &&
+            (fb->rollback_lists[it->first.osd_num].size() >= FLUSH_BATCH ||
+                fb->stable_lists[it->first.osd_num].size() >= FLUSH_BATCH))
         {
             // Stop only at the object boundary
             break;
@@ -75,6 +76,7 @@ void osd_t::handle_flush_op(bool rollback, pool_id_t pool_id, pg_num_t pg_num, p
         // Throw the result away
         return;
     }
+    fb->flush_done++;
     if (retval != 0)
     {
         if (peer_osd == this->osd_num)
@@ -92,12 +94,11 @@ void osd_t::handle_flush_op(bool rollback, pool_id_t pool_id, pg_num_t pg_num, p
             auto fd_it = msgr.osd_peer_fds.find(peer_osd);
             if (fd_it != msgr.osd_peer_fds.end())
             {
+                // Will repeer/stop this PG
                 msgr.stop_client(fd_it->second);
             }
-            return;
         }
     }
-    fb->flush_done++;
     if (fb->flush_done == fb->flush_ops)
     {
         // This flush batch is done
@@ -251,10 +252,18 @@ bool osd_t::pick_next_recovery(osd_recovery_op_t &op)
             auto mask = recovery_last_degraded ? (PG_ACTIVE | PG_HAS_DEGRADED) : (PG_ACTIVE | PG_DEGRADED | PG_HAS_MISPLACED);
             auto check = recovery_last_degraded ? (PG_ACTIVE | PG_HAS_DEGRADED) : (PG_ACTIVE | PG_HAS_MISPLACED);
             // Restart scanning from the same PG as the last time
+        restart:
             for (auto pg_it = pgs.lower_bound(recovery_last_pg); pg_it != pgs.end(); pg_it++)
             {
                 if ((pg_it->second.state & mask) == check)
                 {
+                    auto pool_it = st_cli.pool_config.find(pg_it->first.pool_id);
+                    if (pool_it != st_cli.pool_config.end() && pool_it->second.backfillfull)
+                    {
+                        // Skip the pool
+                        recovery_last_pg.pool_id++;
+                        goto restart;
+                    }
                     auto & src = recovery_last_degraded ? pg_it->second.degraded_objects : pg_it->second.misplaced_objects;
                     assert(src.size() > 0);
                     // Restart scanning from the next object

src/osd/osd_primary.cpp

@@ -645,6 +645,18 @@ void osd_t::remove_object_from_state(object_id & oid, pg_osd_set_state_t **objec
     {
         throw std::runtime_error("BUG: Invalid object state: "+std::to_string((*object_state)->state));
     }
+    if (changed && immediate_commit != IMMEDIATE_ALL)
+    {
+        // Trigger double automatic sync after changing PG state when we're running with fsyncs.
+        // First autosync commits all written objects and applies copies_to_delete_after_sync;
+        // Second autosync commits all deletions run by the first sync.
+        // Without it, rebalancing in a cluster without load may result in some small amount of
+        // garbage left on "extra" OSDs of the PG, because last deletions are not synced at all.
+        // FIXME: 1000% correct way is to switch PG state only after copies_to_delete_after_sync.
+        // But it's much more complicated.
+        unstable_write_count += autosync_writes;
+        autosync_copies_to_delete = 2;
+    }
     if (changed && report)
     {
         report_pg_state(pg);

src/osd/osd_primary_subops.cpp

@@ -9,6 +9,10 @@ void osd_t::autosync()
 {
     if (immediate_commit != IMMEDIATE_ALL && !autosync_op)
     {
+        if (autosync_copies_to_delete > 0)
+        {
+            autosync_copies_to_delete--;
+        }
         autosync_op = new osd_op_t();
         autosync_op->op_type = OSD_OP_IN;
         autosync_op->peer_fd = SELF_FD;
@@ -29,6 +33,11 @@ void osd_t::autosync()
             }
             delete autosync_op;
             autosync_op = NULL;
+            if (autosync_copies_to_delete > 0)
+            {
+                // Trigger the second "copies_to_delete" autosync
+                autosync();
+            }
         };
         exec_op(autosync_op);
     }

src/osd/osd_primary_sync.cpp

@@ -213,6 +213,15 @@ resume_8:
         {
             goto resume_6;
         }
+        if (immediate_commit == IMMEDIATE_NONE)
+        {
+            // Mark OSDs as dirty because deletions have to be synced too!
+            for (int i = 0; i < op_data->copies_to_delete_count; i++)
+            {
+                auto & chunk = op_data->copies_to_delete[i];
+                this->dirty_osds.insert(chunk.osd_num);
+            }
+        }
     }
     for (int i = 0; i < op_data->dirty_pg_count; i++)
     {
@@ -227,7 +236,7 @@ resume_8:
             start_pg_peering(pg);
         }
     }
-    // FIXME: Free those in the destructor?
+    // FIXME: Free those in the destructor (not here)?
     free(op_data->dirty_pgs);
     op_data->dirty_pgs = NULL;
     op_data->dirty_osds = NULL;