Release 1.10.0

New features:

- Implement basic VitastorFS support in [CSI](https://vitastor.io/docs/installation/kubernetes.html)
- Implement [NFS RDMA](https://vitastor.io/docs/usage/nfs.html#rdma) support
- Pause pool rebalance when monitor detects that it can lead to any OSD becoming full ([osd_backfillfull_ratio](https://vitastor.io/docs/config/monitor.html#osd_backfillfull_ratio))
- Auto-select correct [RDMA device and GID](https://vitastor.io/docs/config/network.html#rdma_device) based on osd_network and RoCEv2 priority
- Report slow ops in OSD stats in etcd and show them in vitastor-cli status

Bug fixes:

- Fix possibly incorrect linked list deserialization in NFS
- Fix possible crash in vitastor-nfs --block READDIR operation
- Map netlink after forking to show correct PID in vitastor-nbd ls
- Simplify and fix create-pool OSD count checks for the case of hosts split into sub-nodes
- Make monitor print "Waiting to become master" just once, not every 5s
- Take out_size from oimg if not specified in vitastor-cli dd
- Do not report OSDs with empty statistics as "full" in status
- Trigger double autosync when switching PG state to prevent leaving garbage in non-immediate_commit clusters
- Fix a lack of connection timeout for etcd websockets in OSD leading to slower etcd failover (~70s instead of ~10s)
- Fix a rare OSD crash during client disconnect
- Fix PGs sometimes sticking until OSD restart in the "has_unclean" state with EC pools
- Fix metadata partition zeroing in vitastor-disk prepare
- Add patches for qemu 9.1 and pve-qemu 9.0 and 9.1
- Fix libvirt 8 patch

.gitea/workflows/buildenv.Dockerfile

@@ -22,7 +22,7 @@ RUN apt-get update
 RUN apt-get -y install etcd qemu-system-x86 qemu-block-extra qemu-utils fio libasan5 \
     liburing1 liburing-dev libgoogle-perftools-dev devscripts libjerasure-dev cmake libibverbs-dev libisal-dev
 RUN apt-get -y build-dep fio qemu=`dpkg -s qemu-system-x86|grep ^Version:|awk '{print $2}'`
-RUN apt-get -y install jq lp-solve sudo nfs-common
+RUN apt-get update && apt-get -y install jq lp-solve sudo nfs-common fdisk parted
 RUN apt-get --download-only source fio qemu=`dpkg -s qemu-system-x86|grep ^Version:|awk '{print $2}'`
 
 RUN set -ex; \

.gitea/workflows/test.yml

@@ -288,6 +288,24 @@ jobs:
           echo ""
         done
 
+  test_create_halfhost:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_create_halfhost.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_failure_domain:
     runs-on: ubuntu-latest
     needs: build
@@ -828,6 +846,42 @@ jobs:
           echo ""
         done
 
+  test_resize:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_resize.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_resize_auto:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_resize_auto.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_snapshot_pool2:
     runs-on: ubuntu-latest
     needs: build

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "1.9.2")
+set(VITASTOR_VERSION "1.10.0")
 
 add_subdirectory(src)

csi/Dockerfile

@@ -22,6 +22,8 @@ RUN apt-get update && \
     (echo "APT::Install-Recommends false;" > /etc/apt/apt.conf) && \
     apt-get update && \
     apt-get install -y e2fsprogs xfsprogs kmod iproute2 \
+        # NFS mount dependencies
+        nfs-common netbase \
         # dependencies of qemu-storage-daemon
         libnuma1 liburing2 libglib2.0-0 libfuse3-3 libaio1 libzstd1 libnettle8 \
         libgmp10 libhogweed6 libp11-kit0 libidn2-0 libunistring2 libtasn1-6 libpcre2-8-0 libffi8 && \

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v1.9.2
+VITASTOR_VERSION ?= v1.10.0
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v1.9.2
+          image: vitalif/vitastor-csi:v1.10.0
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v1.9.2
+          image: vitalif/vitastor-csi:v1.10.0
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/009-storage-class.yaml

@@ -9,8 +9,16 @@ metadata:
 provisioner: csi.vitastor.io
 volumeBindingMode: Immediate
 parameters:
-  etcdVolumePrefix: ""
-  poolId: "1"
+  # CSI driver can create block-based volumes and VitastorFS-based volumes
+  # only VitastorFS-based volumes and raw block volumes (without FS) support ReadWriteMany mode
+  # set this parameter to VitastorFS metadata volume name to use VitastorFS
+  # if unset, block-based volumes will be created
+  vitastorfs: ""
+  # for block-based storage classes, pool ID may be either a string (name) or a number (ID)
+  # for vitastorFS-based storage classes it must be a string - name of the default pool for FS data
+  poolId: "testpool"
+  # volume name prefix for block-based storage classes or NFS subdirectory (including /) for FS-based volumes
+  volumePrefix: ""
   # you can choose other configuration file if you have it in the config map
   # different etcd URLs and prefixes should also be put in the config
   #configPath: "/etc/vitastor/vitastor.conf"

csi/deploy/example-storage-class-fs.yaml

@@ -0,0 +1,25 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  namespace: vitastor-system
+  name: vitastor
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+provisioner: csi.vitastor.io
+volumeBindingMode: Immediate
+parameters:
+  # CSI driver can create block-based volumes and VitastorFS-based volumes
+  # only VitastorFS-based volumes and raw block volumes (without FS) support ReadWriteMany mode
+  # set this parameter to VitastorFS metadata volume name to use VitastorFS
+  # if unset, block-based volumes will be created
+  vitastorfs: "testfs"
+  # for block-based storage classes, pool ID may be either a string (name) or a number (ID)
+  # for vitastorFS-based storage classes it must be a string - name of the default pool for FS data
+  poolId: "testpool"
+  # volume name prefix for block-based storage classes or NFS subdirectory (including /) for FS-based volumes
+  volumePrefix: "k8s/"
+  # you can choose other configuration file if you have it in the config map
+  # different etcd URLs and prefixes should also be put in the config
+  #configPath: "/etc/vitastor/vitastor.conf"
+allowVolumeExpansion: true

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "1.9.2"
+    vitastorCSIDriverVersion = "1.10.0"
 )
 
 // Config struct fills the parameters of request or user input

csi/src/controllerserver.go

@@ -8,11 +8,8 @@ import (
     "encoding/json"
     "fmt"
     "strings"
-    "bytes"
-    "strconv"
     "time"
     "os"
-    "os/exec"
     "io/ioutil"
 
     "github.com/kubernetes-csi/csi-lib-utils/protosanitizer"
@@ -70,9 +67,10 @@ func GetConnectionParams(params map[string]string) (map[string]string, error)
     {
         configPath = "/etc/vitastor/vitastor.conf"
     }
-    else
+    ctxVars["configPath"] = configPath
+    if (params["vitastorfs"] != "")
     {
-        ctxVars["configPath"] = configPath
+        ctxVars["vitastorfs"] = params["vitastorfs"]
     }
     config := make(map[string]interface{})
     configFD, err := os.Open(configPath)
@@ -114,22 +112,6 @@ func GetConnectionParams(params map[string]string) (map[string]string, error)
     return ctxVars, nil
 }
 
-func system(program string, args ...string) ([]byte, []byte, error)
-{
-    klog.Infof("Running "+program+" "+strings.Join(args, " "))
-    c := exec.Command(program, args...)
-    var stdout, stderr bytes.Buffer
-    c.Stdout, c.Stderr = &stdout, &stderr
-    err := c.Run()
-    if (err != nil)
-    {
-        stdoutStr, stderrStr := string(stdout.Bytes()), string(stderr.Bytes())
-        klog.Errorf(program+" "+strings.Join(args, " ")+" failed: %s, status %s\n", stdoutStr+stderrStr, err)
-        return nil, nil, status.Error(codes.Internal, stdoutStr+stderrStr+" (status "+err.Error()+")")
-    }
-    return stdout.Bytes(), stderr.Bytes(), nil
-}
-
 func invokeCLI(ctxVars map[string]string, args []string) ([]byte, error)
 {
     if (ctxVars["configPath"] != "")
@@ -158,33 +140,57 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
         return nil, status.Error(codes.InvalidArgument, "volume capabilities is a required field")
     }
 
-    err := cs.checkCaps(volumeCapabilities)
-    if (err != nil)
-    {
-        return nil, err
-    }
-
-    etcdVolumePrefix := req.Parameters["etcdVolumePrefix"]
-    poolId, _ := strconv.ParseUint(req.Parameters["poolId"], 10, 64)
-    if (poolId == 0)
-    {
-        return nil, status.Error(codes.InvalidArgument, "poolId is missing in storage class configuration")
-    }
-
-    volName := etcdVolumePrefix + req.GetName()
-    volSize := 1 * GB
-    if capRange := req.GetCapacityRange(); capRange != nil
-    {
-        volSize = ((capRange.GetRequiredBytes() + MB - 1) / MB) * MB
-    }
-
     ctxVars, err := GetConnectionParams(req.Parameters)
     if (err != nil)
     {
         return nil, err
     }
 
-    args := []string{ "create", volName, "-s", fmt.Sprintf("%v", volSize), "--pool", fmt.Sprintf("%v", poolId) }
+    err = cs.checkCaps(volumeCapabilities, ctxVars["vitastorfs"] != "")
+    if (err != nil)
+    {
+        return nil, err
+    }
+
+    pool := req.Parameters["poolId"]
+    if (pool == "")
+    {
+        return nil, status.Error(codes.InvalidArgument, "poolId is missing in storage class configuration")
+    }
+    volumePrefix := req.Parameters["volumePrefix"]
+    if (volumePrefix == "")
+    {
+        // Old name
+        volumePrefix = req.Parameters["etcdVolumePrefix"]
+    }
+    volName := volumePrefix + req.GetName()
+    volSize := 1 * GB
+    if capRange := req.GetCapacityRange(); capRange != nil
+    {
+        volSize = ((capRange.GetRequiredBytes() + MB - 1) / MB) * MB
+    }
+
+    if (ctxVars["vitastorfs"] != "")
+    {
+        // Nothing to create, subdirectories are created during mounting
+        // FIXME: It would be cool to support quotas some day and set it here
+        if (req.VolumeContentSource.GetSnapshot() != nil)
+        {
+            return nil, status.Error(codes.InvalidArgument, "VitastorFS doesn't support snapshots")
+        }
+        ctxVars["name"] = volName
+        ctxVars["pool"] = pool
+        volumeIdJson, _ := json.Marshal(ctxVars)
+        return &csi.CreateVolumeResponse{
+            Volume: &csi.Volume{
+                // Ugly, but VolumeContext isn't passed to DeleteVolume :-(
+                VolumeId: string(volumeIdJson),
+                CapacityBytes: volSize,
+            },
+        }, nil
+    }
+
+    args := []string{ "create", volName, "-s", fmt.Sprintf("%v", volSize), "--pool", pool }
 
     // Support creation from snapshot
     var src *csi.VolumeContentSource
@@ -267,6 +273,12 @@ func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol
         return nil, err
     }
 
+    if (ctxVars["vitastorfs"] != "")
+    {
+        // FIXME: Delete FS subdirectory
+        return &csi.DeleteVolumeResponse{}, nil
+    }
+
     _, err = invokeCLI(ctxVars, []string{ "rm", volName })
     if (err != nil)
     {
@@ -301,13 +313,25 @@ func (cs *ControllerServer) ValidateVolumeCapabilities(ctx context.Context, req
     {
         return nil, status.Error(codes.InvalidArgument, "volumeId is nil")
     }
+    volVars := make(map[string]string)
+    err := json.Unmarshal([]byte(volumeID), &volVars)
+    if (err != nil)
+    {
+        return nil, status.Error(codes.Internal, "volume ID not in JSON format")
+    }
+    ctxVars, err := GetConnectionParams(volVars)
+    if (err != nil)
+    {
+        return nil, err
+    }
+
     volumeCapabilities := req.GetVolumeCapabilities()
     if (volumeCapabilities == nil)
     {
         return nil, status.Error(codes.InvalidArgument, "volumeCapabilities is nil")
     }
 
-    err := cs.checkCaps(volumeCapabilities)
+    err = cs.checkCaps(volumeCapabilities, ctxVars["vitastorfs"] != "")
     if (err != nil)
     {
         return nil, err
@@ -320,7 +344,7 @@ func (cs *ControllerServer) ValidateVolumeCapabilities(ctx context.Context, req
     }, nil
 }
 
-func (cs *ControllerServer) checkCaps(volumeCapabilities []*csi.VolumeCapability) error
+func (cs *ControllerServer) checkCaps(volumeCapabilities []*csi.VolumeCapability, fs bool) error
 {
     var volumeCapabilityAccessModes []*csi.VolumeCapability_AccessMode
     for _, mode := range []csi.VolumeCapability_AccessMode_Mode{
@@ -336,6 +360,10 @@ func (cs *ControllerServer) checkCaps(volumeCapabilities []*csi.VolumeCapability
     {
         if (capability.GetBlock() != nil)
         {
+            if (fs)
+            {
+                return status.Errorf(codes.InvalidArgument, "%v not supported with FS-based volumes", capability)
+            }
             for _, mode := range []csi.VolumeCapability_AccessMode_Mode{
                 csi.VolumeCapability_AccessMode_MULTI_NODE_SINGLE_WRITER,
                 csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER,
@@ -346,6 +374,12 @@ func (cs *ControllerServer) checkCaps(volumeCapabilities []*csi.VolumeCapability
         }
     }
 
+    if (fs)
+    {
+        // All access modes including RWX are supported with FS-based volumes
+        return nil
+    }
+
     capabilitySupport := false
     for _, capability := range volumeCapabilities
     {
@@ -360,7 +394,7 @@ func (cs *ControllerServer) checkCaps(volumeCapabilities []*csi.VolumeCapability
 
     if (!capabilitySupport)
     {
-        return status.Errorf(codes.NotFound, "%v not supported", volumeCapabilities)
+        return status.Errorf(codes.InvalidArgument, "%v not supported", volumeCapabilities)
     }
 
     return nil
@@ -452,6 +486,12 @@ func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS
     {
         return nil, status.Error(codes.Internal, "volume ID not in JSON format")
     }
+
+    if (ctxVars["vitastorfs"] != "")
+    {
+        return nil, status.Error(codes.InvalidArgument, "VitastorFS doesn't support snapshots")
+    }
+
     volName := ctxVars["name"]
 
     // Create image using vitastor-cli
@@ -510,6 +550,11 @@ func (cs *ControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteS
         return nil, err
     }
 
+    if (ctxVars["vitastorfs"] != "")
+    {
+        return nil, status.Error(codes.InvalidArgument, "VitastorFS doesn't support snapshots")
+    }
+
     _, err = invokeCLI(ctxVars, []string{ "rm", volName+"@"+snapName })
     if (err != nil)
     {
@@ -541,6 +586,11 @@ func (cs *ControllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnap
         return nil, err
     }
 
+    if (ctxVars["vitastorfs"] != "")
+    {
+        return nil, status.Error(codes.InvalidArgument, "VitastorFS doesn't support snapshots")
+    }
+
     inodeCfg, err := invokeList(ctxVars, volName+"@*", false)
     if (err != nil)
     {
@@ -604,6 +654,16 @@ func (cs *ControllerServer) ControllerExpandVolume(ctx context.Context, req *csi
         return nil, err
     }
 
+    if (ctxVars["vitastorfs"] != "")
+    {
+        // Nothing to change
+        // FIXME: Support quotas and change quota here
+        return &csi.ControllerExpandVolumeResponse{
+            CapacityBytes: req.CapacityRange.RequiredBytes,
+            NodeExpansionRequired: false,
+        }, nil
+    }
+
     inodeCfg, err := invokeList(ctxVars, volName, true)
     if (err != nil)
     {

csi/src/nodeserver.go

@@ -5,11 +5,15 @@ package vitastor
 
 import (
     "context"
+    "crypto/sha1"
+    "encoding/hex"
     "encoding/json"
     "fmt"
     "os"
     "os/exec"
     "path/filepath"
+    "regexp"
+    "strconv"
     "strings"
     "sync"
     "syscall"
@@ -29,13 +33,14 @@ import (
 type NodeServer struct
 {
     *Driver
-    useVduse bool
-    stateDir string
-    mounter mount.Interface
+    useVduse        bool
+    stateDir        string
+    nfsStageDir     string
+    mounter         mount.Interface
     restartInterval time.Duration
-    mu sync.Mutex
-    cond *sync.Cond
-    volumeLocks map[string]bool
+    mu              sync.Mutex
+    cond            *sync.Cond
+    volumeLocks     map[string]bool
 }
 
 type DeviceState struct
@@ -48,6 +53,15 @@ type DeviceState struct
     PidFile    string `json:"pidFile"`
 }
 
+type NfsState struct
+{
+    ConfigPath string `json:"configPath"`
+    FsName     string `json:"fsName"`
+    Pool       string `json:"pool"`
+    Path       string `json:"path"`
+    Port       int    `json:"port"`
+}
+
 // NewNodeServer create new instance node
 func NewNodeServer(driver *Driver) *NodeServer
 {
@@ -60,11 +74,17 @@ func NewNodeServer(driver *Driver) *NodeServer
     {
         stateDir += "/"
     }
+    nfsStageDir := os.Getenv("NFS_STAGE_DIR")
+    if (nfsStageDir == "")
+    {
+        nfsStageDir = "/var/lib/kubelet/plugins/csi.vitastor.io/nfs"
+    }
     ns := &NodeServer{
-        Driver: driver,
-        useVduse: checkVduseSupport(),
-        stateDir: stateDir,
-        mounter: mount.New(""),
+        Driver:      driver,
+        useVduse:    checkVduseSupport(),
+        stateDir:    stateDir,
+        nfsStageDir: nfsStageDir,
+        mounter:     mount.New(""),
         volumeLocks: make(map[string]bool),
     }
     ns.cond = sync.NewCond(&ns.mu)
@@ -123,12 +143,12 @@ func (ns *NodeServer) restarter()
 func (ns *NodeServer) restoreVduseDaemons()
 {
     pattern := ns.stateDir+"vitastor-vduse-*.json"
-    matches, err := filepath.Glob(pattern)
+    stateFiles, err := filepath.Glob(pattern)
     if (err != nil)
     {
         klog.Errorf("failed to list %s: %v", pattern, err)
     }
-    if (len(matches) == 0)
+    if (len(stateFiles) == 0)
     {
         return
     }
@@ -146,59 +166,162 @@ func (ns *NodeServer) restoreVduseDaemons()
         klog.Errorf("/sbin/vdpa -j dev list returned bad JSON (error %v): %v", err, string(devListJSON))
         return
     }
-    for _, stateFile := range matches
+    for _, stateFile := range stateFiles
     {
-        vdpaId := filepath.Base(stateFile)
-        vdpaId = vdpaId[0:len(vdpaId)-5]
-        // Check if VDPA device is still added to the bus
-        if (devs[vdpaId] == nil)
-        {
-            // Unused, clean it up
-            unmapVduseById(ns.stateDir, vdpaId)
-            continue
-        }
+        ns.checkVduseState(stateFile, devs)
+    }
+}
 
-        stateJSON, err := os.ReadFile(stateFile)
+func (ns *NodeServer) checkVduseState(stateFile string, devs map[string]interface{})
+{
+    // Check if VDPA device is still added to the bus
+    vdpaId := filepath.Base(stateFile)
+    vdpaId = vdpaId[0:len(vdpaId)-5]
+    if (devs[vdpaId] == nil)
+    {
+        // Unused, clean it up
+        unmapVduseById(ns.stateDir, vdpaId)
+        return
+    }
+
+    // Read state file
+    stateJSON, err := os.ReadFile(stateFile)
+    if (err != nil)
+    {
+        klog.Warningf("error reading state file %v: %v", stateFile, err)
+        return
+    }
+    var state DeviceState
+    err = json.Unmarshal(stateJSON, &state)
+    if (err != nil)
+    {
+        klog.Warningf("state file %v contains invalid JSON (error %v): %v", stateFile, err, string(stateJSON))
+        return
+    }
+
+    // Lock volume
+    ns.lockVolume(state.ConfigPath+":block:"+state.Image)
+    defer ns.unlockVolume(state.ConfigPath+":block:"+state.Image)
+
+    // Recheck state file after locking
+    _, err = os.ReadFile(stateFile)
+    if (err != nil)
+    {
+        klog.Warningf("state file %v disappeared, skipping volume", stateFile)
+        return
+    }
+
+    // Check if the storage daemon is still active
+    pidFile := ns.stateDir + vdpaId + ".pid"
+    exists := false
+    proc, err := findByPidFile(pidFile)
+    if (err == nil)
+    {
+        exists = proc.Signal(syscall.Signal(0)) == nil
+    }
+    if (!exists)
+    {
+        // Restart daemon
+        klog.Warningf("restarting storage daemon for volume %v (VDPA ID %v)", state.Image, vdpaId)
+        err = startStorageDaemon(vdpaId, state.Image, pidFile, state.ConfigPath, state.Readonly)
         if (err != nil)
         {
-            klog.Warningf("error reading state file %v: %v", stateFile, err)
-            continue
+            klog.Warningf("failed to restart storage daemon for volume %v: %v", state.Image, err)
         }
-        var state DeviceState
-        err = json.Unmarshal(stateJSON, &state)
+    }
+}
+
+func (ns *NodeServer) restoreNfsDaemons()
+{
+    pattern := ns.stateDir+"vitastor-nfs-*.json"
+    stateFiles, err := filepath.Glob(pattern)
+    if (err != nil)
+    {
+        klog.Errorf("failed to list %s: %v", pattern, err)
+    }
+    if (len(stateFiles) == 0)
+    {
+        return
+    }
+    activeNFS, err := ns.listActiveNFS()
+    if (err != nil)
+    {
+        return
+    }
+    // Check all state files and try to restore active mounts
+    for _, stateFile := range stateFiles
+    {
+        ns.checkNfsState(stateFile, activeNFS)
+    }
+}
+
+func (ns *NodeServer) readNfsState(stateFile string, allowNotExists bool) (*NfsState, error)
+{
+    stateJSON, err := os.ReadFile(stateFile)
+    if (err != nil)
+    {
+        if (allowNotExists && os.IsNotExist(err))
+        {
+            return nil, nil
+        }
+        klog.Warningf("error reading state file %v: %v", stateFile, err)
+        return nil, err
+    }
+    var state NfsState
+    err = json.Unmarshal(stateJSON, &state)
+    if (err != nil)
+    {
+        klog.Warningf("state file %v contains invalid JSON (error %v): %v", stateFile, err, string(stateJSON))
+        return nil, err
+    }
+    return &state, nil
+}
+
+func (ns *NodeServer) checkNfsState(stateFile string, activeNfs map[int][]string)
+{
+    // Read state file
+    state, err := ns.readNfsState(stateFile, false)
+    if (err != nil)
+    {
+        return
+    }
+    // Lock FS
+    ns.lockVolume(state.ConfigPath+":fs:"+state.FsName)
+    defer ns.unlockVolume(state.ConfigPath+":fs:"+state.FsName)
+    // Check if NFS at this port is still mounted
+    pidFile := ns.stateDir + filepath.Base(stateFile)
+    pidFile = pidFile[0:len(pidFile)-5] + ".pid"
+    if (len(activeNfs[state.Port]) == 0)
+    {
+        // this is a stale state file, remove it
+        klog.Warningf("state file %v contains stale mount at port %d, removing it", stateFile, state.Port)
+        ns.stopNFS(stateFile, pidFile)
+        return
+    }
+    // Check PID file
+    exists := false
+    proc, err := findByPidFile(pidFile)
+    if (err == nil)
+    {
+        exists = proc.Signal(syscall.Signal(0)) == nil
+    }
+    if (!exists)
+    {
+        // Restart vitastor-nfs server
+        klog.Warningf("restarting NFS server for FS %v at port %v", state.FsName, state.Port)
+        _, _, err := system(
+            "/usr/bin/vitastor-nfs", "start",
+            "--pidfile", pidFile,
+            "--bind", "127.0.0.1",
+            "--port", fmt.Sprintf("%d", state.Port),
+            "--fs", state.FsName,
+            "--pool", state.Pool,
+            "--portmap", "0",
+        )
         if (err != nil)
         {
-            klog.Warningf("state file %v contains invalid JSON (error %v): %v", stateFile, err, string(stateJSON))
-            continue
+            klog.Warningf("failed to restart NFS server for FS %v: %v", state.FsName, err)
         }
-
-        ns.lockVolume(state.ConfigPath+":"+state.Image)
-
-        // Recheck state file after locking
-        _, err = os.ReadFile(stateFile)
-        if (err != nil)
-        {
-            klog.Warningf("state file %v disappeared, skipping volume", stateFile)
-            ns.unlockVolume(state.ConfigPath+":"+state.Image)
-            continue
-        }
-
-        // Check if the storage daemon is still active
-        pidFile := ns.stateDir + vdpaId + ".pid"
-        exists := false
-        proc, err := findByPidFile(pidFile)
-        if (err == nil)
-        {
-            exists = proc.Signal(syscall.Signal(0)) == nil
-        }
-        if (!exists)
-        {
-            // Restart daemon
-            klog.Warningf("restarting storage daemon for volume %v (VDPA ID %v)", state.Image, vdpaId)
-            _ = startStorageDaemon(vdpaId, state.Image, pidFile, state.ConfigPath, state.Readonly)
-        }
-
-        ns.unlockVolume(state.ConfigPath+":"+state.Image)
     }
 }
 
@@ -220,16 +343,26 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
     }
     volName := ctxVars["name"]
 
-    ns.lockVolume(ctxVars["configPath"]+":"+volName)
-    defer ns.unlockVolume(ctxVars["configPath"]+":"+volName)
+    if (ctxVars["vitastorfs"] != "")
+    {
+        return &csi.NodeStageVolumeResponse{}, nil
+    }
+
+    ns.lockVolume(ctxVars["configPath"]+":block:"+volName)
+    defer ns.unlockVolume(ctxVars["configPath"]+":block:"+volName)
 
     targetPath := req.GetStagingTargetPath()
     isBlock := req.GetVolumeCapability().GetBlock() != nil
 
     // Check that it's not already mounted
-    _, err = mount.IsNotMountPoint(ns.mounter, targetPath)
+    notmnt, err := mount.IsNotMountPoint(ns.mounter, targetPath)
     if (err == nil)
     {
+        if (!notmnt)
+        {
+            klog.Errorf("target path %s is already mounted", targetPath)
+            return nil, fmt.Errorf("target path %s is already mounted", targetPath)
+        }
         var finfo os.FileInfo
         finfo, err = os.Stat(targetPath)
         if (err != nil)
@@ -300,6 +433,7 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
     diskMounter := &mount.SafeFormatAndMount{Interface: ns.mounter, Exec: utilexec.New()}
     if (isBlock)
     {
+        klog.Infof("bind-mounting %s to %s", devicePath, targetPath)
         err = diskMounter.Mount(devicePath, targetPath, "", []string{"bind"})
     }
     else
@@ -329,39 +463,40 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
         readOnly := Contains(opt, "ro")
         if (existingFormat == "" && !readOnly)
         {
-            var cmdOut []byte
             switch fsType
             {
                 case "ext4":
                     args := []string{"-m0", "-Enodiscard,lazy_itable_init=1,lazy_journal_init=1", devicePath}
-                    cmdOut, err = diskMounter.Exec.Command("mkfs.ext4", args...).CombinedOutput()
+                    _, err = systemCombined("mkfs.ext4", args...)
                 case "xfs":
-                    cmdOut, err = diskMounter.Exec.Command("mkfs.xfs", "-K", devicePath).CombinedOutput()
+                    _, err = systemCombined("mkfs.xfs", "-K", devicePath)
             }
             if (err != nil)
             {
-                klog.Errorf("failed to run mkfs error: %v, output: %v", err, string(cmdOut))
                 goto unmap
             }
         }
 
+        klog.Infof("formatting and mounting %s to %s with FS %s, options: %v", devicePath, targetPath, fsType, opt)
         err = diskMounter.FormatAndMount(devicePath, targetPath, fsType, opt)
+        if (err == nil)
+        {
+            klog.Infof("successfully mounted %s to %s", devicePath, targetPath)
+        }
 
         // Try to run online resize on mount.
         // FIXME: Implement online resize. It requires online resize support in vitastor-nbd.
         if (err == nil && existingFormat != "" && !readOnly)
         {
-            var cmdOut []byte
             switch (fsType)
             {
                 case "ext4":
-                    cmdOut, err = diskMounter.Exec.Command("resize2fs", devicePath).CombinedOutput()
+                    _, err = systemCombined("resize2fs", devicePath)
                 case "xfs":
-                    cmdOut, err = diskMounter.Exec.Command("xfs_growfs", devicePath).CombinedOutput()
+                    _, err = systemCombined("xfs_growfs", devicePath)
             }
             if (err != nil)
             {
-                klog.Errorf("failed to run resizefs error: %v, output: %v", err, string(cmdOut))
                 goto unmap
             }
         }
@@ -401,8 +536,13 @@ func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag
     }
     volName := ctxVars["name"]
 
-    ns.lockVolume(ctxVars["configPath"]+":"+volName)
-    defer ns.unlockVolume(ctxVars["configPath"]+":"+volName)
+    if (ctxVars["vitastorfs"] != "")
+    {
+        return &csi.NodeUnstageVolumeResponse{}, nil
+    }
+
+    ns.lockVolume(ctxVars["configPath"]+":block:"+volName)
+    defer ns.unlockVolume(ctxVars["configPath"]+":block:"+volName)
 
     targetPath := req.GetStagingTargetPath()
     devicePath, _, err := mount.GetDeviceNameFromMount(ns.mounter, targetPath)
@@ -455,6 +595,153 @@ func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag
     return &csi.NodeUnstageVolumeResponse{}, nil
 }
 
+// Mount or check if NFS is already mounted
+func (ns *NodeServer) mountNFS(ctxVars map[string]string) (string, error)
+{
+    sum := sha1.Sum([]byte(ctxVars["configPath"]+":fs:"+ctxVars["vitastorfs"]))
+    nfsHash := hex.EncodeToString(sum[:])
+    stateFile := ns.stateDir+"vitastor-nfs-"+nfsHash+".json"
+    pidFile := ns.stateDir+"vitastor-nfs-"+nfsHash+".pid"
+    mountPath := ns.nfsStageDir+"/"+nfsHash
+    state, err := ns.readNfsState(stateFile, true)
+    if (state != nil)
+    {
+        return state.Path, nil
+    }
+    if (err != nil)
+    {
+        return "", err
+    }
+    err = os.MkdirAll(mountPath, 0777)
+    if (err != nil)
+    {
+        return "", err
+    }
+    // Create a new mount
+    state = &NfsState{
+        ConfigPath: ctxVars["configPath"],
+        FsName:     ctxVars["vitastorfs"],
+        Pool:       ctxVars["pool"],
+        Path:       mountPath,
+    }
+    klog.Infof("starting new NFS server for FS %v", state.FsName)
+    stdout, _, err := system(
+        "/usr/bin/vitastor-nfs", "start",
+        "--pidfile", pidFile,
+        "--bind", "127.0.0.1",
+        "--port", "auto",
+        "--fs", state.FsName,
+        "--pool", state.Pool,
+        "--portmap", "0",
+    )
+    if (err != nil)
+    {
+        return "", err
+    }
+    match := regexp.MustCompile("Port: (\\d+)").FindStringSubmatch(string(stdout))
+    if (match == nil)
+    {
+        klog.Errorf("failed to find port in vitastor-nfs output: %v", string(stdout))
+        ns.stopNFS(stateFile, pidFile)
+        return "", fmt.Errorf("failed to find port in vitastor-nfs output (bad vitastor-nfs version?)")
+    }
+    port, _ := strconv.ParseUint(match[1], 0, 16)
+    state.Port = int(port)
+    // Write state file
+    stateJSON, _ := json.Marshal(state)
+    err = os.WriteFile(stateFile, stateJSON, 0600)
+    if (err != nil)
+    {
+        klog.Errorf("failed to write state file %v", stateFile)
+        ns.stopNFS(stateFile, pidFile)
+        return "", err
+    }
+    // Mount NFS
+    _, _, err = system(
+        "mount", "-t", "nfs", "127.0.0.1:/", state.Path,
+        "-o", fmt.Sprintf("port=%d,mountport=%d,nfsvers=3,soft,nolock,tcp", port, port),
+    )
+    if (err != nil)
+    {
+        ns.stopNFS(stateFile, pidFile)
+        return "", err
+    }
+    return state.Path, nil
+}
+
+// Mount or check if NFS is already mounted
+func (ns *NodeServer) checkStopNFS(ctxVars map[string]string)
+{
+    sum := sha1.Sum([]byte(ctxVars["configPath"]+":fs:"+ctxVars["vitastorfs"]))
+    nfsHash := hex.EncodeToString(sum[:])
+    stateFile := ns.stateDir+"vitastor-nfs-"+nfsHash+".json"
+    pidFile := ns.stateDir+"vitastor-nfs-"+nfsHash+".pid"
+    mountPath := ns.nfsStageDir+"/"+nfsHash
+    state, err := ns.readNfsState(stateFile, true)
+    if (state == nil)
+    {
+        return
+    }
+    activeNFS, err := ns.listActiveNFS()
+    if (err != nil)
+    {
+        return
+    }
+    if (len(activeNFS[state.Port]) > 0)
+    {
+        return
+    }
+    // All volume mounts are detached, unmount the root mount and kill the server
+    err = mount.CleanupMountPoint(mountPath, ns.mounter, false)
+    if (err != nil)
+    {
+        klog.Errorf("failed to unmount %v: %v", mountPath, err)
+        return
+    }
+    ns.stopNFS(stateFile, pidFile)
+}
+
+func (ns *NodeServer) stopNFS(stateFile, pidFile string)
+{
+    err := killByPidFile(pidFile)
+    if (err != nil)
+    {
+        klog.Errorf("failed to kill process with pid from %v: %v", pidFile, err)
+    }
+    os.Remove(pidFile)
+    os.Remove(stateFile)
+}
+
+func (ns *NodeServer) listActiveNFS() (map[int][]string, error)
+{
+    mounts, err := mount.ParseMountInfo("/proc/self/mountinfo")
+    if (err != nil)
+    {
+        klog.Errorf("failed to list mounts: %v", err)
+        return nil, err
+    }
+    activeNFS := make(map[int][]string)
+    for _, mount := range mounts
+    {
+        // Volume mounts always refer to subpaths
+        if (mount.FsType == "nfs" && mount.Root != "/")
+        {
+            for _, opt := range mount.MountOptions
+            {
+                if (strings.HasPrefix(opt, "port="))
+                {
+                    port64, err := strconv.ParseUint(opt[5:], 10, 16)
+                    if (err == nil)
+                    {
+                        activeNFS[int(port64)] = append(activeNFS[int(port64)], mount.MountPoint)
+                    }
+                }
+            }
+        }
+    }
+    return activeNFS, nil
+}
+
 // NodePublishVolume mounts the volume mounted to the staging path to the target path
 func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error)
 {
@@ -473,23 +760,39 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
     }
     volName := ctxVars["name"]
 
-    ns.lockVolume(ctxVars["configPath"]+":"+volName)
-    defer ns.unlockVolume(ctxVars["configPath"]+":"+volName)
+    if (ctxVars["vitastorfs"] != "")
+    {
+        ns.lockVolume(ctxVars["configPath"]+":fs:"+ctxVars["vitastorfs"])
+        defer ns.unlockVolume(ctxVars["configPath"]+":fs:"+ctxVars["vitastorfs"])
+    }
+    else
+    {
+        ns.lockVolume(ctxVars["configPath"]+":block:"+volName)
+        defer ns.unlockVolume(ctxVars["configPath"]+":block:"+volName)
+    }
 
     stagingTargetPath := req.GetStagingTargetPath()
     targetPath := req.GetTargetPath()
     isBlock := req.GetVolumeCapability().GetBlock() != nil
 
-    // Check that stagingTargetPath is mounted
-    _, err = mount.IsNotMountPoint(ns.mounter, stagingTargetPath)
-    if (err != nil)
+    if (ctxVars["vitastorfs"] == "")
     {
-        klog.Errorf("staging path %v is not mounted: %v", stagingTargetPath, err)
-        return nil, fmt.Errorf("staging path %v is not mounted: %v", stagingTargetPath, err)
+        // Check that stagingTargetPath is mounted
+        notmnt, err := mount.IsNotMountPoint(ns.mounter, stagingTargetPath)
+        if (err != nil)
+        {
+            klog.Errorf("staging path %v is not mounted: %w", stagingTargetPath, err)
+            return nil, fmt.Errorf("staging path %v is not mounted: %w", stagingTargetPath, err)
+        }
+        else if (notmnt)
+        {
+            klog.Errorf("staging path %v is not mounted", stagingTargetPath)
+            return nil, fmt.Errorf("staging path %v is not mounted", stagingTargetPath)
+        }
     }
 
     // Check that targetPath is not already mounted
-    _, err = mount.IsNotMountPoint(ns.mounter, targetPath)
+    notmnt, err := mount.IsNotMountPoint(ns.mounter, targetPath)
     if (err != nil)
     {
         if (os.IsNotExist(err))
@@ -524,6 +827,29 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
             return nil, err
         }
     }
+    else if (!notmnt)
+    {
+        klog.Errorf("target path %s is already mounted", targetPath)
+        return nil, fmt.Errorf("target path %s is already mounted", targetPath)
+    }
+
+    if (ctxVars["vitastorfs"] != "")
+    {
+        nfspath, err := ns.mountNFS(ctxVars)
+        if (err != nil)
+        {
+            ns.checkStopNFS(ctxVars)
+            return nil, err
+        }
+        // volName should include prefix
+        stagingTargetPath = nfspath+"/"+volName
+        err = os.MkdirAll(stagingTargetPath, 0777)
+        if (err != nil && !os.IsExist(err))
+        {
+            ns.checkStopNFS(ctxVars)
+            return nil, err
+        }
+    }
 
     execArgs := []string{"--bind", stagingTargetPath, targetPath}
     if (req.GetReadonly())
@@ -536,6 +862,10 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
     out, err := cmd.Output()
     if (err != nil)
     {
+        if (ctxVars["vitastorfs"] != "")
+        {
+            ns.checkStopNFS(ctxVars)
+        }
         return nil, fmt.Errorf("Error running mount %v: %s", strings.Join(execArgs, " "), out)
     }
 
@@ -555,8 +885,16 @@ func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
     }
     volName := ctxVars["name"]
 
-    ns.lockVolume(ctxVars["configPath"]+":"+volName)
-    defer ns.unlockVolume(ctxVars["configPath"]+":"+volName)
+    if (ctxVars["vitastorfs"] != "")
+    {
+        ns.lockVolume(ctxVars["configPath"]+":fs:"+ctxVars["vitastorfs"])
+        defer ns.unlockVolume(ctxVars["configPath"]+":fs:"+ctxVars["vitastorfs"])
+    }
+    else
+    {
+        ns.lockVolume(ctxVars["configPath"]+":block:"+volName)
+        defer ns.unlockVolume(ctxVars["configPath"]+":block:"+volName)
+    }
 
     targetPath := req.GetTargetPath()
     devicePath, _, err := mount.GetDeviceNameFromMount(ns.mounter, targetPath)
@@ -583,6 +921,11 @@ func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
         return nil, err
     }
 
+    if (ctxVars["vitastorfs"] != "")
+    {
+        ns.checkStopNFS(ctxVars)
+    }
+
     return &csi.NodeUnpublishVolumeResponse{}, nil
 }
 

csi/src/utils.go

@@ -4,6 +4,7 @@
 package vitastor
 
 import (
+    "bytes"
     "errors"
     "encoding/json"
     "fmt"
@@ -15,6 +16,8 @@ import (
     "syscall"
 
     "k8s.io/klog"
+    "google.golang.org/grpc/codes"
+    "google.golang.org/grpc/status"
 )
 
 func Contains(list []string, s string) bool
@@ -73,6 +76,10 @@ func checkVduseSupport() bool
             " For VDUSE you need at least Linux 5.15 and the following kernel modules: vdpa, virtio-vdpa, vduse.",
         )
     }
+    else
+    {
+        klog.Infof("VDUSE support enabled successfully")
+    }
     return vduse
 }
 
@@ -97,6 +104,7 @@ func mapNbd(volName string, ctxVars map[string]string, readonly bool) (string, e
     {
         return "", fmt.Errorf("vitastor-nbd did not return the name of NBD device. output: %s", stderr)
     }
+    klog.Infof("Attached volume %s via NBD as %s", volName, dev)
     return dev, err
 }
 
@@ -217,6 +225,7 @@ func mapVduse(stateDir string, volName string, ctxVars map[string]string, readon
                     err = os.WriteFile(stateFile, stateJSON, 0600)
                     if (err == nil)
                     {
+                        klog.Infof("Attached volume %s via VDUSE as %s (VDPA ID %s)", volName, blockdev, vdpaId)
                         return blockdev, vdpaId, nil
                     }
                 }
@@ -299,3 +308,35 @@ func unmapVduseById(stateDir, vdpaId string)
         os.Remove(pidFile)
     }
 }
+
+func system(program string, args ...string) ([]byte, []byte, error)
+{
+    klog.Infof("Running "+program+" "+strings.Join(args, " "))
+    c := exec.Command(program, args...)
+    var stdout, stderr bytes.Buffer
+    c.Stdout, c.Stderr = &stdout, &stderr
+    err := c.Run()
+    if (err != nil)
+    {
+        stdoutStr, stderrStr := string(stdout.Bytes()), string(stderr.Bytes())
+        klog.Errorf(program+" "+strings.Join(args, " ")+" failed: %s\nOutput:\n%s", err, stdoutStr+stderrStr)
+        return nil, nil, status.Error(codes.Internal, stdoutStr+stderrStr+" (status "+err.Error()+")")
+    }
+    return stdout.Bytes(), stderr.Bytes(), nil
+}
+
+func systemCombined(program string, args ...string) ([]byte, error)
+{
+    klog.Infof("Running "+program+" "+strings.Join(args, " "))
+    c := exec.Command(program, args...)
+    var out bytes.Buffer
+    c.Stdout, c.Stderr = &out, &out
+    err := c.Run()
+    if (err != nil)
+    {
+        outStr := string(out.Bytes())
+        klog.Errorf(program+" "+strings.Join(args, " ")+" failed: %s, status %s\n", outStr, err)
+        return nil, status.Error(codes.Internal, outStr+" (status "+err.Error()+")")
+    }
+    return out.Bytes(), nil
+}

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (1.9.2-1) unstable; urgency=medium
+vitastor (1.10.0-1) unstable; urgency=medium
 
   * Bugfixes
 

debian/vitastor.Dockerfile

@@ -21,10 +21,10 @@ RUN set -e -x; \
     echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf; \
     echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
 
-RUN apt-get update
-RUN apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts libjerasure-dev cmake libibverbs-dev libisal-dev libnl-3-dev libnl-genl-3-dev curl
-RUN apt-get -y build-dep fio
-RUN apt-get --download-only source fio
+RUN apt-get update && \
+    apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts libjerasure-dev cmake libibverbs-dev librdmacm-dev libisal-dev libnl-3-dev libnl-genl-3-dev curl && \
+    apt-get -y build-dep fio && \
+    apt-get --download-only source fio
 
 ADD . /root/vitastor
 RUN set -e -x; \

docs/config/layout-osd.en.md

@@ -118,12 +118,13 @@ Physical block size of the journal device. Must be a multiple of
 - Type: boolean
 - Default: false
 
-Do not issue fsyncs to the data device, i.e. do not flush its cache.
-Safe ONLY if your data device has write-through cache. If you disable
-the cache yourself using `hdparm` or `scsi_disk/cache_type` then make sure
-that the cache disable command is run every time before starting Vitastor
-OSD, for example, in the systemd unit. See also `immediate_commit` option
-for the instructions to disable cache and how to benefit from it.
+Do not issue fsyncs to the data device, i.e. do not force it to flush cache.
+Safe ONLY if your data device has write-through cache or if write-back
+cache is disabled. If you disable drive cache manually with `hdparm` or
+writing to `/sys/.../scsi_disk/cache_type` then make sure that you do it
+every time before starting Vitastor OSD (vitastor-disk does it automatically).
+See also [immediate_commit](layout-cluster.en.md#immediate_commit)
+for information about how to benefit from disabled cache.
 
 ## disable_meta_fsync
 
@@ -171,8 +172,7 @@ size, it actually has to write the whole 4 KB sector.
 
 Because of this it can actually be beneficial to use SSDs which work well
 with 512 byte sectors and use 512 byte disk_alignment, journal_block_size
-and meta_block_size. But the only SSD that may fit into this category is
-Intel Optane (probably, not tested yet).
+and meta_block_size. But at the moment, no such SSDs are known...
 
 Clients don't need to be aware of disk_alignment, so it's not required to
 put a modified value into etcd key /vitastor/config/global.

docs/config/layout-osd.ru.md

@@ -122,13 +122,14 @@ SSD-диске, иначе производительность пострада
 - Тип: булево (да/нет)
 - Значение по умолчанию: false
 
-Не отправлять fsync-и устройству данных, т.е. не сбрасывать его кэш.
+Не отправлять fsync-и устройству данных, т.е. не заставлять его сбрасывать кэш.
 Безопасно, ТОЛЬКО если ваше устройство данных имеет кэш со сквозной
-записью (write-through). Если вы отключаете кэш через `hdparm` или
-`scsi_disk/cache_type`, то удостоверьтесь, что команда отключения кэша
-выполняется перед каждым запуском Vitastor OSD, например, в systemd unit-е.
-Смотрите также опцию `immediate_commit` для инструкций по отключению кэша
-и о том, как из этого извлечь выгоду.
+записью (write-through) или если кэш с отложенной записью (write-back) отключён.
+Если вы отключаете кэш вручную через `hdparm` или запись в `/sys/.../scsi_disk/cache_type`,
+то удостоверьтесь, что вы делаете это каждый раз перед запуском Vitastor OSD
+(vitastor-disk делает это автоматически). Смотрите также опцию
+[immediate_commit](layout-cluster.ru.md#immediate_commit) для информации о том,
+как извлечь выгоду из отключённого кэша.
 
 ## disable_meta_fsync
 
@@ -179,9 +180,8 @@ SSD и HDD диски используют 4 КБ физические сект
 
 Поэтому, на самом деле, может быть выгодно найти SSD, хорошо работающие с
 меньшими, 512-байтными, блоками и использовать 512-байтные disk_alignment,
-journal_block_size и meta_block_size. Однако единственные SSD, которые
-теоретически могут попасть в эту категорию - это Intel Optane (но и это
-пока не проверялось автором).
+journal_block_size и meta_block_size. Однако на данный момент такие SSD
+не известны...
 
 Клиентам не обязательно знать про disk_alignment, так что помещать значение
 этого параметра в etcd в /vitastor/config/global не нужно.

docs/config/monitor.en.md

@@ -24,6 +24,7 @@ These parameters only apply to Monitors.
 - [osd_out_time](#osd_out_time)
 - [placement_levels](#placement_levels)
 - [use_old_pg_combinator](#use_old_pg_combinator)
+- [osd_backfillfull_ratio](#osd_backfillfull_ratio)
 
 ## use_antietcd
 
@@ -175,3 +176,18 @@ present in the configuration, then it is defined with the default priority
 
 Use the old PG combination generator which doesn't support [level_placement](pool.en.md#level_placement)
 and [raw_placement](pool.en.md#raw_placement) for pools which don't use this features.
+
+## osd_backfillfull_ratio
+
+- Type: number
+- Default: 0.99
+
+Monitors try to prevent OSDs becoming 100% full during rebalance or recovery by
+calculating how much space will be occupied on every OSD after all rebalance
+and recovery operations finish, and pausing rebalance and recovery if that
+amount of space exceeds OSD capacity multiplied by the value of this
+configuration parameter.
+
+Future used space is calculated by summing space used by all user data blocks
+(objects) in all PGs placed on a specific OSD, even if some of these objects
+currently reside on a different set of OSDs.

docs/config/monitor.ru.md

@@ -24,6 +24,7 @@
 - [osd_out_time](#osd_out_time)
 - [placement_levels](#placement_levels)
 - [use_old_pg_combinator](#use_old_pg_combinator)
+- [osd_backfillfull_ratio](#osd_backfillfull_ratio)
 
 ## use_antietcd
 
@@ -178,3 +179,19 @@ OSD перед обновлением агрегированной статис
 
 Использовать старый генератор комбинаций PG, не поддерживающий [level_placement](pool.ru.md#level_placement)
 и [raw_placement](pool.ru.md#raw_placement) для пулов, которые не используют данные функции.
+
+## osd_backfillfull_ratio
+
+- Тип: число
+- Значение по умолчанию: 0.99
+
+Мониторы стараются предотвратить 100% заполнение OSD в процессе ребаланса
+или восстановления, рассчитывая, сколько места будет занято на каждом OSD после
+завершения всех операций ребаланса и восстановления, и приостанавливая
+ребаланс и восстановление, если рассчитанный объём превышает ёмкость OSD,
+умноженную на значение данного параметра.
+
+Будущее занятое место рассчитывается сложением места, занятого всеми
+пользовательскими блоками данных (объектами) во всех PG, расположенных
+на конкретном OSD, даже если часть этих объектов в данный момент находится
+на другом наборе OSD.

docs/config/network.en.md

@@ -68,11 +68,17 @@ but they are not connected to the cluster.
 - Type: string
 
 RDMA device name to use for Vitastor OSD communications (for example,
-"rocep5s0f0"). Now Vitastor supports all adapters, even ones without
-ODP support, like Mellanox ConnectX-3 and non-Mellanox cards.
+"rocep5s0f0"). If not specified, Vitastor will try to find an RoCE
+device matching [osd_network](osd.en.md#osd_network), preferring RoCEv2,
+or choose the first available RDMA device if no RoCE devices are
+found or if `osd_network` is not specified. Auto-selection is also
+unsupported with old libibverbs < v32, like in Debian 10 Buster or
+CentOS 7.
 
-Versions up to Vitastor 1.2.0 required ODP which is only present in
-Mellanox ConnectX >= 4. See also [rdma_odp](#rdma_odp).
+Vitastor supports all adapters, even ones without ODP support, like
+Mellanox ConnectX-3 and non-Mellanox cards. Versions up to Vitastor
+1.2.0 required ODP which is only present in Mellanox ConnectX >= 4.
+See also [rdma_odp](#rdma_odp).
 
 Run `ibv_devinfo -v` as root to list available RDMA devices and their
 features.
@@ -95,15 +101,17 @@ your device has.
 ## rdma_gid_index
 
 - Type: integer
-- Default: 0
 
 Global address identifier index of the RDMA device to use. Different GID
 indexes may correspond to different protocols like RoCEv1, RoCEv2 and iWARP.
 Search for "GID" in `ibv_devinfo -v` output to determine which GID index
 you need.
 
-**IMPORTANT:** If you want to use RoCEv2 (as recommended) then the correct
-rdma_gid_index is usually 1 (IPv6) or 3 (IPv4).
+If not specified, Vitastor will try to auto-select a RoCEv2 IPv4 GID, then
+RoCEv2 IPv6 GID, then RoCEv1 IPv4 GID, then RoCEv1 IPv6 GID, then IB GID.
+GID auto-selection is unsupported with libibverbs < v32.
+
+A correct rdma_gid_index for RoCEv2 is usually 1 (IPv6) or 3 (IPv4).
 
 ## rdma_mtu
 

docs/config/network.ru.md

@@ -71,12 +71,17 @@ RDMA может быть нужно только если у клиентов е
 - Тип: строка
 
 Название RDMA-устройства для связи с Vitastor OSD (например, "rocep5s0f0").
-Сейчас Vitastor поддерживает все модели адаптеров, включая те, у которых
-нет поддержки ODP, то есть вы можете использовать RDMA с ConnectX-3 и
-картами производства не Mellanox.
+Если не указано, Vitastor попробует найти RoCE-устройство, соответствующее
+[osd_network](osd.en.md#osd_network), предпочитая RoCEv2, или выбрать первое
+попавшееся RDMA-устройство, если RoCE-устройств нет или если сеть `osd_network`
+не задана. Также автовыбор не поддерживается со старыми версиями библиотеки
+libibverbs < v32, например в Debian 10 Buster или CentOS 7.
 
-Версии Vitastor до 1.2.0 включительно требовали ODP, который есть только
-на Mellanox ConnectX 4 и более новых. См. также [rdma_odp](#rdma_odp).
+Vitastor поддерживает все модели адаптеров, включая те, у которых
+нет поддержки ODP, то есть вы можете использовать RDMA с ConnectX-3 и
+картами производства не Mellanox. Версии Vitastor до 1.2.0 включительно
+требовали ODP, который есть только на Mellanox ConnectX 4 и более новых.
+См. также [rdma_odp](#rdma_odp).
 
 Запустите `ibv_devinfo -v` от имени суперпользователя, чтобы посмотреть
 список доступных RDMA-устройств, их параметры и возможности.
@@ -101,15 +106,18 @@ Control) и ECN (Explicit Congestion Notification).
 ## rdma_gid_index
 
 - Тип: целое число
-- Значение по умолчанию: 0
 
 Номер глобального идентификатора адреса RDMA-устройства, который следует
 использовать. Разным gid_index могут соответствовать разные протоколы связи:
 RoCEv1, RoCEv2, iWARP. Чтобы понять, какой нужен вам - смотрите строчки со
 словом "GID" в выводе команды `ibv_devinfo -v`.
 
-**ВАЖНО:** Если вы хотите использовать RoCEv2 (как мы и рекомендуем), то
-правильный rdma_gid_index, как правило, 1 (IPv6) или 3 (IPv4).
+Если не указан, Vitastor попробует автоматически выбрать сначала GID,
+соответствующий RoCEv2 IPv4, потом RoCEv2 IPv6, потом RoCEv1 IPv4, потом
+RoCEv1 IPv6, потом IB. Авто-выбор GID не поддерживается со старыми версиями
+libibverbs < v32.
+
+Правильный rdma_gid_index для RoCEv2, как правило, 1 (IPv6) или 3 (IPv4).
 
 ## rdma_mtu
 

docs/config/src/layout-osd.yml

@@ -110,20 +110,22 @@
   type: bool
   default: false
   info: |
-    Do not issue fsyncs to the data device, i.e. do not flush its cache.
-    Safe ONLY if your data device has write-through cache. If you disable
-    the cache yourself using `hdparm` or `scsi_disk/cache_type` then make sure
-    that the cache disable command is run every time before starting Vitastor
-    OSD, for example, in the systemd unit. See also `immediate_commit` option
-    for the instructions to disable cache and how to benefit from it.
+    Do not issue fsyncs to the data device, i.e. do not force it to flush cache.
+    Safe ONLY if your data device has write-through cache or if write-back
+    cache is disabled. If you disable drive cache manually with `hdparm` or
+    writing to `/sys/.../scsi_disk/cache_type` then make sure that you do it
+    every time before starting Vitastor OSD (vitastor-disk does it automatically).
+    See also [immediate_commit](layout-cluster.en.md#immediate_commit)
+    for information about how to benefit from disabled cache.
   info_ru: |
-    Не отправлять fsync-и устройству данных, т.е. не сбрасывать его кэш.
+    Не отправлять fsync-и устройству данных, т.е. не заставлять его сбрасывать кэш.
     Безопасно, ТОЛЬКО если ваше устройство данных имеет кэш со сквозной
-    записью (write-through). Если вы отключаете кэш через `hdparm` или
-    `scsi_disk/cache_type`, то удостоверьтесь, что команда отключения кэша
-    выполняется перед каждым запуском Vitastor OSD, например, в systemd unit-е.
-    Смотрите также опцию `immediate_commit` для инструкций по отключению кэша
-    и о том, как из этого извлечь выгоду.
+    записью (write-through) или если кэш с отложенной записью (write-back) отключён.
+    Если вы отключаете кэш вручную через `hdparm` или запись в `/sys/.../scsi_disk/cache_type`,
+    то удостоверьтесь, что вы делаете это каждый раз перед запуском Vitastor OSD
+    (vitastor-disk делает это автоматически). Смотрите также опцию
+    [immediate_commit](layout-cluster.ru.md#immediate_commit) для информации о том,
+    как извлечь выгоду из отключённого кэша.
 - name: disable_meta_fsync
   type: bool
   default: false
@@ -179,8 +181,7 @@
 
     Because of this it can actually be beneficial to use SSDs which work well
     with 512 byte sectors and use 512 byte disk_alignment, journal_block_size
-    and meta_block_size. But the only SSD that may fit into this category is
-    Intel Optane (probably, not tested yet).
+    and meta_block_size. But at the moment, no such SSDs are known...
 
     Clients don't need to be aware of disk_alignment, so it's not required to
     put a modified value into etcd key /vitastor/config/global.
@@ -198,9 +199,8 @@
 
     Поэтому, на самом деле, может быть выгодно найти SSD, хорошо работающие с
     меньшими, 512-байтными, блоками и использовать 512-байтные disk_alignment,
-    journal_block_size и meta_block_size. Однако единственные SSD, которые
-    теоретически могут попасть в эту категорию - это Intel Optane (но и это
-    пока не проверялось автором).
+    journal_block_size и meta_block_size. Однако на данный момент такие SSD
+    не известны...
 
     Клиентам не обязательно знать про disk_alignment, так что помещать значение
     этого параметра в etcd в /vitastor/config/global не нужно.

docs/config/src/monitor.yml

@@ -172,3 +172,27 @@
   info_ru: |
     Использовать старый генератор комбинаций PG, не поддерживающий [level_placement](pool.ru.md#level_placement)
     и [raw_placement](pool.ru.md#raw_placement) для пулов, которые не используют данные функции.
+- name: osd_backfillfull_ratio
+  type: float
+  default: 0.99
+  info: |
+    Monitors try to prevent OSDs becoming 100% full during rebalance or recovery by
+    calculating how much space will be occupied on every OSD after all rebalance
+    and recovery operations finish, and pausing rebalance and recovery if that
+    amount of space exceeds OSD capacity multiplied by the value of this
+    configuration parameter.
+
+    Future used space is calculated by summing space used by all user data blocks
+    (objects) in all PGs placed on a specific OSD, even if some of these objects
+    currently reside on a different set of OSDs.
+  info_ru: |
+    Мониторы стараются предотвратить 100% заполнение OSD в процессе ребаланса
+    или восстановления, рассчитывая, сколько места будет занято на каждом OSD после
+    завершения всех операций ребаланса и восстановления, и приостанавливая
+    ребаланс и восстановление, если рассчитанный объём превышает ёмкость OSD,
+    умноженную на значение данного параметра.
+
+    Будущее занятое место рассчитывается сложением места, занятого всеми
+    пользовательскими блоками данных (объектами) во всех PG, расположенных
+    на конкретном OSD, даже если часть этих объектов в данный момент находится
+    на другом наборе OSD.

docs/config/src/network.yml

@@ -48,11 +48,17 @@
   type: string
   info: |
     RDMA device name to use for Vitastor OSD communications (for example,
-    "rocep5s0f0"). Now Vitastor supports all adapters, even ones without
-    ODP support, like Mellanox ConnectX-3 and non-Mellanox cards.
+    "rocep5s0f0"). If not specified, Vitastor will try to find an RoCE
+    device matching [osd_network](osd.en.md#osd_network), preferring RoCEv2,
+    or choose the first available RDMA device if no RoCE devices are
+    found or if `osd_network` is not specified. Auto-selection is also
+    unsupported with old libibverbs < v32, like in Debian 10 Buster or
+    CentOS 7.
 
-    Versions up to Vitastor 1.2.0 required ODP which is only present in
-    Mellanox ConnectX >= 4. See also [rdma_odp](#rdma_odp).
+    Vitastor supports all adapters, even ones without ODP support, like
+    Mellanox ConnectX-3 and non-Mellanox cards. Versions up to Vitastor
+    1.2.0 required ODP which is only present in Mellanox ConnectX >= 4.
+    See also [rdma_odp](#rdma_odp).
 
     Run `ibv_devinfo -v` as root to list available RDMA devices and their
     features.
@@ -64,12 +70,17 @@
     PFC (Priority Flow Control) and ECN (Explicit Congestion Notification).
   info_ru: |
     Название RDMA-устройства для связи с Vitastor OSD (например, "rocep5s0f0").
-    Сейчас Vitastor поддерживает все модели адаптеров, включая те, у которых
-    нет поддержки ODP, то есть вы можете использовать RDMA с ConnectX-3 и
-    картами производства не Mellanox.
+    Если не указано, Vitastor попробует найти RoCE-устройство, соответствующее
+    [osd_network](osd.en.md#osd_network), предпочитая RoCEv2, или выбрать первое
+    попавшееся RDMA-устройство, если RoCE-устройств нет или если сеть `osd_network`
+    не задана. Также автовыбор не поддерживается со старыми версиями библиотеки
+    libibverbs < v32, например в Debian 10 Buster или CentOS 7.
 
-    Версии Vitastor до 1.2.0 включительно требовали ODP, который есть только
-    на Mellanox ConnectX 4 и более новых. См. также [rdma_odp](#rdma_odp).
+    Vitastor поддерживает все модели адаптеров, включая те, у которых
+    нет поддержки ODP, то есть вы можете использовать RDMA с ConnectX-3 и
+    картами производства не Mellanox. Версии Vitastor до 1.2.0 включительно
+    требовали ODP, который есть только на Mellanox ConnectX 4 и более новых.
+    См. также [rdma_odp](#rdma_odp).
 
     Запустите `ibv_devinfo -v` от имени суперпользователя, чтобы посмотреть
     список доступных RDMA-устройств, их параметры и возможности.
@@ -94,23 +105,29 @@
     `ibv_devinfo -v`.
 - name: rdma_gid_index
   type: int
-  default: 0
   info: |
     Global address identifier index of the RDMA device to use. Different GID
     indexes may correspond to different protocols like RoCEv1, RoCEv2 and iWARP.
     Search for "GID" in `ibv_devinfo -v` output to determine which GID index
     you need.
 
-    **IMPORTANT:** If you want to use RoCEv2 (as recommended) then the correct
-    rdma_gid_index is usually 1 (IPv6) or 3 (IPv4).
+    If not specified, Vitastor will try to auto-select a RoCEv2 IPv4 GID, then
+    RoCEv2 IPv6 GID, then RoCEv1 IPv4 GID, then RoCEv1 IPv6 GID, then IB GID.
+    GID auto-selection is unsupported with libibverbs < v32.
+
+    A correct rdma_gid_index for RoCEv2 is usually 1 (IPv6) or 3 (IPv4).
   info_ru: |
     Номер глобального идентификатора адреса RDMA-устройства, который следует
     использовать. Разным gid_index могут соответствовать разные протоколы связи:
     RoCEv1, RoCEv2, iWARP. Чтобы понять, какой нужен вам - смотрите строчки со
     словом "GID" в выводе команды `ibv_devinfo -v`.
 
-    **ВАЖНО:** Если вы хотите использовать RoCEv2 (как мы и рекомендуем), то
-    правильный rdma_gid_index, как правило, 1 (IPv6) или 3 (IPv4).
+    Если не указан, Vitastor попробует автоматически выбрать сначала GID,
+    соответствующий RoCEv2 IPv4, потом RoCEv2 IPv6, потом RoCEv1 IPv4, потом
+    RoCEv1 IPv6, потом IB. Авто-выбор GID не поддерживается со старыми версиями
+    libibverbs < v32.
+
+    Правильный rdma_gid_index для RoCEv2, как правило, 1 (IPv6) или 3 (IPv4).
 - name: rdma_mtu
   type: int
   default: 4096

docs/installation/kubernetes.en.md

@@ -6,9 +6,18 @@
 
 # Kubernetes CSI
 
-Vitastor has a CSI plugin for Kubernetes which supports RWO (and block RWX) volumes.
+Vitastor has a CSI plugin for Kubernetes which supports block-based and VitastorFS-based volumes.
 
-To deploy it, take manifests from [csi/deploy/](../../csi/deploy/) directory, put your
+Block-based volumes may be formatted and mounted with a normal FS (ext4 or xfs). Such volumes
+only support RWO (ReadWriteOnce) mode.
+
+Block-based volumes may also be left without FS and attached into the container as a block
+device. Such volumes also support RWX (ReadWriteMany) mode.
+
+VitastorFS-based volumes use a clustered file system and support FS-based RWX (ReadWriteMany)
+mode. However, such volumes don't support quotas and snapshots.
+
+To deploy the CSI plugin, take manifests from [csi/deploy/](../../csi/deploy/) directory, put your
 Vitastor configuration in [001-csi-config-map.yaml](../../csi/deploy/001-csi-config-map.yaml),
 configure storage class in [009-storage-class.yaml](../../csi/deploy/009-storage-class.yaml)
 and apply all `NNN-*.yaml` manifests to your Kubernetes installation:
@@ -23,16 +32,16 @@ After that you'll be able to create PersistentVolumes.
 kernel modules enabled (vdpa, vduse, virtio-vdpa). If your distribution doesn't
 have them pre-built - build them yourself ([instructions](../usage/qemu.en.md#vduse)),
 I promise it's worth it :-). When VDUSE is unavailable, CSI driver uses [NBD](../usage/nbd.en.md)
-to map Vitastor devices. NBD is slower and prone to timeout issues: if Vitastor
-cluster becomes unresponsible for more than [nbd_timeout](../config/client.en.md#nbd_timeout),
-the NBD device detaches and breaks pods using it.
+to map Vitastor devices. NBD is slower and, with kernels older than 5.19, unmountable
+if the cluster becomes unresponsible.
 
 ## Features
 
 Vitastor CSI supports:
 - Kubernetes starting with 1.20 (or 1.17 for older vitastor-csi <= 1.1.0)
-- Filesystem RWO (ReadWriteOnce) volumes. Example: [PVC](../../csi/deploy/example-pvc.yaml), [pod](../../csi/deploy/example-test-pod.yaml)
+- Block-based FS-formatted RWO (ReadWriteOnce) volumes. Example: [PVC](../../csi/deploy/example-pvc.yaml), [pod](../../csi/deploy/example-test-pod.yaml)
 - Raw block RWX (ReadWriteMany) volumes. Example: [PVC](../../csi/deploy/example-pvc-block.yaml), [pod](../../csi/deploy/example-test-pod-block.yaml)
+- VitastorFS-based volumes RWX (ReadWriteMany) volumes. Example: [storage class](../../csi/deploy/example-storage-class-fs.yaml)
 - Volume expansion
 - Volume snapshots. Example: [snapshot class](../../csi/deploy/example-snapshot-class.yaml), [snapshot](../../csi/deploy/example-snapshot.yaml), [clone](../../csi/deploy/example-snapshot-clone.yaml)
 - [VDUSE](../usage/qemu.en.md#vduse) (preferred) and [NBD](../usage/nbd.en.md) device mapping methods

docs/installation/kubernetes.ru.md

@@ -6,7 +6,17 @@
 
 # Kubernetes CSI
 
-У Vitastor есть CSI-плагин для Kubernetes, поддерживающий RWO, а также блочные RWX, тома.
+У Vitastor есть CSI-плагин для Kubernetes, поддерживающий блочные тома и тома на основе
+кластерной ФС VitastorFS.
+
+Блочные тома могут быть отформатированы и примонтированы со стандартной ФС (ext4 или xfs).
+Такие тома поддерживают только режим RWO (ReadWriteOnce, одновременный доступ с одного узла).
+
+Блочные тома также могут не форматироваться и подключаться в контейнер в виде блочного устройства.
+В таком случае их можно подключать в режиме RWX (ReadWriteMany, одновременный доступ с многих узлов).
+
+Тома на основе VitastorFS используют кластерную ФС и поэтому также поддерживают режим RWX
+(ReadWriteMany). Однако, такие тома не поддерживают ограничение размера и снимки.
 
 Для установки возьмите манифесты из директории [csi/deploy/](../../csi/deploy/), поместите
 вашу конфигурацию подключения к Vitastor в [csi/deploy/001-csi-config-map.yaml](../../csi/deploy/001-csi-config-map.yaml),
@@ -33,6 +43,7 @@ CSI-плагин Vitastor поддерживает:
 - Версии Kubernetes, начиная с 1.20 (или с 1.17 для более старых vitastor-csi <= 1.1.0)
 - Файловые RWO (ReadWriteOnce) тома. Пример: [PVC](../../csi/deploy/example-pvc.yaml), [под](../../csi/deploy/example-test-pod.yaml)
 - Сырые блочные RWX (ReadWriteMany) тома. Пример: [PVC](../../csi/deploy/example-pvc-block.yaml), [под](../../csi/deploy/example-test-pod-block.yaml)
+- Основанные на VitastorFS RWX (ReadWriteMany) тома. Пример: [класс хранения](../../csi/deploy/example-storage-class-fs.yaml)
 - Расширение размера томов
 - Снимки томов. Пример: [класс снимков](../../csi/deploy/example-snapshot-class.yaml), [снимок](../../csi/deploy/example-snapshot.yaml), [клон снимка](../../csi/deploy/example-snapshot-clone.yaml)
 - Способы подключения устройств [VDUSE](../usage/qemu.ru.md#vduse) (предпочитаемый) и [NBD](../usage/nbd.ru.md)

docs/intro/architecture.en.md

@@ -6,19 +6,151 @@
 
 # Architecture
 
+- [Server-side components](#server-side-components)
 - [Basic concepts](#basic-concepts)
+- [Client-side components](#client-side-components)
+- [Additional utilities](#additional-utilities)
+- [Overall read/write process](#overall-read-write-process)
+  - [Nuances of request handling](#nuances-of-request-handling)
 - [Similarities to Ceph](#similarities-to-ceph)
 - [Differences from Ceph](#differences-from-ceph)
 - [Implementation Principles](#implementation-principles)
 
+## Server-side components
+
+- **OSD** (Object Storage Daemon) is a process that directly works with the disk, stores data
+  and serves read/write requests. One OSD serves one disk (or one partition). OSDs talk to etcd
+  and to each other — they receive cluster state from etcd, and send read/write requests for
+  secondary copies of data to other OSDs.
+- **etcd** — clustered key/value database, used as a reliable storage for configuration
+  and high-level cluster state. Etcd is the component that prevents splitbrain in the cluster.
+  Data blocks are not stored in etcd, etcd doesn't participate in data write or read path.
+- **Монитор** — a separate node.js based daemon which monitors the cluster, calculates
+  required configuration changes and saves them to etcd, thus commanding OSDs to apply these
+  changes. Monitor also aggregates cluster statistics. OSD don't talk to monitor, monitor
+  only sends and receives data from etcd.
+
 ## Basic concepts
 
-- OSD (Object Storage Daemon) is a process that stores data and serves read/write requests.
-- PG (Placement Group) is a "shard" of the cluster, group of data stored on one set of replicas.
-- Pool is a container for data that has equal redundancy scheme and placement rules.
-- Monitor is a separate daemon that watches cluster state and handles failures.
-- Failure Domain is a group of OSDs that you allow to fail. It's "host" by default.
-- Placement Tree groups OSDs in a hierarchy to later split them into Failure Domains.
+- **Pool** is a container for data that has equal redundancy scheme and disk placement rules.
+- **PG (Placement Group)** is a "shard" of the cluster, subdivision unit that has its own
+  set of OSDs for data storage.
+- **Failure Domain** is a group of OSDs, from the simultaneous failure of which you are
+  protected by Vitastor. Default failure domain is "host" (server), but you choose a
+  larger (for example, a rack of servers) or smaller (a single drive) failure domain
+  for every pool.
+- **Placement Tree** (similar to Ceph CRUSH Tree) groups OSDs in a hierarchy to later
+  split them into Failure Domains.
+
+## Client-side components
+
+- **Client library** encapsulates client I/O logic. Client library connects to etcd and to all OSDs,
+  receives cluster state from etcd, sends read and write requests directly to all OSDs. Due
+  to the symmetric distributed architecture, all data blocks (each 128 KB by default) are placed
+  to different OSDs, but clients always know where each data block is stored and connect directly
+  to the right OSD.
+
+All other client-side components are based on the client library:
+
+- **[vitastor-cli](../usage/cli.en.md)** — command-line utility for cluster management.
+  Allows to view cluster state, manage pools and images, i.e. create, modify and remove
+  virtual disks, their snapshots and clones.
+- **[QEMU driver](../usage/qemu.en.md)** — pluggable QEMU module allowing QEMU/KVM virtual
+  machines work with virtual Vitastor disks directly from userspace through the client library,
+  without the need to attach disks as kernel block devices. However, if you want to attach
+  disks, you can also do that with the same driver and [VDUSE](../usage/qemu.en.md#vduse).
+- **[vitastor-nbd](../usage/nbd.en.md)** — utility that allows to attach Vitastor disks as
+  kernel block devices using NBD (Network Block Device), which works more like "BUSE"
+  (Block Device In Userspace). Vitastor doesn't have Linux kernel modules for the same task
+  (at least by now). NBD is an older, non-recommended way to attach disks — you should use
+  VDUSE whenever you can.
+- **[CSI driver](../installation/kubernetes.en.md)** — driver for attaching Vitastor images
+  and VitastorFS subdirectories as Kubernetes persistent volumes. Block-based CSI uses
+  VDUSE (when available) or NBD — images are attached as kernel block devices and mounted
+  into containers. FS-based CSI uses **[vitastor-nfs](../usage/nfs.en.md)**.
+- **Drivers for Proxmox, OpenStack and so on** — pluggable modules for corresponding systems,
+  allowing to use Vitastor as storage in them.
+- **[vitastor-nfs](../usage/nfs.en.md)** — NFS 3.0 server allowing export of two file system variants:
+  the first is a simplified pseudo-FS for file-based access to Vitastor block images (for non-QEMU
+  hypervisors with NFS support), the second is **VitastorFS**, full-featured clustered POSIX FS.
+  Both variants support parallel access from multiple vitastor-nfs servers. In fact, you are
+  not required to setup separate NFS servers at all and use vitastor-nfs mount command on every
+  client node — it starts the NFS server and mounts the FS locally.
+- **[fio driver](../usage/fio.en.md)** — pluggable module for fio disk benchmarking tool for
+  running performance tests on your Vitastor cluster.
+- **vitastor-kv** — client for a key-value DB working over shared block volumes (usual
+  vitastor images). VitastorFS metadata is stored in vitastor-kv.
+
+## Additional utilities
+
+- **vitastor-disk** — a Vitastor OSD disk management tool. You can create, remove,
+  resize and move OSD partitions with it.
+
+## Overall read/write process
+
+- Vitastor stores virtual disks, also named "images" or "inodes".
+- Each image is stored in some pool. Pool specifies storage parameters such as redundancy
+  scheme (replication or EC — erasure codes, i.e. error correction codes), failure domain
+  and restrictions on OSD selection for image data placement. See [Pool configuration](../config/pool.en.md) for details.
+- Each image is split into objects/blocks of fixed size, equal to [block_size](../config/layout-cluster.en.md#block_size)
+  (128 KB by default), multiplied by data part count for EC or 1 for replicas. That is,
+  if a pool uses EC 4+2 coding scheme (4 data parts + 2 parity parts), then, with the
+  default block_size, images are split into 512 KB objects.
+- Client read/write requests are split into parts at object boundaries.
+- Each object is mapped to a PG number it belongs to, by simply taking a remainder of
+  division of its offset by PG count of the image's pool.
+- Client reads primary OSD for all PGs from etcd. Primary OSD for each PG is assigned
+  by the monitor during cluster operation, along with the full PG OSD set.
+- If not already connected, client connects to primary OSDs of all PGs involved in a
+  read/write request and sends parts of the request to them.
+- If a primary OSD is unavailable, client retries connection attempts indefinitely
+  either until it becomes available or until the monitor assigns another OSD as primary
+  for that PG.
+- Client also retries requests if the primary OSD replies with error code EPIPE, meaning
+  that the PG is inactive at this OSD at the moment - for example, when the primary OSD
+  is switched, or if the primary OSD itself loses connection to replicas during request
+  handling.
+- Primary OSD determines where the parts of the object are stored. By default, all objects
+  are assumed to be stored at the target OSD set of a PG, but some of them may be present
+  at a different OSD set if they are degraded or moved, or if the data rebalancing process
+  is active. OSDs doesn't do any network requests, if calculates locations of all objects
+  during PG activation and stores it in memory.
+- Primary OSD handles the request locally when it can - for example, when it's a read
+  from a replicated pool or when it's a read from a EC pool involving only one data part
+  stored on the OSD's local disk.
+- When a request requires reads or writes to additional OSDs, primary OSD uses already
+  established connections to secondary OSDs of the PG to execute these requests. This happens
+  in parallel to local disk operations. All such connections are guaranteed to be already
+  established when the PG is active, and if any of them is dropped, PG is restarted and
+  all current read/write operations to it fail with EPIPE error and are retried by clients.
+- After completing all secondary read/write requests, primary OSD sends the response to
+  the client.
+
+### Nuances of request handling
+
+- If a pool uses erasure codes and some of the OSDs are unavailable, primary OSDs recover
+  data from the remaining parts during read.
+- Each object has a version number. During write, primary OSD first determines the current
+  version of the object. As primary OSD usually stores the object or its part itself, most
+  of the time version is read from the memory of the OSD itself. However, if primary OSD
+  doesn't contain parts of the object, it requests the version number from a secondary OSD
+  which has that part. Such request still doesn't involve reading from the disk though,
+  because object metadata, including version number, is always stored in OSD memory.
+- If a pool uses erasure codes, partial writes of an object require reading other parts of
+  it from secondary OSDs or from the local disk of the primary OSD itself. This is called
+  "read-modify-write" process.
+- If a pool uses erasure codes, two-phase write process is used to get rid of the Write Hole
+  problem: first a new version of object parts is written to all secondary OSDs without
+  removing the previous version, and then, after receiving successful write confirmations
+  from all OSDs, new version is committed and the old one is allowed to be removed.
+- In a pool doesn't use immediate_commit mode, then write requests sent by clients aren't
+  treated as committed to physical media instantly. Clients have to send separate type of
+  requests (SYNC) to commit changes, and before it isn't sent, new versions of data are
+  allowed to be lost if some OSDs die. Thus, when immediate_commit is disabled, clients
+  store copies of all write requests in memory and repeat them from there when the
+  connection to primary OSD is lost. This in-memory copy is removed after a successful
+  SYNC, and to prevent excessive memory usage, clients also do an automatic SYNC
+  every [client_dirty_limit](../config/network.en.md#client_dirty_limit) written bytes.
 
 ## Similarities to Ceph
 
@@ -87,5 +219,5 @@
 - Deleting images in a degraded cluster may currently lead to objects reappearing
   after dead OSDs come back, and in case of erasure-coded pools, they may even
   reappear as incomplete. Just repeat the removal request again in this case.
-  This problem will be fixed in the nearest future, the fix is already implemented
-  in the "epoch-deletions" branch.
+  This problem will be fixed in the future, along with the metadata disk storage
+  format update.

docs/intro/architecture.ru.md

@@ -11,6 +11,7 @@
 - [Серверные компоненты](#серверные-компоненты)
 - [Базовые понятия](#базовые-понятия)
 - [Клиентские компоненты](#клиентские-компоненты)
+- [Дополнительные утилиты](#дополнительные-утилиты)
 - [Общий процесс записи и чтения](#общий-процесс-записи-и-чтения)
   - [Особенности обработки запросов](#особенности-обработки-запросов)
 - [Схожесть с Ceph](#схожесть-с-ceph)
@@ -23,8 +24,8 @@
   Один OSD управляет одним диском (или разделом). OSD общаются с etcd и друг с другом — от etcd они
   получают состояние кластера, а друг другу передают запросы записи и чтения вторичных копий данных.
 - **etcd** — кластерная key/value база данных, используется для хранения настроек и верхнеуровневого
-  состояния кластера, а также предотвращения разделения сознания. Блоки данных в etcd не хранятся,
-  в обработке клиентских запросов чтения и записи etcd не участвует.
+  состояния кластера, а также предотвращения разделения сознания (splitbrain). Блоки данных в etcd не
+  хранятся, в обработке клиентских запросов чтения и записи etcd не участвует.
 - **Монитор** — отдельный демон на node.js, рассчитывающий необходимые изменения в конфигурацию
   кластера, сохраняющий эту информацию в etcd и таким образом командующий OSD применить эти изменения.
   Также агрегирует статистику. Контактирует только с etcd, OSD с монитором не общаются.
@@ -34,40 +35,56 @@
 - **Пул (Pool)** — контейнер для данных, имеющих одну и ту же схему избыточности и правила распределения по OSD.
 - **PG (Placement Group)** — "шард", единица деления пулов в кластере, которой назначается свой набор
   OSD для хранения данных (копий или частей объектов).
-- **Домен отказа (Failure Domain)** — группа OSD, одновременное падение которых рассматривается
-  как вероятное. По умолчанию это "host" (сервер).
+- **Домен отказа (Failure Domain)** — группа OSD, от одновременного падения которых должен защищать
+  Vitastor. По умолчанию домен отказа — "host" (сервер), но вы можете установить для пула как больший
+  домен отказа (например, стойку серверов), так и меньший (например, отдельный диск).
 - **Дерево распределения** (Placement Tree, в Ceph CRUSH Tree) — иерархическая группировка OSD
   в узлы, которые далее можно использовать как домены отказа.
 
 ## Клиентские компоненты
 
-- **Клиентская библиотека** — инкапсулирует логику на стороне клиента. Соединяются с etcd и со всеми OSD,
-  от etcd получают состояние кластера, команды чтения и записи отправляют на все OSD напрямую.
+- **Клиентская библиотека** — инкапсулирует логику на стороне клиента. Соединяется с etcd и со всеми OSD,
+  от etcd получает состояние кластера, команды чтения и записи отправляет на все OSD напрямую.
   В силу архитектуры все отдельные блоки данных (по умолчанию по 128 КБ) располагается на разных
   OSD, но клиент устроен так, что всегда точно знает, к какому OSD обращаться, и подключается
   к нему напрямую.
 
 На базе клиентской библиотеки реализованы все остальные клиенты:
 
-- **vitastor-cli** — утилита командной строки для управления кластером. В данный момент позволяет
-  просматривать общее состояние кластера и управлять образами — т.е. создавать, менять и удалять
-  виртуальные диски, их снимки и клоны.
-- **Драйвер QEMU** — подключаемый модуль QEMU, позволяющий QEMU/KVM виртуальным машинам работать
-  с виртуальными дисками Vitastor напрямую из пространства пользователя с помощью клиентской
-  библиотеки, без необходимости отображения дисков в виде блочных устройств. Тот же драйвер
-  позволяет подключать диски в систему через [VDUSE](../usage/qemu.ru.md#vduse).
-- **vitastor-nbd** — утилита, позволяющая монтировать образы Vitastor в виде блочных устройств
-  с помощью NBD (Network Block Device), на самом деле скорее работающего как "BUSE"
-  (Block Device In Userspace). Модуля ядра Linux для выполнения той же задачи в Vitastor нет
-  (по крайней мере, пока).
-- **CSI драйвер** — драйвер для подключения Vitastor-образов в виде персистентных томов (PV) Kubernetes.
-  Работает через vitastor-nbd — образы отражаются в виде блочных устройств и монтируются
-  в контейнеры.
+- **[vitastor-cli](../usage/cli.ru.md)** — утилита командной строки для управления кластером.
+  Позволяет просматривать общее состояние кластера, управлять пулами и образами — то есть
+  создавать, менять и удалять виртуальные диски, их снимки и клоны.
+- **[Драйвер QEMU](../usage/qemu.ru.md)** — подключаемый модуль QEMU, позволяющий QEMU/KVM
+  виртуальным машинам работать с виртуальными дисками Vitastor напрямую из пространства пользователя
+  с помощью клиентской библиотеки, без необходимости подключения дисков в виде блочных устройств
+  Linux. Если, однако, вы хотите подключать диски в виде блочных устройств, то вы тоже можете
+  сделать это с помощью того же самого драйвера и [VDUSE](../usage/qemu.ru.md#vduse).
+- **[vitastor-nbd](../usage/nbd.ru.md)** — утилита, позволяющая монтировать образы Vitastor
+  в виде блочных устройств с помощью NBD (Network Block Device), на самом деле скорее работающего
+  как "BUSE" (Block Device In Userspace). Модуля ядра Linux для выполнения той же задачи в
+  Vitastor нет (по крайней мере, пока). NBD — более старый и нерекомендуемый способ подключения
+  дисков — вам следует использовать VDUSE всегда, когда это возможно.
+- **[CSI драйвер](../installation/kubernetes.ru.md)** — драйвер для подключения Vitastor-образов
+  и поддиректорий VitastorFS в виде персистентных томов (PV) Kubernetes. Блочный CSI работает через
+  VDUSE (когда это возможно) или через NBD — образы отражаются в виде блочных устройств и монтируются
+  в контейнеры. Файловый CSI использует **[vitastor-nfs](../usage/nfs.ru.md)**.
 - **Драйвера Proxmox, OpenStack и т.п.** — подключаемые модули для соответствующих систем,
   позволяющие использовать Vitastor как хранилище в оных.
-- **vitastor-nfs** — утилита, предоставляющая файловый доступ к образам в кластере Vitastor
-  по протоколу NFS 3.0. Предназначена для гипервизоров, не основанных на QEMU и Linux, но при
-  этом поддерживающих NFS.
+- **[vitastor-nfs](../usage/nfs.ru.md)** — NFS 3.0 сервер, предоставляющий два варианта файловой системы:
+  первая — упрощённая для файлового доступа к блочным образам (для не-QEMU гипервизоров, поддерживающих NFS),
+  вторая — VitastorFS, полноценная кластерная POSIX ФС. Оба варианта поддерживают параллельный
+  доступ с нескольких vitastor-nfs серверов. На самом деле можно вообще не выделять
+  отдельные NFS-серверы, а вместо этого использовать команду vitastor-nfs mount, запускающую
+  NFS-сервер прямо на клиентской машине и монтирующую ФС локально.
+- **[Драйвер fio](../usage/fio.ru.md)** — подключаемый модуль для утилиты тестирования
+  производительности дисков fio, позволяющий тестировать Vitastor-кластеры.
+- **vitastor-kv** — клиент для key-value базы данных, работающей поверх разделяемого блочного
+  образа (обычного блочного образа vitastor). Метаданные VitastorFS хранятся именно в vitastor-kv.
+
+## Дополнительные утилиты
+
+- **vitastor-disk** — утилита для разметки дисков под Vitastor OSD. С её помощью можно
+  создавать, удалять, менять размеры или перемещать разделы OSD.
 
 ## Общий процесс записи и чтения
 
@@ -98,16 +115,22 @@
   находиться на других OSD, если эти объекты деградированы или перемещены, или идёт процесс
   ребаланса. Запросы для проверки по сети не отправляются, информация о местоположении всех
   объектов рассчитывается первичным OSD при активации PG и хранится в памяти.
-- Первичный OSD соединяется (если ещё не соединён) с вторичными OSD, на которых располагаются
-  части объекта, и отправляет им запросы чтения/записи, а также читает/пишет из/в своё локальное
-  хранилище, если сам входит в набор.
+- Когда это возможно, первичный OSD обрабатывает запрос локально. Например, так происходит
+  при чтениях объектов из пулов с репликацией или при чтении из EC пула, затрагивающего
+  только часть, хранимую на диске самого первичного OSD.
+- Когда запрос требует записи или чтения с вторичных OSD, первичный OSD использует заранее
+  установленные соединения с ними для выполнения этих запросов. Это происходит параллельно
+  локальным операциям чтения/записи с диска самого OSD. Так как соединения к вторичным OSD PG
+  устанавливаются при её запуске, то они уже гарантированно установлены, когда PG активна,
+  и если любое из этих соединений отключается, PG перезапускается, а все текущие запросы чтения
+  и записи в неё завершаются с ошибкой EPIPE, после чего повторяются клиентами.
 - После завершения всех вторичных операций чтения/записи первичный OSD отправляет ответ клиенту.
 
 ### Особенности обработки запросов
 
 - Если в пуле используются коды коррекции ошибок и при этом часть OSD недоступна, первичный
   OSD при чтении восстанавливает данные из оставшихся частей.
-- Каждый объект имеет номер версии. При записи объекта первичный OSD сначала читает из номер
+- Каждый объект имеет номер версии. При записи объекта первичный OSD сначала получает номер
   версии объекта. Так как первичный OSD обычно сам хранит копию или часть объекта, номер
   версии обычно читается из памяти самого OSD. Однако, если ни одна часть обновляемого объекта
   не находится на первичном OSD, для получения номера версии он обращается к одному из вторичных
@@ -115,20 +138,20 @@
   так как метаданные объектов, включая номер версии, все OSD хранят в памяти.
 - Если в пуле используются коды коррекции ошибок, перед частичной записью объекта для вычисления
   чётности зачастую требуется чтение частей объекта с вторичных OSD или с локального диска
-  самого первичного OSD.
-- Также, если в пуле используются коды коррекции ошибок, для закрытия Write Hole применяется
+  самого первичного OSD. Это называется процессом "чтение-модификация-запись" (read-modify-write).
+- Если в пуле используются коды коррекции ошибок, для закрытия Write Hole применяется
   двухфазный алгоритм записи: сначала на все вторичные OSD записывается новая версия частей
   объекта, но при этом старая версия не удаляется, а потом, после получения подтверждения
   успешной записи от всех вторичных OSD, новая версия фиксируется и разрешается удаление старой.
-- Если в кластере не включён режим immediate_commit, то запросы записи, отправляемые клиентами,
+- Если в пуле не включён режим immediate_commit, то запросы записи, отправляемые клиентами,
   не считаются зафиксированными на физических накопителях сразу. Для фиксации данных клиенты
   должны отдельно отправлять запросы SYNC (отдельный от чтения и записи вид запроса),
   а пока такой запрос не отправлен, считается, что записанные данные могут исчезнуть,
   если соответствующий OSD упадёт. Поэтому, когда режим immediate_commit отключён, все
   запросы записи клиенты копируют в памяти и при потере соединения и повторном соединении
-  с OSD повторяют из памяти. Скопированные в память данные удаляются при успешном fsync,
+  с OSD повторяют из памяти. Скопированные в память данные удаляются при успешном SYNC,
   а чтобы хранение этих данных не приводило к чрезмерному потреблению памяти, клиенты
-  автоматически выполняют fsync каждые [client_dirty_limit](../config/network.ru.md#client_dirty_limit)
+  автоматически выполняют SYNC каждые [client_dirty_limit](../config/network.ru.md#client_dirty_limit)
   записанных байт.
 
 ## Схожесть с Ceph
@@ -205,5 +228,5 @@
 - Удаление образов в деградированном кластере может в данный момент приводить к повторному
   "появлению" удалённых объектов после поднятия отключённых OSD, причём в случае EC-пулов,
   объекты могут появиться в виде "неполных". Если вы столкнётесь с такой ситуацией, просто
-  повторите запрос удаления. Исправление этой проблемы уже реализовано в ветке "epoch-deletions"
-  и вскоре будет включено в релиз.
+  повторите запрос удаления. Данная проблема будет исправлена в будущем вместе с обновлением
+  дискового формата хранения метаданных.

docs/intro/features.en.md

@@ -36,6 +36,7 @@
 - [Clustered file system](../usage/nfs.en.md#vitastorfs)
 - [Experimental internal etcd replacement - antietcd](../config/monitor.en.md#use_antietcd)
 - [Built-in Prometheus metric exporter](../config/monitor.en.md#enable_prometheus)
+- [NFS RDMA support](../usage/nfs.en.md#rdma) (probably also usable for GPUDirect)
 
 ## Plugins and tools
 

docs/intro/features.ru.md

@@ -38,6 +38,7 @@
 - [Кластерная файловая система](../usage/nfs.ru.md#vitastorfs)
 - [Экспериментальная встроенная замена etcd - antietcd](../config/monitor.ru.md#use_antietcd)
 - [Встроенный Prometheus-экспортер метрик](../config/monitor.ru.md#enable_prometheus)
+- [Поддержка NFS RDMA](../usage/nfs.ru.md#rdma) (вероятно, также подходящая для GPUDirect)
 
 ## Драйверы и инструменты
 

docs/intro/quickstart.en.md

@@ -32,7 +32,7 @@
 
 - SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel D3-S4510/4520/4610/4620, Kingston DC500M
 - NVMe: Micron 9100/9200/9300/9400, Micron 7300/7450, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
-  Intel DC-P3700/P4500/P4600, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
+  Intel DC-P3700/P4500/P4600, Intel D5-P4320, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
 - HDD: HGST Ultrastar, Toshiba MG, Seagate EXOS
 
 ## Configure monitors

docs/intro/quickstart.ru.md

@@ -32,7 +32,7 @@
 
 - SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel D3-S4510/4520/4610/4620, Kingston DC500M
 - NVMe: Micron 9100/9200/9300/9400, Micron 7300/7450, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
-  Intel DC-P3700/P4500/P4600, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
+  Intel DC-P3700/P4500/P4600, Intel D5-P4320, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
 - HDD: HGST Ultrastar, Toshiba MG, Seagate EXOS
 
 ## Настройте мониторы

docs/usage/admin.en.md

@@ -58,8 +58,9 @@ and during switching primary OSD of PGs.
 
 **starting**, **repeering**, **stopping** states normally almost aren't visible at all.
 If you notice them for any noticeable time — chances are some operations on some OSDs hung.
-Search for "slow op" in OSD logs to find them — operations hung for more than
-[slow_log_interval](../config/osd.en.md#slow_log_interval) are logged as "slow ops".
+Check `vitastor-cli status` and search for "slow op" in OSD logs to find them — operations
+hung for more than [slow_log_interval](../config/osd.en.md#slow_log_interval) are logged as
+"slow ops" and displayed in `status`.
 
 State transition diagram:
 
@@ -171,7 +172,14 @@ to make them use the new version of the client library.
 
 ### 1.7.x to 1.8.0
 
-After upgrading version <= 1.7.x to version >= 1.8.0, BUT <= 1.9.0: restart all clients
+It's recommended to upgrade from version <= 1.7.x to version >= 1.8.0 with full downtime,
+i.e. you should first stop clients and then the cluster (OSDs and monitor), because 1.8.0
+includes a fix for etcd event stream inconsistency which could lead to "incomplete" objects
+appearing in EC pools, and in rare cases, probably, even to data corruption during mass OSD
+restarts. It doesn't mean that you WILL hit this problem if you upgrade without full downtime,
+but it's better to secure yourself against it.
+
+Also, if you upgrade version from <= 1.7.x to version >= 1.8.0, BUT <= 1.9.0: restart all clients
 (VMs and so on), otherwise they will hang when monitor clears old PG configuration key,
 which happens 24 hours after upgrade.
 

docs/usage/admin.ru.md

@@ -56,9 +56,9 @@ OSD, на протяжении небольшого периода времен
 Состояния **starting**, **repeering**, **stopping** в норме практически не заметны вообще,
 PG должны очень быстро переходить из них в другие. Если эти состояния заметны
 хоть сколько-то значительное время — вероятно, какие-то операции на каких-то OSD зависли.
-Чтобы найти их, ищите "slow op" в журналах OSD — операции, зависшие дольше,
-чем на [slow_log_interval](../config/osd.ru.md#slow_log_interval), записываются в
-журналы OSD как "slow op".
+Чтобы найти их, посморите `vitastor-cli status` и поищите слова "slow op" в журналах OSD —
+операции, зависшие дольше, чем на [slow_log_interval](../config/osd.ru.md#slow_log_interval),
+записываются в журналы OSD как "slow op" и отображаются в `status`.
 
 Диаграмма переходов:
 
@@ -168,7 +168,14 @@ done
 
 ### 1.7.x -> 1.8.0
 
-После обновления с версий <= 1.7.x до версий >= 1.8.0, НО <= 1.9.0: перезапустите всех
+Обновляться с версий <= 1.7.x до версий >= 1.8.0 рекомендуется с полной остановкой
+сначала клиентов, а затем кластера, так как в 1.8.0 исправлена проблема (неконсистентность
+потоков событий от etcd), способная приводить к появлению incomplete объектов в EC-пулах
+и, хоть и редко, но даже к повреждению данных при массовых перезапусках OSD. Если вы
+обновляетесь без полной остановки - это не значит, что вы обязательно столкнётесь с этой
+проблемой, но лучше подстраховаться.
+
+Также, если вы обновляетесь с версии <= 1.7.x до версии >= 1.8.0, НО <= 1.9.0: перезапустите всех
 клиентов (процессы виртуальных машин можно перезапустить путём миграции на другой сервер),
 иначе они зависнут, когда монитор удалит старый ключ конфигурации PG, что происходит через
 24 часа после обновления.

docs/usage/disk.en.md

@@ -51,12 +51,16 @@ Options (automatic mode):
 --osd_per_disk <N>
   Create <N> OSDs on each disk (default 1)
 --hybrid
-  Prepare hybrid (HDD+SSD) OSDs using provided devices. SSDs will be used for
-  journals and metadata, HDDs will be used for data. Partitions for journals and
-  metadata will be created automatically. Whether disks are SSD or HDD is decided
-  by the `/sys/block/.../queue/rotational` flag. In hybrid mode, default object
-  size is 1 MB instead of 128 KB, default journal size is 1 GB instead of 32 MB,
-  and throttle_small_writes is enabled by default.
+  Prepare hybrid (HDD+SSD, NVMe+SATA or etc) OSDs using provided devices. By default,
+  any passed SSDs will be used for journals and metadata, HDDs will be used for data,
+  but you can override this behaviour with --fast-devices option. Journal and metadata
+  partitions will be created automatically. In the default mode, SSD and HDD disks
+  are distinguished by the `/sys/block/.../queue/rotational` flag. When HDDs are used
+  for data in hybrid mode, default block_size is 1 MB instead of 128 KB, default journal
+  size is 1 GB instead of 32 MB, and throttle_small_writes is enabled by default.
+--fast-devices /dev/nvmeX,/dev/nvmeY
+  In --hybrid mode, use these devices for journal and metadata instead of auto-detecting
+  and extracting them from the main [devices...] list.
 --disable_data_fsync auto
   Disable data device cache and fsync (1/yes/true = on, default auto)
 --disable_meta_fsync auto

docs/usage/disk.ru.md

@@ -51,12 +51,17 @@ vitastor-disk - инструмент командной строки для уп
 --osd_per_disk <N>
   Создавать по несколько (<N>) OSD на каждом диске (по умолчанию 1)
 --hybrid
-  Инициализировать гибридные (HDD+SSD) OSD на указанных дисках. SSD будут
-  использованы для журналов и метаданных, а HDD - для данных. Разделы для журналов
-  и метаданных будут созданы автоматически. Является ли диск SSD или HDD, определяется
-  по флагу `/sys/block/.../queue/rotational`. В гибридном режиме по умолчанию
-  используется размер объекта 1 МБ вместо 128 КБ, размер журнала 1 ГБ вместо 32 МБ
-  и включённый throttle_small_writes.
+  Инициализировать гибридные (HDD+SSD, NVMe+SATA и т.п.) OSD на указанных дисках.
+  По умолчанию, SSD будут использованы для журналов и метаданных, а HDD - для данных,
+  но вы можете поменять это поведение опцией --fast-devices. Разделы для журналов
+  и метаданных будут созданы автоматически. В режиме по умолчанию SSD и HDD-диски
+  различаются по флагу `/sys/block/.../queue/rotational`. Когда в гибридном режиме
+  для данных используются HDD, по умолчанию размер блока устанавливается 1 МБ вместо
+  128 КБ, размер журнала 1 ГБ вместо 32 МБ, и throttle_small_writes включается по
+  умолчанию.
+--fast-devices /dev/nvmeX,/dev/nvmeY
+  Использовать данные диски для журналов и метаданных в гибридном режиме вместо их
+  автоопределения и извлечения из основного списка [devices...].
 --disable_data_fsync auto
   Отключать кэш и fsync-и для устройств данных. (1/yes/true = да, по умолчанию автоопределение)
 --disable_meta_fsync auto

docs/usage/nbd.en.md

@@ -96,7 +96,7 @@ Example output (JSON format):
 vitastor-nbd netlink-map [/dev/nbdN] (--image <image> | --pool <pool> --inode <inode> --size <size in bytes>)
 ```
 
-On recent kernel versions it's also possinle to map NBD devices using netlink interface.
+On recent kernel versions it's also possible to map NBD devices using netlink interface.
 
 This is an experimental feature because it doesn't solve all issues of NBD. Differences from regular ioctl-based 'map':
 

docs/usage/nfs.en.md

@@ -111,6 +111,21 @@ settings, because Vitastor NFS proxy doesn't keep uncommitted data in memory
 with these settings. But it may even work without `immediate_commit=all` because
 the Linux NFS client repeats all uncommitted writes if it loses the connection.
 
+## RDMA
+
+vitastor-nfs supports NFS over RDMA, which, in theory, should also allow to use
+VitastorFS from GPUDirect.
+
+You can test NFS-RDMA even if you don't have an RDMA NIC using SoftROCE:
+
+1. First, add SoftROCE device on both servers: `rdma link add rxe0 type rxe netdev eth0`.
+   Here, `rdma` utility is a part the iproute2 package, and `eth0` should be replaced with
+   the name of your Ethernet NIC.
+
+2. Start vitastor-nfs with RDMA: `vitastor-nfs start (--fs <NAME> | --block) --pool <POOL> --port 20049 --nfs_rdma 20049 --portmap 0`
+
+3. Mount the FS: `mount 192.168.0.10:/mnt/test/ /mnt/vita/ -o port=20049,mountport=20049,nfsvers=3,soft,nolock,rdma`
+
 ## Commands
 
 ### mount
@@ -131,11 +146,16 @@ The server will be automatically stopped when the FS is unmounted.
 
 Start network NFS server. Options:
 
-| <!-- -->        | <!-- -->                                                   |
-|-----------------|------------------------------------------------------------|
-| `--bind <IP>`   | bind service to \<IP> address (default 0.0.0.0)            |
-| `--port <PORT>` | use port \<PORT> for NFS services (default is 2049)        |
-| `--portmap 0`   | do not listen on port 111 (portmap/rpcbind, requires root) |
+| <!-- -->               | <!-- -->                                                                                                                    |
+|------------------------|-----------------------------------------------------------------------------------------------------------------------------|
+| `--bind <IP>`          | bind service to \<IP> address (default 0.0.0.0)                                                                             |
+| `--port <PORT>`        | use port \<PORT> for NFS services (default is 2049). Specify "auto" to auto-select and print port                           |
+| `--portmap 0`          | do not listen on port 111 (portmap/rpcbind, requires root)                                                                  |
+| `--nfs_rdma <PORT>`    | enable NFS-RDMA at RDMA-CM port \<PORT> (you can try 20049). If RDMA is enabled and --port is set to 0, TCP will be disabled |
+| `--nfs_rdma_credit 16` | maximum operation credit for RDMA clients (max iodepth)                                                                     |
+| `--nfs_rdma_send 1024` | maximum RDMA send operation count (should be larger than iodepth)                                                           |
+| `--nfs_rdma_alloc 1M`  | RDMA memory allocation rounding                                                                                             |
+| `--nfs_rdma_gc 64M`    | maximum unused RDMA buffers                                                                                                 |
 
 ### upgrade
 

docs/usage/nfs.ru.md

@@ -116,6 +116,21 @@ JSON-формате :-). Для инспекции содержимого БД
 даже без `immediate_commit=all`, потому что NFS-клиент ядра Linux повторяет все
 незафиксированные запросы при потере соединения.
 
+## RDMA
+
+vitastor-nfs поддерживает NFS через RDMA. В теории это также должно позволять использовать
+VitastorFS из GPUDirect.
+
+Вы можете протестировать NFS-RDMA, даже если у вас нет RDMA-карты, с помощью SoftROCE:
+
+1. Сначала создайте SoftROCE устройства на обоих тестовых серверах: `rdma link add rxe0 type rxe netdev eth0`.
+   Утилита `rdma` входит в состав пакета iproute2, а `eth0` вам нужно заменить на имя своей
+   сетевой карты.
+
+2. Запустите vitastor-nfs с RDMA: `vitastor-nfs start (--fs <NAME> | --block) --pool <POOL> --port 20049 --nfs_rdma 20049 --portmap 0`
+
+3. Смонтируйте ФС: `mount 192.168.0.10:/mnt/test/ /mnt/vita/ -o port=20049,mountport=20049,nfsvers=3,soft,nolock,rdma`
+
 ## Команды
 
 ### mount
@@ -136,11 +151,16 @@ JSON-формате :-). Для инспекции содержимого БД
 
 Запустить сетевой NFS-сервер. Опции:
 
-| <!-- -->        | <!-- -->                                                              |
-|-----------------|-----------------------------------------------------------------------|
-| `--bind <IP>`   | принимать соединения по адресу \<IP> (по умолчанию 0.0.0.0 - на всех) |
-| `--port <PORT>` | использовать порт \<PORT> для NFS-сервисов (по умолчанию 2049)        |
-| `--portmap 0`   | отключить сервис portmap/rpcbind на порту 111 (по умолчанию включён и требует root привилегий) |
+| <!-- -->               | <!-- -->                                                                                                                    |
+|------------------------|-----------------------------------------------------------------------------------------------------------------------------|
+| `--bind <IP>`          | принимать соединения по адресу \<IP> (по умолчанию 0.0.0.0 - на всех)                                                       |
+| `--port <PORT>`        | использовать порт \<PORT> для NFS-сервисов (по умолчанию 2049). Укажите "auto", чтобы выбрать и напечатать случайный порт   |
+| `--portmap 0`          | отключить сервис portmap/rpcbind на порту 111 (по умолчанию включён и требует root привилегий)                              |
+| `--nfs_rdma <PORT>`    | включить NFS-RDMA на порту RDMA-CM \<PORT> (попробуйте 20049). Если RDMA включено и указано `--port 0`, TCP будет отключено |
+| `--nfs_rdma_credit 16` | максимальный "кредит", глубина очереди для NFS-клиентов                                                                     |
+| `--nfs_rdma_send 1024` | максимальное число операций RDMA отправки (должно быть больше nfs_rdma_credit)                                              |
+| `--nfs_rdma_alloc 1M`  | округление выделения памяти для RDMA-клиентов                                                                               |
+| `--nfs_rdma_gc 64M`    | максимальный объём неиспользуемой памяти RDMA-клиентом перед освобождением                                                  |
 
 ### upgrade
 

mon/etcd_adapter.js

@@ -232,6 +232,7 @@ class EtcdAdapter
     async become_master()
     {
         const state = { ...this.mon.get_mon_state(), id: ''+this.mon.etcd_lease_id };
+        console.log('Waiting to become master');
         // eslint-disable-next-line no-constant-condition
         while (1)
         {
@@ -243,7 +244,6 @@ class EtcdAdapter
             {
                 break;
             }
-            console.log('Waiting to become master');
             await new Promise(ok => setTimeout(ok, this.mon.config.etcd_start_timeout));
         }
         console.log('Became master');

mon/etcd_schema.js

@@ -56,6 +56,7 @@ const etcd_tree = {
             osd_out_time: 600, // seconds. min: 0
             placement_levels: { datacenter: 1, rack: 2, host: 3, osd: 4, ... },
             use_old_pg_combinator: false,
+            osd_backfillfull_ratio: 0.99,
             // client and osd
             tcp_header_buffer_size: 65536,
             use_sync_send_recv: false,

mon/mon.js

@@ -74,6 +74,7 @@ class Mon
         this.state = JSON.parse(JSON.stringify(etcd_tree));
         this.prev_stats = { osd_stats: {}, osd_diff: {} };
         this.recheck_pgs_active = false;
+        this.updating_total_stats = false;
         this.watcher_active = false;
         this.old_pg_config = false;
         this.old_pg_stats_seen = false;
@@ -567,6 +568,7 @@ class Mon
 
     async apply_pool_pgs(results, up_osds, osd_tree, tree_hash)
     {
+        const etcd_request = { compare: [], success: [] };
         for (const pool_id in (this.state.pg.config||{}).items||{})
         {
             // We should stop all PGs when deleting a pool or changing its PG count
@@ -579,9 +581,24 @@ class Mon
                     return false;
                 }
             }
+            if (!this.state.config.pools[pool_id])
+            {
+                // Delete PG history and stats of the deleted pool
+                etcd_request.success.push({ requestDeleteRange: {
+                    key: b64(this.config.etcd_prefix+'/pg/history/'+pool_id+'/'),
+                    range_end: b64(this.config.etcd_prefix+'/pg/history/'+pool_id+'0'),
+                } });
+                etcd_request.success.push({ requestDeleteRange: {
+                    key: b64(this.config.etcd_prefix+'/pg/stats/'+pool_id+'/'),
+                    range_end: b64(this.config.etcd_prefix+'/pg/stats/'+pool_id+'0'),
+                } });
+                etcd_request.success.push({ requestDeleteRange: {
+                    key: b64(this.config.etcd_prefix+'/pgstats/'+pool_id+'/'),
+                    range_end: b64(this.config.etcd_prefix+'/pgstats/'+pool_id+'0'),
+                } });
+            }
         }
         const new_pg_config = JSON.parse(JSON.stringify(this.state.pg.config));
-        const etcd_request = { compare: [], success: [] };
         for (const pool_id in (new_pg_config||{}).items||{})
         {
             if (!this.state.config.pools[pool_id])
@@ -642,7 +659,19 @@ class Mon
                 this.etcd_watch_revision, pool_id, up_osds, osd_tree, real_prev_pgs, pool_res.pgs, pg_history);
         }
         new_pg_config.hash = tree_hash;
-        return await this.save_pg_config(new_pg_config, etcd_request);
+        const { backfillfull_pools, backfillfull_osds } = sum_object_counts(
+            { ...this.state, pg: { ...this.state.pg, config: new_pg_config } }, this.config
+        );
+        if (backfillfull_pools.join(',') != ((this.state.pg.config||{}).backfillfull_pools||[]).join(','))
+        {
+            this.log_backfillfull(backfillfull_osds, backfillfull_pools);
+        }
+        new_pg_config.backfillfull_pools = backfillfull_pools.length ? backfillfull_pools : undefined;
+        if (!await this.save_pg_config(new_pg_config, etcd_request))
+        {
+            return false;
+        }
+        return true;
     }
 
     async save_pg_config(new_pg_config, etcd_request = { compare: [], success: [] })
@@ -714,7 +743,7 @@ class Mon
     async update_total_stats()
     {
         const txn = [];
-        const { object_counts, object_bytes } = sum_object_counts(this.state, this.config);
+        const { object_counts, object_bytes, backfillfull_pools, backfillfull_osds } = sum_object_counts(this.state, this.config);
         let stats = sum_op_stats(this.state.osd, this.prev_stats);
         let { inode_stats, seen_pools } = sum_inode_stats(this.state, this.prev_stats);
         stats.object_counts = object_counts;
@@ -767,6 +796,27 @@ class Mon
         {
             await this.etcd.etcd_call('/kv/txn', { success: txn }, this.config.etcd_mon_timeout, 0);
         }
+        if (!this.recheck_pgs_active &&
+            backfillfull_pools.join(',') != ((this.state.pg.config||{}).backfillfull_pools||[]).join(','))
+        {
+            this.log_backfillfull(backfillfull_osds, backfillfull_pools);
+            const new_pg_config = { ...this.state.pg.config, backfillfull_pools: backfillfull_pools.length ? backfillfull_pools : undefined };
+            await this.save_pg_config(new_pg_config);
+        }
+    }
+
+    log_backfillfull(osds, pools)
+    {
+        for (const osd in osds)
+        {
+            const bf = osds[osd];
+            console.log('OSD '+osd+' may fill up during rebalance: capacity '+(bf.cap/1024n/1024n)+
+                ' MB, target user data '+(bf.clean/1024n/1024n)+' MB');
+        }
+        console.log(
+            (pools.length ? 'Pool(s) '+pools.join(', ') : 'No pools')+
+            ' are backfillfull now, applying rebalance configuration'
+        );
     }
 
     schedule_update_stats()
@@ -778,7 +828,21 @@ class Mon
         this.stats_timer = setTimeout(() =>
         {
             this.stats_timer = null;
-            this.update_total_stats().catch(console.error);
+            if (this.updating_total_stats)
+            {
+                this.schedule_update_stats();
+                return;
+            }
+            this.updating_total_stats = true;
+            try
+            {
+                this.update_total_stats().catch(console.error);
+            }
+            catch (e)
+            {
+                console.error(e);
+            }
+            this.updating_total_stats = false;
         }, this.config.mon_stats_timeout);
     }
 

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "1.9.2",
+  "version": "1.10.0",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {

mon/stats.js

@@ -109,6 +109,8 @@ function sum_object_counts(state, global_config)
             pgstats[pool_id] = { ...(state.pg.stats[pool_id] || {}), ...(pgstats[pool_id] || {}) };
         }
     }
+    const pool_per_osd = {};
+    const clean_per_osd = {};
     for (const pool_id in pgstats)
     {
         let object_size = 0;
@@ -143,10 +145,45 @@ function sum_object_counts(state, global_config)
                         object_bytes[k] += BigInt(st[k+'_count']) * object_size;
                     }
                 }
+                if (st.object_count)
+                {
+                    for (const pg_osd of (((state.pg.config.items||{})[pool_id]||{})[pg_num]||{}).osd_set||[])
+                    {
+                        if (!(pg_osd in clean_per_osd))
+                        {
+                            clean_per_osd[pg_osd] = 0n;
+                        }
+                        clean_per_osd[pg_osd] += BigInt(st.object_count);
+                        pool_per_osd[pg_osd] = pool_per_osd[pg_osd]||{};
+                        pool_per_osd[pg_osd][pool_id] = true;
+                    }
+                }
             }
         }
     }
-    return { object_counts, object_bytes };
+    // If clean_per_osd[osd] is larger than osd capacity then it will fill up during rebalance
+    let backfillfull_pools = {};
+    let backfillfull_osds = {};
+    for (const osd in clean_per_osd)
+    {
+        const st = state.osd.stats[osd];
+        if (!st || !st.size || !st.data_block_size)
+        {
+            continue;
+        }
+        let cap = BigInt(st.size)/BigInt(st.data_block_size);
+        cap = cap * BigInt((global_config.osd_backfillfull_ratio||0.99)*1000000) / 1000000n;
+        if (cap < clean_per_osd[osd])
+        {
+            backfillfull_osds[osd] = { cap: BigInt(st.size), clean: clean_per_osd[osd]*BigInt(st.data_block_size) };
+            for (const pool_id in pool_per_osd[osd])
+            {
+                backfillfull_pools[pool_id] = true;
+            }
+        }
+    }
+    backfillfull_pools = Object.keys(backfillfull_pools).sort();
+    return { object_counts, object_bytes, backfillfull_pools, backfillfull_osds };
 }
 
 // sum_inode_stats(this.state, this.prev_stats)

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '1.9.2'
+VITASTOR_VERSION = '1.10.0'
 
 LOG = logging.getLogger(__name__)
 

patches/libvirt-8.0-vitastor.diff

@@ -306,12 +306,12 @@ index e5ff653a60..884ecc79ea 100644
 +        etcd = virBufferContentAndReset(&buf);
 +    }
 +
-+    if (virJSONValueObjectCreate(&ret,
-+                                 "S:etcd-host", etcd,
-+                                 "S:etcd-prefix", src->query,
-+                                 "S:config-path", src->configFile,
-+                                 "s:image", src->path,
-+                                 NULL) < 0)
++    if (virJSONValueObjectAdd(&ret,
++                              "S:etcd-host", etcd,
++                              "S:etcd-prefix", src->query,
++                              "S:config-path", src->configFile,
++                              "s:image", src->path,
++                              NULL) < 0)
 +        return NULL;
 +
 +    return ret;

patches/pve-qemu-9.0-vitastor.patch

@@ -0,0 +1,193 @@
+Index: pve-qemu-kvm-9.0.0/block/meson.build
+===================================================================
+--- pve-qemu-kvm-9.0.0.orig/block/meson.build
++++ pve-qemu-kvm-9.0.0/block/meson.build
+@@ -126,6 +126,7 @@ foreach m : [
+   [libnfs, 'nfs', files('nfs.c')],
+   [libssh, 'ssh', files('ssh.c')],
+   [rbd, 'rbd', files('rbd.c')],
++  [vitastor, 'vitastor', files('vitastor.c')],
+ ]
+   if m[0].found()
+     module_ss = ss.source_set()
+Index: pve-qemu-kvm-9.0.0/meson.build
+===================================================================
+--- pve-qemu-kvm-9.0.0.orig/meson.build
++++ pve-qemu-kvm-9.0.0/meson.build
+@@ -1452,6 +1452,26 @@ if not get_option('rbd').auto() or have_
+   endif
+ endif
+ 
++vitastor = not_found
++if not get_option('vitastor').auto() or have_block
++  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
++    required: get_option('vitastor'))
++  if libvitastor_client.found()
++    if cc.links('''
++      #include <vitastor_c.h>
++      int main(void) {
++        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
++        return 0;
++      }''', dependencies: libvitastor_client)
++      vitastor = declare_dependency(dependencies: libvitastor_client)
++    elif get_option('vitastor').enabled()
++      error('could not link libvitastor_client')
++    else
++      warning('could not link libvitastor_client, disabling')
++    endif
++  endif
++endif
++
+ glusterfs = not_found
+ glusterfs_ftruncate_has_stat = false
+ glusterfs_iocb_has_stat = false
+@@ -2254,6 +2274,7 @@ endif
+ config_host_data.set('CONFIG_OPENGL', opengl.found())
+ config_host_data.set('CONFIG_PLUGIN', get_option('plugins'))
+ config_host_data.set('CONFIG_RBD', rbd.found())
++config_host_data.set('CONFIG_VITASTOR', vitastor.found())
+ config_host_data.set('CONFIG_RDMA', rdma.found())
+ config_host_data.set('CONFIG_RELOCATABLE', get_option('relocatable'))
+ config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack'))
+@@ -4454,6 +4475,7 @@ summary_info += {'fdt support':       fd
+ summary_info += {'libcap-ng support': libcap_ng}
+ summary_info += {'bpf support':       libbpf}
+ summary_info += {'rbd support':       rbd}
++summary_info += {'vitastor support':  vitastor}
+ summary_info += {'smartcard support': cacard}
+ summary_info += {'U2F support':       u2f}
+ summary_info += {'libusb':            libusb}
+Index: pve-qemu-kvm-9.0.0/meson_options.txt
+===================================================================
+--- pve-qemu-kvm-9.0.0.orig/meson_options.txt
++++ pve-qemu-kvm-9.0.0/meson_options.txt
+@@ -194,6 +194,8 @@ option('lzo', type : 'feature', value :
+        description: 'lzo compression support')
+ option('rbd', type : 'feature', value : 'auto',
+        description: 'Ceph block device driver')
++option('vitastor', type : 'feature', value : 'auto',
++       description: 'Vitastor block device driver')
+ option('opengl', type : 'feature', value : 'auto',
+        description: 'OpenGL support')
+ option('rdma', type : 'feature', value : 'auto',
+Index: pve-qemu-kvm-9.0.0/qapi/block-core.json
+===================================================================
+--- pve-qemu-kvm-9.0.0.orig/qapi/block-core.json
++++ pve-qemu-kvm-9.0.0/qapi/block-core.json
+@@ -3481,7 +3481,7 @@
+             'raw', 'rbd',
+             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
+             'pbs',
+-            'ssh', 'throttle', 'vdi', 'vhdx',
++            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
+             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
+@@ -4591,6 +4591,28 @@
+             '*server': ['InetSocketAddressBase'] } }
+ 
+ ##
++# @BlockdevOptionsVitastor:
++#
++# Driver specific block device options for vitastor
++#
++# @image:       Image name
++# @inode:       Inode number
++# @pool:        Pool ID
++# @size:        Desired image size in bytes
++# @config-path: Path to Vitastor configuration
++# @etcd-host:   etcd connection address(es)
++# @etcd-prefix: etcd key/value prefix
++##
++{ 'struct': 'BlockdevOptionsVitastor',
++  'data': { '*inode': 'uint64',
++            '*pool': 'uint64',
++            '*size': 'uint64',
++            '*image': 'str',
++            '*config-path': 'str',
++            '*etcd-host': 'str',
++            '*etcd-prefix': 'str' } }
++
++##
+ # @ReplicationMode:
+ #
+ # An enumeration of replication modes.
+@@ -5053,6 +5075,7 @@
+       'throttle':   'BlockdevOptionsThrottle',
+       'vdi':        'BlockdevOptionsGenericFormat',
+       'vhdx':       'BlockdevOptionsGenericFormat',
++      'vitastor':   'BlockdevOptionsVitastor',
+       'virtio-blk-vfio-pci':
+                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
+                       'if': 'CONFIG_BLKIO' },
+@@ -5498,6 +5521,20 @@
+             '*encrypt' :        'RbdEncryptionCreateOptions' } }
+ 
+ ##
++# @BlockdevCreateOptionsVitastor:
++#
++# Driver specific image creation options for Vitastor.
++#
++# @location: Where to store the new image file.  This location cannot
++#     point to a snapshot.
++#
++# @size: Size of the virtual disk in bytes
++##
++{ 'struct': 'BlockdevCreateOptionsVitastor',
++  'data': { 'location':         'BlockdevOptionsVitastor',
++            'size':             'size' } }
++
++##
+ # @BlockdevVmdkSubformat:
+ #
+ # Subformat options for VMDK images
+@@ -5719,6 +5753,7 @@
+       'ssh':            'BlockdevCreateOptionsSsh',
+       'vdi':            'BlockdevCreateOptionsVdi',
+       'vhdx':           'BlockdevCreateOptionsVhdx',
++      'vitastor':       'BlockdevCreateOptionsVitastor',
+       'vmdk':           'BlockdevCreateOptionsVmdk',
+       'vpc':            'BlockdevCreateOptionsVpc'
+   } }
+Index: pve-qemu-kvm-9.0.0/scripts/ci/org.centos/stream/8/x86_64/configure
+===================================================================
+--- pve-qemu-kvm-9.0.0.orig/scripts/ci/org.centos/stream/8/x86_64/configure
++++ pve-qemu-kvm-9.0.0/scripts/ci/org.centos/stream/8/x86_64/configure
+@@ -30,7 +30,7 @@
+ --with-suffix="qemu-kvm" \
+ --firmwarepath=/usr/share/qemu-firmware \
+ --target-list="x86_64-softmmu" \
+---block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
++--block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,vitastor,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
+ --audio-drv-list="" \
+ --block-drv-ro-whitelist="vmdk,vhdx,vpc,https,ssh" \
+ --with-coroutine=ucontext \
+@@ -176,6 +176,7 @@
+ --enable-opengl \
+ --enable-pie \
+ --enable-rbd \
++--enable-vitastor \
+ --enable-rdma \
+ --enable-seccomp \
+ --enable-snappy \
+Index: pve-qemu-kvm-9.0.0/scripts/meson-buildoptions.sh
+===================================================================
+--- pve-qemu-kvm-9.0.0.orig/scripts/meson-buildoptions.sh
++++ pve-qemu-kvm-9.0.0/scripts/meson-buildoptions.sh
+@@ -168,6 +168,7 @@ meson_options_help() {
+   printf "%s\n" '  qed             qed image format support'
+   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
+   printf "%s\n" '  rbd             Ceph block device driver'
++  printf "%s\n" '  vitastor        Vitastor block device driver'
+   printf "%s\n" '  rdma            Enable RDMA-based migration'
+   printf "%s\n" '  replication     replication support'
+   printf "%s\n" '  rutabaga-gfx    rutabaga_gfx support'
+@@ -445,6 +446,8 @@ _meson_option_parse() {
+     --disable-qom-cast-debug) printf "%s" -Dqom_cast_debug=false ;;
+     --enable-rbd) printf "%s" -Drbd=enabled ;;
+     --disable-rbd) printf "%s" -Drbd=disabled ;;
++    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
++    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
+     --enable-rdma) printf "%s" -Drdma=enabled ;;
+     --disable-rdma) printf "%s" -Drdma=disabled ;;
+     --enable-relocatable) printf "%s" -Drelocatable=true ;;

patches/pve-qemu-9.1-vitastor.patch

@@ -0,0 +1,172 @@
+Index: pve-qemu-kvm-9.1.2/block/meson.build
+===================================================================
+--- pve-qemu-kvm-9.1.2.orig/block/meson.build
++++ pve-qemu-kvm-9.1.2/block/meson.build
+@@ -126,6 +126,7 @@ foreach m : [
+   [libnfs, 'nfs', files('nfs.c')],
+   [libssh, 'ssh', files('ssh.c')],
+   [rbd, 'rbd', files('rbd.c')],
++  [vitastor, 'vitastor', files('vitastor.c')],
+ ]
+   if m[0].found()
+     module_ss = ss.source_set()
+Index: pve-qemu-kvm-9.1.2/meson.build
+===================================================================
+--- pve-qemu-kvm-9.1.2.orig/meson.build
++++ pve-qemu-kvm-9.1.2/meson.build
+@@ -1516,6 +1516,26 @@ if not get_option('rbd').auto() or have_
+   endif
+ endif
+ 
++vitastor = not_found
++if not get_option('vitastor').auto() or have_block
++  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
++    required: get_option('vitastor'))
++  if libvitastor_client.found()
++    if cc.links('''
++      #include <vitastor_c.h>
++      int main(void) {
++        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
++        return 0;
++      }''', dependencies: libvitastor_client)
++      vitastor = declare_dependency(dependencies: libvitastor_client)
++    elif get_option('vitastor').enabled()
++      error('could not link libvitastor_client')
++    else
++      warning('could not link libvitastor_client, disabling')
++    endif
++  endif
++endif
++
+ glusterfs = not_found
+ glusterfs_ftruncate_has_stat = false
+ glusterfs_iocb_has_stat = false
+@@ -2367,6 +2387,7 @@ endif
+ config_host_data.set('CONFIG_OPENGL', opengl.found())
+ config_host_data.set('CONFIG_PLUGIN', get_option('plugins'))
+ config_host_data.set('CONFIG_RBD', rbd.found())
++config_host_data.set('CONFIG_VITASTOR', vitastor.found())
+ config_host_data.set('CONFIG_RDMA', rdma.found())
+ config_host_data.set('CONFIG_RELOCATABLE', get_option('relocatable'))
+ config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack'))
+@@ -4534,6 +4555,7 @@ summary_info += {'fdt support':       fd
+ summary_info += {'libcap-ng support': libcap_ng}
+ summary_info += {'bpf support':       libbpf}
+ summary_info += {'rbd support':       rbd}
++summary_info += {'vitastor support':  vitastor}
+ summary_info += {'smartcard support': cacard}
+ summary_info += {'U2F support':       u2f}
+ summary_info += {'libusb':            libusb}
+Index: pve-qemu-kvm-9.1.2/meson_options.txt
+===================================================================
+--- pve-qemu-kvm-9.1.2.orig/meson_options.txt
++++ pve-qemu-kvm-9.1.2/meson_options.txt
+@@ -194,6 +194,8 @@ option('lzo', type : 'feature', value :
+        description: 'lzo compression support')
+ option('rbd', type : 'feature', value : 'auto',
+        description: 'Ceph block device driver')
++option('vitastor', type : 'feature', value : 'auto',
++       description: 'Vitastor block device driver')
+ option('opengl', type : 'feature', value : 'auto',
+        description: 'OpenGL support')
+ option('rdma', type : 'feature', value : 'auto',
+Index: pve-qemu-kvm-9.1.2/qapi/block-core.json
+===================================================================
+--- pve-qemu-kvm-9.1.2.orig/qapi/block-core.json
++++ pve-qemu-kvm-9.1.2/qapi/block-core.json
+@@ -3477,7 +3477,7 @@
+             'raw', 'rbd',
+             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
+             'pbs',
+-            'ssh', 'throttle', 'vdi', 'vhdx',
++            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
+             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
+@@ -4588,6 +4588,28 @@
+             '*server': ['InetSocketAddressBase'] } }
+ 
+ ##
++# @BlockdevOptionsVitastor:
++#
++# Driver specific block device options for vitastor
++#
++# @image:       Image name
++# @inode:       Inode number
++# @pool:        Pool ID
++# @size:        Desired image size in bytes
++# @config-path: Path to Vitastor configuration
++# @etcd-host:   etcd connection address(es)
++# @etcd-prefix: etcd key/value prefix
++##
++{ 'struct': 'BlockdevOptionsVitastor',
++  'data': { '*inode': 'uint64',
++            '*pool': 'uint64',
++            '*size': 'uint64',
++            '*image': 'str',
++            '*config-path': 'str',
++            '*etcd-host': 'str',
++            '*etcd-prefix': 'str' } }
++
++##
+ # @ReplicationMode:
+ #
+ # An enumeration of replication modes.
+@@ -5050,6 +5072,7 @@
+       'throttle':   'BlockdevOptionsThrottle',
+       'vdi':        'BlockdevOptionsGenericFormat',
+       'vhdx':       'BlockdevOptionsGenericFormat',
++      'vitastor':   'BlockdevOptionsVitastor',
+       'virtio-blk-vfio-pci':
+                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
+                       'if': 'CONFIG_BLKIO' },
+@@ -5497,6 +5520,20 @@
+             '*encrypt' :        'RbdEncryptionCreateOptions' } }
+ 
+ ##
++# @BlockdevCreateOptionsVitastor:
++#
++# Driver specific image creation options for Vitastor.
++#
++# @location: Where to store the new image file.  This location cannot
++#     point to a snapshot.
++#
++# @size: Size of the virtual disk in bytes
++##
++{ 'struct': 'BlockdevCreateOptionsVitastor',
++  'data': { 'location':         'BlockdevOptionsVitastor',
++            'size':             'size' } }
++
++##
+ # @BlockdevVmdkSubformat:
+ #
+ # Subformat options for VMDK images
+@@ -5718,6 +5755,7 @@
+       'ssh':            'BlockdevCreateOptionsSsh',
+       'vdi':            'BlockdevCreateOptionsVdi',
+       'vhdx':           'BlockdevCreateOptionsVhdx',
++      'vitastor':       'BlockdevCreateOptionsVitastor',
+       'vmdk':           'BlockdevCreateOptionsVmdk',
+       'vpc':            'BlockdevCreateOptionsVpc'
+   } }
+Index: pve-qemu-kvm-9.1.2/scripts/meson-buildoptions.sh
+===================================================================
+--- pve-qemu-kvm-9.1.2.orig/scripts/meson-buildoptions.sh
++++ pve-qemu-kvm-9.1.2/scripts/meson-buildoptions.sh
+@@ -168,6 +168,7 @@ meson_options_help() {
+   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
+   printf "%s\n" '  qpl             Query Processing Library support'
+   printf "%s\n" '  rbd             Ceph block device driver'
++  printf "%s\n" '  vitastor        Vitastor block device driver'
+   printf "%s\n" '  rdma            Enable RDMA-based migration'
+   printf "%s\n" '  replication     replication support'
+   printf "%s\n" '  rutabaga-gfx    rutabaga_gfx support'
+@@ -444,6 +445,8 @@ _meson_option_parse() {
+     --disable-qpl) printf "%s" -Dqpl=disabled ;;
+     --enable-rbd) printf "%s" -Drbd=enabled ;;
+     --disable-rbd) printf "%s" -Drbd=disabled ;;
++    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
++    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
+     --enable-rdma) printf "%s" -Drdma=enabled ;;
+     --disable-rdma) printf "%s" -Drdma=disabled ;;
+     --enable-relocatable) printf "%s" -Drelocatable=true ;;

patches/qemu-9.1-vitastor.patch

@@ -0,0 +1,172 @@
+diff --git a/block/meson.build b/block/meson.build
+index f1262ec2ba..3cf3e23f16 100644
+--- a/block/meson.build
++++ b/block/meson.build
+@@ -114,6 +114,7 @@ foreach m : [
+   [libnfs, 'nfs', files('nfs.c')],
+   [libssh, 'ssh', files('ssh.c')],
+   [rbd, 'rbd', files('rbd.c')],
++  [vitastor, 'vitastor', files('vitastor.c')],
+ ]
+   if m[0].found()
+     module_ss = ss.source_set()
+diff --git a/meson.build b/meson.build
+index fbda17c987..3edac22aff 100644
+--- a/meson.build
++++ b/meson.build
+@@ -1510,6 +1510,26 @@ if not get_option('rbd').auto() or have_block
+   endif
+ endif
+ 
++vitastor = not_found
++if not get_option('vitastor').auto() or have_block
++  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
++    required: get_option('vitastor'))
++  if libvitastor_client.found()
++    if cc.links('''
++      #include <vitastor_c.h>
++      int main(void) {
++        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
++        return 0;
++      }''', dependencies: libvitastor_client)
++      vitastor = declare_dependency(dependencies: libvitastor_client)
++    elif get_option('vitastor').enabled()
++      error('could not link libvitastor_client')
++    else
++      warning('could not link libvitastor_client, disabling')
++    endif
++  endif
++endif
++
+ glusterfs = not_found
+ glusterfs_ftruncate_has_stat = false
+ glusterfs_iocb_has_stat = false
+@@ -2351,6 +2371,7 @@ endif
+ config_host_data.set('CONFIG_OPENGL', opengl.found())
+ config_host_data.set('CONFIG_PLUGIN', get_option('plugins'))
+ config_host_data.set('CONFIG_RBD', rbd.found())
++config_host_data.set('CONFIG_VITASTOR', vitastor.found())
+ config_host_data.set('CONFIG_RDMA', rdma.found())
+ config_host_data.set('CONFIG_RELOCATABLE', get_option('relocatable'))
+ config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack'))
+@@ -4510,6 +4531,7 @@ summary_info += {'fdt support':       fdt_opt == 'internal' ? 'internal' : fdt}
+ summary_info += {'libcap-ng support': libcap_ng}
+ summary_info += {'bpf support':       libbpf}
+ summary_info += {'rbd support':       rbd}
++summary_info += {'vitastor support':  vitastor}
+ summary_info += {'smartcard support': cacard}
+ summary_info += {'U2F support':       u2f}
+ summary_info += {'libusb':            libusb}
+diff --git a/meson_options.txt b/meson_options.txt
+index 0269fa0f16..4740ffdc27 100644
+--- a/meson_options.txt
++++ b/meson_options.txt
+@@ -194,6 +194,8 @@ option('lzo', type : 'feature', value : 'auto',
+        description: 'lzo compression support')
+ option('rbd', type : 'feature', value : 'auto',
+        description: 'Ceph block device driver')
++option('vitastor', type : 'feature', value : 'auto',
++       description: 'Vitastor block device driver')
+ option('opengl', type : 'feature', value : 'auto',
+        description: 'OpenGL support')
+ option('rdma', type : 'feature', value : 'auto',
+diff --git a/qapi/block-core.json b/qapi/block-core.json
+index aa40d44f1d..bbee6a0e9c 100644
+--- a/qapi/block-core.json
++++ b/qapi/block-core.json
+@@ -3203,7 +3203,7 @@
+             'parallels', 'preallocate', 'qcow', 'qcow2', 'qed', 'quorum',
+             'raw', 'rbd',
+             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
+-            'ssh', 'throttle', 'vdi', 'vhdx',
++            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
+             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
+@@ -4286,6 +4286,28 @@
+             '*key-secret': 'str',
+             '*server': ['InetSocketAddressBase'] } }
+ 
++##
++# @BlockdevOptionsVitastor:
++#
++# Driver specific block device options for vitastor
++#
++# @image:       Image name
++# @inode:       Inode number
++# @pool:        Pool ID
++# @size:        Desired image size in bytes
++# @config-path: Path to Vitastor configuration
++# @etcd-host:   etcd connection address(es)
++# @etcd-prefix: etcd key/value prefix
++##
++{ 'struct': 'BlockdevOptionsVitastor',
++  'data': { '*inode': 'uint64',
++            '*pool': 'uint64',
++            '*size': 'uint64',
++            '*image': 'str',
++            '*config-path': 'str',
++            '*etcd-host': 'str',
++            '*etcd-prefix': 'str' } }
++
+ ##
+ # @ReplicationMode:
+ #
+@@ -4742,6 +4764,7 @@
+       'throttle':   'BlockdevOptionsThrottle',
+       'vdi':        'BlockdevOptionsGenericFormat',
+       'vhdx':       'BlockdevOptionsGenericFormat',
++      'vitastor':   'BlockdevOptionsVitastor',
+       'virtio-blk-vfio-pci':
+                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
+                       'if': 'CONFIG_BLKIO' },
+@@ -5183,6 +5206,20 @@
+             '*cluster-size' :   'size',
+             '*encrypt' :        'RbdEncryptionCreateOptions' } }
+ 
++##
++# @BlockdevCreateOptionsVitastor:
++#
++# Driver specific image creation options for Vitastor.
++#
++# @location: Where to store the new image file.  This location cannot
++#     point to a snapshot.
++#
++# @size: Size of the virtual disk in bytes
++##
++{ 'struct': 'BlockdevCreateOptionsVitastor',
++  'data': { 'location':         'BlockdevOptionsVitastor',
++            'size':             'size' } }
++
+ ##
+ # @BlockdevVmdkSubformat:
+ #
+@@ -5405,6 +5442,7 @@
+       'ssh':            'BlockdevCreateOptionsSsh',
+       'vdi':            'BlockdevCreateOptionsVdi',
+       'vhdx':           'BlockdevCreateOptionsVhdx',
++      'vitastor':       'BlockdevCreateOptionsVitastor',
+       'vmdk':           'BlockdevCreateOptionsVmdk',
+       'vpc':            'BlockdevCreateOptionsVpc'
+   } }
+diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh
+index c97079a38c..4623f552ec 100644
+--- a/scripts/meson-buildoptions.sh
++++ b/scripts/meson-buildoptions.sh
+@@ -168,6 +168,7 @@ meson_options_help() {
+   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
+   printf "%s\n" '  qpl             Query Processing Library support'
+   printf "%s\n" '  rbd             Ceph block device driver'
++  printf "%s\n" '  vitastor        Vitastor block device driver'
+   printf "%s\n" '  rdma            Enable RDMA-based migration'
+   printf "%s\n" '  replication     replication support'
+   printf "%s\n" '  rutabaga-gfx    rutabaga_gfx support'
+@@ -444,6 +445,8 @@ _meson_option_parse() {
+     --disable-qpl) printf "%s" -Dqpl=disabled ;;
+     --enable-rbd) printf "%s" -Drbd=enabled ;;
+     --disable-rbd) printf "%s" -Drbd=disabled ;;
++    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
++    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
+     --enable-rdma) printf "%s" -Drdma=enabled ;;
+     --disable-rdma) printf "%s" -Drdma=disabled ;;
+     --enable-relocatable) printf "%s" -Drelocatable=true ;;

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.9.2
+Version:        1.10.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.9.2.el7.tar.gz
+Source0:        vitastor-1.10.0.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel
@@ -15,7 +15,7 @@ BuildRequires:  rh-nodejs12-npm
 BuildRequires:  jerasure-devel
 BuildRequires:  libisa-l-devel
 BuildRequires:  gf-complete-devel
-BuildRequires:  libibverbs-devel
+BuildRequires:  rdma-core-devel
 BuildRequires:  cmake3
 BuildRequires:  libnl3-devel
 Requires:       vitastor-osd = %{version}-%{release}

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.9.2
+Version:        1.10.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.9.2.el8.tar.gz
+Source0:        vitastor-1.10.0.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel
@@ -14,7 +14,7 @@ BuildRequires:  nodejs >= 10
 BuildRequires:  jerasure-devel
 BuildRequires:  libisa-l-devel
 BuildRequires:  gf-complete-devel
-BuildRequires:  libibverbs-devel
+BuildRequires:  rdma-core-devel
 BuildRequires:  cmake
 BuildRequires:  libnl3-devel
 Requires:       vitastor-osd = %{version}-%{release}

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.9.2
+Version:        1.10.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.9.2.el9.tar.gz
+Source0:        vitastor-1.10.0.el9.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel
@@ -74,7 +74,7 @@ Vitastor library headers for development.
 Summary:        Vitastor - fio drivers
 Group:          Development/Libraries
 Requires:       vitastor-client = %{version}-%{release}
-Requires:       fio = 3.27-8.el9
+Requires:       fio = 3.35-1.el9
 
 
 %description -n vitastor-fio

src/CMakeLists.txt

@@ -19,7 +19,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVITASTOR_VERSION="1.9.2")
+add_definitions(-DVITASTOR_VERSION="1.10.0")
 add_definitions(-D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})
@@ -61,6 +61,10 @@ pkg_check_modules(ISAL libisal)
 if (ISAL_LIBRARIES)
 	add_definitions(-DWITH_ISAL)
 endif (ISAL_LIBRARIES)
+pkg_check_modules(RDMACM librdmacm)
+if (RDMACM_LIBRARIES)
+	add_definitions(-DWITH_RDMACM)
+endif (RDMACM_LIBRARIES)
 
 add_custom_target(build_tests)
 add_custom_target(test

src/client/etcd_state_client.cpp

@@ -176,7 +176,7 @@ void etcd_state_client_t::add_etcd_url(std::string addr)
             exit(1);
         }
         if (!local_ips.size())
-            local_ips = getifaddr_list();
+            local_ips = getifaddr_list(std::vector<std::string>(), true);
         std::string check_addr;
         int pos = addr.find('/');
         int pos2 = addr.find(':');
@@ -785,7 +785,7 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
         }
         for (auto & pool_item: value.object_items())
         {
-            pool_config_t pc;
+            pool_config_t pc = {};
             // ID
             pool_id_t pool_id;
             char null_byte = 0;
@@ -931,12 +931,28 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
             // Ignore old key if the new one is present
             return;
         }
+        for (auto & pool_id_json: value["backfillfull_pools"].array_items())
+        {
+            auto pool_id = pool_id_json.uint64_value();
+            auto pool_it = this->pool_config.find(pool_id);
+            if (pool_it != this->pool_config.end())
+            {
+                pool_it->second.backfillfull |= 2;
+            }
+        }
         for (auto & pool_item: this->pool_config)
         {
             for (auto & pg_item: pool_item.second.pg_config)
             {
                 pg_item.second.config_exists = false;
             }
+            // 3 = was 1 and became 1, 0 = was 0 and became 0
+            if (pool_item.second.backfillfull == 2 || pool_item.second.backfillfull == 1)
+            {
+                if (on_change_backfillfull_hook)
+                    on_change_backfillfull_hook(pool_item.first);
+            }
+            pool_item.second.backfillfull = pool_item.second.backfillfull >> 1;
         }
         for (auto & pool_item: value["items"].object_items())
         {

src/client/etcd_state_client.h

@@ -62,6 +62,7 @@ struct pool_config_t
     std::map<pg_num_t, pg_config_t> pg_config;
     uint64_t scrub_interval;
     std::string used_for_fs;
+    int backfillfull;
 };
 
 struct inode_config_t
@@ -131,6 +132,7 @@ public:
     std::function<json11::Json()> load_pgs_checks_hook;
     std::function<void(bool)> on_load_pgs_hook;
     std::function<void()> on_change_pool_config_hook;
+    std::function<void(pool_id_t)> on_change_backfillfull_hook;
     std::function<void(pool_id_t, pg_num_t, osd_num_t)> on_change_pg_state_hook;
     std::function<void(pool_id_t, pg_num_t)> on_change_pg_history_hook;
     std::function<void(osd_num_t)> on_change_osd_state_hook;

src/client/http_client.cpp

@@ -62,6 +62,7 @@ struct http_co_t
     inline void end() { ended = true; if (!onstack) { delete this; } }
     void run_cb_and_clear();
     void start_connection();
+    void start_ws_connection();
     void close_connection();
     void next_request();
     void handle_events();
@@ -112,7 +113,7 @@ http_co_t* open_websocket(timerfd_manager_t *tfd, const std::string & host, cons
     handler->keepalive = false;
     handler->request = request;
     handler->response_callback = response_callback;
-    handler->start_connection();
+    handler->start_ws_connection();
     return handler;
 }
 
@@ -282,6 +283,27 @@ void http_co_t::close_connection()
     epoll_events = 0;
 }
 
+void http_co_t::start_ws_connection()
+{
+    stackin();
+    start_connection();
+    if (request_timeout > 0)
+    {
+        timeout_id = tfd->set_timer(request_timeout, false, [this](int timer_id)
+        {
+            stackin();
+            if (state != HTTP_CO_WEBSOCKET)
+            {
+                close_connection();
+                parsed = { .error = "Websocket connection timed out" };
+                run_cb_and_clear();
+            }
+            stackout();
+        });
+    }
+    stackout();
+}
+
 void http_co_t::start_connection()
 {
     stackin();

src/client/messenger.cpp

@@ -121,7 +121,7 @@ void osd_messenger_t::init()
     if (use_rdma)
     {
         rdma_context = msgr_rdma_context_t::create(
-            rdma_device != "" ? rdma_device.c_str() : NULL,
+            osd_networks, rdma_device != "" ? rdma_device.c_str() : NULL,
             rdma_port_num, rdma_gid_index, rdma_mtu, rdma_odp, log_level
         );
         if (!rdma_context)
@@ -266,7 +266,8 @@ void osd_messenger_t::parse_config(const json11::Json & config)
     this->rdma_port_num = (uint8_t)config["rdma_port_num"].uint64_value();
     if (!this->rdma_port_num)
         this->rdma_port_num = 1;
-    this->rdma_gid_index = (uint8_t)config["rdma_gid_index"].uint64_value();
+    if (!config["rdma_gid_index"].is_null())
+        this->rdma_gid_index = (uint8_t)config["rdma_gid_index"].uint64_value();
     this->rdma_mtu = (uint32_t)config["rdma_mtu"].uint64_value();
     this->rdma_max_sge = config["rdma_max_sge"].uint64_value();
     if (!this->rdma_max_sge)
@@ -281,6 +282,15 @@ void osd_messenger_t::parse_config(const json11::Json & config)
     if (!this->rdma_max_msg || this->rdma_max_msg > 128*1024*1024)
         this->rdma_max_msg = 129*1024;
     this->rdma_odp = config["rdma_odp"].bool_value();
+    std::vector<std::string> mask;
+    if (config["bind_address"].is_string())
+        mask.push_back(config["bind_address"].string_value());
+    else if (config["osd_network"].is_string())
+        mask.push_back(config["osd_network"].string_value());
+    else
+        for (auto v: config["osd_network"].array_items())
+            mask.push_back(v.string_value());
+    this->osd_networks = mask;
 #endif
     if (!osd_num)
         this->iothread_count = (uint32_t)config["client_iothread_count"].uint64_value();

src/client/messenger.h

@@ -165,8 +165,10 @@ protected:
 
 #ifdef WITH_RDMA
     bool use_rdma = true;
+    std::vector<std::string> osd_networks;
     std::string rdma_device;
-    uint64_t rdma_port_num = 1, rdma_gid_index = 0, rdma_mtu = 0;
+    uint64_t rdma_port_num = 1, rdma_mtu = 0;
+    int rdma_gid_index = -1;
     msgr_rdma_context_t *rdma_context = NULL;
     uint64_t rdma_max_sge = 0, rdma_max_send = 0, rdma_max_recv = 0;
     uint64_t rdma_max_msg = 0;
@@ -177,7 +179,7 @@ protected:
     std::vector<int> read_ready_clients;
     std::vector<int> write_ready_clients;
     // We don't use ringloop->set_immediate here because we may have no ringloop in client :)
-    std::vector<std::function<void()>> set_immediate;
+    std::vector<osd_op_t*> set_immediate_ops;
 
 public:
     timerfd_manager_t *tfd;
@@ -237,6 +239,8 @@ protected:
     void handle_op_hdr(osd_client_t *cl);
     bool handle_reply_hdr(osd_client_t *cl);
     void handle_reply_ready(osd_op_t *op);
+    void handle_immediate_ops();
+    void clear_immediate_ops(int peer_fd);
 
 #ifdef WITH_RDMA
     void try_send_rdma(osd_client_t *cl);

src/client/msgr_rdma.cpp

@@ -3,6 +3,7 @@
 
 #include <stdio.h>
 #include <stdlib.h>
+#include "addr_util.h"
 #include "msgr_rdma.h"
 #include "messenger.h"
 
@@ -69,7 +70,138 @@ msgr_rdma_connection_t::~msgr_rdma_connection_t()
     send_out_size = 0;
 }
 
-msgr_rdma_context_t *msgr_rdma_context_t::create(const char *ib_devname, uint8_t ib_port, uint8_t gid_index, uint32_t mtu, bool odp, int log_level)
+#ifdef IBV_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT
+static bool is_ipv4_gid(ibv_gid_entry *gidx)
+{
+    return (((uint64_t*)gidx->gid.raw)[0] == 0 &&
+        ((uint32_t*)gidx->gid.raw)[2] == 0xffff0000);
+}
+
+static bool match_gid(ibv_gid_entry *gidx, addr_mask_t *networks, int nnet)
+{
+    if (gidx->gid_type != IBV_GID_TYPE_ROCE_V1 &&
+        gidx->gid_type != IBV_GID_TYPE_ROCE_V2 ||
+        ((uint64_t*)gidx->gid.raw)[0] == 0 &&
+        ((uint64_t*)gidx->gid.raw)[1] == 0)
+    {
+        return false;
+    }
+    if (is_ipv4_gid(gidx))
+    {
+        for (int i = 0; i < nnet; i++)
+        {
+            if (networks[i].family == AF_INET && cidr_match(*(in_addr*)(gidx->gid.raw+12), networks[i].ipv4, networks[i].bits))
+                return true;
+        }
+    }
+    else
+    {
+        for (int i = 0; i < nnet; i++)
+        {
+            if (networks[i].family == AF_INET6 && cidr6_match(*(in6_addr*)gidx->gid.raw, networks[i].ipv6, networks[i].bits))
+                return true;
+        }
+    }
+    return false;
+}
+
+struct matched_dev
+{
+    int dev = -1;
+    int port = -1;
+    int gid = -1;
+    bool rocev2 = false;
+};
+
+static void log_rdma_dev_port_gid(ibv_device *dev, int ib_port, int gid_index, ibv_gid_entry & gidx)
+{
+    bool is4 = ((uint64_t*)gidx.gid.raw)[0] == 0 && ((uint32_t*)gidx.gid.raw)[2] == 0xffff0000;
+    char buf[256];
+    inet_ntop(is4 ? AF_INET : AF_INET6, is4 ? gidx.gid.raw+12 : gidx.gid.raw, buf, sizeof(buf));
+    fprintf(
+        stderr, "Auto-selected RDMA device %s port %d GID %d - ROCEv%d IPv%d %s\n",
+        ibv_get_device_name(dev), ib_port, gid_index,
+        gidx.gid_type == IBV_GID_TYPE_ROCE_V2 ? 2 : 1, is4 ? 4 : 6, buf
+    );
+}
+
+static matched_dev match_device(ibv_device **dev_list, addr_mask_t *networks, int nnet, int log_level)
+{
+    matched_dev best;
+    ibv_device_attr attr;
+    ibv_port_attr portinfo;
+    ibv_gid_entry best_gidx;
+    int res;
+    bool have_non_roce = false, have_roce = false;
+    for (int i = 0; dev_list[i]; ++i)
+    {
+        auto dev = dev_list[i];
+        ibv_context *context = ibv_open_device(dev_list[i]);
+        if ((res = ibv_query_device(context, &attr)) != 0)
+        {
+            fprintf(stderr, "Couldn't query RDMA device %s for its features: %s\n", ibv_get_device_name(dev_list[i]), strerror(res));
+            goto cleanup;
+        }
+        for (int j = 1; j <= attr.phys_port_cnt; j++)
+        {
+            // Try to find a port with matching address
+            if ((res = ibv_query_port(context, j, &portinfo)) != 0)
+            {
+                fprintf(stderr, "Couldn't get RDMA device %s port %d info: %s\n", ibv_get_device_name(dev), j, strerror(res));
+                goto cleanup;
+            }
+            for (int k = 0; k < portinfo.gid_tbl_len; k++)
+            {
+                ibv_gid_entry gidx;
+                if ((res = ibv_query_gid_ex(context, j, k, &gidx, 0)) != 0)
+                {
+                    if (res != ENODATA)
+                    {
+                        fprintf(stderr, "Couldn't read RDMA device %s GID index %d: %s\n", ibv_get_device_name(dev), k, strerror(res));
+                        goto cleanup;
+                    }
+                    else
+                        break;
+                }
+                if (gidx.gid_type != IBV_GID_TYPE_ROCE_V1 &&
+                    gidx.gid_type != IBV_GID_TYPE_ROCE_V2)
+                    have_non_roce = true;
+                else
+                    have_roce = true;
+                if (match_gid(&gidx, networks, nnet))
+                {
+                    // Prefer RoCEv2
+                    if (!best.rocev2)
+                    {
+                        best.dev = i;
+                        best.port = j;
+                        best.gid = k;
+                        best.rocev2 = (gidx.gid_type == IBV_GID_TYPE_ROCE_V2);
+                        best_gidx = gidx;
+                    }
+                }
+            }
+        }
+cleanup:
+        ibv_close_device(context);
+        if (best.rocev2)
+        {
+            break;
+        }
+    }
+    if (best.dev >= 0 && log_level > 0)
+    {
+        log_rdma_dev_port_gid(dev_list[best.dev], best.port, best.gid, best_gidx);
+    }
+    if (best.dev < 0 && have_non_roce && !have_roce)
+    {
+        best.dev = -2;
+    }
+    return best;
+}
+#endif
+
+msgr_rdma_context_t *msgr_rdma_context_t::create(std::vector<std::string> osd_networks, const char *ib_devname, uint8_t ib_port, int gid_index, uint32_t mtu, bool odp, int log_level)
 {
     int res;
     ibv_device **dev_list = NULL;
@@ -80,28 +212,23 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(const char *ib_devname, uint8_t
     clock_gettime(CLOCK_REALTIME, &tv);
     srand48(tv.tv_sec*1000000000 + tv.tv_nsec);
     dev_list = ibv_get_device_list(NULL);
-    if (!dev_list)
+    if (!dev_list || !*dev_list)
     {
         if (errno == -ENOSYS || errno == ENOSYS)
         {
             if (log_level > 0)
                 fprintf(stderr, "No RDMA devices found (RDMA device list returned ENOSYS)\n");
         }
+        else if (!*dev_list)
+        {
+            if (log_level > 0)
+                fprintf(stderr, "No RDMA devices found\n");
+        }
         else
             fprintf(stderr, "Failed to get RDMA device list: %s\n", strerror(errno));
         goto cleanup;
     }
-    if (!ib_devname)
-    {
-        ctx->dev = *dev_list;
-        if (!ctx->dev)
-        {
-            if (log_level > 0)
-                fprintf(stderr, "No RDMA devices found\n");
-            goto cleanup;
-        }
-    }
-    else
+    if (ib_devname)
     {
         int i;
         for (i = 0; dev_list[i]; ++i)
@@ -114,6 +241,39 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(const char *ib_devname, uint8_t
             goto cleanup;
         }
     }
+#ifdef IBV_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT
+    else if (osd_networks.size())
+    {
+        std::vector<addr_mask_t> nets;
+        for (auto & netstr: osd_networks)
+        {
+            nets.push_back(cidr_parse(netstr));
+        }
+        auto best = match_device(dev_list, nets.data(), nets.size(), log_level);
+        if (best.dev == -2)
+        {
+            best.dev = 0;
+            if (log_level > 0)
+                fprintf(stderr, "No RoCE devices found, using first available RDMA device %s\n", ibv_get_device_name(*dev_list));
+        }
+        else if (best.dev < 0)
+        {
+            if (log_level > 0)
+                fprintf(stderr, "RDMA device matching osd_network is not found, disabling RDMA\n");
+            goto cleanup;
+        }
+        else
+        {
+            ib_port = best.port;
+            gid_index = best.gid;
+        }
+        ctx->dev = dev_list[best.dev];
+    }
+#endif
+    else
+    {
+        ctx->dev = *dev_list;
+    }
 
     ctx->context = ibv_open_device(ctx->dev);
     if (!ctx->context)
@@ -123,7 +283,6 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(const char *ib_devname, uint8_t
     }
 
     ctx->ib_port = ib_port;
-    ctx->gid_index = gid_index;
     if ((res = ibv_query_port(ctx->context, ib_port, &ctx->portinfo)) != 0)
     {
         fprintf(stderr, "Couldn't get RDMA device %s port %d info: %s\n", ibv_get_device_name(ctx->dev), ib_port, strerror(res));
@@ -135,11 +294,55 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(const char *ib_devname, uint8_t
         fprintf(stderr, "RDMA device %s must have local LID because it's not Ethernet, but LID is zero\n", ibv_get_device_name(ctx->dev));
         goto cleanup;
     }
-    if (ibv_query_gid(ctx->context, ib_port, gid_index, &ctx->my_gid))
+
+#ifdef IBV_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT
+    if (gid_index != -1)
+#endif
     {
-        fprintf(stderr, "Couldn't read RDMA device %s GID index %d\n", ibv_get_device_name(ctx->dev), gid_index);
-        goto cleanup;
+        ctx->gid_index = gid_index < 0 ? 0 : gid_index;
+        if (ibv_query_gid(ctx->context, ib_port, gid_index, &ctx->my_gid))
+        {
+            fprintf(stderr, "Couldn't read RDMA device %s GID index %d\n", ibv_get_device_name(ctx->dev), gid_index);
+            goto cleanup;
+        }
     }
+#ifdef IBV_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT
+    else
+    {
+        // Auto-guess GID
+        ibv_gid_entry best_gidx;
+        for (int k = 0; k < ctx->portinfo.gid_tbl_len; k++)
+        {
+            ibv_gid_entry gidx;
+            if (ibv_query_gid_ex(ctx->context, ib_port, k, &gidx, 0) != 0)
+            {
+                fprintf(stderr, "Couldn't read RDMA device %s GID index %d\n", ibv_get_device_name(ctx->dev), k);
+                goto cleanup;
+            }
+            // Skip empty GID
+            if (((uint64_t*)gidx.gid.raw)[0] == 0 &&
+                ((uint64_t*)gidx.gid.raw)[1] == 0)
+            {
+                continue;
+            }
+            // Prefer IPv4 RoCEv2 -> IPv6 RoCEv2 -> IPv4 RoCEv1 -> IPv6 RoCEv1 -> IB
+            if (gid_index == -1 ||
+                gidx.gid_type == IBV_GID_TYPE_ROCE_V2 && best_gidx.gid_type != IBV_GID_TYPE_ROCE_V2 ||
+                gidx.gid_type == IBV_GID_TYPE_ROCE_V1 && best_gidx.gid_type == IBV_GID_TYPE_IB ||
+                gidx.gid_type == best_gidx.gid_type && is_ipv4_gid(&gidx))
+            {
+                gid_index = k;
+                best_gidx = gidx;
+            }
+        }
+        ctx->gid_index = gid_index = (gid_index == -1 ? 0 : gid_index);
+        if (log_level > 0)
+        {
+            log_rdma_dev_port_gid(ctx->dev, ctx->ib_port, ctx->gid_index, best_gidx);
+        }
+        ctx->my_gid = best_gidx.gid;
+    }
+#endif
 
     ctx->pd = ibv_alloc_pd(ctx->context);
     if (!ctx->pd)
@@ -398,7 +601,7 @@ static int try_send_rdma_copy(osd_client_t *cl, uint8_t *dst, int dst_len)
         iovec & iov = cl->send_list[rc->send_pos];
         uint32_t len = (uint32_t)(iov.iov_len-rc->send_buf_pos < dst_len
             ? iov.iov_len-rc->send_buf_pos : dst_len);
-        memcpy(dst, iov.iov_base+rc->send_buf_pos, len);
+        memcpy(dst, (uint8_t*)iov.iov_base+rc->send_buf_pos, len);
         dst += len;
         dst_len -= len;
         rc->send_buf_pos += len;
@@ -598,6 +801,7 @@ void osd_messenger_t::handle_rdma_events()
                 }
                 fprintf(stderr, " with status: %s, stopping client\n", ibv_wc_status_str(wc[i].status));
                 stop_client(client_id);
+                clear_immediate_ops(client_id);
                 continue;
             }
             if (!is_send)
@@ -606,6 +810,7 @@ void osd_messenger_t::handle_rdma_events()
                 if (!handle_read_buffer(cl, rc->recv_buffers[rc->next_recv_buf].buf, wc[i].byte_len))
                 {
                     // handle_read_buffer may stop the client
+                    clear_immediate_ops(client_id);
                     continue;
                 }
                 try_recv_rdma_wr(cl, rc->recv_buffers[rc->next_recv_buf]);
@@ -666,9 +871,5 @@ void osd_messenger_t::handle_rdma_events()
             }
         }
     } while (event_count > 0);
-    for (auto cb: set_immediate)
-    {
-        cb();
-    }
-    set_immediate.clear();
+    handle_immediate_ops();
 }

src/client/msgr_rdma.h

@@ -36,7 +36,7 @@ struct msgr_rdma_context_t
     int max_cqe = 0;
     int used_max_cqe = 0;
 
-    static msgr_rdma_context_t *create(const char *ib_devname, uint8_t ib_port, uint8_t gid_index, uint32_t mtu, bool odp, int log_level);
+    static msgr_rdma_context_t *create(std::vector<std::string> osd_networks, const char *ib_devname, uint8_t ib_port, int gid_index, uint32_t mtu, bool odp, int log_level);
     ~msgr_rdma_context_t();
 };
 

src/client/msgr_receive.cpp

@@ -65,6 +65,7 @@ void osd_messenger_t::read_requests()
 bool osd_messenger_t::handle_read(int result, osd_client_t *cl)
 {
     bool ret = false;
+    int peer_fd = cl->peer_fd;
     cl->read_msg.msg_iovlen = 0;
     cl->refs--;
     if (cl->peer_state == PEER_STOPPED)
@@ -101,7 +102,8 @@ bool osd_messenger_t::handle_read(int result, osd_client_t *cl)
         {
             if (!handle_read_buffer(cl, cl->in_buf, result))
             {
-                goto fin;
+                clear_immediate_ops(peer_fd);
+                return false;
             }
         }
         else
@@ -113,7 +115,8 @@ bool osd_messenger_t::handle_read(int result, osd_client_t *cl)
             {
                 if (!handle_finished_read(cl))
                 {
-                    goto fin;
+                    clear_immediate_ops(peer_fd);
+                    return false;
                 }
             }
         }
@@ -122,15 +125,47 @@ bool osd_messenger_t::handle_read(int result, osd_client_t *cl)
             ret = true;
         }
     }
-fin:
-    for (auto cb: set_immediate)
-    {
-        cb();
-    }
-    set_immediate.clear();
+    handle_immediate_ops();
     return ret;
 }
 
+void osd_messenger_t::clear_immediate_ops(int peer_fd)
+{
+    size_t i = 0, j = 0;
+    while (i < set_immediate_ops.size())
+    {
+        if (set_immediate_ops[i]->peer_fd == peer_fd)
+        {
+            delete set_immediate_ops[i];
+        }
+        else
+        {
+            if (i != j)
+                set_immediate_ops[j] = set_immediate_ops[i];
+            j++;
+        }
+        i++;
+    }
+    set_immediate_ops.resize(j);
+}
+
+void osd_messenger_t::handle_immediate_ops()
+{
+    for (auto op: set_immediate_ops)
+    {
+        if (op->op_type == OSD_OP_IN)
+        {
+            exec_op(op);
+        }
+        else
+        {
+            // Copy lambda to be unaffected by `delete op`
+            std::function<void(osd_op_t*)>(op->callback)(op);
+        }
+    }
+    set_immediate_ops.clear();
+}
+
 bool osd_messenger_t::handle_read_buffer(osd_client_t *cl, void *curbuf, int remain)
 {
     // Compose operation(s) from the buffer
@@ -199,7 +234,7 @@ bool osd_messenger_t::handle_finished_read(osd_client_t *cl)
     {
         // Operation is ready
         cl->received_ops.push_back(cl->read_op);
-        set_immediate.push_back([this, op = cl->read_op]() { exec_op(op); });
+        set_immediate_ops.push_back(cl->read_op);
         cl->read_op = NULL;
         cl->read_state = 0;
     }
@@ -295,7 +330,7 @@ void osd_messenger_t::handle_op_hdr(osd_client_t *cl)
     {
         // Operation is ready
         cl->received_ops.push_back(cur_op);
-        set_immediate.push_back([this, cur_op]() { exec_op(cur_op); });
+        set_immediate_ops.push_back(cur_op);
         cl->read_op = NULL;
         cl->read_state = 0;
     }
@@ -416,9 +451,5 @@ void osd_messenger_t::handle_reply_ready(osd_op_t *op)
         (tv_end.tv_sec - op->tv_begin.tv_sec)*1000000 +
         (tv_end.tv_nsec - op->tv_begin.tv_nsec)/1000
     );
-    set_immediate.push_back([op]()
-    {
-        // Copy lambda to be unaffected by `delete op`
-        std::function<void(osd_op_t*)>(op->callback)(op);
-    });
+    set_immediate_ops.push_back(op);
 }

src/client/nbd_proxy.cpp

@@ -64,7 +64,7 @@ static void netlink_sock_alloc(struct netlink_ctx *ctx)
     if (nl_driver_id < 0)
     {
         nl_socket_free(sk);
-        fail("Couldn't resolve the nbd netlink family\n");
+        fail("Couldn't resolve the nbd netlink family: %s (code %d)\n", nl_geterror(nl_driver_id), nl_driver_id);
     }
 
     ctx->driver_id = nl_driver_id;
@@ -555,7 +555,12 @@ help:
         fcntl(sockfd[0], F_SETFL, fcntl(sockfd[0], F_GETFL, 0) | O_NONBLOCK);
         nbd_fd = sockfd[0];
         load_module();
+
         bool bg = cfg["foreground"].is_null();
+        if (cfg["logfile"].string_value() != "")
+        {
+            logfile = cfg["logfile"].string_value();
+        }
 
         if (netlink)
         {
@@ -579,6 +584,10 @@ help:
             if (!cfg["nbd_disconnect_on_close"].is_null())
                 cflags |= NBD_CFLAG_DISCONNECT_ON_CLOSE;
 #endif
+            if (bg)
+            {
+                daemonize_fork();
+            }
             int err = netlink_configure(sockfd + 1, 1, devnum, device_size, 4096, flags, cflags, nbd_timeout, nbd_conn_timeout, NULL, revive);
             if (err < 0)
             {
@@ -588,6 +597,10 @@ help:
             }
             close(sockfd[1]);
             printf("/dev/nbd%d\n", err);
+            if (bg)
+            {
+                daemonize_reopen_stdio();
+            }
 #else
             fprintf(stderr, "netlink support is disabled in this build\n");
             exit(1);
@@ -631,14 +644,10 @@ help:
                     }
                 }
             }
-        }
-        if (cfg["logfile"].string_value() != "")
-        {
-            logfile = cfg["logfile"].string_value();
-        }
-        if (bg)
-        {
-            daemonize();
+            if (bg)
+            {
+                daemonize();
+            }
         }
         // Initialize read state
         read_state = CL_READ_HDR;
@@ -716,13 +725,17 @@ help:
         }
     }
 
-    void daemonize()
+    void daemonize_fork()
     {
         if (fork())
             exit(0);
         setsid();
         if (fork())
             exit(0);
+    }
+
+    void daemonize_reopen_stdio()
+    {
         close(0);
         close(1);
         close(2);
@@ -733,6 +746,12 @@ help:
             fprintf(stderr, "Warning: Failed to chdir into /\n");
     }
 
+    void daemonize()
+    {
+        daemonize_fork();
+        daemonize_reopen_stdio();
+    }
+
     json11::Json::object list_mapped()
     {
         const char *self_filename = exe_name;
@@ -783,8 +802,9 @@ help:
                 if (!strcmp(pid_filename, self_filename))
                 {
                     json11::Json::object cfg = nbd_proxy::parse_args(argv.size(), argv.data());
-                    if (cfg["command"] == "map")
+                    if (cfg["command"] == "map" || cfg["command"] == "netlink-map")
                     {
+                        cfg["interface"] = (cfg["command"] == "netlink-map") ? "netlink" : "nbd";
                         cfg.erase("command");
                         cfg["pid"] = pid;
                         mapped["/dev/nbd"+std::to_string(dev_num)] = cfg;

src/client/qemu_driver.c

@@ -16,7 +16,6 @@
 #include "qapi/error.h"
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qerror.h"
-#include "qemu/uri.h"
 #include "qemu/error-report.h"
 #include "qemu/module.h"
 #include "qemu/option.h"
@@ -1021,7 +1020,11 @@ static BlockDriver bdrv_vitastor = {
     // FIXME: Implement it along with per-inode statistics
     //.bdrv_get_allocated_file_size   = vitastor_get_allocated_file_size,
 
+#if QEMU_VERSION_MAJOR > 9 || QEMU_VERSION_MAJOR == 9 && QEMU_VERSION_MINOR > 0
+    .bdrv_open                      = vitastor_file_open,
+#else
     .bdrv_file_open                 = vitastor_file_open,
+#endif
     .bdrv_close                     = vitastor_close,
 
     // Option list for the create operation

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 1.9.2
+Version: 1.10.0
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/cmd/cli_dd.cpp

@@ -261,6 +261,7 @@ struct dd_out_info_t
             else
             {
                 // ok
+                out_size = owatch->cfg.size;
                 return true;
             }
             // Wait for sub-command
@@ -323,7 +324,7 @@ resume_2:
                 cluster_op_t *sync_op = new cluster_op_t;
                 sync_op->opcode = OSD_OP_SYNC;
                 parent->waiting++;
-                sync_op->callback = [this, parent](cluster_op_t *sync_op)
+                sync_op->callback = [parent](cluster_op_t *sync_op)
                 {
                     parent->waiting--;
                     delete sync_op;
@@ -431,7 +432,7 @@ struct cli_dd_t
             if (read_op->retval < 0)
             {
                 fprintf(
-                    stderr, "Failed to read bitmap for %lu bytes from image %s at offset %lu: %s (code %d)\n",
+                    stderr, "Failed to read bitmap for %ju bytes from image %s at offset %ju: %s (code %d)\n",
                     read_op->len, iinfo.iimg.c_str(), read_op->offset,
                     strerror(read_op->retval < 0 ? -read_op->retval : EIO), read_op->retval
                 );
@@ -476,7 +477,7 @@ struct cli_dd_t
             if (read_op->retval != read_op->len)
             {
                 fprintf(
-                    stderr, "Failed to read %lu bytes from image %s at offset %lu: %s (code %d)\n",
+                    stderr, "Failed to read %ju bytes from image %s at offset %ju: %s (code %d)\n",
                     read_op->len, iinfo.iimg.c_str(), read_op->offset,
                     strerror(read_op->retval < 0 ? -read_op->retval : EIO), read_op->retval
                 );
@@ -538,7 +539,7 @@ struct cli_dd_t
                     in_eof = true;
             }
             ring_data_t *data = ((ring_data_t*)sqe->user_data);
-            data->iov = (iovec){ cur_read->buf + cur_read->len, cur_read->max - cur_read->len };
+            data->iov = (iovec){ (uint8_t*)cur_read->buf + cur_read->len, cur_read->max - cur_read->len };
             my_uring_prep_readv(sqe, iinfo.ifd, &data->iov, 1, iinfo.in_seekable ? iseek + cur_read->offset + cur_read->len : -1);
             in_waiting++;
             data->callback = [this, cur_read](ring_data_t *data)
@@ -547,7 +548,7 @@ struct cli_dd_t
                 if (data->res < 0)
                 {
                     fprintf(
-                        stderr, "Failed to read %lu bytes from %s at offset %lu: %s (code %d)\n",
+                        stderr, "Failed to read %ju bytes from %s at offset %ju: %s (code %d)\n",
                         data->iov.iov_len, iinfo.ifile == "" ? "stdin" : iinfo.ifile.c_str(), cur_read->offset,
                         strerror(-data->res), data->res
                     );
@@ -644,7 +645,7 @@ struct cli_dd_t
                 if (write_op->retval != write_op->len)
                 {
                     fprintf(
-                        stderr, "Failed to write %lu bytes to image %s at offset %lu: %s (code %d)\n",
+                        stderr, "Failed to write %ju bytes to image %s at offset %ju: %s (code %d)\n",
                         write_op->len, oinfo.oimg.c_str(), write_op->offset,
                         strerror(write_op->retval < 0 ? -write_op->retval : EIO), write_op->retval
                     );
@@ -671,7 +672,7 @@ struct cli_dd_t
                 return false;
             }
             ring_data_t *data = ((ring_data_t*)sqe->user_data);
-            data->iov = (iovec){ .iov_base = cur_read->buf+cur_read->len, .iov_len = cur_read->max-cur_read->len };
+            data->iov = (iovec){ .iov_base = (uint8_t*)cur_read->buf+cur_read->len, .iov_len = cur_read->max-cur_read->len };
             my_uring_prep_writev(sqe, oinfo.ofd, &data->iov, 1, oinfo.out_seekable ? cur_read->offset+cur_read->len+oseek : -1);
             out_waiting++;
             data->callback = [this, cur_read](ring_data_t *data)
@@ -680,7 +681,7 @@ struct cli_dd_t
                 if (data->res < 0)
                 {
                     fprintf(
-                        stderr, "Failed to write %lu bytes to %s at offset %lu: %s (code %d)\n",
+                        stderr, "Failed to write %ju bytes to %s at offset %ju: %s (code %d)\n",
                         data->iov.iov_len, oinfo.ofile == "" ? "stdout" : oinfo.ofile.c_str(),
                         oinfo.out_seekable ? cur_read->offset+cur_read->len+oseek : 0,
                         strerror(-data->res), data->res
@@ -727,7 +728,7 @@ struct cli_dd_t
         {
             char buf[256];
             snprintf(
-                buf, sizeof(buf), "%lu bytes (%s) copied, %.1f s, %sB/s",
+                buf, sizeof(buf), "%ju bytes (%s) copied, %.1f s, %sB/s",
                 written_size, format_size(written_size).c_str(), sec_total,
                 format_size((uint64_t)(written_size/sec_total), true).c_str()
             );
@@ -749,7 +750,7 @@ struct cli_dd_t
         else
         {
             fprintf(
-                stderr, "\r%lu bytes (%s) copied, %.1f s, %sB/s, avg %sB/s\033[K",
+                stderr, "\r%ju bytes (%s) copied, %.1f s, %sB/s, avg %sB/s\033[K",
                 written_size, format_size(written_size).c_str(), sec_total,
                 format_size((uint64_t)(delta/sec_delta), true).c_str(),
                 format_size((uint64_t)(written_size/sec_total), true).c_str()
@@ -882,6 +883,8 @@ resume_2:
         oinfo.end_fsync = oinfo.end_fsync && oinfo.out_seekable;
         read_offset = 0;
         read_end = iinfo.in_seekable ? iinfo.in_size-iseek : 0;
+        if (oinfo.out_size && (!read_end || read_end > oinfo.out_size-oseek))
+            read_end = oinfo.out_size-oseek;
         if (bytelimit && (!read_end || read_end > bytelimit))
             read_end = bytelimit;
         clock_gettime(CLOCK_REALTIME, &tv_begin);

src/cmd/cli_osd_tree.cpp

@@ -216,7 +216,7 @@ resume_1:
                 for (uint64_t osd_num: node.child_osds)
                 {
                     auto & osd = placement_tree->osds.at(osd_num);
-                    fmt_items.push_back(json11::Json::object{
+                    auto json_osd = json11::Json::object{
                         { "type", "osd" },
                         { "name", osd.num },
                         { "parent", node.name },
@@ -230,7 +230,16 @@ resume_1:
                         { "bitmap", (uint64_t)osd.bitmap_granularity },
                         { "commit", osd.immediate_commit == IMMEDIATE_NONE ? "none" : (osd.immediate_commit == IMMEDIATE_ALL ? "all" : "small") },
                         { "op_stats", osd_stats[osd_num]["op_stats"] },
-                    });
+                    };
+                    if (osd_stats[osd_num]["slow_ops_primary"].uint64_value() > 0)
+                    {
+                        json_osd["slow_ops_primary"] = osd_stats[osd_num]["slow_ops_primary"];
+                    }
+                    if (osd_stats[osd_num]["slow_ops_secondary"].uint64_value() > 0)
+                    {
+                        json_osd["slow_ops_secondary"] = osd_stats[osd_num]["slow_ops_secondary"];
+                    }
+                    fmt_items.push_back(json_osd);
                 }
             }
             result.data = fmt_items;

src/cmd/cli_pool_create.cpp

@@ -35,6 +35,7 @@ struct pool_creator_t
     uint64_t new_pools_mod_rev;
     json11::Json state_node_tree;
     json11::Json new_pools;
+    std::map<osd_num_t, json11::Json> osd_stats;
 
     bool is_done() { return state == 100; }
 
@@ -46,8 +47,6 @@ struct pool_creator_t
             goto resume_2;
         else if (state == 3)
             goto resume_3;
-        else if (state == 4)
-            goto resume_4;
         else if (state == 5)
             goto resume_5;
         else if (state == 6)
@@ -90,13 +89,19 @@ resume_1:
         // If not forced, check that we have enough osds for pg_size
         if (!force)
         {
-            // Get node_placement configuration from etcd
+            // Get node_placement configuration from etcd and OSD stats
             parent->etcd_txn(json11::Json::object {
                 { "success", json11::Json::array {
                     json11::Json::object {
                         { "request_range", json11::Json::object {
                             { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/node_placement") },
-                        } }
+                        } },
+                    },
+                    json11::Json::object {
+                        { "request_range", json11::Json::object {
+                            { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/osd/stats/") },
+                            { "range_end", base64_encode(parent->cli->st_cli.etcd_prefix+"/osd/stats0") },
+                        } },
                     },
                 } },
             });
@@ -112,10 +117,21 @@ resume_2:
                 return;
             }
 
-            // Get state_node_tree based on node_placement and osd peer states
+            // Get state_node_tree based on node_placement and osd stats
             {
-                auto kv = parent->cli->st_cli.parse_etcd_kv(parent->etcd_result["responses"][0]["response_range"]["kvs"][0]);
-                state_node_tree = get_state_node_tree(kv.value.object_items());
+                auto node_placement_kv = parent->cli->st_cli.parse_etcd_kv(parent->etcd_result["responses"][0]["response_range"]["kvs"][0]);
+                timespec tv_now;
+                clock_gettime(CLOCK_REALTIME, &tv_now);
+                uint64_t osd_out_time = parent->cli->config["osd_out_time"].uint64_value();
+                if (!osd_out_time)
+                    osd_out_time = 600;
+                osd_stats.clear();
+                parent->iterate_kvs_1(parent->etcd_result["responses"][1]["response_range"]["kvs"], "/osd/stats/", [&](uint64_t cur_osd, json11::Json value)
+                {
+                    if ((uint64_t)value["time"].number_value()+osd_out_time >= tv_now.tv_sec)
+                        osd_stats[cur_osd] = value;
+                });
+                state_node_tree = get_state_node_tree(node_placement_kv.value.object_items(), osd_stats);
             }
 
             // Skip tag checks, if pool has none
@@ -158,42 +174,18 @@ resume_3:
                 }
             }
 
-            // Get stats (for block_size, bitmap_granularity, ...) of osds in state_node_tree
-            {
-                json11::Json::array osd_stats;
-
-                for (auto osd_num: state_node_tree["osds"].array_items())
-                {
-                    osd_stats.push_back(json11::Json::object {
-                        { "request_range", json11::Json::object {
-                            { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/osd/stats/"+osd_num.as_string()) },
-                        } }
-                    });
-                }
-
-                parent->etcd_txn(json11::Json::object{ { "success", osd_stats } });
-            }
-
-            state = 4;
-resume_4:
-            if (parent->waiting > 0)
-                return;
-            if (parent->etcd_err.err)
-            {
-                result = parent->etcd_err;
-                state = 100;
-                return;
-            }
-
             // Filter osds from state_node_tree based on pool parameters and osd stats
             {
-                std::vector<json11::Json> osd_stats;
-                for (auto & ocr: parent->etcd_result["responses"].array_items())
+                std::vector<json11::Json> filtered_osd_stats;
+                for (auto & osd_num: state_node_tree["osds"].array_items())
                 {
-                    auto kv = parent->cli->st_cli.parse_etcd_kv(ocr["response_range"]["kvs"][0]);
-                    osd_stats.push_back(kv.value);
+                    auto st_it = osd_stats.find(osd_num.uint64_value());
+                    if (st_it != osd_stats.end())
+                    {
+                        filtered_osd_stats.push_back(st_it->second);
+                    }
                 }
-                guess_block_size(osd_stats);
+                guess_block_size(filtered_osd_stats);
                 state_node_tree = filter_state_node_tree_by_stats(state_node_tree, osd_stats);
             }
 
@@ -201,8 +193,7 @@ resume_4:
             {
                 auto failure_domain = cfg["failure_domain"].string_value() == ""
                     ? "host" : cfg["failure_domain"].string_value();
-                uint64_t max_pg_size = get_max_pg_size(state_node_tree["nodes"].object_items(),
-                    failure_domain, cfg["root_node"].string_value());
+                uint64_t max_pg_size = get_max_pg_size(state_node_tree, failure_domain, cfg["root_node"].string_value());
 
                 if (cfg["pg_size"].uint64_value() > max_pg_size)
                 {
@@ -358,56 +349,50 @@ resume_8:
 
     // Returns a JSON object of form {"nodes": {...}, "osds": [...]} that
     // contains: all nodes (osds, hosts, ...) based on node_placement config
-    // and current peer state, and a list of active peer osds.
-    json11::Json get_state_node_tree(json11::Json::object node_placement)
+    // and current osd stats.
+    json11::Json get_state_node_tree(json11::Json::object node_placement, std::map<osd_num_t, json11::Json> & osd_stats)
     {
-        // Erase non-peer osd nodes from node_placement
+        // Erase non-existing osd nodes from node_placement
         for (auto np_it = node_placement.begin(); np_it != node_placement.end();)
         {
             // Numeric nodes are osds
             osd_num_t osd_num = stoull_full(np_it->first);
 
-            // If node is osd and it is not in peer states, erase it
-            if (osd_num > 0 &&
-                parent->cli->st_cli.peer_states.find(osd_num) == parent->cli->st_cli.peer_states.end())
-            {
+            // If node is osd and its stats do not exist, erase it
+            if (osd_num > 0 && osd_stats.find(osd_num) == osd_stats.end())
                 node_placement.erase(np_it++);
-            }
             else
                 np_it++;
         }
 
-        // List of peer osds
-        std::vector<std::string> peer_osds;
+        // List of osds
+        std::vector<std::string> existing_osds;
 
-        // Record peer osds and add missing osds/hosts to np
-        for (auto & ps: parent->cli->st_cli.peer_states)
+        // Record osds and add missing osds/hosts to np
+        for (auto & ps: osd_stats)
         {
             std::string osd_num = std::to_string(ps.first);
 
-            // Record peer osd
-            peer_osds.push_back(osd_num);
+            // Record osd
+            existing_osds.push_back(osd_num);
 
-            // Add osd, if necessary
-            if (node_placement.find(osd_num) == node_placement.end())
+            // Add host if necessary
+            std::string osd_host = ps.second["host"].as_string();
+            if (node_placement.find(osd_host) == node_placement.end())
             {
-                std::string osd_host = ps.second["host"].as_string();
-
-                // Add host, if necessary
-                if (node_placement.find(osd_host) == node_placement.end())
-                {
-                    node_placement[osd_host] = json11::Json::object {
-                        { "level", "host" }
-                    };
-                }
-
-                node_placement[osd_num] = json11::Json::object {
-                    { "parent", osd_host }
+                node_placement[osd_host] = json11::Json::object {
+                    { "level", "host" }
                 };
             }
+
+            // Add osd
+            node_placement[osd_num] = json11::Json::object {
+                { "parent", node_placement[osd_num]["parent"].is_null() ? osd_host : node_placement[osd_num]["parent"] },
+                { "level", "osd" },
+            };
         }
 
-        return json11::Json::object { { "osds", peer_osds }, { "nodes", node_placement } };
+        return json11::Json::object { { "osds", existing_osds }, { "nodes", node_placement } };
     }
 
     // Returns new state_node_tree based on given state_node_tree with osds
@@ -534,15 +519,13 @@ resume_8:
     // filtered out by stats parameters (block_size, bitmap_granularity) in
     // given osd_stats and current pool config.
     // Requires: state_node_tree["osds"] must match osd_stats 1-1
-    json11::Json filter_state_node_tree_by_stats(const json11::Json & state_node_tree, std::vector<json11::Json> & osd_stats)
+    json11::Json filter_state_node_tree_by_stats(const json11::Json & state_node_tree, std::map<osd_num_t, json11::Json> & osd_stats)
     {
-        auto & osds = state_node_tree["osds"].array_items();
-
         // Accepted state_node_tree nodes
         auto accepted_nodes = state_node_tree["nodes"].object_items();
 
         // List of accepted osds
-        std::vector<std::string> accepted_osds;
+        json11::Json::array accepted_osds;
 
         block_size = cfg["block_size"].uint64_value()
             ? cfg["block_size"].uint64_value()
@@ -554,21 +537,25 @@ resume_8:
             ? etcd_state_client_t::parse_immediate_commit(cfg["immediate_commit"].string_value(), IMMEDIATE_ALL)
             : parent->cli->st_cli.global_immediate_commit;
 
-        for (size_t i = 0; i < osd_stats.size(); i++)
+        for (auto osd_num_json: state_node_tree["osds"].array_items())
         {
-            auto & os = osd_stats[i];
-            // Get osd number
-            auto osd_num = osds[i].as_string();
+            auto osd_num = osd_num_json.uint64_value();
+            auto os_it = osd_stats.find(osd_num);
+            if (os_it == osd_stats.end())
+            {
+                continue;
+            }
+            auto & os = os_it->second;
             if (!os["data_block_size"].is_null() && os["data_block_size"] != block_size ||
                 !os["bitmap_granularity"].is_null() && os["bitmap_granularity"] != bitmap_granularity ||
                 !os["immediate_commit"].is_null() &&
                 etcd_state_client_t::parse_immediate_commit(os["immediate_commit"].string_value(), IMMEDIATE_NONE) < immediate_commit)
             {
-                accepted_nodes.erase(osd_num);
+                accepted_nodes.erase(osd_num_json.as_string());
             }
             else
             {
-                accepted_osds.push_back(osd_num);
+                accepted_osds.push_back(osd_num_json);
             }
         }
 
@@ -576,87 +563,28 @@ resume_8:
     }
 
     // Returns maximum pg_size possible for given node_tree and failure_domain, starting at parent_node
-    uint64_t get_max_pg_size(json11::Json::object node_tree, const std::string & level, const std::string & parent_node)
+    uint64_t get_max_pg_size(json11::Json state_node_tree, const std::string & level, const std::string & root_node)
     {
-        uint64_t max_pg_sz = 0;
-
-        std::vector<std::string> nodes;
-
-        // Check if parent node is an osd (numeric)
-        if (parent_node != "" && stoull_full(parent_node))
+        std::set<std::string> level_seen;
+        for (auto & osd: state_node_tree["osds"].array_items())
         {
-            // Add it to node list if osd is in node tree
-            if (node_tree.find(parent_node) != node_tree.end())
-                nodes.push_back(parent_node);
-        }
-        // If parent node given, ...
-        else if (parent_node != "")
-        {
-            // ... look for children nodes of this parent
-            for (auto & sn: node_tree)
+            // find OSD parent at <level>, but stop at <root_node>
+            auto cur_id = osd.string_value();
+            auto cur = state_node_tree["nodes"][cur_id];
+            while (!cur.is_null())
             {
-                auto & props = sn.second.object_items();
-
-                auto parent_prop = props.find("parent");
-                if (parent_prop != props.end() && (parent_prop->second.as_string() == parent_node))
+                if (cur["level"] == level)
                 {
-                    nodes.push_back(sn.first);
-
-                    // If we're not looking for all osds, we only need a single
-                    // child osd node
-                    if (level != "osd" && stoull_full(sn.first))
-                        break;
+                    level_seen.insert(cur_id);
+                    break;
                 }
+                if (cur_id == root_node)
+                    break;
+                cur_id = cur["parent"].string_value();
+                cur = state_node_tree["nodes"][cur_id];
             }
         }
-        // No parent node given, and we're not looking for all osds
-        else if (level != "osd")
-        {
-            // ... look for all level nodes
-            for (auto & sn: node_tree)
-            {
-                auto & props = sn.second.object_items();
-
-                auto level_prop = props.find("level");
-                if (level_prop != props.end() && (level_prop->second.as_string() == level))
-                {
-                    nodes.push_back(sn.first);
-                }
-            }
-        }
-        // Otherwise, ...
-        else
-        {
-            // ... we're looking for osd nodes only
-            for (auto & sn: node_tree)
-            {
-                if (stoull_full(sn.first))
-                {
-                    nodes.push_back(sn.first);
-                }
-            }
-        }
-
-        // Process gathered nodes
-        for (auto & node: nodes)
-        {
-            // Check for osd node, return constant max size
-            if (stoull_full(node))
-            {
-                max_pg_sz += 1;
-            }
-            // Otherwise, ...
-            else
-            {
-                // ... exclude parent node from tree, and ...
-                node_tree.erase(parent_node);
-
-                // ... descend onto the resulting tree
-                max_pg_sz += get_max_pg_size(node_tree, level, node);
-            }
-        }
-
-        return max_pg_sz;
+        return level_seen.size();
     }
 
     json11::Json create_pool(const etcd_kv_t & kv)

src/cmd/cli_status.cpp

@@ -134,6 +134,7 @@ resume_2:
         }
         int osd_count = 0, osd_up = 0;
         uint64_t total_raw = 0, free_raw = 0, free_down_raw = 0, down_raw = 0;
+        std::vector<uint64_t> slow_op_primary_osds, slow_op_secondary_osds;
         parent->iterate_kvs_1(osd_stats, "/osd/stats/", [&](uint64_t stat_osd_num, json11::Json value)
         {
             osd_count++;
@@ -141,18 +142,29 @@ resume_2:
             auto osd_free = value["free"].uint64_value();
             total_raw += osd_size;
             free_raw += osd_free;
-            if (!osd_free)
+            if (osd_size)
             {
-                osds_full++;
-            }
-            else if (osd_free < (uint64_t)(osd_size*(1-osd_nearfull_ratio)))
-            {
-                osds_nearfull++;
+                if (!osd_free)
+                {
+                    osds_full++;
+                }
+                else if (osd_free < (uint64_t)(osd_size*(1-osd_nearfull_ratio)))
+                {
+                    osds_nearfull++;
+                }
             }
             auto peer_it = parent->cli->st_cli.peer_states.find(stat_osd_num);
             if (peer_it != parent->cli->st_cli.peer_states.end())
             {
                 osd_up++;
+                if (value["slow_ops_primary"].uint64_value() > 0)
+                {
+                    slow_op_primary_osds.push_back(stat_osd_num);
+                }
+                if (value["slow_ops_secondary"].uint64_value() > 0)
+                {
+                    slow_op_secondary_osds.push_back(stat_osd_num);
+                }
             }
             else
             {
@@ -216,6 +228,10 @@ resume_2:
                 { "mon_master", mon_master },
                 { "osd_up", osd_up },
                 { "osd_count", osd_count },
+                { "osds_full", osds_full },
+                { "osds_nearfull", osds_nearfull },
+                { "osds_primary_slow_ops", slow_op_primary_osds },
+                { "osds_secondary_slow_ops", slow_op_secondary_osds },
                 { "total_raw", total_raw },
                 { "free_raw", free_raw },
                 { "down_raw", down_raw },
@@ -300,6 +316,28 @@ resume_2:
             warning_str += "    "+std::to_string(osds_nearfull)+
                 (osds_nearfull > 1 ? " osds are almost full\n" : " osd is almost full\n");
         }
+        if (slow_op_primary_osds.size() > 0)
+        {
+            warning_str += "    "+std::to_string(slow_op_primary_osds.size());
+            warning_str += (slow_op_primary_osds.size() > 1 ? " osds have" : " osd has");
+            warning_str += " slow client ops: ";
+            for (int i = 0; i < slow_op_primary_osds.size(); i++)
+            {
+                warning_str += (i > 0 ? ", " : "")+std::to_string(slow_op_primary_osds[i]);
+            }
+            warning_str += "\n";
+        }
+        if (slow_op_secondary_osds.size() > 0)
+        {
+            warning_str += "    "+std::to_string(slow_op_secondary_osds.size());
+            warning_str += (slow_op_secondary_osds.size() > 1 ? " osds have" : " osd has");
+            warning_str += " slow replication ops: ";
+            for (int i = 0; i < slow_op_secondary_osds.size(); i++)
+            {
+                warning_str += (i > 0 ? ", " : "")+std::to_string(slow_op_secondary_osds[i]);
+            }
+            warning_str += "\n";
+        }
         if (warning_str != "")
         {
             warning_str = "\n  warning:\n"+warning_str;

src/disk_tool/disk_tool.cpp

@@ -27,12 +27,16 @@ static const char *help_text =
     "    --osd_per_disk <N>\n"
     "      Create <N> OSDs on each disk (default 1)\n"
     "    --hybrid\n"
-    "      Prepare hybrid (HDD+SSD) OSDs using provided devices. SSDs will be used for\n"
-    "      journals and metadata, HDDs will be used for data. Partitions for journals and\n"
-    "      metadata will be created automatically. Whether disks are SSD or HDD is decided\n"
-    "      by the `/sys/block/.../queue/rotational` flag. In hybrid mode, default object\n"
-    "      size is 1 MB instead of 128 KB, default journal size is 1 GB instead of 32 MB,\n"
-    "      and throttle_small_writes is enabled by default.\n"
+    "      Prepare hybrid (HDD+SSD, NVMe+SATA or etc) OSDs using provided devices. By default,\n"
+    "      any passed SSDs will be used for journals and metadata, HDDs will be used for data,\n"
+    "      but you can override this behaviour with --fast-devices option. Journal and metadata\n"
+    "      partitions will be created automatically. In the default mode, SSD and HDD disks\n"
+    "      are distinguished by the `/sys/block/.../queue/rotational` flag. When HDDs are used\n"
+    "      for data in hybrid mode, default block_size is 1 MB instead of 128 KB, default journal\n"
+    "      size is 1 GB instead of 32 MB, and throttle_small_writes is enabled by default.\n"
+    "    --fast-devices /dev/nvmeX,/dev/nvmeY\n"
+    "      In --hybrid mode, use these devices for journal and metadata instead of auto-detecting\n"
+    "      and extracting them from the main [devices...] list.\n"
     "    --disable_data_fsync auto\n"
     "      Disable data device cache and fsync (1/yes/true = on, default auto)\n"
     "    --disable_meta_fsync auto\n"
@@ -196,6 +200,7 @@ static const char *help_text =
     "    --device_size 0          Set device size\n"
     "    --format text            Result format: json, options, env, or text\n"
     "\n"
+    "Default I/O mode for commands involving disk I/O is O_DIRECT. If you don't want it, add --io cached.\n"
     "Use vitastor-disk --help <command> for command details or vitastor-disk --help --all for all details.\n"
 ;
 
@@ -220,6 +225,10 @@ int main(int argc, char *argv[])
         cmd.push_back((char*)"dump-journal");
         aliased = true;
     }
+    else if (!strcmp(exe_name, "vitastor-disk-test"))
+    {
+        self.test_mode = true;
+    }
     for (int i = 1; i < argc; i++)
     {
         if (!strcmp(argv[i], "--all"))
@@ -314,6 +323,7 @@ int main(int argc, char *argv[])
             // First argument is an OSD device - take metadata layout parameters from it
             if (self.dump_load_check_superblock(self.new_journal_device))
                 return 1;
+            self.new_journal_device = self.dsk.journal_device;
             self.new_journal_offset = self.dsk.journal_offset;
             self.new_journal_len = self.dsk.journal_len;
         }
@@ -379,6 +389,7 @@ int main(int argc, char *argv[])
             // First argument is an OSD device - take metadata layout parameters from it
             if (self.dump_load_check_superblock(self.new_meta_device))
                 return 1;
+            self.new_meta_device = self.dsk.meta_device;
             self.new_meta_offset = self.dsk.meta_offset;
             self.new_meta_len = self.dsk.meta_len;
         }

src/disk_tool/disk_tool.h

@@ -22,6 +22,7 @@
 #define VITASTOR_DISK_MAX_SB_SIZE 128*1024
 #define VITASTOR_PART_TYPE "e7009fac-a5a1-4d72-af72-53de13059903"
 #define DEFAULT_HYBRID_JOURNAL "1G"
+#define DEFAULT_HYBRID_SSD_JOURNAL "128M"
 
 struct resizer_data_moving_t;
 
@@ -40,6 +41,7 @@ struct disk_tool_t
     /**** Parameters ****/
 
     std::map<std::string, std::string> options;
+    bool test_mode = false;
     bool all, json, now;
     bool dump_with_blocks, dump_with_data;
     blockstore_disk_t dsk;
@@ -126,7 +128,8 @@ struct disk_tool_t
     uint32_t write_osd_superblock(std::string device, json11::Json params);
 
     int prepare_one(std::map<std::string, std::string> options, int is_hdd = -1);
-    int check_existing_partition(const std::string & dev);
+    int check_existing_partition(std::string & dev_by_uuid);
+    int fix_partition_type(std::string & dev_by_uuid);
     int prepare(std::vector<std::string> devices);
     std::vector<vitastor_dev_info_t> collect_devices(const std::vector<std::string> & devices);
     json11::Json add_partitions(vitastor_dev_info_t & devinfo, std::vector<std::string> sizes);
@@ -148,6 +151,6 @@ int write_zero(int fd, uint64_t offset, uint64_t size);
 json11::Json read_parttable(std::string dev);
 uint64_t dev_size_from_parttable(json11::Json pt);
 uint64_t free_from_parttable(json11::Json pt);
-int fix_partition_type(std::string dev_by_uuid);
+int fix_partition_type_uuid(std::string & dev_by_uuid, const std::string & type_uuid);
 std::string csum_type_str(uint32_t data_csum_type);
 uint32_t csum_type_from_str(std::string data_csum_type);

src/disk_tool/disk_tool_journal.cpp

@@ -18,7 +18,7 @@ int disk_tool_t::dump_journal()
         printf("[\n");
     if (all)
     {
-        dsk.journal_fd = open(dsk.journal_device.c_str(), O_DIRECT|O_RDONLY);
+        dsk.journal_fd = open(dsk.journal_device.c_str(), (options["io"] == "cached" ? 0 : O_DIRECT) | O_RDONLY);
         if (dsk.journal_fd < 0)
         {
             fprintf(stderr, "Failed to open journal device %s: %s\n", dsk.journal_device.c_str(), strerror(errno));
@@ -121,7 +121,7 @@ int disk_tool_t::dump_journal()
 
 int disk_tool_t::process_journal(std::function<int(void*)> block_fn)
 {
-    dsk.journal_fd = open(dsk.journal_device.c_str(), O_DIRECT|O_RDONLY);
+    dsk.journal_fd = open(dsk.journal_device.c_str(), (options["io"] == "cached" ? 0 : O_DIRECT) | O_RDONLY);
     if (dsk.journal_fd < 0)
     {
         fprintf(stderr, "Failed to open journal device %s: %s\n", dsk.journal_device.c_str(), strerror(errno));

src/disk_tool/disk_tool_meta.cpp

@@ -14,7 +14,7 @@ int disk_tool_t::process_meta(std::function<void(blockstore_meta_header_v2_t *)>
         fprintf(stderr, "Invalid metadata block size: is not a multiple of %d\n", DIRECT_IO_ALIGNMENT);
         return 1;
     }
-    dsk.meta_fd = open(dsk.meta_device.c_str(), O_DIRECT|O_RDONLY);
+    dsk.meta_fd = open(dsk.meta_device.c_str(), (options["io"] == "cached" ? 0 : O_DIRECT) | O_RDONLY);
     if (dsk.meta_fd < 0)
     {
         fprintf(stderr, "Failed to open metadata device %s: %s\n", dsk.meta_device.c_str(), strerror(errno));
@@ -159,7 +159,7 @@ int disk_tool_t::dump_load_check_superblock(const std::string & device)
     {
         auto cfg = json_to_string_map(sb["params"].object_items());
         dsk.parse_config(cfg);
-        dsk.data_io = dsk.meta_io = dsk.journal_io = "direct";
+        dsk.data_io = dsk.meta_io = dsk.journal_io = "cached";
         dsk.open_data();
         dsk.open_meta();
         dsk.open_journal();
@@ -315,8 +315,7 @@ int disk_tool_t::write_json_meta(json11::Json meta)
                 fromhexstr(e["data_csum"].string_value(), new_data_csum_size,
                     ((uint8_t*)new_entry) + sizeof(clean_disk_entry) + 2*new_clean_entry_bitmap_size);
             }
-            uint32_t *new_entry_csum = (uint32_t*)(((uint8_t*)new_entry) + sizeof(clean_disk_entry) +
-                2*new_clean_entry_bitmap_size + new_data_csum_size);
+            uint32_t *new_entry_csum = (uint32_t*)(((uint8_t*)new_entry) + new_clean_entry_size - 4);
             *new_entry_csum = crc32c(0, new_entry, new_clean_entry_size - 4);
         }
     }

src/disk_tool/disk_tool_prepare.cpp

@@ -29,18 +29,12 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
     };
     if (options.find("force") == options.end())
     {
-        std::vector<std::string> all_devs = { options["data_device"], options["meta_device"], options["journal_device"] };
-        for (int i = 0; i < all_devs.size(); i++)
+        std::string* all_devs[] = { &options["data_device"], &options["meta_device"], &options["journal_device"] };
+        for (int i = 0; i < 3; i++)
         {
-            const auto & dev = all_devs[i];
+            auto & dev = *all_devs[i];
             if (dev == "")
                 continue;
-            if (dev.substr(0, 22) != "/dev/disk/by-partuuid/")
-            {
-                // Partitions should be identified by GPT partition UUID
-                fprintf(stderr, "%s does not start with /dev/disk/by-partuuid/. Partitions should be identified by GPT partition UUIDs\n", dev.c_str());
-                return 1;
-            }
             std::string real_dev = realpath_str(dev, false);
             if (real_dev == "")
                 return 1;
@@ -114,7 +108,11 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
     try
     {
         dsk.parse_config(options);
-        dsk.data_io = dsk.meta_io = dsk.journal_io = "direct";
+        // Set all offsets to 4096 to calculate metadata size with excess
+        dsk.journal_offset = 4096;
+        dsk.meta_offset = 4096;
+        dsk.data_offset = 4096;
+        dsk.data_io = dsk.meta_io = dsk.journal_io = (options["io"] == "cached" ? "cached" : "direct");
         dsk.open_data();
         dsk.open_meta();
         dsk.open_journal();
@@ -159,7 +157,11 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
         return 1;
     }
     std::string osd_num_str;
-    if (shell_exec({ "vitastor-cli", "alloc-osd" }, "", &osd_num_str, NULL) != 0)
+    if (test_mode && options.find("osd_num") != options.end())
+    {
+        osd_num_str = options["osd_num"];
+    }
+    else if (shell_exec({ "vitastor-cli", "alloc-osd" }, "", &osd_num_str, NULL) != 0)
     {
         dsk.close_all();
         return 1;
@@ -173,8 +175,8 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
     }
     sb["osd_num"] = osd_num;
     // Zero out metadata and journal
-    if (write_zero(dsk.meta_fd, dsk.meta_offset, dsk.meta_len) != 0 ||
-        write_zero(dsk.journal_fd, dsk.journal_offset, dsk.journal_len) != 0)
+    if (write_zero(dsk.meta_fd, sb["meta_offset"].uint64_value(), dsk.meta_len) != 0 ||
+        write_zero(dsk.journal_fd, sb["journal_offset"].uint64_value(), dsk.journal_len) != 0)
     {
         fprintf(stderr, "Failed to zero out metadata or journal: %s\n", strerror(errno));
         dsk.close_all();
@@ -199,15 +201,18 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
     if (sep_j)
         desc += (sep_m ? " and journal on " : " with journal on ") + realpath_str(options["journal_device"]);
     fprintf(stderr, "Initialized OSD %ju on %s\n", osd_num, desc.c_str());
-    if (shell_exec({ "systemctl", "enable", "--now", "vitastor-osd@"+std::to_string(osd_num) }, "", NULL, NULL) != 0)
+    if (!test_mode || options.find("no_init") == options.end())
     {
-        fprintf(stderr, "Failed to enable systemd unit vitastor-osd@%ju\n", osd_num);
-        return 1;
+        if (shell_exec({ "systemctl", "enable", "--now", "vitastor-osd@"+std::to_string(osd_num) }, "", NULL, NULL) != 0)
+        {
+            fprintf(stderr, "Failed to enable systemd unit vitastor-osd@%ju\n", osd_num);
+            return 1;
+        }
     }
     return 0;
 }
 
-int disk_tool_t::check_existing_partition(const std::string & dev)
+int disk_tool_t::check_existing_partition(std::string & dev)
 {
     std::string out;
     if (shell_exec({ "wipefs", dev }, "", &out, NULL) != 0 || out != "")
@@ -229,11 +234,27 @@ int disk_tool_t::check_existing_partition(const std::string & dev)
     return 0;
 }
 
+int disk_tool_t::fix_partition_type(std::string & dev)
+{
+    std::string type_uuid = VITASTOR_PART_TYPE;
+    if (test_mode && options.find("part_type_uuid") != options.end())
+    {
+        type_uuid = options["part_type_uuid"];
+    }
+    return fix_partition_type_uuid(dev, type_uuid);
+}
+
 std::vector<vitastor_dev_info_t> disk_tool_t::collect_devices(const std::vector<std::string> & devices)
 {
     std::vector<vitastor_dev_info_t> devinfo;
+    std::set<std::string> seen;
     for (auto & dev: devices)
     {
+        if (seen.find(dev) != seen.end())
+        {
+            fprintf(stderr, "%s is specified multiple times, ignoring\n", dev.c_str());
+            continue;
+        }
         // Check if the device is a whole disk
         if (dev.substr(0, 5) != "/dev/")
         {
@@ -294,10 +315,6 @@ std::vector<vitastor_dev_info_t> disk_tool_t::collect_devices(const std::vector<
             .free = !pt.is_null() ? free_from_parttable(pt) : dev_size,
         });
     }
-    if (!devinfo.size())
-    {
-        fprintf(stderr, "No suitable devices found\n");
-    }
     return devinfo;
 }
 
@@ -348,47 +365,12 @@ json11::Json disk_tool_t::add_partitions(vitastor_dev_info_t & devinfo, std::vec
         fprintf(stderr, "Failed to add %zu partition(s) with sfdisk: new partitions not found in table\n", sizes.size());
         return {};
     }
-    // Check if new nodes exist and run partprobe if not
+    // Check if new devices exist, run partprobe if not, then wait until they appear
     // FIXME: We could use parted instead of sfdisk because partprobe is already a part of parted
-    int iter = 0, r;
-    while (true)
-    {
-        for (const auto & part: new_parts)
-        {
-            std::string link_path = "/dev/disk/by-partuuid/"+strtolower(part["uuid"].string_value());
-            struct stat st;
-            if (lstat(link_path.c_str(), &st) < 0)
-            {
-                if (errno == ENOENT)
-                {
-                    iter++;
-                    // Run partprobe
-                    std::string out;
-                    if (iter > 1 || (r = shell_exec({ "partprobe", devinfo.path }, "", &out, NULL)) != 0)
-                    {
-                        fprintf(
-                            stderr, iter == 1 && r == 255
-                                ? "partprobe utility is required to reread partition table while disk %s is in use\n"
-                                : "partprobe failed to re-read partition table while disk %s is in use\n",
-                            devinfo.path.c_str()
-                        );
-                        return {};
-                    }
-                    break;
-                }
-                else
-                {
-                    fprintf(stderr, "Failed to lstat %s: %s\n", link_path.c_str(), strerror(errno));
-                    return {};
-                }
-            }
-        }
-        break;
-    }
-    // Wait until device symlinks in /dev/disk/by-partuuid/ appear
     bool exists = false;
     const int max_iter = 300; // max 30 sec
-    iter = 0;
+    int iter = 0;
+    int r = 0;
     while (!exists && iter < max_iter)
     {
         exists = true;
@@ -396,28 +378,48 @@ json11::Json disk_tool_t::add_partitions(vitastor_dev_info_t & devinfo, std::vec
         {
             std::string link_path = "/dev/disk/by-partuuid/"+strtolower(part["uuid"].string_value());
             struct stat st;
-            if (lstat(link_path.c_str(), &st) < 0)
+            if (stat(part["node"].string_value().c_str(), &st) < 0 ||
+                lstat(link_path.c_str(), &st) < 0)
             {
                 if (errno == ENOENT)
                 {
                     exists = false;
                     if (iter == 4)
                     {
+                        // Print message after 400ms
                         fprintf(stderr, "Waiting for %s to appear for up to %d sec...\n", link_path.c_str(), max_iter/10);
                     }
                 }
                 else
                 {
-                    fprintf(stderr, "Failed to lstat %s: %s\n", link_path.c_str(), strerror(errno));
+                    fprintf(stderr, "Failed to stat %s or lstat %s: %s\n", part["node"].string_value().c_str(),
+                        link_path.c_str(), strerror(errno));
                     return {};
                 }
             }
         }
-        if (!exists)
+        if (exists)
         {
-            struct timespec ts = { .tv_sec = 0, .tv_nsec = 100000000 }; // 100ms
-            iter += (nanosleep(&ts, NULL) == 0);
+            break;
         }
+        if (!exists && iter == 0)
+        {
+            // Run partprobe
+            std::string out;
+            r = shell_exec({ "partprobe", devinfo.path }, "", &out, NULL);
+            if (r != 0)
+            {
+                fprintf(
+                    stderr, r == 255
+                        ? "partprobe utility is required to reread partition table while disk %s is in use\n"
+                        : "partprobe failed to re-read partition table while disk %s is in use\n",
+                    devinfo.path.c_str()
+                );
+                return {};
+            }
+        }
+        struct timespec ts = { .tv_sec = 0, .tv_nsec = 100000000 }; // 100ms
+        iter += (nanosleep(&ts, NULL) == 0 || !iter);
     }
     devinfo.pt = newpt;
     devinfo.osd_part_count += sizes.size();
@@ -500,7 +502,10 @@ int disk_tool_t::get_meta_partition(std::vector<vitastor_dev_info_t> & ssds, std
     {
         blockstore_disk_t dsk;
         dsk.parse_config(options);
-        dsk.data_io = dsk.meta_io = dsk.journal_io = "direct";
+        dsk.journal_offset = 4096;
+        dsk.meta_offset = 4096;
+        dsk.data_offset = 4096;
+        dsk.data_io = dsk.meta_io = dsk.journal_io = "cached";
         dsk.open_data();
         dsk.open_meta();
         dsk.open_journal();
@@ -510,6 +515,7 @@ int disk_tool_t::get_meta_partition(std::vector<vitastor_dev_info_t> & ssds, std
     }
     catch (std::exception & e)
     {
+        dsk.close_all();
         fprintf(stderr, "%s\n", e.what());
         return 1;
     }
@@ -564,9 +570,12 @@ int disk_tool_t::prepare(std::vector<std::string> devices)
 {
     if (options.find("data_device") != options.end() && options["data_device"] != "")
     {
-        if (options.find("hybrid") != options.end() || options.find("osd_per_disk") != options.end() || devices.size())
+        if (options.find("hybrid") != options.end() ||
+            options.find("fast_devices") != options.end() ||
+            options.find("osd_per_disk") != options.end() ||
+            devices.size())
         {
-            fprintf(stderr, "Device list (positional arguments) and --hybrid are incompatible with --data_device\n");
+            fprintf(stderr, "Device list (positional arguments), --osd_per_disk, --hybrid and --fast-devices are incompatible with --data_device\n");
             return 1;
         }
         return prepare_one(options, options.find("hdd") != options.end() ? 1 : 0);
@@ -583,8 +592,10 @@ int disk_tool_t::prepare(std::vector<std::string> devices)
     auto devinfo = collect_devices(devices);
     if (!devinfo.size())
     {
+        fprintf(stderr, "No suitable devices found\n");
         return 1;
     }
+    bool explicit_fast = options.find("fast_devices") != options.end();
     uint64_t osd_per_disk = stoull_full(options["osd_per_disk"]);
     if (!osd_per_disk)
         osd_per_disk = 1;
@@ -603,21 +614,55 @@ int disk_tool_t::prepare(std::vector<std::string> devices)
         if (options.find("disable_meta_fsync") == options.end())
             options["disable_meta_fsync"] = "auto";
         options["disable_journal_fsync"] = options["disable_meta_fsync"];
-        for (auto & dev: devinfo)
-            if (!dev.is_hdd)
-                ssds.push_back(dev);
-        if (!ssds.size())
+        if (explicit_fast)
         {
-            fprintf(stderr, "No SSDs found\n");
-            return 1;
+            auto fast = explode(",", options["fast_devices"], true);
+            ssds = collect_devices(fast);
+            if (!ssds.size())
+            {
+                fprintf(stderr, "No fast devices found\n");
+                return 1;
+            }
+            if (options["journal_size"] == "")
+            {
+                auto auto_journal_size = DEFAULT_HYBRID_SSD_JOURNAL;
+                for (auto & dev: devinfo)
+                {
+                    if (dev.is_hdd)
+                    {
+                        auto_journal_size = DEFAULT_HYBRID_JOURNAL;
+                        break;
+                    }
+                }
+                options["journal_size"] = auto_journal_size;
+            }
         }
-        else if (ssds.size() == devinfo.size())
+        else
         {
-            fprintf(stderr, "No HDDs found\n");
-            return 1;
+            std::vector<vitastor_dev_info_t> hdds;
+            for (auto & dev: devinfo)
+            {
+                if (!dev.is_hdd)
+                    ssds.push_back(dev);
+                else
+                    hdds.push_back(dev);
+            }
+            if (!ssds.size())
+            {
+                fprintf(stderr, "No SSDs found\n");
+                return 1;
+            }
+            if (!hdds.size())
+            {
+                fprintf(stderr, "No HDDs found\n");
+                return 1;
+            }
+            devinfo = hdds;
+            if (options["journal_size"] == "")
+            {
+                options["journal_size"] = DEFAULT_HYBRID_JOURNAL;
+            }
         }
-        if (options["journal_size"] == "")
-            options["journal_size"] = DEFAULT_HYBRID_JOURNAL;
     }
     else
     {
@@ -627,31 +672,28 @@ int disk_tool_t::prepare(std::vector<std::string> devices)
     auto journal_size = options["journal_size"];
     for (auto & dev: devinfo)
     {
-        if (!hybrid || dev.is_hdd)
+        // Select new partitions and create an OSD on each of them
+        for (const auto & uuid: get_new_data_parts(dev, osd_per_disk, max_other_percent))
         {
-            // Select new partitions and create an OSD on each of them
-            for (const auto & uuid: get_new_data_parts(dev, osd_per_disk, max_other_percent))
+            options["force"] = true;
+            options["data_device"] = "/dev/disk/by-partuuid/"+strtolower(uuid);
+            if (hybrid)
             {
-                options["force"] = true;
-                options["data_device"] = "/dev/disk/by-partuuid/"+strtolower(uuid);
-                if (hybrid)
+                // Select/create journal and metadata partitions
+                int r = get_meta_partition(ssds, options);
+                if (r != 0)
                 {
-                    // Select/create journal and metadata partitions
-                    int r = get_meta_partition(ssds, options);
-                    if (r != 0)
-                    {
-                        return 1;
-                    }
-                    options.erase("journal_size");
-                }
-                // Treat all disks as SSDs if not in the hybrid mode
-                prepare_one(options, dev.is_hdd ? 1 : 0);
-                if (hybrid)
-                {
-                    options["journal_size"] = journal_size;
-                    options.erase("journal_device");
-                    options.erase("meta_device");
+                    return 1;
                 }
+                options.erase("journal_size");
+            }
+            // Treat all disks as SSDs if not in the hybrid mode
+            prepare_one(options, dev.is_hdd ? 1 : 0);
+            if (hybrid)
+            {
+                options["journal_size"] = journal_size;
+                options.erase("journal_device");
+                options.erase("meta_device");
             }
         }
     }

src/disk_tool/disk_tool_resize.cpp

@@ -91,7 +91,7 @@ int disk_tool_t::resize_parse_params()
     try
     {
         dsk.parse_config(options);
-        dsk.data_io = dsk.meta_io = dsk.journal_io = "direct";
+        dsk.data_io = dsk.meta_io = dsk.journal_io = "cached";
         dsk.open_data();
         dsk.open_meta();
         dsk.open_journal();
@@ -114,7 +114,10 @@ int disk_tool_t::resize_parse_params()
     new_data_offset = options.find("new_data_offset") != options.end()
         ? parse_size(options["new_data_offset"]) : dsk.data_offset;
     new_data_len = options.find("new_data_len") != options.end()
-        ? parse_size(options["new_data_len"]) : dsk.data_len;
+        ? parse_size(options["new_data_len"])
+        : (options.find("new_data_offset") != options.end()
+            ? dsk.data_device_size-new_data_offset
+            : dsk.data_len);
     new_meta_offset = options.find("new_meta_offset") != options.end()
         ? parse_size(options["new_meta_offset"]) : dsk.meta_offset;
     new_meta_len = options.find("new_meta_len") != options.end()
@@ -123,6 +126,14 @@ int disk_tool_t::resize_parse_params()
         ? parse_size(options["new_journal_offset"]) : dsk.journal_offset;
     new_journal_len = options.find("new_journal_len") != options.end()
         ? parse_size(options["new_journal_len"]) : dsk.journal_len;
+    if (new_data_len+new_data_offset > dsk.data_device_size)
+        new_data_len = dsk.data_device_size-new_data_offset;
+    if (new_meta_device == dsk.data_device && new_data_offset < new_meta_offset &&
+        new_data_len+new_data_offset > new_meta_offset)
+        new_data_len = new_meta_offset-new_data_offset;
+    if (new_journal_device == dsk.data_device && new_data_offset < new_journal_offset &&
+        new_data_len+new_data_offset > new_journal_offset)
+        new_data_len = new_journal_offset-new_data_offset;
     if (new_meta_device == dsk.meta_device &&
         new_journal_device == dsk.journal_device &&
         new_data_offset == dsk.data_offset &&
@@ -159,10 +170,10 @@ void disk_tool_t::resize_init(blockstore_meta_header_v2_t *hdr)
         dsk.data_csum_type = hdr->data_csum_type;
         dsk.csum_block_size = hdr->csum_block_size;
     }
-    if (((new_data_len-dsk.data_len) % dsk.data_block_size) ||
-        ((new_data_offset-dsk.data_offset) % dsk.data_block_size))
+    if (((new_data_offset-dsk.data_offset) % dsk.data_block_size))
     {
-        fprintf(stderr, "Data alignment mismatch\n");
+        fprintf(stderr, "Data alignment mismatch: old data offset is 0x%jx, new is 0x%jx, but alignment on %x should be equal\n",
+            dsk.data_offset, new_data_offset, dsk.data_block_size);
         exit(1);
     }
     data_idx_diff = ((int64_t)(dsk.data_offset-new_data_offset)) / dsk.data_block_size;
@@ -220,10 +231,10 @@ int disk_tool_t::resize_remap_blocks()
     }
     for (uint64_t i = 0; i < free_last; i++)
     {
-        if (data_alloc->get(total_blocks-i))
-            data_remap[total_blocks-i] = 0;
+        if (data_alloc->get(total_blocks-i-1))
+            data_remap[total_blocks-i-1] = 0;
         else
-            data_alloc->set(total_blocks-i, true);
+            data_alloc->set(total_blocks-i-1, true);
     }
     for (auto & p: data_remap)
     {
@@ -246,7 +257,7 @@ int disk_tool_t::resize_copy_data()
         iodepth = 32;
     }
     ringloop = new ring_loop_t(iodepth < RINGLOOP_DEFAULT_SIZE ? RINGLOOP_DEFAULT_SIZE : iodepth);
-    dsk.data_fd = open(dsk.data_device.c_str(), O_DIRECT|O_RDWR);
+    dsk.data_fd = open(dsk.data_device.c_str(), (options["io"] == "cached" ? 0 : O_DIRECT) | O_RDWR);
     if (dsk.data_fd < 0)
     {
         fprintf(stderr, "Failed to open data device %s: %s\n", dsk.data_device.c_str(), strerror(errno));
@@ -441,7 +452,7 @@ int disk_tool_t::resize_rewrite_journal()
 
 int disk_tool_t::resize_write_new_journal()
 {
-    new_journal_fd = open(new_journal_device.c_str(), O_DIRECT|O_RDWR);
+    new_journal_fd = open(new_journal_device.c_str(), (options["io"] == "cached" ? 0 : O_DIRECT) | O_RDWR);
     if (new_journal_fd < 0)
     {
         fprintf(stderr, "Failed to open new journal device %s: %s\n", new_journal_device.c_str(), strerror(errno));
@@ -467,12 +478,13 @@ int disk_tool_t::resize_rewrite_meta()
             blockstore_meta_header_v2_t *new_hdr = (blockstore_meta_header_v2_t *)new_meta_buf;
             new_hdr->zero = 0;
             new_hdr->magic = BLOCKSTORE_META_MAGIC_V1;
-            new_hdr->version = BLOCKSTORE_META_FORMAT_V1;
+            new_hdr->version = BLOCKSTORE_META_FORMAT_V2;
             new_hdr->meta_block_size = dsk.meta_block_size;
             new_hdr->data_block_size = dsk.data_block_size;
             new_hdr->bitmap_granularity = dsk.bitmap_granularity ? dsk.bitmap_granularity : 4096;
             new_hdr->data_csum_type = dsk.data_csum_type;
             new_hdr->csum_block_size = dsk.csum_block_size;
+            new_hdr->header_csum = crc32c(0, new_hdr, sizeof(*new_hdr));
         },
         [this](uint64_t block_num, clean_disk_entry *entry, uint8_t *bitmap)
         {
@@ -481,7 +493,7 @@ int disk_tool_t::resize_rewrite_meta()
                 block_num = remap_it->second;
             if (block_num < free_first || block_num >= total_blocks-free_last)
             {
-                fprintf(stderr, "BUG: remapped block not in range\n");
+                fprintf(stderr, "BUG: remapped block %ju not in range %ju..%ju\n", block_num, free_first, total_blocks-free_last);
                 exit(1);
             }
             block_num += data_idx_diff;
@@ -494,6 +506,8 @@ int disk_tool_t::resize_rewrite_meta()
                 memcpy(new_entry->bitmap, bitmap, 2*new_clean_entry_bitmap_size + new_data_csum_size);
             else
                 memset(new_entry->bitmap, 0xff, 2*new_clean_entry_bitmap_size);
+            uint32_t *new_entry_csum = (uint32_t*)(((uint8_t*)new_entry) + new_clean_entry_size - 4);
+            *new_entry_csum = crc32c(0, new_entry, new_clean_entry_size - 4);
         }
     );
     if (r != 0)
@@ -507,7 +521,7 @@ int disk_tool_t::resize_rewrite_meta()
 
 int disk_tool_t::resize_write_new_meta()
 {
-    new_meta_fd = open(new_meta_device.c_str(), O_DIRECT|O_RDWR);
+    new_meta_fd = open(new_meta_device.c_str(), (options["io"] == "cached" ? 0 : O_DIRECT) | O_RDWR);
     if (new_meta_fd < 0)
     {
         fprintf(stderr, "Failed to open new metadata device %s: %s\n", new_meta_device.c_str(), strerror(errno));

src/disk_tool/disk_tool_resize_auto.cpp

@@ -37,6 +37,8 @@ int disk_tool_t::resize_data(std::string device)
         fprintf(stderr, "%s\n", e.what());
         return 1;
     }
+    // Save FD numbers because calc_lengths() relies on them
+    int old_journal_fd = dsk.journal_fd, old_meta_fd = dsk.meta_fd, old_data_fd = dsk.data_fd;
     dsk.close_all();
     bool dry_run = options.find("dry_run") != options.end();
     auto old_journal_device = dsk.journal_device;
@@ -48,6 +50,22 @@ int disk_tool_t::resize_data(std::string device)
         if (options.find("move_journal") == options.end())
             options["move_journal"] = dsk.journal_device == dsk.data_device ? "" : dsk.journal_device;
     }
+    uint64_t new_data_dev_size = 0;
+    if (options.find("data_size") != options.end())
+    {
+        new_data_dev_size = parse_size(options["data_size"]);
+        new_data_dev_size = options["data_size"] == "max" || new_data_dev_size > dsk.data_device_size
+            ? dsk.data_device_size : new_data_dev_size;
+        dsk.data_device_size = new_data_dev_size;
+        dsk.cfg_data_size = 0;
+        dsk.journal_fd = old_journal_fd;
+        dsk.meta_fd = old_meta_fd;
+        dsk.data_fd = old_data_fd;
+        dsk.calc_lengths(true);
+        dsk.journal_fd = -1;
+        dsk.meta_fd = -1;
+        dsk.data_fd = -1;
+    }
     std::map<std::string, std::string> move_options;
     if (options.find("move_journal") != options.end())
     {
@@ -69,14 +87,8 @@ int disk_tool_t::resize_data(std::string device)
     new_data_offset += ((dsk.data_offset-new_data_offset) % dsk.data_block_size);
     if (new_data_offset != dsk.data_offset)
         move_options["new_data_offset"] = std::to_string(new_data_offset);
-    if (options.find("data_size") != options.end())
-    {
-        auto new_data_dev_size = parse_size(options["data_size"]);
-        new_data_dev_size = options["data_size"] == "max" || new_data_dev_size > dsk.data_device_size
-            ? dsk.data_device_size : new_data_dev_size;
-        if (new_data_dev_size-dsk.data_offset != dsk.data_len)
-            move_options["new_data_len"] = std::to_string(new_data_dev_size-new_data_offset);
-    }
+    if (new_data_dev_size != 0)
+        move_options["new_data_len"] = std::to_string(new_data_dev_size-new_data_offset);
     new_meta_offset = 4096 + (new_meta_device == new_journal_device ? new_journal_len : 0);
     if (new_meta_offset != dsk.meta_offset)
         move_options["new_meta_offset"] = std::to_string(new_meta_offset);
@@ -188,17 +200,12 @@ int disk_tool_t::resize_parse_move_journal(std::map<std::string, std::string> &
             else
                 options["move_journal"] = "<new journal partition on "+parent_dev+">";
         }
-        else if (options["move_journal"].substr(0, 22) != "/dev/disk/by-partuuid/")
-        {
-            // Partitions should be identified by GPT partition UUID
-            fprintf(stderr, "%s does not start with /dev/disk/by-partuuid/. Partitions should be identified by GPT partition UUIDs\n", options["move_journal"].c_str());
-            return 1;
-        }
         else
         {
             // already a partition - check that it's a GPT partition with correct type
-            if (options.find("force") == options.end() &&
-                check_existing_partition(real_dev) != 0)
+            if ((options.find("force") == options.end()
+                ? check_existing_partition(options["move_journal"])
+                : fix_partition_type(options["move_journal"])) != 0)
             {
                 return 1;
             }
@@ -269,17 +276,12 @@ int disk_tool_t::resize_parse_move_meta(std::map<std::string, std::string> & mov
             else
                 options["move_meta"] = "<new metadata partition on "+parent_dev+">";
         }
-        else if (options["move_meta"].substr(0, 22) != "/dev/disk/by-partuuid/")
-        {
-            // Partitions should be identified by GPT partition UUID
-            fprintf(stderr, "%s does not start with /dev/disk/by-partuuid/. Partitions should be identified by GPT partition UUIDs\n", options["move_meta"].c_str());
-            return 1;
-        }
         else
         {
             // already a partition - check that it's a GPT partition with correct type
-            if (options.find("force") == options.end() &&
-                check_existing_partition(real_dev) != 0)
+            if ((options.find("force") == options.end()
+                ? check_existing_partition(options["move_meta"])
+                : fix_partition_type(options["move_meta"])) != 0)
             {
                 return 1;
             }

src/disk_tool/disk_tool_udev.cpp

@@ -122,7 +122,7 @@ uint32_t disk_tool_t::write_osd_superblock(std::string device, json11::Json para
     sb->size = sb_size;
     memcpy(sb->json_data, json_data.c_str(), json_data.size());
     sb->crc32c = crc32c(0, &sb->size, sb->size - ((uint8_t*)&sb->size - buf));
-    int fd = open(device.c_str(), O_DIRECT|O_RDWR);
+    int fd = open(device.c_str(), (options["io"] == "cached" ? 0 : O_DIRECT) | O_RDWR);
     if (fd < 0)
     {
         fprintf(stderr, "Failed to open device %s: %s\n", device.c_str(), strerror(errno));
@@ -150,7 +150,7 @@ json11::Json disk_tool_t::read_osd_superblock(std::string device, bool expect_ex
     json11::Json osd_params;
     std::string json_err;
     std::string real_device, device_type, real_data, real_meta, real_journal;
-    int r, fd = open(device.c_str(), O_DIRECT|O_RDWR);
+    int r, fd = open(device.c_str(), (options["io"] == "cached" ? 0 : O_DIRECT) | O_RDWR);
     if (fd < 0)
     {
         fprintf(stderr, "Failed to open device %s: %s\n", device.c_str(), strerror(errno));
@@ -385,7 +385,7 @@ int disk_tool_t::pre_exec_osd(std::string device)
 int disk_tool_t::clear_osd_superblock(const std::string & dev)
 {
     uint8_t *buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, 4096);
-    int fd = -1, r = open(dev.c_str(), O_DIRECT|O_RDWR);
+    int fd = -1, r = open(dev.c_str(), (options["io"] == "cached" ? 0 : O_DIRECT) | O_RDWR);
     if (r >= 0)
     {
         fd = r;

src/disk_tool/disk_tool_utils.cpp

@@ -343,23 +343,42 @@ uint64_t free_from_parttable(json11::Json pt)
     return free;
 }
 
-int fix_partition_type(std::string dev_by_uuid)
+int fix_partition_type_uuid(std::string & dev_by_uuid, const std::string & type_uuid)
 {
-    auto uuid = strtolower(dev_by_uuid.substr(dev_by_uuid.rfind('/')+1));
-    std::string parent_dev = get_parent_device(realpath_str(dev_by_uuid, false));
+    bool is_partuuid = dev_by_uuid.substr(0, 22) == "/dev/disk/by-partuuid/";
+    auto uuid = is_partuuid ? strtolower(dev_by_uuid.substr(22)) : "";
+    auto node = realpath_str(dev_by_uuid, false);
+    std::string parent_dev = get_parent_device(node);
     if (parent_dev == "")
         return 1;
     auto pt = read_parttable(parent_dev);
     if (pt.is_null() || pt.is_bool())
         return 1;
+    bool found = false;
     std::string script = "label: gpt\n\n";
     for (const auto & part: pt["partitions"].array_items())
     {
-        bool this_part = (strtolower(part["uuid"].string_value()) == uuid);
-        if (this_part && strtolower(part["type"].string_value()) == "e7009fac-a5a1-4d72-af72-53de13059903")
+        bool this_part = (part["node"].string_value() == node) &&
+            (!is_partuuid || strtolower(part["uuid"].string_value()) == uuid);
+        if (this_part)
         {
-            // Already correct type
-            return 0;
+            found = true;
+            if (!is_partuuid)
+            {
+                if (part["uuid"] == "")
+                {
+                    fprintf(stderr, "Could not determine partition UUID for %s. Please use GPT partitions\n", dev_by_uuid.c_str());
+                    return 1;
+                }
+                auto new_dev = "/dev/disk/by-partuuid/"+strtolower(part["uuid"].string_value());
+                fprintf(stderr, "Using %s instead of %s\n", new_dev.c_str(), dev_by_uuid.c_str());
+                dev_by_uuid = new_dev;
+            }
+            if (strtolower(part["type"].string_value()) == type_uuid)
+            {
+                // Already correct type
+                return 0;
+            }
         }
         script += part["node"].string_value()+": ";
         bool first = true;
@@ -369,13 +388,18 @@ int fix_partition_type(std::string dev_by_uuid)
             {
                 script += (first ? "" : ", ")+kv.first+"="+
                     (kv.first == "type" && this_part
-                        ? "e7009fac-a5a1-4d72-af72-53de13059903"
+                        ? type_uuid
                         : (kv.second.is_string() ? kv.second.string_value() : kv.second.dump()));
                 first = false;
             }
         }
         script += "\n";
     }
+    if (!found)
+    {
+        fprintf(stderr, "Could not find partition table entry for %s\n", dev_by_uuid.c_str());
+        return 1;
+    }
     std::string out;
     return shell_exec({ "sfdisk", "--no-reread", "--no-tell-kernel", "--force", parent_dev }, script, &out, NULL);
 }

src/kv/kv_cli.cpp

@@ -546,7 +546,7 @@ void kv_cli_t::handle_cmd(const std::vector<std::string> & cmd, std::function<vo
             auto & key = cmd[1];
             if (opname == "get")
             {
-                db->get(key, [this, cb](int res, const std::string & value)
+                db->get(key, [cb](int res, const std::string & value)
                 {
                     if (res < 0)
                         fprintf(stderr, "Error: %s (code %d)\n", strerror(-res), res);

src/kv/kv_db.cpp

@@ -1448,7 +1448,7 @@ void kv_op_t::update()
 
 void kv_op_t::update_find()
 {
-    get_block(db, cur_block, cur_level, recheck_policy, [=, checked_block = cur_block](int res, int refresh)
+    get_block(db, cur_block, cur_level, recheck_policy, [=](int res, int refresh)
     {
         res = handle_block(res, refresh, true);
         if (res == -EAGAIN)

src/nfs/CMakeLists.txt

@@ -5,6 +5,7 @@ project(vitastor)
 # vitastor-nfs
 add_executable(vitastor-nfs
 	nfs_proxy.cpp
+	nfs_proxy_rdma.cpp
 	nfs_block.cpp
 	nfs_kv.cpp
 	nfs_kv_create.cpp
@@ -21,8 +22,10 @@ add_executable(vitastor-nfs
 	nfs_fsstat.cpp
 	nfs_mount.cpp
 	nfs_portmap.cpp
+	rdma_alloc.cpp
 	../util/sha256.c
 	proto/xdr_impl.cpp
+	proto/rpc_rdma_xdr.cpp
 	proto/rpc_xdr.cpp
 	proto/portmap_xdr.cpp
 	proto/nfs_xdr.cpp
@@ -30,4 +33,5 @@ add_executable(vitastor-nfs
 target_link_libraries(vitastor-nfs
 	vitastor_client
 	vitastor_kv
+	${RDMACM_LIBRARIES}
 )

src/nfs/nfs_block.cpp

@@ -315,8 +315,7 @@ static int block_nfs3_read_proc(void *opaque, rpc_op_t *rop)
     if (aligned_count % alignment)
         aligned_count = aligned_count + alignment - (aligned_count % alignment);
     aligned_count -= aligned_offset;
-    void *buf = malloc_or_die(aligned_count);
-    xdr_add_malloc(rop->xdrs, buf);
+    void *buf = self->malloc_or_rdma(rop, aligned_count);
     cluster_op_t *op = new cluster_op_t;
     op->opcode = OSD_OP_READ;
     op->inode = ino_it->second;
@@ -335,10 +334,22 @@ static int block_nfs3_read_proc(void *opaque, rpc_op_t *rop)
         }
         else
         {
+            nfs_client_t *self = (nfs_client_t*)rop->client;
             auto & reply_ok = reply->resok;
             // reply_ok.data.data is already set above
             reply_ok.count = reply_ok.data.size;
             reply_ok.eof = 0;
+            if (self->rdma_conn)
+            {
+                // FIXME Linux NFS RDMA transport has a bug - when the reply
+                // doesn't contain post_op_attr, the data gets offsetted by
+                // 84 bytes (size of attributes)...
+                // So we have to fill it with RDMA. :-(
+                reply_ok.file_attributes = (post_op_attr){
+                    .attributes_follow = 1,
+                    .attributes = get_file_attributes(self, op->inode),
+                };
+            }
         }
         rpc_queue_reply(rop);
         delete op;
@@ -1057,22 +1068,25 @@ static void block_nfs3_readdir_common(void *opaque, rpc_op_t *rop, bool is_plus)
     void *prev = NULL;
     for (auto it = entries.begin(); it != entries.end();)
     {
-        entryplus3 *entry = &it->second;
-        // First fields of entry3 and entryplus3 are the same: fileid, name, cookie
-        entry->name = xdr_copy_string(rop->xdrs, it->first);
-        entry->cookie = idx++;
-        if (prev)
-        {
-            if (is_plus)
-                ((entryplus3*)prev)->nextentry = entry;
-            else
-                ((entry3*)prev)->nextentry = (entry3*)entry;
-        }
-        prev = entry;
-        if (args->cookie > 0 && entry->cookie == args->cookie)
-            entries.erase(entries.begin(), ++it);
+        if (args->cookie > 0 && idx <= args->cookie)
+            entries.erase(it++);
         else
+        {
+            entryplus3 *entry = &it->second;
+            // First fields of entry3 and entryplus3 are the same: fileid, name, cookie
+            entry->name = xdr_copy_string(rop->xdrs, it->first);
+            entry->cookie = idx;
+            if (prev)
+            {
+                if (is_plus)
+                    ((entryplus3*)prev)->nextentry = entry;
+                else
+                    ((entry3*)prev)->nextentry = (entry3*)entry;
+            }
+            prev = entry;
             it++;
+        }
+        idx++;
     }
     // Now limit results based on maximum reply size
     // Sadly we have to calculate reply size by hand

src/nfs/nfs_kv.cpp

@@ -65,7 +65,7 @@ int kv_map_type(const std::string & type)
         (type == "fifo" ? NF3FIFO : -1)))))));
 }
 
-fattr3 get_kv_attributes(nfs_client_t *self, uint64_t ino, json11::Json attrs)
+fattr3 get_kv_attributes(nfs_proxy_t *proxy, uint64_t ino, json11::Json attrs)
 {
     auto type = kv_map_type(attrs["type"].string_value());
     auto mode = attrs["mode"].uint64_value();
@@ -86,7 +86,7 @@ fattr3 get_kv_attributes(nfs_client_t *self, uint64_t ino, json11::Json attrs)
         .rdev = (type == NF3BLK || type == NF3CHR
             ? (specdata3){ (uint32_t)attrs["major"].uint64_value(), (uint32_t)attrs["minor"].uint64_value() }
             : (specdata3){}),
-        .fsid = self->parent->fsid,
+        .fsid = proxy->fsid,
         .fileid = ino,
         .atime = atime,
         .mtime = mtime,
@@ -119,7 +119,7 @@ std::string kv_direntry_filename(const std::string & key)
 std::string kv_inode_prefix_key(uint64_t ino, const char *prefix)
 {
     int max = 32+strlen(prefix);
-    char key[max] = { 0 };
+    char key[max];
     snprintf(key, max, "%s%x", prefix, INODE_POOL(ino));
     int n = strnlen(key, max-1);
     snprintf(key+n+1, max-n-1, "%jx", INODE_NO_POOL(ino));
@@ -349,6 +349,27 @@ kv_fs_state_t::~kv_fs_state_t()
     }
 }
 
+void kv_fs_state_t::write_inode(inode_t ino, json11::Json value, bool hack_cache, std::function<void(int)> cb, std::function<bool(int, const std::string &)> cas_cb)
+{
+    if (!proxy->rdma_context)
+    {
+        proxy->db->set(kv_inode_key(ino), value.dump(), cb, cas_cb);
+        return;
+    }
+    // FIXME Linux NFS RDMA transport has a bug - it corrupts the data (by offsetting it 84 bytes)
+    // when the READ reply doesn't contain post_op_attr. So we have to fill post_op_attr with RDMA. :-(
+    // So we at least cache it to not repeat K/V requests every read.
+    read_hack_cache.erase(ino);
+    proxy->db->set(kv_inode_key(ino), value.dump(), [=](int res)
+    {
+        if (hack_cache || res != 0)
+            read_hack_cache.erase(ino);
+        else
+            read_hack_cache[ino] = value;
+        cb(res);
+    }, cas_cb);
+}
+
 void kv_fs_state_t::update_inode(inode_t ino, bool allow_cache, std::function<void(json11::Json::object &)> change, std::function<void(int)> cb)
 {
     // FIXME: Use "update" query
@@ -356,12 +377,15 @@ void kv_fs_state_t::update_inode(inode_t ino, bool allow_cache, std::function<vo
     {
         if (!res)
         {
+            read_hack_cache.erase(ino);
             auto ientry = attrs.object_items();
             change(ientry);
             bool *found = new bool;
             *found = true;
-            proxy->db->set(kv_inode_key(ino), json11::Json(ientry).dump(), [=](int res)
+            json11::Json ientry_json(ientry);
+            proxy->db->set(kv_inode_key(ino), ientry_json.dump(), [=](int res)
             {
+                read_hack_cache.erase(ino);
                 if (!*found)
                     res = -ENOENT;
                 delete found;
@@ -384,6 +408,8 @@ void kv_fs_state_t::update_inode(inode_t ino, bool allow_cache, std::function<vo
 
 void kv_fs_state_t::touch_inodes()
 {
+    // Clear RDMA read fattr3 "hack" cache every second
+    read_hack_cache.clear();
     std::set<inode_t> q = std::move(touch_queue);
     for (auto ino: q)
     {

src/nfs/nfs_kv.h

@@ -75,6 +75,7 @@ struct kv_fs_state_t
     std::map<inode_t, kv_inode_extend_t> extends;
     std::set<inode_t> touch_queue;
     std::map<inode_t, uint64_t> volume_removed;
+    std::map<inode_t, json11::Json> read_hack_cache;
     uint64_t volume_stats_ctr = 0;
     uint64_t volume_touch_ctr = 0;
 
@@ -87,6 +88,7 @@ struct kv_fs_state_t
     void upgrade_db(std::function<void(int)> cb);
     void defrag_all(json11::Json cfg, std::function<void(int)> cb);
     void defrag_volume(inode_t ino, bool no_rm, bool dry_run, std::function<void(int, uint64_t, uint64_t, uint64_t)> cb);
+    void write_inode(inode_t ino, json11::Json value, bool hack_cache, std::function<void(int)> cb, std::function<bool(int, const std::string &)> cas_cb);
     ~kv_fs_state_t();
 };
 
@@ -116,7 +118,7 @@ nfstime3 nfstime_from_str(const std::string & s);
 std::string nfstime_to_str(nfstime3 t);
 std::string nfstime_now_str();
 int kv_map_type(const std::string & type);
-fattr3 get_kv_attributes(nfs_client_t *self, uint64_t ino, json11::Json attrs);
+fattr3 get_kv_attributes(nfs_proxy_t *proxy, uint64_t ino, json11::Json attrs);
 std::string kv_direntry_key(uint64_t dir_ino, const std::string & filename);
 std::string kv_direntry_filename(const std::string & key);
 std::string kv_inode_prefix_key(uint64_t ino, const char *prefix);

src/nfs/nfs_kv_create.cpp

@@ -143,11 +143,11 @@ resume_2:
         cb(st->res);
         return;
     }
-    st->self->parent->db->set(kv_inode_key(st->new_id), st->attrs.dump().c_str(), [st](int res)
+    st->self->parent->kvfs->write_inode(st->new_id, st->attrs, false, [st](int res)
     {
         st->res = res;
         kv_continue_create(st, 3);
-    }, [st](int res, const std::string & value)
+    }, [](int res, const std::string & value)
     {
         return res == -ENOENT;
     });
@@ -267,7 +267,7 @@ template<class T, class Tok> static void kv_create_reply(kv_create_state *st, in
                 },
                 .obj_attributes = {
                     .attributes_follow = 1,
-                    .attributes = get_kv_attributes(st->self, st->new_id, st->attrs),
+                    .attributes = get_kv_attributes(st->self->parent, st->new_id, st->attrs),
                 },
             },
         };

src/nfs/nfs_kv_defrag.cpp

@@ -81,12 +81,12 @@ void kv_fs_defrag_t::read()
             empty = true;
             for (; bitmap_pos < bitmap_size; bitmap_pos += 8)
             {
-                if (*((uint64_t*)(op->bitmap_buf + bitmap_pos)))
+                if (*((uint64_t*)((uint8_t*)op->bitmap_buf + bitmap_pos)))
                     empty = false;
             }
             for (; bitmap_pos < bitmap_size; bitmap_pos++)
             {
-                if (*((uint8_t*)(op->bitmap_buf + bitmap_pos)))
+                if (*((uint8_t*)((uint8_t*)op->bitmap_buf + bitmap_pos)))
                     empty = false;
             }
             buf_pos = 0;

src/nfs/nfs_kv_getattr.cpp

@@ -68,7 +68,7 @@ int kv_nfs3_getattr_proc(void *opaque, rpc_op_t *rop)
             *reply = (GETATTR3res){
                 .status = NFS3_OK,
                 .resok = (GETATTR3resok){
-                    .obj_attributes = get_kv_attributes(self, ino, attrs),
+                    .obj_attributes = get_kv_attributes(self->parent, ino, attrs),
                 },
             };
         }

src/nfs/nfs_kv_link.cpp

@@ -81,7 +81,7 @@ resume_1:
         {
             st->res = res;
             nfs_kv_continue_link(st, 2);
-        }, [st](int res, const std::string & old_value)
+        }, [](int res, const std::string & old_value)
         {
             return res == -ENOENT;
         });
@@ -102,7 +102,7 @@ resume_2:
         new_ientry["ctime"] = nfstime_now_str();
         st->ientry = new_ientry;
     }
-    st->self->parent->db->set(kv_inode_key(st->ino), st->ientry.dump(), [st](int res)
+    st->self->parent->kvfs->write_inode(st->ino, st->ientry, false, [st](int res)
     {
         st->res = res;
         nfs_kv_continue_link(st, 3);
@@ -180,7 +180,7 @@ int kv_nfs3_link_proc(void *opaque, rpc_op_t *rop)
                 .resok = (LINK3resok){
                     .file_attributes = (post_op_attr){
                         .attributes_follow = 1,
-                        .attributes = get_kv_attributes(st->self, st->ino, st->ientry),
+                        .attributes = get_kv_attributes(st->self->parent, st->ino, st->ientry),
                     },
                 },
             };

src/nfs/nfs_kv_lookup.cpp

@@ -55,7 +55,7 @@ int kv_nfs3_lookup_proc(void *opaque, rpc_op_t *rop)
                     .object = xdr_copy_string(rop->xdrs, kv_fh(ino)),
                     .obj_attributes = {
                         .attributes_follow = 1,
-                        .attributes = get_kv_attributes(self, ino, ientry),
+                        .attributes = get_kv_attributes(self->parent, ino, ientry),
                     },
                 },
             };
@@ -91,10 +91,14 @@ int kv_nfs3_readlink_proc(void *opaque, rpc_op_t *rop)
         }
         else
         {
+            std::string link_target = attrs["symlink"].string_value();
+            char *cp = (char*)self->malloc_or_rdma(rop, link_target.size()+1);
+            memcpy(cp, link_target.data(), link_target.size());
+            cp[link_target.size()] = 0;
             *reply = (READLINK3res){
                 .status = NFS3_OK,
                 .resok = (READLINK3resok){
-                    .data = xdr_copy_string(rop->xdrs, attrs["symlink"].string_value()),
+                    .data = (xdr_string_t){ link_target.size(), cp },
                 },
             };
         }

src/nfs/nfs_kv_read.cpp

@@ -37,6 +37,7 @@ static void nfs_kv_continue_read(nfs_kv_read_state *st, int state)
     else if (state == 1) goto resume_1;
     else if (state == 2) goto resume_2;
     else if (state == 3) goto resume_3;
+    else if (state == 4) goto resume_4;
     else
     {
         fprintf(stderr, "BUG: invalid state in nfs_kv_continue_read()");
@@ -96,7 +97,7 @@ resume_1:
                 }
                 read_size += sizeof(shared_file_header_t);
                 assert(!st->aligned_buf);
-                st->aligned_buf = (uint8_t*)malloc_or_die(read_size);
+                st->aligned_buf = (uint8_t*)st->self->malloc_or_rdma(st->rop, read_size);
                 st->buf = st->aligned_buf + sizeof(shared_file_header_t) + st->offset;
                 st->op->iov.push_back(st->aligned_buf, read_size);
                 st->op->len = align_up(read_offset+read_size) - st->op->offset;
@@ -106,7 +107,7 @@ resume_1:
                         st->op->offset+st->op->len - (read_offset+read_size));
                 }
             }
-            st->op->callback = [st, state](cluster_op_t *op)
+            st->op->callback = [st](cluster_op_t *op)
             {
                 st->res = op->retval == op->len ? 0 : op->retval;
                 delete op;
@@ -117,7 +118,7 @@ resume_1:
 resume_2:
             if (st->res < 0)
             {
-                free(st->aligned_buf);
+                st->self->free_or_rdma(st->rop, st->aligned_buf);
                 st->aligned_buf = NULL;
                 auto cb = std::move(st->cb);
                 cb(st->res);
@@ -131,7 +132,7 @@ resume_2:
                     " 0x%jx offset 0x%jx: probably a read/write conflict, retrying\n",
                     st->ino, st->ientry["shared_ino"].uint64_value(), st->ientry["shared_offset"].uint64_value());
                 st->retry++;
-                free(st->aligned_buf);
+                st->self->free_or_rdma(st->rop, st->aligned_buf);
                 st->aligned_buf = NULL;
                 st->allow_cache = false;
                 goto resume_0;
@@ -141,10 +142,39 @@ resume_2:
             return;
         }
     }
+    else if (st->self->rdma_conn)
+    {
+        // Take ientry from read_hack_cache for RDMA connections
+        {
+            auto rh_it = st->self->parent->kvfs->read_hack_cache.find(st->ino);
+            if (rh_it != st->self->parent->kvfs->read_hack_cache.end())
+            {
+                st->ientry = rh_it->second;
+            }
+        }
+        if (st->ientry.is_null())
+        {
+            kv_read_inode(st->self->parent, st->ino, [st](int res, const std::string & value, json11::Json attrs)
+            {
+                st->res = res;
+                st->ientry = attrs;
+                nfs_kv_continue_read(st, 4);
+            }, st->allow_cache);
+            return;
+resume_4:
+            if (st->res < 0 || kv_map_type(st->ientry["type"].string_value()) != NF3REG)
+            {
+                auto cb = std::move(st->cb);
+                cb(st->res < 0 ? st->res : -EINVAL);
+                return;
+            }
+            st->self->parent->kvfs->read_hack_cache[st->ino] = st->ientry;
+        }
+    }
     st->aligned_offset = align_down(st->offset);
     st->aligned_size = align_up(st->offset+st->size) - st->aligned_offset;
     assert(!st->aligned_buf);
-    st->aligned_buf = (uint8_t*)malloc_or_die(st->aligned_size);
+    st->aligned_buf = (uint8_t*)st->self->malloc_or_rdma(st->rop, st->aligned_size);
     st->buf = st->aligned_buf + st->offset - st->aligned_offset;
     st->op = new cluster_op_t;
     st->op->opcode = OSD_OP_READ;
@@ -163,7 +193,7 @@ resume_2:
 resume_3:
     if (st->res < 0)
     {
-        free(st->aligned_buf);
+        st->self->free_or_rdma(st->rop, st->aligned_buf);
         st->aligned_buf = NULL;
     }
     auto cb = std::move(st->cb);
@@ -194,11 +224,21 @@ int kv_nfs3_read_proc(void *opaque, rpc_op_t *rop)
         *reply = (READ3res){ .status = vitastor_nfs_map_err(res) };
         if (res == 0)
         {
-            xdr_add_malloc(st->rop->xdrs, st->aligned_buf);
             reply->resok.data.data = (char*)st->buf;
             reply->resok.data.size = st->size;
             reply->resok.count = st->size;
             reply->resok.eof = st->eof;
+            if (st->self->rdma_conn)
+            {
+                // FIXME Linux NFS RDMA transport has a bug - when the reply
+                // doesn't contain post_op_attr, the data gets offsetted by
+                // 84 bytes (size of attributes)...
+                // So we have to fill it with RDMA. :-(
+                reply->resok.file_attributes = (post_op_attr){
+                    .attributes_follow = 1,
+                    .attributes = get_kv_attributes(st->self->parent, st->ino, st->ientry),
+                };
+            }
         }
         rpc_queue_reply(st->rop);
         delete st;

src/nfs/nfs_kv_readdir.cpp

@@ -57,7 +57,7 @@ static void kv_getattr_next(nfs_kv_readdir_state *st)
                 st->entries[idx].name_attributes = (post_op_attr){
                     // FIXME: maybe do not read parent attributes and leave them to a GETATTR?
                     .attributes_follow = 1,
-                    .attributes = get_kv_attributes(st->self, st->entries[idx].fileid, ientry),
+                    .attributes = get_kv_attributes(st->self->parent, st->entries[idx].fileid, ientry),
                 };
             }
             st->getattr_running--;
@@ -126,7 +126,7 @@ resume_1:
             dot.fileid = st->dir_ino;
             dot.name_attributes = (post_op_attr){
                 .attributes_follow = 1,
-                .attributes = get_kv_attributes(st->self, st->dir_ino, st->ientry),
+                .attributes = get_kv_attributes(st->self->parent, st->dir_ino, st->ientry),
             };
             dot.name_handle = (post_op_fh3){
                 .handle_follows = 1,
@@ -169,7 +169,7 @@ resume_2:
         dotdot.name_attributes = (post_op_attr){
             // FIXME: maybe do not read parent attributes and leave them to a GETATTR?
             .attributes_follow = 1,
-            .attributes = get_kv_attributes(st->self,
+            .attributes = get_kv_attributes(st->self->parent,
                 st->parent_ino ? st->parent_ino : st->dir_ino,
                 st->parent_ino ? st->parent_ientry : st->ientry),
         };

src/nfs/nfs_kv_remove.cpp

@@ -169,7 +169,7 @@ resume_4:
             {
                 st->res2 = res;
                 nfs_kv_continue_delete(st, 5);
-            }, [st](int res, const std::string & value)
+            }, [](int res, const std::string & value)
             {
                 return res == -ENOENT;
             });
@@ -197,7 +197,7 @@ resume_5:
         auto copy = st->ientry.object_items();
         copy["nlink"] = st->ientry["nlink"].uint64_value()-1;
         copy["ctime"] = nfstime_now_str();
-        st->self->parent->db->set(kv_inode_key(st->ino), json11::Json(copy).dump(), [st](int res)
+        st->self->parent->kvfs->write_inode(st->ino, copy, false, [st](int res)
         {
             st->res = res;
             nfs_kv_continue_delete(st, 6);

src/nfs/nfs_kv_rename.cpp

@@ -240,7 +240,7 @@ resume_7:
                 copy["nlink"] = st->new_ientry["nlink"].uint64_value()-1;
                 copy["ctime"] = nfstime_now_str();
                 copy.erase("verf");
-                st->self->parent->db->set(kv_inode_key(st->new_direntry["ino"].uint64_value()), json11::Json(copy).dump(), [st](int res)
+                st->self->parent->kvfs->write_inode(st->new_direntry["ino"].uint64_value(), copy, false, [st](int res)
                 {
                     st->res = res;
                     nfs_kv_continue_rename(st, 8);
@@ -328,7 +328,7 @@ resume_11:
             ientry_new["parent_ino"] = st->new_dir_ino;
             ientry_new["ctime"] = nfstime_now_str();
             ientry_new.erase("verf");
-            st->self->parent->db->set(kv_inode_key(st->old_direntry["ino"].uint64_value()), json11::Json(ientry_new).dump(), [st](int res)
+            st->self->parent->kvfs->write_inode(st->old_direntry["ino"].uint64_value(), ientry_new, false, [st](int res)
             {
                 st->res = res;
                 nfs_kv_continue_rename(st, 12);

src/nfs/nfs_kv_setattr.cpp

@@ -84,7 +84,7 @@ resume_1:
     }
     st->new_attrs.erase("verf");
     st->new_attrs["ctime"] = nfstime_now_str();
-    st->self->parent->db->set(kv_inode_key(st->ino), json11::Json(st->new_attrs).dump(), [st](int res)
+    st->self->parent->kvfs->write_inode(st->ino, st->new_attrs, false, [st](int res)
     {
         st->res = res;
         nfs_kv_continue_setattr(st, 2);
@@ -190,7 +190,7 @@ int kv_nfs3_setattr_proc(void *opaque, rpc_op_t *rop)
                     .obj_wcc = (wcc_data){
                         .after = (post_op_attr){
                             .attributes_follow = 1,
-                            .attributes = get_kv_attributes(st->self, st->ino, st->new_attrs),
+                            .attributes = get_kv_attributes(st->self->parent, st->ino, st->new_attrs),
                         },
                     },
                 },

src/nfs/nfs_kv_write.cpp

@@ -553,7 +553,7 @@ static void nfs_do_align_write(nfs_kv_write_state *st, uint64_t ino, uint64_t of
     }
 }
 
-static std::string new_normal_ientry(nfs_kv_write_state *st)
+static json11::Json new_normal_ientry(nfs_kv_write_state *st)
 {
     auto ni = st->ientry.object_items();
     ni.erase("empty");
@@ -564,10 +564,10 @@ static std::string new_normal_ientry(nfs_kv_write_state *st)
     ni["size"] = st->ext->cur_extend;
     ni["ctime"] = ni["mtime"] = nfstime_now_str();
     ni.erase("verf");
-    return json11::Json(ni).dump();
+    return ni;
 }
 
-static std::string new_moved_ientry(nfs_kv_write_state *st)
+static json11::Json new_moved_ientry(nfs_kv_write_state *st)
 {
     auto ni = st->ientry.object_items();
     ni.erase("empty");
@@ -578,10 +578,10 @@ static std::string new_moved_ientry(nfs_kv_write_state *st)
     ni["size"] = st->new_size;
     ni["ctime"] = ni["mtime"] = nfstime_now_str();
     ni.erase("verf");
-    return json11::Json(ni).dump();
+    return ni;
 }
 
-static std::string new_shared_ientry(nfs_kv_write_state *st)
+static json11::Json new_shared_ientry(nfs_kv_write_state *st)
 {
     auto ni = st->ientry.object_items();
     ni.erase("empty");
@@ -589,10 +589,10 @@ static std::string new_shared_ientry(nfs_kv_write_state *st)
     ni["ctime"] = ni["mtime"] = nfstime_now_str();
     ni["shared_ver"] = ni["shared_ver"].uint64_value()+1;
     ni.erase("verf");
-    return json11::Json(ni).dump();
+    return ni;
 }
 
-static std::string new_unshared_ientry(nfs_kv_write_state *st)
+static json11::Json new_unshared_ientry(nfs_kv_write_state *st)
 {
     auto ni = st->ientry.object_items();
     ni.erase("empty");
@@ -602,7 +602,7 @@ static std::string new_unshared_ientry(nfs_kv_write_state *st)
     ni.erase("shared_ver");
     ni["ctime"] = ni["mtime"] = nfstime_now_str();
     ni.erase("verf");
-    return json11::Json(ni).dump();
+    return ni;
 }
 
 static void nfs_kv_extend_inode(nfs_kv_write_state *st, int state, int base_state)
@@ -612,7 +612,7 @@ static void nfs_kv_extend_inode(nfs_kv_write_state *st, int state, int base_stat
     st->ext->cur_extend = st->ext->next_extend;
     st->ext->next_extend = 0;
     st->res2 = -EAGAIN;
-    st->proxy->db->set(kv_inode_key(st->ino), new_normal_ientry(st), [st, base_state](int res)
+    st->proxy->kvfs->write_inode(st->ino, new_normal_ientry(st), true, [st, base_state](int res)
     {
         st->res = res;
         nfs_kv_continue_write(st, base_state+1);
@@ -838,7 +838,7 @@ resume_4:
                 cb(st->res);
                 return;
             }
-            st->proxy->db->set(kv_inode_key(st->ino), new_moved_ientry(st), [st](int res)
+            st->proxy->kvfs->write_inode(st->ino, new_moved_ientry(st), true, [st](int res)
             {
                 st->res = res;
                 nfs_kv_continue_write(st, 5);
@@ -881,7 +881,7 @@ resume_7:
             }
 resume_8:
             // We always have to change inode entry on shared writes
-            st->proxy->db->set(kv_inode_key(st->ino), new_shared_ientry(st), [st](int res)
+            st->proxy->kvfs->write_inode(st->ino, new_shared_ientry(st), true, [st](int res)
             {
                 st->res = res;
                 nfs_kv_continue_write(st, 9);
@@ -930,7 +930,7 @@ resume_11:
                 return;
             }
         }
-        st->proxy->db->set(kv_inode_key(st->ino), new_unshared_ientry(st), [st](int res)
+        st->proxy->kvfs->write_inode(st->ino, new_unshared_ientry(st), true, [st](int res)
         {
             st->res = res;
             nfs_kv_continue_write(st, 12);
@@ -953,7 +953,7 @@ resume_12:
         }
         // Record removed part of the shared inode as obsolete in statistics
         st->proxy->kvfs->volume_removed[st->ientry["shared_ino"].uint64_value()] += st->ientry["shared_alloc"].uint64_value();
-        st->ientry_text = new_unshared_ientry(st);
+        st->ientry_text = new_unshared_ientry(st).dump();
     }
     // Non-shared write
     nfs_do_align_write(st, st->ino, st->offset, 0, 13);

src/nfs/nfs_proxy.cpp

@@ -34,6 +34,9 @@ const char *exe_name = NULL;
 
 nfs_proxy_t::~nfs_proxy_t()
 {
+#ifdef WITH_RDMACM
+    destroy_rdma();
+#endif
     if (kvfs)
         delete kvfs;
     if (blockfs)
@@ -65,9 +68,16 @@ static const char* help_text =
     "\n"
     "vitastor-nfs (--fs <NAME> | --block) start\n"
     "  Start network NFS server. Options:\n"
-    "  --bind <IP>       bind service to <IP> address (default 0.0.0.0)\n"
-    "  --port <PORT>     use port <PORT> for NFS services (default is 2049)\n"
-    "  --portmap 0       do not listen on port 111 (portmap/rpcbind, requires root)\n"
+    "  --bind <IP>           bind service to <IP> address (default 0.0.0.0)\n"
+    "  --port <PORT>         use port <PORT> for NFS services (default is 2049)\n"
+    "                        specify \"auto\" to auto-select and print port\n"
+    "  --portmap 0           do not listen on port 111 (portmap/rpcbind, requires root)\n"
+    "  --nfs_rdma <PORT>     enable NFS-RDMA at RDMA-CM port <PORT> (you can try 20049)\n"
+    "                        if RDMA is enabled and --port is set to 0, TCP will be disabled\n"
+    "  --nfs_rdma_credit 16  maximum operation credit for RDMA clients (max iodepth)\n"
+    "  --nfs_rdma_send 1024  maximum RDMA send operation count (should be larger than iodepth)\n"
+    "  --nfs_rdma_alloc 1M   RDMA memory allocation rounding\n"
+    "  --nfs_rdma_gc 64M     maximum unused RDMA buffers\n"
     "\n"
     "vitastor-nfs --fs <NAME> upgrade\n"
     "  Upgrade FS metadata. Can be run online, but server(s) should be restarted\n"
@@ -184,6 +194,7 @@ void nfs_proxy_t::run(json11::Json cfg)
     srand48(tv.tv_sec*1000000000 + tv.tv_nsec);
     server_id = (uint64_t)lrand48() | ((uint64_t)lrand48() << 31) | ((uint64_t)lrand48() << 62);
     // Parse options
+    mountpoint = cfg["mount"].string_value();
     if (cfg["logfile"].string_value() != "")
         logfile = cfg["logfile"].string_value();
     pidfile = cfg["pidfile"].string_value();
@@ -194,8 +205,24 @@ void nfs_proxy_t::run(json11::Json cfg)
     default_pool = cfg["pool"].as_string();
     portmap_enabled = !json_is_false(cfg["portmap"]);
     nfs_port = cfg["port"].uint64_value() & 0xffff;
+    nfs_rdma_port = cfg["nfs_rdma"].uint64_value() & 0xffff;
+    // Allow RDMA-only mode if port is explicitly set to 0
+    // Allow port auto-selection in server mode if explicitly set to --port auto
+    nfs_port_auto = cfg["port"] == "auto";
     if (!nfs_port)
-        nfs_port = 2049;
+        nfs_port = nfs_port_auto ? 0 : (!cfg["port"].is_null() && nfs_rdma_port ? -1 : 2049);
+    nfs_rdma_credit = cfg["nfs_rdma_credit"].uint64_value();
+    if (!nfs_rdma_credit)
+        nfs_rdma_credit = 16;
+    nfs_rdma_max_send = cfg["nfs_rdma_send"].uint64_value();
+    if (!nfs_rdma_max_send)
+        nfs_rdma_max_send = 1024;
+    nfs_rdma_alloc = cfg["nfs_rdma_alloc"].uint64_value();
+    if (!nfs_rdma_alloc)
+        nfs_rdma_alloc = 1048576;
+    nfs_rdma_gc = cfg["nfs_rdma_gc"].uint64_value();
+    if (!nfs_rdma_gc)
+        nfs_rdma_gc = 64*1048576;
     export_root = cfg["nfspath"].string_value();
     if (!export_root.size())
         export_root = "/";
@@ -207,7 +234,6 @@ void nfs_proxy_t::run(json11::Json cfg)
         obj["client_writeback_allowed"] = true;
         cfg = obj;
     }
-    mountpoint = cfg["mount"].string_value();
     if (mountpoint != "")
     {
         bind_address = "127.0.0.1";
@@ -292,49 +318,60 @@ void nfs_proxy_t::run(json11::Json cfg)
 
 void nfs_proxy_t::run_server(json11::Json cfg)
 {
+    if (nfs_port != -1)
+    {
+        // Create NFS socket and add it to epoll
+        int nfs_socket = create_and_bind_socket(bind_address, nfs_port, 128, &listening_port);
+        fcntl(nfs_socket, F_SETFL, fcntl(nfs_socket, F_GETFL, 0) | O_NONBLOCK);
+        epmgr->tfd->set_fd_handler(nfs_socket, false, [this](int nfs_socket, int epoll_events)
+        {
+            if (epoll_events & EPOLLRDHUP)
+            {
+                fprintf(stderr, "Listening portmap socket disconnected, exiting\n");
+                exit(1);
+            }
+            else
+            {
+                do_accept(nfs_socket);
+            }
+        });
+        if (nfs_port_auto)
+        {
+            printf("Port: %d\n", listening_port);
+        }
+    }
+    else
+    {
+        listening_port = nfs_rdma_port;
+    }
     // Self-register portmap and NFS
     pmap.reg_ports.insert((portmap_id_t){
         .prog = PMAP_PROGRAM,
         .vers = PMAP_V2,
-        .port = portmap_enabled ? 111 : nfs_port,
+        .port = (unsigned)(portmap_enabled ? 111 : listening_port),
         .owner = "portmapper-service",
-        .addr = portmap_enabled ? "0.0.0.0.0.111" : ("0.0.0.0.0."+std::to_string(nfs_port)),
+        .addr = portmap_enabled ? "0.0.0.0.0.111" : ("0.0.0.0.0."+std::to_string(listening_port)),
     });
     pmap.reg_ports.insert((portmap_id_t){
         .prog = PMAP_PROGRAM,
         .vers = PMAP_V3,
-        .port = portmap_enabled ? 111 : nfs_port,
+        .port = (unsigned)(portmap_enabled ? 111 : listening_port),
         .owner = "portmapper-service",
-        .addr = portmap_enabled ? "0.0.0.0.0.111" : ("0.0.0.0.0."+std::to_string(nfs_port)),
+        .addr = portmap_enabled ? "0.0.0.0.0.111" : ("0.0.0.0.0."+std::to_string(listening_port)),
     });
     pmap.reg_ports.insert((portmap_id_t){
         .prog = NFS_PROGRAM,
         .vers = NFS_V3,
-        .port = nfs_port,
+        .port = (unsigned)listening_port,
         .owner = "nfs-server",
-        .addr = "0.0.0.0.0."+std::to_string(nfs_port),
+        .addr = "0.0.0.0.0."+std::to_string(listening_port),
     });
     pmap.reg_ports.insert((portmap_id_t){
         .prog = MOUNT_PROGRAM,
         .vers = MOUNT_V3,
-        .port = nfs_port,
+        .port = (unsigned)listening_port,
         .owner = "rpc.mountd",
-        .addr = "0.0.0.0.0."+std::to_string(nfs_port),
-    });
-    // Create NFS socket and add it to epoll
-    int nfs_socket = create_and_bind_socket(bind_address, nfs_port, 128, &listening_port);
-    fcntl(nfs_socket, F_SETFL, fcntl(nfs_socket, F_GETFL, 0) | O_NONBLOCK);
-    epmgr->tfd->set_fd_handler(nfs_socket, false, [this](int nfs_socket, int epoll_events)
-    {
-        if (epoll_events & EPOLLRDHUP)
-        {
-            fprintf(stderr, "Listening portmap socket disconnected, exiting\n");
-            exit(1);
-        }
-        else
-        {
-            do_accept(nfs_socket);
-        }
+        .addr = "0.0.0.0.0."+std::to_string(listening_port),
     });
     if (portmap_enabled)
     {
@@ -354,6 +391,10 @@ void nfs_proxy_t::run_server(json11::Json cfg)
             }
         });
     }
+    if (nfs_rdma_port)
+    {
+        rdma_context = create_rdma(bind_address, nfs_rdma_port, nfs_rdma_credit, nfs_rdma_max_send, nfs_rdma_alloc, nfs_rdma_gc);
+    }
     if (mountpoint != "")
     {
         mount_fs();
@@ -499,6 +540,20 @@ void nfs_proxy_t::check_default_pool()
     }
 }
 
+nfs_client_t *nfs_proxy_t::create_client()
+{
+    auto cli = new nfs_client_t();
+    cli->parent = this;
+    if (kvfs)
+        nfs_kv_procs(cli);
+    else
+        nfs_block_procs(cli);
+    for (auto & fn: pmap.proc_table)
+        cli->proc_table.insert(fn);
+    rpc_clients.insert(cli);
+    return cli;
+}
+
 void nfs_proxy_t::do_accept(int listen_fd)
 {
     struct sockaddr_storage addr;
@@ -512,18 +567,8 @@ void nfs_proxy_t::do_accept(int listen_fd)
         fcntl(nfs_fd, F_SETFL, fcntl(nfs_fd, F_GETFL, 0) | O_NONBLOCK);
         int one = 1;
         setsockopt(nfs_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
-        auto cli = new nfs_client_t();
-        if (kvfs)
-            nfs_kv_procs(cli);
-        else
-            nfs_block_procs(cli);
-        cli->parent = this;
+        auto cli = this->create_client();
         cli->nfs_fd = nfs_fd;
-        for (auto & fn: pmap.proc_table)
-        {
-            cli->proc_table.insert(fn);
-        }
-        rpc_clients[nfs_fd] = cli;
         epmgr->tfd->set_fd_handler(nfs_fd, true, [cli](int nfs_fd, int epoll_events)
         {
             // Handle incoming event
@@ -780,11 +825,17 @@ void nfs_client_t::stop()
     stopped = true;
     if (refs <= 0)
     {
+#ifdef WITH_RDMACM
+        destroy_rdma_conn();
+#endif
         auto parent = this->parent;
-        parent->rpc_clients.erase(nfs_fd);
+        parent->rpc_clients.erase(this);
         parent->active_connections--;
-        parent->epmgr->tfd->set_fd_handler(nfs_fd, true, NULL);
-        close(nfs_fd);
+        if (nfs_fd >= 0)
+        {
+            parent->epmgr->tfd->set_fd_handler(nfs_fd, true, NULL);
+            close(nfs_fd);
+        }
         delete this;
         parent->check_exit();
     }
@@ -813,8 +864,7 @@ void nfs_client_t::handle_send(int result)
                 if (rop)
                 {
                     // Reply fully sent
-                    xdr_reset(rop->xdrs);
-                    parent->xdr_pool.push_back(rop->xdrs);
+                    parent->free_xdr(rop->xdrs);
                     if (rop->buffer && rop->referenced)
                     {
                         // Dereference the buffer
@@ -831,7 +881,7 @@ void nfs_client_t::handle_send(int result)
                             {
                                 // FIXME Maybe put free_buffers into parent
                                 free_buffers.push_back((rpc_free_buffer_t){
-                                    .buf = rop->buffer,
+                                    .buf = (uint8_t*)rop->buffer,
                                     .size = ub.size,
                                 });
                                 used_buffers.erase(rop->buffer);
@@ -876,8 +926,6 @@ void nfs_client_t::handle_send(int result)
 void rpc_queue_reply(rpc_op_t *rop)
 {
     nfs_client_t *self = (nfs_client_t*)rop->client;
-    iovec *iov_list = NULL;
-    unsigned iov_count = 0;
     int r = xdr_encode(rop->xdrs, (xdrproc_t)xdr_rpc_msg, &rop->out_msg);
     assert(r);
     if (rop->reply_fn != NULL)
@@ -885,55 +933,78 @@ void rpc_queue_reply(rpc_op_t *rop)
         r = xdr_encode(rop->xdrs, rop->reply_fn, rop->reply);
         assert(r);
     }
-    xdr_encode_finish(rop->xdrs, &iov_list, &iov_count);
-    assert(iov_count > 0);
-    rop->reply_marker = 0;
-    for (unsigned i = 0; i < iov_count; i++)
+#ifdef WITH_RDMACM
+    if (!self->rdma_conn)
+#endif
     {
-        rop->reply_marker += iov_list[i].iov_len;
-    }
-    rop->reply_marker = htobe32(rop->reply_marker | 0x80000000);
-    auto & to_send_list = self->write_msg.msg_iovlen ? self->next_send_list : self->send_list;
-    auto & to_outbox = self->write_msg.msg_iovlen ? self->next_outbox : self->outbox;
-    to_send_list.push_back((iovec){ .iov_base = &rop->reply_marker, .iov_len = 4 });
-    to_outbox.push_back(NULL);
-    for (unsigned i = 0; i < iov_count; i++)
-    {
-        to_send_list.push_back(iov_list[i]);
+        iovec *iov_list = NULL;
+        unsigned iov_count = 0;
+        xdr_encode_finish(rop->xdrs, &iov_list, &iov_count);
+        assert(iov_count > 0);
+        rop->reply_marker = 0;
+        for (unsigned i = 0; i < iov_count; i++)
+        {
+            rop->reply_marker += iov_list[i].iov_len;
+        }
+        rop->reply_marker = htobe32(rop->reply_marker | 0x80000000);
+        auto & to_send_list = self->write_msg.msg_iovlen ? self->next_send_list : self->send_list;
+        auto & to_outbox = self->write_msg.msg_iovlen ? self->next_outbox : self->outbox;
+        to_send_list.push_back((iovec){ .iov_base = &rop->reply_marker, .iov_len = 4 });
         to_outbox.push_back(NULL);
+        for (unsigned i = 0; i < iov_count; i++)
+        {
+            to_send_list.push_back(iov_list[i]);
+            to_outbox.push_back(NULL);
+        }
+        to_outbox[to_outbox.size()-1] = rop;
+        self->submit_send();
     }
-    to_outbox[to_outbox.size()-1] = rop;
-    self->submit_send();
+#ifdef WITH_RDMACM
+    else
+    {
+        self->rdma_queue_reply(rop);
+    }
+#endif
 }
 
-int nfs_client_t::handle_rpc_message(void *base_buf, void *msg_buf, uint32_t msg_len)
+XDR *nfs_proxy_t::get_xdr()
 {
     // Take an XDR object from the pool
     XDR *xdrs;
-    if (parent->xdr_pool.size())
+    if (xdr_pool.size())
     {
-        xdrs = parent->xdr_pool.back();
-        parent->xdr_pool.pop_back();
+        xdrs = xdr_pool.back();
+        xdr_pool.pop_back();
     }
     else
     {
         xdrs = xdr_create();
     }
+    return xdrs;
+}
+
+void nfs_proxy_t::free_xdr(XDR *xdrs)
+{
+    xdr_reset(xdrs);
+    xdr_pool.push_back(xdrs);
+}
+
+int nfs_client_t::handle_rpc_message(void *base_buf, void *msg_buf, uint32_t msg_len)
+{
+    XDR *xdrs = parent->get_xdr();
     // Decode the RPC header
     char inmsg_data[sizeof(rpc_msg)];
     rpc_msg *inmsg = (rpc_msg*)&inmsg_data;
     if (!xdr_decode(xdrs, msg_buf, msg_len, (xdrproc_t)xdr_rpc_msg, inmsg))
     {
         // Invalid message, ignore it
-        xdr_reset(xdrs);
-        parent->xdr_pool.push_back(xdrs);
+        parent->free_xdr(xdrs);
         return 0;
     }
     if (inmsg->body.dir != RPC_CALL)
     {
         // Reply sent to the server? Strange thing. Also ignore it
-        xdr_reset(xdrs);
-        parent->xdr_pool.push_back(xdrs);
+        parent->free_xdr(xdrs);
         return 0;
     }
     if (inmsg->body.cbody.rpcvers != RPC_MSG_VERSION)
@@ -968,6 +1039,17 @@ int nfs_client_t::handle_rpc_message(void *base_buf, void *msg_buf, uint32_t msg
         // Incoming buffer isn't needed to handle request, so return 0
         return 0;
     }
+    auto rop = create_rpc_op(xdrs, base_buf, inmsg, NULL);
+    if (!rop)
+    {
+        // No such procedure
+        return 0;
+    }
+    return handle_rpc_op(rop);
+}
+
+rpc_op_t *nfs_client_t::create_rpc_op(XDR *xdrs, void *buffer, rpc_msg *inmsg, rdma_msg *rmsg)
+{
     // Find decoder for the request
     auto proc_it = proc_table.find((rpc_service_proc_t){
         .prog = inmsg->body.cbody.prog,
@@ -995,6 +1077,7 @@ int nfs_client_t::handle_rpc_message(void *base_buf, void *msg_buf, uint32_t msg
         rpc_op_t *rop = (rpc_op_t*)malloc_or_die(sizeof(rpc_op_t));
         *rop = (rpc_op_t){
             .client = this,
+            .buffer = buffer,
             .xdrs = xdrs,
             .out_msg = (rpc_msg){
                 .xid = inmsg->xid,
@@ -1017,9 +1100,15 @@ int nfs_client_t::handle_rpc_message(void *base_buf, void *msg_buf, uint32_t msg
                 },
             },
         };
+        // FIXME: malloc and avoid copy?
+        memcpy(&rop->in_msg, inmsg, sizeof(rpc_msg));
+        if (rmsg)
+        {
+            memcpy(&rop->in_rdma_msg, rmsg, sizeof(rdma_msg));
+        }
         rpc_queue_reply(rop);
         // Incoming buffer isn't needed to handle request, so return 0
-        return 0;
+        return NULL;
     }
     // Allocate memory
     rpc_op_t *rop = (rpc_op_t*)malloc_or_die(
@@ -1028,7 +1117,7 @@ int nfs_client_t::handle_rpc_message(void *base_buf, void *msg_buf, uint32_t msg
     rpc_reply_stat x = RPC_MSG_ACCEPTED;
     *rop = (rpc_op_t){
         .client = this,
-        .buffer = (uint8_t*)base_buf,
+        .buffer = buffer,
         .xdrs = xdrs,
         .out_msg = (rpc_msg){
             .xid = inmsg->xid,
@@ -1045,10 +1134,25 @@ int nfs_client_t::handle_rpc_message(void *base_buf, void *msg_buf, uint32_t msg
         .request = ((uint8_t*)rop) + sizeof(rpc_op_t),
         .reply = ((uint8_t*)rop) + sizeof(rpc_op_t) + proc_it->req_size,
     };
+    // FIXME: malloc and avoid copy?
     memcpy(&rop->in_msg, inmsg, sizeof(rpc_msg));
+    if (rmsg)
+    {
+        memcpy(&rop->in_rdma_msg, rmsg, sizeof(rdma_msg));
+    }
+    return rop;
+}
+
+int nfs_client_t::handle_rpc_op(rpc_op_t *rop)
+{
     // Try to decode the request
     // req_fn may be NULL, that means function has no arguments
-    if (proc_it->req_fn && !proc_it->req_fn(xdrs, rop->request))
+    auto proc_it = proc_table.find((rpc_service_proc_t){
+        .prog = rop->in_msg.body.cbody.prog,
+        .vers = rop->in_msg.body.cbody.vers,
+        .proc = rop->in_msg.body.cbody.proc,
+    });
+    if (proc_it == proc_table.end() || proc_it->req_fn && !proc_it->req_fn(rop->xdrs, rop->request))
     {
         // Invalid request
         rop->out_msg.body.rbody.areply.reply_data.stat = RPC_GARBAGE_ARGS;
@@ -1058,18 +1162,55 @@ int nfs_client_t::handle_rpc_message(void *base_buf, void *msg_buf, uint32_t msg
     }
     rop->out_msg.body.rbody.areply.reply_data.stat = RPC_SUCCESS;
     rop->reply_fn = proc_it->resp_fn;
+    rop->referenced = 0;
     int ref = proc_it->handler_fn(proc_it->opaque, rop);
-    rop->referenced = ref ? 1 : 0;
+    if (ref)
+        rop->referenced = 1;
     return ref;
 }
 
+void *nfs_client_t::malloc_or_rdma(rpc_op_t *rop, size_t size)
+{
+#ifdef WITH_RDMACM
+    if (!rdma_conn)
+    {
+#endif
+        void *buf = malloc_or_die(size);
+        xdr_add_malloc(rop->xdrs, buf);
+        return buf;
+#ifdef WITH_RDMACM
+    }
+    void *buf = rdma_malloc(size);
+    xdr_set_rdma_chunk(rop->xdrs, buf);
+    return buf;
+#endif
+}
+
+void nfs_client_t::free_or_rdma(rpc_op_t *rop, void *buf)
+{
+#ifdef WITH_RDMACM
+    if (!rdma_conn)
+    {
+#endif
+        xdr_del_malloc(rop->xdrs, buf);
+        free(buf);
+#ifdef WITH_RDMACM
+    }
+    else
+    {
+        xdr_set_rdma_chunk(rop->xdrs, NULL);
+        rdma_free(buf);
+    }
+#endif
+}
+
 void nfs_proxy_t::daemonize()
 {
     // Stop all clients because client I/O sometimes breaks during daemonize
     // I.e. the new process stops receiving events on the old FD
     // It doesn't happen if we call sleep(1) here, but we don't want to call sleep(1)...
-    for (auto & clp: rpc_clients)
-        clp.second->stop();
+    for (auto & cli: rpc_clients)
+        cli->stop();
     if (fork())
         exit(0);
     setsid();

src/nfs/nfs_proxy.h

@@ -22,6 +22,7 @@ class cli_tool_t;
 struct kv_fs_state_t;
 struct block_fs_state_t;
 class nfs_client_t;
+struct nfs_rdma_context_t;
 
 class nfs_proxy_t
 {
@@ -33,7 +34,13 @@ public:
     std::string default_pool;
     std::string export_root;
     bool portmap_enabled;
-    unsigned nfs_port;
+    bool nfs_port_auto = false;
+    unsigned nfs_port = 0;
+    unsigned nfs_rdma_port = 0;
+    uint32_t nfs_rdma_credit = 16;
+    uint32_t nfs_rdma_max_send = 1024;
+    uint64_t nfs_rdma_alloc = 1048576;
+    uint64_t nfs_rdma_gc = 500*1048576;
     int trace = 0;
     std::string logfile = "/dev/null";
     std::string pidfile;
@@ -55,7 +62,8 @@ public:
     vitastorkv_dbw_t *db = NULL;
     kv_fs_state_t *kvfs = NULL;
     block_fs_state_t *blockfs = NULL;
-    std::map<int, nfs_client_t*> rpc_clients;
+    nfs_rdma_context_t* rdma_context = NULL;
+    std::set<nfs_client_t*> rpc_clients;
 
     std::vector<XDR*> xdr_pool;
 
@@ -72,12 +80,20 @@ public:
     void watch_stats();
     void parse_stats(etcd_kv_t & kv);
     void check_default_pool();
+    nfs_client_t* create_client();
     void do_accept(int listen_fd);
     void daemonize();
     void write_pid();
     void mount_fs();
     void check_already_mounted();
     void check_exit();
+
+    nfs_rdma_context_t* create_rdma(const std::string & bind_address, int rdmacm_port,
+        uint32_t max_iodepth, uint32_t max_send_wr, uint64_t rdma_malloc_round_to, uint64_t rdma_max_unused_buffers);
+    void destroy_rdma();
+
+    XDR *get_xdr();
+    void free_xdr(XDR *xdrs);
 };
 
 struct rpc_cur_buffer_t
@@ -101,19 +117,24 @@ struct rpc_free_buffer_t
     unsigned size;
 };
 
+struct nfs_rdma_conn_t;
+
 class nfs_client_t
 {
 public:
     nfs_proxy_t *parent = NULL;
-    int nfs_fd;
-    int epoll_events = 0;
     int refs = 0;
     bool stopped = false;
     std::set<rpc_service_proc_t> proc_table;
+    nfs_rdma_conn_t *rdma_conn = NULL;
+
+    // <TCP>
+    int nfs_fd = -1;
+    int epoll_events = 0;
 
     // Read state
     rpc_cur_buffer_t cur_buffer = { 0 };
-    std::map<uint8_t*, rpc_used_buffer_t> used_buffers;
+    std::map<void*, rpc_used_buffer_t> used_buffers;
     std::vector<rpc_free_buffer_t> free_buffers;
 
     iovec read_iov;
@@ -130,7 +151,16 @@ public:
     void submit_send();
     void handle_send(int result);
     int handle_rpc_message(void *base_buf, void *msg_buf, uint32_t msg_len);
+    // </TCP>
 
+    rpc_op_t *create_rpc_op(XDR *xdrs, void *buffer, rpc_msg *inmsg, rdma_msg *rmsg);
+    int handle_rpc_op(rpc_op_t *rop);
     bool deref();
     void stop();
+    void *malloc_or_rdma(rpc_op_t *rop, size_t size);
+    void free_or_rdma(rpc_op_t *rop, void *buf);
+    void *rdma_malloc(size_t size);
+    void rdma_free(void *buf);
+    void rdma_queue_reply(rpc_op_t *rop);
+    void destroy_rdma_conn();
 };

src/nfs/proto/nfs.x

@@ -168,7 +168,7 @@ struct WRITE3args {
 	offset3     offset;
 	count3      count;
 	stable_how  stable;
-	opaque      data<>;
+	opaque      data<>; /* RDMA DDP-eligible */
 };
 
 typedef opaque writeverf3[NFS3_WRITEVERFSIZE];
@@ -409,7 +409,7 @@ struct READ3resok {
 	post_op_attr   file_attributes;
 	count3         count;
 	bool           eof;
-	opaque         data<>;
+	opaque         data<>; /* RDMA DDP-eligible */
 };
 
 struct READ3resfail {
@@ -514,7 +514,7 @@ typedef string nfspath3<>;
 
 struct symlinkdata3 {
 	sattr3    symlink_attributes;
-	nfspath3  symlink_data;
+	nfspath3  symlink_data; /* RDMA DDP-eligible */
 };
 
 struct SYMLINK3args {
@@ -546,7 +546,7 @@ struct READLINK3args {
 
 struct READLINK3resok {
 	post_op_attr   symlink_attributes;
-	nfspath3       data;
+	nfspath3       data; /* RDMA DDP-eligible */
 };
 
 struct READLINK3resfail {

src/nfs/proto/nfs_xdr.cpp

@@ -272,7 +272,7 @@ xdr_WRITE3args (XDR *xdrs, WRITE3args *objp)
 		 return FALSE;
 	 if (!xdr_stable_how (xdrs, &objp->stable))
 		 return FALSE;
-	 if (!xdr_bytes(xdrs, &objp->data, ~0))
+	 if (!xdr_bytes(xdrs, &objp->data, ~0, true))
 		 return FALSE;
 	return TRUE;
 }
@@ -829,7 +829,7 @@ xdr_READ3resok (XDR *xdrs, READ3resok *objp)
 		 return FALSE;
 	 if (!xdr_bool (xdrs, &objp->eof))
 		 return FALSE;
-	 if (!xdr_bytes(xdrs, &objp->data, ~0))
+	 if (!xdr_bytes(xdrs, &objp->data, ~0, true))
 		 return FALSE;
 	return TRUE;
 }
@@ -1173,10 +1173,10 @@ xdr_PATHCONF3res (XDR *xdrs, PATHCONF3res *objp)
 }
 
 bool_t
-xdr_nfspath3 (XDR *xdrs, nfspath3 *objp)
+xdr_nfspath3 (XDR *xdrs, nfspath3 *objp, bool rdma_chunk)
 {
 	
-	 if (!xdr_string (xdrs, objp, ~0))
+	 if (!xdr_string (xdrs, objp, ~0, rdma_chunk))
 		 return FALSE;
 	return TRUE;
 }
@@ -1187,7 +1187,7 @@ xdr_symlinkdata3 (XDR *xdrs, symlinkdata3 *objp)
 	
 	 if (!xdr_sattr3 (xdrs, &objp->symlink_attributes))
 		 return FALSE;
-	 if (!xdr_nfspath3 (xdrs, &objp->symlink_data))
+	 if (!xdr_nfspath3 (xdrs, &objp->symlink_data, true))
 		 return FALSE;
 	return TRUE;
 }
@@ -1259,7 +1259,7 @@ xdr_READLINK3resok (XDR *xdrs, READLINK3resok *objp)
 	
 	 if (!xdr_post_op_attr (xdrs, &objp->symlink_attributes))
 		 return FALSE;
-	 if (!xdr_nfspath3 (xdrs, &objp->data))
+	 if (!xdr_nfspath3 (xdrs, &objp->data, true))
 		 return FALSE;
 	return TRUE;
 }

src/nfs/proto/nfs_xdr.cpp.diff

@@ -0,0 +1,53 @@
+diff --git a/src/nfs/proto/nfs_xdr.cpp b/src/nfs/proto/nfs_xdr.cpp
+index 87451293..5897e6ad 100644
+--- a/src/nfs/proto/nfs_xdr.cpp
++++ b/src/nfs/proto/nfs_xdr.cpp
+@@ -272,7 +272,7 @@ xdr_WRITE3args (XDR *xdrs, WRITE3args *objp)
+ 		 return FALSE;
+ 	 if (!xdr_stable_how (xdrs, &objp->stable))
+ 		 return FALSE;
+-	 if (!xdr_bytes(xdrs, &objp->data, ~0))
++	 if (!xdr_bytes(xdrs, &objp->data, ~0, true))
+ 		 return FALSE;
+ 	return TRUE;
+ }
+@@ -829,7 +829,7 @@ xdr_READ3resok (XDR *xdrs, READ3resok *objp)
+ 		 return FALSE;
+ 	 if (!xdr_bool (xdrs, &objp->eof))
+ 		 return FALSE;
+-	 if (!xdr_bytes(xdrs, &objp->data, ~0))
++	 if (!xdr_bytes(xdrs, &objp->data, ~0, true))
+ 		 return FALSE;
+ 	return TRUE;
+ }
+@@ -1173,10 +1173,10 @@ xdr_PATHCONF3res (XDR *xdrs, PATHCONF3res *objp)
+ }
+ 
+ bool_t
+-xdr_nfspath3 (XDR *xdrs, nfspath3 *objp)
++xdr_nfspath3 (XDR *xdrs, nfspath3 *objp, bool rdma_chunk)
+ {
+ 	
+-	 if (!xdr_string (xdrs, objp, ~0))
++	 if (!xdr_string (xdrs, objp, ~0, rdma_chunk))
+ 		 return FALSE;
+ 	return TRUE;
+ }
+@@ -1187,7 +1187,7 @@ xdr_symlinkdata3 (XDR *xdrs, symlinkdata3 *objp)
+ 	
+ 	 if (!xdr_sattr3 (xdrs, &objp->symlink_attributes))
+ 		 return FALSE;
+-	 if (!xdr_nfspath3 (xdrs, &objp->symlink_data))
++	 if (!xdr_nfspath3 (xdrs, &objp->symlink_data, true))
+ 		 return FALSE;
+ 	return TRUE;
+ }
+@@ -1259,7 +1259,7 @@ xdr_READLINK3resok (XDR *xdrs, READLINK3resok *objp)
+ 	
+ 	 if (!xdr_post_op_attr (xdrs, &objp->symlink_attributes))
+ 		 return FALSE;
+-	 if (!xdr_nfspath3 (xdrs, &objp->data))
++	 if (!xdr_nfspath3 (xdrs, &objp->data, true))
+ 		 return FALSE;
+ 	return TRUE;
+ }