Support UBLK in CSI and make it default

csi/src/nodeserver.go

@@ -33,7 +33,7 @@ import (
 type NodeServer struct
 {
     *Driver
-    useVduse        bool
+    method          MountMethod
     stateDir        string
     nfsStageDir     string
     mounter         mount.Interface
@@ -81,16 +81,23 @@ func NewNodeServer(driver *Driver) *NodeServer
     }
     ns := &NodeServer{
         Driver:      driver,
-        useVduse:    checkVduseSupport(),
+        method:      selectMountMethod(),
         stateDir:    stateDir,
         nfsStageDir: nfsStageDir,
         mounter:     mount.New(""),
         volumeLocks: make(map[string]bool),
     }
     ns.cond = sync.NewCond(&ns.mu)
-    if (ns.useVduse)
+    if (ns.method == MOUNT_VDUSE)
     {
         ns.restoreVduseDaemons()
+    }
+    else if (ns.method == MOUNT_UBLK)
+    {
+        ns.restoreUblkDaemons()
+    }
+    if (ns.method == MOUNT_VDUSE || ns.method == MOUNT_UBLK)
+    {
         dur, err := time.ParseDuration(os.Getenv("RESTART_INTERVAL"))
         if (err != nil)
         {
@@ -136,7 +143,14 @@ func (ns *NodeServer) restarter()
     for
     {
         <-ticker.C
-        ns.restoreVduseDaemons()
+        if (ns.method == MOUNT_VDUSE)
+        {
+            ns.restoreVduseDaemons()
+        }
+        else if (ns.method == MOUNT_UBLK)
+        {
+            ns.restoreUblkDaemons()
+        }
     }
 }
 
@@ -231,6 +245,78 @@ func (ns *NodeServer) checkVduseState(stateFile string, devs map[string]interfac
     }
 }
 
+func (ns *NodeServer) restoreUblkDaemons()
+{
+    pattern := ns.stateDir+"vitastor-ublk-*.json"
+    stateFiles, err := filepath.Glob(pattern)
+    if (err != nil)
+    {
+        klog.Errorf("failed to list %s: %v", pattern, err)
+    }
+    if (len(stateFiles) == 0)
+    {
+        return
+    }
+    for _, stateFile := range stateFiles
+    {
+        deviceNum := stateFile[len(ns.stateDir) + len("vitastor-ublk-") :]
+        deviceNum = deviceNum[0:len(deviceNum)-5]
+        ns.checkUblkState(deviceNum)
+    }
+}
+
+func (ns *NodeServer) checkUblkState(deviceNum string)
+{
+    // Check if the ublk daemon is still active
+
+    // Read state file
+    stateFile := ns.stateDir + "vitastor-ublk-" + deviceNum + ".json"
+    stateJSON, err := os.ReadFile(stateFile)
+    if (err != nil)
+    {
+        klog.Warningf("error reading state file %v: %v", stateFile, err)
+        return
+    }
+    var state DeviceState
+    err = json.Unmarshal(stateJSON, &state)
+    if (err != nil)
+    {
+        klog.Warningf("state file %v contains invalid JSON (error %v): %v", stateFile, err, string(stateJSON))
+        return
+    }
+
+    // Lock volume
+    ns.lockVolume(state.ConfigPath+":block:"+state.Image)
+    defer ns.unlockVolume(state.ConfigPath+":block:"+state.Image)
+
+    // Recheck state file after locking
+    _, err = os.ReadFile(stateFile)
+    if (err != nil)
+    {
+        klog.Warningf("state file %v disappeared, skipping volume", stateFile)
+        return
+    }
+
+    // Check if the vitastor-ublk process is still active
+    pidFile := ns.stateDir + "vitastor-ublk-" + deviceNum + ".pid"
+    exists := false
+    proc, err := findByPidFile(pidFile)
+    if (err == nil)
+    {
+        exists = proc.Signal(syscall.Signal(0)) == nil
+    }
+    if (!exists)
+    {
+        // Restart daemon
+        klog.Warningf("recovering UBLK device /dev/ublkb%v for volume %v", deviceNum, state.Image)
+        _, err = mapUblk(ns.stateDir, state.Image, state.ConfigPath, state.Readonly, "/dev/ublkb"+deviceNum)
+        if (err != nil)
+        {
+            klog.Warningf("failed to recover ublk device for volume %v: %v", state.Image, err)
+        }
+    }
+}
+
 func (ns *NodeServer) restoreNfsDaemons()
 {
     pattern := ns.stateDir+"vitastor-nfs-*.json"
@@ -417,14 +503,18 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
     }
 
     var devicePath, vdpaId string
-    if (!ns.useVduse)
+    if (ns.method == MOUNT_UBLK)
     {
-        devicePath, err = mapNbd(volName, ctxVars, false)
+        devicePath, err = mapUblk(ns.stateDir, volName, ctxVars["configPath"], false, "")
     }
-    else
+    else if (ns.method == MOUNT_VDUSE)
     {
         devicePath, vdpaId, err = mapVduse(ns.stateDir, volName, ctxVars, false)
     }
+    else /* if (ns.method == MOUNT_NBD) */
+    {
+        devicePath, err = mapNbd(volName, ctxVars, false)
+    }
     if (err != nil)
     {
         return nil, err
@@ -496,10 +586,6 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
                 case "xfs":
                     _, err = systemCombined("xfs_growfs", devicePath)
             }
-            if (err != nil)
-            {
-                goto unmap
-            }
         }
     }
     if (err != nil)
@@ -513,14 +599,18 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
     return &csi.NodeStageVolumeResponse{}, nil
 
 unmap:
-    if (!ns.useVduse || len(devicePath) >= 8 && devicePath[0:8] == "/dev/nbd")
+    if (ns.method == MOUNT_UBLK)
     {
-        unmapNbd(devicePath)
+        unmapUblk(ns.stateDir, devicePath)
     }
-    else
+    else if (ns.method == MOUNT_VDUSE)
     {
         unmapVduseById(ns.stateDir, vdpaId)
     }
+    else /* if (ns.method == MOUNT_NBD) */
+    {
+        unmapNbd(devicePath)
+    }
     return nil, err
 }
 
@@ -583,14 +673,18 @@ func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag
     // unmap device
     if (len(refList) == 0)
     {
-        if (!ns.useVduse)
+        if (ns.method == MOUNT_UBLK)
         {
-            unmapNbd(devicePath)
+            unmapUblk(ns.stateDir, devicePath)
         }
-        else
+        else if (ns.method == MOUNT_VDUSE)
         {
             unmapVduse(ns.stateDir, devicePath)
         }
+        else /* if (ns.method == MOUNT_NBD) */
+        {
+            unmapNbd(devicePath)
+        }
     }
 
     return &csi.NodeUnstageVolumeResponse{}, nil

csi/src/utils.go

@@ -22,6 +22,14 @@ import (
     "google.golang.org/grpc/status"
 )
 
+type MountMethod int
+
+const (
+    MOUNT_NBD   MountMethod = 0
+    MOUNT_VDUSE MountMethod = 1
+    MOUNT_UBLK  MountMethod = 2
+)
+
 func Contains(list []string, s string) bool
 {
     for i := 0; i < len(list); i++
@@ -34,29 +42,26 @@ func Contains(list []string, s string) bool
     return false
 }
 
-func checkVduseSupport() bool
+func selectMountMethod() MountMethod
 {
+    // Check UBLK support (ublk_drv kernel module)
+    if (checkModule("ublk_drv"))
+    {
+        klog.Infof("UBLK support enabled successfully")
+        return MOUNT_UBLK
+    }
+    klog.Errorf(
+        "Your host apparently has no UBLK support. UBLK support disabled."+
+        " For UBLK you need at least Linux 6.0 and the ublk_drv kernel module.",
+    )
     // Check VDUSE support (vdpa, vduse, virtio-vdpa kernel modules)
     vduse := true
     for _, mod := range []string{"vdpa", "vduse", "virtio-vdpa"}
     {
-        _, err := os.Stat("/sys/module/"+mod)
+        if (!checkModule(mod))
-        if (err != nil)
         {
-            if (!errors.Is(err, os.ErrNotExist))
+            vduse = false
-            {
+            break
-                klog.Errorf("failed to check /sys/module/%s: %v", mod, err)
-            }
-            c := exec.Command("/sbin/modprobe", mod)
-            c.Stdout = os.Stderr
-            c.Stderr = os.Stderr
-            err := c.Run()
-            if (err != nil)
-            {
-                klog.Errorf("/sbin/modprobe %s failed: %v", mod, err)
-                vduse = false
-                break
-            }
         }
     }
     // Check that vdpa tool functions
@@ -71,18 +76,38 @@ func checkVduseSupport() bool
             vduse = false
         }
     }
-    if (!vduse)
+    if (vduse)
-    {
-        klog.Errorf(
-            "Your host apparently has no VDUSE support. VDUSE support disabled, NBD will be used to map devices."+
-            " For VDUSE you need at least Linux 5.15 and the following kernel modules: vdpa, virtio-vdpa, vduse.",
-        )
-    }
-    else
     {
         klog.Infof("VDUSE support enabled successfully")
+        return MOUNT_VDUSE
     }
-    return vduse
+    klog.Errorf(
+        "Your host apparently has no VDUSE support. VDUSE support disabled, NBD will be used to map devices."+
+        " For VDUSE you need at least Linux 5.15 and the following kernel modules: vdpa, virtio-vdpa, vduse.",
+    )
+    return MOUNT_NBD
+}
+
+func checkModule(mod string) bool
+{
+    _, err := os.Stat("/sys/module/"+mod)
+    if (err != nil)
+    {
+        if (!errors.Is(err, os.ErrNotExist))
+        {
+            klog.Errorf("failed to check /sys/module/%s: %v", mod, err)
+        }
+        c := exec.Command("/sbin/modprobe", mod)
+        c.Stdout = os.Stderr
+        c.Stderr = os.Stderr
+        err := c.Run()
+        if (err != nil)
+        {
+            klog.Errorf("/sbin/modprobe %s failed: %v", mod, err)
+            return false
+        }
+    }
+    return true
 }
 
 func mapNbd(volName string, ctxVars map[string]string, readonly bool) (string, error)
@@ -219,6 +244,7 @@ func mapVduse(stateDir string, volName string, ctxVars map[string]string, readon
                     stateJSON, _ := json.Marshal(&DeviceState{
                         ConfigPath: ctxVars["configPath"],
                         VdpaId:     vdpaId,
+
                         Image:      volName,
                         Blockdev:   blockdev,
                         Readonly:   readonly,
@@ -311,6 +337,117 @@ func unmapVduseById(stateDir, vdpaId string)
     }
 }
 
+func mapUblk(stateDir string, volName string, configPath string, readonly bool, recoverDev string) (string, error)
+{
+    pidFile := ""
+    if (recoverDev != "")
+    {
+        if (len(recoverDev) < 10 || recoverDev[0:10] != "/dev/ublkb")
+        {
+            return "", fmt.Errorf("recover: %s does not start with /dev/ublkb", recoverDev)
+        }
+        pidFile = stateDir + "vitastor-ublk-" + recoverDev[10:] + ".pid"
+    }
+    else
+    {
+        pidFd, err := os.CreateTemp(stateDir, "vitastor-tmp-*.pid")
+        if (err != nil)
+        {
+            return "", err
+        }
+        pidFile = pidFd.Name()
+        pidFd.Close()
+    }
+    // Map device via vitastor-ublk
+    args := []string{
+        "map", "--image", volName, "--pidfile", pidFile,
+    }
+    if (configPath != "")
+    {
+        args = append(args, "--config_path", configPath)
+    }
+    if (readonly)
+    {
+        args = append(args, "--readonly")
+    }
+    if (recoverDev != "")
+    {
+        args = append(args, "--recover", recoverDev)
+    }
+    stdout, stderr, err := system("/usr/bin/vitastor-ublk", args...)
+    if (err != nil)
+    {
+        return "", err
+    }
+    devicePath := strings.TrimSpace(string(stdout))
+    if (devicePath == "")
+    {
+        return "", fmt.Errorf("vitastor-ublk did not return the name of the device. output: %s", stderr)
+    }
+    if (len(devicePath) >= 10 && devicePath[0:10] == "/dev/ublkb")
+    {
+        // Generate state file
+        devNum := devicePath[10:]
+        pidNew := stateDir + "vitastor-ublk-" + devNum + ".pid"
+        if (pidFile != pidNew)
+        {
+            err := os.Rename(pidFile, pidNew)
+            if (err != nil)
+            {
+                klog.Errorf("Failed to rename PID file %s to %s: %v", pidFile, pidNew, err)
+            }
+            else
+            {
+                pidFile = pidNew
+            }
+        }
+        stateFile := stateDir + "vitastor-ublk-" + devNum + ".json"
+        stateJSON, _ := json.Marshal(&DeviceState{
+            ConfigPath: configPath,
+            Image:      volName,
+            Readonly:   readonly,
+            PidFile:    pidFile,
+        })
+        err = os.WriteFile(stateFile, stateJSON, 0600)
+        if (err == nil)
+        {
+            klog.Infof("Attached volume %s via UBLK as %s", volName, devicePath)
+            return devicePath, nil
+        }
+        os.Remove(stateFile)
+    }
+    killErr := killByPidFile(pidFile)
+    if (killErr != nil)
+    {
+        klog.Errorf("Failed to kill started vitastor-ublk: %v", killErr)
+    }
+    os.Remove(pidFile)
+    return "", err
+}
+
+func unmapUblk(stateDir, devicePath string)
+{
+    if (len(devicePath) < 10 || devicePath[0:10] != "/dev/ublkb")
+    {
+        klog.Errorf("%s does not start with /dev/ublkb", devicePath)
+        return
+    }
+    unmapOut, unmapErr := exec.Command("/usr/bin/vitastor-ublk", "unmap", devicePath).CombinedOutput()
+    if (unmapErr != nil)
+    {
+        klog.Errorf("failed to unmap UBLK device %s: %s, error: %v", devicePath, unmapOut, unmapErr)
+    }
+    for _, ext := range []string{"json", "pid"}
+    {
+        fn := stateDir + "vitastor-ublk-" + devicePath[10:] + "." + ext
+        err := os.Remove(fn)
+        if (err != nil)
+        {
+            klog.Errorf("failed to remove %s: %v", fn, err)
+        }
+    }
+}
+
 func system(program string, args ...string) ([]byte, []byte, error)
 {
     klog.Infof("Running "+program+" "+strings.Join(args, " "))

src/client/nbd_proxy.cpp

@@ -272,6 +272,8 @@ const char *help_text =
     "  --dev_num N\n"
     "    Use the specified device /dev/nbdN instead of automatic selection (alternative syntax\n"
     "    to /dev/nbdN positional parameter).\n"
+    "  --readonly\n"
+    "    Make the device read-only.\n"
     "  --foreground 1\n"
     "    Stay in foreground, do not daemonize.\n"
     "\n"
@@ -372,7 +374,7 @@ public:
             else if (args[i][0] == '-' && args[i][1] == '-')
             {
                 const char *opt = args[i]+2;
-                cfg[opt] = !strcmp(opt, "json") || !strcmp(opt, "all") ||
+                cfg[opt] = !strcmp(opt, "json") || !strcmp(opt, "all") || !strcmp(opt, "readonly") ||
                     !strcmp(opt, "force") || i == narg-1 ? "1" : args[++i];
             }
             else if (pos == 0)

src/client/ublk_server.cpp

@@ -40,12 +40,14 @@ const char *help_text =
     "    Make the device read-only.\n"
     "  --hdd\n"
     "    Mark the device as rotational.\n"
-    "  --logfile /path/to/log/file.txt\n"
-    "    Write log messages to the specified file instead of dropping them (in background mode)\n"
-    "    or printing them to the standard output (in foreground mode).\n"
     "  --dev_num N\n"
     "    Use the specified device /dev/ublkbN instead of automatic selection (alternative syntax\n"
     "    to /dev/ublkbN positional parameter).\n"
+    "  --pidfile /run/ublk_pid_file.pid\n"
+    "    Write process ID to the specified file.\n"
+    "  --logfile /path/to/log/file.txt\n"
+    "    Write log messages to the specified file instead of dropping them (in background mode)\n"
+    "    or printing them to the standard output (in foreground mode).\n"
     "  --foreground 1\n"
     "    Stay in foreground, do not daemonize.\n"
     "\n"
@@ -79,6 +81,7 @@ protected:
     inode_watch_t *watch = NULL;
 
     std::string logfile = "/dev/null";
+    std::string pidfile;
 
 public:
     ublk_server()
@@ -300,10 +303,8 @@ help:
         load_module();
 
         bool bg = cfg["foreground"].is_null();
-        if (cfg["logfile"].string_value() != "")
+        logfile = cfg["logfile"].string_value();
-        {
+        pidfile = cfg["pidfile"].string_value();
-            logfile = cfg["logfile"].string_value();
-        }
 
         open_control();
         if (recover)
@@ -331,6 +332,8 @@ help:
             close(notifyfd[0]);
         }
         start_device(recover);
+        if (pidfile != "")
+            write_pid();
         if (bg)
         {
             daemonize_reopen_stdio();
@@ -399,6 +402,22 @@ help:
             fprintf(stderr, "Warning: Failed to chdir into /\n");
     }
 
+    void write_pid()
+    {
+        int fd = open(pidfile.c_str(), O_WRONLY|O_CREAT|O_TRUNC, 0666);
+        if (fd < 0)
+        {
+            fprintf(stderr, "Failed to create pid file %s: %s (code %d)\n", pidfile.c_str(), strerror(errno), errno);
+            return;
+        }
+        auto pid = std::to_string(getpid());
+        if (write(fd, pid.c_str(), pid.size()) < 0)
+        {
+            fprintf(stderr, "Failed to write pid to %s: %s (code %d)\n", pidfile.c_str(), strerror(errno), errno);
+        }
+        close(fd);
+    }
+
     json11::Json::object list_mapped()
     {
         int n_in_dev = 0;