diff --git a/csi/src/nodeserver.go b/csi/src/nodeserver.go index ce764d71..3d72702f 100644 --- a/csi/src/nodeserver.go +++ b/csi/src/nodeserver.go @@ -33,7 +33,7 @@ import ( type NodeServer struct { *Driver - useVduse bool + method MountMethod stateDir string nfsStageDir string mounter mount.Interface @@ -81,16 +81,23 @@ func NewNodeServer(driver *Driver) *NodeServer } ns := &NodeServer{ Driver: driver, - useVduse: checkVduseSupport(), + method: selectMountMethod(), stateDir: stateDir, nfsStageDir: nfsStageDir, mounter: mount.New(""), volumeLocks: make(map[string]bool), } ns.cond = sync.NewCond(&ns.mu) - if (ns.useVduse) + if (ns.method == MOUNT_VDUSE) { ns.restoreVduseDaemons() + } + else if (ns.method == MOUNT_UBLK) + { + ns.restoreUblkDaemons() + } + if (ns.method == MOUNT_VDUSE || ns.method == MOUNT_UBLK) + { dur, err := time.ParseDuration(os.Getenv("RESTART_INTERVAL")) if (err != nil) { @@ -136,7 +143,14 @@ func (ns *NodeServer) restarter() for { <-ticker.C - ns.restoreVduseDaemons() + if (ns.method == MOUNT_VDUSE) + { + ns.restoreVduseDaemons() + } + else if (ns.method == MOUNT_UBLK) + { + ns.restoreUblkDaemons() + } } } @@ -231,6 +245,78 @@ func (ns *NodeServer) checkVduseState(stateFile string, devs map[string]interfac } } +func (ns *NodeServer) restoreUblkDaemons() +{ + pattern := ns.stateDir+"vitastor-ublk-*.json" + stateFiles, err := filepath.Glob(pattern) + if (err != nil) + { + klog.Errorf("failed to list %s: %v", pattern, err) + } + if (len(stateFiles) == 0) + { + return + } + for _, stateFile := range stateFiles + { + deviceNum := stateFile[len(ns.stateDir) + len("vitastor-ublk-") :] + deviceNum = deviceNum[0:len(deviceNum)-5] + ns.checkUblkState(deviceNum) + } +} + +func (ns *NodeServer) checkUblkState(deviceNum string) +{ + // Check if the ublk daemon is still active + + // Read state file + stateFile := ns.stateDir + "vitastor-ublk-" + deviceNum + ".json" + stateJSON, err := os.ReadFile(stateFile) + if (err != nil) + { + klog.Warningf("error reading state file %v: %v", stateFile, err) + return + } + var state DeviceState + err = json.Unmarshal(stateJSON, &state) + if (err != nil) + { + klog.Warningf("state file %v contains invalid JSON (error %v): %v", stateFile, err, string(stateJSON)) + return + } + + // Lock volume + ns.lockVolume(state.ConfigPath+":block:"+state.Image) + defer ns.unlockVolume(state.ConfigPath+":block:"+state.Image) + + // Recheck state file after locking + _, err = os.ReadFile(stateFile) + if (err != nil) + { + klog.Warningf("state file %v disappeared, skipping volume", stateFile) + return + } + + // Check if the vitastor-ublk process is still active + pidFile := ns.stateDir + "vitastor-ublk-" + deviceNum + ".pid" + exists := false + proc, err := findByPidFile(pidFile) + if (err == nil) + { + exists = proc.Signal(syscall.Signal(0)) == nil + } + if (!exists) + { + // Restart daemon + klog.Warningf("recovering UBLK device /dev/ublkb%v for volume %v", deviceNum, state.Image) + _, err = mapUblk(ns.stateDir, state.Image, state.ConfigPath, state.Readonly, "/dev/ublkb"+deviceNum) + if (err != nil) + { + klog.Warningf("failed to recover ublk device for volume %v: %v", state.Image, err) + } + } +} + func (ns *NodeServer) restoreNfsDaemons() { pattern := ns.stateDir+"vitastor-nfs-*.json" @@ -417,14 +503,18 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol } var devicePath, vdpaId string - if (!ns.useVduse) + if (ns.method == MOUNT_UBLK) { - devicePath, err = mapNbd(volName, ctxVars, false) + devicePath, err = mapUblk(ns.stateDir, volName, ctxVars["configPath"], false, "") } - else + else if (ns.method == MOUNT_VDUSE) { devicePath, vdpaId, err = mapVduse(ns.stateDir, volName, ctxVars, false) } + else /* if (ns.method == MOUNT_NBD) */ + { + devicePath, err = mapNbd(volName, ctxVars, false) + } if (err != nil) { return nil, err @@ -496,10 +586,6 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol case "xfs": _, err = systemCombined("xfs_growfs", devicePath) } - if (err != nil) - { - goto unmap - } } } if (err != nil) @@ -513,14 +599,18 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol return &csi.NodeStageVolumeResponse{}, nil unmap: - if (!ns.useVduse || len(devicePath) >= 8 && devicePath[0:8] == "/dev/nbd") + if (ns.method == MOUNT_UBLK) { - unmapNbd(devicePath) + unmapUblk(ns.stateDir, devicePath) } - else + else if (ns.method == MOUNT_VDUSE) { unmapVduseById(ns.stateDir, vdpaId) } + else /* if (ns.method == MOUNT_NBD) */ + { + unmapNbd(devicePath) + } return nil, err } @@ -583,14 +673,18 @@ func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag // unmap device if (len(refList) == 0) { - if (!ns.useVduse) + if (ns.method == MOUNT_UBLK) { - unmapNbd(devicePath) + unmapUblk(ns.stateDir, devicePath) } - else + else if (ns.method == MOUNT_VDUSE) { unmapVduse(ns.stateDir, devicePath) } + else /* if (ns.method == MOUNT_NBD) */ + { + unmapNbd(devicePath) + } } return &csi.NodeUnstageVolumeResponse{}, nil diff --git a/csi/src/utils.go b/csi/src/utils.go index 147b1b5f..37ba4936 100644 --- a/csi/src/utils.go +++ b/csi/src/utils.go @@ -22,6 +22,14 @@ import ( "google.golang.org/grpc/status" ) +type MountMethod int + +const ( + MOUNT_NBD MountMethod = 0 + MOUNT_VDUSE MountMethod = 1 + MOUNT_UBLK MountMethod = 2 +) + func Contains(list []string, s string) bool { for i := 0; i < len(list); i++ @@ -34,29 +42,26 @@ func Contains(list []string, s string) bool return false } -func checkVduseSupport() bool +func selectMountMethod() MountMethod { + // Check UBLK support (ublk_drv kernel module) + if (checkModule("ublk_drv")) + { + klog.Infof("UBLK support enabled successfully") + return MOUNT_UBLK + } + klog.Errorf( + "Your host apparently has no UBLK support. UBLK support disabled."+ + " For UBLK you need at least Linux 6.0 and the ublk_drv kernel module.", + ) // Check VDUSE support (vdpa, vduse, virtio-vdpa kernel modules) vduse := true for _, mod := range []string{"vdpa", "vduse", "virtio-vdpa"} { - _, err := os.Stat("/sys/module/"+mod) - if (err != nil) + if (!checkModule(mod)) { - if (!errors.Is(err, os.ErrNotExist)) - { - klog.Errorf("failed to check /sys/module/%s: %v", mod, err) - } - c := exec.Command("/sbin/modprobe", mod) - c.Stdout = os.Stderr - c.Stderr = os.Stderr - err := c.Run() - if (err != nil) - { - klog.Errorf("/sbin/modprobe %s failed: %v", mod, err) - vduse = false - break - } + vduse = false + break } } // Check that vdpa tool functions @@ -71,18 +76,38 @@ func checkVduseSupport() bool vduse = false } } - if (!vduse) - { - klog.Errorf( - "Your host apparently has no VDUSE support. VDUSE support disabled, NBD will be used to map devices."+ - " For VDUSE you need at least Linux 5.15 and the following kernel modules: vdpa, virtio-vdpa, vduse.", - ) - } - else + if (vduse) { klog.Infof("VDUSE support enabled successfully") + return MOUNT_VDUSE } - return vduse + klog.Errorf( + "Your host apparently has no VDUSE support. VDUSE support disabled, NBD will be used to map devices."+ + " For VDUSE you need at least Linux 5.15 and the following kernel modules: vdpa, virtio-vdpa, vduse.", + ) + return MOUNT_NBD +} + +func checkModule(mod string) bool +{ + _, err := os.Stat("/sys/module/"+mod) + if (err != nil) + { + if (!errors.Is(err, os.ErrNotExist)) + { + klog.Errorf("failed to check /sys/module/%s: %v", mod, err) + } + c := exec.Command("/sbin/modprobe", mod) + c.Stdout = os.Stderr + c.Stderr = os.Stderr + err := c.Run() + if (err != nil) + { + klog.Errorf("/sbin/modprobe %s failed: %v", mod, err) + return false + } + } + return true } func mapNbd(volName string, ctxVars map[string]string, readonly bool) (string, error) @@ -219,6 +244,7 @@ func mapVduse(stateDir string, volName string, ctxVars map[string]string, readon stateJSON, _ := json.Marshal(&DeviceState{ ConfigPath: ctxVars["configPath"], VdpaId: vdpaId, + Image: volName, Blockdev: blockdev, Readonly: readonly, @@ -311,6 +337,117 @@ func unmapVduseById(stateDir, vdpaId string) } } +func mapUblk(stateDir string, volName string, configPath string, readonly bool, recoverDev string) (string, error) +{ + pidFile := "" + if (recoverDev != "") + { + if (len(recoverDev) < 10 || recoverDev[0:10] != "/dev/ublkb") + { + return "", fmt.Errorf("recover: %s does not start with /dev/ublkb", recoverDev) + } + pidFile = stateDir + "vitastor-ublk-" + recoverDev[10:] + ".pid" + } + else + { + pidFd, err := os.CreateTemp(stateDir, "vitastor-tmp-*.pid") + if (err != nil) + { + return "", err + } + pidFile = pidFd.Name() + pidFd.Close() + } + // Map device via vitastor-ublk + args := []string{ + "map", "--image", volName, "--pidfile", pidFile, + } + if (configPath != "") + { + args = append(args, "--config_path", configPath) + } + if (readonly) + { + args = append(args, "--readonly") + } + if (recoverDev != "") + { + args = append(args, "--recover", recoverDev) + } + stdout, stderr, err := system("/usr/bin/vitastor-ublk", args...) + if (err != nil) + { + return "", err + } + devicePath := strings.TrimSpace(string(stdout)) + if (devicePath == "") + { + return "", fmt.Errorf("vitastor-ublk did not return the name of the device. output: %s", stderr) + } + if (len(devicePath) >= 10 && devicePath[0:10] == "/dev/ublkb") + { + // Generate state file + devNum := devicePath[10:] + pidNew := stateDir + "vitastor-ublk-" + devNum + ".pid" + if (pidFile != pidNew) + { + err := os.Rename(pidFile, pidNew) + if (err != nil) + { + klog.Errorf("Failed to rename PID file %s to %s: %v", pidFile, pidNew, err) + } + else + { + pidFile = pidNew + } + } + stateFile := stateDir + "vitastor-ublk-" + devNum + ".json" + stateJSON, _ := json.Marshal(&DeviceState{ + ConfigPath: configPath, + Image: volName, + Readonly: readonly, + PidFile: pidFile, + }) + err = os.WriteFile(stateFile, stateJSON, 0600) + if (err == nil) + { + klog.Infof("Attached volume %s via UBLK as %s", volName, devicePath) + return devicePath, nil + } + os.Remove(stateFile) + } + killErr := killByPidFile(pidFile) + if (killErr != nil) + { + klog.Errorf("Failed to kill started vitastor-ublk: %v", killErr) + } + os.Remove(pidFile) + return "", err +} + +func unmapUblk(stateDir, devicePath string) +{ + if (len(devicePath) < 10 || devicePath[0:10] != "/dev/ublkb") + { + klog.Errorf("%s does not start with /dev/ublkb", devicePath) + return + } + unmapOut, unmapErr := exec.Command("/usr/bin/vitastor-ublk", "unmap", devicePath).CombinedOutput() + if (unmapErr != nil) + { + klog.Errorf("failed to unmap UBLK device %s: %s, error: %v", devicePath, unmapOut, unmapErr) + } + for _, ext := range []string{"json", "pid"} + { + fn := stateDir + "vitastor-ublk-" + devicePath[10:] + "." + ext + err := os.Remove(fn) + if (err != nil) + { + klog.Errorf("failed to remove %s: %v", fn, err) + } + } +} + func system(program string, args ...string) ([]byte, []byte, error) { klog.Infof("Running "+program+" "+strings.Join(args, " ")) diff --git a/src/client/nbd_proxy.cpp b/src/client/nbd_proxy.cpp index 2f090c5f..c905c955 100644 --- a/src/client/nbd_proxy.cpp +++ b/src/client/nbd_proxy.cpp @@ -272,6 +272,8 @@ const char *help_text = " --dev_num N\n" " Use the specified device /dev/nbdN instead of automatic selection (alternative syntax\n" " to /dev/nbdN positional parameter).\n" + " --readonly\n" + " Make the device read-only.\n" " --foreground 1\n" " Stay in foreground, do not daemonize.\n" "\n" @@ -372,7 +374,7 @@ public: else if (args[i][0] == '-' && args[i][1] == '-') { const char *opt = args[i]+2; - cfg[opt] = !strcmp(opt, "json") || !strcmp(opt, "all") || + cfg[opt] = !strcmp(opt, "json") || !strcmp(opt, "all") || !strcmp(opt, "readonly") || !strcmp(opt, "force") || i == narg-1 ? "1" : args[++i]; } else if (pos == 0) diff --git a/src/client/ublk_server.cpp b/src/client/ublk_server.cpp index c3779865..9bbf9a6c 100644 --- a/src/client/ublk_server.cpp +++ b/src/client/ublk_server.cpp @@ -40,12 +40,14 @@ const char *help_text = " Make the device read-only.\n" " --hdd\n" " Mark the device as rotational.\n" - " --logfile /path/to/log/file.txt\n" - " Write log messages to the specified file instead of dropping them (in background mode)\n" - " or printing them to the standard output (in foreground mode).\n" " --dev_num N\n" " Use the specified device /dev/ublkbN instead of automatic selection (alternative syntax\n" " to /dev/ublkbN positional parameter).\n" + " --pidfile /run/ublk_pid_file.pid\n" + " Write process ID to the specified file.\n" + " --logfile /path/to/log/file.txt\n" + " Write log messages to the specified file instead of dropping them (in background mode)\n" + " or printing them to the standard output (in foreground mode).\n" " --foreground 1\n" " Stay in foreground, do not daemonize.\n" "\n" @@ -79,6 +81,7 @@ protected: inode_watch_t *watch = NULL; std::string logfile = "/dev/null"; + std::string pidfile; public: ublk_server() @@ -300,10 +303,8 @@ help: load_module(); bool bg = cfg["foreground"].is_null(); - if (cfg["logfile"].string_value() != "") - { - logfile = cfg["logfile"].string_value(); - } + logfile = cfg["logfile"].string_value(); + pidfile = cfg["pidfile"].string_value(); open_control(); if (recover) @@ -331,6 +332,8 @@ help: close(notifyfd[0]); } start_device(recover); + if (pidfile != "") + write_pid(); if (bg) { daemonize_reopen_stdio(); @@ -399,6 +402,22 @@ help: fprintf(stderr, "Warning: Failed to chdir into /\n"); } + void write_pid() + { + int fd = open(pidfile.c_str(), O_WRONLY|O_CREAT|O_TRUNC, 0666); + if (fd < 0) + { + fprintf(stderr, "Failed to create pid file %s: %s (code %d)\n", pidfile.c_str(), strerror(errno), errno); + return; + } + auto pid = std::to_string(getpid()); + if (write(fd, pid.c_str(), pid.size()) < 0) + { + fprintf(stderr, "Failed to write pid to %s: %s (code %d)\n", pidfile.c_str(), strerror(errno), errno); + } + close(fd); + } + json11::Json::object list_mapped() { int n_in_dev = 0;