thebestvibedev/tromcho.net
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support UBLK in CSI and make it default

Browse Source
This commit is contained in:
Vitaliy Filippov
2025-09-24 02:09:39 +03:00
parent e1715c33bb
commit 2c6bddd831
4 changed files with 303 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
csi/src/nodeserver.go
+111 -17
View File
@@ -33,7 +33,7 @@ import (
type NodeServer struct
{
*Driver
useVduse bool
method MountMethod
stateDir string
nfsStageDir string
mounter mount.Interface
@@ -81,16 +81,23 @@ func NewNodeServer(driver *Driver) *NodeServer
}
ns := &NodeServer{
Driver: driver,
useVduse: checkVduseSupport(),
method: selectMountMethod(),
stateDir: stateDir,
nfsStageDir: nfsStageDir,
mounter: mount.New(""),
volumeLocks: make(map[string]bool),
}
ns.cond = sync.NewCond(&ns.mu)
if (ns.useVduse)
if (ns.method == MOUNT_VDUSE)
{
ns.restoreVduseDaemons()
}
else if (ns.method == MOUNT_UBLK)
{
ns.restoreUblkDaemons()
}
if (ns.method == MOUNT_VDUSE || ns.method == MOUNT_UBLK)
{
dur, err := time.ParseDuration(os.Getenv("RESTART_INTERVAL"))
if (err != nil)
{
@@ -136,7 +143,14 @@ func (ns *NodeServer) restarter()
for
{
<-ticker.C
ns.restoreVduseDaemons()
if (ns.method == MOUNT_VDUSE)
{
ns.restoreVduseDaemons()
}
else if (ns.method == MOUNT_UBLK)
{
ns.restoreUblkDaemons()
}
}
}
@@ -231,6 +245,78 @@ func (ns *NodeServer) checkVduseState(stateFile string, devs map[string]interfac
}
}
func (ns *NodeServer) restoreUblkDaemons()
{
pattern := ns.stateDir+"vitastor-ublk-*.json"
stateFiles, err := filepath.Glob(pattern)
if (err != nil)
{
klog.Errorf("failed to list %s: %v", pattern, err)
}
if (len(stateFiles) == 0)
{
return
}
for _, stateFile := range stateFiles
{
deviceNum := stateFile[len(ns.stateDir) + len("vitastor-ublk-") :]
deviceNum = deviceNum[0:len(deviceNum)-5]
ns.checkUblkState(deviceNum)
}
}
func (ns *NodeServer) checkUblkState(deviceNum string)
{
// Check if the ublk daemon is still active
// Read state file
stateFile := ns.stateDir + "vitastor-ublk-" + deviceNum + ".json"
stateJSON, err := os.ReadFile(stateFile)
if (err != nil)
{
klog.Warningf("error reading state file %v: %v", stateFile, err)
return
}
var state DeviceState
err = json.Unmarshal(stateJSON, &state)
if (err != nil)
{
klog.Warningf("state file %v contains invalid JSON (error %v): %v", stateFile, err, string(stateJSON))
return
}
// Lock volume
ns.lockVolume(state.ConfigPath+":block:"+state.Image)
defer ns.unlockVolume(state.ConfigPath+":block:"+state.Image)
// Recheck state file after locking
_, err = os.ReadFile(stateFile)
if (err != nil)
{
klog.Warningf("state file %v disappeared, skipping volume", stateFile)
return
}
// Check if the vitastor-ublk process is still active
pidFile := ns.stateDir + "vitastor-ublk-" + deviceNum + ".pid"
exists := false
proc, err := findByPidFile(pidFile)
if (err == nil)
{
exists = proc.Signal(syscall.Signal(0)) == nil
}
if (!exists)
{
// Restart daemon
klog.Warningf("recovering UBLK device /dev/ublkb%v for volume %v", deviceNum, state.Image)
_, err = mapUblk(ns.stateDir, state.Image, state.ConfigPath, state.Readonly, "/dev/ublkb"+deviceNum)
if (err != nil)
{
klog.Warningf("failed to recover ublk device for volume %v: %v", state.Image, err)
}
}
}
func (ns *NodeServer) restoreNfsDaemons()
{
pattern := ns.stateDir+"vitastor-nfs-*.json"
@@ -417,14 +503,18 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
}
var devicePath, vdpaId string
if (!ns.useVduse)
if (ns.method == MOUNT_UBLK)
{
devicePath, err = mapNbd(volName, ctxVars, false)
devicePath, err = mapUblk(ns.stateDir, volName, ctxVars["configPath"], false, "")
}
else
else if (ns.method == MOUNT_VDUSE)
{
devicePath, vdpaId, err = mapVduse(ns.stateDir, volName, ctxVars, false)
}
else /* if (ns.method == MOUNT_NBD) */
{
devicePath, err = mapNbd(volName, ctxVars, false)
}
if (err != nil)
{
return nil, err
@@ -496,10 +586,6 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
case "xfs":
_, err = systemCombined("xfs_growfs", devicePath)
}
if (err != nil)
{
goto unmap
}
}
}
if (err != nil)
@@ -513,14 +599,18 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
return &csi.NodeStageVolumeResponse{}, nil
unmap:
if (!ns.useVduse || len(devicePath) >= 8 && devicePath[0:8] == "/dev/nbd")
if (ns.method == MOUNT_UBLK)
{
unmapNbd(devicePath)
unmapUblk(ns.stateDir, devicePath)
}
else
else if (ns.method == MOUNT_VDUSE)
{
unmapVduseById(ns.stateDir, vdpaId)
}
else /* if (ns.method == MOUNT_NBD) */
{
unmapNbd(devicePath)
}
return nil, err
}
@@ -583,14 +673,18 @@ func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag
// unmap device
if (len(refList) == 0)
{
if (!ns.useVduse)
if (ns.method == MOUNT_UBLK)
{
unmapNbd(devicePath)
unmapUblk(ns.stateDir, devicePath)
}
else
else if (ns.method == MOUNT_VDUSE)
{
unmapVduse(ns.stateDir, devicePath)
}
else /* if (ns.method == MOUNT_NBD) */
{
unmapNbd(devicePath)
}
}
return &csi.NodeUnstageVolumeResponse{}, nil
csi/src/utils.go
+163 -26
View File
@@ -22,6 +22,14 @@ import (
"google.golang.org/grpc/status"
)
type MountMethod int
const (
MOUNT_NBD MountMethod = 0
MOUNT_VDUSE MountMethod = 1
MOUNT_UBLK MountMethod = 2
)
func Contains(list []string, s string) bool
{
for i := 0; i < len(list); i++
@@ -34,29 +42,26 @@ func Contains(list []string, s string) bool
return false
}
func checkVduseSupport() bool
func selectMountMethod() MountMethod
{
// Check UBLK support (ublk_drv kernel module)
if (checkModule("ublk_drv"))
{
klog.Infof("UBLK support enabled successfully")
return MOUNT_UBLK
}
klog.Errorf(
"Your host apparently has no UBLK support. UBLK support disabled."+
" For UBLK you need at least Linux 6.0 and the ublk_drv kernel module.",
)
// Check VDUSE support (vdpa, vduse, virtio-vdpa kernel modules)
vduse := true
for _, mod := range []string{"vdpa", "vduse", "virtio-vdpa"}
{
_, err := os.Stat("/sys/module/"+mod)
if (err != nil)
if (!checkModule(mod))
{
if (!errors.Is(err, os.ErrNotExist))
{
klog.Errorf("failed to check /sys/module/%s: %v", mod, err)
}
c := exec.Command("/sbin/modprobe", mod)
c.Stdout = os.Stderr
c.Stderr = os.Stderr
err := c.Run()
if (err != nil)
{
klog.Errorf("/sbin/modprobe %s failed: %v", mod, err)
vduse = false
break
}
vduse = false
break
}
}
// Check that vdpa tool functions
@@ -71,18 +76,38 @@ func checkVduseSupport() bool
vduse = false
}
}
if (!vduse)
{
klog.Errorf(
"Your host apparently has no VDUSE support. VDUSE support disabled, NBD will be used to map devices."+
" For VDUSE you need at least Linux 5.15 and the following kernel modules: vdpa, virtio-vdpa, vduse.",
)
}
else
if (vduse)
{
klog.Infof("VDUSE support enabled successfully")
return MOUNT_VDUSE
}
return vduse
klog.Errorf(
"Your host apparently has no VDUSE support. VDUSE support disabled, NBD will be used to map devices."+
" For VDUSE you need at least Linux 5.15 and the following kernel modules: vdpa, virtio-vdpa, vduse.",
)
return MOUNT_NBD
}
func checkModule(mod string) bool
{
_, err := os.Stat("/sys/module/"+mod)
if (err != nil)
{
if (!errors.Is(err, os.ErrNotExist))
{
klog.Errorf("failed to check /sys/module/%s: %v", mod, err)
}
c := exec.Command("/sbin/modprobe", mod)
c.Stdout = os.Stderr
c.Stderr = os.Stderr
err := c.Run()
if (err != nil)
{
klog.Errorf("/sbin/modprobe %s failed: %v", mod, err)
return false
}
}
return true
}
func mapNbd(volName string, ctxVars map[string]string, readonly bool) (string, error)
@@ -219,6 +244,7 @@ func mapVduse(stateDir string, volName string, ctxVars map[string]string, readon
stateJSON, _ := json.Marshal(&DeviceState{
ConfigPath: ctxVars["configPath"],
VdpaId: vdpaId,
Image: volName,
Blockdev: blockdev,
Readonly: readonly,
@@ -311,6 +337,117 @@ func unmapVduseById(stateDir, vdpaId string)
}
}
func mapUblk(stateDir string, volName string, configPath string, readonly bool, recoverDev string) (string, error)
{
pidFile := ""
if (recoverDev != "")
{
if (len(recoverDev) < 10 || recoverDev[0:10] != "/dev/ublkb")
{
return "", fmt.Errorf("recover: %s does not start with /dev/ublkb", recoverDev)
}
pidFile = stateDir + "vitastor-ublk-" + recoverDev[10:] + ".pid"
}
else
{
pidFd, err := os.CreateTemp(stateDir, "vitastor-tmp-*.pid")
if (err != nil)
{
return "", err
}
pidFile = pidFd.Name()
pidFd.Close()
}
// Map device via vitastor-ublk
args := []string{
"map", "--image", volName, "--pidfile", pidFile,
}
if (configPath != "")
{
args = append(args, "--config_path", configPath)
}
if (readonly)
{
args = append(args, "--readonly")
}
if (recoverDev != "")
{
args = append(args, "--recover", recoverDev)
}
stdout, stderr, err := system("/usr/bin/vitastor-ublk", args...)
if (err != nil)
{
return "", err
}
devicePath := strings.TrimSpace(string(stdout))
if (devicePath == "")
{
return "", fmt.Errorf("vitastor-ublk did not return the name of the device. output: %s", stderr)
}
if (len(devicePath) >= 10 && devicePath[0:10] == "/dev/ublkb")
{
// Generate state file
devNum := devicePath[10:]
pidNew := stateDir + "vitastor-ublk-" + devNum + ".pid"
if (pidFile != pidNew)
{
err := os.Rename(pidFile, pidNew)
if (err != nil)
{
klog.Errorf("Failed to rename PID file %s to %s: %v", pidFile, pidNew, err)
}
else
{
pidFile = pidNew
}
}
stateFile := stateDir + "vitastor-ublk-" + devNum + ".json"
stateJSON, _ := json.Marshal(&DeviceState{
ConfigPath: configPath,
Image: volName,
Readonly: readonly,
PidFile: pidFile,
})
err = os.WriteFile(stateFile, stateJSON, 0600)
if (err == nil)
{
klog.Infof("Attached volume %s via UBLK as %s", volName, devicePath)
return devicePath, nil
}
os.Remove(stateFile)
}
killErr := killByPidFile(pidFile)
if (killErr != nil)
{
klog.Errorf("Failed to kill started vitastor-ublk: %v", killErr)
}
os.Remove(pidFile)
return "", err
}
func unmapUblk(stateDir, devicePath string)
{
if (len(devicePath) < 10 || devicePath[0:10] != "/dev/ublkb")
{
klog.Errorf("%s does not start with /dev/ublkb", devicePath)
return
}
unmapOut, unmapErr := exec.Command("/usr/bin/vitastor-ublk", "unmap", devicePath).CombinedOutput()
if (unmapErr != nil)
{
klog.Errorf("failed to unmap UBLK device %s: %s, error: %v", devicePath, unmapOut, unmapErr)
}
for _, ext := range []string{"json", "pid"}
{
fn := stateDir + "vitastor-ublk-" + devicePath[10:] + "." + ext
err := os.Remove(fn)
if (err != nil)
{
klog.Errorf("failed to remove %s: %v", fn, err)
}
}
}
func system(program string, args ...string) ([]byte, []byte, error)
{
klog.Infof("Running "+program+" "+strings.Join(args, " "))
src/client/nbd_proxy.cpp
+3 -1
View File
@@ -272,6 +272,8 @@ const char *help_text =
" --dev_num N\n"
" Use the specified device /dev/nbdN instead of automatic selection (alternative syntax\n"
" to /dev/nbdN positional parameter).\n"
" --readonly\n"
" Make the device read-only.\n"
" --foreground 1\n"
" Stay in foreground, do not daemonize.\n"
"\n"
@@ -372,7 +374,7 @@ public:
else if (args[i][0] == '-' && args[i][1] == '-')
{
const char *opt = args[i]+2;
cfg[opt] = !strcmp(opt, "json") || !strcmp(opt, "all") ||
cfg[opt] = !strcmp(opt, "json") || !strcmp(opt, "all") || !strcmp(opt, "readonly") ||
!strcmp(opt, "force") || i == narg-1 ? "1" : args[++i];
}
else if (pos == 0)
src/client/ublk_server.cpp
+26 -7
View File
@@ -40,12 +40,14 @@ const char *help_text =
" Make the device read-only.\n"
" --hdd\n"
" Mark the device as rotational.\n"
" --logfile /path/to/log/file.txt\n"
" Write log messages to the specified file instead of dropping them (in background mode)\n"
" or printing them to the standard output (in foreground mode).\n"
" --dev_num N\n"
" Use the specified device /dev/ublkbN instead of automatic selection (alternative syntax\n"
" to /dev/ublkbN positional parameter).\n"
" --pidfile /run/ublk_pid_file.pid\n"
" Write process ID to the specified file.\n"
" --logfile /path/to/log/file.txt\n"
" Write log messages to the specified file instead of dropping them (in background mode)\n"
" or printing them to the standard output (in foreground mode).\n"
" --foreground 1\n"
" Stay in foreground, do not daemonize.\n"
"\n"
@@ -79,6 +81,7 @@ protected:
inode_watch_t *watch = NULL;
std::string logfile = "/dev/null";
std::string pidfile;
public:
ublk_server()
@@ -300,10 +303,8 @@ help:
load_module();
bool bg = cfg["foreground"].is_null();
if (cfg["logfile"].string_value() != "")
{
logfile = cfg["logfile"].string_value();
}
logfile = cfg["logfile"].string_value();
pidfile = cfg["pidfile"].string_value();
open_control();
if (recover)
@@ -331,6 +332,8 @@ help:
close(notifyfd[0]);
}
start_device(recover);
if (pidfile != "")
write_pid();
if (bg)
{
daemonize_reopen_stdio();
@@ -399,6 +402,22 @@ help:
fprintf(stderr, "Warning: Failed to chdir into /\n");
}
void write_pid()
{
int fd = open(pidfile.c_str(), O_WRONLY|O_CREAT|O_TRUNC, 0666);
if (fd < 0)
{
fprintf(stderr, "Failed to create pid file %s: %s (code %d)\n", pidfile.c_str(), strerror(errno), errno);
return;
}
auto pid = std::to_string(getpid());
if (write(fd, pid.c_str(), pid.size()) < 0)
{
fprintf(stderr, "Failed to write pid to %s: %s (code %d)\n", pidfile.c_str(), strerror(errno), errno);
}
close(fd);
}
json11::Json::object list_mapped()
{
int n_in_dev = 0;