From 31b7021330475eb06570e213d78d4bea41f6fd39 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sun, 17 Aug 2025 15:38:22 +0300 Subject: [PATCH] Implement Vitastor ublk server --- debian/vitastor-client.install | 1 + docs/usage/ublk.en.md | 90 ++++ docs/usage/ublk.ru.md | 94 ++++ rpm/vitastor-el7.spec | 1 + rpm/vitastor-el8.spec | 1 + rpm/vitastor-el9.spec | 1 + src/CMakeLists.txt | 2 +- src/client/CMakeLists.txt | 6 + src/client/ublk_server.cpp | 900 ++++++++++++++++++++++++++++++++ src/liburing/README | 2 + src/liburing/include/ublk_cmd.h | 602 +++++++++++++++++++++ src/util/ringloop.cpp | 14 +- src/util/ringloop.h | 2 +- 13 files changed, 1710 insertions(+), 6 deletions(-) create mode 100644 docs/usage/ublk.en.md create mode 100644 docs/usage/ublk.ru.md create mode 100644 src/client/ublk_server.cpp create mode 100644 src/liburing/include/ublk_cmd.h diff --git a/debian/vitastor-client.install b/debian/vitastor-client.install index 6d64cecd..1d86cccb 100644 --- a/debian/vitastor-client.install +++ b/debian/vitastor-client.install @@ -2,6 +2,7 @@ usr/bin/vita usr/bin/vitastor-cli usr/bin/vitastor-rm usr/bin/vitastor-nbd +usr/bin/vitastor-ublk usr/bin/vitastor-nfs usr/bin/vitastor-kv usr/bin/vitastor-kv-stress diff --git a/docs/usage/ublk.en.md b/docs/usage/ublk.en.md new file mode 100644 index 00000000..c0bede5c --- /dev/null +++ b/docs/usage/ublk.en.md @@ -0,0 +1,90 @@ +[Documentation](../../README.md#documentation) → Usage → ublk + +----- + +[Читать на русском](ublk.ru.md) + +# UBLK + +[ublk](https://docs.kernel.org/block/ublk.html) is a new io_uring-based Linux interface +for user-space block device drivers, available since Linux 6.0. + +It's still not zero-copy, but so far it's the fastest userspace block device interface, +outperforming both [NBD](nbd.en.md) and [VDUSE](qemu.en.md#vduse). It also allows to recover +devices even if the server (vitastor-ublk process) dies. + +Supports the following commands: + +- [map](#map) +- [unmap](#unmap) +- [ls](#ls) + +## map + +To create a local block device for a Vitastor image run: + +``` +vitastor-ublk map [/dev/ublkbN] --image testimg +``` + +It will output a block device name like /dev/ublkb0 which you can then use as a normal disk. + +You can also use `--pool --inode --size ` instead of `--image ` if you want. + +vitastor-ublk supports all usual Vitastor configuration options like `--config_path ` plus ublk-specific: + +* `--recover` \ + Recover a mapped device if the previous ublk server is dead. +* `--queue_depth 256` \ + Maximum queue size for the device. +* `--max_io_size 1M` \ + Maximum single I/O size for the device. Default: `max(1 MB, pool block size * EC part count)`. +* `--readonly` \ + Make the device read-only. +* `--hdd` \ + Mark the device as rotational. +* `--logfile /path/to/log/file.txt` \ + Write log messages to the specified file instead of dropping them (in background mode) + or printing them to the standard output (in foreground mode). +* `--dev_num N` \ + Use the specified device /dev/ublkbN instead of automatic selection (alternative syntax + to /dev/ublkbN positional parameter). +* `--foreground 1` \ + Stay in foreground, do not daemonize. + +Note that `ublk_queue_depth` and `ublk_max_io_size` may also be specified +in `/etc/vitastor/vitastor.conf` or in other configuration file specified with `--config_path`. + +## unmap + +To unmap the device run: + +``` +vitastor-ublk unmap /dev/ublkb0 +``` + +## ls + +``` +vitastor-ublk ls [--json] +``` + +List mapped images. + +Example output (normal format): + +``` +/dev/ublkb0 +image: bench +pid: 584536 + +/dev/ublkb1 +image: bench1 +pid: 584546 +``` + +Example output (JSON format): + +``` +{"/dev/ublkb0": {"image": "bench", "pid": 584536}, "/dev/ublkb1": {"image": "bench1", "pid": 584546}} +``` diff --git a/docs/usage/ublk.ru.md b/docs/usage/ublk.ru.md new file mode 100644 index 00000000..e22dc1fb --- /dev/null +++ b/docs/usage/ublk.ru.md @@ -0,0 +1,94 @@ +[Документация](../../README-ru.md#документация) → Использование → ublk + +----- + +[Read in English](ublk.en.md) + +# UBLK + +[ublk](https://docs.kernel.org/block/ublk.html) - это новый Linux-интерфейс на основе io_uring +для реализации блочных устройств в пространстве пользователя, доступный, начиная с Linux 6.0. + +ublk тоже копирует память (т.е. не является zero-copy), но всё равно на данный момент является +самым быстрым интерфейсом ядерного блочного устройства и обгоняет и [NBD](nbd.ru.md), и [VDUSE](qemu.ru.md#vduse). +Также он позволяет оживлять устройства, у которых умер сервер (процесс-обработчик vitastor-ublk). + +Поддерживаются следующие команды: + +- [map](#map) +- [unmap](#unmap) +- [ls](#ls) + +## map + +Чтобы создать локальное блочное устройство для образа, выполните команду: + +``` +vitastor-ublk map [/dev/ublkbN] --image testimg +``` + +Команда напечатает название блочного устройства вида /dev/ublkb0, которое потом можно +будет использовать как обычный диск. + +Для обращения по номеру инода, аналогично другим командам, можно использовать опции +`--pool --inode --size ` вместо `--image testimg`. + +vitastor-ublk поддерживает все обычные опции Vitastor, например, `--config_path `, +плюс специфичные для ublk: + +* `--recover` \ + Восстановить ранее подключённое устройство, у которого умер обработчик. +* `--queue_depth 256` \ + Максимальная глубина очереди устройства. +* `--max_io_size 1M` \ + Максимальный размер запроса ввода-вывода для устройства. По умолчанию: `max(1 MB, блок данных пула * число частей данных EC)`. +* `--readonly` \ + Подключить устройство в режиме только для чтения. +* `--hdd` \ + Пометить устройство как вращающийся жёсткий диск (флаг rotational). +* `--logfile /path/to/log/file.txt` \ + Писать сообщения о процессе работы в заданный файл, вместо пропуска их + при фоновом режиме запуска или печати на стандартный вывод при запуске + в консоли с `--foreground 1`. +* `--dev_num N` \ + Использовать заданное устройство `/dev/ublkbN` вместо автоматического подбора. +* `--foreground 1` \ + Не уводить процесс в фоновый режим. + +Обратите внимание, что опции `ublk_queue_depth` и `ublk_max_io_size` можно +также задавать в `/etc/vitastor/vitastor.conf` или в другом файле конфигурации, +заданном опцией `--config_path`. + +## unmap + +Для отключения устройства выполните: + +``` +vitastor-ublk unmap /dev/ublkb0 +``` + +## ls + +``` +vitastor-ublk ls [--json] +``` + +Вывести подключённые устройства. + +Пример вывода в обычном формате: + +``` +/dev/ublkb0 +image: bench +pid: 584536 + +/dev/ublkb1 +image: bench1 +pid: 584546 +``` + +Пример вывода в JSON-формате: + +``` +{"/dev/ublkb0": {"image": "bench", "pid": 584536}, "/dev/ublkb1": {"image": "bench1", "pid": 584546}} +``` diff --git a/rpm/vitastor-el7.spec b/rpm/vitastor-el7.spec index 3c0dc227..08571340 100644 --- a/rpm/vitastor-el7.spec +++ b/rpm/vitastor-el7.spec @@ -164,6 +164,7 @@ chown vitastor:vitastor /var/lib/vitastor %files -n vitastor-client %_bindir/vitastor-nbd +%_bindir/vitastor-ublk %_bindir/vitastor-nfs %_bindir/vitastor-cli %_bindir/vitastor-rm diff --git a/rpm/vitastor-el8.spec b/rpm/vitastor-el8.spec index 7cd44416..418323c2 100644 --- a/rpm/vitastor-el8.spec +++ b/rpm/vitastor-el8.spec @@ -161,6 +161,7 @@ chown vitastor:vitastor /var/lib/vitastor %files -n vitastor-client %_bindir/vitastor-nbd +%_bindir/vitastor-ublk %_bindir/vitastor-nfs %_bindir/vitastor-cli %_bindir/vitastor-rm diff --git a/rpm/vitastor-el9.spec b/rpm/vitastor-el9.spec index 377a267a..c23f2aaa 100644 --- a/rpm/vitastor-el9.spec +++ b/rpm/vitastor-el9.spec @@ -158,6 +158,7 @@ chown vitastor:vitastor /var/lib/vitastor %files -n vitastor-client %_bindir/vitastor-nbd +%_bindir/vitastor-ublk %_bindir/vitastor-nfs %_bindir/vitastor-cli %_bindir/vitastor-rm diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 101fe36a..03f3c833 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -112,7 +112,7 @@ add_subdirectory(test) ### Install -install(TARGETS vitastor-osd vitastor-disk vitastor-nbd vitastor-nfs vitastor-cli vitastor-kv vitastor-kv-stress RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +install(TARGETS vitastor-osd vitastor-disk vitastor-nbd vitastor-ublk vitastor-nfs vitastor-cli vitastor-kv vitastor-kv-stress RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) install_symlink(vitastor-disk ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/vitastor-dump-journal) install_symlink(vitastor-cli ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/vitastor-rm) install_symlink(vitastor-cli ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/vita) diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 76368ae1..e93600cd 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -68,6 +68,12 @@ if (HAVE_NBD_NETLINK_H AND NL3_LIBRARIES) target_compile_definitions(vitastor-nbd PUBLIC HAVE_NBD_NETLINK_H) endif (HAVE_NBD_NETLINK_H AND NL3_LIBRARIES) +# vitastor-ublk +add_executable(vitastor-ublk + ublk_server.cpp +) +target_link_libraries(vitastor-ublk vitastor_client) + if (${WITH_QEMU}) # qemu_driver.so add_library(qemu_vitastor SHARED diff --git a/src/client/ublk_server.cpp b/src/client/ublk_server.cpp new file mode 100644 index 00000000..4c4b3829 --- /dev/null +++ b/src/client/ublk_server.cpp @@ -0,0 +1,900 @@ +// Copyright (c) Vitaliy Filippov, 2019+ +// License: VNPL-1.1 (see README.md for details) +// ublk-based Vitastor block device in userspace + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "../liburing/include/ublk_cmd.h" +#include "cluster_client.h" +#include "epoll_manager.h" +#include "str_util.h" + +const char *exe_name = NULL; + +const char *help_text = + "Vitastor ublk server " VITASTOR_VERSION "\n" + "(c) Vitaliy Filippov, 2025+ (VNPL-1.1)\n" + "\n" + "COMMANDS:\n" + "\n" + "vitastor-ublk map [OPTIONS] (--image | --pool --inode --size )\n" + " Map a ublk device. Options:\n" + " --recover\n" + " Recover a mapped device if the previous ublk server is dead.\n" + " --queue_depth 256\n" + " Maximum queue size for the device.\n" + " --max_io_size 1M\n" + " Maximum single I/O size for the device. Default: max(1 MB, pool block size * EC part count).\n" + " --readonly\n" + " Make the device read-only.\n" + " --hdd\n" + " Mark the device as rotational.\n" + " --logfile /path/to/log/file.txt\n" + " Write log messages to the specified file instead of dropping them (in background mode)\n" + " or printing them to the standard output (in foreground mode).\n" + " --dev_num N\n" + " Use the specified device /dev/ublkbN instead of automatic selection (alternative syntax\n" + " to /dev/ublkbN positional parameter).\n" + " --foreground 1\n" + " Stay in foreground, do not daemonize.\n" + "\n" + "vitastor-ublk unmap [--force] /dev/ublkb\n" + " Unmap a Vitastor ublk device. Do not check if it's actually mapped if --force is specified.\n" + "\n" + "vitastor-ublk ls [--json]\n" + " List mapped Vitastor ublk devices, optionally in JSON format.\n" + "\n" + "Use vitastor-ublk --help for command details or vitastor-ublk --help --all for all details.\n" + "\n" + "All usual Vitastor config options like --config_path may also be specified in CLI.\n" +; + +class ublk_server +{ +protected: + std::string image_name; + uint64_t inode = 0; + uint64_t device_size = 0; + int req_dev_num = -1; + bool readonly = false; + bool hdd = false; + bool recover = false; + uint16_t queue_depth = 256; + uint32_t max_io_size = 0; + + ring_loop_t *ringloop = NULL; + epoll_manager_t *epmgr = NULL; + cluster_client_t *cli = NULL; + inode_watch_t *watch = NULL; + + std::string logfile = "/dev/null"; + +public: + ublk_server() + { + ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE, false, true); + } + + ~ublk_server() + { + if (ctrl_fd >= 0) + { + close(ctrl_fd); + ctrl_fd = -1; + } + if (cdev_fd >= 0) + { + close(cdev_fd); + cdev_fd = -1; + } + for (auto & buf: buffers) + { + free(buf); + } + buffers.clear(); + if (ringloop) + { + delete ringloop; + ringloop = NULL; + } + } + + static json11::Json::object parse_args(int narg, const char *args[]) + { + json11::Json::object cfg; + int pos = 0; + for (int i = 1; i < narg; i++) + { + if (!strcmp(args[i], "-h") || !strcmp(args[i], "--help")) + { + cfg["help"] = 1; + } + else if (args[i][0] == '-' && args[i][1] == '-') + { + const char *opt = args[i]+2; + cfg[opt] = !strcmp(opt, "json") || !strcmp(opt, "all") || + !strcmp(opt, "readonly") || !strcmp(opt, "hdd") || !strcmp(opt, "recover") || + !strcmp(opt, "force") || i == narg-1 ? "1" : args[++i]; + } + else if (pos == 0) + { + cfg["command"] = args[i]; + pos++; + } + else if (pos == 1) + { + char c = 0; + int n = 0; + if (sscanf(args[i], "/dev/ublkb%d%c", &n, &c) == 1) + cfg["dev_num"] = n; + else if (sscanf(args[i], "/dev/ublkc%d%c", &n, &c) == 1) + cfg["dev_num"] = n; + else + cfg["dev_num"] = args[i]; + pos++; + } + } + return cfg; + } + + void exec(json11::Json cfg) + { + if (cfg["help"].bool_value()) + { + goto help; + } + if (cfg["command"] == "map") + { + start(cfg); + } + else if (cfg["command"] == "unmap") + { + if (!cfg["dev_num"].is_number() && + cfg["dev_num"].string_value() != "0" && + !cfg["dev_num"].uint64_value()) + { + fprintf(stderr, "device name or number is missing\n"); + exit(1); + } + open_control(); + unmap_device(cfg["dev_num"].uint64_value(), cfg["unpriv"].bool_value(), cfg["wait"].bool_value()); + } + else if (cfg["command"] == "ls" || cfg["command"] == "list" || cfg["command"] == "list-mapped") + { + auto mapped = list_mapped(); + print_mapped(mapped, !cfg["json"].is_null()); + } + else + { +help: + print_help(help_text, "vitastor-ublk", cfg["command"].string_value(), cfg["all"].bool_value()); + exit(0); + } + } + + void start(json11::Json cfg) + { + // Check options + if (cfg["dev_num"].string_value() != "" || cfg["dev_num"].is_number()) + { + req_dev_num = cfg["dev_num"].uint64_value(); + } + if (cfg["image"].string_value() != "") + { + // Use image name + image_name = cfg["image"].string_value(); + inode = 0; + } + else + { + // Use pool, inode number and size + device_size = cfg["size"].is_string() + ? parse_size(cfg["size"].string_value()) + : cfg["size"].uint64_value(); + if (!device_size) + { + fprintf(stderr, "device size is missing\n"); + exit(1); + } + inode = cfg["inode"].uint64_value(); + uint64_t pool = cfg["pool"].uint64_value(); + if (pool) + { + inode = INODE_WITH_POOL(pool, inode); + } + if (!INODE_POOL(inode)) + { + fprintf(stderr, "pool is missing\n"); + exit(1); + } + } + if (cfg["client_writeback_allowed"].is_null()) + { + // ublk is always aware of fsync, so we allow write-back cache + // by default if it's enabled + auto obj = cfg.object_items(); + obj["client_writeback_allowed"] = true; + cfg = obj; + } + readonly = cfg["readonly"].bool_value(); + hdd = cfg["hdd"].bool_value(); + recover = cfg["recover"].bool_value(); + if (recover && req_dev_num < 0) + { + fprintf(stderr, "device is missing\n"); + exit(1); + } + + // Create client + epmgr = new epoll_manager_t(ringloop); + cli = new cluster_client_t(ringloop, epmgr->tfd, cfg); + + // cli->config contains merged config + if (!cfg["queue_depth"].is_null()) + { + queue_depth = cfg["queue_depth"].uint64_value(); + } + else if (cli->config.find("ublk_queue_depth") != cli->config.end()) + { + queue_depth = cli->config["ublk_queue_depth"].uint64_value(); + } + if (!cfg["max_io_size"].is_null()) + { + max_io_size = parse_size(cfg["max_io_size"].string_value()); + } + else if (cli->config.find("ublk_max_io_size") != cli->config.end()) + { + max_io_size = cli->config["ublk_max_io_size"].is_string() + ? parse_size(cli->config["ublk_max_io_size"].string_value()) + : cli->config["ublk_max_io_size"].uint64_value(); + } + + // Load image metadata + while (!cli->is_ready()) + { + ringloop->loop(); + if (cli->is_ready()) + break; + ringloop->wait(); + } + if (!inode) + { + watch = cli->st_cli.watch_inode(image_name); + device_size = watch->cfg.size; + if (!watch->cfg.num || !device_size) + { + // Image does not exist + fprintf(stderr, "Image %s does not exist\n", image_name.c_str()); + exit(1); + } + } + const bool writeback = cli->get_immediate_commit(inode); + auto pool_it = cli->st_cli.pool_config.find(INODE_POOL(inode ? inode : watch->cfg.num)); + if (pool_it == cli->st_cli.pool_config.end()) + { + fprintf(stderr, "Pool %u does not exist\n", INODE_POOL(inode ? inode : watch->cfg.num)); + exit(1); + } + auto & pool_cfg = pool_it->second; + uint32_t pg_data_size = pool_cfg.data_block_size * (pool_cfg.scheme == POOL_SCHEME_REPLICATED + ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks); + if (max_io_size & (max_io_size-1)) + { + fprintf(stderr, "max_io_size must be a power of 2\n"); + exit(1); + } + uint32_t buf_size = max_io_size ? max_io_size : (1024*1024 < pg_data_size ? pg_data_size : 1024*1024); + uint32_t bitmap_granularity = pool_cfg.bitmap_granularity; + + load_module(); + + bool bg = cfg["foreground"].is_null(); + if (cfg["logfile"].string_value() != "") + { + logfile = cfg["logfile"].string_value(); + } + + open_control(); + if (recover) + { + recover_device(req_dev_num); + } + else + { + add_device( + req_dev_num, + (writeback ? UBLK_ATTR_VOLATILE_CACHE : 0) | + (readonly ? UBLK_ATTR_READ_ONLY : 0) | (hdd ? UBLK_ATTR_ROTATIONAL : 0), + queue_depth, bitmap_granularity, buf_size, pg_data_size, device_size + ); + } + int notifyfd[2] = { -1, -1 }; + if (bg) + { + if (socketpair(AF_UNIX, SOCK_STREAM, 0, notifyfd) < 0) + { + perror("socketpair"); + exit(1); + } + daemonize_fork(notifyfd); + close(notifyfd[0]); + } + start_device(recover); + if (bg) + { + daemonize_reopen_stdio(); + int ok = 0; + write(notifyfd[1], &ok, sizeof(ok)); + close(notifyfd[1]); + } + else + printf("/dev/ublkb%d\n", ublk_dev.dev_id); + stop = false; + while (!stop) + { + ringloop->loop(); + ringloop->wait(); + } + cluster_op_t *close_sync = new cluster_op_t; + close_sync->opcode = OSD_OP_SYNC; + close_sync->callback = [this](cluster_op_t *op) + { + stop = true; + delete op; + }; + cli->execute(close_sync); + while (!stop) + { + ringloop->loop(); + ringloop->wait(); + } + cli->flush(); + delete cli; + delete epmgr; + cli = NULL; + epmgr = NULL; + } + + void load_module() + { + if (access("/sys/module/ublk_drv", F_OK) == 0) + { + return; + } + int r; + if ((r = system("modprobe ublk_drv")) != 0) + { + if (r < 0) + perror("Failed to load ublk_drv kernel module"); + else + fprintf(stderr, "Failed to load ublk_drv kernel module\n"); + exit(1); + } + } + + void daemonize_fork(int *notifyfd) + { + if (fork()) + { + // Parent - check status + close(notifyfd[1]); + int child_errno = 1; + read(notifyfd[0], &child_errno, sizeof(child_errno)); + if (!child_errno) + printf("/dev/ublkb%d\n", ublk_dev.dev_id); + exit(child_errno); + } + setsid(); + if (fork()) + exit(0); + } + + void daemonize_reopen_stdio() + { + close(0); + close(1); + close(2); + open("/dev/null", O_RDONLY); + open(logfile.c_str(), O_WRONLY|O_APPEND|O_CREAT, 0666); + open(logfile.c_str(), O_WRONLY|O_APPEND|O_CREAT, 0666); + if (chdir("/") != 0) + fprintf(stderr, "Warning: Failed to chdir into /\n"); + } + + json11::Json::object list_mapped() + { + int n_in_dev = 0; + DIR *d = opendir("/dev"); + if (!d) + { + fprintf(stderr, "Failed to list /dev: %s (code %d)\n", strerror(errno), errno); + exit(1); + } + dirent *ent; + while ((ent = readdir(d)) != NULL) + { + if (!strncmp(ent->d_name, "ublkc", strlen("ublkc"))) + n_in_dev++; + } + closedir(d); + json11::Json::object mapped; + const char *self_filename = exe_name; + for (int i = 0; exe_name[i] != 0; i++) + { + if (exe_name[i] == '/') + self_filename = exe_name+i+1; + } + char path[64] = { 0 }; + int dev_num = -1, n_in_ctrl = 0; + open_control(); + while (true) + { + dev_num++; + int res = get_dev_info(dev_num, false); + if (res == -ENODEV) + { + if (n_in_ctrl >= n_in_dev) + break; + continue; + } + n_in_ctrl++; + sprintf(path, "/proc/%d/cmdline", ublk_dev.ublksrv_pid); + std::string cmdline = read_file(path); + if (cmdline == "") + { + // Process is dead + mapped["/dev/ublkb"+std::to_string(dev_num)] = json11::Json::object{{"dead", true}}; + continue; + } + std::vector argv; + int last = 0; + for (int i = 0; i < cmdline.size(); i++) + { + if (cmdline[i] == 0) + { + argv.push_back(cmdline.c_str()+last); + last = i+1; + } + } + if (argv.size() > 0) + { + const char *pid_filename = argv[0]; + for (int i = 0; argv[0][i] != 0; i++) + { + if (argv[0][i] == '/') + pid_filename = argv[0]+i+1; + } + if (!strcmp(pid_filename, self_filename)) + { + json11::Json::object cfg = ublk_server::parse_args(argv.size(), argv.data()); + if (cfg["command"] == "map") + { + cfg.erase("command"); + cfg["pid"] = ublk_dev.ublksrv_pid; + mapped["/dev/ublkb"+std::to_string(dev_num)] = cfg; + } + } + } + } + return mapped; + } + + void print_mapped(json11::Json mapped, bool json) + { + if (json) + { + printf("%s\n", mapped.dump().c_str()); + } + else + { + for (auto & dev: mapped.object_items()) + { + printf("%s\n", dev.first.c_str()); + for (auto & k: dev.second.object_items()) + { + printf("%s: %s\n", k.first.c_str(), k.second.as_string().c_str()); + } + printf("\n"); + } + } + } + + std::string read_file(char *path) + { + int fd = open(path, O_RDONLY); + if (fd < 0) + { + if (errno == ENOENT) + return ""; + auto err = "open "+std::string(path); + perror(err.c_str()); + exit(1); + } + std::string r; + while (true) + { + int l = r.size(); + r.resize(l + 1024); + int rd = read(fd, (void*)(r.c_str() + l), 1024); + if (rd <= 0) + { + r.resize(l); + break; + } + r.resize(l + rd); + } + close(fd); + return r; + } + +protected: + bool stop = false; + bool new_opcodes = true; + uint64_t ublk_features = 0; + int max_wait_time_ms = 5000; + int ctrl_fd = -1, cdev_fd = -1; + ublksrv_ctrl_dev_info ublk_dev = {}; + ublksrv_io_desc *ublk_queue = NULL; + std::vector buffers; + + void open_control() + { + ctrl_fd = open("/dev/ublk-control", O_RDWR); + if (ctrl_fd < 0) + { + fprintf(stderr, "Failed to open /dev/ublk-control: %s (code %d)\n", strerror(errno), errno); + exit(1); + } + // Check features + int res = sync_ublk_cmd(UBLK_U_CMD_GET_FEATURES, &ublk_features, 8, 0); + if (res == -EOPNOTSUPP) + { + new_opcodes = false; + } + else if (res != 0) + { + fprintf(stderr, "Failed to get ublk features: %s (code %d)\n", strerror(-res), res); + exit(1); + } + } + + void add_device(int32_t dev_num, uint32_t attrs, uint16_t queue_depth, uint32_t phys_block_size, + uint32_t max_io_buf_bytes, uint64_t opt_block_size, uint64_t device_size) + { + // Add device + ublk_dev.dev_id = dev_num; + ublk_dev.nr_hw_queues = 1; + ublk_dev.queue_depth = queue_depth; + ublk_dev.max_io_buf_bytes = max_io_buf_bytes; + ublk_dev.flags = UBLK_F_USER_RECOVERY | UBLK_F_USER_RECOVERY_REISSUE; + int res = sync_ublk_cmd(new_opcodes ? UBLK_U_CMD_ADD_DEV : UBLK_CMD_ADD_DEV, &ublk_dev, sizeof(ublk_dev)); + if (res != 0) + { + fprintf(stderr, "Failed to add ublk device: %s (code %d)\n", strerror(-res), res); + exit(1); + } + // Wait until the device appears + std::string ublkc_path = "/dev/ublkc"+std::to_string(ublk_dev.dev_id); + int wait_time = 0; + while (wait_time < max_wait_time_ms) + { + cdev_fd = open(ublkc_path.c_str(), O_RDWR); + if (cdev_fd >= 0) + break; + else if (errno != ENOENT) + { + fprintf(stderr, "Failed to open %s: %s (code %d)", ublkc_path.c_str(), strerror(errno), errno); + exit(1); + } + usleep(100); + wait_time += 100; + } + close(cdev_fd); + cdev_fd = -1; + // Set device params + uint8_t io_opt_shift = 0; + while ((opt_block_size >> io_opt_shift) > 1) + { + io_opt_shift++; + } + uint8_t phys_shift = 0; + while ((phys_block_size >> phys_shift) > 1) + { + phys_shift++; + } + ublk_params params = { + .len = sizeof(ublk_params), + .types = UBLK_PARAM_TYPE_BASIC, + .basic = { + .attrs = attrs, // UBLK_ATTR_READ_ONLY | UBLK_ATTR_ROTATIONAL | UBLK_ATTR_VOLATILE_CACHE | UBLK_ATTR_FUA + .logical_bs_shift = 9, + .physical_bs_shift = phys_shift, + .io_opt_shift = io_opt_shift, + .io_min_shift = phys_shift, + .max_sectors = max_io_buf_bytes / phys_block_size, + .chunk_sectors = 0, + .dev_sectors = device_size / phys_block_size, + .virt_boundary_mask = 0, + }, + .discard = { + .discard_alignment = 0, + .discard_granularity = 0, + .max_discard_sectors = 0, + .max_write_zeroes_sectors = 0, + .max_discard_segments = 0, + }, + }; + res = sync_unpriv_cmd(false, new_opcodes ? UBLK_U_CMD_SET_PARAMS : UBLK_CMD_SET_PARAMS, ¶ms, sizeof(params)); + if (res != 0) + { + fprintf(stderr, "Failed to set ublk device params: %s (code %d)\n", strerror(-res), res); + exit(1); + } + } + + void map_ublk_queue() + { + const unsigned page_sz = getpagesize(); + size_t cmd_buf_size = (ublk_dev.queue_depth * sizeof(ublksrv_io_desc) + page_sz-1) / page_sz * page_sz; + //const unsigned queue_offset = (UBLK_MAX_QUEUE_DEPTH * sizeof(ublksrv_io_desc) + page_sz-1) / page_sz * page_sz; + //off = q_id * queue_offset; + ublk_queue = (ublksrv_io_desc*)mmap(0, cmd_buf_size, PROT_READ, MAP_SHARED | MAP_POPULATE, cdev_fd, 0); + if ((void*)ublk_queue == MAP_FAILED) + { + fprintf(stderr, "Failed to mmap() ublk queue buffer\n"); + exit(1); + } + } + + void recover_device(uint32_t dev_num) + { + ublk_dev.dev_id = dev_num; + int res = sync_ublk_cmd(new_opcodes ? UBLK_U_CMD_GET_DEV_INFO : UBLK_CMD_GET_DEV_INFO, &ublk_dev, sizeof(ublk_dev)); + if (res != 0) + { + fprintf(stderr, "Failed to get /dev/ublkb%u device info: %s (code %d)\n", dev_num, strerror(-res), res); + exit(1); + } + if (ublk_dev.nr_hw_queues != 1) + { + fprintf(stderr, "Device /dev/ublkb%u is not supported because it has %d queues\n", dev_num, ublk_dev.nr_hw_queues); + exit(1); + } + if (ublk_dev.ublksrv_pid != 0) + { + res = kill(ublk_dev.ublksrv_pid, 0); + if (res == 0) + { + fprintf(stderr, "Device /dev/ublkb%u is still alive, daemon PID is %u\n", dev_num, ublk_dev.ublksrv_pid); + exit(1); + } + else if (errno != ESRCH) + { + fprintf(stderr, "Device /dev/ublkb%u is still alive, failed to check if the daemon with PID %u is running: %s (code %d)\n", + dev_num, ublk_dev.ublksrv_pid, strerror(errno), errno); + exit(1); + } + } + // Send the "start recovery" command + res = sync_unpriv_cmd(false, new_opcodes ? UBLK_U_CMD_START_USER_RECOVERY : UBLK_CMD_START_USER_RECOVERY, NULL, 0); + if (res != 0) + { + fprintf(stderr, "Failed to start /dev/ublkb%u device recovery: %s (code %d)\n", dev_num, strerror(-res), res); + exit(1); + } + } + + void start_device(bool recover) + { + std::string ublkc_path = "/dev/ublkc"+std::to_string(ublk_dev.dev_id); + cdev_fd = open(ublkc_path.c_str(), O_RDWR|O_NONBLOCK); + if (cdev_fd < 0) + { + fprintf(stderr, "Failed to open %s: %s (code %d)", ublkc_path.c_str(), strerror(errno), errno); + exit(1); + } + // FIXME Here we could optionally do ublk_get_queue_affinity + // Map queue command buffer + map_ublk_queue(); + // submit initial fetch requests to ublk driver + for (int i = 0; i < ublk_dev.queue_depth; i++) + { + buffers.push_back((uint8_t*)memalign_or_die(MEM_ALIGNMENT, ublk_dev.max_io_buf_bytes)); + submit_request(new_opcodes ? UBLK_U_IO_FETCH_REQ : UBLK_IO_FETCH_REQ, i, 0); + } + ringloop->submit(); + // start device + ublk_dev.ublksrv_pid = getpid(); + int res = sync_unpriv_cmd(false, (recover + ? (new_opcodes ? UBLK_U_CMD_END_USER_RECOVERY : UBLK_CMD_END_USER_RECOVERY) + : (new_opcodes ? UBLK_U_CMD_START_DEV : UBLK_CMD_START_DEV)), NULL, 0, ublk_dev.ublksrv_pid); + if (res != 0) + { + fprintf(stderr, "Failed to start ublk device: %s (code %d)\n", strerror(-res), res); + exit(1); + } + close(ctrl_fd); + ctrl_fd = -1; + } + + void submit_request(uint64_t ublk_cmd, int i, int res) + { + io_uring_sqe *sqe = ringloop->get_sqe(); + ring_data_t* data = ((ring_data_t*)sqe->user_data); + sqe->fd = cdev_fd; + sqe->opcode = IORING_OP_URING_CMD; + //sqe->flags = IOSQE_FIXED_FILE; + sqe->flags = 0; + sqe->rw_flags = 0; + sqe->off = ublk_cmd; + ublksrv_io_cmd *cmd = (ublksrv_io_cmd *)&sqe->addr3; // sqe128 command buffer address + cmd->q_id = 0; + cmd->tag = i; + cmd->addr = (uint64_t)buffers[i]; + cmd->result = res; + data->callback = [this, i](ring_data_t *data) { exec_request(data->res, i); }; + } + + void exec_request(int res, int i) + { + if (res != 0) + { + // Note: res may be also UBLK_IO_RES_NEED_GET_DATA if UBLK_F_NEED_GET_DATA is enabled, + // in this case you should submit_request(UBLK_IO_NEED_GET_DATA, i) again with buffer + if (res == -ENODEV) + { + // ublk device is removed + stop = true; + return; + } + fprintf(stderr, "Fetching ublk request failed: %s (code %d)\n", strerror(-res), res); + exit(1); + } + ublksrv_io_desc *iod = &ublk_queue[i]; + uint8_t opcode = ublksrv_get_op(iod); + if (opcode == UBLK_IO_OP_FLUSH) + { + cluster_op_t *op = new cluster_op_t; + op->opcode = OSD_OP_SYNC; + op->callback = [this, i](cluster_op_t *op) + { + submit_request(new_opcodes ? UBLK_U_IO_COMMIT_AND_FETCH_REQ : UBLK_IO_COMMIT_AND_FETCH_REQ, i, op->retval); + delete op; + }; + cli->execute(op); + } + else if (opcode == UBLK_IO_OP_WRITE_ZEROES || opcode == UBLK_IO_OP_DISCARD) + { + submit_request(new_opcodes ? UBLK_U_IO_COMMIT_AND_FETCH_REQ : UBLK_IO_COMMIT_AND_FETCH_REQ, i, -EINVAL); + } + else if (opcode == UBLK_IO_OP_READ || opcode == UBLK_IO_OP_WRITE) + { + cluster_op_t *op = new cluster_op_t; + op->opcode = opcode == UBLK_IO_OP_READ ? OSD_OP_READ : OSD_OP_WRITE; + op->inode = inode ? inode : watch->cfg.num; + op->offset = iod->start_sector * 512; + op->len = iod->nr_sectors * 512; + op->iov.push_back(buffers[i], op->len); + op->callback = [this, i](cluster_op_t *op) + { + submit_request(new_opcodes ? UBLK_U_IO_COMMIT_AND_FETCH_REQ : UBLK_IO_COMMIT_AND_FETCH_REQ, i, op->retval); + delete op; + }; + cli->execute(op); + } + else + { + submit_request(new_opcodes ? UBLK_U_IO_COMMIT_AND_FETCH_REQ : UBLK_IO_COMMIT_AND_FETCH_REQ, i, -EINVAL); + } + } + + int get_dev_info(int dev_num, bool unpriv) + { + // Get device info + ublk_dev.dev_id = dev_num; + int res = unpriv + ? sync_unpriv_cmd(true, new_opcodes ? UBLK_U_CMD_GET_DEV_INFO2 : UBLK_CMD_GET_DEV_INFO2, &ublk_dev, sizeof(ublk_dev)) + : sync_ublk_cmd(new_opcodes ? UBLK_U_CMD_GET_DEV_INFO : UBLK_CMD_GET_DEV_INFO, &ublk_dev, sizeof(ublk_dev)); + if (res != 0 && res != -ENODEV) + { + fprintf(stderr, "Failed to get device info from /dev/ublkc%d: %s (code %d)\n", dev_num, strerror(-res), res); + exit(1); + } + return res; + } + + void unmap_device(int dev_num, bool unpriv, bool wait) + { + int res = 0; + // Stop the device + ublk_dev.dev_id = dev_num; + res = sync_unpriv_cmd(unpriv, new_opcodes ? UBLK_U_CMD_STOP_DEV : UBLK_CMD_STOP_DEV, NULL, 0); + if (res != 0) + { + fprintf(stderr, "Failed to stop device /dev/ublkc%d: %s (code %d)\n", dev_num, strerror(-res), res); + exit(1); + } + // Delete the device + res = sync_unpriv_cmd(unpriv, new_opcodes ? (wait ? UBLK_U_CMD_DEL_DEV : UBLK_U_CMD_DEL_DEV_ASYNC) : UBLK_CMD_DEL_DEV, NULL, 0); + if (res != 0) + { + fprintf(stderr, "Failed to delete device /dev/ublkc%d: %s (code %d)\n", dev_num, strerror(-res), res); + exit(1); + } + } + + int sync_unpriv_cmd(bool unpriv, uint32_t cmd_op, void *addr, uint32_t len, uint64_t data0 = 0) + { + int res; + if (unpriv) + { + static const int path_max = 64; + char buf[path_max + len]; + memset(buf, 0, path_max); + memcpy(buf + path_max, addr, len); + snprintf(buf, path_max, "/dev/ublkc%d", ublk_dev.dev_id); + res = sync_ublk_cmd(cmd_op, buf, sizeof(buf), path_max, data0); + if (!res) + memcpy(addr, buf + path_max, len); + } + else + { + res = sync_ublk_cmd(cmd_op, addr, len, 0, data0); + } + return res; + } + + int sync_ublk_cmd(uint32_t cmd_op, void *addr, uint32_t len, uint16_t dev_path_len = 0, uint64_t data0 = 0) + { + io_uring_sqe *sqe = ringloop->get_sqe(); + sqe->fd = ctrl_fd; + sqe->opcode = IORING_OP_URING_CMD; + sqe->ioprio = 0; + sqe->off = cmd_op; + ublksrv_ctrl_cmd *cmd = (ublksrv_ctrl_cmd *)&sqe->addr3; // sqe128 command buffer address + cmd->dev_id = ublk_dev.dev_id; + cmd->queue_id = -1; + cmd->addr = (uint64_t)addr; + cmd->len = len; + cmd->data[0] = data0; + cmd->dev_path_len = dev_path_len; + ring_data_t* data = ((ring_data_t*)sqe->user_data); + bool done = false; + int res = 0; + data->callback = [&](ring_data_t *data) + { + res = data->res; + done = true; + }; + ringloop->submit(); + while (!done) + { + ringloop->loop(); + if (!done) + ringloop->wait(); + } + return res; + } +}; + +int main(int narg, const char *args[]) +{ + setvbuf(stdout, NULL, _IONBF, 0); + setvbuf(stderr, NULL, _IONBF, 0); + exe_name = args[0]; + ublk_server *p = new ublk_server(); + p->exec(ublk_server::parse_args(narg, args)); + delete p; + return 0; +} diff --git a/src/liburing/README b/src/liburing/README index 2fd69e22..e517c66c 100644 --- a/src/liburing/README +++ b/src/liburing/README @@ -3,6 +3,8 @@ liburing This is a copy if liburing 2.11 from https://github.com/axboe/liburing/ +ublk_cmd.h is a copy from https://github.com/ublk-org/ublksrv/blob/master/include/ublk_cmd.h + For more info on io_uring, please see: https://kernel.dk/io_uring.pdf diff --git a/src/liburing/include/ublk_cmd.h b/src/liburing/include/ublk_cmd.h new file mode 100644 index 00000000..470ca4db --- /dev/null +++ b/src/liburing/include/ublk_cmd.h @@ -0,0 +1,602 @@ +/* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) or MIT */ +#ifndef USER_BLK_DRV_CMD_INC_H +#define USER_BLK_DRV_CMD_INC_H + +#include + +/* ublk server command definition */ + +/* + * Admin commands, issued by ublk server, and handled by ublk driver. + * + * Legacy command definition, don't use in new application, and don't + * add new such definition any more + */ +#define UBLK_CMD_GET_QUEUE_AFFINITY 0x01 +#define UBLK_CMD_GET_DEV_INFO 0x02 +#define UBLK_CMD_ADD_DEV 0x04 +#define UBLK_CMD_DEL_DEV 0x05 +#define UBLK_CMD_START_DEV 0x06 +#define UBLK_CMD_STOP_DEV 0x07 +#define UBLK_CMD_SET_PARAMS 0x08 +#define UBLK_CMD_GET_PARAMS 0x09 +#define UBLK_CMD_START_USER_RECOVERY 0x10 +#define UBLK_CMD_END_USER_RECOVERY 0x11 +#define UBLK_CMD_GET_DEV_INFO2 0x12 + +/* Any new ctrl command should encode by __IO*() */ +#define UBLK_U_CMD_GET_QUEUE_AFFINITY \ + _IOR('u', UBLK_CMD_GET_QUEUE_AFFINITY, struct ublksrv_ctrl_cmd) +#define UBLK_U_CMD_GET_DEV_INFO \ + _IOR('u', UBLK_CMD_GET_DEV_INFO, struct ublksrv_ctrl_cmd) +#define UBLK_U_CMD_ADD_DEV \ + _IOWR('u', UBLK_CMD_ADD_DEV, struct ublksrv_ctrl_cmd) +#define UBLK_U_CMD_DEL_DEV \ + _IOWR('u', UBLK_CMD_DEL_DEV, struct ublksrv_ctrl_cmd) +#define UBLK_U_CMD_START_DEV \ + _IOWR('u', UBLK_CMD_START_DEV, struct ublksrv_ctrl_cmd) +#define UBLK_U_CMD_STOP_DEV \ + _IOWR('u', UBLK_CMD_STOP_DEV, struct ublksrv_ctrl_cmd) +#define UBLK_U_CMD_SET_PARAMS \ + _IOWR('u', UBLK_CMD_SET_PARAMS, struct ublksrv_ctrl_cmd) +#define UBLK_U_CMD_GET_PARAMS \ + _IOR('u', UBLK_CMD_GET_PARAMS, struct ublksrv_ctrl_cmd) +#define UBLK_U_CMD_START_USER_RECOVERY \ + _IOWR('u', UBLK_CMD_START_USER_RECOVERY, struct ublksrv_ctrl_cmd) +#define UBLK_U_CMD_END_USER_RECOVERY \ + _IOWR('u', UBLK_CMD_END_USER_RECOVERY, struct ublksrv_ctrl_cmd) +#define UBLK_U_CMD_GET_DEV_INFO2 \ + _IOR('u', UBLK_CMD_GET_DEV_INFO2, struct ublksrv_ctrl_cmd) +#define UBLK_U_CMD_GET_FEATURES \ + _IOR('u', 0x13, struct ublksrv_ctrl_cmd) +#define UBLK_U_CMD_DEL_DEV_ASYNC \ + _IOR('u', 0x14, struct ublksrv_ctrl_cmd) +#define UBLK_U_CMD_UPDATE_SIZE \ + _IOWR('u', 0x15, struct ublksrv_ctrl_cmd) +#define UBLK_U_CMD_QUIESCE_DEV \ + _IOWR('u', 0x16, struct ublksrv_ctrl_cmd) + +/* + * 64bits are enough now, and it should be easy to extend in case of + * running out of feature flags + */ +#define UBLK_FEATURES_LEN 8 + +/* + * IO commands, issued by ublk server, and handled by ublk driver. + * + * FETCH_REQ: issued via sqe(URING_CMD) beforehand for fetching IO request + * from ublk driver, should be issued only when starting device. After + * the associated cqe is returned, request's tag can be retrieved via + * cqe->userdata. + * + * COMMIT_AND_FETCH_REQ: issued via sqe(URING_CMD) after ublkserver handled + * this IO request, request's handling result is committed to ublk + * driver, meantime FETCH_REQ is piggyback, and FETCH_REQ has to be + * handled before completing io request. + * + * NEED_GET_DATA: only used for write requests to set io addr and copy data + * When NEED_GET_DATA is set, ublksrv has to issue UBLK_IO_NEED_GET_DATA + * command after ublk driver returns UBLK_IO_RES_NEED_GET_DATA. + * + * It is only used if ublksrv set UBLK_F_NEED_GET_DATA flag + * while starting a ublk device. + */ + +/* + * Legacy IO command definition, don't use in new application, and don't + * add new such definition any more + */ +#define UBLK_IO_FETCH_REQ 0x20 +#define UBLK_IO_COMMIT_AND_FETCH_REQ 0x21 +#define UBLK_IO_NEED_GET_DATA 0x22 + +/* Any new IO command should encode by __IOWR() */ +#define UBLK_U_IO_FETCH_REQ \ + _IOWR('u', UBLK_IO_FETCH_REQ, struct ublksrv_io_cmd) +#define UBLK_U_IO_COMMIT_AND_FETCH_REQ \ + _IOWR('u', UBLK_IO_COMMIT_AND_FETCH_REQ, struct ublksrv_io_cmd) +#define UBLK_U_IO_NEED_GET_DATA \ + _IOWR('u', UBLK_IO_NEED_GET_DATA, struct ublksrv_io_cmd) +#define UBLK_U_IO_REGISTER_IO_BUF \ + _IOWR('u', 0x23, struct ublksrv_io_cmd) +#define UBLK_U_IO_UNREGISTER_IO_BUF \ + _IOWR('u', 0x24, struct ublksrv_io_cmd) + +/* only ABORT means that no re-fetch */ +#define UBLK_IO_RES_OK 0 +#define UBLK_IO_RES_NEED_GET_DATA 1 +#define UBLK_IO_RES_ABORT (-ENODEV) + +#define UBLKSRV_CMD_BUF_OFFSET 0 +#define UBLKSRV_IO_BUF_OFFSET 0x80000000 + +/* tag bit is 16bit, so far limit at most 4096 IOs for each queue */ +#define UBLK_MAX_QUEUE_DEPTH 4096 + +/* single IO buffer max size is 32MB */ +#define UBLK_IO_BUF_OFF 0 +#define UBLK_IO_BUF_BITS 25 +#define UBLK_IO_BUF_BITS_MASK ((1ULL << UBLK_IO_BUF_BITS) - 1) + +/* so at most 64K IOs for each queue */ +#define UBLK_TAG_OFF UBLK_IO_BUF_BITS +#define UBLK_TAG_BITS 16 +#define UBLK_TAG_BITS_MASK ((1ULL << UBLK_TAG_BITS) - 1) + +/* max 4096 queues */ +#define UBLK_QID_OFF (UBLK_TAG_OFF + UBLK_TAG_BITS) +#define UBLK_QID_BITS 12 +#define UBLK_QID_BITS_MASK ((1ULL << UBLK_QID_BITS) - 1) + +#define UBLK_MAX_NR_QUEUES (1U << UBLK_QID_BITS) + +#define UBLKSRV_IO_BUF_TOTAL_BITS (UBLK_QID_OFF + UBLK_QID_BITS) +#define UBLKSRV_IO_BUF_TOTAL_SIZE (1ULL << UBLKSRV_IO_BUF_TOTAL_BITS) + +/* + * zero copy requires 4k block size, and can remap ublk driver's io + * request into ublksrv's vm space + */ +#define UBLK_F_SUPPORT_ZERO_COPY (1ULL << 0) + +/* + * Force to complete io cmd via io_uring_cmd_complete_in_task so that + * performance comparison is done easily with using task_work_add + */ +#define UBLK_F_URING_CMD_COMP_IN_TASK (1ULL << 1) + +/* + * User should issue io cmd again for write requests to + * set io buffer address and copy data from bio vectors + * to the userspace io buffer. + * + * In this mode, task_work is not used. + */ +#define UBLK_F_NEED_GET_DATA (1UL << 2) + +/* + * - Block devices are recoverable if ublk server exits and restarts + * - Outstanding I/O when ublk server exits is met with errors + * - I/O issued while there is no ublk server queues + */ +#define UBLK_F_USER_RECOVERY (1UL << 3) + +/* + * - Block devices are recoverable if ublk server exits and restarts + * - Outstanding I/O when ublk server exits is reissued + * - I/O issued while there is no ublk server queues + */ +#define UBLK_F_USER_RECOVERY_REISSUE (1UL << 4) + +/* + * Unprivileged user can create /dev/ublkcN and /dev/ublkbN. + * + * /dev/ublk-control needs to be available for unprivileged user, and it + * can be done via udev rule to make all control commands available to + * unprivileged user. Except for the command of UBLK_CMD_ADD_DEV, all + * other commands are only allowed for the owner of the specified device. + * + * When userspace sends UBLK_CMD_ADD_DEV, the device pair's owner_uid and + * owner_gid are stored to ublksrv_ctrl_dev_info by kernel, so far only + * the current user's uid/gid is stored, that said owner of the created + * device is always the current user. + * + * We still need udev rule to apply OWNER/GROUP with the stored owner_uid + * and owner_gid. + * + * Then ublk server can be run as unprivileged user, and /dev/ublkbN can + * be accessed and managed by its owner represented by owner_uid/owner_gid. + */ +#define UBLK_F_UNPRIVILEGED_DEV (1UL << 5) + +/* use ioctl encoding for uring command */ +#define UBLK_F_CMD_IOCTL_ENCODE (1UL << 6) + +/* + * Copy between request and user buffer by pread()/pwrite() + * + * Not available for UBLK_F_UNPRIVILEGED_DEV, otherwise userspace may + * deceive us by not filling request buffer, then kernel uninitialized + * data may be leaked. + */ +#define UBLK_F_USER_COPY (1UL << 7) + +/* + * User space sets this flag when setting up the device to request zoned storage support. Kernel may + * deny the request by returning an error. + */ +#define UBLK_F_ZONED (1ULL << 8) + +/* + * - Block devices are recoverable if ublk server exits and restarts + * - Outstanding I/O when ublk server exits is met with errors + * - I/O issued while there is no ublk server is met with errors + */ +#define UBLK_F_USER_RECOVERY_FAIL_IO (1ULL << 9) + +/* + * Resizing a block device is possible with UBLK_U_CMD_UPDATE_SIZE + * New size is passed in cmd->data[0] and is in units of sectors + */ +#define UBLK_F_UPDATE_SIZE (1ULL << 10) + +/* + * request buffer is registered automatically to uring_cmd's io_uring + * context before delivering this io command to ublk server, meantime + * it is un-registered automatically when completing this io command. + * + * For using this feature: + * + * - ublk server has to create sparse buffer table on the same `io_ring_ctx` + * for issuing `UBLK_IO_FETCH_REQ` and `UBLK_IO_COMMIT_AND_FETCH_REQ`. + * If uring_cmd isn't issued on same `io_ring_ctx`, it is ublk server's + * responsibility to unregister the buffer by issuing `IO_UNREGISTER_IO_BUF` + * manually, otherwise this ublk request won't complete. + * + * - ublk server passes auto buf register data via uring_cmd's sqe->addr, + * `struct ublk_auto_buf_reg` is populated from sqe->addr, please see + * the definition of ublk_sqe_addr_to_auto_buf_reg() + * + * - pass buffer index from `ublk_auto_buf_reg.index` + * + * - all reserved fields in `ublk_auto_buf_reg` need to be zeroed + * + * - pass flags from `ublk_auto_buf_reg.flags` if needed + * + * This way avoids extra cost from two uring_cmd, but also simplifies backend + * implementation, such as, the dependency on IO_REGISTER_IO_BUF and + * IO_UNREGISTER_IO_BUF becomes not necessary. + * + * If wrong data or flags are provided, both IO_FETCH_REQ and + * IO_COMMIT_AND_FETCH_REQ are failed, for the latter, the ublk IO request + * won't be completed until new IO_COMMIT_AND_FETCH_REQ command is issued + * successfully + */ +#define UBLK_F_AUTO_BUF_REG (1ULL << 11) + +/* + * Control command `UBLK_U_CMD_QUIESCE_DEV` is added for quiescing device, + * which state can be transitioned to `UBLK_S_DEV_QUIESCED` or + * `UBLK_S_DEV_FAIL_IO` finally, and it needs ublk server cooperation for + * handling `UBLK_IO_RES_ABORT` correctly. + * + * Typical use case is for supporting to upgrade ublk server application, + * meantime keep ublk block device persistent during the period. + * + * This feature is only available when UBLK_F_USER_RECOVERY is enabled. + * + * Note, this command returns -EBUSY in case that all IO commands are being + * handled by ublk server and not completed in specified time period which + * is passed from the control command parameter. + */ +#define UBLK_F_QUIESCE (1ULL << 12) + +/* + * If this feature is set, ublk_drv supports each (qid,tag) pair having + * its own independent daemon task that is responsible for handling it. + * If it is not set, daemons are per-queue instead, so for two pairs + * (qid1,tag1) and (qid2,tag2), if qid1 == qid2, then the same task must + * be responsible for handling (qid1,tag1) and (qid2,tag2). + */ +#define UBLK_F_PER_IO_DAEMON (1ULL << 13) + +/* device state */ +#define UBLK_S_DEV_DEAD 0 +#define UBLK_S_DEV_LIVE 1 +#define UBLK_S_DEV_QUIESCED 2 +#define UBLK_S_DEV_FAIL_IO 3 + +/* shipped via sqe->cmd of io_uring command */ +struct ublksrv_ctrl_cmd { + /* sent to which device, must be valid */ + __u32 dev_id; + + /* sent to which queue, must be -1 if the cmd isn't for queue */ + __u16 queue_id; + /* + * cmd specific buffer, can be IN or OUT. + */ + __u16 len; + __u64 addr; + + /* inline data */ + __u64 data[1]; + + /* + * Used for UBLK_F_UNPRIVILEGED_DEV and UBLK_CMD_GET_DEV_INFO2 + * only, include null char + */ + __u16 dev_path_len; + __u16 pad; + __u32 reserved; +}; + +struct ublksrv_ctrl_dev_info { + __u16 nr_hw_queues; + __u16 queue_depth; + __u16 state; + __u16 pad0; + + __u32 max_io_buf_bytes; + __u32 dev_id; + + __s32 ublksrv_pid; + __u32 pad1; + + __u64 flags; + + /* For ublksrv internal use, invisible to ublk driver */ + __u64 ublksrv_flags; + + __u32 owner_uid; /* store by kernel */ + __u32 owner_gid; /* store by kernel */ + __u64 reserved1; + __u64 reserved2; +}; + +#define UBLK_IO_OP_READ 0 +#define UBLK_IO_OP_WRITE 1 +#define UBLK_IO_OP_FLUSH 2 +#define UBLK_IO_OP_DISCARD 3 +#define UBLK_IO_OP_WRITE_SAME 4 +#define UBLK_IO_OP_WRITE_ZEROES 5 +#define UBLK_IO_OP_ZONE_OPEN 10 +#define UBLK_IO_OP_ZONE_CLOSE 11 +#define UBLK_IO_OP_ZONE_FINISH 12 +#define UBLK_IO_OP_ZONE_APPEND 13 +#define UBLK_IO_OP_ZONE_RESET_ALL 14 +#define UBLK_IO_OP_ZONE_RESET 15 +/* + * Construct a zone report. The report request is carried in `struct + * ublksrv_io_desc`. The `start_sector` field must be the first sector of a zone + * and shall indicate the first zone of the report. The `nr_zones` shall + * indicate how many zones should be reported at most. The report shall be + * delivered as a `struct blk_zone` array. To report fewer zones than requested, + * zero the last entry of the returned array. + * + * Related definitions(blk_zone, blk_zone_cond, blk_zone_type, ...) in + * include/uapi/linux/blkzoned.h are part of ublk UAPI. + */ +#define UBLK_IO_OP_REPORT_ZONES 18 + +#define UBLK_IO_F_FAILFAST_DEV (1U << 8) +#define UBLK_IO_F_FAILFAST_TRANSPORT (1U << 9) +#define UBLK_IO_F_FAILFAST_DRIVER (1U << 10) +#define UBLK_IO_F_META (1U << 11) +#define UBLK_IO_F_FUA (1U << 13) +#define UBLK_IO_F_NOUNMAP (1U << 15) +#define UBLK_IO_F_SWAP (1U << 16) +/* + * For UBLK_F_AUTO_BUF_REG & UBLK_AUTO_BUF_REG_FALLBACK only. + * + * This flag is set if auto buffer register is failed & ublk server passes + * UBLK_AUTO_BUF_REG_FALLBACK, and ublk server need to register buffer + * manually for handling the delivered IO command if this flag is observed + * + * ublk server has to check this flag if UBLK_AUTO_BUF_REG_FALLBACK is + * passed in. + */ +#define UBLK_IO_F_NEED_REG_BUF (1U << 17) + +/* + * io cmd is described by this structure, and stored in share memory, indexed + * by request tag. + * + * The data is stored by ublk driver, and read by ublksrv after one fetch command + * returns. + */ +struct ublksrv_io_desc { + /* op: bit 0-7, flags: bit 8-31 */ + __u32 op_flags; + + union { + __u32 nr_sectors; + __u32 nr_zones; /* for UBLK_IO_OP_REPORT_ZONES */ + }; + + /* start sector for this io */ + __u64 start_sector; + + /* buffer address in ublksrv daemon vm space, from ublk driver */ + __u64 addr; +}; + +static inline __u8 ublksrv_get_op(const struct ublksrv_io_desc *iod) +{ + return iod->op_flags & 0xff; +} + +static inline __u32 ublksrv_get_flags(const struct ublksrv_io_desc *iod) +{ + return iod->op_flags >> 8; +} + +/* + * If this flag is set, fallback by completing the uring_cmd and setting + * `UBLK_IO_F_NEED_REG_BUF` in case of auto-buf-register failure; + * otherwise the client ublk request is failed silently + * + * If ublk server passes this flag, it has to check if UBLK_IO_F_NEED_REG_BUF + * is set in `ublksrv_io_desc.op_flags`. If UBLK_IO_F_NEED_REG_BUF is set, + * ublk server needs to register io buffer manually for handling IO command. + */ +#define UBLK_AUTO_BUF_REG_FALLBACK (1 << 0) +#define UBLK_AUTO_BUF_REG_F_MASK UBLK_AUTO_BUF_REG_FALLBACK + +struct ublk_auto_buf_reg { + /* index for registering the delivered request buffer */ + __u16 index; + __u8 flags; + __u8 reserved0; + + /* + * io_ring FD can be passed via the reserve field in future for + * supporting to register io buffer to external io_uring + */ + __u32 reserved1; +}; + +/* + * For UBLK_F_AUTO_BUF_REG, auto buffer register data is carried via + * uring_cmd's sqe->addr: + * + * - bit0 ~ bit15: buffer index + * - bit16 ~ bit23: flags + * - bit24 ~ bit31: reserved0 + * - bit32 ~ bit63: reserved1 + */ +static inline struct ublk_auto_buf_reg ublk_sqe_addr_to_auto_buf_reg( + __u64 sqe_addr) +{ + struct ublk_auto_buf_reg reg = { + .index = (__u16)(sqe_addr & 0xffff), + .flags = (__u8)((sqe_addr >> 16) & 0xff), + .reserved0 = (__u8)((sqe_addr >> 24) & 0xff), + .reserved1 = (__u8)(sqe_addr >> 32), + }; + + return reg; +} + +static inline __u64 +ublk_auto_buf_reg_to_sqe_addr(const struct ublk_auto_buf_reg *buf) +{ + __u64 addr = buf->index | (__u64)buf->flags << 16 | (__u64)buf->reserved0 << 24 | + (__u64)buf->reserved1 << 32; + + return addr; +} + +/* issued to ublk driver via /dev/ublkcN */ +struct ublksrv_io_cmd { + __u16 q_id; + + /* for fetch/commit which result */ + __u16 tag; + + /* io result, it is valid for COMMIT* command only */ + __s32 result; + + union { + /* + * userspace buffer address in ublksrv daemon process, valid for + * FETCH* command only + * + * `addr` should not be used when UBLK_F_USER_COPY is enabled, + * because userspace handles data copy by pread()/pwrite() over + * /dev/ublkcN. But in case of UBLK_F_ZONED, this union is + * re-used to pass back the allocated LBA for + * UBLK_IO_OP_ZONE_APPEND which actually depends on + * UBLK_F_USER_COPY + */ + __u64 addr; + __u64 zone_append_lba; + }; +}; + +struct ublk_param_basic { +#define UBLK_ATTR_READ_ONLY (1 << 0) +#define UBLK_ATTR_ROTATIONAL (1 << 1) +#define UBLK_ATTR_VOLATILE_CACHE (1 << 2) +#define UBLK_ATTR_FUA (1 << 3) + __u32 attrs; + __u8 logical_bs_shift; + __u8 physical_bs_shift; + __u8 io_opt_shift; + __u8 io_min_shift; + + __u32 max_sectors; + __u32 chunk_sectors; + + __u64 dev_sectors; + __u64 virt_boundary_mask; +}; + +struct ublk_param_discard { + __u32 discard_alignment; + + __u32 discard_granularity; + __u32 max_discard_sectors; + + __u32 max_write_zeroes_sectors; + __u16 max_discard_segments; + __u16 reserved0; +}; + +/* + * read-only, can't set via UBLK_CMD_SET_PARAMS, disk_devt is available + * after device is started + */ +struct ublk_param_devt { + __u32 char_major; + __u32 char_minor; + __u32 disk_major; + __u32 disk_minor; +}; + +struct ublk_param_zoned { + __u32 max_open_zones; + __u32 max_active_zones; + __u32 max_zone_append_sectors; + + __u32 reserved; + __u64 reserved2; + __u64 reserved3; +}; + +struct ublk_param_dma_align { + __u32 alignment; + + __u32 pad; +}; + +#define UBLK_MIN_SEGMENT_SIZE 4096 +/* + * If any one of the three segment parameter is set as 0, the behavior is + * undefined. + */ +struct ublk_param_segment { + /* + * seg_boundary_mask + 1 needs to be power_of_2(), and the sum has + * to be >= UBLK_MIN_SEGMENT_SIZE(4096) + */ + __u64 seg_boundary_mask; + + /* + * max_segment_size could be override by virt_boundary_mask, so be + * careful when setting both. + * + * max_segment_size has to be >= UBLK_MIN_SEGMENT_SIZE(4096) + */ + __u32 max_segment_size; + __u16 max_segments; + + __u16 pad; +}; + +struct ublk_params { + /* + * Total length of parameters, userspace has to set 'len' for both + * SET_PARAMS and GET_PARAMS command, and driver may update len + * if two sides use different version of 'ublk_params', same with + * 'types' fields. + */ + __u32 len; +#define UBLK_PARAM_TYPE_BASIC (1 << 0) +#define UBLK_PARAM_TYPE_DISCARD (1 << 1) +#define UBLK_PARAM_TYPE_DEVT (1 << 2) +#define UBLK_PARAM_TYPE_ZONED (1 << 3) +#define UBLK_PARAM_TYPE_DMA_ALIGN (1 << 4) +#define UBLK_PARAM_TYPE_SEGMENT (1 << 5) + __u32 types; /* types of parameter included */ + + struct ublk_param_basic basic; + struct ublk_param_discard discard; + struct ublk_param_devt devt; + struct ublk_param_zoned zoned; + struct ublk_param_dma_align dma; + struct ublk_param_segment seg; +}; + +#endif diff --git a/src/util/ringloop.cpp b/src/util/ringloop.cpp index db082108..72810a73 100644 --- a/src/util/ringloop.cpp +++ b/src/util/ringloop.cpp @@ -10,10 +10,15 @@ #include "ringloop.h" -ring_loop_t::ring_loop_t(int qd, bool multithreaded) +ring_loop_t::ring_loop_t(int qd, bool multithreaded, bool sqe128) { mt = multithreaded; - int ret = io_uring_queue_init(qd, &ring, 0); + io_uring_params params = {}; + if (sqe128) + { + params.flags = IORING_SETUP_SQE128; + } + int ret = io_uring_queue_init_params(qd, &ring, ¶ms); if (ret < 0) { throw std::runtime_error(std::string("io_uring_queue_init: ") + strerror(-ret)); @@ -179,7 +184,8 @@ unsigned ring_loop_t::save() void ring_loop_t::restore(unsigned sqe_tail) { assert(ring.sq.sqe_tail >= sqe_tail); - for (unsigned i = sqe_tail; i < ring.sq.sqe_tail; i++) + unsigned inc = (1 << io_uring_sqe_shift(&ring)); + for (unsigned i = sqe_tail; i < ring.sq.sqe_tail; i += inc) { free_ring_data[free_ring_data_ptr++] = ((ring_data_t*)ring.sq.sqes[i & *ring.sq.kring_mask].user_data) - ring_datas; } @@ -191,7 +197,7 @@ int ring_loop_t::sqes_left() struct io_uring_sq *sq = &ring.sq; unsigned int head = io_uring_smp_load_acquire(sq->khead); unsigned int next = sq->sqe_tail + 1; - int left = *sq->kring_entries - (next - head); + int left = (*sq->kring_entries - (next - head)) >> io_uring_sqe_shift(&ring); if (left > free_ring_data_ptr) { // return min(sqes left, ring_datas left) diff --git a/src/util/ringloop.h b/src/util/ringloop.h index 0990d729..e40c9033 100644 --- a/src/util/ringloop.h +++ b/src/util/ringloop.h @@ -47,7 +47,7 @@ class __attribute__((visibility("default"))) ring_loop_t int ring_eventfd = -1; bool support_zc = false; public: - ring_loop_t(int qd, bool multithreaded = false); + ring_loop_t(int qd, bool multithreaded = false, bool sqe128 = false); ~ring_loop_t(); void register_consumer(ring_consumer_t *consumer); void unregister_consumer(ring_consumer_t *consumer);