diff --git a/docs/usage/disk.en.md b/docs/usage/disk.en.md index 076d9bc8..5fc23ebc 100644 --- a/docs/usage/disk.en.md +++ b/docs/usage/disk.en.md @@ -51,12 +51,16 @@ Options (automatic mode): --osd_per_disk Create OSDs on each disk (default 1) --hybrid - Prepare hybrid (HDD+SSD) OSDs using provided devices. SSDs will be used for - journals and metadata, HDDs will be used for data. Partitions for journals and - metadata will be created automatically. Whether disks are SSD or HDD is decided - by the `/sys/block/.../queue/rotational` flag. In hybrid mode, default object - size is 1 MB instead of 128 KB, default journal size is 1 GB instead of 32 MB, - and throttle_small_writes is enabled by default. + Prepare hybrid (HDD+SSD, NVMe+SATA or etc) OSDs using provided devices. By default, + any passed SSDs will be used for journals and metadata, HDDs will be used for data, + but you can override this behaviour with --fast-devices option. Journal and metadata + partitions will be created automatically. In the default mode, SSD and HDD disks + are distinguished by the `/sys/block/.../queue/rotational` flag. When HDDs are used + for data in hybrid mode, default block_size is 1 MB instead of 128 KB, default journal + size is 1 GB instead of 32 MB, and throttle_small_writes is enabled by default. +--fast-devices /dev/nvmeX,/dev/nvmeY + In --hybrid mode, use these devices for journal and metadata instead of auto-detecting + and extracting them from the main [devices...] list. --disable_data_fsync auto Disable data device cache and fsync (1/yes/true = on, default auto) --disable_meta_fsync auto diff --git a/docs/usage/disk.ru.md b/docs/usage/disk.ru.md index a2170a3b..acaab595 100644 --- a/docs/usage/disk.ru.md +++ b/docs/usage/disk.ru.md @@ -51,12 +51,17 @@ vitastor-disk - инструмент командной строки для уп --osd_per_disk Создавать по несколько () OSD на каждом диске (по умолчанию 1) --hybrid - Инициализировать гибридные (HDD+SSD) OSD на указанных дисках. SSD будут - использованы для журналов и метаданных, а HDD - для данных. Разделы для журналов - и метаданных будут созданы автоматически. Является ли диск SSD или HDD, определяется - по флагу `/sys/block/.../queue/rotational`. В гибридном режиме по умолчанию - используется размер объекта 1 МБ вместо 128 КБ, размер журнала 1 ГБ вместо 32 МБ - и включённый throttle_small_writes. + Инициализировать гибридные (HDD+SSD, NVMe+SATA и т.п.) OSD на указанных дисках. + По умолчанию, SSD будут использованы для журналов и метаданных, а HDD - для данных, + но вы можете поменять это поведение опцией --fast-devices. Разделы для журналов + и метаданных будут созданы автоматически. В режиме по умолчанию SSD и HDD-диски + различаются по флагу `/sys/block/.../queue/rotational`. Когда в гибридном режиме + для данных используются HDD, по умолчанию размер блока устанавливается 1 МБ вместо + 128 КБ, размер журнала 1 ГБ вместо 32 МБ, и throttle_small_writes включается по + умолчанию. +--fast-devices /dev/nvmeX,/dev/nvmeY + Использовать данные диски для журналов и метаданных в гибридном режиме вместо их + автоопределения и извлечения из основного списка [devices...]. --disable_data_fsync auto Отключать кэш и fsync-и для устройств данных. (1/yes/true = да, по умолчанию автоопределение) --disable_meta_fsync auto diff --git a/src/disk_tool/disk_tool.cpp b/src/disk_tool/disk_tool.cpp index 295f0d16..586bd61b 100644 --- a/src/disk_tool/disk_tool.cpp +++ b/src/disk_tool/disk_tool.cpp @@ -27,12 +27,16 @@ static const char *help_text = " --osd_per_disk \n" " Create OSDs on each disk (default 1)\n" " --hybrid\n" - " Prepare hybrid (HDD+SSD) OSDs using provided devices. SSDs will be used for\n" - " journals and metadata, HDDs will be used for data. Partitions for journals and\n" - " metadata will be created automatically. Whether disks are SSD or HDD is decided\n" - " by the `/sys/block/.../queue/rotational` flag. In hybrid mode, default object\n" - " size is 1 MB instead of 128 KB, default journal size is 1 GB instead of 32 MB,\n" - " and throttle_small_writes is enabled by default.\n" + " Prepare hybrid (HDD+SSD, NVMe+SATA or etc) OSDs using provided devices. By default,\n" + " any passed SSDs will be used for journals and metadata, HDDs will be used for data,\n" + " but you can override this behaviour with --fast-devices option. Journal and metadata\n" + " partitions will be created automatically. In the default mode, SSD and HDD disks\n" + " are distinguished by the `/sys/block/.../queue/rotational` flag. When HDDs are used\n" + " for data in hybrid mode, default block_size is 1 MB instead of 128 KB, default journal\n" + " size is 1 GB instead of 32 MB, and throttle_small_writes is enabled by default.\n" + " --fast-devices /dev/nvmeX,/dev/nvmeY\n" + " In --hybrid mode, use these devices for journal and metadata instead of auto-detecting\n" + " and extracting them from the main [devices...] list.\n" " --disable_data_fsync auto\n" " Disable data device cache and fsync (1/yes/true = on, default auto)\n" " --disable_meta_fsync auto\n" diff --git a/src/disk_tool/disk_tool.h b/src/disk_tool/disk_tool.h index 70167a3e..cd45412b 100644 --- a/src/disk_tool/disk_tool.h +++ b/src/disk_tool/disk_tool.h @@ -22,6 +22,7 @@ #define VITASTOR_DISK_MAX_SB_SIZE 128*1024 #define VITASTOR_PART_TYPE "e7009fac-a5a1-4d72-af72-53de13059903" #define DEFAULT_HYBRID_JOURNAL "1G" +#define DEFAULT_HYBRID_SSD_JOURNAL "128M" struct resizer_data_moving_t; diff --git a/src/disk_tool/disk_tool_prepare.cpp b/src/disk_tool/disk_tool_prepare.cpp index 06e0df61..058483c6 100644 --- a/src/disk_tool/disk_tool_prepare.cpp +++ b/src/disk_tool/disk_tool_prepare.cpp @@ -232,8 +232,14 @@ int disk_tool_t::check_existing_partition(const std::string & dev) std::vector disk_tool_t::collect_devices(const std::vector & devices) { std::vector devinfo; + std::set seen; for (auto & dev: devices) { + if (seen.find(dev) != seen.end()) + { + fprintf(stderr, "%s is specified multiple times, ignoring\n", dev.c_str()); + continue; + } // Check if the device is a whole disk if (dev.substr(0, 5) != "/dev/") { @@ -294,10 +300,6 @@ std::vector disk_tool_t::collect_devices(const std::vector< .free = !pt.is_null() ? free_from_parttable(pt) : dev_size, }); } - if (!devinfo.size()) - { - fprintf(stderr, "No suitable devices found\n"); - } return devinfo; } @@ -549,9 +551,12 @@ int disk_tool_t::prepare(std::vector devices) { if (options.find("data_device") != options.end() && options["data_device"] != "") { - if (options.find("hybrid") != options.end() || options.find("osd_per_disk") != options.end() || devices.size()) + if (options.find("hybrid") != options.end() || + options.find("fast_devices") != options.end() || + options.find("osd_per_disk") != options.end() || + devices.size()) { - fprintf(stderr, "Device list (positional arguments), --osd_per_disk and --hybrid are incompatible with --data_device\n"); + fprintf(stderr, "Device list (positional arguments), --osd_per_disk, --hybrid and --fast-devices are incompatible with --data_device\n"); return 1; } return prepare_one(options, options.find("hdd") != options.end() ? 1 : 0); @@ -568,8 +573,10 @@ int disk_tool_t::prepare(std::vector devices) auto devinfo = collect_devices(devices); if (!devinfo.size()) { + fprintf(stderr, "No suitable devices found\n"); return 1; } + bool explicit_fast = options.find("fast_devices") != options.end(); uint64_t osd_per_disk = stoull_full(options["osd_per_disk"]); if (!osd_per_disk) osd_per_disk = 1; @@ -588,21 +595,55 @@ int disk_tool_t::prepare(std::vector devices) if (options.find("disable_meta_fsync") == options.end()) options["disable_meta_fsync"] = "auto"; options["disable_journal_fsync"] = options["disable_meta_fsync"]; - for (auto & dev: devinfo) - if (!dev.is_hdd) - ssds.push_back(dev); - if (!ssds.size()) + if (explicit_fast) { - fprintf(stderr, "No SSDs found\n"); - return 1; + auto fast = explode(",", options["fast_devices"], true); + ssds = collect_devices(fast); + if (!ssds.size()) + { + fprintf(stderr, "No fast devices found\n"); + return 1; + } + if (options["journal_size"] == "") + { + auto auto_journal_size = DEFAULT_HYBRID_SSD_JOURNAL; + for (auto & dev: devinfo) + { + if (dev.is_hdd) + { + auto_journal_size = DEFAULT_HYBRID_JOURNAL; + break; + } + } + options["journal_size"] = auto_journal_size; + } } - else if (ssds.size() == devinfo.size()) + else { - fprintf(stderr, "No HDDs found\n"); - return 1; + std::vector hdds; + for (auto & dev: devinfo) + { + if (!dev.is_hdd) + ssds.push_back(dev); + else + hdds.push_back(dev); + } + if (!ssds.size()) + { + fprintf(stderr, "No SSDs found\n"); + return 1; + } + if (!hdds.size()) + { + fprintf(stderr, "No HDDs found\n"); + return 1; + } + devinfo = hdds; + if (options["journal_size"] == "") + { + options["journal_size"] = DEFAULT_HYBRID_JOURNAL; + } } - if (options["journal_size"] == "") - options["journal_size"] = DEFAULT_HYBRID_JOURNAL; } else { @@ -612,31 +653,28 @@ int disk_tool_t::prepare(std::vector devices) auto journal_size = options["journal_size"]; for (auto & dev: devinfo) { - if (!hybrid || dev.is_hdd) + // Select new partitions and create an OSD on each of them + for (const auto & uuid: get_new_data_parts(dev, osd_per_disk, max_other_percent)) { - // Select new partitions and create an OSD on each of them - for (const auto & uuid: get_new_data_parts(dev, osd_per_disk, max_other_percent)) + options["force"] = true; + options["data_device"] = "/dev/disk/by-partuuid/"+strtolower(uuid); + if (hybrid) { - options["force"] = true; - options["data_device"] = "/dev/disk/by-partuuid/"+strtolower(uuid); - if (hybrid) + // Select/create journal and metadata partitions + int r = get_meta_partition(ssds, options); + if (r != 0) { - // Select/create journal and metadata partitions - int r = get_meta_partition(ssds, options); - if (r != 0) - { - return 1; - } - options.erase("journal_size"); - } - // Treat all disks as SSDs if not in the hybrid mode - prepare_one(options, dev.is_hdd ? 1 : 0); - if (hybrid) - { - options["journal_size"] = journal_size; - options.erase("journal_device"); - options.erase("meta_device"); + return 1; } + options.erase("journal_size"); + } + // Treat all disks as SSDs if not in the hybrid mode + prepare_one(options, dev.is_hdd ? 1 : 0); + if (hybrid) + { + options["journal_size"] = journal_size; + options.erase("journal_device"); + options.erase("meta_device"); } } }