Support custom hybrid OSD creation (--hybrid --fast-devices /dev/xxx,/dev/yyy)

docs/usage/disk.en.md

@@ -51,12 +51,16 @@ Options (automatic mode):
 --osd_per_disk <N>
   Create <N> OSDs on each disk (default 1)
 --hybrid
-  Prepare hybrid (HDD+SSD) OSDs using provided devices. SSDs will be used for
+  Prepare hybrid (HDD+SSD, NVMe+SATA or etc) OSDs using provided devices. By default,
-  journals and metadata, HDDs will be used for data. Partitions for journals and
+  any passed SSDs will be used for journals and metadata, HDDs will be used for data,
-  metadata will be created automatically. Whether disks are SSD or HDD is decided
+  but you can override this behaviour with --fast-devices option. Journal and metadata
-  by the `/sys/block/.../queue/rotational` flag. In hybrid mode, default object
+  partitions will be created automatically. In the default mode, SSD and HDD disks
-  size is 1 MB instead of 128 KB, default journal size is 1 GB instead of 32 MB,
+  are distinguished by the `/sys/block/.../queue/rotational` flag. When HDDs are used
-  and throttle_small_writes is enabled by default.
+  for data in hybrid mode, default block_size is 1 MB instead of 128 KB, default journal
+  size is 1 GB instead of 32 MB, and throttle_small_writes is enabled by default.
+--fast-devices /dev/nvmeX,/dev/nvmeY
+  In --hybrid mode, use these devices for journal and metadata instead of auto-detecting
+  and extracting them from the main [devices...] list.
 --disable_data_fsync auto
   Disable data device cache and fsync (1/yes/true = on, default auto)
 --disable_meta_fsync auto

docs/usage/disk.ru.md

@@ -51,12 +51,17 @@ vitastor-disk - инструмент командной строки для уп
 --osd_per_disk <N>
   Создавать по несколько (<N>) OSD на каждом диске (по умолчанию 1)
 --hybrid
-  Инициализировать гибридные (HDD+SSD) OSD на указанных дисках. SSD будут
+  Инициализировать гибридные (HDD+SSD, NVMe+SATA и т.п.) OSD на указанных дисках.
-  использованы для журналов и метаданных, а HDD - для данных. Разделы для журналов
+  По умолчанию, SSD будут использованы для журналов и метаданных, а HDD - для данных,
-  и метаданных будут созданы автоматически. Является ли диск SSD или HDD, определяется
+  но вы можете поменять это поведение опцией --fast-devices. Разделы для журналов
-  по флагу `/sys/block/.../queue/rotational`. В гибридном режиме по умолчанию
+  и метаданных будут созданы автоматически. В режиме по умолчанию SSD и HDD-диски
-  используется размер объекта 1 МБ вместо 128 КБ, размер журнала 1 ГБ вместо 32 МБ
+  различаются по флагу `/sys/block/.../queue/rotational`. Когда в гибридном режиме
-  и включённый throttle_small_writes.
+  для данных используются HDD, по умолчанию размер блока устанавливается 1 МБ вместо
+  128 КБ, размер журнала 1 ГБ вместо 32 МБ, и throttle_small_writes включается по
+  умолчанию.
+--fast-devices /dev/nvmeX,/dev/nvmeY
+  Использовать данные диски для журналов и метаданных в гибридном режиме вместо их
+  автоопределения и извлечения из основного списка [devices...].
 --disable_data_fsync auto
   Отключать кэш и fsync-и для устройств данных. (1/yes/true = да, по умолчанию автоопределение)
 --disable_meta_fsync auto

src/disk_tool/disk_tool.cpp

@@ -27,12 +27,16 @@ static const char *help_text =
     "    --osd_per_disk <N>\n"
     "      Create <N> OSDs on each disk (default 1)\n"
     "    --hybrid\n"
-    "      Prepare hybrid (HDD+SSD) OSDs using provided devices. SSDs will be used for\n"
+    "      Prepare hybrid (HDD+SSD, NVMe+SATA or etc) OSDs using provided devices. By default,\n"
-    "      journals and metadata, HDDs will be used for data. Partitions for journals and\n"
+    "      any passed SSDs will be used for journals and metadata, HDDs will be used for data,\n"
-    "      metadata will be created automatically. Whether disks are SSD or HDD is decided\n"
+    "      but you can override this behaviour with --fast-devices option. Journal and metadata\n"
-    "      by the `/sys/block/.../queue/rotational` flag. In hybrid mode, default object\n"
+    "      partitions will be created automatically. In the default mode, SSD and HDD disks\n"
-    "      size is 1 MB instead of 128 KB, default journal size is 1 GB instead of 32 MB,\n"
+    "      are distinguished by the `/sys/block/.../queue/rotational` flag. When HDDs are used\n"
-    "      and throttle_small_writes is enabled by default.\n"
+    "      for data in hybrid mode, default block_size is 1 MB instead of 128 KB, default journal\n"
+    "      size is 1 GB instead of 32 MB, and throttle_small_writes is enabled by default.\n"
+    "    --fast-devices /dev/nvmeX,/dev/nvmeY\n"
+    "      In --hybrid mode, use these devices for journal and metadata instead of auto-detecting\n"
+    "      and extracting them from the main [devices...] list.\n"
     "    --disable_data_fsync auto\n"
     "      Disable data device cache and fsync (1/yes/true = on, default auto)\n"
     "    --disable_meta_fsync auto\n"

src/disk_tool/disk_tool.h

@@ -22,6 +22,7 @@
 #define VITASTOR_DISK_MAX_SB_SIZE 128*1024
 #define VITASTOR_PART_TYPE "e7009fac-a5a1-4d72-af72-53de13059903"
 #define DEFAULT_HYBRID_JOURNAL "1G"
+#define DEFAULT_HYBRID_SSD_JOURNAL "128M"
 
 struct resizer_data_moving_t;
 

src/disk_tool/disk_tool_prepare.cpp

@@ -232,8 +232,14 @@ int disk_tool_t::check_existing_partition(const std::string & dev)
 std::vector<vitastor_dev_info_t> disk_tool_t::collect_devices(const std::vector<std::string> & devices)
 {
     std::vector<vitastor_dev_info_t> devinfo;
+    std::set<std::string> seen;
     for (auto & dev: devices)
     {
+        if (seen.find(dev) != seen.end())
+        {
+            fprintf(stderr, "%s is specified multiple times, ignoring\n", dev.c_str());
+            continue;
+        }
         // Check if the device is a whole disk
         if (dev.substr(0, 5) != "/dev/")
         {
@@ -294,10 +300,6 @@ std::vector<vitastor_dev_info_t> disk_tool_t::collect_devices(const std::vector<
             .free = !pt.is_null() ? free_from_parttable(pt) : dev_size,
         });
     }
-    if (!devinfo.size())
-    {
-        fprintf(stderr, "No suitable devices found\n");
-    }
     return devinfo;
 }
 
@@ -549,9 +551,12 @@ int disk_tool_t::prepare(std::vector<std::string> devices)
 {
     if (options.find("data_device") != options.end() && options["data_device"] != "")
     {
-        if (options.find("hybrid") != options.end() || options.find("osd_per_disk") != options.end() || devices.size())
+        if (options.find("hybrid") != options.end() ||
+            options.find("fast_devices") != options.end() ||
+            options.find("osd_per_disk") != options.end() ||
+            devices.size())
         {
-            fprintf(stderr, "Device list (positional arguments), --osd_per_disk and --hybrid are incompatible with --data_device\n");
+            fprintf(stderr, "Device list (positional arguments), --osd_per_disk, --hybrid and --fast-devices are incompatible with --data_device\n");
             return 1;
         }
         return prepare_one(options, options.find("hdd") != options.end() ? 1 : 0);
@@ -568,8 +573,10 @@ int disk_tool_t::prepare(std::vector<std::string> devices)
     auto devinfo = collect_devices(devices);
     if (!devinfo.size())
     {
+        fprintf(stderr, "No suitable devices found\n");
         return 1;
     }
+    bool explicit_fast = options.find("fast_devices") != options.end();
     uint64_t osd_per_disk = stoull_full(options["osd_per_disk"]);
     if (!osd_per_disk)
         osd_per_disk = 1;
@@ -588,21 +595,55 @@ int disk_tool_t::prepare(std::vector<std::string> devices)
         if (options.find("disable_meta_fsync") == options.end())
             options["disable_meta_fsync"] = "auto";
         options["disable_journal_fsync"] = options["disable_meta_fsync"];
-        for (auto & dev: devinfo)
+        if (explicit_fast)
-            if (!dev.is_hdd)
-                ssds.push_back(dev);
-        if (!ssds.size())
         {
-            fprintf(stderr, "No SSDs found\n");
+            auto fast = explode(",", options["fast_devices"], true);
-            return 1;
+            ssds = collect_devices(fast);
+            if (!ssds.size())
+            {
+                fprintf(stderr, "No fast devices found\n");
+                return 1;
+            }
+            if (options["journal_size"] == "")
+            {
+                auto auto_journal_size = DEFAULT_HYBRID_SSD_JOURNAL;
+                for (auto & dev: devinfo)
+                {
+                    if (dev.is_hdd)
+                    {
+                        auto_journal_size = DEFAULT_HYBRID_JOURNAL;
+                        break;
+                    }
+                }
+                options["journal_size"] = auto_journal_size;
+            }
         }
-        else if (ssds.size() == devinfo.size())
+        else
         {
-            fprintf(stderr, "No HDDs found\n");
+            std::vector<vitastor_dev_info_t> hdds;
-            return 1;
+            for (auto & dev: devinfo)
+            {
+                if (!dev.is_hdd)
+                    ssds.push_back(dev);
+                else
+                    hdds.push_back(dev);
+            }
+            if (!ssds.size())
+            {
+                fprintf(stderr, "No SSDs found\n");
+                return 1;
+            }
+            if (!hdds.size())
+            {
+                fprintf(stderr, "No HDDs found\n");
+                return 1;
+            }
+            devinfo = hdds;
+            if (options["journal_size"] == "")
+            {
+                options["journal_size"] = DEFAULT_HYBRID_JOURNAL;
+            }
         }
-        if (options["journal_size"] == "")
-            options["journal_size"] = DEFAULT_HYBRID_JOURNAL;
     }
     else
     {
@@ -612,31 +653,28 @@ int disk_tool_t::prepare(std::vector<std::string> devices)
     auto journal_size = options["journal_size"];
     for (auto & dev: devinfo)
     {
-        if (!hybrid || dev.is_hdd)
+        // Select new partitions and create an OSD on each of them
+        for (const auto & uuid: get_new_data_parts(dev, osd_per_disk, max_other_percent))
         {
-            // Select new partitions and create an OSD on each of them
+            options["force"] = true;
-            for (const auto & uuid: get_new_data_parts(dev, osd_per_disk, max_other_percent))
+            options["data_device"] = "/dev/disk/by-partuuid/"+strtolower(uuid);
+            if (hybrid)
             {
-                options["force"] = true;
+                // Select/create journal and metadata partitions
-                options["data_device"] = "/dev/disk/by-partuuid/"+strtolower(uuid);
+                int r = get_meta_partition(ssds, options);
-                if (hybrid)
+                if (r != 0)
                 {
-                    // Select/create journal and metadata partitions
+                    return 1;
-                    int r = get_meta_partition(ssds, options);
-                    if (r != 0)
-                    {
-                        return 1;
-                    }
-                    options.erase("journal_size");
-                }
-                // Treat all disks as SSDs if not in the hybrid mode
-                prepare_one(options, dev.is_hdd ? 1 : 0);
-                if (hybrid)
-                {
-                    options["journal_size"] = journal_size;
-                    options.erase("journal_device");
-                    options.erase("meta_device");
                 }
+                options.erase("journal_size");
+            }
+            // Treat all disks as SSDs if not in the hybrid mode
+            prepare_one(options, dev.is_hdd ? 1 : 0);
+            if (hybrid)
+            {
+                options["journal_size"] = journal_size;
+                options.erase("journal_device");
+                options.erase("meta_device");
             }
         }
     }