Auto-guess atomic_write_size from /sys

src/blockstore/blockstore_disk.cpp

@@ -52,6 +52,8 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     meta_format = stoull_full(config["meta_format"]);
     atomic_write_size = (config.find("atomic_write_size") != config.end()
         ? parse_size(config["atomic_write_size"]) : 4096);
+    use_atomic_flag = config.find("use_atomic_flag") != config.end() &&
+        (config["use_atomic_flag"] == "true" || config["use_atomic_flag"] == "1" || config["use_atomic_flag"] == "yes");
     if (config.find("data_io") == config.end() &&
         config.find("meta_io") == config.end() &&
         config.find("journal_io") == config.end())

src/blockstore/blockstore_disk.h

@@ -36,6 +36,8 @@ struct blockstore_disk_t
     uint32_t meta_block_size = 4096;
     // Atomic write size of the data block device
     uint32_t atomic_write_size = 4096;
+    // Whether we should set RWF_ATOMIC on atomic writes
+    bool use_atomic_flag = false;
     // Sparse write tracking granularity. 4 KB is a good choice. Must be a multiple of disk_alignment
     uint32_t bitmap_granularity = 4096;
     // Data checksum type, BLOCKSTORE_CSUM_NONE or BLOCKSTORE_CSUM_CRC32C

src/blockstore/blockstore_internal.h

@@ -48,3 +48,7 @@
 #define COPY_BUF_COALESCED  0x10
 #define COPY_BUF_PADDED     0x20
 #define COPY_BUF_SKIP_CSUM  0x40
+
+#ifndef RWF_ATOMIC
+#define RWF_ATOMIC 0x40
+#endif

src/blockstore/blockstore_write.cpp

@@ -444,6 +444,8 @@ resume_10:
     data->iov = (struct iovec){ op->buf, op->len };
     data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
     io_uring_prep_writev(sqe, dsk.data_fd, &data->iov, 1, dsk.data_offset + PRIV(op)->location + op->offset);
+    if (dsk.use_atomic_flag)
+        sqe->rw_flags = RWF_ATOMIC;
     PRIV(op)->pending_ops++;
     PRIV(op)->op_state = 7;
     return 1;

src/disk_tool/disk_tool.h

@@ -169,7 +169,8 @@ void disk_tool_simple_offsets(json11::Json cfg, bool json_output);
 
 uint64_t sscanf_json(const char *fmt, const json11::Json & str);
 void fromhexstr(const std::string & from, int bytes, uint8_t *to);
-int disable_cache(std::string dev);
+int disable_cache(const std::string & dev);
+uint64_t get_atomic_write_size(const std::string & dev);
 uint64_t get_device_size(const std::string & dev, bool should_exist = false);
 std::string get_parent_device(std::string dev);
 int shell_exec(const std::vector<std::string> & cmd, const std::string & in, std::string *out, std::string *err);

src/disk_tool/disk_tool_prepare.cpp

@@ -32,6 +32,8 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
         "discard_on_start",
         "min_discard_size",
         "discard_granularity",
+        "atomic_write_size",
+        "use_atomic_flag",
     };
     if (options.find("force") == options.end())
     {
@@ -58,6 +60,18 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
                 return 1;
         }
     }
+    if (options.find("atomic_write_size") == options.end())
+    {
+        auto data_dev = realpath_str(options["data_device"], false);
+        uint64_t atomic_write_size = get_atomic_write_size(data_dev);
+        if (atomic_write_size > 4096)
+        {
+            fprintf(stderr, "Data device %s supports atomic writes up to %ju bytes, enabling. Enjoy faster writes!\n",
+                data_dev.c_str(), atomic_write_size);
+            options["atomic_write_size"] = std::to_string(atomic_write_size);
+            options["use_atomic_flag"] = "1";
+        }
+    }
     for (auto dev: std::vector<std::string>{"data", "meta", "journal"})
     {
         if (options[dev+"_device"] != "" && options["disable_"+dev+"_fsync"] == "auto")

src/disk_tool/disk_tool_udev.cpp

@@ -367,6 +367,19 @@ int disk_tool_t::pre_exec_osd(std::string device)
         {
             return 1;
         }
+        if (sb["params"]["atomic_write_size"].uint64_value() > 4096 &&
+            sb["params"]["use_atomic_flag"].bool_value())
+        {
+            uint64_t atomic_write_size = get_atomic_write_size(sb["real_data_device"].string_value());
+            if (atomic_write_size < sb["params"]["atomic_write_size"].uint64_value())
+            {
+                fprintf(stderr, "Atomic write size is set to %ju in the OSD superblock but data device %s only supports %ju."
+                    " Did you enable IOMMU? Linux has a hardcoded max_hw_sectors_kb value for NVMe drives.\n",
+                    sb["params"]["atomic_write_size"].uint64_value(),
+                    sb["real_data_device"].string_value().c_str(), atomic_write_size);
+                return 1;
+            }
+        }
         if (json_is_true(sb["params"]["disable_meta_fsync"]) &&
             sb["real_meta_device"].string_value() != "" && sb["real_meta_device"] != sb["real_data_device"] &&
             check_disabled_cache(sb["real_meta_device"].string_value()) != 0)

src/disk_tool/disk_tool_utils.cpp

@@ -56,8 +56,16 @@ static int check_queue_cache(std::string dev, std::string parent_dev)
     return trim(r) == "write through" ? 0 : -1;
 }
 
+uint64_t get_atomic_write_size(const std::string & dev)
+{
+    auto parent_dev = get_parent_device(dev);
+    if (parent_dev == "")
+        return 0;
+    return stoull_full(trim(read_file("/sys/block/"+parent_dev.substr(5)+"/queue/atomic_write_max_bytes")));
+}
+
 // returns 1 = warning, -1 = error, 0 = success
-int disable_cache(std::string dev)
+int disable_cache(const std::string & dev)
 {
     auto parent_dev = get_parent_device(dev);
     if (parent_dev == "")