diff --git a/src/blockstore/blockstore_disk.cpp b/src/blockstore/blockstore_disk.cpp index 9f1bdf4d..11400803 100644 --- a/src/blockstore/blockstore_disk.cpp +++ b/src/blockstore/blockstore_disk.cpp @@ -52,6 +52,8 @@ void blockstore_disk_t::parse_config(std::map & config meta_format = stoull_full(config["meta_format"]); atomic_write_size = (config.find("atomic_write_size") != config.end() ? parse_size(config["atomic_write_size"]) : 4096); + use_atomic_flag = config.find("use_atomic_flag") != config.end() && + (config["use_atomic_flag"] == "true" || config["use_atomic_flag"] == "1" || config["use_atomic_flag"] == "yes"); if (config.find("data_io") == config.end() && config.find("meta_io") == config.end() && config.find("journal_io") == config.end()) diff --git a/src/blockstore/blockstore_disk.h b/src/blockstore/blockstore_disk.h index 24398d64..91e291c5 100644 --- a/src/blockstore/blockstore_disk.h +++ b/src/blockstore/blockstore_disk.h @@ -36,6 +36,8 @@ struct blockstore_disk_t uint32_t meta_block_size = 4096; // Atomic write size of the data block device uint32_t atomic_write_size = 4096; + // Whether we should set RWF_ATOMIC on atomic writes + bool use_atomic_flag = false; // Sparse write tracking granularity. 4 KB is a good choice. Must be a multiple of disk_alignment uint32_t bitmap_granularity = 4096; // Data checksum type, BLOCKSTORE_CSUM_NONE or BLOCKSTORE_CSUM_CRC32C diff --git a/src/blockstore/blockstore_internal.h b/src/blockstore/blockstore_internal.h index 89569d43..0694587d 100644 --- a/src/blockstore/blockstore_internal.h +++ b/src/blockstore/blockstore_internal.h @@ -48,3 +48,7 @@ #define COPY_BUF_COALESCED 0x10 #define COPY_BUF_PADDED 0x20 #define COPY_BUF_SKIP_CSUM 0x40 + +#ifndef RWF_ATOMIC +#define RWF_ATOMIC 0x40 +#endif diff --git a/src/blockstore/blockstore_write.cpp b/src/blockstore/blockstore_write.cpp index d2791de7..0df432cf 100644 --- a/src/blockstore/blockstore_write.cpp +++ b/src/blockstore/blockstore_write.cpp @@ -444,6 +444,8 @@ resume_10: data->iov = (struct iovec){ op->buf, op->len }; data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); }; io_uring_prep_writev(sqe, dsk.data_fd, &data->iov, 1, dsk.data_offset + PRIV(op)->location + op->offset); + if (dsk.use_atomic_flag) + sqe->rw_flags = RWF_ATOMIC; PRIV(op)->pending_ops++; PRIV(op)->op_state = 7; return 1; diff --git a/src/disk_tool/disk_tool.h b/src/disk_tool/disk_tool.h index 5a343641..b3a801d9 100644 --- a/src/disk_tool/disk_tool.h +++ b/src/disk_tool/disk_tool.h @@ -169,7 +169,8 @@ void disk_tool_simple_offsets(json11::Json cfg, bool json_output); uint64_t sscanf_json(const char *fmt, const json11::Json & str); void fromhexstr(const std::string & from, int bytes, uint8_t *to); -int disable_cache(std::string dev); +int disable_cache(const std::string & dev); +uint64_t get_atomic_write_size(const std::string & dev); uint64_t get_device_size(const std::string & dev, bool should_exist = false); std::string get_parent_device(std::string dev); int shell_exec(const std::vector & cmd, const std::string & in, std::string *out, std::string *err); diff --git a/src/disk_tool/disk_tool_prepare.cpp b/src/disk_tool/disk_tool_prepare.cpp index 8def1f74..abb60845 100644 --- a/src/disk_tool/disk_tool_prepare.cpp +++ b/src/disk_tool/disk_tool_prepare.cpp @@ -32,6 +32,8 @@ int disk_tool_t::prepare_one(std::map options, int is_ "discard_on_start", "min_discard_size", "discard_granularity", + "atomic_write_size", + "use_atomic_flag", }; if (options.find("force") == options.end()) { @@ -58,6 +60,18 @@ int disk_tool_t::prepare_one(std::map options, int is_ return 1; } } + if (options.find("atomic_write_size") == options.end()) + { + auto data_dev = realpath_str(options["data_device"], false); + uint64_t atomic_write_size = get_atomic_write_size(data_dev); + if (atomic_write_size > 4096) + { + fprintf(stderr, "Data device %s supports atomic writes up to %ju bytes, enabling. Enjoy faster writes!\n", + data_dev.c_str(), atomic_write_size); + options["atomic_write_size"] = std::to_string(atomic_write_size); + options["use_atomic_flag"] = "1"; + } + } for (auto dev: std::vector{"data", "meta", "journal"}) { if (options[dev+"_device"] != "" && options["disable_"+dev+"_fsync"] == "auto") diff --git a/src/disk_tool/disk_tool_udev.cpp b/src/disk_tool/disk_tool_udev.cpp index faf7b538..7491afb6 100644 --- a/src/disk_tool/disk_tool_udev.cpp +++ b/src/disk_tool/disk_tool_udev.cpp @@ -367,6 +367,19 @@ int disk_tool_t::pre_exec_osd(std::string device) { return 1; } + if (sb["params"]["atomic_write_size"].uint64_value() > 4096 && + sb["params"]["use_atomic_flag"].bool_value()) + { + uint64_t atomic_write_size = get_atomic_write_size(sb["real_data_device"].string_value()); + if (atomic_write_size < sb["params"]["atomic_write_size"].uint64_value()) + { + fprintf(stderr, "Atomic write size is set to %ju in the OSD superblock but data device %s only supports %ju." + " Did you enable IOMMU? Linux has a hardcoded max_hw_sectors_kb value for NVMe drives.\n", + sb["params"]["atomic_write_size"].uint64_value(), + sb["real_data_device"].string_value().c_str(), atomic_write_size); + return 1; + } + } if (json_is_true(sb["params"]["disable_meta_fsync"]) && sb["real_meta_device"].string_value() != "" && sb["real_meta_device"] != sb["real_data_device"] && check_disabled_cache(sb["real_meta_device"].string_value()) != 0) diff --git a/src/disk_tool/disk_tool_utils.cpp b/src/disk_tool/disk_tool_utils.cpp index 8966ae35..142c3048 100644 --- a/src/disk_tool/disk_tool_utils.cpp +++ b/src/disk_tool/disk_tool_utils.cpp @@ -56,8 +56,16 @@ static int check_queue_cache(std::string dev, std::string parent_dev) return trim(r) == "write through" ? 0 : -1; } +uint64_t get_atomic_write_size(const std::string & dev) +{ + auto parent_dev = get_parent_device(dev); + if (parent_dev == "") + return 0; + return stoull_full(trim(read_file("/sys/block/"+parent_dev.substr(5)+"/queue/atomic_write_max_bytes"))); +} + // returns 1 = warning, -1 = error, 0 = success -int disable_cache(std::string dev) +int disable_cache(const std::string & dev) { auto parent_dev = get_parent_device(dev); if (parent_dev == "")