From c5a58c2e81e03b13a4d079fe11ac014d4078f772 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sun, 6 Oct 2024 14:19:33 +0300 Subject: [PATCH] Support reading parameters automatically from the superblock in vitastor-disk {dump,write}-{meta,journal} --- src/client/CMakeLists.txt | 2 +- src/disk_tool/CMakeLists.txt | 2 +- src/disk_tool/disk_tool.cpp | 85 ++++++++++++++++++++++------- src/disk_tool/disk_tool.h | 2 + src/disk_tool/disk_tool_journal.cpp | 6 ++ src/disk_tool/disk_tool_meta.cpp | 26 +++++++++ src/osd/osd.cpp | 18 +----- src/util/json_util.cpp | 17 ++++++ src/util/json_util.h | 11 ++++ 9 files changed, 133 insertions(+), 36 deletions(-) create mode 100644 src/util/json_util.cpp create mode 100644 src/util/json_util.h diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 84b20488..174023d4 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -10,7 +10,7 @@ endif (IBVERBS_LIBRARIES) add_library(vitastor_common STATIC ../util/epoll_manager.cpp etcd_state_client.cpp messenger.cpp ../util/addr_util.cpp msgr_stop.cpp msgr_op.cpp msgr_send.cpp msgr_receive.cpp ../util/ringloop.cpp ../../json11/json11.cpp - http_client.cpp osd_ops.cpp pg_states.cpp ../util/timerfd_manager.cpp ../util/str_util.cpp ${MSGR_RDMA} + http_client.cpp osd_ops.cpp pg_states.cpp ../util/timerfd_manager.cpp ../util/str_util.cpp ../util/json_util.cpp ${MSGR_RDMA} ) target_link_libraries(vitastor_common pthread) target_compile_options(vitastor_common PUBLIC -fPIC) diff --git a/src/disk_tool/CMakeLists.txt b/src/disk_tool/CMakeLists.txt index bb1689b4..b54145e1 100644 --- a/src/disk_tool/CMakeLists.txt +++ b/src/disk_tool/CMakeLists.txt @@ -6,7 +6,7 @@ project(vitastor) add_executable(vitastor-disk disk_tool.cpp disk_simple_offsets.cpp disk_tool_journal.cpp disk_tool_meta.cpp disk_tool_prepare.cpp disk_tool_resize.cpp disk_tool_udev.cpp disk_tool_utils.cpp disk_tool_upgrade.cpp - ../util/crc32c.c ../util/str_util.cpp ../../json11/json11.cpp ../util/rw_blocking.cpp ../util/allocator.cpp ../util/ringloop.cpp ../blockstore/blockstore_disk.cpp + ../util/crc32c.c ../util/str_util.cpp ../util/json_util.cpp ../../json11/json11.cpp ../util/rw_blocking.cpp ../util/allocator.cpp ../util/ringloop.cpp ../blockstore/blockstore_disk.cpp ) target_link_libraries(vitastor-disk tcmalloc_minimal diff --git a/src/disk_tool/disk_tool.cpp b/src/disk_tool/disk_tool.cpp index 6021d97c..fb1c0c53 100644 --- a/src/disk_tool/disk_tool.cpp +++ b/src/disk_tool/disk_tool.cpp @@ -143,8 +143,10 @@ static const char *help_text = " For now, this only checks that device cache is in write-through mode if fsync is disabled.\n" " Intended for use from startup scripts (i.e. from systemd units).\n" "\n" + "vitastor-disk dump-journal [OPTIONS] \n" "vitastor-disk dump-journal [OPTIONS] \n" - " Dump journal in human-readable or JSON (if --json is specified) format.\n" + " Dump journal in text or JSON (if --json is specified) format.\n" + " You can specify any OSD device (data, metadata or journal), or the layout manually.\n" " Options:\n" " --all Scan the whole journal area for entries and dump them, even outdated ones\n" " --json Dump journal in JSON format\n" @@ -152,16 +154,21 @@ static const char *help_text = " --format data Same as \"entries\", but also include small write data\n" " --format blocks Dump as an array of journal blocks each containing array of entries\n" "\n" + "vitastor-disk write-journal \n" "vitastor-disk write-journal \n" " Write journal from JSON taken from standard input in the same format as produced by\n" " `dump-journal --json --format data`.\n" + " You can specify any OSD device (data, metadata or journal), or the layout manually.\n" "\n" + "vitastor-disk dump-meta \n" "vitastor-disk dump-meta \n" " Dump metadata in JSON format.\n" + " You can specify any OSD device (data, metadata or journal), or the layout manually.\n" "\n" + "vitastor-disk write-meta \n" "vitastor-disk write-meta \n" - " Write metadata from JSON taken from standard input in the same format as produced by\n" - " `dump-meta`. Intended for debugging.\n" + " Write metadata from JSON taken from standard input in the same format as produced by `dump-meta`.\n" + " You can specify any OSD device (data, metadata or journal), or the layout manually.\n" "\n" "vitastor-disk simple-offsets \n" " Calculate offsets for old simple&stupid (no superblock) OSD deployment. Options:\n" @@ -249,29 +256,49 @@ int main(int argc, char *argv[]) } if (!strcmp(cmd[0], "dump-journal")) { - if (cmd.size() < 5) + if (cmd.size() != 2 && cmd.size() < 5) { print_help(help_text, aliased ? "vitastor-dump-journal" : "vitastor-disk", cmd[0], false); return 1; } self.dsk.journal_device = cmd[1]; - self.dsk.journal_block_size = strtoul(cmd[2], NULL, 10); - self.dsk.journal_offset = strtoull(cmd[3], NULL, 10); - self.dsk.journal_len = strtoull(cmd[4], NULL, 10); + if (cmd.size() > 2) + { + self.dsk.journal_block_size = strtoul(cmd[2], NULL, 10); + self.dsk.journal_offset = strtoull(cmd[3], NULL, 10); + self.dsk.journal_len = strtoull(cmd[4], NULL, 10); + } + else + { + // First argument is an OSD device - take metadata layout parameters from it + if (self.dump_load_check_superblock(self.dsk.journal_device)) + return 1; + } return self.dump_journal(); } else if (!strcmp(cmd[0], "write-journal")) { - if (cmd.size() < 6) + if (cmd.size() != 2 && cmd.size() < 6) { print_help(help_text, "vitastor-disk", cmd[0], false); return 1; } self.new_journal_device = cmd[1]; - self.dsk.journal_block_size = strtoul(cmd[2], NULL, 10); - self.dsk.clean_entry_bitmap_size = strtoul(cmd[3], NULL, 10); - self.new_journal_offset = strtoull(cmd[4], NULL, 10); - self.new_journal_len = strtoull(cmd[5], NULL, 10); + if (cmd.size() > 2) + { + self.dsk.journal_block_size = strtoul(cmd[2], NULL, 10); + self.dsk.clean_entry_bitmap_size = strtoul(cmd[3], NULL, 10); + self.new_journal_offset = strtoull(cmd[4], NULL, 10); + self.new_journal_len = strtoull(cmd[5], NULL, 10); + } + else + { + // First argument is an OSD device - take metadata layout parameters from it + if (self.dump_load_check_superblock(self.new_journal_device)) + return 1; + self.new_journal_offset = self.dsk.journal_offset; + self.new_journal_len = self.dsk.journal_len; + } std::string json_err; json11::Json entries = json11::Json::parse(read_all_fd(0), json_err); if (json_err != "") @@ -296,27 +323,47 @@ int main(int argc, char *argv[]) } else if (!strcmp(cmd[0], "dump-meta")) { - if (cmd.size() < 5) + if (cmd.size() != 2 && cmd.size() < 5) { print_help(help_text, "vitastor-disk", cmd[0], false); return 1; } self.dsk.meta_device = cmd[1]; - self.dsk.meta_block_size = strtoul(cmd[2], NULL, 10); - self.dsk.meta_offset = strtoull(cmd[3], NULL, 10); - self.dsk.meta_len = strtoull(cmd[4], NULL, 10); + if (cmd.size() > 2) + { + self.dsk.meta_block_size = strtoul(cmd[2], NULL, 10); + self.dsk.meta_offset = strtoull(cmd[3], NULL, 10); + self.dsk.meta_len = strtoull(cmd[4], NULL, 10); + } + else + { + // First argument is an OSD device - take metadata layout parameters from it + if (self.dump_load_check_superblock(self.dsk.meta_device)) + return 1; + } return self.dump_meta(); } else if (!strcmp(cmd[0], "write-meta")) { - if (cmd.size() < 4) + if (cmd.size() != 2 && cmd.size() < 4) { print_help(help_text, "vitastor-disk", cmd[0], false); return 1; } self.new_meta_device = cmd[1]; - self.new_meta_offset = strtoull(cmd[2], NULL, 10); - self.new_meta_len = strtoull(cmd[3], NULL, 10); + if (cmd.size() > 2) + { + self.new_meta_offset = strtoull(cmd[2], NULL, 10); + self.new_meta_len = strtoull(cmd[3], NULL, 10); + } + else + { + // First argument is an OSD device - take metadata layout parameters from it + if (self.dump_load_check_superblock(self.new_meta_device)) + return 1; + self.new_meta_offset = self.dsk.meta_offset; + self.new_meta_len = self.dsk.meta_len; + } std::string json_err; json11::Json meta = json11::Json::parse(read_all_fd(0), json_err); if (json_err != "") diff --git a/src/disk_tool/disk_tool.h b/src/disk_tool/disk_tool.h index 477aed3c..80549054 100644 --- a/src/disk_tool/disk_tool.h +++ b/src/disk_tool/disk_tool.h @@ -93,6 +93,8 @@ struct disk_tool_t void dump_meta_header(blockstore_meta_header_v2_t *hdr); void dump_meta_entry(uint64_t block_num, clean_disk_entry *entry, uint8_t *bitmap); + int dump_load_check_superblock(const std::string & device); + int write_json_journal(json11::Json entries); int write_json_meta(json11::Json meta); diff --git a/src/disk_tool/disk_tool_journal.cpp b/src/disk_tool/disk_tool_journal.cpp index 340e49c1..0b4d608b 100644 --- a/src/disk_tool/disk_tool_journal.cpp +++ b/src/disk_tool/disk_tool_journal.cpp @@ -517,6 +517,12 @@ int disk_tool_t::write_json_journal(json11::Json entries) uint32_t data_csum_size = !dsk.data_csum_type ? 0 : ne->small_write.len/dsk.csum_block_size*(dsk.data_csum_type & 0xFF); fromhexstr(rec["bitmap"].string_value(), dsk.clean_entry_bitmap_size, ((uint8_t*)ne) + sizeof(journal_entry_small_write) + data_csum_size); fromhexstr(rec["data"].string_value(), ne->small_write.len, new_journal_data); + if (ne->small_write.len > 0 && !rec["data"].is_string()) + { + fprintf(stderr, "Error: entry data is missing, please generate the dump with --json --format data\n"); + free(new_journal_buf); + return 1; + } if (dsk.data_csum_type) fromhexstr(rec["block_csums"].string_value(), data_csum_size, ((uint8_t*)ne) + sizeof(journal_entry_small_write)); if (rec["data"].is_string()) diff --git a/src/disk_tool/disk_tool_meta.cpp b/src/disk_tool/disk_tool_meta.cpp index 98fa4ac6..fe9258d9 100644 --- a/src/disk_tool/disk_tool_meta.cpp +++ b/src/disk_tool/disk_tool_meta.cpp @@ -4,6 +4,7 @@ #include "disk_tool.h" #include "rw_blocking.h" #include "osd_id.h" +#include "json_util.h" int disk_tool_t::process_meta(std::function hdr_fn, std::function record_fn) @@ -149,6 +150,31 @@ int disk_tool_t::process_meta(std::function return 0; } +int disk_tool_t::dump_load_check_superblock(const std::string & device) +{ + json11::Json sb = read_osd_superblock(device, true, false); + if (sb.is_null()) + return 1; + try + { + auto cfg = json_to_string_map(sb["params"].object_items()); + dsk.parse_config(cfg); + dsk.data_io = dsk.meta_io = dsk.journal_io = "cached"; + dsk.open_data(); + dsk.open_meta(); + dsk.open_journal(); + dsk.calc_lengths(true); + } + catch (std::exception & e) + { + dsk.close_all(); + fprintf(stderr, "%s\n", e.what()); + return 1; + } + dsk.close_all(); + return 0; +} + int disk_tool_t::dump_meta() { int r = process_meta( diff --git a/src/osd/osd.cpp b/src/osd/osd.cpp index 75d27359..f9d2cca5 100644 --- a/src/osd/osd.cpp +++ b/src/osd/osd.cpp @@ -14,19 +14,7 @@ #include "osd.h" #include "http_client.h" #include "str_util.h" - -static blockstore_config_t json_to_bs(const json11::Json::object & config) -{ - blockstore_config_t bs; - for (auto kv: config) - { - if (kv.second.is_string()) - bs[kv.first] = kv.second.string_value(); - else if (!kv.second.is_null()) - bs[kv.first] = kv.second.dump(); - } - return bs; -} +#include "json_util.h" osd_t::osd_t(const json11::Json & config, ring_loop_t *ringloop) { @@ -46,7 +34,7 @@ osd_t::osd_t(const json11::Json & config, ring_loop_t *ringloop) if (!json_is_true(this->config["disable_blockstore"])) { - auto bs_cfg = json_to_bs(this->config); + auto bs_cfg = json_to_string_map(this->config); this->bs = new blockstore_t(bs_cfg, ringloop, tfd); // Wait for blockstore initialisation before actually starting OSD logic // to prevent peering timeouts during restart with filled databases @@ -151,7 +139,7 @@ void osd_t::parse_config(bool init) } if (bs) { - auto bs_cfg = json_to_bs(config); + auto bs_cfg = json_to_string_map(config); bs->parse_config(bs_cfg); } st_cli.parse_config(config); diff --git a/src/util/json_util.cpp b/src/util/json_util.cpp new file mode 100644 index 00000000..2ba87bd6 --- /dev/null +++ b/src/util/json_util.cpp @@ -0,0 +1,17 @@ +// Copyright (c) Vitaliy Filippov, 2019+ +// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details) + +#include "json_util.h" + +std::map json_to_string_map(const json11::Json::object & config) +{ + std::map bs; + for (auto kv: config) + { + if (kv.second.is_string()) + bs[kv.first] = kv.second.string_value(); + else if (!kv.second.is_null()) + bs[kv.first] = kv.second.dump(); + } + return bs; +} diff --git a/src/util/json_util.h b/src/util/json_util.h new file mode 100644 index 00000000..0fbe25b3 --- /dev/null +++ b/src/util/json_util.h @@ -0,0 +1,11 @@ +// Copyright (c) Vitaliy Filippov, 2019+ +// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details) + +#pragma once + +#include +#include + +#include "json11/json11.hpp" + +std::map json_to_string_map(const json11::Json::object & config);