thebestvibedev/tromcho.net
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support reading parameters automatically from the superblock in vitastor-disk {dump,write}-{meta,journal}

Browse Source
This commit is contained in:
Vitaliy Filippov
2024-10-07 02:21:58 +03:00
parent 30e7c2ad1e
commit c5a58c2e81
9 changed files with 133 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/client/CMakeLists.txt
+1 -1
View File
@@ -10,7 +10,7 @@ endif (IBVERBS_LIBRARIES)
add_library(vitastor_common STATIC
../util/epoll_manager.cpp etcd_state_client.cpp messenger.cpp ../util/addr_util.cpp
msgr_stop.cpp msgr_op.cpp msgr_send.cpp msgr_receive.cpp ../util/ringloop.cpp ../../json11/json11.cpp
http_client.cpp osd_ops.cpp pg_states.cpp ../util/timerfd_manager.cpp ../util/str_util.cpp ${MSGR_RDMA}
http_client.cpp osd_ops.cpp pg_states.cpp ../util/timerfd_manager.cpp ../util/str_util.cpp ../util/json_util.cpp ${MSGR_RDMA}
)
target_link_libraries(vitastor_common pthread)
target_compile_options(vitastor_common PUBLIC -fPIC)
src/disk_tool/CMakeLists.txt
+1 -1
View File
@@ -6,7 +6,7 @@ project(vitastor)
add_executable(vitastor-disk
disk_tool.cpp disk_simple_offsets.cpp
disk_tool_journal.cpp disk_tool_meta.cpp disk_tool_prepare.cpp disk_tool_resize.cpp disk_tool_udev.cpp disk_tool_utils.cpp disk_tool_upgrade.cpp
../util/crc32c.c ../util/str_util.cpp ../../json11/json11.cpp ../util/rw_blocking.cpp ../util/allocator.cpp ../util/ringloop.cpp ../blockstore/blockstore_disk.cpp
../util/crc32c.c ../util/str_util.cpp ../util/json_util.cpp ../../json11/json11.cpp ../util/rw_blocking.cpp ../util/allocator.cpp ../util/ringloop.cpp ../blockstore/blockstore_disk.cpp
)
target_link_libraries(vitastor-disk
tcmalloc_minimal
src/disk_tool/disk_tool.cpp
+66 -19
View File
@@ -143,8 +143,10 @@ static const char *help_text =
" For now, this only checks that device cache is in write-through mode if fsync is disabled.\n"
" Intended for use from startup scripts (i.e. from systemd units).\n"
"\n"
"vitastor-disk dump-journal [OPTIONS] <osd_device>\n"
"vitastor-disk dump-journal [OPTIONS] <journal_file> <journal_block_size> <offset> <size>\n"
" Dump journal in human-readable or JSON (if --json is specified) format.\n"
" Dump journal in text or JSON (if --json is specified) format.\n"
" You can specify any OSD device (data, metadata or journal), or the layout manually.\n"
" Options:\n"
" --all Scan the whole journal area for entries and dump them, even outdated ones\n"
" --json Dump journal in JSON format\n"
@@ -152,16 +154,21 @@ static const char *help_text =
" --format data Same as \"entries\", but also include small write data\n"
" --format blocks Dump as an array of journal blocks each containing array of entries\n"
"\n"
"vitastor-disk write-journal <osd_device>\n"
"vitastor-disk write-journal <journal_file> <journal_block_size> <bitmap_size> <offset> <size>\n"
" Write journal from JSON taken from standard input in the same format as produced by\n"
" `dump-journal --json --format data`.\n"
" You can specify any OSD device (data, metadata or journal), or the layout manually.\n"
"\n"
"vitastor-disk dump-meta <osd_device>\n"
"vitastor-disk dump-meta <meta_file> <meta_block_size> <offset> <size>\n"
" Dump metadata in JSON format.\n"
" You can specify any OSD device (data, metadata or journal), or the layout manually.\n"
"\n"
"vitastor-disk write-meta <osd_device>\n"
"vitastor-disk write-meta <meta_file> <offset> <size>\n"
" Write metadata from JSON taken from standard input in the same format as produced by\n"
" `dump-meta`. Intended for debugging.\n"
" Write metadata from JSON taken from standard input in the same format as produced by `dump-meta`.\n"
" You can specify any OSD device (data, metadata or journal), or the layout manually.\n"
"\n"
"vitastor-disk simple-offsets <device>\n"
" Calculate offsets for old simple&stupid (no superblock) OSD deployment. Options:\n"
@@ -249,29 +256,49 @@ int main(int argc, char *argv[])
}
if (!strcmp(cmd[0], "dump-journal"))
{
if (cmd.size() < 5)
if (cmd.size() != 2 && cmd.size() < 5)
{
print_help(help_text, aliased ? "vitastor-dump-journal" : "vitastor-disk", cmd[0], false);
return 1;
}
self.dsk.journal_device = cmd[1];
self.dsk.journal_block_size = strtoul(cmd[2], NULL, 10);
self.dsk.journal_offset = strtoull(cmd[3], NULL, 10);
self.dsk.journal_len = strtoull(cmd[4], NULL, 10);
if (cmd.size() > 2)
{
self.dsk.journal_block_size = strtoul(cmd[2], NULL, 10);
self.dsk.journal_offset = strtoull(cmd[3], NULL, 10);
self.dsk.journal_len = strtoull(cmd[4], NULL, 10);
}
else
{
// First argument is an OSD device - take metadata layout parameters from it
if (self.dump_load_check_superblock(self.dsk.journal_device))
return 1;
}
return self.dump_journal();
}
else if (!strcmp(cmd[0], "write-journal"))
{
if (cmd.size() < 6)
if (cmd.size() != 2 && cmd.size() < 6)
{
print_help(help_text, "vitastor-disk", cmd[0], false);
return 1;
}
self.new_journal_device = cmd[1];
self.dsk.journal_block_size = strtoul(cmd[2], NULL, 10);
self.dsk.clean_entry_bitmap_size = strtoul(cmd[3], NULL, 10);
self.new_journal_offset = strtoull(cmd[4], NULL, 10);
self.new_journal_len = strtoull(cmd[5], NULL, 10);
if (cmd.size() > 2)
{
self.dsk.journal_block_size = strtoul(cmd[2], NULL, 10);
self.dsk.clean_entry_bitmap_size = strtoul(cmd[3], NULL, 10);
self.new_journal_offset = strtoull(cmd[4], NULL, 10);
self.new_journal_len = strtoull(cmd[5], NULL, 10);
}
else
{
// First argument is an OSD device - take metadata layout parameters from it
if (self.dump_load_check_superblock(self.new_journal_device))
return 1;
self.new_journal_offset = self.dsk.journal_offset;
self.new_journal_len = self.dsk.journal_len;
}
std::string json_err;
json11::Json entries = json11::Json::parse(read_all_fd(0), json_err);
if (json_err != "")
@@ -296,27 +323,47 @@ int main(int argc, char *argv[])
}
else if (!strcmp(cmd[0], "dump-meta"))
{
if (cmd.size() < 5)
if (cmd.size() != 2 && cmd.size() < 5)
{
print_help(help_text, "vitastor-disk", cmd[0], false);
return 1;
}
self.dsk.meta_device = cmd[1];
self.dsk.meta_block_size = strtoul(cmd[2], NULL, 10);
self.dsk.meta_offset = strtoull(cmd[3], NULL, 10);
self.dsk.meta_len = strtoull(cmd[4], NULL, 10);
if (cmd.size() > 2)
{
self.dsk.meta_block_size = strtoul(cmd[2], NULL, 10);
self.dsk.meta_offset = strtoull(cmd[3], NULL, 10);
self.dsk.meta_len = strtoull(cmd[4], NULL, 10);
}
else
{
// First argument is an OSD device - take metadata layout parameters from it
if (self.dump_load_check_superblock(self.dsk.meta_device))
return 1;
}
return self.dump_meta();
}
else if (!strcmp(cmd[0], "write-meta"))
{
if (cmd.size() < 4)
if (cmd.size() != 2 && cmd.size() < 4)
{
print_help(help_text, "vitastor-disk", cmd[0], false);
return 1;
}
self.new_meta_device = cmd[1];
self.new_meta_offset = strtoull(cmd[2], NULL, 10);
self.new_meta_len = strtoull(cmd[3], NULL, 10);
if (cmd.size() > 2)
{
self.new_meta_offset = strtoull(cmd[2], NULL, 10);
self.new_meta_len = strtoull(cmd[3], NULL, 10);
}
else
{
// First argument is an OSD device - take metadata layout parameters from it
if (self.dump_load_check_superblock(self.new_meta_device))
return 1;
self.new_meta_offset = self.dsk.meta_offset;
self.new_meta_len = self.dsk.meta_len;
}
std::string json_err;
json11::Json meta = json11::Json::parse(read_all_fd(0), json_err);
if (json_err != "")
src/disk_tool/disk_tool.h
+2
View File
@@ -93,6 +93,8 @@ struct disk_tool_t
void dump_meta_header(blockstore_meta_header_v2_t *hdr);
void dump_meta_entry(uint64_t block_num, clean_disk_entry *entry, uint8_t *bitmap);
int dump_load_check_superblock(const std::string & device);
int write_json_journal(json11::Json entries);
int write_json_meta(json11::Json meta);
src/disk_tool/disk_tool_journal.cpp
+6
View File
@@ -517,6 +517,12 @@ int disk_tool_t::write_json_journal(json11::Json entries)
uint32_t data_csum_size = !dsk.data_csum_type ? 0 : ne->small_write.len/dsk.csum_block_size*(dsk.data_csum_type & 0xFF);
fromhexstr(rec["bitmap"].string_value(), dsk.clean_entry_bitmap_size, ((uint8_t*)ne) + sizeof(journal_entry_small_write) + data_csum_size);
fromhexstr(rec["data"].string_value(), ne->small_write.len, new_journal_data);
if (ne->small_write.len > 0 && !rec["data"].is_string())
{
fprintf(stderr, "Error: entry data is missing, please generate the dump with --json --format data\n");
free(new_journal_buf);
return 1;
}
if (dsk.data_csum_type)
fromhexstr(rec["block_csums"].string_value(), data_csum_size, ((uint8_t*)ne) + sizeof(journal_entry_small_write));
if (rec["data"].is_string())
src/disk_tool/disk_tool_meta.cpp
+26
View File
@@ -4,6 +4,7 @@
#include "disk_tool.h"
#include "rw_blocking.h"
#include "osd_id.h"
#include "json_util.h"
int disk_tool_t::process_meta(std::function<void(blockstore_meta_header_v2_t *)> hdr_fn,
std::function<void(uint64_t, clean_disk_entry*, uint8_t*)> record_fn)
@@ -149,6 +150,31 @@ int disk_tool_t::process_meta(std::function<void(blockstore_meta_header_v2_t *)>
return 0;
}
int disk_tool_t::dump_load_check_superblock(const std::string & device)
{
json11::Json sb = read_osd_superblock(device, true, false);
if (sb.is_null())
return 1;
try
{
auto cfg = json_to_string_map(sb["params"].object_items());
dsk.parse_config(cfg);
dsk.data_io = dsk.meta_io = dsk.journal_io = "cached";
dsk.open_data();
dsk.open_meta();
dsk.open_journal();
dsk.calc_lengths(true);
}
catch (std::exception & e)
{
dsk.close_all();
fprintf(stderr, "%s\n", e.what());
return 1;
}
dsk.close_all();
return 0;
}
int disk_tool_t::dump_meta()
{
int r = process_meta(
src/osd/osd.cpp
+3 -15
View File
@@ -14,19 +14,7 @@
#include "osd.h"
#include "http_client.h"
#include "str_util.h"
static blockstore_config_t json_to_bs(const json11::Json::object & config)
{
blockstore_config_t bs;
for (auto kv: config)
{
if (kv.second.is_string())
bs[kv.first] = kv.second.string_value();
else if (!kv.second.is_null())
bs[kv.first] = kv.second.dump();
}
return bs;
}
#include "json_util.h"
osd_t::osd_t(const json11::Json & config, ring_loop_t *ringloop)
{
@@ -46,7 +34,7 @@ osd_t::osd_t(const json11::Json & config, ring_loop_t *ringloop)
if (!json_is_true(this->config["disable_blockstore"]))
{
auto bs_cfg = json_to_bs(this->config);
auto bs_cfg = json_to_string_map(this->config);
this->bs = new blockstore_t(bs_cfg, ringloop, tfd);
// Wait for blockstore initialisation before actually starting OSD logic
// to prevent peering timeouts during restart with filled databases
@@ -151,7 +139,7 @@ void osd_t::parse_config(bool init)
}
if (bs)
{
auto bs_cfg = json_to_bs(config);
auto bs_cfg = json_to_string_map(config);
bs->parse_config(bs_cfg);
}
st_cli.parse_config(config);
src/util/json_util.cpp
+17
View File
@@ -0,0 +1,17 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
#include "json_util.h"
std::map<std::string, std::string> json_to_string_map(const json11::Json::object & config)
{
std::map<std::string, std::string> bs;
for (auto kv: config)
{
if (kv.second.is_string())
bs[kv.first] = kv.second.string_value();
else if (!kv.second.is_null())
bs[kv.first] = kv.second.dump();
}
return bs;
}
src/util/json_util.h
+11
View File
@@ -0,0 +1,11 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
#pragma once
#include <map>
#include <string>
#include "json11/json11.hpp"
std::map<std::string, std::string> json_to_string_map(const json11::Json::object & config);