From 1badc6ad132fc1e18dbe67046d8f9eae752e2e42 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sun, 23 Nov 2025 01:33:12 +0300 Subject: [PATCH] Remove include of blockstore_impl.h from disk_tool and osd --- src/blockstore/blockstore.cpp | 11 ++ src/blockstore/blockstore.h | 14 +-- src/blockstore/blockstore_disk.h | 5 + src/blockstore/blockstore_impl.h | 42 +------ src/blockstore/blockstore_journal.h | 141 --------------------- src/blockstore/ondisk_formats.h | 186 ++++++++++++++++++++++++++++ src/disk_tool/disk_tool.h | 5 +- src/disk_tool/disk_tool_journal.cpp | 3 + src/disk_tool/disk_tool_meta.cpp | 1 + src/disk_tool/disk_tool_resize.cpp | 1 + src/disk_tool/disk_tool_udev.cpp | 1 + src/disk_tool/disk_tool_utils.cpp | 2 + src/osd/osd.cpp | 9 +- 13 files changed, 222 insertions(+), 199 deletions(-) create mode 100644 src/blockstore/ondisk_formats.h diff --git a/src/blockstore/blockstore.cpp b/src/blockstore/blockstore.cpp index 6e4cc931..645580c3 100644 --- a/src/blockstore/blockstore.cpp +++ b/src/blockstore/blockstore.cpp @@ -58,6 +58,17 @@ void blockstore_t::dump_diagnostics() return impl->dump_diagnostics(); } +std::string blockstore_t::get_op_diag(blockstore_op_t *op) +{ + char buf[256]; + auto priv = PRIV(op); + if (priv->wait_for) + snprintf(buf, sizeof(buf), "state=%d wait=%d (detail=%ju)", priv->op_state, priv->wait_for, priv->wait_detail); + else + snprintf(buf, sizeof(buf), "state=%d", priv->op_state); + return std::string(buf); +} + uint32_t blockstore_t::get_block_size() { return impl->get_block_size(); diff --git a/src/blockstore/blockstore.h b/src/blockstore/blockstore.h index e8d46e55..f72d9326 100644 --- a/src/blockstore/blockstore.h +++ b/src/blockstore/blockstore.h @@ -17,11 +17,6 @@ #include "ringloop.h" #include "timerfd_manager.h" -// Memory alignment for direct I/O (usually 512 bytes) -#ifndef DIRECT_IO_ALIGNMENT -#define DIRECT_IO_ALIGNMENT 512 -#endif - // Default block size is 128 KB, current allowed range is 4K - 128M #define DEFAULT_DATA_BLOCK_ORDER 17 #define MIN_DATA_BLOCK_SIZE 4*1024 @@ -30,12 +25,6 @@ #define MIN_JOURNAL_SIZE 1024*1024 -// "VITAstor" -#define BLOCKSTORE_META_MAGIC_V1 0x726F747341544956l -#define BLOCKSTORE_META_FORMAT_V1 1 -#define BLOCKSTORE_META_FORMAT_V2 2 -#define BLOCKSTORE_META_FORMAT_HEAP 3 - #define BS_OP_MIN 1 #define BS_OP_READ 1 #define BS_OP_WRITE 2 @@ -226,6 +215,9 @@ public: // Print diagnostics to stdout void dump_diagnostics(); + // Get diagnostic string for an operation + std::string get_op_diag(blockstore_op_t *op); + uint32_t get_block_size(); uint64_t get_block_count(); uint64_t get_free_block_count(); diff --git a/src/blockstore/blockstore_disk.h b/src/blockstore/blockstore_disk.h index db2dc71d..a3d45b1d 100644 --- a/src/blockstore/blockstore_disk.h +++ b/src/blockstore/blockstore_disk.h @@ -8,6 +8,11 @@ #include #include +// Memory alignment for direct I/O (usually 512 bytes) +#ifndef DIRECT_IO_ALIGNMENT +#define DIRECT_IO_ALIGNMENT 512 +#endif + #define BLOCKSTORE_CSUM_NONE 0 // Lower byte of checksum type is its length #define BLOCKSTORE_CSUM_CRC32C 0x104 diff --git a/src/blockstore/blockstore_impl.h b/src/blockstore/blockstore_impl.h index e6dc69d9..465df7fc 100644 --- a/src/blockstore/blockstore_impl.h +++ b/src/blockstore/blockstore_impl.h @@ -5,6 +5,7 @@ #include "blockstore.h" #include "blockstore_disk.h" +#include "ondisk_formats.h" #include #include @@ -89,47 +90,6 @@ #include "blockstore_journal.h" -// "VITAstor" -#define BLOCKSTORE_META_MAGIC_V1 0x726F747341544956l -#define BLOCKSTORE_META_FORMAT_V1 1 -#define BLOCKSTORE_META_FORMAT_V2 2 - -// metadata header (superblock) -struct __attribute__((__packed__)) blockstore_meta_header_v1_t -{ - uint64_t zero; - uint64_t magic; - uint64_t version; - uint32_t meta_block_size; - uint32_t data_block_size; - uint32_t bitmap_granularity; -}; - -struct __attribute__((__packed__)) blockstore_meta_header_v2_t -{ - uint64_t zero; - uint64_t magic; - uint64_t version; - uint32_t meta_block_size; - uint32_t data_block_size; - uint32_t bitmap_granularity; - uint32_t data_csum_type; - uint32_t csum_block_size; - uint32_t header_csum; -}; - -// 32 bytes = 24 bytes + block bitmap (4 bytes by default) + external attributes (also bitmap, 4 bytes by default) -// per "clean" entry on disk with fixed metadata tables -struct __attribute__((__packed__)) clean_disk_entry -{ - object_id oid; - uint64_t version; - uint8_t bitmap[]; - // Two more fields come after bitmap in metadata version 2: - // uint32_t data_csum[]; - // uint32_t entry_csum; -}; - // 32 = 16 + 16 bytes per "clean" entry in memory (object_id => clean_entry) struct __attribute__((__packed__)) clean_entry { diff --git a/src/blockstore/blockstore_journal.h b/src/blockstore/blockstore_journal.h index ff93d1d3..5929c1c3 100644 --- a/src/blockstore/blockstore_journal.h +++ b/src/blockstore/blockstore_journal.h @@ -3,147 +3,6 @@ #pragma once -#include "crc32c.h" -#include - -#define JOURNAL_MAGIC 0x4A33 -#define JOURNAL_VERSION_V1 1 -#define JOURNAL_VERSION_V2 2 -#define JOURNAL_BUFFER_SIZE 4*1024*1024 -#define JOURNAL_ENTRY_HEADER_SIZE 16 - -// Journal entries -// Journal entries are linked to each other by their crc32 value -// The journal is almost a blockchain, because object versions constantly increase -#define JE_MIN 0x01 -#define JE_START 0x01 -#define JE_SMALL_WRITE 0x02 -#define JE_BIG_WRITE 0x03 -#define JE_STABLE 0x04 -#define JE_DELETE 0x05 -#define JE_ROLLBACK 0x06 -#define JE_SMALL_WRITE_INSTANT 0x07 -#define JE_BIG_WRITE_INSTANT 0x08 -#define JE_MAX 0x08 - -// crc32c comes first to ease calculation -struct __attribute__((__packed__)) journal_entry_start -{ - uint32_t crc32; - uint16_t magic; - uint16_t type; - uint32_t size; - uint32_t reserved; - uint64_t journal_start; - uint64_t version; - uint32_t data_csum_type; - uint32_t csum_block_size; -}; -#define JE_START_V0_SIZE 24 -#define JE_START_V1_SIZE 32 -#define JE_START_V2_SIZE 40 - -struct __attribute__((__packed__)) journal_entry_small_write -{ - uint32_t crc32; - uint16_t magic; - uint16_t type; - uint32_t size; - uint32_t crc32_prev; - object_id oid; - uint64_t version; - uint32_t offset; - uint32_t len; - // small_write entries contain bytes of data which is stored in next sectors - // data_offset is its offset within journal - uint64_t data_offset; - uint32_t crc32_data; // zero when data_csum_type != 0 - // small_write and big_write entries are followed by the "external" bitmap - // its size is dynamic and included in journal entry's field - uint8_t bitmap[]; - // and then data checksums if data_csum_type != 0 - // uint32_t data_crc32c[]; -}; - -struct __attribute__((__packed__)) journal_entry_big_write -{ - uint32_t crc32; - uint16_t magic; - uint16_t type; - uint32_t size; - uint32_t crc32_prev; - object_id oid; - uint64_t version; - uint32_t offset; - uint32_t len; - uint64_t location; - // small_write and big_write entries are followed by the "external" bitmap - // its size is dynamic and included in journal entry's field - uint8_t bitmap[]; - // and then data checksums if data_csum_type != 0 - // uint32_t data_crc32c[]; -}; - -struct __attribute__((__packed__)) journal_entry_stable -{ - uint32_t crc32; - uint16_t magic; - uint16_t type; - uint32_t size; - uint32_t crc32_prev; - object_id oid; - uint64_t version; -}; - -struct __attribute__((__packed__)) journal_entry_rollback -{ - uint32_t crc32; - uint16_t magic; - uint16_t type; - uint32_t size; - uint32_t crc32_prev; - object_id oid; - uint64_t version; -}; - -struct __attribute__((__packed__)) journal_entry_del -{ - uint32_t crc32; - uint16_t magic; - uint16_t type; - uint32_t size; - uint32_t crc32_prev; - object_id oid; - uint64_t version; -}; - -struct __attribute__((__packed__)) journal_entry -{ - union - { - struct __attribute__((__packed__)) - { - uint32_t crc32; - uint16_t magic; - uint16_t type; - uint32_t size; - uint32_t crc32_prev; - }; - journal_entry_start start; - journal_entry_small_write small_write; - journal_entry_big_write big_write; - journal_entry_stable stable; - journal_entry_rollback rollback; - journal_entry_del del; - }; -}; - -inline uint32_t je_crc32(journal_entry *je) -{ - // 0x48674bc7 = crc32(4 zero bytes) - return crc32c(0x48674bc7, ((uint8_t*)je)+4, je->size-4); -} - struct journal_sector_info_t { uint64_t offset; diff --git a/src/blockstore/ondisk_formats.h b/src/blockstore/ondisk_formats.h new file mode 100644 index 00000000..9b85127c --- /dev/null +++ b/src/blockstore/ondisk_formats.h @@ -0,0 +1,186 @@ +// Metadata on-disk structures +// Copyright (c) Vitaliy Filippov, 2019+ +// License: VNPL-1.1 (see README.md for details) + +#pragma once + +#include "crc32c.h" + +#define JOURNAL_MAGIC 0x4A33 +#define JOURNAL_VERSION_V1 1 +#define JOURNAL_VERSION_V2 2 +#define JOURNAL_BUFFER_SIZE 4*1024*1024 +#define JOURNAL_ENTRY_HEADER_SIZE 16 + +// Journal entries +// Journal entries are linked to each other by their crc32 value +// The journal is almost a blockchain, because object versions constantly increase +#define JE_MIN 0x01 +#define JE_START 0x01 +#define JE_SMALL_WRITE 0x02 +#define JE_BIG_WRITE 0x03 +#define JE_STABLE 0x04 +#define JE_DELETE 0x05 +#define JE_ROLLBACK 0x06 +#define JE_SMALL_WRITE_INSTANT 0x07 +#define JE_BIG_WRITE_INSTANT 0x08 +#define JE_MAX 0x08 + +// crc32c comes first to ease calculation +struct __attribute__((__packed__)) journal_entry_start +{ + uint32_t crc32; + uint16_t magic; + uint16_t type; + uint32_t size; + uint32_t reserved; + uint64_t journal_start; + uint64_t version; + uint32_t data_csum_type; + uint32_t csum_block_size; +}; +#define JE_START_V0_SIZE 24 +#define JE_START_V1_SIZE 32 +#define JE_START_V2_SIZE 40 + +struct __attribute__((__packed__)) journal_entry_small_write +{ + uint32_t crc32; + uint16_t magic; + uint16_t type; + uint32_t size; + uint32_t crc32_prev; + object_id oid; + uint64_t version; + uint32_t offset; + uint32_t len; + // small_write entries contain bytes of data which is stored in next sectors + // data_offset is its offset within journal + uint64_t data_offset; + uint32_t crc32_data; // zero when data_csum_type != 0 + // small_write and big_write entries are followed by the "external" bitmap + // its size is dynamic and included in journal entry's field + uint8_t bitmap[]; + // and then data checksums if data_csum_type != 0 + // uint32_t data_crc32c[]; +}; + +struct __attribute__((__packed__)) journal_entry_big_write +{ + uint32_t crc32; + uint16_t magic; + uint16_t type; + uint32_t size; + uint32_t crc32_prev; + object_id oid; + uint64_t version; + uint32_t offset; + uint32_t len; + uint64_t location; + // small_write and big_write entries are followed by the "external" bitmap + // its size is dynamic and included in journal entry's field + uint8_t bitmap[]; + // and then data checksums if data_csum_type != 0 + // uint32_t data_crc32c[]; +}; + +struct __attribute__((__packed__)) journal_entry_stable +{ + uint32_t crc32; + uint16_t magic; + uint16_t type; + uint32_t size; + uint32_t crc32_prev; + object_id oid; + uint64_t version; +}; + +struct __attribute__((__packed__)) journal_entry_rollback +{ + uint32_t crc32; + uint16_t magic; + uint16_t type; + uint32_t size; + uint32_t crc32_prev; + object_id oid; + uint64_t version; +}; + +struct __attribute__((__packed__)) journal_entry_del +{ + uint32_t crc32; + uint16_t magic; + uint16_t type; + uint32_t size; + uint32_t crc32_prev; + object_id oid; + uint64_t version; +}; + +struct __attribute__((__packed__)) journal_entry +{ + union + { + struct __attribute__((__packed__)) + { + uint32_t crc32; + uint16_t magic; + uint16_t type; + uint32_t size; + uint32_t crc32_prev; + }; + journal_entry_start start; + journal_entry_small_write small_write; + journal_entry_big_write big_write; + journal_entry_stable stable; + journal_entry_rollback rollback; + journal_entry_del del; + }; +}; + +inline uint32_t je_crc32(journal_entry *je) +{ + // 0x48674bc7 = crc32(4 zero bytes) + return crc32c(0x48674bc7, ((uint8_t*)je)+4, je->size-4); +} + +// "VITAstor" +#define BLOCKSTORE_META_MAGIC_V1 0x726F747341544956l +#define BLOCKSTORE_META_FORMAT_V1 1 +#define BLOCKSTORE_META_FORMAT_V2 2 + +// metadata header (superblock) +struct __attribute__((__packed__)) blockstore_meta_header_v1_t +{ + uint64_t zero; + uint64_t magic; + uint64_t version; + uint32_t meta_block_size; + uint32_t data_block_size; + uint32_t bitmap_granularity; +}; + +struct __attribute__((__packed__)) blockstore_meta_header_v2_t +{ + uint64_t zero; + uint64_t magic; + uint64_t version; + uint32_t meta_block_size; + uint32_t data_block_size; + uint32_t bitmap_granularity; + uint32_t data_csum_type; + uint32_t csum_block_size; + uint32_t header_csum; +}; + +// 32 bytes = 24 bytes + block bitmap (4 bytes by default) + external attributes (also bitmap, 4 bytes by default) +// per "clean" entry on disk with fixed metadata tables +struct __attribute__((__packed__)) clean_disk_entry +{ + object_id oid; + uint64_t version; + uint8_t bitmap[]; + // Two more fields come after bitmap in metadata version 2: + // uint32_t data_csum[]; + // uint32_t entry_csum; +}; diff --git a/src/disk_tool/disk_tool.h b/src/disk_tool/disk_tool.h index c149704a..9c4733ca 100644 --- a/src/disk_tool/disk_tool.h +++ b/src/disk_tool/disk_tool.h @@ -8,14 +8,17 @@ #endif #include +#include #include #include #include #include "json11/json11.hpp" #include "blockstore_disk.h" -#include "blockstore_impl.h" +#include "blockstore.h" +#include "ondisk_formats.h" #include "crc32c.h" +#include "allocator.h" // vITADisk #define VITASTOR_DISK_MAGIC 0x6b73694441544976 diff --git a/src/disk_tool/disk_tool_journal.cpp b/src/disk_tool/disk_tool_journal.cpp index 3c24aed6..62e90982 100644 --- a/src/disk_tool/disk_tool_journal.cpp +++ b/src/disk_tool/disk_tool_journal.cpp @@ -1,7 +1,10 @@ // Copyright (c) Vitaliy Filippov, 2019+ // License: VNPL-1.1 (see README.md for details) +#include + #include "disk_tool.h" +#include "malloc_or_die.h" int disk_tool_t::dump_journal() { diff --git a/src/disk_tool/disk_tool_meta.cpp b/src/disk_tool/disk_tool_meta.cpp index 253d09f3..fde838db 100644 --- a/src/disk_tool/disk_tool_meta.cpp +++ b/src/disk_tool/disk_tool_meta.cpp @@ -5,6 +5,7 @@ #include "rw_blocking.h" #include "osd_id.h" #include "json_util.h" +#include "malloc_or_die.h" int disk_tool_t::process_meta(std::function hdr_fn, std::function record_fn, bool do_open) diff --git a/src/disk_tool/disk_tool_resize.cpp b/src/disk_tool/disk_tool_resize.cpp index 99e90bc1..ed2e9f67 100644 --- a/src/disk_tool/disk_tool_resize.cpp +++ b/src/disk_tool/disk_tool_resize.cpp @@ -4,6 +4,7 @@ #include "disk_tool.h" #include "rw_blocking.h" #include "str_util.h" +#include "malloc_or_die.h" #define DM_ST_EMPTY 0 #define DM_ST_TO_READ 1 diff --git a/src/disk_tool/disk_tool_udev.cpp b/src/disk_tool/disk_tool_udev.cpp index a506518f..faf7b538 100644 --- a/src/disk_tool/disk_tool_udev.cpp +++ b/src/disk_tool/disk_tool_udev.cpp @@ -7,6 +7,7 @@ #include "rw_blocking.h" #include "str_util.h" #include "json_util.h" +#include "malloc_or_die.h" struct __attribute__((__packed__)) vitastor_disk_superblock_t { diff --git a/src/disk_tool/disk_tool_utils.cpp b/src/disk_tool/disk_tool_utils.cpp index c0f33672..8966ae35 100644 --- a/src/disk_tool/disk_tool_utils.cpp +++ b/src/disk_tool/disk_tool_utils.cpp @@ -2,11 +2,13 @@ // License: VNPL-1.1 (see README.md for details) #include +#include #include #include "disk_tool.h" #include "rw_blocking.h" #include "str_util.h" +#include "malloc_or_die.h" uint64_t sscanf_json(const char *fmt, const json11::Json & str) { diff --git a/src/osd/osd.cpp b/src/osd/osd.cpp index 367e6ab6..86f1a734 100644 --- a/src/osd/osd.cpp +++ b/src/osd/osd.cpp @@ -9,7 +9,6 @@ #include #include "addr_util.h" -#include "blockstore_impl.h" #include "osd_primary.h" #include "osd.h" #include "http_client.h" @@ -650,11 +649,11 @@ void osd_t::print_slow() op->req.hdr.opcode == OSD_OP_SEC_READ_BMP) { cur_slow_op_secondary++; - bufprintf(" state=%d", op->bs_op ? PRIV(op->bs_op)->op_state : -1); - int wait_for = op->bs_op ? PRIV(op->bs_op)->wait_for : 0; - if (wait_for) + if (op->bs_op) { - bufprintf(" wait=%d (detail=%ju)", wait_for, PRIV(op->bs_op)->wait_detail); + auto diag = bs->get_op_diag(op->bs_op); + if (diag != "") + bufprintf(" %s", diag.c_str()); } } else if (op->req.hdr.opcode == OSD_OP_READ || op->req.hdr.opcode == OSD_OP_WRITE ||