thebestvibedev/tromcho.net
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove include of blockstore_impl.h from disk_tool and osd

Browse Source
This commit is contained in:
Vitaliy Filippov
2025-11-23 01:48:44 +03:00
parent be1858848e
commit 1badc6ad13
13 changed files with 222 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/blockstore/blockstore.cpp
+11
View File
@@ -58,6 +58,17 @@ void blockstore_t::dump_diagnostics()
return impl->dump_diagnostics();
}
std::string blockstore_t::get_op_diag(blockstore_op_t *op)
{
char buf[256];
auto priv = PRIV(op);
if (priv->wait_for)
snprintf(buf, sizeof(buf), "state=%d wait=%d (detail=%ju)", priv->op_state, priv->wait_for, priv->wait_detail);
else
snprintf(buf, sizeof(buf), "state=%d", priv->op_state);
return std::string(buf);
}
uint32_t blockstore_t::get_block_size()
{
return impl->get_block_size();
src/blockstore/blockstore.h
+3 -11
View File
@@ -17,11 +17,6 @@
#include "ringloop.h"
#include "timerfd_manager.h"
// Memory alignment for direct I/O (usually 512 bytes)
#ifndef DIRECT_IO_ALIGNMENT
#define DIRECT_IO_ALIGNMENT 512
#endif
// Default block size is 128 KB, current allowed range is 4K - 128M
#define DEFAULT_DATA_BLOCK_ORDER 17
#define MIN_DATA_BLOCK_SIZE 4*1024
@@ -30,12 +25,6 @@
#define MIN_JOURNAL_SIZE 1024*1024
// "VITAstor"
#define BLOCKSTORE_META_MAGIC_V1 0x726F747341544956l
#define BLOCKSTORE_META_FORMAT_V1 1
#define BLOCKSTORE_META_FORMAT_V2 2
#define BLOCKSTORE_META_FORMAT_HEAP 3
#define BS_OP_MIN 1
#define BS_OP_READ 1
#define BS_OP_WRITE 2
@@ -226,6 +215,9 @@ public:
// Print diagnostics to stdout
void dump_diagnostics();
// Get diagnostic string for an operation
std::string get_op_diag(blockstore_op_t *op);
uint32_t get_block_size();
uint64_t get_block_count();
uint64_t get_free_block_count();
src/blockstore/blockstore_disk.h
+5
View File
@@ -8,6 +8,11 @@
#include <string>
#include <map>
// Memory alignment for direct I/O (usually 512 bytes)
#ifndef DIRECT_IO_ALIGNMENT
#define DIRECT_IO_ALIGNMENT 512
#endif
#define BLOCKSTORE_CSUM_NONE 0
// Lower byte of checksum type is its length
#define BLOCKSTORE_CSUM_CRC32C 0x104
src/blockstore/blockstore_impl.h
+1 -41
View File
@@ -5,6 +5,7 @@
#include "blockstore.h"
#include "blockstore_disk.h"
#include "ondisk_formats.h"
#include <sys/types.h>
#include <sys/ioctl.h>
@@ -89,47 +90,6 @@
#include "blockstore_journal.h"
// "VITAstor"
#define BLOCKSTORE_META_MAGIC_V1 0x726F747341544956l
#define BLOCKSTORE_META_FORMAT_V1 1
#define BLOCKSTORE_META_FORMAT_V2 2
// metadata header (superblock)
struct __attribute__((__packed__)) blockstore_meta_header_v1_t
{
uint64_t zero;
uint64_t magic;
uint64_t version;
uint32_t meta_block_size;
uint32_t data_block_size;
uint32_t bitmap_granularity;
};
struct __attribute__((__packed__)) blockstore_meta_header_v2_t
{
uint64_t zero;
uint64_t magic;
uint64_t version;
uint32_t meta_block_size;
uint32_t data_block_size;
uint32_t bitmap_granularity;
uint32_t data_csum_type;
uint32_t csum_block_size;
uint32_t header_csum;
};
// 32 bytes = 24 bytes + block bitmap (4 bytes by default) + external attributes (also bitmap, 4 bytes by default)
// per "clean" entry on disk with fixed metadata tables
struct __attribute__((__packed__)) clean_disk_entry
{
object_id oid;
uint64_t version;
uint8_t bitmap[];
// Two more fields come after bitmap in metadata version 2:
// uint32_t data_csum[];
// uint32_t entry_csum;
};
// 32 = 16 + 16 bytes per "clean" entry in memory (object_id => clean_entry)
struct __attribute__((__packed__)) clean_entry
{
src/blockstore/blockstore_journal.h
-141
View File
@@ -3,147 +3,6 @@
#pragma once
#include "crc32c.h"
#include <set>
#define JOURNAL_MAGIC 0x4A33
#define JOURNAL_VERSION_V1 1
#define JOURNAL_VERSION_V2 2
#define JOURNAL_BUFFER_SIZE 4*1024*1024
#define JOURNAL_ENTRY_HEADER_SIZE 16
// Journal entries
// Journal entries are linked to each other by their crc32 value
// The journal is almost a blockchain, because object versions constantly increase
#define JE_MIN 0x01
#define JE_START 0x01
#define JE_SMALL_WRITE 0x02
#define JE_BIG_WRITE 0x03
#define JE_STABLE 0x04
#define JE_DELETE 0x05
#define JE_ROLLBACK 0x06
#define JE_SMALL_WRITE_INSTANT 0x07
#define JE_BIG_WRITE_INSTANT 0x08
#define JE_MAX 0x08
// crc32c comes first to ease calculation
struct __attribute__((__packed__)) journal_entry_start
{
uint32_t crc32;
uint16_t magic;
uint16_t type;
uint32_t size;
uint32_t reserved;
uint64_t journal_start;
uint64_t version;
uint32_t data_csum_type;
uint32_t csum_block_size;
};
#define JE_START_V0_SIZE 24
#define JE_START_V1_SIZE 32
#define JE_START_V2_SIZE 40
struct __attribute__((__packed__)) journal_entry_small_write
{
uint32_t crc32;
uint16_t magic;
uint16_t type;
uint32_t size;
uint32_t crc32_prev;
object_id oid;
uint64_t version;
uint32_t offset;
uint32_t len;
// small_write entries contain <len> bytes of data which is stored in next sectors
// data_offset is its offset within journal
uint64_t data_offset;
uint32_t crc32_data; // zero when data_csum_type != 0
// small_write and big_write entries are followed by the "external" bitmap
// its size is dynamic and included in journal entry's <size> field
uint8_t bitmap[];
// and then data checksums if data_csum_type != 0
// uint32_t data_crc32c[];
};
struct __attribute__((__packed__)) journal_entry_big_write
{
uint32_t crc32;
uint16_t magic;
uint16_t type;
uint32_t size;
uint32_t crc32_prev;
object_id oid;
uint64_t version;
uint32_t offset;
uint32_t len;
uint64_t location;
// small_write and big_write entries are followed by the "external" bitmap
// its size is dynamic and included in journal entry's <size> field
uint8_t bitmap[];
// and then data checksums if data_csum_type != 0
// uint32_t data_crc32c[];
};
struct __attribute__((__packed__)) journal_entry_stable
{
uint32_t crc32;
uint16_t magic;
uint16_t type;
uint32_t size;
uint32_t crc32_prev;
object_id oid;
uint64_t version;
};
struct __attribute__((__packed__)) journal_entry_rollback
{
uint32_t crc32;
uint16_t magic;
uint16_t type;
uint32_t size;
uint32_t crc32_prev;
object_id oid;
uint64_t version;
};
struct __attribute__((__packed__)) journal_entry_del
{
uint32_t crc32;
uint16_t magic;
uint16_t type;
uint32_t size;
uint32_t crc32_prev;
object_id oid;
uint64_t version;
};
struct __attribute__((__packed__)) journal_entry
{
union
{
struct __attribute__((__packed__))
{
uint32_t crc32;
uint16_t magic;
uint16_t type;
uint32_t size;
uint32_t crc32_prev;
};
journal_entry_start start;
journal_entry_small_write small_write;
journal_entry_big_write big_write;
journal_entry_stable stable;
journal_entry_rollback rollback;
journal_entry_del del;
};
};
inline uint32_t je_crc32(journal_entry *je)
{
// 0x48674bc7 = crc32(4 zero bytes)
return crc32c(0x48674bc7, ((uint8_t*)je)+4, je->size-4);
}
struct journal_sector_info_t
{
uint64_t offset;
src/blockstore/ondisk_formats.h
+186
View File
@@ -0,0 +1,186 @@
// Metadata on-disk structures
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
#pragma once
#include "crc32c.h"
#define JOURNAL_MAGIC 0x4A33
#define JOURNAL_VERSION_V1 1
#define JOURNAL_VERSION_V2 2
#define JOURNAL_BUFFER_SIZE 4*1024*1024
#define JOURNAL_ENTRY_HEADER_SIZE 16
// Journal entries
// Journal entries are linked to each other by their crc32 value
// The journal is almost a blockchain, because object versions constantly increase
#define JE_MIN 0x01
#define JE_START 0x01
#define JE_SMALL_WRITE 0x02
#define JE_BIG_WRITE 0x03
#define JE_STABLE 0x04
#define JE_DELETE 0x05
#define JE_ROLLBACK 0x06
#define JE_SMALL_WRITE_INSTANT 0x07
#define JE_BIG_WRITE_INSTANT 0x08
#define JE_MAX 0x08
// crc32c comes first to ease calculation
struct __attribute__((__packed__)) journal_entry_start
{
uint32_t crc32;
uint16_t magic;
uint16_t type;
uint32_t size;
uint32_t reserved;
uint64_t journal_start;
uint64_t version;
uint32_t data_csum_type;
uint32_t csum_block_size;
};
#define JE_START_V0_SIZE 24
#define JE_START_V1_SIZE 32
#define JE_START_V2_SIZE 40
struct __attribute__((__packed__)) journal_entry_small_write
{
uint32_t crc32;
uint16_t magic;
uint16_t type;
uint32_t size;
uint32_t crc32_prev;
object_id oid;
uint64_t version;
uint32_t offset;
uint32_t len;
// small_write entries contain <len> bytes of data which is stored in next sectors
// data_offset is its offset within journal
uint64_t data_offset;
uint32_t crc32_data; // zero when data_csum_type != 0
// small_write and big_write entries are followed by the "external" bitmap
// its size is dynamic and included in journal entry's <size> field
uint8_t bitmap[];
// and then data checksums if data_csum_type != 0
// uint32_t data_crc32c[];
};
struct __attribute__((__packed__)) journal_entry_big_write
{
uint32_t crc32;
uint16_t magic;
uint16_t type;
uint32_t size;
uint32_t crc32_prev;
object_id oid;
uint64_t version;
uint32_t offset;
uint32_t len;
uint64_t location;
// small_write and big_write entries are followed by the "external" bitmap
// its size is dynamic and included in journal entry's <size> field
uint8_t bitmap[];
// and then data checksums if data_csum_type != 0
// uint32_t data_crc32c[];
};
struct __attribute__((__packed__)) journal_entry_stable
{
uint32_t crc32;
uint16_t magic;
uint16_t type;
uint32_t size;
uint32_t crc32_prev;
object_id oid;
uint64_t version;
};
struct __attribute__((__packed__)) journal_entry_rollback
{
uint32_t crc32;
uint16_t magic;
uint16_t type;
uint32_t size;
uint32_t crc32_prev;
object_id oid;
uint64_t version;
};
struct __attribute__((__packed__)) journal_entry_del
{
uint32_t crc32;
uint16_t magic;
uint16_t type;
uint32_t size;
uint32_t crc32_prev;
object_id oid;
uint64_t version;
};
struct __attribute__((__packed__)) journal_entry
{
union
{
struct __attribute__((__packed__))
{
uint32_t crc32;
uint16_t magic;
uint16_t type;
uint32_t size;
uint32_t crc32_prev;
};
journal_entry_start start;
journal_entry_small_write small_write;
journal_entry_big_write big_write;
journal_entry_stable stable;
journal_entry_rollback rollback;
journal_entry_del del;
};
};
inline uint32_t je_crc32(journal_entry *je)
{
// 0x48674bc7 = crc32(4 zero bytes)
return crc32c(0x48674bc7, ((uint8_t*)je)+4, je->size-4);
}
// "VITAstor"
#define BLOCKSTORE_META_MAGIC_V1 0x726F747341544956l
#define BLOCKSTORE_META_FORMAT_V1 1
#define BLOCKSTORE_META_FORMAT_V2 2
// metadata header (superblock)
struct __attribute__((__packed__)) blockstore_meta_header_v1_t
{
uint64_t zero;
uint64_t magic;
uint64_t version;
uint32_t meta_block_size;
uint32_t data_block_size;
uint32_t bitmap_granularity;
};
struct __attribute__((__packed__)) blockstore_meta_header_v2_t
{
uint64_t zero;
uint64_t magic;
uint64_t version;
uint32_t meta_block_size;
uint32_t data_block_size;
uint32_t bitmap_granularity;
uint32_t data_csum_type;
uint32_t csum_block_size;
uint32_t header_csum;
};
// 32 bytes = 24 bytes + block bitmap (4 bytes by default) + external attributes (also bitmap, 4 bytes by default)
// per "clean" entry on disk with fixed metadata tables
struct __attribute__((__packed__)) clean_disk_entry
{
object_id oid;
uint64_t version;
uint8_t bitmap[];
// Two more fields come after bitmap in metadata version 2:
// uint32_t data_csum[];
// uint32_t entry_csum;
};
src/disk_tool/disk_tool.h
+4 -1
View File
@@ -8,14 +8,17 @@
#endif
#include <map>
#include <set>
#include <vector>
#include <string>
#include <functional>
#include "json11/json11.hpp"
#include "blockstore_disk.h"
#include "blockstore_impl.h"
#include "blockstore.h"
#include "ondisk_formats.h"
#include "crc32c.h"
#include "allocator.h"
// vITADisk
#define VITASTOR_DISK_MAGIC 0x6b73694441544976
src/disk_tool/disk_tool_journal.cpp
+3
View File
@@ -1,7 +1,10 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
#include <unistd.h>
#include "disk_tool.h"
#include "malloc_or_die.h"
int disk_tool_t::dump_journal()
{
src/disk_tool/disk_tool_meta.cpp
+1
View File
@@ -5,6 +5,7 @@
#include "rw_blocking.h"
#include "osd_id.h"
#include "json_util.h"
#include "malloc_or_die.h"
int disk_tool_t::process_meta(std::function<void(blockstore_meta_header_v2_t *)> hdr_fn,
std::function<void(uint64_t, clean_disk_entry*, uint8_t*)> record_fn, bool do_open)
src/disk_tool/disk_tool_resize.cpp
+1
View File
@@ -4,6 +4,7 @@
#include "disk_tool.h"
#include "rw_blocking.h"
#include "str_util.h"
#include "malloc_or_die.h"
#define DM_ST_EMPTY 0
#define DM_ST_TO_READ 1
src/disk_tool/disk_tool_udev.cpp
+1
View File
@@ -7,6 +7,7 @@
#include "rw_blocking.h"
#include "str_util.h"
#include "json_util.h"
#include "malloc_or_die.h"
struct __attribute__((__packed__)) vitastor_disk_superblock_t
{
src/disk_tool/disk_tool_utils.cpp
+2
View File
@@ -2,11 +2,13 @@
// License: VNPL-1.1 (see README.md for details)
#include <sys/wait.h>
#include <sys/ioctl.h>
#include <dirent.h>
#include "disk_tool.h"
#include "rw_blocking.h"
#include "str_util.h"
#include "malloc_or_die.h"
uint64_t sscanf_json(const char *fmt, const json11::Json & str)
{
src/osd/osd.cpp
+4 -5
View File
@@ -9,7 +9,6 @@
#include <arpa/inet.h>
#include "addr_util.h"
#include "blockstore_impl.h"
#include "osd_primary.h"
#include "osd.h"
#include "http_client.h"
@@ -650,11 +649,11 @@ void osd_t::print_slow()
op->req.hdr.opcode == OSD_OP_SEC_READ_BMP)
{
cur_slow_op_secondary++;
bufprintf(" state=%d", op->bs_op ? PRIV(op->bs_op)->op_state : -1);
int wait_for = op->bs_op ? PRIV(op->bs_op)->wait_for : 0;
if (wait_for)
if (op->bs_op)
{
bufprintf(" wait=%d (detail=%ju)", wait_for, PRIV(op->bs_op)->wait_detail);
auto diag = bs->get_op_diag(op->bs_op);
if (diag != "")
bufprintf(" %s", diag.c_str());
}
}
else if (op->req.hdr.opcode == OSD_OP_READ || op->req.hdr.opcode == OSD_OP_WRITE ||