thebestvibedev/tromcho.net
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add a dump|load test, fix multiple bugs in both new&old dump/load utils

Browse Source 
Details:
- New store dump/write-meta didn't use actual metadata parameters from the header
- New store write-meta calculated small entry sizes incorrectly
- Old store write-journal imported entries with checksums incorrectly
- Old store write-meta didn't import block_csums at all (it was using a wrong json key)
This commit is contained in:
Vitaliy Filippov
2026-06-17 01:19:12 +03:00
parent c491db699c
commit 430d3cfb6f
5 changed files with 145 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/disk_tool/disk_tool_journal.cpp
+12 -14
View File
@@ -477,11 +477,15 @@ int disk_tool_t::write_json_journal(json11::Json entries)
uint16_t type = t_it->second;
if (type == JE_START)
continue;
uint32_t offset = (uint32_t)rec["offset"].uint64_value();
uint32_t len = (uint32_t)rec["len"].uint64_value();
uint32_t data_csum_blocks = !dsk.data_csum_type || !len ? 0 :
(((offset + len - 1)/dsk.csum_block_size - offset/dsk.csum_block_size + 1));
uint32_t data_csum_size = data_csum_blocks*(dsk.data_csum_type & 0xFF);
uint32_t entry_size = (type == JE_START
? sizeof(journal_entry_start)
: (type == JE_SMALL_WRITE || type == JE_SMALL_WRITE_INSTANT
? sizeof(journal_entry_small_write) + dsk.clean_entry_bitmap_size +
(dsk.data_csum_type ? rec["len"].uint64_value()/dsk.csum_block_size*(dsk.data_csum_type & 0xFF) : 0)
? sizeof(journal_entry_small_write) + dsk.clean_entry_bitmap_size + data_csum_size
: (type == JE_BIG_WRITE || type == JE_BIG_WRITE_INSTANT
? sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size +
(dsk.data_csum_type ? rec["len"].uint64_value()/dsk.csum_block_size*(dsk.data_csum_type & 0xFF) : 0)
@@ -523,15 +527,12 @@ int disk_tool_t::write_json_journal(json11::Json entries)
.stripe = sscanf_json(NULL, rec["stripe"]),
},
.version = rec["ver"].uint64_value(),
.offset = (uint32_t)rec["offset"].uint64_value(),
.len = (uint32_t)rec["len"].uint64_value(),
.offset = offset,
.len = len,
.data_offset = (uint64_t)(new_journal_data-new_journal_buf),
.crc32_data = !dsk.data_csum_type ? 0 : (uint32_t)sscanf_json("%x", rec["data_crc32"]),
};
uint32_t data_csum_blocks = !dsk.data_csum_type ? 0 :
(((ne->small_write.offset+ne->small_write.len)/dsk.csum_block_size - ne->small_write.len/dsk.csum_block_size));
uint32_t data_csum_size = data_csum_blocks*(dsk.data_csum_type & 0xFF);
fromhexstr(rec["bitmap"].string_value(), dsk.clean_entry_bitmap_size, ((uint8_t*)ne) + sizeof(journal_entry_small_write) + data_csum_size);
fromhexstr(rec["bitmap"].string_value(), dsk.clean_entry_bitmap_size, ((uint8_t*)ne) + sizeof(journal_entry_small_write));
fromhexstr(rec["data"].string_value(), ne->small_write.len, new_journal_data);
if (ne->small_write.len > 0 && !rec["data"].is_string())
{
@@ -545,7 +546,7 @@ int disk_tool_t::write_json_journal(json11::Json entries)
ne->small_write.crc32_data = crc32c(0, new_journal_data, ne->small_write.len);
else if (dsk.data_csum_type == BLOCKSTORE_CSUM_CRC32C)
{
uint32_t *block_csums = (uint32_t*)(((uint8_t*)ne) + sizeof(journal_entry_small_write));
uint32_t *block_csums = (uint32_t*)(((uint8_t*)ne) + sizeof(journal_entry_small_write) + dsk.clean_entry_bitmap_size);
for (uint32_t i = 0; i < data_csum_blocks; i++)
{
uint32_t block_begin = (ne->small_write.offset/dsk.csum_block_size + i) * dsk.csum_block_size;
@@ -574,12 +575,9 @@ int disk_tool_t::write_json_journal(json11::Json entries)
.len = (uint32_t)rec["len"].uint64_value(),
.location = sscanf_json(NULL, rec["loc"]),
};
uint32_t data_csum_blocks = !dsk.data_csum_type ? 0 :
(((ne->small_write.offset+ne->small_write.len)/dsk.csum_block_size - ne->small_write.len/dsk.csum_block_size));
uint32_t data_csum_size = data_csum_blocks*(dsk.data_csum_type & 0xFF);
fromhexstr(rec["bitmap"].string_value(), dsk.clean_entry_bitmap_size, ((uint8_t*)ne) + sizeof(journal_entry_big_write) + data_csum_size);
fromhexstr(rec["bitmap"].string_value(), dsk.clean_entry_bitmap_size, ((uint8_t*)ne) + sizeof(journal_entry_big_write));
if (dsk.data_csum_type)
fromhexstr(rec["block_csums"].string_value(), data_csum_size, ((uint8_t*)ne) + sizeof(journal_entry_big_write));
fromhexstr(rec["block_csums"].string_value(), data_csum_size, ((uint8_t*)ne) + sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size);
}
else if (type == JE_STABLE || type == JE_ROLLBACK || type == JE_DELETE)
{
src/disk_tool/disk_tool_meta.cpp
+16 -3
View File
@@ -95,6 +95,12 @@ close_error:
journal_pos += read_len;
}
}
dsk.meta_format = hdr->version;
dsk.data_block_size = hdr->data_block_size;
dsk.csum_block_size = hdr->csum_block_size;
dsk.data_csum_type = hdr->data_csum_type;
dsk.bitmap_granularity = hdr->bitmap_granularity;
dsk.clean_entry_bitmap_size = (hdr->data_block_size / hdr->bitmap_granularity + 7) / 8;
blockstore_heap_t *heap = new blockstore_heap_t(&dsk, buffer_area, log_level);
// Load heap and just iterate it in memory
hdr_fn(hdr);
@@ -575,7 +581,7 @@ int disk_tool_t::write_json_meta(json11::Json meta)
{
if (new_data_csum_size)
{
fromhexstr(e["data_csum"].string_value(), new_data_csum_size,
fromhexstr(e["block_csums"].string_value(), new_data_csum_size,
((uint8_t*)new_entry) + sizeof(clean_disk_entry) + 2*new_clean_entry_bitmap_size);
}
uint32_t *new_entry_csum = (uint32_t*)(((uint8_t*)new_entry) + new_clean_entry_size - 4);
@@ -616,6 +622,12 @@ int disk_tool_t::write_json_heap(json11::Json meta, json11::Json journal)
new_data_csum_size = (new_meta_hdr->csum_block_size
? ((new_meta_hdr->data_block_size+new_meta_hdr->csum_block_size-1)/new_meta_hdr->csum_block_size*(new_meta_hdr->data_csum_type & 0xFF))
: 0);
dsk.meta_format = new_meta_hdr->version;
dsk.data_block_size = new_meta_hdr->data_block_size;
dsk.csum_block_size = new_meta_hdr->csum_block_size;
dsk.data_csum_type = new_meta_hdr->data_csum_type;
dsk.bitmap_granularity = new_meta_hdr->bitmap_granularity;
dsk.clean_entry_bitmap_size = (new_meta_hdr->data_block_size / new_meta_hdr->bitmap_granularity + 7) / 8;
new_journal_buf = NULL;
if (new_journal_len)
{
@@ -696,7 +708,7 @@ close_err0:
wr->entry_type = wr_type | (write_entry["stable"].bool_value() ? BS_HEAP_STABLE : 0);
wr->lsn = write_entry["lsn"].uint64_value();
wr->version = write_entry["version"].uint64_value();
wr->size = wr->get_size(&heap);
wr->size = wr_size;
if (wr_type == BS_HEAP_SMALL_WRITE || wr_type == BS_HEAP_INTENT_WRITE)
{
wr->small().offset = wr_offset;
@@ -736,6 +748,7 @@ close_err0:
bi.offset = wr_offset;
bi.len = wr_len;
}
wr->size = wr->get_size(&heap);
if (write_entry["bitmap"].is_string() && wr->get_int_bitmap(&heap))
{
fromhexstr(write_entry["bitmap"].string_value(), new_clean_entry_bitmap_size, wr->get_int_bitmap(&heap));
@@ -794,7 +807,7 @@ close_err:
fromhexstr(meta_entry["bitmap"].string_value(), new_clean_entry_bitmap_size, wr->get_int_bitmap(&heap));
fromhexstr(meta_entry["ext_bitmap"].string_value(), new_clean_entry_bitmap_size, wr->get_ext_bitmap(&heap));
if (new_data_csum_size)
fromhexstr(meta_entry["data_csum"].string_value(), new_data_csum_size, wr->get_checksums(&heap));
fromhexstr(meta_entry["block_csums"].string_value(), new_data_csum_size, wr->get_checksums(&heap));
wr->crc32c = wr->calc_crc32c();
assert((uint8_t*)wr + wr->size == new_meta_buf + meta_offset + used_space);
auto j_it = journal_by_object.find(oid);