thebestvibedev/tromcho.net
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support heap format in simple-offsets

Browse Source
This commit is contained in:
Vitaliy Filippov
2025-12-02 01:52:12 +03:00
parent 2d7da127ad
commit 6a30e6653a
6 changed files with 49 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/blockstore/blockstore_init.cpp
+1
View File
@@ -69,6 +69,7 @@ resume_1:
}
if (is_zero((uint64_t*)bs->meta_superblock, bs->dsk.meta_block_size))
{
bs->dsk.check_lengths();
{
blockstore_meta_header_v3_t *hdr = (blockstore_meta_header_v3_t *)bs->meta_superblock;
hdr->zero = 0;
src/disk_tool/disk_simple_offsets.cpp
+41 -7
View File
@@ -11,6 +11,7 @@
#include "str_util.h"
#include "blockstore.h"
#include "blockstore_disk.h"
#include "blockstore_heap.h"
// Calculate offsets for a block device and print OSD command line parameters
void disk_tool_simple_offsets(json11::Json cfg, bool json_output)
@@ -23,6 +24,9 @@ void disk_tool_simple_offsets(json11::Json cfg, bool json_output)
uint64_t journal_offset = parse_size(cfg["journal_offset"].string_value());
uint64_t device_size = parse_size(cfg["device_size"].string_value());
uint32_t csum_block_size = parse_size(cfg["csum_block_size"].string_value());
uint32_t meta_format = cfg["meta_format"].uint64_value();
if (!meta_format)
meta_format = BLOCKSTORE_META_FORMAT_HEAP;
uint32_t data_csum_type = BLOCKSTORE_CSUM_NONE;
if (cfg["data_csum_type"] == "crc32c")
data_csum_type = BLOCKSTORE_CSUM_CRC32C;
@@ -123,15 +127,45 @@ void disk_tool_simple_offsets(json11::Json cfg, bool json_output)
uint64_t meta_offset = journal_offset + ((journal_size+device_block_size-1)/device_block_size)*device_block_size;
uint64_t data_csum_size = (data_csum_type ? data_block_size/csum_block_size*(data_csum_type & 0xFF) : 0);
uint64_t clean_entry_bitmap_size = data_block_size/bitmap_granularity/8;
uint64_t clean_entry_size = 24 /*sizeof(clean_disk_entry)*/ + 2*clean_entry_bitmap_size + data_csum_size + 4 /*entry_csum*/;
uint64_t entries_per_block = device_block_size / clean_entry_size;
uint64_t object_count = ((device_size-meta_offset)/data_block_size);
uint64_t meta_size = (1 + (object_count+entries_per_block-1)/entries_per_block) * device_block_size;
uint64_t meta_size;
if (meta_format == BLOCKSTORE_META_FORMAT_HEAP)
{
uint32_t min_object_size = sizeof(heap_object_t)+sizeof(heap_write_t)+data_csum_size+2*clean_entry_bitmap_size;
uint32_t meta_block_target_free_space = cfg["meta_block_target_free_space"].uint64_value();
if (!meta_block_target_free_space || meta_block_target_free_space > device_block_size-min_object_size)
meta_block_target_free_space = 800;
double meta_reserve = cfg["meta_reserve"].number_value();
if (!meta_reserve)
meta_reserve = 1.5;
else if (meta_reserve < 1)
meta_reserve = 1;
uint32_t entries_per_block = (device_block_size-meta_block_target_free_space) / min_object_size;
meta_size = device_block_size * (uint64_t)((object_count+entries_per_block-1) / entries_per_block * meta_reserve);
}
else if (meta_format == BLOCKSTORE_META_FORMAT_V2)
{
uint64_t clean_entry_size = 24 /*sizeof(clean_disk_entry)*/ + 2*clean_entry_bitmap_size + data_csum_size + 4 /*entry_csum*/;
uint64_t entries_per_block = device_block_size / clean_entry_size;
meta_size = (1 + (object_count+entries_per_block-1)/entries_per_block) * device_block_size;
}
else if (meta_format == BLOCKSTORE_META_FORMAT_V1)
{
uint64_t clean_entry_size = 24 /*sizeof(clean_disk_entry)*/ + 2*clean_entry_bitmap_size;
uint64_t entries_per_block = device_block_size / clean_entry_size;
meta_size = (1 + (object_count+entries_per_block-1)/entries_per_block) * device_block_size;
}
else
{
fprintf(stderr, "meta_format %u is not supported\n", meta_format);
exit(1);
}
uint64_t data_offset = meta_offset + meta_size;
if (format == "json")
{
// JSON
printf("%s\n", json11::Json(json11::Json::object {
{ "meta_format", (uint64_t)meta_format },
{ "meta_block_size", device_block_size },
{ "journal_block_size", device_block_size },
{ "data_size", device_size-data_offset },
@@ -145,9 +179,9 @@ void disk_tool_simple_offsets(json11::Json cfg, bool json_output)
{
// Env
printf(
"meta_block_size=%ju\njournal_block_size=%ju\ndata_size=%ju\n"
"meta_format=%u\nmeta_block_size=%ju\njournal_block_size=%ju\ndata_size=%ju\n"
"data_device=%s\njournal_offset=%ju\nmeta_offset=%ju\ndata_offset=%ju\n",
device_block_size, device_block_size, device_size-data_offset,
meta_format, device_block_size, device_block_size, device_size-data_offset,
device.c_str(), journal_offset, meta_offset, data_offset
);
}
@@ -167,8 +201,8 @@ void disk_tool_simple_offsets(json11::Json cfg, bool json_output)
printf("--data_size %ju\n", device_size-data_offset);
}
printf(
"--data_device %s\n--journal_offset %ju\n--meta_offset %ju\n--data_offset %ju\n",
device.c_str(), journal_offset, meta_offset, data_offset
"--meta_format %u\n--data_device %s\n--journal_offset %ju\n--meta_offset %ju\n--data_offset %ju\n",
meta_format, device.c_str(), journal_offset, meta_offset, data_offset
);
}
}
tests/test_move_reappear.sh
+2 -1
View File
@@ -9,7 +9,8 @@ OSD_COUNT=5
OSD_ARGS="$OSD_ARGS"
for i in $(seq 1 $OSD_COUNT); do
dd if=/dev/zero of=./testdata/bin/test_osd$i.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
build/src/osd/vitastor-osd --log_level 10 --osd_num $i --bind_address 127.0.0.1 --etcd_stats_interval 5 $OSD_ARGS --etcd_address $ETCD_URL $(build/src/disk_tool/vitastor-disk simple-offsets --format options ./testdata/bin/test_osd$i.bin 2>/dev/null) >>./testdata/osd$i.log 2>&1 &
build/src/osd/vitastor-osd --log_level 10 --osd_num $i --bind_address 127.0.0.1 --etcd_stats_interval 5 $OSD_ARGS \
--etcd_address $ETCD_URL $(build/src/disk_tool/vitastor-disk simple-offsets --format options ./testdata/bin/test_osd$i.bin $OFFSET_ARGS 2>/dev/null) >>./testdata/osd$i.log 2>&1 &
eval OSD${i}_PID=$!
done
tests/test_resize.sh
+2
View File
@@ -25,6 +25,7 @@ done
for i in $(seq 1 $OSD_COUNT); do
offsets=$(build/src/disk_tool/vitastor-disk simple-offsets --format json ./testdata/bin/test_osd$i.bin)
meta_format=$(echo $offsets | jq -r .meta_format)
meta_offset=$(echo $offsets | jq -r .meta_offset)
data_offset=$(echo $offsets | jq -r .data_offset)
build/src/disk_tool/vitastor-disk dump-journal --io cached --json ./testdata/bin/test_osd$i.bin 4096 0 $meta_offset >./testdata/journal_before_resize.json
@@ -51,6 +52,7 @@ $ETCDCTL del --prefix /vitastor/osd/state/
for i in $(seq 1 $OSD_COUNT); do
build/src/osd/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $NO_SAME $OSD_ARGS --etcd_address $ETCD_URL \
--meta_format $meta_format \
--data_device ./testdata/bin/test_osd$i.bin \
--meta_offset 0 \
--journal_offset $((1024*1024)) \
tests/test_splitbrain.sh
+1 -1
View File
@@ -23,7 +23,7 @@ LD_PRELOAD="build/src/client/libfio_vitastor.so" \
kill $OSD2_PID
build/src/osd/vitastor-osd --osd_num 1 --bind_address 127.0.0.1 $NO_SAME $OSD_ARGS --etcd_address $ETCD_URL \
$(build/src/disk_tool/vitastor-disk simple-offsets --format options --device ./testdata/bin/test_osd2.bin 2>/dev/null) >>./testdata/osd2.log 2>&1 &
$(build/src/disk_tool/vitastor-disk simple-offsets --format options --device ./testdata/bin/test_osd2.bin $OFFSET_ARGS 2>/dev/null) >>./testdata/osd2.log 2>&1 &
sleep 2
# Check PG state - it should NOT become active
tests/test_vm_cont.sh
+2 -1
View File
@@ -15,7 +15,8 @@ OSD_COUNT=3
OSD_ARGS="$OSD_ARGS"
OFFSET_ARGS="$OFFSET_ARGS"
for i in $(seq 1 $OSD_COUNT); do
build/src/osd/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $OSD_ARGS --etcd_address $ETCD_URL $(build/src/disk_tool/vitastor-disk simple-offsets --format options ./testdata/bin/test_osd$i.bin 2>/dev/null) >>./testdata/osd$i.log 2>&1 &
build/src/osd/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $OSD_ARGS --etcd_address $ETCD_URL \
$(build/src/disk_tool/vitastor-disk simple-offsets --format options ./testdata/bin/test_osd$i.bin $OFFSET_ARGS 2>/dev/null) >>./testdata/osd$i.log 2>&1 &
eval OSD${i}_PID=$!
done