From 1de68c30afd0fa0398ba1ac7f24d83a86800b30e Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sat, 13 Dec 2025 13:26:20 +0000 Subject: [PATCH] Fix cross-pool snapshots and fix the test for it --- src/client/cluster_client.cpp | 22 +++++++++++++--------- src/client/cluster_client_impl.h | 1 + tests/test_snapshot_pool2.sh | 11 ++++++++--- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/client/cluster_client.cpp b/src/client/cluster_client.cpp index 07bc7a29..f4e79185 100644 --- a/src/client/cluster_client.cpp +++ b/src/client/cluster_client.cpp @@ -1253,9 +1253,9 @@ void cluster_client_t::slice_rw(cluster_op_t *op) { op->bitmap_buf = realloc_or_die(op->bitmap_buf, bitmap_mem); op->part_bitmaps = (uint8_t*)op->bitmap_buf + object_bitmap_size; + memset(op->bitmap_buf+op->bitmap_buf_size, 0, bitmap_mem-op->bitmap_buf_size); op->bitmap_buf_size = bitmap_mem; } - memset(op->bitmap_buf, 0, bitmap_mem); } int iov_idx = 0; size_t iov_pos = 0; @@ -1401,7 +1401,7 @@ int cluster_client_t::try_send(cluster_op_t *op, int i, std::functionsecond; - part->flags |= PART_SENT; + part->flags |= PART_SENT|PART_VALID; op->inflight_count++; uint64_t pg_bitmap_size = (pool_cfg.data_block_size / pool_cfg.bitmap_granularity / 8) * ( pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks @@ -1614,14 +1614,11 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part) dirty_osds.insert(part->osd_num); part->flags |= PART_DONE; op->done_count++; - if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP) + if ((op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP) + && op->inode == op->cur_inode) { - copy_part_bitmap(op, part); - if (op->inode == op->cur_inode) - { - // Read only returns the version of the uppermost layer - op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0; - } + // Read only returns the version of the uppermost layer + op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0; } else if (op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) { @@ -1629,6 +1626,13 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part) } if (op->inflight_count == 0 && !op->retry_after) { + // Copy part bitmaps only after finishing all part reads + if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP) + { + for (auto & part: op->parts) + if (part.flags == (PART_SENT|PART_VALID|PART_DONE)) + copy_part_bitmap(op, &part); + } if (op->opcode == OSD_OP_SYNC) continue_sync(op); else diff --git a/src/client/cluster_client_impl.h b/src/client/cluster_client_impl.h index 93bc1131..2e3a3c87 100644 --- a/src/client/cluster_client_impl.h +++ b/src/client/cluster_client_impl.h @@ -10,6 +10,7 @@ #define PART_DONE 2 #define PART_ERROR 4 #define PART_RETRY 8 +#define PART_VALID 16 #define CACHE_DIRTY 1 #define CACHE_WRITTEN 2 #define CACHE_FLUSHING 3 diff --git a/tests/test_snapshot_pool2.sh b/tests/test_snapshot_pool2.sh index 25c6791a..15e7e402 100755 --- a/tests/test_snapshot_pool2.sh +++ b/tests/test_snapshot_pool2.sh @@ -13,13 +13,18 @@ build/src/cmd/vitastor-cli --etcd_address $ETCD_URL create -s 128M testchain -p LD_PRELOAD="build/src/client/libfio_vitastor.so" \ fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=1M -direct=1 -iodepth=4 -fsync=1 -rw=write \ - -etcd=$ETCD_URL -image=testchain -mirror_file=./testdata/bin/mirror.bin -buffer_pattern=0xabcd + -etcd=$ETCD_URL -image=testchain -mirror_file=./testdata/bin/mirror.bin -randrepeat=0 build/src/cmd/vitastor-cli --etcd_address $ETCD_URL snap-create testchain@snap1 -p testpool2 LD_PRELOAD="build/src/client/libfio_vitastor.so" \ fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4k -direct=1 -iodepth=4 -end_fsync=1 -rw=randwrite -number_ios=32 \ - -etcd=$ETCD_URL -image=testchain -mirror_file=./testdata/bin/mirror.bin -buffer_pattern=0xabcd + -etcd=$ETCD_URL -image=testchain -mirror_file=./testdata/bin/mirror.bin -randrepeat=0 + +# Read from the first snapshot + +build/src/cmd/vitastor-cli --etcd_address $ETCD_URL dd iimg=testchain of=./testdata/bin/res.bin bs=128k iodepth=4 --log_level 10 +cmp ./testdata/bin/res.bin ./testdata/bin/mirror.bin # Create a second snapshot - there was a bug where snapshotted reads from another pool # were working only when the image and the snapshot were modified in the same revision @@ -28,7 +33,7 @@ build/src/cmd/vitastor-cli --etcd_address $ETCD_URL snap-create testchain@snap2 LD_PRELOAD="build/src/client/libfio_vitastor.so" \ fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4k -direct=1 -iodepth=4 -end_fsync=1 -rw=randwrite -number_ios=32 \ - -etcd=$ETCD_URL -image=testchain -mirror_file=./testdata/bin/mirror.bin -buffer_pattern=0xabcd + -etcd=$ETCD_URL -image=testchain -mirror_file=./testdata/bin/mirror.bin -randrepeat=0 build/src/cmd/vitastor-cli --etcd_address $ETCD_URL dd iimg=testchain of=./testdata/bin/res.bin bs=128k iodepth=4 --log_level 10