Fix cross-pool snapshots and fix the test for it

src/client/cluster_client.cpp

@@ -1253,9 +1253,9 @@ void cluster_client_t::slice_rw(cluster_op_t *op)
         {
             op->bitmap_buf = realloc_or_die(op->bitmap_buf, bitmap_mem);
             op->part_bitmaps = (uint8_t*)op->bitmap_buf + object_bitmap_size;
+            memset(op->bitmap_buf+op->bitmap_buf_size, 0, bitmap_mem-op->bitmap_buf_size);
             op->bitmap_buf_size = bitmap_mem;
         }
-        memset(op->bitmap_buf, 0, bitmap_mem);
     }
     int iov_idx = 0;
     size_t iov_pos = 0;
@@ -1401,7 +1401,7 @@ int cluster_client_t::try_send(cluster_op_t *op, int i, std::function<void(osd_o
         if (peer_it != msgr.osd_peer_fds.end())
         {
             int peer_fd = peer_it->second;
-            part->flags |= PART_SENT;
+            part->flags |= PART_SENT|PART_VALID;
             op->inflight_count++;
             uint64_t pg_bitmap_size = (pool_cfg.data_block_size / pool_cfg.bitmap_granularity / 8) * (
                 pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks
@@ -1614,14 +1614,11 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
             dirty_osds.insert(part->osd_num);
         part->flags |= PART_DONE;
         op->done_count++;
-        if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
+        if ((op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
+            && op->inode == op->cur_inode)
         {
-            copy_part_bitmap(op, part);
+            // Read only returns the version of the uppermost layer
-            if (op->inode == op->cur_inode)
+            op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0;
-            {
-                // Read only returns the version of the uppermost layer
-                op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0;
-            }
         }
         else if (op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE)
         {
@@ -1629,6 +1626,13 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
         }
         if (op->inflight_count == 0 && !op->retry_after)
         {
+            // Copy part bitmaps only after finishing all part reads
+            if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
+            {
+                for (auto & part: op->parts)
+                    if (part.flags == (PART_SENT|PART_VALID|PART_DONE))
+                        copy_part_bitmap(op, &part);
+            }
             if (op->opcode == OSD_OP_SYNC)
                 continue_sync(op);
             else

src/client/cluster_client_impl.h

@@ -10,6 +10,7 @@
 #define PART_DONE 2
 #define PART_ERROR 4
 #define PART_RETRY 8
+#define PART_VALID 16
 #define CACHE_DIRTY 1
 #define CACHE_WRITTEN 2
 #define CACHE_FLUSHING 3

tests/test_snapshot_pool2.sh

@@ -13,13 +13,18 @@ build/src/cmd/vitastor-cli --etcd_address $ETCD_URL create -s 128M testchain -p
 
 LD_PRELOAD="build/src/client/libfio_vitastor.so" \
     fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=1M -direct=1 -iodepth=4 -fsync=1 -rw=write \
-        -etcd=$ETCD_URL -image=testchain -mirror_file=./testdata/bin/mirror.bin -buffer_pattern=0xabcd
+        -etcd=$ETCD_URL -image=testchain -mirror_file=./testdata/bin/mirror.bin -randrepeat=0
 
 build/src/cmd/vitastor-cli --etcd_address $ETCD_URL snap-create testchain@snap1 -p testpool2
 
 LD_PRELOAD="build/src/client/libfio_vitastor.so" \
     fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4k -direct=1 -iodepth=4 -end_fsync=1 -rw=randwrite -number_ios=32 \
-        -etcd=$ETCD_URL -image=testchain -mirror_file=./testdata/bin/mirror.bin -buffer_pattern=0xabcd
+        -etcd=$ETCD_URL -image=testchain -mirror_file=./testdata/bin/mirror.bin -randrepeat=0
+
+# Read from the first snapshot
+
+build/src/cmd/vitastor-cli --etcd_address $ETCD_URL dd iimg=testchain of=./testdata/bin/res.bin bs=128k iodepth=4 --log_level 10
+cmp ./testdata/bin/res.bin ./testdata/bin/mirror.bin
 
 # Create a second snapshot - there was a bug where snapshotted reads from another pool
 # were working only when the image and the snapshot were modified in the same revision
@@ -28,7 +33,7 @@ build/src/cmd/vitastor-cli --etcd_address $ETCD_URL snap-create testchain@snap2
 
 LD_PRELOAD="build/src/client/libfio_vitastor.so" \
     fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4k -direct=1 -iodepth=4 -end_fsync=1 -rw=randwrite -number_ios=32 \
-        -etcd=$ETCD_URL -image=testchain -mirror_file=./testdata/bin/mirror.bin -buffer_pattern=0xabcd
+        -etcd=$ETCD_URL -image=testchain -mirror_file=./testdata/bin/mirror.bin -randrepeat=0
 
 build/src/cmd/vitastor-cli --etcd_address $ETCD_URL dd iimg=testchain of=./testdata/bin/res.bin bs=128k iodepth=4 --log_level 10