From bcde273ca145da2c4a6e97c74203b15a7038223b Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Wed, 16 Jul 2025 00:44:09 +0300 Subject: [PATCH] Test calc_checksums --- src/blockstore/blockstore_heap.cpp | 38 ++++++++++++++--------------- src/blockstore/blockstore_write.cpp | 3 +++ src/test/test_heap.cpp | 30 +++++++++++++++-------- 3 files changed, 42 insertions(+), 29 deletions(-) diff --git a/src/blockstore/blockstore_heap.cpp b/src/blockstore/blockstore_heap.cpp index c5b0e1dc..b6b8c22a 100644 --- a/src/blockstore/blockstore_heap.cpp +++ b/src/blockstore/blockstore_heap.cpp @@ -563,24 +563,33 @@ bool blockstore_heap_t::calc_block_checksums(uint32_t *block_csums, uint8_t *dat uint32_t pos = start; uint32_t block_end = (start/dsk->csum_block_size + 1)*dsk->csum_block_size; uint32_t block_crc = 0; + bool isset = false; while (pos < end) { if (bitmap) { while (pos < end && pos < block_end) { - if (!bitmap[pos/dsk->bitmap_granularity/8] & (1 << ((pos/dsk->bitmap_granularity) % 8))) - block_crc = crc32c_pad(block_crc, NULL, 0, dsk->bitmap_granularity, 0); - else - block_crc = crc32c(block_crc, data, dsk->bitmap_granularity); - data += dsk->bitmap_granularity; - pos += dsk->bitmap_granularity; + uint32_t prev = pos; + while (pos < end && pos < block_end && !(bitmap[pos/dsk->bitmap_granularity/8] & (1 << ((pos/dsk->bitmap_granularity) % 8)))) + pos += dsk->bitmap_granularity; + if (pos > prev && (isset || pos < block_end)) + block_crc = crc32c_pad(block_crc, NULL, 0, pos-prev, 0); + prev = pos; + while (pos < end && pos < block_end && (bitmap[pos/dsk->bitmap_granularity/8] & (1 << ((pos/dsk->bitmap_granularity) % 8)))) + pos += dsk->bitmap_granularity; + if (pos > prev) + { + block_crc = crc32c(block_crc, data+prev-start, pos-prev); + isset = true; + } } } else { - block_crc = crc32c(block_crc, data, (end > block_end ? block_end : end) - pos); + block_crc = crc32c(block_crc, data+pos-start, (end > block_end ? block_end : end) - pos); pos = (end > block_end ? block_end : end); + isset = true; } if (set) { @@ -596,6 +605,9 @@ bool blockstore_heap_t::calc_block_checksums(uint32_t *block_csums, uint8_t *dat else return false; } + block_end += dsk->csum_block_size; + block_crc = 0; + isset = false; block_csums++; } return res; @@ -1143,12 +1155,6 @@ int blockstore_heap_t::add_object(object_id oid, heap_write_t *wr, uint32_t *mod new_wr->lsn = ++next_lsn; wr->lsn = new_wr->lsn; push_inflight_lsn(oid, new_wr->lsn, new_wr->needs_compact(this) ? HEAP_INFLIGHT_COMPACTABLE : 0); - if ((wr->flags & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE) - { - uint8_t *int_bitmap = new_wr->get_int_bitmap(this); - memset(int_bitmap, 0, dsk->clean_entry_bitmap_size); - bitmap_set(int_bitmap, wr->offset, wr->len, dsk->bitmap_granularity); - } new_entry->size = sizeof(heap_object_t); new_entry->crc32c = new_entry->calc_crc32c(); add_used_space(block_num, sizeof(heap_object_t) + wr_size); @@ -1360,12 +1366,6 @@ int blockstore_heap_t::update_object(uint32_t block_num, heap_object_t *obj, hea } wr->lsn = new_wr->lsn; push_inflight_lsn(oid, new_wr->lsn, new_wr->needs_compact(this) ? HEAP_INFLIGHT_COMPACTABLE : 0); - if ((wr->flags & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE) - { - uint8_t *int_bitmap = new_wr->get_int_bitmap(this); - memset(int_bitmap, 0, dsk->clean_entry_bitmap_size); - bitmap_set(int_bitmap, wr->offset, wr->len, dsk->bitmap_granularity); - } obj->write_pos = offset - ((uint8_t*)obj - inf.data); obj->crc32c = obj->calc_crc32c(); // Change block free space diff --git a/src/blockstore/blockstore_write.cpp b/src/blockstore/blockstore_write.cpp index 00462706..02f2b09c 100644 --- a/src/blockstore/blockstore_write.cpp +++ b/src/blockstore/blockstore_write.cpp @@ -3,6 +3,7 @@ #include "blockstore_impl.h" #include "blockstore_internal.h" +#include "allocator.h" bool blockstore_impl_t::enqueue_write(blockstore_op_t *op) { @@ -289,6 +290,8 @@ bool blockstore_impl_t::make_big_write(blockstore_op_t *op, uint32_t offset, uin wr->flags = BS_HEAP_BIG_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0); if (op->bitmap) memcpy(wr->get_ext_bitmap(heap), op->bitmap, dsk.clean_entry_bitmap_size); + memset(wr->get_int_bitmap(heap), 0, dsk.clean_entry_bitmap_size); + bitmap_set(wr->get_int_bitmap(heap), offset, len, dsk.bitmap_granularity); heap->calc_checksums(wr, (uint8_t*)op->buf, true); int res = heap->post_write(op->oid, wr, modified_block); if (res == ENOSPC) diff --git a/src/test/test_heap.cpp b/src/test/test_heap.cpp index 4370e31e..f88de36f 100644 --- a/src/test/test_heap.cpp +++ b/src/test/test_heap.cpp @@ -58,7 +58,7 @@ int count_free_fragments(blockstore_heap_t & heap, blockstore_disk_t & dsk, uint } int _test_do_big_write(blockstore_heap_t & heap, blockstore_disk_t & dsk, uint64_t inode, uint64_t stripe, uint64_t version, uint64_t location, - bool stable = true, uint32_t offset = 0, uint32_t len = 0) + bool stable = true, uint32_t offset = 0, uint32_t len = 0, uint32_t *checksums = NULL) { if (!offset && !len) len = dsk.data_block_size; @@ -74,17 +74,24 @@ int _test_do_big_write(blockstore_heap_t & heap, blockstore_disk_t & dsk, uint64 assert(wr->get_size(&heap) == sizeof(heap_write_t) + 2*dsk.clean_entry_bitmap_size + (dsk.csum_block_size ? dsk.data_block_size/dsk.csum_block_size*4 : 0)); memset(wr->get_ext_bitmap(&heap), 0xff, dsk.clean_entry_bitmap_size); + memset(wr->get_int_bitmap(&heap), 0, dsk.clean_entry_bitmap_size); + bitmap_set(wr->get_int_bitmap(&heap), offset, len, dsk.bitmap_granularity); if (dsk.csum_block_size) - memset(wr->get_checksums(&heap), 0xde, dsk.data_block_size/dsk.csum_block_size*4); + { + if (checksums) + memcpy(wr->get_checksums(&heap), checksums, dsk.data_block_size/dsk.csum_block_size*4); + else + memset(wr->get_checksums(&heap), 0xde, dsk.data_block_size/dsk.csum_block_size*4); + } uint32_t mblock; return heap.post_write(oid, wr, &mblock); } void _test_big_write(blockstore_heap_t & heap, blockstore_disk_t & dsk, uint64_t inode, uint64_t stripe, uint64_t version, uint64_t location, - bool stable = true, uint32_t offset = 0, uint32_t len = 0) + bool stable = true, uint32_t offset = 0, uint32_t len = 0, uint32_t *checksums = NULL) { heap.use_data(INODE_WITH_POOL(1, inode), location); // blocks are allocated before write and outside the heap_t - int res = _test_do_big_write(heap, dsk, inode, stripe, version, location, stable, offset, len); + int res = _test_do_big_write(heap, dsk, inode, stripe, version, location, stable, offset, len, checksums); assert(res == 0); assert(heap.is_data_used(location)); } @@ -323,7 +330,10 @@ void test_compact(bool csum, bool stable) blockstore_heap_t heap(&dsk, buffer_area.data()); heap.finish_load(); - _test_big_write(heap, dsk, 1, 0, 1, 0x20000, true, 0, 4096); + memset(buffer_area.data(), 0x19, 4096); + uint32_t csums[dsk.data_block_size/(dsk.csum_block_size ? dsk.csum_block_size : 4096)] = {}; + csums[0] = crc32c(0, buffer_area.data(), 4096); + _test_big_write(heap, dsk, 1, 0, 1, 0x20000, true, 0, 4096, csums); // write unstable - stabilize - compact object_id oid = { .inode = INODE_WITH_POOL(1, 1), .stripe = 0 }; @@ -338,7 +348,9 @@ void test_compact(bool csum, bool stable) assert(!memcmp(obj->get_writes()->get_int_bitmap(&heap), ref_int_bitmap, dsk.clean_entry_bitmap_size)); uint64_t old_size = obj->size + obj->get_writes()->size; - _test_small_write(heap, dsk, 1, 0, 3, 8192, 4096, 16384, stable); + memset(buffer_area.data()+8192, 0xAA, 4096); + csums[2] = crc32c(0, buffer_area.data()+8192, 4096); + _test_small_write(heap, dsk, 1, 0, 3, 8192, 4096, 16384, stable, &csums[2]); obj = heap.read_entry(oid, NULL); uint64_t wr_size = obj->get_writes()->get_size(&heap); assert(obj->get_writes()->lsn == 2); @@ -413,10 +425,8 @@ void test_compact(bool csum, bool stable) assert(!memcmp(obj->get_writes()->get_int_bitmap(&heap), ref_int_bitmap, dsk.clean_entry_bitmap_size)); if (csum) { - uint8_t ref_csums[dsk.data_block_size/dsk.csum_block_size*4]; - memset(ref_csums, 0xde, sizeof(ref_csums)); - memset(ref_csums+8, 0xab, 4); - assert(!memcmp(obj->get_writes()->get_checksums(&heap), ref_csums, sizeof(ref_csums))); + assert(heap.calc_checksums(obj->get_writes(), buffer_area.data(), false)); + assert(!memcmp(obj->get_writes()->get_checksums(&heap), csums, dsk.data_block_size/dsk.csum_block_size*4)); } obj = heap.read_entry({ .inode = INODE_WITH_POOL(1, 2), .stripe = 0 }, NULL);