thebestvibedev/tromcho.net
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Test calc_checksums

Browse Source
This commit is contained in:
Vitaliy Filippov
2025-12-02 01:52:12 +03:00
parent 87fe1bc00f
commit bcde273ca1
3 changed files with 42 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/blockstore/blockstore_heap.cpp
+19 -19
View File
@@ -563,24 +563,33 @@ bool blockstore_heap_t::calc_block_checksums(uint32_t *block_csums, uint8_t *dat
uint32_t pos = start;
uint32_t block_end = (start/dsk->csum_block_size + 1)*dsk->csum_block_size;
uint32_t block_crc = 0;
bool isset = false;
while (pos < end)
{
if (bitmap)
{
while (pos < end && pos < block_end)
{
if (!bitmap[pos/dsk->bitmap_granularity/8] & (1 << ((pos/dsk->bitmap_granularity) % 8)))
block_crc = crc32c_pad(block_crc, NULL, 0, dsk->bitmap_granularity, 0);
else
block_crc = crc32c(block_crc, data, dsk->bitmap_granularity);
data += dsk->bitmap_granularity;
pos += dsk->bitmap_granularity;
uint32_t prev = pos;
while (pos < end && pos < block_end && !(bitmap[pos/dsk->bitmap_granularity/8] & (1 << ((pos/dsk->bitmap_granularity) % 8))))
pos += dsk->bitmap_granularity;
if (pos > prev && (isset || pos < block_end))
block_crc = crc32c_pad(block_crc, NULL, 0, pos-prev, 0);
prev = pos;
while (pos < end && pos < block_end && (bitmap[pos/dsk->bitmap_granularity/8] & (1 << ((pos/dsk->bitmap_granularity) % 8))))
pos += dsk->bitmap_granularity;
if (pos > prev)
{
block_crc = crc32c(block_crc, data+prev-start, pos-prev);
isset = true;
}
}
}
else
{
block_crc = crc32c(block_crc, data, (end > block_end ? block_end : end) - pos);
block_crc = crc32c(block_crc, data+pos-start, (end > block_end ? block_end : end) - pos);
pos = (end > block_end ? block_end : end);
isset = true;
}
if (set)
{
@@ -596,6 +605,9 @@ bool blockstore_heap_t::calc_block_checksums(uint32_t *block_csums, uint8_t *dat
else
return false;
}
block_end += dsk->csum_block_size;
block_crc = 0;
isset = false;
block_csums++;
}
return res;
@@ -1143,12 +1155,6 @@ int blockstore_heap_t::add_object(object_id oid, heap_write_t *wr, uint32_t *mod
new_wr->lsn = ++next_lsn;
wr->lsn = new_wr->lsn;
push_inflight_lsn(oid, new_wr->lsn, new_wr->needs_compact(this) ? HEAP_INFLIGHT_COMPACTABLE : 0);
if ((wr->flags & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE)
{
uint8_t *int_bitmap = new_wr->get_int_bitmap(this);
memset(int_bitmap, 0, dsk->clean_entry_bitmap_size);
bitmap_set(int_bitmap, wr->offset, wr->len, dsk->bitmap_granularity);
}
new_entry->size = sizeof(heap_object_t);
new_entry->crc32c = new_entry->calc_crc32c();
add_used_space(block_num, sizeof(heap_object_t) + wr_size);
@@ -1360,12 +1366,6 @@ int blockstore_heap_t::update_object(uint32_t block_num, heap_object_t *obj, hea
}
wr->lsn = new_wr->lsn;
push_inflight_lsn(oid, new_wr->lsn, new_wr->needs_compact(this) ? HEAP_INFLIGHT_COMPACTABLE : 0);
if ((wr->flags & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE)
{
uint8_t *int_bitmap = new_wr->get_int_bitmap(this);
memset(int_bitmap, 0, dsk->clean_entry_bitmap_size);
bitmap_set(int_bitmap, wr->offset, wr->len, dsk->bitmap_granularity);
}
obj->write_pos = offset - ((uint8_t*)obj - inf.data);
obj->crc32c = obj->calc_crc32c();
// Change block free space
src/blockstore/blockstore_write.cpp
+3
View File
@@ -3,6 +3,7 @@
#include "blockstore_impl.h"
#include "blockstore_internal.h"
#include "allocator.h"
bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
{
@@ -289,6 +290,8 @@ bool blockstore_impl_t::make_big_write(blockstore_op_t *op, uint32_t offset, uin
wr->flags = BS_HEAP_BIG_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0);
if (op->bitmap)
memcpy(wr->get_ext_bitmap(heap), op->bitmap, dsk.clean_entry_bitmap_size);
memset(wr->get_int_bitmap(heap), 0, dsk.clean_entry_bitmap_size);
bitmap_set(wr->get_int_bitmap(heap), offset, len, dsk.bitmap_granularity);
heap->calc_checksums(wr, (uint8_t*)op->buf, true);
int res = heap->post_write(op->oid, wr, modified_block);
if (res == ENOSPC)
src/test/test_heap.cpp
+20 -10
View File
@@ -58,7 +58,7 @@ int count_free_fragments(blockstore_heap_t & heap, blockstore_disk_t & dsk, uint
}
int _test_do_big_write(blockstore_heap_t & heap, blockstore_disk_t & dsk, uint64_t inode, uint64_t stripe, uint64_t version, uint64_t location,
bool stable = true, uint32_t offset = 0, uint32_t len = 0)
bool stable = true, uint32_t offset = 0, uint32_t len = 0, uint32_t *checksums = NULL)
{
if (!offset && !len)
len = dsk.data_block_size;
@@ -74,17 +74,24 @@ int _test_do_big_write(blockstore_heap_t & heap, blockstore_disk_t & dsk, uint64
assert(wr->get_size(&heap) == sizeof(heap_write_t) + 2*dsk.clean_entry_bitmap_size + (dsk.csum_block_size
? dsk.data_block_size/dsk.csum_block_size*4 : 0));
memset(wr->get_ext_bitmap(&heap), 0xff, dsk.clean_entry_bitmap_size);
memset(wr->get_int_bitmap(&heap), 0, dsk.clean_entry_bitmap_size);
bitmap_set(wr->get_int_bitmap(&heap), offset, len, dsk.bitmap_granularity);
if (dsk.csum_block_size)
memset(wr->get_checksums(&heap), 0xde, dsk.data_block_size/dsk.csum_block_size*4);
{
if (checksums)
memcpy(wr->get_checksums(&heap), checksums, dsk.data_block_size/dsk.csum_block_size*4);
else
memset(wr->get_checksums(&heap), 0xde, dsk.data_block_size/dsk.csum_block_size*4);
}
uint32_t mblock;
return heap.post_write(oid, wr, &mblock);
}
void _test_big_write(blockstore_heap_t & heap, blockstore_disk_t & dsk, uint64_t inode, uint64_t stripe, uint64_t version, uint64_t location,
bool stable = true, uint32_t offset = 0, uint32_t len = 0)
bool stable = true, uint32_t offset = 0, uint32_t len = 0, uint32_t *checksums = NULL)
{
heap.use_data(INODE_WITH_POOL(1, inode), location); // blocks are allocated before write and outside the heap_t
int res = _test_do_big_write(heap, dsk, inode, stripe, version, location, stable, offset, len);
int res = _test_do_big_write(heap, dsk, inode, stripe, version, location, stable, offset, len, checksums);
assert(res == 0);
assert(heap.is_data_used(location));
}
@@ -323,7 +330,10 @@ void test_compact(bool csum, bool stable)
blockstore_heap_t heap(&dsk, buffer_area.data());
heap.finish_load();
_test_big_write(heap, dsk, 1, 0, 1, 0x20000, true, 0, 4096);
memset(buffer_area.data(), 0x19, 4096);
uint32_t csums[dsk.data_block_size/(dsk.csum_block_size ? dsk.csum_block_size : 4096)] = {};
csums[0] = crc32c(0, buffer_area.data(), 4096);
_test_big_write(heap, dsk, 1, 0, 1, 0x20000, true, 0, 4096, csums);
// write unstable - stabilize - compact
object_id oid = { .inode = INODE_WITH_POOL(1, 1), .stripe = 0 };
@@ -338,7 +348,9 @@ void test_compact(bool csum, bool stable)
assert(!memcmp(obj->get_writes()->get_int_bitmap(&heap), ref_int_bitmap, dsk.clean_entry_bitmap_size));
uint64_t old_size = obj->size + obj->get_writes()->size;
_test_small_write(heap, dsk, 1, 0, 3, 8192, 4096, 16384, stable);
memset(buffer_area.data()+8192, 0xAA, 4096);
csums[2] = crc32c(0, buffer_area.data()+8192, 4096);
_test_small_write(heap, dsk, 1, 0, 3, 8192, 4096, 16384, stable, &csums[2]);
obj = heap.read_entry(oid, NULL);
uint64_t wr_size = obj->get_writes()->get_size(&heap);
assert(obj->get_writes()->lsn == 2);
@@ -413,10 +425,8 @@ void test_compact(bool csum, bool stable)
assert(!memcmp(obj->get_writes()->get_int_bitmap(&heap), ref_int_bitmap, dsk.clean_entry_bitmap_size));
if (csum)
{
uint8_t ref_csums[dsk.data_block_size/dsk.csum_block_size*4];
memset(ref_csums, 0xde, sizeof(ref_csums));
memset(ref_csums+8, 0xab, 4);
assert(!memcmp(obj->get_writes()->get_checksums(&heap), ref_csums, sizeof(ref_csums)));
assert(heap.calc_checksums(obj->get_writes(), buffer_area.data(), false));
assert(!memcmp(obj->get_writes()->get_checksums(&heap), csums, dsk.data_block_size/dsk.csum_block_size*4));
}
obj = heap.read_entry({ .inode = INODE_WITH_POOL(1, 2), .stripe = 0 }, NULL);