thebestvibedev/tromcho.net
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Test calc_checksums

Browse Source
This commit is contained in:
Vitaliy Filippov
2025-12-02 01:52:12 +03:00
parent 87fe1bc00f
commit bcde273ca1
3 changed files with 42 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/blockstore/blockstore_heap.cpp
+19 -19
View File
@@ -563,24 +563,33 @@ bool blockstore_heap_t::calc_block_checksums(uint32_t *block_csums, uint8_t *dat
uint32_t pos = start; uint32_t pos = start;
uint32_t block_end = (start/dsk->csum_block_size + 1)*dsk->csum_block_size; uint32_t block_end = (start/dsk->csum_block_size + 1)*dsk->csum_block_size;
uint32_t block_crc = 0; uint32_t block_crc = 0;
bool isset = false;
while (pos < end) while (pos < end)
{ {
if (bitmap) if (bitmap)
{ {
while (pos < end && pos < block_end) while (pos < end && pos < block_end)
{ {
if (!bitmap[pos/dsk->bitmap_granularity/8] & (1 << ((pos/dsk->bitmap_granularity) % 8))) uint32_t prev = pos;
block_crc = crc32c_pad(block_crc, NULL, 0, dsk->bitmap_granularity, 0); while (pos < end && pos < block_end && !(bitmap[pos/dsk->bitmap_granularity/8] & (1 << ((pos/dsk->bitmap_granularity) % 8))))
else pos += dsk->bitmap_granularity;
block_crc = crc32c(block_crc, data, dsk->bitmap_granularity); if (pos > prev && (isset || pos < block_end))
data += dsk->bitmap_granularity; block_crc = crc32c_pad(block_crc, NULL, 0, pos-prev, 0);
pos += dsk->bitmap_granularity; prev = pos;
while (pos < end && pos < block_end && (bitmap[pos/dsk->bitmap_granularity/8] & (1 << ((pos/dsk->bitmap_granularity) % 8))))
pos += dsk->bitmap_granularity;
if (pos > prev)
{
block_crc = crc32c(block_crc, data+prev-start, pos-prev);
isset = true;
}
} }
} }
else else
{ {
block_crc = crc32c(block_crc, data, (end > block_end ? block_end : end) - pos); block_crc = crc32c(block_crc, data+pos-start, (end > block_end ? block_end : end) - pos);
pos = (end > block_end ? block_end : end); pos = (end > block_end ? block_end : end);
isset = true;
} }
if (set) if (set)
{ {
@@ -596,6 +605,9 @@ bool blockstore_heap_t::calc_block_checksums(uint32_t *block_csums, uint8_t *dat
else else
return false; return false;
} }
block_end += dsk->csum_block_size;
block_crc = 0;
isset = false;
block_csums++; block_csums++;
} }
return res; return res;
@@ -1143,12 +1155,6 @@ int blockstore_heap_t::add_object(object_id oid, heap_write_t *wr, uint32_t *mod
new_wr->lsn = ++next_lsn; new_wr->lsn = ++next_lsn;
wr->lsn = new_wr->lsn; wr->lsn = new_wr->lsn;
push_inflight_lsn(oid, new_wr->lsn, new_wr->needs_compact(this) ? HEAP_INFLIGHT_COMPACTABLE : 0); push_inflight_lsn(oid, new_wr->lsn, new_wr->needs_compact(this) ? HEAP_INFLIGHT_COMPACTABLE : 0);
if ((wr->flags & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE)
{
uint8_t *int_bitmap = new_wr->get_int_bitmap(this);
memset(int_bitmap, 0, dsk->clean_entry_bitmap_size);
bitmap_set(int_bitmap, wr->offset, wr->len, dsk->bitmap_granularity);
}
new_entry->size = sizeof(heap_object_t); new_entry->size = sizeof(heap_object_t);
new_entry->crc32c = new_entry->calc_crc32c(); new_entry->crc32c = new_entry->calc_crc32c();
add_used_space(block_num, sizeof(heap_object_t) + wr_size); add_used_space(block_num, sizeof(heap_object_t) + wr_size);
@@ -1360,12 +1366,6 @@ int blockstore_heap_t::update_object(uint32_t block_num, heap_object_t *obj, hea
} }
wr->lsn = new_wr->lsn; wr->lsn = new_wr->lsn;
push_inflight_lsn(oid, new_wr->lsn, new_wr->needs_compact(this) ? HEAP_INFLIGHT_COMPACTABLE : 0); push_inflight_lsn(oid, new_wr->lsn, new_wr->needs_compact(this) ? HEAP_INFLIGHT_COMPACTABLE : 0);
if ((wr->flags & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE)
{
uint8_t *int_bitmap = new_wr->get_int_bitmap(this);
memset(int_bitmap, 0, dsk->clean_entry_bitmap_size);
bitmap_set(int_bitmap, wr->offset, wr->len, dsk->bitmap_granularity);
}
obj->write_pos = offset - ((uint8_t*)obj - inf.data); obj->write_pos = offset - ((uint8_t*)obj - inf.data);
obj->crc32c = obj->calc_crc32c(); obj->crc32c = obj->calc_crc32c();
// Change block free space // Change block free space
src/blockstore/blockstore_write.cpp
+3
View File
@@ -3,6 +3,7 @@
#include "blockstore_impl.h" #include "blockstore_impl.h"
#include "blockstore_internal.h" #include "blockstore_internal.h"
#include "allocator.h"
bool blockstore_impl_t::enqueue_write(blockstore_op_t *op) bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
{ {
@@ -289,6 +290,8 @@ bool blockstore_impl_t::make_big_write(blockstore_op_t *op, uint32_t offset, uin
wr->flags = BS_HEAP_BIG_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0); wr->flags = BS_HEAP_BIG_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0);
if (op->bitmap) if (op->bitmap)
memcpy(wr->get_ext_bitmap(heap), op->bitmap, dsk.clean_entry_bitmap_size); memcpy(wr->get_ext_bitmap(heap), op->bitmap, dsk.clean_entry_bitmap_size);
memset(wr->get_int_bitmap(heap), 0, dsk.clean_entry_bitmap_size);
bitmap_set(wr->get_int_bitmap(heap), offset, len, dsk.bitmap_granularity);
heap->calc_checksums(wr, (uint8_t*)op->buf, true); heap->calc_checksums(wr, (uint8_t*)op->buf, true);
int res = heap->post_write(op->oid, wr, modified_block); int res = heap->post_write(op->oid, wr, modified_block);
if (res == ENOSPC) if (res == ENOSPC)
src/test/test_heap.cpp
+20 -10
View File
@@ -58,7 +58,7 @@ int count_free_fragments(blockstore_heap_t & heap, blockstore_disk_t & dsk, uint
} }
int _test_do_big_write(blockstore_heap_t & heap, blockstore_disk_t & dsk, uint64_t inode, uint64_t stripe, uint64_t version, uint64_t location, int _test_do_big_write(blockstore_heap_t & heap, blockstore_disk_t & dsk, uint64_t inode, uint64_t stripe, uint64_t version, uint64_t location,
bool stable = true, uint32_t offset = 0, uint32_t len = 0) bool stable = true, uint32_t offset = 0, uint32_t len = 0, uint32_t *checksums = NULL)
{ {
if (!offset && !len) if (!offset && !len)
len = dsk.data_block_size; len = dsk.data_block_size;
@@ -74,17 +74,24 @@ int _test_do_big_write(blockstore_heap_t & heap, blockstore_disk_t & dsk, uint64
assert(wr->get_size(&heap) == sizeof(heap_write_t) + 2*dsk.clean_entry_bitmap_size + (dsk.csum_block_size assert(wr->get_size(&heap) == sizeof(heap_write_t) + 2*dsk.clean_entry_bitmap_size + (dsk.csum_block_size
? dsk.data_block_size/dsk.csum_block_size*4 : 0)); ? dsk.data_block_size/dsk.csum_block_size*4 : 0));
memset(wr->get_ext_bitmap(&heap), 0xff, dsk.clean_entry_bitmap_size); memset(wr->get_ext_bitmap(&heap), 0xff, dsk.clean_entry_bitmap_size);
memset(wr->get_int_bitmap(&heap), 0, dsk.clean_entry_bitmap_size);
bitmap_set(wr->get_int_bitmap(&heap), offset, len, dsk.bitmap_granularity);
if (dsk.csum_block_size) if (dsk.csum_block_size)
memset(wr->get_checksums(&heap), 0xde, dsk.data_block_size/dsk.csum_block_size*4); {
if (checksums)
memcpy(wr->get_checksums(&heap), checksums, dsk.data_block_size/dsk.csum_block_size*4);
else
memset(wr->get_checksums(&heap), 0xde, dsk.data_block_size/dsk.csum_block_size*4);
}
uint32_t mblock; uint32_t mblock;
return heap.post_write(oid, wr, &mblock); return heap.post_write(oid, wr, &mblock);
} }
void _test_big_write(blockstore_heap_t & heap, blockstore_disk_t & dsk, uint64_t inode, uint64_t stripe, uint64_t version, uint64_t location, void _test_big_write(blockstore_heap_t & heap, blockstore_disk_t & dsk, uint64_t inode, uint64_t stripe, uint64_t version, uint64_t location,
bool stable = true, uint32_t offset = 0, uint32_t len = 0) bool stable = true, uint32_t offset = 0, uint32_t len = 0, uint32_t *checksums = NULL)
{ {
heap.use_data(INODE_WITH_POOL(1, inode), location); // blocks are allocated before write and outside the heap_t heap.use_data(INODE_WITH_POOL(1, inode), location); // blocks are allocated before write and outside the heap_t
int res = _test_do_big_write(heap, dsk, inode, stripe, version, location, stable, offset, len); int res = _test_do_big_write(heap, dsk, inode, stripe, version, location, stable, offset, len, checksums);
assert(res == 0); assert(res == 0);
assert(heap.is_data_used(location)); assert(heap.is_data_used(location));
} }
@@ -323,7 +330,10 @@ void test_compact(bool csum, bool stable)
blockstore_heap_t heap(&dsk, buffer_area.data()); blockstore_heap_t heap(&dsk, buffer_area.data());
heap.finish_load(); heap.finish_load();
_test_big_write(heap, dsk, 1, 0, 1, 0x20000, true, 0, 4096); memset(buffer_area.data(), 0x19, 4096);
uint32_t csums[dsk.data_block_size/(dsk.csum_block_size ? dsk.csum_block_size : 4096)] = {};
csums[0] = crc32c(0, buffer_area.data(), 4096);
_test_big_write(heap, dsk, 1, 0, 1, 0x20000, true, 0, 4096, csums);
// write unstable - stabilize - compact // write unstable - stabilize - compact
object_id oid = { .inode = INODE_WITH_POOL(1, 1), .stripe = 0 }; object_id oid = { .inode = INODE_WITH_POOL(1, 1), .stripe = 0 };
@@ -338,7 +348,9 @@ void test_compact(bool csum, bool stable)
assert(!memcmp(obj->get_writes()->get_int_bitmap(&heap), ref_int_bitmap, dsk.clean_entry_bitmap_size)); assert(!memcmp(obj->get_writes()->get_int_bitmap(&heap), ref_int_bitmap, dsk.clean_entry_bitmap_size));
uint64_t old_size = obj->size + obj->get_writes()->size; uint64_t old_size = obj->size + obj->get_writes()->size;
_test_small_write(heap, dsk, 1, 0, 3, 8192, 4096, 16384, stable); memset(buffer_area.data()+8192, 0xAA, 4096);
csums[2] = crc32c(0, buffer_area.data()+8192, 4096);
_test_small_write(heap, dsk, 1, 0, 3, 8192, 4096, 16384, stable, &csums[2]);
obj = heap.read_entry(oid, NULL); obj = heap.read_entry(oid, NULL);
uint64_t wr_size = obj->get_writes()->get_size(&heap); uint64_t wr_size = obj->get_writes()->get_size(&heap);
assert(obj->get_writes()->lsn == 2); assert(obj->get_writes()->lsn == 2);
@@ -413,10 +425,8 @@ void test_compact(bool csum, bool stable)
assert(!memcmp(obj->get_writes()->get_int_bitmap(&heap), ref_int_bitmap, dsk.clean_entry_bitmap_size)); assert(!memcmp(obj->get_writes()->get_int_bitmap(&heap), ref_int_bitmap, dsk.clean_entry_bitmap_size));
if (csum) if (csum)
{ {
uint8_t ref_csums[dsk.data_block_size/dsk.csum_block_size*4]; assert(heap.calc_checksums(obj->get_writes(), buffer_area.data(), false));
memset(ref_csums, 0xde, sizeof(ref_csums)); assert(!memcmp(obj->get_writes()->get_checksums(&heap), csums, dsk.data_block_size/dsk.csum_block_size*4));
memset(ref_csums+8, 0xab, 4);
assert(!memcmp(obj->get_writes()->get_checksums(&heap), ref_csums, sizeof(ref_csums)));
} }
obj = heap.read_entry({ .inode = INODE_WITH_POOL(1, 2), .stripe = 0 }, NULL); obj = heap.read_entry({ .inode = INODE_WITH_POOL(1, 2), .stripe = 0 }, NULL);