From f580cee9363ea33aed5ea929b30c49f89b9b90f5 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sat, 28 Jun 2025 02:17:30 +0300 Subject: [PATCH] Implement a really crazy "multi-linked-list" allocator for buffered data --- src/blockstore/blockstore_heap.cpp | 281 ++++++++++++++++++++++------- src/blockstore/blockstore_heap.h | 37 ++-- src/test/test_heap.cpp | 125 ++++++++++--- 3 files changed, 330 insertions(+), 113 deletions(-) diff --git a/src/blockstore/blockstore_heap.cpp b/src/blockstore/blockstore_heap.cpp index 2cbc2279..f931c147 100644 --- a/src/blockstore/blockstore_heap.cpp +++ b/src/blockstore/blockstore_heap.cpp @@ -122,6 +122,193 @@ uint32_t heap_object_t::calc_crc32c() return res; } +multilist_alloc_t::multilist_alloc_t(uint32_t count, uint32_t maxn): + count(count), maxn(maxn) +{ + // not-so-memory-efficient: 16 MB memory per 1 GB buffer space, but buffer spaces are small, so OK + assert(count > 1 && count < 0x80000000); + sizes.resize(count); + nexts.resize(count); // nexts[i] = 0 -> area is used; nexts[i] = 1 -> no next; nexts[i] >= 2 -> next item + prevs.resize(count); + heads.resize(maxn); // heads[i] = 0 -> empty list; heads[i] >= 1 -> list head + sizes[0] = count; + sizes[count-1] = -count; // end + nexts[0] = 1; + heads[maxn-1] = 1; +#ifdef MULTILIST_TEST + print(); +#endif +} + +bool multilist_alloc_t::is_free(uint32_t pos) +{ + assert(pos < count); + if (sizes[pos] < 0) + pos += sizes[pos]+1; + while (pos > 0 && !sizes[pos]) + pos--; + return nexts[pos] > 0; +} + +uint32_t multilist_alloc_t::allocate(uint32_t size) +{ + assert(size > 0); + assert(size <= maxn); + for (uint32_t i = size-1; i < maxn; i++) + { + if (heads[i]) + { + uint32_t res = heads[i]-1; + assert(nexts[res] >= 1); + heads[i] = nexts[res]-1; + uint32_t area = sizes[res]; + assert(area >= size); + if (area == size) + { + nexts[res] = 0; + } + else + { + uint32_t ni = area-size; + ni = (ni < maxn ? ni : maxn)-1; + sizes[res+size-1] = -size; + sizes[res] = size; + sizes[res+size-1] = -area+size; + sizes[res+size] = area-size; + nexts[res+size] = heads[ni]+1; + prevs[heads[ni]-1] = res+size+1; + assert(!prevs[res+size]); + heads[ni] = res+size+1; + } +#ifdef MULTILIST_TEST + print(); +#endif + return res; + } + } + return UINT32_MAX; +} + +uint32_t multilist_alloc_t::find(uint32_t size) +{ + assert(size > 0); + assert(size <= maxn); + for (uint32_t i = size-1; i < maxn; i++) + { + if (heads[i]) + { + return heads[i]-1; + } + } + return UINT32_MAX; +} + +void multilist_alloc_t::use(uint32_t pos, uint32_t size) +{ + assert(pos < count); + if (sizes[pos] <= 0) + { + uint32_t start = pos; + if (sizes[start] < 0) + start += sizes[start]+1; + else + while (start > 0 && !sizes[start]) + start--; + assert(sizes[start] >= size); + use_full(start); + uint32_t full = sizes[start]; + sizes[pos-1] = -pos+start; + sizes[start] = pos-start; + free(start); + sizes[pos+size-1] = -size; + sizes[pos] = size; + if (pos+size < start+full) + { + sizes[start+full-1] = -(full-pos-size); + sizes[pos+size] = full-pos-size; + free(pos+size); + } + } + else + { + assert(sizes[pos] >= size); + use_full(pos); + if (sizes[pos] > size) + { + uint32_t full = sizes[pos]; + sizes[pos+size-1] = -size; + sizes[pos] = size; + sizes[pos+full-1] = -full+size; + sizes[pos+size] = full-size; + free(pos+size); + } + } +#ifdef MULTILIST_TEST + print(); +#endif +} + +void multilist_alloc_t::use_full(uint32_t pos) +{ + uint32_t prevsize = sizes[pos]; + assert(prevsize); + assert(nexts[pos]); + uint32_t pi = (prevsize < maxn ? prevsize : maxn)-1; + if (heads[pi] == pos+1) + heads[pi] = nexts[pos]-1; + if (prevs[pos]) + nexts[prevs[pos]-1] = nexts[pos]; + if (nexts[pos] >= 2) + prevs[nexts[pos]-2] = prevs[pos]; + prevs[pos] = 0; + nexts[pos] = 0; +} + +void multilist_alloc_t::free(uint32_t pos) +{ + do_free(pos); +#ifdef MULTILIST_TEST + print(); +#endif +} + +void multilist_alloc_t::do_free(uint32_t pos) +{ + assert(!nexts[pos]); + uint32_t size = sizes[pos]; + assert(size > 0); + // merge with previous? + if (pos > 0 && nexts[pos+(sizes[pos-1] == 1 ? -1 : sizes[pos-1])] > 0) + { + assert(sizes[pos-1] < 0 || sizes[pos-1] == 1); + uint32_t prevsize = sizes[pos-1] < 0 ? -sizes[pos-1] : 1; + use_full(pos-prevsize); + sizes[pos] = 0; + sizes[pos-1] = 0; + size += prevsize; + pos -= prevsize; + sizes[pos+size-1] = -size; + sizes[pos] = size; + } + // merge with next? + if (pos+size < count && nexts[pos+size] >= 1) + { + uint32_t nextsize = sizes[pos+size]; + use_full(pos+size); + sizes[pos+size] = 0; + sizes[pos+size-1] = 0; + size += nextsize; + sizes[pos+size-1] = -size; + sizes[pos] = size; + } + uint32_t ni = (size < maxn ? size : maxn)-1; // FIXME ni -> nb (next bucket) + nexts[pos] = heads[ni]+1; + prevs[pos] = 0; + if (heads[ni]) + prevs[heads[ni]-1] = pos+1; + heads[ni] = pos+1; +} + uint64_t blockstore_heap_t::get_pg_id(inode_t inode, uint64_t stripe) { uint64_t pg_num = 0; @@ -149,11 +336,10 @@ blockstore_heap_t::blockstore_heap_t(blockstore_disk_t *dsk, uint8_t *buffer_are assert(sizeof(heap_object_t) < sizeof(heap_write_t)); meta_alloc = new allocator_t(meta_block_count); block_info.resize(meta_block_count); - buffer_by_end.insert((heap_extent_t){ .start = 0, .end = dsk->journal_len }); - buffer_by_size.insert((heap_extent_t){ .start = 0, .end = dsk->journal_len }); data_alloc = new allocator_t(dsk->block_count); if (!target_block_free_space) target_block_free_space = 800; + buffer_alloc = new multilist_alloc_t(dsk->journal_len / dsk->bitmap_granularity, dsk->data_block_size / dsk->bitmap_granularity - 1); } blockstore_heap_t::~blockstore_heap_t() @@ -182,6 +368,10 @@ blockstore_heap_t::~blockstore_heap_t() { delete data_alloc; } + if (buffer_alloc) + { + delete buffer_alloc; + } } // set initially compacted lsn - should be done before loading @@ -1678,87 +1868,44 @@ void blockstore_heap_t::use_data(inode_t inode, uint64_t location) uint64_t blockstore_heap_t::find_free_buffer_area(uint64_t size) { - auto free_it = buffer_by_size.lower_bound((heap_extent_t){ .start = 0, .end = size }); - if (free_it == buffer_by_size.end()) + assert(!(size % dsk->bitmap_granularity)); + uint32_t pos = buffer_alloc->find(size / dsk->bitmap_granularity); + if (pos == UINT32_MAX) { return UINT64_MAX; } - return free_it->start; + return pos * dsk->bitmap_granularity; } bool blockstore_heap_t::is_buffer_area_free(uint64_t location, uint64_t size) { - auto free_it = buffer_by_end.lower_bound((heap_extent_t){ .end = location+size }); - return (free_it != buffer_by_end.end() && free_it->start <= location); + assert(!(location % dsk->bitmap_granularity)); + return buffer_alloc->is_free(location / dsk->bitmap_granularity); +} + +uint64_t blockstore_heap_t::alloc_buffer_area(inode_t inode, uint64_t size) +{ + assert(!(size % dsk->bitmap_granularity)); + uint32_t res = buffer_alloc->allocate(size / dsk->bitmap_granularity); + if (res == UINT32_MAX) + { + return UINT64_MAX; + } + buffer_area_used_space += size; + return res * dsk->bitmap_granularity; } void blockstore_heap_t::use_buffer_area(inode_t inode, uint64_t location, uint64_t size) { - auto free_it = buffer_by_end.lower_bound((heap_extent_t){ .end = location+size }); - assert(free_it != buffer_by_end.end() && free_it->start <= location && free_it->end >= location+size); - heap_extent_t extent = *free_it; - buffer_by_end.erase(free_it); - buffer_by_size.erase(extent); - if (extent.start == location) - { - extent.start += size; - buffer_by_end.insert(extent); - buffer_by_size.insert(extent); - } - else if (extent.end == location+size) - { - extent.end -= size; - buffer_by_end.insert(extent); - buffer_by_size.insert(extent); - } - else - { - buffer_by_end.insert((heap_extent_t){ .start = extent.start, .end = location }); - buffer_by_size.insert((heap_extent_t){ .start = extent.start, .end = location }); - buffer_by_end.insert((heap_extent_t){ .start = location+size, .end = extent.end }); - buffer_by_size.insert((heap_extent_t){ .start = location+size, .end = extent.end }); - } + assert(!(size % dsk->bitmap_granularity)); + buffer_alloc->use(location / dsk->bitmap_granularity, size / dsk->bitmap_granularity); buffer_area_used_space += size; } void blockstore_heap_t::free_buffer_area(inode_t inode, uint64_t location, uint64_t size) { - auto next_it = buffer_by_end.lower_bound((heap_extent_t){ .end = location+size }); - auto prev_it = next_it == buffer_by_end.begin() ? buffer_by_end.end() : std::prev(next_it); - assert(next_it == buffer_by_end.end() || next_it->start >= location+size); - assert(prev_it == buffer_by_end.end() || prev_it->end <= location); - bool merge_prev = (prev_it != buffer_by_end.end() && prev_it->end == location); - bool merge_next = (next_it != buffer_by_end.end() && next_it->start == location+size); - uint64_t prev_start = merge_prev ? prev_it->start : 0; - uint64_t next_end = merge_next ? next_it->end : 0; - if (merge_prev && merge_next) - { - buffer_by_size.erase(*prev_it); - buffer_by_size.erase(*next_it); - buffer_by_end.erase(prev_it); - buffer_by_end.erase(next_it); - buffer_by_end.insert((heap_extent_t){ .start = prev_start, .end = next_end }); - buffer_by_size.insert((heap_extent_t){ .start = prev_start, .end = next_end }); - } - else if (merge_prev) - { - buffer_by_size.erase(*prev_it); - buffer_by_end.erase(prev_it); - buffer_by_end.insert((heap_extent_t){ .start = prev_start, .end = location+size }); - buffer_by_size.insert((heap_extent_t){ .start = prev_start, .end = location+size }); - } - else if (merge_next) - { - buffer_by_size.erase(*next_it); - buffer_by_end.erase(next_it); - buffer_by_end.insert((heap_extent_t){ .start = location, .end = next_end }); - buffer_by_size.insert((heap_extent_t){ .start = location, .end = next_end }); - } - else - { - buffer_by_end.insert((heap_extent_t){ .start = location, .end = location+size }); - buffer_by_size.insert((heap_extent_t){ .start = location, .end = location+size }); - } + assert(!(location % dsk->bitmap_granularity)); + buffer_alloc->free(location / dsk->bitmap_granularity); buffer_area_used_space -= size; } diff --git a/src/blockstore/blockstore_heap.h b/src/blockstore/blockstore_heap.h index 407ab0aa..764a924c 100644 --- a/src/blockstore/blockstore_heap.h +++ b/src/blockstore/blockstore_heap.h @@ -107,26 +107,23 @@ inline bool operator < (const heap_block_free_t & a, const heap_block_free_t & b return a.free_space > b.free_space || a.free_space == b.free_space && a.block_num < b.block_num; } -struct heap_extent_t +struct multilist_alloc_t { - uint64_t start = 0; - uint64_t end = 0; -}; + const uint32_t count, maxn; + std::vector sizes; + std::vector nexts, prevs, heads; -struct heap_less_extent_by_end -{ - const bool operator()(const heap_extent_t & a, const heap_extent_t & b) const - { - return a.end < b.end; - } -}; - -struct heap_less_extent_by_size -{ - const bool operator()(const heap_extent_t & a, const heap_extent_t & b) const - { - return a.end-a.start < b.end-b.start || a.end-a.start == b.end-b.start && a.start < b.start; - } + multilist_alloc_t(uint32_t count, uint32_t maxn); + bool is_free(uint32_t pos); + uint32_t allocate(uint32_t size); + uint32_t find(uint32_t size); + void use_full(uint32_t pos); + void use(uint32_t pos, uint32_t size); + void do_free(uint32_t pos); + void free(uint32_t pos); +#ifdef MULTILIST_TEST + void print(); +#endif }; class blockstore_heap_t @@ -154,8 +151,7 @@ class blockstore_heap_t allocator_t *meta_alloc = NULL; uint32_t meta_alloc_count = 0; uint64_t meta_used_space = 0; - std::set buffer_by_end; - std::set buffer_by_size; + multilist_alloc_t *buffer_alloc = NULL; std::set used_alloc_queue; std::map object_mvcc; std::map mvcc_data_refs; @@ -255,6 +251,7 @@ public: // buffer device allocator functions uint64_t find_free_buffer_area(uint64_t size); bool is_buffer_area_free(uint64_t location, uint64_t size); + uint64_t alloc_buffer_area(inode_t inode, uint64_t size); void use_buffer_area(inode_t inode, uint64_t location, uint64_t size); void free_buffer_area(inode_t inode, uint64_t location, uint64_t size); uint64_t get_buffer_area_used_space(); diff --git a/src/test/test_heap.cpp b/src/test/test_heap.cpp index 7ae223e9..7aafc6e9 100644 --- a/src/test/test_heap.cpp +++ b/src/test/test_heap.cpp @@ -9,6 +9,74 @@ #include "blockstore_heap.h" #include "../util/crc32c.h" +#ifdef MULTILIST_TEST +void multilist_alloc_t::print() +{ + printf("heads:"); + for (int i = 0; i < maxn; i++) + if (heads[i]) + printf(" %u=%u", i, heads[i]); + for (int i = 0; i < maxn; i++) + if (heads[i]) + assert(i < maxn-1 ? sizes[heads[i]-1] == i+1 : (sizes[heads[i]-1] >= i+1)); + printf("\n"); + printf("sizes:"); + for (int i = 0; i < count; i++) + if (sizes[i]) + printf(" %d=%d", i, sizes[i]); + printf("\n"); + printf("prevs:"); + for (int i = 0; i < count; i++) + if (prevs[i]) + printf(" %d=%d", i, prevs[i]); + printf("\n"); + printf("nexts:"); + for (int i = 0; i < count; i++) + if (nexts[i]) + printf(" %d=%d", i, nexts[i]); + printf("\n"); + printf("items:"); + for (int i = 0; i < count; ) + { + if (sizes[i]) + { + printf(" %u=(s:%d,n:%u,p:%u)", i, sizes[i], nexts[i], prevs[i]); + assert(i+sizes[i] <= count); + if (sizes[i] > 1 && sizes[i+sizes[i]-1] != -sizes[i]) + { + printf(" ERROR: start/end mismatch\n"); + abort(); + } + for (int j = i+1; j < i+sizes[i]-1; j++) + { + if (sizes[j]) + { + printf(" ERROR: internal non-zero at %d: %d\n", j, sizes[j]); + abort(); + } + } + if (nexts[i] >= 2) + { + if (nexts[i] >= 2+count) + { + printf(" ERROR: next out of range\n"); + abort(); + } + if (prevs[nexts[i]-2] != i+1) + { + printf(" ERROR: prev[next] != this"); + abort(); + } + } + i += sizes[i]; + } + else + i++; + } + printf("\n"); +} +#endif + static int count_writes(heap_object_t *obj) { int n = 0; @@ -1015,39 +1083,44 @@ void test_alloc_buffer() uint64_t pos; - pos = heap.find_free_buffer_area(4*1024*1024+1); + for (int i = 0; i < 4096/64; i++) + { + pos = heap.find_free_buffer_area(64*1024); + assert(pos == i*64*1024); + heap.use_buffer_area(1, pos, 64*1024); + assert(heap.get_buffer_area_used_space() == (i+1)*64*1024); + assert(!heap.is_buffer_area_free(i*64*1024+4096, 4096)); + if (i < 4096/64-1) + assert(heap.is_buffer_area_free((i+1)*64*1024, 64*1024)); + } + + pos = heap.find_free_buffer_area(4096); assert(pos == UINT64_MAX); - pos = heap.find_free_buffer_area(4*1024*1024); - assert(pos == 0); + for (int i = 0; i < 4096/64/2; i++) + { + heap.free_buffer_area(1, i*2*64*1024, 64*1024); + assert(heap.get_buffer_area_used_space() == 4096*1024-(i+1)*64*1024); + } - pos = heap.find_free_buffer_area(1024*1024); - assert(pos == 0); - heap.use_buffer_area(1, 0, 1024*1024); - assert(heap.get_buffer_area_used_space() == 1024*1024); + for (int i = 0; i < 4096/64/2*16; i++) + { + pos = heap.find_free_buffer_area(4096); + assert(pos != UINT64_MAX); + heap.use_buffer_area(1, pos, 4096); + } - assert(!heap.is_buffer_area_free(1024, 1024)); - assert(heap.is_buffer_area_free(1024*1024+1024, 1024)); - - pos = heap.find_free_buffer_area(1024*1024); - assert(pos == 1024*1024); - heap.use_buffer_area(1, 1024*1024, 1024*1024); - assert(heap.get_buffer_area_used_space() == 2*1024*1024); - - heap.free_buffer_area(1, 0, 1024*1024); - assert(heap.get_buffer_area_used_space() == 1024*1024); - - pos = heap.find_free_buffer_area(2*1024*1024+1); + assert(heap.get_buffer_area_used_space() == 4096*1024); + pos = heap.find_free_buffer_area(4096); assert(pos == UINT64_MAX); - heap.free_buffer_area(1, 1024*1024, 1024*1024); - assert(heap.get_buffer_area_used_space() == 0); + for (int i = 0; i < 4096/64/2*16; i++) + { + heap.free_buffer_area(1, (i/16)*2*64*1024+4096*(i%16), 4096); + } - pos = heap.find_free_buffer_area(4*1024*1024); - assert(pos == 0); - - heap.use_buffer_area(1, 3*1024*1024, 1024*1024); - heap.free_buffer_area(1, 3*1024*1024, 1024*1024); + pos = heap.find_free_buffer_area(64*1024); + assert(pos != UINT64_MAX); printf("OK test_alloc_buffer\n"); }