Implement a really crazy "multi-linked-list" allocator for buffered data

This commit is contained in:
Vitaliy Filippov
2025-12-02 01:52:11 +03:00
parent 6d0460500a
commit f580cee936
3 changed files with 330 additions and 113 deletions
+214 -67
View File
@@ -122,6 +122,193 @@ uint32_t heap_object_t::calc_crc32c()
return res;
}
multilist_alloc_t::multilist_alloc_t(uint32_t count, uint32_t maxn):
count(count), maxn(maxn)
{
// not-so-memory-efficient: 16 MB memory per 1 GB buffer space, but buffer spaces are small, so OK
assert(count > 1 && count < 0x80000000);
sizes.resize(count);
nexts.resize(count); // nexts[i] = 0 -> area is used; nexts[i] = 1 -> no next; nexts[i] >= 2 -> next item
prevs.resize(count);
heads.resize(maxn); // heads[i] = 0 -> empty list; heads[i] >= 1 -> list head
sizes[0] = count;
sizes[count-1] = -count; // end
nexts[0] = 1;
heads[maxn-1] = 1;
#ifdef MULTILIST_TEST
print();
#endif
}
bool multilist_alloc_t::is_free(uint32_t pos)
{
assert(pos < count);
if (sizes[pos] < 0)
pos += sizes[pos]+1;
while (pos > 0 && !sizes[pos])
pos--;
return nexts[pos] > 0;
}
uint32_t multilist_alloc_t::allocate(uint32_t size)
{
assert(size > 0);
assert(size <= maxn);
for (uint32_t i = size-1; i < maxn; i++)
{
if (heads[i])
{
uint32_t res = heads[i]-1;
assert(nexts[res] >= 1);
heads[i] = nexts[res]-1;
uint32_t area = sizes[res];
assert(area >= size);
if (area == size)
{
nexts[res] = 0;
}
else
{
uint32_t ni = area-size;
ni = (ni < maxn ? ni : maxn)-1;
sizes[res+size-1] = -size;
sizes[res] = size;
sizes[res+size-1] = -area+size;
sizes[res+size] = area-size;
nexts[res+size] = heads[ni]+1;
prevs[heads[ni]-1] = res+size+1;
assert(!prevs[res+size]);
heads[ni] = res+size+1;
}
#ifdef MULTILIST_TEST
print();
#endif
return res;
}
}
return UINT32_MAX;
}
uint32_t multilist_alloc_t::find(uint32_t size)
{
assert(size > 0);
assert(size <= maxn);
for (uint32_t i = size-1; i < maxn; i++)
{
if (heads[i])
{
return heads[i]-1;
}
}
return UINT32_MAX;
}
void multilist_alloc_t::use(uint32_t pos, uint32_t size)
{
assert(pos < count);
if (sizes[pos] <= 0)
{
uint32_t start = pos;
if (sizes[start] < 0)
start += sizes[start]+1;
else
while (start > 0 && !sizes[start])
start--;
assert(sizes[start] >= size);
use_full(start);
uint32_t full = sizes[start];
sizes[pos-1] = -pos+start;
sizes[start] = pos-start;
free(start);
sizes[pos+size-1] = -size;
sizes[pos] = size;
if (pos+size < start+full)
{
sizes[start+full-1] = -(full-pos-size);
sizes[pos+size] = full-pos-size;
free(pos+size);
}
}
else
{
assert(sizes[pos] >= size);
use_full(pos);
if (sizes[pos] > size)
{
uint32_t full = sizes[pos];
sizes[pos+size-1] = -size;
sizes[pos] = size;
sizes[pos+full-1] = -full+size;
sizes[pos+size] = full-size;
free(pos+size);
}
}
#ifdef MULTILIST_TEST
print();
#endif
}
void multilist_alloc_t::use_full(uint32_t pos)
{
uint32_t prevsize = sizes[pos];
assert(prevsize);
assert(nexts[pos]);
uint32_t pi = (prevsize < maxn ? prevsize : maxn)-1;
if (heads[pi] == pos+1)
heads[pi] = nexts[pos]-1;
if (prevs[pos])
nexts[prevs[pos]-1] = nexts[pos];
if (nexts[pos] >= 2)
prevs[nexts[pos]-2] = prevs[pos];
prevs[pos] = 0;
nexts[pos] = 0;
}
void multilist_alloc_t::free(uint32_t pos)
{
do_free(pos);
#ifdef MULTILIST_TEST
print();
#endif
}
void multilist_alloc_t::do_free(uint32_t pos)
{
assert(!nexts[pos]);
uint32_t size = sizes[pos];
assert(size > 0);
// merge with previous?
if (pos > 0 && nexts[pos+(sizes[pos-1] == 1 ? -1 : sizes[pos-1])] > 0)
{
assert(sizes[pos-1] < 0 || sizes[pos-1] == 1);
uint32_t prevsize = sizes[pos-1] < 0 ? -sizes[pos-1] : 1;
use_full(pos-prevsize);
sizes[pos] = 0;
sizes[pos-1] = 0;
size += prevsize;
pos -= prevsize;
sizes[pos+size-1] = -size;
sizes[pos] = size;
}
// merge with next?
if (pos+size < count && nexts[pos+size] >= 1)
{
uint32_t nextsize = sizes[pos+size];
use_full(pos+size);
sizes[pos+size] = 0;
sizes[pos+size-1] = 0;
size += nextsize;
sizes[pos+size-1] = -size;
sizes[pos] = size;
}
uint32_t ni = (size < maxn ? size : maxn)-1; // FIXME ni -> nb (next bucket)
nexts[pos] = heads[ni]+1;
prevs[pos] = 0;
if (heads[ni])
prevs[heads[ni]-1] = pos+1;
heads[ni] = pos+1;
}
uint64_t blockstore_heap_t::get_pg_id(inode_t inode, uint64_t stripe)
{
uint64_t pg_num = 0;
@@ -149,11 +336,10 @@ blockstore_heap_t::blockstore_heap_t(blockstore_disk_t *dsk, uint8_t *buffer_are
assert(sizeof(heap_object_t) < sizeof(heap_write_t));
meta_alloc = new allocator_t(meta_block_count);
block_info.resize(meta_block_count);
buffer_by_end.insert((heap_extent_t){ .start = 0, .end = dsk->journal_len });
buffer_by_size.insert((heap_extent_t){ .start = 0, .end = dsk->journal_len });
data_alloc = new allocator_t(dsk->block_count);
if (!target_block_free_space)
target_block_free_space = 800;
buffer_alloc = new multilist_alloc_t(dsk->journal_len / dsk->bitmap_granularity, dsk->data_block_size / dsk->bitmap_granularity - 1);
}
blockstore_heap_t::~blockstore_heap_t()
@@ -182,6 +368,10 @@ blockstore_heap_t::~blockstore_heap_t()
{
delete data_alloc;
}
if (buffer_alloc)
{
delete buffer_alloc;
}
}
// set initially compacted lsn - should be done before loading
@@ -1678,87 +1868,44 @@ void blockstore_heap_t::use_data(inode_t inode, uint64_t location)
uint64_t blockstore_heap_t::find_free_buffer_area(uint64_t size)
{
auto free_it = buffer_by_size.lower_bound((heap_extent_t){ .start = 0, .end = size });
if (free_it == buffer_by_size.end())
assert(!(size % dsk->bitmap_granularity));
uint32_t pos = buffer_alloc->find(size / dsk->bitmap_granularity);
if (pos == UINT32_MAX)
{
return UINT64_MAX;
}
return free_it->start;
return pos * dsk->bitmap_granularity;
}
bool blockstore_heap_t::is_buffer_area_free(uint64_t location, uint64_t size)
{
auto free_it = buffer_by_end.lower_bound((heap_extent_t){ .end = location+size });
return (free_it != buffer_by_end.end() && free_it->start <= location);
assert(!(location % dsk->bitmap_granularity));
return buffer_alloc->is_free(location / dsk->bitmap_granularity);
}
uint64_t blockstore_heap_t::alloc_buffer_area(inode_t inode, uint64_t size)
{
assert(!(size % dsk->bitmap_granularity));
uint32_t res = buffer_alloc->allocate(size / dsk->bitmap_granularity);
if (res == UINT32_MAX)
{
return UINT64_MAX;
}
buffer_area_used_space += size;
return res * dsk->bitmap_granularity;
}
void blockstore_heap_t::use_buffer_area(inode_t inode, uint64_t location, uint64_t size)
{
auto free_it = buffer_by_end.lower_bound((heap_extent_t){ .end = location+size });
assert(free_it != buffer_by_end.end() && free_it->start <= location && free_it->end >= location+size);
heap_extent_t extent = *free_it;
buffer_by_end.erase(free_it);
buffer_by_size.erase(extent);
if (extent.start == location)
{
extent.start += size;
buffer_by_end.insert(extent);
buffer_by_size.insert(extent);
}
else if (extent.end == location+size)
{
extent.end -= size;
buffer_by_end.insert(extent);
buffer_by_size.insert(extent);
}
else
{
buffer_by_end.insert((heap_extent_t){ .start = extent.start, .end = location });
buffer_by_size.insert((heap_extent_t){ .start = extent.start, .end = location });
buffer_by_end.insert((heap_extent_t){ .start = location+size, .end = extent.end });
buffer_by_size.insert((heap_extent_t){ .start = location+size, .end = extent.end });
}
assert(!(size % dsk->bitmap_granularity));
buffer_alloc->use(location / dsk->bitmap_granularity, size / dsk->bitmap_granularity);
buffer_area_used_space += size;
}
void blockstore_heap_t::free_buffer_area(inode_t inode, uint64_t location, uint64_t size)
{
auto next_it = buffer_by_end.lower_bound((heap_extent_t){ .end = location+size });
auto prev_it = next_it == buffer_by_end.begin() ? buffer_by_end.end() : std::prev(next_it);
assert(next_it == buffer_by_end.end() || next_it->start >= location+size);
assert(prev_it == buffer_by_end.end() || prev_it->end <= location);
bool merge_prev = (prev_it != buffer_by_end.end() && prev_it->end == location);
bool merge_next = (next_it != buffer_by_end.end() && next_it->start == location+size);
uint64_t prev_start = merge_prev ? prev_it->start : 0;
uint64_t next_end = merge_next ? next_it->end : 0;
if (merge_prev && merge_next)
{
buffer_by_size.erase(*prev_it);
buffer_by_size.erase(*next_it);
buffer_by_end.erase(prev_it);
buffer_by_end.erase(next_it);
buffer_by_end.insert((heap_extent_t){ .start = prev_start, .end = next_end });
buffer_by_size.insert((heap_extent_t){ .start = prev_start, .end = next_end });
}
else if (merge_prev)
{
buffer_by_size.erase(*prev_it);
buffer_by_end.erase(prev_it);
buffer_by_end.insert((heap_extent_t){ .start = prev_start, .end = location+size });
buffer_by_size.insert((heap_extent_t){ .start = prev_start, .end = location+size });
}
else if (merge_next)
{
buffer_by_size.erase(*next_it);
buffer_by_end.erase(next_it);
buffer_by_end.insert((heap_extent_t){ .start = location, .end = next_end });
buffer_by_size.insert((heap_extent_t){ .start = location, .end = next_end });
}
else
{
buffer_by_end.insert((heap_extent_t){ .start = location, .end = location+size });
buffer_by_size.insert((heap_extent_t){ .start = location, .end = location+size });
}
assert(!(location % dsk->bitmap_granularity));
buffer_alloc->free(location / dsk->bitmap_granularity);
buffer_area_used_space -= size;
}
+17 -20
View File
@@ -107,26 +107,23 @@ inline bool operator < (const heap_block_free_t & a, const heap_block_free_t & b
return a.free_space > b.free_space || a.free_space == b.free_space && a.block_num < b.block_num;
}
struct heap_extent_t
struct multilist_alloc_t
{
uint64_t start = 0;
uint64_t end = 0;
};
const uint32_t count, maxn;
std::vector<int32_t> sizes;
std::vector<uint32_t> nexts, prevs, heads;
struct heap_less_extent_by_end
{
const bool operator()(const heap_extent_t & a, const heap_extent_t & b) const
{
return a.end < b.end;
}
};
struct heap_less_extent_by_size
{
const bool operator()(const heap_extent_t & a, const heap_extent_t & b) const
{
return a.end-a.start < b.end-b.start || a.end-a.start == b.end-b.start && a.start < b.start;
}
multilist_alloc_t(uint32_t count, uint32_t maxn);
bool is_free(uint32_t pos);
uint32_t allocate(uint32_t size);
uint32_t find(uint32_t size);
void use_full(uint32_t pos);
void use(uint32_t pos, uint32_t size);
void do_free(uint32_t pos);
void free(uint32_t pos);
#ifdef MULTILIST_TEST
void print();
#endif
};
class blockstore_heap_t
@@ -154,8 +151,7 @@ class blockstore_heap_t
allocator_t *meta_alloc = NULL;
uint32_t meta_alloc_count = 0;
uint64_t meta_used_space = 0;
std::set<heap_extent_t, heap_less_extent_by_end> buffer_by_end;
std::set<heap_extent_t, heap_less_extent_by_size> buffer_by_size;
multilist_alloc_t *buffer_alloc = NULL;
std::set<heap_block_free_t> used_alloc_queue;
std::map<heap_object_lsn_t, heap_object_mvcc_t> object_mvcc;
std::map<uint64_t, uint32_t> mvcc_data_refs;
@@ -255,6 +251,7 @@ public:
// buffer device allocator functions
uint64_t find_free_buffer_area(uint64_t size);
bool is_buffer_area_free(uint64_t location, uint64_t size);
uint64_t alloc_buffer_area(inode_t inode, uint64_t size);
void use_buffer_area(inode_t inode, uint64_t location, uint64_t size);
void free_buffer_area(inode_t inode, uint64_t location, uint64_t size);
uint64_t get_buffer_area_used_space();
+99 -26
View File
@@ -9,6 +9,74 @@
#include "blockstore_heap.h"
#include "../util/crc32c.h"
#ifdef MULTILIST_TEST
void multilist_alloc_t::print()
{
printf("heads:");
for (int i = 0; i < maxn; i++)
if (heads[i])
printf(" %u=%u", i, heads[i]);
for (int i = 0; i < maxn; i++)
if (heads[i])
assert(i < maxn-1 ? sizes[heads[i]-1] == i+1 : (sizes[heads[i]-1] >= i+1));
printf("\n");
printf("sizes:");
for (int i = 0; i < count; i++)
if (sizes[i])
printf(" %d=%d", i, sizes[i]);
printf("\n");
printf("prevs:");
for (int i = 0; i < count; i++)
if (prevs[i])
printf(" %d=%d", i, prevs[i]);
printf("\n");
printf("nexts:");
for (int i = 0; i < count; i++)
if (nexts[i])
printf(" %d=%d", i, nexts[i]);
printf("\n");
printf("items:");
for (int i = 0; i < count; )
{
if (sizes[i])
{
printf(" %u=(s:%d,n:%u,p:%u)", i, sizes[i], nexts[i], prevs[i]);
assert(i+sizes[i] <= count);
if (sizes[i] > 1 && sizes[i+sizes[i]-1] != -sizes[i])
{
printf(" ERROR: start/end mismatch\n");
abort();
}
for (int j = i+1; j < i+sizes[i]-1; j++)
{
if (sizes[j])
{
printf(" ERROR: internal non-zero at %d: %d\n", j, sizes[j]);
abort();
}
}
if (nexts[i] >= 2)
{
if (nexts[i] >= 2+count)
{
printf(" ERROR: next out of range\n");
abort();
}
if (prevs[nexts[i]-2] != i+1)
{
printf(" ERROR: prev[next] != this");
abort();
}
}
i += sizes[i];
}
else
i++;
}
printf("\n");
}
#endif
static int count_writes(heap_object_t *obj)
{
int n = 0;
@@ -1015,39 +1083,44 @@ void test_alloc_buffer()
uint64_t pos;
pos = heap.find_free_buffer_area(4*1024*1024+1);
for (int i = 0; i < 4096/64; i++)
{
pos = heap.find_free_buffer_area(64*1024);
assert(pos == i*64*1024);
heap.use_buffer_area(1, pos, 64*1024);
assert(heap.get_buffer_area_used_space() == (i+1)*64*1024);
assert(!heap.is_buffer_area_free(i*64*1024+4096, 4096));
if (i < 4096/64-1)
assert(heap.is_buffer_area_free((i+1)*64*1024, 64*1024));
}
pos = heap.find_free_buffer_area(4096);
assert(pos == UINT64_MAX);
pos = heap.find_free_buffer_area(4*1024*1024);
assert(pos == 0);
for (int i = 0; i < 4096/64/2; i++)
{
heap.free_buffer_area(1, i*2*64*1024, 64*1024);
assert(heap.get_buffer_area_used_space() == 4096*1024-(i+1)*64*1024);
}
pos = heap.find_free_buffer_area(1024*1024);
assert(pos == 0);
heap.use_buffer_area(1, 0, 1024*1024);
assert(heap.get_buffer_area_used_space() == 1024*1024);
for (int i = 0; i < 4096/64/2*16; i++)
{
pos = heap.find_free_buffer_area(4096);
assert(pos != UINT64_MAX);
heap.use_buffer_area(1, pos, 4096);
}
assert(!heap.is_buffer_area_free(1024, 1024));
assert(heap.is_buffer_area_free(1024*1024+1024, 1024));
pos = heap.find_free_buffer_area(1024*1024);
assert(pos == 1024*1024);
heap.use_buffer_area(1, 1024*1024, 1024*1024);
assert(heap.get_buffer_area_used_space() == 2*1024*1024);
heap.free_buffer_area(1, 0, 1024*1024);
assert(heap.get_buffer_area_used_space() == 1024*1024);
pos = heap.find_free_buffer_area(2*1024*1024+1);
assert(heap.get_buffer_area_used_space() == 4096*1024);
pos = heap.find_free_buffer_area(4096);
assert(pos == UINT64_MAX);
heap.free_buffer_area(1, 1024*1024, 1024*1024);
assert(heap.get_buffer_area_used_space() == 0);
for (int i = 0; i < 4096/64/2*16; i++)
{
heap.free_buffer_area(1, (i/16)*2*64*1024+4096*(i%16), 4096);
}
pos = heap.find_free_buffer_area(4*1024*1024);
assert(pos == 0);
heap.use_buffer_area(1, 3*1024*1024, 1024*1024);
heap.free_buffer_area(1, 3*1024*1024, 1024*1024);
pos = heap.find_free_buffer_area(64*1024);
assert(pos != UINT64_MAX);
printf("OK test_alloc_buffer\n");
}