Implement a really crazy "multi-linked-list" allocator for buffered data
This commit is contained in:
@@ -122,6 +122,193 @@ uint32_t heap_object_t::calc_crc32c()
|
||||
return res;
|
||||
}
|
||||
|
||||
multilist_alloc_t::multilist_alloc_t(uint32_t count, uint32_t maxn):
|
||||
count(count), maxn(maxn)
|
||||
{
|
||||
// not-so-memory-efficient: 16 MB memory per 1 GB buffer space, but buffer spaces are small, so OK
|
||||
assert(count > 1 && count < 0x80000000);
|
||||
sizes.resize(count);
|
||||
nexts.resize(count); // nexts[i] = 0 -> area is used; nexts[i] = 1 -> no next; nexts[i] >= 2 -> next item
|
||||
prevs.resize(count);
|
||||
heads.resize(maxn); // heads[i] = 0 -> empty list; heads[i] >= 1 -> list head
|
||||
sizes[0] = count;
|
||||
sizes[count-1] = -count; // end
|
||||
nexts[0] = 1;
|
||||
heads[maxn-1] = 1;
|
||||
#ifdef MULTILIST_TEST
|
||||
print();
|
||||
#endif
|
||||
}
|
||||
|
||||
bool multilist_alloc_t::is_free(uint32_t pos)
|
||||
{
|
||||
assert(pos < count);
|
||||
if (sizes[pos] < 0)
|
||||
pos += sizes[pos]+1;
|
||||
while (pos > 0 && !sizes[pos])
|
||||
pos--;
|
||||
return nexts[pos] > 0;
|
||||
}
|
||||
|
||||
uint32_t multilist_alloc_t::allocate(uint32_t size)
|
||||
{
|
||||
assert(size > 0);
|
||||
assert(size <= maxn);
|
||||
for (uint32_t i = size-1; i < maxn; i++)
|
||||
{
|
||||
if (heads[i])
|
||||
{
|
||||
uint32_t res = heads[i]-1;
|
||||
assert(nexts[res] >= 1);
|
||||
heads[i] = nexts[res]-1;
|
||||
uint32_t area = sizes[res];
|
||||
assert(area >= size);
|
||||
if (area == size)
|
||||
{
|
||||
nexts[res] = 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
uint32_t ni = area-size;
|
||||
ni = (ni < maxn ? ni : maxn)-1;
|
||||
sizes[res+size-1] = -size;
|
||||
sizes[res] = size;
|
||||
sizes[res+size-1] = -area+size;
|
||||
sizes[res+size] = area-size;
|
||||
nexts[res+size] = heads[ni]+1;
|
||||
prevs[heads[ni]-1] = res+size+1;
|
||||
assert(!prevs[res+size]);
|
||||
heads[ni] = res+size+1;
|
||||
}
|
||||
#ifdef MULTILIST_TEST
|
||||
print();
|
||||
#endif
|
||||
return res;
|
||||
}
|
||||
}
|
||||
return UINT32_MAX;
|
||||
}
|
||||
|
||||
uint32_t multilist_alloc_t::find(uint32_t size)
|
||||
{
|
||||
assert(size > 0);
|
||||
assert(size <= maxn);
|
||||
for (uint32_t i = size-1; i < maxn; i++)
|
||||
{
|
||||
if (heads[i])
|
||||
{
|
||||
return heads[i]-1;
|
||||
}
|
||||
}
|
||||
return UINT32_MAX;
|
||||
}
|
||||
|
||||
void multilist_alloc_t::use(uint32_t pos, uint32_t size)
|
||||
{
|
||||
assert(pos < count);
|
||||
if (sizes[pos] <= 0)
|
||||
{
|
||||
uint32_t start = pos;
|
||||
if (sizes[start] < 0)
|
||||
start += sizes[start]+1;
|
||||
else
|
||||
while (start > 0 && !sizes[start])
|
||||
start--;
|
||||
assert(sizes[start] >= size);
|
||||
use_full(start);
|
||||
uint32_t full = sizes[start];
|
||||
sizes[pos-1] = -pos+start;
|
||||
sizes[start] = pos-start;
|
||||
free(start);
|
||||
sizes[pos+size-1] = -size;
|
||||
sizes[pos] = size;
|
||||
if (pos+size < start+full)
|
||||
{
|
||||
sizes[start+full-1] = -(full-pos-size);
|
||||
sizes[pos+size] = full-pos-size;
|
||||
free(pos+size);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
assert(sizes[pos] >= size);
|
||||
use_full(pos);
|
||||
if (sizes[pos] > size)
|
||||
{
|
||||
uint32_t full = sizes[pos];
|
||||
sizes[pos+size-1] = -size;
|
||||
sizes[pos] = size;
|
||||
sizes[pos+full-1] = -full+size;
|
||||
sizes[pos+size] = full-size;
|
||||
free(pos+size);
|
||||
}
|
||||
}
|
||||
#ifdef MULTILIST_TEST
|
||||
print();
|
||||
#endif
|
||||
}
|
||||
|
||||
void multilist_alloc_t::use_full(uint32_t pos)
|
||||
{
|
||||
uint32_t prevsize = sizes[pos];
|
||||
assert(prevsize);
|
||||
assert(nexts[pos]);
|
||||
uint32_t pi = (prevsize < maxn ? prevsize : maxn)-1;
|
||||
if (heads[pi] == pos+1)
|
||||
heads[pi] = nexts[pos]-1;
|
||||
if (prevs[pos])
|
||||
nexts[prevs[pos]-1] = nexts[pos];
|
||||
if (nexts[pos] >= 2)
|
||||
prevs[nexts[pos]-2] = prevs[pos];
|
||||
prevs[pos] = 0;
|
||||
nexts[pos] = 0;
|
||||
}
|
||||
|
||||
void multilist_alloc_t::free(uint32_t pos)
|
||||
{
|
||||
do_free(pos);
|
||||
#ifdef MULTILIST_TEST
|
||||
print();
|
||||
#endif
|
||||
}
|
||||
|
||||
void multilist_alloc_t::do_free(uint32_t pos)
|
||||
{
|
||||
assert(!nexts[pos]);
|
||||
uint32_t size = sizes[pos];
|
||||
assert(size > 0);
|
||||
// merge with previous?
|
||||
if (pos > 0 && nexts[pos+(sizes[pos-1] == 1 ? -1 : sizes[pos-1])] > 0)
|
||||
{
|
||||
assert(sizes[pos-1] < 0 || sizes[pos-1] == 1);
|
||||
uint32_t prevsize = sizes[pos-1] < 0 ? -sizes[pos-1] : 1;
|
||||
use_full(pos-prevsize);
|
||||
sizes[pos] = 0;
|
||||
sizes[pos-1] = 0;
|
||||
size += prevsize;
|
||||
pos -= prevsize;
|
||||
sizes[pos+size-1] = -size;
|
||||
sizes[pos] = size;
|
||||
}
|
||||
// merge with next?
|
||||
if (pos+size < count && nexts[pos+size] >= 1)
|
||||
{
|
||||
uint32_t nextsize = sizes[pos+size];
|
||||
use_full(pos+size);
|
||||
sizes[pos+size] = 0;
|
||||
sizes[pos+size-1] = 0;
|
||||
size += nextsize;
|
||||
sizes[pos+size-1] = -size;
|
||||
sizes[pos] = size;
|
||||
}
|
||||
uint32_t ni = (size < maxn ? size : maxn)-1; // FIXME ni -> nb (next bucket)
|
||||
nexts[pos] = heads[ni]+1;
|
||||
prevs[pos] = 0;
|
||||
if (heads[ni])
|
||||
prevs[heads[ni]-1] = pos+1;
|
||||
heads[ni] = pos+1;
|
||||
}
|
||||
|
||||
uint64_t blockstore_heap_t::get_pg_id(inode_t inode, uint64_t stripe)
|
||||
{
|
||||
uint64_t pg_num = 0;
|
||||
@@ -149,11 +336,10 @@ blockstore_heap_t::blockstore_heap_t(blockstore_disk_t *dsk, uint8_t *buffer_are
|
||||
assert(sizeof(heap_object_t) < sizeof(heap_write_t));
|
||||
meta_alloc = new allocator_t(meta_block_count);
|
||||
block_info.resize(meta_block_count);
|
||||
buffer_by_end.insert((heap_extent_t){ .start = 0, .end = dsk->journal_len });
|
||||
buffer_by_size.insert((heap_extent_t){ .start = 0, .end = dsk->journal_len });
|
||||
data_alloc = new allocator_t(dsk->block_count);
|
||||
if (!target_block_free_space)
|
||||
target_block_free_space = 800;
|
||||
buffer_alloc = new multilist_alloc_t(dsk->journal_len / dsk->bitmap_granularity, dsk->data_block_size / dsk->bitmap_granularity - 1);
|
||||
}
|
||||
|
||||
blockstore_heap_t::~blockstore_heap_t()
|
||||
@@ -182,6 +368,10 @@ blockstore_heap_t::~blockstore_heap_t()
|
||||
{
|
||||
delete data_alloc;
|
||||
}
|
||||
if (buffer_alloc)
|
||||
{
|
||||
delete buffer_alloc;
|
||||
}
|
||||
}
|
||||
|
||||
// set initially compacted lsn - should be done before loading
|
||||
@@ -1678,87 +1868,44 @@ void blockstore_heap_t::use_data(inode_t inode, uint64_t location)
|
||||
|
||||
uint64_t blockstore_heap_t::find_free_buffer_area(uint64_t size)
|
||||
{
|
||||
auto free_it = buffer_by_size.lower_bound((heap_extent_t){ .start = 0, .end = size });
|
||||
if (free_it == buffer_by_size.end())
|
||||
assert(!(size % dsk->bitmap_granularity));
|
||||
uint32_t pos = buffer_alloc->find(size / dsk->bitmap_granularity);
|
||||
if (pos == UINT32_MAX)
|
||||
{
|
||||
return UINT64_MAX;
|
||||
}
|
||||
return free_it->start;
|
||||
return pos * dsk->bitmap_granularity;
|
||||
}
|
||||
|
||||
bool blockstore_heap_t::is_buffer_area_free(uint64_t location, uint64_t size)
|
||||
{
|
||||
auto free_it = buffer_by_end.lower_bound((heap_extent_t){ .end = location+size });
|
||||
return (free_it != buffer_by_end.end() && free_it->start <= location);
|
||||
assert(!(location % dsk->bitmap_granularity));
|
||||
return buffer_alloc->is_free(location / dsk->bitmap_granularity);
|
||||
}
|
||||
|
||||
uint64_t blockstore_heap_t::alloc_buffer_area(inode_t inode, uint64_t size)
|
||||
{
|
||||
assert(!(size % dsk->bitmap_granularity));
|
||||
uint32_t res = buffer_alloc->allocate(size / dsk->bitmap_granularity);
|
||||
if (res == UINT32_MAX)
|
||||
{
|
||||
return UINT64_MAX;
|
||||
}
|
||||
buffer_area_used_space += size;
|
||||
return res * dsk->bitmap_granularity;
|
||||
}
|
||||
|
||||
void blockstore_heap_t::use_buffer_area(inode_t inode, uint64_t location, uint64_t size)
|
||||
{
|
||||
auto free_it = buffer_by_end.lower_bound((heap_extent_t){ .end = location+size });
|
||||
assert(free_it != buffer_by_end.end() && free_it->start <= location && free_it->end >= location+size);
|
||||
heap_extent_t extent = *free_it;
|
||||
buffer_by_end.erase(free_it);
|
||||
buffer_by_size.erase(extent);
|
||||
if (extent.start == location)
|
||||
{
|
||||
extent.start += size;
|
||||
buffer_by_end.insert(extent);
|
||||
buffer_by_size.insert(extent);
|
||||
}
|
||||
else if (extent.end == location+size)
|
||||
{
|
||||
extent.end -= size;
|
||||
buffer_by_end.insert(extent);
|
||||
buffer_by_size.insert(extent);
|
||||
}
|
||||
else
|
||||
{
|
||||
buffer_by_end.insert((heap_extent_t){ .start = extent.start, .end = location });
|
||||
buffer_by_size.insert((heap_extent_t){ .start = extent.start, .end = location });
|
||||
buffer_by_end.insert((heap_extent_t){ .start = location+size, .end = extent.end });
|
||||
buffer_by_size.insert((heap_extent_t){ .start = location+size, .end = extent.end });
|
||||
}
|
||||
assert(!(size % dsk->bitmap_granularity));
|
||||
buffer_alloc->use(location / dsk->bitmap_granularity, size / dsk->bitmap_granularity);
|
||||
buffer_area_used_space += size;
|
||||
}
|
||||
|
||||
void blockstore_heap_t::free_buffer_area(inode_t inode, uint64_t location, uint64_t size)
|
||||
{
|
||||
auto next_it = buffer_by_end.lower_bound((heap_extent_t){ .end = location+size });
|
||||
auto prev_it = next_it == buffer_by_end.begin() ? buffer_by_end.end() : std::prev(next_it);
|
||||
assert(next_it == buffer_by_end.end() || next_it->start >= location+size);
|
||||
assert(prev_it == buffer_by_end.end() || prev_it->end <= location);
|
||||
bool merge_prev = (prev_it != buffer_by_end.end() && prev_it->end == location);
|
||||
bool merge_next = (next_it != buffer_by_end.end() && next_it->start == location+size);
|
||||
uint64_t prev_start = merge_prev ? prev_it->start : 0;
|
||||
uint64_t next_end = merge_next ? next_it->end : 0;
|
||||
if (merge_prev && merge_next)
|
||||
{
|
||||
buffer_by_size.erase(*prev_it);
|
||||
buffer_by_size.erase(*next_it);
|
||||
buffer_by_end.erase(prev_it);
|
||||
buffer_by_end.erase(next_it);
|
||||
buffer_by_end.insert((heap_extent_t){ .start = prev_start, .end = next_end });
|
||||
buffer_by_size.insert((heap_extent_t){ .start = prev_start, .end = next_end });
|
||||
}
|
||||
else if (merge_prev)
|
||||
{
|
||||
buffer_by_size.erase(*prev_it);
|
||||
buffer_by_end.erase(prev_it);
|
||||
buffer_by_end.insert((heap_extent_t){ .start = prev_start, .end = location+size });
|
||||
buffer_by_size.insert((heap_extent_t){ .start = prev_start, .end = location+size });
|
||||
}
|
||||
else if (merge_next)
|
||||
{
|
||||
buffer_by_size.erase(*next_it);
|
||||
buffer_by_end.erase(next_it);
|
||||
buffer_by_end.insert((heap_extent_t){ .start = location, .end = next_end });
|
||||
buffer_by_size.insert((heap_extent_t){ .start = location, .end = next_end });
|
||||
}
|
||||
else
|
||||
{
|
||||
buffer_by_end.insert((heap_extent_t){ .start = location, .end = location+size });
|
||||
buffer_by_size.insert((heap_extent_t){ .start = location, .end = location+size });
|
||||
}
|
||||
assert(!(location % dsk->bitmap_granularity));
|
||||
buffer_alloc->free(location / dsk->bitmap_granularity);
|
||||
buffer_area_used_space -= size;
|
||||
}
|
||||
|
||||
|
||||
@@ -107,26 +107,23 @@ inline bool operator < (const heap_block_free_t & a, const heap_block_free_t & b
|
||||
return a.free_space > b.free_space || a.free_space == b.free_space && a.block_num < b.block_num;
|
||||
}
|
||||
|
||||
struct heap_extent_t
|
||||
struct multilist_alloc_t
|
||||
{
|
||||
uint64_t start = 0;
|
||||
uint64_t end = 0;
|
||||
};
|
||||
const uint32_t count, maxn;
|
||||
std::vector<int32_t> sizes;
|
||||
std::vector<uint32_t> nexts, prevs, heads;
|
||||
|
||||
struct heap_less_extent_by_end
|
||||
{
|
||||
const bool operator()(const heap_extent_t & a, const heap_extent_t & b) const
|
||||
{
|
||||
return a.end < b.end;
|
||||
}
|
||||
};
|
||||
|
||||
struct heap_less_extent_by_size
|
||||
{
|
||||
const bool operator()(const heap_extent_t & a, const heap_extent_t & b) const
|
||||
{
|
||||
return a.end-a.start < b.end-b.start || a.end-a.start == b.end-b.start && a.start < b.start;
|
||||
}
|
||||
multilist_alloc_t(uint32_t count, uint32_t maxn);
|
||||
bool is_free(uint32_t pos);
|
||||
uint32_t allocate(uint32_t size);
|
||||
uint32_t find(uint32_t size);
|
||||
void use_full(uint32_t pos);
|
||||
void use(uint32_t pos, uint32_t size);
|
||||
void do_free(uint32_t pos);
|
||||
void free(uint32_t pos);
|
||||
#ifdef MULTILIST_TEST
|
||||
void print();
|
||||
#endif
|
||||
};
|
||||
|
||||
class blockstore_heap_t
|
||||
@@ -154,8 +151,7 @@ class blockstore_heap_t
|
||||
allocator_t *meta_alloc = NULL;
|
||||
uint32_t meta_alloc_count = 0;
|
||||
uint64_t meta_used_space = 0;
|
||||
std::set<heap_extent_t, heap_less_extent_by_end> buffer_by_end;
|
||||
std::set<heap_extent_t, heap_less_extent_by_size> buffer_by_size;
|
||||
multilist_alloc_t *buffer_alloc = NULL;
|
||||
std::set<heap_block_free_t> used_alloc_queue;
|
||||
std::map<heap_object_lsn_t, heap_object_mvcc_t> object_mvcc;
|
||||
std::map<uint64_t, uint32_t> mvcc_data_refs;
|
||||
@@ -255,6 +251,7 @@ public:
|
||||
// buffer device allocator functions
|
||||
uint64_t find_free_buffer_area(uint64_t size);
|
||||
bool is_buffer_area_free(uint64_t location, uint64_t size);
|
||||
uint64_t alloc_buffer_area(inode_t inode, uint64_t size);
|
||||
void use_buffer_area(inode_t inode, uint64_t location, uint64_t size);
|
||||
void free_buffer_area(inode_t inode, uint64_t location, uint64_t size);
|
||||
uint64_t get_buffer_area_used_space();
|
||||
|
||||
+99
-26
@@ -9,6 +9,74 @@
|
||||
#include "blockstore_heap.h"
|
||||
#include "../util/crc32c.h"
|
||||
|
||||
#ifdef MULTILIST_TEST
|
||||
void multilist_alloc_t::print()
|
||||
{
|
||||
printf("heads:");
|
||||
for (int i = 0; i < maxn; i++)
|
||||
if (heads[i])
|
||||
printf(" %u=%u", i, heads[i]);
|
||||
for (int i = 0; i < maxn; i++)
|
||||
if (heads[i])
|
||||
assert(i < maxn-1 ? sizes[heads[i]-1] == i+1 : (sizes[heads[i]-1] >= i+1));
|
||||
printf("\n");
|
||||
printf("sizes:");
|
||||
for (int i = 0; i < count; i++)
|
||||
if (sizes[i])
|
||||
printf(" %d=%d", i, sizes[i]);
|
||||
printf("\n");
|
||||
printf("prevs:");
|
||||
for (int i = 0; i < count; i++)
|
||||
if (prevs[i])
|
||||
printf(" %d=%d", i, prevs[i]);
|
||||
printf("\n");
|
||||
printf("nexts:");
|
||||
for (int i = 0; i < count; i++)
|
||||
if (nexts[i])
|
||||
printf(" %d=%d", i, nexts[i]);
|
||||
printf("\n");
|
||||
printf("items:");
|
||||
for (int i = 0; i < count; )
|
||||
{
|
||||
if (sizes[i])
|
||||
{
|
||||
printf(" %u=(s:%d,n:%u,p:%u)", i, sizes[i], nexts[i], prevs[i]);
|
||||
assert(i+sizes[i] <= count);
|
||||
if (sizes[i] > 1 && sizes[i+sizes[i]-1] != -sizes[i])
|
||||
{
|
||||
printf(" ERROR: start/end mismatch\n");
|
||||
abort();
|
||||
}
|
||||
for (int j = i+1; j < i+sizes[i]-1; j++)
|
||||
{
|
||||
if (sizes[j])
|
||||
{
|
||||
printf(" ERROR: internal non-zero at %d: %d\n", j, sizes[j]);
|
||||
abort();
|
||||
}
|
||||
}
|
||||
if (nexts[i] >= 2)
|
||||
{
|
||||
if (nexts[i] >= 2+count)
|
||||
{
|
||||
printf(" ERROR: next out of range\n");
|
||||
abort();
|
||||
}
|
||||
if (prevs[nexts[i]-2] != i+1)
|
||||
{
|
||||
printf(" ERROR: prev[next] != this");
|
||||
abort();
|
||||
}
|
||||
}
|
||||
i += sizes[i];
|
||||
}
|
||||
else
|
||||
i++;
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
#endif
|
||||
|
||||
static int count_writes(heap_object_t *obj)
|
||||
{
|
||||
int n = 0;
|
||||
@@ -1015,39 +1083,44 @@ void test_alloc_buffer()
|
||||
|
||||
uint64_t pos;
|
||||
|
||||
pos = heap.find_free_buffer_area(4*1024*1024+1);
|
||||
for (int i = 0; i < 4096/64; i++)
|
||||
{
|
||||
pos = heap.find_free_buffer_area(64*1024);
|
||||
assert(pos == i*64*1024);
|
||||
heap.use_buffer_area(1, pos, 64*1024);
|
||||
assert(heap.get_buffer_area_used_space() == (i+1)*64*1024);
|
||||
assert(!heap.is_buffer_area_free(i*64*1024+4096, 4096));
|
||||
if (i < 4096/64-1)
|
||||
assert(heap.is_buffer_area_free((i+1)*64*1024, 64*1024));
|
||||
}
|
||||
|
||||
pos = heap.find_free_buffer_area(4096);
|
||||
assert(pos == UINT64_MAX);
|
||||
|
||||
pos = heap.find_free_buffer_area(4*1024*1024);
|
||||
assert(pos == 0);
|
||||
for (int i = 0; i < 4096/64/2; i++)
|
||||
{
|
||||
heap.free_buffer_area(1, i*2*64*1024, 64*1024);
|
||||
assert(heap.get_buffer_area_used_space() == 4096*1024-(i+1)*64*1024);
|
||||
}
|
||||
|
||||
pos = heap.find_free_buffer_area(1024*1024);
|
||||
assert(pos == 0);
|
||||
heap.use_buffer_area(1, 0, 1024*1024);
|
||||
assert(heap.get_buffer_area_used_space() == 1024*1024);
|
||||
for (int i = 0; i < 4096/64/2*16; i++)
|
||||
{
|
||||
pos = heap.find_free_buffer_area(4096);
|
||||
assert(pos != UINT64_MAX);
|
||||
heap.use_buffer_area(1, pos, 4096);
|
||||
}
|
||||
|
||||
assert(!heap.is_buffer_area_free(1024, 1024));
|
||||
assert(heap.is_buffer_area_free(1024*1024+1024, 1024));
|
||||
|
||||
pos = heap.find_free_buffer_area(1024*1024);
|
||||
assert(pos == 1024*1024);
|
||||
heap.use_buffer_area(1, 1024*1024, 1024*1024);
|
||||
assert(heap.get_buffer_area_used_space() == 2*1024*1024);
|
||||
|
||||
heap.free_buffer_area(1, 0, 1024*1024);
|
||||
assert(heap.get_buffer_area_used_space() == 1024*1024);
|
||||
|
||||
pos = heap.find_free_buffer_area(2*1024*1024+1);
|
||||
assert(heap.get_buffer_area_used_space() == 4096*1024);
|
||||
pos = heap.find_free_buffer_area(4096);
|
||||
assert(pos == UINT64_MAX);
|
||||
|
||||
heap.free_buffer_area(1, 1024*1024, 1024*1024);
|
||||
assert(heap.get_buffer_area_used_space() == 0);
|
||||
for (int i = 0; i < 4096/64/2*16; i++)
|
||||
{
|
||||
heap.free_buffer_area(1, (i/16)*2*64*1024+4096*(i%16), 4096);
|
||||
}
|
||||
|
||||
pos = heap.find_free_buffer_area(4*1024*1024);
|
||||
assert(pos == 0);
|
||||
|
||||
heap.use_buffer_area(1, 3*1024*1024, 1024*1024);
|
||||
heap.free_buffer_area(1, 3*1024*1024, 1024*1024);
|
||||
pos = heap.find_free_buffer_area(64*1024);
|
||||
assert(pos != UINT64_MAX);
|
||||
|
||||
printf("OK test_alloc_buffer\n");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user