diff --git a/src/blockstore/CMakeLists.txt b/src/blockstore/CMakeLists.txt index 39511308..a9434117 100644 --- a/src/blockstore/CMakeLists.txt +++ b/src/blockstore/CMakeLists.txt @@ -5,7 +5,7 @@ project(vitastor) # libvitastor_blk.so add_library(vitastor_blk SHARED ../util/allocator.cpp ../util/crc32c.c ../util/ringloop.cpp - blockstore_heap.cpp blockstore_disk.cpp + multilist.cpp blockstore_heap.cpp blockstore_disk.cpp blockstore.cpp blockstore_impl.cpp blockstore_init.cpp blockstore_open.cpp blockstore_flush.cpp blockstore_read.cpp blockstore_stable.cpp blockstore_sync.cpp blockstore_write.cpp v1/flush.cpp v1/impl.cpp v1/init.cpp v1/journal.cpp v1/open.cpp v1/read.cpp v1/rollback.cpp v1/stable.cpp v1/sync.cpp v1/write.cpp diff --git a/src/blockstore/blockstore_heap.cpp b/src/blockstore/blockstore_heap.cpp index d8501b06..51c96b05 100644 --- a/src/blockstore/blockstore_heap.cpp +++ b/src/blockstore/blockstore_heap.cpp @@ -135,275 +135,6 @@ uint32_t heap_object_t::calc_crc32c() return res; } -multilist_alloc_t::multilist_alloc_t(uint32_t count, uint32_t maxn): - count(count), maxn(maxn) -{ - // not-so-memory-efficient: 16 MB memory per 1 GB buffer space, but buffer spaces are small, so OK - assert(count > 1 && count < 0x80000000); - sizes.resize(count); - nexts.resize(count); // nexts[i] = 0 -> area is used; nexts[i] = 1 -> no next; nexts[i] >= 2 -> next item - prevs.resize(count); - heads.resize(maxn); // heads[i] = 0 -> empty list; heads[i] >= 1 -> list head - sizes[0] = count; - sizes[count-1] = -count; // end - nexts[0] = 1; - heads[maxn-1] = 1; -#ifdef MULTILIST_TRACE - print(); -#endif -} - -bool multilist_alloc_t::is_free(uint32_t pos) -{ - assert(pos < count); - if (sizes[pos] < 0) - pos += sizes[pos]+1; - while (pos > 0 && !sizes[pos]) - pos--; - return nexts[pos] > 0; -} - -uint32_t multilist_alloc_t::find(uint32_t size) -{ - assert(size > 0); - assert(size <= maxn); - for (uint32_t i = size-1; i < maxn; i++) - { - if (heads[i]) - { - return heads[i]-1; - } - } - return UINT32_MAX; -} - -void multilist_alloc_t::verify() -{ - std::set reachable; - for (int i = 0; i < maxn; i++) - { - uint32_t cur = heads[i]; - while (cur) - { - if (!nexts[cur-1]) - { - fprintf(stderr, "ERROR: item %d from freelist %d is not free\n", cur-1, i); - print(); - abort(); - } - if (nexts[cur-1] >= count+2) - { - fprintf(stderr, "ERROR: next out of range at %d: %d\n", cur-1, nexts[cur-1]); - print(); - abort(); - } - if (!(i < maxn-1 ? sizes[cur-1] == i+1 : (sizes[cur-1] >= i+1))) - { - fprintf(stderr, "ERROR: item %d is in wrong freelist: expected size %d, but actual size is %d\n", cur-1, i+1, sizes[cur-1]); - print(); - abort(); - } - if (reachable.find(cur-1) != reachable.end()) - { - fprintf(stderr, "ERROR: doubly-claimed item %d\n", cur-1); - print(); - abort(); - } - reachable.insert(cur-1); - cur = nexts[cur-1]-1; - } - } - for (int i = 0; i < count; ) - { - if (sizes[i]) - { - assert(i+sizes[i] <= count); - if (sizes[i] > 1 && sizes[i+sizes[i]-1] != -sizes[i]) - { - fprintf(stderr, "ERROR: start/end mismatch at %d: sizes[%d] should be %d, but is %d\n", i, i+sizes[i]-1, -sizes[i], sizes[i+sizes[i]-1]); - print(); - abort(); - } - for (int j = i+1; j < i+sizes[i]-1; j++) - { - if (sizes[j]) - { - fprintf(stderr, "ERROR: internal non-zero at %d: %d\n", j, sizes[j]); - print(); - abort(); - } - } - if (nexts[i] && reachable.find(i) == reachable.end()) - { - fprintf(stderr, "ERROR: %d is unreachable from heads\n", i); - print(); - abort(); - } - if (nexts[i] >= 2) - { - if (nexts[i] >= 2+count) - { - fprintf(stderr, "ERROR: next out of range at %d: %d\n", i, nexts[i]); - print(); - abort(); - } - if (prevs[nexts[i]-2] != i+1) - { - fprintf(stderr, "ERROR: prev[next] (%d) != this (%d) at %d", prevs[nexts[i]-2], i+1, i); - print(); - abort(); - } - } - i += (sizes[i] > 1 ? sizes[i] : 1); - } - else - i++; - } -} - -void multilist_alloc_t::print() -{ - printf("heads:"); - for (int i = 0; i < maxn; i++) - if (heads[i]) - printf(" %u=%u", i, heads[i]); - printf("\n"); - printf("sizes:"); - for (int i = 0; i < count; i++) - if (sizes[i]) - printf(" %d=%d", i, sizes[i]); - printf("\n"); - printf("prevs:"); - for (int i = 0; i < count; i++) - if (prevs[i]) - printf(" %d=%d", i, prevs[i]); - printf("\n"); - printf("nexts:"); - for (int i = 0; i < count; i++) - if (nexts[i]) - printf(" %d=%d", i, nexts[i]); - printf("\n"); - printf("items:"); - for (int i = 0; i < count; ) - { - if (sizes[i]) - { - printf(" %u=(s:%d,n:%u,p:%u)", i, sizes[i], nexts[i], prevs[i]); - assert(i+sizes[i] <= count); - i += (sizes[i] > 1 ? sizes[i] : 1); - } - else - i++; - } - printf("\n"); -} - -void multilist_alloc_t::use(uint32_t pos, uint32_t size) -{ - assert(pos < count); - if (sizes[pos] <= 0) - { - uint32_t start = pos; - if (sizes[start] < 0) - start += sizes[start]+1; - else - while (start > 0 && !sizes[start]) - start--; - assert(sizes[start] >= size); - use_full(start); - uint32_t full = sizes[start]; - sizes[pos-1] = -pos+start; - sizes[start] = pos-start; - free(start); - sizes[pos+size-1] = -size; - sizes[pos] = size; - if (pos+size < start+full) - { - sizes[start+full-1] = -(start+full-pos-size); - sizes[pos+size] = start+full-pos-size; - free(pos+size); - } - } - else - { - assert(sizes[pos] >= size); - use_full(pos); - if (sizes[pos] > size) - { - uint32_t full = sizes[pos]; - sizes[pos+size-1] = -size; - sizes[pos] = size; - sizes[pos+full-1] = -full+size; - sizes[pos+size] = full-size; - free(pos+size); - } - } -#ifdef MULTILIST_TRACE - print(); -#endif -} - -void multilist_alloc_t::use_full(uint32_t pos) -{ - uint32_t prevsize = sizes[pos]; - assert(prevsize); - assert(nexts[pos]); - uint32_t pi = (prevsize < maxn ? prevsize : maxn)-1; - if (heads[pi] == pos+1) - heads[pi] = nexts[pos]-1; - if (prevs[pos]) - nexts[prevs[pos]-1] = nexts[pos]; - if (nexts[pos] >= 2) - prevs[nexts[pos]-2] = prevs[pos]; - prevs[pos] = 0; - nexts[pos] = 0; -} - -void multilist_alloc_t::free(uint32_t pos) -{ - do_free(pos); -#ifdef MULTILIST_TRACE - print(); -#endif -} - -void multilist_alloc_t::do_free(uint32_t pos) -{ - assert(!nexts[pos]); - uint32_t size = sizes[pos]; - assert(size > 0); - // merge with previous? - if (pos > 0 && nexts[pos+(sizes[pos-1] == 1 ? -1 : sizes[pos-1])] > 0) - { - assert(sizes[pos-1] < 0 || sizes[pos-1] == 1); - uint32_t prevsize = sizes[pos-1] < 0 ? -sizes[pos-1] : 1; - use_full(pos-prevsize); - sizes[pos] = 0; - sizes[pos-1] = 0; - size += prevsize; - pos -= prevsize; - sizes[pos+size-1] = -size; - sizes[pos] = size; - } - // merge with next? - if (pos+size < count && nexts[pos+size] >= 1) - { - uint32_t nextsize = sizes[pos+size]; - use_full(pos+size); - sizes[pos+size] = 0; - sizes[pos+size-1] = 0; - size += nextsize; - sizes[pos+size-1] = -size; - sizes[pos] = size; - } - uint32_t ni = (size < maxn ? size : maxn)-1; // FIXME ni -> nb (next bucket) - nexts[pos] = heads[ni]+1; - prevs[pos] = 0; - if (heads[ni]) - prevs[heads[ni]-1] = pos+1; - heads[ni] = pos+1; -} - uint64_t blockstore_heap_t::get_pg_id(inode_t inode, uint64_t stripe) { uint64_t pg_num = 0; diff --git a/src/blockstore/blockstore_heap.h b/src/blockstore/blockstore_heap.h index 2c0bf417..eb874d92 100644 --- a/src/blockstore/blockstore_heap.h +++ b/src/blockstore/blockstore_heap.h @@ -12,6 +12,7 @@ #include "../client/object_id.h" #include "../../cpp-btree/btree_map.h" #include "blockstore_disk.h" +#include "multilist.h" struct pool_shard_settings_t { @@ -110,23 +111,6 @@ struct heap_inflight_lsn_t uint64_t flags; }; -struct multilist_alloc_t -{ - const uint32_t count, maxn; - std::vector sizes; - std::vector nexts, prevs, heads; - - multilist_alloc_t(uint32_t count, uint32_t maxn); - bool is_free(uint32_t pos); - uint32_t find(uint32_t size); - void use_full(uint32_t pos); - void use(uint32_t pos, uint32_t size); - void do_free(uint32_t pos); - void free(uint32_t pos); - void verify(); - void print(); -}; - class blockstore_heap_t { friend class heap_write_t; diff --git a/src/blockstore/multilist.cpp b/src/blockstore/multilist.cpp new file mode 100644 index 00000000..8027f526 --- /dev/null +++ b/src/blockstore/multilist.cpp @@ -0,0 +1,278 @@ +// Variable-length O(1) disk space allocator +// Copyright (c) Vitaliy Filippov, 2025+ +// License: VNPL-1.1 (see README.md for details) + +#include +#include +#include +#include +#include "multilist.h" + +multilist_alloc_t::multilist_alloc_t(uint32_t count, uint32_t maxn): + count(count), maxn(maxn) +{ + // not-so-memory-efficient: 16 MB memory per 1 GB buffer space, but buffer spaces are small, so OK + assert(count > 1 && count < 0x80000000); + sizes.resize(count); + nexts.resize(count); // nexts[i] = 0 -> area is used; nexts[i] = 1 -> no next; nexts[i] >= 2 -> next item + prevs.resize(count); + heads.resize(maxn); // heads[i] = 0 -> empty list; heads[i] >= 1 -> list head + sizes[0] = count; + sizes[count-1] = -count; // end + nexts[0] = 1; + heads[maxn-1] = 1; +#ifdef MULTILIST_TRACE + print(); +#endif +} + +bool multilist_alloc_t::is_free(uint32_t pos) +{ + assert(pos < count); + if (sizes[pos] < 0) + pos += sizes[pos]+1; + while (pos > 0 && !sizes[pos]) + pos--; + return nexts[pos] > 0; +} + +uint32_t multilist_alloc_t::find(uint32_t size) +{ + assert(size > 0); + assert(size <= maxn); + for (uint32_t i = size-1; i < maxn; i++) + { + if (heads[i]) + { + return heads[i]-1; + } + } + return UINT32_MAX; +} + +void multilist_alloc_t::verify() +{ + std::set reachable; + for (int i = 0; i < maxn; i++) + { + uint32_t cur = heads[i]; + while (cur) + { + if (!nexts[cur-1]) + { + fprintf(stderr, "ERROR: item %d from freelist %d is not free\n", cur-1, i); + print(); + abort(); + } + if (nexts[cur-1] >= count+2) + { + fprintf(stderr, "ERROR: next out of range at %d: %d\n", cur-1, nexts[cur-1]); + print(); + abort(); + } + if (!(i < maxn-1 ? sizes[cur-1] == i+1 : (sizes[cur-1] >= i+1))) + { + fprintf(stderr, "ERROR: item %d is in wrong freelist: expected size %d, but actual size is %d\n", cur-1, i+1, sizes[cur-1]); + print(); + abort(); + } + if (reachable.find(cur-1) != reachable.end()) + { + fprintf(stderr, "ERROR: doubly-claimed item %d\n", cur-1); + print(); + abort(); + } + reachable.insert(cur-1); + cur = nexts[cur-1]-1; + } + } + for (int i = 0; i < count; ) + { + if (sizes[i]) + { + assert(i+sizes[i] <= count); + if (sizes[i] > 1 && sizes[i+sizes[i]-1] != -sizes[i]) + { + fprintf(stderr, "ERROR: start/end mismatch at %d: sizes[%d] should be %d, but is %d\n", i, i+sizes[i]-1, -sizes[i], sizes[i+sizes[i]-1]); + print(); + abort(); + } + for (int j = i+1; j < i+sizes[i]-1; j++) + { + if (sizes[j]) + { + fprintf(stderr, "ERROR: internal non-zero at %d: %d\n", j, sizes[j]); + print(); + abort(); + } + } + if (nexts[i] && reachable.find(i) == reachable.end()) + { + fprintf(stderr, "ERROR: %d is unreachable from heads\n", i); + print(); + abort(); + } + if (nexts[i] >= 2) + { + if (nexts[i] >= 2+count) + { + fprintf(stderr, "ERROR: next out of range at %d: %d\n", i, nexts[i]); + print(); + abort(); + } + if (prevs[nexts[i]-2] != i+1) + { + fprintf(stderr, "ERROR: prev[next] (%d) != this (%d) at %d", prevs[nexts[i]-2], i+1, i); + print(); + abort(); + } + } + i += (sizes[i] > 1 ? sizes[i] : 1); + } + else + i++; + } +} + +void multilist_alloc_t::print() +{ + printf("heads:"); + for (int i = 0; i < maxn; i++) + if (heads[i]) + printf(" %u=%u", i, heads[i]); + printf("\n"); + printf("sizes:"); + for (int i = 0; i < count; i++) + if (sizes[i]) + printf(" %d=%d", i, sizes[i]); + printf("\n"); + printf("prevs:"); + for (int i = 0; i < count; i++) + if (prevs[i]) + printf(" %d=%d", i, prevs[i]); + printf("\n"); + printf("nexts:"); + for (int i = 0; i < count; i++) + if (nexts[i]) + printf(" %d=%d", i, nexts[i]); + printf("\n"); + printf("items:"); + for (int i = 0; i < count; ) + { + if (sizes[i]) + { + printf(" %u=(s:%d,n:%u,p:%u)", i, sizes[i], nexts[i], prevs[i]); + assert(i+sizes[i] <= count); + i += (sizes[i] > 1 ? sizes[i] : 1); + } + else + i++; + } + printf("\n"); +} + +void multilist_alloc_t::use(uint32_t pos, uint32_t size) +{ + assert(pos < count); + if (sizes[pos] <= 0) + { + uint32_t start = pos; + if (sizes[start] < 0) + start += sizes[start]+1; + else + while (start > 0 && !sizes[start]) + start--; + assert(sizes[start] >= size); + use_full(start); + uint32_t full = sizes[start]; + sizes[pos-1] = -pos+start; + sizes[start] = pos-start; + free(start); + sizes[pos+size-1] = -size; + sizes[pos] = size; + if (pos+size < start+full) + { + sizes[start+full-1] = -(start+full-pos-size); + sizes[pos+size] = start+full-pos-size; + free(pos+size); + } + } + else + { + assert(sizes[pos] >= size); + use_full(pos); + if (sizes[pos] > size) + { + uint32_t full = sizes[pos]; + sizes[pos+size-1] = -size; + sizes[pos] = size; + sizes[pos+full-1] = -full+size; + sizes[pos+size] = full-size; + free(pos+size); + } + } +#ifdef MULTILIST_TRACE + print(); +#endif +} + +void multilist_alloc_t::use_full(uint32_t pos) +{ + uint32_t prevsize = sizes[pos]; + assert(prevsize); + assert(nexts[pos]); + uint32_t pi = (prevsize < maxn ? prevsize : maxn)-1; + if (heads[pi] == pos+1) + heads[pi] = nexts[pos]-1; + if (prevs[pos]) + nexts[prevs[pos]-1] = nexts[pos]; + if (nexts[pos] >= 2) + prevs[nexts[pos]-2] = prevs[pos]; + prevs[pos] = 0; + nexts[pos] = 0; +} + +void multilist_alloc_t::free(uint32_t pos) +{ + do_free(pos); +#ifdef MULTILIST_TRACE + print(); +#endif +} + +void multilist_alloc_t::do_free(uint32_t pos) +{ + assert(!nexts[pos]); + uint32_t size = sizes[pos]; + assert(size > 0); + // merge with previous? + if (pos > 0 && nexts[pos+(sizes[pos-1] == 1 ? -1 : sizes[pos-1])] > 0) + { + assert(sizes[pos-1] < 0 || sizes[pos-1] == 1); + uint32_t prevsize = sizes[pos-1] < 0 ? -sizes[pos-1] : 1; + use_full(pos-prevsize); + sizes[pos] = 0; + sizes[pos-1] = 0; + size += prevsize; + pos -= prevsize; + sizes[pos+size-1] = -size; + sizes[pos] = size; + } + // merge with next? + if (pos+size < count && nexts[pos+size] >= 1) + { + uint32_t nextsize = sizes[pos+size]; + use_full(pos+size); + sizes[pos+size] = 0; + sizes[pos+size-1] = 0; + size += nextsize; + sizes[pos+size-1] = -size; + sizes[pos] = size; + } + uint32_t ni = (size < maxn ? size : maxn)-1; // FIXME ni -> nb (next bucket) + nexts[pos] = heads[ni]+1; + prevs[pos] = 0; + if (heads[ni]) + prevs[heads[ni]-1] = pos+1; + heads[ni] = pos+1; +} diff --git a/src/blockstore/multilist.h b/src/blockstore/multilist.h new file mode 100644 index 00000000..8079faa1 --- /dev/null +++ b/src/blockstore/multilist.h @@ -0,0 +1,25 @@ +// Variable-length O(1) disk space allocator +// Copyright (c) Vitaliy Filippov, 2025+ +// License: VNPL-1.1 (see README.md for details) + +#pragma once + +#include +#include + +struct multilist_alloc_t +{ + const uint32_t count, maxn; + std::vector sizes; + std::vector nexts, prevs, heads; + + multilist_alloc_t(uint32_t count, uint32_t maxn); + bool is_free(uint32_t pos); + uint32_t find(uint32_t size); + void use_full(uint32_t pos); + void use(uint32_t pos, uint32_t size); + void do_free(uint32_t pos); + void free(uint32_t pos); + void verify(); + void print(); +}; diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 59a74fbd..a40ca535 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -37,6 +37,7 @@ add_test(NAME test_allocator COMMAND test_allocator) # test_heap add_executable(test_heap test_heap.cpp + ../blockstore/multilist.cpp ../blockstore/blockstore_heap.cpp ../util/crc32c.c ../util/allocator.cpp