thebestvibedev/tromcho.net
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add missing list_stable_limit support

Browse Source
This commit is contained in:
Vitaliy Filippov
2025-12-02 01:52:12 +03:00
parent c148f97ee4
commit 97dfbfad75
10 changed files with 44 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/blockstore/blockstore_heap.cpp
+8 -4
View File
@@ -1965,15 +1965,15 @@ void blockstore_heap_t::add_used_space(uint32_t block_num, int32_t used_delta)
meta_alloc_count++;
}
int blockstore_heap_t::list_objects(uint32_t pg_num, uint64_t min_inode, uint64_t max_inode,
int blockstore_heap_t::list_objects(uint32_t pg_num, object_id min_oid, object_id max_oid,
obj_ver_id **result_list, size_t *stable_count, size_t *unstable_count)
{
obj_ver_id *res = NULL;
size_t res_size = 0, res_alloc = 0;
obj_ver_id *unstable = NULL;
size_t unstable_size = 0, unstable_alloc = 0;
uint64_t pool_id = (min_inode >> (64-POOL_ID_BITS));
if (pool_id == 0 || pool_id != (max_inode >> (64-POOL_ID_BITS)))
uint64_t pool_id = (min_oid.inode >> (64-POOL_ID_BITS));
if (pool_id == 0 || pool_id != (max_oid.inode >> (64-POOL_ID_BITS)))
{
return EINVAL;
}
@@ -1988,13 +1988,17 @@ int blockstore_heap_t::list_objects(uint32_t pg_num, uint64_t min_inode, uint64_
auto last_it = block_index[pool_pg_id].end();
for (auto inode_it = first_it; inode_it != last_it; inode_it++)
{
if (inode_it->first < min_inode || inode_it->first > max_inode)
if (inode_it->first < min_oid.inode || inode_it->first > max_oid.inode)
{
continue;
}
for (auto & stripe_pair: inode_it->second)
{
auto oid = (object_id){ .inode = inode_it->first, .stripe = stripe_pair.first };
if (oid < min_oid || max_oid < oid)
{
continue;
}
const uint64_t block_pos = stripe_pair.second;
const uint32_t block_num = block_pos / dsk->meta_block_size;
heap_object_t *obj = (heap_object_t*)(block_info[block_num].data + (block_pos % dsk->meta_block_size));
src/blockstore/blockstore_heap.h
+1 -1
View File
@@ -291,7 +291,7 @@ public:
// mark an object as compacted up to the given lsn
int compact_object(object_id oid, uint64_t lsn, uint8_t *new_csums);
// retrieve object listing from a PG
int list_objects(uint32_t pg_num, uint64_t min_inode, uint64_t max_inode,
int list_objects(uint32_t pg_num, object_id min_oid, object_id max_oid,
obj_ver_id **result_list, size_t *stable_count, size_t *unstable_count);
// set a block number for a new object and returns error status: 0, EAGAIN or ENOSPC
int get_block_for_new_object(uint32_t & out_block_num, uint32_t size = 0);
src/blockstore/blockstore_impl.cpp
+12 -1
View File
@@ -325,7 +325,18 @@ void blockstore_impl_t::process_list(blockstore_op_t *op)
heap->reshard(INODE_POOL(min_inode), pg_count, pg_stripe_size);
obj_ver_id *result = NULL;
size_t stable_count = 0, unstable_count = 0;
int res = heap->list_objects(list_pg, min_inode, max_inode, &result, &stable_count, &unstable_count);
int res = heap->list_objects(list_pg, op->min_oid, op->max_oid, &result, &stable_count, &unstable_count);
if (op->list_stable_limit)
{
// Ordered result is expected - used by scrub
// We use an unordered map
std::sort(result, result + stable_count);
if (stable_count > op->list_stable_limit)
{
memmove(result + op->list_stable_limit, result + stable_count, unstable_count);
stable_count = op->list_stable_limit;
}
}
op->version = stable_count;
op->retval = res == 0 ? stable_count+unstable_count : -res;
op->buf = (uint8_t*)result;
src/osd/osd.cpp
+1 -1
View File
@@ -291,7 +291,7 @@ void osd_t::parse_config(bool init)
scrub_sleep_ms = config["scrub_sleep"].uint64_value();
scrub_list_limit = config["scrub_list_limit"].uint64_value();
if (!scrub_list_limit)
scrub_list_limit = 1000;
scrub_list_limit = 262144;
if (!old_auto_scrub && auto_scrub)
{
// Schedule scrubbing
src/osd/osd.h
+1 -1
View File
@@ -149,7 +149,7 @@ class osd_t
uint64_t global_scrub_interval = 30*86400;
uint64_t scrub_queue_depth = 1;
uint64_t scrub_sleep_ms = 0;
uint32_t scrub_list_limit = 1000;
uint32_t scrub_list_limit = 262144;
bool scrub_find_best = true;
uint64_t scrub_ec_max_bruteforce = 100;
bool enable_pg_locks = false;
src/test/test_heap.cpp
+16 -8
View File
@@ -776,21 +776,26 @@ void test_reshard_list()
obj_ver_id *listing = NULL;
size_t stable_count = 0, unstable_count = 0;
res = heap.list_objects(1, INODE_WITH_POOL(0, 1), INODE_WITH_POOL(1, 1), &listing, &stable_count, &unstable_count);
res = heap.list_objects(1, (object_id){ .inode = INODE_WITH_POOL(0, 1) },
(object_id){ .inode = INODE_WITH_POOL(1, 1), .stripe = UINT64_MAX }, &listing, &stable_count, &unstable_count);
assert(res == EINVAL);
res = heap.list_objects(1, INODE_WITH_POOL(1, 1), INODE_WITH_POOL(2, 1), &listing, &stable_count, &unstable_count);
res = heap.list_objects(1, (object_id){ .inode = INODE_WITH_POOL(1, 1) },
(object_id){ .inode = INODE_WITH_POOL(2, 1), .stripe = UINT64_MAX }, &listing, &stable_count, &unstable_count);
assert(res == EINVAL);
res = heap.list_objects(2, INODE_WITH_POOL(1, 1), INODE_WITH_POOL(1, 1), &listing, &stable_count, &unstable_count);
res = heap.list_objects(2, (object_id){ .inode = INODE_WITH_POOL(1, 1) },
(object_id){ .inode = INODE_WITH_POOL(1, 1), .stripe = UINT64_MAX }, &listing, &stable_count, &unstable_count);
assert(res == EINVAL);
res = heap.list_objects(1, INODE_WITH_POOL(1, 1), INODE_WITH_POOL(1, UINT64_MAX), &listing, &stable_count, &unstable_count);
res = heap.list_objects(1, (object_id){ .inode = INODE_WITH_POOL(1, 1) },
(object_id){ .inode = INODE_WITH_POOL(1, UINT64_MAX), .stripe = UINT64_MAX }, &listing, &stable_count, &unstable_count);
assert(res == 0);
assert(stable_count == 4);
assert(unstable_count == 2);
free(listing);
listing = NULL;
res = heap.list_objects(1, INODE_WITH_POOL(1, 1), INODE_WITH_POOL(1, 1), &listing, &stable_count, &unstable_count);
res = heap.list_objects(1, (object_id){ .inode = INODE_WITH_POOL(1, 1) },
(object_id){ .inode = INODE_WITH_POOL(1, 1), .stripe = UINT64_MAX }, &listing, &stable_count, &unstable_count);
assert(res == 0);
assert(stable_count == 3);
assert(unstable_count == 0);
@@ -805,16 +810,19 @@ void test_reshard_list()
assert(heap.read_entry((object_id){ .inode = INODE_WITH_POOL(1, 2), .stripe = 0x60000 }, NULL));
assert(!heap.read_entry((object_id){ .inode = INODE_WITH_POOL(1, 2), .stripe = 0x80000 }, NULL));
res = heap.list_objects(3, INODE_WITH_POOL(1, 1), INODE_WITH_POOL(1, 1), &listing, &stable_count, &unstable_count);
res = heap.list_objects(3, (object_id){ .inode = INODE_WITH_POOL(1, 1) },
(object_id){ .inode = INODE_WITH_POOL(1, 1), .stripe = UINT64_MAX }, &listing, &stable_count, &unstable_count);
assert(res == EINVAL);
res = heap.list_objects(1, INODE_WITH_POOL(1, 1), INODE_WITH_POOL(1, 1), &listing, &stable_count, &unstable_count);
res = heap.list_objects(1, (object_id){ .inode = INODE_WITH_POOL(1, 1) },
(object_id){ .inode = INODE_WITH_POOL(1, 1), .stripe = UINT64_MAX }, &listing, &stable_count, &unstable_count);
assert(res == 0);
assert(stable_count == 2);
assert(unstable_count == 0);
free(listing);
listing = NULL;
res = heap.list_objects(2, INODE_WITH_POOL(1, 1), INODE_WITH_POOL(1, UINT64_MAX), &listing, &stable_count, &unstable_count);
res = heap.list_objects(2, (object_id){ .inode = INODE_WITH_POOL(1, 1) },
(object_id){ .inode = INODE_WITH_POOL(1, UINT64_MAX), .stripe = UINT64_MAX }, &listing, &stable_count, &unstable_count);
assert(res == 0);
assert(stable_count == 2);
assert(unstable_count == 2);