From 97dfbfad758020dcea0909cf3f6793624cb0fca8 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Mon, 1 Sep 2025 18:08:07 +0300 Subject: [PATCH] Add missing list_stable_limit support --- docs/config/osd.en.md | 2 +- docs/config/osd.ru.md | 2 +- docs/config/src/osd.yml | 2 +- src/blockstore/blockstore_heap.cpp | 12 ++++++++---- src/blockstore/blockstore_heap.h | 2 +- src/blockstore/blockstore_impl.cpp | 13 ++++++++++++- src/osd/osd.cpp | 2 +- src/osd/osd.h | 2 +- src/test/test_heap.cpp | 24 ++++++++++++++++-------- tests/test_scrub.sh | 2 ++ 10 files changed, 44 insertions(+), 19 deletions(-) diff --git a/docs/config/osd.en.md b/docs/config/osd.en.md index 7baadf1a..9937185b 100644 --- a/docs/config/osd.en.md +++ b/docs/config/osd.en.md @@ -491,7 +491,7 @@ Can be used to slow down scrubbing if it affects user load too much. ## scrub_list_limit - Type: integer -- Default: 1000 +- Default: 262144 - Can be changed online: yes Number of objects to list in one listing operation during scrub. diff --git a/docs/config/osd.ru.md b/docs/config/osd.ru.md index ead4f369..f3edb106 100644 --- a/docs/config/osd.ru.md +++ b/docs/config/osd.ru.md @@ -514,7 +514,7 @@ fsync небезопасным даже с режимом "directsync". ## scrub_list_limit - Тип: целое число -- Значение по умолчанию: 1000 +- Значение по умолчанию: 262144 - Можно менять на лету: да Размер загружаемых за одну операцию списков объектов в процессе фоновой diff --git a/docs/config/src/osd.yml b/docs/config/src/osd.yml index 50f1ad58..f474a1a9 100644 --- a/docs/config/src/osd.yml +++ b/docs/config/src/osd.yml @@ -566,7 +566,7 @@ сильно влияет на пользовательскую нагрузку. - name: scrub_list_limit type: int - default: 1000 + default: 262144 online: true info: | Number of objects to list in one listing operation during scrub. diff --git a/src/blockstore/blockstore_heap.cpp b/src/blockstore/blockstore_heap.cpp index 5864e5f1..ee4fcee7 100644 --- a/src/blockstore/blockstore_heap.cpp +++ b/src/blockstore/blockstore_heap.cpp @@ -1965,15 +1965,15 @@ void blockstore_heap_t::add_used_space(uint32_t block_num, int32_t used_delta) meta_alloc_count++; } -int blockstore_heap_t::list_objects(uint32_t pg_num, uint64_t min_inode, uint64_t max_inode, +int blockstore_heap_t::list_objects(uint32_t pg_num, object_id min_oid, object_id max_oid, obj_ver_id **result_list, size_t *stable_count, size_t *unstable_count) { obj_ver_id *res = NULL; size_t res_size = 0, res_alloc = 0; obj_ver_id *unstable = NULL; size_t unstable_size = 0, unstable_alloc = 0; - uint64_t pool_id = (min_inode >> (64-POOL_ID_BITS)); - if (pool_id == 0 || pool_id != (max_inode >> (64-POOL_ID_BITS))) + uint64_t pool_id = (min_oid.inode >> (64-POOL_ID_BITS)); + if (pool_id == 0 || pool_id != (max_oid.inode >> (64-POOL_ID_BITS))) { return EINVAL; } @@ -1988,13 +1988,17 @@ int blockstore_heap_t::list_objects(uint32_t pg_num, uint64_t min_inode, uint64_ auto last_it = block_index[pool_pg_id].end(); for (auto inode_it = first_it; inode_it != last_it; inode_it++) { - if (inode_it->first < min_inode || inode_it->first > max_inode) + if (inode_it->first < min_oid.inode || inode_it->first > max_oid.inode) { continue; } for (auto & stripe_pair: inode_it->second) { auto oid = (object_id){ .inode = inode_it->first, .stripe = stripe_pair.first }; + if (oid < min_oid || max_oid < oid) + { + continue; + } const uint64_t block_pos = stripe_pair.second; const uint32_t block_num = block_pos / dsk->meta_block_size; heap_object_t *obj = (heap_object_t*)(block_info[block_num].data + (block_pos % dsk->meta_block_size)); diff --git a/src/blockstore/blockstore_heap.h b/src/blockstore/blockstore_heap.h index e4da238d..b3b67367 100644 --- a/src/blockstore/blockstore_heap.h +++ b/src/blockstore/blockstore_heap.h @@ -291,7 +291,7 @@ public: // mark an object as compacted up to the given lsn int compact_object(object_id oid, uint64_t lsn, uint8_t *new_csums); // retrieve object listing from a PG - int list_objects(uint32_t pg_num, uint64_t min_inode, uint64_t max_inode, + int list_objects(uint32_t pg_num, object_id min_oid, object_id max_oid, obj_ver_id **result_list, size_t *stable_count, size_t *unstable_count); // set a block number for a new object and returns error status: 0, EAGAIN or ENOSPC int get_block_for_new_object(uint32_t & out_block_num, uint32_t size = 0); diff --git a/src/blockstore/blockstore_impl.cpp b/src/blockstore/blockstore_impl.cpp index ec0f7b4b..2eb4ff5e 100644 --- a/src/blockstore/blockstore_impl.cpp +++ b/src/blockstore/blockstore_impl.cpp @@ -325,7 +325,18 @@ void blockstore_impl_t::process_list(blockstore_op_t *op) heap->reshard(INODE_POOL(min_inode), pg_count, pg_stripe_size); obj_ver_id *result = NULL; size_t stable_count = 0, unstable_count = 0; - int res = heap->list_objects(list_pg, min_inode, max_inode, &result, &stable_count, &unstable_count); + int res = heap->list_objects(list_pg, op->min_oid, op->max_oid, &result, &stable_count, &unstable_count); + if (op->list_stable_limit) + { + // Ordered result is expected - used by scrub + // We use an unordered map + std::sort(result, result + stable_count); + if (stable_count > op->list_stable_limit) + { + memmove(result + op->list_stable_limit, result + stable_count, unstable_count); + stable_count = op->list_stable_limit; + } + } op->version = stable_count; op->retval = res == 0 ? stable_count+unstable_count : -res; op->buf = (uint8_t*)result; diff --git a/src/osd/osd.cpp b/src/osd/osd.cpp index e766e367..ffd6f6f2 100644 --- a/src/osd/osd.cpp +++ b/src/osd/osd.cpp @@ -291,7 +291,7 @@ void osd_t::parse_config(bool init) scrub_sleep_ms = config["scrub_sleep"].uint64_value(); scrub_list_limit = config["scrub_list_limit"].uint64_value(); if (!scrub_list_limit) - scrub_list_limit = 1000; + scrub_list_limit = 262144; if (!old_auto_scrub && auto_scrub) { // Schedule scrubbing diff --git a/src/osd/osd.h b/src/osd/osd.h index e7ecb6cd..1b52ccfe 100644 --- a/src/osd/osd.h +++ b/src/osd/osd.h @@ -149,7 +149,7 @@ class osd_t uint64_t global_scrub_interval = 30*86400; uint64_t scrub_queue_depth = 1; uint64_t scrub_sleep_ms = 0; - uint32_t scrub_list_limit = 1000; + uint32_t scrub_list_limit = 262144; bool scrub_find_best = true; uint64_t scrub_ec_max_bruteforce = 100; bool enable_pg_locks = false; diff --git a/src/test/test_heap.cpp b/src/test/test_heap.cpp index c4c094bb..c011c9ce 100644 --- a/src/test/test_heap.cpp +++ b/src/test/test_heap.cpp @@ -776,21 +776,26 @@ void test_reshard_list() obj_ver_id *listing = NULL; size_t stable_count = 0, unstable_count = 0; - res = heap.list_objects(1, INODE_WITH_POOL(0, 1), INODE_WITH_POOL(1, 1), &listing, &stable_count, &unstable_count); + res = heap.list_objects(1, (object_id){ .inode = INODE_WITH_POOL(0, 1) }, + (object_id){ .inode = INODE_WITH_POOL(1, 1), .stripe = UINT64_MAX }, &listing, &stable_count, &unstable_count); assert(res == EINVAL); - res = heap.list_objects(1, INODE_WITH_POOL(1, 1), INODE_WITH_POOL(2, 1), &listing, &stable_count, &unstable_count); + res = heap.list_objects(1, (object_id){ .inode = INODE_WITH_POOL(1, 1) }, + (object_id){ .inode = INODE_WITH_POOL(2, 1), .stripe = UINT64_MAX }, &listing, &stable_count, &unstable_count); assert(res == EINVAL); - res = heap.list_objects(2, INODE_WITH_POOL(1, 1), INODE_WITH_POOL(1, 1), &listing, &stable_count, &unstable_count); + res = heap.list_objects(2, (object_id){ .inode = INODE_WITH_POOL(1, 1) }, + (object_id){ .inode = INODE_WITH_POOL(1, 1), .stripe = UINT64_MAX }, &listing, &stable_count, &unstable_count); assert(res == EINVAL); - res = heap.list_objects(1, INODE_WITH_POOL(1, 1), INODE_WITH_POOL(1, UINT64_MAX), &listing, &stable_count, &unstable_count); + res = heap.list_objects(1, (object_id){ .inode = INODE_WITH_POOL(1, 1) }, + (object_id){ .inode = INODE_WITH_POOL(1, UINT64_MAX), .stripe = UINT64_MAX }, &listing, &stable_count, &unstable_count); assert(res == 0); assert(stable_count == 4); assert(unstable_count == 2); free(listing); listing = NULL; - res = heap.list_objects(1, INODE_WITH_POOL(1, 1), INODE_WITH_POOL(1, 1), &listing, &stable_count, &unstable_count); + res = heap.list_objects(1, (object_id){ .inode = INODE_WITH_POOL(1, 1) }, + (object_id){ .inode = INODE_WITH_POOL(1, 1), .stripe = UINT64_MAX }, &listing, &stable_count, &unstable_count); assert(res == 0); assert(stable_count == 3); assert(unstable_count == 0); @@ -805,16 +810,19 @@ void test_reshard_list() assert(heap.read_entry((object_id){ .inode = INODE_WITH_POOL(1, 2), .stripe = 0x60000 }, NULL)); assert(!heap.read_entry((object_id){ .inode = INODE_WITH_POOL(1, 2), .stripe = 0x80000 }, NULL)); - res = heap.list_objects(3, INODE_WITH_POOL(1, 1), INODE_WITH_POOL(1, 1), &listing, &stable_count, &unstable_count); + res = heap.list_objects(3, (object_id){ .inode = INODE_WITH_POOL(1, 1) }, + (object_id){ .inode = INODE_WITH_POOL(1, 1), .stripe = UINT64_MAX }, &listing, &stable_count, &unstable_count); assert(res == EINVAL); - res = heap.list_objects(1, INODE_WITH_POOL(1, 1), INODE_WITH_POOL(1, 1), &listing, &stable_count, &unstable_count); + res = heap.list_objects(1, (object_id){ .inode = INODE_WITH_POOL(1, 1) }, + (object_id){ .inode = INODE_WITH_POOL(1, 1), .stripe = UINT64_MAX }, &listing, &stable_count, &unstable_count); assert(res == 0); assert(stable_count == 2); assert(unstable_count == 0); free(listing); listing = NULL; - res = heap.list_objects(2, INODE_WITH_POOL(1, 1), INODE_WITH_POOL(1, UINT64_MAX), &listing, &stable_count, &unstable_count); + res = heap.list_objects(2, (object_id){ .inode = INODE_WITH_POOL(1, 1) }, + (object_id){ .inode = INODE_WITH_POOL(1, UINT64_MAX), .stripe = UINT64_MAX }, &listing, &stable_count, &unstable_count); assert(res == 0); assert(stable_count == 2); assert(unstable_count == 2); diff --git a/tests/test_scrub.sh b/tests/test_scrub.sh index b14cb637..0f458d4a 100755 --- a/tests/test_scrub.sh +++ b/tests/test_scrub.sh @@ -7,6 +7,8 @@ if [[ ("$SCHEME" = "" || "$SCHEME" = "replicated") && ("$PG_SIZE" = "" || "$PG_S OSD_COUNT=2 fi +OSD_ARGS="--scrub_list_limit 1000 $OSD_ARGS" + . `dirname $0`/run_3osds.sh check_qemu