diff --git a/docs/config/osd.en.md b/docs/config/osd.en.md index 3fc095b2..8aab9eb5 100644 --- a/docs/config/osd.en.md +++ b/docs/config/osd.en.md @@ -67,6 +67,8 @@ with an OSD restart or, for some of them, even without restarting by updating co - [pg_lock_retry_interval_ms](#pg_lock_retry_interval_ms) - [atomic_write_size](#atomic_write_size) - [use_atomic_flag](#use_atomic_flag) +- [pg_reshard_chunk_size](#pg_reshard_chunk_size) +- [pg_reshard_chunk_pause_ms](#pg_reshard_chunk_pause_ms) ## bind_address @@ -713,3 +715,21 @@ even though the NVMe specification allows them. For NVMe disks with `scheduler=none` writes aren't fragmented anyway so it's not a big deal. However, you can rebuild your kernel with [this patch](../../patches/linux-fix-atomic-write-checks.diff) and turn this option on. It will make your atomic writes a bit safer. + +## pg_reshard_chunk_size + +- Type: integer +- Default: 100000 + +Pool PG count change is a CPU-intensive operation because OSDs store the full object database +in memory and have to move all entries between old and new PGs. Thus it's performed in chunks, +with pauses between chunks to prevent blocking OSD's event loop and other clients' operations. +This option sets the maximum number of object is a chunk. Moving 100k objects usually takes +50-100ms. Chunk size equal to 0 means unlimited. + +## pg_reshard_chunk_pause_ms + +- Type: milliseconds +- Default: 100 + +This option sets the interval between handling two PG count change chunks. diff --git a/docs/config/osd.ru.md b/docs/config/osd.ru.md index 97cdd60a..ae60f773 100644 --- a/docs/config/osd.ru.md +++ b/docs/config/osd.ru.md @@ -68,6 +68,8 @@ - [pg_lock_retry_interval_ms](#pg_lock_retry_interval_ms) - [atomic_write_size](#atomic_write_size) - [use_atomic_flag](#use_atomic_flag) +- [pg_reshard_chunk_size](#pg_reshard_chunk_size) +- [pg_reshard_chunk_pause_ms](#pg_reshard_chunk_pause_ms) ## bind_address @@ -751,3 +753,22 @@ pg_minsize OSD во время переключений, что может по Для NVMe-дисков с `scheduler=none` запросы записи и так не фрагментируются, так что это не так уж и важно, однако вы можете пересобрать своё ядро с [этим патчем](../../patches/linux-fix-atomic-write-checks.diff) и включить данную опцию. Это сделает вашу атомарную запись капельку безопаснее. + +## pg_reshard_chunk_size + +- Тип: целое число +- Значение по умолчанию: 100000 + +Изменение числа PG в пуле заметно загружает процессор, так как OSD хранят полную базу данных +объектов в памяти и им приходится перемещать все записи объектов между старыми и новыми PG. +Поэтому изменение применяется порциями, с паузами между порциями, чтобы не блокировать обработку +событий OSD и операции остальных клиентов. Данная опция задаёт максимальное число объектов +в порции. Перемещение 100 тысяч объектов (значение по умолчанию) обычно занимает порядка +50-100 миллисекунд. Значение опции 0 отключает лимит размера порции. + +## pg_reshard_chunk_pause_ms + +- Тип: миллисекунды +- Значение по умолчанию: 100 + +Данная опция задаёт интервал между обработкой двух порций изменения числа PG пулов. diff --git a/docs/config/src/osd.yml b/docs/config/src/osd.yml index 1d4399b1..19e40411 100644 --- a/docs/config/src/osd.yml +++ b/docs/config/src/osd.yml @@ -883,3 +883,26 @@ Для NVMe-дисков с `scheduler=none` запросы записи и так не фрагментируются, так что это не так уж и важно, однако вы можете пересобрать своё ядро с [этим патчем](../../patches/linux-fix-atomic-write-checks.diff) и включить данную опцию. Это сделает вашу атомарную запись капельку безопаснее. +- name: pg_reshard_chunk_size + type: int + default: 100000 + info: | + Pool PG count change is a CPU-intensive operation because OSDs store the full object database + in memory and have to move all entries between old and new PGs. Thus it's performed in chunks, + with pauses between chunks to prevent blocking OSD's event loop and other clients' operations. + This option sets the maximum number of object is a chunk. Moving 100k objects usually takes + 50-100ms. Chunk size equal to 0 means unlimited. + info_ru: | + Изменение числа PG в пуле заметно загружает процессор, так как OSD хранят полную базу данных + объектов в памяти и им приходится перемещать все записи объектов между старыми и новыми PG. + Поэтому изменение применяется порциями, с паузами между порциями, чтобы не блокировать обработку + событий OSD и операции остальных клиентов. Данная опция задаёт максимальное число объектов + в порции. Перемещение 100 тысяч объектов (значение по умолчанию) обычно занимает порядка + 50-100 миллисекунд. Значение опции 0 отключает лимит размера порции. +- name: pg_reshard_chunk_pause_ms + type: ms + default: 100 + info: | + This option sets the interval between handling two PG count change chunks. + info_ru: | + Данная опция задаёт интервал между обработкой двух порций изменения числа PG пулов. diff --git a/src/client/etcd_state_client.h b/src/client/etcd_state_client.h index c1e0f97f..010e556b 100644 --- a/src/client/etcd_state_client.h +++ b/src/client/etcd_state_client.h @@ -68,6 +68,11 @@ struct pool_config_t std::string used_for_app; int backfillfull = 0; int local_reads = 0; + + // runtime data, used only by OSD: + uint64_t applied_pg_count = 0; + uint64_t applied_pg_stripe_size = 0; + void *reshard_state = NULL; }; struct inode_config_t diff --git a/src/osd/osd.cpp b/src/osd/osd.cpp index 10ff4a94..7c0e9727 100644 --- a/src/osd/osd.cpp +++ b/src/osd/osd.cpp @@ -311,6 +311,12 @@ void osd_t::parse_config(bool init) scrub_list_limit = config["scrub_list_limit"].uint64_value(); if (!scrub_list_limit) scrub_list_limit = 262144; + pg_reshard_chunk_size = config["pg_reshard_chunk_size"].uint64_value(); + if (!pg_reshard_chunk_size) + pg_reshard_chunk_size = 100000; + pg_reshard_chunk_pause_ms = config["pg_reshard_chunk_pause_ms"].uint64_value(); + if (!pg_reshard_chunk_pause_ms) + pg_reshard_chunk_pause_ms = 100; if (!old_auto_scrub && auto_scrub) { // Schedule scrubbing diff --git a/src/osd/osd.h b/src/osd/osd.h index cefc40db..b8cd63a7 100644 --- a/src/osd/osd.h +++ b/src/osd/osd.h @@ -148,6 +148,8 @@ class osd_t bool enable_pg_locks = false; bool pg_locks_localize_only = false; uint64_t pg_lock_retry_interval_ms = 100; + uint64_t pg_reshard_chunk_size = 100000; + uint64_t pg_reshard_chunk_pause_ms = 100; // cluster state @@ -173,6 +175,8 @@ class osd_t std::map pgs; std::set dirty_pgs; std::set dirty_osds; + std::vector reshard_pools; + int reshard_timer_id = -1; int copies_to_delete_after_sync_count = 0; uint64_t misplaced_objects = 0, degraded_objects = 0, incomplete_objects = 0, inconsistent_objects = 0, corrupted_objects = 0; int peering_state = 0; @@ -271,6 +275,7 @@ class osd_t void apply_no_inode_stats(); void apply_pg_count(); void apply_pg_config(); + void reshard_continue(); // event loop, socket read/write void loop(); diff --git a/src/osd/osd_cluster.cpp b/src/osd/osd_cluster.cpp index 768de27f..663b3319 100644 --- a/src/osd/osd_cluster.cpp +++ b/src/osd/osd_cluster.cpp @@ -727,8 +727,9 @@ void osd_t::apply_pg_count() { for (auto & pool_item: st_cli.pool_config) { - if (pool_item.second.real_pg_count != 0 && - pool_item.second.real_pg_count != pg_counts[pool_item.first]) + auto & pool_cfg = pool_item.second; + if (pool_cfg.real_pg_count != 0 && + pool_cfg.real_pg_count != pg_counts[pool_item.first]) { // Check that all pool PGs are offline. It is not allowed to change PG count when any PGs are online // The external tool must wait for all PGs to come down before changing PG count @@ -753,14 +754,79 @@ void osd_t::apply_pg_count() printf( "[OSD %ju] PG count change detected for pool %u (new is %ju, old is %u)," " but %u PG(s) are still active as primary and %u as secondary. This is not allowed. Exiting\n", - this->osd_num, pool_item.first, pool_item.second.real_pg_count, pg_counts[pool_item.first], + this->osd_num, pool_item.first, pool_cfg.real_pg_count, pg_counts[pool_item.first], still_active_primary, still_active_secondary ); force_stop(1); return; } } - this->pg_counts[pool_item.first] = pool_item.second.real_pg_count; + if (bs && (pool_cfg.real_pg_count != pool_cfg.applied_pg_count || + pool_cfg.pg_stripe_size != pool_cfg.applied_pg_stripe_size) && + !pool_cfg.reshard_state) + { + pool_cfg.applied_pg_count = pool_cfg.real_pg_count; + pool_cfg.applied_pg_stripe_size = pool_cfg.pg_stripe_size; + pool_cfg.reshard_state = bs->reshard_start(pool_item.first, pool_cfg.real_pg_count, pool_cfg.pg_stripe_size, pg_reshard_chunk_size); + if (pool_cfg.reshard_state) + { + reshard_pools.push_back(pool_item.first); + } + } + this->pg_counts[pool_item.first] = pool_cfg.real_pg_count; + } + if (reshard_pools.size() && reshard_timer_id < 0) + { + reshard_timer_id = tfd->set_timer(pg_reshard_chunk_pause_ms, false, [this](int) + { + reshard_continue(); + }); + } +} + +void osd_t::reshard_continue() +{ +again: + auto pool_id = reshard_pools[0]; + auto pool_it = st_cli.pool_config.find(pool_id); + if (pool_it == st_cli.pool_config.end() || !pool_it->second.reshard_state) + { + reshard_pools.erase(reshard_pools.begin()); + goto again; + } + auto & pool_cfg = pool_it->second; + bool done = false; + if (pool_cfg.real_pg_count != pool_cfg.applied_pg_count || + pool_cfg.pg_stripe_size != pool_cfg.applied_pg_stripe_size) + { + // PG count changed again, reshard again + bs->reshard_abort(pool_cfg.reshard_state); + pool_cfg.applied_pg_count = pool_cfg.real_pg_count; + pool_cfg.applied_pg_stripe_size = pool_cfg.pg_stripe_size; + pool_cfg.reshard_state = bs->reshard_start(pool_id, pool_cfg.real_pg_count, pool_cfg.pg_stripe_size, pg_reshard_chunk_size); + done = !pool_cfg.reshard_state; + } + else + { + done = bs->reshard_continue(pool_cfg.reshard_state, pg_reshard_chunk_size); + } + if (done) + { + // Pool is resharded + pool_cfg.reshard_state = NULL; + reshard_pools.erase(reshard_pools.begin()); + apply_pg_config(); + } + if (reshard_pools.size()) + { + reshard_timer_id = tfd->set_timer(pg_reshard_chunk_pause_ms, false, [this](int) + { + reshard_continue(); + }); + } + else + { + reshard_timer_id = -1; } } @@ -769,9 +835,15 @@ void osd_t::apply_pg_config() bool all_applied = true; for (auto & pool_item: st_cli.pool_config) { + auto & pool_cfg = pool_item.second; + if (pool_cfg.reshard_state) + { + // Can't apply anything for pools being resharded + continue; + } bool warned_block_size = false; auto pool_id = pool_item.first; - for (auto & kv: pool_item.second.pg_config) + for (auto & kv: pool_cfg.pg_config) { pg_num_t pg_num = kv.first; auto & pg_cfg = kv.second; @@ -780,8 +852,8 @@ void osd_t::apply_pg_config() auto pg_it = this->pgs.find({ .pool_id = pool_id, .pg_num = pg_num }); bool currently_taken = pg_it != this->pgs.end() && pg_it->second.state != PG_OFFLINE; // Check pool block size and bitmap granularity - if (take && this->bs_block_size != pool_item.second.data_block_size || - this->bs_bitmap_granularity != pool_item.second.bitmap_granularity) + if (take && this->bs_block_size != pool_cfg.data_block_size || + this->bs_bitmap_granularity != pool_cfg.bitmap_granularity) { if (!warned_block_size) { @@ -789,7 +861,7 @@ void osd_t::apply_pg_config() "[OSD %ju] My block_size and bitmap_granularity are %u/%u" ", but pool %u has %u/%u. Refusing to start PGs of this pool\n", this->osd_num, bs_block_size, bs_bitmap_granularity, - pool_id, pool_item.second.data_block_size, pool_item.second.bitmap_granularity + pool_id, pool_cfg.data_block_size, pool_cfg.bitmap_granularity ); } warned_block_size = true; @@ -882,12 +954,12 @@ void osd_t::apply_pg_config() } auto & pg = this->pgs[{ .pool_id = pool_id, .pg_num = pg_num }]; pg.state = pg_cfg.cur_primary == this->osd_num ? PG_PEERING : PG_STARTING; - pg.scheme = pool_item.second.scheme; + pg.scheme = pool_cfg.scheme; pg.pg_cursize = 0; - pg.pg_size = pool_item.second.pg_size; - pg.pg_minsize = pool_item.second.pg_minsize; - pg.pg_data_size = pool_item.second.scheme == POOL_SCHEME_REPLICATED - ? 1 : pool_item.second.pg_size - pool_item.second.parity_chunks; + pg.pg_size = pool_cfg.pg_size; + pg.pg_minsize = pool_cfg.pg_minsize; + pg.pg_data_size = pool_cfg.scheme == POOL_SCHEME_REPLICATED + ? 1 : pool_cfg.pg_size - pool_cfg.parity_chunks; pg.pool_id = pool_id; pg.pg_num = pg_num; pg.reported_epoch = pg_cfg.epoch; @@ -896,8 +968,8 @@ void osd_t::apply_pg_config() pg.next_scrub = pg_cfg.next_scrub; pg.target_set = pg_cfg.target_set; pg.disable_pg_locks = pg_locks_localize_only && - (pool_item.second.scheme != POOL_SCHEME_REPLICATED || - pool_item.second.local_reads == POOL_LOCAL_READ_PRIMARY); + (pool_cfg.scheme != POOL_SCHEME_REPLICATED || + pool_cfg.local_reads == POOL_LOCAL_READ_PRIMARY); if (pg.scheme == POOL_SCHEME_EC) { use_ec(pg.pg_size, pg.pg_data_size, true); diff --git a/src/test/test_heap.cpp b/src/test/test_heap.cpp index 8d5ac22a..e902cd64 100644 --- a/src/test/test_heap.cpp +++ b/src/test/test_heap.cpp @@ -899,7 +899,8 @@ void test_reshard_list() free(listing); listing = NULL; - heap.reshard(1, 2, 0x20000); + void *st = heap.reshard_start(1, 2, 0x20000, 0); + assert(st == NULL); assert(heap.read_entry((object_id){ .inode = INODE_WITH_POOL(1, 1), .stripe = 0 })); assert(heap.read_entry((object_id){ .inode = INODE_WITH_POOL(1, 1), .stripe = 0x20000 })); diff --git a/tests/test_change_pg_count.sh b/tests/test_change_pg_count.sh index 247971f5..08669f53 100755 --- a/tests/test_change_pg_count.sh +++ b/tests/test_change_pg_count.sh @@ -1,5 +1,6 @@ #!/bin/bash -ex +GLOBAL_CONFIG=',"pg_reshard_chunk_size":100' OSD_COUNT=${OSD_COUNT:-6} PG_COUNT=16 diff --git a/tests/test_write_no_same.sh b/tests/test_write_no_same.sh index 22c55b7e..2a7c9142 100755 --- a/tests/test_write_no_same.sh +++ b/tests/test_write_no_same.sh @@ -2,7 +2,7 @@ # Test the `no_same_sector_overwrites` mode OSD_ARGS="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all $OSD_ARGS" -GLOBAL_CONF='{"immediate_commit":"all"}' +GLOBAL_CONFIG='{"immediate_commit":"all"}' . `dirname $0`/run_3osds.sh