Support chunked resharding in OSD

docs/config/osd.en.md

@@ -67,6 +67,8 @@ with an OSD restart or, for some of them, even without restarting by updating co
 - [pg_lock_retry_interval_ms](#pg_lock_retry_interval_ms)
 - [atomic_write_size](#atomic_write_size)
 - [use_atomic_flag](#use_atomic_flag)
+- [pg_reshard_chunk_size](#pg_reshard_chunk_size)
+- [pg_reshard_chunk_pause_ms](#pg_reshard_chunk_pause_ms)
 
 ## bind_address
 
@@ -713,3 +715,21 @@ even though the NVMe specification allows them.
 For NVMe disks with `scheduler=none` writes aren't fragmented anyway so it's not a big deal.
 However, you can rebuild your kernel with [this patch](../../patches/linux-fix-atomic-write-checks.diff)
 and turn this option on. It will make your atomic writes a bit safer.
+
+## pg_reshard_chunk_size
+
+- Type: integer
+- Default: 100000
+
+Pool PG count change is a CPU-intensive operation because OSDs store the full object database
+in memory and have to move all entries between old and new PGs. Thus it's performed in chunks,
+with pauses between chunks to prevent blocking OSD's event loop and other clients' operations.
+This option sets the maximum number of object is a chunk. Moving 100k objects usually takes
+50-100ms. Chunk size equal to 0 means unlimited.
+
+## pg_reshard_chunk_pause_ms
+
+- Type: milliseconds
+- Default: 100
+
+This option sets the interval between handling two PG count change chunks.

docs/config/osd.ru.md

@@ -68,6 +68,8 @@
 - [pg_lock_retry_interval_ms](#pg_lock_retry_interval_ms)
 - [atomic_write_size](#atomic_write_size)
 - [use_atomic_flag](#use_atomic_flag)
+- [pg_reshard_chunk_size](#pg_reshard_chunk_size)
+- [pg_reshard_chunk_pause_ms](#pg_reshard_chunk_pause_ms)
 
 ## bind_address
 
@@ -751,3 +753,22 @@ pg_minsize OSD во время переключений, что может по
 Для NVMe-дисков с `scheduler=none` запросы записи и так не фрагментируются, так что это не так
 уж и важно, однако вы можете пересобрать своё ядро с [этим патчем](../../patches/linux-fix-atomic-write-checks.diff)
 и включить данную опцию. Это сделает вашу атомарную запись капельку безопаснее.
+
+## pg_reshard_chunk_size
+
+- Тип: целое число
+- Значение по умолчанию: 100000
+
+Изменение числа PG в пуле заметно загружает процессор, так как OSD хранят полную базу данных
+объектов в памяти и им приходится перемещать все записи объектов между старыми и новыми PG.
+Поэтому изменение применяется порциями, с паузами между порциями, чтобы не блокировать обработку
+событий OSD и операции остальных клиентов. Данная опция задаёт максимальное число объектов
+в порции. Перемещение 100 тысяч объектов (значение по умолчанию) обычно занимает порядка
+50-100 миллисекунд. Значение опции 0 отключает лимит размера порции.
+
+## pg_reshard_chunk_pause_ms
+
+- Тип: миллисекунды
+- Значение по умолчанию: 100
+
+Данная опция задаёт интервал между обработкой двух порций изменения числа PG пулов.

docs/config/src/osd.yml

@@ -883,3 +883,26 @@
     Для NVMe-дисков с `scheduler=none` запросы записи и так не фрагментируются, так что это не так
     уж и важно, однако вы можете пересобрать своё ядро с [этим патчем](../../patches/linux-fix-atomic-write-checks.diff)
     и включить данную опцию. Это сделает вашу атомарную запись капельку безопаснее.
+- name: pg_reshard_chunk_size
+  type: int
+  default: 100000
+  info: |
+    Pool PG count change is a CPU-intensive operation because OSDs store the full object database
+    in memory and have to move all entries between old and new PGs. Thus it's performed in chunks,
+    with pauses between chunks to prevent blocking OSD's event loop and other clients' operations.
+    This option sets the maximum number of object is a chunk. Moving 100k objects usually takes
+    50-100ms. Chunk size equal to 0 means unlimited.
+  info_ru: |
+    Изменение числа PG в пуле заметно загружает процессор, так как OSD хранят полную базу данных
+    объектов в памяти и им приходится перемещать все записи объектов между старыми и новыми PG.
+    Поэтому изменение применяется порциями, с паузами между порциями, чтобы не блокировать обработку
+    событий OSD и операции остальных клиентов. Данная опция задаёт максимальное число объектов
+    в порции. Перемещение 100 тысяч объектов (значение по умолчанию) обычно занимает порядка
+    50-100 миллисекунд. Значение опции 0 отключает лимит размера порции.
+- name: pg_reshard_chunk_pause_ms
+  type: ms
+  default: 100
+  info: |
+    This option sets the interval between handling two PG count change chunks.
+  info_ru: |
+    Данная опция задаёт интервал между обработкой двух порций изменения числа PG пулов.

src/client/etcd_state_client.h

@@ -68,6 +68,11 @@ struct pool_config_t
     std::string used_for_app;
     int backfillfull = 0;
     int local_reads = 0;
+
+    // runtime data, used only by OSD:
+    uint64_t applied_pg_count = 0;
+    uint64_t applied_pg_stripe_size = 0;
+    void *reshard_state = NULL;
 };
 
 struct inode_config_t

src/osd/osd.cpp

@@ -311,6 +311,12 @@ void osd_t::parse_config(bool init)
     scrub_list_limit = config["scrub_list_limit"].uint64_value();
     if (!scrub_list_limit)
         scrub_list_limit = 262144;
+    pg_reshard_chunk_size = config["pg_reshard_chunk_size"].uint64_value();
+    if (!pg_reshard_chunk_size)
+        pg_reshard_chunk_size = 100000;
+    pg_reshard_chunk_pause_ms = config["pg_reshard_chunk_pause_ms"].uint64_value();
+    if (!pg_reshard_chunk_pause_ms)
+        pg_reshard_chunk_pause_ms = 100;
     if (!old_auto_scrub && auto_scrub)
     {
         // Schedule scrubbing

src/osd/osd.h

@@ -148,6 +148,8 @@ class osd_t
     bool enable_pg_locks = false;
     bool pg_locks_localize_only = false;
     uint64_t pg_lock_retry_interval_ms = 100;
+    uint64_t pg_reshard_chunk_size = 100000;
+    uint64_t pg_reshard_chunk_pause_ms = 100;
 
     // cluster state
 
@@ -173,6 +175,8 @@ class osd_t
     std::map<pool_pg_num_t, pg_t> pgs;
     std::set<pool_pg_num_t> dirty_pgs;
     std::set<osd_num_t> dirty_osds;
+    std::vector<pool_id_t> reshard_pools;
+    int reshard_timer_id = -1;
     int copies_to_delete_after_sync_count = 0;
     uint64_t misplaced_objects = 0, degraded_objects = 0, incomplete_objects = 0, inconsistent_objects = 0, corrupted_objects = 0;
     int peering_state = 0;
@@ -271,6 +275,7 @@ class osd_t
     void apply_no_inode_stats();
     void apply_pg_count();
     void apply_pg_config();
+    void reshard_continue();
 
     // event loop, socket read/write
     void loop();

src/osd/osd_cluster.cpp

@@ -727,8 +727,9 @@ void osd_t::apply_pg_count()
 {
     for (auto & pool_item: st_cli.pool_config)
     {
-        if (pool_item.second.real_pg_count != 0 &&
+        auto & pool_cfg = pool_item.second;
-            pool_item.second.real_pg_count != pg_counts[pool_item.first])
+        if (pool_cfg.real_pg_count != 0 &&
+            pool_cfg.real_pg_count != pg_counts[pool_item.first])
         {
             // Check that all pool PGs are offline. It is not allowed to change PG count when any PGs are online
             // The external tool must wait for all PGs to come down before changing PG count
@@ -753,14 +754,79 @@ void osd_t::apply_pg_count()
                 printf(
                     "[OSD %ju] PG count change detected for pool %u (new is %ju, old is %u),"
                     " but %u PG(s) are still active as primary and %u as secondary. This is not allowed. Exiting\n",
-                    this->osd_num, pool_item.first, pool_item.second.real_pg_count, pg_counts[pool_item.first],
+                    this->osd_num, pool_item.first, pool_cfg.real_pg_count, pg_counts[pool_item.first],
                     still_active_primary, still_active_secondary
                 );
                 force_stop(1);
                 return;
             }
         }
-        this->pg_counts[pool_item.first] = pool_item.second.real_pg_count;
+        if (bs && (pool_cfg.real_pg_count != pool_cfg.applied_pg_count ||
+            pool_cfg.pg_stripe_size != pool_cfg.applied_pg_stripe_size) &&
+            !pool_cfg.reshard_state)
+        {
+            pool_cfg.applied_pg_count = pool_cfg.real_pg_count;
+            pool_cfg.applied_pg_stripe_size = pool_cfg.pg_stripe_size;
+            pool_cfg.reshard_state = bs->reshard_start(pool_item.first, pool_cfg.real_pg_count, pool_cfg.pg_stripe_size, pg_reshard_chunk_size);
+            if (pool_cfg.reshard_state)
+            {
+                reshard_pools.push_back(pool_item.first);
+            }
+        }
+        this->pg_counts[pool_item.first] = pool_cfg.real_pg_count;
+    }
+    if (reshard_pools.size() && reshard_timer_id < 0)
+    {
+        reshard_timer_id = tfd->set_timer(pg_reshard_chunk_pause_ms, false, [this](int)
+        {
+            reshard_continue();
+        });
+    }
+}
+
+void osd_t::reshard_continue()
+{
+again:
+    auto pool_id = reshard_pools[0];
+    auto pool_it = st_cli.pool_config.find(pool_id);
+    if (pool_it == st_cli.pool_config.end() || !pool_it->second.reshard_state)
+    {
+        reshard_pools.erase(reshard_pools.begin());
+        goto again;
+    }
+    auto & pool_cfg = pool_it->second;
+    bool done = false;
+    if (pool_cfg.real_pg_count != pool_cfg.applied_pg_count ||
+        pool_cfg.pg_stripe_size != pool_cfg.applied_pg_stripe_size)
+    {
+        // PG count changed again, reshard again
+        bs->reshard_abort(pool_cfg.reshard_state);
+        pool_cfg.applied_pg_count = pool_cfg.real_pg_count;
+        pool_cfg.applied_pg_stripe_size = pool_cfg.pg_stripe_size;
+        pool_cfg.reshard_state = bs->reshard_start(pool_id, pool_cfg.real_pg_count, pool_cfg.pg_stripe_size, pg_reshard_chunk_size);
+        done = !pool_cfg.reshard_state;
+    }
+    else
+    {
+        done = bs->reshard_continue(pool_cfg.reshard_state, pg_reshard_chunk_size);
+    }
+    if (done)
+    {
+        // Pool is resharded
+        pool_cfg.reshard_state = NULL;
+        reshard_pools.erase(reshard_pools.begin());
+        apply_pg_config();
+    }
+    if (reshard_pools.size())
+    {
+        reshard_timer_id = tfd->set_timer(pg_reshard_chunk_pause_ms, false, [this](int)
+        {
+            reshard_continue();
+        });
+    }
+    else
+    {
+        reshard_timer_id = -1;
     }
 }
 
@@ -769,9 +835,15 @@ void osd_t::apply_pg_config()
     bool all_applied = true;
     for (auto & pool_item: st_cli.pool_config)
     {
+        auto & pool_cfg = pool_item.second;
+        if (pool_cfg.reshard_state)
+        {
+            // Can't apply anything for pools being resharded
+            continue;
+        }
         bool warned_block_size = false;
         auto pool_id = pool_item.first;
-        for (auto & kv: pool_item.second.pg_config)
+        for (auto & kv: pool_cfg.pg_config)
         {
             pg_num_t pg_num = kv.first;
             auto & pg_cfg = kv.second;
@@ -780,8 +852,8 @@ void osd_t::apply_pg_config()
             auto pg_it = this->pgs.find({ .pool_id = pool_id, .pg_num = pg_num });
             bool currently_taken = pg_it != this->pgs.end() && pg_it->second.state != PG_OFFLINE;
             // Check pool block size and bitmap granularity
-            if (take && this->bs_block_size != pool_item.second.data_block_size ||
+            if (take && this->bs_block_size != pool_cfg.data_block_size ||
-                this->bs_bitmap_granularity != pool_item.second.bitmap_granularity)
+                this->bs_bitmap_granularity != pool_cfg.bitmap_granularity)
             {
                 if (!warned_block_size)
                 {
@@ -789,7 +861,7 @@ void osd_t::apply_pg_config()
                         "[OSD %ju] My block_size and bitmap_granularity are %u/%u"
                         ", but pool %u has %u/%u. Refusing to start PGs of this pool\n",
                         this->osd_num, bs_block_size, bs_bitmap_granularity,
-                        pool_id, pool_item.second.data_block_size, pool_item.second.bitmap_granularity
+                        pool_id, pool_cfg.data_block_size, pool_cfg.bitmap_granularity
                     );
                 }
                 warned_block_size = true;
@@ -882,12 +954,12 @@ void osd_t::apply_pg_config()
                 }
                 auto & pg = this->pgs[{ .pool_id = pool_id, .pg_num = pg_num }];
                 pg.state = pg_cfg.cur_primary == this->osd_num ? PG_PEERING : PG_STARTING;
-                pg.scheme = pool_item.second.scheme;
+                pg.scheme = pool_cfg.scheme;
                 pg.pg_cursize = 0;
-                pg.pg_size = pool_item.second.pg_size;
+                pg.pg_size = pool_cfg.pg_size;
-                pg.pg_minsize = pool_item.second.pg_minsize;
+                pg.pg_minsize = pool_cfg.pg_minsize;
-                pg.pg_data_size = pool_item.second.scheme == POOL_SCHEME_REPLICATED
+                pg.pg_data_size = pool_cfg.scheme == POOL_SCHEME_REPLICATED
-                     ? 1 : pool_item.second.pg_size - pool_item.second.parity_chunks;
+                     ? 1 : pool_cfg.pg_size - pool_cfg.parity_chunks;
                 pg.pool_id = pool_id;
                 pg.pg_num = pg_num;
                 pg.reported_epoch = pg_cfg.epoch;
@@ -896,8 +968,8 @@ void osd_t::apply_pg_config()
                 pg.next_scrub = pg_cfg.next_scrub;
                 pg.target_set = pg_cfg.target_set;
                 pg.disable_pg_locks = pg_locks_localize_only &&
-                    (pool_item.second.scheme != POOL_SCHEME_REPLICATED ||
+                    (pool_cfg.scheme != POOL_SCHEME_REPLICATED ||
-                    pool_item.second.local_reads == POOL_LOCAL_READ_PRIMARY);
+                    pool_cfg.local_reads == POOL_LOCAL_READ_PRIMARY);
                 if (pg.scheme == POOL_SCHEME_EC)
                 {
                     use_ec(pg.pg_size, pg.pg_data_size, true);

src/test/test_heap.cpp

@@ -899,7 +899,8 @@ void test_reshard_list()
         free(listing);
         listing = NULL;
 
-        heap.reshard(1, 2, 0x20000);
+        void *st = heap.reshard_start(1, 2, 0x20000, 0);
+        assert(st == NULL);
 
         assert(heap.read_entry((object_id){ .inode = INODE_WITH_POOL(1, 1), .stripe = 0 }));
         assert(heap.read_entry((object_id){ .inode = INODE_WITH_POOL(1, 1), .stripe = 0x20000 }));

tests/test_change_pg_count.sh

@@ -1,5 +1,6 @@
 #!/bin/bash -ex
 
+GLOBAL_CONFIG=',"pg_reshard_chunk_size":100'
 OSD_COUNT=${OSD_COUNT:-6}
 PG_COUNT=16
 

tests/test_write_no_same.sh

@@ -2,7 +2,7 @@
 # Test the `no_same_sector_overwrites` mode
 
 OSD_ARGS="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all $OSD_ARGS"
-GLOBAL_CONF='{"immediate_commit":"all"}'
+GLOBAL_CONFIG='{"immediate_commit":"all"}'
 
 . `dirname $0`/run_3osds.sh