Initialize blockstore after loading pool configuration to pre-shard the DB correctly on start

src/osd/osd.cpp

@@ -31,25 +31,6 @@ osd_t::osd_t(const json11::Json & config, ring_loop_t *ringloop)
     // FIXME: Use timerfd_interval based directly on io_uring
     this->tfd = epmgr->tfd;
 
-    if (!json_is_true(this->config["disable_blockstore"]))
-    {
-        auto bs_cfg = json_to_string_map(this->config);
-        this->bs = blockstore_i::create(bs_cfg, ringloop, tfd);
-        // Wait for blockstore initialisation before actually starting OSD logic
-        // to prevent peering timeouts during restart with filled databases
-        while (!bs->is_started())
-        {
-            ringloop->loop();
-            if (bs->is_started())
-                break;
-            ringloop->wait();
-        }
-        // Autosync based on the number of unstable writes to prevent stalls due to insufficient journal space
-        uint64_t max_autosync = bs->get_journal_size() / bs->get_block_size() / 2;
-        if (autosync_writes > max_autosync)
-            autosync_writes = max_autosync;
-    }
-
     if (json_is_true(this->config["osd_memlock"]))
     {
         // Lock all OSD memory if requested
@@ -117,6 +98,7 @@ osd_t::~osd_t()
         autosync_timer_id = -1;
     }
     ringloop->unregister_consumer(&consumer);
+    ringloop->unregister_consumer(&init_consumer);
     delete epmgr;
     if (bs)
         delete bs;
@@ -131,6 +113,42 @@ osd_t::~osd_t()
     free(zero_buffer);
 }
 
+void osd_t::init_blockstore(std::function<void()> on_init)
+{
+    if (!json_is_true(this->config["disable_blockstore"]))
+    {
+        auto bs_cfg = json_to_string_map(this->config);
+        this->bs = blockstore_i::create(bs_cfg, ringloop, tfd);
+        // Pre-configure pool PG shards
+        for (auto & pool_item: st_cli.pool_config)
+        {
+            bs->reshard(pool_item.first, pool_item.second.pg_count, pool_item.second.pg_stripe_size);
+        }
+        // Autosync based on the number of unstable writes to prevent stalls due to insufficient journal space
+        uint64_t max_autosync = bs->get_journal_size() / bs->get_block_size() / 2;
+        if (autosync_writes > max_autosync)
+            autosync_writes = max_autosync;
+        if (on_init)
+        {
+            init_consumer.loop = [this, on_init]()
+            {
+                // Wait for blockstore initialisation before actually starting OSD logic
+                // to prevent peering timeouts during restart with filled databases
+                if (bs->is_started())
+                {
+                    ringloop->set_immediate([this, on_init] { init_consumer.loop = NULL; on_init(); });
+                    ringloop->unregister_consumer(&init_consumer);
+                }
+            };
+            ringloop->register_consumer(&init_consumer);
+        }
+    }
+    else if (on_init)
+    {
+        on_init();
+    }
+}
+
 void osd_t::parse_config(bool init)
 {
     config = msgr.merge_configs(cli_config, file_config, etcd_global_config, etcd_osd_config);

src/osd/osd.h

@@ -158,6 +158,7 @@ class osd_t
     json11::Json self_state;
     bool loading_peer_config = false;
     std::set<pool_pg_num_t> pg_state_dirty;
+    bool etcd_global_config_loaded = false;
     bool pg_config_applied = false;
     bool etcd_reporting_pg_state = false;
     bool etcd_reporting_stats = false;
@@ -206,7 +207,7 @@ class osd_t
     void *zero_buffer = NULL;
     uint64_t zero_buffer_size = 0;
     uint32_t bs_block_size, bs_bitmap_granularity, clean_entry_bitmap_size;
-    ring_loop_t *ringloop;
+    ring_loop_t *ringloop = NULL;
     timerfd_manager_t *tfd = NULL;
     epoll_manager_t *epmgr = NULL;
 
@@ -218,6 +219,7 @@ class osd_t
     int rdmacm_port = 0;
 #endif
     ring_consumer_t consumer;
+    ring_consumer_t init_consumer;
 
     // op statistics
     osd_op_stats_t prev_stats, prev_report_stats;
@@ -241,6 +243,7 @@ class osd_t
 
     // cluster connection
     void parse_config(bool init);
+    void init_blockstore(std::function<void()> on_init);
     void init_cluster();
     void on_change_osd_state_hook(osd_num_t peer_osd);
     void on_change_backfillfull_hook(pool_id_t pool_id);

src/osd/osd_cluster.cpp

@@ -18,6 +18,7 @@ void osd_t::init_cluster()
 {
     if (!st_cli.address_count())
     {
+        init_blockstore(NULL);
         if (run_primary)
         {
             // Test version of clustering code with 1 pool, 1 PG and 2 peers
@@ -422,7 +423,10 @@ void osd_t::on_change_osd_state_hook(osd_num_t peer_osd)
 
 void osd_t::on_change_pool_config_hook()
 {
-    apply_pg_locks_localize_only();
+    if (etcd_global_config_loaded)
+    {
+        apply_pg_locks_localize_only();
+    }
 }
 
 void osd_t::apply_pg_locks_localize_only()
@@ -484,9 +488,13 @@ void osd_t::on_load_config_hook(json11::Json::object & global_config)
 {
     etcd_global_config = global_config;
     parse_config(true);
-    bind_socket();
-    acquire_lease();
     st_cli.on_load_config_hook = [this](json11::Json::object & cfg) { on_reload_config_hook(cfg); };
+    etcd_global_config_loaded = true;
+    init_blockstore([this]()
+    {
+        bind_socket();
+        acquire_lease();
+    });
 }
 
 void osd_t::on_reload_config_hook(json11::Json::object & global_config)