Rename /config/pgs to /pg/config and /pg/stats/* to /pgstats/*

docs/usage/admin.en.md

@@ -42,7 +42,7 @@ PG state always includes exactly 1 of the following base states:
 - **offline** — PG isn't activated by any OSD at all. Either primary OSD isn't set for
   this PG at all (if the pool is just created), or an unavailable OSD is set as primary,
   or the primary OSD refuses to start this PG (for example, because of wrong block_size),
-  or the PG is stopped by the monitor using `pause: true` flag in `/vitastor/config/pgs` in etcd.
+  or the PG is stopped by the monitor using `pause: true` flag in `/vitastor/pg/config` in etcd.
 - **starting** — primary OSD has acquired PG lock in etcd, PG is starting.
 - **peering** — primary OSD requests PG object listings from secondary OSDs and calculates
   the PG state.

docs/usage/admin.ru.md

@@ -42,7 +42,7 @@
 - **offline** — PG вообще не активирована ни одним OSD. Либо первичный OSD не назначен вообще
   (если пул только создан), либо в качестве первичного назначен недоступный OSD, либо
   назначенный OSD отказывается запускать эту PG (например, из-за несовпадения block_size),
-  либо PG остановлена монитором через флаг `pause: true` в `/vitastor/config/pgs` в etcd.
+  либо PG остановлена монитором через флаг `pause: true` в `/vitastor/pg/config` в etcd.
 - **starting** — первичный OSD захватил блокировку PG в etcd, PG запускается.
 - **peering** — первичный OSD опрашивает вторичные OSD на предмет списков объектов данной PG и рассчитывает её состояние.
 - **repeering** — PG ожидает завершения текущих операций ввода-вывода, после чего перейдёт в состояние **peering**.

mon/etcd_schema.js

@@ -6,7 +6,7 @@ const etcd_nonempty_keys = {
     'config/global': 1,
     'config/node_placement': 1,
     'config/pools': 1,
-    'config/pgs': 1,
+    'pg/config': 1,
     'history/last_clean_pgs': 1,
     'stats': 1,
 };
@@ -15,7 +15,8 @@ const etcd_allow = new RegExp('^'+[
     'config/node_placement',
     'config/pools',
     'config/osd/[1-9]\\d*',
-    'config/pgs',
+    'config/pgs', // old name
+    'pg/config',
     'config/inode/[1-9]\\d*/[1-9]\\d*',
     'osd/state/[1-9]\\d*',
     'osd/stats/[1-9]\\d*',
@@ -24,7 +25,8 @@ const etcd_allow = new RegExp('^'+[
     'mon/master',
     'mon/member/[a-f0-9]+',
     'pg/state/[1-9]\\d*/[1-9]\\d*',
-    'pg/stats/[1-9]\\d*/[1-9]\\d*',
+    'pg/stats/[1-9]\\d*/[1-9]\\d*', // old name
+    'pgstats/[1-9]\\d*/[1-9]\\d*',
     'pg/history/[1-9]\\d*/[1-9]\\d*',
     'history/last_clean_pgs',
     'inode/stats/[1-9]\\d*/\\d+',
@@ -205,19 +207,6 @@ const etcd_tree = {
         osd: {
             /* <id>: { reweight?: 1, tags?: [ 'nvme', ... ], noout?: true }, ... */
         },
-        /* pgs: {
-            hash: string,
-            items: {
-                <pool_id>: {
-                    <pg_id>: {
-                        osd_set: [ 1, 2, 3 ],
-                        primary: 1,
-                        pause: false,
-                    }
-                }
-            }
-        }, */
-        pgs: {},
         /* inode: {
             <pool_id>: {
                 <inode_t>: {
@@ -290,6 +279,19 @@ const etcd_tree = {
         },
     },
     pg: {
+        /* config: {
+            hash: string,
+            items: {
+                <pool_id>: {
+                    <pg_id>: {
+                        osd_set: [ 1, 2, 3 ],
+                        primary: 1,
+                        pause: false,
+                    }
+                }
+            }
+        }, */
+        config: {},
         state: {
             /* <pool_id>: {
                 <pg_id>: {
@@ -300,18 +302,6 @@ const etcd_tree = {
                 }
             }, */
         },
-        stats: {
-            /* <pool_id>: {
-                <pg_id>: {
-                    object_count: uint64_t,
-                    clean_count: uint64_t,
-                    misplaced_count: uint64_t,
-                    degraded_count: uint64_t,
-                    incomplete_count: uint64_t,
-                    write_osd_set: osd_num_t[],
-                },
-            }, */
-        },
         history: {
             /* <pool_id>: {
                 <pg_id>: {
@@ -323,6 +313,18 @@ const etcd_tree = {
             }, */
         },
     },
+    pgstats: {
+        /* <pool_id>: {
+            <pg_id>: {
+                object_count: uint64_t,
+                clean_count: uint64_t,
+                misplaced_count: uint64_t,
+                degraded_count: uint64_t,
+                incomplete_count: uint64_t,
+                write_osd_set: osd_num_t[],
+            },
+        }, */
+    },
     inode: {
         stats: {
             /* <pool_id>: {

mon/mon.js

@@ -122,7 +122,7 @@ class Mon
                 !Number(this.state.pool.stats[pool_id].pg_real_size))
             {
                 // Generate missing data in etcd
-                this.state.config.pgs.hash = null;
+                this.state.pg.config.hash = null;
                 break;
             }
         }
@@ -201,7 +201,7 @@ class Mon
                 stats_changed = true;
                 changed = true;
             }
-            else if (key.substr(0, 11) == '/osd/stats/' || key.substr(0, 10) == '/pg/stats/' || key.substr(0, 16) == '/osd/inodestats/')
+            else if (key.substr(0, 11) == '/osd/stats/' || key.substr(0, 9) == '/pgstats/' || key.substr(0, 16) == '/osd/inodestats/')
             {
                 stats_changed = true;
             }
@@ -285,7 +285,7 @@ class Mon
                     continue next_pool;
                 }
             }
-            new_clean_pgs.items[pool_id] = this.state.config.pgs.items[pool_id];
+            new_clean_pgs.items[pool_id] = this.state.pg.config.items[pool_id];
         }
         this.state.history.last_clean_pgs = new_clean_pgs;
         await this.etcd.etcd_call('/kv/txn', {
@@ -406,7 +406,7 @@ class Mon
     async stop_all_pgs(pool_id)
     {
         let has_online = false, paused = true;
-        for (const pg in this.state.config.pgs.items[pool_id]||{})
+        for (const pg in this.state.pg.config.items[pool_id]||{})
         {
             // FIXME: Change all (||{}) to ?. (optional chaining) at some point
             const cur_state = (((this.state.pg.state[pool_id]||{})[pg]||{}).state||[]).join(',');
@@ -414,7 +414,7 @@ class Mon
             {
                 has_online = true;
             }
-            if (!this.state.config.pgs.items[pool_id][pg].pause)
+            if (!this.state.pg.config.items[pool_id][pg].pause)
             {
                 paused = false;
             }
@@ -422,7 +422,7 @@ class Mon
         if (!paused)
         {
             console.log('Stopping all PGs for pool '+pool_id+' before changing PG count');
-            const new_cfg = JSON.parse(JSON.stringify(this.state.config.pgs));
+            const new_cfg = JSON.parse(JSON.stringify(this.state.pg.config));
             for (const pg in new_cfg.items[pool_id])
             {
                 new_cfg.items[pool_id][pg].pause = true;
@@ -439,11 +439,11 @@ class Mon
             await this.etcd.etcd_call('/kv/txn', {
                 compare: [
                     { key: b64(this.config.etcd_prefix+'/mon/master'), target: 'LEASE', lease: ''+this.etcd_lease_id },
-                    { key: b64(this.config.etcd_prefix+'/config/pgs'), target: 'MOD', mod_revision: ''+this.etcd_watch_revision, result: 'LESS' },
+                    { key: b64(this.config.etcd_prefix+'/pg/config'), target: 'MOD', mod_revision: ''+this.etcd_watch_revision, result: 'LESS' },
                     ...checks,
                 ],
                 success: [
-                    { requestPut: { key: b64(this.config.etcd_prefix+'/config/pgs'), value: b64(JSON.stringify(new_cfg)) } },
+                    { requestPut: { key: b64(this.config.etcd_prefix+'/pg/config'), value: b64(JSON.stringify(new_cfg)) } },
                 ],
             }, this.config.etcd_mon_timeout, 0);
             return false;
@@ -473,7 +473,7 @@ class Mon
             pools: this.state.config.pools,
         };
         const tree_hash = sha1hex(stableStringify(tree_cfg));
-        if (this.state.config.pgs.hash != tree_hash)
+        if (this.state.pg.config.hash != tree_hash)
         {
             // Something has changed
             console.log('Pool configuration or OSD tree changed, re-optimizing');
@@ -514,10 +514,10 @@ class Mon
         else
         {
             // Nothing changed, but we still want to recheck the distribution of primaries
-            let new_config_pgs = recheck_primary(this.state, this.config, up_osds, osd_tree);
+            let new_pg_config = recheck_primary(this.state, this.config, up_osds, osd_tree);
-            if (new_config_pgs)
+            if (new_pg_config)
             {
-                const ok = await this.save_pg_config(new_config_pgs);
+                const ok = await this.save_pg_config(new_pg_config);
                 if (ok)
                     console.log('PG configuration successfully changed');
                 else
@@ -532,12 +532,12 @@ class Mon
 
     async apply_pool_pgs(results, up_osds, osd_tree, tree_hash)
     {
-        for (const pool_id in (this.state.config.pgs||{}).items||{})
+        for (const pool_id in (this.state.pg.config||{}).items||{})
         {
             // We should stop all PGs when deleting a pool or changing its PG count
             if (!this.state.config.pools[pool_id] ||
-                this.state.config.pgs.items[pool_id] && this.state.config.pools[pool_id].pg_count !=
+                this.state.pg.config.items[pool_id] && this.state.config.pools[pool_id].pg_count !=
-                Object.keys(this.state.config.pgs.items[pool_id]).reduce((a, c) => (a < (0|c) ? (0|c) : a), 0))
+                Object.keys(this.state.pg.config.items[pool_id]).reduce((a, c) => (a < (0|c) ? (0|c) : a), 0))
             {
                 if (!await this.stop_all_pgs(pool_id))
                 {
@@ -545,22 +545,22 @@ class Mon
                 }
             }
         }
-        const new_config_pgs = JSON.parse(JSON.stringify(this.state.config.pgs));
+        const new_pg_config = JSON.parse(JSON.stringify(this.state.pg.config));
         const etcd_request = { compare: [], success: [] };
-        for (const pool_id in (new_config_pgs||{}).items||{})
+        for (const pool_id in (new_pg_config||{}).items||{})
         {
             if (!this.state.config.pools[pool_id])
             {
                 const prev_pgs = [];
-                for (const pg in new_config_pgs.items[pool_id]||{})
+                for (const pg in new_pg_config.items[pool_id]||{})
                 {
-                    prev_pgs[pg-1] = new_config_pgs.items[pool_id][pg].osd_set;
+                    prev_pgs[pg-1] = new_pg_config.items[pool_id][pg].osd_set;
                 }
                 // Also delete pool statistics
                 etcd_request.success.push({ requestDeleteRange: {
                     key: b64(this.config.etcd_prefix+'/pool/stats/'+pool_id),
                 } });
-                save_new_pgs_txn(new_config_pgs, etcd_request, this.state, this.config.etcd_prefix,
+                save_new_pgs_txn(new_pg_config, etcd_request, this.state, this.config.etcd_prefix,
                     this.etcd_watch_revision, pool_id, up_osds, osd_tree, prev_pgs, [], []);
             }
         }
@@ -569,7 +569,7 @@ class Mon
             const pool_id = pool_res.pool_id;
             const pool_cfg = this.state.config.pools[pool_id];
             let pg_history = [];
-            for (const pg in ((this.state.config.pgs.items||{})[pool_id]||{}))
+            for (const pg in ((this.state.pg.config.items||{})[pool_id]||{}))
             {
                 if (this.state.pg.history[pool_id] &&
                     this.state.pg.history[pool_id][pg])
@@ -578,9 +578,9 @@ class Mon
                 }
             }
             const real_prev_pgs = [];
-            for (const pg in ((this.state.config.pgs.items||{})[pool_id]||{}))
+            for (const pg in ((this.state.pg.config.items||{})[pool_id]||{}))
             {
-                real_prev_pgs[pg-1] = [ ...this.state.config.pgs.items[pool_id][pg].osd_set ];
+                real_prev_pgs[pg-1] = [ ...this.state.pg.config.items[pool_id][pg].osd_set ];
             }
             if (real_prev_pgs.length > 0 && real_prev_pgs.length != pool_res.pgs.length)
             {
@@ -591,8 +591,8 @@ class Mon
                 pg_history = scale_pg_history(pg_history, real_prev_pgs, pool_res.pgs);
                 // Drop stats
                 etcd_request.success.push({ requestDeleteRange: {
-                    key: b64(this.config.etcd_prefix+'/pg/stats/'+pool_id+'/'),
+                    key: b64(this.config.etcd_prefix+'/pgstats/'+pool_id+'/'),
-                    range_end: b64(this.config.etcd_prefix+'/pg/stats/'+pool_id+'0'),
+                    range_end: b64(this.config.etcd_prefix+'/pgstats/'+pool_id+'0'),
                 } });
             }
             const stats = {
@@ -603,21 +603,21 @@ class Mon
                 key: b64(this.config.etcd_prefix+'/pool/stats/'+pool_id),
                 value: b64(JSON.stringify(stats)),
             } });
-            save_new_pgs_txn(new_config_pgs, etcd_request, this.state, this.config.etcd_prefix,
+            save_new_pgs_txn(new_pg_config, etcd_request, this.state, this.config.etcd_prefix,
                 this.etcd_watch_revision, pool_id, up_osds, osd_tree, real_prev_pgs, pool_res.pgs, pg_history);
         }
-        new_config_pgs.hash = tree_hash;
+        new_pg_config.hash = tree_hash;
-        return await this.save_pg_config(new_config_pgs, etcd_request);
+        return await this.save_pg_config(new_pg_config, etcd_request);
     }
 
-    async save_pg_config(new_config_pgs, etcd_request = { compare: [], success: [] })
+    async save_pg_config(new_pg_config, etcd_request = { compare: [], success: [] })
     {
         etcd_request.compare.push(
             { key: b64(this.config.etcd_prefix+'/mon/master'), target: 'LEASE', lease: ''+this.etcd_lease_id },
-            { key: b64(this.config.etcd_prefix+'/config/pgs'), target: 'MOD', mod_revision: ''+this.etcd_watch_revision, result: 'LESS' },
+            { key: b64(this.config.etcd_prefix+'/pg/config'), target: 'MOD', mod_revision: ''+this.etcd_watch_revision, result: 'LESS' },
         );
         etcd_request.success.push(
-            { requestPut: { key: b64(this.config.etcd_prefix+'/config/pgs'), value: b64(JSON.stringify(new_config_pgs)) } },
+            { requestPut: { key: b64(this.config.etcd_prefix+'/pg/config'), value: b64(JSON.stringify(new_pg_config)) } },
         );
         const txn_res = await this.etcd.etcd_call('/kv/txn', etcd_request, this.config.etcd_mon_timeout, 0);
         return txn_res.succeeded;

mon/pg_gen.js

@@ -57,7 +57,7 @@ function pick_primary(pool_config, osd_set, up_osds, aff_osds)
 
 function recheck_primary(state, global_config, up_osds, osd_tree)
 {
-    let new_config_pgs;
+    let new_pg_config;
     for (const pool_id in state.config.pools)
     {
         const pool_cfg = state.config.pools[pool_id];
@@ -69,30 +69,30 @@ function recheck_primary(state, global_config, up_osds, osd_tree)
         reset_rng();
         for (let pg_num = 1; pg_num <= pool_cfg.pg_count; pg_num++)
         {
-            if (!state.config.pgs.items[pool_id])
+            if (!state.pg.config.items[pool_id])
             {
                 continue;
             }
-            const pg_cfg = state.config.pgs.items[pool_id][pg_num];
+            const pg_cfg = state.pg.config.items[pool_id][pg_num];
             if (pg_cfg)
             {
                 const new_primary = pick_primary(state.config.pools[pool_id], pg_cfg.osd_set, up_osds, aff_osds);
                 if (pg_cfg.primary != new_primary)
                 {
-                    if (!new_config_pgs)
+                    if (!new_pg_config)
                     {
-                        new_config_pgs = JSON.parse(JSON.stringify(state.config.pgs));
+                        new_pg_config = JSON.parse(JSON.stringify(state.pg.config));
                     }
                     console.log(
                         `Moving pool ${pool_id} (${pool_cfg.name || 'unnamed'}) PG ${pg_num}`+
                         ` primary OSD from ${pg_cfg.primary} to ${new_primary}`
                     );
-                    new_config_pgs.items[pool_id][pg_num].primary = new_primary;
+                    new_pg_config.items[pool_id][pg_num].primary = new_primary;
                 }
             }
         }
     }
-    return new_config_pgs;
+    return new_pg_config;
 }
 
 function save_new_pgs_txn(save_to, request, state, etcd_prefix, etcd_watch_revision, pool_id, up_osds, osd_tree, prev_pgs, new_pgs, pg_history)
@@ -185,10 +185,10 @@ async function generate_pool_pgs(state, global_config, pool_id, osd_tree, levels
     }
     if (!prev_pgs.length)
     {
-        // Fall back to config/pgs if it's empty
+        // Fall back to pg/config if it's empty
-        for (const pg in ((state.config.pgs.items||{})[pool_id]||{}))
+        for (const pg in ((state.pg.config.items||{})[pool_id]||{}))
         {
-            prev_pgs[pg-1] = [ ...state.config.pgs.items[pool_id][pg].osd_set ];
+            prev_pgs[pg-1] = [ ...state.pg.config.items[pool_id][pg].osd_set ];
         }
     }
     const old_pg_count = prev_pgs.length;
@@ -205,8 +205,8 @@ async function generate_pool_pgs(state, global_config, pool_id, osd_tree, levels
         ordered: pool_cfg.scheme != 'replicated',
     };
     let optimize_result;
-    // Re-shuffle PGs if config/pgs.hash is empty
+    // Re-shuffle PGs if pg/config.hash is empty
-    if (old_pg_count > 0 && state.config.pgs.hash)
+    if (old_pg_count > 0 && state.pg.config.hash)
     {
         if (prev_pgs.length != pool_cfg.pg_count)
         {

mon/prometheus.js

@@ -166,7 +166,7 @@ function export_prometheus_metrics(st)
         res += `vitastor_pool_used_raw_tb{${pool_label}} ${pool_stat.used_raw_tb||0}\n`;
 
         // PG states and pool up/down status
-        const real_pg_count = (Object.keys(((st.config.pgs||{}).items||{})[pool_id]||{}).length) || (0|pool_cfg.pg_count);
+        const real_pg_count = (Object.keys(((st.pg.config||{}).items||{})[pool_id]||{}).length) || (0|pool_cfg.pg_count);
         const per_state = {
             active: 0,
             starting: 0,

mon/stats.js

@@ -100,10 +100,10 @@ function sum_object_counts(state, global_config)
 {
     const object_counts = { object: 0n, clean: 0n, misplaced: 0n, degraded: 0n, incomplete: 0n };
     const object_bytes = { object: 0n, clean: 0n, misplaced: 0n, degraded: 0n, incomplete: 0n };
-    for (const pool_id in state.pg.stats)
+    for (const pool_id in state.pgstats)
     {
         let object_size = 0;
-        for (const osd_num of state.pg.stats[pool_id].write_osd_set||[])
+        for (const osd_num of state.pgstats[pool_id].write_osd_set||[])
         {
             if (osd_num && state.osd.stats[osd_num] && state.osd.stats[osd_num].block_size)
             {
@@ -121,9 +121,9 @@ function sum_object_counts(state, global_config)
             object_size *= ((pool_cfg.pg_size||0) - (pool_cfg.parity_chunks||0));
         }
         object_size = BigInt(object_size);
-        for (const pg_num in state.pg.stats[pool_id])
+        for (const pg_num in state.pgstats[pool_id])
         {
-            const st = state.pg.stats[pool_id][pg_num];
+            const st = state.pgstats[pool_id][pg_num];
             if (st)
             {
                 for (const k in object_counts)

mon/vitastor_persist_filter.js

@@ -35,7 +35,8 @@ function vitastor_persist_filter(cfg)
         }
         else if (key.substr(0, prefix.length+'/osd/'.length) == prefix+'/osd/' ||
             key.substr(0, prefix.length+'/inode/stats/'.length) == prefix+'/inode/stats/' ||
-            key.substr(0, prefix.length+'/pg/stats/'.length) == prefix+'/pg/stats/' ||
+            key.substr(0, prefix.length+'/pg/stats/'.length) == prefix+'/pg/stats/' || // old name
+            key.substr(0, prefix.length+'/pgstats/'.length) == prefix+'/pgstats/' ||
             key.substr(0, prefix.length+'/pool/stats/'.length) == prefix+'/pool/stats/' ||
             key == prefix+'/stats')
         {

src/client/etcd_state_client.cpp

@@ -353,7 +353,6 @@ void etcd_state_client_t::start_etcd_watcher()
                     uint64_t watch_id = data["result"]["watch_id"].uint64_value();
                     if (watch_id == ETCD_CONFIG_WATCH_ID ||
                         watch_id == ETCD_PG_STATE_WATCH_ID ||
-                        watch_id == ETCD_PG_HISTORY_WATCH_ID ||
                         watch_id == ETCD_OSD_STATE_WATCH_ID)
                         etcd_watches_initialised++;
                     if (etcd_watches_initialised == ETCD_TOTAL_WATCHES && this->log_level > 0)
@@ -395,8 +394,8 @@ void etcd_state_client_t::start_etcd_watcher()
                 }
                 if (etcd_watches_initialised == ETCD_TOTAL_WATCHES && !data["result"]["header"]["revision"].is_null())
                 {
-                    // Protect against a revision beign split into multiple messages and some
+                    // Protect against a revision being split into multiple messages and some
-                    // of them being lost. Even though I'm not sure if etcd actually splits them
+                    // of them being lost.
                     // Also sometimes etcd sends something without a header, like:
                     // {"error": {"grpc_code": 14, "http_code": 503, "http_status": "Service Unavailable", "message": "error reading from server: EOF"}}
                     etcd_watch_revision = data["result"]["header"]["revision"].uint64_value();
@@ -473,22 +472,14 @@ void etcd_state_client_t::start_etcd_watcher()
     }).dump());
     http_post_message(etcd_watch_ws, WS_TEXT, json11::Json(json11::Json::object {
         { "create_request", json11::Json::object {
-            { "key", base64_encode(etcd_prefix+"/pg/state/") },
+            { "key", base64_encode(etcd_prefix+"/pg/") },
-            { "range_end", base64_encode(etcd_prefix+"/pg/state0") },
+            { "range_end", base64_encode(etcd_prefix+"/pg0") },
             { "start_revision", etcd_watch_revision },
             { "watch_id", ETCD_PG_STATE_WATCH_ID },
             { "progress_notify", true },
         } }
     }).dump());
-    http_post_message(etcd_watch_ws, WS_TEXT, json11::Json(json11::Json::object {
+    // FIXME: Do not watch /pg/history/ at all in client code (not in OSD)
-        { "create_request", json11::Json::object {
-            { "key", base64_encode(etcd_prefix+"/pg/history/") },
-            { "range_end", base64_encode(etcd_prefix+"/pg/history0") },
-            { "start_revision", etcd_watch_revision },
-            { "watch_id", ETCD_PG_HISTORY_WATCH_ID },
-            { "progress_notify", true },
-        } }
-    }).dump());
     if (on_start_watcher_hook)
     {
         on_start_watcher_hook(etcd_watch_ws);
@@ -591,6 +582,11 @@ void etcd_state_client_t::load_pgs()
                 { "key", base64_encode(etcd_prefix+"/config/pgs") },
             } }
         },
+        json11::Json::object {
+            { "request_range", json11::Json::object {
+                { "key", base64_encode(etcd_prefix+"/pg/config") },
+            } }
+        },
         json11::Json::object {
             { "request_range", json11::Json::object {
                 { "key", base64_encode(etcd_prefix+"/config/inode/") },
@@ -895,8 +891,17 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
             on_change_pool_config_hook();
         }
     }
-    else if (key == etcd_prefix+"/config/pgs")
+    else if (key == etcd_prefix+"/pg/config" || key == etcd_prefix+"/config/pgs")
     {
+        if (key == etcd_prefix+"/pg/config")
+        {
+            new_pg_config = !value.is_null();
+        }
+        else if (new_pg_config)
+        {
+            // Ignore old key if the new one is present
+            return;
+        }
         for (auto & pool_item: this->pool_config)
         {
             for (auto & pg_item: pool_item.second.pg_config)

src/client/etcd_state_client.h

@@ -10,10 +10,9 @@
 #include "timerfd_manager.h"
 
 #define ETCD_CONFIG_WATCH_ID 1
-#define ETCD_PG_STATE_WATCH_ID 2
+#define ETCD_OSD_STATE_WATCH_ID 2
-#define ETCD_PG_HISTORY_WATCH_ID 3
+#define ETCD_PG_STATE_WATCH_ID 3
-#define ETCD_OSD_STATE_WATCH_ID 4
+#define ETCD_TOTAL_WATCHES 3
-#define ETCD_TOTAL_WATCHES 4
 
 #define DEFAULT_BLOCK_SIZE 128*1024
 #define MIN_DATA_BLOCK_SIZE 4*1024
@@ -96,6 +95,7 @@ protected:
     std::vector<std::string> addresses_to_try;
     std::vector<inode_watch_t*> watches;
     http_co_t *etcd_watch_ws = NULL, *keepalive_client = NULL;
+    bool new_pg_config = false;
     int ws_keepalive_timer = -1;
     int ws_alive = 0;
     bool rand_initialized = false;

src/cmd/cli_pg_ls.cpp

@@ -49,8 +49,8 @@ struct pg_lister_t
             { "success", json11::Json::array {
                 json11::Json::object {
                     { "request_range", json11::Json::object {
-                        { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/pg/stats"+(pool_id ? "/"+std::to_string(pool_id)+"/" : "/")) },
+                        { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/pgstats"+(pool_id ? "/"+std::to_string(pool_id)+"/" : "/")) },
-                        { "range_end", base64_encode(parent->cli->st_cli.etcd_prefix+"/pg/stats"+(pool_id ? "/"+std::to_string(pool_id)+"0" : "0")) },
+                        { "range_end", base64_encode(parent->cli->st_cli.etcd_prefix+"/pgstats"+(pool_id ? "/"+std::to_string(pool_id)+"0" : "0")) },
                     } },
                 },
             } },
@@ -65,7 +65,7 @@ resume_1:
             state = 100;
             return;
         }
-        parent->iterate_kvs_2(parent->etcd_result["responses"][0]["response_range"]["kvs"], "/pg/stats/", [&](pool_id_t pool_id, uint64_t pg_num, json11::Json value)
+        parent->iterate_kvs_2(parent->etcd_result["responses"][0]["response_range"]["kvs"], "/pgstats/", [&](pool_id_t pool_id, uint64_t pg_num, json11::Json value)
         {
             pg_stats[(pool_pg_num_t){ .pool_id = pool_id, .pg_num = (pg_num_t)pg_num }] = value;
         });

src/cmd/cli_pool_ls.cpp

@@ -214,10 +214,10 @@ resume_1:
                 json11::Json::object {
                     { "request_range", json11::Json::object {
                         { "key", base64_encode(
-                            parent->cli->st_cli.etcd_prefix+"/pg/stats/"
+                            parent->cli->st_cli.etcd_prefix+"/pgstats/"
                         ) },
                         { "range_end", base64_encode(
-                            parent->cli->st_cli.etcd_prefix+"/pg/stats0"
+                            parent->cli->st_cli.etcd_prefix+"/pgstats0"
                         ) },
                     } },
                 },
@@ -235,7 +235,7 @@ resume_1:
         }
         // Calculate recovery percent
         std::map<pool_id_t, object_counts_t> counts;
-        parent->iterate_kvs_2(parent->etcd_result["responses"][0]["response_range"]["kvs"], "/pg/stats/",
+        parent->iterate_kvs_2(parent->etcd_result["responses"][0]["response_range"]["kvs"], "/pgstats/",
             [&](pool_id_t pool_id, uint64_t pg_num, json11::Json value)
         {
             auto & cnt = counts[pool_id];

src/cmd/cli_rm_osd.cpp

@@ -176,7 +176,7 @@ struct rm_osd_t
             json11::Json::object {
                 { "request_range", json11::Json::object {
                     { "key", base64_encode(
-                        parent->cli->st_cli.etcd_prefix+"/config/pgs"
+                        parent->cli->st_cli.etcd_prefix+"/pg/config"
                     ) },
                 } },
             },
@@ -229,7 +229,7 @@ struct rm_osd_t
             }
             if (!new_pgs.is_null())
             {
-                auto pgs_key = base64_encode(parent->cli->st_cli.etcd_prefix+"/config/pgs");
+                auto pgs_key = base64_encode(parent->cli->st_cli.etcd_prefix+"/pg/config");
                 rm_items.push_back(json11::Json::object {
                     { "request_put", json11::Json::object {
                         { "key", pgs_key },

src/osd/osd_cluster.cpp

@@ -371,7 +371,7 @@ void osd_t::report_statistics()
         pg_stats["write_osd_set"] = pg.cur_set;
         txn.push_back(json11::Json::object {
             { "request_put", json11::Json::object {
-                { "key", base64_encode(st_cli.etcd_prefix+"/pg/stats/"+std::to_string(pg.pool_id)+"/"+std::to_string(pg.pg_num)) },
+                { "key", base64_encode(st_cli.etcd_prefix+"/pgstats/"+std::to_string(pg.pool_id)+"/"+std::to_string(pg.pg_num)) },
                 { "value", base64_encode(json11::Json(pg_stats).dump()) },
             } }
         });
@@ -418,7 +418,7 @@ void osd_t::on_change_etcd_state_hook(std::map<std::string, etcd_kv_t> & changes
     }
     if (run_primary)
     {
-        bool pgs = changes.find(st_cli.etcd_prefix+"/config/pgs") != changes.end();
+        bool pgs = changes.find(st_cli.etcd_prefix+"/pg/config") != changes.end();
         if (pools || pgs)
         {
             apply_pg_count();

src/test/test_cluster_client.cpp

@@ -22,7 +22,7 @@ void configure_single_pg_pool(cluster_client_t *cli)
         },
     });
     cli->st_cli.parse_state((etcd_kv_t){
-        .key = "/config/pgs",
+        .key = "/pg/config",
         .value = json11::Json::object {
             { "items", json11::Json::object {
                 { "1", json11::Json::object {

tests/run_3osds.sh

@@ -89,7 +89,7 @@ wait_up()
     local i=0
     local configured=0
     while [[ $i -lt $sec ]]; do
-        if $ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(. | length) != 0 and ([ .[0].items["1"][] |
+        if $ETCDCTL get /vitastor/pg/config --print-value-only | jq -s -e '(. | length) != 0 and ([ .[0].items["1"][] |
             select(((.osd_set | select(. != 0) | sort | unique) | length) == '$PG_SIZE') ] | length) == '$PG_COUNT; then
             configured=1
             if $ETCDCTL get /vitastor/pg/state/1/ --prefix --print-value-only | jq -s -e '[ .[] | select(.state == ["active"]) ] | length == '$PG_COUNT; then

tests/test_add_osd.sh

@@ -13,14 +13,14 @@ start_osd 4
 sleep 2
 
 for i in {1..30}; do
-    ($ETCDCTL get /vitastor/config/pgs --print-value-only |\
+    ($ETCDCTL get /vitastor/pg/config --print-value-only |\
         jq -s -e '([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3","4"])') && \
         ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$PG_COUNT) && \
         break
     sleep 1
 done
 
-if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only |\
+if ! ($ETCDCTL get /vitastor/pg/config --print-value-only |\
     jq -s -e '([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3","4"])'); then
     format_error "FAILED: OSD NOT ADDED INTO DISTRIBUTION"
 fi
@@ -35,14 +35,14 @@ build/src/cmd/vitastor-cli --etcd_address $ETCD_URL rm-osd --force 4
 sleep 2
 
 for i in {1..30}; do
-    ($ETCDCTL get /vitastor/config/pgs --print-value-only |\
+    ($ETCDCTL get /vitastor/pg/config --print-value-only |\
         jq -s -e '([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3"])') && \
         ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"] or .state == ["active", "left_on_dead"]) ] | length) == '$PG_COUNT'') && \
         break
     sleep 1
 done
 
-if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only |\
+if ! ($ETCDCTL get /vitastor/pg/config --print-value-only |\
     jq -s -e '([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3"])'); then
     format_error "FAILED: OSD NOT REMOVED FROM DISTRIBUTION"
 fi

tests/test_change_pg_count.sh

@@ -23,7 +23,7 @@ try_change()
     $ETCDCTL put /vitastor/config/pools '{"1":{'$POOLCFG',"pg_size":'$PG_SIZE',"pg_minsize":'$PG_MINSIZE',"pg_count":'$n'}}'
 
     for i in {1..60}; do
-        ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(.[0].items["1"] | map((.osd_set | select(. > 0)) | length == 2) | length) == '$n) && \
+        ($ETCDCTL get /vitastor/pg/config --print-value-only | jq -s -e '(.[0].items["1"] | map((.osd_set | select(. > 0)) | length == 2) | length) == '$n) && \
             ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"] or .state == ["active", "has_misplaced"]) ] | length) == '$n'') && \
             break
         sleep 1
@@ -36,14 +36,14 @@ try_change()
         sleep 1
     done
 
-    if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(.[0].items["1"] | map((.osd_set | select(. > 0)) | length == 2) | length) == '$n); then
+    if ! ($ETCDCTL get /vitastor/pg/config --print-value-only | jq -s -e '(.[0].items["1"] | map((.osd_set | select(. > 0)) | length == 2) | length) == '$n); then
-        $ETCDCTL get /vitastor/config/pgs
+        $ETCDCTL get /vitastor/pg/config
         $ETCDCTL get --prefix /vitastor/pg/state/
         format_error "FAILED: $n PGS NOT CONFIGURED"
     fi
 
     if ! ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$n); then
-        $ETCDCTL get /vitastor/config/pgs
+        $ETCDCTL get /vitastor/pg/config
         $ETCDCTL get --prefix /vitastor/pg/state/
         format_error "FAILED: $n PGS NOT UP"
     fi
@@ -53,7 +53,7 @@ try_change()
     nobj=0
     waittime=0
     while [[ $nobj -ne $NOBJ && $waittime -lt 7 ]]; do
-        nobj=`$ETCDCTL get --prefix '/vitastor/pg/stats' --print-value-only | jq -s '[ .[].object_count ] | reduce .[] as $num (0; .+$num)'`
+        nobj=`$ETCDCTL get --prefix '/vitastor/pgstats' --print-value-only | jq -s '[ .[].object_count ] | reduce .[] as $num (0; .+$num)'`
         if [[ $nobj -ne $NOBJ ]]; then
             waittime=$((waittime+1))
             sleep 1

tests/test_change_pg_size.sh

@@ -13,7 +13,7 @@ try_change()
     $ETCDCTL put /vitastor/config/pools '{"1":{'$POOLCFG',"pg_size":'$s',"pg_minsize":'$PG_MINSIZE',"pg_count":'$n'}}'
 
     for i in {1..10}; do
-        ($ETCDCTL get /vitastor/config/pgs --print-value-only |\
+        ($ETCDCTL get /vitastor/pg/config --print-value-only |\
             jq -s -e '(.[0].items["1"] | map(  ([ .osd_set[] | select(. != 0) ] | length) == '$s'  ) | length == '$n')
                 and ([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3"])') && \
             ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$n'') && \
@@ -21,16 +21,16 @@ try_change()
         sleep 1
     done
 
-    if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only |\
+    if ! ($ETCDCTL get /vitastor/pg/config --print-value-only |\
         jq -s -e '(.[0].items["1"] | map(  ([ .osd_set[] | select(. != 0) ] | length) == '$s'  ) | length == '$n')
             and ([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3"])'); then
-        $ETCDCTL get /vitastor/config/pgs
+        $ETCDCTL get /vitastor/pg/config
         $ETCDCTL get --prefix /vitastor/pg/state/
         format_error "FAILED: PG SIZE NOT CHANGED OR SOME OSDS DO NOT HAVE PGS"
     fi
 
     if ! ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$n); then
-        $ETCDCTL get /vitastor/config/pgs
+        $ETCDCTL get /vitastor/pg/config
         $ETCDCTL get --prefix /vitastor/pg/state/
         format_error "FAILED: PGS NOT UP AFTER PG SIZE CHANGE"
     fi

tests/test_failure_domain.sh

@@ -28,7 +28,7 @@ $ETCDCTL get --print-value-only /vitastor/config/pools | jq -s -e '. == [{"1":{"
 
 sleep 2
 
-$ETCDCTL get --prefix /vitastor/config/pgs --print-value-only | \
+$ETCDCTL get --prefix /vitastor/pg/config --print-value-only | \
     jq -s -e '([ .[0].items["1"] | .[].osd_set | map_values(. | tonumber) | select((.[0] <= 4) != (.[1] <= 4)) ] | length) == 4'
 
 format_green OK

tests/test_interrupted_rebalance.sh

@@ -43,7 +43,7 @@ wait_finish_rebalance 300
 #fi
 
 # Check that no objects are lost !
-nobj=`$ETCDCTL get --prefix '/vitastor/pg/stats' --print-value-only | jq -s '[ .[].object_count ] | reduce .[] as $num (0; .+$num)'`
+nobj=`$ETCDCTL get --prefix '/vitastor/pgstats' --print-value-only | jq -s '[ .[].object_count ] | reduce .[] as $num (0; .+$num)'`
 if [ "$nobj" -ne $((IMG_SIZE*8/PG_DATA_SIZE)) ]; then
     format_error "Data lost after multiple interrupted rebalancings"
 fi

tests/test_move_reappear.sh

@@ -15,7 +15,7 @@ done
 
 $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":1,"failure_domain":"osd","immediate_commit":"none"}}'
 
-$ETCDCTL put /vitastor/config/pgs '{"items":{"1":{"1":{"osd_set":[1,0],"primary":1}}}}'
+$ETCDCTL put /vitastor/pg/config '{"items":{"1":{"1":{"osd_set":[1,0],"primary":1}}}}'
 
 for i in {1..30}; do
     sleep 1
@@ -30,7 +30,7 @@ LD_PRELOAD="build/src/client/libfio_vitastor.so" \
 fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \
     -etcd=$ETCD_URL -pool=1 -inode=2 -size=32M -cluster_log_level=10
 
-$ETCDCTL put /vitastor/config/pgs '{"items":{"1":{"1":{"osd_set":[1,0],"primary":0}}}}'
+$ETCDCTL put /vitastor/pg/config '{"items":{"1":{"1":{"osd_set":[1,0],"primary":0}}}}'
 
 for i in {1..30}; do
     sleep 1
@@ -43,7 +43,7 @@ done
 
 $ETCDCTL put /vitastor/pg/history/1/1 '{"all_peers":[1,2,3]}'
 
-$ETCDCTL put /vitastor/config/pgs '{"items":{"1":{"1":{"osd_set":[4,5],"primary":4}}}}'
+$ETCDCTL put /vitastor/pg/config '{"items":{"1":{"1":{"osd_set":[4,5],"primary":4}}}}'
 
 sleep 5
 for i in {1..30}; do
@@ -60,7 +60,7 @@ LD_PRELOAD="build/src/client/libfio_vitastor.so" \
 fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4k -direct=1 -iodepth=1 -fsync=1 -number_ios=2 -rw=write \
     -etcd=$ETCD_URL -pool=1 -inode=2 -size=32M -cluster_log_level=10
 
-$ETCDCTL put /vitastor/config/pgs '{"items":{"1":{"1":{"osd_set":[4,5],"primary":0}}}}'
+$ETCDCTL put /vitastor/pg/config '{"items":{"1":{"1":{"osd_set":[4,5],"primary":0}}}}'
 
 $ETCDCTL put /vitastor/pg/history/1/1 '{"all_peers":[1,2,3]}'
 
@@ -76,7 +76,7 @@ done
 cp testdata/osd4.log testdata/osd4_pre.log
 >testdata/osd4.log
 
-$ETCDCTL put /vitastor/config/pgs '{"items":{"1":{"1":{"osd_set":[4,5],"primary":4}}}}'
+$ETCDCTL put /vitastor/pg/config '{"items":{"1":{"1":{"osd_set":[4,5],"primary":4}}}}'
 
 for i in {1..30}; do
     sleep 1

tests/test_osd_tags.sh

@@ -27,9 +27,9 @@ $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicate
 
 sleep 2
 
-$ETCDCTL get --prefix /vitastor/config/pgs --print-value-only
+$ETCDCTL get --prefix /vitastor/pg/config --print-value-only
 
-if ! ($ETCDCTL get --prefix /vitastor/config/pgs --print-value-only | \
+if ! ($ETCDCTL get --prefix /vitastor/pg/config --print-value-only | \
     jq -s -e '[ [ .[] | select(has("items")) | .items["1"] | .[].osd_set | map(. | select(. != "" and (.|tonumber) < 5)) ][] | select((. | length) == 2) ] | length == 16'); then
     format_error "Some PGs missing replicas"
 fi

tests/test_parity_change.sh

@@ -16,7 +16,7 @@ try_change()
     s=$2
 
     for i in {1..10}; do
-        ($ETCDCTL get /vitastor/config/pgs --print-value-only |\
+        ($ETCDCTL get /vitastor/pg/config --print-value-only |\
             jq -s -e '(.[0].items["1"] | map(  ([ .osd_set[] | select(. != 0) ] | length) == '$s'  ) | length == '$n')
                 and ([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3","4","5","6","7"])') && \
             ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$n'') && \
@@ -24,16 +24,16 @@ try_change()
         sleep 1
     done
 
-    if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only |\
+    if ! ($ETCDCTL get /vitastor/pg/config --print-value-only |\
         jq -s -e '(.[0].items["1"] | map(  ([ .osd_set[] | select(. != 0) ] | length) == '$s'  ) | length == '$n')
             and ([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3","4","5","6","7"])'); then
-        $ETCDCTL get /vitastor/config/pgs
+        $ETCDCTL get /vitastor/pg/config
         $ETCDCTL get --prefix /vitastor/pg/state/
         format_error "FAILED: PG SIZE NOT CHANGED OR SOME OSDS DO NOT HAVE PGS"
     fi
 
     if ! ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$n); then
-        $ETCDCTL get /vitastor/config/pgs
+        $ETCDCTL get /vitastor/pg/config
         $ETCDCTL get --prefix /vitastor/pg/state/
         format_error "FAILED: PGS NOT UP AFTER PG SIZE CHANGE"
     fi

tests/test_root_node.sh

@@ -21,9 +21,7 @@ $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicate
 
 sleep 2
 
-$ETCDCTL get --prefix /vitastor/config/pgs --print-value-only
+if ! ($ETCDCTL get --prefix /vitastor/pg/config --print-value-only | \
-
-if ! ($ETCDCTL get --prefix /vitastor/config/pgs --print-value-only | \
     jq -s -e '[ [ .[0].items["1"] | .[].osd_set | map(. | select(. != "" and (.|tonumber) < 5)) ][] | select((. | length) == 2) ] | length == 16'); then
     format_error "Some PGs missing replicas"
 fi

tests/test_scrub.sh

@@ -21,7 +21,7 @@ LD_PRELOAD="build/src/client/libfio_vitastor.so" \
         -mirror_file=./testdata/mirror.bin -end_fsync=1 -rw=write -etcd=$ETCD_URL -image=testimg
 
 # Save PG primary
-primary=$($ETCDCTL get --print-value-only /vitastor/config/pgs | jq -r '.items["1"]["1"].primary')
+primary=$($ETCDCTL get --print-value-only /vitastor/pg/config | jq -r '.items["1"]["1"].primary')
 
 # Intentionally corrupt OSD data and restart it
 zero_osd_pid=OSD${ZERO_OSD}_PID
@@ -38,7 +38,7 @@ start_osd $ZERO_OSD
 wait_up 10
 
 # Wait until PG is back on the same primary
-wait_condition 10 "$ETCDCTL"$' get --print-value-only /vitastor/config/pgs | jq -s -e \'.[0].items["1"]["1"].primary == "'$primary'"'"'"
+wait_condition 10 "$ETCDCTL"$' get --print-value-only /vitastor/pg/config | jq -s -e \'.[0].items["1"]["1"].primary == "'$primary'"'"'"
 
 # Trigger scrub
 $ETCDCTL put /vitastor/pg/history/1/1 `$ETCDCTL get --print-value-only /vitastor/pg/history/1/1 | jq -s -c '(.[0] // {}) + {"next_scrub":1}'`

tests/test_switch_primary.sh

@@ -2,14 +2,14 @@
 
 . `dirname $0`/run_3osds.sh
 
-primary=$($ETCDCTL get --print-value-only /vitastor/config/pgs | jq -r '.items["1"]["1"].primary')
+primary=$($ETCDCTL get --print-value-only /vitastor/pg/config | jq -r '.items["1"]["1"].primary')
 primary_pid=OSD${primary}_PID
 kill -9 ${!primary_pid}
 
 sleep 15
-wait_condition 10 "$ETCDCTL get --print-value-only /vitastor/config/pgs | jq -s -e '.[0].items[\"1\"][\"1\"].primary != \"$primary\"'"
+wait_condition 10 "$ETCDCTL get --print-value-only /vitastor/pg/config | jq -s -e '.[0].items[\"1\"][\"1\"].primary != \"$primary\"'"
 
-newprim=$($ETCDCTL get --print-value-only /vitastor/config/pgs | jq -r '.items["1"]["1"].primary')
+newprim=$($ETCDCTL get --print-value-only /vitastor/pg/config | jq -r '.items["1"]["1"].primary')
 
 if [ "$newprim" = "$primary" ]; then
     format_error Primary not switched