From 5b0aebada4ce04a343a19b065a297bb850929739 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Fri, 19 Jul 2024 10:40:00 +0300 Subject: [PATCH] Rename /config/pgs to /pg/config and /pg/stats/* to /pgstats/* --- docs/usage/admin.en.md | 2 +- docs/usage/admin.ru.md | 2 +- mon/etcd_schema.js | 58 ++++++++++++++------------- mon/mon.js | 62 ++++++++++++++--------------- mon/pg_gen.js | 24 +++++------ mon/prometheus.js | 2 +- mon/stats.js | 8 ++-- mon/vitastor_persist_filter.js | 3 +- src/client/etcd_state_client.cpp | 35 +++++++++------- src/client/etcd_state_client.h | 8 ++-- src/cmd/cli_pg_ls.cpp | 6 +-- src/cmd/cli_pool_ls.cpp | 6 +-- src/cmd/cli_rm_osd.cpp | 4 +- src/osd/osd_cluster.cpp | 4 +- src/test/test_cluster_client.cpp | 2 +- tests/run_3osds.sh | 2 +- tests/test_add_osd.sh | 8 ++-- tests/test_change_pg_count.sh | 10 ++--- tests/test_change_pg_size.sh | 8 ++-- tests/test_failure_domain.sh | 2 +- tests/test_interrupted_rebalance.sh | 2 +- tests/test_move_reappear.sh | 10 ++--- tests/test_osd_tags.sh | 4 +- tests/test_parity_change.sh | 8 ++-- tests/test_root_node.sh | 4 +- tests/test_scrub.sh | 4 +- tests/test_switch_primary.sh | 6 +-- 27 files changed, 150 insertions(+), 144 deletions(-) diff --git a/docs/usage/admin.en.md b/docs/usage/admin.en.md index 71a99fdf..32baf168 100644 --- a/docs/usage/admin.en.md +++ b/docs/usage/admin.en.md @@ -42,7 +42,7 @@ PG state always includes exactly 1 of the following base states: - **offline** — PG isn't activated by any OSD at all. Either primary OSD isn't set for this PG at all (if the pool is just created), or an unavailable OSD is set as primary, or the primary OSD refuses to start this PG (for example, because of wrong block_size), - or the PG is stopped by the monitor using `pause: true` flag in `/vitastor/config/pgs` in etcd. + or the PG is stopped by the monitor using `pause: true` flag in `/vitastor/pg/config` in etcd. - **starting** — primary OSD has acquired PG lock in etcd, PG is starting. - **peering** — primary OSD requests PG object listings from secondary OSDs and calculates the PG state. diff --git a/docs/usage/admin.ru.md b/docs/usage/admin.ru.md index 8ee080df..6d6f9a58 100644 --- a/docs/usage/admin.ru.md +++ b/docs/usage/admin.ru.md @@ -42,7 +42,7 @@ - **offline** — PG вообще не активирована ни одним OSD. Либо первичный OSD не назначен вообще (если пул только создан), либо в качестве первичного назначен недоступный OSD, либо назначенный OSD отказывается запускать эту PG (например, из-за несовпадения block_size), - либо PG остановлена монитором через флаг `pause: true` в `/vitastor/config/pgs` в etcd. + либо PG остановлена монитором через флаг `pause: true` в `/vitastor/pg/config` в etcd. - **starting** — первичный OSD захватил блокировку PG в etcd, PG запускается. - **peering** — первичный OSD опрашивает вторичные OSD на предмет списков объектов данной PG и рассчитывает её состояние. - **repeering** — PG ожидает завершения текущих операций ввода-вывода, после чего перейдёт в состояние **peering**. diff --git a/mon/etcd_schema.js b/mon/etcd_schema.js index a6cec0d8..f3497603 100644 --- a/mon/etcd_schema.js +++ b/mon/etcd_schema.js @@ -6,7 +6,7 @@ const etcd_nonempty_keys = { 'config/global': 1, 'config/node_placement': 1, 'config/pools': 1, - 'config/pgs': 1, + 'pg/config': 1, 'history/last_clean_pgs': 1, 'stats': 1, }; @@ -15,7 +15,8 @@ const etcd_allow = new RegExp('^'+[ 'config/node_placement', 'config/pools', 'config/osd/[1-9]\\d*', - 'config/pgs', + 'config/pgs', // old name + 'pg/config', 'config/inode/[1-9]\\d*/[1-9]\\d*', 'osd/state/[1-9]\\d*', 'osd/stats/[1-9]\\d*', @@ -24,7 +25,8 @@ const etcd_allow = new RegExp('^'+[ 'mon/master', 'mon/member/[a-f0-9]+', 'pg/state/[1-9]\\d*/[1-9]\\d*', - 'pg/stats/[1-9]\\d*/[1-9]\\d*', + 'pg/stats/[1-9]\\d*/[1-9]\\d*', // old name + 'pgstats/[1-9]\\d*/[1-9]\\d*', 'pg/history/[1-9]\\d*/[1-9]\\d*', 'history/last_clean_pgs', 'inode/stats/[1-9]\\d*/\\d+', @@ -205,19 +207,6 @@ const etcd_tree = { osd: { /* : { reweight?: 1, tags?: [ 'nvme', ... ], noout?: true }, ... */ }, - /* pgs: { - hash: string, - items: { - : { - : { - osd_set: [ 1, 2, 3 ], - primary: 1, - pause: false, - } - } - } - }, */ - pgs: {}, /* inode: { : { : { @@ -290,6 +279,19 @@ const etcd_tree = { }, }, pg: { + /* config: { + hash: string, + items: { + : { + : { + osd_set: [ 1, 2, 3 ], + primary: 1, + pause: false, + } + } + } + }, */ + config: {}, state: { /* : { : { @@ -300,18 +302,6 @@ const etcd_tree = { } }, */ }, - stats: { - /* : { - : { - object_count: uint64_t, - clean_count: uint64_t, - misplaced_count: uint64_t, - degraded_count: uint64_t, - incomplete_count: uint64_t, - write_osd_set: osd_num_t[], - }, - }, */ - }, history: { /* : { : { @@ -323,6 +313,18 @@ const etcd_tree = { }, */ }, }, + pgstats: { + /* : { + : { + object_count: uint64_t, + clean_count: uint64_t, + misplaced_count: uint64_t, + degraded_count: uint64_t, + incomplete_count: uint64_t, + write_osd_set: osd_num_t[], + }, + }, */ + }, inode: { stats: { /* : { diff --git a/mon/mon.js b/mon/mon.js index a2702769..97bb2552 100644 --- a/mon/mon.js +++ b/mon/mon.js @@ -122,7 +122,7 @@ class Mon !Number(this.state.pool.stats[pool_id].pg_real_size)) { // Generate missing data in etcd - this.state.config.pgs.hash = null; + this.state.pg.config.hash = null; break; } } @@ -201,7 +201,7 @@ class Mon stats_changed = true; changed = true; } - else if (key.substr(0, 11) == '/osd/stats/' || key.substr(0, 10) == '/pg/stats/' || key.substr(0, 16) == '/osd/inodestats/') + else if (key.substr(0, 11) == '/osd/stats/' || key.substr(0, 9) == '/pgstats/' || key.substr(0, 16) == '/osd/inodestats/') { stats_changed = true; } @@ -285,7 +285,7 @@ class Mon continue next_pool; } } - new_clean_pgs.items[pool_id] = this.state.config.pgs.items[pool_id]; + new_clean_pgs.items[pool_id] = this.state.pg.config.items[pool_id]; } this.state.history.last_clean_pgs = new_clean_pgs; await this.etcd.etcd_call('/kv/txn', { @@ -406,7 +406,7 @@ class Mon async stop_all_pgs(pool_id) { let has_online = false, paused = true; - for (const pg in this.state.config.pgs.items[pool_id]||{}) + for (const pg in this.state.pg.config.items[pool_id]||{}) { // FIXME: Change all (||{}) to ?. (optional chaining) at some point const cur_state = (((this.state.pg.state[pool_id]||{})[pg]||{}).state||[]).join(','); @@ -414,7 +414,7 @@ class Mon { has_online = true; } - if (!this.state.config.pgs.items[pool_id][pg].pause) + if (!this.state.pg.config.items[pool_id][pg].pause) { paused = false; } @@ -422,7 +422,7 @@ class Mon if (!paused) { console.log('Stopping all PGs for pool '+pool_id+' before changing PG count'); - const new_cfg = JSON.parse(JSON.stringify(this.state.config.pgs)); + const new_cfg = JSON.parse(JSON.stringify(this.state.pg.config)); for (const pg in new_cfg.items[pool_id]) { new_cfg.items[pool_id][pg].pause = true; @@ -439,11 +439,11 @@ class Mon await this.etcd.etcd_call('/kv/txn', { compare: [ { key: b64(this.config.etcd_prefix+'/mon/master'), target: 'LEASE', lease: ''+this.etcd_lease_id }, - { key: b64(this.config.etcd_prefix+'/config/pgs'), target: 'MOD', mod_revision: ''+this.etcd_watch_revision, result: 'LESS' }, + { key: b64(this.config.etcd_prefix+'/pg/config'), target: 'MOD', mod_revision: ''+this.etcd_watch_revision, result: 'LESS' }, ...checks, ], success: [ - { requestPut: { key: b64(this.config.etcd_prefix+'/config/pgs'), value: b64(JSON.stringify(new_cfg)) } }, + { requestPut: { key: b64(this.config.etcd_prefix+'/pg/config'), value: b64(JSON.stringify(new_cfg)) } }, ], }, this.config.etcd_mon_timeout, 0); return false; @@ -473,7 +473,7 @@ class Mon pools: this.state.config.pools, }; const tree_hash = sha1hex(stableStringify(tree_cfg)); - if (this.state.config.pgs.hash != tree_hash) + if (this.state.pg.config.hash != tree_hash) { // Something has changed console.log('Pool configuration or OSD tree changed, re-optimizing'); @@ -514,10 +514,10 @@ class Mon else { // Nothing changed, but we still want to recheck the distribution of primaries - let new_config_pgs = recheck_primary(this.state, this.config, up_osds, osd_tree); - if (new_config_pgs) + let new_pg_config = recheck_primary(this.state, this.config, up_osds, osd_tree); + if (new_pg_config) { - const ok = await this.save_pg_config(new_config_pgs); + const ok = await this.save_pg_config(new_pg_config); if (ok) console.log('PG configuration successfully changed'); else @@ -532,12 +532,12 @@ class Mon async apply_pool_pgs(results, up_osds, osd_tree, tree_hash) { - for (const pool_id in (this.state.config.pgs||{}).items||{}) + for (const pool_id in (this.state.pg.config||{}).items||{}) { // We should stop all PGs when deleting a pool or changing its PG count if (!this.state.config.pools[pool_id] || - this.state.config.pgs.items[pool_id] && this.state.config.pools[pool_id].pg_count != - Object.keys(this.state.config.pgs.items[pool_id]).reduce((a, c) => (a < (0|c) ? (0|c) : a), 0)) + this.state.pg.config.items[pool_id] && this.state.config.pools[pool_id].pg_count != + Object.keys(this.state.pg.config.items[pool_id]).reduce((a, c) => (a < (0|c) ? (0|c) : a), 0)) { if (!await this.stop_all_pgs(pool_id)) { @@ -545,22 +545,22 @@ class Mon } } } - const new_config_pgs = JSON.parse(JSON.stringify(this.state.config.pgs)); + const new_pg_config = JSON.parse(JSON.stringify(this.state.pg.config)); const etcd_request = { compare: [], success: [] }; - for (const pool_id in (new_config_pgs||{}).items||{}) + for (const pool_id in (new_pg_config||{}).items||{}) { if (!this.state.config.pools[pool_id]) { const prev_pgs = []; - for (const pg in new_config_pgs.items[pool_id]||{}) + for (const pg in new_pg_config.items[pool_id]||{}) { - prev_pgs[pg-1] = new_config_pgs.items[pool_id][pg].osd_set; + prev_pgs[pg-1] = new_pg_config.items[pool_id][pg].osd_set; } // Also delete pool statistics etcd_request.success.push({ requestDeleteRange: { key: b64(this.config.etcd_prefix+'/pool/stats/'+pool_id), } }); - save_new_pgs_txn(new_config_pgs, etcd_request, this.state, this.config.etcd_prefix, + save_new_pgs_txn(new_pg_config, etcd_request, this.state, this.config.etcd_prefix, this.etcd_watch_revision, pool_id, up_osds, osd_tree, prev_pgs, [], []); } } @@ -569,7 +569,7 @@ class Mon const pool_id = pool_res.pool_id; const pool_cfg = this.state.config.pools[pool_id]; let pg_history = []; - for (const pg in ((this.state.config.pgs.items||{})[pool_id]||{})) + for (const pg in ((this.state.pg.config.items||{})[pool_id]||{})) { if (this.state.pg.history[pool_id] && this.state.pg.history[pool_id][pg]) @@ -578,9 +578,9 @@ class Mon } } const real_prev_pgs = []; - for (const pg in ((this.state.config.pgs.items||{})[pool_id]||{})) + for (const pg in ((this.state.pg.config.items||{})[pool_id]||{})) { - real_prev_pgs[pg-1] = [ ...this.state.config.pgs.items[pool_id][pg].osd_set ]; + real_prev_pgs[pg-1] = [ ...this.state.pg.config.items[pool_id][pg].osd_set ]; } if (real_prev_pgs.length > 0 && real_prev_pgs.length != pool_res.pgs.length) { @@ -591,8 +591,8 @@ class Mon pg_history = scale_pg_history(pg_history, real_prev_pgs, pool_res.pgs); // Drop stats etcd_request.success.push({ requestDeleteRange: { - key: b64(this.config.etcd_prefix+'/pg/stats/'+pool_id+'/'), - range_end: b64(this.config.etcd_prefix+'/pg/stats/'+pool_id+'0'), + key: b64(this.config.etcd_prefix+'/pgstats/'+pool_id+'/'), + range_end: b64(this.config.etcd_prefix+'/pgstats/'+pool_id+'0'), } }); } const stats = { @@ -603,21 +603,21 @@ class Mon key: b64(this.config.etcd_prefix+'/pool/stats/'+pool_id), value: b64(JSON.stringify(stats)), } }); - save_new_pgs_txn(new_config_pgs, etcd_request, this.state, this.config.etcd_prefix, + save_new_pgs_txn(new_pg_config, etcd_request, this.state, this.config.etcd_prefix, this.etcd_watch_revision, pool_id, up_osds, osd_tree, real_prev_pgs, pool_res.pgs, pg_history); } - new_config_pgs.hash = tree_hash; - return await this.save_pg_config(new_config_pgs, etcd_request); + new_pg_config.hash = tree_hash; + return await this.save_pg_config(new_pg_config, etcd_request); } - async save_pg_config(new_config_pgs, etcd_request = { compare: [], success: [] }) + async save_pg_config(new_pg_config, etcd_request = { compare: [], success: [] }) { etcd_request.compare.push( { key: b64(this.config.etcd_prefix+'/mon/master'), target: 'LEASE', lease: ''+this.etcd_lease_id }, - { key: b64(this.config.etcd_prefix+'/config/pgs'), target: 'MOD', mod_revision: ''+this.etcd_watch_revision, result: 'LESS' }, + { key: b64(this.config.etcd_prefix+'/pg/config'), target: 'MOD', mod_revision: ''+this.etcd_watch_revision, result: 'LESS' }, ); etcd_request.success.push( - { requestPut: { key: b64(this.config.etcd_prefix+'/config/pgs'), value: b64(JSON.stringify(new_config_pgs)) } }, + { requestPut: { key: b64(this.config.etcd_prefix+'/pg/config'), value: b64(JSON.stringify(new_pg_config)) } }, ); const txn_res = await this.etcd.etcd_call('/kv/txn', etcd_request, this.config.etcd_mon_timeout, 0); return txn_res.succeeded; diff --git a/mon/pg_gen.js b/mon/pg_gen.js index bf09654c..a0b00dc9 100644 --- a/mon/pg_gen.js +++ b/mon/pg_gen.js @@ -57,7 +57,7 @@ function pick_primary(pool_config, osd_set, up_osds, aff_osds) function recheck_primary(state, global_config, up_osds, osd_tree) { - let new_config_pgs; + let new_pg_config; for (const pool_id in state.config.pools) { const pool_cfg = state.config.pools[pool_id]; @@ -69,30 +69,30 @@ function recheck_primary(state, global_config, up_osds, osd_tree) reset_rng(); for (let pg_num = 1; pg_num <= pool_cfg.pg_count; pg_num++) { - if (!state.config.pgs.items[pool_id]) + if (!state.pg.config.items[pool_id]) { continue; } - const pg_cfg = state.config.pgs.items[pool_id][pg_num]; + const pg_cfg = state.pg.config.items[pool_id][pg_num]; if (pg_cfg) { const new_primary = pick_primary(state.config.pools[pool_id], pg_cfg.osd_set, up_osds, aff_osds); if (pg_cfg.primary != new_primary) { - if (!new_config_pgs) + if (!new_pg_config) { - new_config_pgs = JSON.parse(JSON.stringify(state.config.pgs)); + new_pg_config = JSON.parse(JSON.stringify(state.pg.config)); } console.log( `Moving pool ${pool_id} (${pool_cfg.name || 'unnamed'}) PG ${pg_num}`+ ` primary OSD from ${pg_cfg.primary} to ${new_primary}` ); - new_config_pgs.items[pool_id][pg_num].primary = new_primary; + new_pg_config.items[pool_id][pg_num].primary = new_primary; } } } } - return new_config_pgs; + return new_pg_config; } function save_new_pgs_txn(save_to, request, state, etcd_prefix, etcd_watch_revision, pool_id, up_osds, osd_tree, prev_pgs, new_pgs, pg_history) @@ -185,10 +185,10 @@ async function generate_pool_pgs(state, global_config, pool_id, osd_tree, levels } if (!prev_pgs.length) { - // Fall back to config/pgs if it's empty - for (const pg in ((state.config.pgs.items||{})[pool_id]||{})) + // Fall back to pg/config if it's empty + for (const pg in ((state.pg.config.items||{})[pool_id]||{})) { - prev_pgs[pg-1] = [ ...state.config.pgs.items[pool_id][pg].osd_set ]; + prev_pgs[pg-1] = [ ...state.pg.config.items[pool_id][pg].osd_set ]; } } const old_pg_count = prev_pgs.length; @@ -205,8 +205,8 @@ async function generate_pool_pgs(state, global_config, pool_id, osd_tree, levels ordered: pool_cfg.scheme != 'replicated', }; let optimize_result; - // Re-shuffle PGs if config/pgs.hash is empty - if (old_pg_count > 0 && state.config.pgs.hash) + // Re-shuffle PGs if pg/config.hash is empty + if (old_pg_count > 0 && state.pg.config.hash) { if (prev_pgs.length != pool_cfg.pg_count) { diff --git a/mon/prometheus.js b/mon/prometheus.js index c4558054..6b5d0096 100644 --- a/mon/prometheus.js +++ b/mon/prometheus.js @@ -166,7 +166,7 @@ function export_prometheus_metrics(st) res += `vitastor_pool_used_raw_tb{${pool_label}} ${pool_stat.used_raw_tb||0}\n`; // PG states and pool up/down status - const real_pg_count = (Object.keys(((st.config.pgs||{}).items||{})[pool_id]||{}).length) || (0|pool_cfg.pg_count); + const real_pg_count = (Object.keys(((st.pg.config||{}).items||{})[pool_id]||{}).length) || (0|pool_cfg.pg_count); const per_state = { active: 0, starting: 0, diff --git a/mon/stats.js b/mon/stats.js index 4e1bbab2..c63e10bc 100644 --- a/mon/stats.js +++ b/mon/stats.js @@ -100,10 +100,10 @@ function sum_object_counts(state, global_config) { const object_counts = { object: 0n, clean: 0n, misplaced: 0n, degraded: 0n, incomplete: 0n }; const object_bytes = { object: 0n, clean: 0n, misplaced: 0n, degraded: 0n, incomplete: 0n }; - for (const pool_id in state.pg.stats) + for (const pool_id in state.pgstats) { let object_size = 0; - for (const osd_num of state.pg.stats[pool_id].write_osd_set||[]) + for (const osd_num of state.pgstats[pool_id].write_osd_set||[]) { if (osd_num && state.osd.stats[osd_num] && state.osd.stats[osd_num].block_size) { @@ -121,9 +121,9 @@ function sum_object_counts(state, global_config) object_size *= ((pool_cfg.pg_size||0) - (pool_cfg.parity_chunks||0)); } object_size = BigInt(object_size); - for (const pg_num in state.pg.stats[pool_id]) + for (const pg_num in state.pgstats[pool_id]) { - const st = state.pg.stats[pool_id][pg_num]; + const st = state.pgstats[pool_id][pg_num]; if (st) { for (const k in object_counts) diff --git a/mon/vitastor_persist_filter.js b/mon/vitastor_persist_filter.js index dbbb6100..f2668409 100644 --- a/mon/vitastor_persist_filter.js +++ b/mon/vitastor_persist_filter.js @@ -35,7 +35,8 @@ function vitastor_persist_filter(cfg) } else if (key.substr(0, prefix.length+'/osd/'.length) == prefix+'/osd/' || key.substr(0, prefix.length+'/inode/stats/'.length) == prefix+'/inode/stats/' || - key.substr(0, prefix.length+'/pg/stats/'.length) == prefix+'/pg/stats/' || + key.substr(0, prefix.length+'/pg/stats/'.length) == prefix+'/pg/stats/' || // old name + key.substr(0, prefix.length+'/pgstats/'.length) == prefix+'/pgstats/' || key.substr(0, prefix.length+'/pool/stats/'.length) == prefix+'/pool/stats/' || key == prefix+'/stats') { diff --git a/src/client/etcd_state_client.cpp b/src/client/etcd_state_client.cpp index d954532b..76ad6cf2 100644 --- a/src/client/etcd_state_client.cpp +++ b/src/client/etcd_state_client.cpp @@ -353,7 +353,6 @@ void etcd_state_client_t::start_etcd_watcher() uint64_t watch_id = data["result"]["watch_id"].uint64_value(); if (watch_id == ETCD_CONFIG_WATCH_ID || watch_id == ETCD_PG_STATE_WATCH_ID || - watch_id == ETCD_PG_HISTORY_WATCH_ID || watch_id == ETCD_OSD_STATE_WATCH_ID) etcd_watches_initialised++; if (etcd_watches_initialised == ETCD_TOTAL_WATCHES && this->log_level > 0) @@ -395,8 +394,8 @@ void etcd_state_client_t::start_etcd_watcher() } if (etcd_watches_initialised == ETCD_TOTAL_WATCHES && !data["result"]["header"]["revision"].is_null()) { - // Protect against a revision beign split into multiple messages and some - // of them being lost. Even though I'm not sure if etcd actually splits them + // Protect against a revision being split into multiple messages and some + // of them being lost. // Also sometimes etcd sends something without a header, like: // {"error": {"grpc_code": 14, "http_code": 503, "http_status": "Service Unavailable", "message": "error reading from server: EOF"}} etcd_watch_revision = data["result"]["header"]["revision"].uint64_value(); @@ -473,22 +472,14 @@ void etcd_state_client_t::start_etcd_watcher() }).dump()); http_post_message(etcd_watch_ws, WS_TEXT, json11::Json(json11::Json::object { { "create_request", json11::Json::object { - { "key", base64_encode(etcd_prefix+"/pg/state/") }, - { "range_end", base64_encode(etcd_prefix+"/pg/state0") }, + { "key", base64_encode(etcd_prefix+"/pg/") }, + { "range_end", base64_encode(etcd_prefix+"/pg0") }, { "start_revision", etcd_watch_revision }, { "watch_id", ETCD_PG_STATE_WATCH_ID }, { "progress_notify", true }, } } }).dump()); - http_post_message(etcd_watch_ws, WS_TEXT, json11::Json(json11::Json::object { - { "create_request", json11::Json::object { - { "key", base64_encode(etcd_prefix+"/pg/history/") }, - { "range_end", base64_encode(etcd_prefix+"/pg/history0") }, - { "start_revision", etcd_watch_revision }, - { "watch_id", ETCD_PG_HISTORY_WATCH_ID }, - { "progress_notify", true }, - } } - }).dump()); + // FIXME: Do not watch /pg/history/ at all in client code (not in OSD) if (on_start_watcher_hook) { on_start_watcher_hook(etcd_watch_ws); @@ -591,6 +582,11 @@ void etcd_state_client_t::load_pgs() { "key", base64_encode(etcd_prefix+"/config/pgs") }, } } }, + json11::Json::object { + { "request_range", json11::Json::object { + { "key", base64_encode(etcd_prefix+"/pg/config") }, + } } + }, json11::Json::object { { "request_range", json11::Json::object { { "key", base64_encode(etcd_prefix+"/config/inode/") }, @@ -895,8 +891,17 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv) on_change_pool_config_hook(); } } - else if (key == etcd_prefix+"/config/pgs") + else if (key == etcd_prefix+"/pg/config" || key == etcd_prefix+"/config/pgs") { + if (key == etcd_prefix+"/pg/config") + { + new_pg_config = !value.is_null(); + } + else if (new_pg_config) + { + // Ignore old key if the new one is present + return; + } for (auto & pool_item: this->pool_config) { for (auto & pg_item: pool_item.second.pg_config) diff --git a/src/client/etcd_state_client.h b/src/client/etcd_state_client.h index 4d66cb00..fb21771e 100644 --- a/src/client/etcd_state_client.h +++ b/src/client/etcd_state_client.h @@ -10,10 +10,9 @@ #include "timerfd_manager.h" #define ETCD_CONFIG_WATCH_ID 1 -#define ETCD_PG_STATE_WATCH_ID 2 -#define ETCD_PG_HISTORY_WATCH_ID 3 -#define ETCD_OSD_STATE_WATCH_ID 4 -#define ETCD_TOTAL_WATCHES 4 +#define ETCD_OSD_STATE_WATCH_ID 2 +#define ETCD_PG_STATE_WATCH_ID 3 +#define ETCD_TOTAL_WATCHES 3 #define DEFAULT_BLOCK_SIZE 128*1024 #define MIN_DATA_BLOCK_SIZE 4*1024 @@ -96,6 +95,7 @@ protected: std::vector addresses_to_try; std::vector watches; http_co_t *etcd_watch_ws = NULL, *keepalive_client = NULL; + bool new_pg_config = false; int ws_keepalive_timer = -1; int ws_alive = 0; bool rand_initialized = false; diff --git a/src/cmd/cli_pg_ls.cpp b/src/cmd/cli_pg_ls.cpp index 4c60915d..281e7383 100644 --- a/src/cmd/cli_pg_ls.cpp +++ b/src/cmd/cli_pg_ls.cpp @@ -49,8 +49,8 @@ struct pg_lister_t { "success", json11::Json::array { json11::Json::object { { "request_range", json11::Json::object { - { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/pg/stats"+(pool_id ? "/"+std::to_string(pool_id)+"/" : "/")) }, - { "range_end", base64_encode(parent->cli->st_cli.etcd_prefix+"/pg/stats"+(pool_id ? "/"+std::to_string(pool_id)+"0" : "0")) }, + { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/pgstats"+(pool_id ? "/"+std::to_string(pool_id)+"/" : "/")) }, + { "range_end", base64_encode(parent->cli->st_cli.etcd_prefix+"/pgstats"+(pool_id ? "/"+std::to_string(pool_id)+"0" : "0")) }, } }, }, } }, @@ -65,7 +65,7 @@ resume_1: state = 100; return; } - parent->iterate_kvs_2(parent->etcd_result["responses"][0]["response_range"]["kvs"], "/pg/stats/", [&](pool_id_t pool_id, uint64_t pg_num, json11::Json value) + parent->iterate_kvs_2(parent->etcd_result["responses"][0]["response_range"]["kvs"], "/pgstats/", [&](pool_id_t pool_id, uint64_t pg_num, json11::Json value) { pg_stats[(pool_pg_num_t){ .pool_id = pool_id, .pg_num = (pg_num_t)pg_num }] = value; }); diff --git a/src/cmd/cli_pool_ls.cpp b/src/cmd/cli_pool_ls.cpp index ad807715..e9071fc4 100644 --- a/src/cmd/cli_pool_ls.cpp +++ b/src/cmd/cli_pool_ls.cpp @@ -214,10 +214,10 @@ resume_1: json11::Json::object { { "request_range", json11::Json::object { { "key", base64_encode( - parent->cli->st_cli.etcd_prefix+"/pg/stats/" + parent->cli->st_cli.etcd_prefix+"/pgstats/" ) }, { "range_end", base64_encode( - parent->cli->st_cli.etcd_prefix+"/pg/stats0" + parent->cli->st_cli.etcd_prefix+"/pgstats0" ) }, } }, }, @@ -235,7 +235,7 @@ resume_1: } // Calculate recovery percent std::map counts; - parent->iterate_kvs_2(parent->etcd_result["responses"][0]["response_range"]["kvs"], "/pg/stats/", + parent->iterate_kvs_2(parent->etcd_result["responses"][0]["response_range"]["kvs"], "/pgstats/", [&](pool_id_t pool_id, uint64_t pg_num, json11::Json value) { auto & cnt = counts[pool_id]; diff --git a/src/cmd/cli_rm_osd.cpp b/src/cmd/cli_rm_osd.cpp index 9a4a5a52..4c118a22 100644 --- a/src/cmd/cli_rm_osd.cpp +++ b/src/cmd/cli_rm_osd.cpp @@ -176,7 +176,7 @@ struct rm_osd_t json11::Json::object { { "request_range", json11::Json::object { { "key", base64_encode( - parent->cli->st_cli.etcd_prefix+"/config/pgs" + parent->cli->st_cli.etcd_prefix+"/pg/config" ) }, } }, }, @@ -229,7 +229,7 @@ struct rm_osd_t } if (!new_pgs.is_null()) { - auto pgs_key = base64_encode(parent->cli->st_cli.etcd_prefix+"/config/pgs"); + auto pgs_key = base64_encode(parent->cli->st_cli.etcd_prefix+"/pg/config"); rm_items.push_back(json11::Json::object { { "request_put", json11::Json::object { { "key", pgs_key }, diff --git a/src/osd/osd_cluster.cpp b/src/osd/osd_cluster.cpp index d75b0bb5..adb19e8c 100644 --- a/src/osd/osd_cluster.cpp +++ b/src/osd/osd_cluster.cpp @@ -371,7 +371,7 @@ void osd_t::report_statistics() pg_stats["write_osd_set"] = pg.cur_set; txn.push_back(json11::Json::object { { "request_put", json11::Json::object { - { "key", base64_encode(st_cli.etcd_prefix+"/pg/stats/"+std::to_string(pg.pool_id)+"/"+std::to_string(pg.pg_num)) }, + { "key", base64_encode(st_cli.etcd_prefix+"/pgstats/"+std::to_string(pg.pool_id)+"/"+std::to_string(pg.pg_num)) }, { "value", base64_encode(json11::Json(pg_stats).dump()) }, } } }); @@ -418,7 +418,7 @@ void osd_t::on_change_etcd_state_hook(std::map & changes } if (run_primary) { - bool pgs = changes.find(st_cli.etcd_prefix+"/config/pgs") != changes.end(); + bool pgs = changes.find(st_cli.etcd_prefix+"/pg/config") != changes.end(); if (pools || pgs) { apply_pg_count(); diff --git a/src/test/test_cluster_client.cpp b/src/test/test_cluster_client.cpp index db552159..9dc00c6f 100644 --- a/src/test/test_cluster_client.cpp +++ b/src/test/test_cluster_client.cpp @@ -22,7 +22,7 @@ void configure_single_pg_pool(cluster_client_t *cli) }, }); cli->st_cli.parse_state((etcd_kv_t){ - .key = "/config/pgs", + .key = "/pg/config", .value = json11::Json::object { { "items", json11::Json::object { { "1", json11::Json::object { diff --git a/tests/run_3osds.sh b/tests/run_3osds.sh index d588596a..166b9b70 100644 --- a/tests/run_3osds.sh +++ b/tests/run_3osds.sh @@ -89,7 +89,7 @@ wait_up() local i=0 local configured=0 while [[ $i -lt $sec ]]; do - if $ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(. | length) != 0 and ([ .[0].items["1"][] | + if $ETCDCTL get /vitastor/pg/config --print-value-only | jq -s -e '(. | length) != 0 and ([ .[0].items["1"][] | select(((.osd_set | select(. != 0) | sort | unique) | length) == '$PG_SIZE') ] | length) == '$PG_COUNT; then configured=1 if $ETCDCTL get /vitastor/pg/state/1/ --prefix --print-value-only | jq -s -e '[ .[] | select(.state == ["active"]) ] | length == '$PG_COUNT; then diff --git a/tests/test_add_osd.sh b/tests/test_add_osd.sh index 0d166cfb..5a1d298c 100755 --- a/tests/test_add_osd.sh +++ b/tests/test_add_osd.sh @@ -13,14 +13,14 @@ start_osd 4 sleep 2 for i in {1..30}; do - ($ETCDCTL get /vitastor/config/pgs --print-value-only |\ + ($ETCDCTL get /vitastor/pg/config --print-value-only |\ jq -s -e '([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3","4"])') && \ ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$PG_COUNT) && \ break sleep 1 done -if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only |\ +if ! ($ETCDCTL get /vitastor/pg/config --print-value-only |\ jq -s -e '([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3","4"])'); then format_error "FAILED: OSD NOT ADDED INTO DISTRIBUTION" fi @@ -35,14 +35,14 @@ build/src/cmd/vitastor-cli --etcd_address $ETCD_URL rm-osd --force 4 sleep 2 for i in {1..30}; do - ($ETCDCTL get /vitastor/config/pgs --print-value-only |\ + ($ETCDCTL get /vitastor/pg/config --print-value-only |\ jq -s -e '([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3"])') && \ ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"] or .state == ["active", "left_on_dead"]) ] | length) == '$PG_COUNT'') && \ break sleep 1 done -if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only |\ +if ! ($ETCDCTL get /vitastor/pg/config --print-value-only |\ jq -s -e '([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3"])'); then format_error "FAILED: OSD NOT REMOVED FROM DISTRIBUTION" fi diff --git a/tests/test_change_pg_count.sh b/tests/test_change_pg_count.sh index 56c8cf68..6ebc323b 100755 --- a/tests/test_change_pg_count.sh +++ b/tests/test_change_pg_count.sh @@ -23,7 +23,7 @@ try_change() $ETCDCTL put /vitastor/config/pools '{"1":{'$POOLCFG',"pg_size":'$PG_SIZE',"pg_minsize":'$PG_MINSIZE',"pg_count":'$n'}}' for i in {1..60}; do - ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(.[0].items["1"] | map((.osd_set | select(. > 0)) | length == 2) | length) == '$n) && \ + ($ETCDCTL get /vitastor/pg/config --print-value-only | jq -s -e '(.[0].items["1"] | map((.osd_set | select(. > 0)) | length == 2) | length) == '$n) && \ ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"] or .state == ["active", "has_misplaced"]) ] | length) == '$n'') && \ break sleep 1 @@ -36,14 +36,14 @@ try_change() sleep 1 done - if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(.[0].items["1"] | map((.osd_set | select(. > 0)) | length == 2) | length) == '$n); then - $ETCDCTL get /vitastor/config/pgs + if ! ($ETCDCTL get /vitastor/pg/config --print-value-only | jq -s -e '(.[0].items["1"] | map((.osd_set | select(. > 0)) | length == 2) | length) == '$n); then + $ETCDCTL get /vitastor/pg/config $ETCDCTL get --prefix /vitastor/pg/state/ format_error "FAILED: $n PGS NOT CONFIGURED" fi if ! ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$n); then - $ETCDCTL get /vitastor/config/pgs + $ETCDCTL get /vitastor/pg/config $ETCDCTL get --prefix /vitastor/pg/state/ format_error "FAILED: $n PGS NOT UP" fi @@ -53,7 +53,7 @@ try_change() nobj=0 waittime=0 while [[ $nobj -ne $NOBJ && $waittime -lt 7 ]]; do - nobj=`$ETCDCTL get --prefix '/vitastor/pg/stats' --print-value-only | jq -s '[ .[].object_count ] | reduce .[] as $num (0; .+$num)'` + nobj=`$ETCDCTL get --prefix '/vitastor/pgstats' --print-value-only | jq -s '[ .[].object_count ] | reduce .[] as $num (0; .+$num)'` if [[ $nobj -ne $NOBJ ]]; then waittime=$((waittime+1)) sleep 1 diff --git a/tests/test_change_pg_size.sh b/tests/test_change_pg_size.sh index c2ad712b..9e7c7615 100755 --- a/tests/test_change_pg_size.sh +++ b/tests/test_change_pg_size.sh @@ -13,7 +13,7 @@ try_change() $ETCDCTL put /vitastor/config/pools '{"1":{'$POOLCFG',"pg_size":'$s',"pg_minsize":'$PG_MINSIZE',"pg_count":'$n'}}' for i in {1..10}; do - ($ETCDCTL get /vitastor/config/pgs --print-value-only |\ + ($ETCDCTL get /vitastor/pg/config --print-value-only |\ jq -s -e '(.[0].items["1"] | map( ([ .osd_set[] | select(. != 0) ] | length) == '$s' ) | length == '$n') and ([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3"])') && \ ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$n'') && \ @@ -21,16 +21,16 @@ try_change() sleep 1 done - if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only |\ + if ! ($ETCDCTL get /vitastor/pg/config --print-value-only |\ jq -s -e '(.[0].items["1"] | map( ([ .osd_set[] | select(. != 0) ] | length) == '$s' ) | length == '$n') and ([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3"])'); then - $ETCDCTL get /vitastor/config/pgs + $ETCDCTL get /vitastor/pg/config $ETCDCTL get --prefix /vitastor/pg/state/ format_error "FAILED: PG SIZE NOT CHANGED OR SOME OSDS DO NOT HAVE PGS" fi if ! ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$n); then - $ETCDCTL get /vitastor/config/pgs + $ETCDCTL get /vitastor/pg/config $ETCDCTL get --prefix /vitastor/pg/state/ format_error "FAILED: PGS NOT UP AFTER PG SIZE CHANGE" fi diff --git a/tests/test_failure_domain.sh b/tests/test_failure_domain.sh index a1e23d2c..d626b940 100755 --- a/tests/test_failure_domain.sh +++ b/tests/test_failure_domain.sh @@ -28,7 +28,7 @@ $ETCDCTL get --print-value-only /vitastor/config/pools | jq -s -e '. == [{"1":{" sleep 2 -$ETCDCTL get --prefix /vitastor/config/pgs --print-value-only | \ +$ETCDCTL get --prefix /vitastor/pg/config --print-value-only | \ jq -s -e '([ .[0].items["1"] | .[].osd_set | map_values(. | tonumber) | select((.[0] <= 4) != (.[1] <= 4)) ] | length) == 4' format_green OK diff --git a/tests/test_interrupted_rebalance.sh b/tests/test_interrupted_rebalance.sh index 070c1aca..f6b9bfdd 100755 --- a/tests/test_interrupted_rebalance.sh +++ b/tests/test_interrupted_rebalance.sh @@ -43,7 +43,7 @@ wait_finish_rebalance 300 #fi # Check that no objects are lost ! -nobj=`$ETCDCTL get --prefix '/vitastor/pg/stats' --print-value-only | jq -s '[ .[].object_count ] | reduce .[] as $num (0; .+$num)'` +nobj=`$ETCDCTL get --prefix '/vitastor/pgstats' --print-value-only | jq -s '[ .[].object_count ] | reduce .[] as $num (0; .+$num)'` if [ "$nobj" -ne $((IMG_SIZE*8/PG_DATA_SIZE)) ]; then format_error "Data lost after multiple interrupted rebalancings" fi diff --git a/tests/test_move_reappear.sh b/tests/test_move_reappear.sh index af331b30..174bffe4 100755 --- a/tests/test_move_reappear.sh +++ b/tests/test_move_reappear.sh @@ -15,7 +15,7 @@ done $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":1,"failure_domain":"osd","immediate_commit":"none"}}' -$ETCDCTL put /vitastor/config/pgs '{"items":{"1":{"1":{"osd_set":[1,0],"primary":1}}}}' +$ETCDCTL put /vitastor/pg/config '{"items":{"1":{"1":{"osd_set":[1,0],"primary":1}}}}' for i in {1..30}; do sleep 1 @@ -30,7 +30,7 @@ LD_PRELOAD="build/src/client/libfio_vitastor.so" \ fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \ -etcd=$ETCD_URL -pool=1 -inode=2 -size=32M -cluster_log_level=10 -$ETCDCTL put /vitastor/config/pgs '{"items":{"1":{"1":{"osd_set":[1,0],"primary":0}}}}' +$ETCDCTL put /vitastor/pg/config '{"items":{"1":{"1":{"osd_set":[1,0],"primary":0}}}}' for i in {1..30}; do sleep 1 @@ -43,7 +43,7 @@ done $ETCDCTL put /vitastor/pg/history/1/1 '{"all_peers":[1,2,3]}' -$ETCDCTL put /vitastor/config/pgs '{"items":{"1":{"1":{"osd_set":[4,5],"primary":4}}}}' +$ETCDCTL put /vitastor/pg/config '{"items":{"1":{"1":{"osd_set":[4,5],"primary":4}}}}' sleep 5 for i in {1..30}; do @@ -60,7 +60,7 @@ LD_PRELOAD="build/src/client/libfio_vitastor.so" \ fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4k -direct=1 -iodepth=1 -fsync=1 -number_ios=2 -rw=write \ -etcd=$ETCD_URL -pool=1 -inode=2 -size=32M -cluster_log_level=10 -$ETCDCTL put /vitastor/config/pgs '{"items":{"1":{"1":{"osd_set":[4,5],"primary":0}}}}' +$ETCDCTL put /vitastor/pg/config '{"items":{"1":{"1":{"osd_set":[4,5],"primary":0}}}}' $ETCDCTL put /vitastor/pg/history/1/1 '{"all_peers":[1,2,3]}' @@ -76,7 +76,7 @@ done cp testdata/osd4.log testdata/osd4_pre.log >testdata/osd4.log -$ETCDCTL put /vitastor/config/pgs '{"items":{"1":{"1":{"osd_set":[4,5],"primary":4}}}}' +$ETCDCTL put /vitastor/pg/config '{"items":{"1":{"1":{"osd_set":[4,5],"primary":4}}}}' for i in {1..30}; do sleep 1 diff --git a/tests/test_osd_tags.sh b/tests/test_osd_tags.sh index 40e678af..6def3332 100755 --- a/tests/test_osd_tags.sh +++ b/tests/test_osd_tags.sh @@ -27,9 +27,9 @@ $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicate sleep 2 -$ETCDCTL get --prefix /vitastor/config/pgs --print-value-only +$ETCDCTL get --prefix /vitastor/pg/config --print-value-only -if ! ($ETCDCTL get --prefix /vitastor/config/pgs --print-value-only | \ +if ! ($ETCDCTL get --prefix /vitastor/pg/config --print-value-only | \ jq -s -e '[ [ .[] | select(has("items")) | .items["1"] | .[].osd_set | map(. | select(. != "" and (.|tonumber) < 5)) ][] | select((. | length) == 2) ] | length == 16'); then format_error "Some PGs missing replicas" fi diff --git a/tests/test_parity_change.sh b/tests/test_parity_change.sh index a6cbe7f8..0453c291 100755 --- a/tests/test_parity_change.sh +++ b/tests/test_parity_change.sh @@ -16,7 +16,7 @@ try_change() s=$2 for i in {1..10}; do - ($ETCDCTL get /vitastor/config/pgs --print-value-only |\ + ($ETCDCTL get /vitastor/pg/config --print-value-only |\ jq -s -e '(.[0].items["1"] | map( ([ .osd_set[] | select(. != 0) ] | length) == '$s' ) | length == '$n') and ([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3","4","5","6","7"])') && \ ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$n'') && \ @@ -24,16 +24,16 @@ try_change() sleep 1 done - if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only |\ + if ! ($ETCDCTL get /vitastor/pg/config --print-value-only |\ jq -s -e '(.[0].items["1"] | map( ([ .osd_set[] | select(. != 0) ] | length) == '$s' ) | length == '$n') and ([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3","4","5","6","7"])'); then - $ETCDCTL get /vitastor/config/pgs + $ETCDCTL get /vitastor/pg/config $ETCDCTL get --prefix /vitastor/pg/state/ format_error "FAILED: PG SIZE NOT CHANGED OR SOME OSDS DO NOT HAVE PGS" fi if ! ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$n); then - $ETCDCTL get /vitastor/config/pgs + $ETCDCTL get /vitastor/pg/config $ETCDCTL get --prefix /vitastor/pg/state/ format_error "FAILED: PGS NOT UP AFTER PG SIZE CHANGE" fi diff --git a/tests/test_root_node.sh b/tests/test_root_node.sh index 288d4ad1..c215aaff 100755 --- a/tests/test_root_node.sh +++ b/tests/test_root_node.sh @@ -21,9 +21,7 @@ $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicate sleep 2 -$ETCDCTL get --prefix /vitastor/config/pgs --print-value-only - -if ! ($ETCDCTL get --prefix /vitastor/config/pgs --print-value-only | \ +if ! ($ETCDCTL get --prefix /vitastor/pg/config --print-value-only | \ jq -s -e '[ [ .[0].items["1"] | .[].osd_set | map(. | select(. != "" and (.|tonumber) < 5)) ][] | select((. | length) == 2) ] | length == 16'); then format_error "Some PGs missing replicas" fi diff --git a/tests/test_scrub.sh b/tests/test_scrub.sh index 3d6949b9..66ae19d3 100755 --- a/tests/test_scrub.sh +++ b/tests/test_scrub.sh @@ -21,7 +21,7 @@ LD_PRELOAD="build/src/client/libfio_vitastor.so" \ -mirror_file=./testdata/mirror.bin -end_fsync=1 -rw=write -etcd=$ETCD_URL -image=testimg # Save PG primary -primary=$($ETCDCTL get --print-value-only /vitastor/config/pgs | jq -r '.items["1"]["1"].primary') +primary=$($ETCDCTL get --print-value-only /vitastor/pg/config | jq -r '.items["1"]["1"].primary') # Intentionally corrupt OSD data and restart it zero_osd_pid=OSD${ZERO_OSD}_PID @@ -38,7 +38,7 @@ start_osd $ZERO_OSD wait_up 10 # Wait until PG is back on the same primary -wait_condition 10 "$ETCDCTL"$' get --print-value-only /vitastor/config/pgs | jq -s -e \'.[0].items["1"]["1"].primary == "'$primary'"'"'" +wait_condition 10 "$ETCDCTL"$' get --print-value-only /vitastor/pg/config | jq -s -e \'.[0].items["1"]["1"].primary == "'$primary'"'"'" # Trigger scrub $ETCDCTL put /vitastor/pg/history/1/1 `$ETCDCTL get --print-value-only /vitastor/pg/history/1/1 | jq -s -c '(.[0] // {}) + {"next_scrub":1}'` diff --git a/tests/test_switch_primary.sh b/tests/test_switch_primary.sh index 78dcde78..3b5067a7 100755 --- a/tests/test_switch_primary.sh +++ b/tests/test_switch_primary.sh @@ -2,14 +2,14 @@ . `dirname $0`/run_3osds.sh -primary=$($ETCDCTL get --print-value-only /vitastor/config/pgs | jq -r '.items["1"]["1"].primary') +primary=$($ETCDCTL get --print-value-only /vitastor/pg/config | jq -r '.items["1"]["1"].primary') primary_pid=OSD${primary}_PID kill -9 ${!primary_pid} sleep 15 -wait_condition 10 "$ETCDCTL get --print-value-only /vitastor/config/pgs | jq -s -e '.[0].items[\"1\"][\"1\"].primary != \"$primary\"'" +wait_condition 10 "$ETCDCTL get --print-value-only /vitastor/pg/config | jq -s -e '.[0].items[\"1\"][\"1\"].primary != \"$primary\"'" -newprim=$($ETCDCTL get --print-value-only /vitastor/config/pgs | jq -r '.items["1"]["1"].primary') +newprim=$($ETCDCTL get --print-value-only /vitastor/pg/config | jq -r '.items["1"]["1"].primary') if [ "$newprim" = "$primary" ]; then format_error Primary not switched