From 0daa8ea39bd3b32cc724215d56f8bcf67b03fcc1 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Fri, 19 Jul 2024 14:19:07 +0300 Subject: [PATCH] Support seamless upgrade to new PG config and stats etcd key names --- mon/mon.js | 75 ++++++++++++++++++++++++++++++++++++++++++++++------ mon/stats.js | 17 +++++++++--- 2 files changed, 80 insertions(+), 12 deletions(-) diff --git a/mon/mon.js b/mon/mon.js index 97bb2552..53365062 100644 --- a/mon/mon.js +++ b/mon/mon.js @@ -75,6 +75,8 @@ class Mon this.prev_stats = { osd_stats: {}, osd_diff: {} }; this.recheck_pgs_active = false; this.watcher_active = false; + this.old_pg_config = false; + this.old_pg_stats_seen = false; } async start() @@ -205,6 +207,11 @@ class Mon { stats_changed = true; } + else if (key.substr(0, 10) == '/pg/stats/') + { + this.old_pg_stats_seen = true; + stats_changed = true; + } else if (key.substr(0, 10) == '/pg/state/') { pg_states_changed = true; @@ -396,6 +403,50 @@ class Mon this.parse_kv(kv); } } + if (Object.keys((this.state.config.pgs||{}).items||{}).length) + { + // Support seamless upgrade to new OSDs + if (!Object.keys((this.state.pg.config||{}).items||{}).length) + { + const pgs = JSON.stringify(this.state.config.pgs); + this.state.pg.config = JSON.parse(pgs); + const res = await this.etcd.etcd_call('/kv/txn', { + success: [ + { requestPut: { key: b64(this.config.etcd_prefix+'/pg/config'), value: b64(pgs) } }, + ], + compare: [ + { key: b64(this.config.etcd_prefix+'/pg/config'), target: 'MOD', mod_revision: ''+this.etcd_watch_revision, result: 'LESS' }, + ], + }, this.config.etcd_mon_timeout, this.config.etcd_mon_retries); + if (!res.succeeded) + throw new Error('Failed to duplicate old PG config to new PG config'); + } + this.old_pg_config = true; + this.old_pg_config_timer = setInterval(() => this.check_clear_old_config().catch(console.error), + this.config.old_pg_config_clear_interval||3600000); + } + } + + async check_clear_old_config() + { + if (this.old_pg_config && this.old_pg_stats_seen) + { + this.old_pg_stats_seen = false; + return; + } + if (this.old_pg_config) + { + await this.etcd.etcd_call('/kv/txn', { success: [ + { requestDeleteRange: { key: b64(this.config.etcd_prefix+'/config/pgs') } }, + { requestDeleteRange: { key: b64(this.config.etcd_prefix+'/pg/stats/'), range_end: b64(this.config.etcd_prefix+'/pg/stats0') } }, + ] }, this.config.etcd_mon_timeout, this.config.etcd_mon_retries); + this.old_pg_config = false; + } + if (this.old_pg_config_timer) + { + clearInterval(this.old_pg_config_timer); + this.old_pg_config_timer = null; + } } all_osds() @@ -430,22 +481,26 @@ class Mon // Check that no OSDs change their state before we pause PGs // Doing this we make sure that OSDs don't wake up in the middle of our "transaction" // and can't see the old PG configuration - const checks = []; + const checks = [ + { key: b64(this.config.etcd_prefix+'/mon/master'), target: 'LEASE', lease: ''+this.etcd_lease_id }, + { key: b64(this.config.etcd_prefix+'/pg/config'), target: 'MOD', mod_revision: ''+this.etcd_watch_revision, result: 'LESS' }, + ]; for (const osd_num of this.all_osds()) { const key = b64(this.config.etcd_prefix+'/osd/state/'+osd_num); checks.push({ key, target: 'MOD', result: 'LESS', mod_revision: ''+this.etcd_watch_revision }); } - await this.etcd.etcd_call('/kv/txn', { - compare: [ - { key: b64(this.config.etcd_prefix+'/mon/master'), target: 'LEASE', lease: ''+this.etcd_lease_id }, - { key: b64(this.config.etcd_prefix+'/pg/config'), target: 'MOD', mod_revision: ''+this.etcd_watch_revision, result: 'LESS' }, - ...checks, - ], + const txn = { + compare: checks, success: [ { requestPut: { key: b64(this.config.etcd_prefix+'/pg/config'), value: b64(JSON.stringify(new_cfg)) } }, ], - }, this.config.etcd_mon_timeout, 0); + }; + if (this.old_pg_config) + { + txn.success.push({ requestPut: { key: b64(this.config.etcd_prefix+'/config/pgs'), value: b64(JSON.stringify(new_cfg)) } }); + } + await this.etcd.etcd_call('/kv/txn', txn, this.config.etcd_mon_timeout, 0); return false; } return !has_online; @@ -619,6 +674,10 @@ class Mon etcd_request.success.push( { requestPut: { key: b64(this.config.etcd_prefix+'/pg/config'), value: b64(JSON.stringify(new_pg_config)) } }, ); + if (this.old_pg_config) + { + etcd_request.success.push({ requestPut: { key: b64(this.config.etcd_prefix+'/config/pgs'), value: b64(JSON.stringify(new_pg_config)) } }); + } const txn_res = await this.etcd.etcd_call('/kv/txn', etcd_request, this.config.etcd_mon_timeout, 0); return txn_res.succeeded; } diff --git a/mon/stats.js b/mon/stats.js index c63e10bc..28222995 100644 --- a/mon/stats.js +++ b/mon/stats.js @@ -100,10 +100,19 @@ function sum_object_counts(state, global_config) { const object_counts = { object: 0n, clean: 0n, misplaced: 0n, degraded: 0n, incomplete: 0n }; const object_bytes = { object: 0n, clean: 0n, misplaced: 0n, degraded: 0n, incomplete: 0n }; - for (const pool_id in state.pgstats) + let pgstats = state.pgstats; + if (state.pg.stats) + { + // Merge with old stats for seamless transition to new stats + for (const pool_id in state.pg.stats) + { + pgstats[pool_id] = { ...(state.pg.stats[pool_id] || {}), ...(pgstats[pool_id] || {}) }; + } + } + for (const pool_id in pgstats) { let object_size = 0; - for (const osd_num of state.pgstats[pool_id].write_osd_set||[]) + for (const osd_num of pgstats[pool_id].write_osd_set||[]) { if (osd_num && state.osd.stats[osd_num] && state.osd.stats[osd_num].block_size) { @@ -121,9 +130,9 @@ function sum_object_counts(state, global_config) object_size *= ((pool_cfg.pg_size||0) - (pool_cfg.parity_chunks||0)); } object_size = BigInt(object_size); - for (const pg_num in state.pgstats[pool_id]) + for (const pg_num in pgstats[pool_id]) { - const st = state.pgstats[pool_id][pg_num]; + const st = pgstats[pool_id][pg_num]; if (st) { for (const k in object_counts)