thebestvibedev/tromcho.net
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Put all configuration to Mon.config

Browse Source
This commit is contained in:
Vitaliy Filippov
2024-06-07 00:20:38 +03:00
parent cf60b6818c
commit 4eabebd245
2 changed files with 48 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mon/etcd_adapter.js
+8 -8
View File
@@ -98,9 +98,9 @@ class EtcdAdapter
const cur_addr = this.pick_next_etcd(); const cur_addr = this.pick_next_etcd();
const base = 'ws'+cur_addr.substr(4); const base = 'ws'+cur_addr.substr(4);
let now = Date.now(); let now = Date.now();
if (tried[base] && now-tried[base] < this.mon.etcd_start_timeout) if (tried[base] && now-tried[base] < this.mon.config.etcd_start_timeout)
{ {
await new Promise(ok => setTimeout(ok, this.mon.etcd_start_timeout-(now-tried[base]))); await new Promise(ok => setTimeout(ok, this.mon.config.etcd_start_timeout-(now-tried[base])));
now = Date.now(); now = Date.now();
} }
tried[base] = now; tried[base] = now;
@@ -155,8 +155,8 @@ class EtcdAdapter
this.ws.on('error', () => this.restart_watcher(cur_addr)); this.ws.on('error', () => this.restart_watcher(cur_addr));
this.ws.send(JSON.stringify({ this.ws.send(JSON.stringify({
create_request: { create_request: {
key: b64(this.mon.etcd_prefix+'/'), key: b64(this.mon.config.etcd_prefix+'/'),
range_end: b64(this.mon.etcd_prefix+'0'), range_end: b64(this.mon.config.etcd_prefix+'0'),
start_revision: ''+this.mon.etcd_watch_revision, start_revision: ''+this.mon.etcd_watch_revision,
watch_id: 1, watch_id: 1,
progress_notify: true, progress_notify: true,
@@ -207,15 +207,15 @@ class EtcdAdapter
while (1) while (1)
{ {
const res = await this.etcd_call('/kv/txn', { const res = await this.etcd_call('/kv/txn', {
compare: [ { target: 'CREATE', create_revision: 0, key: b64(this.mon.etcd_prefix+'/mon/master') } ], compare: [ { target: 'CREATE', create_revision: 0, key: b64(this.mon.config.etcd_prefix+'/mon/master') } ],
success: [ { requestPut: { key: b64(this.mon.etcd_prefix+'/mon/master'), value: b64(JSON.stringify(state)), lease: ''+this.mon.etcd_lease_id } } ], success: [ { requestPut: { key: b64(this.mon.config.etcd_prefix+'/mon/master'), value: b64(JSON.stringify(state)), lease: ''+this.mon.etcd_lease_id } } ],
}, this.mon.etcd_start_timeout, 0); }, this.mon.config.etcd_start_timeout, 0);
if (res.succeeded) if (res.succeeded)
{ {
break; break;
} }
console.log('Waiting to become master'); console.log('Waiting to become master');
await new Promise(ok => setTimeout(ok, this.mon.etcd_start_timeout)); await new Promise(ok => setTimeout(ok, this.mon.config.etcd_start_timeout));
} }
console.log('Became master'); console.log('Became master');
} }
mon/mon.js
+40 -42
View File
@@ -19,19 +19,14 @@ class Mon
{ {
this.failconnect = (e) => this._die(e, 2); this.failconnect = (e) => this._die(e, 2);
this.die = (e) => this._die(e, 1); this.die = (e) => this._die(e, 1);
this.fileConfig = {};
if (fs.existsSync(config.config_path||'/etc/vitastor/vitastor.conf')) if (fs.existsSync(config.config_path||'/etc/vitastor/vitastor.conf'))
{ {
config = { this.fileConfig = JSON.parse(fs.readFileSync(config.config_path||'/etc/vitastor/vitastor.conf', { encoding: 'utf-8' }));
...JSON.parse(fs.readFileSync(config.config_path||'/etc/vitastor/vitastor.conf', { encoding: 'utf-8' })),
...config,
};
} }
this.verbose = config.verbose || 0; this.cliConfig = config;
this.initConfig = config; this.config = { ...this.fileConfig, ...this.cliConfig };
this.config = { ...config }; this.check_config();
this.etcd_prefix = config.etcd_prefix || '/vitastor';
this.etcd_prefix = this.etcd_prefix.replace(/\/\/+/g, '/').replace(/^\/?(.*[^\/])\/?$/, '/$1');
this.etcd_start_timeout = (config.etcd_start_timeout || 5) * 1000;
this.state = JSON.parse(JSON.stringify(etcd_tree)); this.state = JSON.parse(JSON.stringify(etcd_tree));
this.prev_stats = { osd_stats: {}, osd_diff: {} }; this.prev_stats = { osd_stats: {}, osd_diff: {} };
this.signals_set = false; this.signals_set = false;
@@ -65,17 +60,19 @@ class Mon
async load_config() async load_config()
{ {
const res = await this.etcd.etcd_call('/kv/txn', { success: [ const res = await this.etcd.etcd_call('/kv/txn', { success: [
{ requestRange: { key: b64(this.etcd_prefix+'/config/global') } } { requestRange: { key: b64(this.config.etcd_prefix+'/config/global') } }
] }, this.etcd_start_timeout, -1); ] }, this.config.etcd_start_timeout, -1);
if (res.responses[0].response_range.kvs) if (res.responses[0].response_range.kvs)
{ {
this.parse_kv(res.responses[0].response_range.kvs[0]); this.parse_kv(res.responses[0].response_range.kvs[0]);
} }
this.check_config();
} }
check_config() check_config()
{ {
this.config.etcd_prefix = this.config.etcd_prefix || '/vitastor';
this.config.etcd_prefix = this.config.etcd_prefix.replace(/\/\/+/g, '/').replace(/^\/?(.*[^\/])\/?$/, '/$1');
this.config.etcd_start_timeout = (this.config.etcd_start_timeout || 5) * 1000;
this.config.etcd_mon_ttl = Number(this.config.etcd_mon_ttl) || 5; this.config.etcd_mon_ttl = Number(this.config.etcd_mon_ttl) || 5;
if (this.config.etcd_mon_ttl < 1) if (this.config.etcd_mon_ttl < 1)
{ {
@@ -117,7 +114,7 @@ class Mon
on_message(msg) on_message(msg)
{ {
let stats_changed = false, changed = false, pg_states_changed = false; let stats_changed = false, changed = false, pg_states_changed = false;
if (this.verbose) if (this.config.verbose)
{ {
console.log('Revision '+msg.header.revision+' events: '); console.log('Revision '+msg.header.revision+' events: ');
} }
@@ -125,7 +122,7 @@ class Mon
for (const e of msg.events||[]) for (const e of msg.events||[])
{ {
this.parse_kv(e.kv); this.parse_kv(e.kv);
const key = e.kv.key.substr(this.etcd_prefix.length); const key = e.kv.key.substr(this.config.etcd_prefix.length);
if (key.substr(0, 11) == '/osd/state/') if (key.substr(0, 11) == '/osd/state/')
{ {
stats_changed = true; stats_changed = true;
@@ -143,7 +140,7 @@ class Mon
{ {
changed = true; changed = true;
} }
if (this.verbose) if (this.config.verbose)
{ {
console.log(JSON.stringify(e)); console.log(JSON.stringify(e));
} }
@@ -216,10 +213,10 @@ class Mon
this.state.history.last_clean_pgs = new_clean_pgs; this.state.history.last_clean_pgs = new_clean_pgs;
await this.etcd.etcd_call('/kv/txn', { await this.etcd.etcd_call('/kv/txn', {
success: [ { requestPut: { success: [ { requestPut: {
key: b64(this.etcd_prefix+'/history/last_clean_pgs'), key: b64(this.config.etcd_prefix+'/history/last_clean_pgs'),
value: b64(JSON.stringify(this.state.history.last_clean_pgs)) value: b64(JSON.stringify(this.state.history.last_clean_pgs))
} } ], } } ],
}, this.etcd_start_timeout, 0); }, this.config.etcd_start_timeout, 0);
this.save_last_clean_running = false; this.save_last_clean_running = false;
} }
@@ -237,10 +234,10 @@ class Mon
// Register in /mon/member, just for the information // Register in /mon/member, just for the information
const state = this.get_mon_state(); const state = this.get_mon_state();
res = await this.etcd.etcd_call('/kv/put', { res = await this.etcd.etcd_call('/kv/put', {
key: b64(this.etcd_prefix+'/mon/member/'+this.etcd_lease_id), key: b64(this.config.etcd_prefix+'/mon/member/'+this.etcd_lease_id),
value: b64(JSON.stringify(state)), value: b64(JSON.stringify(state)),
lease: ''+this.etcd_lease_id lease: ''+this.etcd_lease_id
}, this.etcd_start_timeout, 0); }, this.config.etcd_start_timeout, 0);
// Set refresh timer // Set refresh timer
this.lease_timer = setInterval(async () => this.lease_timer = setInterval(async () =>
{ {
@@ -268,8 +265,8 @@ class Mon
async load_cluster_state() async load_cluster_state()
{ {
const res = await this.etcd.etcd_call('/kv/txn', { success: [ const res = await this.etcd.etcd_call('/kv/txn', { success: [
{ requestRange: { key: b64(this.etcd_prefix+'/'), range_end: b64(this.etcd_prefix+'0') } }, { requestRange: { key: b64(this.config.etcd_prefix+'/'), range_end: b64(this.config.etcd_prefix+'0') } },
] }, this.etcd_start_timeout, -1); ] }, this.config.etcd_start_timeout, -1);
this.etcd_watch_revision = BigInt(res.header.revision)+BigInt(1); this.etcd_watch_revision = BigInt(res.header.revision)+BigInt(1);
this.state = JSON.parse(JSON.stringify(etcd_tree)); this.state = JSON.parse(JSON.stringify(etcd_tree));
for (const response of res.responses) for (const response of res.responses)
@@ -316,17 +313,17 @@ class Mon
const checks = []; const checks = [];
for (const osd_num of this.all_osds()) for (const osd_num of this.all_osds())
{ {
const key = b64(this.etcd_prefix+'/osd/state/'+osd_num); const key = b64(this.config.etcd_prefix+'/osd/state/'+osd_num);
checks.push({ key, target: 'MOD', result: 'LESS', mod_revision: ''+this.etcd_watch_revision }); checks.push({ key, target: 'MOD', result: 'LESS', mod_revision: ''+this.etcd_watch_revision });
} }
await this.etcd.etcd_call('/kv/txn', { await this.etcd.etcd_call('/kv/txn', {
compare: [ compare: [
{ key: b64(this.etcd_prefix+'/mon/master'), target: 'LEASE', lease: ''+this.etcd_lease_id }, { key: b64(this.config.etcd_prefix+'/mon/master'), target: 'LEASE', lease: ''+this.etcd_lease_id },
{ key: b64(this.etcd_prefix+'/config/pgs'), target: 'MOD', mod_revision: ''+this.etcd_watch_revision, result: 'LESS' }, { key: b64(this.config.etcd_prefix+'/config/pgs'), target: 'MOD', mod_revision: ''+this.etcd_watch_revision, result: 'LESS' },
...checks, ...checks,
], ],
success: [ success: [
{ requestPut: { key: b64(this.etcd_prefix+'/config/pgs'), value: b64(JSON.stringify(new_cfg)) } }, { requestPut: { key: b64(this.config.etcd_prefix+'/config/pgs'), value: b64(JSON.stringify(new_cfg)) } },
], ],
}, this.config.etcd_mon_timeout, 0); }, this.config.etcd_mon_timeout, 0);
return false; return false;
@@ -437,9 +434,9 @@ class Mon
} }
// Also delete pool statistics // Also delete pool statistics
etcd_request.success.push({ requestDeleteRange: { etcd_request.success.push({ requestDeleteRange: {
key: b64(this.etcd_prefix+'/pool/stats/'+pool_id), key: b64(this.config.etcd_prefix+'/pool/stats/'+pool_id),
} }); } });
save_new_pgs_txn(new_config_pgs, etcd_request, this.state, this.etcd_prefix, save_new_pgs_txn(new_config_pgs, etcd_request, this.state, this.config.etcd_prefix,
this.etcd_watch_revision, pool_id, up_osds, osd_tree, prev_pgs, [], []); this.etcd_watch_revision, pool_id, up_osds, osd_tree, prev_pgs, [], []);
} }
} }
@@ -470,8 +467,8 @@ class Mon
pg_history = scale_pg_history(pg_history, real_prev_pgs, pool_res.pgs); pg_history = scale_pg_history(pg_history, real_prev_pgs, pool_res.pgs);
// Drop stats // Drop stats
etcd_request.success.push({ requestDeleteRange: { etcd_request.success.push({ requestDeleteRange: {
key: b64(this.etcd_prefix+'/pg/stats/'+pool_id+'/'), key: b64(this.config.etcd_prefix+'/pg/stats/'+pool_id+'/'),
range_end: b64(this.etcd_prefix+'/pg/stats/'+pool_id+'0'), range_end: b64(this.config.etcd_prefix+'/pg/stats/'+pool_id+'0'),
} }); } });
} }
const stats = { const stats = {
@@ -479,10 +476,10 @@ class Mon
...pool_res.stats, ...pool_res.stats,
}; };
etcd_request.success.push({ requestPut: { etcd_request.success.push({ requestPut: {
key: b64(this.etcd_prefix+'/pool/stats/'+pool_id), key: b64(this.config.etcd_prefix+'/pool/stats/'+pool_id),
value: b64(JSON.stringify(stats)), value: b64(JSON.stringify(stats)),
} }); } });
save_new_pgs_txn(new_config_pgs, etcd_request, this.state, this.etcd_prefix, save_new_pgs_txn(new_config_pgs, etcd_request, this.state, this.config.etcd_prefix,
this.etcd_watch_revision, pool_id, up_osds, osd_tree, real_prev_pgs, pool_res.pgs, pg_history); this.etcd_watch_revision, pool_id, up_osds, osd_tree, real_prev_pgs, pool_res.pgs, pg_history);
} }
new_config_pgs.hash = tree_hash; new_config_pgs.hash = tree_hash;
@@ -492,11 +489,11 @@ class Mon
async save_pg_config(new_config_pgs, etcd_request = { compare: [], success: [] }) async save_pg_config(new_config_pgs, etcd_request = { compare: [], success: [] })
{ {
etcd_request.compare.push( etcd_request.compare.push(
{ key: b64(this.etcd_prefix+'/mon/master'), target: 'LEASE', lease: ''+this.etcd_lease_id }, { key: b64(this.config.etcd_prefix+'/mon/master'), target: 'LEASE', lease: ''+this.etcd_lease_id },
{ key: b64(this.etcd_prefix+'/config/pgs'), target: 'MOD', mod_revision: ''+this.etcd_watch_revision, result: 'LESS' }, { key: b64(this.config.etcd_prefix+'/config/pgs'), target: 'MOD', mod_revision: ''+this.etcd_watch_revision, result: 'LESS' },
); );
etcd_request.success.push( etcd_request.success.push(
{ requestPut: { key: b64(this.etcd_prefix+'/config/pgs'), value: b64(JSON.stringify(new_config_pgs)) } }, { requestPut: { key: b64(this.config.etcd_prefix+'/config/pgs'), value: b64(JSON.stringify(new_config_pgs)) } },
); );
const txn_res = await this.etcd.etcd_call('/kv/txn', etcd_request, this.config.etcd_mon_timeout, 0); const txn_res = await this.etcd.etcd_call('/kv/txn', etcd_request, this.config.etcd_mon_timeout, 0);
return txn_res.succeeded; return txn_res.succeeded;
@@ -553,13 +550,13 @@ class Mon
stats.object_bytes = object_bytes; stats.object_bytes = object_bytes;
stats = serialize_bigints(stats); stats = serialize_bigints(stats);
inode_stats = serialize_bigints(inode_stats); inode_stats = serialize_bigints(inode_stats);
txn.push({ requestPut: { key: b64(this.etcd_prefix+'/stats'), value: b64(JSON.stringify(stats)) } }); txn.push({ requestPut: { key: b64(this.config.etcd_prefix+'/stats'), value: b64(JSON.stringify(stats)) } });
for (const pool_id in inode_stats) for (const pool_id in inode_stats)
{ {
for (const inode_num in inode_stats[pool_id]) for (const inode_num in inode_stats[pool_id])
{ {
txn.push({ requestPut: { txn.push({ requestPut: {
key: b64(this.etcd_prefix+'/inode/stats/'+pool_id+'/'+inode_num), key: b64(this.config.etcd_prefix+'/inode/stats/'+pool_id+'/'+inode_num),
value: b64(JSON.stringify(inode_stats[pool_id][inode_num])), value: b64(JSON.stringify(inode_stats[pool_id][inode_num])),
} }); } });
} }
@@ -571,7 +568,7 @@ class Mon
if (!inode_stats[pool_id] || !inode_stats[pool_id][inode_num]) if (!inode_stats[pool_id] || !inode_stats[pool_id][inode_num])
{ {
txn.push({ requestDeleteRange: { txn.push({ requestDeleteRange: {
key: b64(this.etcd_prefix+'/inode/stats/'+pool_id+'/'+inode_num), key: b64(this.config.etcd_prefix+'/inode/stats/'+pool_id+'/'+inode_num),
} }); } });
} }
} }
@@ -581,7 +578,7 @@ class Mon
if (!seen_pools[pool_id]) if (!seen_pools[pool_id])
{ {
txn.push({ requestDeleteRange: { txn.push({ requestDeleteRange: {
key: b64(this.etcd_prefix+'/pool/stats/'+pool_id), key: b64(this.config.etcd_prefix+'/pool/stats/'+pool_id),
} }); } });
delete this.state.pool.stats[pool_id]; delete this.state.pool.stats[pool_id];
} }
@@ -590,7 +587,7 @@ class Mon
const pool_stats = { ...this.state.pool.stats[pool_id] }; const pool_stats = { ...this.state.pool.stats[pool_id] };
serialize_bigints(pool_stats); serialize_bigints(pool_stats);
txn.push({ requestPut: { txn.push({ requestPut: {
key: b64(this.etcd_prefix+'/pool/stats/'+pool_id), key: b64(this.config.etcd_prefix+'/pool/stats/'+pool_id),
value: b64(JSON.stringify(pool_stats)), value: b64(JSON.stringify(pool_stats)),
} }); } });
} }
@@ -622,7 +619,7 @@ class Mon
} }
kv.key = de64(kv.key); kv.key = de64(kv.key);
kv.value = kv.value ? de64(kv.value) : null; kv.value = kv.value ? de64(kv.value) : null;
let key = kv.key.substr(this.etcd_prefix.length+1); let key = kv.key.substr(this.config.etcd_prefix.length+1);
if (!etcd_allow.exec(key)) if (!etcd_allow.exec(key))
{ {
console.log('Bad key in etcd: '+kv.key+' = '+kv.value); console.log('Bad key in etcd: '+kv.key+' = '+kv.value);
@@ -652,8 +649,9 @@ class Mon
cur[key_parts[key_parts.length-1]] = kv.value; cur[key_parts[key_parts.length-1]] = kv.value;
if (key === 'config/global') if (key === 'config/global')
{ {
this.config = { ...this.initConfig, ...this.state.config.global }; this.config = { ...this.fileConfig, ...this.state.config.global, ...this.cliConfig };
this.check_config(); this.check_config();
this.etcd.parse_config(this.config);
for (const osd_num in this.state.osd.stats) for (const osd_num in this.state.osd.stats)
{ {
// Recheck PGs <osd_out_time> later // Recheck PGs <osd_out_time> later