From a2278be84d47f60071354169bfa58afd8b68ebd4 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sun, 13 Apr 2025 18:46:22 +0300 Subject: [PATCH] Improve data distribution: solve LP task on failure domains instead of individual OSDs This greatly speeds up PG placement and makes it more uniform both because the LP task becomes simpler and because the distribution of individual OSDs is optimised manually --- mon/lp_optimizer/dsl_pgs.js | 8 +- mon/lp_optimizer/fold.js | 244 +++++++++++++++++++++++++ mon/lp_optimizer/lp_optimizer.js | 2 + mon/lp_optimizer/test-optimize-fold.js | 96 ++++++++++ mon/osd_tree.js | 4 +- mon/pg_gen.js | 24 ++- 6 files changed, 364 insertions(+), 14 deletions(-) create mode 100644 mon/lp_optimizer/fold.js create mode 100644 mon/lp_optimizer/test-optimize-fold.js diff --git a/mon/lp_optimizer/dsl_pgs.js b/mon/lp_optimizer/dsl_pgs.js index 14f382fc..f492d61c 100644 --- a/mon/lp_optimizer/dsl_pgs.js +++ b/mon/lp_optimizer/dsl_pgs.js @@ -253,7 +253,7 @@ function random_custom_combinations(osd_tree, rules, count, ordered) for (let i = 1; i < rules.length; i++) { const filtered = filter_tree_by_rules(osd_tree, rules[i], selected); - const idx = select_murmur3(filtered.length, i => 'p:'+f.id+':'+filtered[i].id); + const idx = select_murmur3(filtered.length, i => 'p:'+f.id+':'+(filtered[i].name || filtered[i].id)); selected.push(idx == null ? { levels: {}, id: null } : filtered[idx]); } const size = selected.filter(s => s.id !== null).length; @@ -270,7 +270,7 @@ function random_custom_combinations(osd_tree, rules, count, ordered) for (const item_rules of rules) { const filtered = selected.length ? filter_tree_by_rules(osd_tree, item_rules, selected) : first; - const idx = select_murmur3(filtered.length, i => n+':'+filtered[i].id); + const idx = select_murmur3(filtered.length, i => n+':'+(filtered[i].name || filtered[i].id)); selected.push(idx == null ? { levels: {}, id: null } : filtered[idx]); } const size = selected.filter(s => s.id !== null).length; @@ -340,9 +340,9 @@ function filter_tree_by_rules(osd_tree, rules, selected) } // Convert from -// node_list = { id: string|number, level: string, size?: number, parent?: string|number }[] +// node_list = { id: string|number, name?: string, level: string, size?: number, parent?: string|number }[] // to -// node_tree = { [node_id]: { id, level, size?, parent?, children?: child_node[], levels: { [level]: id, ... } } } +// node_tree = { [node_id]: { id, name?, level, size?, parent?, children?: child_node[], levels: { [level]: id, ... } } } function index_tree(node_list) { const tree = { '': { children: [], levels: {} } }; diff --git a/mon/lp_optimizer/fold.js b/mon/lp_optimizer/fold.js new file mode 100644 index 00000000..9a70c4d3 --- /dev/null +++ b/mon/lp_optimizer/fold.js @@ -0,0 +1,244 @@ +// Copyright (c) Vitaliy Filippov, 2019+ +// License: VNPL-1.1 (see README.md for details) + +// Extract OSDs from the lowest affected tree level into a separate (flat) map +// to run PG optimisation on failure domains instead of individual OSDs +// +// node_list = same input as for index_tree() +// rules = [ level, operator, value ][][] +// returns { nodes: new_node_list, leaves: { new_folded_node_id: [ extracted_leaf_nodes... ] } } +function fold_failure_domains(node_list, rules) +{ + const interest = {}; + for (const level_rules of rules) + { + for (const rule of level_rules) + interest[rule[0]] = true; + } + const max_numeric_id = node_list.reduce((a, c) => a < (0|c.id) ? (0|c.id) : a, 0); + let next_id = max_numeric_id; + const node_map = node_list.reduce((a, c) => { a[c.id||''] = c; return a; }, {}); + const old_ids_by_new = {}; + const extracted_nodes = {}; + let folded = true; + while (folded) + { + const per_parent = {}; + for (const node_id in node_map) + { + const node = node_map[node_id]; + const p = node.parent || ''; + per_parent[p] = per_parent[p]||[]; + per_parent[p].push(node); + } + folded = false; + for (const node_id in per_parent) + { + const fold_node = per_parent[node_id].filter(child => per_parent[child.id||''] || interest[child.level]).length == 0; + if (fold_node) + { + const old_node = node_map[node_id]; + const new_id = ++next_id; + node_map[new_id] = { + ...old_node, + id: new_id, + name: node_id, // for use in murmur3 hashes + size: per_parent[node_id].reduce((a, c) => a + (Number(c.size)||0), 0), + }; + delete node_map[node_id]; + old_ids_by_new[new_id] = node_id; + extracted_nodes[new_id] = []; + for (const child of per_parent[node_id]) + { + if (old_ids_by_new[child.id]) + { + extracted_nodes[new_id].push(...extracted_nodes[child.id]); + delete extracted_nodes[child.id]; + } + else + extracted_nodes[new_id].push(child); + delete node_map[child.id]; + } + folded = true; + } + } + } + return { nodes: Object.values(node_map), leaves: extracted_nodes }; +} + +// Distribute PGs mapped to "folded" nodes to individual OSDs according to their weights +// folded_pgs = optimize_result.int_pgs before folding +// prev_pgs = optional previous PGs from optimize_change() input +// extracted_nodes = output from fold_failure_domains +function unfold_failure_domains(folded_pgs, prev_pgs, extracted_nodes) +{ + const maps = {}; + let found = false; + for (const new_id in extracted_nodes) + { + const weights = {}; + for (const sub_node of extracted_nodes[new_id]) + { + weights[sub_node.id] = sub_node.size; + } + maps[new_id] = { weights, prev: [], next: [], pos: 0 }; + found = true; + } + if (!found) + { + return folded_pgs; + } + for (let i = 0; i < folded_pgs.length; i++) + { + for (let j = 0; j < folded_pgs[i].length; j++) + { + if (maps[folded_pgs[i][j]]) + { + maps[folded_pgs[i][j]].prev.push(prev_pgs && prev_pgs[i] && prev_pgs[i][j] || 0); + } + } + } + for (const new_id in maps) + { + maps[new_id].next = adjust_distribution(maps[new_id].weights, maps[new_id].prev); + } + const mapped_pgs = []; + for (let i = 0; i < folded_pgs.length; i++) + { + mapped_pgs.push(folded_pgs[i].map(osd => (maps[osd] ? maps[osd].next[maps[osd].pos++] : osd))); + } + return mapped_pgs; +} + +// Return the new array of items re-distributed as close as possible to weights in wanted_weights +// wanted_weights = { [key]: weight } +// cur_items = key[] +function adjust_distribution(wanted_weights, cur_items) +{ + const item_map = {}; + for (let i = 0; i < cur_items.length; i++) + { + const item = cur_items[i]; + item_map[item] = (item_map[item] || { target: 0, cur: [] }); + item_map[item].cur.push(i); + } + let total_weight = 0; + for (const item in wanted_weights) + { + total_weight += Number(wanted_weights[item]) || 0; + } + for (const item in wanted_weights) + { + const weight = wanted_weights[item] / total_weight * cur_items.length; + if (weight > 0) + { + item_map[item] = (item_map[item] || { target: 0, cur: [] }); + item_map[item].target = weight; + } + } + const diff = (item) => (item_map[item].cur.length - item_map[item].target); + const most_underweighted = Object.keys(item_map) + .filter(item => item_map[item].target > 0) + .sort((a, b) => diff(a) - diff(b)); + // Items with zero target weight MUST never be selected - remove them + // and remap each of them to a most underweighted item + for (const item in item_map) + { + if (!item_map[item].target) + { + const prev = item_map[item]; + delete item_map[item]; + for (const idx of prev.cur) + { + const move_to = most_underweighted[0]; + item_map[move_to].cur.push(idx); + move_leftmost(most_underweighted, diff); + } + } + } + // Other over-weighted items are only moved if it improves the distribution + while (most_underweighted.length > 1) + { + const first = most_underweighted[0]; + const last = most_underweighted[most_underweighted.length-1]; + const first_diff = diff(first); + const last_diff = diff(last); + if (Math.abs(first_diff+1)+Math.abs(last_diff-1) < Math.abs(first_diff)+Math.abs(last_diff)) + { + item_map[first].cur.push(item_map[last].cur.pop()); + move_leftmost(most_underweighted, diff); + move_rightmost(most_underweighted, diff); + } + else + { + break; + } + } + const new_items = new Array(cur_items.length); + for (const item in item_map) + { + for (const idx of item_map[item].cur) + { + new_items[idx] = item; + } + } + return new_items; +} + +function move_leftmost(sorted_array, diff) +{ + // Re-sort by moving the leftmost item to the right if it changes position + const first = sorted_array[0]; + const new_diff = diff(first); + let r = 0; + while (r < sorted_array.length-1 && diff(sorted_array[r+1]) <= new_diff) + r++; + if (r > 0) + { + for (let i = 0; i < r; i++) + sorted_array[i] = sorted_array[i+1]; + sorted_array[r] = first; + } +} + +function move_rightmost(sorted_array, diff) +{ + // Re-sort by moving the rightmost item to the left if it changes position + const last = sorted_array[sorted_array.length-1]; + const new_diff = diff(last); + let r = sorted_array.length-1; + while (r > 0 && diff(sorted_array[r-1]) > new_diff) + r--; + if (r < sorted_array.length-1) + { + for (let i = sorted_array.length-1; i > r; i--) + sorted_array[i] = sorted_array[i-1]; + sorted_array[r] = last; + } +} + +// map previous PGs to folded nodes +function fold_prev_pgs(pgs, extracted_nodes) +{ + const unmap = {}; + for (const new_id in extracted_nodes) + { + for (const sub_node of extracted_nodes[new_id]) + { + unmap[sub_node.id] = new_id; + } + } + const mapped_pgs = []; + for (let i = 0; i < pgs.length; i++) + { + mapped_pgs.push(pgs[i].map(osd => (unmap[osd] || osd))); + } + return mapped_pgs; +} + +module.exports = { + fold_failure_domains, + unfold_failure_domains, + adjust_distribution, + fold_prev_pgs, +}; diff --git a/mon/lp_optimizer/lp_optimizer.js b/mon/lp_optimizer/lp_optimizer.js index f26441df..dbb0cf30 100644 --- a/mon/lp_optimizer/lp_optimizer.js +++ b/mon/lp_optimizer/lp_optimizer.js @@ -98,6 +98,7 @@ async function optimize_initial({ osd_weights, combinator, pg_count, pg_size = 3 score: lp_result.score, weights: lp_result.vars, int_pgs, + pg_effsize, space: eff * pg_effsize, total_space: total_weight, }; @@ -409,6 +410,7 @@ async function optimize_change({ prev_pgs: prev_int_pgs, osd_weights, combinator int_pgs: new_pgs, differs, osd_differs, + pg_effsize, space: pg_effsize * pg_list_space_efficiency(new_pgs, osd_weights, pg_minsize, parity_space), total_space: total_weight, }; diff --git a/mon/lp_optimizer/test-optimize-fold.js b/mon/lp_optimizer/test-optimize-fold.js new file mode 100644 index 00000000..aac5a3d2 --- /dev/null +++ b/mon/lp_optimizer/test-optimize-fold.js @@ -0,0 +1,96 @@ +// Copyright (c) Vitaliy Filippov, 2019+ +// License: VNPL-1.1 (see README.md for details) + +const assert = require('assert'); +const { fold_failure_domains, unfold_failure_domains, adjust_distribution } = require('./fold.js'); +const DSL = require('./dsl_pgs.js'); +const LPOptimizer = require('./lp_optimizer.js'); +const stableStringify = require('../stable-stringify.js'); + +async function run() +{ + // Test run adjust_distribution + console.log('adjust_distribution'); + const rand = []; + for (let i = 0; i < 100; i++) + { + rand.push(1 + Math.floor(10*Math.random())); + // or rand.push(0); + } + const adj = adjust_distribution({ 1: 1, 2: 1, 3: 1, 4: 1, 5: 1, 6: 1, 7: 1, 8: 1, 9: 1, 10: 1 }, rand); + //console.log(rand.join(' ')); + console.log(rand.reduce((a, c) => { a[c] = (a[c]||0)+1; return a; }, {})); + //console.log(adj.join(' ')); + console.log(adj.reduce((a, c) => { a[c] = (a[c]||0)+1; return a; }, {})); + console.log('Movement: '+rand.reduce((a, c, i) => a+(rand[i] != adj[i] ? 1 : 0), 0)+'/'+rand.length); + + console.log('\nfold_failure_domains'); + console.log(JSON.stringify(fold_failure_domains( + [ + { id: 1, level: 'osd', size: 1, parent: 'disk1' }, + { id: 2, level: 'osd', size: 2, parent: 'disk1' }, + { id: 'disk1', level: 'disk', parent: 'host1' }, + { id: 'host1', level: 'host', parent: 'dc1' }, + { id: 'dc1', level: 'dc' }, + ], + [ [ [ 'dc' ], [ 'host' ] ] ] + ), 0, 2)); + + console.log('\noptimize_folded'); + // 5 DCs, 2 hosts per DC, 10 OSD per host + const nodes = []; + for (let i = 1; i <= 100; i++) + { + nodes.push({ id: i, level: 'osd', size: 1, parent: 'host'+(1+(0|((i-1)/10))) }); + } + for (let i = 1; i <= 10; i++) + { + nodes.push({ id: 'host'+i, level: 'host', parent: 'dc'+(1+(0|((i-1)/2))) }); + } + for (let i = 1; i <= 5; i++) + { + nodes.push({ id: 'dc'+i, level: 'dc' }); + } + + // Check rules + const rules = DSL.parse_level_indexes({ dc: '112233', host: '123456' }, [ 'dc', 'host', 'osd' ]); + assert.deepEqual(rules, [[],[["dc","=",1],["host","!=",[1]]],[["dc","!=",[1]]],[["dc","=",3],["host","!=",[3]]],[["dc","!=",[1,3]]],[["dc","=",5],["host","!=",[5]]]]); + + // Check tree folding + const { nodes: folded_nodes, leaves: folded_leaves } = fold_failure_domains(nodes, rules); + const expected_folded = []; + const expected_leaves = {}; + for (let i = 1; i <= 10; i++) + { + expected_folded.push({ id: 100+i, name: 'host'+i, level: 'host', size: 10, parent: 'dc'+(1+(0|((i-1)/2))) }); + expected_leaves[100+i] = [ ...new Array(10).keys() ].map(k => ({ id: 10*(i-1)+k+1, level: 'osd', size: 1, parent: 'host'+i })); + } + for (let i = 1; i <= 5; i++) + { + expected_folded.push({ id: 'dc'+i, level: 'dc' }); + } + assert.equal(stableStringify(folded_nodes), stableStringify(expected_folded)); + assert.equal(stableStringify(folded_leaves), stableStringify(expected_leaves)); + + // Now optimise it + console.log('1000 PGs, EC 112233'); + const leaf_weights = folded_nodes.reduce((a, c) => { if (Number(c.id)) { a[c.id] = c.size; } return a; }, {}); + let res = await LPOptimizer.optimize_initial({ + osd_weights: leaf_weights, + combinator: new DSL.RuleCombinator(folded_nodes, rules, 10000, false), + pg_size: 6, + pg_count: 1000, + ordered: false, + }); + LPOptimizer.print_change_stats(res, false); + assert.equal(res.space, 100, 'Initial distribution'); + + const unfolded_res = { ...res }; + unfolded_res.int_pgs = unfold_failure_domains(res.int_pgs, null, folded_leaves); + const osd_weights = nodes.reduce((a, c) => { if (Number(c.id)) { a[c.id] = c.size; } return a; }, {}); + unfolded_res.space = unfolded_res.pg_effsize * LPOptimizer.pg_list_space_efficiency(unfolded_res.int_pgs, osd_weights, 0, 1); + LPOptimizer.print_change_stats(unfolded_res, false); + assert.equal(res.space, 100, 'Initial distribution'); +} + +run().catch(console.error); diff --git a/mon/osd_tree.js b/mon/osd_tree.js index d4f7e610..a0fa9b4e 100644 --- a/mon/osd_tree.js +++ b/mon/osd_tree.js @@ -87,7 +87,7 @@ function make_hier_tree(global_config, tree) tree[''] = { children: [] }; for (const node_id in tree) { - if (node_id === '' || tree[node_id].level === 'osd' && (!tree[node_id].size || tree[node_id].size <= 0)) + if (node_id === '' || !(tree[node_id].children||[]).length && (tree[node_id].size||0) <= 0) { continue; } @@ -107,7 +107,7 @@ function make_hier_tree(global_config, tree) deleted = 0; for (const node_id in tree) { - if (tree[node_id].level !== 'osd' && (!tree[node_id].children || !tree[node_id].children.length)) + if (!(tree[node_id].children||[]).length && (tree[node_id].size||0) <= 0) { const parent = tree[node_id].parent; if (parent) diff --git a/mon/pg_gen.js b/mon/pg_gen.js index cabdf7e0..25814f6b 100644 --- a/mon/pg_gen.js +++ b/mon/pg_gen.js @@ -3,6 +3,7 @@ const { RuleCombinator } = require('./lp_optimizer/dsl_pgs.js'); const { SimpleCombinator, flatten_tree } = require('./lp_optimizer/simple_pgs.js'); +const { fold_failure_domains, unfold_failure_domains, fold_prev_pgs } = require('./lp_optimizer/fold.js'); const { validate_pool_cfg, get_pg_rules } = require('./pool_config.js'); const LPOptimizer = require('./lp_optimizer/lp_optimizer.js'); const { scale_pg_count } = require('./pg_utils.js'); @@ -160,7 +161,6 @@ async function generate_pool_pgs(state, global_config, pool_id, osd_tree, levels pool_cfg.bitmap_granularity || global_config.bitmap_granularity || 4096, pool_cfg.immediate_commit || global_config.immediate_commit || 'all' ); - pool_tree = make_hier_tree(global_config, pool_tree); // First try last_clean_pgs to minimize data movement let prev_pgs = []; for (const pg in ((state.history.last_clean_pgs.items||{})[pool_id]||{})) @@ -175,14 +175,19 @@ async function generate_pool_pgs(state, global_config, pool_id, osd_tree, levels prev_pgs[pg-1] = [ ...state.pg.config.items[pool_id][pg].osd_set ]; } } + const use_rules = !global_config.use_old_pg_combinator || pool_cfg.level_placement || pool_cfg.raw_placement; + const rules = use_rules ? get_pg_rules(pool_id, pool_cfg, global_config.placement_levels) : null; + const folded = fold_failure_domains(Object.values(pool_tree), use_rules ? rules : [ [ [ pool_cfg.failure_domain ] ] ]); + // FIXME: Remove/merge make_hier_tree() step somewhere, however it's needed to remove empty nodes + const folded_tree = make_hier_tree(global_config, folded.nodes); const old_pg_count = prev_pgs.length; const optimize_cfg = { - osd_weights: Object.values(pool_tree).filter(item => item.level === 'osd').reduce((a, c) => { a[c.id] = c.size; return a; }, {}), - combinator: !global_config.use_old_pg_combinator || pool_cfg.level_placement || pool_cfg.raw_placement + osd_weights: folded.nodes.reduce((a, c) => { if (Number(c.id)) { a[c.id] = c.size; } return a; }, {}), + combinator: use_rules // new algorithm: - ? new RuleCombinator(pool_tree, get_pg_rules(pool_id, pool_cfg, global_config.placement_levels), pool_cfg.max_osd_combinations) + ? new RuleCombinator(folded_tree, rules, pool_cfg.max_osd_combinations) // old algorithm: - : new SimpleCombinator(flatten_tree(pool_tree[''].children, levels, pool_cfg.failure_domain, 'osd'), pool_cfg.pg_size, pool_cfg.max_osd_combinations), + : new SimpleCombinator(flatten_tree(folded_tree[''].children, levels, pool_cfg.failure_domain, 'osd'), pool_cfg.pg_size, pool_cfg.max_osd_combinations), pg_count: pool_cfg.pg_count, pg_size: pool_cfg.pg_size, pg_minsize: pool_cfg.pg_minsize, @@ -202,12 +207,11 @@ async function generate_pool_pgs(state, global_config, pool_id, osd_tree, levels for (const pg of prev_pgs) { while (pg.length < pool_cfg.pg_size) - { pg.push(0); - } } + const folded_prev_pgs = fold_prev_pgs(prev_pgs, folded.leaves); optimize_result = await LPOptimizer.optimize_change({ - prev_pgs, + prev_pgs: folded_prev_pgs, ...optimize_cfg, }); } @@ -215,6 +219,10 @@ async function generate_pool_pgs(state, global_config, pool_id, osd_tree, levels { optimize_result = await LPOptimizer.optimize_initial(optimize_cfg); } + optimize_result.int_pgs = unfold_failure_domains(optimize_result.int_pgs, prev_pgs, folded.leaves); + const osd_weights = Object.values(pool_tree).reduce((a, c) => { if (c.level === 'osd') { a[c.id] = c.size; } return a; }, {}); + optimize_result.space = optimize_result.pg_effsize * LPOptimizer.pg_list_space_efficiency(optimize_result.int_pgs, + osd_weights, optimize_cfg.pg_minsize, 1); console.log(`Pool ${pool_id} (${pool_cfg.name || 'unnamed'}):`); LPOptimizer.print_change_stats(optimize_result); let pg_effsize = pool_cfg.pg_size;