diff --git a/mon/lp_optimizer/dsl_pgs.js b/mon/lp_optimizer/dsl_pgs.js index 14f382fc..f492d61c 100644 --- a/mon/lp_optimizer/dsl_pgs.js +++ b/mon/lp_optimizer/dsl_pgs.js @@ -253,7 +253,7 @@ function random_custom_combinations(osd_tree, rules, count, ordered) for (let i = 1; i < rules.length; i++) { const filtered = filter_tree_by_rules(osd_tree, rules[i], selected); - const idx = select_murmur3(filtered.length, i => 'p:'+f.id+':'+filtered[i].id); + const idx = select_murmur3(filtered.length, i => 'p:'+f.id+':'+(filtered[i].name || filtered[i].id)); selected.push(idx == null ? { levels: {}, id: null } : filtered[idx]); } const size = selected.filter(s => s.id !== null).length; @@ -270,7 +270,7 @@ function random_custom_combinations(osd_tree, rules, count, ordered) for (const item_rules of rules) { const filtered = selected.length ? filter_tree_by_rules(osd_tree, item_rules, selected) : first; - const idx = select_murmur3(filtered.length, i => n+':'+filtered[i].id); + const idx = select_murmur3(filtered.length, i => n+':'+(filtered[i].name || filtered[i].id)); selected.push(idx == null ? { levels: {}, id: null } : filtered[idx]); } const size = selected.filter(s => s.id !== null).length; @@ -340,9 +340,9 @@ function filter_tree_by_rules(osd_tree, rules, selected) } // Convert from -// node_list = { id: string|number, level: string, size?: number, parent?: string|number }[] +// node_list = { id: string|number, name?: string, level: string, size?: number, parent?: string|number }[] // to -// node_tree = { [node_id]: { id, level, size?, parent?, children?: child_node[], levels: { [level]: id, ... } } } +// node_tree = { [node_id]: { id, name?, level, size?, parent?, children?: child_node[], levels: { [level]: id, ... } } } function index_tree(node_list) { const tree = { '': { children: [], levels: {} } }; diff --git a/mon/lp_optimizer/fold.js b/mon/lp_optimizer/fold.js new file mode 100644 index 00000000..9a70c4d3 --- /dev/null +++ b/mon/lp_optimizer/fold.js @@ -0,0 +1,244 @@ +// Copyright (c) Vitaliy Filippov, 2019+ +// License: VNPL-1.1 (see README.md for details) + +// Extract OSDs from the lowest affected tree level into a separate (flat) map +// to run PG optimisation on failure domains instead of individual OSDs +// +// node_list = same input as for index_tree() +// rules = [ level, operator, value ][][] +// returns { nodes: new_node_list, leaves: { new_folded_node_id: [ extracted_leaf_nodes... ] } } +function fold_failure_domains(node_list, rules) +{ + const interest = {}; + for (const level_rules of rules) + { + for (const rule of level_rules) + interest[rule[0]] = true; + } + const max_numeric_id = node_list.reduce((a, c) => a < (0|c.id) ? (0|c.id) : a, 0); + let next_id = max_numeric_id; + const node_map = node_list.reduce((a, c) => { a[c.id||''] = c; return a; }, {}); + const old_ids_by_new = {}; + const extracted_nodes = {}; + let folded = true; + while (folded) + { + const per_parent = {}; + for (const node_id in node_map) + { + const node = node_map[node_id]; + const p = node.parent || ''; + per_parent[p] = per_parent[p]||[]; + per_parent[p].push(node); + } + folded = false; + for (const node_id in per_parent) + { + const fold_node = per_parent[node_id].filter(child => per_parent[child.id||''] || interest[child.level]).length == 0; + if (fold_node) + { + const old_node = node_map[node_id]; + const new_id = ++next_id; + node_map[new_id] = { + ...old_node, + id: new_id, + name: node_id, // for use in murmur3 hashes + size: per_parent[node_id].reduce((a, c) => a + (Number(c.size)||0), 0), + }; + delete node_map[node_id]; + old_ids_by_new[new_id] = node_id; + extracted_nodes[new_id] = []; + for (const child of per_parent[node_id]) + { + if (old_ids_by_new[child.id]) + { + extracted_nodes[new_id].push(...extracted_nodes[child.id]); + delete extracted_nodes[child.id]; + } + else + extracted_nodes[new_id].push(child); + delete node_map[child.id]; + } + folded = true; + } + } + } + return { nodes: Object.values(node_map), leaves: extracted_nodes }; +} + +// Distribute PGs mapped to "folded" nodes to individual OSDs according to their weights +// folded_pgs = optimize_result.int_pgs before folding +// prev_pgs = optional previous PGs from optimize_change() input +// extracted_nodes = output from fold_failure_domains +function unfold_failure_domains(folded_pgs, prev_pgs, extracted_nodes) +{ + const maps = {}; + let found = false; + for (const new_id in extracted_nodes) + { + const weights = {}; + for (const sub_node of extracted_nodes[new_id]) + { + weights[sub_node.id] = sub_node.size; + } + maps[new_id] = { weights, prev: [], next: [], pos: 0 }; + found = true; + } + if (!found) + { + return folded_pgs; + } + for (let i = 0; i < folded_pgs.length; i++) + { + for (let j = 0; j < folded_pgs[i].length; j++) + { + if (maps[folded_pgs[i][j]]) + { + maps[folded_pgs[i][j]].prev.push(prev_pgs && prev_pgs[i] && prev_pgs[i][j] || 0); + } + } + } + for (const new_id in maps) + { + maps[new_id].next = adjust_distribution(maps[new_id].weights, maps[new_id].prev); + } + const mapped_pgs = []; + for (let i = 0; i < folded_pgs.length; i++) + { + mapped_pgs.push(folded_pgs[i].map(osd => (maps[osd] ? maps[osd].next[maps[osd].pos++] : osd))); + } + return mapped_pgs; +} + +// Return the new array of items re-distributed as close as possible to weights in wanted_weights +// wanted_weights = { [key]: weight } +// cur_items = key[] +function adjust_distribution(wanted_weights, cur_items) +{ + const item_map = {}; + for (let i = 0; i < cur_items.length; i++) + { + const item = cur_items[i]; + item_map[item] = (item_map[item] || { target: 0, cur: [] }); + item_map[item].cur.push(i); + } + let total_weight = 0; + for (const item in wanted_weights) + { + total_weight += Number(wanted_weights[item]) || 0; + } + for (const item in wanted_weights) + { + const weight = wanted_weights[item] / total_weight * cur_items.length; + if (weight > 0) + { + item_map[item] = (item_map[item] || { target: 0, cur: [] }); + item_map[item].target = weight; + } + } + const diff = (item) => (item_map[item].cur.length - item_map[item].target); + const most_underweighted = Object.keys(item_map) + .filter(item => item_map[item].target > 0) + .sort((a, b) => diff(a) - diff(b)); + // Items with zero target weight MUST never be selected - remove them + // and remap each of them to a most underweighted item + for (const item in item_map) + { + if (!item_map[item].target) + { + const prev = item_map[item]; + delete item_map[item]; + for (const idx of prev.cur) + { + const move_to = most_underweighted[0]; + item_map[move_to].cur.push(idx); + move_leftmost(most_underweighted, diff); + } + } + } + // Other over-weighted items are only moved if it improves the distribution + while (most_underweighted.length > 1) + { + const first = most_underweighted[0]; + const last = most_underweighted[most_underweighted.length-1]; + const first_diff = diff(first); + const last_diff = diff(last); + if (Math.abs(first_diff+1)+Math.abs(last_diff-1) < Math.abs(first_diff)+Math.abs(last_diff)) + { + item_map[first].cur.push(item_map[last].cur.pop()); + move_leftmost(most_underweighted, diff); + move_rightmost(most_underweighted, diff); + } + else + { + break; + } + } + const new_items = new Array(cur_items.length); + for (const item in item_map) + { + for (const idx of item_map[item].cur) + { + new_items[idx] = item; + } + } + return new_items; +} + +function move_leftmost(sorted_array, diff) +{ + // Re-sort by moving the leftmost item to the right if it changes position + const first = sorted_array[0]; + const new_diff = diff(first); + let r = 0; + while (r < sorted_array.length-1 && diff(sorted_array[r+1]) <= new_diff) + r++; + if (r > 0) + { + for (let i = 0; i < r; i++) + sorted_array[i] = sorted_array[i+1]; + sorted_array[r] = first; + } +} + +function move_rightmost(sorted_array, diff) +{ + // Re-sort by moving the rightmost item to the left if it changes position + const last = sorted_array[sorted_array.length-1]; + const new_diff = diff(last); + let r = sorted_array.length-1; + while (r > 0 && diff(sorted_array[r-1]) > new_diff) + r--; + if (r < sorted_array.length-1) + { + for (let i = sorted_array.length-1; i > r; i--) + sorted_array[i] = sorted_array[i-1]; + sorted_array[r] = last; + } +} + +// map previous PGs to folded nodes +function fold_prev_pgs(pgs, extracted_nodes) +{ + const unmap = {}; + for (const new_id in extracted_nodes) + { + for (const sub_node of extracted_nodes[new_id]) + { + unmap[sub_node.id] = new_id; + } + } + const mapped_pgs = []; + for (let i = 0; i < pgs.length; i++) + { + mapped_pgs.push(pgs[i].map(osd => (unmap[osd] || osd))); + } + return mapped_pgs; +} + +module.exports = { + fold_failure_domains, + unfold_failure_domains, + adjust_distribution, + fold_prev_pgs, +}; diff --git a/mon/lp_optimizer/lp_optimizer.js b/mon/lp_optimizer/lp_optimizer.js index f26441df..dbb0cf30 100644 --- a/mon/lp_optimizer/lp_optimizer.js +++ b/mon/lp_optimizer/lp_optimizer.js @@ -98,6 +98,7 @@ async function optimize_initial({ osd_weights, combinator, pg_count, pg_size = 3 score: lp_result.score, weights: lp_result.vars, int_pgs, + pg_effsize, space: eff * pg_effsize, total_space: total_weight, }; @@ -409,6 +410,7 @@ async function optimize_change({ prev_pgs: prev_int_pgs, osd_weights, combinator int_pgs: new_pgs, differs, osd_differs, + pg_effsize, space: pg_effsize * pg_list_space_efficiency(new_pgs, osd_weights, pg_minsize, parity_space), total_space: total_weight, }; diff --git a/mon/lp_optimizer/test-optimize-fold.js b/mon/lp_optimizer/test-optimize-fold.js new file mode 100644 index 00000000..aac5a3d2 --- /dev/null +++ b/mon/lp_optimizer/test-optimize-fold.js @@ -0,0 +1,96 @@ +// Copyright (c) Vitaliy Filippov, 2019+ +// License: VNPL-1.1 (see README.md for details) + +const assert = require('assert'); +const { fold_failure_domains, unfold_failure_domains, adjust_distribution } = require('./fold.js'); +const DSL = require('./dsl_pgs.js'); +const LPOptimizer = require('./lp_optimizer.js'); +const stableStringify = require('../stable-stringify.js'); + +async function run() +{ + // Test run adjust_distribution + console.log('adjust_distribution'); + const rand = []; + for (let i = 0; i < 100; i++) + { + rand.push(1 + Math.floor(10*Math.random())); + // or rand.push(0); + } + const adj = adjust_distribution({ 1: 1, 2: 1, 3: 1, 4: 1, 5: 1, 6: 1, 7: 1, 8: 1, 9: 1, 10: 1 }, rand); + //console.log(rand.join(' ')); + console.log(rand.reduce((a, c) => { a[c] = (a[c]||0)+1; return a; }, {})); + //console.log(adj.join(' ')); + console.log(adj.reduce((a, c) => { a[c] = (a[c]||0)+1; return a; }, {})); + console.log('Movement: '+rand.reduce((a, c, i) => a+(rand[i] != adj[i] ? 1 : 0), 0)+'/'+rand.length); + + console.log('\nfold_failure_domains'); + console.log(JSON.stringify(fold_failure_domains( + [ + { id: 1, level: 'osd', size: 1, parent: 'disk1' }, + { id: 2, level: 'osd', size: 2, parent: 'disk1' }, + { id: 'disk1', level: 'disk', parent: 'host1' }, + { id: 'host1', level: 'host', parent: 'dc1' }, + { id: 'dc1', level: 'dc' }, + ], + [ [ [ 'dc' ], [ 'host' ] ] ] + ), 0, 2)); + + console.log('\noptimize_folded'); + // 5 DCs, 2 hosts per DC, 10 OSD per host + const nodes = []; + for (let i = 1; i <= 100; i++) + { + nodes.push({ id: i, level: 'osd', size: 1, parent: 'host'+(1+(0|((i-1)/10))) }); + } + for (let i = 1; i <= 10; i++) + { + nodes.push({ id: 'host'+i, level: 'host', parent: 'dc'+(1+(0|((i-1)/2))) }); + } + for (let i = 1; i <= 5; i++) + { + nodes.push({ id: 'dc'+i, level: 'dc' }); + } + + // Check rules + const rules = DSL.parse_level_indexes({ dc: '112233', host: '123456' }, [ 'dc', 'host', 'osd' ]); + assert.deepEqual(rules, [[],[["dc","=",1],["host","!=",[1]]],[["dc","!=",[1]]],[["dc","=",3],["host","!=",[3]]],[["dc","!=",[1,3]]],[["dc","=",5],["host","!=",[5]]]]); + + // Check tree folding + const { nodes: folded_nodes, leaves: folded_leaves } = fold_failure_domains(nodes, rules); + const expected_folded = []; + const expected_leaves = {}; + for (let i = 1; i <= 10; i++) + { + expected_folded.push({ id: 100+i, name: 'host'+i, level: 'host', size: 10, parent: 'dc'+(1+(0|((i-1)/2))) }); + expected_leaves[100+i] = [ ...new Array(10).keys() ].map(k => ({ id: 10*(i-1)+k+1, level: 'osd', size: 1, parent: 'host'+i })); + } + for (let i = 1; i <= 5; i++) + { + expected_folded.push({ id: 'dc'+i, level: 'dc' }); + } + assert.equal(stableStringify(folded_nodes), stableStringify(expected_folded)); + assert.equal(stableStringify(folded_leaves), stableStringify(expected_leaves)); + + // Now optimise it + console.log('1000 PGs, EC 112233'); + const leaf_weights = folded_nodes.reduce((a, c) => { if (Number(c.id)) { a[c.id] = c.size; } return a; }, {}); + let res = await LPOptimizer.optimize_initial({ + osd_weights: leaf_weights, + combinator: new DSL.RuleCombinator(folded_nodes, rules, 10000, false), + pg_size: 6, + pg_count: 1000, + ordered: false, + }); + LPOptimizer.print_change_stats(res, false); + assert.equal(res.space, 100, 'Initial distribution'); + + const unfolded_res = { ...res }; + unfolded_res.int_pgs = unfold_failure_domains(res.int_pgs, null, folded_leaves); + const osd_weights = nodes.reduce((a, c) => { if (Number(c.id)) { a[c.id] = c.size; } return a; }, {}); + unfolded_res.space = unfolded_res.pg_effsize * LPOptimizer.pg_list_space_efficiency(unfolded_res.int_pgs, osd_weights, 0, 1); + LPOptimizer.print_change_stats(unfolded_res, false); + assert.equal(res.space, 100, 'Initial distribution'); +} + +run().catch(console.error); diff --git a/mon/osd_tree.js b/mon/osd_tree.js index d4f7e610..a0fa9b4e 100644 --- a/mon/osd_tree.js +++ b/mon/osd_tree.js @@ -87,7 +87,7 @@ function make_hier_tree(global_config, tree) tree[''] = { children: [] }; for (const node_id in tree) { - if (node_id === '' || tree[node_id].level === 'osd' && (!tree[node_id].size || tree[node_id].size <= 0)) + if (node_id === '' || !(tree[node_id].children||[]).length && (tree[node_id].size||0) <= 0) { continue; } @@ -107,7 +107,7 @@ function make_hier_tree(global_config, tree) deleted = 0; for (const node_id in tree) { - if (tree[node_id].level !== 'osd' && (!tree[node_id].children || !tree[node_id].children.length)) + if (!(tree[node_id].children||[]).length && (tree[node_id].size||0) <= 0) { const parent = tree[node_id].parent; if (parent) diff --git a/mon/pg_gen.js b/mon/pg_gen.js index cabdf7e0..25814f6b 100644 --- a/mon/pg_gen.js +++ b/mon/pg_gen.js @@ -3,6 +3,7 @@ const { RuleCombinator } = require('./lp_optimizer/dsl_pgs.js'); const { SimpleCombinator, flatten_tree } = require('./lp_optimizer/simple_pgs.js'); +const { fold_failure_domains, unfold_failure_domains, fold_prev_pgs } = require('./lp_optimizer/fold.js'); const { validate_pool_cfg, get_pg_rules } = require('./pool_config.js'); const LPOptimizer = require('./lp_optimizer/lp_optimizer.js'); const { scale_pg_count } = require('./pg_utils.js'); @@ -160,7 +161,6 @@ async function generate_pool_pgs(state, global_config, pool_id, osd_tree, levels pool_cfg.bitmap_granularity || global_config.bitmap_granularity || 4096, pool_cfg.immediate_commit || global_config.immediate_commit || 'all' ); - pool_tree = make_hier_tree(global_config, pool_tree); // First try last_clean_pgs to minimize data movement let prev_pgs = []; for (const pg in ((state.history.last_clean_pgs.items||{})[pool_id]||{})) @@ -175,14 +175,19 @@ async function generate_pool_pgs(state, global_config, pool_id, osd_tree, levels prev_pgs[pg-1] = [ ...state.pg.config.items[pool_id][pg].osd_set ]; } } + const use_rules = !global_config.use_old_pg_combinator || pool_cfg.level_placement || pool_cfg.raw_placement; + const rules = use_rules ? get_pg_rules(pool_id, pool_cfg, global_config.placement_levels) : null; + const folded = fold_failure_domains(Object.values(pool_tree), use_rules ? rules : [ [ [ pool_cfg.failure_domain ] ] ]); + // FIXME: Remove/merge make_hier_tree() step somewhere, however it's needed to remove empty nodes + const folded_tree = make_hier_tree(global_config, folded.nodes); const old_pg_count = prev_pgs.length; const optimize_cfg = { - osd_weights: Object.values(pool_tree).filter(item => item.level === 'osd').reduce((a, c) => { a[c.id] = c.size; return a; }, {}), - combinator: !global_config.use_old_pg_combinator || pool_cfg.level_placement || pool_cfg.raw_placement + osd_weights: folded.nodes.reduce((a, c) => { if (Number(c.id)) { a[c.id] = c.size; } return a; }, {}), + combinator: use_rules // new algorithm: - ? new RuleCombinator(pool_tree, get_pg_rules(pool_id, pool_cfg, global_config.placement_levels), pool_cfg.max_osd_combinations) + ? new RuleCombinator(folded_tree, rules, pool_cfg.max_osd_combinations) // old algorithm: - : new SimpleCombinator(flatten_tree(pool_tree[''].children, levels, pool_cfg.failure_domain, 'osd'), pool_cfg.pg_size, pool_cfg.max_osd_combinations), + : new SimpleCombinator(flatten_tree(folded_tree[''].children, levels, pool_cfg.failure_domain, 'osd'), pool_cfg.pg_size, pool_cfg.max_osd_combinations), pg_count: pool_cfg.pg_count, pg_size: pool_cfg.pg_size, pg_minsize: pool_cfg.pg_minsize, @@ -202,12 +207,11 @@ async function generate_pool_pgs(state, global_config, pool_id, osd_tree, levels for (const pg of prev_pgs) { while (pg.length < pool_cfg.pg_size) - { pg.push(0); - } } + const folded_prev_pgs = fold_prev_pgs(prev_pgs, folded.leaves); optimize_result = await LPOptimizer.optimize_change({ - prev_pgs, + prev_pgs: folded_prev_pgs, ...optimize_cfg, }); } @@ -215,6 +219,10 @@ async function generate_pool_pgs(state, global_config, pool_id, osd_tree, levels { optimize_result = await LPOptimizer.optimize_initial(optimize_cfg); } + optimize_result.int_pgs = unfold_failure_domains(optimize_result.int_pgs, prev_pgs, folded.leaves); + const osd_weights = Object.values(pool_tree).reduce((a, c) => { if (c.level === 'osd') { a[c.id] = c.size; } return a; }, {}); + optimize_result.space = optimize_result.pg_effsize * LPOptimizer.pg_list_space_efficiency(optimize_result.int_pgs, + osd_weights, optimize_cfg.pg_minsize, 1); console.log(`Pool ${pool_id} (${pool_cfg.name || 'unnamed'}):`); LPOptimizer.print_change_stats(optimize_result); let pg_effsize = pool_cfg.pg_size;