Move monitor to mon/

mon/PGUtil.js

@@ -0,0 +1,109 @@
+module.exports = {
+    scale_pg_count,
+};
+
+function scale_pg_count(prev_pgs, prev_pg_history, new_pg_history, new_pg_count)
+{
+    const old_pg_count = prev_pgs.length;
+    // Add all possibly intersecting PGs into the history of new PGs
+    if (!(new_pg_count % old_pg_count))
+    {
+        // New PG count is a multiple of the old PG count
+        const mul = (new_pg_count / old_pg_count);
+        for (let i = 0; i < new_pg_count; i++)
+        {
+            const old_i = Math.floor(new_pg_count / mul);
+            new_pg_history[i] = JSON.parse(JSON.stringify(prev_pg_history[1+old_i]));
+        }
+    }
+    else if (!(old_pg_count % new_pg_count))
+    {
+        // Old PG count is a multiple of the new PG count
+        const mul = (old_pg_count / new_pg_count);
+        for (let i = 0; i < new_pg_count; i++)
+        {
+            new_pg_history[i] = {
+                osd_sets: [],
+                all_peers: [],
+                epoch: 0,
+            };
+            for (let j = 0; j < mul; j++)
+            {
+                new_pg_history[i].osd_sets.push(prev_pgs[i*mul]);
+                const hist = prev_pg_history[1+i*mul+j];
+                if (hist && hist.osd_sets && hist.osd_sets.length)
+                {
+                    Array.prototype.push.apply(new_pg_history[i].osd_sets, hist.osd_sets);
+                }
+                if (hist && hist.all_peers && hist.all_peers.length)
+                {
+                    Array.prototype.push.apply(new_pg_history[i].all_peers, hist.all_peers);
+                }
+                if (hist && hist.epoch)
+                {
+                    new_pg_history[i].epoch = new_pg_history[i].epoch < hist.epoch ? hist.epoch : new_pg_history[i].epoch;
+                }
+            }
+        }
+    }
+    else
+    {
+        // Any PG may intersect with any PG after non-multiple PG count change
+        // So, merge ALL PGs history
+        let all_sets = {};
+        let all_peers = {};
+        let max_epoch = 0;
+        for (const pg of prev_pgs)
+        {
+            all_sets[pg.join(' ')] = pg;
+        }
+        for (const pg in prev_pg_history)
+        {
+            const hist = prev_pg_history[pg];
+            if (hist && hist.osd_sets)
+            {
+                for (const pg of hist.osd_sets)
+                {
+                    all_sets[pg.join(' ')] = pg;
+                }
+            }
+            if (hist && hist.all_peers)
+            {
+                for (const osd_num of hist.all_peers)
+                {
+                    all_peers[osd_num] = Number(osd_num);
+                }
+            }
+            if (hist && hist.epoch)
+            {
+                max_epoch = max_epoch < hist.epoch ? hist.epoch : max_epoch;
+            }
+        }
+        all_sets = Object.values(all_sets);
+        all_peers = Object.values(all_peers);
+        for (let i = 0; i < new_pg_count; i++)
+        {
+            new_pg_history[i] = { osd_sets: all_sets, all_peers, epoch: max_epoch };
+        }
+    }
+    // Mark history keys for removed PGs as removed
+    for (let i = new_pg_count; i < old_pg_count; i++)
+    {
+        new_pg_history[i] = null;
+    }
+    if (old_pg_count < new_pg_count)
+    {
+        for (let i = new_pg_count-1; i >= 0; i--)
+        {
+            prev_pgs[i] = prev_pgs[Math.floor(i/new_pg_count*old_pg_count)];
+        }
+    }
+    else if (old_pg_count > new_pg_count)
+    {
+        for (let i = 0; i < new_pg_count; i++)
+        {
+            prev_pgs[i] = prev_pgs[Math.round(i/new_pg_count*old_pg_count)];
+        }
+        prev_pgs.splice(new_pg_count, old_pg_count-new_pg_count);
+    }
+}

mon/lp-optimizer.js

@@ -0,0 +1,685 @@
+// Data distribution optimizer using linear programming (lp_solve)
+
+const child_process = require('child_process');
+
+const NO_OSD = 'Z';
+
+async function lp_solve(text)
+{
+    const cp = child_process.spawn('lp_solve');
+    let stdout = '', stderr = '', finish_cb;
+    cp.stdout.on('data', buf => stdout += buf.toString());
+    cp.stderr.on('data', buf => stderr += buf.toString());
+    cp.on('exit', () => finish_cb && finish_cb());
+    cp.stdin.write(text);
+    cp.stdin.end();
+    if (cp.exitCode == null)
+    {
+        await new Promise(ok => finish_cb = ok);
+    }
+    if (!stdout.trim())
+    {
+        return null;
+    }
+    let score = 0;
+    let vars = {};
+    for (const line of stdout.split(/\n/))
+    {
+        let m = /^(^Value of objective function: (-?[\d\.]+)|Actual values of the variables:)\s*$/.exec(line);
+        if (m)
+        {
+            if (m[2])
+            {
+                score = m[2];
+            }
+            continue;
+        }
+        else if (/This problem is (infeasible|unbounded)/.exec(line))
+        {
+            return null;
+        }
+        let [ k, v ] = line.trim().split(/\s+/, 2);
+        if (v)
+        {
+            vars[k] = v;
+        }
+    }
+    return { score, vars };
+}
+
+async function optimize_initial({ osd_tree, pg_count, pg_size = 3, pg_minsize = 2, max_combinations = 10000, parity_space = 1 })
+{
+    if (!pg_count || !osd_tree)
+    {
+        return null;
+    }
+    const all_weights = Object.assign({}, ...Object.values(osd_tree));
+    const total_weight = Object.values(all_weights).reduce((a, c) => Number(a) + Number(c), 0);
+    const all_pgs = Object.values(random_combinations(osd_tree, pg_size, max_combinations));
+    const pg_per_osd = {};
+    for (const pg of all_pgs)
+    {
+        for (let i = 0; i < pg.length; i++)
+        {
+            const osd = pg[i];
+            pg_per_osd[osd] = pg_per_osd[osd] || [];
+            pg_per_osd[osd].push((i >= pg_minsize ? parity_space+'*' : '')+"pg_"+pg.join("_"));
+        }
+    }
+    const pg_effsize = Math.min(pg_minsize, Object.keys(osd_tree).length)
+        + Math.max(0, Math.min(pg_size, Object.keys(osd_tree).length) - pg_minsize) * parity_space;
+    let lp = '';
+    lp += "max: "+all_pgs.map(pg => 'pg_'+pg.join('_')).join(' + ')+";\n";
+    for (const osd in pg_per_osd)
+    {
+        if (osd !== NO_OSD)
+        {
+            let osd_pg_count = all_weights[osd]/total_weight*pg_effsize*pg_count;
+            lp += pg_per_osd[osd].join(' + ')+' <= '+osd_pg_count+';\n';
+        }
+    }
+    for (const pg of all_pgs)
+    {
+        lp += 'pg_'+pg.join('_')+" >= 0;\n";
+    }
+    lp += "sec "+all_pgs.map(pg => 'pg_'+pg.join('_')).join(', ')+";\n";
+    const lp_result = await lp_solve(lp);
+    if (!lp_result)
+    {
+        console.log(lp);
+        throw new Error('Problem is infeasible or unbounded - is it a bug?');
+    }
+    const int_pgs = make_int_pgs(lp_result.vars, pg_count);
+    const eff = pg_list_space_efficiency(int_pgs, all_weights, pg_minsize, parity_space);
+    const res = {
+        score: lp_result.score,
+        weights: lp_result.vars,
+        int_pgs,
+        space: eff * pg_effsize,
+        total_space: total_weight,
+    };
+    return res;
+}
+
+function make_int_pgs(weights, pg_count)
+{
+    const total_weight = Object.values(weights).reduce((a, c) => Number(a) + Number(c), 0);
+    let int_pgs = [];
+    let pg_left = pg_count;
+    let weight_left = total_weight;
+    for (const pg_name in weights)
+    {
+        let n = Math.round(weights[pg_name] / weight_left * pg_left);
+        for (let i = 0; i < n; i++)
+        {
+            int_pgs.push(pg_name.substr(3).split('_'));
+        }
+        weight_left -= weights[pg_name];
+        pg_left -= n;
+    }
+    return int_pgs;
+}
+
+function calc_intersect_weights(pg_size, pg_count, prev_weights, all_pgs)
+{
+    const move_weights = {};
+    if ((1 << pg_size) < pg_count)
+    {
+        const intersect = {};
+        for (const pg_name in prev_weights)
+        {
+            const pg = pg_name.substr(3).split(/_/);
+            for (let omit = 1; omit < (1 << pg_size); omit++)
+            {
+                let pg_omit = [ ...pg ];
+                let intersect_count = pg_size;
+                for (let i = 0; i < pg_size; i++)
+                {
+                    if (omit & (1 << i))
+                    {
+                        pg_omit[i] = '';
+                        intersect_count--;
+                    }
+                }
+                pg_omit = pg_omit.join(':');
+                intersect[pg_omit] = Math.max(intersect[pg_omit] || 0, intersect_count);
+            }
+        }
+        for (const pg of all_pgs)
+        {
+            let max_int = 0;
+            for (let omit = 1; omit < (1 << pg_size); omit++)
+            {
+                let pg_omit = [ ...pg ];
+                for (let i = 0; i < pg_size; i++)
+                {
+                    if (omit & (1 << i))
+                    {
+                        pg_omit[i] = '';
+                    }
+                }
+                pg_omit = pg_omit.join(':');
+                max_int = Math.max(max_int, intersect[pg_omit] || 0);
+            }
+            move_weights['pg_'+pg.join('_')] = pg_size-max_int;
+        }
+    }
+    else
+    {
+        const prev_pg_hashed = Object.keys(prev_weights).map(pg_name => pg_name.substr(3).split(/_/).reduce((a, c) => { a[c] = 1; return a; }, {}));
+        for (const pg of all_pgs)
+        {
+            if (!prev_weights['pg_'+pg.join('_')])
+            {
+                let max_int = 0;
+                for (const prev_hash in prev_pg_hashed)
+                {
+                    const intersect_count = pg.reduce((a, osd) => a + (prev_hash[osd] ? 1 : 0), 0);
+                    if (max_int < intersect_count)
+                    {
+                        max_int = intersect_count;
+                        if (max_int >= pg_size)
+                        {
+                            break;
+                        }
+                    }
+                }
+                move_weights['pg_'+pg.join('_')] = pg_size-max_int;
+            }
+        }
+    }
+    return move_weights;
+}
+
+function add_valid_previous(osd_tree, prev_weights, all_pgs)
+{
+    // Add previous combinations that are still valid
+    const hosts = Object.keys(osd_tree).sort();
+    const host_per_osd = {};
+    for (const host in osd_tree)
+    {
+        for (const osd in osd_tree[host])
+        {
+            host_per_osd[osd] = host;
+        }
+    }
+    skip_pg: for (const pg_name in prev_weights)
+    {
+        const seen_hosts = {};
+        const pg = pg_name.substr(3).split(/_/);
+        for (const osd of pg)
+        {
+            if (!host_per_osd[osd] || seen_hosts[host_per_osd[osd]])
+            {
+                continue skip_pg;
+            }
+            seen_hosts[host_per_osd[osd]] = true;
+        }
+        if (!all_pgs[pg_name])
+        {
+            all_pgs[pg_name] = pg;
+        }
+    }
+}
+
+// Try to minimize data movement
+async function optimize_change({ prev_pgs: prev_int_pgs, osd_tree, pg_size = 3, pg_minsize = 2, max_combinations = 10000, parity_space = 1 })
+{
+    if (!osd_tree)
+    {
+        return null;
+    }
+    const pg_effsize = Math.min(pg_minsize, Object.keys(osd_tree).length)
+        + Math.max(0, Math.min(pg_size, Object.keys(osd_tree).length) - pg_minsize) * parity_space;
+    const pg_count = prev_int_pgs.length;
+    const prev_weights = {};
+    const prev_pg_per_osd = {};
+    for (const pg of prev_int_pgs)
+    {
+        const pg_name = 'pg_'+pg.join('_');
+        prev_weights[pg_name] = (prev_weights[pg_name]||0) + 1;
+        for (let i = 0; i < pg.length; i++)
+        {
+            const osd = pg[i];
+            prev_pg_per_osd[osd] = prev_pg_per_osd[osd] || [];
+            prev_pg_per_osd[osd].push([ pg_name, (i >= pg_minsize ? parity_space : 1) ]);
+        }
+    }
+    // Get all combinations
+    let all_pgs = random_combinations(osd_tree, pg_size, max_combinations);
+    add_valid_previous(osd_tree, prev_weights, all_pgs);
+    all_pgs = Object.values(all_pgs);
+    const pg_per_osd = {};
+    for (const pg of all_pgs)
+    {
+        const pg_name = 'pg_'+pg.join('_');
+        for (let i = 0; i < pg.length; i++)
+        {
+            const osd = pg[i];
+            pg_per_osd[osd] = pg_per_osd[osd] || [];
+            pg_per_osd[osd].push([ pg_name, (i >= pg_minsize ? parity_space : 1) ]);
+        }
+    }
+    // Penalize PGs based on their similarity to old PGs
+    const move_weights = calc_intersect_weights(pg_size, pg_count, prev_weights, all_pgs);
+    // Calculate total weight - old PG weights
+    const all_pg_names = all_pgs.map(pg => 'pg_'+pg.join('_'));
+    const all_pgs_hash = all_pg_names.reduce((a, c) => { a[c] = true; return a; }, {});
+    const all_weights = Object.assign({}, ...Object.values(osd_tree));
+    const total_weight = Object.values(all_weights).reduce((a, c) => Number(a) + Number(c), 0);
+    // Generate the LP problem
+    let lp = '';
+    lp += 'max: '+all_pg_names.map(pg_name => (
+        prev_weights[pg_name] ? `${pg_size+1}*add_${pg_name} - ${pg_size+1}*del_${pg_name}` : `${pg_size+1-move_weights[pg_name]}*${pg_name}`
+    )).join(' + ')+';\n';
+    for (const osd in pg_per_osd)
+    {
+        if (osd !== NO_OSD)
+        {
+            const osd_sum = (pg_per_osd[osd]||[]).map(([ pg_name, space ]) => (
+                prev_weights[pg_name] ? `${space} * add_${pg_name} - ${space} * del_${pg_name}` : `${space} * ${pg_name}`
+            )).join(' + ');
+            const rm_osd_pg_count = (prev_pg_per_osd[osd]||[])
+                .reduce((a, [ old_pg_name, space ]) => (a + (all_pgs_hash[old_pg_name] ? space : 0)), 0);
+            const osd_pg_count = all_weights[osd]*pg_effsize/total_weight*pg_count - rm_osd_pg_count;
+            lp += osd_sum + ' <= ' + osd_pg_count + ';\n';
+        }
+    }
+    let pg_vars = [];
+    for (const pg_name of all_pg_names)
+    {
+        if (prev_weights[pg_name])
+        {
+            pg_vars.push(`add_${pg_name}`, `del_${pg_name}`);
+            // Can't add or remove less than zero
+            lp += `add_${pg_name} >= 0;\n`;
+            lp += `del_${pg_name} >= 0;\n`;
+            // Can't remove more than the PG already has
+            lp += `add_${pg_name} - del_${pg_name} >= -${prev_weights[pg_name]};\n`;
+        }
+        else
+        {
+            pg_vars.push(pg_name);
+            lp += `${pg_name} >= 0;\n`;
+        }
+    }
+    lp += 'sec '+pg_vars.join(', ')+';\n';
+    // Solve it
+    const lp_result = await lp_solve(lp);
+    if (!lp_result)
+    {
+        console.log(lp);
+        throw new Error('Problem is infeasible or unbounded - is it a bug?');
+    }
+    // Generate the new distribution
+    const weights = { ...prev_weights };
+    for (const k in prev_weights)
+    {
+        if (!all_pgs_hash[k])
+        {
+            delete weights[k];
+        }
+    }
+    for (const k in lp_result.vars)
+    {
+        if (k.substr(0, 4) === 'add_')
+        {
+            weights[k.substr(4)] = (weights[k.substr(4)] || 0) + Number(lp_result.vars[k]);
+        }
+        else if (k.substr(0, 4) === 'del_')
+        {
+            weights[k.substr(4)] = (weights[k.substr(4)] || 0) - Number(lp_result.vars[k]);
+        }
+        else if (k.substr(0, 3) === 'pg_')
+        {
+            weights[k] = Number(lp_result.vars[k]);
+        }
+    }
+    for (const k in weights)
+    {
+        if (!weights[k])
+        {
+            delete weights[k];
+        }
+    }
+    const int_pgs = make_int_pgs(weights, pg_count);
+    // Align them with most similar previous PGs
+    const new_pgs = align_pgs(prev_int_pgs, int_pgs);
+    let differs = 0, osd_differs = 0;
+    for (let i = 0; i < pg_count; i++)
+    {
+        if (new_pgs[i].join('_') != prev_int_pgs[i].join('_'))
+        {
+            differs++;
+        }
+        for (let j = 0; j < pg_size; j++)
+        {
+            if (new_pgs[i][j] != prev_int_pgs[i][j])
+            {
+                osd_differs++;
+            }
+        }
+    }
+    return {
+        prev_pgs: prev_int_pgs,
+        score: lp_result.score,
+        weights,
+        int_pgs: new_pgs,
+        differs,
+        osd_differs,
+        space: pg_effsize * pg_list_space_efficiency(new_pgs, all_weights, pg_minsize, parity_space),
+        total_space: total_weight,
+    };
+}
+
+function print_change_stats(retval, detailed)
+{
+    const new_pgs = retval.int_pgs;
+    const prev_int_pgs = retval.prev_pgs;
+    if (prev_int_pgs)
+    {
+        if (detailed)
+        {
+            for (let i = 0; i < new_pgs.length; i++)
+            {
+                if (new_pgs[i].join('_') != prev_int_pgs[i].join('_'))
+                {
+                    console.log("pg "+i+": "+prev_int_pgs[i].join(' ')+" -> "+new_pgs[i].join(' '));
+                }
+            }
+        }
+        console.log(
+            "Data movement: "+retval.differs+" pgs, "+
+            retval.osd_differs+" pg*osds = "+Math.round(retval.osd_differs / prev_int_pgs.length / 3 * 10000)/100+" %"
+        );
+    }
+    console.log(
+        "Total space (raw): "+Math.round(retval.space*100)/100+" TB, space efficiency: "+
+        Math.round(retval.space/(retval.total_space||1)*10000)/100+" %"
+    );
+}
+
+function align_pgs(prev_int_pgs, int_pgs)
+{
+    const aligned_pgs = [];
+    put_aligned_pgs(aligned_pgs, int_pgs, prev_int_pgs, (pg) => [ pg.join(':') ]);
+    put_aligned_pgs(aligned_pgs, int_pgs, prev_int_pgs, (pg) => [ pg[0]+'::'+pg[2], ':'+pg[1]+':'+pg[2], pg[0]+':'+pg[1]+':' ]);
+    put_aligned_pgs(aligned_pgs, int_pgs, prev_int_pgs, (pg) => [ pg[0]+'::', ':'+pg[1]+':', '::'+pg[2] ]);
+    const free_slots = prev_int_pgs.map((pg, i) => !aligned_pgs[i] ? i : null).filter(i => i != null);
+    for (const pg of int_pgs)
+    {
+        if (!free_slots.length)
+        {
+            throw new Error("Can't place unaligned PG");
+        }
+        aligned_pgs[free_slots.shift()] = pg;
+    }
+    return aligned_pgs;
+}
+
+function put_aligned_pgs(aligned_pgs, int_pgs, prev_int_pgs, keygen)
+{
+    let prev_indexes = {};
+    for (let i = 0; i < prev_int_pgs.length; i++)
+    {
+        for (let k of keygen(prev_int_pgs[i]))
+        {
+            prev_indexes[k] = prev_indexes[k] || [];
+            prev_indexes[k].push(i);
+        }
+    }
+    PG: for (let i = int_pgs.length-1; i >= 0; i--)
+    {
+        let pg = int_pgs[i];
+        let keys = keygen(int_pgs[i]);
+        for (let k of keys)
+        {
+            while (prev_indexes[k] && prev_indexes[k].length)
+            {
+                let idx = prev_indexes[k].shift();
+                if (!aligned_pgs[idx])
+                {
+                    aligned_pgs[idx] = pg;
+                    int_pgs.splice(i, 1);
+                    continue PG;
+                }
+            }
+        }
+    }
+}
+
+// Convert multi-level osd_tree = { level: number|string, id?: string, size?: number, children?: osd_tree }[]
+// levels = { string: number }
+// to a two-level osd_tree suitable for all_combinations()
+function flatten_tree(osd_tree, levels, failure_domain_level, osd_level, domains = {}, i = { i: 1 })
+{
+    osd_level = levels[osd_level] || osd_level;
+    failure_domain_level = levels[failure_domain_level] || failure_domain_level;
+    for (const node of osd_tree)
+    {
+        if ((levels[node.level] || node.level) < failure_domain_level)
+        {
+            flatten_tree(node.children||[], levels, failure_domain_level, osd_level, domains, i);
+        }
+        else
+        {
+            domains['dom'+(i.i++)] = extract_osds([ node ], levels, osd_level);
+        }
+    }
+    return domains;
+}
+
+function extract_osds(osd_tree, levels, osd_level, osds = {})
+{
+    for (const node of osd_tree)
+    {
+        if ((levels[node.level] || node.level) >= osd_level)
+        {
+            osds[node.id] = node.size;
+        }
+        else
+        {
+            extract_osds(node.children||[], levels, osd_level, osds);
+        }
+    }
+    return osds;
+}
+
+function random_combinations(osd_tree, pg_size, count)
+{
+    let seed = 0x5f020e43;
+    let rng = () =>
+    {
+        seed ^= seed << 13;
+        seed ^= seed >> 17;
+        seed ^= seed << 5;
+        return seed + 2147483648;
+    };
+    const hosts = Object.keys(osd_tree).sort();
+    const osds = Object.keys(osd_tree).reduce((a, c) => { a[c] = Object.keys(osd_tree[c]).sort(); return a; }, {});
+    const r = {};
+    // Generate random combinations including each OSD at least once
+    for (let h = 0; h < hosts.length; h++)
+    {
+        for (let o = 0; o < osds[hosts[h]].length; o++)
+        {
+            const pg = [ osds[hosts[h]][o] ];
+            const cur_hosts = [ ...hosts ];
+            cur_hosts.splice(h, 1);
+            for (let i = 1; i < pg_size && i < hosts.length; i++)
+            {
+                const next_host = rng() % cur_hosts.length;
+                const next_osd = rng() % osds[cur_hosts[next_host]].length;
+                pg.push(osds[cur_hosts[next_host]][next_osd]);
+                cur_hosts.splice(next_host, 1);
+            }
+            while (pg.length < pg_size)
+            {
+                pg.push(NO_OSD);
+            }
+            r['pg_'+pg.join('_')] = pg;
+        }
+    }
+    // Generate purely random combinations
+    restart: while (count > 0)
+    {
+        let host_idx = [];
+        for (let i = 0; i < pg_size && i < hosts.length; i++)
+        {
+            let start = i > 0 ? host_idx[i-1]+1 : 0;
+            if (start >= hosts.length)
+            {
+                continue restart;
+            }
+            host_idx[i] = start + rng() % (hosts.length-start);
+        }
+        let pg = host_idx.map(h => osds[hosts[h]][rng() % osds[hosts[h]].length]);
+        while (pg.length < pg_size)
+        {
+            pg.push(NO_OSD);
+        }
+        r['pg_'+pg.join('_')] = pg;
+        count--;
+    }
+    return r;
+}
+
+// Super-stupid algorithm. Given the current OSD tree, generate all possible OSD combinations
+// osd_tree = { failure_domain1: { osd1: size1, ... }, ... }
+// ordered = return combinations without duplicates having different order
+function all_combinations(osd_tree, pg_size, ordered, count)
+{
+    const hosts = Object.keys(osd_tree).sort();
+    const osds = Object.keys(osd_tree).reduce((a, c) => { a[c] = Object.keys(osd_tree[c]).sort(); return a; }, {});
+    while (hosts.length < pg_size)
+    {
+        osds[NO_OSD] = [ NO_OSD ];
+        hosts.push(NO_OSD);
+    }
+    let host_idx = [];
+    let osd_idx = [];
+    for (let i = 0; i < pg_size; i++)
+    {
+        host_idx.push(i);
+        osd_idx.push(0);
+    }
+    const r = [];
+    while (!count || count < 0 || r.length < count)
+    {
+        r.push(host_idx.map((hi, i) => osds[hosts[hi]][osd_idx[i]]));
+        let inc = pg_size-1;
+        while (inc >= 0)
+        {
+            osd_idx[inc]++;
+            if (osd_idx[inc] >= osds[hosts[host_idx[inc]]].length)
+            {
+                osd_idx[inc] = 0;
+                inc--;
+            }
+            else
+            {
+                break;
+            }
+        }
+        if (inc < 0)
+        {
+            // no osds left in the current host combination, select the next one
+            inc = pg_size-1;
+            same_again: while (inc >= 0)
+            {
+                host_idx[inc]++;
+                for (let prev_host = 0; prev_host < inc; prev_host++)
+                {
+                    if (host_idx[prev_host] == host_idx[inc])
+                    {
+                        continue same_again;
+                    }
+                }
+                if (host_idx[inc] < (ordered ? hosts.length-(pg_size-1-inc) : hosts.length))
+                {
+                    while ((++inc) < pg_size)
+                    {
+                        host_idx[inc] = (ordered ? host_idx[inc-1]+1 : 0);
+                    }
+                    break;
+                }
+                else
+                {
+                    inc--;
+                }
+            }
+            if (inc < 0)
+            {
+                break;
+            }
+        }
+    }
+    return r;
+}
+
+function pg_weights_space_efficiency(weights, pg_count, osd_sizes)
+{
+    const per_osd = {};
+    for (const pg_name in weights)
+    {
+        for (const osd of pg_name.substr(3).split(/_/))
+        {
+            per_osd[osd] = (per_osd[osd]||0) + weights[pg_name];
+        }
+    }
+    return pg_per_osd_space_efficiency(per_osd, pg_count, osd_sizes);
+}
+
+function pg_list_space_efficiency(pgs, osd_sizes, pg_minsize, parity_space)
+{
+    const per_osd = {};
+    for (const pg of pgs)
+    {
+        for (let i = 0; i < pg.length; i++)
+        {
+            const osd = pg[i];
+            per_osd[osd] = (per_osd[osd]||0) + (i >= pg_minsize ? (parity_space||1) : 1);
+        }
+    }
+    return pg_per_osd_space_efficiency(per_osd, pgs.length, osd_sizes);
+}
+
+function pg_per_osd_space_efficiency(per_osd, pg_count, osd_sizes)
+{
+    // each PG gets randomly selected in 1/N cases
+    // & there are x PGs per OSD
+    // => an OSD is selected in x/N cases
+    // => total space * x/N <= OSD size
+    // => total space <= OSD size * N/x
+    let space;
+    for (let osd in per_osd)
+    {
+        if (osd in osd_sizes)
+        {
+            const space_estimate = osd_sizes[osd] * pg_count / per_osd[osd];
+            if (space == null || space > space_estimate)
+            {
+                space = space_estimate;
+            }
+        }
+    }
+    return space == null ? 0 : space;
+}
+
+module.exports = {
+    NO_OSD,
+
+    optimize_initial,
+    optimize_change,
+    print_change_stats,
+    pg_weights_space_efficiency,
+    pg_list_space_efficiency,
+    pg_per_osd_space_efficiency,
+    flatten_tree,
+
+    lp_solve,
+    make_int_pgs,
+    align_pgs,
+    random_combinations,
+    all_combinations,
+};

mon/make-units.sh

@@ -0,0 +1,134 @@
+#!/bin/bash
+# Example startup script generator
+# Of course this isn't a production solution yet, this is just for tests
+
+IP=`ip -json a s | jq -r '.[].addr_info[] | select(.broadcast == "10.115.0.255") | .local'`
+
+[ "$IP" != "" ] || exit 1
+
+useradd vitastor
+chmod 755 /root
+
+BASE=${IP/*./}
+BASE=$(((BASE-10)*12))
+
+cat >/etc/systemd/system/vitastor.target <<EOF
+[Unit]
+Description=vitastor target
+[Install]
+WantedBy=multi-user.target
+EOF
+
+i=1
+for DEV in `ls /dev/disk/by-id/ | grep ata-INTEL_SSDSC2KB`; do
+    dd if=/dev/zero of=/dev/disk/by-id/$DEV bs=1048576 count=$(((427814912+1048575)/1048576+2))
+    dd if=/dev/zero of=/dev/disk/by-id/$DEV bs=1048576 count=$(((427814912+1048575)/1048576+2)) seek=$((1920377991168/1048576))
+cat >/etc/systemd/system/vitastor-osd$((BASE+i)).service <<EOF
+[Unit]
+Description=Vitastor object storage daemon osd.$((BASE+i))
+After=network-online.target local-fs.target time-sync.target
+Wants=network-online.target local-fs.target time-sync.target
+PartOf=vitastor.target
+
+[Service]
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitMEMLOCK=infinity
+ExecStart=/root/vitastor/osd \\
+    --etcd_address $IP:2379/v3 \\
+    --bind_address $IP \\
+    --osd_num $((BASE+i)) \\
+    --disable_data_fsync 1 \\
+    --disable_device_lock 1 \\
+    --immediate_commit all \\
+    --flusher_count 8 \\
+    --disk_alignment 4096 --journal_block_size 4096 --meta_block_size 4096 \\
+    --journal_no_same_sector_overwrites true \\
+    --journal_sector_buffer_count 1024 \\
+    --journal_offset 0 \\
+    --meta_offset 16777216 \\
+    --data_offset 427814912 \\
+    --data_size $((1920377991168-427814912)) \\
+    --data_device /dev/disk/by-id/$DEV
+WorkingDirectory=/root/vitastor
+ExecStartPre=+chown vitastor:vitastor /dev/disk/by-id/$DEV
+User=vitastor
+PrivateTmp=false
+TasksMax=infinity
+Restart=always
+StartLimitInterval=0
+StartLimitIntervalSec=0
+RestartSec=10
+
+[Install]
+WantedBy=vitastor.target
+EOF
+    systemctl enable vitastor-osd$((BASE+i))
+    i=$((i+1))
+cat >/etc/systemd/system/vitastor-osd$((BASE+i)).service <<EOF
+[Unit]
+Description=Vitastor object storage daemon osd.$((BASE+i))
+After=network-online.target local-fs.target time-sync.target
+Wants=network-online.target local-fs.target time-sync.target
+PartOf=vitastor.target
+
+[Service]
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitMEMLOCK=infinity
+ExecStart=/root/vitastor/osd \\
+    --etcd_address $IP:2379/v3 \\
+    --bind_address $IP \\
+    --osd_num $((BASE+i)) \\
+    --disable_data_fsync 1 \\
+    --immediate_commit all \\
+    --flusher_count 8 \\
+    --disk_alignment 4096 --journal_block_size 4096 --meta_block_size 4096 \\
+    --journal_no_same_sector_overwrites true \\
+    --journal_sector_buffer_count 1024 \\
+    --journal_offset 1920377991168 \\
+    --meta_offset $((1920377991168+16777216)) \\
+    --data_offset $((1920377991168+427814912)) \\
+    --data_size $((1920377991168-427814912)) \\
+    --data_device /dev/disk/by-id/$DEV
+WorkingDirectory=/root/vitastor
+ExecStartPre=+chown vitastor:vitastor /dev/disk/by-id/$DEV
+User=vitastor
+PrivateTmp=false
+TasksMax=infinity
+Restart=always
+StartLimitInterval=0
+StartLimitIntervalSec=0
+RestartSec=10
+
+[Install]
+WantedBy=vitastor.target
+EOF
+    systemctl enable vitastor-osd$((BASE+i))
+    i=$((i+1))
+done
+
+exit
+
+node mon-main.js --etcd_url 'http://10.115.0.10:2379,http://10.115.0.11:2379,http://10.115.0.12:2379,http://10.115.0.13:2379' --etcd_prefix '/vitastor' --etcd_start_timeout 5
+
+podman run -d --network host --restart always -v /var/lib/etcd0.etcd:/etcd0.etcd --name etcd quay.io/coreos/etcd:v3.4.13 etcd -name etcd0 \
+    -advertise-client-urls http://10.115.0.10:2379 -listen-client-urls http://10.115.0.10:2379 \
+    -initial-advertise-peer-urls http://10.115.0.10:2380 -listen-peer-urls http://10.115.0.10:2380 \
+    -initial-cluster-token vitastor-etcd-1 -initial-cluster etcd0=http://10.115.0.10:2380,etcd1=http://10.115.0.11:2380,etcd2=http://10.115.0.12:2380,etcd3=http://10.115.0.13:2380 \
+    -initial-cluster-state new --max-txn-ops=100000 --auto-compaction-retention=10 --auto-compaction-mode=revision
+
+etcdctl --endpoints http://10.115.0.10:2379 put /vitastor/config/global '{"immediate_commit":"all"}'
+
+etcdctl --endpoints http://10.115.0.10:2379 put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":48,"failure_domain":"host"}}'
+
+#let pgs = {};
+#for (let n = 0; n < 48; n++) { let i = n/2 | 0; pgs[1+n] = { osd_set: [ (1+i%12+(i/12 | 0)*24), (1+12+i%12+(i/12 | 0)*24) ], primary: (1+(n%2)*12+i%12+(i/12 | 0)*24) }; };
+#console.log(JSON.stringify({ items: { 1: pgs } }));
+#etcdctl --endpoints http://10.115.0.10:2379 put /vitastor/config/pgs ...
+
+#    --disk_alignment 4096 --journal_block_size 4096 --meta_block_size 4096 \\
+#    --data_offset 427814912 \\
+
+#    --disk_alignment 4096 --journal_block_size 512 --meta_block_size 512 \\
+#    --data_offset 433434624 \\

mon/mon-main.js

@@ -0,0 +1,22 @@
+#!/usr/bin/node
+
+const Mon = require('./mon.js');
+
+const options = {};
+
+for (let i = 2; i < process.argv.length; i++)
+{
+    if (process.argv[i].substr(0, 2) == '--')
+    {
+        options[process.argv[i].substr(2)] = process.argv[i+1];
+        i++;
+    }
+}
+
+if (!options.etcd_url)
+{
+    console.error('USAGE: '+process.argv[0]+' '+process.argv[1]+' --etcd_url "http://127.0.0.1:2379,..." --etcd_prefix "/vitastor" --etcd_start_timeout 5 [--verbose 1]');
+    process.exit();
+}
+
+new Mon(options).start().catch(e => { console.error(e); process.exit(); });

mon/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "vitastor-mon",
+  "version": "1.0.0",
+  "description": "Vitastor SDS monitor service",
+  "main": "mon-main.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "Vitaliy Filippov",
+  "license": "UNLICENSED",
+  "dependencies": {
+    "ws": "^7.2.5"
+  }
+}

mon/stable-stringify.js

@@ -0,0 +1,75 @@
+function stableStringify(obj, opts)
+{
+    if (!opts)
+        opts = {};
+    if (typeof opts === 'function')
+        opts = { cmp: opts };
+    let space = opts.space || '';
+    if (typeof space === 'number')
+        space = Array(space+1).join(' ');
+    const cycles = (typeof opts.cycles === 'boolean') ? opts.cycles : false;
+    const cmp = opts.cmp && (function (f)
+    {
+        return function (node)
+        {
+            return function (a, b)
+            {
+                let aobj = { key: a, value: node[a] };
+                let bobj = { key: b, value: node[b] };
+                return f(aobj, bobj);
+            };
+        };
+    })(opts.cmp);
+    const seen = new Map();
+    return (function stringify (parent, key, node, level)
+    {
+        const indent = space ? ('\n' + new Array(level + 1).join(space)) : '';
+        const colonSeparator = space ? ': ' : ':';
+        if (node === undefined)
+        {
+            return;
+        }
+        if (typeof node !== 'object' || node === null)
+        {
+            return JSON.stringify(node);
+        }
+        if (node instanceof Array)
+        {
+            const out = [];
+            for (let i = 0; i < node.length; i++)
+            {
+                const item = stringify(node, i, node[i], level+1) || JSON.stringify(null);
+                out.push(indent + space + item);
+            }
+            return '[' + out.join(',') + indent + ']';
+        }
+        else
+        {
+            if (seen.has(node))
+            {
+                if (cycles)
+                    return JSON.stringify('__cycle__');
+                throw new TypeError('Converting circular structure to JSON');
+            }
+            else
+                seen.set(node, true);
+            const keys = Object.keys(node).sort(cmp && cmp(node));
+            const out = [];
+            for (let i = 0; i < keys.length; i++)
+            {
+                const key = keys[i];
+                const value = stringify(node, key, node[key], level+1);
+                if (!value)
+                    continue;
+                const keyValue = JSON.stringify(key)
+                    + colonSeparator
+                    + value;
+                out.push(indent + space + keyValue);
+            }
+            seen.delete(node);
+            return '{' + out.join(',') + indent + '}';
+        }
+    })({ '': obj }, '', obj, 0);
+}
+
+module.exports = stableStringify;

mon/test-nonuniform.js

@@ -0,0 +1,127 @@
+// Interesting real-world example coming from Ceph with EC and compression enabled.
+// EC parity chunks can't be compressed as efficiently as data chunks,
+// thus they occupy more space (2.26x more space) in OSD object stores.
+// This leads to really uneven OSD fill ratio in Ceph even when PGs are perfectly balanced.
+// But we support this case with the "parity_space" parameter in optimize_initial()/optimize_change().
+
+const LPOptimizer = require('./lp-optimizer.js');
+
+const osd_tree = {
+    ripper5: {
+        osd0: 3.493144989013672,
+        osd1: 3.493144989013672,
+        osd2: 3.454082489013672,
+        osd12: 3.461894989013672,
+    },
+    ripper7: {
+        osd4: 3.638690948486328,
+        osd5: 3.638690948486328,
+        osd6: 3.638690948486328,
+    },
+    ripper4: {
+        osd9: 3.4609375,
+        osd10: 3.4609375,
+        osd11: 3.4609375,
+    },
+    ripper6: {
+        osd3: 3.5849609375,
+        osd7: 3.5859336853027344,
+        osd8: 3.638690948486328,
+        osd13: 3.461894989013672
+    },
+};
+
+const prev_pgs = [[12,7,5],[6,11,12],[3,6,9],[10,0,5],[2,5,13],[9,8,6],[3,4,12],[7,4,12],[12,11,13],[13,6,0],[4,13,10],[9,7,6],[7,10,0],[10,8,0],[3,10,2],[3,0,4],[6,13,0],[13,10,0],[13,10,5],[8,11,6],[3,9,2],[2,8,5],[8,9,5],[3,12,11],[0,7,4],[13,11,1],[11,3,12],[12,8,10],[7,5,12],[2,13,5],[7,11,0],[13,2,6],[0,6,8],[13,1,6],[0,13,4],[0,8,10],[4,10,0],[8,12,4],[8,12,9],[12,7,4],[13,9,5],[3,2,11],[1,9,7],[1,8,5],[5,12,9],[3,5,12],[2,8,10],[0,8,4],[1,4,11],[7,10,2],[12,13,5],[3,1,11],[7,1,4],[4,12,8],[7,0,9],[11,1,8],[3,0,5],[11,13,0],[1,13,5],[12,7,10],[12,8,4],[11,13,5],[0,11,6],[2,11,3],[13,1,11],[2,7,10],[7,10,12],[7,12,10],[12,11,5],[13,12,10],[2,3,9],[4,3,9],[13,2,5],[7,12,6],[12,10,13],[9,8,1],[13,1,5],[9,5,12],[5,11,7],[6,2,9],[8,11,6],[12,5,8],[6,13,1],[7,6,11],[2,3,6],[8,5,9],[1,13,6],[9,3,2],[7,11,1],[3,10,1],[0,11,7],[3,0,5],[1,3,6],[6,0,9],[3,11,4],[8,10,2],[13,1,9],[12,6,9],[3,12,9],[12,8,9],[7,5,0],[8,12,5],[0,11,3],[12,11,13],[0,7,11],[0,3,10],[1,3,11],[2,7,11],[13,2,6],[9,12,13],[8,2,4],[0,7,4],[5,13,0],[13,12,9],[1,9,8],[0,10,3],[3,5,10],[7,12,9],[2,13,4],[12,7,5],[9,2,7],[3,2,9],[6,2,7],[3,1,9],[4,3,2],[5,3,11],[0,7,6],[1,6,13],[7,10,2],[12,4,8],[13,12,6],[7,5,11],[6,2,3],[2,7,6],[2,3,10],[2,7,10],[11,12,6],[0,13,5],[10,2,4],[13,0,11],[7,0,6],[8,9,4],[8,4,11],[7,11,2],[3,4,2],[6,1,3],[7,2,11],[8,9,4],[11,4,8],[10,3,1],[2,10,13],[1,7,11],[13,11,12],[2,6,9],[10,0,13],[7,10,4],[0,11,13],[13,10,1],[7,5,0],[7,12,10],[3,1,4],[7,1,5],[3,11,5],[7,5,0],[1,3,5],[10,5,12],[0,3,9],[7,1,11],[11,8,12],[3,6,2],[7,12,9],[7,11,12],[4,11,3],[0,11,13],[13,2,5],[1,5,8],[0,11,8],[3,5,1],[11,0,6],[3,11,2],[11,8,12],[4,1,3],[10,13,4],[13,9,6],[2,3,10],[12,7,9],[10,0,4],[10,13,2],[3,11,1],[7,2,9],[1,7,4],[13,1,4],[7,0,6],[5,3,9],[10,0,7],[0,7,10],[3,6,10],[13,0,5],[8,4,1],[3,1,10],[2,10,13],[13,0,5],[13,10,2],[12,7,9],[6,8,10],[6,1,8],[10,8,1],[13,5,0],[5,11,3],[7,6,1],[8,5,9],[2,13,11],[10,12,4],[13,4,1],[2,13,4],[11,7,0],[2,9,7],[1,7,6],[8,0,4],[8,1,9],[7,10,12],[13,9,6],[7,6,11],[13,0,4],[1,8,4],[3,12,5],[10,3,1],[10,2,13],[2,4,8],[6,2,3],[3,0,10],[6,7,12],[8,12,5],[3,0,6],[13,12,10],[11,3,6],[9,0,13],[10,0,6],[7,5,2],[1,3,11],[7,10,2],[2,9,8],[11,13,12],[0,8,4],[8,12,11],[6,0,3],[1,13,4],[11,8,2],[12,3,6],[4,7,1],[7,6,12],[3,10,6],[0,10,7],[8,9,1],[0,10,6],[8,10,1]]
+    .map(pg => pg.map(n => 'osd'+n));
+
+const by_osd = {};
+
+for (let i = 0; i < prev_pgs.length; i++)
+{
+    for (let j = 0; j < prev_pgs[i].length; j++)
+    {
+        by_osd[prev_pgs[i][j]] = by_osd[prev_pgs[i][j]] || [];
+        by_osd[prev_pgs[i][j]][j] = (by_osd[prev_pgs[i][j]][j] || 0) + 1;
+    }
+}
+
+/*
+
+This set of PGs was balanced by hand, by heavily tuning OSD weights in Ceph:
+
+{
+  osd0: 4.2,
+  osd1: 3.5,
+  osd2: 3.45409,
+  osd3: 4.5,
+  osd4: 1.4,
+  osd5: 1.4,
+  osd6: 1.75,
+  osd7: 4.5,
+  osd8: 4.4,
+  osd9: 2.2,
+  osd10: 2.7,
+  osd11: 2,
+  osd12: 3.4,
+  osd13: 3.4,
+}
+
+EC+compression is a nightmare in Ceph, yeah :))
+
+To calculate the average ratio between data chunks and parity chunks we
+calculate the number of PG chunks for each chunk role for each OSD:
+
+{
+  osd12: [ 18, 22, 17 ],
+  osd7: [ 35, 22, 8 ],
+  osd5: [ 6, 17, 27 ],
+  osd6: [ 13, 12, 28 ],
+  osd11: [ 13, 26, 20 ],
+  osd3: [ 30, 20, 10 ],
+  osd9: [ 8, 12, 26 ],
+  osd10: [ 15, 23, 20 ],
+  osd0: [ 22, 22, 14 ],
+  osd2: [ 22, 16, 16 ],
+  osd13: [ 29, 19, 13 ],
+  osd8: [ 20, 18, 12 ],
+  osd4: [ 8, 10, 28 ],
+  osd1: [ 17, 17, 17 ]
+}
+
+And now we can pick a pair of OSDs and determine the ratio by solving the following:
+
+osd5 = 23*X + 27*Y = 3249728140
+osd13 = 48*X + 13*Y = 2991675992
+
+=>
+
+osd5 - 27/13*osd13 = 23*X - 27/13*48*X = -76.6923076923077*X = -2963752766.46154
+
+=>
+
+X = 38644720.1243731
+Y = (osd5-23*X)/27 = 87440725.0792377
+Y/X = 2.26268232239284 ~= 2.26
+
+Which means that parity chunks are compressed ~2.26 times worse than data chunks.
+
+Fine, let's try to optimize for it.
+
+*/
+
+async function run()
+{
+    const all_weights = Object.assign({}, ...Object.values(osd_tree));
+    const total_weight = Object.values(all_weights).reduce((a, c) => Number(a) + Number(c), 0);
+    const eff = LPOptimizer.pg_list_space_efficiency(prev_pgs, all_weights, 2, 2.26);
+    const orig = eff*4.26 / total_weight;
+    console.log('Original efficiency was: '+Math.round(orig*10000)/100+' %');
+
+    let prev = await LPOptimizer.optimize_initial({ osd_tree, pg_size: 3, pg_count: 256, parity_space: 2.26 });
+    LPOptimizer.print_change_stats(prev);
+
+    let next = await LPOptimizer.optimize_change({ prev_pgs, osd_tree, pg_size: 3, max_combinations: 10000, parity_space: 2.26 });
+    LPOptimizer.print_change_stats(next);
+}
+
+run().catch(console.error);

mon/test-optimize-undersized.js

@@ -0,0 +1,71 @@
+const LPOptimizer = require('./lp-optimizer.js');
+
+const crush_tree = [
+    { level: 1, children: [
+        { level: 2, children: [
+            { level: 3, id: 1, size: 3 },
+            { level: 3, id: 2, size: 3 },
+        ] },
+        { level: 2, children: [
+            { level: 3, id: 3, size: 3 },
+            { level: 3, id: 4, size: 3 },
+        ] },
+    ] },
+    { level: 1, children: [
+        { level: 2, children: [
+            { level: 3, id: 5, size: 3 },
+            { level: 3, id: 6, size: 3 },
+        ] },
+        { level: 2, children: [
+            { level: 3, id: 7, size: 3 },
+            { level: 3, id: 8, size: 3 },
+        ] },
+    ] },
+    { level: 1, children: [
+        { level: 2, children: [
+            { level: 3, id: 9, size: 3 },
+            { level: 3, id: 10, size: 3 },
+        ] },
+        { level: 2, children: [
+            { level: 3, id: 11, size: 3 },
+            { level: 3, id: 12, size: 3 },
+        ] },
+    ] },
+];
+
+const osd_tree = LPOptimizer.flatten_tree(crush_tree, {}, 1, 3);
+console.log(osd_tree);
+
+async function run()
+{
+    const cur_tree = {};
+    console.log('Empty tree:');
+    let res = await LPOptimizer.optimize_initial({ osd_tree: cur_tree, pg_size: 3, pg_count: 256 });
+    LPOptimizer.print_change_stats(res, false);
+    console.log('\nAdding 1st failure domain:');
+    cur_tree['dom1'] = osd_tree['dom1'];
+    res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 });
+    LPOptimizer.print_change_stats(res, false);
+    console.log('\nAdding 2nd failure domain:');
+    cur_tree['dom2'] = osd_tree['dom2'];
+    res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 });
+    LPOptimizer.print_change_stats(res, false);
+    console.log('\nAdding 3rd failure domain:');
+    cur_tree['dom3'] = osd_tree['dom3'];
+    res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 });
+    LPOptimizer.print_change_stats(res, false);
+    console.log('\nRemoving 3rd failure domain:');
+    delete cur_tree['dom3'];
+    res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 });
+    LPOptimizer.print_change_stats(res, false);
+    console.log('\nRemoving 2nd failure domain:');
+    delete cur_tree['dom2'];
+    res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 });
+    LPOptimizer.print_change_stats(res, false);
+    console.log('\nRemoving 1st failure domain:');
+    delete cur_tree['dom1'];
+    res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 });
+    LPOptimizer.print_change_stats(res, false);
+}
+
+run().catch(console.error);

mon/test-optimize.js

@@ -0,0 +1,112 @@
+const LPOptimizer = require('./lp-optimizer.js');
+
+const osd_tree = {
+    100: {
+        7: 3.63869,
+    },
+    300: {
+        10: 3.46089,
+        11: 3.46089,
+        12: 3.46089,
+    },
+    400: {
+        1: 3.49309,
+        2: 3.49309,
+        3: 3.49309,
+    },
+    500: {
+        4: 3.58498,
+//        8: 3.58589,
+        9: 3.63869,
+    },
+    600: {
+        5: 3.63869,
+        6: 3.63869,
+    },
+/*    100: {
+        1: 2.72800,
+    },
+    200: {
+        2: 2.72900,
+    },
+    300: {
+        3: 1.87000,
+    },
+    400: {
+        4: 1.87000,
+    },
+    500: {
+        5: 3.63869,
+    },*/
+};
+
+const crush_tree = [
+    { level: 1, children: [
+        { level: 2, children: [
+            { level: 3, id: 1, size: 3 },
+            { level: 3, id: 2, size: 2 },
+        ] },
+        { level: 2, children: [
+            { level: 3, id: 3, size: 4 },
+            { level: 3, id: 4, size: 4 },
+        ] },
+    ] },
+    { level: 1, children: [
+        { level: 2, children: [
+            { level: 3, id: 5, size: 4 },
+            { level: 3, id: 6, size: 1 },
+        ] },
+        { level: 2, children: [
+            { level: 3, id: 7, size: 3 },
+            { level: 3, id: 8, size: 5 },
+        ] },
+    ] },
+    { level: 1, children: [
+        { level: 2, children: [
+            { level: 3, id: 9, size: 5 },
+            { level: 3, id: 10, size: 2 },
+        ] },
+        { level: 2, children: [
+            { level: 3, id: 11, size: 3 },
+            { level: 3, id: 12, size: 3 },
+        ] },
+    ] },
+];
+
+async function run()
+{
+    let res;
+
+    // Test: add 1 OSD of almost the same size. Ideal data movement could be 1/12 = 8.33%. Actual is ~13%
+    // Space efficiency is ~99% in all cases.
+
+    console.log('256 PGs, size=2');
+    res = await LPOptimizer.optimize_initial({ osd_tree, pg_size: 2, pg_count: 256 });
+    LPOptimizer.print_change_stats(res, false);
+    console.log('\nAdding osd.8');
+    osd_tree[500][8] = 3.58589;
+    res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree, pg_size: 2 });
+    LPOptimizer.print_change_stats(res, false);
+    console.log('\nRemoving osd.8');
+    delete osd_tree[500][8];
+    res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree, pg_size: 2 });
+    LPOptimizer.print_change_stats(res, false);
+
+    console.log('\n256 PGs, size=3');
+    res = await LPOptimizer.optimize_initial({ osd_tree, pg_size: 3, pg_count: 256 });
+    LPOptimizer.print_change_stats(res, false);
+    console.log('\nAdding osd.8');
+    osd_tree[500][8] = 3.58589;
+    res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree, pg_size: 3 });
+    LPOptimizer.print_change_stats(res, false);
+    console.log('\nRemoving osd.8');
+    delete osd_tree[500][8];
+    res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree, pg_size: 3 });
+    LPOptimizer.print_change_stats(res, false);
+
+    console.log('\n256 PGs, size=3, failure domain=rack');
+    res = await LPOptimizer.optimize_initial({ osd_tree: LPOptimizer.flatten_tree(crush_tree, {}, 1, 3), pg_size: 3, pg_count: 256 });
+    LPOptimizer.print_change_stats(res, false);
+}
+
+run().catch(console.error);