Move scripts

mon/scripts/90-vitastor.rules

@@ -0,0 +1,7 @@
+SUBSYSTEM=="block", ENV{ID_PART_ENTRY_TYPE}=="e7009fac-a5a1-4d72-af72-53de13059903", \
+    OWNER="vitastor", GROUP="vitastor", \
+    IMPORT{program}="/usr/bin/vitastor-disk udev $devnode", \
+    SYMLINK+="vitastor/$env{VITASTOR_ALIAS}"
+
+ENV{VITASTOR_OSD_NUM}!="", ACTION=="add", RUN{program}+="/usr/bin/systemctl enable --now --no-block vitastor-osd@$env{VITASTOR_OSD_NUM}"
+ENV{VITASTOR_OSD_NUM}!="", ACTION=="remove", RUN{program}+="/usr/bin/systemctl disable --now --no-block vitastor-osd@$env{VITASTOR_OSD_NUM}"

mon/scripts/afr.js

@@ -0,0 +1,89 @@
+// Functions to calculate Annualized Failure Rate of your cluster
+// if you know AFR of your drives, number of drives, expected rebalance time
+// and replication factor
+// License: VNPL-1.1 (see https://yourcmc.ru/git/vitalif/vitastor/src/branch/master/README.md for details) or AGPL-3.0
+// Author: Vitaliy Filippov, 2020+
+
+module.exports = {
+    cluster_afr_fullmesh,
+    failure_rate_fullmesh,
+    cluster_afr,
+    c_n_k,
+};
+
+/******** "FULL MESH": ASSUME EACH OSD COMMUNICATES WITH ALL OTHER OSDS ********/
+
+// Estimate AFR of the cluster
+// n - number of drives
+// afr - annualized failure rate of a single drive
+// l - expected rebalance time in days after a single drive failure
+// k - replication factor / number of drives that must fail at the same time for the cluster to fail
+function cluster_afr_fullmesh(n, afr, l, k)
+{
+    return 1 - (1 - afr * failure_rate_fullmesh(n-(k-1), afr*l/365, k-1)) ** (n-(k-1));
+}
+
+// Probability of at least <f> failures in a cluster with <n> drives with AFR=<a>
+function failure_rate_fullmesh(n, a, f)
+{
+    if (f <= 0)
+    {
+        return (1-a)**n;
+    }
+    let p = 1;
+    for (let i = 0; i < f; i++)
+    {
+        p -= c_n_k(n, i) * (1-a)**(n-i) * a**i;
+    }
+    return p;
+}
+
+/******** PGS: EACH OSD ONLY COMMUNICATES WITH <pgs> OTHER OSDs ********/
+
+// <n> hosts of <m> drives of <capacity> GB, each able to backfill at <speed> GB/s,
+// <k> replicas, <pgs> unique peer PGs per OSD (~50 for 100 PG-per-OSD in a big cluster)
+//
+// For each of n*m drives: P(drive fails in a year) * P(any of its peers fail in <l*365> next days).
+// More peers per OSD increase rebalance speed (more drives work together to resilver) if you
+// let them finish rebalance BEFORE replacing the failed drive (degraded_replacement=false).
+// At the same time, more peers per OSD increase probability of any of them to fail!
+// osd_rm=true means that failed OSDs' data is rebalanced over all other hosts,
+// not over the same host as it's in Ceph by default (dead OSDs are marked 'out').
+//
+// Probability of all except one drives in a replica group to fail is (AFR^(k-1)).
+// So with <x> PGs it becomes ~ (x * (AFR*L/365)^(k-1)). Interesting but reasonable consequence
+// is that, with k=2, total failure rate doesn't depend on number of peers per OSD,
+// because it gets increased linearly by increased number of peers to fail
+// and decreased linearly by reduced rebalance time.
+function cluster_afr({ n_hosts, n_drives, afr_drive, afr_host, capacity, speed, ec, ec_data, ec_parity, replicas, pgs = 1, osd_rm, degraded_replacement, down_out_interval = 600 })
+{
+    const pg_size = (ec ? ec_data+ec_parity : replicas);
+    pgs = Math.min(pgs, (n_hosts-1)*n_drives/(pg_size-1));
+    const host_pgs = Math.min(pgs*n_drives, (n_hosts-1)*n_drives/(pg_size-1));
+    const resilver_disk = n_drives == 1 || osd_rm ? pgs : (n_drives-1);
+    const disk_heal_time = (down_out_interval + capacity/(degraded_replacement ? 1 : resilver_disk)/speed)/86400/365;
+    const host_heal_time = (down_out_interval + n_drives*capacity/pgs/speed)/86400/365;
+    const disk_heal_fail = ((afr_drive+afr_host/n_drives)*disk_heal_time);
+    const host_heal_fail = ((afr_drive+afr_host/n_drives)*host_heal_time);
+    const disk_pg_fail = ec
+        ? failure_rate_fullmesh(ec_data+ec_parity-1, disk_heal_fail, ec_parity)
+        : disk_heal_fail**(replicas-1);
+    const host_pg_fail = ec
+        ? failure_rate_fullmesh(ec_data+ec_parity-1, host_heal_fail, ec_parity)
+        : host_heal_fail**(replicas-1);
+    return 1 - ((1 - afr_drive * (1-(1-disk_pg_fail)**pgs)) ** (n_hosts*n_drives))
+        * ((1 - afr_host * (1-(1-host_pg_fail)**host_pgs)) ** n_hosts);
+}
+
+/******** UTILITY ********/
+
+// Combination count
+function c_n_k(n, k)
+{
+    let r = 1;
+    for (let i = 0; i < k; i++)
+    {
+        r *= (n-i) / (i+1);
+    }
+    return r;
+}

mon/scripts/afr_test.js

@@ -0,0 +1,28 @@
+const { sprintf } = require('sprintf-js');
+const { cluster_afr } = require('./afr.js');
+
+print_cluster_afr({ n_hosts: 4, n_drives: 6, afr_drive: 0.03, afr_host: 0.05, capacity: 4000, speed: 0.1, replicas: 2 });
+print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, afr_host: 0, capacity: 4000, speed: 0.1, replicas: 2 });
+print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, afr_host: 0.05, capacity: 4000, speed: 0.1, replicas: 2 });
+print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, afr_host: 0, capacity: 4000, speed: 0.1, ec: true, ec_data: 2, ec_parity: 1 });
+print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, afr_host: 0.05, capacity: 4000, speed: 0.1, ec: true, ec_data: 2, ec_parity: 1 });
+print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0, capacity: 8000, speed: 0.02, replicas: 2 });
+print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 2 });
+print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0, capacity: 8000, speed: 0.02, replicas: 3 });
+print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 3 });
+print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0, capacity: 8000, speed: 0.02, replicas: 3, pgs: 100 });
+print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 3, pgs: 100 });
+print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 3, pgs: 100, degraded_replacement: 1 });
+
+function print_cluster_afr(config)
+{
+    console.log(
+        `${config.n_hosts} nodes with ${config.n_drives} ${sprintf("%.1f", config.capacity/1000)}TB drives`+
+        `, capable to backfill at ${sprintf("%.1f", config.speed*1000)} MB/s, drive AFR ${sprintf("%.1f", config.afr_drive*100)}%`+
+        (config.afr_host ? `, host AFR ${sprintf("%.1f", config.afr_host*100)}%` : '')+
+        (config.ec ? `, EC ${config.ec_data}+${config.ec_parity}` : `, ${config.replicas} replicas`)+
+        `, ${config.pgs||1} PG per OSD`+
+        (config.degraded_replacement ? `\n...and you don't let the rebalance finish before replacing drives` : '')
+    );
+    console.log('-> '+sprintf("%.7f%%", 100*cluster_afr(config))+'\n');
+}

mon/scripts/make-etcd

@@ -0,0 +1,111 @@
+#!/usr/bin/node
+// Simple systemd unit generator for etcd
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: MIT
+
+// USAGE:
+// 1) Put the same etcd_address into /etc/vitastor/vitastor.conf on all monitor nodes
+// 2) Run ./make-etcd.js. It will create the etcd service on one of specified IPs
+
+const child_process = require('child_process');
+const fs = require('fs');
+const os = require('os');
+
+run().catch(e => { console.error(e); process.exit(1); });
+
+async function run()
+{
+    const config_path = process.argv[2] || '/etc/vitastor/vitastor.conf';
+    if (config_path == '-h' || config_path == '--help')
+    {
+        console.log(
+            'Initialize systemd etcd service for Vitastor\n'+
+            '(c) Vitaliy Filippov, 2019+ (MIT)\n'+
+            '\n'+
+            'USAGE:\n'+
+            '1) Put the same etcd_address into /etc/vitastor/vitastor.conf on all monitor nodes\n'+
+            '2) Run '+process.argv[1]+' [config_path]\n'
+        );
+        process.exit(0);
+    }
+    if (!fs.existsSync(config_path))
+    {
+        console.log(config_path+' is missing');
+        process.exit(1);
+    }
+    if (fs.existsSync("/etc/systemd/system/etcd.service"))
+    {
+        console.log("/etc/systemd/system/etcd.service already exists");
+        process.exit(1);
+    }
+    const config = JSON.parse(fs.readFileSync(config_path, { encoding: 'utf-8' }));
+    if (!config.etcd_address)
+    {
+        console.log("etcd_address is missing in "+config_path);
+        process.exit(1);
+    }
+    const etcds = (config.etcd_address instanceof Array ? config.etcd_address : (''+config.etcd_address).split(/,/))
+        .map(s => (''+s).replace(/^https?:\/\/\[?|\]?(:\d+)?(\/.*)?$/g, '').toLowerCase());
+    const num = select_local_etcd(etcds);
+    if (num < 0)
+    {
+        console.log('No matching IPs in etcd_address from '+config_path);
+        process.exit(0);
+    }
+    const etcd_cluster = etcds.map((e, i) => `etcd${i}=http://${e}:2380`).join(',');
+    await system(`mkdir -p /var/lib/etcd${num}.etcd`);
+    fs.writeFileSync(
+        "/etc/systemd/system/etcd.service",
+`[Unit]
+Description=etcd for vitastor
+After=network-online.target local-fs.target time-sync.target
+Wants=network-online.target local-fs.target time-sync.target
+
+[Service]
+Restart=always
+Environment=GOGC=50
+ExecStart=etcd -name etcd${num} --data-dir /var/lib/etcd${num}.etcd \\
+    --snapshot-count 10000 --advertise-client-urls http://${etcds[num]}:2379 --listen-client-urls http://${etcds[num]}:2379 \\
+    --initial-advertise-peer-urls http://${etcds[num]}:2380 --listen-peer-urls http://${etcds[num]}:2380 \\
+    --initial-cluster-token vitastor-etcd-1 --initial-cluster ${etcd_cluster} \\
+    --initial-cluster-state new --max-txn-ops=100000 --max-request-bytes=104857600 \\
+    --auto-compaction-retention=10 --auto-compaction-mode=revision
+WorkingDirectory=/var/lib/etcd${num}.etcd
+ExecStartPre=+chown -R etcd /var/lib/etcd${num}.etcd
+User=etcd
+PrivateTmp=false
+TasksMax=infinity
+Restart=always
+StartLimitInterval=0
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
+`);
+    await system(`useradd etcd`);
+    await system(`systemctl daemon-reload`);
+    await system(`systemctl enable etcd`);
+    await system(`systemctl start etcd`);
+    process.exit(0);
+}
+
+function select_local_etcd(etcds)
+{
+    const ifaces = os.networkInterfaces();
+    for (const ifname in ifaces)
+        for (const iface of ifaces[ifname])
+            for (let i = 0; i < etcds.length; i++)
+                if (etcds[i] == iface.address.toLowerCase())
+                    return i;
+    return -1;
+}
+
+async function system(cmd)
+{
+    const cp = child_process.spawn(cmd, { shell: true, stdio: [ 0, 1, 2 ] });
+    let finish_cb;
+    cp.on('exit', () => finish_cb && finish_cb());
+    if (cp.exitCode == null)
+        await new Promise(ok => finish_cb = ok);
+    return cp.exitCode;
+}

mon/scripts/simple-offsets.js

@@ -0,0 +1,97 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: MIT
+
+// Simple tool to calculate journal and metadata offsets for a single device
+// Will be replaced by smarter tools in the future
+
+const fs = require('fs').promises;
+const child_process = require('child_process');
+
+async function run()
+{
+    const options = {
+        object_size: 128*1024,
+        bitmap_granularity: 4096,
+        journal_size: 16*1024*1024,
+        device_block_size: 4096,
+        journal_offset: 0,
+        device_size: 0,
+        format: 'text',
+    };
+    for (let i = 2; i < process.argv.length; i++)
+    {
+        if (process.argv[i].substr(0, 2) == '--')
+        {
+            options[process.argv[i].substr(2)] = process.argv[i+1];
+            i++;
+        }
+    }
+    if (!options.device)
+    {
+        process.stderr.write('USAGE: nodejs '+process.argv[1]+' --device /dev/sdXXX\n');
+        process.exit(1);
+    }
+    options.device_size = Number(options.device_size);
+    let device_size = options.device_size;
+    if (!device_size)
+    {
+        const st = await fs.stat(options.device);
+        options.device_block_size = st.blksize;
+        if (st.isBlockDevice())
+            device_size = Number(await system("/sbin/blockdev --getsize64 "+options.device));
+        else
+            device_size = st.size;
+    }
+    if (!device_size)
+    {
+        process.stderr.write('Failed to get device size\n');
+        process.exit(1);
+    }
+    options.journal_offset = Math.ceil(options.journal_offset/options.device_block_size)*options.device_block_size;
+    const meta_offset = options.journal_offset + Math.ceil(options.journal_size/options.device_block_size)*options.device_block_size;
+    const meta_entry_size = 24 + 2*options.object_size/options.bitmap_granularity/8;
+    const entries_per_block = Math.floor(options.device_block_size / meta_entry_size);
+    const object_count = Math.floor((device_size-meta_offset)/options.object_size);
+    const meta_size = Math.ceil(1 + object_count / entries_per_block) * options.device_block_size;
+    const data_offset = meta_offset + meta_size;
+    const meta_size_fmt = (meta_size > 1024*1024*1024 ? Math.round(meta_size/1024/1024/1024*100)/100+" GB"
+        : Math.round(meta_size/1024/1024*100)/100+" MB");
+    if (options.format == 'text' || options.format == 'options')
+    {
+        if (options.format == 'text')
+        {
+            process.stderr.write(
+                `Metadata size: ${meta_size_fmt}\n`+
+                `Options for the OSD:\n`
+            );
+        }
+        process.stdout.write(
+            (options.device_block_size != 4096 ?
+                `    --meta_block_size ${options.device}\n`+
+                `    --journal_block-size ${options.device}\n` : '')+
+            `    --data_device ${options.device}\n`+
+            `    --journal_offset ${options.journal_offset}\n`+
+            `    --meta_offset ${meta_offset}\n`+
+            `    --data_offset ${data_offset}\n`+
+            (options.device_size ? `    --data_size ${device_size-data_offset}\n` : '')
+        );
+    }
+    else if (options.format == 'env')
+    {
+        process.stdout.write(
+            `journal_offset=${options.journal_offset}\n`+
+            `meta_offset=${meta_offset}\n`+
+            `data_offset=${data_offset}\n`+
+            `data_size=${device_size-data_offset}\n`
+        );
+    }
+    else
+        process.stdout.write('Unknown format: '+options.format);
+}
+
+function system(cmd)
+{
+    return new Promise((ok, no) => child_process.exec(cmd, { maxBuffer: 64*1024*1024 }, (err, stdout/*, stderr*/) => (err ? no(err.message) : ok(stdout))));
+}
+
+run().catch(err => { console.error(err); process.exit(1); });

mon/scripts/vitastor-mon.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=Vitastor monitor
+After=network-online.target local-fs.target time-sync.target
+Wants=network-online.target local-fs.target time-sync.target
+
+[Service]
+Restart=always
+ExecStart=node /usr/lib/vitastor/mon/mon-main.js
+WorkingDirectory=/
+User=vitastor
+PrivateTmp=false
+TasksMax=infinity
+Restart=always
+StartLimitInterval=0
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target

mon/scripts/vitastor-osd@.service

@@ -0,0 +1,24 @@
+[Unit]
+Description=Vitastor object storage daemon osd.%i
+After=network-online.target local-fs.target time-sync.target
+Wants=network-online.target local-fs.target time-sync.target
+PartOf=vitastor.target
+
+[Service]
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitMEMLOCK=infinity
+# Use the following for direct logs to files
+#ExecStart=bash -c 'exec vitastor-disk exec-osd /dev/vitastor/osd%i-data >>/var/log/vitastor/osd%i.log 2>&1'
+ExecStart=vitastor-disk exec-osd /dev/vitastor/osd%i-data
+ExecStartPre=+vitastor-disk pre-exec /dev/vitastor/osd%i-data
+WorkingDirectory=/
+User=vitastor
+PrivateTmp=false
+TasksMax=infinity
+Restart=always
+StartLimitInterval=0
+RestartSec=10
+
+[Install]
+WantedBy=vitastor.target

mon/scripts/vitastor.target

@@ -0,0 +1,4 @@
+[Unit]
+Description=vitastor target
+[Install]
+WantedBy=multi-user.target