Run tests with antietcd

A bunch of monitor fixes

- Add noout flag for OSDs (/vitastor/config/osd/xx)
- Fix "effective" size of degraded PGs (and thus "used space") calculation in monitor
- Fix monitor not clearing PGs of deleted pools
- Fix incorrect PG generation with hosts with 0 OSDs
- Fix monitor crashing during primary OSD recheck when pool has no PGs
- Fix monitor crashing when node_placement included non-existing OSDs
- Fix possible data movement after removing OSDs reweighted to 0
- Remove extra empty keys from pool configurations created by vitastor-cli create-pool
- Fix 32-bit build

.gitea/workflows/test.yml

@@ -64,6 +64,13 @@ jobs:
     # leak sanitizer sometimes crashes
     - run: cd /root/vitastor/build && ASAN_OPTIONS=detect_leaks=0 make -j16 test
 
+  npm_lint:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - run: cd /root/vitastor/mon && npm run lint
+
   test_add_osd:
     runs-on: ubuntu-latest
     needs: build

.gitignore

@@ -3,16 +3,3 @@
 package-lock.json
 fio
 qemu
-osd
-stub_osd
-stub_uring_osd
-stub_bench
-osd_test
-osd_peering_pg_test
-dump_journal
-nbd_proxy
-rm_inode
-test_allocator
-test_blockstore
-test_shit
-osd_rmw_test

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VERSION "1.6.0")
+set(VERSION "1.6.1")
 
 add_subdirectory(src)

copy-fio-includes.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-gcc -I. -E -o fio_headers.i src/fio_headers.h
+gcc -I. -E -o fio_headers.i src/util/fio_headers.h
 
 rm -rf fio-copy
 for i in `grep -Po 'fio/[^"]+' fio_headers.i | sort | uniq`; do

copy-qemu-includes.sh

@@ -5,7 +5,7 @@
 #cd b/qemu; make qapi
 
 gcc -I qemu/b/qemu `pkg-config glib-2.0 --cflags` \
-    -I qemu/include -E -o qemu_driver.i src/qemu_driver.c
+    -I qemu/include -E -o qemu_driver.i src/client/qemu_driver.c
 
 rm -rf qemu-copy
 for i in `grep -Po 'qemu/[^"]+' qemu_driver.i | sort | uniq`; do

csi/Makefile

@@ -1,4 +1,4 @@
-VERSION ?= v1.6.0
+VERSION ?= v1.6.1
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v1.6.0
+          image: vitalif/vitastor-csi:v1.6.1
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v1.6.0
+          image: vitalif/vitastor-csi:v1.6.1
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "1.6.0"
+    vitastorCSIDriverVersion = "1.6.1"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (1.6.0-1) unstable; urgency=medium
+vitastor (1.6.1-1) unstable; urgency=medium
 
   * Bugfixes
 

debian/patched-qemu.Dockerfile

@@ -27,7 +27,7 @@ RUN apt-get -y build-dep qemu
 RUN apt-get --download-only source qemu
 
 ADD patches /root/vitastor/patches
-ADD src/qemu_driver.c /root/vitastor/src/qemu_driver.c
+ADD src/client/qemu_driver.c /root/qemu_driver.c
 
 #RUN set -e; \
 #    apt-get install -y wget; \
@@ -52,7 +52,7 @@ RUN set -e; \
     cd /root/packages/qemu-$REL/qemu-*/; \
     quilt push -a; \
     quilt add block/vitastor.c; \
-    cp /root/vitastor/src/qemu_driver.c block/vitastor.c; \
+    cp /root/qemu_driver.c block/vitastor.c; \
     quilt refresh; \
     V=$(head -n1 debian/changelog | perl -pe 's/5\.2\+dfsg-9/5.2+dfsg-11/; s/^.*\((.*?)(~bpo[\d\+]*)?\).*$/$1/')+vitastor4; \
     if [ "$REL" = bullseye ]; then V=${V}bullseye; fi; \

debian/vitastor.Dockerfile

@@ -37,8 +37,8 @@ RUN set -e -x; \
     mkdir -p /root/packages/vitastor-$REL; \
     rm -rf /root/packages/vitastor-$REL/*; \
     cd /root/packages/vitastor-$REL; \
-    cp -r /root/vitastor vitastor-1.6.0; \
+    cp -r /root/vitastor vitastor-1.6.1; \
-    cd vitastor-1.6.0; \
+    cd vitastor-1.6.1; \
     ln -s /root/fio-build/fio-*/ ./fio; \
     FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
@@ -51,8 +51,8 @@ RUN set -e -x; \
     rm -rf a b; \
     echo "dep:fio=$FIO" > debian/fio_version; \
     cd /root/packages/vitastor-$REL; \
-    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_1.6.0.orig.tar.xz vitastor-1.6.0; \
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_1.6.1.orig.tar.xz vitastor-1.6.1; \
-    cd vitastor-1.6.0; \
+    cd vitastor-1.6.1; \
     V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \
     DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \

docs/config/pool.en.md

@@ -86,7 +86,11 @@ Parent node reference is required for intermediate tree nodes.
 Separate OSD settings are set in etc keys `/vitastor/config/osd/<number>`
 in JSON format `{"<key>":<value>}`.
 
-As of now, two settings are supported:
+As of now, the following settings are supported:
+
+- [reweight](#reweight)
+- [tags](#tags)
+- [noout](#noout)
 
 ## reweight
 
@@ -109,6 +113,14 @@ subsets and then use a specific subset for pool instead of all OSDs.
 For example you can mark SSD OSDs with tag "ssd" and HDD OSDs with "hdd" and
 such tags will work as device classes.
 
+## noout
+
+- Type: boolean
+- Default: false
+
+If set to true, [osd_out_time](monitor.en.md#osd_out_time) is ignored for this
+OSD and it's never removed from data distribution by the monitor.
+
 # Pool parameters
 
 ## name

docs/config/pool.ru.md

@@ -85,10 +85,11 @@
 Настройки отдельных OSD задаются в ключах etcd `/vitastor/config/osd/<number>`
 в JSON-формате `{"<key>":<value>}`.
 
-На данный момент поддерживаются две настройки:
+На данный момент поддерживаются следующие настройки:
 
 - [reweight](#reweight)
 - [tags](#tags)
+- [noout](#noout)
 
 ## reweight
 
@@ -112,6 +113,14 @@
 всех. Можно, например, пометить SSD OSD тегом "ssd", а HDD тегом "hdd", в
 этом смысле теги работают аналогично классам устройств.
 
+## noout
+
+- Тип: булево (да/нет)
+- Значение по умолчанию: false
+
+Если установлено в true, то [osd_out_time](monitor.ru.md#osd_out_time) для этого
+OSD игнорируется и OSD не удаляется из распределения данных монитором.
+
 # Параметры
 
 ## name

docs/installation/source.en.md

@@ -41,7 +41,7 @@ It's recommended to build the QEMU driver (qemu_driver.c) in-tree, as a part of
 QEMU build process. To do that:
 - Install vitastor client library headers (from source or from vitastor-client-dev package)
 - Take a corresponding patch from `patches/qemu-*-vitastor.patch` and apply it to QEMU source
-- Copy `src/qemu_driver.c` to QEMU source directory as `block/vitastor.c`
+- Copy `src/client/qemu_driver.c` to QEMU source directory as `block/vitastor.c`
 - Build QEMU as usual
 
 But it is also possible to build it out-of-tree. To do that:

docs/installation/source.ru.md

@@ -41,7 +41,7 @@ cmake .. && make -j8 install
 Драйвер QEMU (qemu_driver.c) рекомендуется собирать вместе с самим QEMU. Для этого:
 - Установите заголовки клиентской библиотеки Vitastor (из исходников или из пакета vitastor-client-dev)
 - Возьмите соответствующий патч из `patches/qemu-*-vitastor.patch` и примените его к исходникам QEMU
-- Скопируйте [src/qemu_driver.c](../../src/qemu_driver.c) в директорию исходников QEMU как `block/vitastor.c`
+- Скопируйте [src/client/qemu_driver.c](../../src/client/qemu_driver.c) в директорию исходников QEMU как `block/vitastor.c`
 - Соберите QEMU как обычно
 
 Однако в целях отладки драйвер также можно собирать отдельно от QEMU. Для этого:

mon/.eslintrc.js

@@ -0,0 +1,49 @@
+module.exports = {
+    "env": {
+        "es6": true,
+        "node": true
+    },
+    "extends": [
+        "eslint:recommended",
+        "plugin:node/recommended"
+    ],
+    "parserOptions": {
+        "ecmaVersion": 2020
+    },
+    "plugins": [
+    ],
+    "rules": {
+        "indent": [
+            "error",
+            4
+        ],
+        "brace-style": [
+            "error",
+            "allman",
+            { "allowSingleLine": true }
+        ],
+        "linebreak-style": [
+            "error",
+            "unix"
+        ],
+        "semi": [
+            "error",
+            "always"
+        ],
+        "no-useless-escape": [
+            "off"
+        ],
+        "no-control-regex": [
+            "off"
+        ],
+        "no-empty": [
+            "off"
+        ],
+        "no-process-exit": [
+            "off"
+        ],
+        "node/shebang": [
+            "off"
+        ]
+    }
+};

mon/PGUtil.js

@@ -97,7 +97,6 @@ function scale_pg_history(prev_pg_history, prev_pgs, new_pgs)
 
 function scale_pg_count(prev_pgs, new_pg_count)
 {
-    const old_pg_count = prev_pgs.length;
     // Just for the lp_solve optimizer - pick a "previous" PG for each "new" one
     if (prev_pgs.length < new_pg_count)
     {

mon/dsl_pgs.js

@@ -152,6 +152,7 @@ function parse_pg_dsl(text)
             else if (rule[2] === '(')
             {
                 rule[2] = [];
+                // eslint-disable-next-line no-constant-condition
                 while (true)
                 {
                     if (i > tokens.length-1)

mon/lp-optimizer.js

@@ -77,7 +77,7 @@ async function optimize_initial({ osd_weights, combinator, pg_count, pg_size = 3
     {
         if (osd !== NO_OSD)
         {
-            let osd_pg_count = (osd_weights[osd]||0)/total_weight*pg_effsize*pg_count;
+            let osd_pg_count = osd_weights[osd]/total_weight*pg_effsize*pg_count;
             lp += pg_per_osd[osd].join(' + ')+' <= '+osd_pg_count+';\n';
         }
     }
@@ -215,7 +215,7 @@ function calc_intersect_weights(old_pg_size, pg_size, pg_count, prev_weights, al
                 {
                     const intersect_count = ordered
                         ? pg.reduce((a, osd, i) => a + (prev_hash[osd] == 1+i ? 1 : 0), 0)
-                        : pg.reduce((a, osd, i) => a + (prev_hash[osd] ? 1 : 0), 0);
+                        : pg.reduce((a, osd) => a + (prev_hash[osd] ? 1 : 0), 0);
                     if (max_int < intersect_count)
                     {
                         max_int = intersect_count;
@@ -299,7 +299,7 @@ async function optimize_change({ prev_pgs: prev_int_pgs, osd_weights, combinator
             )).join(' + ');
             const rm_osd_pg_count = (prev_pg_per_osd[osd]||[])
                 .reduce((a, [ old_pg_name, space ]) => (a + (all_pgs_hash[old_pg_name] ? space : 0)), 0);
-            const osd_pg_count = (osd_weights[osd]||0)*pg_effsize/total_weight*pg_count - rm_osd_pg_count;
+            const osd_pg_count = osd_weights[osd]*pg_effsize/total_weight*pg_count - rm_osd_pg_count;
             lp += osd_sum + ' <= ' + osd_pg_count + ';\n';
         }
     }

mon/mon.js

@@ -37,7 +37,6 @@ const etcd_allow = new RegExp('^'+[
     'pg/state/[1-9]\\d*/[1-9]\\d*',
     'pg/stats/[1-9]\\d*/[1-9]\\d*',
     'pg/history/[1-9]\\d*/[1-9]\\d*',
-    'pool/stats/[1-9]\\d*',
     'history/last_clean_pgs',
     'inode/stats/[1-9]\\d*/\\d+',
     'pool/stats/[1-9]\\d*',
@@ -215,7 +214,7 @@ const etcd_tree = {
         }, */
         pools: {},
         osd: {
-            /* <id>: { reweight?: 1, tags?: [ 'nvme', ... ] }, ... */
+            /* <id>: { reweight?: 1, tags?: [ 'nvme', ... ], noout?: true }, ... */
         },
         /* pgs: {
             hash: string,
@@ -585,7 +584,7 @@ class Mon
                 now = Date.now();
             }
             tried[base] = now;
-            const ok = await new Promise((ok, no) =>
+            const ok = await new Promise(ok =>
             {
                 const timer_id = setTimeout(() =>
                 {
@@ -747,6 +746,7 @@ class Mon
         this.save_last_clean_running = true;
         // last_clean_pgs is used to avoid extra data move when observing a series of changes in the cluster
         const new_clean_pgs = { items: {} };
+        // eslint-disable-next-line indent
     next_pool:
         for (const pool_id in this.state.config.pools)
         {
@@ -829,6 +829,7 @@ class Mon
     async become_master()
     {
         const state = { ...this.get_mon_state(), id: ''+this.etcd_lease_id };
+        // eslint-disable-next-line no-constant-condition
         while (1)
         {
             const res = await this.etcd_call('/kv/txn', {
@@ -873,32 +874,19 @@ class Mon
         levels.osd = levels.osd || 101;
         const tree = {};
         let up_osds = {};
-        for (const node_id in this.state.config.node_placement||{})
-        {
-            const node_cfg = this.state.config.node_placement[node_id];
-            if (/^\d+$/.exec(node_id))
-            {
-                node_cfg.level = 'osd';
-            }
-            if (!node_id || !node_cfg.level || !levels[node_cfg.level])
-            {
-                // All nodes must have non-empty IDs and valid levels
-                continue;
-            }
-            tree[node_id] = { id: node_id, level: node_cfg.level, parent: node_cfg.parent, children: [] };
-        }
         // This requires monitor system time to be in sync with OSD system times (at least to some extent)
         const down_time = Date.now()/1000 - this.config.osd_out_time;
         for (const osd_num of this.all_osds().sort((a, b) => a - b))
         {
             const stat = this.state.osd.stats[osd_num];
-            if (stat && stat.size && (this.state.osd.state[osd_num] || Number(stat.time) >= down_time))
+            const osd_cfg = this.state.config.osd[osd_num];
+            let reweight = osd_cfg == null ? 1 : Number(osd_cfg.reweight);
+            if (reweight < 0 || isNaN(reweight))
+                reweight = 1;
+            if (stat && stat.size && reweight && (this.state.osd.state[osd_num] || Number(stat.time) >= down_time ||
+                osd_cfg && osd_cfg.noout))
             {
                 // Numeric IDs are reserved for OSDs
-                const osd_cfg = this.state.config.osd[osd_num];
-                let reweight = osd_cfg == null ? 1 : Number(osd_cfg.reweight);
-                if (reweight < 0 || isNaN(reweight))
-                    reweight = 1;
                 if (this.state.osd.state[osd_num] && reweight > 0)
                 {
                     // React to down OSDs immediately
@@ -926,6 +914,29 @@ class Mon
                 }
             }
         }
+        for (const node_id in this.state.config.node_placement||{})
+        {
+            const node_cfg = this.state.config.node_placement[node_id];
+            if (/^\d+$/.exec(node_id))
+            {
+                node_cfg.level = 'osd';
+            }
+            if (!node_id || !node_cfg.level || !levels[node_cfg.level] ||
+                node_cfg.level === 'osd' && !tree[node_id])
+            {
+                // All nodes must have non-empty IDs and valid levels
+                // OSDs have to actually exist
+                continue;
+            }
+            tree[node_id] = tree[node_id] || {};
+            tree[node_id].id = node_id;
+            tree[node_id].level = node_cfg.level;
+            tree[node_id].parent = node_cfg.parent;
+            if (node_cfg.level !== 'osd')
+            {
+                tree[node_id].children = [];
+            }
+        }
         return { up_osds, levels, osd_tree: tree };
     }
 
@@ -955,6 +966,25 @@ class Mon
             const parent = parent_level && parent_level < node_level ? node_cfg.parent : '';
             tree[parent].children.push(tree[node_id]);
         }
+        // Delete empty nodes
+        let deleted = 0;
+        do
+        {
+            deleted = 0;
+            for (const node_id in tree)
+            {
+                if (tree[node_id].level !== 'osd' && (!tree[node_id].children || !tree[node_id].children.length))
+                {
+                    const parent = tree[node_id].parent;
+                    if (parent)
+                    {
+                        tree[parent].children = tree[parent].children.filter(c => c != tree[node_id]);
+                    }
+                    deleted++;
+                    delete tree[node_id];
+                }
+            }
+        } while (deleted > 0);
         return tree;
     }
 
@@ -991,7 +1021,7 @@ class Mon
                 const key = b64(this.etcd_prefix+'/osd/state/'+osd_num);
                 checks.push({ key, target: 'MOD', result: 'LESS', mod_revision: ''+this.etcd_watch_revision });
             }
-            const res = await this.etcd_call('/kv/txn', {
+            await this.etcd_call('/kv/txn', {
                 compare: [
                     { key: b64(this.etcd_prefix+'/mon/master'), target: 'LEASE', lease: ''+this.etcd_lease_id },
                     { key: b64(this.etcd_prefix+'/config/pgs'), target: 'MOD', mod_revision: ''+this.etcd_watch_revision, result: 'LESS' },
@@ -1315,7 +1345,7 @@ class Mon
             for (const k in rules)
             {
                 if (!levels[k] || typeof rules[k] !== 'string' &&
-                    (!rules[k] instanceof Array ||
+                    (!(rules[k] instanceof Array) ||
                     rules[k].filter(s => typeof s !== 'string' && typeof s !== 'number').length > 0))
                 {
                     if (warn)
@@ -1429,7 +1459,15 @@ class Mon
         }
         console.log(`Pool ${pool_id} (${pool_cfg.name || 'unnamed'}):`);
         LPOptimizer.print_change_stats(optimize_result);
-        const pg_effsize = Math.min(pool_cfg.pg_size, Object.keys(pool_tree).length);
+        let pg_effsize = pool_cfg.pg_size;
+        for (const pg of optimize_result.int_pgs)
+        {
+            const this_pg_size = pg.filter(osd => osd != LPOptimizer.NO_OSD).length;
+            if (this_pg_size && this_pg_size < pg_effsize)
+            {
+                pg_effsize = this_pg_size;
+            }
+        }
         return {
             pool_id,
             pgs: optimize_result.int_pgs,
@@ -1511,11 +1549,14 @@ class Mon
                 {
                     continue;
                 }
-                const replicated = pool_cfg.scheme === 'replicated';
                 const aff_osds = this.get_affinity_osds(pool_cfg, up_osds, osd_tree);
                 this.reset_rng();
                 for (let pg_num = 1; pg_num <= pool_cfg.pg_count; pg_num++)
                 {
+                    if (!this.state.config.pgs.items[pool_id])
+                    {
+                        continue;
+                    }
                     const pg_cfg = this.state.config.pgs.items[pool_id][pg_num];
                     if (pg_cfg)
                     {
@@ -1553,10 +1594,6 @@ class Mon
 
     async apply_pool_pgs(results, up_osds, osd_tree, tree_hash)
     {
-        if (!results.length)
-        {
-            return true;
-        }
         for (const pool_id in (this.state.config.pgs||{}).items||{})
         {
             // We should stop all PGs when deleting a pool or changing its PG count
@@ -1689,7 +1726,6 @@ class Mon
 
     derive_osd_stats(st, prev, prev_diff)
     {
-        const zero_stats = { op: { bps: 0n, iops: 0n, lat: 0n }, subop: { iops: 0n, lat: 0n }, recovery: { bps: 0n, iops: 0n } };
         const diff = { op_stats: {}, subop_stats: {}, recovery_stats: {}, inode_stats: {} };
         if (!st || !st.time || !prev || !prev.time || prev.time >= st.time)
         {
@@ -1729,7 +1765,7 @@ class Mon
         }
         for (const pool_id in st.inode_stats||{})
         {
-            const pool_diff = diff.inode_stats[pool_id] = {};
+            diff.inode_stats[pool_id] = {};
             for (const inode_num in st.inode_stats[pool_id])
             {
                 const inode_diff = diff.inode_stats[pool_id][inode_num] = {};
@@ -2147,7 +2183,7 @@ class Mon
     _die(err, code)
     {
         // In fact we can just try to rejoin
-        console.error(new Error(err || 'Cluster connection failed'));
+        console.error(err instanceof Error ? err : new Error(err || 'Cluster connection failed'));
         process.exit(code || 2);
     }
 
@@ -2171,7 +2207,7 @@ class Mon
 
 function POST(url, body, timeout)
 {
-    return new Promise((ok, no) =>
+    return new Promise(ok =>
     {
         const body_text = Buffer.from(JSON.stringify(body));
         let timer_id = timeout > 0 ? setTimeout(() =>

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "1.6.0",
+  "version": "1.6.1",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {
@@ -11,5 +11,15 @@
   "dependencies": {
     "sprintf-js": "^1.1.2",
     "ws": "^7.2.5"
+  },
+  "devDependencies": {
+    "eslint": "^8.0.0",
+    "eslint-plugin-node": "^11.1.0"
+  },
+  "engines": {
+    "node": ">=12.0.0"
+  },
+  "scripts": {
+    "lint": "eslint *.js"
   }
 }

mon/simple-offsets.js

@@ -38,7 +38,7 @@ async function run()
         const st = await fs.stat(options.device);
         options.device_block_size = st.blksize;
         if (st.isBlockDevice())
-            device_size = Number(await system("/sbin/blockdev --getsize64 "+options.device))
+            device_size = Number(await system("/sbin/blockdev --getsize64 "+options.device));
         else
             device_size = st.size;
     }
@@ -91,7 +91,7 @@ async function run()
 
 function system(cmd)
 {
-    return new Promise((ok, no) => child_process.exec(cmd, { maxBuffer: 64*1024*1024 }, (err, stdout, stderr) => (err ? no(err.message) : ok(stdout))));
+    return new Promise((ok, no) => child_process.exec(cmd, { maxBuffer: 64*1024*1024 }, (err, stdout/*, stderr*/) => (err ? no(err.message) : ok(stdout))));
 }
 
 run().catch(err => { console.error(err); process.exit(1); });

mon/simple_pgs.js

@@ -198,7 +198,6 @@ function all_combinations(osd_tree, pg_size, ordered, count)
 
 function check_combinations(osd_tree, pgs)
 {
-    const hosts = Object.keys(osd_tree).sort();
     const host_per_osd = {};
     for (const host in osd_tree)
     {
@@ -235,6 +234,7 @@ function compat(params)
 
 module.exports = {
     flatten_tree,
+    all_combinations,
     SimpleCombinator,
     compat,
     NO_OSD,

mon/test-optimize.js

@@ -20,7 +20,7 @@ const osd_tree = {
     },
     500: {
         4: 3.58498,
-//        8: 3.58589,
+        /*8: 3.58589,*/
         9: 3.63869,
     },
     600: {

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VERSION = '1.6.0'
+VERSION = '1.6.1'
 
 LOG = logging.getLogger(__name__)
 

patches/qemu-make-patches.sh

@@ -9,7 +9,7 @@ for i in "$DIR"/qemu-*-vitastor.patch "$DIR"/pve-qemu-*-vitastor.patch; do
         echo '===================================================================' >> $i
         echo '--- /dev/null' >> $i
         echo '+++ a/block/vitastor.c' >> $i
-        echo '@@ -0,0 +1,'$(wc -l "$DIR"/../src/qemu_driver.c | cut -d ' ' -f 1)' @@' >> $i
+        echo '@@ -0,0 +1,'$(wc -l "$DIR"/../src/client/qemu_driver.c | cut -d ' ' -f 1)' @@' >> $i
-        cat "$DIR"/../src/qemu_driver.c | sed 's/^/+/' >> $i
+        cat "$DIR"/../src/client/qemu_driver.c | sed 's/^/+/' >> $i
     fi
 done

rpm/build-tarball.sh

@@ -24,4 +24,4 @@ rm fio
 mv fio-copy fio
 FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
 perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec
-tar --transform 's#^#vitastor-1.6.0/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-1.6.0$(rpm --eval '%dist').tar.gz *
+tar --transform 's#^#vitastor-1.6.1/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-1.6.1$(rpm --eval '%dist').tar.gz *

rpm/qemu-el8.Dockerfile

@@ -1,5 +1,5 @@
 # This is an attempt to automatically build patched RPM specs
-# More or less broken, better use *.spec.patch for now (and copy src/qemu_driver.c to SOURCES/qemu-vitastor.c)
+# More or less broken, better use *.spec.patch for now (and copy src/client/qemu_driver.c to SOURCES/qemu-vitastor.c)
 
 # Build packages for CentOS 8 inside a container
 # cd ..; podman build -t qemu-el8 -v `pwd`/packages:/root/packages -f rpm/qemu-el8.Dockerfile .

rpm/vitastor-el7.Dockerfile

@@ -36,7 +36,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-1.6.0.el7.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-1.6.1.el7.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el7.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.6.0
+Version:        1.6.1
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.6.0.el7.tar.gz
+Source0:        vitastor-1.6.1.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel
@@ -104,7 +104,7 @@ rm -rf $RPM_BUILD_ROOT
 %make_install
 . /opt/rh/rh-nodejs12/enable
 cd mon
-npm install
+npm install --production
 cd ..
 mkdir -p %buildroot/usr/lib/vitastor
 cp -r mon %buildroot/usr/lib/vitastor

rpm/vitastor-el8.Dockerfile

@@ -35,7 +35,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-1.6.0.el8.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-1.6.1.el8.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el8.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.6.0
+Version:        1.6.1
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.6.0.el8.tar.gz
+Source0:        vitastor-1.6.1.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel
@@ -101,7 +101,7 @@ Vitastor fio drivers for benchmarking.
 rm -rf $RPM_BUILD_ROOT
 %make_install
 cd mon
-npm install
+npm install --production
 cd ..
 mkdir -p %buildroot/usr/lib/vitastor
 cp -r mon %buildroot/usr/lib/vitastor

rpm/vitastor-el9.Dockerfile

@@ -18,7 +18,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-1.6.0.el9.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-1.6.1.el9.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el9.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.6.0
+Version:        1.6.1
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.6.0.el9.tar.gz
+Source0:        vitastor-1.6.1.el9.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel
@@ -94,7 +94,7 @@ Vitastor fio drivers for benchmarking.
 rm -rf $RPM_BUILD_ROOT
 %cmake_install
 cd mon
-npm install
+npm install --production
 cd ..
 mkdir -p %buildroot/usr/lib/vitastor
 cp -r mon %buildroot/usr/lib/vitastor

src/CMakeLists.txt

@@ -19,7 +19,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVERSION="1.6.0")
+add_definitions(-DVERSION="1.6.1")
 add_definitions(-D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})
@@ -72,300 +72,28 @@ add_dependencies(test build_tests)
 
 include_directories(
 	../
+	${CMAKE_SOURCE_DIR}/src/blockstore
+	${CMAKE_SOURCE_DIR}/src/cmd
+	${CMAKE_SOURCE_DIR}/src/client
+	${CMAKE_SOURCE_DIR}/src/disk_tool
+	${CMAKE_SOURCE_DIR}/src/kv
+	${CMAKE_SOURCE_DIR}/src/nfs
+	${CMAKE_SOURCE_DIR}/src/osd
+	${CMAKE_SOURCE_DIR}/src/test
+	${CMAKE_SOURCE_DIR}/src/util
 	/usr/include/jerasure
 	${LIBURING_INCLUDE_DIRS}
 	${IBVERBS_INCLUDE_DIRS}
 )
 
-# libvitastor_blk.so
+add_subdirectory(blockstore)
-add_library(vitastor_blk SHARED
+add_subdirectory(cmd)
-	allocator.cpp blockstore.cpp blockstore_impl.cpp blockstore_disk.cpp blockstore_init.cpp blockstore_open.cpp blockstore_journal.cpp blockstore_read.cpp
+add_subdirectory(client)
-	blockstore_write.cpp blockstore_sync.cpp blockstore_stable.cpp blockstore_rollback.cpp blockstore_flush.cpp crc32c.c ringloop.cpp
+add_subdirectory(disk_tool)
-)
+add_subdirectory(kv)
-target_link_libraries(vitastor_blk
+add_subdirectory(nfs)
-	${LIBURING_LIBRARIES}
+add_subdirectory(osd)
-	tcmalloc_minimal
+add_subdirectory(test)
-	# for timerfd_manager
-	vitastor_common
-)
-set_target_properties(vitastor_blk PROPERTIES VERSION ${VERSION} SOVERSION 0)
-
-if (${WITH_FIO})
-	# libfio_vitastor_blk.so
-	add_library(fio_vitastor_blk SHARED
-		fio_engine.cpp
-		../json11/json11.cpp
-	)
-	target_link_libraries(fio_vitastor_blk
-		vitastor_blk
-	)
-endif (${WITH_FIO})
-
-# libvitastor_common.a
-set(MSGR_RDMA "")
-if (IBVERBS_LIBRARIES)
-	set(MSGR_RDMA "msgr_rdma.cpp")
-endif (IBVERBS_LIBRARIES)
-add_library(vitastor_common STATIC
-	epoll_manager.cpp etcd_state_client.cpp messenger.cpp addr_util.cpp
-	msgr_stop.cpp msgr_op.cpp msgr_send.cpp msgr_receive.cpp ringloop.cpp ../json11/json11.cpp
-	http_client.cpp osd_ops.cpp pg_states.cpp timerfd_manager.cpp str_util.cpp ${MSGR_RDMA}
-)
-target_compile_options(vitastor_common PUBLIC -fPIC)
-
-# vitastor-osd
-add_executable(vitastor-osd
-	osd_main.cpp osd.cpp osd_secondary.cpp osd_peering.cpp osd_flush.cpp osd_peering_pg.cpp
-	osd_primary.cpp osd_primary_chain.cpp osd_primary_sync.cpp osd_primary_write.cpp osd_primary_subops.cpp
-	osd_cluster.cpp osd_rmw.cpp osd_scrub.cpp osd_primary_describe.cpp
-)
-target_link_libraries(vitastor-osd
-	vitastor_common
-	vitastor_blk
-	Jerasure
-	${ISAL_LIBRARIES}
-	${IBVERBS_LIBRARIES}
-)
-
-if (${WITH_FIO})
-	# libfio_vitastor_sec.so
-	add_library(fio_vitastor_sec SHARED
-		fio_sec_osd.cpp
-		rw_blocking.cpp
-		addr_util.cpp
-	)
-	target_link_libraries(fio_vitastor_sec
-		tcmalloc_minimal
-	)
-endif (${WITH_FIO})
-
-# libvitastor_client.so
-add_library(vitastor_client SHARED
-	cluster_client.cpp
-	cluster_client_list.cpp
-	cluster_client_wb.cpp
-	vitastor_c.cpp
-	cli_common.cpp
-	cli_alloc_osd.cpp
-	cli_status.cpp
-	cli_describe.cpp
-	cli_fix.cpp
-	cli_ls.cpp
-	cli_create.cpp
-	cli_modify.cpp
-	cli_flatten.cpp
-	cli_merge.cpp
-	cli_rm_data.cpp
-	cli_rm.cpp
-	cli_rm_osd.cpp
-	cli_pool_cfg.cpp
-	cli_pool_create.cpp
-	cli_pool_ls.cpp
-	cli_pool_modify.cpp
-	cli_pool_rm.cpp
-)
-set_target_properties(vitastor_client PROPERTIES PUBLIC_HEADER "vitastor_c.h")
-target_link_libraries(vitastor_client
-	vitastor_common
-	${LIBURING_LIBRARIES}
-	${IBVERBS_LIBRARIES}
-)
-set_target_properties(vitastor_client PROPERTIES VERSION ${VERSION} SOVERSION 0)
-
-if (${WITH_FIO})
-	# libfio_vitastor.so
-	add_library(fio_vitastor SHARED
-		fio_cluster.cpp
-	)
-	target_link_libraries(fio_vitastor
-		vitastor_client
-	)
-endif (${WITH_FIO})
-
-# vitastor-nbd
-pkg_check_modules(NL3 libnl-3.0 libnl-genl-3.0)
-add_executable(vitastor-nbd
-	nbd_proxy.cpp
-)
-target_include_directories(vitastor-nbd PUBLIC ${NL3_INCLUDE_DIRS})
-target_link_libraries(vitastor-nbd vitastor_client ${NL3_LIBRARIES})
-if (HAVE_NBD_NETLINK_H AND NL3_LIBRARIES)
-	target_compile_definitions(vitastor-nbd PUBLIC HAVE_NBD_NETLINK_H)
-endif (HAVE_NBD_NETLINK_H AND NL3_LIBRARIES)
-
-# libvitastor_kv.so
-add_library(vitastor_kv SHARED
-	kv_db.cpp
-	kv_db.h
-)
-target_link_libraries(vitastor_kv
-	vitastor_client
-)
-set_target_properties(vitastor_kv PROPERTIES VERSION ${VERSION} SOVERSION 0)
-
-# vitastor-kv
-add_executable(vitastor-kv
-	kv_cli.cpp
-)
-target_link_libraries(vitastor-kv
-	vitastor_kv
-)
-
-add_executable(vitastor-kv-stress
-	kv_stress.cpp
-)
-target_link_libraries(vitastor-kv-stress
-	vitastor_kv
-)
-
-# vitastor-nfs
-add_executable(vitastor-nfs
-	nfs_proxy.cpp
-	nfs_block.cpp
-	nfs_kv.cpp
-	nfs_kv_create.cpp
-	nfs_kv_getattr.cpp
-	nfs_kv_link.cpp
-	nfs_kv_lookup.cpp
-	nfs_kv_read.cpp
-	nfs_kv_readdir.cpp
-	nfs_kv_remove.cpp
-	nfs_kv_rename.cpp
-	nfs_kv_setattr.cpp
-	nfs_kv_write.cpp
-	nfs_fsstat.cpp
-	nfs_mount.cpp
-	nfs_portmap.cpp
-	sha256.c
-	nfs/xdr_impl.cpp
-	nfs/rpc_xdr.cpp
-	nfs/portmap_xdr.cpp
-	nfs/nfs_xdr.cpp
-)
-target_link_libraries(vitastor-nfs
-	vitastor_client
-	vitastor_kv
-)
-
-# vitastor-cli
-add_executable(vitastor-cli
-	cli.cpp
-)
-target_link_libraries(vitastor-cli
-	vitastor_client
-)
-configure_file(vitastor.pc.in vitastor.pc @ONLY)
-
-# vitastor-disk
-add_executable(vitastor-disk
-	disk_tool.cpp disk_simple_offsets.cpp
-	disk_tool_journal.cpp disk_tool_meta.cpp disk_tool_prepare.cpp disk_tool_resize.cpp disk_tool_udev.cpp disk_tool_utils.cpp disk_tool_upgrade.cpp
-	crc32c.c str_util.cpp ../json11/json11.cpp rw_blocking.cpp allocator.cpp ringloop.cpp blockstore_disk.cpp
-)
-target_link_libraries(vitastor-disk
-	tcmalloc_minimal
-	${LIBURING_LIBRARIES}
-)
-
-if (${WITH_QEMU})
-	# qemu_driver.so
-	add_library(qemu_vitastor SHARED
-		qemu_driver.c
-	)
-	target_compile_options(qemu_vitastor PUBLIC -DVITASTOR_SOURCE_TREE)
-	target_include_directories(qemu_vitastor PUBLIC
-		../qemu/b/qemu
-		../qemu/include
-		${GLIB_INCLUDE_DIRS}
-	)
-	target_link_libraries(qemu_vitastor
-		vitastor_client
-	)
-	set_target_properties(qemu_vitastor PROPERTIES
-		PREFIX ""
-		OUTPUT_NAME "block-vitastor"
-	)
-endif (${WITH_QEMU})
-
-### Test stubs
-
-# stub_osd, stub_bench, osd_test
-add_executable(stub_osd stub_osd.cpp rw_blocking.cpp addr_util.cpp)
-target_link_libraries(stub_osd tcmalloc_minimal)
-add_executable(stub_bench stub_bench.cpp rw_blocking.cpp addr_util.cpp)
-target_link_libraries(stub_bench tcmalloc_minimal)
-add_executable(osd_test osd_test.cpp rw_blocking.cpp addr_util.cpp)
-target_link_libraries(osd_test tcmalloc_minimal)
-
-# osd_rmw_test
-add_executable(osd_rmw_test EXCLUDE_FROM_ALL osd_rmw_test.cpp allocator.cpp)
-target_link_libraries(osd_rmw_test Jerasure ${ISAL_LIBRARIES} tcmalloc_minimal)
-add_dependencies(build_tests osd_rmw_test)
-add_test(NAME osd_rmw_test COMMAND osd_rmw_test)
-
-if (ISAL_LIBRARIES)
-	add_executable(osd_rmw_test_je EXCLUDE_FROM_ALL osd_rmw_test.cpp allocator.cpp)
-	target_compile_definitions(osd_rmw_test_je PUBLIC -DNO_ISAL)
-	target_link_libraries(osd_rmw_test_je Jerasure tcmalloc_minimal)
-	add_dependencies(build_tests osd_rmw_test_je)
-	add_test(NAME osd_rmw_test_jerasure COMMAND osd_rmw_test_je)
-endif (ISAL_LIBRARIES)
-
-# stub_uring_osd
-add_executable(stub_uring_osd
-	stub_uring_osd.cpp
-)
-target_link_libraries(stub_uring_osd
-	vitastor_common
-	${LIBURING_LIBRARIES}
-	${IBVERBS_LIBRARIES}
-	tcmalloc_minimal
-)
-
-# osd_peering_pg_test
-add_executable(osd_peering_pg_test EXCLUDE_FROM_ALL osd_peering_pg_test.cpp osd_peering_pg.cpp)
-target_link_libraries(osd_peering_pg_test tcmalloc_minimal)
-add_dependencies(build_tests osd_peering_pg_test)
-add_test(NAME osd_peering_pg_test COMMAND osd_peering_pg_test)
-
-# test_allocator
-add_executable(test_allocator EXCLUDE_FROM_ALL test_allocator.cpp allocator.cpp)
-add_dependencies(build_tests test_allocator)
-add_test(NAME test_allocator COMMAND test_allocator)
-
-# test_cas
-add_executable(test_cas
-	test_cas.cpp
-)
-target_link_libraries(test_cas
-	vitastor_client
-)
-
-# test_crc32
-add_executable(test_crc32
-	test_crc32.cpp
-)
-target_link_libraries(test_crc32
-	vitastor_blk
-)
-
-# test_cluster_client
-add_executable(test_cluster_client
-	EXCLUDE_FROM_ALL
-	test_cluster_client.cpp
-	pg_states.cpp osd_ops.cpp cluster_client.cpp cluster_client_list.cpp cluster_client_wb.cpp msgr_op.cpp mock/messenger.cpp msgr_stop.cpp
-	etcd_state_client.cpp timerfd_manager.cpp str_util.cpp ../json11/json11.cpp
-)
-target_compile_definitions(test_cluster_client PUBLIC -D__MOCK__)
-target_include_directories(test_cluster_client PUBLIC ${CMAKE_SOURCE_DIR}/src/mock)
-add_dependencies(build_tests test_cluster_client)
-add_test(NAME test_cluster_client COMMAND test_cluster_client)
-
-## test_blockstore, test_shit
-#add_executable(test_blockstore test_blockstore.cpp)
-#target_link_libraries(test_blockstore blockstore)
-#add_executable(test_shit test_shit.cpp osd_peering_pg.cpp)
-#target_link_libraries(test_shit ${LIBURING_LIBRARIES} m)
 
 ### Install
 
@@ -378,7 +106,7 @@ install(
 	LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
 	PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
 )
-install(FILES ${CMAKE_CURRENT_BINARY_DIR}/vitastor.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
+install(FILES ${CMAKE_CURRENT_BINARY_DIR}/client/vitastor.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
 if (${WITH_FIO})
 	install(TARGETS fio_vitastor fio_vitastor_blk fio_vitastor_sec LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
 endif (${WITH_FIO})

src/blockstore/CMakeLists.txt

@@ -0,0 +1,27 @@
+cmake_minimum_required(VERSION 2.8.12)
+
+project(vitastor)
+
+# libvitastor_blk.so
+add_library(vitastor_blk SHARED
+	../util/allocator.cpp blockstore.cpp blockstore_impl.cpp blockstore_disk.cpp blockstore_init.cpp blockstore_open.cpp blockstore_journal.cpp blockstore_read.cpp
+	blockstore_write.cpp blockstore_sync.cpp blockstore_stable.cpp blockstore_rollback.cpp blockstore_flush.cpp ../util/crc32c.c ../util/ringloop.cpp
+)
+target_link_libraries(vitastor_blk
+	${LIBURING_LIBRARIES}
+	tcmalloc_minimal
+	# for timerfd_manager
+	vitastor_common
+)
+set_target_properties(vitastor_blk PROPERTIES VERSION ${VERSION} SOVERSION 0)
+
+if (${WITH_FIO})
+	# libfio_vitastor_blk.so
+	add_library(fio_vitastor_blk SHARED
+		fio_engine.cpp
+		../../json11/json11.cpp
+	)
+	target_link_libraries(fio_vitastor_blk
+		vitastor_blk
+	)
+endif (${WITH_FIO})

src/client/CMakeLists.txt

@@ -0,0 +1,95 @@
+cmake_minimum_required(VERSION 2.8.12)
+
+project(vitastor)
+
+# libvitastor_common.a
+set(MSGR_RDMA "")
+if (IBVERBS_LIBRARIES)
+	set(MSGR_RDMA "msgr_rdma.cpp")
+endif (IBVERBS_LIBRARIES)
+add_library(vitastor_common STATIC
+	../util/epoll_manager.cpp etcd_state_client.cpp messenger.cpp ../util/addr_util.cpp
+	msgr_stop.cpp msgr_op.cpp msgr_send.cpp msgr_receive.cpp ../util/ringloop.cpp ../../json11/json11.cpp
+	http_client.cpp osd_ops.cpp pg_states.cpp ../util/timerfd_manager.cpp ../util/str_util.cpp ${MSGR_RDMA}
+)
+target_compile_options(vitastor_common PUBLIC -fPIC)
+
+# libvitastor_client.so
+add_library(vitastor_client SHARED
+	cluster_client.cpp
+	cluster_client_list.cpp
+	cluster_client_wb.cpp
+	vitastor_c.cpp
+)
+set_target_properties(vitastor_client PROPERTIES PUBLIC_HEADER "client/vitastor_c.h")
+target_link_libraries(vitastor_client
+	vitastor_common
+	vitastor_cli
+	${LIBURING_LIBRARIES}
+	${IBVERBS_LIBRARIES}
+)
+set_target_properties(vitastor_client PROPERTIES VERSION ${VERSION} SOVERSION 0)
+configure_file(vitastor.pc.in vitastor.pc @ONLY)
+
+if (${WITH_FIO})
+	# libfio_vitastor.so
+	add_library(fio_vitastor SHARED
+		fio_cluster.cpp
+	)
+	target_link_libraries(fio_vitastor
+		vitastor_client
+	)
+
+	# libfio_vitastor_sec.so
+	add_library(fio_vitastor_sec SHARED
+		fio_sec_osd.cpp
+		../util/rw_blocking.cpp
+		../util/addr_util.cpp
+	)
+	target_link_libraries(fio_vitastor_sec
+		tcmalloc_minimal
+	)
+endif (${WITH_FIO})
+
+# vitastor-nbd
+pkg_check_modules(NL3 libnl-3.0 libnl-genl-3.0)
+add_executable(vitastor-nbd
+	nbd_proxy.cpp
+)
+target_include_directories(vitastor-nbd PUBLIC ${NL3_INCLUDE_DIRS})
+target_link_libraries(vitastor-nbd vitastor_client ${NL3_LIBRARIES})
+if (HAVE_NBD_NETLINK_H AND NL3_LIBRARIES)
+	target_compile_definitions(vitastor-nbd PUBLIC HAVE_NBD_NETLINK_H)
+endif (HAVE_NBD_NETLINK_H AND NL3_LIBRARIES)
+
+if (${WITH_QEMU})
+	# qemu_driver.so
+	add_library(qemu_vitastor SHARED
+		qemu_driver.c
+	)
+	target_compile_options(qemu_vitastor PUBLIC -DVITASTOR_SOURCE_TREE)
+	target_include_directories(qemu_vitastor PUBLIC
+		../../qemu/b/qemu
+		../../qemu/include
+		${GLIB_INCLUDE_DIRS}
+	)
+	target_link_libraries(qemu_vitastor
+		vitastor_client
+	)
+	set_target_properties(qemu_vitastor PROPERTIES
+		PREFIX ""
+		OUTPUT_NAME "block-vitastor"
+	)
+endif (${WITH_QEMU})
+
+# test_cluster_client
+add_executable(test_cluster_client
+	EXCLUDE_FROM_ALL
+	../test/test_cluster_client.cpp
+	pg_states.cpp osd_ops.cpp cluster_client.cpp cluster_client_list.cpp cluster_client_wb.cpp msgr_op.cpp ../test/mock/messenger.cpp msgr_stop.cpp
+	etcd_state_client.cpp ../util/timerfd_manager.cpp ../util/str_util.cpp ../../json11/json11.cpp
+)
+target_compile_definitions(test_cluster_client PUBLIC -D__MOCK__)
+target_include_directories(test_cluster_client BEFORE PUBLIC ${CMAKE_SOURCE_DIR}/src/test/mock)
+add_dependencies(build_tests test_cluster_client)
+add_test(NAME test_cluster_client COMMAND test_cluster_client)

src/client/etcd_state_client.h

@@ -20,9 +20,11 @@
 #define MAX_DATA_BLOCK_SIZE 128*1024*1024
 #define DEFAULT_BITMAP_GRANULARITY 4096
 
+#ifndef IMMEDIATE_NONE
 #define IMMEDIATE_NONE 0
 #define IMMEDIATE_SMALL 1
 #define IMMEDIATE_ALL 2
+#endif
 
 struct etcd_kv_t
 {

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 1.6.0
+Version: 1.6.1
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/client/vitastor_c.cpp

@@ -200,6 +200,11 @@ vitastor_c *vitastor_c_create_epoll_json(const char **options, int options_len)
     return self;
 }
 
+void* vitastor_c_get_internal_client(vitastor_c *client)
+{
+    return client->cli;
+}
+
 void vitastor_c_destroy(vitastor_c *client)
 {
     delete client->cli;
@@ -379,4 +384,12 @@ int vitastor_c_inode_get_readonly(void *handle)
     return watch->cfg.readonly;
 }
 
+uint32_t vitastor_c_inode_get_immediate_commit(vitastor_c *client, uint64_t inode_num)
+{
+    auto pool_it = client->cli->st_cli.pool_config.find(INODE_POOL(inode_num));
+    if (pool_it == client->cli->st_cli.pool_config.end())
+        return 0;
+    return pool_it->second.immediate_commit;
+}
+
 }

src/client/vitastor_c.h

@@ -7,11 +7,18 @@
 #define VITASTOR_QEMU_PROXY_H
 
 // C API wrapper version
-#define VITASTOR_C_API_VERSION 3
+#define VITASTOR_C_API_VERSION 4
 
 #ifndef POOL_ID_BITS
 #define POOL_ID_BITS 16
 #endif
+
+#ifndef IMMEDIATE_NONE
+#define IMMEDIATE_NONE 0
+#define IMMEDIATE_SMALL 1
+#define IMMEDIATE_ALL 2
+#endif
+
 #include <stdint.h>
 #include <sys/uio.h>
 
@@ -41,6 +48,7 @@ vitastor_c *vitastor_c_create_uring(const char *config_path, const char *etcd_ho
     int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level);
 vitastor_c *vitastor_c_create_uring_json(const char **options, int options_len);
 vitastor_c *vitastor_c_create_epoll_json(const char **options, int options_len);
+void* vitastor_c_get_internal_client(vitastor_c *client);
 void vitastor_c_destroy(vitastor_c *client);
 int vitastor_c_is_ready(vitastor_c *client);
 int vitastor_c_uring_register_eventfd(vitastor_c *client);
@@ -64,6 +72,8 @@ uint64_t vitastor_c_inode_get_num(void *handle);
 uint32_t vitastor_c_inode_get_block_size(vitastor_c *client, uint64_t inode_num);
 uint32_t vitastor_c_inode_get_bitmap_granularity(vitastor_c *client, uint64_t inode_num);
 int vitastor_c_inode_get_readonly(void *handle);
+// Returns IMMEDIATE_ALL, IMMEDIATE_SMALL or IMMEDIATE_NONE
+uint32_t vitastor_c_inode_get_immediate_commit(vitastor_c *client, uint64_t inode_num);
 
 #ifdef __cplusplus
 }

src/cmd/CMakeLists.txt

@@ -0,0 +1,34 @@
+cmake_minimum_required(VERSION 2.8.12)
+
+project(vitastor)
+
+# libvitastor_cli.a
+add_library(vitastor_cli STATIC
+	cli_common.cpp
+	cli_alloc_osd.cpp
+	cli_status.cpp
+	cli_describe.cpp
+	cli_fix.cpp
+	cli_ls.cpp
+	cli_create.cpp
+	cli_modify.cpp
+	cli_flatten.cpp
+	cli_merge.cpp
+	cli_rm_data.cpp
+	cli_rm.cpp
+	cli_rm_osd.cpp
+	cli_pool_cfg.cpp
+	cli_pool_create.cpp
+	cli_pool_ls.cpp
+	cli_pool_modify.cpp
+	cli_pool_rm.cpp
+)
+target_compile_options(vitastor_cli PUBLIC -fPIC)
+
+# vitastor-cli
+add_executable(vitastor-cli
+	cli.cpp
+)
+target_link_libraries(vitastor-cli
+	vitastor_client
+)