WIP Batch encrypted records to reduce EncryptInit/EncryptFinal costs

Try to run tests with AES-GCM
Implement direct AES-256-GCM with a static key for benchmark
2026-04-19 12:29:27 +00:00 · 2026-04-18 20:41:49 +03:00 · 2026-04-18 20:41:49 +03:00 · 2026-04-18 14:22:42 +03:00 · 2026-04-17 21:22:47 +03:00 · 2026-04-17 21:22:47 +03:00
111 changed files with 5952 additions and 1638 deletions
@@ -63,7 +63,7 @@ jobs:
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    # leak sanitizer sometimes crashes
-    - run: cd /root/vitastor/build && ASAN_OPTIONS=detect_leaks=0 make -j16 test
+    - run: cd /root/vitastor/build && ASAN_OPTIONS=detect_leaks=0 make -j16 build_tests test

  npm_lint:
    runs-on: ubuntu-latest
@@ -1,20 +1,20 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 2.8...3.30)

 project(vitastor)

-set(VITASTOR_VERSION "3.0.5")
+set(VITASTOR_VERSION "3.0.9")

 include(CTest)

 add_custom_target(build_tests)
-add_custom_target(test
-	COMMAND
-	echo leak:tcmalloc > ${CMAKE_CURRENT_BINARY_DIR}/lsan-suppress.txt &&
-	env LSAN_OPTIONS=suppressions=${CMAKE_CURRENT_BINARY_DIR}/lsan-suppress.txt ${CMAKE_CTEST_COMMAND}
+set_property(TEST PROPERTY ENVIRONMENT LSAN_OPTIONS=suppressions=${CMAKE_CURRENT_BINARY_DIR}/lsan-suppress.txt)
+add_test(gen_lsan_suppress
+    ${CMAKE_COMMAND} -E echo leak:tcmalloc > "${CMAKE_CURRENT_BINARY_DIR}/lsan-suppress.txt"
 )
+set_tests_properties(gen_lsan_suppress PROPERTIES FIXTURES_SETUP f_lsan_suppress)
+set_property(TEST PROPERTY FIXTURES_REQUIRED f_lsan_suppress)
 # make -j16 -C ../../build test_heap && ../../build/src/test/test_heap
 # make -j16 -C ../../build test_heap && rm -f $(find ../../build -name '*.gcda') && ctest -V -T test -T coverage -R heap --test-dir ../../build && (cd ../../build; gcovr -f ../src --html --html-nested -o coverage/index.html; cd ../src/test)
 # make -j16 -C ../../build test_blockstore && rm -f $(find ../../build -name '*.gcda') && ctest -V -T test -T coverage -R blockstore --test-dir ../../build && (cd ../../build; gcovr -f ../src --html --html-nested -o coverage/index.html; cd ../src/test)
 # kcov --include-path=../../../src ../../kcov ./test_blockstore
-add_dependencies(test build_tests)
 add_subdirectory(src)
@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.5
+VITASTOR_VERSION ?= v3.0.9

 all: build push

@@ -49,7 +49,7 @@ spec:
            capabilities:
              add: ["SYS_ADMIN"]
            allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v3.0.5
+          image: vitalif/vitastor-csi:v3.0.9
          args:
            - "--node=$(NODE_ID)"
            - "--endpoint=$(CSI_ENDPOINT)"
@@ -121,7 +121,7 @@ spec:
            privileged: true
            capabilities:
              add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v3.0.5
+          image: vitalif/vitastor-csi:v3.0.9
          args:
            - "--node=$(NODE_ID)"
            - "--endpoint=$(CSI_ENDPOINT)"
@@ -5,7 +5,7 @@ package vitastor

 const (
    vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "3.0.5"
+    vitastorCSIDriverVersion = "3.0.9"
 )

 // Config struct fills the parameters of request or user input
@@ -1,4 +1,4 @@
-vitastor (3.0.5-1) unstable; urgency=medium
+vitastor (3.0.9-1) unstable; urgency=medium

  * Bugfixes

@@ -44,7 +44,7 @@ curl -s https://git.yourcmc.ru/vitalif/antietcd/archive/master.tar.gz | tar -zx
 curl -s https://git.yourcmc.ru/vitalif/tinyraft/archive/master.tar.gz | tar -zx

 cd /root/vitastor/packages/vitastor-$REL
-if [[ "$REL" = "trixie" && -e ../vitastor-bookworm/vitastor_$VER.orig.tar.xz ]]; then
+if [[ ( "$REL" = "trixie" || "$REL" = "resolute" ) && -e ../vitastor-bookworm/vitastor_$VER.orig.tar.xz ]]; then
    # Fucking shit, archives differ between bookworm (xz 5.4.1) and trixie (xz 5.8.1)
    cp ../vitastor-bookworm/vitastor_$VER.orig.tar.xz .
 else
@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.5
+VITASTOR_VERSION ?= v3.0.9

 all: build push

@@ -4,7 +4,7 @@
 #

 # Desired Vitastor version
-VITASTOR_VERSION=v3.0.5
+VITASTOR_VERSION=v3.0.9

 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here
@@ -70,6 +70,7 @@ with an OSD restart or, for some of them, even without restarting by updating co
 - [use_atomic_flag](#use_atomic_flag)
 - [pg_reshard_chunk_size](#pg_reshard_chunk_size)
 - [pg_reshard_chunk_pause_ms](#pg_reshard_chunk_pause_ms)
+- [gc_on_start](#gc_on_start)

 ## bind_address

@@ -753,3 +754,9 @@ This option sets the maximum number of object is a chunk. Moving 100k objects us
 - Default: 100

 This option sets the interval between handling two PG count change chunks.
+
+## gc_on_start
+
+- Type: boolean
+
+Forcibly clean all garbage entries in the new store on every OSD restart.
@@ -71,6 +71,7 @@
 - [use_atomic_flag](#use_atomic_flag)
 - [pg_reshard_chunk_size](#pg_reshard_chunk_size)
 - [pg_reshard_chunk_pause_ms](#pg_reshard_chunk_pause_ms)
+- [gc_on_start](#gc_on_start)

 ## bind_address

@@ -793,3 +794,9 @@ pg_minsize OSD во время переключений, что может по
 - Значение по умолчанию: 100

 Данная опция задаёт интервал между обработкой двух порций изменения числа PG пулов.
+
+## gc_on_start
+
+- Тип: булево (да/нет)
+
+Принудительно очищать все мусорные записи в новом хранилище при каждом запуске OSD.
@@ -938,3 +938,7 @@
    This option sets the interval between handling two PG count change chunks.
  info_ru: |
    Данная опция задаёт интервал между обработкой двух порций изменения числа PG пулов.
+- name: gc_on_start
+  type: bool
+  info: Forcibly clean all garbage entries in the new store on every OSD restart.
+  info_ru: Принудительно очищать все мусорные записи в новом хранилище при каждом запуске OSD.
@@ -26,9 +26,9 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.

 1. Download a Docker image of the desired version: \
-   `docker pull vitalif/vitastor:v3.0.5`
+   `docker pull vitalif/vitastor:v3.0.9`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.5 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.9 install.sh`
 3. Reload udev rules: \
   `udevadm control --reload-rules`
 4. Enable the vitastor-host service: \
@@ -25,9 +25,9 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.

 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitalif/vitastor:v3.0.5`
+   `docker pull vitalif/vitastor:v3.0.9`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.5 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.9 install.sh`
 3. Перезагрузите правила udev: \
   `udevadm control --reload-rules`
 4. Включите сервис vitastor-host: \
@@ -17,6 +17,7 @@
  - Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
  - Ubuntu 22.04 (Jammy): `deb https://vitastor.io/debian jammy main`
  - Ubuntu 24.04 (Noble): `deb https://vitastor.io/debian noble main`
+  - Ubuntu 26.04 (Resolute): `deb https://vitastor.io/debian resolute main`
  - Add `-oldstable` to bookworm/bullseye/buster in this line to install the last
    stable version from 0.9.x branch instead of 1.x
 - To always prefer vitastor-patched QEMU and Libvirt versions, add the following to `/etc/apt/preferences`:
@@ -17,6 +17,7 @@
  - Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
  - Ubuntu 22.04 (Jammy): `deb https://vitastor.io/debian jammy main`
  - Ubuntu 24.04 (Noble): `deb https://vitastor.io/debian noble main`
+  - Ubuntu 26.04 (Resolute): `deb https://vitastor.io/debian resolute main`
  - Добавьте `-oldstable` к слову bookworm/bullseye/buster в этой строке, чтобы
    установить последнюю стабильную версию из ветки 0.9.x вместо 1.x
 - Чтобы всегда предпочитались версии пакетов QEMU и Libvirt с патчами Vitastor, добавьте в `/etc/apt/preferences`:
@@ -49,6 +49,17 @@ class AntiEtcdAdapter
                    stale_read: 1,
                    log_level: 1,
                };
+                if (config.use_auth)
+                {
+                    antietcd_config.client_cert_auth = true;
+                    antietcd_config.auth_filter = require('./vitastor_auth_filter.js');
+                    antietcd_config.peer_ca = config.antietcd_server_ca;
+                    if (!config.antietcd_server_ca || config.antietcd_server_ca == config.etcd_ca)
+                    {
+                        console.error('Secure setup requires separate antietcd_server_ca (for signing antietcd server certificates) and etcd_ca (for signing client certificates)');
+                        process.exit(1);
+                    }
+                }
                for (const key in config)
                {
                    if (key.substr(0, 9) === 'antietcd_')
@@ -173,7 +184,7 @@ class AntiEtcdAdapter
                    await new Promise(ok => setTimeout(ok, timeout-(Date.now()-prev)));
                }
                prev = Date.now();
-                const res = await this.antietcd.api(path.replace(/^\/+/, '').replace(/\/+$/, '').replace(/\/+/g, '_'), body);
+                const res = await this.antietcd.api(path.replace(/^\/+/, '').replace(/\/+$/, '').replace(/\/+/g, '_'), body, { username: 'root' });
                if (res.error)
                {
                    console.error('Failed to query antietcd '+path+' (retry '+retry+'/'+retries+'): '+res.error);
@@ -16,6 +16,7 @@ const etcd_allow = new RegExp('^'+[
    'config/pools',
    'config/osd/[1-9]\\d*',
    'config/pgs', // old name
+    'config/user/.*',
    'pg/config',
    'config/inode/[1-9]\\d*/[1-9]\\d*',
    'osd/state/[1-9]\\d*',
@@ -208,6 +209,8 @@ const etcd_tree = {
                primary_affinity_tags?: 'nvme' | [ 'nvme', ... ],
                // scrub interval
                scrub_interval?: '30d',
+                // users allowed to create images in this pool
+                creator_group?: '',
            },
            ...
        }, */
@@ -225,11 +228,20 @@ const etcd_tree = {
                    readonly?: boolean,
                    deleted?: boolean,
                    enc_key?: string,
-                    meta?: any,
+                    owner?: string,
+                    owner_group?: string,
+                    reader_group?: string,
                }
            }
        }, */
        inode: {},
+        /* user: {
+            <username>: {
+                type: 'osd'|'mon'|'admin'|'client',
+                groups: string[],
+            },
+        }, */
+        user: {},
    },
    osd: {
        state: {
@@ -392,7 +404,6 @@ const etcd_tree = {
            /* <name>: {
                id: uint64_t,
                pool_id: uint64_t,
-                // ...plus a copy of everything from config/inode/x/y
            }, */
        },
        maxid: {
@@ -1,6 +1,6 @@
 {
  "name": "vitastor-mon",
-  "version": "3.0.5",
+  "version": "3.0.9",
  "description": "Vitastor SDS monitor service",
  "main": "mon-main.js",
  "scripts": {
@@ -37,6 +37,7 @@ function derive_osd_stats(st, prev, prev_diff)
        const n = c.count - BigInt(pr && pr.count||0);
        diff.recovery_stats[op] = { ...c, bps: n > 0 ? b*1000n/timediff : 0n, iops: n > 0 ? n*1000n/timediff : 0n };
    }
+    diff.inode_stats = {};
    for (const pool_id in st.inode_stats||{})
    {
        diff.inode_stats[pool_id] = {};
@@ -0,0 +1,471 @@
+// AntiEtcd authentication filter for Vitastor
+// (c) Vitaliy Filippov, 2026
+// License: Mozilla Public License 2.0 or Vitastor Network Public License 1.1
+
+// Permissions are based on:
+// 1. Users.
+//    Stored in /vitastor/config/user/<username>.
+//    Has 2 properties:
+//    - type, one of: osd, mon, admin, client.
+//      osd, mon types should be used by OSDs/monitors.
+//      admin should be used for administrative access from vitastor-cli.
+//      client should be used for regular clients.
+//    - groups, a list of group names the user is included in.
+// 2. Images.
+//    Stored in /vitastor/config/inode/<pool>/<inode>. Has the following properties:
+//    - owner (user name)
+//    - owner_group (group name)
+//    - reader_group
+
+const static_perms = {
+    invalid: {
+        keys: {},
+        prefixes: {},
+    },
+    osd: {
+        keys: { '/pg/config': false },
+        prefixes: { '/osd/': true, '/pg/state/': true, '/pg/history/': true, '/pgstats/': true },
+    },
+    mon: {
+        keys: { '/pg/config': true, '/stats': true, '/history/last_clean_pgs': true },
+        prefixes: {
+            '/config/': false, '/osd/': false, '/mon/': true, '/pg/history/': true,
+            '/pgstats/': false, '/inode/stats/': true, '/pool/stats/': true,
+        },
+    },
+    admin: {
+        keys: { '/stats': false },
+        prefixes: {
+            '/config/': true, '/osd/': true, '/index/': true, '/pg/history/': true,
+            '/mon/': false, '/pg/': false, '/pgstats/': false, '/inode/stats/': false, '/pool/stats/': false,
+        },
+    },
+    client: {
+        keys: { '/config/global': false, '/config/node_placement': false, '/config/pools': false, '/pg/config': false },
+        prefixes: { '/osd/stats/': false, '/pg/state/': false, '/index/maxid/': false },
+    },
+};
+
+const api_perms = {
+    osd: { lease_grant: true, lease_revoke: true, lease_keepalive: true },
+    mon: { lease_grant: true, lease_revoke: true, lease_keepalive: true },
+    admin: { maintenance_status: true },
+    client: {},
+};
+
+class VitastorAuthFilter
+{
+    constructor(antietcd)
+    {
+        this.cfg = antietcd.cfg;
+        this.antietcd = antietcd;
+        this.prefix = this.cfg.vitastor_prefix || '/vitastor';
+        this.prefix_parts = this.prefix.split('/');
+    }
+
+    _get(path, decode)
+    {
+        let cur = this.antietcd.etctree.state;
+        path = path instanceof Array ? path : path.split('/');
+        for (const p of path)
+        {
+            if (!cur.children)
+            {
+                return null;
+            }
+            cur = cur.children[p];
+            if (!cur)
+            {
+                return null;
+            }
+        }
+        if (decode)
+        {
+            return this._decode(path, cur.value);
+        }
+        return cur;
+    }
+
+    _decode(path, cur)
+    {
+        if (!cur)
+        {
+            return null;
+        }
+        if (cur)
+        {
+            try
+            {
+                cur = JSON.parse(cur);
+            }
+            catch (e)
+            {
+                console.warn('Invalid JSON in '+(path instanceof Array ? path.join('/') : path)+': '+e);
+            }
+        }
+        return cur;
+    }
+
+    // userInfo: { name: string, type: string, perms: static_perms[type], groups: { [string]: true } }
+    _check_compare(check, userInfo, checked)
+    {
+        let key = String(check.key);
+        if (key.substr(0, this.prefix.length) !== this.prefix)
+        {
+            return false;
+        }
+        key = key.substr(this.prefix.length);
+        if (key in userInfo.perms.keys)
+        {
+            return true;
+        }
+        for (const pfx in userInfo.perms.prefixes)
+        {
+            if (key.substr(0, pfx.length) == pfx)
+            {
+                return true;
+            }
+        }
+        if (userInfo.type == 'client')
+        {
+            // Image permissions
+            if (key.substr(0, 14) == '/config/inode/')
+            {
+                // Allowed to check that a key does not exist
+                if (check.target == 'VERSION' && check.version == 0)
+                {
+                    checked['M'+key] = true;
+                    return true;
+                }
+                else if (check.target == 'MOD')
+                {
+                    const data = this._get(check.key);
+                    if (!data || data.mod_revision != check.mod_revision)
+                    {
+                        // Break check to trigger CAS failure
+                        check.mod_revision = '18446744073709551615'; // UINT64_MAX
+                        return true;
+                    }
+                    const inode = this._decode(check.key, data.value);
+                    if (inode && (inode.owner_group && userInfo.groups[inode.owner_group] ||
+                        inode.owner === userInfo.name))
+                    {
+                        checked['M'+key] = true;
+                        return true;
+                    }
+                }
+                return false;
+            }
+            if (key.substr(0, 13) == '/index/image/')
+            {
+                // Allowed to check that a key does not exist
+                if (check.target == 'VERSION' && check.version == 0)
+                {
+                    checked['M'+key] = true;
+                    return true;
+                }
+                else if (check.target == 'MOD')
+                {
+                    let data = this._get(check.key);
+                    if (!data || data.mod_revision != check.mod_revision)
+                    {
+                        // Break check to trigger CAS failure
+                        check.mod_revision = '18446744073709551615'; // UINT64_MAX
+                        return true;
+                    }
+                    data = this._decode(check.key, data.value);
+                    if (data)
+                    {
+                        const inode = this._get([ ...this.prefix_parts, 'config', 'inode', data.pool_id, data.id ], true);
+                        if (inode && (inode.owner_group && userInfo.groups[inode.owner_group] ||
+                            inode.owner === userInfo.name))
+                        {
+                            checked['M'+key] = true;
+                            return true;
+                        }
+                    }
+                }
+                return false;
+            }
+            if (key.substr(0, 13) == '/index/maxid/')
+            {
+                const pool_id = key.substr(13);
+                const pool_cfg = this._get([ ...this.prefix_parts, 'config', 'pools' ], true);
+                if (!pool_cfg || !pool_cfg[pool_id] || !pool_cfg[pool_id].creator_group || !userInfo.groups[pool_cfg[pool_id].creator_group])
+                {
+                    return false;
+                }
+                if (check.target == 'VERSION' && check.version == 0)
+                {
+                    checked['I'+parseInt(key.substr(13))+'_0'] = true;
+                    return true;
+                }
+                else if (check.target == 'MOD')
+                {
+                    const data = this._get(check.key);
+                    if (!data || data.mod_revision != check.mod_revision)
+                    {
+                        // Break check to trigger CAS failure
+                        check.mod_revision = '18446744073709551615'; // UINT64_MAX
+                        return true;
+                    }
+                    checked['I'+parseInt(key.substr(13))+'_'+data.value] = true;
+                    return true;
+                }
+                return false;
+            }
+        }
+        return false;
+    }
+
+    _check_read(kv, userInfo)
+    {
+        let key = String(kv.key);
+        if (key.substr(0, this.prefix.length) !== this.prefix)
+        {
+            return false;
+        }
+        key = key.substr(this.prefix.length);
+        if (key in userInfo.perms.keys)
+        {
+            return true;
+        }
+        for (const pfx in userInfo.perms.prefixes)
+        {
+            if (key.substr(0, pfx.length) == pfx)
+            {
+                return true;
+            }
+        }
+        if (userInfo.type == 'client')
+        {
+            // Image permissions
+            if (key.substr(0, 14) == '/config/inode/')
+            {
+                const inode = this._decode(kv.key, kv.value);
+                if (inode && (inode.reader_group && userInfo.groups[inode.reader_group] ||
+                    inode.owner_group && userInfo.groups[inode.owner_group] ||
+                    inode.owner === userInfo.name))
+                {
+                    return true;
+                }
+                return false;
+            }
+            if (key.substr(0, 13) == '/index/image/')
+            {
+                const data = this._decode(kv.key, kv.value);
+                const inode = this._get([ ...this.prefix_parts, 'config', 'inode', data.pool_id, data.id ], true);
+                if (inode && (inode.reader_group && userInfo.groups[inode.reader_group] ||
+                    inode.owner_group && userInfo.groups[inode.owner_group] ||
+                    inode.owner === userInfo.name))
+                {
+                    return true;
+                }
+                return false;
+            }
+        }
+        return false;
+    }
+
+    _check_write(put, userInfo, checked)
+    {
+        let key = String(put.key);
+        if (key.substr(0, this.prefix.length) !== this.prefix)
+        {
+            return false;
+        }
+        key = key.substr(this.prefix.length);
+        if (userInfo.perms.keys[key])
+        {
+            return true;
+        }
+        for (const pfx in userInfo.perms.prefixes)
+        {
+            if (userInfo.perms.prefixes[pfx] && key.substr(0, pfx.length) == pfx)
+            {
+                return true;
+            }
+        }
+        if (checked && userInfo.type == 'client')
+        {
+            if (key.substr(0, 13) == '/index/maxid/' &&
+                checked['I'+parseInt(key.substr(13))+'_'+(put.value-1)])
+            {
+                // Allowed to increment maxid
+                return true;
+            }
+            if (checked['M'+key])
+            {
+                // Allowed to modify known images with CAS checks
+                return true;
+            }
+        }
+        return false;
+    }
+
+    _check_req(req, userInfo, checked)
+    {
+        let r;
+        if ((r = (req.request_range || req.requestRange)))
+        {
+            // All range queries are allowed, but responses are filtered - it's simpler
+        }
+        else if ((r = (req.request_put || req.requestPut)))
+        {
+            if (!this._check_write(r, userInfo, checked))
+                return false;
+        }
+        else if ((r = (req.request_delete_range || req.requestDeleteRange)))
+        {
+            if (!r.range_end || r.range_end === r.key)
+            {
+                if (!this._check_write({ key: r.key }, userInfo))
+                    return false;
+            }
+            else
+            {
+                // All keys in range must satisfy prefix
+                r.range_end = String(r.range_end);
+                if (r.key.length != r.range_end.length ||
+                    r.key[r.key.length-1] != '/' ||
+                    r.range_end[r.range_end.length-1] != '0')
+                {
+                    return false;
+                }
+                let key = r.key.substr(this.prefix.length);
+                let found = false;
+                for (const pfx in userInfo.perms.prefixes)
+                {
+                    if (userInfo.perms.prefixes[pfx] && key.substr(0, pfx.length) == pfx)
+                    {
+                        found = true;
+                        break;
+                    }
+                }
+                if (!found)
+                    return false;
+            }
+        }
+        return true;
+    }
+
+    _get_user(username)
+    {
+        if (!username)
+        {
+            return null;
+        }
+        let userInfo = this._get([ ...this.prefix_parts, 'config', 'user', username ], true);
+        if (!userInfo)
+        {
+            userInfo = { type: 'client' };
+        }
+        userInfo.perms = static_perms[userInfo.type] || static_perms['invalid'];
+        userInfo.name = username;
+        if (userInfo.groups instanceof Array)
+        {
+            userInfo.groups = userInfo.groups.reduce((a, c) => { a[c] = true; return a; }, {});
+        }
+        else
+        {
+            userInfo.groups = {};
+        }
+        return userInfo;
+    }
+
+    filter_api(username, api/*, data*/)
+    {
+        if (username === 'root')
+        {
+            return true;
+        }
+        const userInfo = this._get([ ...this.prefix_parts, 'config', 'user', username ], true);
+        return userInfo && api_perms[userInfo.type] && api_perms[userInfo.type][api];
+    }
+
+    filter_txn(username, txn)
+    {
+        if (username === 'root')
+        {
+            return true;
+        }
+        const userInfo = this._get_user(username);
+        if (!userInfo)
+        {
+            return null;
+        }
+        const checked = {};
+        if (txn.compare)
+        {
+            for (const check of txn.compare)
+            {
+                if (!this._check_compare(check, userInfo, checked))
+                    return null;
+            }
+        }
+        // Special transactions:
+        // 1. create image: create config/inode and index/image, increment index/maxid/<pool> (with CAS)
+        // 2. create snapshot: same as create image but also rename previous to @snap
+        if (txn.success)
+        {
+            for (const req of txn.success)
+            {
+                if (!this._check_req(req, userInfo, checked))
+                    return null;
+            }
+        }
+        if (txn.failure)
+        {
+            for (const req of txn.failure)
+            {
+                if (!this._check_req(req, userInfo, null))
+                    return null;
+            }
+        }
+        return txn;
+    }
+
+    filter_txn_response(username, txn, res)
+    {
+        if (!res.responses || username === 'root')
+        {
+            return;
+        }
+        const userInfo = this._get_user(username);
+        if (!userInfo)
+        {
+            for (const resp of res.responses)
+            {
+                if (resp.response_range && resp.response_range.kvs)
+                {
+                    resp.response_range.kvs = [];
+                }
+            }
+            return;
+        }
+        for (const resp of res.responses)
+        {
+            if (resp.response_range && resp.response_range.kvs)
+            {
+                resp.response_range.kvs = resp.response_range.kvs.filter(kv => this._check_read(kv, userInfo));
+            }
+        }
+    }
+
+    filter_watch_message(username, msg)
+    {
+        if (!msg.result || !msg.result.events || username === 'root')
+        {
+            return;
+        }
+        const userInfo = this._get_user(username);
+        if (!userInfo)
+        {
+            msg.result.events = [];
+            return;
+        }
+        msg.result.events = msg.result.events.filter(ev => this._check_read(ev.kv, userInfo));
+    }
+}
+
+module.exports = VitastorAuthFilter;
@@ -1,6 +1,6 @@
 {
  "name": "vitastor",
-  "version": "3.0.5",
+  "version": "3.0.9",
  "description": "Low-level native bindings to Vitastor client library",
  "main": "index.js",
  "keywords": [
@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils

-VITASTOR_VERSION = '3.0.5'
+VITASTOR_VERSION = '3.0.9'

 LOG = logging.getLogger(__name__)

@@ -0,0 +1,637 @@
+diff --git a/include/libvirt/libvirt-storage.h b/include/libvirt/libvirt-storage.h
+index aaad4a3da1..5f5daa8341 100644
+--- a/include/libvirt/libvirt-storage.h
+++ b/include/libvirt/libvirt-storage.h
+@@ -326,6 +326,7 @@ typedef enum {
+     VIR_CONNECT_LIST_STORAGE_POOLS_ZFS           = 1 << 17, /* (Since: 1.2.8) */
+     VIR_CONNECT_LIST_STORAGE_POOLS_VSTORAGE      = 1 << 18, /* (Since: 3.1.0) */
+     VIR_CONNECT_LIST_STORAGE_POOLS_ISCSI_DIRECT  = 1 << 19, /* (Since: 5.6.0) */
+    VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR      = 1 << 20, /* (Since: 5.0.0) */
+ } virConnectListAllStoragePoolsFlags;
+ 
+ int                     virConnectListAllStoragePools(virConnectPtr conn,
+diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
+index 9ca5c2450c..cc52f00c0c 100644
+--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
+@@ -7453,7 +7453,8 @@ virDomainDiskSourceNetworkParse(xmlNodePtr node,
+     src->configFile = virXPathString("string(./config/@file)", ctxt);
+ 
+     if (src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTP ||
+-        src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTPS)
+        src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTPS ||
+        src->protocol == VIR_STORAGE_NET_PROTOCOL_VITASTOR)
+         src->query = virXMLPropString(node, "query");
+ 
+     if (virDomainStorageNetworkParseHosts(node, ctxt, &src->hosts, &src->nhosts) < 0)
+@@ -32187,6 +32188,7 @@ virDomainStorageSourceTranslateSourcePool(virStorageSource *src,
+ 
+     case VIR_STORAGE_POOL_MPATH:
+     case VIR_STORAGE_POOL_RBD:
+    case VIR_STORAGE_POOL_VITASTOR:
+     case VIR_STORAGE_POOL_SHEEPDOG:
+     case VIR_STORAGE_POOL_GLUSTER:
+     case VIR_STORAGE_POOL_LAST:
+diff --git a/src/conf/domain_validate.c b/src/conf/domain_validate.c
+index 7346a61731..83e94d762e 100644
+--- a/src/conf/domain_validate.c
+++ b/src/conf/domain_validate.c
+@@ -520,6 +520,7 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
+         case VIR_STORAGE_NET_PROTOCOL_RBD:
+             break;
+ 
+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+         case VIR_STORAGE_NET_PROTOCOL_NBD:
+         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
+         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
+@@ -592,7 +593,7 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
+         }
+     }
+ 
+-    /* internal snapshots and config files are currently supported only with rbd: */
+    /* internal snapshots are currently supported only with rbd: */
+     if (virStorageSourceGetActualType(src) != VIR_STORAGE_TYPE_NETWORK &&
+         src->protocol != VIR_STORAGE_NET_PROTOCOL_RBD) {
+         if (src->snapshot) {
+@@ -600,10 +601,14 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
+                            _("<snapshot> element is currently supported only with 'rbd' disks"));
+             return -1;
+         }
+-
+    }
+    /* config files are currently supported only with rbd and vitastor: */
+    if (virStorageSourceGetActualType(src) != VIR_STORAGE_TYPE_NETWORK &&
+        src->protocol != VIR_STORAGE_NET_PROTOCOL_RBD &&
+        src->protocol != VIR_STORAGE_NET_PROTOCOL_VITASTOR) {
+         if (src->configFile) {
+             virReportError(VIR_ERR_XML_ERROR, "%s",
+-                           _("<config> element is currently supported only with 'rbd' disks"));
+                           _("<config> element is currently supported only with 'rbd' and 'vitastor' disks"));
+             return -1;
+         }
+     }
+diff --git a/src/conf/schemas/domaincommon.rng b/src/conf/schemas/domaincommon.rng
+index 114dd3f96f..c71f9a3277 100644
+--- a/src/conf/schemas/domaincommon.rng
+++ b/src/conf/schemas/domaincommon.rng
+@@ -2093,6 +2093,35 @@
+     </element>
+   </define>
+ 
+  <define name="diskSourceNetworkProtocolVitastor">
+    <element name="source">
+      <interleave>
+        <attribute name="protocol">
+          <value>vitastor</value>
+        </attribute>
+        <ref name="diskSourceCommon"/>
+        <optional>
+          <attribute name="name"/>
+        </optional>
+        <optional>
+          <attribute name="query"/>
+        </optional>
+        <zeroOrMore>
+          <ref name="diskSourceNetworkHost"/>
+        </zeroOrMore>
+        <optional>
+          <element name="config">
+            <attribute name="file">
+              <ref name="absFilePath"/>
+            </attribute>
+            <empty/>
+          </element>
+        </optional>
+        <empty/>
+      </interleave>
+    </element>
+  </define>
+
+   <define name="diskSourceNetworkProtocolISCSI">
+     <element name="source">
+       <attribute name="protocol">
+@@ -2443,6 +2472,7 @@
+       <ref name="diskSourceNetworkProtocolSimple"/>
+       <ref name="diskSourceNetworkProtocolVxHS"/>
+       <ref name="diskSourceNetworkProtocolNFS"/>
+      <ref name="diskSourceNetworkProtocolVitastor"/>
+     </choice>
+   </define>
+ 
+diff --git a/src/conf/storage_conf.c b/src/conf/storage_conf.c
+index 1dc9365bf2..a8a736be81 100644
+--- a/src/conf/storage_conf.c
+++ b/src/conf/storage_conf.c
+@@ -56,7 +56,7 @@ VIR_ENUM_IMPL(virStoragePool,
+               "logical", "disk", "iscsi",
+               "iscsi-direct", "scsi", "mpath",
+               "rbd", "sheepdog", "gluster",
+-              "zfs", "vstorage",
+              "zfs", "vstorage", "vitastor",
+ );
+ 
+ VIR_ENUM_IMPL(virStoragePoolFormatFileSystem,
+@@ -242,6 +242,18 @@ static virStoragePoolTypeInfo poolTypeInfo[] = {
+           .formatToString = virStorageFileFormatTypeToString,
+       }
+     },
+    {.poolType = VIR_STORAGE_POOL_VITASTOR,
+     .poolOptions = {
+         .flags = (VIR_STORAGE_POOL_SOURCE_HOST |
+                   VIR_STORAGE_POOL_SOURCE_NETWORK |
+                   VIR_STORAGE_POOL_SOURCE_NAME),
+      },
+      .volOptions = {
+          .defaultFormat = VIR_STORAGE_FILE_RAW,
+          .formatFromString = virStorageVolumeFormatFromString,
+          .formatToString = virStorageFileFormatTypeToString,
+      }
+    },
+     {.poolType = VIR_STORAGE_POOL_SHEEPDOG,
+      .poolOptions = {
+          .flags = (VIR_STORAGE_POOL_SOURCE_HOST |
+@@ -538,6 +550,11 @@ virStoragePoolDefParseSource(xmlXPathContextPtr ctxt,
+                        _("element 'name' is mandatory for RBD pool"));
+         return -1;
+     }
+    if (pool_type == VIR_STORAGE_POOL_VITASTOR && source->name == NULL) {
+        virReportError(VIR_ERR_XML_ERROR, "%s",
+                       _("element 'name' is mandatory for Vitastor pool"));
+        return -1;
+    }
+ 
+     if (options->formatFromString) {
+         g_autofree char *format = NULL;
+@@ -1127,6 +1144,7 @@ virStoragePoolDefFormatBuf(virBuffer *buf,
+     /* RBD, Sheepdog, Gluster and Iscsi-direct devices are not local block devs nor
+      * files, so they don't have a target */
+     if (def->type != VIR_STORAGE_POOL_RBD &&
+        def->type != VIR_STORAGE_POOL_VITASTOR &&
+         def->type != VIR_STORAGE_POOL_SHEEPDOG &&
+         def->type != VIR_STORAGE_POOL_GLUSTER &&
+         def->type != VIR_STORAGE_POOL_ISCSI_DIRECT) {
+diff --git a/src/conf/storage_conf.h b/src/conf/storage_conf.h
+index fc67957cfe..720c07ef74 100644
+--- a/src/conf/storage_conf.h
+++ b/src/conf/storage_conf.h
+@@ -103,6 +103,7 @@ typedef enum {
+     VIR_STORAGE_POOL_GLUSTER,  /* Gluster device */
+     VIR_STORAGE_POOL_ZFS,      /* ZFS */
+     VIR_STORAGE_POOL_VSTORAGE, /* Virtuozzo Storage */
+    VIR_STORAGE_POOL_VITASTOR, /* Vitastor */
+ 
+     VIR_STORAGE_POOL_LAST,
+ } virStoragePoolType;
+@@ -454,6 +455,7 @@ VIR_ENUM_DECL(virStoragePartedFs);
+                  VIR_CONNECT_LIST_STORAGE_POOLS_SCSI     | \
+                  VIR_CONNECT_LIST_STORAGE_POOLS_MPATH    | \
+                  VIR_CONNECT_LIST_STORAGE_POOLS_RBD      | \
+                 VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR | \
+                  VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG | \
+                  VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER  | \
+                  VIR_CONNECT_LIST_STORAGE_POOLS_ZFS      | \
+diff --git a/src/conf/storage_source_conf.c b/src/conf/storage_source_conf.c
+index d7b9bdfecb..38aefd0dd4 100644
+--- a/src/conf/storage_source_conf.c
+++ b/src/conf/storage_source_conf.c
+@@ -90,6 +90,7 @@ VIR_ENUM_IMPL(virStorageNetProtocol,
+               "ssh",
+               "vxhs",
+               "nfs",
+              "vitastor",
+ );
+ 
+ 
+@@ -1317,6 +1318,7 @@ virStorageSourceNetworkDefaultPort(virStorageNetProtocol protocol)
+         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
+             return 24007;
+ 
+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+         case VIR_STORAGE_NET_PROTOCOL_RBD:
+             /* we don't provide a default for RBD */
+             return 0;
+diff --git a/src/conf/storage_source_conf.h b/src/conf/storage_source_conf.h
+index 22c35d420d..f1e32ea83d 100644
+--- a/src/conf/storage_source_conf.h
+++ b/src/conf/storage_source_conf.h
+@@ -131,6 +131,7 @@ typedef enum {
+     VIR_STORAGE_NET_PROTOCOL_SSH,
+     VIR_STORAGE_NET_PROTOCOL_VXHS,
+     VIR_STORAGE_NET_PROTOCOL_NFS,
+    VIR_STORAGE_NET_PROTOCOL_VITASTOR,
+ 
+     VIR_STORAGE_NET_PROTOCOL_LAST
+ } virStorageNetProtocol;
+diff --git a/src/conf/virstorageobj.c b/src/conf/virstorageobj.c
+index 59fa5da372..4739167f5f 100644
+--- a/src/conf/virstorageobj.c
+++ b/src/conf/virstorageobj.c
+@@ -1438,6 +1438,7 @@ virStoragePoolObjSourceFindDuplicateCb(const void *payload,
+             return 1;
+         break;
+ 
+    case VIR_STORAGE_POOL_VITASTOR:
+     case VIR_STORAGE_POOL_ISCSI_DIRECT:
+     case VIR_STORAGE_POOL_RBD:
+     case VIR_STORAGE_POOL_LAST:
+@@ -1921,6 +1922,8 @@ virStoragePoolObjMatch(virStoragePoolObj *obj,
+                (obj->def->type == VIR_STORAGE_POOL_MPATH))   ||
+               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_RBD) &&
+                (obj->def->type == VIR_STORAGE_POOL_RBD))     ||
+              (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR) &&
+               (obj->def->type == VIR_STORAGE_POOL_VITASTOR)) ||
+               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG) &&
+                (obj->def->type == VIR_STORAGE_POOL_SHEEPDOG)) ||
+               (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER) &&
+diff --git a/src/libvirt-storage.c b/src/libvirt-storage.c
+index db7660aac4..561df34709 100644
+--- a/src/libvirt-storage.c
+++ b/src/libvirt-storage.c
+@@ -94,6 +94,7 @@ virStoragePoolGetConnect(virStoragePoolPtr pool)
+  * VIR_CONNECT_LIST_STORAGE_POOLS_SCSI
+  * VIR_CONNECT_LIST_STORAGE_POOLS_MPATH
+  * VIR_CONNECT_LIST_STORAGE_POOLS_RBD
+ * VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR
+  * VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG
+  * VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER
+  * VIR_CONNECT_LIST_STORAGE_POOLS_ZFS
+diff --git a/src/libxl/libxl_conf.c b/src/libxl/libxl_conf.c
+index 2b988157fa..9d0eb47b25 100644
+--- a/src/libxl/libxl_conf.c
+++ b/src/libxl/libxl_conf.c
+@@ -1069,6 +1069,7 @@ libxlMakeNetworkDiskSrcStr(virStorageSource *src,
+     case VIR_STORAGE_NET_PROTOCOL_SSH:
+     case VIR_STORAGE_NET_PROTOCOL_VXHS:
+     case VIR_STORAGE_NET_PROTOCOL_NFS:
+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+     case VIR_STORAGE_NET_PROTOCOL_LAST:
+     case VIR_STORAGE_NET_PROTOCOL_NONE:
+         virReportError(VIR_ERR_NO_SUPPORT,
+diff --git a/src/libxl/xen_xl.c b/src/libxl/xen_xl.c
+index e72e7d7f44..8482c21805 100644
+--- a/src/libxl/xen_xl.c
+++ b/src/libxl/xen_xl.c
+@@ -1461,6 +1461,7 @@ xenFormatXLDiskSrcNet(virStorageSource *src)
+     case VIR_STORAGE_NET_PROTOCOL_SSH:
+     case VIR_STORAGE_NET_PROTOCOL_VXHS:
+     case VIR_STORAGE_NET_PROTOCOL_NFS:
+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+     case VIR_STORAGE_NET_PROTOCOL_LAST:
+     case VIR_STORAGE_NET_PROTOCOL_NONE:
+         virReportError(VIR_ERR_NO_SUPPORT,
+diff --git a/src/qemu/qemu_block.c b/src/qemu/qemu_block.c
+index 9b43279797..459d8e8a65 100644
+--- a/src/qemu/qemu_block.c
+++ b/src/qemu/qemu_block.c
+@@ -743,6 +743,38 @@ qemuBlockStorageSourceGetRBDProps(virStorageSource *src,
+ }
+ 
+ 
+static virJSONValue *
+qemuBlockStorageSourceGetVitastorProps(virStorageSource *src)
+{
+    virJSONValue *ret = NULL;
+    virStorageNetHostDef *host;
+    size_t i;
+    g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;
+    g_autofree char *etcd = NULL;
+
+    for (i = 0; i < src->nhosts; i++) {
+        host = src->hosts + i;
+        if ((virStorageNetHostTransport)host->transport != VIR_STORAGE_NET_HOST_TRANS_TCP) {
+            return NULL;
+        }
+        virBufferAsprintf(&buf, i > 0 ? ",%s:%u" : "%s:%u", host->name, host->port);
+    }
+    if (src->nhosts > 0) {
+        etcd = virBufferContentAndReset(&buf);
+    }
+
+    if (virJSONValueObjectAdd(&ret,
+                              "S:etcd-host", etcd,
+                              "S:etcd-prefix", src->query,
+                              "S:config-path", src->configFile,
+                              "s:image", src->path,
+                              NULL) < 0)
+        return NULL;
+
+    return ret;
+}
+
+
+ static virJSONValue *
+ qemuBlockStorageSourceGetSshProps(virStorageSource *src)
+ {
+@@ -1094,6 +1126,12 @@ qemuBlockStorageSourceGetBackendProps(virStorageSource *src,
+                 return NULL;
+             break;
+ 
+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+            driver = "vitastor";
+            if (!(fileprops = qemuBlockStorageSourceGetVitastorProps(src)))
+                return NULL;
+            break;
+
+         case VIR_STORAGE_NET_PROTOCOL_SSH:
+             driver = "ssh";
+             if (!(fileprops = qemuBlockStorageSourceGetSshProps(src)))
+@@ -1997,6 +2035,7 @@ qemuBlockGetBackingStoreString(virStorageSource *src,
+ 
+             case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
+             case VIR_STORAGE_NET_PROTOCOL_RBD:
+            case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+             case VIR_STORAGE_NET_PROTOCOL_VXHS:
+             case VIR_STORAGE_NET_PROTOCOL_NFS:
+             case VIR_STORAGE_NET_PROTOCOL_SSH:
+@@ -2377,6 +2416,12 @@ qemuBlockStorageSourceCreateGetStorageProps(virStorageSource *src,
+                 return -1;
+             break;
+ 
+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+            driver = "vitastor";
+            if (!(location = qemuBlockStorageSourceGetVitastorProps(src)))
+                return -1;
+            break;
+
+         case VIR_STORAGE_NET_PROTOCOL_SSH:
+             if (srcPriv->nbdkitProcess) {
+                 /* disk creation not yet supported with nbdkit, and even if it
+diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
+index ac56fc7cb4..9e407b4aab 100644
+--- a/src/qemu/qemu_domain.c
+++ b/src/qemu/qemu_domain.c
+@@ -4677,7 +4677,8 @@ qemuDomainValidateStorageSource(virStorageSource *src,
+     if (src->query &&
+         (actualType != VIR_STORAGE_TYPE_NETWORK ||
+          (src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTPS &&
+-          src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTP))) {
+          src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTP &&
+          src->protocol != VIR_STORAGE_NET_PROTOCOL_VITASTOR))) {
+         virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                        _("query is supported only with HTTP(S) protocols"));
+         return -1;
+@@ -9103,6 +9104,7 @@ qemuDomainPrepareStorageSourceTLS(virStorageSource *src,
+         break;
+ 
+     case VIR_STORAGE_NET_PROTOCOL_RBD:
+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+     case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
+     case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
+     case VIR_STORAGE_NET_PROTOCOL_ISCSI:
+diff --git a/src/qemu/qemu_snapshot.c b/src/qemu/qemu_snapshot.c
+index e738afffc3..37d64f469b 100644
+--- a/src/qemu/qemu_snapshot.c
+++ b/src/qemu/qemu_snapshot.c
+@@ -665,6 +665,7 @@ qemuSnapshotPrepareDiskExternalInactive(virDomainSnapshotDiskDef *snapdisk,
+         case VIR_STORAGE_NET_PROTOCOL_NONE:
+         case VIR_STORAGE_NET_PROTOCOL_NBD:
+         case VIR_STORAGE_NET_PROTOCOL_RBD:
+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
+         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
+         case VIR_STORAGE_NET_PROTOCOL_ISCSI:
+@@ -893,6 +894,7 @@ qemuSnapshotPrepareDiskInternal(virDomainDiskDef *disk,
+         case VIR_STORAGE_NET_PROTOCOL_NONE:
+         case VIR_STORAGE_NET_PROTOCOL_NBD:
+         case VIR_STORAGE_NET_PROTOCOL_RBD:
+        case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+         case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
+         case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
+         case VIR_STORAGE_NET_PROTOCOL_ISCSI:
+diff --git a/src/storage/storage_driver.c b/src/storage/storage_driver.c
+index e19e032427..59f91f4710 100644
+--- a/src/storage/storage_driver.c
+++ b/src/storage/storage_driver.c
+@@ -1626,6 +1626,7 @@ storageVolLookupByPathCallback(virStoragePoolObj *obj,
+ 
+         case VIR_STORAGE_POOL_GLUSTER:
+         case VIR_STORAGE_POOL_RBD:
+        case VIR_STORAGE_POOL_VITASTOR:
+         case VIR_STORAGE_POOL_SHEEPDOG:
+         case VIR_STORAGE_POOL_ZFS:
+         case VIR_STORAGE_POOL_LAST:
+diff --git a/src/storage_file/storage_source_backingstore.c b/src/storage_file/storage_source_backingstore.c
+index 821378883c..2211f6891b 100644
+--- a/src/storage_file/storage_source_backingstore.c
+++ b/src/storage_file/storage_source_backingstore.c
+@@ -264,6 +264,75 @@ virStorageSourceParseRBDColonString(const char *rbdstr,
+ }
+ 
+ 
+static int
+virStorageSourceParseVitastorColonString(const char *colonstr,
+                                         virStorageSource *src)
+{
+    char *p, *e, *next;
+    g_autofree char *options = NULL;
+
+    /* optionally skip the "vitastor:" prefix if provided */
+    if (STRPREFIX(colonstr, "vitastor:"))
+        colonstr += strlen("vitastor:");
+
+    options = g_strdup(colonstr);
+
+    p = options;
+    while (*p) {
+        /* find : delimiter or end of string */
+        for (e = p; *e && *e != ':'; ++e) {
+            if (*e == '\\') {
+                e++;
+                if (*e == '\0')
+                    break;
+            }
+        }
+        if (*e == '\0') {
+            next = e;    /* last kv pair */
+        } else {
+            next = e + 1;
+            *e = '\0';
+        }
+
+        if (STRPREFIX(p, "image=")) {
+            src->path = g_strdup(p + strlen("image="));
+        } else if (STRPREFIX(p, "etcd-prefix=")) {
+            src->query = g_strdup(p + strlen("etcd-prefix="));
+        } else if (STRPREFIX(p, "config-path=")) {
+            src->configFile = g_strdup(p + strlen("config-path="));
+        } else if (STRPREFIX(p, "etcd-host=")) {
+            char *h, *sep;
+
+            h = p + strlen("etcd-host=");
+            while (h < e) {
+                for (sep = h; sep < e; ++sep) {
+                    if (*sep == '\\' && (sep[1] == ',' ||
+                                         sep[1] == ';' ||
+                                         sep[1] == ' ')) {
+                        *sep = '\0';
+                        sep += 2;
+                        break;
+                    }
+                }
+
+                if (virStorageSourceRBDAddHost(src, h) < 0)
+                    return -1;
+
+                h = sep;
+            }
+        }
+
+        p = next;
+    }
+
+    if (!src->path) {
+        return -1;
+    }
+
+    return 0;
+}
+
+
+ static int
+ virStorageSourceParseNBDColonString(const char *nbdstr,
+                                     virStorageSource *src)
+@@ -379,6 +448,11 @@ virStorageSourceParseBackingColon(virStorageSource *src,
+             return -1;
+         break;
+ 
+    case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
+        if (virStorageSourceParseVitastorColonString(path, src) < 0)
+            return -1;
+        break;
+
+     case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
+     case VIR_STORAGE_NET_PROTOCOL_LAST:
+     case VIR_STORAGE_NET_PROTOCOL_NONE:
+@@ -953,6 +1027,54 @@ virStorageSourceParseBackingJSONRBD(virStorageSource *src,
+     return 0;
+ }
+ 
+static int
+virStorageSourceParseBackingJSONVitastor(virStorageSource *src,
+                                         virJSONValue *json,
+                                         const char *jsonstr G_GNUC_UNUSED,
+                                         int opaque G_GNUC_UNUSED)
+{
+    const char *filename;
+    const char *image = virJSONValueObjectGetString(json, "image");
+    const char *conf = virJSONValueObjectGetString(json, "config-path");
+    const char *etcd_prefix = virJSONValueObjectGetString(json, "etcd-prefix");
+    virJSONValue *servers = virJSONValueObjectGetArray(json, "server");
+    size_t nservers;
+    size_t i;
+
+    src->type = VIR_STORAGE_TYPE_NETWORK;
+    src->protocol = VIR_STORAGE_NET_PROTOCOL_VITASTOR;
+
+    /* legacy syntax passed via 'filename' option */
+    if ((filename = virJSONValueObjectGetString(json, "filename")))
+        return virStorageSourceParseVitastorColonString(filename, src);
+
+    if (!image) {
+        virReportError(VIR_ERR_INVALID_ARG, "%s",
+                       _("missing image name in Vitastor backing volume "
+                         "JSON specification"));
+        return -1;
+    }
+
+    src->path = g_strdup(image);
+    src->configFile = g_strdup(conf);
+    src->query = g_strdup(etcd_prefix);
+
+    if (servers) {
+        nservers = virJSONValueArraySize(servers);
+
+        src->hosts = g_new0(virStorageNetHostDef, nservers);
+        src->nhosts = nservers;
+
+        for (i = 0; i < nservers; i++) {
+            if (virStorageSourceParseBackingJSONInetSocketAddress(src->hosts + i,
+                                                                  virJSONValueArrayGet(servers, i)) < 0)
+                return -1;
+        }
+    }
+
+    return 0;
+}
+
+ static int
+ virStorageSourceParseBackingJSONRaw(virStorageSource *src,
+                                     virJSONValue *json,
+@@ -1130,6 +1252,7 @@ static const struct virStorageSourceJSONDriverParser jsonParsers[] = {
+     {"sheepdog", false, virStorageSourceParseBackingJSONSheepdog, 0},
+     {"ssh", false, virStorageSourceParseBackingJSONSSH, 0},
+     {"rbd", false, virStorageSourceParseBackingJSONRBD, 0},
+    {"vitastor", false, virStorageSourceParseBackingJSONVitastor, 0},
+     {"raw", true, virStorageSourceParseBackingJSONRaw, 0},
+     {"nfs", false, virStorageSourceParseBackingJSONNFS, 0},
+     {"vxhs", false, virStorageSourceParseBackingJSONVxHS, 0},
+diff --git a/src/test/test_driver.c b/src/test/test_driver.c
+index 1165689de7..bba846351c 100644
+--- a/src/test/test_driver.c
+++ b/src/test/test_driver.c
+@@ -7345,6 +7345,7 @@ testStorageVolumeTypeForPool(int pooltype)
+     case VIR_STORAGE_POOL_ISCSI_DIRECT:
+     case VIR_STORAGE_POOL_GLUSTER:
+     case VIR_STORAGE_POOL_RBD:
+    case VIR_STORAGE_POOL_VITASTOR:
+         return VIR_STORAGE_VOL_NETWORK;
+     case VIR_STORAGE_POOL_LOGICAL:
+     case VIR_STORAGE_POOL_DISK:
+diff --git a/tests/storagepoolcapsschemadata/poolcaps-fs.xml b/tests/storagepoolcapsschemadata/poolcaps-fs.xml
+index eee75af746..8bd0a57bdd 100644
+--- a/tests/storagepoolcapsschemadata/poolcaps-fs.xml
+++ b/tests/storagepoolcapsschemadata/poolcaps-fs.xml
+@@ -204,4 +204,11 @@
+       </enum>
+     </volOptions>
+   </pool>
+  <pool type='vitastor' supported='no'>
+    <volOptions>
+      <defaultFormat type='raw'/>
+      <enum name='targetFormatType'>
+      </enum>
+    </volOptions>
+  </pool>
+ </storagepoolCapabilities>
+diff --git a/tests/storagepoolcapsschemadata/poolcaps-full.xml b/tests/storagepoolcapsschemadata/poolcaps-full.xml
+index 805950a937..852df0de16 100644
+--- a/tests/storagepoolcapsschemadata/poolcaps-full.xml
+++ b/tests/storagepoolcapsschemadata/poolcaps-full.xml
+@@ -204,4 +204,11 @@
+       </enum>
+     </volOptions>
+   </pool>
+  <pool type='vitastor' supported='yes'>
+    <volOptions>
+      <defaultFormat type='raw'/>
+      <enum name='targetFormatType'>
+      </enum>
+    </volOptions>
+  </pool>
+ </storagepoolCapabilities>
+diff --git a/tests/storagepoolxml2argvtest.c b/tests/storagepoolxml2argvtest.c
+index d5c2531ab8..b19308ac38 100644
+--- a/tests/storagepoolxml2argvtest.c
+++ b/tests/storagepoolxml2argvtest.c
+@@ -57,6 +57,7 @@ testCompareXMLToArgvFiles(bool shouldFail,
+     case VIR_STORAGE_POOL_GLUSTER:
+     case VIR_STORAGE_POOL_ZFS:
+     case VIR_STORAGE_POOL_VSTORAGE:
+    case VIR_STORAGE_POOL_VITASTOR:
+     case VIR_STORAGE_POOL_LAST:
+     default:
+         VIR_TEST_DEBUG("pool type '%s' has no xml2argv test", defTypeStr);
+diff --git a/tools/virsh-pool.c b/tools/virsh-pool.c
+index 2010ef1356..072e2ff9e8 100644
+--- a/tools/virsh-pool.c
+++ b/tools/virsh-pool.c
+@@ -1187,6 +1187,9 @@ cmdPoolList(vshControl *ctl, const vshCmd *cmd G_GNUC_UNUSED)
+             case VIR_STORAGE_POOL_VSTORAGE:
+                 flags |= VIR_CONNECT_LIST_STORAGE_POOLS_VSTORAGE;
+                 break;
+            case VIR_STORAGE_POOL_VITASTOR:
+                flags |= VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR;
+                break;
+             case VIR_STORAGE_POOL_LAST:
+                 break;
+             }
@@ -1,29 +1,172 @@
-diff --git a/src/client/qemu_driver.c b/src/client/qemu_driver.c
-index d8356dab..5f4cd50d 100644
--- a/src/client/qemu_driver.c
-+++ b/src/client/qemu_driver.c
-@@ -974,14 +974,21 @@ static void vitastor_co_read_bitmap_cb(void *opaque, long retval, uint8_t *bitma
- #endif
- }
+diff --git a/block/meson.build b/block/meson.build
+index 34b1b2a306..24ca0f1e52 100644
+--- a/block/meson.build
+++ b/block/meson.build
+@@ -114,6 +114,7 @@ foreach m : [
+   [libnfs, 'nfs', files('nfs.c')],
+   [libssh, 'ssh', files('ssh.c')],
+   [rbd, 'rbd', files('rbd.c')],
+  [vitastor, 'vitastor', files('vitastor.c')],
+ ]
+   if m[0].found()
+     module_ss = ss.source_set()
+diff --git a/meson.build b/meson.build
+index 50c774a195..e5c7a3a4b1 100644
+--- a/meson.build
+++ b/meson.build
+@@ -1652,6 +1652,26 @@ if not get_option('rbd').auto() or have_block
+   endif
+ endif
 
-static int coroutine_fn vitastor_co_block_status(
-    BlockDriverState *bs, bool want_zero, int64_t offset, int64_t bytes,
-    int64_t *pnum, int64_t *map, BlockDriverState **file)
-+static int coroutine_fn vitastor_co_block_status(BlockDriverState *bs,
-+#if QEMU_VERSION_MAJOR > 10 || QEMU_VERSION_MAJOR == 10 && QEMU_VERSION_MINOR >= 1
-+    unsigned int mode,
-+#else
-+    bool want_zero,
-+#endif
-+    int64_t offset, int64_t bytes, int64_t *pnum, int64_t *map, BlockDriverState **file)
- {
-     // Allocated => return BDRV_BLOCK_DATA|BDRV_BLOCK_OFFSET_VALID
-     // Not allocated => return 0
-     // Error => return -errno
-     // Set pnum to length of the extent, `*map` = `offset`, `*file` = `bs`
-+#if QEMU_VERSION_MAJOR > 10 || QEMU_VERSION_MAJOR == 10 && QEMU_VERSION_MINOR >= 1
-+    int want_zero = (mode == BDRV_WANT_PRECISE);
-+#endif
-     VitastorRPC task;
-     VitastorClient *client = bs->opaque;
-     uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode;
+vitastor = not_found
+if not get_option('vitastor').auto() or have_block
+  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
+    required: get_option('vitastor'))
+  if libvitastor_client.found()
+    if cc.links('''
+      #include <vitastor_c.h>
+      int main(void) {
+        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+        return 0;
+      }''', dependencies: libvitastor_client)
+      vitastor = declare_dependency(dependencies: libvitastor_client)
+    elif get_option('vitastor').enabled()
+      error('could not link libvitastor_client')
+    else
+      warning('could not link libvitastor_client, disabling')
+    endif
+  endif
+endif
+
+ glusterfs = not_found
+ glusterfs_ftruncate_has_stat = false
+ glusterfs_iocb_has_stat = false
+@@ -2547,6 +2567,7 @@ endif
+ config_host_data.set('CONFIG_OPENGL', opengl.found())
+ config_host_data.set('CONFIG_PLUGIN', get_option('plugins'))
+ config_host_data.set('CONFIG_RBD', rbd.found())
+config_host_data.set('CONFIG_VITASTOR', vitastor.found())
+ config_host_data.set('CONFIG_RDMA', rdma.found())
+ config_host_data.set('CONFIG_RELOCATABLE', get_option('relocatable'))
+ config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack'))
+@@ -4972,6 +4993,7 @@ summary_info += {'fdt support':       fdt_opt == 'internal' ? 'internal' : fdt}
+ summary_info += {'libcap-ng support': libcap_ng}
+ summary_info += {'bpf support':       libbpf}
+ summary_info += {'rbd support':       rbd}
+summary_info += {'vitastor support':  vitastor}
+ summary_info += {'smartcard support': cacard}
+ summary_info += {'U2F support':       u2f}
+ summary_info += {'libusb':            libusb}
+diff --git a/meson_options.txt b/meson_options.txt
+index fff1521e58..f0844c0e00 100644
+--- a/meson_options.txt
+++ b/meson_options.txt
+@@ -202,6 +202,8 @@ option('pvg', type: 'feature', value: 'auto',
+        description: 'macOS paravirtualized graphics support')
+ option('rbd', type : 'feature', value : 'auto',
+        description: 'Ceph block device driver')
+option('vitastor', type : 'feature', value : 'auto',
+       description: 'Vitastor block device driver')
+ option('opengl', type : 'feature', value : 'auto',
+        description: 'OpenGL support')
+ option('rdma', type : 'feature', value : 'auto',
+diff --git a/qapi/block-core.json b/qapi/block-core.json
+index dc6eb4ae23..d043f4340e 100644
+--- a/qapi/block-core.json
+++ b/qapi/block-core.json
+@@ -3280,7 +3280,7 @@
+             'parallels', 'preallocate', 'qcow', 'qcow2', 'qed', 'quorum',
+             'raw', 'rbd',
+             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
+-            'ssh', 'throttle', 'vdi', 'vhdx',
+            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
+             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
+@@ -4363,6 +4363,28 @@
+             '*key-secret': 'str',
+             '*server': ['InetSocketAddressBase'] } }
+ 
+##
+# @BlockdevOptionsVitastor:
+#
+# Driver specific block device options for vitastor
+#
+# @image:       Image name
+# @inode:       Inode number
+# @pool:        Pool ID
+# @size:        Desired image size in bytes
+# @config-path: Path to Vitastor configuration
+# @etcd-host:   etcd connection address(es)
+# @etcd-prefix: etcd key/value prefix
+##
+{ 'struct': 'BlockdevOptionsVitastor',
+  'data': { '*inode': 'uint64',
+            '*pool': 'uint64',
+            '*size': 'uint64',
+            '*image': 'str',
+            '*config-path': 'str',
+            '*etcd-host': 'str',
+            '*etcd-prefix': 'str' } }
+
+ ##
+ # @ReplicationMode:
+ #
+@@ -4831,6 +4853,7 @@
+       'throttle':   'BlockdevOptionsThrottle',
+       'vdi':        'BlockdevOptionsGenericFormat',
+       'vhdx':       'BlockdevOptionsGenericFormat',
+      'vitastor':   'BlockdevOptionsVitastor',
+       'virtio-blk-vfio-pci':
+                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
+                       'if': 'CONFIG_BLKIO' },
+@@ -5304,6 +5327,20 @@
+             '*cluster-size' :   'size',
+             '*encrypt' :        'RbdEncryptionCreateOptions' } }
+ 
+##
+# @BlockdevCreateOptionsVitastor:
+#
+# Driver specific image creation options for Vitastor.
+#
+# @location: Where to store the new image file.  This location cannot
+#     point to a snapshot.
+#
+# @size: Size of the virtual disk in bytes
+##
+{ 'struct': 'BlockdevCreateOptionsVitastor',
+  'data': { 'location':         'BlockdevOptionsVitastor',
+            'size':             'size' } }
+
+ ##
+ # @BlockdevVmdkSubformat:
+ #
+@@ -5526,6 +5563,7 @@
+       'ssh':            'BlockdevCreateOptionsSsh',
+       'vdi':            'BlockdevCreateOptionsVdi',
+       'vhdx':           'BlockdevCreateOptionsVhdx',
+      'vitastor':       'BlockdevCreateOptionsVitastor',
+       'vmdk':           'BlockdevCreateOptionsVmdk',
+       'vpc':            'BlockdevCreateOptionsVpc'
+   } }
+diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh
+index 0ebe6bc52a..2c37ad3892 100644
+--- a/scripts/meson-buildoptions.sh
+++ b/scripts/meson-buildoptions.sh
+@@ -175,6 +175,7 @@ meson_options_help() {
+   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
+   printf "%s\n" '  qpl             Query Processing Library support'
+   printf "%s\n" '  rbd             Ceph block device driver'
+  printf "%s\n" '  vitastor        Vitastor block device driver'
+   printf "%s\n" '  rdma            Enable RDMA-based migration'
+   printf "%s\n" '  replication     replication support'
+   printf "%s\n" '  rust            Rust support'
+@@ -459,6 +460,8 @@ _meson_option_parse() {
+     --disable-qpl) printf "%s" -Dqpl=disabled ;;
+     --enable-rbd) printf "%s" -Drbd=enabled ;;
+     --disable-rbd) printf "%s" -Drbd=disabled ;;
+    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
+    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
+     --enable-rdma) printf "%s" -Drdma=enabled ;;
+     --disable-rdma) printf "%s" -Drdma=disabled ;;
+     --enable-relocatable) printf "%s" -Drelocatable=true ;;
@@ -0,0 +1,172 @@
+diff --git a/block/meson.build b/block/meson.build
+index 34b1b2a306..24ca0f1e52 100644
+--- a/block/meson.build
+++ b/block/meson.build
+@@ -114,6 +114,7 @@ foreach m : [
+   [libnfs, 'nfs', files('nfs.c')],
+   [libssh, 'ssh', files('ssh.c')],
+   [rbd, 'rbd', files('rbd.c')],
+  [vitastor, 'vitastor', files('vitastor.c')],
+ ]
+   if m[0].found()
+     module_ss = ss.source_set()
+diff --git a/meson.build b/meson.build
+index d9293294d8..776a5becc6 100644
+--- a/meson.build
+++ b/meson.build
+@@ -1665,6 +1665,26 @@ if not get_option('rbd').auto() or have_block
+   endif
+ endif
+ 
+vitastor = not_found
+if not get_option('vitastor').auto() or have_block
+  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
+    required: get_option('vitastor'))
+  if libvitastor_client.found()
+    if cc.links('''
+      #include <vitastor_c.h>
+      int main(void) {
+        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+        return 0;
+      }''', dependencies: libvitastor_client)
+      vitastor = declare_dependency(dependencies: libvitastor_client)
+    elif get_option('vitastor').enabled()
+      error('could not link libvitastor_client')
+    else
+      warning('could not link libvitastor_client, disabling')
+    endif
+  endif
+endif
+
+ glusterfs = not_found
+ glusterfs_ftruncate_has_stat = false
+ glusterfs_iocb_has_stat = false
+@@ -2509,6 +2529,7 @@ endif
+ config_host_data.set('CONFIG_OPENGL', opengl.found())
+ config_host_data.set('CONFIG_PLUGIN', get_option('plugins'))
+ config_host_data.set('CONFIG_RBD', rbd.found())
+config_host_data.set('CONFIG_VITASTOR', vitastor.found())
+ config_host_data.set('CONFIG_RDMA', rdma.found())
+ config_host_data.set('CONFIG_RELOCATABLE', get_option('relocatable'))
+ config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack'))
+@@ -4948,6 +4969,7 @@ summary_info += {'fdt support':       fdt_opt == 'internal' ? 'internal' : fdt}
+ summary_info += {'libcap-ng support': libcap_ng}
+ summary_info += {'bpf support':       libbpf}
+ summary_info += {'rbd support':       rbd}
+summary_info += {'vitastor support':  vitastor}
+ summary_info += {'smartcard support': cacard}
+ summary_info += {'U2F support':       u2f}
+ summary_info += {'libusb':            libusb}
+diff --git a/meson_options.txt b/meson_options.txt
+index 2836156257..148086cc6f 100644
+--- a/meson_options.txt
+++ b/meson_options.txt
+@@ -206,6 +206,8 @@ option('pvg', type: 'feature', value: 'auto',
+        description: 'macOS paravirtualized graphics support')
+ option('rbd', type : 'feature', value : 'auto',
+        description: 'Ceph block device driver')
+option('vitastor', type : 'feature', value : 'auto',
+       description: 'Vitastor block device driver')
+ option('opengl', type : 'feature', value : 'auto',
+        description: 'OpenGL support')
+ option('rdma', type : 'feature', value : 'auto',
+diff --git a/qapi/block-core.json b/qapi/block-core.json
+index b82af74256..f25a6f5ce8 100644
+--- a/qapi/block-core.json
+++ b/qapi/block-core.json
+@@ -3351,7 +3351,7 @@
+             'parallels', 'preallocate', 'qcow', 'qcow2', 'qed', 'quorum',
+             'raw', 'rbd',
+             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
+-            'ssh', 'throttle', 'vdi', 'vhdx',
+            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
+             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
+@@ -4434,6 +4434,28 @@
+             '*key-secret': 'str',
+             '*server': ['InetSocketAddressBase'] } }
+ 
+##
+# @BlockdevOptionsVitastor:
+#
+# Driver specific block device options for vitastor
+#
+# @image:       Image name
+# @inode:       Inode number
+# @pool:        Pool ID
+# @size:        Desired image size in bytes
+# @config-path: Path to Vitastor configuration
+# @etcd-host:   etcd connection address(es)
+# @etcd-prefix: etcd key/value prefix
+##
+{ 'struct': 'BlockdevOptionsVitastor',
+  'data': { '*inode': 'uint64',
+            '*pool': 'uint64',
+            '*size': 'uint64',
+            '*image': 'str',
+            '*config-path': 'str',
+            '*etcd-host': 'str',
+            '*etcd-prefix': 'str' } }
+
+ ##
+ # @ReplicationMode:
+ #
+@@ -4902,6 +4924,7 @@
+       'throttle':   'BlockdevOptionsThrottle',
+       'vdi':        'BlockdevOptionsGenericFormat',
+       'vhdx':       'BlockdevOptionsGenericFormat',
+      'vitastor':   'BlockdevOptionsVitastor',
+       'virtio-blk-vfio-pci':
+                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
+                       'if': 'CONFIG_BLKIO' },
+@@ -5376,6 +5399,20 @@
+             '*cluster-size' :   'size',
+             '*encrypt' :        'RbdEncryptionCreateOptions' } }
+ 
+##
+# @BlockdevCreateOptionsVitastor:
+#
+# Driver specific image creation options for Vitastor.
+#
+# @location: Where to store the new image file.  This location cannot
+#     point to a snapshot.
+#
+# @size: Size of the virtual disk in bytes
+##
+{ 'struct': 'BlockdevCreateOptionsVitastor',
+  'data': { 'location':         'BlockdevOptionsVitastor',
+            'size':             'size' } }
+
+ ##
+ # @BlockdevVmdkSubformat:
+ #
+@@ -5598,6 +5635,7 @@
+       'ssh':            'BlockdevCreateOptionsSsh',
+       'vdi':            'BlockdevCreateOptionsVdi',
+       'vhdx':           'BlockdevCreateOptionsVhdx',
+      'vitastor':       'BlockdevCreateOptionsVitastor',
+       'vmdk':           'BlockdevCreateOptionsVmdk',
+       'vpc':            'BlockdevCreateOptionsVpc'
+   } }
+diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh
+index 3d0d132344..65ee8c855e 100644
+--- a/scripts/meson-buildoptions.sh
+++ b/scripts/meson-buildoptions.sh
+@@ -177,6 +177,7 @@ meson_options_help() {
+   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
+   printf "%s\n" '  qpl             Query Processing Library support'
+   printf "%s\n" '  rbd             Ceph block device driver'
+  printf "%s\n" '  vitastor        Vitastor block device driver'
+   printf "%s\n" '  rdma            Enable RDMA-based migration'
+   printf "%s\n" '  replication     replication support'
+   printf "%s\n" '  rust            Rust support'
+@@ -464,6 +465,8 @@ _meson_option_parse() {
+     --disable-qpl) printf "%s" -Dqpl=disabled ;;
+     --enable-rbd) printf "%s" -Drbd=enabled ;;
+     --disable-rbd) printf "%s" -Drbd=disabled ;;
+    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
+    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
+     --enable-rdma) printf "%s" -Drdma=enabled ;;
+     --disable-rdma) printf "%s" -Drdma=disabled ;;
+     --enable-relocatable) printf "%s" -Drelocatable=true ;;
@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.5
+Version:        3.0.9
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage

 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.5.el10.tar.gz
+Source0:        vitastor-3.0.9.el10.tar.gz

 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++
@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.5
+Version:        3.0.9
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage

 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.5.el7.tar.gz
+Source0:        vitastor-3.0.9.el7.tar.gz

 BuildRequires:  gperftools-devel
 BuildRequires:  devtoolset-9-gcc-c++
@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.5
+Version:        3.0.9
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage

 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.5.el8.tar.gz
+Source0:        vitastor-3.0.9.el8.tar.gz

 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-toolset-9-gcc-c++
@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.5
+Version:        3.0.9
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage

 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.5.el9.tar.gz
+Source0:        vitastor-3.0.9.el9.tar.gz

 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++
@@ -1,9 +1,8 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 2.8...3.30)

 project(vitastor)

 include(GNUInstallDirs)
-include(CTest)
 include(CheckIncludeFile)

 find_package(PkgConfig)
@@ -21,7 +20,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 endif()
 set(ENABLE_COVERAGE false CACHE BOOL "Enable code coverage")

-add_definitions(-DVITASTOR_VERSION="3.0.5")
+add_definitions(-DVITASTOR_VERSION="3.0.9")
 add_definitions(-D_GNU_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -fvisibility=hidden -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})
@@ -1,10 +1,10 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 2.8...3.30)

 project(vitastor)

 # libvitastor_blk.a
 add_library(vitastor_blk STATIC
-	../util/allocator.cpp ../util/crc32c.c ../util/xxhash.c ../util/ringloop.cpp
+	../util/allocator.cpp ../util/crc32c.c ../util/ringloop.cpp
 	multilist.cpp blockstore_heap.cpp blockstore_disk.cpp
 	blockstore.cpp blockstore_impl.cpp blockstore_init.cpp blockstore_open.cpp
 	blockstore_flush.cpp blockstore_read.cpp blockstore_stable.cpp blockstore_sync.cpp blockstore_write.cpp
@@ -228,4 +228,9 @@ public:
    virtual uint64_t get_journal_size() = 0;

    virtual uint32_t get_bitmap_granularity() = 0;
+
+    virtual uint64_t get_live_entries() = 0;
+    virtual uint64_t get_live_memory() = 0;
+    virtual uint64_t get_garbage_entries() = 0;
+    virtual uint64_t get_garbage_memory() = 0;
 };
@@ -98,6 +98,8 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
    csum_block_size = parse_size(config["csum_block_size"]);
    discard_on_start = config.find("discard_on_start") != config.end() &&
        (config["discard_on_start"] == "true" || config["discard_on_start"] == "1" || config["discard_on_start"] == "yes");
+    gc_on_start = config.find("gc_on_start") == config.end() ||
+        (config["gc_on_start"] == "true" || config["gc_on_start"] == "1" || config["gc_on_start"] == "yes");
    min_discard_size = parse_size(config["min_discard_size"]);
    if (!min_discard_size)
        min_discard_size = 1024*1024;
@@ -58,6 +58,8 @@ struct blockstore_disk_t
    bool inmemory_journal = true;
    // Data discard granularity and minimum size (for the sake of performance)
    bool discard_on_start = false;
+    // GC on start (new store)
+    bool gc_on_start = true;
    uint64_t min_discard_size = 1024*1024;
    uint64_t discard_granularity = 0;

@@ -12,7 +12,7 @@
 #include "blockstore_heap.h"
 #include "../util/allocator.h"
 #include "../util/crc32c.h"
-#include "../util/xxhash.h"
+#include "../util/xxh_x86dispatch.h"
 #include "../util/malloc_or_die.h"

 #define BS_HEAP_FREE_MVCC 1
@@ -30,6 +30,8 @@
 #define IMAP_MALLOC_LOW_BITS ((size_t)0x0F)
 #define IMAP_MAX_LOW 16

+#define list_item_overhead(a) (((a) + sizeof(heap_list_item_t) - sizeof(heap_entry_t) + sizeof(void*) + 15) & ~15)
+
 void inode_map_put(void* & inode_idx, heap_list_item_t* li);
 void inode_map_get(void *inode_idx, heap_inode_map_t::iterator & li_it, heap_list_item_t* & li, uint64_t stripe);
 void inode_map_free(void* inode_idx);
@@ -254,7 +256,7 @@ blockstore_heap_t::blockstore_heap_t(blockstore_disk_t *dsk, uint8_t *buffer_are
    assert(dsk->meta_block_size < 32768);
    assert(dsk->meta_area_size > 0);
    assert(dsk->journal_len > 0);
-    meta_alloc = new multilist_index_t(meta_block_count, META_ALLOC_LEVELS+1, 0);
+    meta_alloc = new multilist_index_t(meta_block_count, META_ALLOC_LEVELS+1, 2);
    block_info.resize(meta_block_count);
    assert(dsk->block_count <= 0xFFFF0000);
    data_alloc = new allocator_t(dsk->block_count);
@@ -342,19 +344,12 @@ corrupted_block:
                    block_num, block_offset, wr->size, sizeof(heap_entry_t));
                goto corrupted_block;
            }
-            if (wr->is_garbage())
-            {
-                // Garbage collection is only performed when writing new entries into the block
-                // because it needs a fake LSN and modified blocks require consecutive modified LSNs
-                // That's why garbage entries may persist on disk
-                if (log_level > 5)
-                {
-                    fprintf(stderr, "Notice: skipping garbage entry %jx:%jx v%ju l%ju in metadata block %u at %u\n",
-                        wr->inode, wr->stripe, wr->version, wr->lsn, block_num, block_offset);
-                }
-                block_offset += wr->size;
-                continue;
-            }
+            // Garbage collection is only performed when writing new entries into the block
+            // because it needs a fake LSN and modified blocks require consecutive modified LSNs
+            // At the same time, further modifications _after_ putting new entries into the block,
+            // but _before_ writing it, may mark some entries in it as garbage. That's why garbage
+            // entries may still be present on disk.
+            wr->entry_type &= ~BS_HEAP_GARBAGE;
            if ((wr->entry_type & BS_HEAP_TYPE) < BS_HEAP_BIG_WRITE ||
                (wr->entry_type & BS_HEAP_TYPE) > BS_HEAP_ROLLBACK ||
                (wr->entry_type & ~(BS_HEAP_TYPE|BS_HEAP_STABLE)) ||
@@ -436,7 +431,10 @@ int blockstore_heap_t::load_blocks(uint64_t disk_offset, uint64_t size, uint8_t
    entries_loaded = 0;
    return read_blocks(disk_offset, size, buf, allow_corrupted, [&](uint32_t block_num, heap_entry_t *wr_orig)
    {
-        heap_list_item_t *li = (heap_list_item_t*)malloc_or_die(wr_orig->size + sizeof(heap_list_item_t) - sizeof(heap_entry_t));
+        auto alloc_size = wr_orig->size + sizeof(heap_list_item_t) - sizeof(heap_entry_t);
+        heap_list_item_t *li = (heap_list_item_t*)malloc_or_die(alloc_size);
+        live_entries++;
+        live_memory += list_item_overhead(wr_orig->size);
        li->block_num = block_num;
        li->prev = li->next = NULL;
        memcpy(&li->entry, wr_orig, wr_orig->size);
@@ -621,10 +619,11 @@ int blockstore_heap_t::mark_used_blocks()
                if (wr->entry_type == (BS_HEAP_DELETE|BS_HEAP_STABLE) && !li->prev)
                {
                    wr->set_garbage();
+                    garbage_entries++;
+                    garbage_memory += list_item_overhead(wr->size);
                    modify_alloc(li->block_num, [&](heap_block_info_t & inf)
                    {
-                        inf.used_space -= wr->size;
-                        inf.has_garbage = true;
+                        inf.garbage_space += wr->size;
                    });
                    li = NULL;
                }
@@ -634,10 +633,11 @@ int blockstore_heap_t::mark_used_blocks()
                    if (overwritten)
                    {
                        wr->set_garbage();
+                        garbage_entries++;
+                        garbage_memory += list_item_overhead(wr->size);
                        modify_alloc(li->block_num, [&](heap_block_info_t & inf)
                        {
-                            inf.used_space -= wr->size;
-                            inf.has_garbage = true;
+                            inf.garbage_space += wr->size;
                        });
                        continue;
                    }
@@ -676,15 +676,66 @@ int blockstore_heap_t::mark_used_blocks()
            });
        }
    }
+    if (dsk->gc_on_start)
+    {
+        recheck_full_gc();
+    }
    return res;
 }

+void blockstore_heap_t::recheck_full_gc()
+{
+    uint32_t block_num = 0;
+    for (auto & inf: block_info)
+    {
+        // Instantly collect all garbage on restart
+        if (inf.garbage_space > 0)
+        {
+            if (log_level > 5)
+            {
+                fprintf(stderr, "Clearing %u out of %u garbage bytes in block %u\n", inf.garbage_space, inf.used_space, block_num);
+            }
+            uint32_t collected_garbage = 0;
+            size_t i = 0, j = 0;
+            for (; i < inf.entries.size(); i++)
+            {
+                if (inf.entries[i]->entry.is_garbage())
+                {
+                    collected_garbage += inf.entries[i]->entry.size;
+                    remove_list_item(inf.entries[i]);
+                }
+                else
+                {
+                    if (j != i)
+                        inf.entries[j] = inf.entries[i];
+                    j++;
+                }
+            }
+            inf.entries.resize(j);
+            modify_alloc(block_num, [&](heap_block_info_t & inf)
+            {
+                inf.used_space -= collected_garbage;
+                inf.garbage_space -= collected_garbage;
+            });
+            recheck_modified_blocks.insert(block_num);
+        }
+        block_num++;
+    }
+}
+
 void blockstore_heap_t::recheck_buffer(heap_entry_t *cwr, uint8_t *buf)
 {
    auto free_entry = [&](heap_list_item_t *li)
    {
        uint32_t block_num = li->block_num;
        auto wr_size = li->entry.size;
+        if (li->entry.is_garbage())
+        {
+            garbage_entries--;
+            garbage_memory -= list_item_overhead(wr_size);
+        }
+        live_entries--;
+        live_memory -= list_item_overhead(wr_size);
        free(li);
        modify_alloc(block_num, [&](heap_block_info_t & inf)
        {
@@ -722,7 +773,12 @@ void blockstore_heap_t::recheck_buffer(heap_entry_t *cwr, uint8_t *buf)
            assert(li->entry.entry_type == cwr->entry_type);
            auto prev = li->prev;
            li->next = li->prev = NULL;
-            li->entry.set_garbage();
+            if (!li->entry.is_garbage())
+            {
+                garbage_entries++;
+                garbage_memory += list_item_overhead(li->entry.size);
+                li->entry.set_garbage();
+            }
            li = prev;
            rolled_back++;
        }
@@ -1238,7 +1294,7 @@ heap_entry_t *blockstore_heap_t::read_entry(object_id oid)

 void blockstore_heap_t::gc_block(heap_block_info_t & inf)
 {
-    if (inf.has_garbage)
+    if (inf.garbage_space > 0)
    {
        size_t i = 0, j = 0;
        for (; i < inf.entries.size(); i++)
@@ -1261,7 +1317,8 @@ void blockstore_heap_t::gc_block(heap_block_info_t & inf)
            }
        }
        inf.entries.resize(j);
-        inf.has_garbage = false;
+        inf.used_space -= inf.garbage_space;
+        inf.garbage_space = 0;
    }
 }

@@ -1271,11 +1328,12 @@ int blockstore_heap_t::allocate_entry(uint32_t entry_size, uint32_t *block_num,
    {
        // First try to write into the same block as the previous time
        auto & inf = block_info.at(last_allocated_block);
-        auto free_space = dsk->meta_block_size - inf.used_space;
-        if (inf.is_writing || free_space < entry_size ||
+        if (inf.is_writing ||
+            inf.used_space - inf.garbage_space + entry_size > dsk->meta_block_size ||
            // Do not allow to make the last non-nearfull block nearfull
            !allow_last_free && meta_nearfull_blocks >= meta_block_count-1 &&
-            free_space >= max_entry_size && free_space < max_entry_size+entry_size)
+            inf.used_space - inf.garbage_space <= dsk->meta_block_size-max_entry_size &&
+            inf.used_space - inf.garbage_space + entry_size > dsk->meta_block_size-max_entry_size)
        {
            last_allocated_block = UINT32_MAX;
        }
@@ -1292,8 +1350,8 @@ int blockstore_heap_t::allocate_entry(uint32_t entry_size, uint32_t *block_num,
        {
            // Do not allow to make the last non-nearfull block nearfull
            auto & inf = block_info.at(last_allocated_block);
-            auto free_space = dsk->meta_block_size - inf.used_space;
-            if (free_space >= max_entry_size && free_space < max_entry_size+entry_size)
+            if (inf.used_space - inf.garbage_space <= dsk->meta_block_size-max_entry_size &&
+                inf.used_space - inf.garbage_space + entry_size > dsk->meta_block_size-max_entry_size)
            {
                last_allocated_block = UINT32_MAX;
            }
@@ -1304,8 +1362,7 @@ int blockstore_heap_t::allocate_entry(uint32_t entry_size, uint32_t *block_num,
            for (uint32_t b = meta_alloc->find(META_ALLOC_LEVELS-1); b != UINT32_MAX; b = meta_alloc->next(b))
            {
                auto & inf = block_info.at(b);
-                auto free_space = dsk->meta_block_size - inf.used_space;
-                if (free_space >= entry_size)
+                if (inf.used_space - inf.garbage_space + entry_size <= dsk->meta_block_size)
                {
                    last_allocated_block = b;
                    break;
@@ -1322,20 +1379,27 @@ int blockstore_heap_t::allocate_entry(uint32_t entry_size, uint32_t *block_num,
    {
        // Do not allow to make the last non-nearfull block nearfull
        auto & inf = block_info.at(last_allocated_block);
-        if (dsk->meta_block_size-inf.used_space >= max_entry_size &&
-            dsk->meta_block_size-inf.used_space+entry_size < max_entry_size)
+        if (inf.used_space - inf.garbage_space <= dsk->meta_block_size-max_entry_size &&
+            inf.used_space - inf.garbage_space + entry_size > dsk->meta_block_size-max_entry_size)
        {
            last_allocated_block = UINT32_MAX;
            return ENOSPC;
        }
    }
    // Write into the same block
-    auto & inf = block_info.at(last_allocated_block);
-    gc_block(inf);
    *block_num = last_allocated_block;
    modify_alloc(last_allocated_block, [&](heap_block_info_t & inf)
    {
+        // Write just 1 entry to the block to collect garbage
+        if (inf.garbage_space > (inf.used_space-inf.garbage_space)/2)
+            last_allocated_block = UINT32_MAX;
+        gc_block(inf);
        inf.used_space += entry_size;
+        assert(inf.used_space - inf.garbage_space <= dsk->meta_block_size);
+        assert(!inf.mod_lsn_to || inf.mod_lsn_to == next_lsn);
+        ++next_lsn;
+        inf.mod_lsn = inf.mod_lsn ? inf.mod_lsn : next_lsn;
+        inf.mod_lsn_to = next_lsn;
    });
    return 0;
 }
@@ -1394,16 +1458,15 @@ int blockstore_heap_t::add_entry(uint32_t wr_size, uint32_t *modified_block,
        *modified_block = block_num;
    }
    auto li = (heap_list_item_t*)malloc_or_die(wr_size + sizeof(heap_list_item_t) - sizeof(heap_entry_t));
+    live_entries++;
+    live_memory += list_item_overhead(wr_size);
    auto new_wr = &li->entry;
    auto & inf = block_info.at(block_num);
    if (!inf.entries.size())
        inf.entries.reserve(dsk->meta_block_size / max_entry_size);
    inf.entries.push_back(li);
-    assert(!inf.mod_lsn_to || inf.mod_lsn_to == next_lsn);
-    new_wr->lsn = ++next_lsn;
+    new_wr->lsn = next_lsn;
    fill_entry(new_wr);
-    inf.mod_lsn = inf.mod_lsn ? inf.mod_lsn : next_lsn;
-    inf.mod_lsn_to = next_lsn;
    // Remember the object as dirty and remove older entries when this block is written and fsynced
    push_inflight_lsn(next_lsn, new_wr,
        (explicit_complete ? HEAP_INFLIGHT_EXPLICIT : 0) |
@@ -1742,28 +1805,38 @@ int blockstore_heap_t::add_delete(heap_entry_t *obj, uint32_t *modified_block)

 uint32_t blockstore_heap_t::meta_alloc_pos(const heap_block_info_t & inf)
 {
-    if (inf.is_writing || inf.used_space > dsk->meta_block_size-sizeof(heap_entry_t))
+    auto real_used = (inf.used_space-inf.garbage_space);
+    if (inf.is_writing || inf.mod_lsn || real_used > dsk->meta_block_size-sizeof(heap_entry_t))
    {
        // 100% full - no entry can be written into this block at all
        return META_ALLOC_LEVELS;
    }
-    if (inf.used_space > dsk->meta_block_size-max_entry_size)
+    if (real_used > dsk->meta_block_size-max_entry_size)
    {
        // nearfull - big_entries won't fit into this block so it can't be used for compaction
        return META_ALLOC_LEVELS-1;
    }
-    // normal block
-    return inf.used_space / ((dsk->meta_block_size-max_entry_size+META_ALLOC_LEVELS-2) / (META_ALLOC_LEVELS-1));
+    // First we want to write to blocks with most garbage:
+    // >= 2*used, >= used/2
+    // (i.e. 66% garbage, 33% garbage)
+    // Then to mostly free blocks:
+    // >= 75% free, >= 50% free, >= 25% free
+    if (inf.garbage_space > real_used*2)
+        return 0;
+    if (inf.garbage_space > real_used/2)
+        return 1;
+    // META_ALLOC_LEVELS-3 levels left
+    return 2 + real_used / ((dsk->meta_block_size-max_entry_size+META_ALLOC_LEVELS-4) / (META_ALLOC_LEVELS-3));
 }

 void blockstore_heap_t::modify_alloc(uint32_t block_num, std::function<void(heap_block_info_t &)> change_cb)
 {
    auto & inf = block_info.at(block_num);
    uint32_t old_pos = meta_alloc_pos(inf);
-    uint32_t old_used = inf.used_space;
+    uint32_t old_used = inf.used_space-inf.garbage_space;
    change_cb(inf);
    uint32_t new_pos = meta_alloc_pos(inf);
-    uint32_t new_used = inf.used_space;
+    uint32_t new_used = inf.used_space-inf.garbage_space;
    meta_alloc->change(block_num, old_pos, new_pos);
    meta_used_space -= old_used;
    meta_used_space += new_used;
@@ -1775,11 +1848,19 @@ void blockstore_heap_t::modify_alloc(uint32_t block_num, std::function<void(heap

 void blockstore_heap_t::start_block_write(uint32_t block_num)
 {
-    modify_alloc(block_num, [&](heap_block_info_t & inf)
+    auto & inf = block_info.at(block_num);
+    assert(!inf.is_writing);
+    if (!inf.mod_lsn)
+    {
+        modify_alloc(block_num, [&](heap_block_info_t & inf)
+        {
+            inf.is_writing = true;
+        });
+    }
+    else
    {
-        assert(!inf.is_writing);
        inf.is_writing = true;
-    });
+    }
 }

 void blockstore_heap_t::complete_block_write(uint32_t block_num)
@@ -1844,6 +1925,8 @@ void blockstore_heap_t::mark_garbage_up_to(heap_entry_t *wr)
 void blockstore_heap_t::mark_garbage(uint32_t block_num, heap_entry_t *prev_wr, uint32_t used_big)
 {
    prev_wr->set_garbage();
+    garbage_entries++;
+    garbage_memory += list_item_overhead(prev_wr->size);
    // And this is the moment when we can free the data reference
    if (prev_wr->type() == BS_HEAP_SMALL_WRITE && prev_wr->small().len > 0)
    {
@@ -1859,8 +1942,7 @@ void blockstore_heap_t::mark_garbage(uint32_t block_num, heap_entry_t *prev_wr,
    }
    modify_alloc(block_num, [&](heap_block_info_t & inf)
    {
-        inf.used_space -= prev_wr->size;
-        inf.has_garbage = true;
+        inf.garbage_space += prev_wr->size;
    });
 }

@@ -2230,7 +2312,7 @@ void blockstore_heap_t::fill_block_empty_space(uint8_t *buffer, uint32_t pos)
 uint32_t blockstore_heap_t::get_meta_block_used_space(uint32_t block_num)
 {
    auto & inf = block_info.at(block_num);
-    return inf.used_space;
+    return inf.used_space - inf.garbage_space;
 }

 uint64_t blockstore_heap_t::get_data_used_space()
@@ -2273,6 +2355,26 @@ uint64_t blockstore_heap_t::get_compacted_count()
    return compacted_count;
 }

+uint64_t blockstore_heap_t::get_live_entries()
+{
+    return live_entries-garbage_entries;
+}
+
+uint64_t blockstore_heap_t::get_live_memory()
+{
+    return live_memory-garbage_memory;
+}
+
+uint64_t blockstore_heap_t::get_garbage_entries()
+{
+    return garbage_entries;
+}
+
+uint64_t blockstore_heap_t::get_garbage_memory()
+{
+    return garbage_memory;
+}
+
 void blockstore_heap_t::push_inflight_lsn(uint64_t lsn, heap_entry_t *wr, uint64_t flags)
 {
    uint64_t next_inf = first_inflight_lsn + inflight_lsn.size();
@@ -2359,37 +2461,50 @@ void blockstore_heap_t::apply_inflight(heap_inflight_lsn_t & inflight)
    {
        // Remove entry
        auto li = list_item(wr);
-        auto prev = li->prev;
-        auto next = li->next;
-        if (prev)
-        {
-            prev->next = next;
-        }
-        if (!next)
-        {
-            // The last freed entry must be a deletion
-            assert(!prev);
-            assert(wr->entry_type == BS_HEAP_DELETE|BS_HEAP_STABLE);
-            auto & pg_idx = block_index[get_pg_id(wr->inode, wr->stripe)];
-            auto & inode_idx = pg_idx[wr->inode];
-            heap_inode_map_t::iterator li_it;
-            heap_list_item_t *old_li = NULL;
-            inode_map_get(inode_idx, li_it, old_li, wr->stripe);
-            inode_map_erase(pg_idx, inode_idx, li_it, old_li);
-        }
-        else
-        {
-            next->prev = prev;
-            if (!prev && next->entry.entry_type == (BS_HEAP_DELETE|BS_HEAP_STABLE))
-            {
-                // free BS_HEAP_DELETEs when all previous entries are also freed
-                mark_garbage(next->block_num, &next->entry, UINT32_MAX);
-            }
-        }
-        free(li);
+        remove_list_item(li);
    }
 }

+void blockstore_heap_t::remove_list_item(heap_list_item_t *li)
+{
+    auto prev = li->prev;
+    auto next = li->next;
+    if (prev)
+    {
+        prev->next = next;
+    }
+    if (!next)
+    {
+        // The last freed entry must be a deletion
+        assert(!prev);
+        auto wr = &li->entry;
+        assert(wr->entry_type == BS_HEAP_DELETE|BS_HEAP_STABLE);
+        auto & pg_idx = block_index[get_pg_id(wr->inode, wr->stripe)];
+        auto & inode_idx = pg_idx[wr->inode];
+        heap_inode_map_t::iterator li_it;
+        heap_list_item_t *old_li = NULL;
+        inode_map_get(inode_idx, li_it, old_li, wr->stripe);
+        inode_map_erase(pg_idx, inode_idx, li_it, old_li);
+    }
+    else
+    {
+        next->prev = prev;
+        if (!prev && next->entry.entry_type == (BS_HEAP_DELETE|BS_HEAP_STABLE))
+        {
+            // free BS_HEAP_DELETEs when all previous entries are also freed
+            mark_garbage(next->block_num, &next->entry, UINT32_MAX);
+        }
+    }
+    if (li->entry.is_garbage())
+    {
+        garbage_entries--;
+        garbage_memory -= list_item_overhead(li->entry.size);
+    }
+    live_entries--;
+    live_memory -= list_item_overhead(li->entry.size);
+    free(li);
+}
+
 bool blockstore_heap_t::is_lsn_completed(uint64_t lsn)
 {
    if (lsn <= completed_lsn)
@@ -118,10 +118,13 @@ struct heap_object_mvcc_t

 struct heap_block_info_t
 {
-    uint32_t used_space = 0;
+    struct __attribute__((__packed__))
+    {
+        uint32_t used_space = 0;
+        uint32_t garbage_space = 0;
+    };
    uint64_t mod_lsn = 0, mod_lsn_to = 0; // only 1 block write of LSN sequence is allowed at a moment
-    bool is_writing: 1;
-    bool has_garbage: 1;
+    bool is_writing = false;
    std::vector<heap_list_item_t*> entries;
 };

@@ -185,6 +188,11 @@ class blockstore_heap_t
    uint64_t buffer_area_used_space = 0;
    uint64_t data_used_space = 0;

+    uint64_t live_entries = 0;
+    uint64_t live_memory = 0;
+    uint64_t garbage_entries = 0;
+    uint64_t garbage_memory = 0;
+
    uint64_t next_lsn = 0;
    uint32_t last_allocated_block = UINT32_MAX;
    heap_mvcc_map_t object_mvcc;
@@ -213,6 +221,7 @@ class blockstore_heap_t
    bool validate_object(heap_entry_t *obj);
    void fill_recheck_queue();
    int mark_used_blocks();
+    void recheck_full_gc();
    void recheck_buffer(heap_entry_t *cwr, uint8_t *buf);
    void defragment_block(uint32_t block_num);
    void reshard_add(heap_reshard_state_t *st, heap_list_item_t *li);
@@ -220,6 +229,7 @@ class blockstore_heap_t
    void gc_block(heap_block_info_t & inf);
    int allocate_entry(uint32_t entry_size, uint32_t *block_num, bool allow_last_free);
    void insert_list_item(heap_list_item_t *li);
+    void remove_list_item(heap_list_item_t *li);
    int add_entry(uint32_t wr_size, uint32_t *modified_block, bool allow_last_free,
        bool explicit_complete, std::function<void(heap_entry_t *wr)> fill_entry);
    int add_simple(heap_entry_t *obj, uint64_t version, uint32_t *modified_block, uint32_t entry_type);
@@ -346,6 +356,10 @@ public:
    uint32_t get_compact_queue_size();
    uint32_t get_to_compact_count();
    uint64_t get_compacted_count();
+    uint64_t get_live_entries();
+    uint64_t get_live_memory();
+    uint64_t get_garbage_entries();
+    uint64_t get_garbage_memory();

    uint64_t entry_pos(uint32_t block_num, uint32_t offset);
    heap_entry_t *entry_from_pos(uint64_t entry_pos, bool allow_unallocated = false);
@@ -229,4 +229,9 @@ public:
    uint64_t get_free_block_count();
    inline uint32_t get_bitmap_granularity() { return dsk.bitmap_granularity; }
    inline uint64_t get_journal_size() { return dsk.journal_len; }
+
+    inline uint64_t get_live_entries() { return heap->get_live_entries(); }
+    inline uint64_t get_live_memory() { return heap->get_live_memory(); }
+    inline uint64_t get_garbage_entries() { return heap->get_garbage_entries(); }
+    inline uint64_t get_garbage_memory() { return heap->get_garbage_memory(); }
 };
@@ -240,7 +240,7 @@ resume_4:
    }
    // metadata read finished
    bs->heap->finish_load();
-    printf("Metadata entries loaded: %ju, used blocks: %ju / %ju\n", entries_loaded, bs->heap->get_data_used_space() / bs->dsk.data_block_size, bs->dsk.block_count);
+    printf("Metadata entries loaded: %ju, rechecking unfinished writes and garbage entries\n", entries_loaded);
    if (zero_on_init && !bs->dsk.disable_meta_fsync)
    {
        GET_SQE();
@@ -293,6 +293,11 @@ resume_7:
    if (bs->readonly)
    {
        recheck_mod.clear();
+        printf("Actual metadata entries: %ju\n", bs->heap->get_live_entries());
+    }
+    else
+    {
+        printf("Actual metadata entries: %ju, clearing garbage in %zu metadata blocks\n", bs->heap->get_live_entries(), recheck_mod.size());
    }
    for (i = 0; i < recheck_mod.size(); i++)
    {
@@ -306,7 +311,7 @@ resume_8:
        uint32_t block_num = recheck_mod[i];
        uint64_t block_offset = bs->dsk.meta_offset + (uint64_t)(block_num+1) * bs->dsk.meta_block_size;
        data = ((ring_data_t*)sqe->user_data);
-        uint8_t *buf = (uint8_t*)malloc_or_die(bs->dsk.meta_block_size);
+        uint8_t *buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, bs->dsk.meta_block_size);
        bs->heap->get_meta_block(block_num, buf);
        data->iov = { buf, bs->dsk.meta_block_size };
        data->callback = [this, buf, block_offset](ring_data_t *data)
@@ -332,5 +337,9 @@ resume_9:
    }
    free(metadata_buffer);
    metadata_buffer = NULL;
+    printf("Loading finished. Data used: %ju / %ju bytes (%s / %s)\n",
+        bs->heap->get_data_used_space(), bs->dsk.block_count * bs->dsk.data_block_size,
+        format_size(bs->heap->get_data_used_space()).c_str(),
+        format_size(bs->dsk.block_count * bs->dsk.data_block_size).c_str());
    return 0;
 }
@@ -462,6 +462,10 @@ int blockstore_impl_t::read_bitmap(object_id oid, uint64_t target_version, void
        {
            if (target_version >= wr->version)
            {
+                if (wr->type() == BS_HEAP_DELETE)
+                {
+                    return false;
+                }
                found = true;
                if (result_version)
                {
@@ -855,4 +855,29 @@ std::string blockstore_impl_t::get_op_diag(blockstore_op_t *op)
    return std::string(buf);
 }

+uint64_t blockstore_impl_t::get_live_entries()
+{
+    return used_blocks;
+}
+
+uint64_t blockstore_impl_t::get_live_memory()
+{
+    uint64_t used = 0;
+    for (auto & kv: clean_db_shards)
+    {
+        used += kv.second.size() * sizeof(blockstore_clean_db_t::value_type);
+    }
+    return used;
+}
+
+uint64_t blockstore_impl_t::get_garbage_entries()
+{
+    return dirty_db.size();
+}
+
+uint64_t blockstore_impl_t::get_garbage_memory()
+{
+    return (sizeof(obj_ver_id) + sizeof(dirty_entry) + 32) * dirty_db.size();
+}
+
 } // namespace v1
@@ -332,6 +332,10 @@ public:
    inline uint64_t get_free_block_count() { return dsk.block_count - used_blocks; }
    inline uint32_t get_bitmap_granularity() { return dsk.disk_alignment; }
    inline uint64_t get_journal_size() { return dsk.journal_len; }
+    uint64_t get_live_entries();
+    uint64_t get_live_memory();
+    uint64_t get_garbage_entries();
+    uint64_t get_garbage_memory();
 };

 } // namespace v1
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 2.8...3.30)

 project(vitastor)

@@ -12,7 +12,7 @@ if (RDMACM_LIBRARIES)
 	set(MSGR_RDMACM "msgr_rdmacm.cpp")
 endif (RDMACM_LIBRARIES)
 add_library(vitastor_common STATIC
-	../util/epoll_manager.cpp etcd_state_client.cpp messenger.cpp ../util/addr_util.cpp
+	../util/epoll_manager.cpp etcd_state_client.cpp messenger.cpp ../util/addr_util.cpp ../util/xxh_x86dispatch.c
 	msgr_encrypt.cpp msgr_stop.cpp msgr_op.cpp msgr_send.cpp msgr_receive.cpp ../util/ringloop.cpp ../../json11/json11.cpp
 	http_client.cpp osd_ops.cpp pg_states.cpp ../util/timerfd_manager.cpp ../util/str_util.cpp ../util/json_util.cpp ${MSGR_RDMA} ${MSGR_RDMACM}
 )
@@ -101,7 +101,7 @@ add_executable(test_cluster_client
 	EXCLUDE_FROM_ALL
 	../test/test_cluster_client.cpp
 	pg_states.cpp osd_ops.cpp cluster_client.cpp cluster_client_list.cpp cluster_client_wb.cpp cluster_client_icache.cpp msgr_op.cpp ../test/mock/messenger.cpp msgr_stop.cpp msgr_encrypt.cpp
-	etcd_state_client.cpp ../util/timerfd_manager.cpp ../util/addr_util.cpp ../util/str_util.cpp ../util/json_util.cpp ../../json11/json11.cpp
+	etcd_state_client.cpp ../util/timerfd_manager.cpp ../util/addr_util.cpp ../util/str_util.cpp ../util/json_util.cpp ../util/xxh_x86dispatch.c ../../json11/json11.cpp
 )
 target_link_libraries(test_cluster_client ${OPENSSL_LIBRARIES})
 target_compile_definitions(test_cluster_client PUBLIC -D__MOCK__)
@@ -27,7 +27,7 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
    msgr.ringloop = ringloop;
    msgr.repeer_pgs = [this](osd_num_t peer_osd)
    {
-        if (msgr.osd_peer_fds.find(peer_osd) != msgr.osd_peer_fds.end())
+        if (msgr.osd_peers.find(peer_osd) != msgr.osd_peers.end())
        {
            // peer_osd just connected
            continue_ops();
@@ -47,8 +47,8 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
    msgr.exec_op = [this](osd_op_t *op)
    {
        // Garbage in
-        fprintf(stderr, "Incoming garbage from peer %d\n", op->peer_fd);
-        msgr.stop_client(op->peer_fd);
+        fprintf(stderr, "Can't handle incoming operation from client %lu\n", op->client_id);
+        msgr.stop_client(op->client_id);
        delete op;
    };
    msgr.parse_config(config);
@@ -71,9 +71,6 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
        st_cli.infinite_start = config["client_infinite_start"].bool_value();
    }
    st_cli.load_global_config();
-
-    scrap_buffer_size = SCRAP_BUFFER_SIZE;
-    scrap_buffer = malloc_or_die(scrap_buffer_size);
 }

 cluster_client_t::~cluster_client_t()
@@ -96,7 +93,6 @@ cluster_client_t::~cluster_client_t()
    {
        ringloop->unregister_consumer(&consumer);
    }
-    free(scrap_buffer);
    delete wb;
    wb = NULL;
 }
@@ -158,7 +154,7 @@ void cluster_client_t::continue_raw_ops(osd_num_t peer_osd)
    {
        auto op = it->second;
        op->op_type = OSD_OP_OUT;
-        op->peer_fd = msgr.osd_peer_fds.at(peer_osd);
+        op->client_id = msgr.osd_peers.at(peer_osd)->client_id;
        msgr.outbox_push(op);
        raw_ops.erase(it++);
    }
@@ -594,7 +590,7 @@ void cluster_client_t::on_change_pool_config_hook()
        {
            if (log_level > 2 && pg_counts[pool_item.first])
            {
-                printf("Pool %u (%s) PG count changed from %lu to %lu\n", pool_item.first, pool_item.second.name.c_str(),
+                fprintf(stderr, "Pool %u (%s) PG count changed from %lu to %lu\n", pool_item.first, pool_item.second.name.c_str(),
                    pg_counts[pool_item.first], pool_item.second.real_pg_count);
            }
            // At this point, all pool operations should have been suspended
@@ -882,13 +878,13 @@ void cluster_client_t::execute_cas(cluster_op_t *op)
        if (op->retval != expected && op->retval >= 0)
            op->retval = -EIO;
        op->retval = op->retval == -EPIPE ? -EINTR : op->retval;
-        auto peer_it = msgr.osd_peer_fds.find(op->parts[0].osd_num);
+        auto peer_it = msgr.osd_peers.find(op->parts[0].osd_num);
        if (op->retval != 0 || (op->flags & OP_IMMEDIATE_COMMIT))
        {
            auto cb = std::move(op->callback);
            cb(op);
        }
-        else if (peer_it == msgr.osd_peer_fds.end())
+        else if (peer_it == msgr.osd_peers.end())
        {
            // Care must be taken to make sure that the client doesn't reconnect to the OSD
            // before executing the previously completed operation callback (!)
@@ -899,10 +895,10 @@ void cluster_client_t::execute_cas(cluster_op_t *op)
        else
        {
            // CAS writes have a built-in sync
-            auto peer_fd = peer_it->second;
+            osd_client_t *cl = peer_it->second;
            *part = (osd_op_t){
                .op_type = OSD_OP_OUT,
-                .peer_fd = peer_fd,
+                .client_id = cl->client_id,
                .req = {
                    .hdr = {
                        .magic = SECONDARY_OSD_OP_MAGIC,
@@ -1051,11 +1047,11 @@ bool cluster_client_t::check_rw(cluster_op_t *op)

 void cluster_client_t::execute_raw(osd_num_t osd_num, osd_op_t *op)
 {
-    auto fd_it = msgr.osd_peer_fds.find(osd_num);
-    if (fd_it != msgr.osd_peer_fds.end())
+    auto peer_it = msgr.osd_peers.find(osd_num);
+    if (peer_it != msgr.osd_peers.end())
    {
        op->op_type = OSD_OP_OUT;
-        op->peer_fd = fd_it->second;
+        op->client_id = peer_it->second->client_id;
        msgr.outbox_push(op);
    }
    else
@@ -1245,7 +1241,7 @@ resume_2:
    return 0;
 }

-static void add_iov(int size, bool skip, cluster_op_t *op, int &iov_idx, size_t &iov_pos, osd_op_buf_list_t &iov, void *scrap, int scrap_len)
+static void add_iov(int size, int skip, cluster_op_t *op, int &iov_idx, size_t &iov_pos, osd_op_buf_list_t &iov)
 {
    int left = size;
    while (left > 0 && iov_idx < op->iov.count)
@@ -1253,7 +1249,7 @@ static void add_iov(int size, bool skip, cluster_op_t *op, int &iov_idx, size_t
        int cur_left = op->iov.buf[iov_idx].iov_len - iov_pos;
        if (cur_left < left)
        {
-            if (!skip)
+            if (skip == 0)
            {
                iov.push_back((uint8_t*)op->iov.buf[iov_idx].iov_base + iov_pos, cur_left);
            }
@@ -1263,7 +1259,7 @@ static void add_iov(int size, bool skip, cluster_op_t *op, int &iov_idx, size_t
        }
        else
        {
-            if (!skip)
+            if (skip == 0)
            {
                iov.push_back((uint8_t*)op->iov.buf[iov_idx].iov_base + iov_pos, left);
            }
@@ -1272,16 +1268,10 @@ static void add_iov(int size, bool skip, cluster_op_t *op, int &iov_idx, size_t
        }
    }
    assert(left == 0);
-    if (skip && scrap_len > 0)
+    if (skip == 1)
    {
-        // All skipped ranges are read into the same useless buffer
-        left = size;
-        while (left > 0)
-        {
-            int cur_left = scrap_len < left ? scrap_len : left;
-            iov.push_back(scrap, cur_left);
-            left -= cur_left;
-        }
+        // data read into a NULL buffer will be discarded by messenger
+        iov.push_back(NULL, size);
    }
 }

@@ -1304,8 +1294,8 @@ void cluster_client_t::slice_rw(cluster_op_t *op)
        unsigned bitmap_mem = object_bitmap_size +
            op->parts.size() * pg_data_size *
            (pool_cfg.data_block_size / pool_cfg.bitmap_granularity / 8
-            // read chain info - 1 byte per block
-            + (op->enc ? op->len/pool_cfg.bitmap_granularity : 0));
+            // read chain_info - max 4 bytes per block
+            + (op->enc ? osd_op_rw_t::chain_info_bytes(op->enc->chain_size)*op->len/pool_cfg.bitmap_granularity : 0));
        if (!op->bitmap_buf || op->bitmap_buf_size < bitmap_mem)
        {
            op->bitmap_buf = realloc_or_die(op->bitmap_buf, bitmap_mem);
@@ -1347,10 +1337,10 @@ void cluster_client_t::slice_rw(cluster_op_t *op)
                        {
                            begin = cur;
                            // Just advance iov_idx & iov_pos
-                            add_iov(cur-prev, true, op, iov_idx, iov_pos, op->parts[i].iov, NULL, 0);
+                            add_iov(cur-prev, 2, op, iov_idx, iov_pos, op->parts[i].iov);
                        }
                        else
-                            add_iov(cur-prev, skip_prev, op, iov_idx, iov_pos, op->parts[i].iov, scrap_buffer, scrap_buffer_size);
+                            add_iov(cur-prev, skip_prev ? 1 : 0, op, iov_idx, iov_pos, op->parts[i].iov);
                    }
                    skip_prev = skip;
                    prev = cur;
@@ -1361,11 +1351,11 @@ void cluster_client_t::slice_rw(cluster_op_t *op)
            if (skip_prev)
            {
                // Just advance iov_idx & iov_pos
-                add_iov(end-prev, true, op, iov_idx, iov_pos, op->parts[i].iov, NULL, 0);
+                add_iov(end-prev, 2, op, iov_idx, iov_pos, op->parts[i].iov);
                end = prev;
            }
            else
-                add_iov(cur-prev, skip_prev, op, iov_idx, iov_pos, op->parts[i].iov, scrap_buffer, scrap_buffer_size);
+                add_iov(cur-prev, skip_prev ? 1 : 0, op, iov_idx, iov_pos, op->parts[i].iov);
            if (end == begin)
            {
                op->done_count++;
@@ -1374,7 +1364,7 @@ void cluster_client_t::slice_rw(cluster_op_t *op)
        }
        else if (op->opcode != OSD_OP_READ_BITMAP && op->opcode != OSD_OP_READ_CHAIN_BITMAP && op->opcode != OSD_OP_DELETE)
        {
-            add_iov(end-begin, false, op, iov_idx, iov_pos, op->parts[i].iov, NULL, 0);
+            add_iov(end-begin, 0, op, iov_idx, iov_pos, op->parts[i].iov);
        }
        op->parts[i].parent = op;
        op->parts[i].offset = begin;
@@ -1454,15 +1444,18 @@ int cluster_client_t::try_send(cluster_op_t *op, int i, std::function<void(osd_o
                primary_osd = nearest_osd;
        }
        part->osd_num = primary_osd;
-        auto peer_it = msgr.osd_peer_fds.find(primary_osd);
-        if (peer_it != msgr.osd_peer_fds.end())
+        auto peer_it = msgr.osd_peers.find(primary_osd);
+        if (peer_it != msgr.osd_peers.end())
        {
-            int peer_fd = peer_it->second;
+            osd_client_t *cl = peer_it->second;
            part->flags |= PART_SENT|PART_VALID;
            op->inflight_count++;
            uint32_t pg_data_size = (pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks);
            uint64_t pg_bitmap_size = pg_data_size * (pool_cfg.data_block_size / pool_cfg.bitmap_granularity / 8
-                + (op->opcode == OSD_OP_READ && op->enc ? pool_cfg.data_block_size/pool_cfg.bitmap_granularity : 0));
+                // read chain_info - max 4 bytes per block
+                + (op->opcode == OSD_OP_READ && op->enc
+                    ? osd_op_rw_t::chain_info_bytes(op->enc->chain_size)*pool_cfg.data_block_size/pool_cfg.bitmap_granularity
+                    : 0));
            uint64_t meta_rev = 0;
            if (op->opcode != OSD_OP_READ_BITMAP && op->opcode != OSD_OP_DELETE && !op->deoptimise_snapshot)
            {
@@ -1472,7 +1465,7 @@ int cluster_client_t::try_send(cluster_op_t *op, int i, std::function<void(osd_o
            }
            part->op = (osd_op_t){
                .op_type = OSD_OP_OUT,
-                .peer_fd = peer_fd,
+                .client_id = cl->client_id,
                .req = { .rw = {
                    .header = {
                        .magic = SECONDARY_OSD_OP_MAGIC,
@@ -1523,8 +1516,8 @@ int cluster_client_t::continue_sync(cluster_op_t *op)
    for (auto do_it = dirty_osds.begin(); do_it != dirty_osds.end(); )
    {
        osd_num_t sync_osd = *do_it;
-        auto peer_it = msgr.osd_peer_fds.find(sync_osd);
-        if (peer_it == msgr.osd_peer_fds.end())
+        auto peer_it = msgr.osd_peers.find(sync_osd);
+        if (peer_it == msgr.osd_peers.end())
            dirty_osds.erase(do_it++);
        else
            do_it++;
@@ -1577,12 +1570,12 @@ resume_1:

 void cluster_client_t::send_sync(cluster_op_t *op, cluster_op_part_t *part)
 {
-    auto peer_fd = msgr.osd_peer_fds.at(part->osd_num);
+    osd_client_t *cl = msgr.osd_peers.at(part->osd_num);
    part->flags |= PART_SENT;
    op->inflight_count++;
    part->op = (osd_op_t){
        .op_type = OSD_OP_OUT,
-        .peer_fd = peer_fd,
+        .client_id = cl->client_id,
        .req = {
            .hdr = {
                .magic = SECONDARY_OSD_OP_MAGIC,
@@ -1622,10 +1615,10 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
            // Error priority: EIO > ENOSPC > ETIMEDOUT > EPIPE
            op->retval = part->op.reply.hdr.retval;
        }
-        int stop_fd = -1;
+        uint64_t stop_client_id = 0;
        if (op->retval != -EINTR && op->retval != -EIO && op->retval != -ENOSPC)
        {
-            stop_fd = part->op.peer_fd;
+            stop_client_id = part->op.client_id;
            if (op->retval != -EPIPE || log_level > 0)
            {
                fprintf(
@@ -1652,9 +1645,9 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
            op->retry_after = op->retval != -EPIPE ? client_eio_retry_interval : client_retry_interval;
        }
        reset_retry_timer(op->retry_after);
-        if (stop_fd >= 0)
+        if (stop_client_id)
        {
-            msgr.stop_client(stop_fd);
+            msgr.stop_client(stop_client_id);
        }
        op->inflight_count--;
        if (op->inflight_count == 0 && !op->retry_after)
@@ -1689,7 +1682,7 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
            if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
            {
                for (auto & part: op->parts)
-                    if (part.flags == (PART_SENT|PART_VALID|PART_DONE))
+                    if ((part.flags & (PART_SENT|PART_VALID|PART_DONE)) == (PART_SENT|PART_VALID|PART_DONE))
                        copy_part_bitmap(op, &part);
            }
            if (op->opcode == OSD_OP_SYNC)
@@ -84,7 +84,7 @@ class writeback_cache_t;

 struct inode_cache_t
 {
-    std::vector<inode_t> chain;
+    std::vector<inode_t> chain; // only parents from the same pool
    uint8_t *key_data = NULL;
    osd_op_enc_t *op_enc = NULL;
    bool readonly = false;
@@ -153,9 +153,6 @@ public:
    std::set<osd_num_t> dirty_osds;
    uint64_t dirty_bytes = 0, dirty_ops = 0;

-    void *scrap_buffer = NULL;
-    unsigned scrap_buffer_size = 0;
-
    // inodes require some extra state for read/write, it's stored here.
    // moreover, robin_hood access is slightly faster than std::map :)
    robin_hood::unordered_flat_map<inode_t, std::shared_ptr<inode_cache_t>> inode_cache;
@@ -5,7 +5,6 @@

 #include "cluster_client.h"

-#define SCRAP_BUFFER_SIZE 4*1024*1024
 #define PART_SENT 1
 #define PART_DONE 2
 #define PART_ERROR 4
@@ -295,7 +295,7 @@ int cluster_client_t::start_pg_listing(inode_list_pg_t *pg)
    bool conn = true;
    for (osd_num_t peer_osd: all_peers)
    {
-        if (msgr.osd_peer_fds.find(peer_osd) == msgr.osd_peer_fds.end())
+        if (msgr.osd_peers.find(peer_osd) == msgr.osd_peers.end())
        {
            // Initiate connection
            if (st_cli.peer_states[peer_osd].is_null())
@@ -340,7 +340,7 @@ void cluster_client_t::send_list(inode_list_osd_t *cur_list)
    osd_op_t *op = new osd_op_t();
    op->op_type = OSD_OP_OUT;
    // Already checked that it exists above, but anyway
-    op->peer_fd = msgr.osd_peer_fds.at(cur_list->osd_num);
+    op->client_id = msgr.osd_peers.at(cur_list->osd_num)->client_id;
    op->req = (osd_any_op_t){
        .sec_list = {
            .header = {
@@ -262,6 +262,7 @@ void etcd_state_client_t::parse_config(const json11::Json & config)
    }
    this->etcd_ca = config["etcd_ca"].string_value();
    this->etcd_prefix = config["etcd_prefix"].string_value();
+    this->use_auth = config["use_auth"].bool_value();
    if (this->etcd_prefix == "")
    {
        this->etcd_prefix = "/vitastor";
@@ -304,7 +305,6 @@ void etcd_state_client_t::parse_config(const json11::Json & config)
    {
        this->etcd_min_reload_interval = 50;
    }
-    this->use_image_index_for_auth = config["use_image_index_for_auth"].bool_value();
    if (this->etcd_ws_keepalive_interval != old_etcd_ws_keepalive_interval && ws_keepalive_timer >= 0)
    {
 #ifndef __MOCK__
@@ -436,7 +436,6 @@ void etcd_state_client_t::start_etcd_watcher_selected()
        etcd_watch_ws = http_init(get_http_ctx());
    else
        http_close(etcd_watch_ws);
-    etcd_total_watches = (use_image_index_for_auth ? 4 : 3);
    open_websocket(etcd_watch_ws, url.addr, url.hostname, url.path+"/watch", { .timeout = etcd_slow_timeout, .ssl = url.ssl },
        [this, cur_addr = url.addr](http_message_t *msg)
    {
@@ -456,12 +455,11 @@ void etcd_state_client_t::start_etcd_watcher_selected()
                {
                    if (watch_id == ETCD_CONFIG_WATCH_ID ||
                        watch_id == ETCD_PG_STATE_WATCH_ID ||
-                        watch_id == ETCD_OSD_STATE_WATCH_ID ||
-                        watch_id == ETCD_INDEX_WATCH_ID)
+                        watch_id == ETCD_OSD_STATE_WATCH_ID)
                    {
                        etcd_watches_initialised++;
                    }
-                    if (etcd_watches_initialised == etcd_total_watches && this->log_level > 0)
+                    if (etcd_watches_initialised == ETCD_TOTAL_WATCHES && this->log_level > 0)
                    {
                        fprintf(stderr, "Successfully subscribed to etcd at %s, revision %ju/%ju/%ju\n", cur_addr.c_str(),
                            etcd_watch_revision_config, etcd_watch_revision_osd, etcd_watch_revision_pg);
@@ -482,7 +480,7 @@ void etcd_state_client_t::start_etcd_watcher_selected()
                                fprintf(stderr, "Revisions before %ju were compacted by etcd, reloading state\n",
                                    data["result"]["compact_revision"].uint64_value());
                                http_close(etcd_watch_ws);
-                                etcd_watch_revision_config = etcd_watch_revision_osd = etcd_watch_revision_pg = etcd_watch_revision_index = 0;
+                                etcd_watch_revision_config = etcd_watch_revision_osd = etcd_watch_revision_pg = 0;
                                on_reload_hook();
                            }
                            return;
@@ -503,7 +501,7 @@ void etcd_state_client_t::start_etcd_watcher_selected()
                // Save revision only if it's present in the message - because sometimes etcd sends something without a header, like:
                // {"error": {"grpc_code": 14, "http_code": 503, "http_status": "Service Unavailable", "message": "error reading from server: EOF"}}
                // Also don't save revision from the initial created: true messages because they always contain the latest revision
-                if (etcd_watches_initialised == etcd_total_watches &&
+                if (etcd_watches_initialised == ETCD_TOTAL_WATCHES &&
                    !data["result"]["header"]["revision"].is_null() &&
                    !data["result"]["created"].bool_value())
                {
@@ -526,8 +524,6 @@ void etcd_state_client_t::start_etcd_watcher_selected()
                        etcd_watch_revision_pg = watch_rev;
                    else if (watch_id == ETCD_OSD_STATE_WATCH_ID)
                        etcd_watch_revision_osd = watch_rev;
-                    else if (watch_id == ETCD_INDEX_WATCH_ID)
-                        etcd_watch_revision_index = watch_rev;
                    etcd_urls_to_try.clear();
                }
                // First gather all changes into a hash to remove multiple overwrites
@@ -603,18 +599,6 @@ void etcd_state_client_t::start_etcd_watcher_selected()
            { "progress_notify", true },
        } }
    }).dump());
-    if (use_image_index_for_auth)
-    {
-        http_post_message(etcd_watch_ws, WS_TEXT, json11::Json(json11::Json::object {
-            { "create_request", json11::Json::object {
-                { "key", base64_encode(etcd_prefix+"/index/image/") },
-                { "range_end", base64_encode(etcd_prefix+"/index/image0") },
-                { "start_revision", etcd_watch_revision_index },
-                { "watch_id", ETCD_INDEX_WATCH_ID },
-                { "progress_notify", true },
-            } }
-        }).dump());
-    }
    // FIXME: Do not watch /pg/history/ at all in client code (not in OSD)
    if (on_start_watcher_hook)
    {
@@ -638,7 +622,7 @@ void etcd_state_client_t::start_ws_keepalive()
    {
        ws_keepalive_timer = tfd->set_timer(etcd_ws_keepalive_interval*1000, true, [this](int)
        {
-            if (!etcd_watch_ws || etcd_watches_initialised < etcd_total_watches)
+            if (!etcd_watch_ws || etcd_watches_initialised < ETCD_TOTAL_WATCHES)
            {
                // Do nothing
            }
@@ -784,15 +768,6 @@ void etcd_state_client_t::load_pgs()
            } }
        },
    };
-    if (use_image_index_for_auth)
-    {
-        txn.push_back(json11::Json::object {
-            { "request_range", json11::Json::object {
-                { "key", base64_encode(etcd_prefix+"/index/image/") },
-                { "range_end", base64_encode(etcd_prefix+"/index/image0") },
-            } }
-        });
-    }
    json11::Json::object req = { { "success", txn } };
    json11::Json checks = load_pgs_checks_hook != NULL ? load_pgs_checks_hook() : json11::Json();
    if (checks.array_items().size() > 0)
@@ -1043,6 +1018,8 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
                pc.used_for_app = "fs:"+pc.used_for_app;
            else
                pc.used_for_app = pool_item.second["used_for_app"].as_string();
+            // Create group permission
+            pc.creator_group = pool_item.second["creator_group"].string_value();
            // Local Read Configuration
            std::string local_reads = pool_item.second["local_reads"].string_value();
            if (local_reads == "nearest")
@@ -1370,39 +1347,6 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
            }
        }
    }
-    else if (key.substr(0, etcd_prefix.length()+13) == etcd_prefix+"/index/image/")
-    {
-        // <etcd_prefix>/index/image/%s
-        // Used for environments based on etcd authentication to grant client image access
-        // by names - granting by ID is inconvenient because of snapshots
-        std::string inode_name = key.substr(etcd_prefix.length()+13);
-        if (!value.is_object())
-        {
-            auto n_it = this->inode_by_name.find(inode_name);
-            if (n_it != this->inode_by_name.end())
-            {
-                uint64_t inode_num = n_it->second;
-                if (on_inode_change_hook != NULL)
-                {
-                    on_inode_change_hook(inode_num, true);
-                }
-                this->inode_config.erase(inode_num);
-                this->inode_by_name.erase(n_it);
-            }
-        }
-        else if (value["name"] == inode_name)
-        {
-            uint64_t inode_num = INODE_WITH_POOL(value["pool_id"].uint64_value(), value["id"].uint64_value());
-            if (!INODE_POOL(inode_num) || !INODE_NO_POOL(inode_num))
-            {
-                fprintf(stderr, "Invalid inode ID in key %s: %ju\n", key.c_str(), inode_num);
-            }
-            else
-            {
-                insert_inode_config(deserialize_inode_cfg(inode_num, value, kv.mod_revision));
-            }
-        }
-    }
    else if (key == etcd_prefix+"/config/node_placement")
    {
        // <etcd_prefix>/config/node_placement
@@ -1410,6 +1354,14 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
        if (on_change_node_placement_hook)
            on_change_node_placement_hook();
    }
+    else if (use_auth && key.substr(0, etcd_prefix.length()+13) == etcd_prefix+"/config/user/")
+    {
+        // <etcd_prefix>/config/user/<username>
+        if (!value.is_object())
+            user_info.erase(key.substr(etcd_prefix.length()+13));
+        else
+            user_info[key.substr(etcd_prefix.length()+13)] = value;
+    }
 }

 uint32_t etcd_state_client_t::parse_immediate_commit(const std::string & immediate_commit_str, uint32_t default_value)
@@ -1500,6 +1452,18 @@ json11::Json::object etcd_state_client_t::serialize_inode_cfg(inode_config_t *cf
    {
        new_cfg["deleted"] = true;
    }
+    if (!cfg->owner.empty())
+    {
+        new_cfg["owner"] = cfg->owner;
+    }
+    if (!cfg->owner_group.empty())
+    {
+        new_cfg["owner_group"] = cfg->owner_group;
+    }
+    if (!cfg->reader_group.empty())
+    {
+        new_cfg["reader_group"] = cfg->reader_group;
+    }
    if (cfg->meta.is_object())
    {
        new_cfg["meta"] = cfg->meta;
@@ -1546,6 +1510,9 @@ inode_config_t etcd_state_client_t::deserialize_inode_cfg(uint64_t inode_num, js
        .readonly = value["readonly"].bool_value(),
        .deleted = value["deleted"].bool_value(),
        .enc_key = std::move(enc_key),
+        .owner = value["owner"].string_value(),
+        .owner_group = value["owner_group"].string_value(),
+        .reader_group = value["reader_group"].string_value(),
        .meta = value["meta"],
        .mod_revision = mod_revision,
    };
@@ -13,7 +13,7 @@
 #define ETCD_CONFIG_WATCH_ID 1
 #define ETCD_OSD_STATE_WATCH_ID 2
 #define ETCD_PG_STATE_WATCH_ID 3
-#define ETCD_INDEX_WATCH_ID 4
+#define ETCD_TOTAL_WATCHES 3

 #define DEFAULT_BLOCK_SIZE 128*1024
 #define MIN_DATA_BLOCK_SIZE 4*1024
@@ -69,6 +69,7 @@ struct pool_config_t
    std::map<pg_num_t, pg_config_t> pg_config;
    uint64_t scrub_interval = 0;
    std::string used_for_app;
+    std::string creator_group;
    int backfillfull = 0;
    int local_reads = 0;

@@ -87,6 +88,8 @@ struct inode_config_t
    bool readonly = false;
    bool deleted = false;
    std::string enc_key;
+    // Permissions
+    std::string owner, owner_group, reader_group;
    // Arbitrary metadata
    json11::Json meta;
    // Change revision of the metadata in etcd
@@ -126,7 +129,6 @@ protected:
    std::vector<inode_watch_t*> watches;
    std::vector<std::function<void()>> on_resolve_queue;
    bool new_pg_config = false;
-    bool use_image_index_for_auth = false;
    int ws_keepalive_timer = -1;
    int ws_alive = 0;
    bool rand_initialized = false;
@@ -143,6 +145,7 @@ public:
    int etcd_slow_timeout = 5000;
    int etcd_min_reload_interval = 1000;
    bool infinite_start = true;
+    bool use_auth = false;
    uint64_t global_block_size = DEFAULT_BLOCK_SIZE;
    uint32_t global_bitmap_granularity = DEFAULT_BITMAP_GRANULARITY;
    uint32_t global_immediate_commit = IMMEDIATE_NONE;
@@ -158,11 +161,9 @@ public:
    http_context_t *http_ctx = NULL;
    http_co_t *etcd_watch_ws = NULL, *keepalive_client = NULL;
    int etcd_watches_initialised = 0;
-    int etcd_total_watches = 0;
    uint64_t etcd_watch_revision_config = 0;
    uint64_t etcd_watch_revision_osd = 0;
    uint64_t etcd_watch_revision_pg = 0;
-    uint64_t etcd_watch_revision_index = 0;
    timespec etcd_last_reload = {};
    int load_pgs_timer_id = -1;
    std::map<pool_id_t, pool_config_t> pool_config;
@@ -170,6 +171,7 @@ public:
    std::set<osd_num_t> seen_peers;
    std::map<inode_t, inode_config_t> inode_config;
    std::map<std::string, inode_t> inode_by_name;
+    std::map<std::string, json11::Json> user_info;
    json11::Json node_placement;

    std::function<void(std::map<std::string, etcd_kv_t> &)> on_change_hook;
@@ -48,6 +48,7 @@ struct http_context_t

 #ifdef WITH_OPENSSL
    SSL_CTX *ssl_ctx = NULL;
+    std::string ssl_cn;
 #endif

    ~http_context_t()
@@ -163,47 +164,81 @@ void http_ares_cb(void *data, ares_socket_t socket_fd, int readable, int writabl
 }

 #ifdef WITH_OPENSSL
+bool openssl_ctx_add_ca(SSL_CTX *ssl_ctx, const std::string & file_or_pem)
+{
+    std::string pem;
+    BIO *bio = NULL;
+    if (file_or_pem.substr(0, 5) != "-----")
+    {
+        pem = read_file(file_or_pem);
+        bio = BIO_new_mem_buf(pem.data(), pem.size());
+    }
+    else
+        bio = BIO_new_mem_buf(file_or_pem.data(), file_or_pem.size());
+    if (!bio)
+        return false;
+    X509 *x509 = PEM_read_bio_X509(bio, NULL, 0, NULL);
+    bool ok = !!x509;
+    if (x509)
+    {
+        X509_STORE *store = SSL_CTX_get_cert_store(ssl_ctx);
+        X509_STORE_add_cert(store, x509);
+        X509_free(x509);
+    }
+    BIO_free(bio);
+    return ok;
+}
+
 bool openssl_ctx_use_ca(SSL_CTX *ssl_ctx, const std::string & file_or_pem)
 {
    if (file_or_pem.substr(0, 5) == "-----")
    {
-        BIO *bio = BIO_new_mem_buf(file_or_pem.data(), file_or_pem.size());
-        if (!bio)
-            return false;
-        X509 *x509 = PEM_read_bio_X509(bio, NULL, 0, NULL);
-        bool ok = !!x509;
-        if (x509)
-        {
-            X509_STORE *store = SSL_CTX_get_cert_store(ssl_ctx);
-            X509_STORE_add_cert(store, x509);
-            X509_free(x509);
-        }
-        BIO_free(bio);
-        return ok;
+        return openssl_ctx_add_ca(ssl_ctx, file_or_pem);
    }
    return file_or_pem.empty()
        ? !!SSL_CTX_set_default_verify_paths(ssl_ctx)
        : !!SSL_CTX_load_verify_locations(ssl_ctx, file_or_pem.c_str(), NULL);
 }

-bool openssl_ctx_use_cert(SSL_CTX *ssl_ctx, const std::string & file_or_pem)
+std::string openssl_get_cn(X509 *x509)
 {
-    if (file_or_pem.substr(0, 5) == "-----")
+    X509_NAME* subj = X509_get_subject_name(x509);
+    int pos = X509_NAME_get_index_by_NID(subj, NID_commonName, -1);
+    if (pos != -1)
    {
-        BIO *bio = BIO_new_mem_buf(file_or_pem.data(), file_or_pem.size());
-        if (!bio)
-            return false;
-        X509 *x509 = PEM_read_bio_X509(bio, NULL, 0, NULL);
-        bool ok = !!x509;
-        if (x509)
-        {
-            ok = SSL_CTX_use_certificate(ssl_ctx, x509);
-            X509_free(x509);
-        }
-        BIO_free(bio);
-        return ok;
+        X509_NAME_ENTRY* cn = X509_NAME_get_entry(subj, pos);
+        ASN1_STRING* str = X509_NAME_ENTRY_get_data(cn);
+        return std::string((const char*)ASN1_STRING_get0_data(str), ASN1_STRING_length(str));
    }
-    return !!SSL_CTX_use_certificate_file(ssl_ctx, file_or_pem.c_str(), SSL_FILETYPE_PEM);
+    return "";
+}
+
+bool openssl_ctx_use_cert(SSL_CTX *ssl_ctx, const std::string & file_or_pem, std::string & common_name)
+{
+    BIO *bio = NULL;
+    std::string contents;
+    if (file_or_pem.substr(0, 5) == "-----")
+        bio = BIO_new_mem_buf(file_or_pem.data(), file_or_pem.size());
+    else
+    {
+        contents = read_file(file_or_pem);
+        if (!contents.size())
+            return false;
+        bio = BIO_new_mem_buf(contents.data(), contents.size());
+    }
+    if (!bio)
+        return false;
+    X509 *x509 = PEM_read_bio_X509(bio, NULL, 0, NULL);
+    bool ok = !!x509;
+    if (x509)
+    {
+        ok = SSL_CTX_use_certificate(ssl_ctx, x509);
+        if (ok)
+            common_name = openssl_get_cn(x509);
+        X509_free(x509);
+    }
+    BIO_free(bio);
+    return ok;
 }

 bool openssl_ctx_use_key(SSL_CTX *ssl_ctx, const std::string & file_or_pem)
@@ -243,6 +278,7 @@ http_context_t* http_context_init(timerfd_manager_t *tfd, const std::string & ss
    ctx->ssl_key = ssl_key;
    ctx->ssl_ca = ssl_ca;
    ctx->ssl_ctx = ssl_ctx;
+    ctx->ssl_cn = "";
    if (!ssl_ctx)
        goto init_err;
    SSL_CTX_set_verify(ssl_ctx, verify_peer ? SSL_VERIFY_PEER : SSL_VERIFY_NONE, NULL);
@@ -251,7 +287,7 @@ http_context_t* http_context_init(timerfd_manager_t *tfd, const std::string & ss
    if (!openssl_ctx_use_ca(ssl_ctx, ssl_ca))
        goto init_err;
    if (ssl_cert != "" && ssl_key != "" &&
-        (!openssl_ctx_use_cert(ssl_ctx, ssl_cert) ||
+        (!openssl_ctx_use_cert(ssl_ctx, ssl_cert, ctx->ssl_cn) ||
        !openssl_ctx_use_key(ssl_ctx, ssl_key)))
        goto init_err;
 #endif
@@ -262,6 +298,11 @@ init_err:
    return NULL;
 }

+std::string http_context_get_ssl_cn(http_context_t *ctx)
+{
+    return ctx->ssl_cn;
+}
+
 struct http_ctx_resolve_t
 {
    std::function<void(const std::string & error, const std::vector<std::string> & addrs)> cb;
@@ -1112,6 +1153,11 @@ bool http_co_t::handle_read()
            }
            state = HTTP_CO_REQ_HDR_RECEIVED;
            parse_http_headers(response, &parsed, true);
+            if (ssl)
+            {
+                auto x509 = SSL_get0_peer_certificate(ssl_cli);
+                parsed.headers["_tls_common_name"] = openssl_get_cn(x509);
+            }
            auto conn_it = parsed.headers.find("connection");
            keepalive = (conn_it != parsed.headers.end() && conn_it->second == "keep-alive");
            auto enc_it = parsed.headers.find("transfer-encoding");
@@ -8,6 +8,10 @@
 #include <functional>
 #include "json11/json11.hpp"

+#ifdef WITH_OPENSSL
+#include <openssl/types.h>
+#endif
+
 #define WS_CONTINUATION 0
 #define WS_TEXT 1
 #define WS_BINARY 2
@@ -48,6 +52,7 @@ struct http_co_t;

 http_context_t* http_context_init(timerfd_manager_t *tfd, const std::string & ssl_cert, const std::string & ssl_key,
    const std::string & ssl_ca, bool verify_peer, std::string & error);
+std::string http_context_get_ssl_cn(http_context_t *ctx);
 void http_resolve(http_context_t *ctx, bool ssl, std::string host,
    std::function<void(const std::string & error, const std::vector<std::string> & addrs)> cb);
 void http_context_destroy(http_context_t *ctx);
@@ -68,3 +73,11 @@ void http_close(http_co_t *co);
 void http_destroy(http_co_t *co);

 #pragma GCC visibility pop
+
+#ifdef WITH_OPENSSL
+bool openssl_ctx_add_ca(SSL_CTX *ssl_ctx, const std::string & file_or_pem);
+bool openssl_ctx_use_ca(SSL_CTX *ssl_ctx, const std::string & file_or_pem);
+std::string openssl_get_cn(X509 *x509);
+bool openssl_ctx_use_cert(SSL_CTX *ssl_ctx, const std::string & file_or_pem, std::string & common_name);
+bool openssl_ctx_use_key(SSL_CTX *ssl_ctx, const std::string & file_or_pem);
+#endif
@@ -10,10 +10,18 @@
 #include <stdexcept>

 #include "addr_util.h"
+#include "str_util.h"
 #include "messenger.h"
 #ifdef WITH_RDMA
 #include "msgr_rdma.h"
 #endif
+#include "http_client.h"
+#ifdef WITH_OPENSSL
+#include <openssl/bio.h>
+#include <openssl/err.h>
+#include <openssl/pem.h>
+#include <openssl/ssl.h>
+#endif

 #include <sys/poll.h>

@@ -117,6 +125,50 @@ void msgr_iothread_t::run()

 void osd_messenger_t::init()
 {
+    if (!tls_cert.empty() || !tls_key.empty() || !osd_tls_ca.empty() || !client_tls_ca.empty())
+    {
+        // Initialize TLS context
+        // FIXME: require OpenSSL
+#ifndef WITH_OPENSSL
+        fprintf(stderr, "Vitastor is built without OpenSSL support\n");
+        exit(1);
+#else
+        if (tls_cert.empty() || tls_key.empty() || osd_tls_ca.empty() || osd_num && client_tls_ca.empty())
+        {
+            if (osd_num)
+                fprintf(stderr, "Vitastor OSD TLS requires osd_tls_cert, osd_tls_key, osd_tls_ca, client_tls_ca\n");
+            else
+                fprintf(stderr, "Vitastor client TLS requires tls_cert, tls_key and osd_tls_ca\n");
+            exit(1);
+        }
+        else
+        {
+            ssl_ctx = SSL_CTX_new(TLS_method());
+            if (!ssl_ctx)
+            {
+init_err:
+                fprintf(stderr, "OpenSSL initialization failed: %s\n", ERR_error_string(ERR_get_error(), NULL));
+                exit(1);
+            }
+            SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, NULL);
+            bool ok = SSL_CTX_set_min_proto_version(ssl_ctx, TLS1_3_VERSION);
+            ok = ok && openssl_ctx_add_ca(ssl_ctx, osd_tls_ca);
+            if (osd_num)
+            {
+                // OSD uses 2 separate root certificates to distinguish between clients and peer OSDs
+                ok = ok && openssl_ctx_add_ca(ssl_ctx, client_tls_ca);
+            }
+            ok = ok && openssl_ctx_use_cert(ssl_ctx, tls_cert, tls_cn);
+            ok = ok && openssl_ctx_use_key(ssl_ctx, tls_key);
+            if (!ok)
+            {
+                SSL_CTX_free(ssl_ctx);
+                ssl_ctx = NULL;
+                goto init_err;
+            }
+        }
+#endif
+    }
 #ifdef WITH_RDMACM
    if (use_rdmacm)
    {
@@ -182,12 +234,13 @@ void osd_messenger_t::init()
    }
    keepalive_timer_id = tfd->set_timer(1000, true, [this](int)
    {
+        std::vector<uint64_t> clients_to_stop;
+        std::vector<osd_op_t*> ops_to_send;
        auto cl_it = clients.begin();
        while (cl_it != clients.end())
        {
            auto cl = cl_it->second;
            cl_it++;
-            auto peer_fd = cl->peer_fd;
            if (!cl->osd_num && !cl->in_osd_num || cl->peer_state != PEER_CONNECTED && cl->peer_state != PEER_RDMA)
            {
                // Do not run keepalive on regular clients
@@ -199,10 +252,9 @@ void osd_messenger_t::init()
                if (!cl->ping_time_remaining)
                {
                    // Ping timed out, stop the client
-                    fprintf(stderr, "Ping timed out for OSD %ju (client %d), disconnecting peer\n", cl->in_osd_num ? cl->in_osd_num : cl->osd_num, cl->peer_fd);
-                    stop_client(peer_fd, true);
-                    // Restart iterator because it may be invalidated
-                    cl_it = clients.upper_bound(peer_fd);
+                    fprintf(stderr, "Ping timed out for OSD %ju (client %ju), disconnecting peer\n",
+                        cl->in_osd_num ? cl->in_osd_num : cl->osd_num, cl->client_id);
+                    clients_to_stop.push_back(cl->client_id);
                }
            }
            else if (cl->idle_time_remaining > 0)
@@ -213,7 +265,7 @@ void osd_messenger_t::init()
                    // Connection is idle for <osd_idle_time>, send ping
                    osd_op_t *op = new osd_op_t();
                    op->op_type = OSD_OP_OUT;
-                    op->peer_fd = cl->peer_fd;
+                    op->client_id = cl->client_id;
                    op->req = (osd_any_op_t){
                        .hdr = {
                            .magic = SECONDARY_OSD_OP_MAGIC,
@@ -222,28 +274,26 @@ void osd_messenger_t::init()
                    };
                    op->callback = [this, cl](osd_op_t *op)
                    {
-                        auto cl_it = clients.find(op->peer_fd);
-                        if (cl_it == clients.end() || cl_it->second != cl)
+                        auto cl_it = clients.find(op->client_id);
+                        if (cl_it == clients.end())
                        {
                            // client is already dropped
                            delete op;
                            return;
                        }
-                        int fail_fd = (op->reply.hdr.retval != 0 ? op->peer_fd : -1);
+                        uint64_t fail_client_id = (op->reply.hdr.retval != 0 ? op->client_id : 0);
                        auto fail_osd_num = cl->in_osd_num ? cl->in_osd_num : cl->osd_num;
                        cl->ping_time_remaining = 0;
                        delete op;
-                        if (fail_fd >= 0)
+                        if (fail_client_id)
                        {
-                            fprintf(stderr, "Ping failed for OSD %ju (client %d), disconnecting peer\n", fail_osd_num, fail_fd);
-                            stop_client(fail_fd, true);
+                            fprintf(stderr, "Ping failed for OSD %ju (client %ju), disconnecting peer\n", fail_osd_num, fail_client_id);
+                            stop_client(fail_client_id);
                        }
                    };
                    cl->ping_time_remaining = osd_ping_timeout;
                    cl->idle_time_remaining = osd_idle_timeout;
-                    outbox_push(op);
-                    // Restart iterator because it may be invalidated
-                    cl_it = clients.upper_bound(peer_fd);
+                    ops_to_send.push_back(op);
                }
            }
            else
@@ -251,6 +301,14 @@ void osd_messenger_t::init()
                cl->idle_time_remaining = osd_idle_timeout;
            }
        }
+        for (uint64_t client_id: clients_to_stop)
+        {
+            stop_client(client_id);
+        }
+        for (osd_op_t *op: ops_to_send)
+        {
+            outbox_push(op);
+        }
    });
 }

@@ -263,7 +321,7 @@ osd_messenger_t::~osd_messenger_t()
    }
    while (clients.size() > 0)
    {
-        stop_client(clients.begin()->first, true, true);
+        stop_client(clients.begin()->first, true);
    }
    if (iothreads.size())
    {
@@ -297,6 +355,13 @@ osd_messenger_t::~osd_messenger_t()
    {
        destroy_aes_xts_decrypt(decrypt_ctx);
    }
+#ifdef WITH_OPENSSL
+    if (ssl_ctx)
+    {
+        SSL_CTX_free(ssl_ctx);
+        ssl_ctx = NULL;
+    }
+#endif
 }

 void osd_messenger_t::parse_config(const json11::Json & config)
@@ -334,6 +399,30 @@ void osd_messenger_t::parse_config(const json11::Json & config)
    this->max_aes_xts_pool_size = config["max_aes_xts_pool_size"].uint64_value();
    if (!this->max_aes_xts_pool_size)
        this->max_aes_xts_pool_size = 256;
+    if (config["proto_checksums"].is_null())
+        this->use_proto_checksums = MSGR_CSUM_PAYLOAD;
+    else if (config["proto_checksums"].is_bool())
+        this->use_proto_checksums = config["proto_checksums"].bool_value() ? MSGR_CSUM_FULL : 0;
+    else if (config["proto_checksums"].string_value() != "")
+        this->use_proto_checksums = config["proto_checksums"].string_value() == "full" ? MSGR_CSUM_FULL : MSGR_CSUM_PAYLOAD;
+    else
+        this->use_proto_checksums = 0;
+    if (!osd_num)
+    {
+        tls_cert = config["tls_cert"].string_value();
+        tls_key = config["tls_key"].string_value();
+        osd_tls_ca = config["osd_tls_ca"].string_value();
+    }
+    else
+    {
+        tls_cert = config["osd_tls_cert"].string_value();
+        tls_key = config["osd_tls_key"].string_value();
+        osd_tls_ca = config["osd_tls_ca"].string_value();
+        client_tls_ca = config["client_tls_ca"].string_value();
+    }
+    test_osd_aes_key.resize(32);
+    if (fromhexstr(config["test_osd_aes_key"].string_value(), 32, (uint8_t*)test_osd_aes_key.data()) != 32)
+        test_osd_aes_key.clear();
    if (!osd_num)
        this->iothread_count = (uint32_t)config["client_iothread_count"].uint64_value();
    else
@@ -342,7 +431,7 @@ void osd_messenger_t::parse_config(const json11::Json & config)
    if (!this->receive_buffer_size || this->receive_buffer_size > 1024*1024*1024)
        this->receive_buffer_size = 65536;
    this->use_sync_send_recv = config["use_sync_send_recv"].bool_value() ||
-        config["use_sync_send_recv"].uint64_value();
+        config["use_sync_send_recv"].uint64_value() || !ringloop;
    this->min_zerocopy_send_size = config["min_zerocopy_send_size"].is_null()
        ? DEFAULT_MIN_ZEROCOPY_SEND_SIZE
        : (int)config["min_zerocopy_send_size"].int64_value();
@@ -451,7 +540,7 @@ void osd_messenger_t::try_connect_peer(uint64_t peer_osd)
    {
        return;
    }
-    if (osd_peer_fds.find(peer_osd) != osd_peer_fds.end())
+    if (osd_peers.find(peer_osd) != osd_peers.end())
    {
        wanted_peers.erase(peer_osd);
        return;
@@ -478,20 +567,20 @@ void osd_messenger_t::try_connect_peer_tcp(osd_num_t peer_osd, const char *peer_
 #ifdef WITH_RDMACM
    if (disable_tcp)
    {
-        on_connect_peer(peer_osd, -EINVAL);
+        on_connect_peer(peer_osd, -EINVAL, 0);
        return;
    }
 #endif
    struct sockaddr_storage addr;
    if (!string_to_addr(peer_host, 0, peer_port, &addr))
    {
-        on_connect_peer(peer_osd, -EINVAL);
+        on_connect_peer(peer_osd, -EINVAL, 0);
        return;
    }
    int peer_fd = socket(addr.ss_family, SOCK_STREAM, 0);
    if (peer_fd < 0)
    {
-        on_connect_peer(peer_osd, -errno);
+        on_connect_peer(peer_osd, -errno, 0);
        return;
    }
    fcntl(peer_fd, F_SETFL, fcntl(peer_fd, F_GETFL, 0) | O_NONBLOCK);
@@ -499,21 +588,25 @@ void osd_messenger_t::try_connect_peer_tcp(osd_num_t peer_osd, const char *peer_
    if (r < 0 && errno != EINPROGRESS)
    {
        close(peer_fd);
-        on_connect_peer(peer_osd, -errno);
+        on_connect_peer(peer_osd, -errno, 0);
        return;
    }
-    clients[peer_fd] = new osd_client_t();
+    const uint64_t client_id = next_client_id++;
+    osd_client_t *cl = new osd_client_t();
    if (log_level > 0)
    {
-        fprintf(stderr, "Connecting to OSD %ju at %s:%d (client %d)\n", peer_osd, peer_host, peer_port, peer_fd);
+        fprintf(stderr, "Connecting to OSD %ju at %s:%d (client %ju, FD %d)\n", peer_osd, peer_host, peer_port, client_id, peer_fd);
    }
-    clients[peer_fd]->peer_addr = addr;
-    clients[peer_fd]->peer_port = peer_port;
-    clients[peer_fd]->peer_fd = peer_fd;
-    clients[peer_fd]->peer_state = PEER_CONNECTING;
-    clients[peer_fd]->connect_timeout_id = -1;
-    clients[peer_fd]->osd_num = peer_osd;
-    clients[peer_fd]->in_buf = (uint8_t*)malloc_or_die(receive_buffer_size);
+    cl->client_id = client_id;
+    cl->peer_addr = addr;
+    cl->peer_port = peer_port;
+    cl->peer_fd = peer_fd;
+    cl->peer_state = PEER_CONNECTING;
+    cl->connect_timeout_id = -1;
+    cl->osd_num = peer_osd;
+    cl->in_buf = (uint8_t*)malloc_or_die(receive_buffer_size);
+    clients[client_id] = cl;
+    clients_by_fd[peer_fd] = cl;
    tfd->set_fd_handler(peer_fd, true, [this](int peer_fd, int epoll_events)
    {
        // Either OUT (connected) or HUP
@@ -521,11 +614,11 @@ void osd_messenger_t::try_connect_peer_tcp(osd_num_t peer_osd, const char *peer_
    });
    if (peer_connect_timeout > 0)
    {
-        clients[peer_fd]->connect_timeout_id = tfd->set_timer(1000*peer_connect_timeout, false, [this, peer_fd](int timer_id)
+        cl->connect_timeout_id = tfd->set_timer(1000*peer_connect_timeout, false, [this, client_id](int timer_id)
        {
-            osd_num_t peer_osd = clients.at(peer_fd)->osd_num;
-            stop_client(peer_fd, true);
-            on_connect_peer(peer_osd, -EPIPE);
+            osd_num_t peer_osd = clients.at(client_id)->osd_num;
+            stop_client(client_id);
+            on_connect_peer(peer_osd, -EPIPE, 0);
            return;
        });
    }
@@ -533,7 +626,7 @@ void osd_messenger_t::try_connect_peer_tcp(osd_num_t peer_osd, const char *peer_

 void osd_messenger_t::handle_connect_epoll(int peer_fd)
 {
-    auto cl = clients[peer_fd];
+    auto cl = clients_by_fd.at(peer_fd);
    if (cl->connect_timeout_id >= 0)
    {
        tfd->clear_timer(cl->connect_timeout_id);
@@ -548,8 +641,8 @@ void osd_messenger_t::handle_connect_epoll(int peer_fd)
    }
    if (result != 0)
    {
-        stop_client(peer_fd, true);
-        on_connect_peer(peer_osd, -result);
+        stop_client(cl->client_id);
+        on_connect_peer(peer_osd, -result, 0);
        return;
    }
    int one = 1;
@@ -560,29 +653,30 @@ void osd_messenger_t::handle_connect_epoll(int peer_fd)
        handle_peer_epoll(peer_fd, epoll_events);
    });
    // Check OSD number
+    ssl_init(cl, false);
    check_peer_config(cl);
 }

 void osd_messenger_t::handle_peer_epoll(int peer_fd, int epoll_events)
 {
    // Mark client as ready (i.e. some data is available)
+    auto cl = clients_by_fd.at(peer_fd);
    if (epoll_events & EPOLLRDHUP)
    {
        // Stop client
        if (log_level > 0)
        {
-            fprintf(stderr, "[OSD %ju] client %d disconnected\n", this->osd_num, peer_fd);
+            fprintf(stderr, "[OSD %ju] client %ju disconnected\n", this->osd_num, cl->client_id);
        }
-        stop_client(peer_fd, true);
+        stop_client(cl->client_id);
    }
    else if (epoll_events & EPOLLIN)
    {
        // Mark client as ready (i.e. some data is available)
-        auto cl = clients[peer_fd];
        cl->read_ready++;
        if (cl->read_ready == 1)
        {
-            read_ready_clients.push_back(cl->peer_fd);
+            read_ready_clients.push_back(cl->client_id);
            if (ringloop)
                ringloop->wakeup();
            else
@@ -591,13 +685,13 @@ void osd_messenger_t::handle_peer_epoll(int peer_fd, int epoll_events)
    }
 }

-void osd_messenger_t::on_connect_peer(osd_num_t peer_osd, int peer_fd)
+void osd_messenger_t::on_connect_peer(osd_num_t peer_osd, int errcode, uint64_t client_id)
 {
    auto & wp = wanted_peers.at(peer_osd);
    wp.connecting = false;
-    if (peer_fd < 0)
+    if (errcode < 0)
    {
-        fprintf(stderr, "Failed to connect to peer OSD %ju address %s port %d: %s\n", peer_osd, wp.cur_addr.c_str(), wp.cur_port, strerror(-peer_fd));
+        fprintf(stderr, "Failed to connect to peer OSD %ju address %s port %d: %s\n", peer_osd, wp.cur_addr.c_str(), wp.cur_port, strerror(-errcode));
        if (wp.address_changed)
        {
            wp.address_changed = false;
@@ -624,7 +718,7 @@ void osd_messenger_t::on_connect_peer(osd_num_t peer_osd, int peer_fd)
    }
    if (log_level > 0)
    {
-        fprintf(stderr, "[OSD %ju] Connected with peer OSD %ju (client %d)\n", osd_num, peer_osd, peer_fd);
+        fprintf(stderr, "[OSD %ju] Connected with peer OSD %ju (client %ju)\n", osd_num, peer_osd, client_id);
    }
    wanted_peers.erase(peer_osd);
    repeer_pgs(peer_osd);
@@ -634,7 +728,7 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl)
 {
    osd_op_t *op = new osd_op_t();
    op->op_type = OSD_OP_OUT;
-    op->peer_fd = cl->peer_fd;
+    op->client_id = cl->client_id;
    op->req = (osd_any_op_t){
        .show_conf = {
            .header = {
@@ -649,7 +743,12 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl)
        // Inform that we're OSD <osd_num>
        payload["osd_num"] = osd_num;
    }
-    payload["features"] = json11::Json::object{ { "check_sequencing", true } };
+    auto features = json11::Json::object{ { "check_sequencing", true } };
+    if (use_proto_checksums)
+    {
+        features["proto_checksums"] = use_proto_checksums;
+    }
+    payload["features"] = features;
 #ifdef WITH_RDMA
    if (!use_rdmacm && rdma_contexts.size())
    {
@@ -658,7 +757,7 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl)
        if (!selected_ctx)
        {
            if (log_level > 0)
-                fprintf(stderr, "No RDMA context for OSD %ju connection (peer %d), using only TCP\n", cl->osd_num, cl->peer_fd);
+                fprintf(stderr, "No RDMA context for OSD %ju connection (client %ju), using only TCP\n", cl->osd_num, cl->client_id);
        }
        else
        {
@@ -719,11 +818,19 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl)
        if (err)
        {
            osd_num_t peer_osd = cl->osd_num;
-            stop_client(op->peer_fd);
-            on_connect_peer(peer_osd, -EINVAL);
+            stop_client(op->client_id);
+            on_connect_peer(peer_osd, -EINVAL, 0);
            delete op;
            return;
        }
+        if (use_proto_checksums)
+        {
+            auto peer_csums = config["features"]["proto_checksums"].uint64_value();
+            if (peer_csums == MSGR_CSUM_FULL && use_proto_checksums == MSGR_CSUM_FULL)
+                cl->proto_csum_status = MSGR_CSUM_FULL;
+            else if (peer_csums && use_proto_checksums)
+                cl->proto_csum_status = MSGR_CSUM_PAYLOAD;
+        }
 #ifdef WITH_RDMA
        if (!use_rdmacm && cl->rdma_conn && config["rdma_address"].is_string())
        {
@@ -755,8 +862,8 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl)
            }
        }
 #endif
-        osd_peer_fds[cl->osd_num] = cl->peer_fd;
-        on_connect_peer(cl->osd_num, cl->peer_fd);
+        osd_peers[cl->osd_num] = cl;
+        on_connect_peer(cl->osd_num, 0, cl->client_id);
        delete op;
    };
    outbox_push(op);
@@ -771,13 +878,16 @@ void osd_messenger_t::accept_connections(int listen_fd)
    while ((peer_fd = accept(listen_fd, (sockaddr*)&addr, &peer_addr_size)) >= 0)
    {
        assert(peer_fd != 0);
-        fprintf(stderr, "[OSD %ju] new client %d: connection from %s\n", this->osd_num, peer_fd,
+        const uint64_t client_id = next_client_id++;
+        fprintf(stderr, "[OSD %ju] new client %ju (FD %d): connection from %s\n", this->osd_num, client_id, peer_fd,
            addr_to_string(addr).c_str());
        fcntl(peer_fd, F_SETFL, fcntl(peer_fd, F_GETFL, 0) | O_NONBLOCK);
        int one = 1;
        setsockopt(peer_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
        auto cl = new osd_client_t();
-        clients[peer_fd] = cl;
+        cl->client_id = client_id;
+        clients[cl->client_id] = cl;
+        clients_by_fd[peer_fd] = cl;
        cl->is_incoming = true;
        cl->peer_addr = addr;
        cl->peer_addr = addr;
@@ -785,6 +895,7 @@ void osd_messenger_t::accept_connections(int listen_fd)
        cl->peer_fd = peer_fd;
        cl->peer_state = PEER_CONNECTED;
        cl->in_buf = (uint8_t*)malloc_or_die(receive_buffer_size);
+        ssl_init(cl, true);
        // Add FD to epoll
        tfd->set_fd_handler(peer_fd, false, [this](int peer_fd, int epoll_events)
        {
@@ -799,6 +910,52 @@ void osd_messenger_t::accept_connections(int listen_fd)
    }
 }

+void osd_messenger_t::ssl_init(osd_client_t *cl, bool server_mode)
+{
+    if (!tls_cert.empty())
+    {
+        cl->write_to_ssl = BIO_new(BIO_s_mem());
+        cl->read_from_ssl = BIO_new(BIO_s_mem());
+        cl->ssl_cli = SSL_new(ssl_ctx);
+        if (!cl->ssl_cli)
+        {
+            fprintf(stderr, "OpenSSL initialization failed: %s\n", ERR_error_string(ERR_get_error(), NULL));
+            exit(1);
+        }
+        if (server_mode)
+        {
+            SSL_set_accept_state(cl->ssl_cli);
+        }
+        else
+        {
+            SSL_set_connect_state(cl->ssl_cli);
+        }
+        SSL_set_bio(cl->ssl_cli, cl->write_to_ssl, cl->read_from_ssl);
+        bool ok = ssl_do_handshake(cl);
+        assert(ok);
+    }
+    else if (!test_osd_aes_key.empty())
+    {
+        int r;
+        cl->enc_ctx = EVP_CIPHER_CTX_new();
+        assert(cl->enc_ctx);
+        r = EVP_EncryptInit_ex(cl->enc_ctx, EVP_aes_256_gcm(), NULL, NULL, NULL);
+        assert(r == 1);
+        r = EVP_CIPHER_CTX_set_padding(cl->enc_ctx, 0);
+        assert(r == 1);
+        r = EVP_CIPHER_CTX_ctrl(cl->enc_ctx, EVP_CTRL_GCM_SET_IVLEN, 12, NULL);
+        assert(r == 1);
+        cl->dec_ctx = EVP_CIPHER_CTX_new();
+        assert(cl->dec_ctx);
+        r = EVP_DecryptInit_ex(cl->dec_ctx, EVP_aes_256_gcm(), NULL, NULL, NULL);
+        assert(r == 1);
+        r = EVP_CIPHER_CTX_set_padding(cl->dec_ctx, 0);
+        assert(r == 1);
+        r = EVP_CIPHER_CTX_ctrl(cl->dec_ctx, EVP_CTRL_GCM_SET_IVLEN, 12, NULL);
+        assert(r == 1);
+    }
+}
+
 #ifdef WITH_RDMA
 msgr_rdma_context_t* osd_messenger_t::choose_rdma_context(osd_client_t *cl)
 {
@@ -12,6 +12,12 @@
 #include <deque>
 #include <vector>

+#ifdef WITH_OPENSSL
+#include <openssl/types.h>
+#endif
+
+#include "../util/xxh_x86dispatch.h"
+#include "../util/robin_hood.h"
 #include "malloc_or_die.h"
 #include "json11/json11.hpp"
 #include "msgr_op.h"
@@ -30,6 +36,10 @@
 #define PEER_RDMA 4
 #define PEER_STOPPED 5

+#define MSGR_CSUM_PAYLOAD 1
+#define MSGR_CSUM_FULL 2
+#define MSGR_CSUM_NEG 4
+
 #define VITASTOR_CONFIG_PATH "/etc/vitastor/vitastor.conf"

 #define DEFAULT_MIN_ZEROCOPY_SEND_SIZE 32*1024
@@ -50,8 +60,17 @@ struct op_aes_xts_decrypt_t;
 void destroy_aes_xts_encrypt(op_aes_xts_encrypt_t *encrypt_ctx);
 void destroy_aes_xts_decrypt(op_aes_xts_decrypt_t *decrypt_ctx);

+// Standard TLS record header. We are only interested in the record size
+struct __attribute__((__packed__)) msgr_tls_record_hdr_t
+{
+    uint8_t content_type;
+    uint16_t version;
+    uint16_t size;
+};
+
 struct osd_client_t
 {
+    uint64_t client_id = 0;
    int refs = 0;

    sockaddr_storage peer_addr = {};
@@ -71,7 +90,32 @@ struct osd_client_t
    msgr_rdma_connection_t *rdma_conn = NULL;
 #endif

+#ifdef WITH_OPENSSL
+    SSL *ssl_cli = NULL;
+    BIO *write_to_ssl = NULL;
+    // FIXME: use custom bio to avoid 1 more memory copy?
+    BIO *read_from_ssl = NULL;
+    uint8_t *ssl_out_buf = NULL;
+    size_t ssl_out_buf_size = 0, ssl_out_buf_cap = 0;
+    bool ssl_handshake_done = false;
+    msgr_tls_record_hdr_t ssl_read_record;
+    size_t ssl_read_header_size = 0;
+    bool ssl_more_to_buffer = false;
+
+    EVP_CIPHER_CTX *enc_ctx = NULL;
+    uint8_t enc_tag[16];
+    size_t enc_tag_size = 0;
+    bool enc_batch = false;
+    EVP_CIPHER_CTX *dec_ctx = NULL;
+    uint8_t dec_tag[16];
+    size_t dec_tag_size = 0;
+    uint32_t dec_batch_size = 0;
+    size_t dec_batch_size_size = 0;
+    std::vector<osd_op_t*> unverified_ops;
+#endif
+
    // Read state
+    bool io_error = false;
    int read_ready = 0;
    osd_op_t *read_op = NULL;
    size_t read_op_size = 0;
@@ -79,19 +123,21 @@ struct osd_client_t
    iovec read_iov = { 0 };
    msghdr read_msg = { 0 };
    std::vector<iovec> recv_list;
-    size_t recv_list_size = 0;
+    std::vector<int> recv_flags;
    uint64_t read_op_id = 1;
    bool check_sequencing = false;
    bool enable_pg_locks = false;
    op_aes_xts_decrypt_t *decrypt_ctx = NULL;
    size_t read_op_inline_decrypt_pos = 0;
    size_t read_op_inline_decrypt_in = 0;
+    int proto_csum_status = 0;
+    XXH3_state_t* read_csum_state = NULL;

    // Incoming operations
    std::vector<osd_op_t*> received_ops;

    // Outbound operations
-    std::map<uint64_t, osd_op_t*> sent_ops;
+    robin_hood::unordered_flat_map<uint64_t, osd_op_t*> sent_ops;
    uint64_t send_op_id = 0;

    // PGs dirtied by this client's primary-writes
@@ -108,6 +154,7 @@ struct osd_client_t
    std::deque<osd_op_t*> send_free_ops;
    std::vector<osd_op_t*> zc_free_list;
    op_aes_xts_encrypt_t *encrypt_ctx = NULL;
+    XXH3_state_t* write_csum_state = NULL;

    ~osd_client_t();
    void cancel_ops();
@@ -185,9 +232,21 @@ struct osd_messenger_t;
 struct rdmacm_connecting_t;
 #endif

+class msgr_op_reader_t;
+class msgr_op_writer_t;
+
 struct __attribute__((visibility("default"))) osd_messenger_t
 {
 protected:
+    friend class copy_op_reader_t;
+    friend class ssl_op_reader_t;
+    friend class gcm_op_reader_t;
+    friend class get_op_reader_t;
+    friend class copy_op_writer_t;
+    friend class ssl_op_writer_t;
+    friend class gcm_op_writer_t;
+    friend class get_op_writer_t;
+
    int keepalive_timer_id = -1;

    uint32_t receive_buffer_size = 0;
@@ -201,6 +260,12 @@ protected:
    int iothread_count = 0;
    int max_aes_xts_pool_size = 256;

+    std::string tls_cert;
+    std::string tls_key;
+    std::string osd_tls_ca;
+    std::string client_tls_ca;
+    std::string test_osd_aes_key; // FIXME Insecure, only for PoC tests
+
 #ifdef WITH_RDMA
    bool use_rdma = true;
    bool use_rdmacm = false;
@@ -213,13 +278,21 @@ protected:
    uint64_t rdma_max_sge = 0, rdma_max_send = 0, rdma_max_recv = 0;
    uint64_t rdma_max_msg = 0;
    rdma_event_channel *rdmacm_evch = NULL;
-    std::map<rdma_cm_id*, osd_client_t*> rdmacm_connections;
-    std::map<rdma_cm_id*, rdmacm_connecting_t*> rdmacm_connecting;
+    robin_hood::unordered_flat_map<rdma_cm_id*, osd_client_t*> rdmacm_connections;
+    robin_hood::unordered_flat_map<rdma_cm_id*, rdmacm_connecting_t*> rdmacm_connecting;
+#endif
+
+#ifdef WITH_OPENSSL
+    SSL_CTX *ssl_ctx = NULL;
+    std::string tls_cn;
+
+    void ssl_init(osd_client_t *cl, bool server_mode);
+    bool ssl_do_handshake(osd_client_t *cl);
 #endif

    std::vector<msgr_iothread_t*> iothreads;
-    std::vector<int> read_ready_clients;
-    std::vector<int> write_ready_clients;
+    std::vector<uint64_t> read_ready_clients;
+    std::vector<uint64_t> write_ready_clients;
    // We don't use ringloop->set_immediate here because we may have no ringloop in client :)
    std::deque<osd_op_t*> set_immediate_ops;

@@ -231,23 +304,27 @@ public:
    ring_loop_t *ringloop = NULL;
    bool has_sendmsg_zc = false;
    // osd_num_t is only for logging and asserts
+    uint64_t next_client_id = 1;
    osd_num_t osd_num;
-    std::map<int, osd_client_t*> clients;
-    std::map<osd_num_t, osd_wanted_peer_t> wanted_peers;
-    std::map<uint64_t, int> osd_peer_fds;
+    robin_hood::unordered_flat_map<uint64_t, osd_client_t*> clients;
+    robin_hood::unordered_flat_map<uint64_t, osd_client_t*> osd_peers;
+    robin_hood::unordered_flat_map<int, osd_client_t*> clients_by_fd;
+    robin_hood::unordered_flat_map<osd_num_t, osd_wanted_peer_t> wanted_peers;
    std::vector<std::string> osd_networks;
    std::vector<addr_mask_t> osd_network_masks;
    std::vector<std::string> osd_cluster_networks;
    std::vector<addr_mask_t> osd_cluster_network_masks;
    std::vector<std::string> all_osd_networks;
    std::vector<addr_mask_t> all_osd_network_masks;
+    int use_proto_checksums = 0;
    // op statistics
    osd_op_stats_t stats, recovery_stats;

    void init();
    void parse_config(const json11::Json & config);
    void connect_peer(uint64_t osd_num, json11::Json peer_state);
-    void stop_client(int peer_fd, bool force = false, bool force_delete = false);
+    void stop_client(uint64_t client_id, bool force_delete = false);
+    void destroy_client(osd_client_t *cl);
    void outbox_push(osd_op_t *cur_op);
    std::function<void(osd_op_t*)> exec_op;
    std::function<void(osd_num_t)> repeer_pgs;
@@ -266,7 +343,7 @@ public:

 #ifdef WITH_RDMA
    bool is_rdma_enabled();
-    bool connect_rdma(int peer_fd, std::string rdma_address, uint64_t client_max_msg);
+    bool connect_rdma(uint64_t client_id, std::string rdma_address, uint64_t client_max_msg);
 #endif
 #ifdef WITH_RDMACM
    bool is_use_rdmacm();
@@ -282,41 +359,40 @@ protected:
    void try_connect_peer_tcp(osd_num_t peer_osd, const char *peer_host, int peer_port);
    void handle_peer_epoll(int peer_fd, int epoll_events);
    void handle_connect_epoll(int peer_fd);
-    void on_connect_peer(osd_num_t peer_osd, int peer_fd);
+    void on_connect_peer(osd_num_t peer_osd, int errcode, uint64_t client_id);
    void check_peer_config(osd_client_t *cl);
    void cancel_osd_ops(osd_client_t *cl);
    void cancel_op(osd_op_t *op);

    bool try_send(osd_client_t *cl);
    void handle_send(int result, bool prev, bool more, osd_client_t *cl);
-    size_t op_copy_to(osd_client_t *cl, uint8_t *dst, size_t dst_len);
-    void op_get_write_buffers(osd_client_t *cl, std::vector<iovec> & lst);
+    bool op_write_to(osd_client_t *cl, msgr_op_writer_t & wr);
+    void next_write_op(osd_client_t *cl);
+    bool op_write_buf(osd_client_t *cl, uint8_t *src, size_t src_len, uint8_t *dst, size_t dst_len, bool skip_csum, size_t & from, size_t & done);
+    bool op_copy_data_to(osd_client_t *cl, uint8_t *dst, size_t dst_len, size_t & from, size_t & done);
+    size_t copy_ops_to(osd_client_t *cl, uint8_t *dst, size_t dst_len);
+    template<typename T> size_t copy_ops_to_with(osd_client_t *cl, uint8_t *dst, size_t dst_len);

    void handle_read(int result, osd_client_t *cl);
    bool handle_read_buffer(osd_client_t *cl, uint8_t *curbuf, size_t bufsize);
+    template<typename T> bool handle_buffer_with(osd_client_t *cl, uint8_t *curbuf, size_t bufsize);
    bool handle_hdr(osd_client_t *cl);
    bool allocate_op_buffers(osd_client_t *cl);
    bool allocate_reply_buffers(osd_client_t *cl, osd_op_t *op);
-    size_t op_copy_from(osd_client_t *cl, uint8_t *src, size_t src_len, size_t & done);
-    size_t op_get_read_buffers(osd_client_t *cl, std::vector<iovec> & lst);
-    void handle_finished_op(osd_client_t *cl);
+    bool op_read_from(osd_client_t *cl, msgr_op_reader_t & rdr);
+    bool handle_finished_op(osd_client_t *cl);
+    void execute_verified_op(osd_client_t *cl, osd_op_t *op);
+    void handle_immediate_ops();

-    bool op_encrypted_copy_data_to(osd_client_t* cl, uint8_t *buf, size_t len, size_t from, size_t & done);
-    bool op_decrypted_copy_data_from(osd_client_t* cl, uint8_t *buf, size_t len, size_t from, size_t & done);
+    void op_encrypted_copy_buf(osd_client_t *cl, uint8_t *enc_buf, size_t enc_len, uint8_t *plain, size_t plain_len, size_t & done_plain, size_t & done_enc);
+    void op_encrypt_free(osd_client_t* cl);
+    void op_decrypted_copy_buf(osd_client_t *cl, uint8_t *enc_buf, size_t enc_len, uint8_t *plain, size_t plain_len, size_t & done_plain, size_t & done_enc);
    void op_decrypt_start(osd_client_t* cl);
    void op_decrypt_inline(osd_client_t* cl);
    void op_decrypt_free(osd_client_t* cl);

-    bool handle_finished_read(osd_client_t *cl);
-    void handle_op_hdr(osd_client_t *cl);
-    bool handle_reply_hdr(osd_client_t *cl);
-    void handle_reply_ready(osd_op_t *op);
-    void handle_immediate_ops();
-    void clear_immediate_ops(int peer_fd);
-
 #ifdef WITH_RDMA
    void try_send_rdma(osd_client_t *cl);
-    int try_send_rdma_copy(osd_client_t *cl, uint8_t *dst, int dst_len);
    bool init_recv_rdma(osd_client_t *cl);
    void handle_rdma_events(msgr_rdma_context_t *rdma_context);
    msgr_rdma_context_t* choose_rdma_context(osd_client_t *cl);
@@ -81,7 +81,6 @@ void op_aes_xts_encrypt_t::encrypt_block(uint8_t *in, uint8_t *out)
 #endif
 }

-// FIXME: Copy-paste
 void op_aes_xts_encrypt_t::update(uint8_t *in, size_t max_in, uint8_t *out, size_t max_out, size_t & done_in, size_t & done_out)
 {
    // Fucking AES-XTS implementations (all of them) don't have streaming support,
@@ -187,13 +186,14 @@ op_aes_xts_decrypt_t::~op_aes_xts_decrypt_t()
        free(tmp);
 }

-void op_aes_xts_decrypt_t::start(uint8_t **key_chain, size_t chain_size, uint8_t *key_indexes, uint64_t start_offset, size_t block_size)
+void op_aes_xts_decrypt_t::start(uint8_t **key_chain, size_t chain_size, void *key_indexes, uint64_t start_offset, size_t block_size)
 {
    assert(!decrypted);
    this->start_offset = start_offset;
    this->key_chain = chain_size > 1 ? key_chain : 0;
    this->chain_size = chain_size > 1 ? chain_size : 0;
    this->key_indexes = chain_size > 1 ? key_indexes : NULL;
+    this->key_index_bytes = osd_op_rw_t::chain_info_bytes(chain_size);
    assert(chain_size <= 1 || key_indexes != NULL);
    this->block_size = block_size;
    this->offset = 0;
@@ -218,8 +218,15 @@ void op_aes_xts_decrypt_t::decrypt_block(uint8_t *in, uint8_t *out)
    uint8_t *key = NULL;
    if (chain_size)
    {
-        assert(key_indexes[offset/block_size] < chain_size);
-        key = key_chain[key_indexes[offset/block_size]];
+        uint32_t key_index = key_index_bytes == 1
+            ? ((uint8_t*)key_indexes)[offset/block_size]
+            : (key_index_bytes == 2
+                ? ((uint16_t*)key_indexes)[offset/block_size]
+                : (key_index_bytes == 4
+                    ? ((uint32_t*)key_indexes)[offset/block_size]
+                    : UINT32_MAX));
+        assert(key_index < chain_size);
+        key = key_chain[key_index];
        if (!key)
        {
            if (in != out)
@@ -245,6 +252,8 @@ void op_aes_xts_decrypt_t::decrypt_block(uint8_t *in, uint8_t *out)
 #endif
 }

+// out may be NULL, in this case all input is still decrypted to calculate checksums,
+// but part of it is skipped and not copied to out
 void op_aes_xts_decrypt_t::update(uint8_t *in, size_t max_in, uint8_t *out, size_t max_out, size_t & done_in, size_t & done_out)
 {
    // Fucking AES-XTS implementations (all of them) don't have streaming support,
@@ -258,7 +267,8 @@ void op_aes_xts_decrypt_t::update(uint8_t *in, size_t max_in, uint8_t *out, size
        assert(tmp);
        if (max_out > block_size - tmp_pos)
            max_out = block_size - tmp_pos;
-        memcpy(out, tmp + tmp_pos, max_out);
+        if (out)
+            memcpy(out, tmp + tmp_pos, max_out);
        done_out += max_out;
        tmp_pos += max_out;
        if (tmp_pos >= block_size)
@@ -276,7 +286,7 @@ void op_aes_xts_decrypt_t::update(uint8_t *in, size_t max_in, uint8_t *out, size
        done_in += max_in;
        offset += max_in;
    }
-    else if (max_out < block_size)
+    else if (max_out < block_size || !out)
    {
        // Accumulate and decrypt input in <tmp>, then copy part of it to <out>
        if (!tmp)
@@ -288,7 +298,8 @@ void op_aes_xts_decrypt_t::update(uint8_t *in, size_t max_in, uint8_t *out, size
        memcpy(tmp + offset%block_size, in, max_in);
        decrypt_block(tmp, tmp);
        decrypted = true;
-        memcpy(out, tmp, max_out);
+        if (out)
+            memcpy(out, tmp, max_out);
        tmp_pos = max_out;
        done_in += max_in;
        offset += max_in;
@@ -297,7 +308,8 @@ void op_aes_xts_decrypt_t::update(uint8_t *in, size_t max_in, uint8_t *out, size
    else if (!(offset%block_size))
    {
        // Full block - simplest case
-        decrypt_block(in, out);
+        if (out)
+            decrypt_block(in, out);
        done_in += block_size;
        offset += block_size;
        done_out += block_size;
@@ -308,6 +320,7 @@ void op_aes_xts_decrypt_t::update(uint8_t *in, size_t max_in, uint8_t *out, size
        assert(tmp);
        max_in = block_size - offset%block_size;
        memcpy(tmp + offset%block_size, in, max_in);
+        assert(out);
        decrypt_block(tmp, out);
        done_in += max_in;
        offset += max_in;
@@ -320,81 +333,49 @@ void destroy_aes_xts_decrypt(op_aes_xts_decrypt_t *decrypt_ctx)
    delete decrypt_ctx;
 }

-bool osd_messenger_t::op_encrypted_copy_data_to(osd_client_t* cl, uint8_t *enc_buf, size_t enc_len, size_t from, size_t & done)
+void osd_messenger_t::op_encrypted_copy_buf(osd_client_t *cl, uint8_t *enc_buf, size_t enc_len, uint8_t *plain, size_t plain_len, size_t & done_plain, size_t & done_enc)
 {
-    auto op = cl->write_op;
-    auto & op_pos = cl->write_op_pos;
-    assert(op->req.hdr.opcode == OSD_OP_WRITE);
-    if (!from)
+    if (!cl->encrypt_ctx)
    {
-        if (!cl->encrypt_ctx)
+        if (encrypt_ctx_pool.size())
        {
-            if (encrypt_ctx_pool.size())
-            {
-                cl->encrypt_ctx = encrypt_ctx_pool.back();
-                encrypt_ctx_pool.pop_back();
-            }
-            else
-                cl->encrypt_ctx = new op_aes_xts_encrypt_t();
+            cl->encrypt_ctx = encrypt_ctx_pool.back();
+            encrypt_ctx_pool.pop_back();
        }
-        assert(op->enc->key_chain[0]);
-        cl->encrypt_ctx->start(op->enc->key_chain[0], op->req.rw.offset, op->enc->bitmap_granularity);
-    }
-    for (int i = 0; i < op->iov.count; i++)
-    {
-        uint8_t *plain = (uint8_t*)op->iov.buf[i].iov_base;
-        size_t plain_len = op->iov.buf[i].iov_len;
-        while (from < plain_len || cl->encrypt_ctx->has_buffered())
-        {
-            if (done >= enc_len)
-                return false;
-            size_t done_in = 0;
-            size_t done_out = 0;
-            cl->encrypt_ctx->update(plain+from, plain_len-from, enc_buf+done, enc_len-done, done_in, done_out);
-            done += done_out;
-            op_pos += done_in;
-            from += done_in;
-        }
-        from -= plain_len;
-    }
-    if (cl->encrypt_ctx)
-    {
-        if (encrypt_ctx_pool.size() > max_aes_xts_pool_size)
-            delete cl->encrypt_ctx;
        else
-            encrypt_ctx_pool.push_back(cl->encrypt_ctx);
-        cl->encrypt_ctx = NULL;
+            cl->encrypt_ctx = new op_aes_xts_encrypt_t();
+        assert(cl->write_op->enc->key_chain[0]);
+        cl->encrypt_ctx->start(cl->write_op->enc->key_chain[0], cl->write_op->req.rw.offset, cl->write_op->enc->bitmap_granularity);
+    }
+    while (done_enc < enc_len && (done_plain < plain_len || cl->encrypt_ctx->has_buffered()))
+    {
+        size_t done_in = 0;
+        size_t done_out = 0;
+        cl->encrypt_ctx->update(plain+done_plain, plain_len-done_plain, enc_buf+done_enc, enc_len-done_enc, done_in, done_out);
+        if (cl->write_csum_state && done_out > 0)
+            XXH3_64bits_update(cl->write_csum_state, enc_buf+done_enc, done_out);
+        done_enc += done_out;
+        cl->write_op_pos += done_in;
+        done_plain += done_in;
    }
-    return true;
 }

-bool osd_messenger_t::op_decrypted_copy_data_from(osd_client_t* cl, uint8_t *enc_buf, size_t enc_len, size_t from, size_t & done)
+void osd_messenger_t::op_decrypted_copy_buf(osd_client_t *cl, uint8_t *enc_buf, size_t enc_len, uint8_t *plain, size_t plain_len, size_t & done_plain, size_t & done_enc)
 {
    op_decrypt_start(cl);
-    auto op = cl->read_op;
-    assert(op->req.hdr.opcode == OSD_OP_READ);
-    for (int i = 0; i < op->iov.count; i++)
+    while (done_plain < plain_len && done_enc < enc_len)
    {
-        uint8_t *plain = (uint8_t*)op->iov.buf[i].iov_base;
-        size_t plain_len = op->iov.buf[i].iov_len;
-        while (from < plain_len)
-        {
-            if (done >= enc_len)
-                return false;
-            size_t done_in = 0;
-            size_t done_out = 0;
-            cl->decrypt_ctx->update(enc_buf+done, enc_len-done, plain+from, plain_len-from, done_in, done_out);
-            done += done_in;
-            cl->read_op_pos += done_out;
-            cl->read_op_inline_decrypt_in += done_in;
-            from += done_out;
-            if (!done_out)
-                return false;
-        }
-        from -= plain_len;
+        size_t done_in = 0;
+        size_t done_out = 0;
+        // plain == NULL means skip output
+        cl->decrypt_ctx->update(enc_buf+done_enc, enc_len-done_enc, plain ? plain+done_plain : NULL, plain_len-done_plain, done_in, done_out);
+        if (cl->read_csum_state && done_in > 0)
+            XXH3_64bits_update(cl->read_csum_state, enc_buf+done_enc, done_in);
+        done_enc += done_in;
+        cl->read_op_pos += done_out;
+        cl->read_op_inline_decrypt_in += done_in;
+        done_plain += done_out;
    }
-    op_decrypt_free(cl);
-    return true;
 }

 void osd_messenger_t::op_decrypt_start(osd_client_t* cl)
@@ -409,6 +390,7 @@ void osd_messenger_t::op_decrypt_start(osd_client_t* cl)
        else
            cl->decrypt_ctx = new op_aes_xts_decrypt_t();
        auto & enc = cl->read_op->enc;
+        assert(cl->read_op->req.hdr.opcode == OSD_OP_READ);
        cl->decrypt_ctx->start(enc->key_chain, enc->chain_size,
            (cl->read_op->req.rw.flags & OSD_OP_RETURN_CHAIN) ? (uint8_t*)cl->read_op->bitmap + enc->read_chain_bitmap_pos : 0,
            cl->read_op->req.rw.offset, enc->bitmap_granularity);
@@ -458,7 +440,6 @@ void osd_messenger_t::op_decrypt_inline(osd_client_t* cl)
            from_out += done_out;
    }
    assert(j >= op->iov.count);
-    op_decrypt_free(cl);
 }

 void osd_messenger_t::op_decrypt_free(osd_client_t* cl)
@@ -472,3 +453,15 @@ void osd_messenger_t::op_decrypt_free(osd_client_t* cl)
        cl->decrypt_ctx = NULL;
    }
 }
+
+void osd_messenger_t::op_encrypt_free(osd_client_t* cl)
+{
+    if (cl->encrypt_ctx)
+    {
+        if (encrypt_ctx_pool.size() > max_aes_xts_pool_size)
+            delete cl->encrypt_ctx;
+        else
+            encrypt_ctx_pool.push_back(cl->encrypt_ctx);
+        cl->encrypt_ctx = NULL;
+    }
+}
@@ -3,6 +3,7 @@

 #include <stdint.h>

+#include "../util/xxh_x86dispatch.h"
 // WITH_OPENSSL is left to possibly support other crypto libraries
 #ifdef WITH_OPENSSL
 #include <openssl/conf.h>
@@ -45,7 +46,8 @@ class op_aes_xts_decrypt_t
    uint64_t start_offset = 0;
    uint8_t **key_chain = NULL;
    size_t chain_size = 0;
-    uint8_t *key_indexes = NULL;
+    void *key_indexes = NULL;
+    int key_index_bytes = 0;
    size_t offset = 0;
    size_t block_size = 0;
    uint8_t *tmp = NULL;
@@ -60,7 +62,7 @@ public:
    ~op_aes_xts_decrypt_t();

    inline bool has_buffered() { return decrypted; };
-    void start(uint8_t **key_chain, size_t chain_size, uint8_t *key_indexes, uint64_t start_offset, size_t block_size);
+    void start(uint8_t **key_chain, size_t chain_size, void *key_indexes, uint64_t start_offset, size_t block_size);
    void update(uint8_t *in, size_t max_in, uint8_t *out, size_t max_out, size_t & done_in, size_t & done_out);
 };

@@ -8,7 +8,6 @@
 osd_op_t::~osd_op_t()
 {
    assert(!bs_op);
-    assert(!op_data);
    if (bitmap_buf)
    {
        free(bitmap_buf);
@@ -27,6 +26,10 @@ osd_op_t::~osd_op_t()
    {
        free(enc_buf);
    }
+    if (op_data)
+    {
+        free(op_data);
+    }
 }

 bool osd_op_t::is_recovery_related()
@@ -173,7 +173,7 @@ struct __attribute__((visibility("default"))) osd_op_t
 {
    timespec tv_begin = { 0 }, tv_end = { 0 };
    uint64_t op_type = OSD_OP_IN;
-    int peer_fd;
+    uint64_t client_id = 0;
    osd_any_op_t req;
    osd_any_reply_t reply;
    blockstore_op_t *bs_op = NULL;
@@ -182,10 +182,11 @@ struct __attribute__((visibility("default"))) osd_op_t
    void *bitmap = NULL;
    unsigned bitmap_len = 0;
    unsigned bmp_data = 0;
-    void *bitmap_buf = NULL;
+    uint8_t *bitmap_buf = NULL;
    void *rmw_buf = NULL;
    std::shared_ptr<osd_op_enc_t> enc;
    uint8_t *enc_buf = NULL;
+    uint64_t csum = 0; // network layer checksum
    osd_primary_op_data_t* op_data = NULL;
    std::function<void(osd_op_t*)> callback;

@@ -195,4 +196,5 @@ struct __attribute__((visibility("default"))) osd_op_t
    void cancel();

    bool is_recovery_related();
+    uint64_t calc_data_checksum();
 };
@@ -187,6 +187,8 @@ std::vector<msgr_rdma_context_t*> msgr_rdma_context_t::create_all(const std::vec
    ibv_device **raw_dev_list = NULL;
    ibv_device **dev_list = NULL;
    ibv_device *single_list[2] = {};
+    int up_ports = 0;
+    int single_port_num = 0;

    raw_dev_list = dev_list = ibv_get_device_list(NULL);
    if (!dev_list || !*dev_list)
@@ -221,6 +223,7 @@ std::vector<msgr_rdma_context_t*> msgr_rdma_context_t::create_all(const std::vec
        dev_list = single_list;
    }

+retry:
    for (int i = 0; dev_list[i]; ++i)
    {
        auto dev = dev_list[i];
@@ -258,6 +261,9 @@ std::vector<msgr_rdma_context_t*> msgr_rdma_context_t::create_all(const std::vec
                fprintf(stderr, "RDMA device %s port %d GID %d does not exist\n", ibv_get_device_name(dev), port_num, sel_gid_index);
                continue;
            }
+            up_ports++;
+            single_port_num = port_num;
+            single_list[0] = dev;
            uint32_t port_mtu = sel_mtu ? sel_mtu : ibv_mtu_to_bytes(portinfo.active_mtu);
 #ifdef IBV_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT
            if (sel_gid_index < 0)
@@ -298,6 +304,14 @@ cleanup_dev:
        ibv_close_device(context);
    }

+    if (!ret.size() && up_ports == 1 && dev_list != single_list)
+    {
+        // Auto-select the only available device/port if there is only one
+        dev_list = single_list;
+        sel_port_num = single_port_num;
+        goto retry;
+    }
+
 cleanup:
    if (raw_dev_list)
        ibv_free_device_list(raw_dev_list);
@@ -493,7 +507,7 @@ int msgr_rdma_connection_t::connect(msgr_rdma_address_t *dest)
    return 0;
 }

-bool osd_messenger_t::connect_rdma(int peer_fd, std::string rdma_address, uint64_t client_max_msg)
+bool osd_messenger_t::connect_rdma(uint64_t client_id, std::string rdma_address, uint64_t client_max_msg)
 {
    // Try to connect to the peer using RDMA
    msgr_rdma_address_t addr;
@@ -503,12 +517,12 @@ bool osd_messenger_t::connect_rdma(int peer_fd, std::string rdma_address, uint64
        {
            client_max_msg = rdma_max_msg;
        }
-        auto cl = clients.at(peer_fd);
+        auto cl = clients.at(client_id);
        msgr_rdma_context_t *selected_ctx = choose_rdma_context(cl);
        if (!selected_ctx)
        {
            if (log_level > 0)
-                fprintf(stderr, "No RDMA context for peer %d, using only TCP\n", cl->peer_fd);
+                fprintf(stderr, "No RDMA context for peer %ju, using only TCP\n", client_id);
            return false;
        }
        msgr_rdma_connection_t *rdma_conn = msgr_rdma_connection_t::create(selected_ctx, rdma_max_send, rdma_max_recv, rdma_max_sge, client_max_msg);
@@ -519,14 +533,13 @@ bool osd_messenger_t::connect_rdma(int peer_fd, std::string rdma_address, uint64
            {
                delete rdma_conn;
                fprintf(
-                    stderr, "Failed to connect RDMA queue pair to %s (client %d)\n",
-                    addr.to_string().c_str(), peer_fd
+                    stderr, "Failed to connect RDMA queue pair to %s (client %ju)\n",
+                    addr.to_string().c_str(), client_id
                );
            }
            else
            {
                // Remember connection, but switch to RDMA only after sending the configuration response
-                auto cl = clients.at(peer_fd);
                cl->rdma_conn = rdma_conn;
                cl->peer_state = PEER_RDMA_CONNECTING;
                return true;
@@ -540,7 +553,7 @@ static void try_send_rdma_wr(osd_client_t *cl, ibv_sge *sge, int op_sge)
 {
    ibv_send_wr *bad_wr = NULL;
    ibv_send_wr wr = {
-        .wr_id = (uint64_t)(cl->peer_fd*2+1),
+        .wr_id = cl->client_id,
        .sg_list = sge,
        .num_sge = op_sge,
        .opcode = IBV_WR_SEND,
@@ -555,33 +568,6 @@ static void try_send_rdma_wr(osd_client_t *cl, ibv_sge *sge, int op_sge)
    cl->rdma_conn->cur_send++;
 }

-int osd_messenger_t::try_send_rdma_copy(osd_client_t *cl, uint8_t *dst, int dst_len)
-{
-    int total_dst_len = dst_len;
-    while (dst_len > 0 && (cl->write_op || cl->write_ops.size()))
-    {
-        if (!cl->write_op)
-        {
-            cl->write_op = cl->write_ops.front();
-            cl->write_ops.pop_front();
-        }
-        osd_op_t *op = cl->write_op;
-        size_t copied = op_copy_to(cl, dst, dst_len);
-        if (!copied)
-        {
-            break;
-        }
-        dst += copied;
-        dst_len -= copied;
-        if (!cl->write_op && op->op_type == OSD_OP_IN)
-        {
-            // this is a reply, free the op after sending it
-            cl->send_free_ops.push_back(op);
-        }
-    }
-    return total_dst_len-dst_len;
-}
-
 void osd_messenger_t::try_send_rdma(osd_client_t *cl)
 {
    auto rc = cl->rdma_conn;
@@ -604,10 +590,44 @@ void osd_messenger_t::try_send_rdma(osd_client_t *cl)
    while (!rc->send_out_full && copied > 0 && rc->cur_send < rc->max_send)
    {
        dst = (uint8_t*)rc->send_out.buf + rc->send_out_pos;
-        dst_len = (rc->send_out_pos < rc->send_out_size ? rc->send_out_size-rc->send_out_pos : rc->send_done_pos-rc->send_out_pos);
+        if (rc->send_out_pos >= rc->send_done_pos)
+        {
+            dst_len = rc->send_out_size-rc->send_out_pos;
+            if (dst_len < 4096)
+            {
+                // free end of the buffer is too small, skip
+                rc->send_out_pos = 0;
+                if (rc->send_out_pos >= rc->send_done_pos)
+                    rc->send_out_full = true;
+                if (!rc->send_sizes.size())
+                {
+                    rc->send_done_pos += dst_len;
+                    rc->send_out_full = false;
+                    if (rc->send_done_pos == rc->send_out_size)
+                        rc->send_done_pos = 0;
+                }
+                else
+                    rc->send_sizes.back() += dst_len;
+                continue;
+            }
+        }
+        else
+        {
+            dst_len = rc->send_done_pos-rc->send_out_pos;
+            if (dst_len < 4096)
+            {
+                // too small buffer, stop
+                break;
+            }
+        }
        if (dst_len > rc->max_msg)
            dst_len = rc->max_msg;
-        copied = try_send_rdma_copy(cl, dst, dst_len);
+        copied = copy_ops_to(cl, dst, dst_len);
+        if (cl->io_error)
+        {
+            stop_client(cl->client_id);
+            return;
+        }
        if (copied > 0)
        {
            rc->send_out_pos += copied;
@@ -637,7 +657,7 @@ static void try_recv_rdma_wr(osd_client_t *cl, void *buf)
    };
    ibv_recv_wr *bad_wr = NULL;
    ibv_recv_wr wr = {
-        .wr_id = (uint64_t)(cl->peer_fd*2),
+        .wr_id = cl->client_id,
        .sg_list = &sge,
        .num_sge = 1,
    };
@@ -694,8 +714,8 @@ void osd_messenger_t::handle_rdma_events(msgr_rdma_context_t *rdma_context)
        event_count = ibv_poll_cq(rdma_context->cq, RDMA_EVENTS_AT_ONCE, wc);
        for (int i = 0; i < event_count; i++)
        {
-            int client_id = wc[i].wr_id >> 1;
-            bool is_send = wc[i].wr_id & 1;
+            uint64_t client_id = wc[i].wr_id;
+            bool is_send = wc[i].opcode == IBV_WC_SEND;
            auto cl_it = clients.find(client_id);
            if (cl_it == clients.end())
            {
@@ -709,14 +729,13 @@ void osd_messenger_t::handle_rdma_events(msgr_rdma_context_t *rdma_context)
            auto rc = cl->rdma_conn;
            if (wc[i].status != IBV_WC_SUCCESS)
            {
-                fprintf(stderr, "RDMA work request failed for client %d", client_id);
+                fprintf(stderr, "RDMA work request failed for client %ju", client_id);
                if (cl->osd_num)
                {
                    fprintf(stderr, " (OSD %ju)", cl->osd_num);
                }
                fprintf(stderr, " with status: %s, stopping client\n", ibv_wc_status_str(wc[i].status));
                stop_client(client_id);
-                clear_immediate_ops(client_id);
                continue;
            }
            if (!is_send)
@@ -724,8 +743,6 @@ void osd_messenger_t::handle_rdma_events(msgr_rdma_context_t *rdma_context)
                rc->cur_recv--;
                if (!handle_read_buffer(cl, rc->recv_buffers[rc->next_recv_buf], wc[i].byte_len))
                {
-                    // handle_read_buffer may stop the client
-                    clear_immediate_ops(client_id);
                    continue;
                }
                try_recv_rdma_wr(cl, rc->recv_buffers[rc->next_recv_buf]);
@@ -748,6 +765,11 @@ void osd_messenger_t::handle_rdma_events(msgr_rdma_context_t *rdma_context)
                    cl->send_free_ops.pop_front();
                }
                cl->send_free_ops.pop_front();
+                if ((cl->proto_csum_status & MSGR_CSUM_NEG) && !cl->write_op && !cl->write_ops.size())
+                {
+                    // Checksums negotiated, enable
+                    cl->proto_csum_status = cl->proto_csum_status & (~MSGR_CSUM_NEG);
+                }
                try_send_rdma(cl);
            }
        }
@@ -79,7 +79,7 @@ struct msgr_rdma_connection_t
    msgr_rdma_buf_t recv_buf;
    std::deque<uint64_t> send_sizes;
    msgr_rdma_buf_t send_out;
-    int send_out_pos = 0, send_done_pos = 0, send_out_size = 0;
+    size_t send_out_pos = 0, send_done_pos = 0, send_out_size = 0;
    bool send_out_full = false;

    ~msgr_rdma_connection_t();
@@ -11,7 +11,7 @@
 struct rdmacm_connecting_t
 {
    rdma_cm_id *cmid = NULL;
-    int peer_fd = -1;
+    uint64_t client_id = 0;
    osd_num_t peer_osd = 0;
    std::string addr;
    sockaddr_storage parsed_addr = {};
@@ -117,9 +117,9 @@ void osd_messenger_t::handle_rdmacm_events()
            auto cli_it = rdmacm_connections.find(ev->id);
            if (cli_it != rdmacm_connections.end())
            {
-                fprintf(stderr, "Received %s event for peer %d, closing connection\n",
-                    event_type_name, cli_it->second->peer_fd);
-                stop_client(cli_it->second->peer_fd);
+                fprintf(stderr, "Received %s event for client %ju, closing connection\n",
+                    event_type_name, cli_it->second->client_id);
+                stop_client(cli_it->second->client_id);
            }
            else if (rdmacm_connecting.find(ev->id) != rdmacm_connecting.end())
            {
@@ -265,14 +265,6 @@ msgr_rdma_context_t* osd_messenger_t::rdmacm_create_qp(rdma_cm_id *cmid)

 void osd_messenger_t::rdmacm_accept(rdma_cm_event *ev)
 {
-    // Make a fake FD (FIXME: do not use FDs for identifying clients!)
-    int fake_fd = socket(AF_INET, SOCK_STREAM, 0);
-    if (fake_fd < 0)
-    {
-        fprintf(stderr, "Failed to allocate a fake socket for RDMA-CM client: %s (code %d)\n", strerror(errno), errno);
-        rdma_destroy_id(ev->id);
-        return;
-    }
    auto rdma_context = rdmacm_create_qp(ev->id);
    if (!rdma_context)
    {
@@ -297,12 +289,12 @@ void osd_messenger_t::rdmacm_accept(rdma_cm_event *ev)
    // Wait for RDMA_CM_ESTABLISHED, and enable the connection only after it
    auto conn = new rdmacm_connecting_t;
    conn->cmid = ev->id;
-    conn->peer_fd = fake_fd;
+    conn->client_id = next_client_id++;
    conn->parsed_addr = *(sockaddr_storage*)rdma_get_peer_addr(ev->id);
    conn->rdma_context = rdma_context;
    rdmacm_set_conn_timeout(conn);
    rdmacm_connecting[ev->id] = conn;
-    fprintf(stderr, "[OSD %ju] new client %d: connection from %s via RDMA-CM\n", this->osd_num, conn->peer_fd,
+    fprintf(stderr, "[OSD %ju] new client %ju: connection from %s via RDMA-CM\n", this->osd_num, conn->client_id,
        addr_to_string(conn->parsed_addr).c_str());
 }

@@ -332,8 +324,6 @@ void osd_messenger_t::rdmacm_on_connect_peer_error(rdma_cm_id *cmid, int res)
    auto peer_osd = conn->peer_osd;
    if (conn->timeout_id >= 0)
        tfd->clear_timer(conn->timeout_id);
-    if (conn->peer_fd >= 0)
-        close(conn->peer_fd);
    if (conn->rdma_context)
        conn->rdma_context->reserve_cqe(-rdma_max_send-rdma_max_recv);
    if (conn->cmid)
@@ -354,7 +344,7 @@ void osd_messenger_t::rdmacm_on_connect_peer_error(rdma_cm_id *cmid, int res)
        else
        {
            // TCP is disabled
-            on_connect_peer(peer_osd, res == 0 ? -EINVAL : (res > 0 ? -res : res));
+            on_connect_peer(peer_osd, res == 0 ? -EINVAL : (res > 0 ? -res : res), 0);
        }
    }
 }
@@ -365,7 +355,7 @@ void osd_messenger_t::rdmacm_try_connect_peer(uint64_t peer_osd, const std::stri
    if (!string_to_addr(addr, false, rdmacm_port, &sa))
    {
        fprintf(stderr, "Address %s is invalid\n", addr.c_str());
-        on_connect_peer(peer_osd, -EINVAL);
+        on_connect_peer(peer_osd, -EINVAL, 0);
        return;
    }
    rdma_cm_id *cmid = NULL;
@@ -376,17 +366,7 @@ void osd_messenger_t::rdmacm_try_connect_peer(uint64_t peer_osd, const std::stri
        if (!disable_tcp)
            try_connect_peer_tcp(peer_osd, addr.c_str(), fallback_tcp_port);
        else
-            on_connect_peer(peer_osd, res);
-        return;
-    }
-    // Make a fake FD (FIXME: do not use FDs for identifying clients!)
-    int fake_fd = socket(AF_INET, SOCK_STREAM, 0);
-    if (fake_fd < 0)
-    {
-        int res = -errno;
-        rdma_destroy_id(cmid);
-        // Can't create socket, pointless to try TCP
-        on_connect_peer(peer_osd, res);
+            on_connect_peer(peer_osd, res, 0);
        return;
    }
    if (log_level > 0)
@@ -394,7 +374,7 @@ void osd_messenger_t::rdmacm_try_connect_peer(uint64_t peer_osd, const std::stri
    auto conn = new rdmacm_connecting_t;
    rdmacm_connecting[cmid] = conn;
    conn->cmid = cmid;
-    conn->peer_fd = fake_fd;
+    conn->client_id = next_client_id++;
    conn->peer_osd = peer_osd;
    conn->addr = addr;
    conn->parsed_addr = sa;
@@ -511,13 +491,13 @@ void osd_messenger_t::rdmacm_established(rdma_cm_event *ev)
    auto cl = new osd_client_t();
    cl->peer_addr = conn->parsed_addr;
    cl->peer_port = conn->rdmacm_port;
-    cl->peer_fd = conn->peer_fd;
+    cl->client_id = conn->client_id;
    cl->peer_state = PEER_RDMA;
    cl->connect_timeout_id = -1;
    cl->osd_num = peer_osd;
    cl->in_buf = (uint8_t*)malloc_or_die(receive_buffer_size);
    cl->rdma_conn = rc;
-    clients[conn->peer_fd] = cl;
+    clients[conn->client_id] = cl;
    if (conn->timeout_id >= 0)
    {
        tfd->clear_timer(conn->timeout_id);
@@ -5,9 +5,16 @@
 #include <assert.h>

 #include "messenger.h"
+#include "../util/xxh_x86dispatch.h"
 #ifdef WITH_RDMA
 #include "msgr_rdma.h"
 #endif
+#ifdef WITH_OPENSSL
+#include <openssl/bio.h>
+#include <openssl/err.h>
+#include <openssl/pem.h>
+#include <openssl/ssl.h>
+#endif

 void osd_client_t::cancel_ops()
 {
@@ -43,34 +50,40 @@ void osd_op_t::cancel()
    }
 }

-void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
+// force_delete means stop the client anyway, even if there are refs to it in the event loop.
+// the flag should be used in the destructor.
+// why? - because yes, we could close the FD first and let it fail all requests in the event loop,
+// but in that case it can be quickly reopened and we can get old failed responses for the new FD.
+void osd_messenger_t::stop_client(uint64_t client_id, bool force_delete)
 {
-    assert(peer_fd != 0);
-    auto it = clients.find(peer_fd);
-    if (it == clients.end())
+    auto it = clients.find(client_id);
+    if (!client_id || it == clients.end())
    {
        return;
    }
    osd_client_t *cl = it->second;
-    // FIXME: This 'force' flag is probably an ugly reenterability hack - check its logic and maybe remove it
-    if (cl->peer_state == PEER_CONNECTING && !force || cl->peer_state == PEER_STOPPED)
+    if (cl->peer_state == PEER_STOPPED)
    {
+        if (force_delete)
+        {
+            destroy_client(cl);
+        }
        return;
    }
-    clear_immediate_ops(peer_fd);
+    cl->received_ops.clear();
    if (log_level > 0)
    {
        if (cl->osd_num)
        {
-            fprintf(stderr, "[OSD %ju] Stopping client %d (OSD peer %ju)\n", osd_num, peer_fd, cl->osd_num);
+            fprintf(stderr, "[OSD %ju] Stopping client %ju (OSD peer %ju)\n", osd_num, client_id, cl->osd_num);
        }
        else if (cl->in_osd_num)
        {
-            fprintf(stderr, "[OSD %ju] Stopping client %d (incoming OSD peer %ju)\n", osd_num, peer_fd, cl->in_osd_num);
+            fprintf(stderr, "[OSD %ju] Stopping client %ju (incoming OSD peer %ju)\n", osd_num, client_id, cl->in_osd_num);
        }
        else
        {
-            fprintf(stderr, "[OSD %ju] Stopping client %d (regular client)\n", osd_num, peer_fd);
+            fprintf(stderr, "[OSD %ju] Stopping client %ju (regular client)\n", osd_num, client_id);
        }
    }
    if (cl->encrypt_ctx)
@@ -95,11 +108,11 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
    cl->peer_state = PEER_STOPPED;
    if (cl->osd_num)
    {
-        auto osd_it = osd_peer_fds.find(cl->osd_num);
-        if (osd_it != osd_peer_fds.end() && osd_it->second == cl->peer_fd)
+        auto osd_it = osd_peers.find(cl->osd_num);
+        if (osd_it != osd_peers.end() && osd_it->second == cl)
        {
            // ...and forget OSD peer
-            osd_peer_fds.erase(osd_it);
+            osd_peers.erase(osd_it);
        }
    }
 #ifdef WITH_RDMA
@@ -113,29 +126,11 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
    }
 #endif
 #ifndef __MOCK__
-    // Then remove FD from the eventloop so we don't accidentally read something
-    tfd->set_fd_handler(peer_fd, false, NULL);
    if (cl->connect_timeout_id >= 0)
    {
        tfd->clear_timer(cl->connect_timeout_id);
        cl->connect_timeout_id = -1;
    }
-    for (auto rit = read_ready_clients.begin(); rit != read_ready_clients.end(); rit++)
-    {
-        if (*rit == peer_fd)
-        {
-            read_ready_clients.erase(rit);
-            break;
-        }
-    }
-    for (auto wit = write_ready_clients.begin(); wit != write_ready_clients.end(); wit++)
-    {
-        if (*wit == peer_fd)
-        {
-            write_ready_clients.erase(wit);
-            break;
-        }
-    }
 #endif
    if (cl->in_osd_num && break_pg_locks)
    {
@@ -150,19 +145,51 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
        // so do not repeer on it.
        repeer_pgs(cl->osd_num);
    }
-    // Find the item again because it can be invalidated at this point
-    it = clients.find(peer_fd);
-    if (it != clients.end())
+    if (cl->peer_fd >= 0)
    {
-        clients.erase(it);
+        int r = shutdown(cl->peer_fd, SHUT_RDWR);
+        if (r != 0 && errno != ENOTCONN)
+        {
+            fprintf(stderr, "[OSD %ju] failed to shutdown a socket: %s (code %d)\n", osd_num, strerror(errno), errno);
+        }
    }
    cl->refs--;
    if (cl->refs <= 0 || force_delete)
    {
-        delete cl;
+        destroy_client(cl);
    }
 }

+void osd_messenger_t::destroy_client(osd_client_t *cl)
+{
+    // Find the item again because it can be invalidated at this point
+    clients.erase(cl->client_id);
+    if (cl->peer_fd >= 0)
+    {
+#ifndef __MOCK__
+        tfd->set_fd_handler(cl->peer_fd, false, NULL);
+#endif
+        for (auto rit = read_ready_clients.begin(); rit != read_ready_clients.end(); rit++)
+        {
+            if (*rit == cl->client_id)
+            {
+                read_ready_clients.erase(rit);
+                break;
+            }
+        }
+        for (auto wit = write_ready_clients.begin(); wit != write_ready_clients.end(); wit++)
+        {
+            if (*wit == cl->client_id)
+            {
+                write_ready_clients.erase(wit);
+                break;
+            }
+        }
+        clients_by_fd.erase(cl->peer_fd);
+    }
+    delete cl;
+}
+
 osd_client_t::~osd_client_t()
 {
    free(in_buf);
@@ -183,6 +210,12 @@ osd_client_t::~osd_client_t()
        read_op->cancel();
        read_op = NULL;
    }
+    while (unverified_ops.size())
+    {
+        auto op = unverified_ops.back();
+        unverified_ops.pop_back();
+        op->cancel();
+    }
    // Cancel outbound ops
    cancel_ops();
    for (osd_op_t *op: send_free_ops)
@@ -196,7 +229,10 @@ osd_client_t::~osd_client_t()
    {
        if (op)
        {
-            delete op;
+            if (!((size_t)op & 7))
+                delete op;
+            else
+                free((void*)((size_t)op & ~(size_t)7));
        }
    }
 #ifndef __MOCK__
@@ -207,5 +243,39 @@ osd_client_t::~osd_client_t()
        rdma_conn = NULL;
    }
 #endif
+#endif
+    if (read_csum_state)
+    {
+        XXH3_freeState(read_csum_state);
+        read_csum_state = NULL;
+    }
+    if (write_csum_state)
+    {
+        XXH3_freeState(write_csum_state);
+        write_csum_state = NULL;
+    }
+#ifdef WITH_OPENSSL
+    if (enc_ctx)
+    {
+        EVP_CIPHER_CTX_free(enc_ctx);
+        enc_ctx = NULL;
+    }
+    if (dec_ctx)
+    {
+        EVP_CIPHER_CTX_free(dec_ctx);
+        dec_ctx = NULL;
+    }
+    if (ssl_cli)
+    {
+        SSL_free(ssl_cli);
+        ssl_cli = NULL;
+        write_to_ssl = NULL;
+        read_from_ssl = NULL;
+    }
+    if (ssl_out_buf)
+    {
+        free(ssl_out_buf);
+        ssl_out_buf = NULL;
+    }
 #endif
 }
@@ -231,12 +231,25 @@ struct __attribute__((__packed__)) osd_op_rw_t
    uint32_t len;
    // flags
    // OSD_OP_RETURN_CHAIN for chained reads: return parent number in chain for each block
+    // read_chain size comes after bitmap, takes 0 bytes / 1 byte / 2 byte / 4 byte per each block,
+    // depending on the number of parent inodes (0 parents = 0 bytes, up to 255 parents = 1 byte, etc)
    uint32_t flags;
    // inode metadata revision for chained reads
    uint64_t meta_revision;
    // object version for atomic "CAS" (compare-and-set) writes
    // writes and deletes fail with -EINTR if object version differs from (version-1)
    uint64_t version;
+
+    static inline size_t chain_info_bytes(size_t chain_size)
+    {
+        if (chain_size <= 1)
+            return 0;
+        if (chain_size <= 256)
+            return 1;
+        if (chain_size <= 65536)
+            return 2;
+        return 4;
+    }
 };

 struct __attribute__((__packed__)) osd_reply_rw_t
@@ -247,15 +260,6 @@ struct __attribute__((__packed__)) osd_reply_rw_t
    uint32_t pad0;
    // for reads and writes: object version
    uint64_t version;
-    // for reads: chain info size for OSD_OP_RETURN_CHAIN
-    // (parent number is returned as a variable 2^N number of bits)
-    // i.e. 0 bits = everything is read from the inode itself
-    // 1 bit = inode(0) or its parent(1)
-    // 2 bits = inode(0) or its 1-3 parents
-    // 4 bits = inode(0) or its 1-7 parents
-    // and etc
-    // chain size in bits is (op->req.rw.len / bitmap_granularity * chain_bits_per_block + 7) / 8
-    uint32_t chain_bits_per_block;
 };

 struct __attribute__((__packed__)) osd_reply_del_t
@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@

 Name: Vitastor
 Description: Vitastor client library
-Version: 3.0.5
+Version: 3.0.9
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 2.8...3.30)

 project(vitastor)

@@ -15,15 +15,14 @@ add_custom_command(
 add_library(vitastor_cli STATIC
 	cli_common.cpp
 	cli_alloc_osd.cpp
-	cli_status.cpp
 	cli_describe.cpp
 	cli_fix.cpp
 	cli_ls.cpp
 	cli_create.cpp
-	cli_create_user.cpp
 	cli_dd.cpp
 	cli_modify.cpp
 	cli_modify_osd.cpp
+	cli_modify_user.cpp
 	cli_osd_tree.cpp
 	cli_pg_ls.cpp
 	cli_flatten.cpp
@@ -38,6 +37,8 @@ add_library(vitastor_cli STATIC
 	cli_pool_modify.cpp
 	cli_pool_rm.cpp
 	cli_serve.cpp
+	cli_status.cpp
+	cli_user_ls.cpp
 	${OPENAPI_JSON_H}
 )
 target_compile_options(vitastor_cli PUBLIC -fPIC)
@@ -49,6 +49,9 @@ static const char* help_text =
    "  --enc-key random    Generate a new random AES-256-XTS encryption key for the new image.\n"
    "  --enc-key HEX       Set a specified AES-256-XTS key (64 bytes in hex) for the new image.\n"
    "  --enc-key vault:ID  Use an encryption key from an external Vault secret with specified ID.\n"
+    "  --owner username    Set owner (default is the current user from TLS certificate).\n"
+    "  --owner_group name  Set owner group name.\n"
+    "  --reader_group rdr  Set reader group name.\n"
    "\n"
    "vitastor-cli create --snapshot <snapshot> [OPTIONS] <image>\n"
    "vitastor-cli snap-create [OPTIONS] <image>@<snapshot>\n"
@@ -63,10 +66,13 @@ static const char* help_text =
    "  Rename, resize image or change its readonly status. Images with children can't be made read-write.\n"
    "  If the new size is smaller than the old size, extra data will be purged.\n"
    "  You should resize file system in the image, if present, before shrinking it.\n"
-    "  --deleted 1|0  Set/clear 'deleted image' flag (set automatically during unfinished deletes).\n"
-    "  -f|--force     Proceed with shrinking or setting readwrite flag even if the image has children.\n"
-    "  --down-ok      Proceed with shrinking even if some data will be left on unavailable OSDs.\n"
-    "  --enc-key HEX  Change image encryption key (allowed only with --force).\n"
+    "  --deleted 1|0       Set/clear 'deleted image' flag (set automatically during unfinished deletes).\n"
+    "  -f|--force          Proceed with shrinking or setting readwrite flag even if the image has children.\n"
+    "  --down-ok           Proceed with shrinking even if some data will be left on unavailable OSDs.\n"
+    "  --enc-key HEX       Change image encryption key (allowed only with --force).\n"
+    "  --owner username    Change image owner.\n"
+    "  --owner_group name  Change image owner group name.\n"
+    "  --reader_group rdr  Change image reader group name.\n"
    "\n"
    "vitastor-cli dd [iimg=<image> | if=<file>] [oimg=<image> | of=<file>] [bs=1M]\n"
    "    [count=N] [seek/oseek=N] [skip/iseek=M] [iodepth=N] [status=progress]\n"
@@ -205,6 +211,7 @@ static const char* help_text =
    "    --used_for_app s3:<name>      Mark pool as used for S3 location with name <name>\n"
    "    --pg_stripe_size <number>     Increase object grouping stripe\n"
    "    --max_osd_combinations 10000  Maximum number of random combinations for LP solver input\n"
+    "    --creator_group <group>       User group allowed to create images in this pool.\n"
    "    --wait                        Wait for the new pool to come online\n"
    "    -f|--force                    Do not check that cluster has enough OSDs to create the pool\n"
    "  Examples:\n"
@@ -216,7 +223,7 @@ static const char* help_text =
    "    [-s|--pg_size <number>] [--pg_minsize <number>] [-n|--pg_count <count>]\n"
    "    [--failure_domain <level>] [--root_node <node>] [--osd_tags <tags>] [--used_for_app <type>:<name>]\n"
    "    [--max_osd_combinations <number>] [--primary_affinity_tags <tags>] [--scrub_interval <time>]\n"
-    "    [--level_placement <rules>] [--raw_placement <rules>]\n"
+    "    [--level_placement <rules>] [--raw_placement <rules>] [--creator_group <group>]\n"
    "  Non-modifiable parameters (changing them WILL lead to data loss):\n"
    "    [--block_size <size>] [--bitmap_granularity <size>]\n"
    "    [--immediate_commit <all|small|none>] [--pg_stripe_size <size>]\n"
@@ -237,8 +244,16 @@ static const char* help_text =
    "  -r|--reverse    Sort in descending order\n"
    "  -n|--count N    Only list first N items\n"
    "\n"
-    "vitastor-cli create-user --type <type> --name <name> --grants <grants>\n"
-    "  FIXME\n"
+    "vitastor-cli ls-users|user-ls|ls-user|list-users [<name> ...]\n"
+    "  List users (only with specified names if passed).\n"
+    "\n"
+    "vitastor-cli modify-user --type <type> --groups group1,group2,... <username>\n"
+    "  Create or update user permissions. User names match CN of their certificates.\n"
+    "  --type TYPE      Set user type: client, admin, mon or osd. Default is client.\n"
+    "  --groups GROUPS  Set user's groups.\n"
+    "\n"
+    "vitastor-cli rm-user|remove-user|delete-user <username>\n"
+    "  Remove a user.\n"
    "\n"
    "vitastor-cli serve\n"
    "  Start HTTP server able to handle CLI commands over a REST API. Options:\n"
@@ -570,10 +585,34 @@ std::function<bool(cli_result_t &)> cli_tool_t::start(json11::Json::object cfg,
        }
        action_cb = start_pool_ls(cfg);
    }
-    else if (cmd[0] == "create-user")
+    else if (cmd[0] == "user-ls" || cmd[0] == "ls-user" || cmd[0] == "ls-users" || cmd[0] == "list-users")
+    {
+        // List users
+        if (cmd.size() > 1)
+        {
+            cmd.erase(cmd.begin(), cmd.begin()+1);
+            cfg["names"] = cmd;
+        }
+        action_cb = start_user_ls(cfg);
+    }
+    else if (cmd[0] == "modify-user" || cmd[0] == "user-modify")
    {
        // Create/update user
-        action_cb = start_create_user(cfg);
+        if (cmd.size() > 1)
+        {
+            cfg["name"] = cmd[1];
+        }
+        action_cb = start_modify_user(cfg);
+    }
+    else if (cmd[0] == "rm-user" || cmd[0] == "remove-user" || cmd[0] == "delete-user")
+    {
+        // Remove user
+        if (cmd.size() > 1)
+        {
+            cfg["name"] = cmd[1];
+        }
+        cfg["remove"] = true;
+        action_cb = start_modify_user(cfg);
    }
    else if (cmd[0] == "serve")
    {
@@ -9,6 +9,7 @@
 #include "object_id.h"
 #include "ringloop.h"
 #include <functional>
+#include <set>

 struct rm_inode_t;
 struct snap_merger_t;
@@ -26,6 +27,13 @@ struct cli_result_t
    json11::Json data;
 };

+struct cli_user_t
+{
+    std::string name;
+    std::string type;
+    std::set<std::string> groups;
+};
+
 class cli_tool_t
 {
 public:
@@ -37,6 +45,8 @@ public:
    bool is_command_line = false;
    bool color = false;

+    std::unique_ptr<cli_user_t> user; // for http mode
+
    ring_loop_t *ringloop = NULL;
    epoll_manager_t *epmgr = NULL;
    cluster_client_t *cli = NULL;
@@ -53,6 +63,8 @@ public:
    void change_parent(inode_t cur, inode_t new_parent, cli_result_t *result);
    inode_config_t* get_inode_cfg(const std::string & name);

+    bool check_image_perm(const inode_config_t & cfg, bool write);
+
    friend struct rm_inode_t;
    friend struct snap_merger_t;
    friend struct snap_flattener_t;
@@ -61,7 +73,6 @@ public:
    std::function<bool(cli_result_t &)> start(json11::Json::object cfg, cli_result_t & result);
    std::function<bool(cli_result_t &)> start_alloc_osd(json11::Json);
    std::function<bool(cli_result_t &)> start_create(json11::Json);
-    std::function<bool(cli_result_t &)> start_create_user(json11::Json);
    std::function<bool(cli_result_t &)> start_dd(json11::Json);
    std::function<bool(cli_result_t &)> start_describe(json11::Json);
    std::function<bool(cli_result_t &)> start_fix(json11::Json);
@@ -70,6 +81,7 @@ public:
    std::function<bool(cli_result_t &)> start_merge(json11::Json);
    std::function<bool(cli_result_t &)> start_modify(json11::Json);
    std::function<bool(cli_result_t &)> start_modify_osd(json11::Json);
+    std::function<bool(cli_result_t &)> start_modify_user(json11::Json);
    std::function<bool(cli_result_t &)> start_osd_tree(json11::Json);
    std::function<bool(cli_result_t &)> start_pg_list(json11::Json);
    std::function<bool(cli_result_t &)> start_pool_create(json11::Json);
@@ -82,11 +94,11 @@ public:
    std::function<bool(cli_result_t &)> start_rm_wildcard(json11::Json);
    std::function<bool(cli_result_t &)> start_serve(json11::Json);
    std::function<bool(cli_result_t &)> start_status(json11::Json);
+    std::function<bool(cli_result_t &)> start_user_ls(json11::Json);

    // Should be called like loop_and_wait(start_status(), <completion callback>)
    void loop_and_wait(std::function<bool(cli_result_t &)> loop_cb, std::function<void(const cli_result_t &)> complete_cb);

-    void etcd_call(const std::string & api, json11::Json body);
    void etcd_txn(json11::Json txn);

    void iterate_kvs_1(json11::Json kvs, const std::string & prefix, std::function<void(uint64_t num, json11::Json)> cb);
@@ -103,5 +115,3 @@ std::string format_lat(uint64_t lat);
 std::string format_q(double depth);

 bool stupid_glob(const std::string str, const std::string glob);
-
-json11::Json::object merge_json_objects(json11::Json::object obj1, const json11::Json::object & obj2);
@@ -6,6 +6,15 @@
 #include "cluster_client.h"
 #include "cli.h"

+bool cli_tool_t::check_image_perm(const inode_config_t & cfg, bool write)
+{
+    return !user ||
+        user->type == "admin" ||
+        user->name == cfg.owner ||
+        cfg.owner_group != "" && user->groups.find(cfg.owner_group) != user->groups.end() ||
+        !write && cfg.reader_group != "" && user->groups.find(cfg.reader_group) != user->groups.end();
+}
+
 json11::Json::object cli_tool_t::format_image(const inode_config_t & cfg)
 {
    auto pool_it = cli->st_cli.pool_config.find(INODE_POOL(cfg.num));
@@ -20,6 +29,18 @@ json11::Json::object cli_tool_t::format_image(const inode_config_t & cfg)
        { "readonly", cfg.readonly },
        { "deleted", cfg.deleted },
    };
+    if (cfg.owner != "")
+    {
+        img["owner"] = cfg.owner;
+    }
+    if (cfg.owner_group != "")
+    {
+        img["owner_group"] = cfg.owner_group;
+    }
+    if (cfg.reader_group != "")
+    {
+        img["reader_group"] = cfg.reader_group;
+    }
    if (!cfg.enc_key.empty())
    {
        img["encrypted"] = true;
@@ -109,10 +130,10 @@ void cli_tool_t::change_parent(inode_t cur, inode_t new_parent, cli_result_t *re
    });
 }

-void cli_tool_t::etcd_call(const std::string & api, json11::Json body)
+void cli_tool_t::etcd_txn(json11::Json txn)
 {
    waiting++;
-    cli->st_cli.etcd_call(api, body, cli->st_cli.etcd_slow_timeout, cli->st_cli.max_etcd_attempts, 0, [this](std::string err, json11::Json res)
+    cli->st_cli.etcd_txn_slow(txn, [this](std::string err, json11::Json res)
    {
        waiting--;
        if (err != "")
@@ -124,11 +145,6 @@ void cli_tool_t::etcd_call(const std::string & api, json11::Json body)
    });
 }

-void cli_tool_t::etcd_txn(json11::Json txn)
-{
-    etcd_call("/kv/txn", txn);
-}
-
 inode_config_t* cli_tool_t::get_inode_cfg(const std::string & name)
 {
    for (auto & ic: cli->st_cli.inode_config)
@@ -249,10 +265,3 @@ void cli_tool_t::iterate_kvs_2(json11::Json kvs, const std::string & prefix, std
        cb(pool_id, num, kv.value);
    }
 }
-
-json11::Json::object merge_json_objects(json11::Json::object obj1, const json11::Json::object & obj2)
-{
-    for (auto & kv: obj2)
-        obj1[kv.first] = kv.second;
-    return obj1;
-}
@@ -7,6 +7,7 @@

 #include <ctype.h>
 #include "cli.h"
+#include "http_client.h"
 #include "cluster_client.h"
 #include "str_util.h"

@@ -35,6 +36,7 @@ struct image_creator_t
    bool force_size = false;
    std::string enc_key;
    bool set_key = false;
+    std::string new_owner, new_owner_group, new_reader_group;

    pool_id_t old_pool_id = 0;
    inode_t new_parent_id = 0;
@@ -119,6 +121,23 @@ struct image_creator_t
            create_snapshot();
    }

+    bool check_pool_permission()
+    {
+        if (!parent->user || parent->user->type == "admin")
+        {
+            return true;
+        }
+        auto pool_it = parent->cli->st_cli.pool_config.find(new_pool_id);
+        if (pool_it == parent->cli->st_cli.pool_config.end() ||
+            (pool_it->second.creator_group == "" || parent->user->groups.find(pool_it->second.creator_group) == parent->user->groups.end()))
+        {
+            result = (cli_result_t){ .err = EACCES, .text = "Pool image create permission denied" };
+            state = 100;
+            return false;
+        }
+        return true;
+    }
+
    void create_image()
    {
        if (state == 2)
@@ -158,6 +177,10 @@ struct image_creator_t
            state = 100;
            return;
        }
+        if (!check_pool_permission())
+        {
+            return;
+        }
        if (!size && !force_size)
        {
            result = (cli_result_t){ .err = EINVAL, .text = "Image size is missing" };
@@ -249,11 +272,22 @@ resume_3:
                state = 100;
                return;
            }
+            if (!parent->check_image_perm(cur_cfg, true))
+            {
+                result = (cli_result_t){ .err = EACCES, .text = "Image permission denied" };
+                state = 100;
+                return;
+            }
            if (!new_pool_id)
            {
                // Create snapshot in the same pool by default
                new_pool_id = old_pool_id;
            }
+            // Verify pool permissions if the pool is different from the original
+            if (new_pool_id != old_pool_id && !check_pool_permission())
+            {
+                return;
+            }
            attempt_create();
            state = 4;
 resume_4:
@@ -442,6 +476,19 @@ resume_3:
        {
            new_cfg.enc_key = cur_cfg.enc_key;
        }
+        new_cfg.owner = http_context_get_ssl_cn(parent->cli->st_cli.get_http_ctx());
+        if (!new_owner.empty())
+        {
+            new_cfg.owner = new_owner;
+        }
+        if (!new_owner_group.empty())
+        {
+            new_cfg.owner_group = new_owner_group;
+        }
+        if (!new_reader_group.empty())
+        {
+            new_cfg.reader_group = new_reader_group;
+        }
        json11::Json::array checks = json11::Json::array {
            json11::Json::object {
                { "target", "VERSION" },
@@ -480,13 +527,10 @@ resume_3:
            json11::Json::object {
                { "request_put", json11::Json::object {
                    { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/index/image/"+image_name) },
-                    { "value", base64_encode(json11::Json(merge_json_objects(
-                        parent->cli->st_cli.serialize_inode_cfg(&new_cfg),
-                        json11::Json::object{
-                            { "id", new_id },
-                            { "pool_id", (uint64_t)new_pool_id },
-                        }
-                    )).dump()) },
+                    { "value", base64_encode(json11::Json(json11::Json::object{
+                        { "id", new_id },
+                        { "pool_id", (uint64_t)new_pool_id },
+                    }).dump()) },
                } },
            },
            json11::Json::object {
@@ -541,13 +585,10 @@ resume_3:
            success.push_back(json11::Json::object {
                { "request_put", json11::Json::object {
                    { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/index/image/"+image_name+"@"+new_snap) },
-                    { "value", base64_encode(json11::Json(merge_json_objects(
-                        parent->cli->st_cli.serialize_inode_cfg(&snap_cfg),
-                        json11::Json::object{
-                            { "id", old_id },
-                            { "pool_id", (uint64_t)old_pool_id },
-                        }
-                    )).dump()) },
+                    { "value", base64_encode(json11::Json(json11::Json::object{
+                        { "id", old_id },
+                        { "pool_id", (uint64_t)old_pool_id },
+                    }).dump()) },
                } },
            });
        };
@@ -610,6 +651,9 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_create(json11::Json cfg)
            }
        }
    }
+    image_creator->new_owner = cfg["owner"].string_value();
+    image_creator->new_owner_group = cfg["owner_group"].string_value();
+    image_creator->new_reader_group = cfg["reader_group"].string_value();
    image_creator->new_parent = cfg["parent"].string_value();
    if (!cfg["size"].is_null())
    {
@@ -1,393 +0,0 @@
-// Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.1 (see README.md for details)
-
-#include "cli.h"
-#include "cluster_client.h"
-#include "str_util.h"
-#include "json_util.h"
-
-struct etcd_perm_t
-{
-    std::string key;
-    std::string range_end;
-    std::string perm_type;
-};
-
-class etcd_user_sync_t
-{
-protected:
-    int state = 0;
-    std::vector<etcd_perm_t> new_perms;
-    int i = 0;
-    std::map<std::pair<std::string, std::string>, std::string> old_perms;
-    std::map<std::pair<std::string, std::string>, std::string>::iterator old_perm_it;
-
-public:
-    cli_tool_t *parent = NULL;
-    std::string role_name;
-    std::vector<etcd_perm_t> perms;
-    cli_result_t result;
-
-    // returns boolean true when done
-    bool run()
-    {
-        if (state == 1)
-            goto resume_1;
-        else if (state == 2)
-            goto resume_2;
-        else if (state == 3)
-            goto resume_3;
-        else if (state == 4)
-            goto resume_4;
-        else if (state == 5)
-            goto resume_5;
-        else if (state == 6)
-            goto resume_6;
-        else if (state == 100)
-            return true;
-        parent->etcd_call("/auth/user/get", json11::Json::object{
-            { "name", role_name },
-        });
-resume_1:
-        state = 1;
-        if (parent->waiting > 0)
-            return false;
-        // etcd returns HTTP/1.1 400 Bad Request + {"error":"etcdserver: user name not found","code":9}
-        // FIXME Check if it works
-        if (parent->etcd_err.text.find("user name not found") != std::string::npos)
-        {
-            parent->etcd_call("/auth/user/add", json11::Json::object{
-                { "name", role_name },
-                { "options", json11::Json::object{
-                    { "no_password", true }
-                } },
-            });
-        resume_2:
-            state = 2;
-            if (parent->waiting > 0)
-                return false;
-            // etcd returns HTTP/1.1 400 Bad Request + {"error":"etcdserver: role name already exists","code":9}
-            if (parent->etcd_err.err && parent->etcd_err.text.find("role name already exists") == std::string::npos)
-            {
-                result = parent->etcd_err;
-                return true;
-            }
-        }
-        parent->etcd_call("/auth/role/get", json11::Json::object{
-            { "role", role_name },
-        });
-    resume_3:
-        state = 3;
-        if (parent->waiting > 0)
-            return false;
-        // etcd returns HTTP/1.1 400 Bad Request + {"error":"etcdserver: role name not found","code":9}
-        if (parent->etcd_err.text.find("role name not found") != std::string::npos)
-        {
-            parent->etcd_call("/auth/role/add", json11::Json::object{
-                { "name", role_name },
-            });
-        resume_4:
-            state = 4;
-            if (parent->waiting > 0)
-                return false;
-            // etcd returns HTTP/1.1 400 Bad Request + {"error":"etcdserver: role name already exists","code":9}
-            if (parent->etcd_err.err && parent->etcd_err.text.find("role name already exists") == std::string::npos)
-            {
-                state = 100;
-                result = parent->etcd_err;
-                return true;
-            }
-        }
-        else if (parent->etcd_err.err)
-        {
-            state = 100;
-            result = parent->etcd_err;
-            return true;
-        }
-        for (auto & old_perm: parent->etcd_result["perm"].array_items())
-        {
-            old_perms.emplace(std::make_pair(old_perm["key"].string_value(), old_perm["range_end"].string_value()), old_perm["perm_type"].string_value());
-        }
-        for (auto & perm: perms)
-        {
-            auto perm_key = std::make_pair(
-                base64_encode(parent->cli->st_cli.etcd_prefix+perm.key),
-                perm.range_end.empty() ? "" : base64_encode(parent->cli->st_cli.etcd_prefix+perm.range_end)
-            );
-            auto perm_it = old_perms.find(perm_key);
-            if (perm_it == old_perms.end() || perm_it->second != perm.perm_type)
-                new_perms.push_back(perm);
-            else
-                old_perms.erase(perm_it);
-        }
-        for (old_perm_it = old_perms.begin(); old_perm_it != old_perms.end(); old_perm_it++)
-        {
-            parent->etcd_call("/auth/role/revoke", json11::Json::object{
-                { "role", role_name },
-                { "key", old_perm_it->first.first },
-                { "range_end", old_perm_it->first.second },
-            });
-        resume_5:
-            state = 5;
-            if (parent->waiting > 0)
-                return false;
-            if (parent->etcd_err.err)
-            {
-                state = 100;
-                result = parent->etcd_err;
-                return true;
-            }
-        }
-        for (i = 0; i < new_perms.size(); i++)
-        {
-            {
-                auto & perm = new_perms[i];
-                json11::Json::object obj;
-                obj["key"] = base64_encode(parent->cli->st_cli.etcd_prefix+perm.key);
-                if (!perm.range_end.empty())
-                    obj["range_end"] = base64_encode(parent->cli->st_cli.etcd_prefix+perm.range_end);
-                obj["perm_type"] = perm.perm_type;
-                parent->etcd_call("/auth/role/grant", json11::Json::object{
-                    { "name", role_name },
-                    { "perm", obj },
-                });
-            }
-        resume_6:
-            state = 6;
-            if (parent->waiting > 0)
-                return false;
-            if (parent->etcd_err.err)
-            {
-                state = 100;
-                result = parent->etcd_err;
-                return true;
-            }
-        }
-        state = 100;
-        result = {};
-        return true;
-    }
-};
-
-// Create/update a user
-struct cli_create_user_t
-{
-    cli_tool_t *parent;
-
-    std::string user_name;
-    std::string user_type;
-    json11::Json grants;
-
-    etcd_user_sync_t *user_sync = NULL;
-
-    int state = 0;
-    cli_result_t result;
-
-    bool is_done()
-    {
-        return state == 100;
-    }
-
-    void loop()
-    {
-        if (state == 1)
-            goto resume_1;
-        else if (state == 2)
-            goto resume_2;
-        user_sync = new etcd_user_sync_t();
-        user_sync->parent = parent;
-        user_sync->role_name = user_name;
-        if (user_type == "osd")
-        {
-            user_sync->perms = {
-                { "/config/", "/config0", "READ" },
-                { "/osd/", "/osd0", "READWRITE" },
-                { "/pg/config", "", "READ" },
-                { "/pg/state/", "/pg/state0", "READWRITE" },
-                { "/pg/history/", "/pg/history0", "READWRITE" },
-                { "/pgstats/", "/pgstats0", "READWRITE" },
-            };
-        }
-        else if (user_type == "mon")
-        {
-            user_sync->perms = {
-                { "/config/", "/config0", "READ" },
-                { "/osd/", "/osd0", "READ" },
-                { "/mon/", "/mon0", "READWRITE" },
-                { "/pg/config", "", "READWRITE" },
-                { "/pg/history/", "/pg/history0", "READWRITE" },
-                { "/pgstats/", "/pgstats0", "READ" },
-                { "/inode/stats/", "/inode/stats0", "READWRITE" },
-                { "/pool/stats/", "/pool/stats0", "READWRITE" },
-                { "/stats/", "/stats0", "READWRITE" },
-                { "/history/last_clean_pgs", "", "READWRITE" },
-                { "/index/", "/index0", "READ" },
-            };
-        }
-        else if (user_type == "admin")
-        {
-            // CLI admin + global I/O client
-            user_sync->perms = {
-                { "/config/", "/config0", "READWRITE" },
-                { "/osd", "/osd0", "READWRITE" },
-                { "/mon/", "/mon0", "READ" },
-                { "/pg/", "/pg0", "READ" },
-                { "/pg/history/", "/pg/history0", "READWRITE" },
-                { "/pgstats/", "/pgstats0", "READ" },
-                { "/inode/stats/", "/inode/stats0", "READ" },
-                { "/pool/stats/", "/pool/stats0", "READ" },
-                { "/stats/", "/stats0", "READ" },
-                { "/index/", "/index0", "READWRITE" },
-            };
-        }
-        else if (user_type == "client")
-        {
-            // I/O client without etcd write permissions
-            user_sync->perms = {
-                { "/config/global", "", "READ" },
-                { "/config/node_placement", "", "READ" },
-                { "/config/pools", "", "READ" },
-                { "/osd/state", "/osd/state0", "READ" },
-                { "/pg/config", "", "READ" },
-                { "/pg/state/", "/pg/state0", "READ" },
-                //{ "/index/image/<name>", "", "READ" },
-            };
-            {
-                std::string error;
-                grants = parse_grants(grants, error);
-                if (error != "")
-                {
-                    result = (cli_result_t){ .err = EINVAL, .text = error };
-                    state = 100;
-                    return;
-                }
-            }
-            for (auto & grant: grants.array_items())
-            {
-                user_sync->perms.push_back({
-                    "/index/image/"+grant["name"].string_value()+"/",
-                    grant["prefix"].bool_value() ? "/index/image/"+grant["name"].string_value()+"0" : "",
-                    "READ"
-                });
-            }
-            // Also write client's image permissions to etcd, for future vitastor-cli API support
-            parent->etcd_txn(json11::Json::object {
-                { "success", json11::Json::array { json11::Json::object {
-                    { "request_put", json11::Json::object {
-                        { "key", base64_encode(
-                            parent->cli->st_cli.etcd_prefix+"/config/user/"+user_name
-                        ) },
-                        { "value", base64_encode(
-                            json11::Json(json11::Json::object{
-                                { "type", user_type },
-                                { "grants", grants },
-                            }).dump()
-                        ) },
-                    } },
-                } } }
-            });
-            state = 2;
-resume_2:
-            if (parent->waiting > 0)
-                return;
-            if (parent->etcd_err.err)
-            {
-                result = parent->etcd_err;
-                state = 100;
-                return;
-            }
-        }
-        else
-        {
-            result = (cli_result_t){ .err = EINVAL, .text = "Invalid user type: "+user_type };
-            state = 100;
-            return;
-        }
-    resume_1:
-        state = 1;
-        while (!user_sync->run())
-        {
-            return;
-        }
-        if (user_sync->result.err)
-        {
-            result = user_sync->result;
-            state = 100;
-            return;
-        }
-        delete user_sync;
-        user_sync = NULL;
-        state = 100;
-    }
-
-    json11::Json::array parse_grants(json11::Json grants, std::string & error)
-    {
-        json11::Json::array parsed;
-        if (grants.is_string())
-        {
-            // [rwx]:[^ ]+*( +[rwx]:name*)
-            grants = explode(" ", grants.string_value(), true);
-        }
-        for (auto & grant: grants.array_items())
-        {
-            if (grant.is_string())
-            {
-                const auto & str = grant.string_value();
-                if (str.size() < 3 || str[0] != 'r' && str[0] != 'w' && str[0] != 'x' || str[1] != ':')
-                {
-                    error = "Invalid image permission: "+str;
-                    return json11::Json::array();
-                }
-                else
-                {
-                    parsed.push_back(json11::Json::object{
-                        { "type", str[0] == 'r' ? "read" : (str[0] == 'w' ? "write" : "admin") },
-                        { "name", str.substr(2, str.size() - (str[str.size()-1] == '*' ? 3 : 2)) },
-                        { "prefix", str[str.size()-1] == '*' },
-                    });
-                }
-            }
-            else if (!grant.is_object())
-            {
-                error = "Image permissions should be strings or objects";
-                return json11::Json::array();
-            }
-            else
-            {
-                const auto & obj = grant.object_items();
-                for (auto & kv: obj)
-                {
-                    if (!(kv.first == "type" && (kv.second == "read" || kv.second == "write" || kv.second == "admin") ||
-                        kv.first == "name" && kv.second.is_string() ||
-                        kv.first == "prefix" && kv.second.is_bool()))
-                    {
-                        error = "Invalid image permission: "+grant.dump();
-                        return json11::Json::array();
-                    }
-                }
-                parsed.push_back(grant);
-            }
-        }
-        return parsed;
-    }
-};
-
-std::function<bool(cli_result_t &)> cli_tool_t::start_create_user(json11::Json cfg)
-{
-    auto creator = new cli_create_user_t();
-    creator->parent = this;
-    creator->user_type = cfg["type"].string_value();
-    creator->user_name = cfg["name"].string_value();
-    creator->grants = cfg["grants"];
-    return [creator](cli_result_t & result)
-    {
-        creator->loop();
-        if (creator->is_done())
-        {
-            result = creator->result;
-            delete creator;
-            return true;
-        }
-        return false;
-    };
-}
@@ -35,6 +35,12 @@ struct snap_flattener_t
            state = 100;
            return;
        }
+        if (!parent->check_image_perm(*target_cfg, true))
+        {
+            result = (cli_result_t){ .err = EACCES, .text = "Image permission denied" };
+            state = 100;
+            return;
+        }
        target_id = target_cfg->num;
        std::vector<inode_t> chain_list;
        inode_config_t *cur = target_cfg;
@@ -53,13 +53,19 @@ struct image_lister_t
                return;
            }
        }
-        for (auto & ic: parent->cli->st_cli.inode_config)
+        auto begin_it = list_pool_id
+            ? parent->cli->st_cli.inode_config.lower_bound(INODE_WITH_POOL(list_pool_id, 0))
+            : parent->cli->st_cli.inode_config.begin();
+        auto end_it = list_pool_id
+            ? parent->cli->st_cli.inode_config.lower_bound(INODE_WITH_POOL(list_pool_id+1, 0))
+            : parent->cli->st_cli.inode_config.end();
+        for (auto it = begin_it; it != end_it; it++)
        {
-            if (list_pool_id && INODE_POOL(ic.second.num) != list_pool_id)
+            if (!parent->check_image_perm(it->second, false))
            {
                continue;
            }
-            stats[ic.second.num] = parent->format_image(ic.second);
+            stats[it->second.num] = parent->format_image(it->second);
        }
    }

@@ -106,6 +112,7 @@ resume_1:
            state = 100;
            return;
        }
+        // FIXME: Do not always read everything
        space_info = parent->etcd_result;
        std::map<pool_id_t, uint64_t> pool_pg_real_size;
        for (auto & kv_item: space_info["responses"][0]["response_range"]["kvs"].array_items())
@@ -139,6 +146,11 @@ resume_1:
            }
            inode_t inode_num = INODE_WITH_POOL(pool_id, only_inode_num);
            uint64_t used_size = kv.value["raw_used"].uint64_value();
+            auto stat_it = stats.find(inode_num);
+            if (parent->user && parent->user->type != "admin" && stat_it == stats.end())
+            {
+                continue;
+            }
            // save stats
            auto pool_it = parent->cli->st_cli.pool_config.find(pool_id);
            if (pool_it != parent->cli->st_cli.pool_config.end())
@@ -147,7 +159,6 @@ resume_1:
                used_size = used_size / (pool_pg_real_size[pool_id] ? pool_pg_real_size[pool_id] : 1)
                    * (pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks);
            }
-            auto stat_it = stats.find(inode_num);
            if (stat_it == stats.end())
            {
                stats[inode_num] = json11::Json::object {
@@ -19,6 +19,7 @@ struct image_changer_t
    bool set_deleted = false, new_deleted = false;
    bool set_key = false;
    std::string enc_key;
+    json11::Json new_owner, new_owner_group, new_reader_group;
    bool down_ok = false;
    // interval between fsyncs
    int fsync_interval = 128;
@@ -76,6 +77,12 @@ struct image_changer_t
            state = 100;
            return;
        }
+        if (!parent->check_image_perm(cfg, true))
+        {
+            result = (cli_result_t){ .err = EACCES, .text = "Image permission denied" };
+            state = 100;
+            return;
+        }
        for (auto & ic: parent->cli->st_cli.inode_config)
        {
            if (ic.second.parent_id == inode_num)
@@ -151,6 +158,18 @@ resume_1:
        {
            cfg.name = new_name;
        }
+        if (new_owner.is_string())
+        {
+            cfg.owner = new_owner.string_value();
+        }
+        if (new_owner_group.is_string())
+        {
+            cfg.owner_group = new_owner_group.string_value();
+        }
+        if (new_reader_group.is_string())
+        {
+            cfg.reader_group = new_reader_group.string_value();
+        }
        if (set_key)
        {
            if (!force)
@@ -188,6 +207,7 @@ resume_1:
                } }
            });
        }
+        if (new_name != "")
        {
            std::string old_idx_key = base64_encode(
                parent->cli->st_cli.etcd_prefix+"/index/image/"+image_name
@@ -201,29 +221,23 @@ resume_1:
                { "result", "LESS" },
                { "mod_revision", cfg.mod_revision+1 },
            });
-            if (new_name != "")
-            {
-                checks.push_back(json11::Json::object {
-                    { "target", "VERSION" },
-                    { "version", 0 },
-                    { "key", new_idx_key },
-                });
-                success.push_back(json11::Json::object {
-                    { "request_delete_range", json11::Json::object {
-                        { "key", old_idx_key },
-                    } }
-                });
-            }
+            checks.push_back(json11::Json::object {
+                { "target", "VERSION" },
+                { "version", 0 },
+                { "key", new_idx_key },
+            });
+            success.push_back(json11::Json::object {
+                { "request_delete_range", json11::Json::object {
+                    { "key", old_idx_key },
+                } }
+            });
            success.push_back(json11::Json::object {
                { "request_put", json11::Json::object {
-                    { "key", new_name.empty() ? old_idx_key : new_idx_key },
-                    { "value", base64_encode(json11::Json(merge_json_objects(
-                        parent->cli->st_cli.serialize_inode_cfg(&cfg),
-                        json11::Json::object{
-                            { "id", INODE_NO_POOL(inode_num) },
-                            { "pool_id", (uint64_t)INODE_POOL(inode_num) },
-                        }
-                    )).dump()) },
+                    { "key", new_idx_key },
+                    { "value", base64_encode(json11::Json(json11::Json::object{
+                        { "id", INODE_NO_POOL(inode_num) },
+                        { "pool_id", (uint64_t)INODE_POOL(inode_num) },
+                    }).dump()) },
                } }
            });
        }
@@ -283,6 +297,9 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_modify(json11::Json cfg)
    if (!changer->fsync_interval)
        changer->fsync_interval = 128;
    changer->down_ok = cfg["down_ok"].bool_value();
+    changer->new_owner = cfg["owner"];
+    changer->new_owner_group = cfg["owner_group"];
+    changer->new_reader_group = cfg["reader_group"];
    // FIXME Check that the image doesn't have children when shrinking
    return [changer](cli_result_t & result)
    {
@@ -0,0 +1,161 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 (see README.md for details)
+
+#include "cli.h"
+#include "cluster_client.h"
+#include "str_util.h"
+
+// Create/update/delete a user
+struct cli_modify_user_t
+{
+    cli_tool_t *parent;
+
+    std::string user_name;
+    std::string user_type;
+    json11::Json groups;
+    bool del = false;
+
+    int state = 0;
+    cli_result_t result;
+    etcd_kv_t kv;
+    json11::Json::object new_cfg;
+
+    bool is_done()
+    {
+        return state == 100;
+    }
+
+    void loop()
+    {
+        if (state == 1)
+            goto resume_1;
+        else if (state == 2)
+            goto resume_2;
+        if (user_type != "client" && user_type != "admin" && user_type != "mon" && user_type != "osd")
+        {
+            result = (cli_result_t){ .err = EINVAL, .text = "Unknown user type: "+user_type };
+            state = 100;
+            return;
+        }
+        if (groups.is_string())
+        {
+            groups = groups == "" ? std::vector<std::string>() : explode(",", groups.string_value(), true);
+        }
+        else if (groups.is_array())
+        {
+            for (auto & gr: groups.array_items())
+            {
+                if (!gr.is_string())
+                {
+                    result = (cli_result_t){ .err = EINVAL, .text = "Group names must be strings" };
+                    state = 100;
+                    return;
+                }
+            }
+        }
+        else if (!groups.is_null())
+        {
+            result = (cli_result_t){ .err = EINVAL, .text = "Group names must be strings" };
+            state = 100;
+            return;
+        }
+        if (user_name == "")
+        {
+            result = (cli_result_t){ .err = EINVAL, .text = "User name must not be empty" };
+            state = 100;
+            return;
+        }
+        parent->etcd_txn(json11::Json::object {
+            { "success", json11::Json::array { json11::Json::object {
+                { "request_range", json11::Json::object {
+                    { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/user/"+user_name) },
+                } },
+            } } }
+        });
+        state = 1;
+resume_1:
+        if (parent->waiting > 0)
+            return;
+        if (parent->etcd_err.err)
+        {
+            result = parent->etcd_err;
+            state = 100;
+            return;
+        }
+        kv = parent->cli->st_cli.parse_etcd_kv(parent->etcd_result["responses"][0]["response_range"]["kvs"][0]);
+        while (true)
+        {
+            new_cfg = kv.value.object_items();
+            if (!kv.mod_revision && del)
+            {
+                result = (cli_result_t){ .err = ENOENT, .text = "User "+user_name+" does not exist" };
+                state = 100;
+                break;
+            }
+            if (!groups.is_null())
+                new_cfg["groups"] = groups;
+            if (user_type != "")
+                new_cfg["type"] = user_type;
+            if (!new_cfg["type"].is_string())
+                new_cfg["type"] = "client";
+            parent->etcd_txn(json11::Json::object {
+                { "compare", json11::Json::array { json11::Json::object {
+                    { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/user/"+user_name) },
+                    { "target", kv.mod_revision ? "MOD" : "VERSION" },
+                    { kv.mod_revision ? "mod_revision" : "version", kv.mod_revision },
+                } } },
+                { "success", json11::Json::array {
+                    del ? json11::Json::object { { "request_delete_range", json11::Json::object {
+                        { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/user/"+user_name) },
+                    } } } : json11::Json::object { { "request_put", json11::Json::object {
+                        { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/user/"+user_name) },
+                        { "value", base64_encode(json11::Json(new_cfg).dump()) }
+                    } } },
+                } },
+                { "failure", json11::Json::array { json11::Json::object {
+                    { "request_range", json11::Json::object {
+                        { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/user/"+user_name) },
+                    } },
+                } } },
+            });
+            state = 2;
+resume_2:
+            if (parent->waiting > 0)
+                return;
+            if (parent->etcd_err.err)
+            {
+                result = parent->etcd_err;
+                state = 100;
+                return;
+            }
+            if (parent->etcd_result["succeeded"].bool_value())
+                break;
+            kv = parent->cli->st_cli.parse_etcd_kv(parent->etcd_result["responses"][0]["response_range"]["kvs"][0]);
+        }
+        state = 100;
+        result.text = del ? "User "+user_name+" removed" : "User "+user_name+" modified";
+        new_cfg["name"] = user_name;
+        result.data = del ? json11::Json::object{ { "ok", true } } : new_cfg;
+    }
+};
+
+std::function<bool(cli_result_t &)> cli_tool_t::start_modify_user(json11::Json cfg)
+{
+    auto creator = new cli_modify_user_t();
+    creator->parent = this;
+    creator->user_name = cfg["name"].string_value();
+    creator->user_type = cfg["type"].string_value();
+    creator->groups = cfg["groups"];
+    creator->del = cfg["remove"].bool_value();
+    return [creator](cli_result_t & result)
+    {
+        creator->loop();
+        if (creator->is_done())
+        {
+            result = creator->result;
+            delete creator;
+            return true;
+        }
+        return false;
+    };
+}
@@ -91,7 +91,7 @@ std::string validate_pool_config(json11::Json::object & new_cfg, json11::Json ol
        }
        else if (key == "name" || key == "scheme" || key == "immediate_commit" ||
            key == "failure_domain" || key == "root_node" || key == "scrub_interval" || key == "used_for_app" ||
-            key == "used_for_fs" || key == "raw_placement" || key == "local_reads")
+            key == "used_for_fs" || key == "raw_placement" || key == "local_reads" || key == "creator_group")
        {
            if (!value.is_string())
            {
@@ -269,10 +269,17 @@ resume_100:
            {
                char buf[1024];
                snprintf(buf, 1024, "Parent inode of layer %s (id 0x%jx) not found", cur->name.c_str(), cur->parent_id);
+                result = (cli_result_t){ .err = ENOENT, .text = buf };
                state = 100;
                return;
            }
            cur = &it->second;
+            if (!parent->check_image_perm(*cur, true))
+            {
+                result = (cli_result_t){ .err = EACCES, .text = "Image permission denied" };
+                state = 100;
+                return;
+            }
            chain_list.push_back(cur->num);
        }
        if (cur->num != from_cfg->num)
@@ -297,6 +304,12 @@ resume_100:
            auto it = sources.find(ic.second.parent_id);
            if (it != sources.end() && sources.find(ic.second.num) == sources.end())
            {
+                if (!parent->check_image_perm(ic.second, true))
+                {
+                    result = (cli_result_t){ .err = EACCES, .text = "Image permission denied" };
+                    state = 100;
+                    return;
+                }
                merge_children.push_back(ic.second.num);
                if (ic.second.readonly || writers_stopped)
                {
@@ -487,13 +500,10 @@ resume_100:
            json11::Json::object {
                { "request_put", json11::Json::object {
                    { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/index/image/"+child_cfg->name) },
-                    { "value", base64_encode(json11::Json(merge_json_objects(
-                        parent->cli->st_cli.serialize_inode_cfg(&new_cfg),
-                        json11::Json::object{
-                            { "id", INODE_NO_POOL(inverse_parent) },
-                            { "pool_id", (uint64_t)INODE_POOL(inverse_parent) },
-                        }
-                    )).dump()) },
+                    { "value", base64_encode(json11::Json({
+                        { "id", INODE_NO_POOL(inverse_parent) },
+                        { "pool_id", (uint64_t)INODE_POOL(inverse_parent) },
+                    }).dump()) },
                } },
            },
        };
@@ -286,6 +286,8 @@ struct rm_inode_t
                    .data = data,
                };
            }
+            // Wakeup callers (otherwise they wake up only on next loop())
+            parent->ringloop->wakeup();
        }
        in_continue = false;
    }
@@ -35,31 +35,35 @@ struct cli_serve_path_t
 {
    std::string cmd;
    bool allow_get;
+    bool allow_client;
 };

 // Serve vitastor-cli commands over HTTP in JSON format
 struct cli_serve_t
 {
    std::map<std::string, cli_serve_path_t> cmd_paths = {
-        {"data/delete",   {"rm-data", false}},
-        {"data/describe", {"describe", true}},
-        {"data/fix",      {"fix", false}},
-        {"data/merge",    {"merge-data", false}},
-        {"image/create",  {"create", false}},
-        {"image/delete",  {"rm", false}},
-        {"image/flatten", {"flatten", false}},
-        {"image/list",    {"ls", true}},
-        {"image/modify",  {"modify", false}},
-        {"osd/alloc",     {"alloc-osd", false}},
-        {"osd/delete",    {"rm-osd", false}},
-        {"osd/list",      {"ls-osd", true}},
-        {"osd/modify",    {"modify-osd", false}},
-        {"pg/list",       {"ls-pgs", true}},
-        {"pool/create",   {"create-pool", false}},
-        {"pool/delete",   {"rm-pool", false}},
-        {"pool/list",     {"pools", true}},
-        {"pool/modify",   {"modify-pool", false}},
-        {"status",        {"status", true}},
+        {"data/delete",   {"rm-data",     false, false}},
+        {"data/describe", {"describe",    true,  false}},
+        {"data/fix",      {"fix",         false, false}},
+        {"data/merge",    {"merge-data",  false, false}},
+        {"image/create",  {"create",      false, true}},
+        {"image/delete",  {"rm",          false, true}},
+        {"image/flatten", {"flatten",     false, true}},
+        {"image/list",    {"ls",          true,  true}},
+        {"image/modify",  {"modify",      false, true}},
+        {"osd/alloc",     {"alloc-osd",   false, false}},
+        {"osd/delete",    {"rm-osd",      false, false}},
+        {"osd/list",      {"ls-osd",      true,  false}},
+        {"osd/modify",    {"modify-osd",  false, false}},
+        {"pg/list",       {"ls-pgs",      true,  false}},
+        {"pool/create",   {"create-pool", false, false}},
+        {"pool/delete",   {"rm-pool",     false, false}},
+        {"pool/list",     {"pools",       true,  false}},
+        {"pool/modify",   {"modify-pool", false, false}},
+        {"user/delete",   {"remove-user", false, false}},
+        {"user/list",     {"ls-user",     false, false}},
+        {"user/modify",   {"modify-user", false, false}},
+        {"status",        {"status",      true,  false}},
    };

    cli_tool_t *parent = NULL;
@@ -200,6 +204,12 @@ struct cli_serve_t
            if (text)
                *text = "Bad Request";
        }
+        else if (err == EACCES)
+        {
+            code = 403;
+            if (text)
+                *text = "Forbidden";
+        }
        else if (err == EOPNOTSUPP)
        {
            code = 404;
@@ -334,6 +344,18 @@ struct cli_serve_t
        conn->request_path = std::move(req_line[1]);
        conn->request_body = std::move(msg->body);
        conn->response_type = "";
+        if (parent->cli->st_cli.use_auth)
+        {
+            auto user = std::make_unique<cli_user_t>();
+            user->name = msg->headers["_tls_common_name"];
+            auto user_it = parent->cli->st_cli.user_info.find(user->name);
+            auto userinfo = user_it == parent->cli->st_cli.user_info.end() ? user_it->second : json11::Json();
+            user->type = user->name == "root" ? "admin" : userinfo["type"].string_value();
+            for (auto & gr: userinfo["groups"].array_items())
+            {
+                user->groups.insert(gr.string_value());
+            }
+        }
        auto ctype = msg->headers["content-type"];
        if (conn->request_method != "GET" && conn->request_method != "POST")
        {
@@ -361,6 +383,32 @@ struct cli_serve_t
            {
                conn->response_type = "application/json";
                conn->result = { .text = openapi_description };
+                if (parent->cli->st_cli.use_auth)
+                {
+                    // Filter available paths by privileges
+                    if (conn->p->user->type == "client")
+                    {
+                        std::string error;
+                        auto openapi = json11::Json::parse(openapi_description, error).object_items();
+                        json11::Json::object paths;
+                        for (auto & kv: openapi["paths"].object_items())
+                        {
+                            auto cmd_it = cmd_paths.find(kv.first.substr(1));
+                            if (cmd_it != cmd_paths.end() && cmd_it->second.allow_client)
+                            {
+                                paths[kv.first] = kv.second;
+                            }
+                        }
+                        openapi["paths"] = paths;
+                        conn->response_type = "application/json";
+                        conn->result = { .text = json11::Json(openapi).dump() };
+                    }
+                    else if (conn->p->user->type != "admin")
+                    {
+                        conn->response_type = "";
+                        conn->result = { .err = EACCES, .text = "Access denied" };
+                    }
+                }
            }
            else if (cmd_it == cmd_paths.end())
            {
@@ -370,6 +418,10 @@ struct cli_serve_t
            {
                conn->result = { .err = ENOSYS, .text = "method /"+uri[0]+" only allows POST requests" };
            }
+            else if (parent->cli->st_cli.use_auth && conn->p->user->type == "client" && !cmd_it->second.allow_client)
+            {
+                conn->result = { .err = EACCES, .text = "Access denied" };
+            }
            else
            {
                std::string error;
@@ -0,0 +1,128 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 (see README.md for details)
+
+#include <algorithm>
+#include "cli.h"
+#include "cluster_client.h"
+#include "str_util.h"
+#include "json_util.h"
+
+// List users
+struct user_lister_t
+{
+    cli_tool_t *parent;
+    std::vector<std::string> only_names;
+
+    int state = 0;
+    cli_result_t result;
+    json11::Json::array users;
+
+    bool is_done()
+    {
+        return state == 100;
+    }
+
+    void loop()
+    {
+        if (state == 1)
+            goto resume_1;
+        if (state == 100)
+            return;
+        {
+            json11::Json::array select;
+            if (!only_names.size())
+            {
+                select.push_back(json11::Json::object {
+                    { "request_range", json11::Json::object {
+                        { "key", base64_encode(
+                            parent->cli->st_cli.etcd_prefix+"/config/user/"
+                        ) },
+                        { "range_end", base64_encode(
+                            parent->cli->st_cli.etcd_prefix+"/config/user0"
+                        ) },
+                    } },
+                });
+            }
+            else
+            {
+                for (auto & name: only_names)
+                {
+                    select.push_back(json11::Json::object {
+                        { "request_range", json11::Json::object {
+                            { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/user/"+name) },
+                        } }
+                    });
+                }
+            }
+            parent->etcd_txn(json11::Json::object {
+                { "success", select },
+            });
+        }
+        state = 1;
+resume_1:
+        if (parent->waiting > 0)
+            return;
+        if (parent->etcd_err.err)
+        {
+            result = parent->etcd_err;
+            state = 100;
+            return;
+        }
+        for (auto & response: parent->etcd_result["responses"].array_items())
+        {
+            for (auto & kv_item: response["response_range"]["kvs"].array_items())
+            {
+                auto kv = parent->cli->st_cli.parse_etcd_kv(kv_item);
+                auto user = kv.value.object_items();
+                user["name"] = kv.key.substr(parent->cli->st_cli.etcd_prefix.size()+13);
+                if (!parent->json_output)
+                    user["groups_fmt"] = implode(",", user["groups"].array_items());
+                users.push_back(std::move(user));
+            }
+        }
+        if (parent->json_output)
+        {
+            // JSON output
+            result.data = users;
+            state = 100;
+            return;
+        }
+        // Table output: name, type, groups
+        json11::Json::array cols;
+        cols.push_back(json11::Json::object{
+            { "key", "name" },
+            { "title", "NAME" },
+        });
+        cols.push_back(json11::Json::object{
+            { "key", "type" },
+            { "title", "TYPE" },
+        });
+        cols.push_back(json11::Json::object{
+            { "key", "groups_fmt" },
+            { "title", "GROUPS" },
+        });
+        result.text = print_table(users, cols, parent->color);
+        state = 100;
+    }
+};
+
+std::function<bool(cli_result_t &)> cli_tool_t::start_user_ls(json11::Json cfg)
+{
+    auto lister = new user_lister_t();
+    lister->parent = this;
+    if (cfg["names"].is_string())
+        lister->only_names.push_back(cfg["names"].string_value());
+    for (auto & item: cfg["names"].array_items())
+        lister->only_names.push_back(item.string_value());
+    return [lister](cli_result_t & result)
+    {
+        lister->loop();
+        if (lister->is_done())
+        {
+            result = lister->result;
+            delete lister;
+            return true;
+        }
+        return false;
+    };
+}
@@ -108,12 +108,16 @@
                            { "type": "string", "enum": [ "", "random" ] },
                            { "type": "string", "pattern": "^[0-9a-fA-F]{128}$|^vault:" }
                        ]
-                    }
+                    },
+                    "owner": { "type": "string", "description": "Set image owner" },
+                    "owner_group": { "type": "string", "description": "Set image owner group" },
+                    "reader_group": { "type": "string", "description": "Set image reader group" }
                }
            } } } },
            "responses": {
                "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Image" } } } },
                "400": { "$ref": "#/components/responses/Invalid" },
+                "403": { "$ref": "#/components/responses/Forbidden" },
                "409": { "$ref": "#/components/responses/UpdateConflict" },
                "412": { "$ref": "#/components/responses/PreconditionFailed" }
            }
@@ -146,6 +150,9 @@
                            { "type": "string", "pattern": "^[0-9a-fA-F]{128}$|^vault:" }
                        ]
                    },
+                    "owner": { "type": "string", "description": "Set image owner" },
+                    "owner_group": { "type": "string", "description": "Set image owner group" },
+                    "reader_group": { "type": "string", "description": "Set image reader group" },
                    "force": { "type": "boolean", "description": "Proceed with shrinking or setting readwrite flag even if the image has children" },
                    "down_ok": { "type": "boolean", "description": "Proceed with shrinking even if some data will be left on unavailable OSDs" }
                }
@@ -153,6 +160,7 @@
            "responses": {
                "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Image" } } } },
                "400": { "$ref": "#/components/responses/Invalid" },
+                "403": { "$ref": "#/components/responses/Forbidden" },
                "409": { "$ref": "#/components/responses/UpdateConflict" },
                "412": { "$ref": "#/components/responses/PreconditionFailed" }
            }
@@ -202,6 +210,7 @@
                    }
                } } } },
                "400": { "$ref": "#/components/responses/Invalid" },
+                "403": { "$ref": "#/components/responses/Forbidden" },
                "409": { "$ref": "#/components/responses/UpdateConflict" },
                "412": { "$ref": "#/components/responses/PreconditionFailed" }
            }
@@ -221,6 +230,7 @@
            "responses": {
                "200": { "content": { "text/plain": { "schema": { "type": "string", "description": "Empty response" } } } },
                "400": { "$ref": "#/components/responses/Invalid" },
+                "403": { "$ref": "#/components/responses/Forbidden" },
                "409": { "$ref": "#/components/responses/UpdateConflict" },
                "412": { "$ref": "#/components/responses/PreconditionFailed" }
            }
@@ -413,7 +423,7 @@
            }
        } },
        "/pool/delete": { "post": {
-            "summary": "Delete an pool",
+            "summary": "Delete a pool",
            "operationId": "poolDelete",
            "consumes": [ "application/json" ],
            "produces": [ "application/json" ],
@@ -435,6 +445,66 @@
                "412": { "$ref": "#/components/responses/PreconditionFailed" }
            }
        } },
+        "/user/list": { "get": {
+            "summary": "List users",
+            "operationId": "userListGet",
+            "produces": [ "application/json" ],
+            "parameters": [ {
+                "name": "params",
+                "in": "query",
+                "schema": { "$ref": "#/components/schemas/UserListParams" },
+                "style": "form",
+                "explode": true
+            } ],
+            "responses": {
+                "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/UserList" } } } }
+            }
+        }, "post": {
+            "summary": "List users",
+            "operationId": "userListPost",
+            "consumes": [ "application/json" ],
+            "produces": [ "application/json" ],
+            "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/UserListParams" } } } },
+            "responses": {
+                "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/UserList" } } } }
+            }
+        } },
+        "/user/modify": { "post": {
+            "summary": "Modify a user",
+            "operationId": "userModify",
+            "consumes": [ "application/json" ],
+            "produces": [ "application/json" ],
+            "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/User" } } } },
+            "responses": {
+                "200": { "content": { "application/json": { "schema": {
+                    "$ref": "#/components/schemas/User"
+                } } } },
+                "400": { "$ref": "#/components/responses/Invalid" },
+                "409": { "$ref": "#/components/responses/UpdateConflict" },
+                "412": { "$ref": "#/components/responses/PreconditionFailed" }
+            }
+        } },
+        "/user/delete": { "post": {
+            "summary": "Delete a user",
+            "operationId": "userDelete",
+            "consumes": [ "application/json" ],
+            "produces": [ "application/json" ],
+            "requestBody": { "content": { "application/json": { "schema": {
+                "type": "object",
+                "properties": {
+                    "name": { "type": "string", "description": "User name to delete" }
+                }
+            } } } },
+            "responses": {
+                "200": { "content": { "application/json": { "schema": {
+                    "type": "object",
+                    "properties": { "ok": { "type": "boolean" } }
+                } } } },
+                "400": { "$ref": "#/components/responses/Invalid" },
+                "409": { "$ref": "#/components/responses/UpdateConflict" },
+                "412": { "$ref": "#/components/responses/PreconditionFailed" }
+            }
+        } },
        "/pg/list": { "get": {
            "summary": "List PGs",
            "operationId": "pgListGet",
@@ -605,6 +675,14 @@
                    }
                }
            },
+            "Forbidden": {
+                "description": "Access Denied (EACCES)",
+                "content": {
+                    "text/plain": {
+                        "schema": { "type": "string", "description": "Error text" }
+                    }
+                }
+            },
            "PreconditionFailed": {
                "description": "Precondition Failed (ENOTEMPTY, EEXIST or ENOENT)",
                "content": {
@@ -812,7 +890,8 @@
                    "scrub_interval": { "type": "string", "pattern": "^\\d+[smhdMy]$", "description": "Automatic scrub interval" },
                    "level_placement": { "type": "string", "description": "Additional failure domain rules" },
                    "raw_placement": { "type": "string", "description": "Raw PG generation rules" },
-                    "max_osd_combinations": { "type": "integer", "format": "uint64", "description": "Maximum number of random combinations during PG generation" }
+                    "max_osd_combinations": { "type": "integer", "format": "uint64", "description": "Maximum number of random combinations during PG generation" },
+                    "creator_group": { "type": "string", "description": "User group allowed to create images in this pool" }
                }
            },
            "PoolList": {
@@ -853,6 +932,24 @@
                    } } ]
                }
            },
+            "UserListParams": {
+                "type": "object",
+                "properties": {
+                    "names": { "type": "array", "items": { "type": "string" }, "description": "User names to list" }
+                }
+            },
+            "UserList": {
+                "type": "array",
+                "items": { "$ref": "#/components/schemas/User" }
+            },
+            "User": {
+                "type": "object",
+                "properties": {
+                    "name": { "type": "string", "description": "User name" },
+                    "type": { "type": "string", "enum": [ "osd", "mon", "admin", "client" ], "description": "User type" },
+                    "groups": { "type": "array", "items": { "type": "string" }, "description": "User group names" }
+                }
+            },
            "PgListParams": {
                "type": "object",
                "properties": {
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 2.8...3.30)

 project(vitastor)

@@ -7,7 +7,7 @@ add_executable(vitastor-disk
 	disk_tool.cpp disk_simple_offsets.cpp
 	disk_tool_discard.cpp disk_tool_journal.cpp disk_tool_meta.cpp disk_tool_prepare.cpp disk_tool_resize.cpp
 	disk_tool_resize_auto.cpp disk_tool_udev.cpp disk_tool_utils.cpp disk_tool_upgrade.cpp
-	../util/crc32c.c ../util/xxhash.c ../util/str_util.cpp ../util/json_util.cpp ../../json11/json11.cpp ../util/rw_blocking.cpp ../util/allocator.cpp ../util/ringloop.cpp
+	../util/crc32c.c ../util/xxh_x86dispatch.c ../util/str_util.cpp ../util/json_util.cpp ../../json11/json11.cpp ../util/rw_blocking.cpp ../util/allocator.cpp ../util/ringloop.cpp
 	../blockstore/blockstore_disk.cpp ../blockstore/blockstore_heap.cpp ../blockstore/multilist.cpp
 )
 target_link_libraries(vitastor-disk
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 2.8...3.30)

 project(vitastor)

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 2.8...3.30)

 project(vitastor)

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 2.8...3.30)

 project(vitastor)

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 2.8...3.30)

 project(vitastor)

@@ -530,6 +530,11 @@ void osd_t::exec_op(osd_op_t *cur_op)

 void osd_t::print_stats()
 {
+    if (bs && log_level > 1)
+    {
+        printf("[OSD %ju] Live entries: %ju (%ju bytes), garbage entries: %ju (%ju bytes)\n", osd_num,
+            bs->get_live_entries(), bs->get_live_memory(), bs->get_garbage_entries(), bs->get_garbage_memory());
+    }
    for (int i = OSD_OP_MIN; i <= OSD_OP_MAX; i++)
    {
        if (msgr.stats.op_stat_count[i] != prev_stats.op_stat_count[i] && i != OSD_OP_PING)
@@ -617,11 +622,11 @@ void osd_t::print_slow()
                bufprintf("[OSD %ju] Slow op %jx", osd_num, (uint64_t)op);
                if (kv.second->osd_num)
                {
-                    bufprintf(" from peer OSD %ju (client %d)", kv.second->osd_num, kv.second->peer_fd);
+                    bufprintf(" from peer OSD %ju (client %ju)", kv.second->osd_num, kv.second->client_id);
                }
                else
                {
-                    bufprintf(" from client %d", kv.second->peer_fd);
+                    bufprintf(" from client %ju", kv.second->client_id);
                }
                bufprintf(": %s id=%ju", osd_op_names[op->req.hdr.opcode], op->req.hdr.id);
                if (op->req.hdr.opcode == OSD_OP_SEC_READ || op->req.hdr.opcode == OSD_OP_SEC_WRITE ||
@@ -347,7 +347,7 @@ class osd_t
    bool check_write_queue(osd_op_t *cur_op, pg_t & pg);
    pg_osd_set_state_t* add_object_to_set(pg_t & pg, const object_id oid, const pg_osd_set_t & osd_set,
        uint64_t old_pg_state, int log_at_level);
-    void remove_object_from_state(object_id & oid, pg_osd_set_state_t **object_state, pg_t &pg, bool report = true);
+    bool remove_object_from_state(object_id & oid, pg_osd_set_state_t **object_state, pg_t &pg, bool report = true);
    pg_osd_set_state_t *mark_object(pg_t & pg, object_id oid, pg_osd_set_state_t *prev_object_state, bool ref,
        std::function<int(pg_osd_set_t & new_set)> calc_set);
    pg_osd_set_state_t *mark_object_corrupted(pg_t & pg, object_id oid, pg_osd_set_state_t *prev_object_state,
@@ -991,7 +991,7 @@ void osd_t::apply_pg_config()
                    // Add peers
                    for (auto pg_osd: all_peers)
                    {
-                        if (pg_osd != this->osd_num && msgr.osd_peer_fds.find(pg_osd) == msgr.osd_peer_fds.end())
+                        if (pg_osd != this->osd_num && msgr.osd_peers.find(pg_osd) == msgr.osd_peers.end())
                        {
                            msgr.connect_peer(pg_osd, st_cli.peer_states[pg_osd]);
                        }
@@ -4,6 +4,7 @@
 #include "osd.h"

 #define FLUSH_BATCH 512
+#define SELF_CLIENT 0

 void osd_t::submit_pg_flush_ops(pg_t & pg)
 {
@@ -91,11 +92,11 @@ void osd_t::handle_flush_op(bool rollback, pool_id_t pool_id, pg_num_t pg_num, p
        else
        {
            printf("Error while doing flush on OSD %ju: %d (%s)\n", osd_num, retval, strerror(-retval));
-            auto fd_it = msgr.osd_peer_fds.find(peer_osd);
-            if (fd_it != msgr.osd_peer_fds.end())
+            auto peer_it = msgr.osd_peers.find(peer_osd);
+            if (peer_it != msgr.osd_peers.end())
            {
                // Will repeer/stop this PG
-                msgr.stop_client(fd_it->second);
+                msgr.stop_client(peer_it->second->client_id);
            }
        }
    }
@@ -212,10 +213,10 @@ bool osd_t::submit_flush_op(pool_id_t pool_id, pg_num_t pg_num, pg_flush_batch_t
            handle_flush_op(op->req.hdr.opcode == OSD_OP_SEC_ROLLBACK, pool_id, pg_num, fb, peer_osd, op->reply.hdr.retval);
            delete op;
        };
-        auto peer_fd_it = msgr.osd_peer_fds.find(peer_osd);
-        if (peer_fd_it != msgr.osd_peer_fds.end())
+        auto peer_it = msgr.osd_peers.find(peer_osd);
+        if (peer_it != msgr.osd_peers.end())
        {
-            op->peer_fd = peer_fd_it->second;
+            op->client_id = peer_it->second->client_id;
            msgr.outbox_push(op);
        }
        else
@@ -307,7 +308,7 @@ void osd_t::submit_recovery_op(osd_recovery_op_t *op)
    {
        printf("Submitting recovery operation for %jx:%jx (%s)\n", op->oid.inode, op->oid.stripe, op->degraded ? "degraded" : "misplaced");
    }
-    op->osd_op->peer_fd = -1;
+    op->osd_op->client_id = SELF_CLIENT;
    op->osd_op->callback = [this, op](osd_op_t *osd_op)
    {
        ringloop->set_immediate([this, op]()
@@ -9,7 +9,7 @@
 #include "str_util.h"
 #include "osd.h"

-#define SELF_FD -1
+#define SELF_CLIENT 0

 // Peering loop
 void osd_t::handle_peers()
@@ -175,17 +175,17 @@ void osd_t::drop_dirty_pg_connections(pool_pg_num_t pg)
 {
    if (immediate_commit != IMMEDIATE_ALL)
    {
-        std::vector<int> to_stop;
+        std::vector<uint64_t> to_stop;
        for (auto & cp: msgr.clients)
        {
            if (cp.second->dirty_pgs.find(pg) != cp.second->dirty_pgs.end())
            {
-                to_stop.push_back(cp.first);
+                to_stop.push_back(cp.second->client_id);
            }
        }
-        for (auto peer_fd: to_stop)
+        for (auto client_id: to_stop)
        {
-            msgr.stop_client(peer_fd);
+            msgr.stop_client(client_id);
        }
    }
 }
@@ -203,7 +203,7 @@ void osd_t::start_pg_peering(pg_t & pg)
    for (auto pg_osd: pg.all_peers)
    {
        if (pg_osd != this->osd_num &&
-            msgr.osd_peer_fds.find(pg_osd) == msgr.osd_peer_fds.end())
+            msgr.osd_peers.find(pg_osd) == msgr.osd_peers.end())
        {
            if (msgr.wanted_peers.find(pg_osd) == msgr.wanted_peers.end())
                msgr.connect_peer(pg_osd, st_cli.peer_states[pg_osd]);
@@ -224,7 +224,7 @@ void osd_t::start_pg_peering(pg_t & pg)
    for (int role = 0; role < pg.target_set.size(); role++)
    {
        pg.cur_set[role] = pg.target_set[role] == this->osd_num ||
-            msgr.osd_peer_fds.find(pg.target_set[role]) != msgr.osd_peer_fds.end() ? pg.target_set[role] : 0;
+            msgr.osd_peers.find(pg.target_set[role]) != msgr.osd_peers.end() ? pg.target_set[role] : 0;
        if (pg.cur_set[role] != 0)
        {
            pg.pg_cursize++;
@@ -246,7 +246,7 @@ void osd_t::start_pg_peering(pg_t & pg)
    std::set<osd_num_t> dead_peers;
    for (auto pg_osd: pg.all_peers)
    {
-        if (pg_osd == this->osd_num || msgr.osd_peer_fds.find(pg_osd) != msgr.osd_peer_fds.end())
+        if (pg_osd == this->osd_num || msgr.osd_peers.find(pg_osd) != msgr.osd_peers.end())
            cur_peers.insert(pg_osd);
        else
            dead_peers.insert(pg_osd);
@@ -266,7 +266,7 @@ void osd_t::start_pg_peering(pg_t & pg)
                {
                    nonzero++;
                    if (history_osd == this->osd_num ||
-                        msgr.osd_peer_fds.find(history_osd) != msgr.osd_peer_fds.end())
+                        msgr.osd_peers.find(history_osd) != msgr.osd_peers.end())
                    {
                        found++;
                    }
@@ -435,8 +435,8 @@ void osd_t::relock_pg(pg_t & pg)
        bool unlock_peer = (i >= relock_osd_count);
        uint64_t new_state = unlock_peer ? 0 : pg.state;
        auto peer_osd = diff_osds[i];
-        auto peer_fd_it = msgr.osd_peer_fds.find(peer_osd);
-        if (peer_fd_it == msgr.osd_peer_fds.end())
+        auto peer_it = msgr.osd_peers.find(peer_osd);
+        if (peer_it == msgr.osd_peers.end())
        {
            if (unlock_peer)
            {
@@ -446,8 +446,7 @@ void osd_t::relock_pg(pg_t & pg)
            }
            continue;
        }
-        int peer_fd = peer_fd_it->second;
-        auto cl = msgr.clients.at(peer_fd);
+        auto cl = peer_it->second;
        if (!cl->enable_pg_locks)
        {
            // Peer does not support locking - just instantly remember the lock as successful
@@ -458,7 +457,7 @@ void osd_t::relock_pg(pg_t & pg)
        pg.inflight_locks++;
        osd_op_t *op = new osd_op_t();
        op->op_type = OSD_OP_OUT;
-        op->peer_fd = peer_fd;
+        op->client_id = cl->client_id;
        op->req = (osd_any_op_t){
            .sec_lock = {
                .header = {
@@ -529,7 +528,7 @@ void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
        // Self
        osd_op_t *op = new osd_op_t();
        op->op_type = 0;
-        op->peer_fd = SELF_FD;
+        op->client_id = SELF_CLIENT;
        clock_gettime(CLOCK_REALTIME, &op->tv_begin);
        op->bs_op = new blockstore_op_t();
        op->bs_op->opcode = BS_OP_LIST;
@@ -567,8 +566,8 @@ void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
    }
    else
    {
-        auto role_fd_it = msgr.osd_peer_fds.find(role_osd);
-        if (role_fd_it == msgr.osd_peer_fds.end())
+        auto peer_it = msgr.osd_peers.find(role_osd);
+        if (peer_it == msgr.osd_peers.end())
        {
            printf("Failed to get object list from OSD %ju because it is disconnected\n", role_osd);
            return;
@@ -576,7 +575,7 @@ void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
        // Peer
        osd_op_t *op = new osd_op_t();
        op->op_type = OSD_OP_OUT;
-        op->peer_fd = role_fd_it->second;
+        op->client_id = peer_it->second->client_id;
        op->req = (osd_any_op_t){
            .sec_list = {
                .header = {
@@ -595,10 +594,10 @@ void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
            if (op->reply.hdr.retval < 0)
            {
                printf("Failed to get object list from OSD %ju (retval=%jd), disconnecting peer\n", role_osd, op->reply.hdr.retval);
-                int fail_fd = op->peer_fd;
+                uint64_t fail_client_id = op->client_id;
                ps->list_ops.erase(role_osd);
                delete op;
-                msgr.stop_client(fail_fd);
+                msgr.stop_client(fail_client_id);
                return;
            }
            printf(
@@ -622,7 +621,7 @@ void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)

 void osd_t::discard_list_subop(osd_op_t *list_op)
 {
-    if (list_op->peer_fd == SELF_FD)
+    if (list_op->client_id == SELF_CLIENT)
    {
        // Self
        list_op->bs_op->callback = [list_op](blockstore_op_t *bs_op)
@@ -96,7 +96,7 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
            if (inode_it->second.parent_id == cur_op->req.rw.inode ||
                inode_it->second.parent_id == inode_it->second.num ||
                chain_size > st_cli.inode_config.size() ||
-                chain_size > 255)
+                chain_size > UINT32_MAX)
            {
                printf("Inode %ju from pool %u has too many parents, returning EINVAL in response to read\n",
                    INODE_NO_POOL(cur_op->req.rw.inode), INODE_POOL(cur_op->req.rw.inode));
@@ -111,7 +111,7 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
            // Add the original inode
            chain_size++;
            chain_info_len = (cur_op->req.rw.flags & OSD_OP_RETURN_CHAIN
-                ? (cur_op->req.rw.len / bs_bitmap_granularity)
+                ? osd_op_rw_t::chain_info_bytes(chain_size) * (cur_op->req.rw.len / bs_bitmap_granularity)
                : 0);
        }
    }
@@ -130,9 +130,7 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
            stripe_count * clean_entry_bitmap_size +
            // - 'missing' flags for chained reads
            (pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 0 : pg_it->second.pg_size)
-        ) +
-        // read chain info
-        chain_info_len
+        )
    );
    void *data_buf = (uint8_t*)op_data + sizeof(osd_primary_op_data_t);
    op_data->pg_num = pg_num;
@@ -140,6 +138,7 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
    op_data->oid = oid;
    op_data->stripes = (osd_rmw_stripe_t*)data_buf;
    op_data->stripe_count = stripe_count;
+    op_data->chain_info = NULL;
    data_buf = (uint8_t*)data_buf + sizeof(osd_rmw_stripe_t) * stripe_count;
    cur_op->op_data = op_data;
    if (cur_op->req.hdr.opcode != OSD_OP_SCRUB)
@@ -147,10 +146,17 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
        split_stripes(pg_data_size, bs_block_size, (uint32_t)(cur_op->req.rw.offset - oid.stripe), cur_op->req.rw.len, op_data->stripes);
        // Resulting bitmaps have to survive op_data and be freed with the op itself
        assert(!cur_op->bitmap_buf);
-        cur_op->bitmap_buf = calloc_or_die(1, clean_entry_bitmap_size * stripe_count);
+        cur_op->bitmap_buf = (uint8_t*)calloc_or_die(1, clean_entry_bitmap_size*stripe_count + chain_info_len);
+        uint8_t *buf = cur_op->bitmap_buf;
        for (int i = 0; i < stripe_count; i++)
        {
-            op_data->stripes[i].bmp_buf = (uint8_t*)cur_op->bitmap_buf + clean_entry_bitmap_size * i;
+            op_data->stripes[i].bmp_buf = buf;
+            buf += clean_entry_bitmap_size;
+            if (i == pg_data_size-1 && chain_info_len)
+            {
+                op_data->chain_info = buf;
+                buf += chain_info_len;
+            }
        }
    }
    op_data->chain_size = chain_size;
@@ -164,11 +170,6 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
        data_buf = (uint8_t*)data_buf + chain_size * stripe_count * clean_entry_bitmap_size;
        op_data->missing_flags = (uint8_t*)data_buf;
        data_buf = (uint8_t*)data_buf + chain_size * (pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 0 : pg_it->second.pg_size);
-        if (chain_info_len)
-        {
-            op_data->chain_info = (uint8_t*)data_buf;
-            data_buf = (uint8_t*)data_buf + chain_info_len;
-        }
        // Copy chain
        int chain_num = 0;
        op_data->read_chain[chain_num] = cur_op->req.rw.inode;
@@ -319,6 +320,7 @@ resume_2:
    }
    cur_op->reply.rw.version = op_data->fact_ver;
    cur_op->reply.rw.bitmap_len = (pg ? pg->pg_data_size : 1) * clean_entry_bitmap_size;
+    cur_op->bitmap = op_data->stripes[0].bmp_buf;
    if (op_data->degraded)
    {
        // Reconstruct missing stripes
@@ -331,7 +333,6 @@ resume_2:
        {
            reconstruct_stripes_ec(stripes, pg->pg_size, pg->pg_data_size, clean_entry_bitmap_size);
        }
-        cur_op->iov.push_back(op_data->stripes[0].bmp_buf, cur_op->reply.rw.bitmap_len);
        for (int role = 0; role < pg->pg_size; role++)
        {
            if (stripes[role].req_end != 0)
@@ -346,7 +347,6 @@ resume_2:
    }
    else
    {
-        cur_op->iov.push_back(op_data->stripes[0].bmp_buf, cur_op->reply.rw.bitmap_len);
        cur_op->iov.push_back(cur_op->buf, cur_op->req.rw.len);
    }
    finish_op(cur_op, cur_op->req.rw.len);
@@ -616,11 +616,11 @@ pg_osd_set_state_t* osd_t::add_object_to_set(pg_t & pg, const object_id oid, con
 }

 // Decrement pg_osd_set_state_t's object_count and change PG state accordingly
-void osd_t::remove_object_from_state(object_id & oid, pg_osd_set_state_t **object_state, pg_t & pg, bool report)
+bool osd_t::remove_object_from_state(object_id & oid, pg_osd_set_state_t **object_state, pg_t & pg, bool report)
 {
    if (!*object_state)
    {
-        return;
+        return false;
    }
    pg_osd_set_state_t *recheck_state = NULL;
    get_object_osd_set(pg, oid, &recheck_state);
@@ -629,7 +629,7 @@ void osd_t::remove_object_from_state(object_id & oid, pg_osd_set_state_t **objec
        recheck_state->ref_count++;
        (*object_state)->ref_count--;
        *object_state = recheck_state;
-        return;
+        return false;
    }
    bool changed = false;
    (*object_state)->object_count--;
@@ -704,6 +704,7 @@ void osd_t::remove_object_from_state(object_id & oid, pg_osd_set_state_t **objec
    {
        report_pg_state(pg);
    }
+    return changed;
 }

 void osd_t::deref_object_state(pg_t & pg, pg_osd_set_state_t **object_state, bool deref)
@@ -806,7 +807,7 @@ resume_5:
        {
            this->dirty_osds.insert(chunk.osd_num);
        }
-        for (auto cl_it = msgr.clients.find(cur_op->peer_fd); cl_it != msgr.clients.end(); )
+        for (auto cl_it = msgr.clients.find(cur_op->client_id); cl_it != msgr.clients.end(); )
        {
            cl_it->second->dirty_pgs.insert({ .pool_id = pg.pool_id, .pg_num = pg.pg_num });
            break;
@@ -56,7 +56,7 @@ struct osd_primary_op_data_t
            int chain_size;
            osd_chain_read_t *chain_reads;
            int chain_read_count;
-            uint8_t *chain_info;
+            void *chain_info;
        };
    };
 };
@@ -312,16 +312,16 @@ int osd_t::submit_bitmap_subops(osd_op_t *cur_op, pg_t & pg)
                    }
                    handle_primary_subop(subop, cur_op);
                };
-                auto peer_fd_it = msgr.osd_peer_fds.find(subop_osd_num);
-                if (peer_fd_it != msgr.osd_peer_fds.end())
+                auto peer_it = msgr.osd_peers.find(subop_osd_num);
+                if (peer_it != msgr.osd_peers.end())
                {
-                    subop->peer_fd = peer_fd_it->second;
+                    subop->client_id = peer_it->second->client_id;
                    msgr.outbox_push(subop);
                }
                else
                {
                    // Fail it immediately
-                    subop->peer_fd = -1;
+                    subop->client_id = 0;
                    subop->reply.hdr.retval = -EPIPE;
                    ringloop->set_immediate([subop]() { std::function<void(osd_op_t*)>(subop->callback)(subop); });
                }
@@ -582,11 +582,10 @@ void osd_t::send_chained_read_results(pg_t *pg, osd_op_t *cur_op)
        }
    }
    // Send bitmap
+    cur_op->bitmap = op_data->stripes[0].bmp_buf;
    cur_op->reply.rw.bitmap_len = (pg ? pg->pg_data_size : 1) * clean_entry_bitmap_size;
-    cur_op->iov.push_back(op_data->stripes[0].bmp_buf, cur_op->reply.rw.bitmap_len);
    if (cur_op->req.rw.flags & OSD_OP_RETURN_CHAIN)
    {
-        cur_op->iov.push_back(op_data->chain_info, (cur_op->req.rw.len / bs_bitmap_granularity));
        cur_op->reply.rw.bitmap_len += (cur_op->req.rw.len / bs_bitmap_granularity);
    }
    // And finally compose the result
@@ -596,6 +595,7 @@ void osd_t::send_chained_read_results(pg_t *pg, osd_op_t *cur_op)
    int prev = (cur_op->req.rw.offset - op_data->oid.stripe) / bs_bitmap_granularity;
    int end = prev + cur_op->req.rw.len/bs_bitmap_granularity;
    int cur = prev;
+    size_t key_index_bytes = osd_op_rw_t::chain_info_bytes(op_data->chain_size);
    while (cur <= end)
    {
        bool has_bit = false;
@@ -607,7 +607,14 @@ void osd_t::send_chained_read_results(pg_t *pg, osd_op_t *cur_op)
                if (has_bit)
                {
                    if (op_data->chain_info)
-                        op_data->chain_info[cur] = pos;
+                    {
+                        if (key_index_bytes == 1)
+                            ((uint8_t*)op_data->chain_info)[cur] = pos;
+                        else if (key_index_bytes == 2)
+                            ((uint16_t*)op_data->chain_info)[cur] = pos;
+                        else
+                            ((uint32_t*)op_data->chain_info)[cur] = pos;
+                    }
                    break;
                }
            }
@@ -616,7 +623,7 @@ void osd_t::send_chained_read_results(pg_t *pg, osd_op_t *cur_op)
        {
            if (cur > prev)
            {
-                // Send buffer in parts to avoid copying
+                 // Send buffer in parts to avoid copying
                if (!prev_set)
                {
                    while ((cur-prev) > zero_buffer_size/bs_bitmap_granularity)
@@ -3,7 +3,7 @@

 #include "osd_primary.h"

-#define SELF_FD -1
+#define SELF_CLIENT 0

 void osd_t::autosync()
 {
@@ -15,7 +15,7 @@ void osd_t::autosync()
        }
        autosync_op = new osd_op_t();
        autosync_op->op_type = OSD_OP_IN;
-        autosync_op->peer_fd = SELF_FD;
+        autosync_op->client_id = SELF_CLIENT;
        autosync_op->req = (osd_any_op_t){
            .sync = {
                .header = {
@@ -83,14 +83,12 @@ void osd_t::finish_op(osd_op_t *cur_op, int retval)
            rm_inflight(pg);
        }
        assert(!cur_op->op_data->subops);
-        free(cur_op->op_data);
-        cur_op->op_data = NULL;
    }
    cur_op->reply.hdr.magic = SECONDARY_OSD_REPLY_MAGIC;
    cur_op->reply.hdr.id = cur_op->req.hdr.id;
    cur_op->reply.hdr.opcode = cur_op->req.hdr.opcode;
    cur_op->reply.hdr.retval = retval;
-    if (cur_op->peer_fd == SELF_FD)
+    if (cur_op->client_id == SELF_CLIENT)
    {
        // Do not include internal primary writes (recovery/rebalance) into client op statistics
        if (cur_op->req.hdr.opcode != OSD_OP_WRITE)
@@ -103,7 +101,7 @@ void osd_t::finish_op(osd_op_t *cur_op, int retval)
    else
    {
        // FIXME add separate magic number for primary ops
-        auto cl_it = msgr.clients.find(cur_op->peer_fd);
+        auto cl_it = msgr.clients.find(cur_op->client_id);
        if (cl_it != msgr.clients.end())
        {
            msgr.outbox_push(cur_op);
@@ -243,7 +241,7 @@ void osd_t::submit_primary_subop(osd_op_t *cur_op, osd_op_t *subop,
            .offset = wr ? si->write_start : si->read_start,
            .len = subop_len,
            .attr_len = wr ? clean_entry_bitmap_size : 0,
-            .flags = cur_op->peer_fd == SELF_FD && cur_op->req.hdr.opcode != OSD_OP_SCRUB ? OSD_OP_RECOVERY_RELATED : 0,
+            .flags = cur_op->client_id == SELF_CLIENT && cur_op->req.hdr.opcode != OSD_OP_SCRUB ? OSD_OP_RECOVERY_RELATED : 0,
        };
 #ifdef OSD_DEBUG
        printf(
@@ -270,16 +268,16 @@ void osd_t::submit_primary_subop(osd_op_t *cur_op, osd_op_t *subop,
        {
            handle_primary_subop(subop, cur_op);
        };
-        auto peer_fd_it = msgr.osd_peer_fds.find(si->osd_num);
-        if (peer_fd_it != msgr.osd_peer_fds.end())
+        auto peer_it = msgr.osd_peers.find(si->osd_num);
+        if (peer_it != msgr.osd_peers.end())
        {
-            subop->peer_fd = peer_fd_it->second;
+            subop->client_id = peer_it->second->client_id;
            msgr.outbox_push(subop);
        }
        else
        {
            // Fail it immediately
-            subop->peer_fd = -1;
+            subop->client_id = 0;
            subop->reply.hdr.retval = -EPIPE;
            ringloop->set_immediate([subop]() { std::function<void(osd_op_t*)>(subop->callback)(subop); });
        }
@@ -326,7 +324,7 @@ void osd_t::handle_primary_bs_subop(osd_op_t *subop)
        }
        throw std::runtime_error("local blockstore modification failed");
    }
-    bool recovery_related = cur_op->peer_fd == SELF_FD && cur_op->req.hdr.opcode != OSD_OP_SCRUB;
+    bool recovery_related = cur_op->client_id == SELF_CLIENT && cur_op->req.hdr.opcode != OSD_OP_SCRUB;
    add_bs_subop_stats(subop, recovery_related);
    subop->req.hdr.opcode = bs_op_to_osd_op[bs_op->opcode];
    subop->reply.hdr.retval = bs_op->retval;
@@ -339,7 +337,7 @@ void osd_t::handle_primary_bs_subop(osd_op_t *subop)
    }
    delete bs_op;
    subop->bs_op = NULL;
-    subop->peer_fd = SELF_FD;
+    subop->client_id = SELF_CLIENT;
    if (recovery_related && recovery_target_sleep_us)
    {
        tfd->set_timer_us(recovery_target_sleep_us, false, [=](int timer_id)
@@ -399,11 +397,15 @@ void osd_t::handle_primary_subop(osd_op_t *subop, osd_op_t *cur_op)
    {
        uint64_t version = subop->reply.sec_rw.version;
 #ifdef OSD_DEBUG
-        int64_t peer_osd = subop->peer_fd == SELF_FD ? osd_num :
-            (msgr.clients.find(subop->peer_fd) != msgr.clients.end()
-                ? msgr.clients[subop->peer_fd]->osd_num : -subop->peer_fd);
-        printf("subop %s %jx:%jx from osd %jd: version = %ju\n", osd_op_names[opcode],
-            subop->req.sec_rw.oid.inode, subop->req.sec_rw.oid.stripe, peer_osd, version);
+        if (subop->client_id == SELF_CLIENT)
+            printf("subop %s %jx:%jx from local: version = %ju\n", osd_op_names[opcode],
+                subop->req.sec_rw.oid.inode, subop->req.sec_rw.oid.stripe, version);
+        else if (msgr.clients.find(subop->client_id) != msgr.clients.end())
+            printf("subop %s %jx:%jx from osd %ju: version = %ju\n", osd_op_names[opcode],
+                subop->req.sec_rw.oid.inode, subop->req.sec_rw.oid.stripe, msgr.clients.at(subop->client_id)->osd_num, version);
+        else
+            printf("subop %s %jx:%jx from client %ju: version = %ju\n", osd_op_names[opcode],
+                subop->req.sec_rw.oid.inode, subop->req.sec_rw.oid.stripe, subop->client_id, version);
 #endif
        if (version != 0 && op_data->fact_ver != UINT64_MAX)
        {
@@ -422,16 +424,16 @@ void osd_t::handle_primary_subop(osd_op_t *subop, osd_op_t *cur_op)
    }
    if (retval != expected)
    {
-        int64_t peer_osd = (msgr.clients.find(subop->peer_fd) != msgr.clients.end()
-            ? msgr.clients[subop->peer_fd]->osd_num : 0);
+        int64_t peer_osd = (msgr.clients.find(subop->client_id) != msgr.clients.end()
+            ? msgr.clients.at(subop->client_id)->osd_num : 0);
        if (opcode == OSD_OP_SEC_READ || opcode == OSD_OP_SEC_WRITE || opcode == OSD_OP_SEC_WRITE_STABLE)
        {
            printf("%s subop to %jx:%jx v%ju failed ", osd_op_names[opcode],
                subop->req.sec_rw.oid.inode, subop->req.sec_rw.oid.stripe, subop->req.sec_rw.version);
-            if (subop->peer_fd >= 0 && peer_osd > 0)
+            if (subop->client_id && peer_osd > 0)
                printf("on osd %ju: retval = %d (expected %d)\n", peer_osd, retval, expected);
-            else if (peer_osd > 0)
-                printf("on peer %d: retval = %d (expected %d)\n", subop->peer_fd, retval, expected);
+            else if (subop->client_id)
+                printf("on client %ju: retval = %d (expected %d)\n", subop->client_id, retval, expected);
            else
                printf("locally: retval = %d (expected %d)\n", retval, expected);
        }
@@ -446,7 +448,7 @@ void osd_t::handle_primary_subop(osd_op_t *subop, osd_op_t *cur_op)
        else
        {
            printf(
-                "%s subop failed on osd %jd: retval = %d (expected %d)\n",
+                "%s subop failed on osd %ju: retval = %d (expected %d)\n",
                osd_op_names[opcode], peer_osd, retval, expected
            );
        }
@@ -460,12 +462,12 @@ void osd_t::handle_primary_subop(osd_op_t *subop, osd_op_t *cur_op)
        {
            op_data->errcode = retval;
        }
-        if (subop->peer_fd >= 0 && retval != -EDOM && retval != -ERANGE &&
+        if (subop->client_id && retval != -EDOM && retval != -ERANGE &&
            (retval != -ENOSPC || opcode != OSD_OP_SEC_WRITE && opcode != OSD_OP_SEC_WRITE_STABLE) &&
            (retval != -EIO || opcode != OSD_OP_SEC_READ))
        {
            // Drop connection on unexpected errors
-            msgr.stop_client(subop->peer_fd);
+            msgr.stop_client(subop->client_id);
            op_data->drops++;
        }
        // Increase op_data->errors after stop_client to prevent >= n_subops running twice
@@ -606,22 +608,22 @@ void osd_t::submit_primary_del_batch(osd_op_t *cur_op, obj_ver_osd_t *chunks_to_
                },
                .oid = chunk.oid,
                .version = chunk.version,
-                .flags = cur_op->peer_fd == SELF_FD && cur_op->req.hdr.opcode != OSD_OP_SCRUB ? OSD_OP_RECOVERY_RELATED : 0,
+                .flags = cur_op->client_id == SELF_CLIENT && cur_op->req.hdr.opcode != OSD_OP_SCRUB ? OSD_OP_RECOVERY_RELATED : 0,
            } };
            subops[i].callback = [cur_op, this](osd_op_t *subop)
            {
                handle_primary_subop(subop, cur_op);
            };
-            auto peer_fd_it = msgr.osd_peer_fds.find(chunk.osd_num);
-            if (peer_fd_it != msgr.osd_peer_fds.end())
+            auto peer_it = msgr.osd_peers.find(chunk.osd_num);
+            if (peer_it != msgr.osd_peers.end())
            {
-                subops[i].peer_fd = peer_fd_it->second;
+                subops[i].client_id = peer_it->second->client_id;
                msgr.outbox_push(&subops[i]);
            }
            else
            {
                // Fail it immediately
-                subops[i].peer_fd = -1;
+                subops[i].client_id = 0;
                subops[i].reply.hdr.retval = -EPIPE;
                ringloop->set_immediate([subop = &subops[i]]() { std::function<void(osd_op_t*)>(subop->callback)(subop); });
            }
@@ -637,7 +639,7 @@ int osd_t::submit_primary_sync_subops(osd_op_t *cur_op)
    op_data->done = op_data->errors = op_data->errcode = 0;
    op_data->n_subops = n_osds;
    op_data->subops = subops;
-    std::map<uint64_t, int>::iterator peer_it;
+    robin_hood::unordered_flat_map<uint64_t, osd_client_t*>::iterator peer_it;
    for (int i = 0; i < n_osds; i++)
    {
        osd_num_t sync_osd = op_data->dirty_osds[i];
@@ -654,16 +656,16 @@ int osd_t::submit_primary_sync_subops(osd_op_t *cur_op)
            });
            bs->enqueue_op(subops[i].bs_op);
        }
-        else if ((peer_it = msgr.osd_peer_fds.find(sync_osd)) != msgr.osd_peer_fds.end())
+        else if ((peer_it = msgr.osd_peers.find(sync_osd)) != msgr.osd_peers.end())
        {
            subops[i].op_type = OSD_OP_OUT;
-            subops[i].peer_fd = peer_it->second;
+            subops[i].client_id = peer_it->second->client_id;
            subops[i].req = (osd_any_op_t){ .sec_sync = {
                .header = {
                    .magic = SECONDARY_OSD_OP_MAGIC,
                    .opcode = OSD_OP_SEC_SYNC,
                },
-                .flags = cur_op->peer_fd == SELF_FD && cur_op->req.hdr.opcode != OSD_OP_SCRUB ? OSD_OP_RECOVERY_RELATED : 0,
+                .flags = cur_op->client_id == SELF_CLIENT && cur_op->req.hdr.opcode != OSD_OP_SCRUB ? OSD_OP_RECOVERY_RELATED : 0,
            } };
            subops[i].callback = [cur_op, this](osd_op_t *subop)
            {
@@ -722,23 +724,23 @@ void osd_t::submit_primary_stab_subops(osd_op_t *cur_op)
                    .opcode = OSD_OP_SEC_STABILIZE,
                },
                .len = (uint64_t)(stab_osd.len * sizeof(obj_ver_id)),
-                .flags = cur_op->peer_fd == SELF_FD && cur_op->req.hdr.opcode != OSD_OP_SCRUB ? OSD_OP_RECOVERY_RELATED : 0,
+                .flags = cur_op->client_id == SELF_CLIENT && cur_op->req.hdr.opcode != OSD_OP_SCRUB ? OSD_OP_RECOVERY_RELATED : 0,
            } };
            subops[i].iov.push_back(op_data->unstable_writes + stab_osd.start, stab_osd.len * sizeof(obj_ver_id));
            subops[i].callback = [cur_op, this](osd_op_t *subop)
            {
                handle_primary_subop(subop, cur_op);
            };
-            auto peer_fd_it = msgr.osd_peer_fds.find(stab_osd.osd_num);
-            if (peer_fd_it != msgr.osd_peer_fds.end())
+            auto peer_it = msgr.osd_peers.find(stab_osd.osd_num);
+            if (peer_it != msgr.osd_peers.end())
            {
-                subops[i].peer_fd = peer_fd_it->second;
+                subops[i].client_id = peer_it->second->client_id;
                msgr.outbox_push(&subops[i]);
            }
            else
            {
                // Fail it immediately
-                subops[i].peer_fd = -1;
+                subops[i].client_id = 0;
                subops[i].reply.hdr.retval = -EPIPE;
                ringloop->set_immediate([subop = &subops[i]]() { std::function<void(osd_op_t*)>(subop->callback)(subop); });
            }
@@ -756,7 +758,7 @@ void osd_t::submit_primary_rollback_subops(osd_op_t *cur_op, const uint64_t* osd
    for (int role = 0; role < op_data->pg->pg_size; role++)
    {
        if (osd_set[role] != 0 && !stripes[role].read_error &&
-            (osd_set[role] == this->osd_num || msgr.osd_peer_fds.find(osd_set[role]) != msgr.osd_peer_fds.end()))
+            (osd_set[role] == this->osd_num || msgr.osd_peers.find(osd_set[role]) != msgr.osd_peers.end()))
        {
            n_subops++;
        }
@@ -773,7 +775,7 @@ void osd_t::submit_primary_rollback_subops(osd_op_t *cur_op, const uint64_t* osd
    for (int role = 0; role < op_data->pg->pg_size; role++)
    {
        if (osd_set[role] != 0 && !stripes[role].read_error &&
-            (osd_set[role] == this->osd_num || msgr.osd_peer_fds.find(osd_set[role]) != msgr.osd_peer_fds.end()))
+            (osd_set[role] == this->osd_num || msgr.osd_peers.find(osd_set[role]) != msgr.osd_peers.end()))
        {
            osd_op_t *subop = &op_data->subops[i];
            op_data->unstable_writes[i] = (obj_ver_id){
@@ -827,7 +829,7 @@ void osd_t::submit_primary_rollback_subops(osd_op_t *cur_op, const uint64_t* osd
                    op_data->oid.inode, op_data->oid.stripe | role, op_data->target_ver-1
                );
 #endif
-                subop->peer_fd = msgr.osd_peer_fds.at(osd_set[role]);
+                subop->client_id = msgr.osd_peers.at(osd_set[role])->client_id;
                msgr.outbox_push(subop);
            }
            i++;
@@ -247,9 +247,9 @@ resume_8:
    else
    {
 finish:
-        if (cur_op->peer_fd)
+        if (cur_op->client_id)
        {
-            auto it = msgr.clients.find(cur_op->peer_fd);
+            auto it = msgr.clients.find(cur_op->client_id);
            if (it != msgr.clients.end())
                it->second->dirty_pgs.clear();
        }
@@ -276,6 +276,7 @@ resume_12:
    {
        // Any kind of a non-clean object can have extra chunks, because we don't record objects
        // as degraded & misplaced or incomplete & misplaced at the same time. So try to remove extra chunks
+        bool changed = false;
        if (immediate_commit != IMMEDIATE_ALL)
        {
            // We can't remove extra chunks yet if fsyncs are explicit, because
@@ -303,13 +304,21 @@ resume_12:
            {
                // PG can't be active+clean until extra copies aren't removed, so mark it as PG_HAS_MISPLACED
                pg.state |= PG_HAS_MISPLACED;
-                //this->pg_state_dirty.insert({ .pool_id = pg.pool_id, .pg_num = pg.pg_num });
+                changed = true;
+                this->pg_state_dirty.insert({ .pool_id = pg.pool_id, .pg_num = pg.pg_num });
            }
        }
        // We must forget the unclean state of the object before deleting it
        // so the next reads don't accidentally read a deleted version
        // And it should be done at the same time as the removal of the version override
-        remove_object_from_state(op_data->oid, &op_data->object_state, pg);
+        if (remove_object_from_state(op_data->oid, &op_data->object_state, pg, false))
+        {
+            changed = true;
+        }
+        if (changed)
+        {
+            report_pg_state(pg);
+        }
        pg.clean_count++;
    }
 resume_6:
@@ -536,7 +545,7 @@ lazy:
        }
        // Remember PG as dirty to drop the connection when PG goes offline
        // (this is required because of the "lazy sync")
-        auto cl_it = msgr.clients.find(cur_op->peer_fd);
+        auto cl_it = msgr.clients.find(cur_op->client_id);
        if (cl_it != msgr.clients.end())
        {
            cl_it->second->dirty_pgs.insert({ .pool_id = pg.pool_id, .pg_num = pg.pg_num });
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Vitaliy Filippov	62826a4627	WIP Batch encrypted records to reduce EncryptInit/EncryptFinal costs	2026-04-19 12:29:27 +00:00
Vitaliy Filippov	35e7bc8aeb	Try to run tests with AES-GCM	2026-04-18 20:41:49 +03:00
Vitaliy Filippov	9a5fabddb4	Implement direct AES-256-GCM with a static key for benchmark	2026-04-18 20:41:49 +03:00
Vitaliy Filippov	088e85a423	Make sure to send all TLS data before continuing	2026-04-18 14:22:42 +03:00
Vitaliy Filippov	43f1d58f29	Parse standard TLS record headers	2026-04-17 21:22:47 +03:00
Vitaliy Filippov	a6b29a01a2	Omit msgr_tls_record_hdr_t for non-tls data	2026-04-17 21:22:47 +03:00
Vitaliy Filippov	5d65d87dfb	Allow 2 and 4 byte per block chain_info (allow more than 255 snapshots with encryption)	2026-04-17 21:22:47 +03:00
Vitaliy Filippov	ddf28feaf8	Implement OSD TLS support	2026-04-17 21:22:47 +03:00
Vitaliy Filippov	a57049be63	Allow to skip checksums for headers	2026-04-17 13:53:41 +03:00
Vitaliy Filippov	7f53e315c5	Implement protocol-level checksums (xxhash3)	2026-04-17 13:53:41 +03:00
Vitaliy Filippov	48756520dd	Include xxhash3 x86dispatch	2026-04-17 13:53:41 +03:00
Vitaliy Filippov	4f23b242f3	Do not use scrap_buffer in the client (it would block protocol checksum support)	2026-04-17 13:53:41 +03:00
Vitaliy Filippov	c371b74e12	Support TLS CN authentication and per-image permissions in vitastor-cli serve	2026-04-17 13:53:41 +03:00
Vitaliy Filippov	c014d20fca	Add VitastorAuthFilter	2026-04-17 13:53:41 +03:00
Vitaliy Filippov	9b2480d552	Implement vitastor-cli ls-user, modify-user, remove-user commands	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	1ae10c21ca	Add image owner/owner_group/reader_group support (for antietcd VitastorAuthFilter)	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	4698009b59	Add security parameter documentation	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	d0e0b70f81	Support inline (string PEM) certificates and pkeys	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	914f42da5c	Show encryption keys (only IDs) in the listing	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	273b641820	Support storing image encryption keys in Vault	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	4f9de7a6fb	Prefer local etcd addresses and correctly cycle over them even when they need resolving Seems slightly overcomplicated...	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	d3c529abd7	Support DNS resolving via libc-ares	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	a8e19ba28a	Batch handle_immediate_ops more	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	64a8cd1f4b	Add vitastor-cli create & modify --enc-key parameter	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	cc6e531410	Support reading from snapshots encrypted with different keys	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	ca608d0b87	Support decryption with multiple keys	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	0e095d2347	Allow to return chain_info in response to reads	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	4fe2a0a3eb	Add basic AES-XTS client-side encryption support	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	1a4d275616	Rework msgr send/receive to allow encryption support	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	803c870493	Move fromhexstr() to str_util	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	0d97fba466	Add openapi description	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	41100260ec	Slightly fix API return and input types	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	07c9606594	Implement vitastor-cli serve command to serve simple HTTP API	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	4ba361d83d	Implement HTTP server support O_o	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	662fb86eae	Rename http_response_t to http_message_t	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	1df8b51abb	Extract common HTTP context	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	d2d01e5183	Support xxhash 32-bit checksums (data_csum_type=xxh3_32)	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	3d96c3907e	Detect block checksums using csum_block_size, not data_csum_type	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	fb483185ff	Add client certificate support	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	d6eb00e18e	Do not re-initialize TLS context every connection	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	34d47bd62a	Add https support to antietcd	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	2846eba3af	Implement etcd SSL support via OpenSSL Maybe I should remove all of this and use libwebsockets :)	2026-04-17 13:53:40 +03:00
Vitaliy Filippov	a76d5ccc0d	Wakeup callers in rm-data	2026-04-17 13:53:32 +03:00
Vitaliy Filippov	8ed1e180e0	Clear inode_stats in mon	2026-04-14 02:44:46 +03:00
Vitaliy Filippov	8832fc3b14	Fix print to stdout in client	2026-04-14 02:43:50 +03:00
Vitaliy Filippov	0134934c99	Do not skip garbage entries on start (rollback change from 3.0.4)	2026-04-11 12:05:02 +03:00
Vitaliy Filippov	2e36f292bd	Fix buffer alignment during metadata clearing on init	2026-04-10 21:56:12 +03:00
Vitaliy Filippov	bcc6419760	Auto-select the only RDMA device/port if there is only one	2026-04-08 15:46:10 +03:00
Vitaliy Filippov	dd5941b9a4	Release 3.0.9 - Fix broken garbage collection in the new store leading to very high memory usage - Make the new store do full garbage collection on OSD startup by default (can be turned off with a new gc_on_start parameter) - Make startup logging with the new store more informative, allow to log amount of garbage with log_level > 1 - Fix clients reading from snapshots incorrectly on retries (for example when OSDs were restarted during read) - Fix a possible "send-after-free" in OSD which could probably also affect snapshot reads - Fix OSD crashing when handling a read_bitmap operation from a deleted object with the new store	2026-04-04 18:38:40 +03:00
Vitaliy Filippov	4005b88865	Do not use node_type	2026-04-04 17:37:47 +03:00
Vitaliy Filippov	280b5cd675	Add gc_on_start docs	2026-04-04 17:19:32 +03:00
Vitaliy Filippov	e5c505eaf4	Write 1 entry to blocks to collect garbage	2026-04-03 18:08:12 +00:00
Vitaliy Filippov	c1d244d4f0	Fix init metadata entry loading logging	2026-04-03 18:08:12 +00:00
Vitaliy Filippov	9b264a212f	Do full metadata GC on start (new store)	2026-04-03 18:08:12 +00:00
Vitaliy Filippov	ff7f5cb4f4	Report live & garbage entry count / memory with log_level > 1	2026-04-02 23:45:13 +00:00
Vitaliy Filippov	25ecca7625	Fix use-after-free in OSD ops (ooops)	2026-04-02 13:48:17 +00:00
Vitaliy Filippov	99c4244004	Try to write into blocks with more garbage first	2026-04-02 13:48:17 +00:00
Vitaliy Filippov	9949b9fb4e	Fix incorrect reads from snapshots on retries	2026-03-31 02:06:53 +03:00
Vitaliy Filippov	e6881ad1d5	Fix read_bitmap from deleted objects with the new store	2026-03-31 01:59:53 +03:00
Vitaliy Filippov	b30635b932	Release 3.0.8 - Surrogate increment-only peer IDs are now used instead of file descriptor numbers, and client cleanup sequence is reworked in another attempt to fix a possible PG hang in 'peering' states. The history of this bug is still unclear and it still hasn't been reproduced in tests, but several users have reported it since 3.0.4 (only with enabled RDMA). - Fix a minor bug where an OSD could skip reporting misplaced PG state when moving an object in non-immediate_commit mode. - Add Ubuntu 26.04 (Resolute Raccoon) support and patches for qemu 10.2 and libvirt 12.0	2026-03-30 02:09:02 +03:00
Vitaliy Filippov	0c1154833c	Add Ubuntu 26.04 support	2026-03-30 02:06:10 +03:00
Vitaliy Filippov	c227bb05b6	Fix incorrect patch contents for qemu 10.1, add patch for qemu 10.2	2026-03-30 02:06:10 +03:00
Vitaliy Filippov	dd85315f22	Update CMake supported versions	2026-03-30 02:06:10 +03:00
Vitaliy Filippov	47d2f4e0be	Add patch for libvirt 12.0	2026-03-30 02:06:10 +03:00
Vitaliy Filippov	2a5028d17f	Use robin_hood_map in messenger	2026-03-30 02:06:10 +03:00
Vitaliy Filippov	07915c2881	Use surrogate peer IDs instead of FDs	2026-03-30 02:06:10 +03:00
Vitaliy Filippov	79141eb383	Fix commented out //this->pg_state_dirty.insert	2026-03-28 20:08:52 +03:00
Vitaliy Filippov	f7cbb6ed56	Remove FD from the event loop in destroy_client, not in stop_client	2026-03-28 14:18:20 +03:00
Vitaliy Filippov	8f8172db99	clear_immediate_ops is now done by stop_client	2026-03-26 01:08:23 +03:00
Vitaliy Filippov	94be147e80	Clear received_ops - they are already freed	2026-03-26 01:08:02 +03:00
Vitaliy Filippov	538620b400	Release 3.0.6 - Fix one more bug in old store checksums O_o - checksum verification wasn't working correctly on reads quickly after small initial offsetted writes (when a BIG_WRITE was read from the journal). - Fix clients possibly not destroyed after being switched to RDMA. Not 100% verified, but in theory it could rarely lead to PGs hanging in peering/stopping states with RDMA enabled. - Use vitastor-cli in the Cinder driver for modifications instead of direct etcd interaction. - Fix mon_https_ca parsing. - Fix Docker install.sh, add instructions for Podman.	2026-03-22 22:49:51 +03:00