Release 3.0.15

- QEMU virtual disk migration with enabled iothread is finally fixed correctly.
- Fixed operation of Proxmox VMs with swTPM without enabling NBD for all disks.
- Debian packages are now again built with stable Antietcd released instead of
  the unstable master branch.
- Antietcd cluster mode previously broken in that master branch is fixed. The
  symptom was Antietcd being unable to elect the leader in a cluster.
- Fixed monitor startup with embedded Antietcd when using IPv6.
- Fixed space statistics calculation for FS and S3 pools in the new storage ([PR #127](https://github.com/vitalif/vitastor/pull/127)).

CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 2.8...3.30)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "3.0.14")
+set(VITASTOR_VERSION "3.0.15")
 
 include(CTest)
 

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.14
+VITASTOR_VERSION ?= v3.0.15
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v3.0.14
+          image: vitalif/vitastor-csi:v3.0.15
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v3.0.14
+          image: vitalif/vitastor-csi:v3.0.15
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "3.0.14"
+    vitastorCSIDriverVersion = "3.0.15"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (3.0.14-1) unstable; urgency=medium
+vitastor (3.0.15-1) unstable; urgency=medium
 
   * Bugfixes
 

debian/rules

@@ -11,6 +11,7 @@ override_dh_install:
 	cp -v node-binding/package.json node-binding/index.js node-binding/addon.cc node-binding/addon.h node-binding/client.cc node-binding/client.h debian/tmp/usr/lib/x86_64-linux-gnu/nodejs/vitastor
 	cp -v node-binding/build/Release/addon.node debian/tmp/usr/lib/x86_64-linux-gnu/nodejs/vitastor/build/Release
 	dh_install
+	cd debian/vitastor-mon/usr/lib/vitastor/mon && npm install --production
 
 override_dh_installdeb:
 	cat debian/fio_version >> debian/vitastor-fio.substvars

debian/vitastor-build.sh

@@ -37,12 +37,6 @@ rm -rf a b
 
 echo "dep:fio=$FIO" > debian/fio_version
 
-cd /root/vitastor/packages/vitastor-$REL/vitastor-$VER
-mkdir mon/node_modules
-cd mon/node_modules
-curl -s https://git.yourcmc.ru/vitalif/antietcd/archive/master.tar.gz | tar -zx
-curl -s https://git.yourcmc.ru/vitalif/tinyraft/archive/master.tar.gz | tar -zx
-
 cd /root/vitastor/packages/vitastor-$REL
 if [[ ( "$REL" = "trixie" || "$REL" = "resolute" ) && -e ../vitastor-bookworm/vitastor_$VER.orig.tar.xz ]]; then
     # Fucking shit, archives differ between bookworm (xz 5.4.1) and trixie (xz 5.8.1)

docker/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.14
+VITASTOR_VERSION ?= v3.0.15
 
 all: build push
 

docker/etc/vitastor/docker.conf

@@ -4,7 +4,7 @@
 #
 
 # Desired Vitastor version
-VITASTOR_VERSION=v3.0.14
+VITASTOR_VERSION=v3.0.15
 
 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here

docs/installation/docker.en.md

@@ -26,9 +26,9 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.
 
 1. Download a Docker image of the desired version: \
-   `docker pull vitalif/vitastor:v3.0.14`
+   `docker pull vitalif/vitastor:v3.0.15`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.14 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.15 install.sh`
 3. Reload udev rules: \
    `udevadm control --reload-rules`
 4. Enable the vitastor-host service: \

docs/installation/docker.ru.md

@@ -25,9 +25,9 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.
 
 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitalif/vitastor:v3.0.14`
+   `docker pull vitalif/vitastor:v3.0.15`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.14 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.15 install.sh`
 3. Перезагрузите правила udev: \
    `udevadm control --reload-rules`
 4. Включите сервис vitastor-host: \

mon/antietcd_adapter.js

@@ -18,7 +18,7 @@ class AntiEtcdAdapter
                 cluster = cluster ? (''+(cluster||'')).split(/,+/) : [];
             cluster = Object.keys(cluster.reduce((a, url) =>
             {
-                a[url.toLowerCase().replace(/^(https?:\/\/)/, '').replace(/\/.*$/, '')] = true;
+                a[url.toLowerCase().replace(/^(https?:\/\/)?(.*?)(\/.*)?$/, (m, m1, m2) => (m1||'http://')+m2)] = true;
                 return a;
             }, {}));
             const cfg_port = config.antietcd_port;
@@ -26,7 +26,18 @@ class AntiEtcdAdapter
             is_local['0.0.0.0'] = true;
             is_local['::'] = true;
             is_local[''] = true;
-            const selected = cluster.map(s => s.split(':', 2)).filter(ip => is_local[ip[0]] && (!cfg_port || ip[1] == cfg_port));
+            // split :, 3 -> <schema>:<//ip>:<port>
+            const selected = [];
+            for (let i = 0; i < cluster.length; i++)
+            {
+                const m = /^(https?:\/\/)?(?:\[(.*)\]|([^\[\:]+))(?::(\d+))?$/.exec(cluster[i]);
+                if (!m)
+                    continue;
+                const ip = m[3] || m[2];
+                const port = m[4] || 2379;
+                if (is_local[ip] && (!cfg_port || port == cfg_port))
+                    selected.push({ idx: i, ip, port });
+            }
             if (selected.length > 1)
             {
                 console.error('More than 1 etcd_address matches local IPs, please specify port');
@@ -35,15 +46,16 @@ class AntiEtcdAdapter
             else if (selected.length == 1)
             {
                 const antietcd_config = {
-                    ip: selected[0][0],
-                    port: selected[0][1],
-                    data: config.antietcd_data_file || ((config.antietcd_data_dir || '/var/lib/vitastor') + '/mon_'+selected[0][1]+'.json.gz'),
+                    ip: selected[0].ip,
+                    port: selected[0].port,
+                    data: config.antietcd_data_file || ((config.antietcd_data_dir || '/var/lib/vitastor') + '/mon_'+selected[0].port+'.json.gz'),
                     persist_filter: vitastor_persist_filter({ vitastor_prefix: config.etcd_prefix || '/vitastor' }),
-                    node_id: selected[0][0]+':'+selected[0][1], // node_id = ip:port
-                    cluster: (cluster.length == 1 ? null : cluster.reduce((a, c) => { a[c] = "http://"+c; return a; }, {})),
+                    node_id: cluster[selected[0].idx].replace(/^(https?:\/\/)/, ''), // same as in <cluster> below
+                    cluster: (cluster.length == 1 ? null : cluster.reduce((a, c) => { a[c.replace(/^(https?:\/\/)/, '')] = c; return a; }, {})),
                     cluster_key: (config.etcd_prefix || '/vitastor'),
                     stale_read: 1,
                     log_level: 1,
+                    logs: { cluster: true },
                 };
                 for (const key in config)
                 {

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "3.0.14",
+  "version": "3.0.15",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {
@@ -9,7 +9,7 @@
   "author": "Vitaliy Filippov",
   "license": "UNLICENSED",
   "dependencies": {
-    "antietcd": "^1.2.4",
+    "antietcd": "^1.3.1",
     "sprintf-js": "^1.1.2",
     "ws": "^7.2.5"
   },

node-binding/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor",
-  "version": "3.0.14",
+  "version": "3.0.15",
   "description": "Low-level native bindings to Vitastor client library",
   "main": "index.js",
   "keywords": [

patches/VitastorPlugin.pm

@@ -366,15 +366,38 @@ sub map_volume
     my $prefix = defined $scfg->{vitastor_prefix} ? $scfg->{vitastor_prefix} : 'pve/';
 
     my ($vtype, $img_name, $vmid) = $class->parse_volname($volname);
-    my $name = $img_name;
+    my $name = $prefix.$img_name;
     $name .= '@'.$snapname if $snapname;
 
     my $mapped = run_cli($scfg, [ 'ls' ], binary => '/usr/bin/vitastor-nbd');
-    my ($kerneldev) = grep { $mapped->{$_}->{image} eq $prefix.$name } keys %$mapped;
-    return $kerneldev if $kerneldev && -b $kerneldev; # already mapped
+    my ($kerneldev) = grep {
+        $mapped->{$_} && $mapped->{$_}->{image} && $mapped->{$_}->{image} eq $name
+    } keys %$mapped;
 
-    $kerneldev = run_cli($scfg, [ 'map', '--image', $prefix.$name ], binary => '/usr/bin/vitastor-nbd', json => 0);
-    return $kerneldev;
+    if ($kerneldev && -b $kerneldev)
+    {
+        my $size = `/usr/sbin/blockdev --getsize64 $kerneldev`;
+        return $kerneldev if $size && $size > 0;
+    }
+
+    my $map_out = run_cli($scfg, [ 'map', '--image', $name ], binary => '/usr/bin/vitastor-nbd', json => 0);
+    $map_out =~ s/^\s+|\s+$//gso;
+
+    # Wait until the device is started
+    for (my $i = 0; $i < 100; $i++)
+    {
+        $mapped = run_cli($scfg, [ 'ls' ], binary => '/usr/bin/vitastor-nbd');
+        ($kerneldev) = grep { $mapped->{$_} && $mapped->{$_}->{image} && $mapped->{$_}->{image} eq $name } keys %$mapped;
+        if ($kerneldev && -b $kerneldev)
+        {
+            my $size = `/usr/sbin/blockdev --getsize64 $kerneldev`;
+            return $kerneldev if $size && $size > 0;
+        }
+        select(undef, undef, undef, 0.1);
+    }
+
+    die "Failed to map Vitastor image $name via NBD".
+        ($map_out ? ", vitastor-nbd map returned '$map_out'" : "")."\n";
 }
 
 sub unmap_volume
@@ -383,13 +406,19 @@ sub unmap_volume
     my $prefix = defined $scfg->{vitastor_prefix} ? $scfg->{vitastor_prefix} : 'pve/';
 
     my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+    $name = $prefix.$name;
     $name .= '@'.$snapname if $snapname;
-
     my $mapped = run_cli($scfg, [ 'ls' ], binary => '/usr/bin/vitastor-nbd');
-    my ($kerneldev) = grep { $mapped->{$_}->{image} eq $prefix.$name } keys %$mapped;
-    if ($kerneldev && -b $kerneldev)
+
+    my @kerneldevs = grep {
+        $mapped->{$_} && $mapped->{$_}->{image} && $mapped->{$_}->{image} eq $name
+    } keys %$mapped;
+
+    for my $kerneldev (@kerneldevs)
     {
-        run_cli($scfg, [ 'unmap', $kerneldev ], binary => '/usr/bin/vitastor-nbd', json => 0);
+        next if !$kerneldev || !-b $kerneldev;
+        eval { run_cli($scfg, [ 'unmap', $kerneldev ], binary => '/usr/bin/vitastor-nbd', json => 0); };
+        warn "Failed to unmap Vitastor image $name from $kerneldev: $@" if $@;
     }
 
     return 1;
@@ -405,7 +434,13 @@ sub activate_volume
 sub deactivate_volume
 {
     my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_;
-    $class->unmap_volume($storeid, $scfg, $volname, $snapname) if $scfg->{vitastor_nbd};
+
+    # Even with vitastor_nbd=0, Proxmox may call map_volume() for special
+    # volumes like tpmstate0 because swtpm needs a local file/block path.
+    # Therefore, always try to unmap an existing NBD mapping here.
+    # unmap_volume() is a no-op if the volume is not currently mapped.
+    $class->unmap_volume($storeid, $scfg, $volname, $snapname);
+
     return 1;
 }
 

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '3.0.14'
+VITASTOR_VERSION = '3.0.15'
 
 LOG = logging.getLogger(__name__)
 

rpm/vitastor-el10.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.14
+Version:        3.0.15
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.14.el10.tar.gz
+Source0:        vitastor-3.0.15.el10.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.14
+Version:        3.0.15
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.14.el7.tar.gz
+Source0:        vitastor-3.0.15.el7.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  devtoolset-9-gcc-c++

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.14
+Version:        3.0.15
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.14.el8.tar.gz
+Source0:        vitastor-3.0.15.el8.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-toolset-9-gcc-c++

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.14
+Version:        3.0.15
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.14.el9.tar.gz
+Source0:        vitastor-3.0.15.el9.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++

src/CMakeLists.txt

@@ -20,7 +20,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 endif()
 set(ENABLE_COVERAGE false CACHE BOOL "Enable code coverage")
 
-add_definitions(-DVITASTOR_VERSION="3.0.14")
+add_definitions(-DVITASTOR_VERSION="3.0.15")
 add_definitions(-D_GNU_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -fvisibility=hidden -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/blockstore/blockstore_heap.cpp

@@ -2300,7 +2300,7 @@ void blockstore_heap_t::use_data(inode_t inode, uint64_t location)
 {
     auto sh_it = pool_shard_settings.find(INODE_POOL(inode));
     if (sh_it != pool_shard_settings.end() && sh_it->second.no_inode_stats)
-        inode = (INODE_POOL(inode) << POOL_ID_BITS);
+        inode = INODE_WITH_POOL(INODE_POOL(inode), 0);
     assert(!data_alloc->get(location / dsk->data_block_size));
     data_alloc->set(location / dsk->data_block_size, true);
     inode_space_stats[inode] += dsk->data_block_size;
@@ -2311,7 +2311,7 @@ void blockstore_heap_t::free_data(inode_t inode, uint64_t location)
 {
     auto sh_it = pool_shard_settings.find(INODE_POOL(inode));
     if (sh_it != pool_shard_settings.end() && sh_it->second.no_inode_stats)
-        inode = (INODE_POOL(inode) << POOL_ID_BITS);
+        inode = INODE_WITH_POOL(INODE_POOL(inode), 0);
     assert(data_alloc->get(location / dsk->data_block_size));
     data_alloc->set(location / dsk->data_block_size, false);
     auto sp_it = inode_space_stats.find(inode);

src/client/qemu_driver.c

@@ -880,6 +880,25 @@ static int vitastor_refresh_limits(BlockDriverState *bs)
 //    return 0;
 //}
 
+// Move the running coroutine to the BlockDriverState's home AioContext.
+//
+// The block-coroutine-wrapper generator sets poll_state.ctx to
+// qemu_get_current_aio_context() in the sync wrappers (bdrv_flush(),
+// bdrv_pread() etc.). When bdrv_flush_all() runs under BQL from outside the
+// bs's iothread (e.g. on the migration thread inside do_vm_stop()), that is
+// the main AioContext, not the iothread that actually owns the bs. The
+// coroutine then runs on the wrong context while completions are delivered on
+// the iothread, and racing aio_co_schedule() vs. qemu_aio_coroutine_enter()
+// on the same coroutine triggers "Co-routine was already scheduled in
+// aio_co_schedule" and aborts the process (observed during live migration).
+//
+// aio_co_reschedule_self() is a no-op when we are already on the target ctx.
+#if QEMU_VERSION_MAJOR > 5 || QEMU_VERSION_MAJOR == 5 && QEMU_VERSION_MINOR >= 2
+#define vitastor_co_pin_to_bs_ctx(bs) aio_co_reschedule_self(bdrv_get_aio_context(bs))
+#else
+#define vitastor_co_pin_to_bs_ctx(bs) ((void)0)
+#endif
+
 static void vitastor_co_init_task(BlockDriverState *bs, VitastorRPC *task)
 {
     *task = (VitastorRPC) {
@@ -938,6 +957,7 @@ static int coroutine_fn vitastor_co_preadv(BlockDriverState *bs,
 {
     VitastorClient *client = bs->opaque;
     VitastorRPC task;
+    vitastor_co_pin_to_bs_ctx(bs);
     vitastor_co_init_task(bs, &task);
     task.iov = iov;
 
@@ -966,6 +986,7 @@ static int coroutine_fn vitastor_co_pwritev(BlockDriverState *bs,
 {
     VitastorClient *client = bs->opaque;
     VitastorRPC task;
+    vitastor_co_pin_to_bs_ctx(bs);
     vitastor_co_init_task(bs, &task);
     task.iov = iov;
 
@@ -1039,6 +1060,7 @@ static int coroutine_fn vitastor_co_block_status(BlockDriverState *bs,
 #endif
     VitastorRPC task;
     VitastorClient *client = bs->opaque;
+    vitastor_co_pin_to_bs_ctx(bs);
     uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode;
     uint8_t bit = 0;
     if (client->last_bitmap && client->last_bitmap_inode == inode &&
@@ -1151,6 +1173,7 @@ static int coroutine_fn vitastor_co_flush(BlockDriverState *bs)
 {
     VitastorClient *client = bs->opaque;
     VitastorRPC task;
+    vitastor_co_pin_to_bs_ctx(bs);
     vitastor_co_init_task(bs, &task);
 
     qemu_mutex_lock(&client->mutex);
@@ -1236,6 +1259,8 @@ static BlockDriver bdrv_vitastor = {
 #endif
     .bdrv_close                     = vitastor_close,
 
+    // Re-register fd handlers when the bs is moved to a different AioContext
+    // (live migration, drain, iothread reassignment).
     .bdrv_detach_aio_context        = vitastor_detach_aio_context,
     .bdrv_attach_aio_context        = vitastor_attach_aio_context,
 

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 3.0.14
+Version: 3.0.15
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

tests/test_etcd_fail.sh

@@ -23,9 +23,13 @@ kill_etcds()
     start_etcd 3
 }
 
-kill_etcds &
-
 $VITASTOR_FIO -bs=4k -direct=1 -iodepth=1 -fsync=1 -rw=randwrite \
-    -pool=1 -inode=1 -size=128M -cluster_log_level=10 -runtime=30
+    -pool=1 -inode=1 -size=128M -cluster_log_level=10 -runtime=30 &
+FIO_PID=$!
+
+# Kill in foreground because otherwise antietcds are not killed by jobs -p
+kill_etcds
+
+wait $FIO_PID
 
 format_green OK