Release 0.9.3

- Add patch for libvirt 9.0
- Add support for Proxmox VE 8.0
- Fix compatibility of the QEMU driver with iothread (QEMU rebuilds are coming)
- Fix vitastor-cli rm-data/rm/merge hanging when some OSDs are down.
  Allow deletions in unclean cluster at the cost of some data possibly
  "reappearing" when those OSDs start back. In that case you can just repeat
  the deletion request using rm-data.
- A bunch of bug fixes for snapshots:
  - Fix snapshot reads often not working at all with snapshot chain size > 2
  - Fix optimized snapshot data merge (children to parent)
  - Fix updating of image name index key during optimized merge
  - Fix auto-selection preventing the use of optimized merge with only 1 snapshot
  - Fix incorrect CAS retries during snapshot merge
  - Fix snapshot merge progress reporting
- Fix primary_read bitmap buffers use-after-free which could lead to
  incorrect allocation map reads
- Remove /usr/local/bin path from make-etcd
- Some documentation fixes

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VERSION "0.9.2")
+set(VERSION "0.9.3")
 
 add_subdirectory(src)

csi/Makefile

@@ -1,4 +1,4 @@
-VERSION ?= v0.9.2
+VERSION ?= v0.9.3
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v0.9.2
+          image: vitalif/vitastor-csi:v0.9.3
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -116,7 +116,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v0.9.2
+          image: vitalif/vitastor-csi:v0.9.3
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "0.9.2"
+    vitastorCSIDriverVersion = "0.9.3"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,10 +1,10 @@
-vitastor (0.9.2-1) unstable; urgency=medium
+vitastor (0.9.3-1) unstable; urgency=medium
 
   * Bugfixes
 
  -- Vitaliy Filippov <vitalif@yourcmc.ru>  Fri, 03 Jun 2022 02:09:44 +0300
 
-vitastor (0.9.2-1) unstable; urgency=medium
+vitastor (0.9.3-1) unstable; urgency=medium
 
   * Implement NFS proxy
   * Add documentation

debian/vitastor.Dockerfile

@@ -35,8 +35,8 @@ RUN set -e -x; \
     mkdir -p /root/packages/vitastor-$REL; \
     rm -rf /root/packages/vitastor-$REL/*; \
     cd /root/packages/vitastor-$REL; \
-    cp -r /root/vitastor vitastor-0.9.2; \
-    cd vitastor-0.9.2; \
+    cp -r /root/vitastor vitastor-0.9.3; \
+    cd vitastor-0.9.3; \
     ln -s /root/fio-build/fio-*/ ./fio; \
     FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
@@ -49,8 +49,8 @@ RUN set -e -x; \
     rm -rf a b; \
     echo "dep:fio=$FIO" > debian/fio_version; \
     cd /root/packages/vitastor-$REL; \
-    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.9.2.orig.tar.xz vitastor-0.9.2; \
-    cd vitastor-0.9.2; \
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.9.3.orig.tar.xz vitastor-0.9.3; \
+    cd vitastor-0.9.3; \
     V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \
     DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VERSION = '0.9.2'
+VERSION = '0.9.3'
 
 LOG = logging.getLogger(__name__)
 

rpm/build-tarball.sh

@@ -24,4 +24,4 @@ rm fio
 mv fio-copy fio
 FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
 perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec
-tar --transform 's#^#vitastor-0.9.2/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.9.2$(rpm --eval '%dist').tar.gz *
+tar --transform 's#^#vitastor-0.9.3/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.9.3$(rpm --eval '%dist').tar.gz *

rpm/vitastor-el7.Dockerfile

@@ -35,7 +35,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.9.2.el7.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.9.3.el7.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el7.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.9.2
+Version:        0.9.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.9.2.el7.tar.gz
+Source0:        vitastor-0.9.3.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.Dockerfile

@@ -35,7 +35,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.9.2.el8.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.9.3.el8.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el8.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.9.2
+Version:        0.9.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.9.2.el8.tar.gz
+Source0:        vitastor-0.9.3.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el9.Dockerfile

@@ -18,7 +18,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.9.2.el9.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.9.3.el9.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el9.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.9.2
+Version:        0.9.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.9.2.el9.tar.gz
+Source0:        vitastor-0.9.3.el9.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -16,7 +16,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVERSION="0.9.2")
+add_definitions(-DVERSION="0.9.3")
 add_definitions(-Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -I ${CMAKE_SOURCE_DIR}/src)
 if (${WITH_ASAN})
 	add_definitions(-fsanitize=address -fno-omit-frame-pointer)

src/messenger.h

@@ -132,7 +132,6 @@ protected:
     uint64_t rdma_max_msg = 0;
 #endif
 
-    bool has_send_loop = false;
     std::vector<int> read_ready_clients;
     std::vector<int> write_ready_clients;
     // We don't use ringloop->set_immediate here because we may have no ringloop in client :)
@@ -160,7 +159,6 @@ public:
     std::function<bool(osd_client_t*, json11::Json)> check_config_hook;
     void read_requests();
     void send_replies();
-    bool can_send();
     void accept_connections(int listen_fd);
     ~osd_messenger_t();
 

src/msgr_send.cpp

@@ -111,28 +111,22 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
         return;
     }
 #endif
-    if (!ringloop && !has_send_loop)
+    if (!ringloop)
     {
-        // "Send loop" should be used, like in QEMU driver, or performance will suffer
-        // due to a large number of sendmsg() syscalls where they can be avoided.
-        // "Send loop" increases 4k T1Q128 randread from ~55k iops to ~90k on a 1 OSD.
-        // The difference is smaller when there are more OSDs though...
+        // FIXME: It's worse because it doesn't allow batching
         while (cl->outbox.size())
         {
             try_send(cl);
         }
     }
-    else if (cl->write_msg.msg_iovlen > 0 || has_send_loop || !try_send(cl))
+    else if (cl->write_msg.msg_iovlen > 0 || !try_send(cl))
     {
         if (cl->write_state == 0)
         {
             cl->write_state = CL_WRITE_READY;
             write_ready_clients.push_back(cur_op->peer_fd);
         }
-        if (ringloop)
-        {
-            ringloop->wakeup();
-        }
+        ringloop->wakeup();
     }
 }
 
@@ -224,12 +218,6 @@ void osd_messenger_t::send_replies()
     write_ready_clients.clear();
 }
 
-bool osd_messenger_t::can_send()
-{
-    has_send_loop = true;
-    return write_ready_clients.size() > 0;
-}
-
 void osd_messenger_t::handle_send(int result, osd_client_t *cl)
 {
     cl->write_msg.msg_iovlen = 0;

src/qemu_driver.c

@@ -77,7 +77,6 @@ typedef struct VitastorClient
     AioContext *ctx;
     VitastorFdData **fds;
     int fd_count, fd_alloc;
-    int bh_send_all_scheduled;
 
     uint64_t last_bitmap_inode, last_bitmap_offset, last_bitmap_len;
     uint32_t last_bitmap_granularity;
@@ -107,14 +106,6 @@ typedef struct VitastorRPC
 #endif
 } VitastorRPC;
 
-#if QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR < 8
-typedef struct VitastorBH
-{
-    VitastorClient *cli;
-    QEMUBH *bh;
-} VitastorBH;
-#endif
-
 static void vitastor_co_init_task(BlockDriverState *bs, VitastorRPC *task);
 static void vitastor_co_generic_cb(void *opaque, long retval);
 static void vitastor_co_read_cb(void *opaque, long retval, uint64_t version);
@@ -232,49 +223,6 @@ out:
     return;
 }
 
-#if defined VITASTOR_C_API_VERSION && VITASTOR_C_API_VERSION >= 2
-static void vitastor_bh_send_all(void *opaque)
-{
-#if QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR < 8
-    VitastorBH *vbh = opaque;
-    VitastorClient *client = vbh->cli;
-#else
-    VitastorClient *client = opaque;
-#endif
-    qemu_mutex_lock(&client->mutex);
-    client->bh_send_all_scheduled = 0;
-    vitastor_c_qemu_send_all(client->proxy);
-    qemu_mutex_unlock(&client->mutex);
-#if QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR < 8
-    qemu_bh_delete(vbh->bh);
-    free(vbh);
-#endif
-}
-#endif
-
-static void vitastor_schedule_send_all(VitastorClient *client)
-{
-#if defined VITASTOR_C_API_VERSION && VITASTOR_C_API_VERSION >= 2
-    if (!client->bh_send_all_scheduled)
-    {
-        client->bh_send_all_scheduled = 1;
-#if QEMU_VERSION_MAJOR > 4 || QEMU_VERSION_MAJOR == 4 && QEMU_VERSION_MINOR >= 2
-        replay_bh_schedule_oneshot_event(client->ctx, vitastor_bh_send_all, client);
-#elif QEMU_VERSION_MAJOR >= 3 || QEMU_VERSION_MAJOR == 2 && QEMU_VERSION_MINOR >= 8
-        aio_bh_schedule_oneshot(client->ctx, vitastor_bh_send_all, client);
-#elif QEMU_VERSION_MAJOR >= 2
-        VitastorBH *vbh = (VitastorBH*)malloc(sizeof(VitastorBH));
-        vbh->cli = client;
-        vbh->bh = aio_bh_new(bdrv_get_aio_context(task->bs), vitastor_bh_send_all, vbh);
-        qemu_bh_schedule(vbh->bh);
-#else
-        client->bh_send_all_scheduled = 0;
-        vitastor_c_qemu_send_all(client->proxy);
-#endif
-    }
-#endif
-}
-
 static void coroutine_fn vitastor_co_get_metadata(VitastorRPC *task)
 {
     BlockDriverState *bs = task->bs;
@@ -283,7 +231,6 @@ static void coroutine_fn vitastor_co_get_metadata(VitastorRPC *task)
 
     qemu_mutex_lock(&client->mutex);
     vitastor_c_watch_inode(client->proxy, client->image, vitastor_co_generic_cb, task);
-    vitastor_schedule_send_all(client);
     qemu_mutex_unlock(&client->mutex);
 
     while (!task->complete)
@@ -297,7 +244,6 @@ static void vitastor_aio_fd_read(void *fddv)
     VitastorFdData *fdd = (VitastorFdData*)fddv;
     qemu_mutex_lock(&fdd->cli->mutex);
     fdd->fd_read(fdd->opaque);
-    vitastor_schedule_send_all(fdd->cli);
     qemu_mutex_unlock(&fdd->cli->mutex);
 }
 
@@ -306,7 +252,6 @@ static void vitastor_aio_fd_write(void *fddv)
     VitastorFdData *fdd = (VitastorFdData*)fddv;
     qemu_mutex_lock(&fdd->cli->mutex);
     fdd->fd_write(fdd->opaque);
-    vitastor_schedule_send_all(fdd->cli);
     qemu_mutex_unlock(&fdd->cli->mutex);
 }
 
@@ -648,7 +593,6 @@ static int coroutine_fn vitastor_co_preadv(BlockDriverState *bs,
     uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode;
     qemu_mutex_lock(&client->mutex);
     vitastor_c_read(client->proxy, inode, offset, bytes, iov->iov, iov->niov, vitastor_co_read_cb, &task);
-    vitastor_schedule_send_all(client);
     qemu_mutex_unlock(&client->mutex);
 
     while (!task.complete)
@@ -682,7 +626,6 @@ static int coroutine_fn vitastor_co_pwritev(BlockDriverState *bs,
     uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode;
     qemu_mutex_lock(&client->mutex);
     vitastor_c_write(client->proxy, inode, offset, bytes, 0, iov->iov, iov->niov, vitastor_co_generic_cb, &task);
-    vitastor_schedule_send_all(client);
     qemu_mutex_unlock(&client->mutex);
 
     while (!task.complete)
@@ -761,7 +704,6 @@ static int coroutine_fn vitastor_co_block_status(
         task.bitmap = client->last_bitmap = NULL;
         qemu_mutex_lock(&client->mutex);
         vitastor_c_read_bitmap(client->proxy, task.inode, task.offset, task.len, !client->skip_parents, vitastor_co_read_bitmap_cb, &task);
-        vitastor_schedule_send_all(client);
         qemu_mutex_unlock(&client->mutex);
         while (!task.complete)
         {
@@ -848,7 +790,6 @@ static int coroutine_fn vitastor_co_flush(BlockDriverState *bs)
 
     qemu_mutex_lock(&client->mutex);
     vitastor_c_sync(client->proxy, vitastor_co_generic_cb, &task);
-    vitastor_schedule_send_all(client);
     qemu_mutex_unlock(&client->mutex);
 
     while (!task.complete)

src/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 0.9.2
+Version: 0.9.3
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/vitastor_c.cpp

@@ -166,14 +166,6 @@ void vitastor_c_uring_wait_events(vitastor_c *client)
     client->ringloop->wait();
 }
 
-void vitastor_c_qemu_send_all(vitastor_c *client)
-{
-    while (client->cli->msgr.can_send())
-    {
-        client->cli->msgr.send_replies();
-    }
-}
-
 void vitastor_c_read(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len,
     struct iovec *iov, int iovcnt, VitastorReadHandler cb, void *opaque)
 {

src/vitastor_c.h

@@ -7,7 +7,7 @@
 #define VITASTOR_QEMU_PROXY_H
 
 // C API wrapper version
-#define VITASTOR_C_API_VERSION 2
+#define VITASTOR_C_API_VERSION 1
 
 #ifndef POOL_ID_BITS
 #define POOL_ID_BITS 16
@@ -42,7 +42,6 @@ int vitastor_c_is_ready(vitastor_c *client);
 void vitastor_c_uring_wait_ready(vitastor_c *client);
 void vitastor_c_uring_handle_events(vitastor_c *client);
 void vitastor_c_uring_wait_events(vitastor_c *client);
-void vitastor_c_qemu_send_all(vitastor_c *client);
 void vitastor_c_read(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len,
     struct iovec *iov, int iovcnt, VitastorReadHandler cb, void *opaque);
 void vitastor_c_write(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, uint64_t check_version,