Release 3.0.11

A single hotfix:

- Fix missing error handling in EC snapshot/cloned image reads potentially
  leading to corrupted reads during OSD/PG restarts

CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 2.8...3.30)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "3.0.9")
+set(VITASTOR_VERSION "3.0.11")
 
 include(CTest)
 

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.9
+VITASTOR_VERSION ?= v3.0.11
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v3.0.9
+          image: vitalif/vitastor-csi:v3.0.11
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v3.0.9
+          image: vitalif/vitastor-csi:v3.0.11
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "3.0.9"
+    vitastorCSIDriverVersion = "3.0.11"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (3.0.9-1) unstable; urgency=medium
+vitastor (3.0.11-1) unstable; urgency=medium
 
   * Bugfixes
 

docker/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.9
+VITASTOR_VERSION ?= v3.0.11
 
 all: build push
 

docker/etc/vitastor/docker.conf

@@ -4,7 +4,7 @@
 #
 
 # Desired Vitastor version
-VITASTOR_VERSION=v3.0.9
+VITASTOR_VERSION=v3.0.11
 
 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here

docs/installation/docker.en.md

@@ -26,9 +26,9 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.
 
 1. Download a Docker image of the desired version: \
-   `docker pull vitalif/vitastor:v3.0.9`
+   `docker pull vitalif/vitastor:v3.0.11`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.9 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.11 install.sh`
 3. Reload udev rules: \
    `udevadm control --reload-rules`
 4. Enable the vitastor-host service: \

docs/installation/docker.ru.md

@@ -25,9 +25,9 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.
 
 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitalif/vitastor:v3.0.9`
+   `docker pull vitalif/vitastor:v3.0.11`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.9 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.11 install.sh`
 3. Перезагрузите правила udev: \
    `udevadm control --reload-rules`
 4. Включите сервис vitastor-host: \

docs/usage/nfs.en.md

@@ -262,3 +262,4 @@ Options:
 | `--logfile <FILE>` | log to the specified file                                |
 | `--enforce 1`      | enforce permissions at the server side (no by default)   |
 | `--foreground 1`   | stay in foreground, do not daemonize                     |
+| `--trace`          | trace all NFS requests                                   |

docs/usage/nfs.ru.md

@@ -274,3 +274,4 @@ VitastorFS из GPUDirect.
 | `--logfile <FILE>` | записывать логи в заданный файл                         |
 | `--enforce 1`      | проверять права доступа на стороне сервера (по умолчанию нет) |
 | `--foreground 1`   | не уходить в фон после запуска                          |
+| `--trace`          | логгировать все запросы NFS                             |

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "3.0.9",
+  "version": "3.0.11",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {

mon/stats.js

@@ -37,6 +37,7 @@ function derive_osd_stats(st, prev, prev_diff)
         const n = c.count - BigInt(pr && pr.count||0);
         diff.recovery_stats[op] = { ...c, bps: n > 0 ? b*1000n/timediff : 0n, iops: n > 0 ? n*1000n/timediff : 0n };
     }
+    diff.inode_stats = {};
     for (const pool_id in st.inode_stats||{})
     {
         diff.inode_stats[pool_id] = {};

node-binding/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor",
-  "version": "3.0.9",
+  "version": "3.0.11",
   "description": "Low-level native bindings to Vitastor client library",
   "main": "index.js",
   "keywords": [

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '3.0.9'
+VITASTOR_VERSION = '3.0.11'
 
 LOG = logging.getLogger(__name__)
 

rpm/vitastor-el10.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.9
+Version:        3.0.11
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.9.el10.tar.gz
+Source0:        vitastor-3.0.11.el10.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.9
+Version:        3.0.11
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.9.el7.tar.gz
+Source0:        vitastor-3.0.11.el7.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  devtoolset-9-gcc-c++

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.9
+Version:        3.0.11
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.9.el8.tar.gz
+Source0:        vitastor-3.0.11.el8.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-toolset-9-gcc-c++

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.9
+Version:        3.0.11
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.9.el9.tar.gz
+Source0:        vitastor-3.0.11.el9.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++

src/CMakeLists.txt

@@ -20,7 +20,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 endif()
 set(ENABLE_COVERAGE false CACHE BOOL "Enable code coverage")
 
-add_definitions(-DVITASTOR_VERSION="3.0.9")
+add_definitions(-DVITASTOR_VERSION="3.0.11")
 add_definitions(-D_GNU_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -fvisibility=hidden -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/blockstore/blockstore_heap.cpp

@@ -334,19 +334,12 @@ corrupted_block:
                     block_num, block_offset, wr->size, sizeof(heap_entry_t));
                 goto corrupted_block;
             }
-            if (wr->is_garbage())
-            {
-                // Garbage collection is only performed when writing new entries into the block
-                // because it needs a fake LSN and modified blocks require consecutive modified LSNs
-                // That's why garbage entries may persist on disk
-                if (log_level > 5)
-                {
-                    fprintf(stderr, "Notice: skipping garbage entry %jx:%jx v%ju l%ju in metadata block %u at %u\n",
-                        wr->inode, wr->stripe, wr->version, wr->lsn, block_num, block_offset);
-                }
-                block_offset += wr->size;
-                continue;
-            }
+            // Garbage collection is only performed when writing new entries into the block
+            // because it needs a fake LSN and modified blocks require consecutive modified LSNs
+            // At the same time, further modifications _after_ putting new entries into the block,
+            // but _before_ writing it, may mark some entries in it as garbage. That's why garbage
+            // entries may still be present on disk.
+            wr->entry_type &= ~BS_HEAP_GARBAGE;
             if ((wr->entry_type & BS_HEAP_TYPE) < BS_HEAP_BIG_WRITE ||
                 (wr->entry_type & BS_HEAP_TYPE) > BS_HEAP_ROLLBACK ||
                 (wr->entry_type & ~(BS_HEAP_TYPE|BS_HEAP_STABLE)) ||

src/blockstore/blockstore_init.cpp

@@ -311,7 +311,7 @@ resume_8:
         uint32_t block_num = recheck_mod[i];
         uint64_t block_offset = bs->dsk.meta_offset + (uint64_t)(block_num+1) * bs->dsk.meta_block_size;
         data = ((ring_data_t*)sqe->user_data);
-        uint8_t *buf = (uint8_t*)malloc_or_die(bs->dsk.meta_block_size);
+        uint8_t *buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, bs->dsk.meta_block_size);
         bs->heap->get_meta_block(block_num, buf);
         data->iov = { buf, bs->dsk.meta_block_size };
         data->callback = [this, buf, block_offset](ring_data_t *data)

src/client/cluster_client.cpp

@@ -590,7 +590,7 @@ void cluster_client_t::on_change_pool_config_hook()
         {
             if (log_level > 2 && pg_counts[pool_item.first])
             {
-                printf("Pool %u (%s) PG count changed from %lu to %lu\n", pool_item.first, pool_item.second.name.c_str(),
+                fprintf(stderr, "Pool %u (%s) PG count changed from %lu to %lu\n", pool_item.first, pool_item.second.name.c_str(),
                     pg_counts[pool_item.first], pool_item.second.real_pg_count);
             }
             // At this point, all pool operations should have been suspended
@@ -1119,6 +1119,13 @@ resume_2:
         // Finished successfully
         // Even if the PG count has changed in meanwhile we treat it as success
         // because if some operations were invalid for the new PG count we'd get errors
+        if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
+        {
+            // Copy part bitmaps only after finishing all part reads
+            for (auto & part: op->parts)
+                if ((part.flags & (PART_SENT|PART_DONE|PART_VALID)) == (PART_SENT|PART_DONE|PART_VALID))
+                    copy_part_bitmap(op, &part);
+        }
         if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
         {
             // Check parent inode
@@ -1630,13 +1637,6 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
         }
         if (op->inflight_count == 0 && !op->retry_after)
         {
-            // Copy part bitmaps only after finishing all part reads
-            if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
-            {
-                for (auto & part: op->parts)
-                    if ((part.flags & (PART_SENT|PART_VALID|PART_DONE)) == (PART_SENT|PART_VALID|PART_DONE))
-                        copy_part_bitmap(op, &part);
-            }
             if (op->opcode == OSD_OP_SYNC)
                 continue_sync(op);
             else

src/client/msgr_op.cpp

@@ -8,6 +8,7 @@
 osd_op_t::~osd_op_t()
 {
     assert(!bs_op);
+    assert(!op_data);
     if (bitmap_buf)
     {
         free(bitmap_buf);
@@ -22,10 +23,6 @@ osd_op_t::~osd_op_t()
         // So we don't reuse it, but free it every time
         free(buf);
     }
-    if (op_data)
-    {
-        free(op_data);
-    }
 }
 
 bool osd_op_t::is_recovery_related()

src/client/msgr_rdma.cpp

@@ -187,6 +187,8 @@ std::vector<msgr_rdma_context_t*> msgr_rdma_context_t::create_all(const std::vec
     ibv_device **raw_dev_list = NULL;
     ibv_device **dev_list = NULL;
     ibv_device *single_list[2] = {};
+    int up_ports = 0;
+    int single_port_num = 0;
 
     raw_dev_list = dev_list = ibv_get_device_list(NULL);
     if (!dev_list || !*dev_list)
@@ -221,6 +223,7 @@ std::vector<msgr_rdma_context_t*> msgr_rdma_context_t::create_all(const std::vec
         dev_list = single_list;
     }
 
+retry:
     for (int i = 0; dev_list[i]; ++i)
     {
         auto dev = dev_list[i];
@@ -258,6 +261,9 @@ std::vector<msgr_rdma_context_t*> msgr_rdma_context_t::create_all(const std::vec
                 fprintf(stderr, "RDMA device %s port %d GID %d does not exist\n", ibv_get_device_name(dev), port_num, sel_gid_index);
                 continue;
             }
+            up_ports++;
+            single_port_num = port_num;
+            single_list[0] = dev;
             uint32_t port_mtu = sel_mtu ? sel_mtu : ibv_mtu_to_bytes(portinfo.active_mtu);
 #ifdef IBV_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT
             if (sel_gid_index < 0)
@@ -298,6 +304,14 @@ cleanup_dev:
         ibv_close_device(context);
     }
 
+    if (!ret.size() && up_ports == 1 && dev_list != single_list)
+    {
+        // Auto-select the only available device/port if there is only one
+        dev_list = single_list;
+        sel_port_num = single_port_num;
+        goto retry;
+    }
+
 cleanup:
     if (raw_dev_list)
         ibv_free_device_list(raw_dev_list);

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 3.0.9
+Version: 3.0.11
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/cmd/cli_rm_data.cpp

@@ -286,6 +286,8 @@ struct rm_inode_t
                     .data = data,
                 };
             }
+            // Wakeup callers (otherwise they wake up only on next loop())
+            parent->ringloop->wakeup();
         }
         in_continue = false;
     }

src/nfs/nfs_proxy.cpp

@@ -121,6 +121,7 @@ static const char* help_text =
     "  --logfile <FILE>  log to the specified file\n"
     "  --enforce 1       enforce permissions at the server side (default is disabled)\n"
     "  --foreground 1    stay in foreground, do not daemonize\n"
+    "  --trace           trace all NFS requests\n"
     "\n"
     "NFS proxy is stateless if you use immediate_commit=all in your cluster and if\n"
     "you do not use client_enable_writeback=true, so you can freely use multiple\n"
@@ -158,7 +159,7 @@ json11::Json::object nfs_proxy_t::parse_args(int narg, const char *args[])
         {
             const char *opt = args[i]+2;
             cfg[str_replace(opt, "-", "_")] = !strcmp(opt, "json") || !strcmp(opt, "block") ||
-                !strcmp(opt, "dry-run") || !strcmp(opt, "recalc-stats") ||
+                !strcmp(opt, "dry-run") || !strcmp(opt, "recalc-stats") || !strcmp(opt, "trace") ||
                 !strcmp(opt, "include-empty") || !strcmp(opt, "no-rm") || i == narg-1 ? "1" : args[++i];
         }
         else

src/osd/osd_primary_chain.cpp

@@ -82,6 +82,7 @@ int osd_t::read_bitmaps(osd_op_t *cur_op, pg_t *pg, int base_state)
         goto resume_0;
     else if (op_data->st == base_state+1)
         goto resume_1;
+    assert(op_data->st < base_state);
     if (!pg || pg->state == PG_ACTIVE && pg->scheme == POOL_SCHEME_REPLICATED)
     {
         // Happy path for clean replicated PGs (all bitmaps are available locally)
@@ -112,6 +113,12 @@ resume_0:
             return 1;
         }
 resume_1:
+        if (op_data->errors > 0)
+        {
+            // Failure
+            finish_op(cur_op, op_data->errcode);
+            return -1;
+        }
         if (pg->scheme != POOL_SCHEME_REPLICATED)
         {
             for (int chain_num = 0; chain_num < op_data->chain_size; chain_num++)
@@ -607,7 +614,7 @@ void osd_t::send_chained_read_results(pg_t *pg, osd_op_t *cur_op)
         {
             if (cur > prev)
             {
-                 // Send buffer in parts to avoid copying
+                // Send buffer in parts to avoid copying
                 if (!prev_set)
                 {
                     while ((cur-prev) > zero_buffer_size/bs_bitmap_granularity)

src/osd/osd_primary_subops.cpp

@@ -83,6 +83,8 @@ void osd_t::finish_op(osd_op_t *cur_op, int retval)
             rm_inflight(pg);
         }
         assert(!cur_op->op_data->subops);
+        free(cur_op->op_data);
+        cur_op->op_data = NULL;
     }
     cur_op->reply.hdr.magic = SECONDARY_OSD_REPLY_MAGIC;
     cur_op->reply.hdr.id = cur_op->req.hdr.id;

src/test/test_heap.cpp

@@ -1893,7 +1893,8 @@ void test_big_intent_csums()
         heap.finish_recheck();
 
         auto mod = heap.get_recheck_modified_blocks();
-        assert(mod.size() == 0);
+        assert(mod.size() == 1);
+        assert(mod[0] == 0);
 
         // read object 1 - big_intent should be there
         object_id oid = { .inode = INODE_WITH_POOL(1, 1), .stripe = 0 };