Release 3.0.3

- Fix csum_block_size > 0 unusable with atomic writes in the new store
  (almost all atomic write requests generated invalid checksums with csum_block_size > 0)
- Fix monitor sometimes randomly failing to optimize PGs with the "problem is
  infeasible or unbounded" message due to not waiting to read full stdout of lp_solve
- Remove reshard_abort optimisation to fix chunked resharding introduced in 3.0.2 possibly
  corrupting in-memory OSD state when handling multiple rapid PG count change requests
- Fix removed inodes not disappearing from OSD statistics in the new store, leading to
  bloating of the statistics with old inodes
- Increase test coverage for the new store and fix several minor bugs:
  - Enabling/disabling of used_for_app was recalculating inode space statistics incorrectly
  - Fix object metadata validation on OSD startup rejecting some correct sequences of entries,
    possibly leading to OSD being unable to start
  - Fix PG activation with EC failing in rare cases with EBUSY when requesting to commit an
    already committed write
  - Fix a theoretically possible metadata writeback issue on ENOSPC during commit
- Fix monitor failing to optimize PGs in presence of a host with name convertible to
  js Number (like 04e278988710) :D
- Fix vitastor-cli dd sometimes (rarely) truncating the image when writing to stdout
- Fix a theoretically possible client connection object leak when io_uring is full
- Fix a leak of RDMA-CM connection objects
- Fix crashes with data_block_size < 32KB (useless setup, but anyway) (#113)

CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "3.0.2")
+set(VITASTOR_VERSION "3.0.3")
 
 include(CTest)
 

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.2
+VITASTOR_VERSION ?= v3.0.3
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v3.0.2
+          image: vitalif/vitastor-csi:v3.0.3
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v3.0.2
+          image: vitalif/vitastor-csi:v3.0.3
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "3.0.2"
+    vitastorCSIDriverVersion = "3.0.3"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (3.0.2-1) unstable; urgency=medium
+vitastor (3.0.3-1) unstable; urgency=medium
 
   * Bugfixes
 

docker/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.2
+VITASTOR_VERSION ?= v3.0.3
 
 all: build push
 

docker/etc/vitastor/docker.conf

@@ -4,7 +4,7 @@
 #
 
 # Desired Vitastor version
-VITASTOR_VERSION=v3.0.2
+VITASTOR_VERSION=v3.0.3
 
 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here

docs/installation/docker.en.md

@@ -26,9 +26,9 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.
 
 1. Download a Docker image of the desired version: \
-   `docker pull vitalif/vitastor:v3.0.2`
+   `docker pull vitalif/vitastor:v3.0.3`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.2 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.3 install.sh`
 3. Reload udev rules: \
    `udevadm control --reload-rules`
 

docs/installation/docker.ru.md

@@ -25,9 +25,9 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.
 
 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitalif/vitastor:v3.0.2`
+   `docker pull vitalif/vitastor:v3.0.3`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.2 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.3 install.sh`
 3. Перезагрузите правила udev: \
    `udevadm control --reload-rules`
 

mon/lp_optimizer/lp_optimizer.js

@@ -10,16 +10,19 @@ const NO_OSD = 'Z';
 async function lp_solve(text)
 {
     const cp = child_process.spawn('lp_solve');
-    let stdout = '', stderr = '', finish_cb;
+    let stdout = '', stderr = '', finish_cb, finished = 0;
     cp.stdout.on('data', buf => stdout += buf.toString());
     cp.stderr.on('data', buf => stderr += buf.toString());
-    cp.on('exit', () => finish_cb && finish_cb());
+    cp.stdout.on('end', () => finish_cb());
+    cp.stderr.on('end', () => finish_cb());
     cp.stdin.write(text);
     cp.stdin.end();
-    if (cp.exitCode == null)
+    await new Promise(ok => (finish_cb = () =>
     {
-        await new Promise(ok => finish_cb = ok);
+        finished++;
-    }
+        if (finished == 2)
+            ok();
+    }));
     if (!stdout.trim())
     {
         return null;

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "3.0.2",
+  "version": "3.0.3",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {

mon/pg_gen.js

@@ -195,7 +195,7 @@ async function generate_pool_pgs(state, global_config, pool_id, osd_tree, levels
     const folded_tree = make_hier_tree(global_config, folded.nodes.reduce((a, c) => { a[c.id] = c; return a; }, {}));
     const old_pg_count = prev_pgs.length;
     const optimize_cfg = {
-        osd_weights: folded.nodes.reduce((a, c) => { if (Number(c.id)) { a[c.id] = c.size; } return a; }, {}),
+        osd_weights: folded.nodes.reduce((a, c) => { if (/^\d+$/.exec(c.id) && c.size != null) { a[c.id] = c.size||0; } return a; }, {}),
         combinator: use_rules
             // new algorithm:
             ? new RuleCombinator(folded_tree, rules, pool_cfg.max_osd_combinations)

node-binding/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor",
-  "version": "3.0.2",
+  "version": "3.0.3",
   "description": "Low-level native bindings to Vitastor client library",
   "main": "index.js",
   "keywords": [

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '3.0.2'
+VITASTOR_VERSION = '3.0.3'
 
 LOG = logging.getLogger(__name__)
 

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.2
+Version:        3.0.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.2.el7.tar.gz
+Source0:        vitastor-3.0.3.el7.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  devtoolset-9-gcc-c++

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.2
+Version:        3.0.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.2.el8.tar.gz
+Source0:        vitastor-3.0.3.el8.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-toolset-9-gcc-c++

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.2
+Version:        3.0.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.2.el9.tar.gz
+Source0:        vitastor-3.0.3.el9.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++

src/CMakeLists.txt

@@ -21,7 +21,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 endif()
 set(ENABLE_COVERAGE false CACHE BOOL "Enable code coverage")
 
-add_definitions(-DVITASTOR_VERSION="3.0.2")
+add_definitions(-DVITASTOR_VERSION="3.0.3")
 add_definitions(-D_GNU_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -fvisibility=hidden -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/blockstore/blockstore.h

@@ -187,7 +187,6 @@ public:
     // MUST be called only when nobody makes any modifications to the DB for this pool
     virtual void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit) = 0;
     virtual bool reshard_continue(void *reshard_state, uint64_t chunk_limit) = 0;
-    virtual void reshard_abort(void *reshard_state) = 0;
 
     // Event loop
     virtual void loop() = 0;

src/blockstore/blockstore_disk.cpp

@@ -171,6 +171,12 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     {
         throw std::runtime_error("Data block size must be a multiple of sparse write tracking granularity");
     }
+    if (data_block_size / bitmap_granularity < 8)
+    {
+        fprintf(stderr, "Warning: block_size (%u) / bitmap_granularity (%u) = %u bits. "
+            "Consider using larger block_size or bitmap_granularity for better performance.\n",
+            data_block_size, bitmap_granularity, data_block_size / bitmap_granularity);
+    }
     if (!data_csum_type)
     {
         csum_block_size = 0;
@@ -259,7 +265,7 @@ void blockstore_disk_t::calc_lengths(bool skip_meta_check)
     }
     // required metadata size
     block_count = data_len / data_block_size;
-    clean_entry_bitmap_size = data_block_size / bitmap_granularity / 8;
+    clean_entry_bitmap_size = (data_block_size / bitmap_granularity + 7) / 8;
     clean_dyn_size = clean_entry_bitmap_size*2 + (csum_block_size
         ? data_block_size/csum_block_size*(data_csum_type & 0xFF) : 0);
 recalc:

src/blockstore/blockstore_flush.h

@@ -58,7 +58,6 @@ class journal_flusher_co
     int i, res;
     bool read_to_fill_incomplete;
     int copy_count;
-    bool do_repeat = false;
 
     friend class journal_flusher_t;
 

src/blockstore/blockstore_heap.cpp

@@ -458,19 +458,22 @@ bool blockstore_heap_t::validate_object(heap_entry_t *obj)
         next_wr = wr;
         if (wr->type() == BS_HEAP_ROLLBACK)
         {
-            if (commit_wr && wr->version > commit_wr->version)
-            {
-                // rollback may not come before commit with a smaller version
-                fprintf(stderr, "Error: rollback entry %jx:%jx v%ju l%ju comes before a commit entry v%ju l%ju\n",
-                    wr->inode, wr->stripe, wr->version, wr->lsn, commit_wr->version, commit_wr->lsn);
-                return false;
-            }
             rollback_wr = wr;
             continue;
         }
         if (wr->type() == BS_HEAP_COMMIT)
         {
-            commit_wr = wr;
+            if (commit_wr && wr->version > commit_wr->version)
+            {
+                // commit may not come before commit with a smaller version
+                fprintf(stderr, "Error: commit entry %jx:%jx v%ju l%ju comes before a commit entry v%ju l%ju\n",
+                    wr->inode, wr->stripe, wr->version, wr->lsn, commit_wr->version, commit_wr->lsn);
+                return false;
+            }
+            if (!commit_wr)
+            {
+                commit_wr = wr;
+            }
             continue;
         }
         if (wr->entry_type & BS_HEAP_STABLE)
@@ -837,7 +840,6 @@ bool blockstore_heap_t::calc_checksums(heap_entry_t *wr, uint8_t *data, bool set
         {
             return true;
         }
-        uint32_t len = 0;
         if (wr->type() == BS_HEAP_SMALL_WRITE || wr->type() == BS_HEAP_INTENT_WRITE)
             len = wr->small().len;
         else if (wr->type() == BS_HEAP_BIG_INTENT)
@@ -854,13 +856,14 @@ bool blockstore_heap_t::calc_checksums(heap_entry_t *wr, uint8_t *data, bool set
     }
     if (wr->type() == BS_HEAP_BIG_WRITE)
     {
+        assert(offset != UINT32_MAX && len != UINT32_MAX);
         return calc_block_checksums((uint32_t*)(wr->get_checksums(this) + offset/dsk->csum_block_size * (dsk->data_csum_type & 0xFF)),
             data, wr->get_int_bitmap(this), offset, offset+len, set, NULL);
     }
     if (wr->type() == BS_HEAP_BIG_INTENT)
     {
         auto & bi = wr->big_intent();
-        return calc_block_checksums((uint32_t*)(wr->get_checksums(this) + offset/dsk->csum_block_size * (dsk->data_csum_type & 0xFF)),
+        return calc_block_checksums((uint32_t*)(wr->get_checksums(this) + bi.offset/dsk->csum_block_size * (dsk->data_csum_type & 0xFF)),
             data, wr->get_int_bitmap(this), bi.offset, bi.offset+bi.len, set, NULL);
     }
     assert(wr->type() == BS_HEAP_SMALL_WRITE || wr->type() == BS_HEAP_INTENT_WRITE);
@@ -1077,16 +1080,6 @@ bool blockstore_heap_t::reshard_check(pool_id_t pool, uint32_t pg_count, uint32_
         set_it->second.pg_stripe_size == pg_stripe_size);
 }
 
-void blockstore_heap_t::reshard_abort(void* reshard_state)
-{
-    heap_reshard_state_t *st = (heap_reshard_state_t*)reshard_state;
-    for (auto sh_it = st->old_shards.begin(); sh_it != st->old_shards.end(); sh_it++)
-    {
-        block_index[sh_it->first] = std::move(sh_it->second);
-    }
-    delete st;
-}
-
 heap_entry_t *blockstore_heap_t::lock_and_read_entry(object_id oid)
 {
     auto obj = read_entry(oid);
@@ -1099,27 +1092,6 @@ heap_entry_t *blockstore_heap_t::lock_and_read_entry(object_id oid)
     return obj;
 }
 
-heap_entry_t *blockstore_heap_t::read_locked_entry(object_id oid, uint64_t lsn)
-{
-    auto obj = read_entry(oid);
-    assert(obj);
-    for (auto wr = obj; wr; wr = prev(wr))
-    {
-        if (wr->is_overwrite())
-        {
-            if (lsn == wr->lsn)
-            {
-                return obj;
-            }
-            else
-            {
-                obj = prev(wr);
-            }
-        }
-    }
-    return NULL;
-}
-
 bool blockstore_heap_t::unlock_entry(object_id oid)
 {
     auto mvcc_it = object_mvcc.find(oid);
@@ -1418,7 +1390,7 @@ int blockstore_heap_t::add_redirect_intent(object_id oid, heap_entry_t **obj_ptr
         bitmap_set(wr->get_int_bitmap(this), offset, len, dsk->bitmap_granularity);
         if (dsk->data_csum_type)
             memset(wr->get_checksums(this), 0, get_csum_size(wr));
-        calc_checksums(wr, (uint8_t*)data, true, offset, len);
+        calc_checksums(wr, (uint8_t*)data, true);
         *obj_ptr = wr;
     });
 }
@@ -1457,11 +1429,11 @@ int blockstore_heap_t::add_big_intent(object_id oid, heap_entry_t **obj_ptr, uin
         if (dsk->data_csum_type)
         {
             if (checksums)
-                memcpy(wr->get_checksums(this), checksums, dsk->clean_entry_bitmap_size);
+                memcpy(wr->get_checksums(this), checksums, get_csum_size(wr));
             else
             {
-                memcpy(wr->get_checksums(this), obj->get_checksums(this), dsk->clean_entry_bitmap_size);
+                memcpy(wr->get_checksums(this), obj->get_checksums(this), get_csum_size(wr));
-                calc_checksums(wr, (uint8_t*)data, true, offset, len);
+                calc_checksums(wr, (uint8_t*)data, true);
             }
         }
         else
@@ -1582,7 +1554,7 @@ int blockstore_heap_t::add_commit(heap_entry_t *obj, uint64_t version, uint32_t
     }
     if (!uncommitted)
     {
-        return EBUSY;
+        return 0;
     }
     return add_simple(obj, version, modified_block, BS_HEAP_COMMIT);
 }
@@ -1592,23 +1564,32 @@ int blockstore_heap_t::add_rollback(heap_entry_t *obj, uint64_t version, uint32_
     heap_entry_t *wr = obj;
     bool found_uncommitted = false;
     uint64_t commit_version = 0;
-    while (wr && !wr->is_overwrite())
+    uint64_t rollback_version = UINT64_MAX;
+    while (wr)
     {
         if (wr->type() == BS_HEAP_ROLLBACK)
         {
-            auto rollback_version = wr->version;
+            if (wr->version <= version)
-            wr = prev(wr);
-            while (wr->version > rollback_version)
             {
-                assert(!(wr->entry_type & BS_HEAP_STABLE));
+                // All previous writes are already rolled back, stop
-                wr = prev(wr);
+                break;
             }
+            rollback_version = wr->version;
+            wr = prev(wr);
             continue;
         }
         if (wr->type() == BS_HEAP_COMMIT)
         {
             if (commit_version < wr->version)
+            {
                 commit_version = wr->version;
+            }
+            wr = prev(wr);
+            continue;
+        }
+        if (wr->version > rollback_version)
+        {
+            // Already rolled back, skip
             wr = prev(wr);
             continue;
         }
@@ -1619,14 +1600,10 @@ int blockstore_heap_t::add_rollback(heap_entry_t *obj, uint64_t version, uint32_
             {
                 return EBUSY;
             }
-            else if (wr->version == version)
+            else
             {
                 break;
             }
-            else if (wr->version < version)
-            {
-                return ENOENT;
-            }
         }
         else if (wr->version > version)
         {
@@ -1807,9 +1784,9 @@ void blockstore_heap_t::iterate_with_stable(heap_entry_t *obj, uint64_t max_lsn,
         }
         else
         {
-            // 1) 1 2 3 ROLLBACK(2) COMMIT(3) -> impossible
+            // 1) 1 2 3 ROLLBACK(2) COMMIT(3) -> 3 is unstable
             // 2) 1 2 3 4 ROLLBACK(3) COMMIT(2) -> OK
-            // 3) 1 2 3 ROLLBACK(2) 3 COMMIT(3) -> first 3 shouldn't be treated as stable
+            // 3) 1 2 3 ROLLBACK(2) 3 COMMIT(3) -> first 3 is unstable
             // 4) 1 2 3 COMMIT(3) ROLLBACK(2) -> impossible
             //    I.e. a rollback always has version >= previous commit
             // 5) 1 2 3 4 5 ROLLBACK(4) 5 ROLLBACK(3)
@@ -2045,7 +2022,13 @@ void blockstore_heap_t::free_data(inode_t inode, uint64_t location)
         inode = (INODE_POOL(inode) << POOL_ID_BITS);
     assert(data_alloc->get(location / dsk->data_block_size));
     data_alloc->set(location / dsk->data_block_size, false);
-    inode_space_stats[inode] -= dsk->data_block_size;
+    auto sp_it = inode_space_stats.find(inode);
+    if (sp_it != inode_space_stats.end())
+    {
+        sp_it->second -= dsk->data_block_size;
+        if (sp_it->second == 0)
+            inode_space_stats.erase(sp_it);
+    }
     data_used_space -= dsk->data_block_size;
 }
 
@@ -2328,7 +2311,7 @@ void blockstore_heap_t::set_no_inode_stats(const std::vector<uint64_t> & pool_id
     {
         // Recalculate if changed
         if (ps.second.no_inode_stats == 2 || ps.second.no_inode_stats == 1)
-            recalc_inode_space_stats(ps.first, ps.second.no_inode_stats == 1);
+            recalc_inode_space_stats(ps.first, ps.second.no_inode_stats == 2);
         ps.second.no_inode_stats &= 1;
     }
 }
@@ -2339,8 +2322,8 @@ void blockstore_heap_t::recalc_inode_space_stats(uint64_t pool_id, bool per_inod
     auto sp_begin = inode_space_stats.lower_bound((pool_id << (64-POOL_ID_BITS)));
     auto sp_end = inode_space_stats.lower_bound(((pool_id+1) << (64-POOL_ID_BITS)));
     inode_space_stats.erase(sp_begin, sp_end);
-    uint32_t pg_count = ps.pg_count ? ps.pg_count : 1;
+    uint32_t pg_count = ps.pg_count;
-    for (uint32_t pg_num = 1; pg_num <= pg_count; pg_num++)
+    for (uint32_t pg_num = pg_count ? 1 : 0; pg_num <= pg_count; pg_num++)
     {
         auto & pg_idx = block_index[(pool_id << (64-POOL_ID_BITS)) | pg_num];
         for (auto & ip: pg_idx)

src/blockstore/blockstore_heap.h

@@ -163,7 +163,7 @@ using heap_mvcc_map_t = robin_hood::unordered_flat_map<object_id, heap_object_mv
 
 class blockstore_heap_t
 {
-    friend class heap_entry_t;
+    friend struct heap_entry_t;
 
     blockstore_disk_t *dsk = NULL;
     uint8_t* buffer_area = NULL;
@@ -248,20 +248,17 @@ public:
     void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
     bool reshard_continue(void* reshard_state, uint64_t chunk_limit);
     bool reshard_check(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size);
-    void reshard_abort(void* reshard_state);
     void set_no_inode_stats(const std::vector<uint64_t> & pool_ids);
     void recalc_inode_space_stats(uint64_t pool_id, bool per_inode);
     // read an object entry and lock it against removal
     // in the future, may become asynchronous
     heap_entry_t *lock_and_read_entry(object_id oid);
-    // re-read a locked object entry with the given lsn (pointer may be invalidated)
-    heap_entry_t *read_locked_entry(object_id oid, uint64_t lsn);
     // read an object entry without locking it
     heap_entry_t *read_entry(object_id oid);
     // unlock an entry
     bool unlock_entry(object_id oid);
     // set or verify checksums in a write request
-    bool calc_checksums(heap_entry_t *wr, uint8_t *data, bool set, uint32_t offset = 0, uint32_t len = 0);
+    bool calc_checksums(heap_entry_t *wr, uint8_t *data, bool set, uint32_t offset = UINT32_MAX, uint32_t len = UINT32_MAX);
     // set or verify raw block checksums
     bool calc_block_checksums(uint32_t *block_csums, uint8_t *data, uint8_t *bitmap, uint32_t start, uint32_t end,
         bool set, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);

src/blockstore/blockstore_impl.cpp

@@ -406,8 +406,3 @@ bool blockstore_impl_t::reshard_continue(void *reshard_state, uint64_t chunk_lim
 {
     return heap->reshard_continue(reshard_state, chunk_limit);
 }
-
-void blockstore_impl_t::reshard_abort(void *reshard_state)
-{
-    return heap->reshard_abort(reshard_state);
-}

src/blockstore/blockstore_impl.h

@@ -191,7 +191,6 @@ public:
 
     void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
     bool reshard_continue(void *reshard_state, uint64_t chunk_limit);
-    void reshard_abort(void *reshard_state);
 
     // Event loop
     void loop();

src/blockstore/blockstore_init.cpp

@@ -269,7 +269,7 @@ resume_6:
         }
         GET_SQE();
         data->iov = (iovec){ buf, len };
-        data->callback = [this, offset, cb](ring_data_t *data)
+        data->callback = [offset, cb](ring_data_t *data)
         {
             if (data->res < 0)
             {

src/blockstore/blockstore_stable.cpp

@@ -57,9 +57,9 @@ int blockstore_impl_t::dequeue_stable(blockstore_op_t *op)
             }
             assert(res == 0);
         }
+resume_1:
         if (priv->modified_block != UINT32_MAX && priv->modified_block2 != priv->modified_block)
         {
-resume_1:
             BS_SUBMIT_CHECK_SQES(1);
             prepare_meta_block_write(priv->modified_block);
 resume_2:

src/blockstore/v1/impl.cpp

@@ -480,17 +480,6 @@ resume_1:
     return true;
 }
 
-void blockstore_impl_t::reshard_abort(void *reshard_state)
-{
-    bs_reshard_state_t *st = (bs_reshard_state_t*)reshard_state;
-    for (auto sh_it = st->old_shards.begin(); sh_it != st->old_shards.end(); sh_it++)
-    {
-        auto & to = clean_db_shards[sh_it->first];
-        to.swap(sh_it->second);
-    }
-    delete st;
-}
-
 void blockstore_impl_t::process_list(blockstore_op_t *op)
 {
     uint32_t list_pg = op->pg_number+1;

src/blockstore/v1/impl.h

@@ -290,7 +290,6 @@ public:
     // Reshard database for a pool
     void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
     bool reshard_continue(void *reshard_state, uint64_t chunk_limit);
-    void reshard_abort(void *reshard_state);
 
     // Event loop
     void loop();

src/client/cluster_client.cpp

@@ -898,7 +898,7 @@ void cluster_client_t::execute_cas(cluster_op_t *op)
                         .opcode = OSD_OP_SYNC,
                     },
                 },
-                .callback = [this, op](osd_op_t *part)
+                .callback = [op](osd_op_t *part)
                 {
                     if (part->reply.hdr.retval != 0)
                     {
@@ -1257,7 +1257,7 @@ void cluster_client_t::slice_rw(cluster_op_t *op)
         {
             op->bitmap_buf = realloc_or_die(op->bitmap_buf, bitmap_mem);
             op->part_bitmaps = (uint8_t*)op->bitmap_buf + object_bitmap_size;
-            memset(op->bitmap_buf+op->bitmap_buf_size, 0, bitmap_mem-op->bitmap_buf_size);
+            memset((uint8_t*)op->bitmap_buf+op->bitmap_buf_size, 0, bitmap_mem-op->bitmap_buf_size);
             op->bitmap_buf_size = bitmap_mem;
         }
     }

src/client/msgr_receive.cpp

@@ -42,6 +42,7 @@ void osd_messenger_t::read_requests()
             }
             if (!sqe)
             {
+                cl->refs--;
                 cl->read_msg.msg_iovlen = 0;
                 read_ready_clients.erase(read_ready_clients.begin(), read_ready_clients.begin() + i);
                 return;

src/client/msgr_stop.cpp

@@ -85,6 +85,16 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
             osd_peer_fds.erase(osd_it);
         }
     }
+#ifdef WITH_RDMA
+    if (cl->rdma_conn && cl->rdma_conn->cmid)
+    {
+        auto rdma_it = rdmacm_connections.find(cl->rdma_conn->cmid);
+        if (rdma_it != rdmacm_connections.end() && rdma_it->second == cl)
+        {
+            rdmacm_connections.erase(rdma_it);
+        }
+    }
+#endif
 #ifndef __MOCK__
     // Then remove FD from the eventloop so we don't accidentally read something
     tfd->set_fd_handler(peer_fd, false, NULL);

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 3.0.2
+Version: 3.0.3
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/cmd/cli_dd.cpp

@@ -890,12 +890,12 @@ resume_2:
         clock_gettime(CLOCK_REALTIME, &tv_begin);
         tv_progress = tv_begin;
 resume_3:
-        while ((ignore_errors || !copy_error) && (!in_eof || read_buffers.size() || in_waiting > 0 || out_waiting > 0))
+        while ((ignore_errors || !copy_error) && (!in_eof || read_buffers.size() || in_waiting > 0 || out_waiting > 0 || short_writes.size()))
         {
             print_progress(false);
             while ((ignore_errors || !copy_error) &&
                 (!in_eof && in_waiting < in_iodepth && read_buffers.size() < out_iodepth ||
-                read_buffers.size() && out_waiting < out_iodepth))
+                (read_buffers.size() || short_writes.size()) && out_waiting < out_iodepth))
             {
                 if (!in_eof && in_waiting < in_iodepth && read_buffers.size() < out_iodepth)
                 {
@@ -904,7 +904,7 @@ resume_3:
                         break;
                     }
                 }
-                if (read_buffers.size() && out_waiting < out_iodepth)
+                if ((read_buffers.size() || short_writes.size()) && out_waiting < out_iodepth)
                 {
                     if (!add_write_op())
                     {

src/disk_tool/disk_tool_discard.cpp

@@ -54,7 +54,7 @@ int disk_tool_t::trim_data(std::string device)
     fprintf(stderr, "Reading metadata\n");
     data_alloc = new allocator_t(dsk.block_count);
     r = process_meta(
-        [this](blockstore_meta_header_v3_t *hdr) {},
+        [](blockstore_meta_header_v3_t *hdr) {},
         [this](blockstore_heap_t *heap, heap_entry_t *obj, uint32_t meta_block_num)
         {
             for (auto wr = obj; wr; wr = heap->prev(wr))

src/osd/osd_cluster.cpp

@@ -809,21 +809,17 @@ again:
         goto again;
     }
     auto & pool_cfg = pool_it->second;
-    bool done = false;
+    bool done = bs->reshard_continue(pool_cfg.reshard_state, pg_reshard_chunk_size);
-    if (pool_cfg.real_pg_count != pool_cfg.applied_pg_count ||
+    if (done &&
-        pool_cfg.pg_stripe_size != pool_cfg.applied_pg_stripe_size)
+        (pool_cfg.real_pg_count != pool_cfg.applied_pg_count ||
+        pool_cfg.pg_stripe_size != pool_cfg.applied_pg_stripe_size))
     {
         // PG count changed again, reshard again
-        bs->reshard_abort(pool_cfg.reshard_state);
         pool_cfg.applied_pg_count = pool_cfg.real_pg_count;
         pool_cfg.applied_pg_stripe_size = pool_cfg.pg_stripe_size;
         pool_cfg.reshard_state = bs->reshard_start(pool_id, pool_cfg.real_pg_count, pool_cfg.pg_stripe_size, pg_reshard_chunk_size);
         done = !pool_cfg.reshard_state;
     }
-    else
-    {
-        done = bs->reshard_continue(pool_cfg.reshard_state, pg_reshard_chunk_size);
-    }
     if (done)
     {
         // Pool is resharded

src/osd/osd_peering.cpp

@@ -584,7 +584,7 @@ void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
                     .opcode = OSD_OP_SEC_LIST,
                 },
                 .list_pg = ps->pg_num,
-                .pg_count = pool_cfg.applied_pg_count,
+                .pg_count = (uint32_t)pool_cfg.applied_pg_count,
                 .pg_stripe_size = pool_cfg.applied_pg_stripe_size,
                 .min_inode = ((uint64_t)(ps->pool_id) << (64 - POOL_ID_BITS)),
                 .max_inode = ((uint64_t)(ps->pool_id+1) << (64 - POOL_ID_BITS)) - 1,

src/test/ringloop_mock.cpp

@@ -199,7 +199,7 @@ void disk_mock_t::erase_buffers(uint64_t begin, uint64_t end)
         {
             // Cut beginning & end & stop
             uint8_t *ce = (uint8_t*)malloc_or_die(be-end);
-            memcpy(ce, it->second.iov_base + (end-bs), be-end);
+            memcpy(ce, (uint8_t*)it->second.iov_base + (end-bs), be-end);
             uint8_t *cs = (uint8_t*)realloc(it->second.iov_base, begin-bs);
             if (!cs)
                 throw std::bad_alloc();
@@ -221,7 +221,7 @@ void disk_mock_t::erase_buffers(uint64_t begin, uint64_t end)
             // Cut end & stop
             assert(be > end);
             uint8_t *ce = (uint8_t*)malloc_or_die(be-end);
-            memcpy(ce, it->second.iov_base + (end-bs), be-end);
+            memcpy(ce, (uint8_t*)it->second.iov_base + (end-bs), be-end);
             buffers[be] = (iovec){ .iov_base = ce, .iov_len = be-end };
             buffers.erase(it);
             break;
@@ -308,7 +308,7 @@ void disk_mock_t::read_item(uint8_t *to, uint64_t offset, uint64_t len)
             last = offset;
         }
         uint64_t cur_end = be < offset+len ? be : offset+len;
-        memcpy(to+last-offset, it->second.iov_base+last-bs, cur_end-last);
+        memcpy(to+last-offset, (uint8_t*)it->second.iov_base+last-bs, cur_end-last);
         last = be;
     }
     if (last < offset+len)

tests/test_small_block_size.sh

@@ -0,0 +1,47 @@
+#!/bin/bash -ex
+
+# Test for Issue #112: Integer division bug causes crash with block_size < 32KB
+# This test verifies that small block sizes (4KB, 8KB, 16KB) work correctly
+# with 4KB bitmap_granularity, both for aligned and unaligned I/O
+
+# Arrange: Set up test environment with small block sizes
+export SCHEME=replicated
+export OSD_COUNT=3
+export PG_COUNT=1
+export PG_SIZE=2
+export OSD_SIZE=256
+
+# Test with 16KB block_size and 4KB bitmap_granularity
+# This should trigger the bug: 16384 / 4096 / 8 = 4 / 8 = 0 bytes
+export OFFSET_ARGS="--data_block_size 16384 --bitmap_granularity 4096"
+
+. `dirname $0`/run_3osds.sh
+
+# Act: Run I/O tests that exercise the bitmap code paths
+
+echo "Test 1: 128KB aligned I/O (may work even with bug)"
+LD_PRELOAD="build/src/client/libfio_vitastor.so" \
+    fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so \
+        -bs=128k -direct=1 -iodepth=4 -rw=randwrite \
+        -etcd=$ETCD_URL -pool=1 -inode=1 -size=64M -runtime=5
+
+echo "Test 2: 4KB random I/O (will crash with bug)"
+LD_PRELOAD="build/src/client/libfio_vitastor.so" \
+    fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so \
+        -bs=4k -direct=1 -iodepth=16 -rw=randwrite \
+        -etcd=$ETCD_URL -pool=1 -inode=1 -size=64M -runtime=5
+
+echo "Test 3: Mixed read/write with 4KB I/O"
+LD_PRELOAD="build/src/client/libfio_vitastor.so" \
+    fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so \
+        -bs=4k -direct=1 -iodepth=16 -rw=randrw -rwmixread=50 \
+        -etcd=$ETCD_URL -pool=1 -inode=1 -size=64M -runtime=5
+
+echo "Test 4: Sequential 4KB writes"
+LD_PRELOAD="build/src/client/libfio_vitastor.so" \
+    fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so \
+        -bs=4k -direct=1 -iodepth=1 -rw=write \
+        -etcd=$ETCD_URL -pool=1 -inode=1 -size=64M
+
+# Assert: If we reach here without crash, test passed
+format_green "OK: 16KB block_size with 4KB bitmap_granularity works correctly"