Fix ref leak on read path when SQE allocation fails

CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "3.0.3")
+set(VITASTOR_VERSION "3.0.2")
 
 include(CTest)
 

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.3
+VITASTOR_VERSION ?= v3.0.2
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v3.0.3
+          image: vitalif/vitastor-csi:v3.0.2
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v3.0.3
+          image: vitalif/vitastor-csi:v3.0.2
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "3.0.3"
+    vitastorCSIDriverVersion = "3.0.2"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (3.0.3-1) unstable; urgency=medium
+vitastor (3.0.2-1) unstable; urgency=medium
 
   * Bugfixes
 

docker/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.3
+VITASTOR_VERSION ?= v3.0.2
 
 all: build push
 

docker/etc/vitastor/docker.conf

@@ -4,7 +4,7 @@
 #
 
 # Desired Vitastor version
-VITASTOR_VERSION=v3.0.3
+VITASTOR_VERSION=v3.0.2
 
 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here

docs/installation/docker.en.md

@@ -26,9 +26,9 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.
 
 1. Download a Docker image of the desired version: \
-   `docker pull vitalif/vitastor:v3.0.3`
+   `docker pull vitalif/vitastor:v3.0.2`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.3 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.2 install.sh`
 3. Reload udev rules: \
    `udevadm control --reload-rules`
 

docs/installation/docker.ru.md

@@ -25,9 +25,9 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.
 
 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitalif/vitastor:v3.0.3`
+   `docker pull vitalif/vitastor:v3.0.2`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.3 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.2 install.sh`
 3. Перезагрузите правила udev: \
    `udevadm control --reload-rules`
 

mon/lp_optimizer/lp_optimizer.js

@@ -10,19 +10,16 @@ const NO_OSD = 'Z';
 async function lp_solve(text)
 {
     const cp = child_process.spawn('lp_solve');
-    let stdout = '', stderr = '', finish_cb, finished = 0;
+    let stdout = '', stderr = '', finish_cb;
     cp.stdout.on('data', buf => stdout += buf.toString());
     cp.stderr.on('data', buf => stderr += buf.toString());
-    cp.stdout.on('end', () => finish_cb());
+    cp.on('exit', () => finish_cb && finish_cb());
-    cp.stderr.on('end', () => finish_cb());
     cp.stdin.write(text);
     cp.stdin.end();
-    await new Promise(ok => (finish_cb = () =>
+    if (cp.exitCode == null)
     {
-        finished++;
+        await new Promise(ok => finish_cb = ok);
-        if (finished == 2)
+    }
-            ok();
-    }));
     if (!stdout.trim())
     {
         return null;

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "3.0.3",
+  "version": "3.0.2",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {

mon/pg_gen.js

@@ -195,7 +195,7 @@ async function generate_pool_pgs(state, global_config, pool_id, osd_tree, levels
     const folded_tree = make_hier_tree(global_config, folded.nodes.reduce((a, c) => { a[c.id] = c; return a; }, {}));
     const old_pg_count = prev_pgs.length;
     const optimize_cfg = {
-        osd_weights: folded.nodes.reduce((a, c) => { if (/^\d+$/.exec(c.id) && c.size != null) { a[c.id] = c.size||0; } return a; }, {}),
+        osd_weights: folded.nodes.reduce((a, c) => { if (Number(c.id)) { a[c.id] = c.size; } return a; }, {}),
         combinator: use_rules
             // new algorithm:
             ? new RuleCombinator(folded_tree, rules, pool_cfg.max_osd_combinations)

node-binding/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor",
-  "version": "3.0.3",
+  "version": "3.0.2",
   "description": "Low-level native bindings to Vitastor client library",
   "main": "index.js",
   "keywords": [

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '3.0.3'
+VITASTOR_VERSION = '3.0.2'
 
 LOG = logging.getLogger(__name__)
 

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.3
+Version:        3.0.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.3.el7.tar.gz
+Source0:        vitastor-3.0.2.el7.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  devtoolset-9-gcc-c++

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.3
+Version:        3.0.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.3.el8.tar.gz
+Source0:        vitastor-3.0.2.el8.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-toolset-9-gcc-c++

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.3
+Version:        3.0.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.3.el9.tar.gz
+Source0:        vitastor-3.0.2.el9.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++

src/CMakeLists.txt

@@ -21,7 +21,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 endif()
 set(ENABLE_COVERAGE false CACHE BOOL "Enable code coverage")
 
-add_definitions(-DVITASTOR_VERSION="3.0.3")
+add_definitions(-DVITASTOR_VERSION="3.0.2")
 add_definitions(-D_GNU_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -fvisibility=hidden -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/blockstore/blockstore.h

@@ -187,6 +187,7 @@ public:
     // MUST be called only when nobody makes any modifications to the DB for this pool
     virtual void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit) = 0;
     virtual bool reshard_continue(void *reshard_state, uint64_t chunk_limit) = 0;
+    virtual void reshard_abort(void *reshard_state) = 0;
 
     // Event loop
     virtual void loop() = 0;

src/blockstore/blockstore_disk.cpp

@@ -171,12 +171,6 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     {
         throw std::runtime_error("Data block size must be a multiple of sparse write tracking granularity");
     }
-    if (data_block_size / bitmap_granularity < 8)
-    {
-        fprintf(stderr, "Warning: block_size (%u) / bitmap_granularity (%u) = %u bits. "
-            "Consider using larger block_size or bitmap_granularity for better performance.\n",
-            data_block_size, bitmap_granularity, data_block_size / bitmap_granularity);
-    }
     if (!data_csum_type)
     {
         csum_block_size = 0;
@@ -265,7 +259,7 @@ void blockstore_disk_t::calc_lengths(bool skip_meta_check)
     }
     // required metadata size
     block_count = data_len / data_block_size;
-    clean_entry_bitmap_size = (data_block_size / bitmap_granularity + 7) / 8;
+    clean_entry_bitmap_size = data_block_size / bitmap_granularity / 8;
     clean_dyn_size = clean_entry_bitmap_size*2 + (csum_block_size
         ? data_block_size/csum_block_size*(data_csum_type & 0xFF) : 0);
 recalc:

src/blockstore/blockstore_flush.h

@@ -58,6 +58,7 @@ class journal_flusher_co
     int i, res;
     bool read_to_fill_incomplete;
     int copy_count;
+    bool do_repeat = false;
 
     friend class journal_flusher_t;
 

src/blockstore/blockstore_heap.cpp

@@ -458,22 +458,19 @@ bool blockstore_heap_t::validate_object(heap_entry_t *obj)
         next_wr = wr;
         if (wr->type() == BS_HEAP_ROLLBACK)
         {
+            if (commit_wr && wr->version > commit_wr->version)
+            {
+                // rollback may not come before commit with a smaller version
+                fprintf(stderr, "Error: rollback entry %jx:%jx v%ju l%ju comes before a commit entry v%ju l%ju\n",
+                    wr->inode, wr->stripe, wr->version, wr->lsn, commit_wr->version, commit_wr->lsn);
+                return false;
+            }
             rollback_wr = wr;
             continue;
         }
         if (wr->type() == BS_HEAP_COMMIT)
         {
-            if (commit_wr && wr->version > commit_wr->version)
+            commit_wr = wr;
-            {
-                // commit may not come before commit with a smaller version
-                fprintf(stderr, "Error: commit entry %jx:%jx v%ju l%ju comes before a commit entry v%ju l%ju\n",
-                    wr->inode, wr->stripe, wr->version, wr->lsn, commit_wr->version, commit_wr->lsn);
-                return false;
-            }
-            if (!commit_wr)
-            {
-                commit_wr = wr;
-            }
             continue;
         }
         if (wr->entry_type & BS_HEAP_STABLE)
@@ -840,6 +837,7 @@ bool blockstore_heap_t::calc_checksums(heap_entry_t *wr, uint8_t *data, bool set
         {
             return true;
         }
+        uint32_t len = 0;
         if (wr->type() == BS_HEAP_SMALL_WRITE || wr->type() == BS_HEAP_INTENT_WRITE)
             len = wr->small().len;
         else if (wr->type() == BS_HEAP_BIG_INTENT)
@@ -856,14 +854,13 @@ bool blockstore_heap_t::calc_checksums(heap_entry_t *wr, uint8_t *data, bool set
     }
     if (wr->type() == BS_HEAP_BIG_WRITE)
     {
-        assert(offset != UINT32_MAX && len != UINT32_MAX);
         return calc_block_checksums((uint32_t*)(wr->get_checksums(this) + offset/dsk->csum_block_size * (dsk->data_csum_type & 0xFF)),
             data, wr->get_int_bitmap(this), offset, offset+len, set, NULL);
     }
     if (wr->type() == BS_HEAP_BIG_INTENT)
     {
         auto & bi = wr->big_intent();
-        return calc_block_checksums((uint32_t*)(wr->get_checksums(this) + bi.offset/dsk->csum_block_size * (dsk->data_csum_type & 0xFF)),
+        return calc_block_checksums((uint32_t*)(wr->get_checksums(this) + offset/dsk->csum_block_size * (dsk->data_csum_type & 0xFF)),
             data, wr->get_int_bitmap(this), bi.offset, bi.offset+bi.len, set, NULL);
     }
     assert(wr->type() == BS_HEAP_SMALL_WRITE || wr->type() == BS_HEAP_INTENT_WRITE);
@@ -1080,6 +1077,16 @@ bool blockstore_heap_t::reshard_check(pool_id_t pool, uint32_t pg_count, uint32_
         set_it->second.pg_stripe_size == pg_stripe_size);
 }
 
+void blockstore_heap_t::reshard_abort(void* reshard_state)
+{
+    heap_reshard_state_t *st = (heap_reshard_state_t*)reshard_state;
+    for (auto sh_it = st->old_shards.begin(); sh_it != st->old_shards.end(); sh_it++)
+    {
+        block_index[sh_it->first] = std::move(sh_it->second);
+    }
+    delete st;
+}
+
 heap_entry_t *blockstore_heap_t::lock_and_read_entry(object_id oid)
 {
     auto obj = read_entry(oid);
@@ -1092,6 +1099,27 @@ heap_entry_t *blockstore_heap_t::lock_and_read_entry(object_id oid)
     return obj;
 }
 
+heap_entry_t *blockstore_heap_t::read_locked_entry(object_id oid, uint64_t lsn)
+{
+    auto obj = read_entry(oid);
+    assert(obj);
+    for (auto wr = obj; wr; wr = prev(wr))
+    {
+        if (wr->is_overwrite())
+        {
+            if (lsn == wr->lsn)
+            {
+                return obj;
+            }
+            else
+            {
+                obj = prev(wr);
+            }
+        }
+    }
+    return NULL;
+}
+
 bool blockstore_heap_t::unlock_entry(object_id oid)
 {
     auto mvcc_it = object_mvcc.find(oid);
@@ -1390,7 +1418,7 @@ int blockstore_heap_t::add_redirect_intent(object_id oid, heap_entry_t **obj_ptr
         bitmap_set(wr->get_int_bitmap(this), offset, len, dsk->bitmap_granularity);
         if (dsk->data_csum_type)
             memset(wr->get_checksums(this), 0, get_csum_size(wr));
-        calc_checksums(wr, (uint8_t*)data, true);
+        calc_checksums(wr, (uint8_t*)data, true, offset, len);
         *obj_ptr = wr;
     });
 }
@@ -1429,11 +1457,11 @@ int blockstore_heap_t::add_big_intent(object_id oid, heap_entry_t **obj_ptr, uin
         if (dsk->data_csum_type)
         {
             if (checksums)
-                memcpy(wr->get_checksums(this), checksums, get_csum_size(wr));
+                memcpy(wr->get_checksums(this), checksums, dsk->clean_entry_bitmap_size);
             else
             {
-                memcpy(wr->get_checksums(this), obj->get_checksums(this), get_csum_size(wr));
+                memcpy(wr->get_checksums(this), obj->get_checksums(this), dsk->clean_entry_bitmap_size);
-                calc_checksums(wr, (uint8_t*)data, true);
+                calc_checksums(wr, (uint8_t*)data, true, offset, len);
             }
         }
         else
@@ -1554,7 +1582,7 @@ int blockstore_heap_t::add_commit(heap_entry_t *obj, uint64_t version, uint32_t
     }
     if (!uncommitted)
     {
-        return 0;
+        return EBUSY;
     }
     return add_simple(obj, version, modified_block, BS_HEAP_COMMIT);
 }
@@ -1564,32 +1592,23 @@ int blockstore_heap_t::add_rollback(heap_entry_t *obj, uint64_t version, uint32_
     heap_entry_t *wr = obj;
     bool found_uncommitted = false;
     uint64_t commit_version = 0;
-    uint64_t rollback_version = UINT64_MAX;
+    while (wr && !wr->is_overwrite())
-    while (wr)
     {
         if (wr->type() == BS_HEAP_ROLLBACK)
         {
-            if (wr->version <= version)
+            auto rollback_version = wr->version;
-            {
-                // All previous writes are already rolled back, stop
-                break;
-            }
-            rollback_version = wr->version;
             wr = prev(wr);
+            while (wr->version > rollback_version)
+            {
+                assert(!(wr->entry_type & BS_HEAP_STABLE));
+                wr = prev(wr);
+            }
             continue;
         }
         if (wr->type() == BS_HEAP_COMMIT)
         {
             if (commit_version < wr->version)
-            {
                 commit_version = wr->version;
-            }
-            wr = prev(wr);
-            continue;
-        }
-        if (wr->version > rollback_version)
-        {
-            // Already rolled back, skip
             wr = prev(wr);
             continue;
         }
@@ -1600,10 +1619,14 @@ int blockstore_heap_t::add_rollback(heap_entry_t *obj, uint64_t version, uint32_
             {
                 return EBUSY;
             }
-            else
+            else if (wr->version == version)
             {
                 break;
             }
+            else if (wr->version < version)
+            {
+                return ENOENT;
+            }
         }
         else if (wr->version > version)
         {
@@ -1784,9 +1807,9 @@ void blockstore_heap_t::iterate_with_stable(heap_entry_t *obj, uint64_t max_lsn,
         }
         else
         {
-            // 1) 1 2 3 ROLLBACK(2) COMMIT(3) -> 3 is unstable
+            // 1) 1 2 3 ROLLBACK(2) COMMIT(3) -> impossible
             // 2) 1 2 3 4 ROLLBACK(3) COMMIT(2) -> OK
-            // 3) 1 2 3 ROLLBACK(2) 3 COMMIT(3) -> first 3 is unstable
+            // 3) 1 2 3 ROLLBACK(2) 3 COMMIT(3) -> first 3 shouldn't be treated as stable
             // 4) 1 2 3 COMMIT(3) ROLLBACK(2) -> impossible
             //    I.e. a rollback always has version >= previous commit
             // 5) 1 2 3 4 5 ROLLBACK(4) 5 ROLLBACK(3)
@@ -2022,13 +2045,7 @@ void blockstore_heap_t::free_data(inode_t inode, uint64_t location)
         inode = (INODE_POOL(inode) << POOL_ID_BITS);
     assert(data_alloc->get(location / dsk->data_block_size));
     data_alloc->set(location / dsk->data_block_size, false);
-    auto sp_it = inode_space_stats.find(inode);
+    inode_space_stats[inode] -= dsk->data_block_size;
-    if (sp_it != inode_space_stats.end())
-    {
-        sp_it->second -= dsk->data_block_size;
-        if (sp_it->second == 0)
-            inode_space_stats.erase(sp_it);
-    }
     data_used_space -= dsk->data_block_size;
 }
 
@@ -2311,7 +2328,7 @@ void blockstore_heap_t::set_no_inode_stats(const std::vector<uint64_t> & pool_id
     {
         // Recalculate if changed
         if (ps.second.no_inode_stats == 2 || ps.second.no_inode_stats == 1)
-            recalc_inode_space_stats(ps.first, ps.second.no_inode_stats == 2);
+            recalc_inode_space_stats(ps.first, ps.second.no_inode_stats == 1);
         ps.second.no_inode_stats &= 1;
     }
 }
@@ -2322,8 +2339,8 @@ void blockstore_heap_t::recalc_inode_space_stats(uint64_t pool_id, bool per_inod
     auto sp_begin = inode_space_stats.lower_bound((pool_id << (64-POOL_ID_BITS)));
     auto sp_end = inode_space_stats.lower_bound(((pool_id+1) << (64-POOL_ID_BITS)));
     inode_space_stats.erase(sp_begin, sp_end);
-    uint32_t pg_count = ps.pg_count;
+    uint32_t pg_count = ps.pg_count ? ps.pg_count : 1;
-    for (uint32_t pg_num = pg_count ? 1 : 0; pg_num <= pg_count; pg_num++)
+    for (uint32_t pg_num = 1; pg_num <= pg_count; pg_num++)
     {
         auto & pg_idx = block_index[(pool_id << (64-POOL_ID_BITS)) | pg_num];
         for (auto & ip: pg_idx)

src/blockstore/blockstore_heap.h

@@ -163,7 +163,7 @@ using heap_mvcc_map_t = robin_hood::unordered_flat_map<object_id, heap_object_mv
 
 class blockstore_heap_t
 {
-    friend struct heap_entry_t;
+    friend class heap_entry_t;
 
     blockstore_disk_t *dsk = NULL;
     uint8_t* buffer_area = NULL;
@@ -248,17 +248,20 @@ public:
     void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
     bool reshard_continue(void* reshard_state, uint64_t chunk_limit);
     bool reshard_check(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size);
+    void reshard_abort(void* reshard_state);
     void set_no_inode_stats(const std::vector<uint64_t> & pool_ids);
     void recalc_inode_space_stats(uint64_t pool_id, bool per_inode);
     // read an object entry and lock it against removal
     // in the future, may become asynchronous
     heap_entry_t *lock_and_read_entry(object_id oid);
+    // re-read a locked object entry with the given lsn (pointer may be invalidated)
+    heap_entry_t *read_locked_entry(object_id oid, uint64_t lsn);
     // read an object entry without locking it
     heap_entry_t *read_entry(object_id oid);
     // unlock an entry
     bool unlock_entry(object_id oid);
     // set or verify checksums in a write request
-    bool calc_checksums(heap_entry_t *wr, uint8_t *data, bool set, uint32_t offset = UINT32_MAX, uint32_t len = UINT32_MAX);
+    bool calc_checksums(heap_entry_t *wr, uint8_t *data, bool set, uint32_t offset = 0, uint32_t len = 0);
     // set or verify raw block checksums
     bool calc_block_checksums(uint32_t *block_csums, uint8_t *data, uint8_t *bitmap, uint32_t start, uint32_t end,
         bool set, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);

src/blockstore/blockstore_impl.cpp

@@ -406,3 +406,8 @@ bool blockstore_impl_t::reshard_continue(void *reshard_state, uint64_t chunk_lim
 {
     return heap->reshard_continue(reshard_state, chunk_limit);
 }
+
+void blockstore_impl_t::reshard_abort(void *reshard_state)
+{
+    return heap->reshard_abort(reshard_state);
+}

src/blockstore/blockstore_impl.h

@@ -191,6 +191,7 @@ public:
 
     void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
     bool reshard_continue(void *reshard_state, uint64_t chunk_limit);
+    void reshard_abort(void *reshard_state);
 
     // Event loop
     void loop();

src/blockstore/blockstore_init.cpp

@@ -269,7 +269,7 @@ resume_6:
         }
         GET_SQE();
         data->iov = (iovec){ buf, len };
-        data->callback = [offset, cb](ring_data_t *data)
+        data->callback = [this, offset, cb](ring_data_t *data)
         {
             if (data->res < 0)
             {

src/blockstore/blockstore_stable.cpp

@@ -57,9 +57,9 @@ int blockstore_impl_t::dequeue_stable(blockstore_op_t *op)
             }
             assert(res == 0);
         }
-resume_1:
         if (priv->modified_block != UINT32_MAX && priv->modified_block2 != priv->modified_block)
         {
+resume_1:
             BS_SUBMIT_CHECK_SQES(1);
             prepare_meta_block_write(priv->modified_block);
 resume_2:

src/blockstore/v1/impl.cpp

@@ -480,6 +480,17 @@ resume_1:
     return true;
 }
 
+void blockstore_impl_t::reshard_abort(void *reshard_state)
+{
+    bs_reshard_state_t *st = (bs_reshard_state_t*)reshard_state;
+    for (auto sh_it = st->old_shards.begin(); sh_it != st->old_shards.end(); sh_it++)
+    {
+        auto & to = clean_db_shards[sh_it->first];
+        to.swap(sh_it->second);
+    }
+    delete st;
+}
+
 void blockstore_impl_t::process_list(blockstore_op_t *op)
 {
     uint32_t list_pg = op->pg_number+1;

src/blockstore/v1/impl.h

@@ -290,6 +290,7 @@ public:
     // Reshard database for a pool
     void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
     bool reshard_continue(void *reshard_state, uint64_t chunk_limit);
+    void reshard_abort(void *reshard_state);
 
     // Event loop
     void loop();

src/client/cluster_client.cpp

@@ -898,7 +898,7 @@ void cluster_client_t::execute_cas(cluster_op_t *op)
                         .opcode = OSD_OP_SYNC,
                     },
                 },
-                .callback = [op](osd_op_t *part)
+                .callback = [this, op](osd_op_t *part)
                 {
                     if (part->reply.hdr.retval != 0)
                     {
@@ -1257,7 +1257,7 @@ void cluster_client_t::slice_rw(cluster_op_t *op)
         {
             op->bitmap_buf = realloc_or_die(op->bitmap_buf, bitmap_mem);
             op->part_bitmaps = (uint8_t*)op->bitmap_buf + object_bitmap_size;
-            memset((uint8_t*)op->bitmap_buf+op->bitmap_buf_size, 0, bitmap_mem-op->bitmap_buf_size);
+            memset(op->bitmap_buf+op->bitmap_buf_size, 0, bitmap_mem-op->bitmap_buf_size);
             op->bitmap_buf_size = bitmap_mem;
         }
     }

src/client/messenger.cpp

@@ -583,7 +583,13 @@ void osd_messenger_t::handle_peer_epoll(int peer_fd, int epoll_events)
 
 void osd_messenger_t::on_connect_peer(osd_num_t peer_osd, int peer_fd)
 {
-    auto & wp = wanted_peers.at(peer_osd);
+    auto wp_it = wanted_peers.find(peer_osd);
+    if (wp_it == wanted_peers.end())
+    {
+        fprintf(stderr, "on_connect_peer: no wanted peer entry for OSD %ju\n", peer_osd);
+        return;
+    }
+    auto & wp = wp_it->second;
     wp.connecting = false;
     if (peer_fd < 0)
     {

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 3.0.3
+Version: 3.0.2
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/cmd/cli_dd.cpp

@@ -890,12 +890,12 @@ resume_2:
         clock_gettime(CLOCK_REALTIME, &tv_begin);
         tv_progress = tv_begin;
 resume_3:
-        while ((ignore_errors || !copy_error) && (!in_eof || read_buffers.size() || in_waiting > 0 || out_waiting > 0 || short_writes.size()))
+        while ((ignore_errors || !copy_error) && (!in_eof || read_buffers.size() || in_waiting > 0 || out_waiting > 0))
         {
             print_progress(false);
             while ((ignore_errors || !copy_error) &&
                 (!in_eof && in_waiting < in_iodepth && read_buffers.size() < out_iodepth ||
-                (read_buffers.size() || short_writes.size()) && out_waiting < out_iodepth))
+                read_buffers.size() && out_waiting < out_iodepth))
             {
                 if (!in_eof && in_waiting < in_iodepth && read_buffers.size() < out_iodepth)
                 {
@@ -904,7 +904,7 @@ resume_3:
                         break;
                     }
                 }
-                if ((read_buffers.size() || short_writes.size()) && out_waiting < out_iodepth)
+                if (read_buffers.size() && out_waiting < out_iodepth)
                 {
                     if (!add_write_op())
                     {

src/disk_tool/disk_tool_discard.cpp

@@ -54,7 +54,7 @@ int disk_tool_t::trim_data(std::string device)
     fprintf(stderr, "Reading metadata\n");
     data_alloc = new allocator_t(dsk.block_count);
     r = process_meta(
-        [](blockstore_meta_header_v3_t *hdr) {},
+        [this](blockstore_meta_header_v3_t *hdr) {},
         [this](blockstore_heap_t *heap, heap_entry_t *obj, uint32_t meta_block_num)
         {
             for (auto wr = obj; wr; wr = heap->prev(wr))

src/osd/osd.h

@@ -328,8 +328,8 @@ class osd_t
     void exec_show_config(osd_op_t *cur_op);
     void exec_secondary(osd_op_t *cur_op);
     void exec_secondary_real(osd_op_t *cur_op);
-    void exec_sec_read_bmp(osd_op_t *cur_op);
+    void exec_sec_read_bmp(osd_op_t *cur_op, osd_client_t *cl);
-    void exec_sec_lock(osd_op_t *cur_op);
+    void exec_sec_lock(osd_op_t *cur_op, osd_client_t *cl);
     void secondary_op_callback(osd_op_t *cur_op);
 
     // primary ops

src/osd/osd_cluster.cpp

@@ -809,17 +809,21 @@ again:
         goto again;
     }
     auto & pool_cfg = pool_it->second;
-    bool done = bs->reshard_continue(pool_cfg.reshard_state, pg_reshard_chunk_size);
+    bool done = false;
-    if (done &&
+    if (pool_cfg.real_pg_count != pool_cfg.applied_pg_count ||
-        (pool_cfg.real_pg_count != pool_cfg.applied_pg_count ||
+        pool_cfg.pg_stripe_size != pool_cfg.applied_pg_stripe_size)
-        pool_cfg.pg_stripe_size != pool_cfg.applied_pg_stripe_size))
     {
         // PG count changed again, reshard again
+        bs->reshard_abort(pool_cfg.reshard_state);
         pool_cfg.applied_pg_count = pool_cfg.real_pg_count;
         pool_cfg.applied_pg_stripe_size = pool_cfg.pg_stripe_size;
         pool_cfg.reshard_state = bs->reshard_start(pool_id, pool_cfg.real_pg_count, pool_cfg.pg_stripe_size, pg_reshard_chunk_size);
         done = !pool_cfg.reshard_state;
     }
+    else
+    {
+        done = bs->reshard_continue(pool_cfg.reshard_state, pg_reshard_chunk_size);
+    }
     if (done)
     {
         // Pool is resharded

src/osd/osd_peering.cpp

@@ -584,7 +584,7 @@ void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
                     .opcode = OSD_OP_SEC_LIST,
                 },
                 .list_pg = ps->pg_num,
-                .pg_count = (uint32_t)pool_cfg.applied_pg_count,
+                .pg_count = pool_cfg.applied_pg_count,
                 .pg_stripe_size = pool_cfg.applied_pg_stripe_size,
                 .min_inode = ((uint64_t)(ps->pool_id) << (64 - POOL_ID_BITS)),
                 .max_inode = ((uint64_t)(ps->pool_id+1) << (64 - POOL_ID_BITS)) - 1,

src/osd/osd_secondary.cpp

@@ -112,6 +112,13 @@ bool osd_t::sec_check_pg_lock(osd_num_t primary_osd, const object_id &oid, uint3
 
 void osd_t::exec_secondary_real(osd_op_t *cur_op)
 {
+    auto cl_it = msgr.clients.find(cur_op->peer_fd);
+    if (cl_it == msgr.clients.end())
+    {
+        finish_op(cur_op, -EPIPE);
+        return;
+    }
+    auto cl = cl_it->second;
     if (cur_op->req.hdr.opcode == OSD_OP_SEC_LIST &&
         (cur_op->req.sec_list.flags & OSD_LIST_PRIMARY))
     {
@@ -120,15 +127,14 @@ void osd_t::exec_secondary_real(osd_op_t *cur_op)
     }
     if (cur_op->req.hdr.opcode == OSD_OP_SEC_READ_BMP)
     {
-        exec_sec_read_bmp(cur_op);
+        exec_sec_read_bmp(cur_op, cl);
         return;
     }
     else if (cur_op->req.hdr.opcode == OSD_OP_SEC_LOCK)
     {
-        exec_sec_lock(cur_op);
+        exec_sec_lock(cur_op, cl);
         return;
     }
-    auto cl = msgr.clients.at(cur_op->peer_fd);
     cur_op->bs_op = new blockstore_op_t();
     cur_op->bs_op->callback = [this, cur_op](blockstore_op_t* bs_op) { secondary_op_callback(cur_op); };
     cur_op->bs_op->opcode = (cur_op->req.hdr.opcode == OSD_OP_SEC_READ ? BS_OP_READ
@@ -247,9 +253,8 @@ void osd_t::exec_secondary_real(osd_op_t *cur_op)
 #endif
 }
 
-void osd_t::exec_sec_read_bmp(osd_op_t *cur_op)
+void osd_t::exec_sec_read_bmp(osd_op_t *cur_op, osd_client_t *cl)
 {
-    auto cl = msgr.clients.at(cur_op->peer_fd);
     int n = cur_op->req.sec_read_bmp.len / sizeof(obj_ver_id);
     if (n > 0)
     {
@@ -275,10 +280,9 @@ void osd_t::exec_sec_read_bmp(osd_op_t *cur_op)
 }
 
 // Lock/Unlock PG
-void osd_t::exec_sec_lock(osd_op_t *cur_op)
+void osd_t::exec_sec_lock(osd_op_t *cur_op, osd_client_t *cl)
 {
     cur_op->reply.sec_lock.cur_primary = 0;
-    auto cl = msgr.clients.at(cur_op->peer_fd);
     if (!cl->in_osd_num ||
         cur_op->req.sec_lock.flags != OSD_SEC_LOCK_PG &&
         cur_op->req.sec_lock.flags != OSD_SEC_UNLOCK_PG ||
@@ -340,7 +344,13 @@ void osd_t::exec_show_config(osd_op_t *cur_op)
         ? json11::Json::parse(std::string((char *)cur_op->buf), json_err)
         : json11::Json();
     auto peer_osd_num = req_json["osd_num"].uint64_value();
-    auto cl = msgr.clients.at(cur_op->peer_fd);
+    auto cl_it = msgr.clients.find(cur_op->peer_fd);
+    if (cl_it == msgr.clients.end())
+    {
+        finish_op(cur_op, -EPIPE);
+        return;
+    }
+    auto cl = cl_it->second;
     cl->in_osd_num = peer_osd_num;
     if (req_json["features"]["check_sequencing"].bool_value())
     {

src/test/ringloop_mock.cpp

@@ -199,7 +199,7 @@ void disk_mock_t::erase_buffers(uint64_t begin, uint64_t end)
         {
             // Cut beginning & end & stop
             uint8_t *ce = (uint8_t*)malloc_or_die(be-end);
-            memcpy(ce, (uint8_t*)it->second.iov_base + (end-bs), be-end);
+            memcpy(ce, it->second.iov_base + (end-bs), be-end);
             uint8_t *cs = (uint8_t*)realloc(it->second.iov_base, begin-bs);
             if (!cs)
                 throw std::bad_alloc();
@@ -221,7 +221,7 @@ void disk_mock_t::erase_buffers(uint64_t begin, uint64_t end)
             // Cut end & stop
             assert(be > end);
             uint8_t *ce = (uint8_t*)malloc_or_die(be-end);
-            memcpy(ce, (uint8_t*)it->second.iov_base + (end-bs), be-end);
+            memcpy(ce, it->second.iov_base + (end-bs), be-end);
             buffers[be] = (iovec){ .iov_base = ce, .iov_len = be-end };
             buffers.erase(it);
             break;
@@ -308,7 +308,7 @@ void disk_mock_t::read_item(uint8_t *to, uint64_t offset, uint64_t len)
             last = offset;
         }
         uint64_t cur_end = be < offset+len ? be : offset+len;
-        memcpy(to+last-offset, (uint8_t*)it->second.iov_base+last-bs, cur_end-last);
+        memcpy(to+last-offset, it->second.iov_base+last-bs, cur_end-last);
         last = be;
     }
     if (last < offset+len)

tests/test_small_block_size.sh

@@ -1,47 +0,0 @@
-#!/bin/bash -ex
-
-# Test for Issue #112: Integer division bug causes crash with block_size < 32KB
-# This test verifies that small block sizes (4KB, 8KB, 16KB) work correctly
-# with 4KB bitmap_granularity, both for aligned and unaligned I/O
-
-# Arrange: Set up test environment with small block sizes
-export SCHEME=replicated
-export OSD_COUNT=3
-export PG_COUNT=1
-export PG_SIZE=2
-export OSD_SIZE=256
-
-# Test with 16KB block_size and 4KB bitmap_granularity
-# This should trigger the bug: 16384 / 4096 / 8 = 4 / 8 = 0 bytes
-export OFFSET_ARGS="--data_block_size 16384 --bitmap_granularity 4096"
-
-. `dirname $0`/run_3osds.sh
-
-# Act: Run I/O tests that exercise the bitmap code paths
-
-echo "Test 1: 128KB aligned I/O (may work even with bug)"
-LD_PRELOAD="build/src/client/libfio_vitastor.so" \
-    fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so \
-        -bs=128k -direct=1 -iodepth=4 -rw=randwrite \
-        -etcd=$ETCD_URL -pool=1 -inode=1 -size=64M -runtime=5
-
-echo "Test 2: 4KB random I/O (will crash with bug)"
-LD_PRELOAD="build/src/client/libfio_vitastor.so" \
-    fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so \
-        -bs=4k -direct=1 -iodepth=16 -rw=randwrite \
-        -etcd=$ETCD_URL -pool=1 -inode=1 -size=64M -runtime=5
-
-echo "Test 3: Mixed read/write with 4KB I/O"
-LD_PRELOAD="build/src/client/libfio_vitastor.so" \
-    fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so \
-        -bs=4k -direct=1 -iodepth=16 -rw=randrw -rwmixread=50 \
-        -etcd=$ETCD_URL -pool=1 -inode=1 -size=64M -runtime=5
-
-echo "Test 4: Sequential 4KB writes"
-LD_PRELOAD="build/src/client/libfio_vitastor.so" \
-    fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so \
-        -bs=4k -direct=1 -iodepth=1 -rw=write \
-        -etcd=$ETCD_URL -pool=1 -inode=1 -size=64M
-
-# Assert: If we reach here without crash, test passed
-format_green "OK: 16KB block_size with 4KB bitmap_granularity works correctly"