Release 3.0.11

A single hotfix:

- Fix missing error handling in EC snapshot/cloned image reads potentially
  leading to corrupted reads during OSD/PG restarts

CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 2.8...3.30)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "3.0.8")
+set(VITASTOR_VERSION "3.0.11")
 
 include(CTest)
 

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.8
+VITASTOR_VERSION ?= v3.0.11
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v3.0.8
+          image: vitalif/vitastor-csi:v3.0.11
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v3.0.8
+          image: vitalif/vitastor-csi:v3.0.11
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "3.0.8"
+    vitastorCSIDriverVersion = "3.0.11"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (3.0.8-1) unstable; urgency=medium
+vitastor (3.0.11-1) unstable; urgency=medium
 
   * Bugfixes
 

docker/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.8
+VITASTOR_VERSION ?= v3.0.11
 
 all: build push
 

docker/etc/vitastor/docker.conf

@@ -4,7 +4,7 @@
 #
 
 # Desired Vitastor version
-VITASTOR_VERSION=v3.0.8
+VITASTOR_VERSION=v3.0.11
 
 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here

docs/config/osd.en.md

@@ -70,6 +70,7 @@ with an OSD restart or, for some of them, even without restarting by updating co
 - [use_atomic_flag](#use_atomic_flag)
 - [pg_reshard_chunk_size](#pg_reshard_chunk_size)
 - [pg_reshard_chunk_pause_ms](#pg_reshard_chunk_pause_ms)
+- [gc_on_start](#gc_on_start)
 
 ## bind_address
 
@@ -753,3 +754,9 @@ This option sets the maximum number of object is a chunk. Moving 100k objects us
 - Default: 100
 
 This option sets the interval between handling two PG count change chunks.
+
+## gc_on_start
+
+- Type: boolean
+
+Forcibly clean all garbage entries in the new store on every OSD restart.

docs/config/osd.ru.md

@@ -71,6 +71,7 @@
 - [use_atomic_flag](#use_atomic_flag)
 - [pg_reshard_chunk_size](#pg_reshard_chunk_size)
 - [pg_reshard_chunk_pause_ms](#pg_reshard_chunk_pause_ms)
+- [gc_on_start](#gc_on_start)
 
 ## bind_address
 
@@ -793,3 +794,9 @@ pg_minsize OSD во время переключений, что может по
 - Значение по умолчанию: 100
 
 Данная опция задаёт интервал между обработкой двух порций изменения числа PG пулов.
+
+## gc_on_start
+
+- Тип: булево (да/нет)
+
+Принудительно очищать все мусорные записи в новом хранилище при каждом запуске OSD.

docs/config/src/osd.yml

@@ -938,3 +938,7 @@
     This option sets the interval between handling two PG count change chunks.
   info_ru: |
     Данная опция задаёт интервал между обработкой двух порций изменения числа PG пулов.
+- name: gc_on_start
+  type: bool
+  info: Forcibly clean all garbage entries in the new store on every OSD restart.
+  info_ru: Принудительно очищать все мусорные записи в новом хранилище при каждом запуске OSD.

docs/installation/docker.en.md

@@ -26,9 +26,9 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.
 
 1. Download a Docker image of the desired version: \
-   `docker pull vitalif/vitastor:v3.0.8`
+   `docker pull vitalif/vitastor:v3.0.11`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.8 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.11 install.sh`
 3. Reload udev rules: \
    `udevadm control --reload-rules`
 4. Enable the vitastor-host service: \

docs/installation/docker.ru.md

@@ -25,9 +25,9 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.
 
 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitalif/vitastor:v3.0.8`
+   `docker pull vitalif/vitastor:v3.0.11`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.8 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.11 install.sh`
 3. Перезагрузите правила udev: \
    `udevadm control --reload-rules`
 4. Включите сервис vitastor-host: \

docs/usage/nfs.en.md

@@ -262,3 +262,4 @@ Options:
 | `--logfile <FILE>` | log to the specified file                                |
 | `--enforce 1`      | enforce permissions at the server side (no by default)   |
 | `--foreground 1`   | stay in foreground, do not daemonize                     |
+| `--trace`          | trace all NFS requests                                   |

docs/usage/nfs.ru.md

@@ -274,3 +274,4 @@ VitastorFS из GPUDirect.
 | `--logfile <FILE>` | записывать логи в заданный файл                         |
 | `--enforce 1`      | проверять права доступа на стороне сервера (по умолчанию нет) |
 | `--foreground 1`   | не уходить в фон после запуска                          |
+| `--trace`          | логгировать все запросы NFS                             |

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "3.0.8",
+  "version": "3.0.11",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {

mon/stats.js

@@ -37,6 +37,7 @@ function derive_osd_stats(st, prev, prev_diff)
         const n = c.count - BigInt(pr && pr.count||0);
         diff.recovery_stats[op] = { ...c, bps: n > 0 ? b*1000n/timediff : 0n, iops: n > 0 ? n*1000n/timediff : 0n };
     }
+    diff.inode_stats = {};
     for (const pool_id in st.inode_stats||{})
     {
         diff.inode_stats[pool_id] = {};

node-binding/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor",
-  "version": "3.0.8",
+  "version": "3.0.11",
   "description": "Low-level native bindings to Vitastor client library",
   "main": "index.js",
   "keywords": [

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '3.0.8'
+VITASTOR_VERSION = '3.0.11'
 
 LOG = logging.getLogger(__name__)
 

rpm/vitastor-el10.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.8
+Version:        3.0.11
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.8.el10.tar.gz
+Source0:        vitastor-3.0.11.el10.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.8
+Version:        3.0.11
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.8.el7.tar.gz
+Source0:        vitastor-3.0.11.el7.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  devtoolset-9-gcc-c++

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.8
+Version:        3.0.11
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.8.el8.tar.gz
+Source0:        vitastor-3.0.11.el8.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-toolset-9-gcc-c++

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.8
+Version:        3.0.11
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.8.el9.tar.gz
+Source0:        vitastor-3.0.11.el9.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++

src/CMakeLists.txt

@@ -20,7 +20,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 endif()
 set(ENABLE_COVERAGE false CACHE BOOL "Enable code coverage")
 
-add_definitions(-DVITASTOR_VERSION="3.0.8")
+add_definitions(-DVITASTOR_VERSION="3.0.11")
 add_definitions(-D_GNU_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -fvisibility=hidden -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/blockstore/blockstore.h

@@ -228,4 +228,9 @@ public:
     virtual uint64_t get_journal_size() = 0;
 
     virtual uint32_t get_bitmap_granularity() = 0;
+
+    virtual uint64_t get_live_entries() = 0;
+    virtual uint64_t get_live_memory() = 0;
+    virtual uint64_t get_garbage_entries() = 0;
+    virtual uint64_t get_garbage_memory() = 0;
 };

src/blockstore/blockstore_disk.cpp

@@ -94,6 +94,8 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     csum_block_size = parse_size(config["csum_block_size"]);
     discard_on_start = config.find("discard_on_start") != config.end() &&
         (config["discard_on_start"] == "true" || config["discard_on_start"] == "1" || config["discard_on_start"] == "yes");
+    gc_on_start = config.find("gc_on_start") == config.end() ||
+        (config["gc_on_start"] == "true" || config["gc_on_start"] == "1" || config["gc_on_start"] == "yes");
     min_discard_size = parse_size(config["min_discard_size"]);
     if (!min_discard_size)
         min_discard_size = 1024*1024;

src/blockstore/blockstore_disk.h

@@ -57,6 +57,8 @@ struct blockstore_disk_t
     bool inmemory_journal = true;
     // Data discard granularity and minimum size (for the sake of performance)
     bool discard_on_start = false;
+    // GC on start (new store)
+    bool gc_on_start = true;
     uint64_t min_discard_size = 1024*1024;
     uint64_t discard_granularity = 0;
 

src/blockstore/blockstore_heap.cpp

@@ -29,6 +29,8 @@
 #define IMAP_MALLOC_LOW_BITS ((size_t)0x0F)
 #define IMAP_MAX_LOW 16
 
+#define list_item_overhead(a) (((a) + sizeof(heap_list_item_t) - sizeof(heap_entry_t) + sizeof(void*) + 15) & ~15)
+
 void inode_map_put(void* & inode_idx, heap_list_item_t* li);
 void inode_map_get(void *inode_idx, heap_inode_map_t::iterator & li_it, heap_list_item_t* & li, uint64_t stripe);
 void inode_map_free(void* inode_idx);
@@ -244,7 +246,7 @@ blockstore_heap_t::blockstore_heap_t(blockstore_disk_t *dsk, uint8_t *buffer_are
     assert(dsk->meta_block_size < 32768);
     assert(dsk->meta_area_size > 0);
     assert(dsk->journal_len > 0);
-    meta_alloc = new multilist_index_t(meta_block_count, META_ALLOC_LEVELS+1, 0);
+    meta_alloc = new multilist_index_t(meta_block_count, META_ALLOC_LEVELS+1, 2);
     block_info.resize(meta_block_count);
     assert(dsk->block_count <= 0xFFFF0000);
     data_alloc = new allocator_t(dsk->block_count);
@@ -332,19 +334,12 @@ corrupted_block:
                     block_num, block_offset, wr->size, sizeof(heap_entry_t));
                 goto corrupted_block;
             }
-            if (wr->is_garbage())
-            {
-                // Garbage collection is only performed when writing new entries into the block
-                // because it needs a fake LSN and modified blocks require consecutive modified LSNs
-                // That's why garbage entries may persist on disk
-                if (log_level > 5)
-                {
-                    fprintf(stderr, "Notice: skipping garbage entry %jx:%jx v%ju l%ju in metadata block %u at %u\n",
-                        wr->inode, wr->stripe, wr->version, wr->lsn, block_num, block_offset);
-                }
-                block_offset += wr->size;
-                continue;
-            }
+            // Garbage collection is only performed when writing new entries into the block
+            // because it needs a fake LSN and modified blocks require consecutive modified LSNs
+            // At the same time, further modifications _after_ putting new entries into the block,
+            // but _before_ writing it, may mark some entries in it as garbage. That's why garbage
+            // entries may still be present on disk.
+            wr->entry_type &= ~BS_HEAP_GARBAGE;
             if ((wr->entry_type & BS_HEAP_TYPE) < BS_HEAP_BIG_WRITE ||
                 (wr->entry_type & BS_HEAP_TYPE) > BS_HEAP_ROLLBACK ||
                 (wr->entry_type & ~(BS_HEAP_TYPE|BS_HEAP_STABLE)) ||
@@ -426,7 +421,10 @@ int blockstore_heap_t::load_blocks(uint64_t disk_offset, uint64_t size, uint8_t
     entries_loaded = 0;
     return read_blocks(disk_offset, size, buf, allow_corrupted, [&](uint32_t block_num, heap_entry_t *wr_orig)
     {
-        heap_list_item_t *li = (heap_list_item_t*)malloc_or_die(wr_orig->size + sizeof(heap_list_item_t) - sizeof(heap_entry_t));
+        auto alloc_size = wr_orig->size + sizeof(heap_list_item_t) - sizeof(heap_entry_t);
+        heap_list_item_t *li = (heap_list_item_t*)malloc_or_die(alloc_size);
+        live_entries++;
+        live_memory += list_item_overhead(wr_orig->size);
         li->block_num = block_num;
         li->prev = li->next = NULL;
         memcpy(&li->entry, wr_orig, wr_orig->size);
@@ -611,10 +609,11 @@ int blockstore_heap_t::mark_used_blocks()
                 if (wr->entry_type == (BS_HEAP_DELETE|BS_HEAP_STABLE) && !li->prev)
                 {
                     wr->set_garbage();
+                    garbage_entries++;
+                    garbage_memory += list_item_overhead(wr->size);
                     modify_alloc(li->block_num, [&](heap_block_info_t & inf)
                     {
-                        inf.used_space -= wr->size;
-                        inf.has_garbage = true;
+                        inf.garbage_space += wr->size;
                     });
                     li = NULL;
                 }
@@ -624,10 +623,11 @@ int blockstore_heap_t::mark_used_blocks()
                     if (overwritten)
                     {
                         wr->set_garbage();
+                        garbage_entries++;
+                        garbage_memory += list_item_overhead(wr->size);
                         modify_alloc(li->block_num, [&](heap_block_info_t & inf)
                         {
-                            inf.used_space -= wr->size;
-                            inf.has_garbage = true;
+                            inf.garbage_space += wr->size;
                         });
                         continue;
                     }
@@ -666,15 +666,66 @@ int blockstore_heap_t::mark_used_blocks()
             });
         }
     }
+    if (dsk->gc_on_start)
+    {
+        recheck_full_gc();
+    }
     return res;
 }
 
+void blockstore_heap_t::recheck_full_gc()
+{
+    uint32_t block_num = 0;
+    for (auto & inf: block_info)
+    {
+        // Instantly collect all garbage on restart
+        if (inf.garbage_space > 0)
+        {
+            if (log_level > 5)
+            {
+                fprintf(stderr, "Clearing %u out of %u garbage bytes in block %u\n", inf.garbage_space, inf.used_space, block_num);
+            }
+            uint32_t collected_garbage = 0;
+            size_t i = 0, j = 0;
+            for (; i < inf.entries.size(); i++)
+            {
+                if (inf.entries[i]->entry.is_garbage())
+                {
+                    collected_garbage += inf.entries[i]->entry.size;
+                    remove_list_item(inf.entries[i]);
+                }
+                else
+                {
+                    if (j != i)
+                        inf.entries[j] = inf.entries[i];
+                    j++;
+                }
+            }
+            inf.entries.resize(j);
+            modify_alloc(block_num, [&](heap_block_info_t & inf)
+            {
+                inf.used_space -= collected_garbage;
+                inf.garbage_space -= collected_garbage;
+            });
+            recheck_modified_blocks.insert(block_num);
+        }
+        block_num++;
+    }
+}
+
 void blockstore_heap_t::recheck_buffer(heap_entry_t *cwr, uint8_t *buf)
 {
     auto free_entry = [&](heap_list_item_t *li)
     {
         uint32_t block_num = li->block_num;
         auto wr_size = li->entry.size;
+        if (li->entry.is_garbage())
+        {
+            garbage_entries--;
+            garbage_memory -= list_item_overhead(wr_size);
+        }
+        live_entries--;
+        live_memory -= list_item_overhead(wr_size);
         free(li);
         modify_alloc(block_num, [&](heap_block_info_t & inf)
         {
@@ -712,7 +763,12 @@ void blockstore_heap_t::recheck_buffer(heap_entry_t *cwr, uint8_t *buf)
             assert(li->entry.entry_type == cwr->entry_type);
             auto prev = li->prev;
             li->next = li->prev = NULL;
-            li->entry.set_garbage();
+            if (!li->entry.is_garbage())
+            {
+                garbage_entries++;
+                garbage_memory += list_item_overhead(li->entry.size);
+                li->entry.set_garbage();
+            }
             li = prev;
             rolled_back++;
         }
@@ -1162,7 +1218,7 @@ heap_entry_t *blockstore_heap_t::read_entry(object_id oid)
 
 void blockstore_heap_t::gc_block(heap_block_info_t & inf)
 {
-    if (inf.has_garbage)
+    if (inf.garbage_space > 0)
     {
         size_t i = 0, j = 0;
         for (; i < inf.entries.size(); i++)
@@ -1185,7 +1241,8 @@ void blockstore_heap_t::gc_block(heap_block_info_t & inf)
             }
         }
         inf.entries.resize(j);
-        inf.has_garbage = false;
+        inf.used_space -= inf.garbage_space;
+        inf.garbage_space = 0;
     }
 }
 
@@ -1195,11 +1252,12 @@ int blockstore_heap_t::allocate_entry(uint32_t entry_size, uint32_t *block_num,
     {
         // First try to write into the same block as the previous time
         auto & inf = block_info.at(last_allocated_block);
-        auto free_space = dsk->meta_block_size - inf.used_space;
-        if (inf.is_writing || free_space < entry_size ||
+        if (inf.is_writing ||
+            inf.used_space - inf.garbage_space + entry_size > dsk->meta_block_size ||
             // Do not allow to make the last non-nearfull block nearfull
             !allow_last_free && meta_nearfull_blocks >= meta_block_count-1 &&
-            free_space >= max_entry_size && free_space < max_entry_size+entry_size)
+            inf.used_space - inf.garbage_space <= dsk->meta_block_size-max_entry_size &&
+            inf.used_space - inf.garbage_space + entry_size > dsk->meta_block_size-max_entry_size)
         {
             last_allocated_block = UINT32_MAX;
         }
@@ -1216,8 +1274,8 @@ int blockstore_heap_t::allocate_entry(uint32_t entry_size, uint32_t *block_num,
         {
             // Do not allow to make the last non-nearfull block nearfull
             auto & inf = block_info.at(last_allocated_block);
-            auto free_space = dsk->meta_block_size - inf.used_space;
-            if (free_space >= max_entry_size && free_space < max_entry_size+entry_size)
+            if (inf.used_space - inf.garbage_space <= dsk->meta_block_size-max_entry_size &&
+                inf.used_space - inf.garbage_space + entry_size > dsk->meta_block_size-max_entry_size)
             {
                 last_allocated_block = UINT32_MAX;
             }
@@ -1228,8 +1286,7 @@ int blockstore_heap_t::allocate_entry(uint32_t entry_size, uint32_t *block_num,
             for (uint32_t b = meta_alloc->find(META_ALLOC_LEVELS-1); b != UINT32_MAX; b = meta_alloc->next(b))
             {
                 auto & inf = block_info.at(b);
-                auto free_space = dsk->meta_block_size - inf.used_space;
-                if (free_space >= entry_size)
+                if (inf.used_space - inf.garbage_space + entry_size <= dsk->meta_block_size)
                 {
                     last_allocated_block = b;
                     break;
@@ -1246,20 +1303,27 @@ int blockstore_heap_t::allocate_entry(uint32_t entry_size, uint32_t *block_num,
     {
         // Do not allow to make the last non-nearfull block nearfull
         auto & inf = block_info.at(last_allocated_block);
-        if (dsk->meta_block_size-inf.used_space >= max_entry_size &&
-            dsk->meta_block_size-inf.used_space+entry_size < max_entry_size)
+        if (inf.used_space - inf.garbage_space <= dsk->meta_block_size-max_entry_size &&
+            inf.used_space - inf.garbage_space + entry_size > dsk->meta_block_size-max_entry_size)
         {
             last_allocated_block = UINT32_MAX;
             return ENOSPC;
         }
     }
     // Write into the same block
-    auto & inf = block_info.at(last_allocated_block);
-    gc_block(inf);
     *block_num = last_allocated_block;
     modify_alloc(last_allocated_block, [&](heap_block_info_t & inf)
     {
+        // Write just 1 entry to the block to collect garbage
+        if (inf.garbage_space > (inf.used_space-inf.garbage_space)/2)
+            last_allocated_block = UINT32_MAX;
+        gc_block(inf);
         inf.used_space += entry_size;
+        assert(inf.used_space - inf.garbage_space <= dsk->meta_block_size);
+        assert(!inf.mod_lsn_to || inf.mod_lsn_to == next_lsn);
+        ++next_lsn;
+        inf.mod_lsn = inf.mod_lsn ? inf.mod_lsn : next_lsn;
+        inf.mod_lsn_to = next_lsn;
     });
     return 0;
 }
@@ -1318,16 +1382,15 @@ int blockstore_heap_t::add_entry(uint32_t wr_size, uint32_t *modified_block,
         *modified_block = block_num;
     }
     auto li = (heap_list_item_t*)malloc_or_die(wr_size + sizeof(heap_list_item_t) - sizeof(heap_entry_t));
+    live_entries++;
+    live_memory += list_item_overhead(wr_size);
     auto new_wr = &li->entry;
     auto & inf = block_info.at(block_num);
     if (!inf.entries.size())
         inf.entries.reserve(dsk->meta_block_size / max_entry_size);
     inf.entries.push_back(li);
-    assert(!inf.mod_lsn_to || inf.mod_lsn_to == next_lsn);
-    new_wr->lsn = ++next_lsn;
+    new_wr->lsn = next_lsn;
     fill_entry(new_wr);
-    inf.mod_lsn = inf.mod_lsn ? inf.mod_lsn : next_lsn;
-    inf.mod_lsn_to = next_lsn;
     // Remember the object as dirty and remove older entries when this block is written and fsynced
     push_inflight_lsn(next_lsn, new_wr,
         (explicit_complete ? HEAP_INFLIGHT_EXPLICIT : 0) |
@@ -1666,28 +1729,38 @@ int blockstore_heap_t::add_delete(heap_entry_t *obj, uint32_t *modified_block)
 
 uint32_t blockstore_heap_t::meta_alloc_pos(const heap_block_info_t & inf)
 {
-    if (inf.is_writing || inf.used_space > dsk->meta_block_size-sizeof(heap_entry_t))
+    auto real_used = (inf.used_space-inf.garbage_space);
+    if (inf.is_writing || inf.mod_lsn || real_used > dsk->meta_block_size-sizeof(heap_entry_t))
     {
         // 100% full - no entry can be written into this block at all
         return META_ALLOC_LEVELS;
     }
-    if (inf.used_space > dsk->meta_block_size-max_entry_size)
+    if (real_used > dsk->meta_block_size-max_entry_size)
     {
         // nearfull - big_entries won't fit into this block so it can't be used for compaction
         return META_ALLOC_LEVELS-1;
     }
-    // normal block
-    return inf.used_space / ((dsk->meta_block_size-max_entry_size+META_ALLOC_LEVELS-2) / (META_ALLOC_LEVELS-1));
+    // First we want to write to blocks with most garbage:
+    // >= 2*used, >= used/2
+    // (i.e. 66% garbage, 33% garbage)
+    // Then to mostly free blocks:
+    // >= 75% free, >= 50% free, >= 25% free
+    if (inf.garbage_space > real_used*2)
+        return 0;
+    if (inf.garbage_space > real_used/2)
+        return 1;
+    // META_ALLOC_LEVELS-3 levels left
+    return 2 + real_used / ((dsk->meta_block_size-max_entry_size+META_ALLOC_LEVELS-4) / (META_ALLOC_LEVELS-3));
 }
 
 void blockstore_heap_t::modify_alloc(uint32_t block_num, std::function<void(heap_block_info_t &)> change_cb)
 {
     auto & inf = block_info.at(block_num);
     uint32_t old_pos = meta_alloc_pos(inf);
-    uint32_t old_used = inf.used_space;
+    uint32_t old_used = inf.used_space-inf.garbage_space;
     change_cb(inf);
     uint32_t new_pos = meta_alloc_pos(inf);
-    uint32_t new_used = inf.used_space;
+    uint32_t new_used = inf.used_space-inf.garbage_space;
     meta_alloc->change(block_num, old_pos, new_pos);
     meta_used_space -= old_used;
     meta_used_space += new_used;
@@ -1699,11 +1772,19 @@ void blockstore_heap_t::modify_alloc(uint32_t block_num, std::function<void(heap
 
 void blockstore_heap_t::start_block_write(uint32_t block_num)
 {
-    modify_alloc(block_num, [&](heap_block_info_t & inf)
+    auto & inf = block_info.at(block_num);
+    assert(!inf.is_writing);
+    if (!inf.mod_lsn)
+    {
+        modify_alloc(block_num, [&](heap_block_info_t & inf)
+        {
+            inf.is_writing = true;
+        });
+    }
+    else
     {
-        assert(!inf.is_writing);
         inf.is_writing = true;
-    });
+    }
 }
 
 void blockstore_heap_t::complete_block_write(uint32_t block_num)
@@ -1768,6 +1849,8 @@ void blockstore_heap_t::mark_garbage_up_to(heap_entry_t *wr)
 void blockstore_heap_t::mark_garbage(uint32_t block_num, heap_entry_t *prev_wr, uint32_t used_big)
 {
     prev_wr->set_garbage();
+    garbage_entries++;
+    garbage_memory += list_item_overhead(prev_wr->size);
     // And this is the moment when we can free the data reference
     if (prev_wr->type() == BS_HEAP_SMALL_WRITE && prev_wr->small().len > 0)
     {
@@ -1783,8 +1866,7 @@ void blockstore_heap_t::mark_garbage(uint32_t block_num, heap_entry_t *prev_wr,
     }
     modify_alloc(block_num, [&](heap_block_info_t & inf)
     {
-        inf.used_space -= prev_wr->size;
-        inf.has_garbage = true;
+        inf.garbage_space += prev_wr->size;
     });
 }
 
@@ -2154,7 +2236,7 @@ void blockstore_heap_t::fill_block_empty_space(uint8_t *buffer, uint32_t pos)
 uint32_t blockstore_heap_t::get_meta_block_used_space(uint32_t block_num)
 {
     auto & inf = block_info.at(block_num);
-    return inf.used_space;
+    return inf.used_space - inf.garbage_space;
 }
 
 uint64_t blockstore_heap_t::get_data_used_space()
@@ -2197,6 +2279,26 @@ uint64_t blockstore_heap_t::get_compacted_count()
     return compacted_count;
 }
 
+uint64_t blockstore_heap_t::get_live_entries()
+{
+    return live_entries-garbage_entries;
+}
+
+uint64_t blockstore_heap_t::get_live_memory()
+{
+    return live_memory-garbage_memory;
+}
+
+uint64_t blockstore_heap_t::get_garbage_entries()
+{
+    return garbage_entries;
+}
+
+uint64_t blockstore_heap_t::get_garbage_memory()
+{
+    return garbage_memory;
+}
+
 void blockstore_heap_t::push_inflight_lsn(uint64_t lsn, heap_entry_t *wr, uint64_t flags)
 {
     uint64_t next_inf = first_inflight_lsn + inflight_lsn.size();
@@ -2283,37 +2385,50 @@ void blockstore_heap_t::apply_inflight(heap_inflight_lsn_t & inflight)
     {
         // Remove entry
         auto li = list_item(wr);
-        auto prev = li->prev;
-        auto next = li->next;
-        if (prev)
-        {
-            prev->next = next;
-        }
-        if (!next)
-        {
-            // The last freed entry must be a deletion
-            assert(!prev);
-            assert(wr->entry_type == BS_HEAP_DELETE|BS_HEAP_STABLE);
-            auto & pg_idx = block_index[get_pg_id(wr->inode, wr->stripe)];
-            auto & inode_idx = pg_idx[wr->inode];
-            heap_inode_map_t::iterator li_it;
-            heap_list_item_t *old_li = NULL;
-            inode_map_get(inode_idx, li_it, old_li, wr->stripe);
-            inode_map_erase(pg_idx, inode_idx, li_it, old_li);
-        }
-        else
-        {
-            next->prev = prev;
-            if (!prev && next->entry.entry_type == (BS_HEAP_DELETE|BS_HEAP_STABLE))
-            {
-                // free BS_HEAP_DELETEs when all previous entries are also freed
-                mark_garbage(next->block_num, &next->entry, UINT32_MAX);
-            }
-        }
-        free(li);
+        remove_list_item(li);
     }
 }
 
+void blockstore_heap_t::remove_list_item(heap_list_item_t *li)
+{
+    auto prev = li->prev;
+    auto next = li->next;
+    if (prev)
+    {
+        prev->next = next;
+    }
+    if (!next)
+    {
+        // The last freed entry must be a deletion
+        assert(!prev);
+        auto wr = &li->entry;
+        assert(wr->entry_type == BS_HEAP_DELETE|BS_HEAP_STABLE);
+        auto & pg_idx = block_index[get_pg_id(wr->inode, wr->stripe)];
+        auto & inode_idx = pg_idx[wr->inode];
+        heap_inode_map_t::iterator li_it;
+        heap_list_item_t *old_li = NULL;
+        inode_map_get(inode_idx, li_it, old_li, wr->stripe);
+        inode_map_erase(pg_idx, inode_idx, li_it, old_li);
+    }
+    else
+    {
+        next->prev = prev;
+        if (!prev && next->entry.entry_type == (BS_HEAP_DELETE|BS_HEAP_STABLE))
+        {
+            // free BS_HEAP_DELETEs when all previous entries are also freed
+            mark_garbage(next->block_num, &next->entry, UINT32_MAX);
+        }
+    }
+    if (li->entry.is_garbage())
+    {
+        garbage_entries--;
+        garbage_memory -= list_item_overhead(li->entry.size);
+    }
+    live_entries--;
+    live_memory -= list_item_overhead(li->entry.size);
+    free(li);
+}
+
 bool blockstore_heap_t::is_lsn_completed(uint64_t lsn)
 {
     if (lsn <= completed_lsn)

src/blockstore/blockstore_heap.h

@@ -117,10 +117,13 @@ struct heap_object_mvcc_t
 
 struct heap_block_info_t
 {
-    uint32_t used_space = 0;
+    struct __attribute__((__packed__))
+    {
+        uint32_t used_space = 0;
+        uint32_t garbage_space = 0;
+    };
     uint64_t mod_lsn = 0, mod_lsn_to = 0; // only 1 block write of LSN sequence is allowed at a moment
-    bool is_writing: 1;
-    bool has_garbage: 1;
+    bool is_writing = false;
     std::vector<heap_list_item_t*> entries;
 };
 
@@ -184,6 +187,11 @@ class blockstore_heap_t
     uint64_t buffer_area_used_space = 0;
     uint64_t data_used_space = 0;
 
+    uint64_t live_entries = 0;
+    uint64_t live_memory = 0;
+    uint64_t garbage_entries = 0;
+    uint64_t garbage_memory = 0;
+
     uint64_t next_lsn = 0;
     uint32_t last_allocated_block = UINT32_MAX;
     heap_mvcc_map_t object_mvcc;
@@ -212,6 +220,7 @@ class blockstore_heap_t
     bool validate_object(heap_entry_t *obj);
     void fill_recheck_queue();
     int mark_used_blocks();
+    void recheck_full_gc();
     void recheck_buffer(heap_entry_t *cwr, uint8_t *buf);
     void defragment_block(uint32_t block_num);
     void reshard_add(heap_reshard_state_t *st, heap_list_item_t *li);
@@ -219,6 +228,7 @@ class blockstore_heap_t
     void gc_block(heap_block_info_t & inf);
     int allocate_entry(uint32_t entry_size, uint32_t *block_num, bool allow_last_free);
     void insert_list_item(heap_list_item_t *li);
+    void remove_list_item(heap_list_item_t *li);
     int add_entry(uint32_t wr_size, uint32_t *modified_block, bool allow_last_free,
         bool explicit_complete, std::function<void(heap_entry_t *wr)> fill_entry);
     int add_simple(heap_entry_t *obj, uint64_t version, uint32_t *modified_block, uint32_t entry_type);
@@ -345,6 +355,10 @@ public:
     uint32_t get_compact_queue_size();
     uint32_t get_to_compact_count();
     uint64_t get_compacted_count();
+    uint64_t get_live_entries();
+    uint64_t get_live_memory();
+    uint64_t get_garbage_entries();
+    uint64_t get_garbage_memory();
 
     uint64_t entry_pos(uint32_t block_num, uint32_t offset);
     heap_entry_t *entry_from_pos(uint64_t entry_pos, bool allow_unallocated = false);

src/blockstore/blockstore_impl.h

@@ -229,4 +229,9 @@ public:
     uint64_t get_free_block_count();
     inline uint32_t get_bitmap_granularity() { return dsk.bitmap_granularity; }
     inline uint64_t get_journal_size() { return dsk.journal_len; }
+
+    inline uint64_t get_live_entries() { return heap->get_live_entries(); }
+    inline uint64_t get_live_memory() { return heap->get_live_memory(); }
+    inline uint64_t get_garbage_entries() { return heap->get_garbage_entries(); }
+    inline uint64_t get_garbage_memory() { return heap->get_garbage_memory(); }
 };

src/blockstore/blockstore_init.cpp

@@ -240,7 +240,7 @@ resume_4:
     }
     // metadata read finished
     bs->heap->finish_load();
-    printf("Metadata entries loaded: %ju, used blocks: %ju / %ju\n", entries_loaded, bs->heap->get_data_used_space() / bs->dsk.data_block_size, bs->dsk.block_count);
+    printf("Metadata entries loaded: %ju, rechecking unfinished writes and garbage entries\n", entries_loaded);
     if (zero_on_init && !bs->dsk.disable_meta_fsync)
     {
         GET_SQE();
@@ -293,6 +293,11 @@ resume_7:
     if (bs->readonly)
     {
         recheck_mod.clear();
+        printf("Actual metadata entries: %ju\n", bs->heap->get_live_entries());
+    }
+    else
+    {
+        printf("Actual metadata entries: %ju, clearing garbage in %zu metadata blocks\n", bs->heap->get_live_entries(), recheck_mod.size());
     }
     for (i = 0; i < recheck_mod.size(); i++)
     {
@@ -306,7 +311,7 @@ resume_8:
         uint32_t block_num = recheck_mod[i];
         uint64_t block_offset = bs->dsk.meta_offset + (uint64_t)(block_num+1) * bs->dsk.meta_block_size;
         data = ((ring_data_t*)sqe->user_data);
-        uint8_t *buf = (uint8_t*)malloc_or_die(bs->dsk.meta_block_size);
+        uint8_t *buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, bs->dsk.meta_block_size);
         bs->heap->get_meta_block(block_num, buf);
         data->iov = { buf, bs->dsk.meta_block_size };
         data->callback = [this, buf, block_offset](ring_data_t *data)
@@ -332,5 +337,9 @@ resume_9:
     }
     free(metadata_buffer);
     metadata_buffer = NULL;
+    printf("Loading finished. Data used: %ju / %ju bytes (%s / %s)\n",
+        bs->heap->get_data_used_space(), bs->dsk.block_count * bs->dsk.data_block_size,
+        format_size(bs->heap->get_data_used_space()).c_str(),
+        format_size(bs->dsk.block_count * bs->dsk.data_block_size).c_str());
     return 0;
 }

src/blockstore/blockstore_read.cpp

@@ -462,6 +462,10 @@ int blockstore_impl_t::read_bitmap(object_id oid, uint64_t target_version, void
         {
             if (target_version >= wr->version)
             {
+                if (wr->type() == BS_HEAP_DELETE)
+                {
+                    return false;
+                }
                 found = true;
                 if (result_version)
                 {

src/blockstore/v1/impl.cpp

@@ -855,4 +855,29 @@ std::string blockstore_impl_t::get_op_diag(blockstore_op_t *op)
     return std::string(buf);
 }
 
+uint64_t blockstore_impl_t::get_live_entries()
+{
+    return used_blocks;
+}
+
+uint64_t blockstore_impl_t::get_live_memory()
+{
+    uint64_t used = 0;
+    for (auto & kv: clean_db_shards)
+    {
+        used += kv.second.size() * sizeof(blockstore_clean_db_t::value_type);
+    }
+    return used;
+}
+
+uint64_t blockstore_impl_t::get_garbage_entries()
+{
+    return dirty_db.size();
+}
+
+uint64_t blockstore_impl_t::get_garbage_memory()
+{
+    return (sizeof(obj_ver_id) + sizeof(dirty_entry) + 32) * dirty_db.size();
+}
+
 } // namespace v1

src/blockstore/v1/impl.h

@@ -332,6 +332,10 @@ public:
     inline uint64_t get_free_block_count() { return dsk.block_count - used_blocks; }
     inline uint32_t get_bitmap_granularity() { return dsk.disk_alignment; }
     inline uint64_t get_journal_size() { return dsk.journal_len; }
+    uint64_t get_live_entries();
+    uint64_t get_live_memory();
+    uint64_t get_garbage_entries();
+    uint64_t get_garbage_memory();
 };
 
 } // namespace v1

src/client/cluster_client.cpp

@@ -590,7 +590,7 @@ void cluster_client_t::on_change_pool_config_hook()
         {
             if (log_level > 2 && pg_counts[pool_item.first])
             {
-                printf("Pool %u (%s) PG count changed from %lu to %lu\n", pool_item.first, pool_item.second.name.c_str(),
+                fprintf(stderr, "Pool %u (%s) PG count changed from %lu to %lu\n", pool_item.first, pool_item.second.name.c_str(),
                     pg_counts[pool_item.first], pool_item.second.real_pg_count);
             }
             // At this point, all pool operations should have been suspended
@@ -1119,6 +1119,13 @@ resume_2:
         // Finished successfully
         // Even if the PG count has changed in meanwhile we treat it as success
         // because if some operations were invalid for the new PG count we'd get errors
+        if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
+        {
+            // Copy part bitmaps only after finishing all part reads
+            for (auto & part: op->parts)
+                if ((part.flags & (PART_SENT|PART_DONE|PART_VALID)) == (PART_SENT|PART_DONE|PART_VALID))
+                    copy_part_bitmap(op, &part);
+        }
         if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
         {
             // Check parent inode
@@ -1630,13 +1637,6 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
         }
         if (op->inflight_count == 0 && !op->retry_after)
         {
-            // Copy part bitmaps only after finishing all part reads
-            if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
-            {
-                for (auto & part: op->parts)
-                    if (part.flags == (PART_SENT|PART_VALID|PART_DONE))
-                        copy_part_bitmap(op, &part);
-            }
             if (op->opcode == OSD_OP_SYNC)
                 continue_sync(op);
             else

src/client/msgr_rdma.cpp

@@ -187,6 +187,8 @@ std::vector<msgr_rdma_context_t*> msgr_rdma_context_t::create_all(const std::vec
     ibv_device **raw_dev_list = NULL;
     ibv_device **dev_list = NULL;
     ibv_device *single_list[2] = {};
+    int up_ports = 0;
+    int single_port_num = 0;
 
     raw_dev_list = dev_list = ibv_get_device_list(NULL);
     if (!dev_list || !*dev_list)
@@ -221,6 +223,7 @@ std::vector<msgr_rdma_context_t*> msgr_rdma_context_t::create_all(const std::vec
         dev_list = single_list;
     }
 
+retry:
     for (int i = 0; dev_list[i]; ++i)
     {
         auto dev = dev_list[i];
@@ -258,6 +261,9 @@ std::vector<msgr_rdma_context_t*> msgr_rdma_context_t::create_all(const std::vec
                 fprintf(stderr, "RDMA device %s port %d GID %d does not exist\n", ibv_get_device_name(dev), port_num, sel_gid_index);
                 continue;
             }
+            up_ports++;
+            single_port_num = port_num;
+            single_list[0] = dev;
             uint32_t port_mtu = sel_mtu ? sel_mtu : ibv_mtu_to_bytes(portinfo.active_mtu);
 #ifdef IBV_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT
             if (sel_gid_index < 0)
@@ -298,6 +304,14 @@ cleanup_dev:
         ibv_close_device(context);
     }
 
+    if (!ret.size() && up_ports == 1 && dev_list != single_list)
+    {
+        // Auto-select the only available device/port if there is only one
+        dev_list = single_list;
+        sel_port_num = single_port_num;
+        goto retry;
+    }
+
 cleanup:
     if (raw_dev_list)
         ibv_free_device_list(raw_dev_list);

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 3.0.8
+Version: 3.0.11
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/cmd/cli_rm_data.cpp

@@ -286,6 +286,8 @@ struct rm_inode_t
                     .data = data,
                 };
             }
+            // Wakeup callers (otherwise they wake up only on next loop())
+            parent->ringloop->wakeup();
         }
         in_continue = false;
     }

src/nfs/nfs_proxy.cpp

@@ -121,6 +121,7 @@ static const char* help_text =
     "  --logfile <FILE>  log to the specified file\n"
     "  --enforce 1       enforce permissions at the server side (default is disabled)\n"
     "  --foreground 1    stay in foreground, do not daemonize\n"
+    "  --trace           trace all NFS requests\n"
     "\n"
     "NFS proxy is stateless if you use immediate_commit=all in your cluster and if\n"
     "you do not use client_enable_writeback=true, so you can freely use multiple\n"
@@ -158,7 +159,7 @@ json11::Json::object nfs_proxy_t::parse_args(int narg, const char *args[])
         {
             const char *opt = args[i]+2;
             cfg[str_replace(opt, "-", "_")] = !strcmp(opt, "json") || !strcmp(opt, "block") ||
-                !strcmp(opt, "dry-run") || !strcmp(opt, "recalc-stats") ||
+                !strcmp(opt, "dry-run") || !strcmp(opt, "recalc-stats") || !strcmp(opt, "trace") ||
                 !strcmp(opt, "include-empty") || !strcmp(opt, "no-rm") || i == narg-1 ? "1" : args[++i];
         }
         else

src/osd/osd.cpp

@@ -529,6 +529,11 @@ void osd_t::exec_op(osd_op_t *cur_op)
 
 void osd_t::print_stats()
 {
+    if (bs && log_level > 1)
+    {
+        printf("[OSD %ju] Live entries: %ju (%ju bytes), garbage entries: %ju (%ju bytes)\n", osd_num,
+            bs->get_live_entries(), bs->get_live_memory(), bs->get_garbage_entries(), bs->get_garbage_memory());
+    }
     for (int i = OSD_OP_MIN; i <= OSD_OP_MAX; i++)
     {
         if (msgr.stats.op_stat_count[i] != prev_stats.op_stat_count[i] && i != OSD_OP_PING)

src/osd/osd_primary_chain.cpp

@@ -82,6 +82,7 @@ int osd_t::read_bitmaps(osd_op_t *cur_op, pg_t *pg, int base_state)
         goto resume_0;
     else if (op_data->st == base_state+1)
         goto resume_1;
+    assert(op_data->st < base_state);
     if (!pg || pg->state == PG_ACTIVE && pg->scheme == POOL_SCHEME_REPLICATED)
     {
         // Happy path for clean replicated PGs (all bitmaps are available locally)
@@ -112,6 +113,12 @@ resume_0:
             return 1;
         }
 resume_1:
+        if (op_data->errors > 0)
+        {
+            // Failure
+            finish_op(cur_op, op_data->errcode);
+            return -1;
+        }
         if (pg->scheme != POOL_SCHEME_REPLICATED)
         {
             for (int chain_num = 0; chain_num < op_data->chain_size; chain_num++)
@@ -607,7 +614,7 @@ void osd_t::send_chained_read_results(pg_t *pg, osd_op_t *cur_op)
         {
             if (cur > prev)
             {
-                 // Send buffer in parts to avoid copying
+                // Send buffer in parts to avoid copying
                 if (!prev_set)
                 {
                     while ((cur-prev) > zero_buffer_size/bs_bitmap_granularity)

src/test/test_heap.cpp

@@ -1893,7 +1893,8 @@ void test_big_intent_csums()
         heap.finish_recheck();
 
         auto mod = heap.get_recheck_modified_blocks();
-        assert(mod.size() == 0);
+        assert(mod.size() == 1);
+        assert(mod[0] == 0);
 
         // read object 1 - big_intent should be there
         object_id oid = { .inode = INODE_WITH_POOL(1, 1), .stripe = 0 };
@@ -1990,6 +1991,7 @@ void test_redirect_intent_csums()
 
     // reload heap to check that the write is still here
     {
+        dsk.gc_on_start = false;
         blockstore_heap_t heap(&dsk, buffer_area.data(), 10);
         uint64_t entries_loaded;
         heap.load_blocks(0, dsk.meta_block_size, tmp.data(), false, entries_loaded);