WIP refactor dequeue_write/continue_write into a single "coroutine"

Test scenarios:
- 128KB aligned I/O (may work despite bug)
- 4KB random I/O (triggers assertion failure)
- 4KB mixed read/write (triggers crash)
- 4KB sequential write (triggers crash)

I accept Vitastor CLA agreement: https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/CLA-en.md

.gitea/workflows/test.yml

@@ -414,6 +414,24 @@ jobs:
           echo ""
         done
 
+  test_level_placement:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_level_placement.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_snapshot:
     runs-on: ubuntu-latest
     needs: build
@@ -1800,6 +1818,24 @@ jobs:
           echo ""
         done
 
+  test_old_partwr_csum:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_partwr_csum.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_heal_old_csum_32k_dmj:
     runs-on: ubuntu-latest
     needs: build
@@ -1926,3 +1962,21 @@ jobs:
           echo ""
         done
 
+  test_nfs_unaligned_append:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_nfs_unaligned_append.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+

CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "3.0.0")
+set(VITASTOR_VERSION "3.0.2")
 
 include(CTest)
 

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.0
+VITASTOR_VERSION ?= v3.0.2
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v3.0.0
+          image: vitalif/vitastor-csi:v3.0.2
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v3.0.0
+          image: vitalif/vitastor-csi:v3.0.2
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "3.0.0"
+    vitastorCSIDriverVersion = "3.0.2"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (3.0.0-1) unstable; urgency=medium
+vitastor (3.0.2-1) unstable; urgency=medium
 
   * Bugfixes
 

docker/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.0
+VITASTOR_VERSION ?= v3.0.2
 
 all: build push
 

docker/etc/vitastor/docker.conf

@@ -4,7 +4,7 @@
 #
 
 # Desired Vitastor version
-VITASTOR_VERSION=v3.0.0
+VITASTOR_VERSION=v3.0.2
 
 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here

docs/config/osd.en.md

@@ -67,6 +67,8 @@ with an OSD restart or, for some of them, even without restarting by updating co
 - [pg_lock_retry_interval_ms](#pg_lock_retry_interval_ms)
 - [atomic_write_size](#atomic_write_size)
 - [use_atomic_flag](#use_atomic_flag)
+- [pg_reshard_chunk_size](#pg_reshard_chunk_size)
+- [pg_reshard_chunk_pause_ms](#pg_reshard_chunk_pause_ms)
 
 ## bind_address
 
@@ -682,7 +684,10 @@ with replicated pools and reach the best possible write performance.
 
 Default value is auto-detected during OSD initialization from
 `/sys/block/xx/queue/atomic_write_max_bytes` or assumed to be 4096 bytes
-because all known disks support 4 KB atomic writes.
+because all known disks support 4 KB atomic writes. Auto-detection is only used for
+NVMe disks because SAS disks require the explicit WRITE ATOMIC command which requires
+RWF_ATOMIC (see below [#use_atomic_flag]) but that flag works incorrectly in current
+Linux versions.
 
 You can also check if your NVMe drives support atomic writes by running
 the command `nvme id-ctrl /dev/nvme0n1 | grep awupf`. If the reported value,
@@ -697,12 +702,34 @@ reducing Write Amplification and improving write performance up to 2 times.
 
 - Type: boolean
 
-This option controls whether the Vitastor OSD uses RWF_ATOMIC write flag with atomic
-writes. This flag is only supported on Linux kernel since 6.11. Atomic writes are
-generally only safe to use with this flag because it tells the kernel to never fragment
-write requests and also to check the write against the actual atomic write capabilities
-of the device.
+This option controls whether Vitastor OSDs use RWF_ATOMIC write flag with atomic writes.
+This flag is supported since Linux 6.11 and adds some safety to atomic writes - the kernel
+guarantees to not fragment write requests with it and also to check them against the actual
+device atomic write capabilities.
 
-This option is enabled by default when atomic_write_size is set to a value larger than 4 KB.
-You can disable it if you're sure that your disks support atomic writes and you want to
-bypass the Linux atomic write checks.
+However, the option is disabled by default because the flag is currently UNUSABLE - Linux
+incorrectly requires writes with that flag to be of power-of-2 length and length-aligned.
+I.e., for example, 12 KB writes and not-8-KB aligned 8 KB writes are forbidden by the kernel,
+even though the NVMe specification allows them.
+
+For NVMe disks with `scheduler=none` writes aren't fragmented anyway so it's not a big deal.
+However, you can rebuild your kernel with [this patch](../../patches/linux-fix-atomic-write-checks.diff)
+and turn this option on. It will make your atomic writes a bit safer.
+
+## pg_reshard_chunk_size
+
+- Type: integer
+- Default: 100000
+
+Pool PG count change is a CPU-intensive operation because OSDs store the full object database
+in memory and have to move all entries between old and new PGs. Thus it's performed in chunks,
+with pauses between chunks to prevent blocking OSD's event loop and other clients' operations.
+This option sets the maximum number of object is a chunk. Moving 100k objects usually takes
+50-100ms. Chunk size equal to 0 means unlimited.
+
+## pg_reshard_chunk_pause_ms
+
+- Type: milliseconds
+- Default: 100
+
+This option sets the interval between handling two PG count change chunks.

docs/config/osd.ru.md

@@ -68,6 +68,8 @@
 - [pg_lock_retry_interval_ms](#pg_lock_retry_interval_ms)
 - [atomic_write_size](#atomic_write_size)
 - [use_atomic_flag](#use_atomic_flag)
+- [pg_reshard_chunk_size](#pg_reshard_chunk_size)
+- [pg_reshard_chunk_pause_ms](#pg_reshard_chunk_pause_ms)
 
 ## bind_address
 
@@ -717,6 +719,9 @@ pg_minsize OSD во время переключений, что может по
 Значение по умолчанию авто-определяется во время инициализации OSD из
 `/sys/block/xx/queue/atomic_write_max_bytes` либо принимается равным 4096,
 так как все известные диски поддерживают атомарную запись 4 КБ блоков.
+Автоопределение применяется только для NVMe-дисков, так как SAS диски требуют
+использования отдельной команды WRITE ATOMIC, а для неё нужен флаг RWF_ATOMIC
+(см. ниже [#use_atomic_flag]), а он в текущих версиях Linux работает некорректно.
 
 Вы также можете проверить, поддерживают ли ваши NVMe-диски атомарную запись,
 с помощью команды `nvme id-ctrl /dev/nvme0n1 | grep awupf`. Если значение awupf
@@ -735,11 +740,35 @@ pg_minsize OSD во время переключений, что может по
 - Тип: булево (да/нет)
 
 Данная опция контролирует использование Vitastor OSD флага RWF_ATOMIC при атомарной записи
-блоков. Этот флаг поддерживается только в ядрах Linux начиная с 6.11. Атомарная запись
-является безопасной только при использовании этого флага, так как он сообщает ядру о том,
-что запрос записи нельзя фрагментировать и о том, что запрос нужно проверить на соответствие
-реальным возможностям атомарной записи устройства.
+блоков. Этот флаг поддерживается, начиная с версии ядра Linux 6.11 и добавляет немного корректности
+атомарным записям - ядро гарантирует отсутствие фрагментации запросов записи с этим флагом и
+проверяет их на соответствие реальным возможностям устройства.
 
-Опция включается по умолчанию, когда atomic_write_size устанавливается в значение больше 4 КБ.
-Вы можете явно отключить её, если уверены, что ваши диски поддерживают атомарную запись и
-хотите обойти проверки уровня ядра.
+Однако, данная опция по умолчанию отключена, так как флаг в текущих версиях Linux работает
+абсолютно НЕКОРРЕКТНО - при нём Linux требует, чтобы запросы записи имели длину, равную
+степени двойки и были выровнены на эту длину. То есть, например, 12 КБ запросы записи, а также
+8 КБ запросы записи по не-кратному 8 КБ смещению запрещаются ядром, хотя спецификация NVMe их
+разрешает.
+
+Для NVMe-дисков с `scheduler=none` запросы записи и так не фрагментируются, так что это не так
+уж и важно, однако вы можете пересобрать своё ядро с [этим патчем](../../patches/linux-fix-atomic-write-checks.diff)
+и включить данную опцию. Это сделает вашу атомарную запись капельку безопаснее.
+
+## pg_reshard_chunk_size
+
+- Тип: целое число
+- Значение по умолчанию: 100000
+
+Изменение числа PG в пуле заметно загружает процессор, так как OSD хранят полную базу данных
+объектов в памяти и им приходится перемещать все записи объектов между старыми и новыми PG.
+Поэтому изменение применяется порциями, с паузами между порциями, чтобы не блокировать обработку
+событий OSD и операции остальных клиентов. Данная опция задаёт максимальное число объектов
+в порции. Перемещение 100 тысяч объектов (значение по умолчанию) обычно занимает порядка
+50-100 миллисекунд. Значение опции 0 отключает лимит размера порции.
+
+## pg_reshard_chunk_pause_ms
+
+- Тип: миллисекунды
+- Значение по умолчанию: 100
+
+Данная опция задаёт интервал между обработкой двух порций изменения числа PG пулов.

docs/config/src/osd.yml

@@ -813,7 +813,10 @@
 
     Default value is auto-detected during OSD initialization from
     `/sys/block/xx/queue/atomic_write_max_bytes` or assumed to be 4096 bytes
-    because all known disks support 4 KB atomic writes.
+    because all known disks support 4 KB atomic writes. Auto-detection is only used for
+    NVMe disks because SAS disks require the explicit WRITE ATOMIC command which requires
+    RWF_ATOMIC (see below [#use_atomic_flag]) but that flag works incorrectly in current
+    Linux versions.
 
     You can also check if your NVMe drives support atomic writes by running
     the command `nvme id-ctrl /dev/nvme0n1 | grep awupf`. If the reported value,
@@ -834,6 +837,9 @@
     Значение по умолчанию авто-определяется во время инициализации OSD из
     `/sys/block/xx/queue/atomic_write_max_bytes` либо принимается равным 4096,
     так как все известные диски поддерживают атомарную запись 4 КБ блоков.
+    Автоопределение применяется только для NVMe-дисков, так как SAS диски требуют
+    использования отдельной команды WRITE ATOMIC, а для неё нужен флаг RWF_ATOMIC
+    (см. ниже [#use_atomic_flag]), а он в текущих версиях Linux работает некорректно.
 
     Вы также можете проверить, поддерживают ли ваши NVMe-диски атомарную запись,
     с помощью команды `nvme id-ctrl /dev/nvme0n1 | grep awupf`. Если значение awupf
@@ -849,22 +855,54 @@
 - name: use_atomic_flag
   type: bool
   info: |
-    This option controls whether the Vitastor OSD uses RWF_ATOMIC write flag with atomic
-    writes. This flag is only supported on Linux kernel since 6.11. Atomic writes are
-    generally only safe to use with this flag because it tells the kernel to never fragment
-    write requests and also to check the write against the actual atomic write capabilities
-    of the device.
+    This option controls whether Vitastor OSDs use RWF_ATOMIC write flag with atomic writes.
+    This flag is supported since Linux 6.11 and adds some safety to atomic writes - the kernel
+    guarantees to not fragment write requests with it and also to check them against the actual
+    device atomic write capabilities.
 
-    This option is enabled by default when atomic_write_size is set to a value larger than 4 KB.
-    You can disable it if you're sure that your disks support atomic writes and you want to
-    bypass the Linux atomic write checks.
+    However, the option is disabled by default because the flag is currently UNUSABLE - Linux
+    incorrectly requires writes with that flag to be of power-of-2 length and length-aligned.
+    I.e., for example, 12 KB writes and not-8-KB aligned 8 KB writes are forbidden by the kernel,
+    even though the NVMe specification allows them.
+
+    For NVMe disks with `scheduler=none` writes aren't fragmented anyway so it's not a big deal.
+    However, you can rebuild your kernel with [this patch](../../patches/linux-fix-atomic-write-checks.diff)
+    and turn this option on. It will make your atomic writes a bit safer.
   info_ru: |
     Данная опция контролирует использование Vitastor OSD флага RWF_ATOMIC при атомарной записи
-    блоков. Этот флаг поддерживается только в ядрах Linux начиная с 6.11. Атомарная запись
-    является безопасной только при использовании этого флага, так как он сообщает ядру о том,
-    что запрос записи нельзя фрагментировать и о том, что запрос нужно проверить на соответствие
-    реальным возможностям атомарной записи устройства.
+    блоков. Этот флаг поддерживается, начиная с версии ядра Linux 6.11 и добавляет немного корректности
+    атомарным записям - ядро гарантирует отсутствие фрагментации запросов записи с этим флагом и
+    проверяет их на соответствие реальным возможностям устройства.
 
-    Опция включается по умолчанию, когда atomic_write_size устанавливается в значение больше 4 КБ.
-    Вы можете явно отключить её, если уверены, что ваши диски поддерживают атомарную запись и
-    хотите обойти проверки уровня ядра.
+    Однако, данная опция по умолчанию отключена, так как флаг в текущих версиях Linux работает
+    абсолютно НЕКОРРЕКТНО - при нём Linux требует, чтобы запросы записи имели длину, равную
+    степени двойки и были выровнены на эту длину. То есть, например, 12 КБ запросы записи, а также
+    8 КБ запросы записи по не-кратному 8 КБ смещению запрещаются ядром, хотя спецификация NVMe их
+    разрешает.
+
+    Для NVMe-дисков с `scheduler=none` запросы записи и так не фрагментируются, так что это не так
+    уж и важно, однако вы можете пересобрать своё ядро с [этим патчем](../../patches/linux-fix-atomic-write-checks.diff)
+    и включить данную опцию. Это сделает вашу атомарную запись капельку безопаснее.
+- name: pg_reshard_chunk_size
+  type: int
+  default: 100000
+  info: |
+    Pool PG count change is a CPU-intensive operation because OSDs store the full object database
+    in memory and have to move all entries between old and new PGs. Thus it's performed in chunks,
+    with pauses between chunks to prevent blocking OSD's event loop and other clients' operations.
+    This option sets the maximum number of object is a chunk. Moving 100k objects usually takes
+    50-100ms. Chunk size equal to 0 means unlimited.
+  info_ru: |
+    Изменение числа PG в пуле заметно загружает процессор, так как OSD хранят полную базу данных
+    объектов в памяти и им приходится перемещать все записи объектов между старыми и новыми PG.
+    Поэтому изменение применяется порциями, с паузами между порциями, чтобы не блокировать обработку
+    событий OSD и операции остальных клиентов. Данная опция задаёт максимальное число объектов
+    в порции. Перемещение 100 тысяч объектов (значение по умолчанию) обычно занимает порядка
+    50-100 миллисекунд. Значение опции 0 отключает лимит размера порции.
+- name: pg_reshard_chunk_pause_ms
+  type: ms
+  default: 100
+  info: |
+    This option sets the interval between handling two PG count change chunks.
+  info_ru: |
+    Данная опция задаёт интервал между обработкой двух порций изменения числа PG пулов.

docs/installation/docker.en.md

@@ -26,9 +26,9 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.
 
 1. Download a Docker image of the desired version: \
-   `docker pull vitalif/vitastor:v3.0.0`
+   `docker pull vitalif/vitastor:v3.0.2`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.0 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.2 install.sh`
 3. Reload udev rules: \
    `udevadm control --reload-rules`
 

docs/installation/docker.ru.md

@@ -25,9 +25,9 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.
 
 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitalif/vitastor:v3.0.0`
+   `docker pull vitalif/vitastor:v3.0.2`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.0 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.2 install.sh`
 3. Перезагрузите правила udev: \
    `udevadm control --reload-rules`
 

docs/installation/proxmox.en.md

@@ -6,7 +6,7 @@
 
 # Proxmox VE
 
-To enable Vitastor support in Proxmox Virtual Environment (6.4-8.x are supported):
+To enable Vitastor support in Proxmox Virtual Environment (6.4-9.x are supported):
 
 - Add the corresponding Vitastor Debian repository into sources.list on Proxmox hosts:
   trixie for 9.0+, bookworm for 8.1+, pve8.0 for 8.0, bullseye for 7.4, pve7.3 for 7.3, pve7.2 for 7.2, pve7.1 for 7.1, buster for 6.4

docs/installation/proxmox.ru.md

@@ -6,7 +6,7 @@
 
 # Proxmox VE
 
-Чтобы подключить Vitastor к Proxmox Virtual Environment (поддерживаются версии 6.4-8.x):
+Чтобы подключить Vitastor к Proxmox Virtual Environment (поддерживаются версии 6.4-9.x):
 
 - Добавьте соответствующий Debian-репозиторий Vitastor в sources.list на хостах Proxmox:
   trixie для 9.0+, bookworm для 8.1+, pve8.0 для 8.0, bullseye для 7.4, pve7.3 для 7.3, pve7.2 для 7.2, pve7.1 для 7.1, buster для 6.4

docs/usage/disk.en.md

@@ -95,6 +95,8 @@ Options (single-device mode):
 Options (both modes):
 
 ```
+--tags tag1,tag2           Set new OSD tag(s)
+--weight <number>          Set new OSD weight (between 0 to 1)
 --journal_size 1G/32M      Set journal size (area or partition size)
 --block_size 1M/128k       Set blockstore object size
 --bitmap_granularity 4k    Set bitmap granularity

docs/usage/disk.ru.md

@@ -96,6 +96,8 @@ vitastor-disk - инструмент командной строки для уп
 Опции для обоих режимов:
 
 ```
+--tags tag1,tag2           Задать теги для новых OSD
+--weight <number>          Задать вес для новых OSD (от 0 до 1)
 --journal_size 1G/32M      Задать размер журнала (области или раздела журнала)
 --block_size 1M/128k       Задать размер объекта хранилища
 --bitmap_granularity 4k    Задать гранулярность битовых карт

mon/osd_tree.js

@@ -87,7 +87,7 @@ function make_hier_tree(global_config, tree)
     tree[''] = { children: [] };
     for (const node_id in tree)
     {
-        if (node_id === '' || !(tree[node_id].children||[]).length && (tree[node_id].size||0) <= 0)
+        if (node_id === '')
         {
             continue;
         }

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "3.0.0",
+  "version": "3.0.2",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {
@@ -9,7 +9,7 @@
   "author": "Vitaliy Filippov",
   "license": "UNLICENSED",
   "dependencies": {
-    "antietcd": "^1.1.3",
+    "antietcd": "^1.2.2",
     "sprintf-js": "^1.1.2",
     "ws": "^7.2.5"
   },

mon/pg_gen.js

@@ -52,6 +52,7 @@ function recheck_primary(state, global_config, up_osds, osd_tree)
             continue;
         }
         const aff_osds = get_affinity_osds(pool_cfg, up_osds, osd_tree);
+        let paused = false;
         for (let pg_num = 1; pg_num <= pool_cfg.pg_count; pg_num++)
         {
             if (!state.pg.config.items[pool_id])
@@ -74,6 +75,19 @@ function recheck_primary(state, global_config, up_osds, osd_tree)
                     );
                     new_pg_config.items[pool_id][pg_num].primary = new_primary;
                 }
+                paused = paused || !!pg_cfg.pause;
+            }
+        }
+        if (paused)
+        {
+            if (!new_pg_config)
+            {
+                new_pg_config = JSON.parse(JSON.stringify(state.pg.config));
+            }
+            console.log(`Resuming paused pool ${pool_id}`);
+            for (const pg in new_pg_config.items[pool_id])
+            {
+                delete new_pg_config.items[pool_id][pg].pause;
             }
         }
     }
@@ -178,7 +192,7 @@ async function generate_pool_pgs(state, global_config, pool_id, osd_tree, levels
     const rules = use_rules ? get_pg_rules(pool_id, pool_cfg, global_config.placement_levels) : null;
     const folded = fold_failure_domains(Object.values(pool_tree), use_rules ? rules : [ [ [ pool_cfg.failure_domain ] ] ]);
     // FIXME: Remove/merge make_hier_tree() step somewhere, however it's needed to remove empty nodes
-    const folded_tree = make_hier_tree(global_config, folded.nodes);
+    const folded_tree = make_hier_tree(global_config, folded.nodes.reduce((a, c) => { a[c.id] = c; return a; }, {}));
     const old_pg_count = prev_pgs.length;
     const optimize_cfg = {
         osd_weights: folded.nodes.reduce((a, c) => { if (Number(c.id)) { a[c.id] = c.size; } return a; }, {}),

node-binding/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor",
-  "version": "3.0.0",
+  "version": "3.0.2",
   "description": "Low-level native bindings to Vitastor client library",
   "main": "index.js",
   "keywords": [

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '3.0.0'
+VITASTOR_VERSION = '3.0.2'
 
 LOG = logging.getLogger(__name__)
 

patches/linux-fix-atomic-write-checks.diff

@@ -0,0 +1,39 @@
+From 98d3f68a40130c438854f61db6025f9e9b099cb6 Mon Sep 17 00:00:00 2001
+From: Vitaliy Filippov <vitalifster@gmail.com>
+Date: Sat, 20 Dec 2025 14:44:35 +0300
+Subject: [PATCH] Do not require atomic writes to be power of 2 sized and
+ aligned on length boundary
+
+It contradicts NVMe specification where alignment is only required when atomic
+write boundary (NABSPF/NABO) is set and highly limits usage of NVMe atomic writes
+
+Signed-off-by: Vitaliy Filippov <vitalifster@gmail.com>
+---
+ fs/read_write.c | 8 --------
+ 1 file changed, 8 deletions(-)
+
+diff --git a/fs/read_write.c b/fs/read_write.c
+index 833bae068770..5467d710108d 100644
+--- a/fs/read_write.c
++++ b/fs/read_write.c
+@@ -1802,17 +1802,9 @@ int generic_file_rw_checks(struct file *file_in, struct file *file_out)
+ 
+ int generic_atomic_write_valid(struct kiocb *iocb, struct iov_iter *iter)
+ {
+-	size_t len = iov_iter_count(iter);
+-
+ 	if (!iter_is_ubuf(iter))
+ 		return -EINVAL;
+ 
+-	if (!is_power_of_2(len))
+-		return -EINVAL;
+-
+-	if (!IS_ALIGNED(iocb->ki_pos, len))
+-		return -EINVAL;
+-
+ 	if (!(iocb->ki_flags & IOCB_DIRECT))
+ 		return -EOPNOTSUPP;
+ 
+-- 
+2.51.0
+

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.0
+Version:        3.0.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.0.el7.tar.gz
+Source0:        vitastor-3.0.2.el7.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  devtoolset-9-gcc-c++

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.0
+Version:        3.0.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.0.el8.tar.gz
+Source0:        vitastor-3.0.2.el8.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-toolset-9-gcc-c++

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.0
+Version:        3.0.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.0.el9.tar.gz
+Source0:        vitastor-3.0.2.el9.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++

src/CMakeLists.txt

@@ -21,7 +21,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 endif()
 set(ENABLE_COVERAGE false CACHE BOOL "Enable code coverage")
 
-add_definitions(-DVITASTOR_VERSION="3.0.0")
+add_definitions(-DVITASTOR_VERSION="3.0.2")
 add_definitions(-D_GNU_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -fvisibility=hidden -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/blockstore/blockstore.h

@@ -183,6 +183,12 @@ public:
     // Update configuration
     virtual void parse_config(blockstore_config_t & config) = 0;
 
+    // Reshard database for a pool in chunks
+    // MUST be called only when nobody makes any modifications to the DB for this pool
+    virtual void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit) = 0;
+    virtual bool reshard_continue(void *reshard_state, uint64_t chunk_limit) = 0;
+    virtual void reshard_abort(void *reshard_state) = 0;
+
     // Event loop
     virtual void loop() = 0;
 

src/blockstore/blockstore_disk.cpp

@@ -171,6 +171,12 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     {
         throw std::runtime_error("Data block size must be a multiple of sparse write tracking granularity");
     }
+    if (data_block_size / bitmap_granularity < 8)
+    {
+        fprintf(stderr, "Warning: block_size (%u) / bitmap_granularity (%u) = %u bits. "
+            "Consider using larger block_size or bitmap_granularity for better performance.\n",
+            data_block_size, bitmap_granularity, data_block_size / bitmap_granularity);
+    }
     if (!data_csum_type)
     {
         csum_block_size = 0;
@@ -259,7 +265,7 @@ void blockstore_disk_t::calc_lengths(bool skip_meta_check)
     }
     // required metadata size
     block_count = data_len / data_block_size;
-    clean_entry_bitmap_size = data_block_size / bitmap_granularity / 8;
+    clean_entry_bitmap_size = (data_block_size / bitmap_granularity + 7) / 8;
     clean_dyn_size = clean_entry_bitmap_size*2 + (csum_block_size
         ? data_block_size/csum_block_size*(data_csum_type & 0xFF) : 0);
 recalc:

src/blockstore/blockstore_flush.h

@@ -58,7 +58,6 @@ class journal_flusher_co
     int i, res;
     bool read_to_fill_incomplete;
     int copy_count;
-    bool do_repeat = false;
 
     friend class journal_flusher_t;
 

src/blockstore/blockstore_heap.cpp

@@ -29,6 +29,15 @@
 #define IMAP_MALLOC_LOW_BITS ((size_t)0x0F)
 #define IMAP_MAX_LOW 16
 
+void inode_map_put(void* & inode_idx, heap_list_item_t* li);
+void inode_map_get(void *inode_idx, heap_inode_map_t::iterator & li_it, heap_list_item_t* & li, uint64_t stripe);
+void inode_map_free(void* inode_idx);
+bool inode_map_is_big(void* & inode_idx);
+void inode_map_iterate(void* & inode_idx, std::function<void(heap_list_item_t*)> cb);
+void inode_map_replace(void* & inode_idx, const heap_inode_map_t::iterator & li_it, heap_list_item_t* new_li);
+void inode_map_erase(robin_hood::unordered_flat_map<inode_t, void*, i64hash_t> & pg_idx, void* & inode_idx,
+    const heap_inode_map_t::iterator & li_it, heap_list_item_t* li);
+
 static inline heap_list_item_t *list_item(heap_entry_t *wr)
 {
     return (heap_list_item_t*)((uint8_t*)wr - offsetof(struct heap_list_item_t, entry));
@@ -658,7 +667,8 @@ void blockstore_heap_t::recheck_buffer(heap_entry_t *cwr, uint8_t *buf)
     else if (!calc_checksums(cwr, buf, false))
     {
         // write entry is invalid, erase it and mark newer entries with garbage bit
-        auto & inode_idx = block_index[get_pg_id(cwr->inode, cwr->stripe)][cwr->inode];
+        auto & pg_idx = block_index[get_pg_id(cwr->inode, cwr->stripe)];
+        auto & inode_idx = pg_idx[cwr->inode];
         heap_inode_map_t::iterator li_it;
         heap_list_item_t *li = NULL;
         inode_map_get(inode_idx, li_it, li, cwr->stripe);
@@ -684,7 +694,7 @@ void blockstore_heap_t::recheck_buffer(heap_entry_t *cwr, uint8_t *buf)
         {
             fprintf(stderr, "Notice: the whole object %jx:%jx only has unfinished writes, rolling back\n",
                 cwr->inode, cwr->stripe);
-            inode_map_erase(inode_idx, li_it, li);
+            inode_map_erase(pg_idx, inode_idx, li_it, li);
         }
         free_entry(li);
     }
@@ -943,44 +953,138 @@ bool blockstore_heap_t::calc_block_checksums(uint32_t *block_csums, uint8_t *bit
     return res;
 }
 
-void blockstore_heap_t::reshard(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size)
+struct heap_reshard_state_t
+{
+    int state = 0;
+    uint64_t pool_id = 0;
+    uint32_t old_pg_count = 0;
+    uint32_t pg_count = 0;
+    uint32_t pg_stripe_size = 0;
+    uint64_t chunk_size = 0;
+    heap_block_index_t new_shards;
+    heap_block_index_t old_shards;
+    heap_block_index_t::iterator sh_it;
+    robin_hood::unordered_flat_map<inode_t, void*, i64hash_t>::iterator inode_it;
+    heap_inode_map_t *stripe_map = NULL;
+    heap_inode_map_t::iterator stripe_it;
+
+    void add(heap_list_item_t *li);
+    bool run(uint64_t chunk_limit);
+};
+
+void heap_reshard_state_t::add(heap_list_item_t *li)
+{
+    // like map_to_pg()
+    uint64_t pg_num = (li->entry.stripe / pg_stripe_size) % pg_count + 1;
+    uint64_t shard_id = (pool_id << (64-POOL_ID_BITS)) | pg_num;
+    inode_map_put(new_shards[shard_id][li->entry.inode], li);
+    chunk_size++;
+}
+
+bool heap_reshard_state_t::run(uint64_t chunk_limit)
+{
+    chunk_size = 0;
+    if (state == 1)
+        goto resume_1;
+    else if (state == 2)
+        goto resume_2;
+    sh_it = old_shards.begin();
+    for (; sh_it != old_shards.end(); sh_it++)
+    {
+        inode_it = sh_it->second.begin();
+        for (; inode_it != sh_it->second.end(); inode_it++)
+        {
+            if (!inode_map_is_big(inode_it->second))
+            {
+                if (chunk_limit > 0 && chunk_size >= chunk_limit)
+                {
+                    state = 1;
+                    return false;
+                }
+resume_1:
+                inode_map_iterate(inode_it->second, [&](heap_list_item_t *li) { add(li); });
+            }
+            else
+            {
+                stripe_map = (heap_inode_map_t*)inode_it->second;
+                stripe_it = stripe_map->begin();
+                for (; stripe_it != stripe_map->end(); stripe_it++)
+                {
+                    if (chunk_limit > 0 && chunk_size >= chunk_limit)
+                    {
+                        state = 2;
+                        return false;
+                    }
+resume_2:
+                    add(*stripe_it);
+                }
+            }
+            inode_map_free(inode_it->second);
+        }
+    }
+    return true;
+}
+
+void* blockstore_heap_t::reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit)
 {
     auto & pool_settings = pool_shard_settings[pool];
     if (pool_settings.pg_count == pg_count && pool_settings.pg_stripe_size == pg_stripe_size)
     {
-        return;
+        return NULL;
     }
-    uint32_t old_pg_count = !pool_settings.pg_count ? 1 : pool_settings.pg_count;
-    uint64_t pool_id = (uint64_t)pool;
-    heap_block_index_t new_shards;
-    for (uint32_t pg_num = 0; pg_num <= old_pg_count; pg_num++)
+    heap_reshard_state_t *st = new heap_reshard_state_t;
+    st->pool_id = (uint64_t)pool;
+    st->pg_count = pg_count;
+    st->pg_stripe_size = pg_stripe_size;
+    st->old_pg_count = !pool_settings.pg_count ? 1 : pool_settings.pg_count;
+    for (uint32_t pg_num = 0; pg_num <= st->old_pg_count; pg_num++)
     {
-        auto sh_it = block_index.find((pool_id << (64-POOL_ID_BITS)) | pg_num);
-        if (sh_it == block_index.end())
+        auto sh_it = block_index.find((st->pool_id << (64-POOL_ID_BITS)) | pg_num);
+        if (sh_it != block_index.end())
         {
-            continue;
+            st->old_shards[pg_num] = std::move(sh_it->second);
+            block_index.erase(sh_it);
         }
-        for (auto & inode_pair: sh_it->second)
-        {
-            inode_map_iterate(inode_pair.second, [&](heap_list_item_t *li)
-            {
-                // like map_to_pg()
-                uint64_t pg_num = (li->entry.stripe / pg_stripe_size) % pg_count + 1;
-                uint64_t shard_id = (pool_id << (64-POOL_ID_BITS)) | pg_num;
-                inode_map_put(new_shards[shard_id][li->entry.inode], li);
-            });
-            inode_map_free(inode_pair.second);
-        }
-        block_index.erase(sh_it);
     }
-    for (auto sh_it = new_shards.begin(); sh_it != new_shards.end(); sh_it++)
+    bool finished = reshard_continue(st, chunk_limit);
+    return finished ? NULL : st;
+}
+
+bool blockstore_heap_t::reshard_continue(void *reshard_state, uint64_t chunk_limit)
+{
+    heap_reshard_state_t *st = (heap_reshard_state_t*)reshard_state;
+    if (!st->run(chunk_limit))
+    {
+        return false;
+    }
+    for (auto sh_it = st->new_shards.begin(); sh_it != st->new_shards.end(); sh_it++)
     {
         block_index[sh_it->first] = std::move(sh_it->second);
     }
-    pool_settings = (pool_shard_settings_t){
-        .pg_count = pg_count,
-        .pg_stripe_size = pg_stripe_size,
+    pool_shard_settings[st->pool_id] = (pool_shard_settings_t){
+        .pg_count = st->pg_count,
+        .pg_stripe_size = st->pg_stripe_size,
     };
+    delete st;
+    return true;
+}
+
+bool blockstore_heap_t::reshard_check(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size)
+{
+    auto set_it = pool_shard_settings.find(pool);
+    return (set_it != pool_shard_settings.end() &&
+        set_it->second.pg_count == pg_count &&
+        set_it->second.pg_stripe_size == pg_stripe_size);
+}
+
+void blockstore_heap_t::reshard_abort(void* reshard_state)
+{
+    heap_reshard_state_t *st = (heap_reshard_state_t*)reshard_state;
+    for (auto sh_it = st->old_shards.begin(); sh_it != st->old_shards.end(); sh_it++)
+    {
+        block_index[sh_it->first] = std::move(sh_it->second);
+    }
+    delete st;
 }
 
 heap_entry_t *blockstore_heap_t::lock_and_read_entry(object_id oid)
@@ -1243,6 +1347,7 @@ int blockstore_heap_t::add_small_write(object_id oid, heap_entry_t **obj_ptr, ui
     // Small writes are written in parallel with buffered data so they require explicit_complete
     return add_entry(wr_size, modified_block, false, true, [&](heap_entry_t *wr)
     {
+        printf("add_small_write t%u %lx:%lx l%lu v%lu %u +%u loc:%lx\n", type, oid.inode, oid.stripe, wr->lsn, version, offset, len, location);
         wr->entry_type = type;
         wr->inode = oid.inode;
         wr->stripe = oid.stripe;
@@ -1272,6 +1377,7 @@ int blockstore_heap_t::add_big_write(object_id oid, heap_entry_t *old_head, bool
     // Big writes are written after writing data so they don't require explicit_complete
     return add_entry(wr_size, modified_block, false, false, [&](heap_entry_t *wr)
     {
+        printf("add_big_write %lx:%lx l%lu v%lu loc:%lx\n", oid.inode, oid.stripe, wr->lsn, version, location);
         wr->entry_type = BS_HEAP_BIG_WRITE | (stable ? BS_HEAP_STABLE : 0);
         wr->inode = oid.inode;
         wr->stripe = oid.stripe;
@@ -1298,6 +1404,7 @@ int blockstore_heap_t::add_redirect_intent(object_id oid, heap_entry_t **obj_ptr
     // Big-redirect intents, just like regular big writes, are written after writing data so they don't require explicit_complete
     return add_entry(wr_size, modified_block, false, false, [&](heap_entry_t *wr)
     {
+        printf("add_redir_intent %lx:%lx l%lu v%lu %u +%u loc:%lx\n", oid.inode, oid.stripe, wr->lsn, version, offset, len, location);
         wr->entry_type = BS_HEAP_BIG_INTENT|BS_HEAP_STABLE;
         wr->inode = oid.inode;
         wr->stripe = oid.stripe;
@@ -1334,6 +1441,7 @@ int blockstore_heap_t::add_big_intent(object_id oid, heap_entry_t **obj_ptr, uin
     // Big intents are written before writing data so they require explicit_complete
     return add_entry(wr_size, modified_block, false, true, [&](heap_entry_t *wr)
     {
+        printf("add_big_intent %lx:%lx l%lu v%lu %u +%u loc:%lx\n", oid.inode, oid.stripe, wr->lsn, version, offset, len, obj->big_location(this));
         wr->entry_type = BS_HEAP_BIG_INTENT | BS_HEAP_STABLE;
         wr->inode = oid.inode;
         wr->stripe = oid.stripe;
@@ -1389,6 +1497,7 @@ int blockstore_heap_t::add_compact(heap_entry_t *obj, uint64_t compact_version,
     // Compaction entry is added after copying data so it doesn't require explicit_complete
     return add_entry(wr_size, modified_block, true, false, [&](heap_entry_t *new_wr)
     {
+        printf("add_compact %lx:%lx l%lu v%lu loc:%lx\n", obj->inode, obj->stripe, compact_lsn, compact_version, compact_location);
         new_wr->entry_type = BS_HEAP_BIG_WRITE|BS_HEAP_STABLE;
         new_wr->inode = obj->inode;
         new_wr->stripe = obj->stripe;
@@ -2171,11 +2280,12 @@ void blockstore_heap_t::apply_inflight(heap_inflight_lsn_t & inflight)
         if (!next)
         {
             assert(!prev);
-            auto & inode_idx = block_index[get_pg_id(wr->inode, wr->stripe)][wr->inode];
+            auto & pg_idx = block_index[get_pg_id(wr->inode, wr->stripe)];
+            auto & inode_idx = pg_idx[wr->inode];
             heap_inode_map_t::iterator li_it;
             heap_list_item_t *old_li = NULL;
             inode_map_get(inode_idx, li_it, old_li, wr->stripe);
-            inode_map_erase(inode_idx, li_it, old_li);
+            inode_map_erase(pg_idx, inode_idx, li_it, old_li);
         }
         else
         {
@@ -2190,6 +2300,15 @@ void blockstore_heap_t::apply_inflight(heap_inflight_lsn_t & inflight)
     }
 }
 
+bool blockstore_heap_t::is_lsn_completed(uint64_t lsn)
+{
+    if (lsn <= completed_lsn)
+        return true;
+    assert(lsn-first_inflight_lsn < inflight_lsn.size());
+    auto it = inflight_lsn.begin() + (lsn-first_inflight_lsn);
+    return (it->flags & HEAP_INFLIGHT_DONE);
+}
+
 uint64_t blockstore_heap_t::get_completed_lsn()
 {
     return completed_lsn;
@@ -2259,7 +2378,7 @@ void blockstore_heap_t::recalc_inode_space_stats(uint64_t pool_id, bool per_inod
 // This is some really crazy shit but it seems to work well :)
 // At the same time it has almost zero overhead and works just as fast for fat inodes.
 
-void blockstore_heap_t::inode_map_get(void *inode_idx, heap_inode_map_t::iterator & li_it, heap_list_item_t* & li, uint64_t stripe)
+void inode_map_get(void *inode_idx, heap_inode_map_t::iterator & li_it, heap_list_item_t* & li, uint64_t stripe)
 {
     size_t map_n = ((size_t)inode_idx & IMAP_MALLOC_LOW_BITS);
     if (!map_n)
@@ -2286,7 +2405,7 @@ void blockstore_heap_t::inode_map_get(void *inode_idx, heap_inode_map_t::iterato
     }
 }
 
-void blockstore_heap_t::inode_map_free(void* inode_idx)
+void inode_map_free(void* inode_idx)
 {
     size_t n = ((size_t)inode_idx & IMAP_MALLOC_LOW_BITS);
     if (!n)
@@ -2299,7 +2418,12 @@ void blockstore_heap_t::inode_map_free(void* inode_idx)
     }
 }
 
-void blockstore_heap_t::inode_map_iterate(void* & inode_idx, std::function<void(heap_list_item_t*)> cb)
+bool inode_map_is_big(void* & inode_idx)
+{
+    return !((size_t)inode_idx & IMAP_MALLOC_LOW_BITS);
+}
+
+void inode_map_iterate(void* & inode_idx, std::function<void(heap_list_item_t*)> cb)
 {
     size_t n = ((size_t)inode_idx & IMAP_MALLOC_LOW_BITS);
     if (!n)
@@ -2326,7 +2450,7 @@ void blockstore_heap_t::inode_map_iterate(void* & inode_idx, std::function<void(
     }
 }
 
-void blockstore_heap_t::inode_map_put(void* & inode_idx, heap_list_item_t* li)
+void inode_map_put(void* & inode_idx, heap_list_item_t* li)
 {
     if (!inode_idx)
     {
@@ -2395,7 +2519,7 @@ void blockstore_heap_t::inode_map_put(void* & inode_idx, heap_list_item_t* li)
     }
 }
 
-void blockstore_heap_t::inode_map_replace(void* & inode_idx, const heap_inode_map_t::iterator & li_it, heap_list_item_t* new_li)
+void inode_map_replace(void* & inode_idx, const heap_inode_map_t::iterator & li_it, heap_list_item_t* new_li)
 {
     size_t map_n = ((size_t)inode_idx & IMAP_MALLOC_LOW_BITS);
     if (!map_n)
@@ -2421,7 +2545,8 @@ void blockstore_heap_t::inode_map_replace(void* & inode_idx, const heap_inode_ma
     }
 }
 
-void blockstore_heap_t::inode_map_erase(void* & inode_idx, const heap_inode_map_t::iterator & li_it, heap_list_item_t* li)
+void inode_map_erase(robin_hood::unordered_flat_map<inode_t, void*, i64hash_t> & pg_idx, void* & inode_idx,
+    const heap_inode_map_t::iterator & li_it, heap_list_item_t* li)
 {
     size_t map_n = ((size_t)inode_idx & IMAP_MALLOC_LOW_BITS);
     if (!map_n)
@@ -2446,7 +2571,7 @@ void blockstore_heap_t::inode_map_erase(void* & inode_idx, const heap_inode_map_
     else if (map_n == 1)
     {
         // Erase
-        block_index[get_pg_id(li->entry.inode, li->entry.stripe)].erase(li->entry.inode);
+        pg_idx.erase(li->entry.inode);
     }
     else
     {

src/blockstore/blockstore_heap.h

@@ -137,6 +137,8 @@ struct heap_compact_t
     bool do_delete;
 };
 
+struct heap_reshard_state_t;
+
 struct heap_li_hash
 {
     size_t operator()(const heap_list_item_t* li) const noexcept
@@ -161,7 +163,7 @@ using heap_mvcc_map_t = robin_hood::unordered_flat_map<object_id, heap_object_mv
 
 class blockstore_heap_t
 {
-    friend class heap_entry_t;
+    friend struct heap_entry_t;
 
     blockstore_disk_t *dsk = NULL;
     uint8_t* buffer_area = NULL;
@@ -205,19 +207,13 @@ class blockstore_heap_t
     std::function<void(bool is_data, uint64_t offset, uint64_t len, uint8_t* buf, std::function<void()>)> recheck_cb;
     int recheck_queue_depth = 0;
 
-    void inode_map_put(void* & inode_idx, heap_list_item_t* li);
-    void inode_map_get(void *inode_idx, heap_inode_map_t::iterator & li_it, heap_list_item_t* & li, uint64_t stripe);
-    void inode_map_free(void* inode_idx);
-    void inode_map_iterate(void* & inode_idx, std::function<void(heap_list_item_t*)> cb);
-    void inode_map_replace(void* & inode_idx, const heap_inode_map_t::iterator & li_it, heap_list_item_t* new_li);
-    void inode_map_erase(void* & inode_idx, const heap_inode_map_t::iterator & li_it, heap_list_item_t* li);
-
     uint64_t get_pg_id(inode_t inode, uint64_t stripe);
     bool validate_object(heap_entry_t *obj);
     void fill_recheck_queue();
     int mark_used_blocks();
     void recheck_buffer(heap_entry_t *cwr, uint8_t *buf);
     void defragment_block(uint32_t block_num);
+    void reshard_add(heap_reshard_state_t *st, heap_list_item_t *li);
 
     int allocate_entry(uint32_t entry_size, uint32_t *block_num, bool allow_last_free);
     void insert_list_item(heap_list_item_t *li);
@@ -249,7 +245,10 @@ public:
     // recheck small write data after reading the database from disk
     bool recheck_small_writes(std::function<void(bool is_data, uint64_t offset, uint64_t len, uint8_t* buf, std::function<void()>)> read_buffer, int queue_depth);
     // reshard database according to the pool's PG count
-    void reshard(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size);
+    void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
+    bool reshard_continue(void* reshard_state, uint64_t chunk_limit);
+    bool reshard_check(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size);
+    void reshard_abort(void* reshard_state);
     void set_no_inode_stats(const std::vector<uint64_t> & pool_ids);
     void recalc_inode_space_stats(uint64_t pool_id, bool per_inode);
     // read an object entry and lock it against removal
@@ -314,6 +313,7 @@ public:
     void start_block_write(uint32_t block_num);
     void complete_block_write(uint32_t block_num);
     void complete_lsn_write(uint64_t lsn);
+    bool is_lsn_completed(uint64_t lsn);
     uint64_t get_completed_lsn();
     uint64_t get_fsynced_lsn();
     void mark_lsn_fsynced(uint64_t lsn);

src/blockstore/blockstore_impl.cpp

@@ -23,6 +23,7 @@ blockstore_impl_t::blockstore_impl_t(blockstore_config_t & config, ring_loop_i *
         dsk.open_meta();
         dsk.open_journal();
         dsk.calc_lengths();
+        dsk.check_lengths();
     }
     catch (std::exception & e)
     {
@@ -31,16 +32,13 @@ blockstore_impl_t::blockstore_impl_t(blockstore_config_t & config, ring_loop_i *
     }
     meta_superblock = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size);
     memset(meta_superblock, 0, dsk.meta_block_size);
-}
-
-void blockstore_impl_t::init()
-{
     flusher = new journal_flusher_t(this);
     if (dsk.inmemory_journal)
     {
         buffer_area = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.journal_len);
     }
     heap = new blockstore_heap_t(&dsk, buffer_area, log_level);
+    ringloop->wakeup();
 }
 
 blockstore_impl_t::~blockstore_impl_t()
@@ -325,9 +323,13 @@ void blockstore_impl_t::process_list(blockstore_op_t *op)
         FINISH_OP(op);
         return;
     }
-    // Check if the DB needs resharding
-    // (we don't know about PGs from the beginning, we only create "shards" here)
-    heap->reshard(INODE_POOL(min_inode), pg_count, pg_stripe_size);
+    // Check if the DB is sharded correctly
+    if (!heap->reshard_check(INODE_POOL(min_inode), pg_count, pg_stripe_size))
+    {
+        op->retval = -EAGAIN;
+        FINISH_OP(op);
+        return;
+    }
     obj_ver_id *result = NULL;
     size_t stable_count = 0, unstable_count = 0;
     int res = heap->list_objects(list_pg, op->min_oid, op->max_oid, &result, &stable_count, &unstable_count);
@@ -394,3 +396,18 @@ std::string blockstore_impl_t::get_op_diag(blockstore_op_t *op)
         snprintf(buf, sizeof(buf), "state=%d", priv->op_state);
     return std::string(buf);
 }
+
+void* blockstore_impl_t::reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit)
+{
+    return heap->reshard_start(pool, pg_count, pg_stripe_size, chunk_limit);
+}
+
+bool blockstore_impl_t::reshard_continue(void *reshard_state, uint64_t chunk_limit)
+{
+    return heap->reshard_continue(reshard_state, chunk_limit);
+}
+
+void blockstore_impl_t::reshard_abort(void *reshard_state)
+{
+    return heap->reshard_abort(reshard_state);
+}

src/blockstore/blockstore_impl.h

@@ -142,7 +142,6 @@ public:
     int metadata_buf_size;
     blockstore_init_meta* metadata_init_reader;
 
-    void init();
     void check_wait(blockstore_op_t *op);
     void init_op(blockstore_op_t *op);
 
@@ -190,6 +189,10 @@ public:
     void parse_config(blockstore_config_t & config);
     void parse_config(blockstore_config_t & config, bool init);
 
+    void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
+    bool reshard_continue(void *reshard_state, uint64_t chunk_limit);
+    void reshard_abort(void *reshard_state);
+
     // Event loop
     void loop();
 

src/blockstore/blockstore_init.cpp

@@ -72,7 +72,6 @@ resume_1:
     }
     if (is_zero((uint64_t*)bs->meta_superblock, bs->dsk.meta_block_size))
     {
-        bs->dsk.check_lengths();
         {
             blockstore_meta_header_v3_t *hdr = (blockstore_meta_header_v3_t *)bs->meta_superblock;
             hdr->zero = 0;
@@ -141,7 +140,7 @@ resume_1:
             hdr->bitmap_granularity != bs->dsk.bitmap_granularity ||
             hdr->data_csum_type != bs->dsk.data_csum_type ||
             hdr->csum_block_size != bs->dsk.csum_block_size ||
-            hdr->meta_area_size > bs->dsk.meta_area_size)
+            hdr->meta_area_size != bs->dsk.meta_area_size)
         {
             printf(
                 "Configuration stored in metadata superblock"
@@ -154,15 +153,7 @@ resume_1:
             );
             exit(1);
         }
-        bs->dsk.meta_area_size = hdr->meta_area_size;
-        if (bs->dsk.meta_format != hdr->version)
-        {
-            bs->dsk.meta_format = hdr->version;
-            bs->dsk.calc_lengths();
-        }
-        bs->dsk.check_lengths();
     }
-    bs->init();
     bs->heap->start_load(((blockstore_meta_header_v3_t *)bs->meta_superblock)->completed_lsn);
     if (bs->dsk.inmemory_journal)
     {
@@ -278,7 +269,7 @@ resume_6:
         }
         GET_SQE();
         data->iov = (iovec){ buf, len };
-        data->callback = [this, offset, cb](ring_data_t *data)
+        data->callback = [offset, cb](ring_data_t *data)
         {
             if (data->res < 0)
             {

src/blockstore/blockstore_read.cpp

@@ -22,7 +22,7 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
     uint64_t result_version = 0;
     bool found = false;
     uint32_t skip_csum = 0;
-    uint32_t blk_start = 0, blk_end = 0;
+    uint32_t blk_start = op->offset, blk_end = op->offset+op->len;
     bool need_skip = dsk.csum_block_size > dsk.bitmap_granularity && !perfect_csum_update;
     if (need_skip)
     {
@@ -32,12 +32,28 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
         if (blk_end % dsk.csum_block_size)
             blk_end += dsk.csum_block_size - (blk_end % dsk.csum_block_size);
     }
+    bool need_wait = false;
     heap->iterate_with_stable(obj, obj->lsn, [&](heap_entry_t *wr, bool stable)
     {
         if (wr->type() == BS_HEAP_DELETE)
         {
             return false;
         }
+        if (!heap->is_lsn_completed(wr->lsn))
+        {
+            if (wr->type() == BS_HEAP_BIG_INTENT && wr->big_intent().offset < blk_end && wr->big_intent().offset+wr->big_intent().len > blk_start ||
+                wr->type() == BS_HEAP_INTENT_WRITE && wr->small().offset < blk_end && wr->small().offset+wr->small().len > blk_start)
+            {
+                // Wait until intent write is completed
+                need_wait = true;
+                return false;
+            }
+            else if (wr->type() == BS_HEAP_SMALL_WRITE && wr->small().offset < blk_end && wr->small().offset+wr->small().len > blk_start)
+            {
+                // Skip entry and read the previous one
+                return true;
+            }
+        }
         if (op->version >= wr->version && !found)
         {
             found = true;
@@ -47,12 +63,6 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
                 memcpy(op->bitmap, wr->get_ext_bitmap(heap), dsk.clean_entry_bitmap_size);
             }
         }
-        if (need_skip && wr->lsn < heap->get_completed_lsn() &&
-            (wr->type() == BS_HEAP_BIG_INTENT && wr->big_intent().offset < blk_end && wr->big_intent().offset+wr->big_intent().len > blk_start ||
-            wr->type() == BS_HEAP_INTENT_WRITE && wr->small().offset < blk_end && wr->small().offset+wr->small().len > blk_start))
-        {
-            skip_csum = COPY_BUF_SKIP_CSUM;
-        }
         if (op->version >= wr->version)
         {
             fulfilled += prepare_read(PRIV(op)->read_vec, obj, wr, op->offset, op->offset+op->len,
@@ -65,13 +75,23 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
                 return false;
             }
         }
-        if (need_skip && (wr->type() == BS_HEAP_SMALL_WRITE || wr->type() == BS_HEAP_INTENT_WRITE) &&
+        if (need_skip && wr->type() == BS_HEAP_SMALL_WRITE &&
             wr->small().offset < blk_end && wr->small().offset+wr->small().len > blk_start)
         {
+            // Small write may mutate big write checksums during flush
             skip_csum = COPY_BUF_SKIP_CSUM;
         }
         return true;
     });
+    if (need_wait)
+    {
+undo_wait:
+        // Need to wait. undo added requests, unlock lsn
+        heap->unlock_entry(op->oid);
+        free_read_buffers(rv);
+        rv.clear();
+        return 0;
+    }
     if (!found)
     {
         // May happen if there are entries but all of them are > requested version
@@ -84,11 +104,7 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
     assert(fulfilled == op->len);
     if (!fulfill_read(op))
     {
-        // Need to wait. undo added requests, unlock lsn
-        heap->unlock_entry(op->oid);
-        free_read_buffers(rv);
-        rv.clear();
-        return 0;
+        goto undo_wait;
     }
     op->version = result_version;
     if (!PRIV(op)->pending_ops)

src/blockstore/blockstore_write.cpp

@@ -13,10 +13,13 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
     return true;
 }
 
-void blockstore_impl_t::prepare_meta_block_write(uint32_t modified_block)
+bool blockstore_impl_t::prepare_meta_block_write(uint32_t modified_block)
 {
-    if (modified_blocks.find(modified_block) != modified_blocks.end())
-        return;
+    auto mod_it = modified_blocks.find(modified_block);
+    if (mod_it != modified_blocks.end())
+    {
+        return !mod_it->second.sent;
+    }
     io_uring_sqe *sqe = get_sqe();
     assert(sqe != NULL);
     ring_data_t *data = ((ring_data_t*)sqe->user_data);
@@ -41,6 +44,7 @@ void blockstore_impl_t::prepare_meta_block_write(uint32_t modified_block)
     unsynced_meta_write_count++;
     pending_modified_blocks.push_back(modified_block);
     modified_blocks[modified_block] = { .sent = false, .buf = buf };
+    return true;
 }
 
 bool blockstore_impl_t::meta_block_is_pending(uint32_t modified_block)
@@ -121,6 +125,7 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
     heap_entry_t *obj = heap->read_entry(op->oid);
     if (op->opcode == BS_OP_DELETE)
     {
+        return continue_delete(op, 0);
         // Delete
         if (!obj || obj->type() == BS_HEAP_DELETE)
         {
@@ -133,17 +138,38 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
         BS_SUBMIT_CHECK_SQES(1);
         int res = heap->add_delete(obj, &PRIV(op)->modified_block);
         if (res == ENOSPC)
+        {
             goto enospc;
+        }
         assert(res == 0);
-        prepare_meta_block_write(PRIV(op)->modified_block);
-        PRIV(op)->pending_ops++;
-        PRIV(op)->op_state = 5;
         write_iodepth++;
+resume_1:
+        while (!prepare_meta_block_write(PRIV(op)->modified_block))
+        {
+            PRIV(op)->op_state = 1;
+            return 1;
+        }
+rseume_2:
+        while (meta_block_is_pending(PRIV(op)->modified_block))
+        {
+            PRIV(op)->op_state = 2;
+            return 1;
+        }
+resume_3:
+resume_4:
+        if (!throttle_write(op, 3))
+        {
+            return 1;
+        }
+        write_iodepth--;
+        ack_write(op);
+        return 2;
     }
     // FIXME: Allow to do initial writes as buffered, not redirected
     // FIXME: Allow to do direct writes over holes
     else if (!obj || obj->type() == BS_HEAP_DELETE || op->offset == 0 && op->len == dsk.data_block_size)
     {
+        return continue_big_write(op, 10);
         // Big (redirect) write
         PRIV(op)->write_type = dsk.disable_data_fsync || op->opcode != BS_OP_WRITE_STABLE ? BS_HEAP_BIG_WRITE : _REDIRECT_INTENT;
         BS_SUBMIT_CHECK_SQES(1);
@@ -163,6 +189,7 @@ enospc:
             flusher->request_trim();
             return 0;
         }
+        write_iodepth++;
         uint64_t loc = PRIV(op)->location;
 #ifdef BLOCKSTORE_DEBUG
         printf(
@@ -176,18 +203,72 @@ enospc:
         data->iov = (struct iovec){ op->buf, op->len };
         data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
         io_uring_prep_writev(sqe, dsk.data_fd, &data->iov, 1, dsk.data_offset + loc + op->offset);
+        if (PRIV(op)->write_type == BS_HEAP_BIG_WRITE)
+            inflight_big++;
         PRIV(op)->pending_ops++;
-        write_iodepth++;
+resume_10:
+        if (PRIV(op)->pending_ops > 0)
+        {
+            PRIV(op)->op_state = 10;
+            return 1;
+        }
         if (PRIV(op)->write_type == BS_HEAP_BIG_WRITE)
         {
-            PRIV(op)->op_state = 1;
-            inflight_big++;
+            inflight_big--;
+resume_11:
+resume_12:
+resume_13:
+            if (!fsync_big_write(op, 11))
+                return 1;
+        }
+        heap_entry_t *obj = heap->read_entry(op->oid);
+        int res = 0;
+        if (PRIV(op)->write_type == _REDIRECT_INTENT)
+        {
+            res = heap->add_redirect_intent(op->oid, &obj, op->version, op->offset, op->len,
+                PRIV(op)->location, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
         }
         else
-            PRIV(op)->op_state = 3;
+        {
+            res = heap->add_big_write(op->oid, obj, op->opcode == BS_OP_WRITE_STABLE,
+                op->version, op->offset, op->len, PRIV(op)->location, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
+        }
+        if (res == ENOSPC)
+        {
+            if (!heap->get_to_compact_count())
+            {
+                // no space
+                heap->free_data(op->oid.inode, PRIV(op)->location);
+                write_iodepth--;
+                op->retval = -ENOSPC;
+                FINISH_OP(op);
+                return 2;
+            }
+            PRIV(op)->wait_for = WAIT_COMPACTION;
+            PRIV(op)->wait_detail = heap->get_compacted_count();
+            flusher->request_trim();
+            return 0;
+        }
+        assert(res == 0);
+resume_14:
+        while (!prepare_meta_block_write(PRIV(op)->modified_block))
+        {
+            PRIV(op)->op_state = 14;
+            return 1;
+        }
+resume_15:
+        while (meta_block_is_pending(PRIV(op)->modified_block))
+        {
+            PRIV(op)->op_state = 15;
+            return 1;
+        }
+        write_iodepth--;
+        ack_write(op);
+        return 2;
     }
     else if (intent_write_allowed(op, obj))
     {
+        return continue_intent_write(op, 20);
         // Direct intent-write
         BS_SUBMIT_CHECK_SQES(1);
         int res = 0;
@@ -223,13 +304,41 @@ enospc:
             assert(res == 0);
             PRIV(op)->lsn = obj->lsn;
         }
-        prepare_meta_block_write(PRIV(op)->modified_block);
-        PRIV(op)->pending_ops++;
-        PRIV(op)->op_state = 9;
         write_iodepth++;
+resume_20:
+        while (!prepare_meta_block_write(PRIV(op)->modified_block))
+        {
+            PRIV(op)->op_state = 20;
+            return 1;
+        }
+resume_21:
+        while (meta_block_is_pending(PRIV(op)->modified_block))
+        {
+            PRIV(op)->op_state = 21;
+            return 1;
+        }
+        // Direct intent-write
+        // LSN is not marked as completed so big_write won't be freed
+        BS_SUBMIT_GET_SQE(sqe, data);
+        data->iov = (struct iovec){ op->buf, op->len };
+        data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
+        io_uring_prep_writev(sqe, dsk.data_fd, &data->iov, 1, dsk.data_offset + PRIV(op)->location + op->offset);
+        if (dsk.use_atomic_flag)
+            sqe->rw_flags = RWF_ATOMIC;
+        PRIV(op)->pending_ops++;
+resume_22:
+        if (PRIV(op)->pending_ops > 0)
+        {
+            PRIV(op)->op_state = 22;
+            return 1;
+        }
+        write_iodepth--;
+        ack_write(op);
+        return 2;
     }
     else
     {
+        return continue_small_write(op, 30);
         // Small (buffered) overwrite
         // First check if there is free buffer space
         PRIV(op)->write_type = BS_HEAP_SMALL_WRITE;
@@ -242,7 +351,7 @@ enospc:
             return 0;
         }
         // There is sufficient space. Check SQE(s)
-        BS_SUBMIT_CHECK_SQES(1 + (op->len > 0 ? 1 : 0));
+        BS_SUBMIT_CHECK_SQES(1 + (op->len > 0 ? 1 : 0)); ---> refactor too
         int res = heap->add_small_write(op->oid, &obj, (BS_HEAP_SMALL_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0)),
             op->version, op->offset, op->len, loc, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
         if (res == ENOSPC)
@@ -311,27 +420,46 @@ again:
         goto again;
     }
 resume_2:
+
+
+resume_4:
+resume_6:
+
+resume_8:
+    ack
+    return 2;
+resume_10:
+    return 1;
+}
+
+bool blockstore_impl_t::fsync_big_write(blockstore_op_t *op, int base_state)
+{
+    if (PRIV(op)->state == base_state)
+        goto resume_0;
+    else if (PRIV(op)->state == base_state+1)
+        goto resume_1;
+    else if (PRIV(op)->state == base_state+2)
+        goto resume_2;
     // We must fsync all big writes to avoid complex write workflows
     // It's OK for all HDDs and for server SSDs, but slightly worse for desktop SSDs
-    inflight_big--;
     if (!dsk.disable_data_fsync)
     {
         // fsync data in a batch
-resume_11:
+resume_0:
         if (inflight_big > 0)
         {
-            PRIV(op)->op_state = 11;
-            return 1;
+            PRIV(op)->op_state = base_state;
+            return false;
         }
         if (fsyncing_data)
         {
-resume_12:
+resume_1:
             if (fsyncing_data)
             {
-                PRIV(op)->op_state = 12;
-                return 1;
+                PRIV(op)->op_state = base_state+1;
+                return false;
             }
-            goto resume_4;
+            return true;
         }
         fsyncing_data = true;
         BS_SUBMIT_GET_SQE(sqe, data);
@@ -343,47 +471,23 @@ resume_12:
             handle_write_event(data, op);
         };
         PRIV(op)->pending_ops++;
-        PRIV(op)->op_state = 3;
-        return 1;
+resume_2:
+        if (PRIV(op)->pending_ops > 0)
+        {
+            PRIV(op)->op_state = base_state+2;
+            return false;
+        }
     }
-resume_4:
-    {
-        auto obj = heap->read_entry(op->oid);
-        int res = 0;
-        if (PRIV(op)->write_type == _REDIRECT_INTENT)
-        {
-            res = heap->add_redirect_intent(op->oid, &obj, op->version, op->offset, op->len,
-                PRIV(op)->location, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
-        }
-        else
-        {
-            res = heap->add_big_write(op->oid, obj, op->opcode == BS_OP_WRITE_STABLE,
-                op->version, op->offset, op->len, PRIV(op)->location, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
-        }
-        if (res == ENOSPC)
-        {
-            if (!heap->get_to_compact_count())
-            {
-                // no space
-                heap->free_data(op->oid.inode, PRIV(op)->location);
-                write_iodepth--;
-                op->retval = -ENOSPC;
-                FINISH_OP(op);
-                return 2;
-            }
-            PRIV(op)->wait_for = WAIT_COMPACTION;
-            PRIV(op)->wait_detail = heap->get_compacted_count();
-            flusher->request_trim();
-            return 0;
-        }
-        assert(res == 0);
-        prepare_meta_block_write(PRIV(op)->modified_block);
-        PRIV(op)->pending_ops++;
-        PRIV(op)->op_state = 5;
-        return 1;
-    }
-resume_6:
+    return true;
+}
+
+bool blockstore_impl_t::throttle_write(blockstore_op_t *op, int base_state)
+{
     // Apply throttling to not fill the journal too quickly for the SSD+HDD case
+    if (PRIV(op)->op_state >= base_state+1)
+    {
+        return true;
+    }
     if (PRIV(op)->write_type == BS_HEAP_SMALL_WRITE && throttle_small_writes)
     {
         // Apply throttling
@@ -404,17 +508,21 @@ resume_6:
         if (ref_us > exec_us + throttle_threshold_us)
         {
             // Pause reply
-            PRIV(op)->op_state = 7;
+            PRIV(op)->op_state = base_state;
             // Remember that the timer can in theory be called right here
             tfd->set_timer_us(ref_us-exec_us, false, [this, op](int timer_id)
             {
-                PRIV(op)->op_state = 8;
+                PRIV(op)->op_state++;
                 ringloop->wakeup();
             });
-            return 1;
+            return false;
         }
     }
-resume_8:
+    return true;
+}
+
+void blockstore_impl_t::ack_write(blockstore_op_t *op)
+{
     // Acknowledge write
 #ifdef BLOCKSTORE_DEBUG
     printf("Ack write %jx:%jx v%ju\n", op->oid.inode, op->oid.stripe, op->version);
@@ -441,21 +549,7 @@ resume_8:
         unsynced_data_write_count++;
         intent_write_counter++;
     }
-    write_iodepth--;
     FINISH_OP(op);
-    return 2;
-resume_10:
-    // Direct intent-write
-    // LSN is not marked as completed so big_write won't be freed
-    BS_SUBMIT_GET_SQE(sqe, data);
-    data->iov = (struct iovec){ op->buf, op->len };
-    data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
-    io_uring_prep_writev(sqe, dsk.data_fd, &data->iov, 1, dsk.data_offset + PRIV(op)->location + op->offset);
-    if (dsk.use_atomic_flag)
-        sqe->rw_flags = RWF_ATOMIC;
-    PRIV(op)->pending_ops++;
-    PRIV(op)->op_state = 7;
-    return 1;
 }
 
 void blockstore_impl_t::handle_write_event(ring_data_t *data, blockstore_op_t *op)

src/blockstore/v1/impl.cpp

@@ -407,32 +407,88 @@ blockstore_clean_db_t& blockstore_impl_t::clean_db_shard(object_id oid)
     return clean_db_shards[(pool_id << (64-POOL_ID_BITS)) | pg_num];
 }
 
-void blockstore_impl_t::reshard_clean_db(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size)
+struct bs_reshard_state_t
 {
-    uint64_t pool_id = (uint64_t)pool;
+    int state = 0;
+    uint64_t pool_id = 0;
+    uint32_t pg_count = 0;
+    uint32_t pg_stripe_size = 0;
+    uint64_t chunk_size = 0;
+    std::map<pool_pg_id_t, blockstore_clean_db_t> old_shards;
     std::map<pool_pg_id_t, blockstore_clean_db_t> new_shards;
-    auto sh_it = clean_db_shards.lower_bound((pool_id << (64-POOL_ID_BITS)));
-    while (sh_it != clean_db_shards.end() &&
-        (sh_it->first >> (64-POOL_ID_BITS)) == pool_id)
+    std::map<pool_pg_id_t, blockstore_clean_db_t>::iterator sh_it;
+    blockstore_clean_db_t::iterator obj_it;
+};
+
+void* blockstore_impl_t::reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit)
+{
+    auto & settings = clean_db_settings[pool];
+    if (settings.pg_count == pg_count && settings.pg_stripe_size == pg_stripe_size)
     {
-        for (auto & pair: sh_it->second)
-        {
-            // like map_to_pg()
-            uint64_t pg_num = (pair.first.stripe / pg_stripe_size) % pg_count + 1;
-            uint64_t shard_id = (pool_id << (64-POOL_ID_BITS)) | pg_num;
-            new_shards[shard_id][pair.first] = pair.second;
-        }
+        return NULL;
+    }
+    bs_reshard_state_t *st = new bs_reshard_state_t;
+    st->state = 0;
+    st->pool_id = pool;
+    st->pg_count = pg_count;
+    st->pg_stripe_size = pg_stripe_size;
+    auto sh_it = clean_db_shards.lower_bound((st->pool_id << (64-POOL_ID_BITS)));
+    while (sh_it != clean_db_shards.end() &&
+        (sh_it->first >> (64-POOL_ID_BITS)) == st->pool_id)
+    {
+        st->old_shards[sh_it->first] = std::move(sh_it->second);
         clean_db_shards.erase(sh_it++);
     }
-    for (sh_it = new_shards.begin(); sh_it != new_shards.end(); sh_it++)
+    bool finished = reshard_continue(st, chunk_limit);
+    return finished ? NULL : st;
+}
+
+bool blockstore_impl_t::reshard_continue(void *reshard_state, uint64_t chunk_limit)
+{
+    bs_reshard_state_t *st = (bs_reshard_state_t*)reshard_state;
+    uint64_t chunk_size = 0;
+    if (st->state == 1)
+        goto resume_1;
+    for (st->sh_it = st->old_shards.begin(); st->sh_it != st->old_shards.end(); )
+    {
+        for (st->obj_it = st->sh_it->second.begin(); st->obj_it != st->sh_it->second.end(); st->obj_it++)
+        {
+            if (chunk_limit > 0 && chunk_size >= chunk_limit)
+            {
+                st->state = 1;
+                return false;
+            }
+resume_1:
+            // like map_to_pg()
+            uint64_t pg_num = (st->obj_it->first.stripe / st->pg_stripe_size) % st->pg_count + 1;
+            uint64_t shard_id = (st->pool_id << (64-POOL_ID_BITS)) | pg_num;
+            st->new_shards[shard_id][st->obj_it->first] = st->obj_it->second;
+            chunk_size++;
+        }
+        st->old_shards.erase(st->sh_it++);
+    }
+    for (auto sh_it = st->new_shards.begin(); sh_it != st->new_shards.end(); sh_it++)
     {
         auto & to = clean_db_shards[sh_it->first];
         to.swap(sh_it->second);
     }
-    clean_db_settings[pool_id] = (pool_shard_settings_t){
-        .pg_count = pg_count,
-        .pg_stripe_size = pg_stripe_size,
+    clean_db_settings[st->pool_id] = (pool_shard_settings_t){
+        .pg_count = st->pg_count,
+        .pg_stripe_size = st->pg_stripe_size,
     };
+    delete st;
+    return true;
+}
+
+void blockstore_impl_t::reshard_abort(void *reshard_state)
+{
+    bs_reshard_state_t *st = (bs_reshard_state_t*)reshard_state;
+    for (auto sh_it = st->old_shards.begin(); sh_it != st->old_shards.end(); sh_it++)
+    {
+        auto & to = clean_db_shards[sh_it->first];
+        to.swap(sh_it->second);
+    }
+    delete st;
 }
 
 void blockstore_impl_t::process_list(blockstore_op_t *op)
@@ -465,7 +521,10 @@ void blockstore_impl_t::process_list(blockstore_op_t *op)
                 sh_it->second.pg_count != pg_count ||
                 sh_it->second.pg_stripe_size != pg_stripe_size)
             {
-                reshard_clean_db(pool_id, pg_count, pg_stripe_size);
+                // Sharding mismatch
+                op->retval = -EAGAIN;
+                FINISH_OP(op);
+                return;
             }
             first_shard = last_shard = ((uint64_t)pool_id << (64-POOL_ID_BITS)) | list_pg;
         }

src/blockstore/v1/impl.h

@@ -202,7 +202,6 @@ class blockstore_impl_t: public blockstore_i
     uint8_t* get_clean_entry_bitmap(uint64_t block_loc, int offset);
 
     blockstore_clean_db_t& clean_db_shard(object_id oid);
-    void reshard_clean_db(pool_id_t pool_id, uint32_t pg_count, uint32_t pg_stripe_size);
     void recalc_inode_space_stats(uint64_t pool_id, bool per_inode);
 
     // Journaling
@@ -288,6 +287,11 @@ public:
     void parse_config(blockstore_config_t & config);
     void parse_config(blockstore_config_t & config, bool init);
 
+    // Reshard database for a pool
+    void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
+    bool reshard_continue(void *reshard_state, uint64_t chunk_limit);
+    void reshard_abort(void *reshard_state);
+
     // Event loop
     void loop();
 

src/blockstore/v1/write.cpp

@@ -183,7 +183,7 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
         uint32_t end = (op->offset+op->len-1) / dsk.csum_block_size;
         auto fn = state & BS_ST_BIG_WRITE ? crc32c_pad : crc32c_nopad;
         if (start == end)
-            data_csums[0] = fn(0, op->buf, op->len, op->offset - start*dsk.csum_block_size, end*dsk.csum_block_size - (op->offset+op->len));
+            data_csums[0] = fn(0, op->buf, op->len, op->offset - start*dsk.csum_block_size, (end+1)*dsk.csum_block_size - (op->offset+op->len));
         else
         {
             // First block

src/client/cluster_client.cpp

@@ -868,7 +868,8 @@ void cluster_client_t::execute_cas(cluster_op_t *op)
     {
         int expected = part->req.hdr.opcode == OSD_OP_DELETE ? 0 : part->req.rw.len;
         op->retval = part->reply.hdr.retval;
-        op->retval = op->retval == expected ? 0 : (op->retval >= 0 ? -EIO : op->retval);
+        if (op->retval != expected && op->retval >= 0)
+            op->retval = -EIO;
         op->retval = op->retval == -EPIPE ? -EINTR : op->retval;
         auto peer_it = msgr.osd_peer_fds.find(op->parts[0].osd_num);
         if (op->retval != 0 || (op->flags & OP_IMMEDIATE_COMMIT))
@@ -897,10 +898,13 @@ void cluster_client_t::execute_cas(cluster_op_t *op)
                         .opcode = OSD_OP_SYNC,
                     },
                 },
-                .callback = [this, op](osd_op_t *part)
+                .callback = [op](osd_op_t *part)
                 {
-                    op->retval = part->reply.hdr.retval;
-                    op->retval = op->retval == -EPIPE ? -EINTR : op->retval;
+                    if (part->reply.hdr.retval != 0)
+                    {
+                        op->retval = part->reply.hdr.retval;
+                        op->retval = op->retval == -EPIPE ? -EINTR : op->retval;
+                    }
                     auto cb = std::move(op->callback);
                     cb(op);
                 },
@@ -1253,9 +1257,9 @@ void cluster_client_t::slice_rw(cluster_op_t *op)
         {
             op->bitmap_buf = realloc_or_die(op->bitmap_buf, bitmap_mem);
             op->part_bitmaps = (uint8_t*)op->bitmap_buf + object_bitmap_size;
+            memset((uint8_t*)op->bitmap_buf+op->bitmap_buf_size, 0, bitmap_mem-op->bitmap_buf_size);
             op->bitmap_buf_size = bitmap_mem;
         }
-        memset(op->bitmap_buf, 0, bitmap_mem);
     }
     int iov_idx = 0;
     size_t iov_pos = 0;
@@ -1401,7 +1405,7 @@ int cluster_client_t::try_send(cluster_op_t *op, int i, std::function<void(osd_o
         if (peer_it != msgr.osd_peer_fds.end())
         {
             int peer_fd = peer_it->second;
-            part->flags |= PART_SENT;
+            part->flags |= PART_SENT|PART_VALID;
             op->inflight_count++;
             uint64_t pg_bitmap_size = (pool_cfg.data_block_size / pool_cfg.bitmap_granularity / 8) * (
                 pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks
@@ -1614,14 +1618,11 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
             dirty_osds.insert(part->osd_num);
         part->flags |= PART_DONE;
         op->done_count++;
-        if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
+        if ((op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
+            && op->inode == op->cur_inode)
         {
-            copy_part_bitmap(op, part);
-            if (op->inode == op->cur_inode)
-            {
-                // Read only returns the version of the uppermost layer
-                op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0;
-            }
+            // Read only returns the version of the uppermost layer
+            op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0;
         }
         else if (op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE)
         {
@@ -1629,6 +1630,13 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
         }
         if (op->inflight_count == 0 && !op->retry_after)
         {
+            // Copy part bitmaps only after finishing all part reads
+            if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
+            {
+                for (auto & part: op->parts)
+                    if (part.flags == (PART_SENT|PART_VALID|PART_DONE))
+                        copy_part_bitmap(op, &part);
+            }
             if (op->opcode == OSD_OP_SYNC)
                 continue_sync(op);
             else

src/client/cluster_client_impl.h

@@ -10,6 +10,7 @@
 #define PART_DONE 2
 #define PART_ERROR 4
 #define PART_RETRY 8
+#define PART_VALID 16
 #define CACHE_DIRTY 1
 #define CACHE_WRITTEN 2
 #define CACHE_FLUSHING 3

src/client/etcd_state_client.cpp

@@ -568,9 +568,19 @@ void etcd_state_client_t::start_ws_keepalive()
 
 void etcd_state_client_t::load_global_config()
 {
-    etcd_call("/kv/range", json11::Json::object {
-        { "key", base64_encode(etcd_prefix+"/config/global") }
-    }, etcd_quick_timeout, max_etcd_attempts, 0, [this](std::string err, json11::Json data)
+    json11::Json::object req = { { "success", json11::Json::array {
+        json11::Json::object {
+            { "request_range", json11::Json::object {
+                { "key", base64_encode(etcd_prefix+"/config/global") },
+            } }
+        },
+        json11::Json::object {
+            { "request_range", json11::Json::object {
+                { "key", base64_encode(etcd_prefix+"/config/pools") },
+            } }
+        },
+    } } };
+    etcd_txn(req, etcd_quick_timeout, max_etcd_attempts, 0, [this](std::string err, json11::Json data)
     {
         if (err != "")
         {
@@ -588,10 +598,12 @@ void etcd_state_client_t::load_global_config()
             }
             return;
         }
+        json11::Json config_kv = data["responses"][0]["response_range"]["kvs"][0];
+        json11::Json pools_kv = data["responses"][1]["response_range"]["kvs"][0];
         json11::Json::object global_config;
-        if (data["kvs"].array_items().size() > 0)
+        if (!config_kv.is_null())
         {
-            auto kv = parse_etcd_kv(data["kvs"][0]);
+            auto kv = parse_etcd_kv(config_kv);
             if (kv.value.is_object())
             {
                 global_config = kv.value.object_items();
@@ -608,6 +620,11 @@ void etcd_state_client_t::load_global_config()
             global_bitmap_granularity = DEFAULT_BITMAP_GRANULARITY;
         }
         global_immediate_commit = parse_immediate_commit(global_config["immediate_commit"].string_value(), IMMEDIATE_ALL);
+        if (!pools_kv.is_null())
+        {
+            auto kv = parse_etcd_kv(pools_kv);
+            parse_state(kv);
+        }
         on_load_config_hook(global_config);
     });
 }
@@ -945,8 +962,12 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
             if (pc.pg_stripe_size < min_stripe_size)
                 pc.pg_stripe_size = min_stripe_size;
             // Save
-            pc.real_pg_count = this->pool_config[pool_id].real_pg_count;
-            std::swap(pc.pg_config, this->pool_config[pool_id].pg_config);
+            auto & old_pc = this->pool_config[pool_id];
+            pc.real_pg_count = old_pc.real_pg_count;
+            pc.applied_pg_count = old_pc.applied_pg_count;
+            pc.applied_pg_stripe_size = old_pc.applied_pg_stripe_size;
+            pc.reshard_state = old_pc.reshard_state;
+            std::swap(pc.pg_config, old_pc.pg_config);
             std::swap(this->pool_config[pool_id], pc);
             auto & parsed_cfg = this->pool_config[pool_id];
             parsed_cfg.exists = true;

src/client/etcd_state_client.h

@@ -68,6 +68,11 @@ struct pool_config_t
     std::string used_for_app;
     int backfillfull = 0;
     int local_reads = 0;
+
+    // runtime data, used only by OSD:
+    uint64_t applied_pg_count = 0;
+    uint64_t applied_pg_stripe_size = 0;
+    void *reshard_state = NULL;
 };
 
 struct inode_config_t

src/client/msgr_receive.cpp

@@ -42,6 +42,7 @@ void osd_messenger_t::read_requests()
             }
             if (!sqe)
             {
+                cl->refs--;
                 cl->read_msg.msg_iovlen = 0;
                 read_ready_clients.erase(read_ready_clients.begin(), read_ready_clients.begin() + i);
                 return;

src/client/msgr_send.cpp

@@ -326,8 +326,10 @@ void osd_messenger_t::handle_send(int result, bool prev, bool more, osd_client_t
             int expected = cl->send_list.size() < IOV_MAX ? cl->send_list.size() : IOV_MAX;
             if (done != expected)
             {
-                fprintf(stderr, "BUG (maybe kernel): Expected to send %d iovecs with MSG_WAITALL but sent %d\n", expected, done);
-                exit(1);
+                fprintf(stderr, "Client %d socket write error: expected to send "
+                    "%d iovecs with MSG_WAITALL but sent %d. Disconnecting client\n", cl->peer_fd, expected, done);
+                stop_client(cl->peer_fd);
+                return;
             }
             cl->zc_free_list.push_back(NULL); // end marker
         }

src/client/msgr_stop.cpp

@@ -85,6 +85,16 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
             osd_peer_fds.erase(osd_it);
         }
     }
+#ifdef WITH_RDMA
+    if (cl->rdma_conn && cl->rdma_conn->cmid)
+    {
+        auto rdma_it = rdmacm_connections.find(cl->rdma_conn->cmid);
+        if (rdma_it != rdmacm_connections.end() && rdma_it->second == cl)
+        {
+            rdmacm_connections.erase(rdma_it);
+        }
+    }
+#endif
 #ifndef __MOCK__
     // Then remove FD from the eventloop so we don't accidentally read something
     tfd->set_fd_handler(peer_fd, false, NULL);
@@ -158,6 +168,13 @@ osd_client_t::~osd_client_t()
     }
     // Cancel outbound ops
     cancel_ops();
+    for (osd_op_t *op: zc_free_list)
+    {
+        if (op)
+        {
+            delete op;
+        }
+    }
 #ifndef __MOCK__
 #ifdef WITH_RDMA
     if (rdma_conn)

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 3.0.0
+Version: 3.0.2
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/cmd/cli.cpp

@@ -245,7 +245,7 @@ static json11::Json::object parse_args(int narg, const char *args[])
     cfg["progress"] = "1";
     for (int i = 1; i < narg; i++)
     {
-        bool argHasValue = (!(i == narg-1) && (args[i+1][0] != '-'));
+        bool argHasValue = (i < narg-1);
         if (args[i][0] == '-' && args[i][1] == 'h' && args[i][2] == 0)
         {
             cfg["help"] = "1";

src/cmd/cli_dd.cpp

@@ -111,7 +111,7 @@ struct dd_in_info_t
             {
                 in_granularity = 512;
             }
-            if (lseek(ifd, 1, SEEK_SET) == (off_t)-1)
+            if (lseek(ifd, 1, SEEK_SET) != (off_t)1)
             {
                 in_seekable = false;
             }
@@ -389,7 +389,7 @@ struct cli_dd_t
     int state = 0;
     int copy_error = 0;
     int in_waiting = 0, out_waiting = 0;
-    cli_result_t result;
+    cli_result_t result = {};
 
     bool is_done()
     {
@@ -890,12 +890,12 @@ resume_2:
         clock_gettime(CLOCK_REALTIME, &tv_begin);
         tv_progress = tv_begin;
 resume_3:
-        while ((ignore_errors || !copy_error) && (!in_eof || read_buffers.size() || in_waiting > 0 || out_waiting > 0))
+        while ((ignore_errors || !copy_error) && (!in_eof || read_buffers.size() || in_waiting > 0 || out_waiting > 0 || short_writes.size()))
         {
             print_progress(false);
             while ((ignore_errors || !copy_error) &&
                 (!in_eof && in_waiting < in_iodepth && read_buffers.size() < out_iodepth ||
-                read_buffers.size() && out_waiting < out_iodepth))
+                (read_buffers.size() || short_writes.size()) && out_waiting < out_iodepth))
             {
                 if (!in_eof && in_waiting < in_iodepth && read_buffers.size() < out_iodepth)
                 {
@@ -904,7 +904,7 @@ resume_3:
                         break;
                     }
                 }
-                if (read_buffers.size() && out_waiting < out_iodepth)
+                if ((read_buffers.size() || short_writes.size()) && out_waiting < out_iodepth)
                 {
                     if (!add_write_op())
                     {
@@ -931,7 +931,8 @@ close_end:
         oinfo.close_output(parent);
         iinfo.close_input(parent);
         // Done
-        result.err = copy_error;
+        if (copy_error)
+            result.err = copy_error;
         state = 100;
     }
 };

src/cmd/cli_modify_osd.cpp

@@ -120,12 +120,23 @@ resume_1:
                 else
                     osd_cfg.erase("noout");
             }
-            compare.push_back(json11::Json::object {
-                { "target", "MOD" },
-                { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/osd/"+std::to_string(osd_num)) },
-                { "result", "LESS" },
-                { "mod_revision", osd_cfg_mod_rev+1 },
-            });
+            if (osd_cfg_mod_rev)
+            {
+                compare.push_back(json11::Json::object {
+                    { "target", "MOD" },
+                    { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/osd/"+std::to_string(osd_num)) },
+                    { "result", "LESS" },
+                    { "mod_revision", osd_cfg_mod_rev+1 },
+                });
+            }
+            else
+            {
+                compare.push_back(json11::Json::object {
+                    { "target", "VERSION" },
+                    { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/osd/"+std::to_string(osd_num)) },
+                    { "version", 0 },
+                });
+            }
             if (!osd_cfg.size())
             {
                 success.push_back(json11::Json::object {

src/disk_tool/disk_tool.cpp

@@ -65,6 +65,8 @@ static const char *help_text =
     "    --force                    Bypass partition safety checks (for emptiness and so on)\n"
     "  \n"
     "  Options (both modes):\n"
+    "    --tags tag1,tag2           Set new OSD tag(s)\n"
+    "    --weight <number>          Set new OSD weight (between 0 and 1)\n"
     "    --journal_size 32M/1G      Set journal size (area or partition size)\n"
     "    --block_size 128k/1M       Set blockstore object size\n"
     "    --bitmap_granularity 4k    Set bitmap granularity\n"

src/disk_tool/disk_tool.h

@@ -88,6 +88,7 @@ struct disk_tool_t
     uint32_t new_meta_format = 0;
     int new_journal_fd = -1, new_meta_fd = -1;
     resizer_data_moving_t *moving_blocks = NULL;
+    bool atomic_warned = false;
 
     bool started = false;
     void *small_write_data = NULL;

src/disk_tool/disk_tool_discard.cpp

@@ -54,7 +54,7 @@ int disk_tool_t::trim_data(std::string device)
     fprintf(stderr, "Reading metadata\n");
     data_alloc = new allocator_t(dsk.block_count);
     r = process_meta(
-        [this](blockstore_meta_header_v3_t *hdr) {},
+        [](blockstore_meta_header_v3_t *hdr) {},
         [this](blockstore_heap_t *heap, heap_entry_t *obj, uint32_t meta_block_num)
         {
             for (auto wr = obj; wr; wr = heap->prev(wr))

src/disk_tool/disk_tool_prepare.cpp

@@ -78,25 +78,37 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
             if (check_existing_partition(dev) != 0)
                 return 1;
         }
+        if (options.find("weight") != options.end())
+        {
+            double reweight = json11::Json(options["weight"]).number_value();
+            if (reweight < 0 || reweight > 1)
+            {
+                fprintf(stderr, "OSD weight must be between 0 and 1\n");
+                return 1;
+            }
+        }
     }
     if (options.find("atomic_write_size") == options.end())
     {
         auto data_dev = realpath_str(options["data_device"], false);
-        uint64_t atomic_write_size = get_atomic_write_size(data_dev);
-        if (atomic_write_size > 4096)
+        if (data_dev.substr(0, 9) == "/dev/nvme")
         {
-            fprintf(stderr, "Data device %s supports atomic writes up to %ju bytes, enabling. Enjoy faster writes!\n",
-                data_dev.c_str(), atomic_write_size);
-            options["atomic_write_size"] = std::to_string(atomic_write_size);
-            options["use_atomic_flag"] = "1";
+            uint64_t atomic_write_size = get_atomic_write_size(data_dev);
+            if (atomic_write_size > 4096)
+            {
+                // FIXME: Enable use_atomic_flag when the kernel does checks correctly
+                fprintf(stderr, "Data device %s supports atomic writes up to %ju bytes, enabling. Enjoy faster writes!\n",
+                    data_dev.c_str(), atomic_write_size);
+                options["atomic_write_size"] = std::to_string(atomic_write_size);
+                if (!atomic_warned)
+                {
+                    fprintf(stderr, "WARNING: RWF_ATOMIC can't be used because Linux checks atomic writes incorrectly.\n"
+                        " Please don't change scheduler from default 'none' and check use_atomic_flag documentation for more details.\n");
+                    atomic_warned = true;
+                }
+            }
         }
     }
-    else if (options.find("use_atomic_flag") == options.end() &&
-        parse_size(options["atomic_write_size"]) > 4096)
-    {
-        fprintf(stderr, "Atomic writes larger than 4 KB are enabled manually, enabling use_atomic_flag too.\n");
-        options["use_atomic_flag"] = "1";
-    }
     for (auto dev: std::vector<std::string>{"data", "meta", "journal"})
     {
         if (options[dev+"_device"] != "" && options["disable_"+dev+"_fsync"] == "auto")
@@ -238,9 +250,28 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
         return 1;
     }
     sb["osd_num"] = osd_num;
-    // Zero out metadata and journal
-    if (write_zero(dsk.meta_fd, sb["meta_offset"].uint64_value(), dsk.meta_area_size) != 0 ||
-        write_zero(dsk.journal_fd, sb["journal_offset"].uint64_value(), dsk.journal_len) != 0)
+    if (options.find("weight") != options.end() || options.find("tags") != options.end())
+    {
+        std::vector<std::string> cmd = { "vitastor-cli", "modify-osd", std::to_string(osd_num) };
+        if (options.find("weight") != options.end())
+        {
+            cmd.push_back("--reweight");
+            cmd.push_back(options["weight"]);
+        }
+        if (options.find("tags") != options.end())
+        {
+            cmd.push_back("--tags");
+            cmd.push_back(options["tags"]);
+        }
+        if (shell_exec(cmd, "", NULL, NULL) != 0)
+        {
+            fprintf(stderr, "Failed to modify OSD %ju tags and/or reweight\n", osd_num);
+            return 1;
+        }
+    }
+    // Zero out the first block of metadata and journal - OSD will zero the rest on the first run
+    if (write_zero(dsk.meta_fd, sb["meta_offset"].uint64_value(), 4096) != 0 ||
+        write_zero(dsk.journal_fd, sb["journal_offset"].uint64_value(), 4096) != 0)
     {
         fprintf(stderr, "Failed to zero out metadata or journal: %s\n", strerror(errno));
         dsk.close_all();

src/nfs/nfs_kv.h

@@ -104,8 +104,10 @@ struct nfs_rmw_t
     nfs_proxy_t *parent = NULL;
     uint64_t ino = 0;
     uint64_t offset = 0;
-    uint8_t *buf = NULL;
-    uint64_t size = 0;
+    uint8_t *buf1 = NULL;
+    uint64_t size1 = 0;
+    uint8_t *buf2 = NULL;
+    uint64_t size2 = 0;
     uint8_t *part_buf = NULL;
     uint64_t version = 0;
     nfs_rmw_t *other = NULL;

src/nfs/nfs_kv_write.cpp

@@ -185,8 +185,8 @@ void nfs_do_rmw(nfs_rmw_t *rmw)
 {
     auto parent = rmw->parent;
     auto align = parent->kvfs->pool_alignment;
-    assert(rmw->size < align);
-    assert((rmw->offset/parent->kvfs->pool_block_size) == ((rmw->offset+rmw->size-1)/parent->kvfs->pool_block_size));
+    assert(rmw->size1+rmw->size2 < align);
+    assert((rmw->offset/parent->kvfs->pool_block_size) == ((rmw->offset+rmw->size1+rmw->size2-1)/parent->kvfs->pool_block_size));
     if (!rmw->part_buf)
     {
         rmw->part_buf = (uint8_t*)malloc_or_die(align);
@@ -221,7 +221,7 @@ void nfs_do_rmw(nfs_rmw_t *rmw)
             }
             auto align = parent->kvfs->pool_alignment;
             bool is_begin = (rmw->offset % align);
-            bool is_end = ((rmw->offset+rmw->size) % align);
+            bool is_end = ((rmw->offset+rmw->size1+rmw->size2) % align);
             auto op = new cluster_op_t;
             op->opcode = OSD_OP_WRITE;
             op->inode = rmw->ino;
@@ -232,10 +232,17 @@ void nfs_do_rmw(nfs_rmw_t *rmw)
             {
                 op->iov.push_back(rmw->part_buf, rmw->offset % align);
             }
-            op->iov.push_back(rmw->buf, rmw->size);
+            if (rmw->buf1)
+            {
+                op->iov.push_back(rmw->buf1, rmw->size1);
+            }
+            if (rmw->buf2)
+            {
+                op->iov.push_back(rmw->buf2, rmw->size2);
+            }
             if (is_end)
             {
-                op->iov.push_back(rmw->part_buf + (rmw->offset % align) + rmw->size, align - (rmw->offset % align) - rmw->size);
+                op->iov.push_back(rmw->part_buf + (rmw->offset % align) + rmw->size1 + rmw->size2, align - (rmw->offset % align) - rmw->size1 - rmw->size2);
             }
             op->callback = [rmw](cluster_op_t *op)
             {
@@ -446,9 +453,9 @@ static void nfs_do_align_write(nfs_kv_write_state *st, uint64_t ino, uint64_t of
     uint64_t good_offset = offset;
     uint64_t good_size = st->size;
     bool begin_shdr = false;
-    uint64_t end_pad = 0;
     st->waiting++;
-    st->rmw[0].buf = st->rmw[1].buf = NULL;
+    st->rmw[0].buf1 = st->rmw[1].buf1 = NULL;
+    st->rmw[0].buf2 = st->rmw[1].buf2 = NULL;
     auto make_rmw_cb = [st, state]()
     {
         return [st, state](nfs_rmw_t *rmw)
@@ -473,7 +480,7 @@ static void nfs_do_align_write(nfs_kv_write_state *st, uint64_t ino, uint64_t of
             };
             begin_shdr = true;
             good_offset -= sizeof(shared_file_header_t);
-            offset = 0;
+            offset -= sizeof(shared_file_header_t);
         }
         else
         {
@@ -492,8 +499,8 @@ static void nfs_do_align_write(nfs_kv_write_state *st, uint64_t ino, uint64_t of
                 .parent = st->proxy,
                 .ino = ino,
                 .offset = offset,
-                .buf = st->buf,
-                .size = s,
+                .buf1 = st->buf,
+                .size1 = s,
                 .cb = make_rmw_cb(),
             };
             st->waiting++;
@@ -501,14 +508,26 @@ static void nfs_do_align_write(nfs_kv_write_state *st, uint64_t ino, uint64_t of
         }
     }
     if ((end % alignment) &&
-        (offset == 0 || end/alignment > (offset-1)/alignment))
+        ((end-1)/alignment > offset/alignment || offset == ((end-1) & ~(alignment-1))))
     {
         // Requires read-modify-write in the end
         assert(st->offset+st->size <= st->new_size);
-        if (st->offset+st->size == st->new_size)
+        if ((end-1)/alignment == offset/alignment && begin_shdr)
         {
-            // rmw can be skipped at end - we can just zero pad the request
-            end_pad = alignment - (end % alignment);
+            // end is at the same moment the beginning with a shared header
+            assert(!(offset % alignment) && good_size == st->size);
+            st->rmw[1] = (nfs_rmw_t){
+                .parent = st->proxy,
+                .ino = ino,
+                .offset = offset,
+                .buf1 = (uint8_t*)&st->shdr,
+                .size1 = sizeof(shared_file_header_t),
+                .buf2 = st->buf,
+                .size2 = st->size,
+                .cb = make_rmw_cb(),
+            };
+            good_size = 0;
+            begin_shdr = false;
         }
         else
         {
@@ -521,29 +540,28 @@ static void nfs_do_align_write(nfs_kv_write_state *st, uint64_t ino, uint64_t of
                 .parent = st->proxy,
                 .ino = ino,
                 .offset = end - s,
-                .buf = st->buf + st->size - s,
-                .size = s,
+                .buf2 = st->buf + st->size - s,
+                .size2 = s,
                 .cb = make_rmw_cb(),
             };
-            if (st->rmw[0].buf)
-            {
-                st->rmw[0].other = &st->rmw[1];
-                st->rmw[1].other = &st->rmw[0];
-            }
-            st->waiting++;
-            nfs_do_rmw(&st->rmw[1]);
         }
+        if (st->rmw[0].buf1)
+        {
+            st->rmw[0].other = &st->rmw[1];
+            st->rmw[1].other = &st->rmw[0];
+        }
+        st->waiting++;
+        nfs_do_rmw(&st->rmw[1]);
     }
-    if (good_size > 0 || end_pad > 0 || begin_shdr)
+    assert(!begin_shdr || good_size > 0);
+    if (good_size > 0)
     {
         // Normal write
-        nfs_do_write(ino, good_offset, (begin_shdr ? sizeof(shared_file_header_t) : 0)+good_size+end_pad, [&](cluster_op_t *op)
+        nfs_do_write(ino, good_offset, (begin_shdr ? sizeof(shared_file_header_t) : 0)+good_size, [&](cluster_op_t *op)
         {
             if (begin_shdr)
                 op->iov.push_back(&st->shdr, sizeof(shared_file_header_t));
             op->iov.push_back(good_buf, good_size);
-            if (end_pad)
-                op->iov.push_back(st->proxy->kvfs->zero_block.data(), end_pad);
         }, st, state);
     }
     st->waiting--;

src/osd/osd.cpp

@@ -31,25 +31,6 @@ osd_t::osd_t(const json11::Json & config, ring_loop_t *ringloop)
     // FIXME: Use timerfd_interval based directly on io_uring
     this->tfd = epmgr->tfd;
 
-    if (!json_is_true(this->config["disable_blockstore"]))
-    {
-        auto bs_cfg = json_to_string_map(this->config);
-        this->bs = blockstore_i::create(bs_cfg, ringloop, tfd);
-        // Wait for blockstore initialisation before actually starting OSD logic
-        // to prevent peering timeouts during restart with filled databases
-        while (!bs->is_started())
-        {
-            ringloop->loop();
-            if (bs->is_started())
-                break;
-            ringloop->wait();
-        }
-        // Autosync based on the number of unstable writes to prevent stalls due to insufficient journal space
-        uint64_t max_autosync = bs->get_journal_size() / bs->get_block_size() / 2;
-        if (autosync_writes > max_autosync)
-            autosync_writes = max_autosync;
-    }
-
     if (json_is_true(this->config["osd_memlock"]))
     {
         // Lock all OSD memory if requested
@@ -117,6 +98,7 @@ osd_t::~osd_t()
         autosync_timer_id = -1;
     }
     ringloop->unregister_consumer(&consumer);
+    ringloop->unregister_consumer(&init_consumer);
     delete epmgr;
     if (bs)
         delete bs;
@@ -131,6 +113,43 @@ osd_t::~osd_t()
     free(zero_buffer);
 }
 
+void osd_t::init_blockstore(std::function<void()> on_init)
+{
+    if (!json_is_true(this->config["disable_blockstore"]))
+    {
+        auto bs_cfg = json_to_string_map(this->config);
+        this->bs = blockstore_i::create(bs_cfg, ringloop, tfd);
+        // Pre-configure pool PG shards
+        for (auto & pool_item: st_cli.pool_config)
+        {
+            auto st = bs->reshard_start(pool_item.first, pool_item.second.pg_count, pool_item.second.pg_stripe_size, 0);
+            assert(!st);
+        }
+        // Autosync based on the number of unstable writes to prevent stalls due to insufficient journal space
+        uint64_t max_autosync = bs->get_journal_size() / bs->get_block_size() / 2;
+        if (autosync_writes > max_autosync)
+            autosync_writes = max_autosync;
+        if (on_init)
+        {
+            init_consumer.loop = [this, on_init]()
+            {
+                // Wait for blockstore initialisation before actually starting OSD logic
+                // to prevent peering timeouts during restart with filled databases
+                if (bs->is_started())
+                {
+                    ringloop->set_immediate([this, on_init] { init_consumer.loop = NULL; on_init(); });
+                    ringloop->unregister_consumer(&init_consumer);
+                }
+            };
+            ringloop->register_consumer(&init_consumer);
+        }
+    }
+    else if (on_init)
+    {
+        on_init();
+    }
+}
+
 void osd_t::parse_config(bool init)
 {
     config = msgr.merge_configs(cli_config, file_config, etcd_global_config, etcd_osd_config);
@@ -292,6 +311,12 @@ void osd_t::parse_config(bool init)
     scrub_list_limit = config["scrub_list_limit"].uint64_value();
     if (!scrub_list_limit)
         scrub_list_limit = 262144;
+    pg_reshard_chunk_size = config["pg_reshard_chunk_size"].uint64_value();
+    if (!pg_reshard_chunk_size)
+        pg_reshard_chunk_size = 100000;
+    pg_reshard_chunk_pause_ms = config["pg_reshard_chunk_pause_ms"].uint64_value();
+    if (!pg_reshard_chunk_pause_ms)
+        pg_reshard_chunk_pause_ms = 100;
     if (!old_auto_scrub && auto_scrub)
     {
         // Schedule scrubbing

src/osd/osd.h

@@ -148,6 +148,8 @@ class osd_t
     bool enable_pg_locks = false;
     bool pg_locks_localize_only = false;
     uint64_t pg_lock_retry_interval_ms = 100;
+    uint64_t pg_reshard_chunk_size = 100000;
+    uint64_t pg_reshard_chunk_pause_ms = 100;
 
     // cluster state
 
@@ -158,6 +160,7 @@ class osd_t
     json11::Json self_state;
     bool loading_peer_config = false;
     std::set<pool_pg_num_t> pg_state_dirty;
+    bool etcd_global_config_loaded = false;
     bool pg_config_applied = false;
     bool etcd_reporting_pg_state = false;
     bool etcd_reporting_stats = false;
@@ -168,10 +171,11 @@ class osd_t
     // peers and PGs
 
     std::map<pool_pg_num_t, osd_pg_lock_t> pg_locks;
-    std::map<pool_id_t, pg_num_t> pg_counts;
     std::map<pool_pg_num_t, pg_t> pgs;
     std::set<pool_pg_num_t> dirty_pgs;
     std::set<osd_num_t> dirty_osds;
+    std::vector<pool_id_t> reshard_pools;
+    int reshard_timer_id = -1;
     int copies_to_delete_after_sync_count = 0;
     uint64_t misplaced_objects = 0, degraded_objects = 0, incomplete_objects = 0, inconsistent_objects = 0, corrupted_objects = 0;
     int peering_state = 0;
@@ -206,7 +210,7 @@ class osd_t
     void *zero_buffer = NULL;
     uint64_t zero_buffer_size = 0;
     uint32_t bs_block_size, bs_bitmap_granularity, clean_entry_bitmap_size;
-    ring_loop_t *ringloop;
+    ring_loop_t *ringloop = NULL;
     timerfd_manager_t *tfd = NULL;
     epoll_manager_t *epmgr = NULL;
 
@@ -218,6 +222,7 @@ class osd_t
     int rdmacm_port = 0;
 #endif
     ring_consumer_t consumer;
+    ring_consumer_t init_consumer;
 
     // op statistics
     osd_op_stats_t prev_stats, prev_report_stats;
@@ -241,6 +246,7 @@ class osd_t
 
     // cluster connection
     void parse_config(bool init);
+    void init_blockstore(std::function<void()> on_init);
     void init_cluster();
     void on_change_osd_state_hook(osd_num_t peer_osd);
     void on_change_backfillfull_hook(pool_id_t pool_id);
@@ -268,6 +274,7 @@ class osd_t
     void apply_no_inode_stats();
     void apply_pg_count();
     void apply_pg_config();
+    void reshard_continue();
 
     // event loop, socket read/write
     void loop();
@@ -317,7 +324,7 @@ class osd_t
     void finish_op(osd_op_t *cur_op, int retval);
 
     // secondary ops
-    bool sec_check_pg_lock(osd_num_t primary_osd, const object_id &oid);
+    bool sec_check_pg_lock(osd_num_t primary_osd, const object_id & oid, uint32_t flags);
     void exec_show_config(osd_op_t *cur_op);
     void exec_secondary(osd_op_t *cur_op);
     void exec_secondary_real(osd_op_t *cur_op);
@@ -376,12 +383,12 @@ class osd_t
     int submit_bitmap_subops(osd_op_t *cur_op, pg_t & pg);
     int read_bitmaps(osd_op_t *cur_op, pg_t *pg, int base_state);
 
-    inline pg_num_t map_to_pg(object_id oid, uint64_t pg_stripe_size)
+    inline pg_num_t map_to_pg(object_id oid)
     {
-        uint64_t pg_count = pg_counts[INODE_POOL(oid.inode)];
-        if (!pg_count)
-            pg_count = 1;
-        return (oid.stripe / pg_stripe_size) % pg_count + 1;
+        auto pool_it = st_cli.pool_config.find(INODE_POOL(oid.inode));
+        if (pool_it == st_cli.pool_config.end())
+            return 1;
+        return (oid.stripe / pool_it->second.applied_pg_stripe_size) % pool_it->second.applied_pg_count + 1;
     }
 
 public:

src/osd/osd_cluster.cpp

@@ -18,6 +18,7 @@ void osd_t::init_cluster()
 {
     if (!st_cli.address_count())
     {
+        init_blockstore(NULL);
         if (run_primary)
         {
             // Test version of clustering code with 1 pool, 1 PG and 2 peers
@@ -54,9 +55,10 @@ void osd_t::init_cluster()
                 .pg_minsize = 2,
                 .pg_count = 1,
                 .real_pg_count = 1,
+                .applied_pg_count = 1,
+                .applied_pg_stripe_size = bs_block_size*2,
             };
             report_pg_state(pgs[{ 1, 1 }]);
-            pg_counts[1] = 1;
         }
         bind_socket();
     }
@@ -422,7 +424,10 @@ void osd_t::on_change_osd_state_hook(osd_num_t peer_osd)
 
 void osd_t::on_change_pool_config_hook()
 {
-    apply_pg_locks_localize_only();
+    if (etcd_global_config_loaded)
+    {
+        apply_pg_locks_localize_only();
+    }
 }
 
 void osd_t::apply_pg_locks_localize_only()
@@ -484,9 +489,13 @@ void osd_t::on_load_config_hook(json11::Json::object & global_config)
 {
     etcd_global_config = global_config;
     parse_config(true);
-    bind_socket();
-    acquire_lease();
     st_cli.on_load_config_hook = [this](json11::Json::object & cfg) { on_reload_config_hook(cfg); };
+    etcd_global_config_loaded = true;
+    init_blockstore([this]()
+    {
+        bind_socket();
+        acquire_lease();
+    });
 }
 
 void osd_t::on_reload_config_hook(json11::Json::object & global_config)
@@ -719,8 +728,13 @@ void osd_t::apply_pg_count()
 {
     for (auto & pool_item: st_cli.pool_config)
     {
-        if (pool_item.second.real_pg_count != 0 &&
-            pool_item.second.real_pg_count != pg_counts[pool_item.first])
+        auto & pool_cfg = pool_item.second;
+        if (pool_cfg.real_pg_count == 0)
+        {
+            continue;
+        }
+        if (pool_cfg.real_pg_count != pool_cfg.applied_pg_count ||
+            pool_cfg.pg_stripe_size != pool_cfg.applied_pg_stripe_size)
         {
             // Check that all pool PGs are offline. It is not allowed to change PG count when any PGs are online
             // The external tool must wait for all PGs to come down before changing PG count
@@ -742,17 +756,91 @@ void osd_t::apply_pg_count()
             }
             if (still_active_primary > 0 || still_active_secondary > 0)
             {
-                printf(
-                    "[OSD %ju] PG count change detected for pool %u (new is %ju, old is %u),"
-                    " but %u PG(s) are still active as primary and %u as secondary. This is not allowed. Exiting\n",
-                    this->osd_num, pool_item.first, pool_item.second.real_pg_count, pg_counts[pool_item.first],
-                    still_active_primary, still_active_secondary
-                );
+                if (pool_cfg.real_pg_count != pool_cfg.applied_pg_count)
+                {
+                    printf(
+                        "[OSD %ju] PG count change detected for pool %u (new is %ju, old is %ju),"
+                        " but %u PG(s) are still active as primary and %u as secondary. This is not allowed. Exiting\n",
+                        this->osd_num, pool_item.first, pool_cfg.real_pg_count, pool_cfg.applied_pg_count,
+                        still_active_primary, still_active_secondary
+                    );
+                }
+                else
+                {
+                    printf(
+                        "[OSD %ju] PG stripe change detected for pool %u (new is %ju, old is %ju),"
+                        " but %u PG(s) are still active as primary and %u as secondary. This is not allowed. Exiting\n",
+                        this->osd_num, pool_item.first, pool_cfg.pg_stripe_size, pool_cfg.applied_pg_stripe_size,
+                        still_active_primary, still_active_secondary
+                    );
+                }
                 force_stop(1);
                 return;
             }
+            pool_cfg.applied_pg_count = pool_cfg.real_pg_count;
+            pool_cfg.applied_pg_stripe_size = pool_cfg.pg_stripe_size;
+            if (bs && !pool_cfg.reshard_state)
+            {
+                pool_cfg.reshard_state = bs->reshard_start(pool_item.first, pool_cfg.real_pg_count, pool_cfg.pg_stripe_size, pg_reshard_chunk_size);
+                if (pool_cfg.reshard_state)
+                {
+                    reshard_pools.push_back(pool_item.first);
+                }
+            }
         }
-        this->pg_counts[pool_item.first] = pool_item.second.real_pg_count;
+    }
+    if (reshard_pools.size() && reshard_timer_id < 0)
+    {
+        reshard_timer_id = tfd->set_timer(pg_reshard_chunk_pause_ms, false, [this](int)
+        {
+            reshard_continue();
+        });
+    }
+}
+
+void osd_t::reshard_continue()
+{
+again:
+    auto pool_id = reshard_pools[0];
+    auto pool_it = st_cli.pool_config.find(pool_id);
+    if (pool_it == st_cli.pool_config.end() || !pool_it->second.reshard_state)
+    {
+        reshard_pools.erase(reshard_pools.begin());
+        goto again;
+    }
+    auto & pool_cfg = pool_it->second;
+    bool done = false;
+    if (pool_cfg.real_pg_count != pool_cfg.applied_pg_count ||
+        pool_cfg.pg_stripe_size != pool_cfg.applied_pg_stripe_size)
+    {
+        // PG count changed again, reshard again
+        bs->reshard_abort(pool_cfg.reshard_state);
+        pool_cfg.applied_pg_count = pool_cfg.real_pg_count;
+        pool_cfg.applied_pg_stripe_size = pool_cfg.pg_stripe_size;
+        pool_cfg.reshard_state = bs->reshard_start(pool_id, pool_cfg.real_pg_count, pool_cfg.pg_stripe_size, pg_reshard_chunk_size);
+        done = !pool_cfg.reshard_state;
+    }
+    else
+    {
+        done = bs->reshard_continue(pool_cfg.reshard_state, pg_reshard_chunk_size);
+    }
+    if (done)
+    {
+        // Pool is resharded
+        pool_cfg.reshard_state = NULL;
+        reshard_pools.erase(reshard_pools.begin());
+        apply_pg_config();
+    }
+    if (reshard_pools.size())
+    {
+        reshard_timer_id = tfd->set_timer(pg_reshard_chunk_pause_ms, false, [this](int)
+        {
+            reshard_continue();
+        });
+    }
+    else
+    {
+        reshard_timer_id = -1;
     }
 }
 
@@ -761,9 +849,15 @@ void osd_t::apply_pg_config()
     bool all_applied = true;
     for (auto & pool_item: st_cli.pool_config)
     {
+        auto & pool_cfg = pool_item.second;
+        if (pool_cfg.reshard_state)
+        {
+            // Can't apply anything for pools being resharded
+            continue;
+        }
         bool warned_block_size = false;
         auto pool_id = pool_item.first;
-        for (auto & kv: pool_item.second.pg_config)
+        for (auto & kv: pool_cfg.pg_config)
         {
             pg_num_t pg_num = kv.first;
             auto & pg_cfg = kv.second;
@@ -772,8 +866,8 @@ void osd_t::apply_pg_config()
             auto pg_it = this->pgs.find({ .pool_id = pool_id, .pg_num = pg_num });
             bool currently_taken = pg_it != this->pgs.end() && pg_it->second.state != PG_OFFLINE;
             // Check pool block size and bitmap granularity
-            if (take && this->bs_block_size != pool_item.second.data_block_size ||
-                this->bs_bitmap_granularity != pool_item.second.bitmap_granularity)
+            if (take && this->bs_block_size != pool_cfg.data_block_size ||
+                this->bs_bitmap_granularity != pool_cfg.bitmap_granularity)
             {
                 if (!warned_block_size)
                 {
@@ -781,7 +875,7 @@ void osd_t::apply_pg_config()
                         "[OSD %ju] My block_size and bitmap_granularity are %u/%u"
                         ", but pool %u has %u/%u. Refusing to start PGs of this pool\n",
                         this->osd_num, bs_block_size, bs_bitmap_granularity,
-                        pool_id, pool_item.second.data_block_size, pool_item.second.bitmap_granularity
+                        pool_id, pool_cfg.data_block_size, pool_cfg.bitmap_granularity
                     );
                 }
                 warned_block_size = true;
@@ -874,12 +968,12 @@ void osd_t::apply_pg_config()
                 }
                 auto & pg = this->pgs[{ .pool_id = pool_id, .pg_num = pg_num }];
                 pg.state = pg_cfg.cur_primary == this->osd_num ? PG_PEERING : PG_STARTING;
-                pg.scheme = pool_item.second.scheme;
+                pg.scheme = pool_cfg.scheme;
                 pg.pg_cursize = 0;
-                pg.pg_size = pool_item.second.pg_size;
-                pg.pg_minsize = pool_item.second.pg_minsize;
-                pg.pg_data_size = pool_item.second.scheme == POOL_SCHEME_REPLICATED
-                     ? 1 : pool_item.second.pg_size - pool_item.second.parity_chunks;
+                pg.pg_size = pool_cfg.pg_size;
+                pg.pg_minsize = pool_cfg.pg_minsize;
+                pg.pg_data_size = pool_cfg.scheme == POOL_SCHEME_REPLICATED
+                     ? 1 : pool_cfg.pg_size - pool_cfg.parity_chunks;
                 pg.pool_id = pool_id;
                 pg.pg_num = pg_num;
                 pg.reported_epoch = pg_cfg.epoch;
@@ -888,8 +982,8 @@ void osd_t::apply_pg_config()
                 pg.next_scrub = pg_cfg.next_scrub;
                 pg.target_set = pg_cfg.target_set;
                 pg.disable_pg_locks = pg_locks_localize_only &&
-                    (pool_item.second.scheme != POOL_SCHEME_REPLICATED ||
-                    pool_item.second.local_reads == POOL_LOCAL_READ_PRIMARY);
+                    (pool_cfg.scheme != POOL_SCHEME_REPLICATED ||
+                    pool_cfg.local_reads == POOL_LOCAL_READ_PRIMARY);
                 if (pg.scheme == POOL_SCHEME_EC)
                 {
                     use_ec(pg.pg_size, pg.pg_data_size, true);

src/osd/osd_flush.cpp

@@ -318,8 +318,7 @@ void osd_t::submit_recovery_op(osd_recovery_op_t *op)
                 // EPIPE is totally harmless (peer is gone), others like EIO/EDOM may be not
                 printf(
                     "[PG %u/%u] Recovery operation failed with object %jx:%jx: error %jd\n",
-                    INODE_POOL(op->oid.inode),
-                    map_to_pg(op->oid, st_cli.pool_config.at(INODE_POOL(op->oid.inode)).pg_stripe_size),
+                    INODE_POOL(op->oid.inode), map_to_pg(op->oid),
                     op->oid.inode, op->oid.stripe, op->osd_op->reply.hdr.retval
                 );
             }

src/osd/osd_peering.cpp

@@ -159,11 +159,10 @@ void osd_t::reset_pg(pg_t & pg)
         cancel_primary_write(p.second);
     }
     pg.write_queue.clear();
-    uint64_t pg_stripe_size = st_cli.pool_config[pg.pool_id].pg_stripe_size;
     for (auto it = unstable_writes.begin(); it != unstable_writes.end(); )
     {
         // Forget this PG's unstable writes
-        if (INODE_POOL(it->first.oid.inode) == pg.pool_id && map_to_pg(it->first.oid, pg_stripe_size) == pg.pg_num)
+        if (INODE_POOL(it->first.oid.inode) == pg.pool_id && map_to_pg(it->first.oid) == pg.pg_num)
             unstable_writes.erase(it++);
         else
             it++;
@@ -524,6 +523,7 @@ void osd_t::relock_pg(pg_t & pg)
 
 void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
 {
+    auto & pool_cfg = st_cli.pool_config.at(ps->pool_id);
     if (role_osd == this->osd_num)
     {
         // Self
@@ -533,11 +533,11 @@ void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
         clock_gettime(CLOCK_REALTIME, &op->tv_begin);
         op->bs_op = new blockstore_op_t();
         op->bs_op->opcode = BS_OP_LIST;
-        op->bs_op->pg_alignment = st_cli.pool_config[ps->pool_id].pg_stripe_size;
+        op->bs_op->pg_alignment = pool_cfg.applied_pg_stripe_size;
         op->bs_op->min_oid.inode = ((uint64_t)ps->pool_id << (64 - POOL_ID_BITS));
         op->bs_op->max_oid.inode = ((uint64_t)(ps->pool_id+1) << (64 - POOL_ID_BITS)) - 1;
         op->bs_op->max_oid.stripe = UINT64_MAX;
-        op->bs_op->pg_count = pg_counts[ps->pool_id];
+        op->bs_op->pg_count = pool_cfg.applied_pg_count;
         op->bs_op->pg_number = ps->pg_num-1;
         op->bs_op->callback = [this, ps, op, role_osd](blockstore_op_t *bs_op)
         {
@@ -584,8 +584,8 @@ void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
                     .opcode = OSD_OP_SEC_LIST,
                 },
                 .list_pg = ps->pg_num,
-                .pg_count = pg_counts[ps->pool_id],
-                .pg_stripe_size = st_cli.pool_config[ps->pool_id].pg_stripe_size,
+                .pg_count = (uint32_t)pool_cfg.applied_pg_count,
+                .pg_stripe_size = pool_cfg.applied_pg_stripe_size,
                 .min_inode = ((uint64_t)(ps->pool_id) << (64 - POOL_ID_BITS)),
                 .max_inode = ((uint64_t)(ps->pool_id+1) << (64 - POOL_ID_BITS)) - 1,
             },

src/osd/osd_primary.cpp

@@ -35,7 +35,14 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
         // oid.stripe = starting offset of the parity stripe
         .stripe = (cur_op->req.rw.offset/pg_block_size)*pg_block_size,
     };
-    pg_num_t pg_num = (oid.stripe/pool_cfg.pg_stripe_size) % pg_counts[pool_id] + 1; // like map_to_pg()
+    auto pg_count = pool_cfg.applied_pg_count;
+    if (!pg_count)
+    {
+        // Pool config is not loaded yet
+        finish_op(cur_op, -EPIPE);
+        return false;
+    }
+    pg_num_t pg_num = (oid.stripe/pool_cfg.applied_pg_stripe_size) % pg_count + 1; // like map_to_pg()
     auto pg_it = pgs.find({ .pool_id = pool_id, .pg_num = pg_num });
     if (pg_it == pgs.end() || pg_it->second.state == PG_OFFLINE)
     {

src/osd/osd_primary_sync.cpp

@@ -171,7 +171,7 @@ resume_6:
                     auto & w = op_data->unstable_writes[unstable_osd.start + i];
                     pool_pg_num_t wpg = {
                         .pool_id = INODE_POOL(w.oid.inode),
-                        .pg_num = map_to_pg(w.oid, st_cli.pool_config.at(INODE_POOL(w.oid.inode)).pg_stripe_size),
+                        .pg_num = map_to_pg(w.oid),
                     };
                     if (pgs.at(wpg).state & PG_ACTIVE)
                     {

src/osd/osd_scrub.cpp

@@ -9,6 +9,7 @@ void osd_t::scrub_list(pool_pg_num_t pg_id, osd_num_t role_osd, object_id min_oi
 {
     pool_id_t pool_id = pg_id.pool_id;
     pg_num_t pg_num = pg_id.pg_num;
+    auto & pool_cfg = st_cli.pool_config.at(pool_id);
     assert(!scrub_list_op);
     if (role_osd == this->osd_num)
     {
@@ -19,7 +20,7 @@ void osd_t::scrub_list(pool_pg_num_t pg_id, osd_num_t role_osd, object_id min_oi
         clock_gettime(CLOCK_REALTIME, &op->tv_begin);
         op->bs_op = new blockstore_op_t();
         op->bs_op->opcode = BS_OP_LIST;
-        op->bs_op->pg_alignment = st_cli.pool_config[pool_id].pg_stripe_size;
+        op->bs_op->pg_alignment = pool_cfg.applied_pg_stripe_size;
         if (min_oid.inode != 0 || min_oid.stripe != 0)
             op->bs_op->min_oid = min_oid;
         else
@@ -30,7 +31,7 @@ void osd_t::scrub_list(pool_pg_num_t pg_id, osd_num_t role_osd, object_id min_oi
         op->bs_op->max_oid.inode = ((uint64_t)(pool_id+1) << (64 - POOL_ID_BITS)) - 1;
         op->bs_op->max_oid.stripe = UINT64_MAX;
         op->bs_op->list_stable_limit = scrub_list_limit;
-        op->bs_op->pg_count = pg_counts[pool_id];
+        op->bs_op->pg_count = pool_cfg.applied_pg_count;
         op->bs_op->pg_number = pg_num-1;
         op->bs_op->callback = [this, op](blockstore_op_t *bs_op)
         {
@@ -68,8 +69,8 @@ void osd_t::scrub_list(pool_pg_num_t pg_id, osd_num_t role_osd, object_id min_oi
                     .opcode = OSD_OP_SEC_LIST,
                 },
                 .list_pg = pg_num,
-                .pg_count = pg_counts[pool_id],
-                .pg_stripe_size = st_cli.pool_config[pool_id].pg_stripe_size,
+                .pg_count = (uint32_t)pool_cfg.applied_pg_count,
+                .pg_stripe_size = pool_cfg.applied_pg_stripe_size,
                 .min_inode = min_oid.inode ? min_oid.inode : ((uint64_t)(pool_id) << (64 - POOL_ID_BITS)),
                 .max_inode = ((uint64_t)(pool_id+1) << (64 - POOL_ID_BITS)) - 1,
                 .min_stripe = min_oid.stripe,
@@ -249,7 +250,7 @@ void osd_t::submit_scrub_op(object_id oid)
             printf(
                 "Scrub failed with object %jx:%jx (PG %u/%u): error %jd\n",
                 oid.inode, oid.stripe, INODE_POOL(oid.inode),
-                map_to_pg(oid, st_cli.pool_config.at(INODE_POOL(oid.inode)).pg_stripe_size),
+                map_to_pg(oid),
                 osd_op->reply.hdr.retval
             );
         }

src/osd/osd_secondary.cpp

@@ -79,12 +79,8 @@ void osd_t::exec_secondary(osd_op_t *op)
     }
 }
 
-bool osd_t::sec_check_pg_lock(osd_num_t primary_osd, const object_id &oid)
+bool osd_t::sec_check_pg_lock(osd_num_t primary_osd, const object_id &oid, uint32_t flags)
 {
-    if (!enable_pg_locks)
-    {
-        return true;
-    }
     pool_id_t pool_id = INODE_POOL(oid.inode);
     auto pool_cfg_it = st_cli.pool_config.find(pool_id);
     if (pool_cfg_it == st_cli.pool_config.end())
@@ -92,11 +88,19 @@ bool osd_t::sec_check_pg_lock(osd_num_t primary_osd, const object_id &oid)
         return false;
     }
     auto & pool_cfg = pool_cfg_it->second;
+    if (pool_cfg.reshard_state)
+    {
+        return false;
+    }
+    if (!enable_pg_locks || (flags & OSD_OP_IGNORE_PG_LOCK))
+    {
+        return true;
+    }
     if (pg_locks_localize_only && (pool_cfg.scheme != POOL_SCHEME_REPLICATED || pool_cfg.local_reads == POOL_LOCAL_READ_PRIMARY))
     {
         return true;
     }
-    auto ppg = (pool_pg_num_t){ .pool_id = pool_id, .pg_num = map_to_pg(oid, pool_cfg_it->second.pg_stripe_size) };
+    auto ppg = (pool_pg_num_t){ .pool_id = pool_id, .pg_num = map_to_pg(oid) };
     auto pg_it = pgs.find(ppg);
     if (pg_it != pgs.end() && pg_it->second.state != PG_OFFLINE)
     {
@@ -140,8 +144,7 @@ void osd_t::exec_secondary_real(osd_op_t *cur_op)
         cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE ||
         cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE_STABLE)
     {
-        if (!(cur_op->req.sec_rw.flags & OSD_OP_IGNORE_PG_LOCK) &&
-            !sec_check_pg_lock(cl->in_osd_num, cur_op->req.sec_rw.oid))
+        if (!sec_check_pg_lock(cl->in_osd_num, cur_op->req.sec_rw.oid, cur_op->req.sec_rw.flags))
         {
             cur_op->bs_op->retval = -EPIPE;
             secondary_op_callback(cur_op);
@@ -169,8 +172,7 @@ void osd_t::exec_secondary_real(osd_op_t *cur_op)
     }
     else if (cur_op->req.hdr.opcode == OSD_OP_SEC_DELETE)
     {
-        if (!(cur_op->req.sec_del.flags & OSD_OP_IGNORE_PG_LOCK) &&
-            !sec_check_pg_lock(cl->in_osd_num, cur_op->req.sec_del.oid))
+        if (!sec_check_pg_lock(cl->in_osd_num, cur_op->req.sec_del.oid, cur_op->req.sec_del.flags))
         {
             cur_op->bs_op->retval = -EPIPE;
             secondary_op_callback(cur_op);
@@ -190,11 +192,11 @@ void osd_t::exec_secondary_real(osd_op_t *cur_op)
 #ifdef OSD_STUB
         cur_op->bs_op->retval = 0;
 #endif
-        if (enable_pg_locks && !(cur_op->req.sec_stab.flags & OSD_OP_IGNORE_PG_LOCK))
+        if (enable_pg_locks)
         {
             for (int i = 0; i < cur_op->bs_op->len; i++)
             {
-                if (!sec_check_pg_lock(cl->in_osd_num, ((obj_ver_id*)cur_op->buf)[i].oid))
+                if (!sec_check_pg_lock(cl->in_osd_num, ((obj_ver_id*)cur_op->buf)[i].oid, cur_op->req.sec_stab.flags))
                 {
                     cur_op->bs_op->retval = -EPIPE;
                     secondary_op_callback(cur_op);
@@ -213,6 +215,14 @@ void osd_t::exec_secondary_real(osd_op_t *cur_op)
             secondary_op_callback(cur_op);
             return;
         }
+        auto pool_id = INODE_POOL(cur_op->bs_op->min_oid.inode);
+        if (pool_id && !sec_check_pg_lock(0, (object_id){ .inode = cur_op->bs_op->min_oid.inode }, OSD_OP_IGNORE_PG_LOCK))
+        {
+            // Check resharding state of the pool
+            cur_op->bs_op->retval = -EPIPE;
+            secondary_op_callback(cur_op);
+            return;
+        }
         cur_op->bs_op->pg_alignment = cur_op->req.sec_list.pg_stripe_size;
         cur_op->bs_op->pg_count = cur_op->req.sec_list.pg_count;
         cur_op->bs_op->pg_number = cur_op->req.sec_list.list_pg - 1;
@@ -248,8 +258,7 @@ void osd_t::exec_sec_read_bmp(osd_op_t *cur_op)
         void *cur_buf = reply_buf;
         for (int i = 0; i < n; i++)
         {
-            if (!sec_check_pg_lock(cl->in_osd_num, ov[i].oid) &&
-                !(cur_op->req.sec_read_bmp.flags & OSD_OP_IGNORE_PG_LOCK))
+            if (!sec_check_pg_lock(cl->in_osd_num, ov[i].oid, cur_op->req.sec_read_bmp.flags))
             {
                 free(reply_buf);
                 cur_op->bs_op->retval = -EPIPE;

src/test/ringloop_mock.cpp

@@ -199,7 +199,7 @@ void disk_mock_t::erase_buffers(uint64_t begin, uint64_t end)
         {
             // Cut beginning & end & stop
             uint8_t *ce = (uint8_t*)malloc_or_die(be-end);
-            memcpy(ce, it->second.iov_base + (end-bs), be-end);
+            memcpy(ce, (uint8_t*)it->second.iov_base + (end-bs), be-end);
             uint8_t *cs = (uint8_t*)realloc(it->second.iov_base, begin-bs);
             if (!cs)
                 throw std::bad_alloc();
@@ -221,7 +221,7 @@ void disk_mock_t::erase_buffers(uint64_t begin, uint64_t end)
             // Cut end & stop
             assert(be > end);
             uint8_t *ce = (uint8_t*)malloc_or_die(be-end);
-            memcpy(ce, it->second.iov_base + (end-bs), be-end);
+            memcpy(ce, (uint8_t*)it->second.iov_base + (end-bs), be-end);
             buffers[be] = (iovec){ .iov_base = ce, .iov_len = be-end };
             buffers.erase(it);
             break;
@@ -308,7 +308,7 @@ void disk_mock_t::read_item(uint8_t *to, uint64_t offset, uint64_t len)
             last = offset;
         }
         uint64_t cur_end = be < offset+len ? be : offset+len;
-        memcpy(to+last-offset, it->second.iov_base+last-bs, cur_end-last);
+        memcpy(to+last-offset, (uint8_t*)it->second.iov_base+last-bs, cur_end-last);
         last = be;
     }
     if (last < offset+len)

src/test/test_heap.cpp

@@ -436,12 +436,13 @@ void test_compact(bool csum, bool stable)
 
     bitmap_set(ref_int_bitmap, 8192, 4096, 4096);
     {
-        uint32_t csums[dsk.data_block_size/(dsk.csum_block_size ? dsk.csum_block_size : 4096)] = {};
+        size_t csum_count = dsk.data_block_size/(dsk.csum_block_size ? dsk.csum_block_size : 4096);
+        std::vector<uint32_t> csums(csum_count);
         csums[0] = crc32c(0, buffer_area.data(), 4096);
         csums[2] = crc32c(0, buffer_area.data()+8192, 4096);
         res = heap.add_compact(obj, compact_info.compact_version, compact_info.compact_lsn,
             compact_info.clean_wr->big_location(&heap), compact_info.do_delete,
-            &mblock, ref_int_bitmap, ref_int_bitmap, (uint8_t*)csums);
+            &mblock, ref_int_bitmap, ref_int_bitmap, (uint8_t*)csums.data());
         assert(res == 0);
     }
     assert(mblock == 0);
@@ -461,10 +462,11 @@ void test_compact(bool csum, bool stable)
     if (csum)
     {
         assert(heap.calc_checksums(obj, buffer_area.data(), false));
-        uint32_t csums[dsk.data_block_size/(dsk.csum_block_size ? dsk.csum_block_size : 4096)] = {};
+        size_t csum_count = dsk.data_block_size/(dsk.csum_block_size ? dsk.csum_block_size : 4096);
+        std::vector<uint32_t> csums(csum_count);
         csums[0] = crc32c(0, buffer_area.data(), 4096);
         csums[2] = crc32c(0, buffer_area.data()+8192, 4096);
-        assert(!memcmp(obj->get_checksums(&heap), csums, dsk.data_block_size/dsk.csum_block_size*4));
+        assert(!memcmp(obj->get_checksums(&heap), csums.data(), dsk.data_block_size/dsk.csum_block_size*4));
     }
 
     obj = heap.read_entry({ .inode = INODE_WITH_POOL(1, 2), .stripe = 0 });
@@ -899,7 +901,8 @@ void test_reshard_list()
         free(listing);
         listing = NULL;
 
-        heap.reshard(1, 2, 0x20000);
+        void *st = heap.reshard_start(1, 2, 0x20000, 0);
+        assert(st == NULL);
 
         assert(heap.read_entry((object_id){ .inode = INODE_WITH_POOL(1, 1), .stripe = 0 }));
         assert(heap.read_entry((object_id){ .inode = INODE_WITH_POOL(1, 1), .stripe = 0x20000 }));

src/util/crc32c.c

@@ -37,6 +37,7 @@
 #include <stdlib.h>
 #include <stdint.h>
 #include <unistd.h>
+#include <assert.h>
 #include "crc32c.h"
 
 #ifdef WITH_ISAL
@@ -394,6 +395,7 @@ static uint8_t zero_page[4096] = {};
 
 uint32_t crc32c_pad(uint32_t prev_crc, const void *buf, size_t len, size_t left_pad, size_t right_pad)
 {
+    assert(left_pad < 0x10000000 && right_pad < 0x10000000);
     uint32_t r = prev_crc;
     while (left_pad >= 4096)
     {

tests/common.sh

@@ -65,7 +65,7 @@ start_etcd()
             --max-txn-ops=100000 --auto-compaction-retention=10 --auto-compaction-mode=revision &>./testdata/etcd$i.log &
         eval ETCD${i}_PID=$!
     else
-        node mon/mon-main.js $MON_PARAMS --antietcd_port $((ETCD_PORT+2*i-2)) --etcd_address $ETCD_URL --etcd_prefix "/vitastor" --verbose 1 >>./testdata/mon$i.log 2>&1 &
+        node mon/mon-main.js $MON_PARAMS --antietcd_port $((ETCD_PORT+2*i-2)) --verbose 1 >>./testdata/mon$i.log 2>&1 &
         eval ETCD${i}_PID=$!
     fi
 }
@@ -108,12 +108,20 @@ wait_condition()
     done
 }
 
+VITASTOR_CFG='"etcd_address":"'$ETCD_URL'"'
+echo "{$VITASTOR_CFG}" > ./testdata/vitastor.conf
+VITASTOR_CFG=./testdata/vitastor.conf
+VITASTOR_CLI="build/src/cmd/vitastor-cli --config_path $VITASTOR_CFG"
+# Preload build/src/client/libfio_vitastor.so so libasan detects all symbols
+VITASTOR_FIO="env LD_PRELOAD=build/src/client/libfio_vitastor.so fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -conf $VITASTOR_CFG"
+OSD_ARGS="$OSD_ARGS --config_path $VITASTOR_CFG"
+MON_PARAMS="$MON_PARAMS --config_path $VITASTOR_CFG"
+
 if [[ -n "$ANTIETCD" ]]; then
     ETCDCTL="node mon/node_modules/.bin/anticli -e $ETCD_URL"
     MON_PARAMS="--use_antietcd 1 --antietcd_data_dir ./testdata --antietcd_persist_interval 500 $MON_PARAMS"
 else
     ETCDCTL="${ETCD}ctl --endpoints=$ETCD_URL --dial-timeout=5s --command-timeout=10s"
-    MON_PARAMS="$MON_PARAMS"
     start_etcd_cluster
 fi
 

tests/run_3osds.sh

@@ -23,7 +23,7 @@ if [[ -n "$ANTIETCD" ]]; then
         start_etcd $i
     done
 else
-    node mon/mon-main.js $MON_PARAMS --etcd_address $ETCD_URL --etcd_prefix "/vitastor" --verbose 1 >>./testdata/mon.log 2>&1 &
+    node mon/mon-main.js $MON_PARAMS --verbose 1 >>./testdata/mon.log 2>&1 &
     MON_PID=$!
 fi
 wait_etcd
@@ -40,7 +40,7 @@ start_osd_on()
 {
     local i=$1
     local dev=$2
-    build/src/osd/vitastor-osd --osd_num $i --bind_address $ETCD_IP $NO_SAME $OSD_ARGS --etcd_address $ETCD_URL \
+    build/src/osd/vitastor-osd --osd_num $i --bind_address $ETCD_IP $NO_SAME $OSD_ARGS \
         $(build/src/disk_tool/vitastor-disk simple-offsets --format options $OFFSET_ARGS $dev $OFFSET_ARGS 2>/dev/null) \
         >>./testdata/osd$i.log 2>&1 &
     eval OSD${i}_PID=$!

tests/run_tests.sh

@@ -33,6 +33,8 @@ OLD=1 SCHEME=ec IMMEDIATE_COMMIT=1 ./test_interrupted_rebalance.sh
 
 ./test_failure_domain.sh
 
+./test_level_placement.sh
+
 ./test_snapshot.sh
 SCHEME=ec ./test_snapshot.sh
 OLD=1 ./test_snapshot.sh
@@ -136,6 +138,7 @@ OLD=1 SCHEME=xor ./test_scrub.sh
 OLD=1 PG_SIZE=3 ./test_scrub.sh
 OLD=1 PG_SIZE=6 PG_MINSIZE=4 OSD_COUNT=6 SCHEME=ec ./test_scrub.sh
 OLD=1 SCHEME=ec ./test_scrub.sh
+OLD=1 ./test_partwr_csum.sh
 
 TEST_NAME=old_csum_32k_dmj OLD=1 OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k --inmemory_metadata false --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS ./test_heal.sh
 TEST_NAME=old_csum_32k_dj  OLD=1 OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS ./test_heal.sh
@@ -145,3 +148,4 @@ TEST_NAME=old_csum_4k_dj   OLD=1 OSD_ARGS="--data_csum_type crc32c --inmemory_jo
 TEST_NAME=old_csum_4k      OLD=1 OSD_ARGS="--data_csum_type crc32c" OFFSET_ARGS=$OSD_ARGS ./test_heal.sh
 
 ./test_nfs.sh
+./test_nfs_unaligned_append.sh

tests/test_add_osd.sh

@@ -4,9 +4,8 @@ PG_COUNT=2048
 GLOBAL_CONFIG=',"osd_out_time":1'
 . `dirname $0`/run_3osds.sh
 
-LD_PRELOAD="build/src/client/libfio_vitastor.so" \
-    fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -end_fsync=1 \
-        -rw=write -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -cluster_log_level=10
+$VITASTOR_FIO -bs=4M -direct=1 -iodepth=1 -end_fsync=1 \
+    -rw=write -pool=1 -inode=1 -size=128M -cluster_log_level=10
 
 start_osd 4
 
@@ -30,7 +29,7 @@ wait_finish_rebalance 60
 sleep 1
 kill -9 $OSD4_PID
 sleep 1
-build/src/cmd/vitastor-cli --etcd_address $ETCD_URL rm-osd --force 4
+$VITASTOR_CLI rm-osd --force 4
 
 sleep 2
 

tests/test_cas.sh

@@ -2,6 +2,6 @@
 
 . `dirname $0`/run_3osds.sh
 
-build/src/test/test_cas --pool_id 1 --inode_id 1 --etcd_address $ETCD_URL
+build/src/test/test_cas --pool_id 1 --inode_id 1 --config_path $VITASTOR_CFG
 
 format_green OK

tests/test_change_pg_count.sh

@@ -1,5 +1,6 @@
 #!/bin/bash -ex
 
+GLOBAL_CONFIG=',"pg_reshard_chunk_size":100'
 OSD_COUNT=${OSD_COUNT:-6}
 PG_COUNT=16
 
@@ -7,9 +8,8 @@ PG_COUNT=16
 
 NOBJ=$(((128*8+PG_DATA_SIZE-1)/PG_DATA_SIZE))
 
-LD_PRELOAD="build/src/client/libfio_vitastor.so" \
-fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \
-    -etcd=$ETCD_URL -pool=1 -inode=2 -size=128M -cluster_log_level=10
+$VITASTOR_FIO -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \
+    -pool=1 -inode=2 -size=128M -cluster_log_level=10
 
 try_change()
 {

tests/test_change_pg_count_online.sh

@@ -6,13 +6,10 @@ GLOBAL_CONFIG=',"client_retry_interval":1000'
 
 . `dirname $0`/run_3osds.sh
 
-LD_PRELOAD="build/src/client/libfio_vitastor.so" \
-fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \
-    -etcd=$ETCD_URL -pool=1 -inode=2 -size=128M
+$VITASTOR_FIO -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write -pool=1 -inode=2 -size=128M
 
-LD_PRELOAD="build/src/client/libfio_vitastor.so" \
-fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4k -direct=1 -iodepth=4 -rw=randrw \
-    -etcd=$ETCD_URL -pool=1 -inode=2 -size=128M -loops=100 -cluster_log_level=3 -runtime=60 &>./testdata/fio.log &
+$VITASTOR_FIO -bs=4k -direct=1 -iodepth=4 -rw=randrw -pool=1 -inode=2 -size=128M -loops=100 \
+    -cluster_log_level=3 -runtime=60 &>./testdata/fio.log &
 FIO_PID=$!
 
 try_change()

tests/test_checksum.sh

@@ -11,9 +11,8 @@ IMG_SIZE=128
 $ETCDCTL put /vitastor/config/inode/1/1 '{"name":"testimg","size":'$((IMG_SIZE*1024*1024))'}'
 
 # Write
-LD_PRELOAD="build/src/client/libfio_vitastor.so" \
-    fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=1M -direct=1 -iodepth=4 \
-        -mirror_file=./testdata/bin/mirror.bin -end_fsync=1 -rw=write -etcd=$ETCD_URL -image=testimg -runtime=10
+$VITASTOR_FIO -bs=1M -direct=1 -iodepth=4 \
+    -mirror_file=./testdata/bin/mirror.bin -end_fsync=1 -rw=write -image=testimg -runtime=10
 
 # Intentionally corrupt OSD data and restart it
 kill $OSD1_PID
@@ -29,7 +28,7 @@ wait_up 10
 
 # Read everything back
 qemu-img convert -S 4096 -p \
-    -f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:image=testimg" \
+    -f raw "vitastor:config_path=$VITASTOR_CFG:image=testimg" \
     -O raw ./testdata/bin/read.bin
 
 diff ./testdata/bin/read.bin ./testdata/bin/mirror.bin

tests/test_create_halfhost.sh

@@ -2,7 +2,7 @@
 
 . `dirname $0`/common.sh
 
-node mon/mon-main.js $MON_PARAMS --etcd_address $ETCD_URL --etcd_prefix "/vitastor" >>./testdata/mon.log 2>&1 &
+node mon/mon-main.js $MON_PARAMS >>./testdata/mon.log 2>&1 &
 MON_PID=$!
 wait_etcd
 
@@ -28,8 +28,8 @@ $ETCDCTL put /vitastor/osd/stats/3 '{"host":"host2","size":1073741824,"time":"'$
 $ETCDCTL put /vitastor/osd/stats/4 '{"host":"host2","size":1073741824,"time":"'$TIME'"}'
 $ETCDCTL put /vitastor/osd/stats/5 '{"host":"host3","size":1073741824,"time":"'$TIME'"}'
 $ETCDCTL put /vitastor/osd/stats/6 '{"host":"host3","size":1073741824,"time":"'$TIME'"}'
-build/src/cmd/vitastor-cli --etcd_address $ETCD_URL osd-tree
+$VITASTOR_CLI osd-tree
 # check that it doesn't fail
-build/src/cmd/vitastor-cli --etcd_address $ETCD_URL create-pool testpool --ec 2+1 -n 32
+$VITASTOR_CLI create-pool testpool --ec 2+1 -n 32
 
 format_green OK

tests/test_create_nomaxid.sh

@@ -6,7 +6,7 @@
 
 $ETCDCTL put /vitastor/config/inode/1/120 '{"name":"testimg","size":'$((1024*1024*1024))'}'
 
-build/src/cmd/vitastor-cli create --etcd_address $ETCD_URL -s 1G testimg2
+$VITASTOR_CLI create -s 1G testimg2
 
 t=$($ETCDCTL get --print-value-only /vitastor/config/inode/1/121 | jq -r .name)
 if [[ "$t" != "testimg2" ]]; then

tests/test_dd.sh

@@ -4,17 +4,17 @@
 
 # pipe in - pipe out
 dd if=/dev/urandom of=./testdata/testfile bs=1M count=128
-build/src/cmd/vitastor-cli --etcd_address $ETCD_URL dd oimg=testimg iodepth=4 bs=1M count=128 < ./testdata/testfile
-build/src/cmd/vitastor-cli --etcd_address $ETCD_URL dd iimg=testimg iodepth=4 bs=1M count=128 > ./testdata/testfile1
+$VITASTOR_CLI dd oimg=testimg iodepth=4 bs=1M count=128 < ./testdata/testfile
+$VITASTOR_CLI dd iimg=testimg iodepth=4 bs=1M count=128 > ./testdata/testfile1
 diff ./testdata/testfile ./testdata/testfile1
 rm ./testdata/testfile1
 
 # snapshot
 dd if=/dev/urandom of=./testdata/over bs=1M count=4
 dd if=./testdata/over of=./testdata/testfile bs=1M seek=17 conv=notrunc
-build/src/cmd/vitastor-cli --etcd_address $ETCD_URL snap-create testimg@snap1
-build/src/cmd/vitastor-cli --etcd_address $ETCD_URL dd iodepth=4 if=./testdata/over oimg=testimg bs=1M seek=17
-build/src/cmd/vitastor-cli --etcd_address $ETCD_URL dd iodepth=4 iimg=testimg of=./testdata/testfile1
+$VITASTOR_CLI snap-create testimg@snap1
+$VITASTOR_CLI dd iodepth=4 if=./testdata/over oimg=testimg bs=1M seek=17
+$VITASTOR_CLI dd iodepth=4 iimg=testimg of=./testdata/testfile1
 diff ./testdata/testfile ./testdata/testfile1
 
 format_green OK

tests/test_degraded.sh

@@ -15,9 +15,8 @@ for i in $(seq 1 $OSD_COUNT); do
 done
 wait_up 120
 
-LD_PRELOAD="build/src/client/libfio_vitastor.so" \
-fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4k -direct=1 -iodepth=4 -rw=randwrite -loops=1000 \
-    -etcd=$ETCD_URL -pool=1 -inode=2 -size=256M -cluster_log_level=10 -runtime=5 &
+$VITASTOR_FIO -bs=4k -direct=1 -iodepth=4 -rw=randwrite -loops=1000 \
+    -pool=1 -inode=2 -size=256M -cluster_log_level=10 -runtime=5 &
 FIO_PID=$!
 
 sleep 15

tests/test_enospc.sh

@@ -5,23 +5,21 @@ GLOBAL_CONFIG=',"client_retry_enospc":false'
 
 . `dirname $0`/run_3osds.sh
 
-export LD_PRELOAD="build/src/client/libfio_vitastor.so"
-
 # Should fail with ENOSPC
-if fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=1M -direct=1 -iodepth=4 \
-    -rw=write -etcd=$ETCD_URL -pool=1 -inode=1 -size=500M -cluster_log_level=10; then
+if $VITASTOR_FIO -bs=1M -direct=1 -iodepth=4 \
+    -rw=write -pool=1 -inode=1 -size=500M -cluster_log_level=10; then
     format_error "Should get ENOSPC, but didn't"
 fi
 
 # Should fail with ENOSPC too (the idea is to try to overwrite first objects to check their rollback)
-if fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=1M -direct=1 -iodepth=32 \
-    -rw=write -etcd=$ETCD_URL -pool=1 -inode=1 -size=500M -cluster_log_level=10; then
+if $VITASTOR_FIO -bs=1M -direct=1 -iodepth=32 \
+    -rw=write -pool=1 -inode=1 -size=500M -cluster_log_level=10; then
     format_error "Should get ENOSPC, but didn't"
 fi
 
 # Should complete OK
-if ! fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4k -direct=1 -iodepth=4 \
-    -rw=randwrite -etcd=$ETCD_URL -pool=1 -inode=1 -size=100M -cluster_log_level=10 -number_ios=4096; then
+if ! $VITASTOR_FIO -bs=4k -direct=1 -iodepth=4 \
+    -rw=randwrite -pool=1 -inode=1 -size=100M -cluster_log_level=10 -number_ios=4096; then
     format_error "Should do random writes over ENOSPC correctly, but got an error"
 fi
 

tests/test_etcd_fail.sh

@@ -5,9 +5,8 @@ ETCD_COUNT=5
 
 . `dirname $0`/run_3osds.sh
 
-LD_PRELOAD="build/src/client/libfio_vitastor.so" \
-    fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=randwrite \
-        -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -cluster_log_level=10
+$VITASTOR_FIO -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=randwrite \
+    -pool=1 -inode=1 -size=128M -cluster_log_level=10
 
 kill_etcds()
 {
@@ -26,8 +25,7 @@ kill_etcds()
 
 kill_etcds &
 
-LD_PRELOAD="build/src/client/libfio_vitastor.so" \
-    fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4k -direct=1 -iodepth=1 -fsync=1 -rw=randwrite \
-        -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -cluster_log_level=10 -runtime=30
+$VITASTOR_FIO -bs=4k -direct=1 -iodepth=1 -fsync=1 -rw=randwrite \
+    -pool=1 -inode=1 -size=128M -cluster_log_level=10 -runtime=30
 
 format_green OK

tests/test_failure_domain.sh

@@ -2,7 +2,7 @@
 
 . `dirname $0`/common.sh
 
-node mon/mon-main.js $MON_PARAMS --etcd_address $ETCD_URL --etcd_prefix "/vitastor" >>./testdata/mon.log 2>&1 &
+node mon/mon-main.js $MON_PARAMS >>./testdata/mon.log 2>&1 &
 MON_PID=$!
 wait_etcd
 
@@ -17,13 +17,13 @@ $ETCDCTL put /vitastor/osd/stats/5 '{"host":"host3","size":1073741824,"time":"'$
 $ETCDCTL put /vitastor/osd/stats/6 '{"host":"host3","size":1073741824,"time":"'$TIME'"}'
 $ETCDCTL put /vitastor/osd/stats/7 '{"host":"host4","size":1073741824,"time":"'$TIME'"}'
 $ETCDCTL put /vitastor/osd/stats/8 '{"host":"host4","size":1073741824,"time":"'$TIME'"}'
-build/src/cmd/vitastor-cli --etcd_address $ETCD_URL create-pool testpool --ec 3+2 -n 32 --failure_domain rack --force
+$VITASTOR_CLI create-pool testpool --ec 3+2 -n 32 --failure_domain rack --force
 $ETCDCTL get --print-value-only /vitastor/config/pools | jq -s -e '. == [{"1": {"failure_domain": "rack", "name": "testpool", "parity_chunks": 2, "pg_count": 32, "pg_minsize": 4, "pg_size": 5, "scheme": "ec"}}]'
-build/src/cmd/vitastor-cli --etcd_address $ETCD_URL modify-pool testpool --ec 3+3 --failure_domain host
+$VITASTOR_CLI modify-pool testpool --ec 3+3 --failure_domain host
 $ETCDCTL get --print-value-only /vitastor/config/pools | jq -s -e '. == [{"1": {"failure_domain": "host", "name": "testpool", "parity_chunks": 3, "pg_count": 32, "pg_minsize": 4, "pg_size": 6, "scheme": "ec"}}]'
-build/src/cmd/vitastor-cli --etcd_address $ETCD_URL rm-pool testpool
+$VITASTOR_CLI rm-pool testpool
 $ETCDCTL get --print-value-only /vitastor/config/pools | jq -s -e '. == [{}]'
-build/src/cmd/vitastor-cli --etcd_address $ETCD_URL create-pool testpool -s 2 -n 4 --failure_domain rack --force
+$VITASTOR_CLI create-pool testpool -s 2 -n 4 --failure_domain rack --force
 $ETCDCTL get --print-value-only /vitastor/config/pools | jq -s -e '. == [{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":4,"failure_domain":"rack"}}]'
 
 sleep 2
@@ -32,9 +32,9 @@ $ETCDCTL get --prefix /vitastor/pg/config --print-value-only | \
     jq -s -e '([ .[0].items["1"] | .[].osd_set | map_values(. | tonumber) | select((.[0] <= 4) != (.[1] <= 4)) ] | length) == 4'
 
 # test pool with size 1
-build/src/cmd/vitastor-cli --etcd_address $ETCD_URL create-pool size1pool -s 1 -n 1 --force
+$VITASTOR_CLI create-pool size1pool -s 1 -n 1 --force
 wait_condition 10 "$ETCDCTL get --prefix /vitastor/pg/config --print-value-only | jq -s -e '.[0].items["'"'"2"'"'"]'"
 
-build/src/cmd/vitastor-cli --etcd_address $ETCD_URL modify-pool size1pool -s 2 --force
+$VITASTOR_CLI modify-pool size1pool -s 2 --force
 
 format_green OK

tests/test_heal.sh

@@ -18,9 +18,8 @@ IMG_SIZE=960
 
 $ETCDCTL put /vitastor/config/inode/1/1 '{"name":"testimg","size":'$((IMG_SIZE*1024*1024))'}'
 
-LD_PRELOAD="build/src/client/libfio_vitastor.so" \
-    fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \
-        -mirror_file=./testdata/bin/mirror.bin -etcd=$ETCD_URL -image=testimg -cluster_log_level=10
+$VITASTOR_FIO -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \
+    -mirror_file=./testdata/bin/mirror.bin -image=testimg -cluster_log_level=10
 
 kill_osds()
 {
@@ -51,12 +50,11 @@ kill_osds()
 
 kill_osds &
 
-LD_PRELOAD="build/src/client/libfio_vitastor.so" \
-    fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bsrange=4k-128k -blockalign=4k -direct=1 -iodepth=32 -fsync=256 -rw=randrw \
-        -serialize_overlap=1 -randrepeat=0 -refill_buffers=1 -mirror_file=./testdata/bin/mirror.bin -etcd=$ETCD_URL -image=testimg -loops=10 -runtime=120
+$VITASTOR_FIO -bsrange=4k-128k -blockalign=4k -direct=1 -iodepth=32 -fsync=256 -rw=randrw \
+    -serialize_overlap=1 -randrepeat=0 -refill_buffers=1 -mirror_file=./testdata/bin/mirror.bin -image=testimg -loops=10 -runtime=120
 
 qemu-img convert -S 4096 -p \
-    -f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:image=testimg" \
+    -f raw "vitastor:config_path=$VITASTOR_CFG:image=testimg" \
     -O raw ./testdata/bin/read.bin
 
 if ! diff -q ./testdata/bin/read.bin ./testdata/bin/mirror.bin; then

tests/test_interrupted_rebalance.sh

@@ -6,9 +6,8 @@ PG_COUNT=32
 
 IMG_SIZE=960
 
-LD_PRELOAD="build/src/client/libfio_vitastor.so" \
-fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4M -direct=1 -iodepth=16 -fsync=16 -rw=write \
-    -etcd=$ETCD_URL -pool=1 -inode=2 -size=${IMG_SIZE}M -cluster_log_level=10
+$VITASTOR_FIO -bs=4M -direct=1 -iodepth=16 -fsync=16 -rw=write \
+    -pool=1 -inode=2 -size=${IMG_SIZE}M -cluster_log_level=10
 
 try_reweight 1 0
 

tests/test_kv_stress.sh

@@ -3,6 +3,6 @@
 PG_COUNT=16
 . `dirname $0`/run_3osds.sh
 
-build/src/kv/vitastor-kv-stress --etcd_address $ETCD_URL --pool_id 1 --inode_id 1 --runtime 30
+build/src/kv/vitastor-kv-stress --config_path $VITASTOR_CFG --pool_id 1 --inode_id 1 --runtime 30
 
 format_green OK

tests/test_level_placement.sh

@@ -0,0 +1,39 @@
+#!/bin/bash -ex
+
+. `dirname $0`/common.sh
+
+node mon/mon-main.js $MON_PARAMS >>./testdata/mon.log 2>&1 &
+MON_PID=$!
+wait_etcd
+
+TIME=$(date '+%s')
+$ETCDCTL put /vitastor/config/global '{"placement_levels":{"rack":1,"host":2,"osd":3},"immediate_commit":"none"}'
+$ETCDCTL put /vitastor/config/node_placement '{"rack1":{"level":"rack"},"rack2":{"level":"rack"},"rack3":{"level":"rack"},"rack4":{"level":"rack"},
+    "host1":{"level":"host","parent":"rack1"},"host2":{"level":"host","parent":"rack1"},
+    "host3":{"level":"host","parent":"rack2"},"host4":{"level":"host","parent":"rack2"},
+    "host5":{"level":"host","parent":"rack3"},"host6":{"level":"host","parent":"rack3"},
+    "host7":{"level":"host","parent":"rack4"},"host8":{"level":"host","parent":"rack4"}}'
+$ETCDCTL put /vitastor/osd/stats/1 '{"host":"host1","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/2 '{"host":"host1","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/3 '{"host":"host2","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/4 '{"host":"host2","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/5 '{"host":"host3","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/6 '{"host":"host3","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/7 '{"host":"host4","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/8 '{"host":"host4","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/9 '{"host":"host5","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/10 '{"host":"host5","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/11 '{"host":"host6","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/12 '{"host":"host6","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/13 '{"host":"host7","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/14 '{"host":"host7","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/15 '{"host":"host8","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/16 '{"host":"host8","size":1073741824,"time":"'$TIME'"}'
+$VITASTOR_CLI create-pool testpool --failure_domain host --level_placement rack=112233 --ec 4+2 -n 32
+sleep 2
+$ETCDCTL get --prefix /vitastor/pg/config --print-value-only | \
+    jq -s -e '([ .[0].items["1"] | .[].osd_set | map_values(. | tonumber) | select(
+        (. | map_values((. - 1) / 2 | floor) | unique | length) == 6 and
+        (. | map_values((. - 1) / 4 | floor) | unique | length) == 3) ] | length) == 32'
+
+format_green OK

tests/test_move_reappear.sh

@@ -10,7 +10,7 @@ OSD_ARGS="$OSD_ARGS"
 for i in $(seq 1 $OSD_COUNT); do
     dd if=/dev/zero of=./testdata/bin/test_osd$i.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
     build/src/osd/vitastor-osd --log_level 10 --osd_num $i --bind_address 127.0.0.1 --etcd_stats_interval 5 $OSD_ARGS \
-        --etcd_address $ETCD_URL $(build/src/disk_tool/vitastor-disk simple-offsets --format options ./testdata/bin/test_osd$i.bin $OFFSET_ARGS 2>/dev/null) >>./testdata/osd$i.log 2>&1 &
+        $(build/src/disk_tool/vitastor-disk simple-offsets --format options ./testdata/bin/test_osd$i.bin $OFFSET_ARGS 2>/dev/null) >>./testdata/osd$i.log 2>&1 &
     eval OSD${i}_PID=$!
 done
 
@@ -27,9 +27,8 @@ for i in {1..30}; do
     fi
 done
 
-LD_PRELOAD="build/src/client/libfio_vitastor.so" \
-fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \
-    -etcd=$ETCD_URL -pool=1 -inode=2 -size=32M -cluster_log_level=10
+$VITASTOR_FIO -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \
+    -pool=1 -inode=2 -size=32M -cluster_log_level=10
 
 $ETCDCTL put /vitastor/pg/config '{"items":{"1":{"1":{"osd_set":[1,0],"primary":0}}}}'
 
@@ -57,9 +56,8 @@ for i in {1..30}; do
 done
 
 # Sync so all moved objects are removed from OSD 1 (they aren't removed without a sync)
-LD_PRELOAD="build/src/client/libfio_vitastor.so" \
-fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4k -direct=1 -iodepth=1 -fsync=1 -number_ios=2 -rw=write \
-    -etcd=$ETCD_URL -pool=1 -inode=2 -size=32M -cluster_log_level=10
+$VITASTOR_FIO -bs=4k -direct=1 -iodepth=1 -fsync=1 -number_ios=2 -rw=write \
+    -pool=1 -inode=2 -size=32M -cluster_log_level=10
 
 $ETCDCTL put /vitastor/pg/config '{"items":{"1":{"1":{"osd_set":[4,5],"primary":0}}}}'
 

tests/test_nfs.sh

@@ -5,9 +5,9 @@ IMMEDIATE_COMMIT=1
 PG_COUNT=16
 . `dirname $0`/run_3osds.sh
 
-build/src/cmd/vitastor-cli --etcd_address $ETCD_URL create -s 10G fsmeta
-build/src/cmd/vitastor-cli --etcd_address $ETCD_URL modify-pool --used-for-app fs:fsmeta testpool
-build/src/nfs/vitastor-nfs start --fs fsmeta --etcd_address $ETCD_URL --portmap 0 --port 2050 --foreground 1 --trace 1 >>./testdata/nfs.log 2>&1 &
+$VITASTOR_CLI create -s 10G fsmeta
+$VITASTOR_CLI modify-pool --used-for-app fs:fsmeta testpool
+build/src/nfs/vitastor-nfs --config_path $VITASTOR_CFG start --fs fsmeta --portmap 0 --port 2050 --foreground 1 --trace 1 >>./testdata/nfs.log 2>&1 &
 NFS_PID=$!
 
 mkdir -p testdata/nfs
@@ -175,22 +175,22 @@ format_green "rename over existing file ok"
 
 # check listing and removal of a bad direntry
 sudo umount ./testdata/nfs/
-build/src/kv/vitastor-kv --etcd_address $ETCD_URL fsmeta set d11/settings.jsonLGNmGn '{"ino": 123}'
+build/src/kv/vitastor-kv --config_path $VITASTOR_CFG fsmeta set d11/settings.jsonLGNmGn '{"ino": 123}'
 sudo mount localhost:/ ./testdata/nfs -o port=2050,mountport=2050,nfsvers=3,soft,nolock,tcp
 ls -l ./testdata/nfs
 ls -l ./testdata/nfs/settings.jsonLGNmGn
 sudo rm ./testdata/nfs/settings.jsonLGNmGn
-build/src/kv/vitastor-kv --etcd_address $ETCD_URL fsmeta get d11/settings.jsonLGNmGn 2>&1 | grep '(code -2)'
+build/src/kv/vitastor-kv --config_path $VITASTOR_CFG fsmeta get d11/settings.jsonLGNmGn 2>&1 | grep '(code -2)'
 ls -l ./testdata/nfs
 
 # repeat with ino=0
 sudo umount ./testdata/nfs/
-build/src/kv/vitastor-kv --etcd_address $ETCD_URL fsmeta set d11/settings.jsonLGNmGn '{"ino": 0}'
+build/src/kv/vitastor-kv --config_path $VITASTOR_CFG fsmeta set d11/settings.jsonLGNmGn '{"ino": 0}'
 sudo mount localhost:/ ./testdata/nfs -o port=2050,mountport=2050,nfsvers=3,soft,nolock,tcp
 ls -l ./testdata/nfs
 ls -l ./testdata/nfs/settings.jsonLGNmGn
 sudo rm ./testdata/nfs/settings.jsonLGNmGn
-build/src/kv/vitastor-kv --etcd_address $ETCD_URL fsmeta get d11/settings.jsonLGNmGn 2>&1 | grep '(code -2)'
+build/src/kv/vitastor-kv --config_path $VITASTOR_CFG fsmeta get d11/settings.jsonLGNmGn 2>&1 | grep '(code -2)'
 ls -l ./testdata/nfs
 
 format_green OK

tests/test_nfs_unaligned_append.sh

@@ -0,0 +1,34 @@
+#!/bin/bash -ex
+
+GLOBAL_CONFIG=',"client_enable_writeback":false'
+IMMEDIATE_COMMIT=1
+PG_COUNT=16
+. `dirname $0`/run_3osds.sh
+
+$VITASTOR_CLI create -s 10G fsmeta
+$VITASTOR_CLI modify-pool --used-for-app fs:fsmeta testpool
+build/src/nfs/vitastor-nfs --config_path $VITASTOR_CFG start --fs fsmeta --portmap 0 --port 2050 --foreground 1 --trace 1 >>./testdata/nfs.log 2>&1 &
+NFS_PID=$!
+
+mkdir -p testdata/nfs
+sudo mount localhost:/ ./testdata/nfs -o port=2050,mountport=2050,nfsvers=3,soft,nolock,tcp
+MNT=$(pwd)/testdata/nfs
+trap "sudo umount -f $MNT"' || true; kill -9 $(jobs -p)' EXIT
+
+# big file
+$VITASTOR_CLI dd if=/dev/urandom of=./testdata/ref_data.bin bs=1M count=32 seek=1024B
+$VITASTOR_CLI dd if=./testdata/ref_data.bin of=$MNT/testfile oflag=direct bs=1M iodepth=4 seek=1024B skip=1024B
+cp $MNT/testfile ./testdata/nfs_data.bin
+if ! diff -q ./testdata/ref_data.bin $MNT/testfile; then
+    format_error 'Data lost during parallel unaligned writes to VitastorFS'
+fi
+
+# small shared file
+$VITASTOR_CLI dd if=/dev/urandom of=./testdata/ref_small.bin bs=10 count=500 seek=15B
+$VITASTOR_CLI dd if=./testdata/ref_small.bin of=$MNT/smallfile oflag=direct bs=10 iodepth=4 seek=15B skip=15B
+cp $MNT/smallfile ./testdata/nfs_small.bin
+if ! diff -q ./testdata/ref_small.bin $MNT/smallfile; then
+    format_error 'Data lost during parallel unaligned writes to a small file in VitastorFS'
+fi
+
+format_green OK

tests/test_osd_tags.sh

@@ -2,7 +2,7 @@
 
 . `dirname $0`/common.sh
 
-node mon/mon-main.js $MON_PARAMS --etcd_address $ETCD_URL --etcd_prefix "/vitastor" >>./testdata/mon.log 2>&1 &
+node mon/mon-main.js $MON_PARAMS >>./testdata/mon.log 2>&1 &
 MON_PID=$!
 wait_etcd
 

tests/test_parity_change.sh

@@ -39,9 +39,7 @@ try_change()
     fi
 }
 
-LD_PRELOAD="build/src/client/libfio_vitastor.so" \
-    fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=1M -direct=1 -iodepth=4 \
-        -rw=write -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -runtime=10
+$VITASTOR_FIO -bs=1M -direct=1 -iodepth=4 -rw=write -pool=1 -inode=1 -size=128M -runtime=10
 
 PG_SIZE=7
 POOLCFG='"name":"testpool","failure_domain":"osd","scheme":"ec","parity_chunks":'$((PG_SIZE-PG_DATA_SIZE))

tests/test_partwr_csum.sh

@@ -0,0 +1,18 @@
+#!/bin/bash -ex
+
+OSD_ARGS="--data_csum_type crc32c"
+OFFSET_ARGS=$OSD_ARGS
+GLOBAL_CONFIG=',"client_eio_retry_interval":0'
+. `dirname $0`/run_3osds.sh
+
+$VITASTOR_CLI create -s 32M testimg
+
+$VITASTOR_FIO -bs=4k -blockalign=128k -direct=1 -iodepth=16 -end_fsync=1 -rw=write -image=testimg
+
+$VITASTOR_CLI dd iimg=testimg of=/dev/null
+
+if grep -q 'Checksum mismatch' ./testdata/osd*.log; then
+    format_error Checksum mismatches detected during test
+fi
+
+format_green OK

tests/test_rebalance_verify.sh

@@ -10,7 +10,7 @@ IMG_SIZE=256
 
 $ETCDCTL put /vitastor/config/inode/1/1 '{"name":"testimg","size":'$((IMG_SIZE*1024*1024))'}'
 
-NBD_DEV=$(sudo build/src/client/vitastor-nbd map --nbd_timeout 180 --etcd_address $ETCD_URL --image testimg --logfile ./testdata/nbd.log &)
+NBD_DEV=$(sudo build/src/client/vitastor-nbd map --nbd_timeout 180 --config_path $VITASTOR_CFG --image testimg --logfile ./testdata/nbd.log &)
 
 trap "sudo build/src/client/vitastor-nbd unmap $NBD_DEV"' || true; kill -9 $(jobs -p)' EXIT
 

tests/test_resize.sh

@@ -5,16 +5,14 @@ PG_COUNT=${PG_COUNT:-32}
 . `dirname $0`/run_3osds.sh
 check_qemu
 
-LD_PRELOAD="build/src/client/libfio_vitastor.so" \
-    fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4M -direct=1 -iodepth=4 \
-        -rw=write -etcd=$ETCD_URL -end_fsync=1 -pool=1 -inode=1 -size=256M -runtime=10
+$VITASTOR_FIO -bs=4M -direct=1 -iodepth=4 \
+    -rw=write -end_fsync=1 -pool=1 -inode=1 -size=256M -runtime=10
 
-LD_PRELOAD="build/src/client/libfio_vitastor.so" \
-    fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4k -direct=1 -iodepth=32 \
-        -rw=randwrite -etcd=$ETCD_URL -end_fsync=1 -pool=1 -inode=1 -size=256M -runtime=10 -number_ios=1024
+$VITASTOR_FIO -bs=4k -direct=1 -iodepth=32 \
+    -rw=randwrite -end_fsync=1 -pool=1 -inode=1 -size=256M -runtime=10 -number_ios=1024
 
 qemu-img convert -S 4096 -p \
-    -f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:pool=1:inode=1:size=$((256*1024*1024))" \
+    -f raw "vitastor:config_path=$VITASTOR_CFG:pool=1:inode=1:size=$((256*1024*1024))" \
     -O raw ./testdata/bin/before.bin
 
 for i in $(seq 1 $OSD_COUNT); do
@@ -72,7 +70,7 @@ done
 $ETCDCTL del --prefix /vitastor/osd/state/
 
 for i in $(seq 1 $OSD_COUNT); do
-    build/src/osd/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $NO_SAME $OSD_ARGS --etcd_address $ETCD_URL \
+    build/src/osd/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $NO_SAME $OSD_ARGS \
         --meta_format $meta_format \
         --data_device ./testdata/bin/test_osd$i.bin \
         --meta_offset 0 \
@@ -82,7 +80,7 @@ for i in $(seq 1 $OSD_COUNT); do
 done
 
 qemu-img convert -S 4096 -p \
-    -f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:pool=1:inode=1:size=$((256*1024*1024))" \
+    -f raw "vitastor:config_path=$VITASTOR_CFG:pool=1:inode=1:size=$((256*1024*1024))" \
     -O raw ./testdata/bin/after.bin
 
 if ! cmp ./testdata/bin/before.bin ./testdata/bin/after.bin; then

tests/test_reweight_half.sh

@@ -2,7 +2,7 @@
 
 . `dirname $0`/common.sh
 
-node mon/mon-main.js $MON_PARAMS --etcd_address $ETCD_URL --etcd_prefix "/vitastor" >>./testdata/mon.log 2>&1 &
+node mon/mon-main.js $MON_PARAMS >>./testdata/mon.log 2>&1 &
 MON_PID=$!
 wait_etcd
 
@@ -11,7 +11,7 @@ $ETCDCTL put /vitastor/osd/stats/1 '{"host":"host1","size":1073741824,"time":"'$
 $ETCDCTL put /vitastor/osd/stats/2 '{"host":"host1","size":1073741824,"time":"'$TIME'"}'
 $ETCDCTL put /vitastor/osd/stats/3 '{"host":"host2","size":1073741824,"time":"'$TIME'"}'
 $ETCDCTL put /vitastor/osd/stats/4 '{"host":"host2","size":1073741824,"time":"'$TIME'"}'
-build/src/cmd/vitastor-cli --etcd_address $ETCD_URL create-pool testpool -s 2 -n 16 --force
+$VITASTOR_CLI create-pool testpool -s 2 -n 16 --force
 
 sleep 2
 
@@ -25,7 +25,7 @@ $ETCDCTL get /vitastor/pg/config --print-value-only | \
 $ETCDCTL get /vitastor/pg/config --print-value-only | \
     jq -s -e '([ .[0].items["1"] | .[].osd_set | map_values(. | tonumber) | select(.[0] == 4 or .[1] == 4) ] | length) == 8'
 
-build/src/cmd/vitastor-cli --etcd_address $ETCD_URL modify-osd --reweight 0.5 3
+$VITASTOR_CLI modify-osd --reweight 0.5 3
 
 sleep 2
 

tests/test_rm.sh

@@ -3,12 +3,11 @@
 PG_COUNT=16
 . `dirname $0`/run_3osds.sh
 
-LD_PRELOAD="build/src/client/libfio_vitastor.so" \
-    fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 \
-        -end_fsync=1 -fsync=1 -rw=write -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -cluster_log_level=10
+$VITASTOR_FIO -bs=4M -direct=1 -iodepth=1 \
+    -end_fsync=1 -fsync=1 -rw=write -pool=1 -inode=1 -size=128M -cluster_log_level=10
 
 $ETCDCTL get --prefix '/vitastor/pg/state'
 
-build/src/cmd/vitastor-cli rm-data --etcd_address $ETCD_URL --pool 1 --inode 1
+$VITASTOR_CLI rm-data --pool 1 --inode 1
 
 format_green OK

tests/test_rm_degraded.sh

@@ -5,20 +5,19 @@ PG_COUNT=16
 PG_MINSIZE=2
 . `dirname $0`/run_3osds.sh
 
-build/src/cmd/vitastor-cli --etcd_address $ETCD_URL create -s 128M testimg
+$VITASTOR_CLI create -s 128M testimg
 
-LD_PRELOAD="build/src/client/libfio_vitastor.so" \
-    fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 \
-        -end_fsync=1 -fsync=1 -rw=write -etcd=$ETCD_URL -image=testimg -size=128M -cluster_log_level=10
+$VITASTOR_FIO -bs=4M -direct=1 -iodepth=1 \
+    -end_fsync=1 -fsync=1 -rw=write -image=testimg -size=128M -cluster_log_level=10
 
 kill -9 $OSD3_PID
 $ETCDCTL del /vitastor/osd/state/3
 
-if build/src/cmd/vitastor-cli --etcd_address $ETCD_URL rm testimg --log_level 10 ; then
+if $VITASTOR_CLI rm testimg --log_level 10 ; then
     format_error "Delete should not be successful with inactive OSDs"
 fi
 
-if ! ( build/src/cmd/vitastor-cli --etcd_address $ETCD_URL ls | grep testimg | grep DEL ) ; then
+if ! ( $VITASTOR_CLI ls | grep testimg | grep DEL ) ; then
     format_error "Image should be marked as partially deleted"
 fi
 
@@ -27,17 +26,16 @@ sleep 5
 
 # Now do the same but without del /vitastor/osd/state
 
-LD_PRELOAD="build/src/client/libfio_vitastor.so" \
-    fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 \
-        -end_fsync=1 -fsync=1 -rw=write -etcd=$ETCD_URL -image=testimg -size=128M -cluster_log_level=10
+$VITASTOR_FIO -bs=4M -direct=1 -iodepth=1 \
+    -end_fsync=1 -fsync=1 -rw=write -image=testimg -size=128M -cluster_log_level=10
 
 kill -9 $OSD3_PID
 
-if build/src/cmd/vitastor-cli --etcd_address $ETCD_URL rm testimg --log_level 10 ; then
+if $VITASTOR_CLI rm testimg --log_level 10 ; then
     format_error "Delete should not be successful with inactive OSDs"
 fi
 
-if ! ( build/src/cmd/vitastor-cli --etcd_address $ETCD_URL ls | grep testimg | grep DEL ) ; then
+if ! ( $VITASTOR_CLI ls | grep testimg | grep DEL ) ; then
     format_error "Image should be marked as partially deleted"
 fi