Release 3.0.1

Important fixes:

- Disable RWF_ATOMIC by default because Linux incorrectly requires all atomic
  writes to be power-of-2-sized and length-aligned. Details: [use_atomic_flag](https://vitastor.io/en/docs/config/osd.html#use_atomic_flag)
- Fix cross-pool snapshots not working at all - always reading old data after taking the snapshot
- Fix level_placement (broken in 2.2.0)
- Fix CAS write return values in the client library (broken in 2.4.4), also breaking unaligned writes in VitastorFS
- Fix VitastorFS possibly losing some of intersecting parallel unaligned writes
- Prevent possible reads of the old data during unfinished intent writes in the new store
- Tests added for all of above problems to prevent future regressions

Other fixes:

- Allow to specify OSD tags and weights during prepare
- Only clear the first block instead of whole OSD metadata and journal areas during vitastor-disk
  prepare - OSD anyway clears them on the first run
- Fix vitastor-cli dd non-seekable detection and error status
- Do not stop OSD on zero-copy tcp short send errors - these aren't a bug, but just caused
  by TCP client disconnections
- Fix modify-osd not working after deleting the OSD configuration key from etcd
- Fix regular antietcd websocket disconnections in OSD due to lack of WS_PONG responses to WS_PING
- Reduce the number of allocated RDMA memory regions
- Allow routed RoCE

.gitea/workflows/test.yml

@@ -414,6 +414,24 @@ jobs:
           echo ""
         done
 
+  test_level_placement:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_level_placement.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_snapshot:
     runs-on: ubuntu-latest
     needs: build
@@ -1926,3 +1944,21 @@ jobs:
           echo ""
         done
 
+  test_nfs_unaligned_append:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_nfs_unaligned_append.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+

CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "3.0.0")
+set(VITASTOR_VERSION "3.0.1")
 
 include(CTest)
 

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.0
+VITASTOR_VERSION ?= v3.0.1
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v3.0.0
+          image: vitalif/vitastor-csi:v3.0.1
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v3.0.0
+          image: vitalif/vitastor-csi:v3.0.1
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "3.0.0"
+    vitastorCSIDriverVersion = "3.0.1"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (3.0.0-1) unstable; urgency=medium
+vitastor (3.0.1-1) unstable; urgency=medium
 
   * Bugfixes
 

docker/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.0
+VITASTOR_VERSION ?= v3.0.1
 
 all: build push
 

docker/etc/vitastor/docker.conf

@@ -4,7 +4,7 @@
 #
 
 # Desired Vitastor version
-VITASTOR_VERSION=v3.0.0
+VITASTOR_VERSION=v3.0.1
 
 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here

docs/config/osd.en.md

@@ -682,7 +682,10 @@ with replicated pools and reach the best possible write performance.
 
 Default value is auto-detected during OSD initialization from
 `/sys/block/xx/queue/atomic_write_max_bytes` or assumed to be 4096 bytes
-because all known disks support 4 KB atomic writes.
+because all known disks support 4 KB atomic writes. Auto-detection is only used for
+NVMe disks because SAS disks require the explicit WRITE ATOMIC command which requires
+RWF_ATOMIC (see below [#use_atomic_flag]) but that flag works incorrectly in current
+Linux versions.
 
 You can also check if your NVMe drives support atomic writes by running
 the command `nvme id-ctrl /dev/nvme0n1 | grep awupf`. If the reported value,
@@ -697,12 +700,16 @@ reducing Write Amplification and improving write performance up to 2 times.
 
 - Type: boolean
 
-This option controls whether the Vitastor OSD uses RWF_ATOMIC write flag with atomic
-writes. This flag is only supported on Linux kernel since 6.11. Atomic writes are
-generally only safe to use with this flag because it tells the kernel to never fragment
-write requests and also to check the write against the actual atomic write capabilities
-of the device.
+This option controls whether Vitastor OSDs use RWF_ATOMIC write flag with atomic writes.
+This flag is supported since Linux 6.11 and adds some safety to atomic writes - the kernel
+guarantees to not fragment write requests with it and also to check them against the actual
+device atomic write capabilities.
 
-This option is enabled by default when atomic_write_size is set to a value larger than 4 KB.
-You can disable it if you're sure that your disks support atomic writes and you want to
-bypass the Linux atomic write checks.
+However, the option is disabled by default because the flag is currently UNUSABLE - Linux
+incorrectly requires writes with that flag to be of power-of-2 length and length-aligned.
+I.e., for example, 12 KB writes and not-8-KB aligned 8 KB writes are forbidden by the kernel,
+even though the NVMe specification allows them.
+
+For NVMe disks with `scheduler=none` writes aren't fragmented anyway so it's not a big deal.
+However, you can rebuild your kernel with [this patch](../../patches/linux-fix-atomic-write-checks.diff)
+and turn this option on. It will make your atomic writes a bit safer.

docs/config/osd.ru.md

@@ -717,6 +717,9 @@ pg_minsize OSD во время переключений, что может по
 Значение по умолчанию авто-определяется во время инициализации OSD из
 `/sys/block/xx/queue/atomic_write_max_bytes` либо принимается равным 4096,
 так как все известные диски поддерживают атомарную запись 4 КБ блоков.
+Автоопределение применяется только для NVMe-дисков, так как SAS диски требуют
+использования отдельной команды WRITE ATOMIC, а для неё нужен флаг RWF_ATOMIC
+(см. ниже [#use_atomic_flag]), а он в текущих версиях Linux работает некорректно.
 
 Вы также можете проверить, поддерживают ли ваши NVMe-диски атомарную запись,
 с помощью команды `nvme id-ctrl /dev/nvme0n1 | grep awupf`. Если значение awupf
@@ -735,11 +738,16 @@ pg_minsize OSD во время переключений, что может по
 - Тип: булево (да/нет)
 
 Данная опция контролирует использование Vitastor OSD флага RWF_ATOMIC при атомарной записи
-блоков. Этот флаг поддерживается только в ядрах Linux начиная с 6.11. Атомарная запись
-является безопасной только при использовании этого флага, так как он сообщает ядру о том,
-что запрос записи нельзя фрагментировать и о том, что запрос нужно проверить на соответствие
-реальным возможностям атомарной записи устройства.
+блоков. Этот флаг поддерживается, начиная с версии ядра Linux 6.11 и добавляет немного корректности
+атомарным записям - ядро гарантирует отсутствие фрагментации запросов записи с этим флагом и
+проверяет их на соответствие реальным возможностям устройства.
 
-Опция включается по умолчанию, когда atomic_write_size устанавливается в значение больше 4 КБ.
-Вы можете явно отключить её, если уверены, что ваши диски поддерживают атомарную запись и
-хотите обойти проверки уровня ядра.
+Однако, данная опция по умолчанию отключена, так как флаг в текущих версиях Linux работает
+абсолютно НЕКОРРЕКТНО - при нём Linux требует, чтобы запросы записи имели длину, равную
+степени двойки и были выровнены на эту длину. То есть, например, 12 КБ запросы записи, а также
+8 КБ запросы записи по не-кратному 8 КБ смещению запрещаются ядром, хотя спецификация NVMe их
+разрешает.
+
+Для NVMe-дисков с `scheduler=none` запросы записи и так не фрагментируются, так что это не так
+уж и важно, однако вы можете пересобрать своё ядро с [этим патчем](../../patches/linux-fix-atomic-write-checks.diff)
+и включить данную опцию. Это сделает вашу атомарную запись капельку безопаснее.

docs/config/src/osd.yml

@@ -813,7 +813,10 @@
 
     Default value is auto-detected during OSD initialization from
     `/sys/block/xx/queue/atomic_write_max_bytes` or assumed to be 4096 bytes
-    because all known disks support 4 KB atomic writes.
+    because all known disks support 4 KB atomic writes. Auto-detection is only used for
+    NVMe disks because SAS disks require the explicit WRITE ATOMIC command which requires
+    RWF_ATOMIC (see below [#use_atomic_flag]) but that flag works incorrectly in current
+    Linux versions.
 
     You can also check if your NVMe drives support atomic writes by running
     the command `nvme id-ctrl /dev/nvme0n1 | grep awupf`. If the reported value,
@@ -834,6 +837,9 @@
     Значение по умолчанию авто-определяется во время инициализации OSD из
     `/sys/block/xx/queue/atomic_write_max_bytes` либо принимается равным 4096,
     так как все известные диски поддерживают атомарную запись 4 КБ блоков.
+    Автоопределение применяется только для NVMe-дисков, так как SAS диски требуют
+    использования отдельной команды WRITE ATOMIC, а для неё нужен флаг RWF_ATOMIC
+    (см. ниже [#use_atomic_flag]), а он в текущих версиях Linux работает некорректно.
 
     Вы также можете проверить, поддерживают ли ваши NVMe-диски атомарную запись,
     с помощью команды `nvme id-ctrl /dev/nvme0n1 | grep awupf`. Если значение awupf
@@ -849,22 +855,31 @@
 - name: use_atomic_flag
   type: bool
   info: |
-    This option controls whether the Vitastor OSD uses RWF_ATOMIC write flag with atomic
-    writes. This flag is only supported on Linux kernel since 6.11. Atomic writes are
-    generally only safe to use with this flag because it tells the kernel to never fragment
-    write requests and also to check the write against the actual atomic write capabilities
-    of the device.
+    This option controls whether Vitastor OSDs use RWF_ATOMIC write flag with atomic writes.
+    This flag is supported since Linux 6.11 and adds some safety to atomic writes - the kernel
+    guarantees to not fragment write requests with it and also to check them against the actual
+    device atomic write capabilities.
 
-    This option is enabled by default when atomic_write_size is set to a value larger than 4 KB.
-    You can disable it if you're sure that your disks support atomic writes and you want to
-    bypass the Linux atomic write checks.
+    However, the option is disabled by default because the flag is currently UNUSABLE - Linux
+    incorrectly requires writes with that flag to be of power-of-2 length and length-aligned.
+    I.e., for example, 12 KB writes and not-8-KB aligned 8 KB writes are forbidden by the kernel,
+    even though the NVMe specification allows them.
+
+    For NVMe disks with `scheduler=none` writes aren't fragmented anyway so it's not a big deal.
+    However, you can rebuild your kernel with [this patch](../../patches/linux-fix-atomic-write-checks.diff)
+    and turn this option on. It will make your atomic writes a bit safer.
   info_ru: |
     Данная опция контролирует использование Vitastor OSD флага RWF_ATOMIC при атомарной записи
-    блоков. Этот флаг поддерживается только в ядрах Linux начиная с 6.11. Атомарная запись
-    является безопасной только при использовании этого флага, так как он сообщает ядру о том,
-    что запрос записи нельзя фрагментировать и о том, что запрос нужно проверить на соответствие
-    реальным возможностям атомарной записи устройства.
+    блоков. Этот флаг поддерживается, начиная с версии ядра Linux 6.11 и добавляет немного корректности
+    атомарным записям - ядро гарантирует отсутствие фрагментации запросов записи с этим флагом и
+    проверяет их на соответствие реальным возможностям устройства.
 
-    Опция включается по умолчанию, когда atomic_write_size устанавливается в значение больше 4 КБ.
-    Вы можете явно отключить её, если уверены, что ваши диски поддерживают атомарную запись и
-    хотите обойти проверки уровня ядра.
+    Однако, данная опция по умолчанию отключена, так как флаг в текущих версиях Linux работает
+    абсолютно НЕКОРРЕКТНО - при нём Linux требует, чтобы запросы записи имели длину, равную
+    степени двойки и были выровнены на эту длину. То есть, например, 12 КБ запросы записи, а также
+    8 КБ запросы записи по не-кратному 8 КБ смещению запрещаются ядром, хотя спецификация NVMe их
+    разрешает.
+
+    Для NVMe-дисков с `scheduler=none` запросы записи и так не фрагментируются, так что это не так
+    уж и важно, однако вы можете пересобрать своё ядро с [этим патчем](../../patches/linux-fix-atomic-write-checks.diff)
+    и включить данную опцию. Это сделает вашу атомарную запись капельку безопаснее.

docs/installation/docker.en.md

@@ -26,9 +26,9 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.
 
 1. Download a Docker image of the desired version: \
-   `docker pull vitalif/vitastor:v3.0.0`
+   `docker pull vitalif/vitastor:v3.0.1`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.0 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.1 install.sh`
 3. Reload udev rules: \
    `udevadm control --reload-rules`
 

docs/installation/docker.ru.md

@@ -25,9 +25,9 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.
 
 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitalif/vitastor:v3.0.0`
+   `docker pull vitalif/vitastor:v3.0.1`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.0 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.1 install.sh`
 3. Перезагрузите правила udev: \
    `udevadm control --reload-rules`
 

docs/installation/proxmox.en.md

@@ -6,7 +6,7 @@
 
 # Proxmox VE
 
-To enable Vitastor support in Proxmox Virtual Environment (6.4-8.x are supported):
+To enable Vitastor support in Proxmox Virtual Environment (6.4-9.x are supported):
 
 - Add the corresponding Vitastor Debian repository into sources.list on Proxmox hosts:
   trixie for 9.0+, bookworm for 8.1+, pve8.0 for 8.0, bullseye for 7.4, pve7.3 for 7.3, pve7.2 for 7.2, pve7.1 for 7.1, buster for 6.4

docs/installation/proxmox.ru.md

@@ -6,7 +6,7 @@
 
 # Proxmox VE
 
-Чтобы подключить Vitastor к Proxmox Virtual Environment (поддерживаются версии 6.4-8.x):
+Чтобы подключить Vitastor к Proxmox Virtual Environment (поддерживаются версии 6.4-9.x):
 
 - Добавьте соответствующий Debian-репозиторий Vitastor в sources.list на хостах Proxmox:
   trixie для 9.0+, bookworm для 8.1+, pve8.0 для 8.0, bullseye для 7.4, pve7.3 для 7.3, pve7.2 для 7.2, pve7.1 для 7.1, buster для 6.4

docs/usage/disk.en.md

@@ -95,6 +95,8 @@ Options (single-device mode):
 Options (both modes):
 
 ```
+--tags tag1,tag2           Set new OSD tag(s)
+--weight <number>          Set new OSD weight (between 0 to 1)
 --journal_size 1G/32M      Set journal size (area or partition size)
 --block_size 1M/128k       Set blockstore object size
 --bitmap_granularity 4k    Set bitmap granularity

docs/usage/disk.ru.md

@@ -96,6 +96,8 @@ vitastor-disk - инструмент командной строки для уп
 Опции для обоих режимов:
 
 ```
+--tags tag1,tag2           Задать теги для новых OSD
+--weight <number>          Задать вес для новых OSD (от 0 до 1)
 --journal_size 1G/32M      Задать размер журнала (области или раздела журнала)
 --block_size 1M/128k       Задать размер объекта хранилища
 --bitmap_granularity 4k    Задать гранулярность битовых карт

mon/osd_tree.js

@@ -87,7 +87,7 @@ function make_hier_tree(global_config, tree)
     tree[''] = { children: [] };
     for (const node_id in tree)
     {
-        if (node_id === '' || !(tree[node_id].children||[]).length && (tree[node_id].size||0) <= 0)
+        if (node_id === '')
         {
             continue;
         }

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "3.0.0",
+  "version": "3.0.1",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {

mon/pg_gen.js

@@ -178,7 +178,7 @@ async function generate_pool_pgs(state, global_config, pool_id, osd_tree, levels
     const rules = use_rules ? get_pg_rules(pool_id, pool_cfg, global_config.placement_levels) : null;
     const folded = fold_failure_domains(Object.values(pool_tree), use_rules ? rules : [ [ [ pool_cfg.failure_domain ] ] ]);
     // FIXME: Remove/merge make_hier_tree() step somewhere, however it's needed to remove empty nodes
-    const folded_tree = make_hier_tree(global_config, folded.nodes);
+    const folded_tree = make_hier_tree(global_config, folded.nodes.reduce((a, c) => { a[c.id] = c; return a; }, {}));
     const old_pg_count = prev_pgs.length;
     const optimize_cfg = {
         osd_weights: folded.nodes.reduce((a, c) => { if (Number(c.id)) { a[c.id] = c.size; } return a; }, {}),

node-binding/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor",
-  "version": "3.0.0",
+  "version": "3.0.1",
   "description": "Low-level native bindings to Vitastor client library",
   "main": "index.js",
   "keywords": [

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '3.0.0'
+VITASTOR_VERSION = '3.0.1'
 
 LOG = logging.getLogger(__name__)
 

patches/linux-fix-atomic-write-checks.diff

@@ -0,0 +1,39 @@
+From 98d3f68a40130c438854f61db6025f9e9b099cb6 Mon Sep 17 00:00:00 2001
+From: Vitaliy Filippov <vitalifster@gmail.com>
+Date: Sat, 20 Dec 2025 14:44:35 +0300
+Subject: [PATCH] Do not require atomic writes to be power of 2 sized and
+ aligned on length boundary
+
+It contradicts NVMe specification where alignment is only required when atomic
+write boundary (NABSPF/NABO) is set and highly limits usage of NVMe atomic writes
+
+Signed-off-by: Vitaliy Filippov <vitalifster@gmail.com>
+---
+ fs/read_write.c | 8 --------
+ 1 file changed, 8 deletions(-)
+
+diff --git a/fs/read_write.c b/fs/read_write.c
+index 833bae068770..5467d710108d 100644
+--- a/fs/read_write.c
++++ b/fs/read_write.c
+@@ -1802,17 +1802,9 @@ int generic_file_rw_checks(struct file *file_in, struct file *file_out)
+ 
+ int generic_atomic_write_valid(struct kiocb *iocb, struct iov_iter *iter)
+ {
+-	size_t len = iov_iter_count(iter);
+-
+ 	if (!iter_is_ubuf(iter))
+ 		return -EINVAL;
+ 
+-	if (!is_power_of_2(len))
+-		return -EINVAL;
+-
+-	if (!IS_ALIGNED(iocb->ki_pos, len))
+-		return -EINVAL;
+-
+ 	if (!(iocb->ki_flags & IOCB_DIRECT))
+ 		return -EOPNOTSUPP;
+ 
+-- 
+2.51.0
+

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.0
+Version:        3.0.1
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.0.el7.tar.gz
+Source0:        vitastor-3.0.1.el7.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  devtoolset-9-gcc-c++

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.0
+Version:        3.0.1
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.0.el8.tar.gz
+Source0:        vitastor-3.0.1.el8.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-toolset-9-gcc-c++

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.0
+Version:        3.0.1
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.0.el9.tar.gz
+Source0:        vitastor-3.0.1.el9.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++

src/CMakeLists.txt

@@ -21,7 +21,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 endif()
 set(ENABLE_COVERAGE false CACHE BOOL "Enable code coverage")
 
-add_definitions(-DVITASTOR_VERSION="3.0.0")
+add_definitions(-DVITASTOR_VERSION="3.0.1")
 add_definitions(-D_GNU_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -fvisibility=hidden -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/blockstore/blockstore_heap.cpp

@@ -2190,6 +2190,15 @@ void blockstore_heap_t::apply_inflight(heap_inflight_lsn_t & inflight)
     }
 }
 
+bool blockstore_heap_t::is_lsn_completed(uint64_t lsn)
+{
+    if (lsn <= completed_lsn)
+        return true;
+    assert(lsn-first_inflight_lsn < inflight_lsn.size());
+    auto it = inflight_lsn.begin() + (lsn-first_inflight_lsn);
+    return (it->flags & HEAP_INFLIGHT_DONE);
+}
+
 uint64_t blockstore_heap_t::get_completed_lsn()
 {
     return completed_lsn;

src/blockstore/blockstore_heap.h

@@ -314,6 +314,7 @@ public:
     void start_block_write(uint32_t block_num);
     void complete_block_write(uint32_t block_num);
     void complete_lsn_write(uint64_t lsn);
+    bool is_lsn_completed(uint64_t lsn);
     uint64_t get_completed_lsn();
     uint64_t get_fsynced_lsn();
     void mark_lsn_fsynced(uint64_t lsn);

src/blockstore/blockstore_read.cpp

@@ -22,7 +22,7 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
     uint64_t result_version = 0;
     bool found = false;
     uint32_t skip_csum = 0;
-    uint32_t blk_start = 0, blk_end = 0;
+    uint32_t blk_start = op->offset, blk_end = op->offset+op->len;
     bool need_skip = dsk.csum_block_size > dsk.bitmap_granularity && !perfect_csum_update;
     if (need_skip)
     {
@@ -32,12 +32,28 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
         if (blk_end % dsk.csum_block_size)
             blk_end += dsk.csum_block_size - (blk_end % dsk.csum_block_size);
     }
+    bool need_wait = false;
     heap->iterate_with_stable(obj, obj->lsn, [&](heap_entry_t *wr, bool stable)
     {
         if (wr->type() == BS_HEAP_DELETE)
         {
             return false;
         }
+        if (!heap->is_lsn_completed(wr->lsn))
+        {
+            if (wr->type() == BS_HEAP_BIG_INTENT && wr->big_intent().offset < blk_end && wr->big_intent().offset+wr->big_intent().len > blk_start ||
+                wr->type() == BS_HEAP_INTENT_WRITE && wr->small().offset < blk_end && wr->small().offset+wr->small().len > blk_start)
+            {
+                // Wait until intent write is completed
+                need_wait = true;
+                return false;
+            }
+            else if (wr->type() == BS_HEAP_SMALL_WRITE && wr->small().offset < blk_end && wr->small().offset+wr->small().len > blk_start)
+            {
+                // Skip entry and read the previous one
+                return true;
+            }
+        }
         if (op->version >= wr->version && !found)
         {
             found = true;
@@ -47,12 +63,6 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
                 memcpy(op->bitmap, wr->get_ext_bitmap(heap), dsk.clean_entry_bitmap_size);
             }
         }
-        if (need_skip && wr->lsn < heap->get_completed_lsn() &&
-            (wr->type() == BS_HEAP_BIG_INTENT && wr->big_intent().offset < blk_end && wr->big_intent().offset+wr->big_intent().len > blk_start ||
-            wr->type() == BS_HEAP_INTENT_WRITE && wr->small().offset < blk_end && wr->small().offset+wr->small().len > blk_start))
-        {
-            skip_csum = COPY_BUF_SKIP_CSUM;
-        }
         if (op->version >= wr->version)
         {
             fulfilled += prepare_read(PRIV(op)->read_vec, obj, wr, op->offset, op->offset+op->len,
@@ -65,13 +75,23 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
                 return false;
             }
         }
-        if (need_skip && (wr->type() == BS_HEAP_SMALL_WRITE || wr->type() == BS_HEAP_INTENT_WRITE) &&
+        if (need_skip && wr->type() == BS_HEAP_SMALL_WRITE &&
             wr->small().offset < blk_end && wr->small().offset+wr->small().len > blk_start)
         {
+            // Small write may mutate big write checksums during flush
             skip_csum = COPY_BUF_SKIP_CSUM;
         }
         return true;
     });
+    if (need_wait)
+    {
+undo_wait:
+        // Need to wait. undo added requests, unlock lsn
+        heap->unlock_entry(op->oid);
+        free_read_buffers(rv);
+        rv.clear();
+        return 0;
+    }
     if (!found)
     {
         // May happen if there are entries but all of them are > requested version
@@ -84,11 +104,7 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
     assert(fulfilled == op->len);
     if (!fulfill_read(op))
     {
-        // Need to wait. undo added requests, unlock lsn
-        heap->unlock_entry(op->oid);
-        free_read_buffers(rv);
-        rv.clear();
-        return 0;
+        goto undo_wait;
     }
     op->version = result_version;
     if (!PRIV(op)->pending_ops)

src/client/cluster_client.cpp

@@ -868,7 +868,8 @@ void cluster_client_t::execute_cas(cluster_op_t *op)
     {
         int expected = part->req.hdr.opcode == OSD_OP_DELETE ? 0 : part->req.rw.len;
         op->retval = part->reply.hdr.retval;
-        op->retval = op->retval == expected ? 0 : (op->retval >= 0 ? -EIO : op->retval);
+        if (op->retval != expected && op->retval >= 0)
+            op->retval = -EIO;
         op->retval = op->retval == -EPIPE ? -EINTR : op->retval;
         auto peer_it = msgr.osd_peer_fds.find(op->parts[0].osd_num);
         if (op->retval != 0 || (op->flags & OP_IMMEDIATE_COMMIT))
@@ -899,8 +900,11 @@ void cluster_client_t::execute_cas(cluster_op_t *op)
                 },
                 .callback = [this, op](osd_op_t *part)
                 {
-                    op->retval = part->reply.hdr.retval;
-                    op->retval = op->retval == -EPIPE ? -EINTR : op->retval;
+                    if (part->reply.hdr.retval != 0)
+                    {
+                        op->retval = part->reply.hdr.retval;
+                        op->retval = op->retval == -EPIPE ? -EINTR : op->retval;
+                    }
                     auto cb = std::move(op->callback);
                     cb(op);
                 },
@@ -1253,9 +1257,9 @@ void cluster_client_t::slice_rw(cluster_op_t *op)
         {
             op->bitmap_buf = realloc_or_die(op->bitmap_buf, bitmap_mem);
             op->part_bitmaps = (uint8_t*)op->bitmap_buf + object_bitmap_size;
+            memset(op->bitmap_buf+op->bitmap_buf_size, 0, bitmap_mem-op->bitmap_buf_size);
             op->bitmap_buf_size = bitmap_mem;
         }
-        memset(op->bitmap_buf, 0, bitmap_mem);
     }
     int iov_idx = 0;
     size_t iov_pos = 0;
@@ -1401,7 +1405,7 @@ int cluster_client_t::try_send(cluster_op_t *op, int i, std::function<void(osd_o
         if (peer_it != msgr.osd_peer_fds.end())
         {
             int peer_fd = peer_it->second;
-            part->flags |= PART_SENT;
+            part->flags |= PART_SENT|PART_VALID;
             op->inflight_count++;
             uint64_t pg_bitmap_size = (pool_cfg.data_block_size / pool_cfg.bitmap_granularity / 8) * (
                 pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks
@@ -1614,14 +1618,11 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
             dirty_osds.insert(part->osd_num);
         part->flags |= PART_DONE;
         op->done_count++;
-        if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
+        if ((op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
+            && op->inode == op->cur_inode)
         {
-            copy_part_bitmap(op, part);
-            if (op->inode == op->cur_inode)
-            {
-                // Read only returns the version of the uppermost layer
-                op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0;
-            }
+            // Read only returns the version of the uppermost layer
+            op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0;
         }
         else if (op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE)
         {
@@ -1629,6 +1630,13 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
         }
         if (op->inflight_count == 0 && !op->retry_after)
         {
+            // Copy part bitmaps only after finishing all part reads
+            if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
+            {
+                for (auto & part: op->parts)
+                    if (part.flags == (PART_SENT|PART_VALID|PART_DONE))
+                        copy_part_bitmap(op, &part);
+            }
             if (op->opcode == OSD_OP_SYNC)
                 continue_sync(op);
             else

src/client/cluster_client_impl.h

@@ -10,6 +10,7 @@
 #define PART_DONE 2
 #define PART_ERROR 4
 #define PART_RETRY 8
+#define PART_VALID 16
 #define CACHE_DIRTY 1
 #define CACHE_WRITTEN 2
 #define CACHE_FLUSHING 3

src/client/http_client.cpp

@@ -23,7 +23,7 @@
 #define READ_BUFFER_SIZE 9000
 
 static std::string ws_format_frame(int type, uint64_t size);
-static bool ws_parse_frame(std::string & buf, int & type, std::string & res);
+static bool ws_parse_frame(std::string & buf, uint8_t & type, std::string & res);
 static void parse_http_headers(std::string & res, http_response_t *parsed);
 
 struct http_co_t
@@ -70,7 +70,7 @@ struct http_co_t
     void submit_read(bool check_timeout);
     void submit_send();
     bool handle_read();
-    void post_message(int type, const std::string & msg);
+    void post_message(uint8_t type, const std::string & msg);
     void send_request(const std::string & host, const std::string & request,
         const http_options_t & options, std::function<void(const http_response_t *response)> response_callback);
 };
@@ -199,12 +199,12 @@ void http_co_t::send_request(const std::string & host, const std::string & reque
     stackout();
 }
 
-void http_post_message(http_co_t *handler, int type, const std::string & msg)
+void http_post_message(http_co_t *handler, uint8_t type, const std::string & msg)
 {
     handler->post_message(type, msg);
 }
 
-void http_co_t::post_message(int type, const std::string & msg)
+void http_co_t::post_message(uint8_t type, const std::string & msg)
 {
     stackin();
     if (state == HTTP_CO_WEBSOCKET)
@@ -608,7 +608,13 @@ bool http_co_t::handle_read()
     {
         while (ws_parse_frame(response, parsed.ws_msg_type, parsed.body))
         {
-            response_callback(&parsed);
+            if (parsed.ws_msg_type == WS_PING)
+            {
+                // Reply with WS_PONG
+                post_message(WS_PONG, "");
+            }
+            else
+                response_callback(&parsed);
             parsed.body = "";
         }
     }
@@ -698,7 +704,7 @@ static std::string ws_format_frame(int type, uint64_t size)
     return res;
 }
 
-static bool ws_parse_frame(std::string & buf, int & type, std::string & res)
+static bool ws_parse_frame(std::string & buf, uint8_t & type, std::string & res)
 {
     uint64_t hdr = 2;
     if (buf.size() < hdr)

src/client/http_client.h

@@ -32,7 +32,7 @@ struct http_response_t
     int status_code = 0;
     std::string status_line;
     std::map<std::string, std::string> headers;
-    int ws_msg_type = -1;
+    uint8_t ws_msg_type = -1;
     std::string body;
 
     void parse_json_response(std::string & error, json11::Json & r) const;
@@ -46,5 +46,5 @@ http_co_t* open_websocket(timerfd_manager_t *tfd, const std::string & host, cons
     int timeout, std::function<void(const http_response_t *msg)> on_message);
 void http_request(http_co_t *handler, const std::string & host, const std::string & request,
     const http_options_t & options, std::function<void(const http_response_t *response)> response_callback);
-void http_post_message(http_co_t *handler, int type, const std::string & msg);
+void http_post_message(http_co_t *handler, uint8_t type, const std::string & msg);
 void http_close(http_co_t *co);

src/client/messenger.cpp

@@ -749,7 +749,7 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl)
                     }
                 });
                 // Add the initial receive request
-                try_recv_rdma(cl);
+                init_recv_rdma(cl);
             }
         }
 #endif

src/client/messenger.h

@@ -289,7 +289,7 @@ protected:
     void try_send_rdma(osd_client_t *cl);
     void try_send_rdma_odp(osd_client_t *cl);
     void try_send_rdma_nodp(osd_client_t *cl);
-    bool try_recv_rdma(osd_client_t *cl);
+    bool init_recv_rdma(osd_client_t *cl);
     void handle_rdma_events(msgr_rdma_context_t *rdma_context);
     msgr_rdma_context_t* choose_rdma_context(osd_client_t *cl);
 #endif

src/client/msgr_rdma.cpp

@@ -67,6 +67,20 @@ msgr_rdma_context_t::~msgr_rdma_context_t()
         ibv_close_device(context);
 }
 
+msgr_rdma_buf_t::~msgr_rdma_buf_t()
+{
+    if (buf)
+    {
+        free(buf);
+        buf = NULL;
+    }
+    if (mr)
+    {
+        ibv_dereg_mr(mr);
+        mr = NULL;
+    }
+}
+
 msgr_rdma_connection_t::~msgr_rdma_connection_t()
 {
     ctx->reserve_cqe(-max_send-max_recv);
@@ -84,26 +98,6 @@ msgr_rdma_connection_t::~msgr_rdma_connection_t()
     if (qp)
         ibv_destroy_qp(qp);
 #endif
-    if (recv_buffers.size())
-    {
-        for (auto b: recv_buffers)
-        {
-            if (b.mr)
-                ibv_dereg_mr(b.mr);
-            free(b.buf);
-        }
-        recv_buffers.clear();
-    }
-    if (send_out.mr)
-    {
-        ibv_dereg_mr(send_out.mr);
-        send_out.mr = NULL;
-    }
-    if (send_out.buf)
-    {
-        free(send_out.buf);
-        send_out.buf = NULL;
-    }
     send_out_size = 0;
 }
 
@@ -493,12 +487,12 @@ int msgr_rdma_connection_t::connect(msgr_rdma_address_t *dest)
             .grh        = {
                 .dgid = dest->gid,
                 .sgid_index = conn->ctx->gid_index,
-                .hop_limit = 1, // FIXME can it vary?
+                .hop_limit = 64, // FIXME can it vary?
             },
             .dlid       = dest->lid,
             .sl         = 0, // service level
             .src_path_bits = 0,
-            .is_global  = (uint8_t)(dest->gid.global.interface_id ? 1 : 0),
+            .is_global  = (uint8_t)(dest->gid.global.subnet_prefix || dest->gid.global.interface_id ? 1 : 0),
             .port_num   = conn->ctx->ib_port,
         },
         .max_rd_atomic  = 1,
@@ -663,7 +657,7 @@ void osd_messenger_t::try_send_rdma_nodp(osd_client_t *cl)
     {
         // Allocate send ring buffer, if not yet
         rc->send_out_size = rc->max_msg*rdma_max_send;
-        rc->send_out.buf = malloc_or_die(rc->send_out_size);
+        rc->send_out.buf = (uint8_t*)malloc_or_die(rc->send_out_size);
         if (!rc->ctx->odp)
         {
             rc->send_out.mr = ibv_reg_mr(rc->ctx->pd, rc->send_out.buf, rc->send_out_size, 0);
@@ -712,12 +706,12 @@ void osd_messenger_t::try_send_rdma(osd_client_t *cl)
         try_send_rdma_nodp(cl);
 }
 
-static void try_recv_rdma_wr(osd_client_t *cl, msgr_rdma_buf_t b)
+static void try_recv_rdma_wr(osd_client_t *cl, void *buf)
 {
     ibv_sge sge = {
-        .addr = (uintptr_t)b.buf,
+        .addr = (uintptr_t)buf,
         .length = (uint32_t)cl->rdma_conn->max_msg,
-        .lkey = cl->rdma_conn->ctx->odp ? cl->rdma_conn->ctx->mr->lkey : b.mr->lkey,
+        .lkey = cl->rdma_conn->ctx->odp ? cl->rdma_conn->ctx->mr->lkey : cl->rdma_conn->recv_buf.mr->lkey,
     };
     ibv_recv_wr *bad_wr = NULL;
     ibv_recv_wr wr = {
@@ -734,22 +728,23 @@ static void try_recv_rdma_wr(osd_client_t *cl, msgr_rdma_buf_t b)
     cl->rdma_conn->cur_recv++;
 }
 
-bool osd_messenger_t::try_recv_rdma(osd_client_t *cl)
+bool osd_messenger_t::init_recv_rdma(osd_client_t *cl)
 {
     auto rc = cl->rdma_conn;
-    while (rc->cur_recv < rc->max_recv)
+    assert(!rc->recv_buf.buf);
+    rc->recv_buf.buf = (uint8_t*)malloc_or_die(rc->max_msg * rc->max_recv);
+    if (!rc->ctx->odp)
     {
-        msgr_rdma_buf_t b;
-        b.buf = malloc_or_die(rc->max_msg);
-        if (!rc->ctx->odp)
+        rc->recv_buf.mr = ibv_reg_mr(rc->ctx->pd, rc->recv_buf.buf, rc->max_msg * rc->max_recv, IBV_ACCESS_LOCAL_WRITE);
+        if (!rc->recv_buf.mr)
         {
-            b.mr = ibv_reg_mr(rc->ctx->pd, b.buf, rc->max_msg, IBV_ACCESS_LOCAL_WRITE);
-            if (!b.mr)
-            {
-                fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
-                exit(1);
-            }
+            fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
+            exit(1);
         }
+    }
+    for (uint32_t i = 0; i < rc->max_recv; i++)
+    {
+        uint8_t *b = rc->recv_buf.buf + i*rc->max_msg;
         rc->recv_buffers.push_back(b);
         try_recv_rdma_wr(cl, b);
     }
@@ -807,7 +802,7 @@ void osd_messenger_t::handle_rdma_events(msgr_rdma_context_t *rdma_context)
                 cl->ping_time_remaining = 0;
                 cl->idle_time_remaining = osd_idle_timeout;
                 rc->cur_recv--;
-                if (!handle_read_buffer(cl, rc->recv_buffers[rc->next_recv_buf].buf, wc[i].byte_len))
+                if (!handle_read_buffer(cl, rc->recv_buffers[rc->next_recv_buf], wc[i].byte_len))
                 {
                     // handle_read_buffer may stop the client
                     clear_immediate_ops(client_id);

src/client/msgr_rdma.h

@@ -54,8 +54,10 @@ struct msgr_rdma_context_t
 
 struct msgr_rdma_buf_t
 {
-    void *buf = NULL;
+    uint8_t *buf = NULL;
     ibv_mr *mr = NULL;
+
+    ~msgr_rdma_buf_t();
 };
 
 struct msgr_rdma_connection_t
@@ -72,7 +74,8 @@ struct msgr_rdma_connection_t
     int cur_send = 0, cur_recv = 0;
     int send_pos = 0, send_buf_pos = 0;
     int next_recv_buf = 0;
-    std::vector<msgr_rdma_buf_t> recv_buffers;
+    std::vector<void*> recv_buffers;
+    msgr_rdma_buf_t recv_buf;
     std::vector<uint64_t> send_sizes;
     msgr_rdma_buf_t send_out;
     int send_out_pos = 0, send_done_pos = 0, send_out_size = 0;

src/client/msgr_rdmacm.cpp

@@ -531,7 +531,7 @@ void osd_messenger_t::rdmacm_established(rdma_cm_event *ev)
         fprintf(stderr, "Successfully connected with OSD %ju using RDMA-CM\n", peer_osd);
     }
     // Add initial receive request(s)
-    try_recv_rdma(cl);
+    init_recv_rdma(cl);
     if (peer_osd)
     {
         check_peer_config(cl);

src/client/msgr_send.cpp

@@ -326,8 +326,10 @@ void osd_messenger_t::handle_send(int result, bool prev, bool more, osd_client_t
             int expected = cl->send_list.size() < IOV_MAX ? cl->send_list.size() : IOV_MAX;
             if (done != expected)
             {
-                fprintf(stderr, "BUG (maybe kernel): Expected to send %d iovecs with MSG_WAITALL but sent %d\n", expected, done);
-                exit(1);
+                fprintf(stderr, "Client %d socket write error: expected to send "
+                    "%d iovecs with MSG_WAITALL but sent %d. Disconnecting client\n", cl->peer_fd, expected, done);
+                stop_client(cl->peer_fd);
+                return;
             }
             cl->zc_free_list.push_back(NULL); // end marker
         }
@@ -363,7 +365,7 @@ void osd_messenger_t::handle_send(int result, bool prev, bool more, osd_client_t
                 }
             });
             // Add the initial receive request
-            try_recv_rdma(cl);
+            init_recv_rdma(cl);
         }
 #endif
     }

src/client/msgr_stop.cpp

@@ -158,6 +158,13 @@ osd_client_t::~osd_client_t()
     }
     // Cancel outbound ops
     cancel_ops();
+    for (osd_op_t *op: zc_free_list)
+    {
+        if (op)
+        {
+            delete op;
+        }
+    }
 #ifndef __MOCK__
 #ifdef WITH_RDMA
     if (rdma_conn)

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 3.0.0
+Version: 3.0.1
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/cmd/cli_dd.cpp

@@ -111,7 +111,7 @@ struct dd_in_info_t
             {
                 in_granularity = 512;
             }
-            if (lseek(ifd, 1, SEEK_SET) == (off_t)-1)
+            if (lseek(ifd, 1, SEEK_SET) != (off_t)1)
             {
                 in_seekable = false;
             }
@@ -389,7 +389,7 @@ struct cli_dd_t
     int state = 0;
     int copy_error = 0;
     int in_waiting = 0, out_waiting = 0;
-    cli_result_t result;
+    cli_result_t result = {};
 
     bool is_done()
     {
@@ -931,7 +931,8 @@ close_end:
         oinfo.close_output(parent);
         iinfo.close_input(parent);
         // Done
-        result.err = copy_error;
+        if (copy_error)
+            result.err = copy_error;
         state = 100;
     }
 };

src/cmd/cli_modify_osd.cpp

@@ -120,12 +120,23 @@ resume_1:
                 else
                     osd_cfg.erase("noout");
             }
-            compare.push_back(json11::Json::object {
-                { "target", "MOD" },
-                { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/osd/"+std::to_string(osd_num)) },
-                { "result", "LESS" },
-                { "mod_revision", osd_cfg_mod_rev+1 },
-            });
+            if (osd_cfg_mod_rev)
+            {
+                compare.push_back(json11::Json::object {
+                    { "target", "MOD" },
+                    { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/osd/"+std::to_string(osd_num)) },
+                    { "result", "LESS" },
+                    { "mod_revision", osd_cfg_mod_rev+1 },
+                });
+            }
+            else
+            {
+                compare.push_back(json11::Json::object {
+                    { "target", "VERSION" },
+                    { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/osd/"+std::to_string(osd_num)) },
+                    { "version", 0 },
+                });
+            }
             if (!osd_cfg.size())
             {
                 success.push_back(json11::Json::object {

src/disk_tool/disk_tool.cpp

@@ -65,6 +65,8 @@ static const char *help_text =
     "    --force                    Bypass partition safety checks (for emptiness and so on)\n"
     "  \n"
     "  Options (both modes):\n"
+    "    --tags tag1,tag2           Set new OSD tag(s)\n"
+    "    --weight <number>          Set new OSD weight (between 0 and 1)\n"
     "    --journal_size 32M/1G      Set journal size (area or partition size)\n"
     "    --block_size 128k/1M       Set blockstore object size\n"
     "    --bitmap_granularity 4k    Set bitmap granularity\n"

src/disk_tool/disk_tool.h

@@ -88,6 +88,7 @@ struct disk_tool_t
     uint32_t new_meta_format = 0;
     int new_journal_fd = -1, new_meta_fd = -1;
     resizer_data_moving_t *moving_blocks = NULL;
+    bool atomic_warned = false;
 
     bool started = false;
     void *small_write_data = NULL;

src/disk_tool/disk_tool_prepare.cpp

@@ -78,25 +78,37 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
             if (check_existing_partition(dev) != 0)
                 return 1;
         }
+        if (options.find("weight") != options.end())
+        {
+            double reweight = json11::Json(options["weight"]).number_value();
+            if (reweight < 0 || reweight > 1)
+            {
+                fprintf(stderr, "OSD weight must be between 0 and 1\n");
+                return 1;
+            }
+        }
     }
     if (options.find("atomic_write_size") == options.end())
     {
         auto data_dev = realpath_str(options["data_device"], false);
-        uint64_t atomic_write_size = get_atomic_write_size(data_dev);
-        if (atomic_write_size > 4096)
+        if (data_dev.substr(0, 9) == "/dev/nvme")
         {
-            fprintf(stderr, "Data device %s supports atomic writes up to %ju bytes, enabling. Enjoy faster writes!\n",
-                data_dev.c_str(), atomic_write_size);
-            options["atomic_write_size"] = std::to_string(atomic_write_size);
-            options["use_atomic_flag"] = "1";
+            uint64_t atomic_write_size = get_atomic_write_size(data_dev);
+            if (atomic_write_size > 4096)
+            {
+                // FIXME: Enable use_atomic_flag when the kernel does checks correctly
+                fprintf(stderr, "Data device %s supports atomic writes up to %ju bytes, enabling. Enjoy faster writes!\n",
+                    data_dev.c_str(), atomic_write_size);
+                options["atomic_write_size"] = std::to_string(atomic_write_size);
+                if (!atomic_warned)
+                {
+                    fprintf(stderr, "WARNING: RWF_ATOMIC can't be used because Linux checks atomic writes incorrectly.\n"
+                        " Please don't change scheduler from default 'none' and check use_atomic_flag documentation for more details.\n");
+                    atomic_warned = true;
+                }
+            }
         }
     }
-    else if (options.find("use_atomic_flag") == options.end() &&
-        parse_size(options["atomic_write_size"]) > 4096)
-    {
-        fprintf(stderr, "Atomic writes larger than 4 KB are enabled manually, enabling use_atomic_flag too.\n");
-        options["use_atomic_flag"] = "1";
-    }
     for (auto dev: std::vector<std::string>{"data", "meta", "journal"})
     {
         if (options[dev+"_device"] != "" && options["disable_"+dev+"_fsync"] == "auto")
@@ -238,9 +250,28 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
         return 1;
     }
     sb["osd_num"] = osd_num;
-    // Zero out metadata and journal
-    if (write_zero(dsk.meta_fd, sb["meta_offset"].uint64_value(), dsk.meta_area_size) != 0 ||
-        write_zero(dsk.journal_fd, sb["journal_offset"].uint64_value(), dsk.journal_len) != 0)
+    if (options.find("weight") != options.end() || options.find("tags") != options.end())
+    {
+        std::vector<std::string> cmd = { "vitastor-cli", "modify-osd", std::to_string(osd_num) };
+        if (options.find("weight") != options.end())
+        {
+            cmd.push_back("--reweight");
+            cmd.push_back(options["weight"]);
+        }
+        if (options.find("tags") != options.end())
+        {
+            cmd.push_back("--tags");
+            cmd.push_back(options["tags"]);
+        }
+        if (shell_exec(cmd, "", NULL, NULL) != 0)
+        {
+            fprintf(stderr, "Failed to modify OSD %ju tags and/or reweight\n", osd_num);
+            return 1;
+        }
+    }
+    // Zero out the first block of metadata and journal - OSD will zero the rest on the first run
+    if (write_zero(dsk.meta_fd, sb["meta_offset"].uint64_value(), 4096) != 0 ||
+        write_zero(dsk.journal_fd, sb["journal_offset"].uint64_value(), 4096) != 0)
     {
         fprintf(stderr, "Failed to zero out metadata or journal: %s\n", strerror(errno));
         dsk.close_all();

src/nfs/nfs_kv.h

@@ -104,8 +104,10 @@ struct nfs_rmw_t
     nfs_proxy_t *parent = NULL;
     uint64_t ino = 0;
     uint64_t offset = 0;
-    uint8_t *buf = NULL;
-    uint64_t size = 0;
+    uint8_t *buf1 = NULL;
+    uint64_t size1 = 0;
+    uint8_t *buf2 = NULL;
+    uint64_t size2 = 0;
     uint8_t *part_buf = NULL;
     uint64_t version = 0;
     nfs_rmw_t *other = NULL;

src/nfs/nfs_kv_write.cpp

@@ -185,8 +185,8 @@ void nfs_do_rmw(nfs_rmw_t *rmw)
 {
     auto parent = rmw->parent;
     auto align = parent->kvfs->pool_alignment;
-    assert(rmw->size < align);
-    assert((rmw->offset/parent->kvfs->pool_block_size) == ((rmw->offset+rmw->size-1)/parent->kvfs->pool_block_size));
+    assert(rmw->size1+rmw->size2 < align);
+    assert((rmw->offset/parent->kvfs->pool_block_size) == ((rmw->offset+rmw->size1+rmw->size2-1)/parent->kvfs->pool_block_size));
     if (!rmw->part_buf)
     {
         rmw->part_buf = (uint8_t*)malloc_or_die(align);
@@ -221,7 +221,7 @@ void nfs_do_rmw(nfs_rmw_t *rmw)
             }
             auto align = parent->kvfs->pool_alignment;
             bool is_begin = (rmw->offset % align);
-            bool is_end = ((rmw->offset+rmw->size) % align);
+            bool is_end = ((rmw->offset+rmw->size1+rmw->size2) % align);
             auto op = new cluster_op_t;
             op->opcode = OSD_OP_WRITE;
             op->inode = rmw->ino;
@@ -232,10 +232,17 @@ void nfs_do_rmw(nfs_rmw_t *rmw)
             {
                 op->iov.push_back(rmw->part_buf, rmw->offset % align);
             }
-            op->iov.push_back(rmw->buf, rmw->size);
+            if (rmw->buf1)
+            {
+                op->iov.push_back(rmw->buf1, rmw->size1);
+            }
+            if (rmw->buf2)
+            {
+                op->iov.push_back(rmw->buf2, rmw->size2);
+            }
             if (is_end)
             {
-                op->iov.push_back(rmw->part_buf + (rmw->offset % align) + rmw->size, align - (rmw->offset % align) - rmw->size);
+                op->iov.push_back(rmw->part_buf + (rmw->offset % align) + rmw->size1 + rmw->size2, align - (rmw->offset % align) - rmw->size1 - rmw->size2);
             }
             op->callback = [rmw](cluster_op_t *op)
             {
@@ -446,9 +453,9 @@ static void nfs_do_align_write(nfs_kv_write_state *st, uint64_t ino, uint64_t of
     uint64_t good_offset = offset;
     uint64_t good_size = st->size;
     bool begin_shdr = false;
-    uint64_t end_pad = 0;
     st->waiting++;
-    st->rmw[0].buf = st->rmw[1].buf = NULL;
+    st->rmw[0].buf1 = st->rmw[1].buf1 = NULL;
+    st->rmw[0].buf2 = st->rmw[1].buf2 = NULL;
     auto make_rmw_cb = [st, state]()
     {
         return [st, state](nfs_rmw_t *rmw)
@@ -473,7 +480,7 @@ static void nfs_do_align_write(nfs_kv_write_state *st, uint64_t ino, uint64_t of
             };
             begin_shdr = true;
             good_offset -= sizeof(shared_file_header_t);
-            offset = 0;
+            offset -= sizeof(shared_file_header_t);
         }
         else
         {
@@ -492,8 +499,8 @@ static void nfs_do_align_write(nfs_kv_write_state *st, uint64_t ino, uint64_t of
                 .parent = st->proxy,
                 .ino = ino,
                 .offset = offset,
-                .buf = st->buf,
-                .size = s,
+                .buf1 = st->buf,
+                .size1 = s,
                 .cb = make_rmw_cb(),
             };
             st->waiting++;
@@ -501,14 +508,26 @@ static void nfs_do_align_write(nfs_kv_write_state *st, uint64_t ino, uint64_t of
         }
     }
     if ((end % alignment) &&
-        (offset == 0 || end/alignment > (offset-1)/alignment))
+        ((end-1)/alignment > offset/alignment || offset == ((end-1) & ~(alignment-1))))
     {
         // Requires read-modify-write in the end
         assert(st->offset+st->size <= st->new_size);
-        if (st->offset+st->size == st->new_size)
+        if ((end-1)/alignment == offset/alignment && begin_shdr)
         {
-            // rmw can be skipped at end - we can just zero pad the request
-            end_pad = alignment - (end % alignment);
+            // end is at the same moment the beginning with a shared header
+            assert(!(offset % alignment) && good_size == st->size);
+            st->rmw[1] = (nfs_rmw_t){
+                .parent = st->proxy,
+                .ino = ino,
+                .offset = offset,
+                .buf1 = (uint8_t*)&st->shdr,
+                .size1 = sizeof(shared_file_header_t),
+                .buf2 = st->buf,
+                .size2 = st->size,
+                .cb = make_rmw_cb(),
+            };
+            good_size = 0;
+            begin_shdr = false;
         }
         else
         {
@@ -521,29 +540,28 @@ static void nfs_do_align_write(nfs_kv_write_state *st, uint64_t ino, uint64_t of
                 .parent = st->proxy,
                 .ino = ino,
                 .offset = end - s,
-                .buf = st->buf + st->size - s,
-                .size = s,
+                .buf2 = st->buf + st->size - s,
+                .size2 = s,
                 .cb = make_rmw_cb(),
             };
-            if (st->rmw[0].buf)
-            {
-                st->rmw[0].other = &st->rmw[1];
-                st->rmw[1].other = &st->rmw[0];
-            }
-            st->waiting++;
-            nfs_do_rmw(&st->rmw[1]);
         }
+        if (st->rmw[0].buf1)
+        {
+            st->rmw[0].other = &st->rmw[1];
+            st->rmw[1].other = &st->rmw[0];
+        }
+        st->waiting++;
+        nfs_do_rmw(&st->rmw[1]);
     }
-    if (good_size > 0 || end_pad > 0 || begin_shdr)
+    assert(!begin_shdr || good_size > 0);
+    if (good_size > 0)
     {
         // Normal write
-        nfs_do_write(ino, good_offset, (begin_shdr ? sizeof(shared_file_header_t) : 0)+good_size+end_pad, [&](cluster_op_t *op)
+        nfs_do_write(ino, good_offset, (begin_shdr ? sizeof(shared_file_header_t) : 0)+good_size, [&](cluster_op_t *op)
         {
             if (begin_shdr)
                 op->iov.push_back(&st->shdr, sizeof(shared_file_header_t));
             op->iov.push_back(good_buf, good_size);
-            if (end_pad)
-                op->iov.push_back(st->proxy->kvfs->zero_block.data(), end_pad);
         }, st, state);
     }
     st->waiting--;

tests/run_tests.sh

@@ -33,6 +33,8 @@ OLD=1 SCHEME=ec IMMEDIATE_COMMIT=1 ./test_interrupted_rebalance.sh
 
 ./test_failure_domain.sh
 
+./test_level_placement.sh
+
 ./test_snapshot.sh
 SCHEME=ec ./test_snapshot.sh
 OLD=1 ./test_snapshot.sh
@@ -145,3 +147,4 @@ TEST_NAME=old_csum_4k_dj   OLD=1 OSD_ARGS="--data_csum_type crc32c --inmemory_jo
 TEST_NAME=old_csum_4k      OLD=1 OSD_ARGS="--data_csum_type crc32c" OFFSET_ARGS=$OSD_ARGS ./test_heal.sh
 
 ./test_nfs.sh
+./test_nfs_unaligned_append.sh

tests/test_level_placement.sh

@@ -0,0 +1,39 @@
+#!/bin/bash -ex
+
+. `dirname $0`/common.sh
+
+node mon/mon-main.js $MON_PARAMS --etcd_address $ETCD_URL --etcd_prefix "/vitastor" >>./testdata/mon.log 2>&1 &
+MON_PID=$!
+wait_etcd
+
+TIME=$(date '+%s')
+$ETCDCTL put /vitastor/config/global '{"placement_levels":{"rack":1,"host":2,"osd":3},"immediate_commit":"none"}'
+$ETCDCTL put /vitastor/config/node_placement '{"rack1":{"level":"rack"},"rack2":{"level":"rack"},"rack3":{"level":"rack"},"rack4":{"level":"rack"},
+    "host1":{"level":"host","parent":"rack1"},"host2":{"level":"host","parent":"rack1"},
+    "host3":{"level":"host","parent":"rack2"},"host4":{"level":"host","parent":"rack2"},
+    "host5":{"level":"host","parent":"rack3"},"host6":{"level":"host","parent":"rack3"},
+    "host7":{"level":"host","parent":"rack4"},"host8":{"level":"host","parent":"rack4"}}'
+$ETCDCTL put /vitastor/osd/stats/1 '{"host":"host1","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/2 '{"host":"host1","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/3 '{"host":"host2","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/4 '{"host":"host2","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/5 '{"host":"host3","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/6 '{"host":"host3","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/7 '{"host":"host4","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/8 '{"host":"host4","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/9 '{"host":"host5","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/10 '{"host":"host5","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/11 '{"host":"host6","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/12 '{"host":"host6","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/13 '{"host":"host7","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/14 '{"host":"host7","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/15 '{"host":"host8","size":1073741824,"time":"'$TIME'"}'
+$ETCDCTL put /vitastor/osd/stats/16 '{"host":"host8","size":1073741824,"time":"'$TIME'"}'
+build/src/cmd/vitastor-cli --etcd_address $ETCD_URL create-pool testpool --failure_domain host --level_placement rack=112233 --ec 4+2 -n 32
+sleep 2
+$ETCDCTL get --prefix /vitastor/pg/config --print-value-only | \
+    jq -s -e '([ .[0].items["1"] | .[].osd_set | map_values(. | tonumber) | select(
+        (. | map_values((. - 1) / 2 | floor) | unique | length) == 6 and
+        (. | map_values((. - 1) / 4 | floor) | unique | length) == 3) ] | length) == 32'
+
+format_green OK

tests/test_nfs_unaligned_append.sh

@@ -0,0 +1,34 @@
+#!/bin/bash -ex
+
+GLOBAL_CONFIG=',"client_enable_writeback":false'
+IMMEDIATE_COMMIT=1
+PG_COUNT=16
+. `dirname $0`/run_3osds.sh
+
+build/src/cmd/vitastor-cli --etcd_address $ETCD_URL create -s 10G fsmeta
+build/src/cmd/vitastor-cli --etcd_address $ETCD_URL modify-pool --used-for-app fs:fsmeta testpool
+build/src/nfs/vitastor-nfs start --fs fsmeta --etcd_address $ETCD_URL --portmap 0 --port 2050 --foreground 1 --trace 1 >>./testdata/nfs.log 2>&1 &
+NFS_PID=$!
+
+mkdir -p testdata/nfs
+sudo mount localhost:/ ./testdata/nfs -o port=2050,mountport=2050,nfsvers=3,soft,nolock,tcp
+MNT=$(pwd)/testdata/nfs
+trap "sudo umount -f $MNT"' || true; kill -9 $(jobs -p)' EXIT
+
+# big file
+build/src/cmd/vitastor-cli --etcd_address $ETCD_URL dd if=/dev/urandom of=./testdata/ref_data.bin bs=1M count=32 seek=1024B
+build/src/cmd/vitastor-cli --etcd_address $ETCD_URL dd if=./testdata/ref_data.bin of=$MNT/testfile oflag=direct bs=1M iodepth=4 seek=1024B skip=1024B
+cp $MNT/testfile ./testdata/nfs_data.bin
+if ! diff -q ./testdata/ref_data.bin $MNT/testfile; then
+    format_error 'Data lost during parallel unaligned writes to VitastorFS'
+fi
+
+# small shared file
+build/src/cmd/vitastor-cli --etcd_address $ETCD_URL dd if=/dev/urandom of=./testdata/ref_small.bin bs=10 count=500 seek=15B
+build/src/cmd/vitastor-cli --etcd_address $ETCD_URL dd if=./testdata/ref_small.bin of=$MNT/smallfile oflag=direct bs=10 iodepth=4 seek=15B skip=15B
+cp $MNT/smallfile ./testdata/nfs_small.bin
+if ! diff -q ./testdata/ref_small.bin $MNT/smallfile; then
+    format_error 'Data lost during parallel unaligned writes to a small file in VitastorFS'
+fi
+
+format_green OK

tests/test_snapshot_pool2.sh

@@ -13,13 +13,18 @@ build/src/cmd/vitastor-cli --etcd_address $ETCD_URL create -s 128M testchain -p
 
 LD_PRELOAD="build/src/client/libfio_vitastor.so" \
     fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=1M -direct=1 -iodepth=4 -fsync=1 -rw=write \
-        -etcd=$ETCD_URL -image=testchain -mirror_file=./testdata/bin/mirror.bin -buffer_pattern=0xabcd
+        -etcd=$ETCD_URL -image=testchain -mirror_file=./testdata/bin/mirror.bin -randrepeat=0
 
 build/src/cmd/vitastor-cli --etcd_address $ETCD_URL snap-create testchain@snap1 -p testpool2
 
 LD_PRELOAD="build/src/client/libfio_vitastor.so" \
     fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4k -direct=1 -iodepth=4 -end_fsync=1 -rw=randwrite -number_ios=32 \
-        -etcd=$ETCD_URL -image=testchain -mirror_file=./testdata/bin/mirror.bin -buffer_pattern=0xabcd
+        -etcd=$ETCD_URL -image=testchain -mirror_file=./testdata/bin/mirror.bin -randrepeat=0
+
+# Read from the first snapshot
+
+build/src/cmd/vitastor-cli --etcd_address $ETCD_URL dd iimg=testchain of=./testdata/bin/res.bin bs=128k iodepth=4 --log_level 10
+cmp ./testdata/bin/res.bin ./testdata/bin/mirror.bin
 
 # Create a second snapshot - there was a bug where snapshotted reads from another pool
 # were working only when the image and the snapshot were modified in the same revision
@@ -28,7 +33,7 @@ build/src/cmd/vitastor-cli --etcd_address $ETCD_URL snap-create testchain@snap2
 
 LD_PRELOAD="build/src/client/libfio_vitastor.so" \
     fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4k -direct=1 -iodepth=4 -end_fsync=1 -rw=randwrite -number_ios=32 \
-        -etcd=$ETCD_URL -image=testchain -mirror_file=./testdata/bin/mirror.bin -buffer_pattern=0xabcd
+        -etcd=$ETCD_URL -image=testchain -mirror_file=./testdata/bin/mirror.bin -randrepeat=0
 
 build/src/cmd/vitastor-cli --etcd_address $ETCD_URL dd iimg=testchain of=./testdata/bin/res.bin bs=128k iodepth=4 --log_level 10