Release 3.0.4

New store bug fixes:

- Fix very slow OSD startup (hours) possible in unlucky cases
- Fix incorrect garbage entry validation on start in some cases, preventing OSD startup
- Fix small_write_throttling not functioning and either hanging or crashing OSDs
- Fix "different fact_versions returned from subops" error possible with EC in some cases
- Add a recovery option for the new store: skip_corrupted_meta_entries
- Disable intent writes (even 4k) for hybrid OSDs by default for better write buffering

Other changes:

- Fix chmod/chown of a new empty VitastorFS root directory not working
- Fix vitastor-dd treating all < 32k reads as always empty
- Fix vitastor-disk always setting 32k csum_block_size for HDDs when checksums are enabled
- Fix OSD missing PG state updates and PGs hanging in "starting" in rare cases
- Fix a possible OSD crash with "map::at" message in rare cases when stopping a client
- Fix a possible OSD crash on start when RDMA is available but fails to initialize
- Fix removal and overwriting of totaled objects (objects with all corrupted copies)
- Do not allow data_block_size < 8*bitmap_granularity
- Remove RDMA ODP (On-Demand Paging) support
- Update antietcd to 1.2.4
- Update Docker build to Debian Trixie
- Add packages for AlmaLinux 10

.gitea/workflows/buildenv.Dockerfile

@@ -1,28 +1,29 @@
-FROM node:16-bullseye
+FROM node:16-bookworm
 
 WORKDIR /root
 
-ADD ./docker/vitastor.gpg /etc/apt/trusted.gpg.d
+ADD ./docker/etc/apt/trusted.gpg.d /etc/apt/trusted.gpg.d
 
-RUN echo 'deb http://deb.debian.org/debian bullseye-backports main' >> /etc/apt/sources.list; \
-    echo 'deb http://vitastor.io/debian bullseye main' >> /etc/apt/sources.list; \
+RUN echo 'deb http://deb.debian.org/debian bookworm-backports main' >> /etc/apt/sources.list; \
+    echo 'deb http://vitastor.io/debian bookworm main' >> /etc/apt/sources.list; \
     echo >> /etc/apt/preferences; \
     echo 'Package: *' >> /etc/apt/preferences; \
-    echo 'Pin: release a=bullseye-backports' >> /etc/apt/preferences; \
+    echo 'Pin: release n=bookworm-backports' >> /etc/apt/preferences; \
     echo 'Pin-Priority: 500' >> /etc/apt/preferences; \
     echo >> /etc/apt/preferences; \
     echo 'Package: *' >> /etc/apt/preferences; \
     echo 'Pin: origin "vitastor.io"' >> /etc/apt/preferences; \
     echo 'Pin-Priority: 1000' >> /etc/apt/preferences; \
+    perl -i -pe 's/Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/debian.sources; \
     grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb/deb-src/' >> /etc/apt/sources.list; \
     echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf; \
     echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
 
 RUN apt-get update
-RUN apt-get -y install etcd qemu-system-x86 qemu-block-extra qemu-utils fio libasan5 \
+RUN apt-get -y install etcd qemu-system-x86 qemu-block-extra qemu-utils fio libasan8 \
     libgoogle-perftools-dev devscripts libjerasure-dev cmake libibverbs-dev libisal-dev
 RUN apt-get -y build-dep fio qemu=`dpkg -s qemu-system-x86|grep ^Version:|awk '{print $2}'`
-RUN apt-get update && apt-get -y install jq lp-solve sudo nfs-common fdisk parted
+RUN apt-get update && apt-get -y install jq lp-solve sudo nfs-common fdisk parted libc-ares-dev udev
 RUN apt-get --download-only source fio qemu=`dpkg -s qemu-system-x86|grep ^Version:|awk '{print $2}'`
 
 RUN set -ex; \

.gitea/workflows/test.yml

@@ -306,6 +306,78 @@ jobs:
           echo ""
         done
 
+  test_old_interrupted_rebalance:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: OLD=1 /root/vitastor/tests/test_interrupted_rebalance.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_interrupted_rebalance_imm:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: OLD=1 IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_interrupted_rebalance.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_interrupted_rebalance_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: OLD=1 SCHEME=ec /root/vitastor/tests/test_interrupted_rebalance.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_interrupted_rebalance_ec_imm:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: OLD=1 SCHEME=ec IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_interrupted_rebalance.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_create_halfhost:
     runs-on: ubuntu-latest
     needs: build
@@ -342,6 +414,24 @@ jobs:
           echo ""
         done
 
+  test_level_placement:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_level_placement.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_snapshot:
     runs-on: ubuntu-latest
     needs: build
@@ -378,6 +468,42 @@ jobs:
           echo ""
         done
 
+  test_old_snapshot:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_snapshot.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_snapshot_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 SCHEME=ec /root/vitastor/tests/test_snapshot.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_minsize_1:
     runs-on: ubuntu-latest
     needs: build
@@ -414,6 +540,24 @@ jobs:
           echo ""
         done
 
+  test_old_move_reappear:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_move_reappear.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_degraded:
     runs-on: ubuntu-latest
     needs: build
@@ -504,6 +648,42 @@ jobs:
           echo ""
         done
 
+  test_old_snapshot_chain:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_snapshot_chain.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_snapshot_chain_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 SCHEME=ec /root/vitastor/tests/test_snapshot_chain.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_snapshot_down:
     runs-on: ubuntu-latest
     needs: build
@@ -540,6 +720,78 @@ jobs:
           echo ""
         done
 
+  test_old_snapshot_down:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_snapshot_down.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_snapshot_down_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 SCHEME=ec /root/vitastor/tests/test_snapshot_down.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_kv_stress:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_kv_stress.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_kv_stress_imm:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_kv_stress.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_splitbrain:
     runs-on: ubuntu-latest
     needs: build
@@ -630,6 +882,78 @@ jobs:
           echo ""
         done
 
+  test_old_rebalance_verify:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: OLD=1 /root/vitastor/tests/test_rebalance_verify.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_rebalance_verify_imm:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: OLD=1 IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_rebalance_verify.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_rebalance_verify_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: OLD=1 SCHEME=ec /root/vitastor/tests/test_rebalance_verify.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_rebalance_verify_ec_imm:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: OLD=1 SCHEME=ec IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_rebalance_verify.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_dd:
     runs-on: ubuntu-latest
     needs: build
@@ -738,7 +1062,7 @@ jobs:
           echo ""
         done
 
-  test_write_no_same:
+  test_old_write:
     runs-on: ubuntu-latest
     needs: build
     container: ${{env.TEST_IMAGE}}:${{github.sha}}
@@ -746,7 +1070,61 @@ jobs:
     - name: Run test
       id: test
       timeout-minutes: 3
-      run: /root/vitastor/tests/test_write_no_same.sh
+      run: OLD=1 /root/vitastor/tests/test_write.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_write_xor:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 SCHEME=xor /root/vitastor/tests/test_write.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_write_old_iothreads:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: TEST_NAME=old_iothreads OLD=1 GLOBAL_CONFIG=',"client_iothread_count":4' /root/vitastor/tests/test_write.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_write_no_same:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_write_no_same.sh
     - name: Print logs
       if: always() && steps.test.outcome == 'failure'
       run: |
@@ -828,6 +1206,78 @@ jobs:
           echo ""
         done
 
+  test_checksum:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_checksum.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_checksum:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_checksum.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_corrupt_all:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_corrupt_all.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_corrupt_all:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_corrupt_all.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_reweight_half:
     runs-on: ubuntu-latest
     needs: build
@@ -1026,6 +1476,42 @@ jobs:
           echo ""
         done
 
+  test_old_resize:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_resize.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_resize_auto:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_resize_auto.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_osd_tags:
     runs-on: ubuntu-latest
     needs: build
@@ -1116,6 +1602,78 @@ jobs:
           echo ""
         done
 
+  test_old_enospc:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_enospc.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_enospc_xor:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 SCHEME=xor /root/vitastor/tests/test_enospc.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_enospc_imm:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_enospc.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_enospc_imm_xor:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 IMMEDIATE_COMMIT=1 SCHEME=xor /root/vitastor/tests/test_enospc.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_scrub:
     runs-on: ubuntu-latest
     needs: build
@@ -1224,6 +1782,240 @@ jobs:
           echo ""
         done
 
+  test_old_scrub:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_scrub.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_scrub_zero_osd_2:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 ZERO_OSD=2 /root/vitastor/tests/test_scrub.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_scrub_xor:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 SCHEME=xor /root/vitastor/tests/test_scrub.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_scrub_pg_size_3:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 PG_SIZE=3 /root/vitastor/tests/test_scrub.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_scrub_pg_size_6_pg_minsize_4_osd_count_6_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 PG_SIZE=6 PG_MINSIZE=4 OSD_COUNT=6 SCHEME=ec /root/vitastor/tests/test_scrub.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_scrub_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 SCHEME=ec /root/vitastor/tests/test_scrub.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_partwr_csum:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_partwr_csum.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_heal_old_csum_32k_dmj:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: TEST_NAME=old_csum_32k_dmj OLD=1 OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k --inmemory_metadata false --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_heal_old_csum_32k_dj:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: TEST_NAME=old_csum_32k_dj  OLD=1 OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_heal_old_csum_32k:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: TEST_NAME=old_csum_32k     OLD=1 OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_heal_old_csum_4k_dmj:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: TEST_NAME=old_csum_4k_dmj  OLD=1 OSD_ARGS="--data_csum_type crc32c --inmemory_metadata false --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_heal_old_csum_4k_dj:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: TEST_NAME=old_csum_4k_dj   OLD=1 OSD_ARGS="--data_csum_type crc32c --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_heal_old_csum_4k:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: TEST_NAME=old_csum_4k      OLD=1 OSD_ARGS="--data_csum_type crc32c" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_nfs:
     runs-on: ubuntu-latest
     needs: build
@@ -1242,3 +2034,21 @@ jobs:
           echo ""
         done
 
+  test_nfs_unaligned_append:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_nfs_unaligned_append.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+

.gitea/workflows/tests-to-yaml.pl

@@ -38,6 +38,10 @@ for my $line (<>)
             {
                 $test_name .= '_antietcd';
             }
+            elsif ($1 eq 'OLD')
+            {
+                $test_name =~ s/^test_/test_old_/s;
+            }
             else
             {
                 $test_name .= '_'.lc($1).'_'.$2;

CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "2.4.3")
+set(VITASTOR_VERSION "3.0.4")
 
 include(CTest)
 

README-ru.md

@@ -26,11 +26,15 @@ Vitastor поддерживает QEMU-драйвер, протоколы UBLK,
 
 ## Презентации и записи докладов
 
+- KuberConf'2025: [видео](https://vitastor.io/presentation/kuberconf.webm)
+- Highload'2025: [видео](https://vitastor.io/presentation/hl2025/hl2025.webm),
+  [на youtube](https://www.youtube.com/watch?v=0R8MLjFtz7g), презентация
+  ([на русском](https://vitastor.io/presentation/hl2025/), [на английском](https://vitastor.io/presentation/hl2025/en.html))
+- Highload'2022: презентация ([на русском](https://vitastor.io/presentation/highload/highload.html)),
+  [видео](https://vitastor.io/presentation/highload/talk.webm)
 - DevOpsConf'2021: презентация ([на русском](https://vitastor.io/presentation/devopsconf/devopsconf.html),
   [на английском](https://vitastor.io/presentation/devopsconf/devopsconf_en.html)),
   [видео](https://vitastor.io/presentation/devopsconf/talk.webm)
-- Highload'2022: презентация ([на русском](https://vitastor.io/presentation/highload/highload.html)),
-  [видео](https://vitastor.io/presentation/highload/talk.webm)
 
 ## Документация
 

README.md

@@ -26,11 +26,15 @@ Read more details in the documentation. You can start from here: [Quick Start](d
 
 ## Talks and presentations
 
+- KuberConf'2025: [video](https://vitastor.io/presentation/kuberconf.webm)
+- Highload'2025: [video](https://vitastor.io/presentation/hl2025/hl2025.webm),
+  [youtube](https://www.youtube.com/watch?v=0R8MLjFtz7g), presentation
+  ([in Russian](https://vitastor.io/presentation/hl2025/), [in English](https://vitastor.io/presentation/hl2025/en.html))
+- Highload'2022: presentation ([in Russian](https://vitastor.io/presentation/highload/highload.html)),
+  [video](https://vitastor.io/presentation/highload/talk.webm)
 - DevOpsConf'2021: presentation ([in Russian](https://vitastor.io/presentation/devopsconf/devopsconf.html),
   [in English](https://vitastor.io/presentation/devopsconf/devopsconf_en.html)),
   [video](https://vitastor.io/presentation/devopsconf/talk.webm)
-- Highload'2022: presentation ([in Russian](https://vitastor.io/presentation/highload/highload.html)),
-  [video](https://vitastor.io/presentation/highload/talk.webm)
 
 ## Documentation
 

csi/Dockerfile

@@ -1,5 +1,5 @@
 # Compile stage
-FROM golang:bookworm AS build
+FROM golang:trixie AS build
 
 ADD go.sum go.mod /app/
 RUN cd /app; CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go mod download -x
@@ -9,7 +9,7 @@ RUN perl -i -e '$/ = undef; while(<>) { s/\n\s*(\{\s*\n)/$1\n/g; s/\}(\s*\n\s*)e
     CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o vitastor-csi
 
 # Final stage
-FROM debian:bookworm
+FROM debian:trixie
 
 LABEL maintainers="Vitaliy Filippov <vitalif@yourcmc.ru>"
 LABEL description="Vitastor CSI Driver"
@@ -25,20 +25,20 @@ RUN apt-get update && \
         # NFS mount dependencies
         nfs-common netbase \
         # dependencies of qemu-storage-daemon
-        libnuma1 liburing2 libglib2.0-0 libfuse3-3 libaio1 libzstd1 libnettle8 \
-        libgmp10 libhogweed6 libp11-kit0 libidn2-0 libunistring2 libtasn1-6 libpcre2-8-0 libffi8 && \
+        libaio1t64 libc6 libfuse3-4 libglib2.0-0t64 libgmp10 libgnutls30t64 \
+        libhogweed6t64 libnettle8t64 libnuma1 libselinux1 liburing2 libzstd1 zlib1g && \
     apt-get clean && \
     (echo options nbd nbds_max=128 > /etc/modprobe.d/nbd.conf)
 
 COPY --from=build /app/vitastor-csi /bin/
 
-RUN (echo deb http://vitastor.io/debian bookworm main > /etc/apt/sources.list.d/vitastor.list) && \
+RUN (echo deb http://vitastor.io/debian trixie main > /etc/apt/sources.list.d/vitastor.list) && \
     ((echo 'Package: *'; echo 'Pin: origin "vitastor.io"'; echo 'Pin-Priority: 1000') > /etc/apt/preferences.d/vitastor.pref) && \
     wget -q -O /etc/apt/trusted.gpg.d/vitastor.gpg https://vitastor.io/debian/pubkey.gpg && \
     apt-get update && \
     apt-get install -y vitastor-client ibverbs-providers && \
-    wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-utils_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
-    wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-block-extra_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-trixie-10.0.2%2Bds-2%2Bvitastor1/qemu-utils_10.0.2%2Bds-2%2Bvitastor1_amd64.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-trixie-10.0.2%2Bds-2%2Bvitastor1/qemu-block-extra_10.0.2%2Bds-2%2Bvitastor1_amd64.deb && \
     dpkg -x qemu-utils*.deb tmp1 && \
     dpkg -x qemu-block-extra*.deb tmp1 && \
     cp -a tmp1/usr/bin/qemu-storage-daemon /usr/bin/ && \

csi/Dockerfile.local

@@ -1,5 +1,5 @@
 # Compile stage
-FROM golang:bookworm AS build
+FROM golang:trixie AS build
 
 ADD go.sum go.mod /app/
 RUN cd /app; CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go mod download -x
@@ -9,7 +9,7 @@ RUN perl -i -e '$/ = undef; while(<>) { s/\n\s*(\{\s*\n)/$1\n/g; s/\}(\s*\n\s*)e
     CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o vitastor-csi
 
 # Final stage
-FROM debian:bookworm
+FROM debian:trixie
 
 LABEL maintainers="Vitaliy Filippov <vitalif@yourcmc.ru>"
 LABEL description="Vitastor CSI Driver"
@@ -36,8 +36,8 @@ ADD deb /deb
 
 RUN apt-get update && \
     apt-get -y install /deb/vitastor-client_*.deb && \
-    wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-utils_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
-    wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-block-extra_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-trixie-9.2.2%2Bds-1%2Bvitastor4/qemu-utils_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-trixie-9.2.2%2Bds-1%2Bvitastor4/qemu-block-extra_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
     dpkg -x qemu-utils*.deb tmp1 && \
     dpkg -x qemu-block-extra*.deb tmp1 && \
     cp -a tmp1/usr/bin/qemu-storage-daemon /usr/bin/ && \

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v2.4.3
+VITASTOR_VERSION ?= v3.0.4
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v2.4.3
+          image: vitalif/vitastor-csi:v3.0.4
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v2.4.3
+          image: vitalif/vitastor-csi:v3.0.4
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "2.4.3"
+    vitastorCSIDriverVersion = "3.0.4"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (2.4.3-1) unstable; urgency=medium
+vitastor (3.0.4-1) unstable; urgency=medium
 
   * Bugfixes
 

docker/Dockerfile

@@ -1,6 +1,6 @@
 # Build Docker image with Vitastor packages
 
-FROM debian:bookworm
+FROM debian:trixie
 
 ADD etc/apt /etc/apt/
 RUN apt-get update && apt-get -y install vitastor ibverbs-providers udev systemd qemu-system-x86 qemu-system-common qemu-block-extra qemu-utils jq nfs-common && apt-get clean

docker/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v2.4.3
+VITASTOR_VERSION ?= v3.0.4
 
 all: build push
 

docker/etc/apt/preferences

@@ -1,3 +1,3 @@
 Package: *
-Pin: release n=bookworm-backports
+Pin: release n=trixie-backports
 Pin-Priority: 500

docker/etc/apt/sources.list.d/vitastor.list

@@ -1,2 +1,2 @@
-deb http://vitastor.io/debian bookworm main
-deb http://http.debian.net/debian/ bookworm-backports main
+deb http://vitastor.io/debian trixie main
+#deb http://http.debian.net/debian/ trixie-backports main

docker/etc/vitastor/docker.conf

@@ -4,7 +4,7 @@
 #
 
 # Desired Vitastor version
-VITASTOR_VERSION=v2.4.3
+VITASTOR_VERSION=v3.0.4
 
 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here

docs/config/layout-osd.en.md

@@ -9,6 +9,7 @@
 These parameters apply to OSDs, are fixed at the moment of OSD drive
 initialization and can't be changed after it without losing data.
 
+- [meta_format](#meta_format)
 - [data_device](#data_device)
 - [meta_device](#meta_device)
 - [journal_device](#journal_device)
@@ -27,6 +28,21 @@ initialization and can't be changed after it without losing data.
 - [data_csum_type](#data_csum_type)
 - [csum_block_size](#csum_block_size)
 
+## meta_format
+
+- Type: integer
+- Default: 3
+
+OSD store implementation version and on-disk metadata format.
+
+Three versions are currently supported: 3, 2 and 1.
+- 3 the new log-structured store, it's overall faster, has lower Write
+  Amplification, which may be even close to 1 (i.e. almost no extra writes)
+  if your SSDs support atomic writes (see [atomic_write_size](osd.en.md#atomic_write_size)).
+- 2 is the old stable store from Vitastor 0.9-2.x.
+- 1 is the same old store but with a legacy metadata format from Vitastor
+  versions to up 0.8.x, without any support for checksums.
+
 ## data_device
 
 - Type: string

docs/config/layout-osd.ru.md

@@ -10,6 +10,7 @@
 дисковые параметры, задаются в момент инициализации дисков OSD и не могут быть
 изменены после этого без потери данных.
 
+- [meta_format](#meta_format)
 - [data_device](#data_device)
 - [meta_device](#meta_device)
 - [journal_device](#journal_device)
@@ -28,6 +29,23 @@
 - [data_csum_type](#data_csum_type)
 - [csum_block_size](#csum_block_size)
 
+## meta_format
+
+- Тип: целое число
+- Значение по умолчанию: 3
+
+Версия реализации дискового хранилища OSD и дискового формата метаданных.
+
+Поддерживаются три версии: 3, 2 и 1.
+- 3 - новое лог-структурированное хранилище, в целом более быстрое, со
+  сниженным фактором амплификации записи, который может составлять около 1
+  (то есть, практически без лишней служебной записи), если ваши SSD
+  поддерживают атомарную запись (см. [atomic_write_size](osd.ru.md#atomic_write_size)).
+- 2 - старое стабильное хранилище из версий Vitastor 0.9-2.x.
+- 1 - то же самое стабильное хранилище, но с ещё более старым форматом
+  метаданных из версий Vitastor до 0.8.x, без какой-либо поддержки
+  контрольных сумм.
+
 ## data_device
 
 - Тип: строка

docs/config/network.en.md

@@ -22,7 +22,6 @@ between clients, OSDs and etcd.
 - [rdma_max_msg](#rdma_max_msg)
 - [rdma_max_recv](#rdma_max_recv)
 - [rdma_max_send](#rdma_max_send)
-- [rdma_odp](#rdma_odp)
 - [peer_connect_interval](#peer_connect_interval)
 - [peer_connect_timeout](#peer_connect_timeout)
 - [osd_idle_timeout](#osd_idle_timeout)
@@ -102,11 +101,6 @@ found or if `osd_network` is not specified. Auto-selection is also
 unsupported with old libibverbs < v32, like in Debian 10 Buster or
 CentOS 7.
 
-Vitastor supports all adapters, even ones without ODP support, like
-Mellanox ConnectX-3 and non-Mellanox cards. Versions up to Vitastor
-1.2.0 required ODP which is only present in Mellanox ConnectX >= 4.
-See also [rdma_odp](#rdma_odp).
-
 Run `ibv_devinfo -v` as root to list available RDMA devices and their
 features.
 
@@ -116,6 +110,23 @@ the manual of your network vendor for details about setting up the switch
 for RoCEv2 correctly. Usually it means setting up Lossless Ethernet with
 PFC (Priority Flow Control) and ECN (Explicit Congestion Notification).
 
+Vitastor supports all adapters, even ones without ODP (On-Demand Paging)
+support, like Mellanox ConnectX-3 and non-Mellanox cards. ODP is only present
+in Mellanox ConnectX >= 4 adapters and allows to skip memory registration
+for RDMA and thus, in theory, avoid memory copying.
+
+Versions up to Vitastor 1.2.0 required ODP, then it was disabled by default,
+but it was still supported up to 3.0.3. Now ODP support is removed because it
+actually only hurts performance: an example 3-node cluster with 8 NVMe in each
+node and 2*25 GBit/s ConnectX-6 RDMA network pushed 3950000 read iops without
+ODP, but only 239000 iops with ODP.
+
+This happens because Mellanox ODP implementation seems to be based on
+message retransmissions when the adapter doesn't know about the buffer yet -
+it likely uses standard "RNR retransmissions" (RNR = receiver not ready)
+which is generally slow in RDMA/RoCE networks. Here's a presentation about
+it from ISPASS-2021 conference: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
+
 ## rdma_port_num
 
 - Type: integer
@@ -187,28 +198,6 @@ less than `rdma_max_recv` so the receiving side doesn't run out of buffers.
 Doesn't affect memory usage - additional memory isn't allocated for send
 operations.
 
-## rdma_odp
-
-- Type: boolean
-- Default: false
-
-Use RDMA with On-Demand Paging. ODP is currently only available on Mellanox
-ConnectX-4 and newer adapters. ODP allows to not register memory explicitly
-for RDMA adapter to be able to use it. This, in turn, allows to skip memory
-copying during sending. One would think this should improve performance, but
-**in reality** RDMA performance with ODP is **drastically** worse. Example
-3-node cluster with 8 NVMe in each node and 2*25 GBit/s ConnectX-6 RDMA network
-without ODP pushes 3950000 read iops, but only 239000 iops with ODP...
-
-This happens because Mellanox ODP implementation seems to be based on
-message retransmissions when the adapter doesn't know about the buffer yet -
-it likely uses standard "RNR retransmissions" (RNR = receiver not ready)
-which is generally slow in RDMA/RoCE networks. Here's a presentation about
-it from ISPASS-2021 conference: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
-
-ODP support is retained in the code just in case a good ODP implementation
-appears one day.
-
 ## peer_connect_interval
 
 - Type: seconds

docs/config/network.ru.md

@@ -22,7 +22,6 @@
 - [rdma_max_msg](#rdma_max_msg)
 - [rdma_max_recv](#rdma_max_recv)
 - [rdma_max_send](#rdma_max_send)
-- [rdma_odp](#rdma_odp)
 - [peer_connect_interval](#peer_connect_interval)
 - [peer_connect_timeout](#peer_connect_timeout)
 - [osd_idle_timeout](#osd_idle_timeout)
@@ -101,12 +100,6 @@ RoCEv1/RoCEv2, и даже позволяет полностью отключи
 не задана. Также автовыбор не поддерживается со старыми версиями библиотеки
 libibverbs < v32, например в Debian 10 Buster или CentOS 7.
 
-Vitastor поддерживает все модели адаптеров, включая те, у которых
-нет поддержки ODP, то есть вы можете использовать RDMA с ConnectX-3 и
-картами производства не Mellanox. Версии Vitastor до 1.2.0 включительно
-требовали ODP, который есть только на Mellanox ConnectX 4 и более новых.
-См. также [rdma_odp](#rdma_odp).
-
 Запустите `ibv_devinfo -v` от имени суперпользователя, чтобы посмотреть
 список доступных RDMA-устройств, их параметры и возможности.
 
@@ -117,6 +110,24 @@ Vitastor поддерживает все модели адаптеров, вкл
 подразумевает настройку сети без потерь на основе PFC (Priority Flow
 Control) и ECN (Explicit Congestion Notification).
 
+Vitastor поддерживает все модели адаптеров, включая те, у которых нет
+поддержки ODP (On-Demand Paging), например, ConnectX-3 и карты производства
+не Mellanox. Функция ODP доступна только на адаптерах Mellanox ConnectX-4 и
+более новых и позволяет не регистрировать память для её использования RDMA-картой,
+благодаря чему в теории можно избежать лишних копирований памяти.
+
+Версии Vitastor до 1.2.0 включительно требовали ODP, потом функция был отключена
+по умолчанию, но поддерживалась вплоть до версии 3.0.3. Сейчас поддержка ODP
+полностью удалена, так как на самом деле она только портит производительность:
+например, на 3-узловом кластере с 8 NVMe в каждом узле и сетью 2*25 Гбит/с на
+чтение с RDMA без ODP удаётся снять 3950000 iops, а с ODP - всего 239000 iops.
+
+Это происходит из-за того, что реализация ODP у Mellanox неоптимальная и
+основана на повторной передаче сообщений, когда карте не известен буфер -
+вероятно, на стандартных "RNR retransmission" (RNR = receiver not ready).
+А данные повторные передачи в RDMA/RoCE - всегда очень медленная штука.
+Презентация на эту тему с конференции ISPASS-2021: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
+
 ## rdma_port_num
 
 - Тип: целое число
@@ -192,29 +203,6 @@ OSD в любом случае согласовывают реальное зн
 Не влияет на потребление памяти - дополнительная память на операции отправки
 не выделяется.
 
-## rdma_odp
-
-- Тип: булево (да/нет)
-- Значение по умолчанию: false
-
-Использовать RDMA с On-Demand Paging. ODP - функция, доступная пока что
-исключительно на адаптерах Mellanox ConnectX-4 и более новых. ODP позволяет
-не регистрировать память для её использования RDMA-картой. Благодаря этому
-можно не копировать данные при отправке их в сеть и, казалось бы, это должно
-улучшать производительность - но **по факту** получается так, что
-производительность только ухудшается, причём сильно. Пример - на 3-узловом
-кластере с 8 NVMe в каждом узле и сетью 2*25 Гбит/с на чтение с RDMA без ODP
-удаётся снять 3950000 iops, а с ODP - всего 239000 iops...
-
-Это происходит из-за того, что реализация ODP у Mellanox неоптимальная и
-основана на повторной передаче сообщений, когда карте не известен буфер -
-вероятно, на стандартных "RNR retransmission" (RNR = receiver not ready).
-А данные повторные передачи в RDMA/RoCE - всегда очень медленная штука.
-Презентация на эту тему с конференции ISPASS-2021: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
-
-Возможность использования ODP сохранена в коде на случай, если вдруг в один
-прекрасный день появится хорошая реализация ODP.
-
 ## peer_connect_interval
 
 - Тип: секунды

docs/config/osd.en.md

@@ -38,6 +38,7 @@ with an OSD restart or, for some of them, even without restarting by updating co
 - [journal_io](#journal_io)
 - [journal_sector_buffer_count](#journal_sector_buffer_count)
 - [journal_no_same_sector_overwrites](#journal_no_same_sector_overwrites)
+- [skip_corrupted_meta_entries](#skip_corrupted_meta_entries)
 - [throttle_small_writes](#throttle_small_writes)
 - [throttle_target_iops](#throttle_target_iops)
 - [throttle_target_mbs](#throttle_target_mbs)
@@ -65,6 +66,10 @@ with an OSD restart or, for some of them, even without restarting by updating co
 - [allow_net_split](#allow_net_split)
 - [enable_pg_locks](#enable_pg_locks)
 - [pg_lock_retry_interval_ms](#pg_lock_retry_interval_ms)
+- [atomic_write_size](#atomic_write_size)
+- [use_atomic_flag](#use_atomic_flag)
+- [pg_reshard_chunk_size](#pg_reshard_chunk_size)
+- [pg_reshard_chunk_pause_ms](#pg_reshard_chunk_pause_ms)
 
 ## bind_address
 
@@ -275,13 +280,19 @@ Maximum number of journal flushers (see above min_flusher_count).
 - Type: boolean
 - Default: true
 
-This parameter makes Vitastor always keep metadata area of the block device
-in memory. It's required for good performance because it allows to avoid
-additional read-modify-write cycles during metadata modifications. Metadata
-area size is currently roughly 224 MB per 1 TB of data. You can turn it off
-to reduce memory usage by this value, but it will hurt performance. This
-restriction is likely to be removed in the future along with the upgrade
-of the metadata storage scheme.
+Only for the old store ([meta_format](layout-osd.en.md#meta_format) 2).
+
+This parameter makes Vitastor keep a copy of metadata area in memory as it is
+on disk, in addition to the metadata database. When the option is enabled, every
+metadata entry is effectively stored in RAM twice. It's required for good performance
+because it allows to avoid additional read-modify-write cycles during metadata
+modifications. Metadata area size with the old store is roughly 224 MB per 1 TB
+of data. You can turn the option off to reduce memory usage by this value, but
+it will reduce performance.
+
+For the new store ([meta_format](layout-osd.en.md#meta_format) 3), the option
+may be changed in the future to support operation without loading full metadata
+database in memory.
 
 ## inmemory_journal
 
@@ -360,6 +371,8 @@ blocks. The only situation when you should increase it to a larger value
 is when you enable journal_no_same_sector_overwrites. In this case set
 it to, for example, 1024.
 
+Not applicable to the new store ([meta_format](layout-osd.en.md#meta_format) 3).
+
 ## journal_no_same_sector_overwrites
 
 - Type: boolean
@@ -373,6 +386,17 @@ journal after writing it instead of possibly overwriting it the second time.
 
 Most (99%) other SSDs don't need this option.
 
+Not applicable to the new store ([meta_format](layout-osd.en.md#meta_format) 3).
+
+## skip_corrupted_meta_entries
+
+- Type: boolean
+- Default: false
+
+Only for the new store ([meta_format](layout-osd.en.md#meta_format) 3).
+Allow OSD to start when some metadata entries or blocks are corrupted by
+skipping them. Should be only used as an emergency measure.
+
 ## throttle_small_writes
 
 - Type: boolean
@@ -666,3 +690,66 @@ Use this parameter to enable or disable this function for all pools.
 - Default: 100
 
 Retry interval for failed PG lock attempts.
+
+## atomic_write_size
+
+- Type: integer
+- Default: 4096
+
+Maximum data device atomic write size allowed for OSD to use.
+
+Atomic writes allow to reduce the Write Amplification factor with the new store
+([meta_format](layout-osd.en.md#meta_format)=3) to almost 1 (i.e. almost no extra writes)
+with replicated pools and reach the best possible write performance.
+
+Default value is auto-detected during OSD initialization from
+`/sys/block/xx/queue/atomic_write_max_bytes` or assumed to be 4096 bytes
+because all known disks support 4 KB atomic writes. Auto-detection is only used for
+NVMe disks because SAS disks require the explicit WRITE ATOMIC command which requires
+RWF_ATOMIC (see below [#use_atomic_flag]) but that flag works incorrectly in current
+Linux versions.
+
+You can also check if your NVMe drives support atomic writes by running
+the command `nvme id-ctrl /dev/nvme0n1 | grep awupf`. If the reported value,
+plus 1, multiplied by the currently selected block size of the NVMe,
+is more than 4 KB, then the new store can utilize it for better performance.
+The only drives known to support it currently are [Micron and Kioxia](../intro/quickstart.en.md).
+
+Atomic writes allow to skip double data writes in replicated pools, thus
+reducing Write Amplification and improving write performance up to 2 times.
+
+## use_atomic_flag
+
+- Type: boolean
+
+This option controls whether Vitastor OSDs use RWF_ATOMIC write flag with atomic writes.
+This flag is supported since Linux 6.11 and adds some safety to atomic writes - the kernel
+guarantees to not fragment write requests with it and also to check them against the actual
+device atomic write capabilities.
+
+However, the option is disabled by default because the flag is currently UNUSABLE - Linux
+incorrectly requires writes with that flag to be of power-of-2 length and length-aligned.
+I.e., for example, 12 KB writes and not-8-KB aligned 8 KB writes are forbidden by the kernel,
+even though the NVMe specification allows them.
+
+For NVMe disks with `scheduler=none` writes aren't fragmented anyway so it's not a big deal.
+However, you can rebuild your kernel with [this patch](../../patches/linux-fix-atomic-write-checks.diff)
+and turn this option on. It will make your atomic writes a bit safer.
+
+## pg_reshard_chunk_size
+
+- Type: integer
+- Default: 100000
+
+Pool PG count change is a CPU-intensive operation because OSDs store the full object database
+in memory and have to move all entries between old and new PGs. Thus it's performed in chunks,
+with pauses between chunks to prevent blocking OSD's event loop and other clients' operations.
+This option sets the maximum number of object is a chunk. Moving 100k objects usually takes
+50-100ms. Chunk size equal to 0 means unlimited.
+
+## pg_reshard_chunk_pause_ms
+
+- Type: milliseconds
+- Default: 100
+
+This option sets the interval between handling two PG count change chunks.

docs/config/osd.ru.md

@@ -39,6 +39,7 @@
 - [journal_io](#journal_io)
 - [journal_sector_buffer_count](#journal_sector_buffer_count)
 - [journal_no_same_sector_overwrites](#journal_no_same_sector_overwrites)
+- [skip_corrupted_meta_entries](#skip_corrupted_meta_entries)
 - [throttle_small_writes](#throttle_small_writes)
 - [throttle_target_iops](#throttle_target_iops)
 - [throttle_target_mbs](#throttle_target_mbs)
@@ -66,6 +67,10 @@
 - [allow_net_split](#allow_net_split)
 - [enable_pg_locks](#enable_pg_locks)
 - [pg_lock_retry_interval_ms](#pg_lock_retry_interval_ms)
+- [atomic_write_size](#atomic_write_size)
+- [use_atomic_flag](#use_atomic_flag)
+- [pg_reshard_chunk_size](#pg_reshard_chunk_size)
+- [pg_reshard_chunk_pause_ms](#pg_reshard_chunk_pause_ms)
 
 ## bind_address
 
@@ -283,13 +288,19 @@ Flusher - это микро-поток (корутина), которая коп
 - Тип: булево (да/нет)
 - Значение по умолчанию: true
 
-Данный параметр заставляет Vitastor всегда держать область метаданных диска
-в памяти. Это нужно, чтобы избегать дополнительных операций чтения с диска
-при записи. Размер области метаданных на данный момент составляет примерно
-224 МБ на 1 ТБ данных. При включении потребление памяти снизится примерно
-на эту величину, но при этом также снизится и производительность. В будущем,
-после обновления схемы хранения метаданных, это ограничение, скорее всего,
-будет ликвидировано.
+Только для старого хранилища ([meta_format](layout-osd.en.md#meta_format) 2).
+
+Данный параметр заставляет Vitastor всегда держать копию области метаданных
+в памяти в том же виде, как она лежит на диске, в дополнение к БД метаданных.
+То есть, с включённой опцией каждая запись метаданных хранится в памяти дважды.
+Это нужно, чтобы избегать дополнительных операций чтения с диска при записи.
+Размер области метаданных в старом хранилище составляет примерно 224 МБ на
+1 ТБ данных. Вы можете отключить опцию, чтобы снизить потребление памяти
+примерно на эту величину, но при этом также снизится и производительность.
+
+Для нового хранилища ([meta_format](layout-osd.en.md#meta_format) 3) опция,
+возможно, будет переработана в будущем для поддержки работы без полной
+загрузки метаданных в памяти.
 
 ## inmemory_journal
 
@@ -372,6 +383,8 @@ fsync небезопасным даже с режимом "directsync".
 нужно менять - это если вы включаете journal_no_same_sector_overwrites. В
 этом случае установите данный параметр, например, в 1024.
 
+Неприменимо к новому хранилищу ([meta_format](layout-osd.en.md#meta_format) 3).
+
 ## journal_no_same_sector_overwrites
 
 - Тип: булево (да/нет)
@@ -387,6 +400,18 @@ fsync небезопасным даже с режимом "directsync".
 
 Почти все другие SSD (99% моделей) не требуют данной опции.
 
+Неприменимо к новому хранилищу ([meta_format](layout-osd.en.md#meta_format) 3).
+
+## skip_corrupted_meta_entries
+
+- Тип: булево (да/нет)
+- Значение по умолчанию: false
+
+Только для нового хранилища ([meta_format](layout-osd.en.md#meta_format) 3).
+Разрешить OSD запускаться, даже если часть блоков или записей метаданных
+повреждена, пропуская их. Опция предназначена для использования только в
+целях аварийного восстановления.
+
 ## throttle_small_writes
 
 - Тип: булево (да/нет)
@@ -699,3 +724,72 @@ pg_minsize OSD во время переключений, что может по
 - Значение по умолчанию: 100
 
 Интервал повтора неудачных попыток блокировки PG.
+
+## atomic_write_size
+
+- Тип: целое число
+- Значение по умолчанию: 4096
+
+Максимальный размер атомарной записи на диск данных, который OSD разрешено использовать.
+
+Поддержка атомарной записи позволяет снизить мультипликатор записи (Write Amplification)
+на диск с новым хранилищем ([meta_format](layout-osd.ru.md#meta_format)=3)
+практически до 1 (то есть, почти до нулевого объёма лишней записи) в реплицированных
+пулах и достигнуть наилучшей возможной производительности записи.
+
+Значение по умолчанию авто-определяется во время инициализации OSD из
+`/sys/block/xx/queue/atomic_write_max_bytes` либо принимается равным 4096,
+так как все известные диски поддерживают атомарную запись 4 КБ блоков.
+Автоопределение применяется только для NVMe-дисков, так как SAS диски требуют
+использования отдельной команды WRITE ATOMIC, а для неё нужен флаг RWF_ATOMIC
+(см. ниже [#use_atomic_flag]), а он в текущих версиях Linux работает некорректно.
+
+Вы также можете проверить, поддерживают ли ваши NVMe-диски атомарную запись,
+с помощью команды `nvme id-ctrl /dev/nvme0n1 | grep awupf`. Если значение awupf
+плюс 1, умноженное на текущий выбранный размер блока NVMe-диска, больше 4 КБ,
+то новое хранилище может использовать атомарные записи для достижения лучшей
+производительности. Единственные известные диски, которые поддерживают это сейчас -
+[Micron и Kioxia](../intro/quickstart.ru.md).
+
+Атомарная запись позволяет не использовать двойную запись данных (в журнал и на
+устройство данных) в реплицированных пулах и таким образом снижает амплификацию
+записи (объём служебной записи на диск) и улучшает производительность записи
+вплоть до 2-х кратного прироста.
+
+## use_atomic_flag
+
+- Тип: булево (да/нет)
+
+Данная опция контролирует использование Vitastor OSD флага RWF_ATOMIC при атомарной записи
+блоков. Этот флаг поддерживается, начиная с версии ядра Linux 6.11 и добавляет немного корректности
+атомарным записям - ядро гарантирует отсутствие фрагментации запросов записи с этим флагом и
+проверяет их на соответствие реальным возможностям устройства.
+
+Однако, данная опция по умолчанию отключена, так как флаг в текущих версиях Linux работает
+абсолютно НЕКОРРЕКТНО - при нём Linux требует, чтобы запросы записи имели длину, равную
+степени двойки и были выровнены на эту длину. То есть, например, 12 КБ запросы записи, а также
+8 КБ запросы записи по не-кратному 8 КБ смещению запрещаются ядром, хотя спецификация NVMe их
+разрешает.
+
+Для NVMe-дисков с `scheduler=none` запросы записи и так не фрагментируются, так что это не так
+уж и важно, однако вы можете пересобрать своё ядро с [этим патчем](../../patches/linux-fix-atomic-write-checks.diff)
+и включить данную опцию. Это сделает вашу атомарную запись капельку безопаснее.
+
+## pg_reshard_chunk_size
+
+- Тип: целое число
+- Значение по умолчанию: 100000
+
+Изменение числа PG в пуле заметно загружает процессор, так как OSD хранят полную базу данных
+объектов в памяти и им приходится перемещать все записи объектов между старыми и новыми PG.
+Поэтому изменение применяется порциями, с паузами между порциями, чтобы не блокировать обработку
+событий OSD и операции остальных клиентов. Данная опция задаёт максимальное число объектов
+в порции. Перемещение 100 тысяч объектов (значение по умолчанию) обычно занимает порядка
+50-100 миллисекунд. Значение опции 0 отключает лимит размера порции.
+
+## pg_reshard_chunk_pause_ms
+
+- Тип: миллисекунды
+- Значение по умолчанию: 100
+
+Данная опция задаёт интервал между обработкой двух порций изменения числа PG пулов.

docs/config/src/layout-osd.yml

@@ -1,3 +1,28 @@
+- name: meta_format
+  type: int
+  default: 3
+  info: |
+    OSD store implementation version and on-disk metadata format.
+
+    Three versions are currently supported: 3, 2 and 1.
+    - 3 the new log-structured store, it's overall faster, has lower Write
+      Amplification, which may be even close to 1 (i.e. almost no extra writes)
+      if your SSDs support atomic writes (see [atomic_write_size](osd.en.md#atomic_write_size)).
+    - 2 is the old stable store from Vitastor 0.9-2.x.
+    - 1 is the same old store but with a legacy metadata format from Vitastor
+      versions to up 0.8.x, without any support for checksums.
+  info_ru: |
+    Версия реализации дискового хранилища OSD и дискового формата метаданных.
+
+    Поддерживаются три версии: 3, 2 и 1.
+    - 3 - новое лог-структурированное хранилище, в целом более быстрое, со
+      сниженным фактором амплификации записи, который может составлять около 1
+      (то есть, практически без лишней служебной записи), если ваши SSD
+      поддерживают атомарную запись (см. [atomic_write_size](osd.ru.md#atomic_write_size)).
+    - 2 - старое стабильное хранилище из версий Vitastor 0.9-2.x.
+    - 1 - то же самое стабильное хранилище, но с ещё более старым форматом
+      метаданных из версий Vitastor до 0.8.x, без какой-либо поддержки
+      контрольных сумм.
 - name: data_device
   type: string
   info: |

docs/config/src/network.yml

@@ -84,11 +84,6 @@
     unsupported with old libibverbs < v32, like in Debian 10 Buster or
     CentOS 7.
 
-    Vitastor supports all adapters, even ones without ODP support, like
-    Mellanox ConnectX-3 and non-Mellanox cards. Versions up to Vitastor
-    1.2.0 required ODP which is only present in Mellanox ConnectX >= 4.
-    See also [rdma_odp](#rdma_odp).
-
     Run `ibv_devinfo -v` as root to list available RDMA devices and their
     features.
 
@@ -97,6 +92,23 @@
     the manual of your network vendor for details about setting up the switch
     for RoCEv2 correctly. Usually it means setting up Lossless Ethernet with
     PFC (Priority Flow Control) and ECN (Explicit Congestion Notification).
+
+    Vitastor supports all adapters, even ones without ODP (On-Demand Paging)
+    support, like Mellanox ConnectX-3 and non-Mellanox cards. ODP is only present
+    in Mellanox ConnectX >= 4 adapters and allows to skip memory registration
+    for RDMA and thus, in theory, avoid memory copying.
+
+    Versions up to Vitastor 1.2.0 required ODP, then it was disabled by default,
+    but it was still supported up to 3.0.3. Now ODP support is removed because it
+    actually only hurts performance: an example 3-node cluster with 8 NVMe in each
+    node and 2*25 GBit/s ConnectX-6 RDMA network pushed 3950000 read iops without
+    ODP, but only 239000 iops with ODP.
+
+    This happens because Mellanox ODP implementation seems to be based on
+    message retransmissions when the adapter doesn't know about the buffer yet -
+    it likely uses standard "RNR retransmissions" (RNR = receiver not ready)
+    which is generally slow in RDMA/RoCE networks. Here's a presentation about
+    it from ISPASS-2021 conference: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
   info_ru: |
     Название RDMA-устройства для связи с Vitastor OSD (например, "rocep5s0f0").
     Если не указано, Vitastor попробует найти RoCE-устройство, соответствующее
@@ -105,12 +117,6 @@
     не задана. Также автовыбор не поддерживается со старыми версиями библиотеки
     libibverbs < v32, например в Debian 10 Buster или CentOS 7.
 
-    Vitastor поддерживает все модели адаптеров, включая те, у которых
-    нет поддержки ODP, то есть вы можете использовать RDMA с ConnectX-3 и
-    картами производства не Mellanox. Версии Vitastor до 1.2.0 включительно
-    требовали ODP, который есть только на Mellanox ConnectX 4 и более новых.
-    См. также [rdma_odp](#rdma_odp).
-
     Запустите `ibv_devinfo -v` от имени суперпользователя, чтобы посмотреть
     список доступных RDMA-устройств, их параметры и возможности.
 
@@ -120,6 +126,24 @@
     коммутатора для RoCEv2 ищите в документации производителя. Обычно это
     подразумевает настройку сети без потерь на основе PFC (Priority Flow
     Control) и ECN (Explicit Congestion Notification).
+
+    Vitastor поддерживает все модели адаптеров, включая те, у которых нет
+    поддержки ODP (On-Demand Paging), например, ConnectX-3 и карты производства
+    не Mellanox. Функция ODP доступна только на адаптерах Mellanox ConnectX-4 и
+    более новых и позволяет не регистрировать память для её использования RDMA-картой,
+    благодаря чему в теории можно избежать лишних копирований памяти.
+
+    Версии Vitastor до 1.2.0 включительно требовали ODP, потом функция был отключена
+    по умолчанию, но поддерживалась вплоть до версии 3.0.3. Сейчас поддержка ODP
+    полностью удалена, так как на самом деле она только портит производительность:
+    например, на 3-узловом кластере с 8 NVMe в каждом узле и сетью 2*25 Гбит/с на
+    чтение с RDMA без ODP удаётся снять 3950000 iops, а с ODP - всего 239000 iops.
+
+    Это происходит из-за того, что реализация ODP у Mellanox неоптимальная и
+    основана на повторной передаче сообщений, когда карте не известен буфер -
+    вероятно, на стандартных "RNR retransmission" (RNR = receiver not ready).
+    А данные повторные передачи в RDMA/RoCE - всегда очень медленная штука.
+    Презентация на эту тему с конференции ISPASS-2021: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
 - name: rdma_port_num
   type: int
   info: |
@@ -218,45 +242,6 @@
     у принимающей стороны в процессе работы не заканчивались буферы на приём.
     Не влияет на потребление памяти - дополнительная память на операции отправки
     не выделяется.
-- name: rdma_odp
-  type: bool
-  default: false
-  online: false
-  info: |
-    Use RDMA with On-Demand Paging. ODP is currently only available on Mellanox
-    ConnectX-4 and newer adapters. ODP allows to not register memory explicitly
-    for RDMA adapter to be able to use it. This, in turn, allows to skip memory
-    copying during sending. One would think this should improve performance, but
-    **in reality** RDMA performance with ODP is **drastically** worse. Example
-    3-node cluster with 8 NVMe in each node and 2*25 GBit/s ConnectX-6 RDMA network
-    without ODP pushes 3950000 read iops, but only 239000 iops with ODP...
-
-    This happens because Mellanox ODP implementation seems to be based on
-    message retransmissions when the adapter doesn't know about the buffer yet -
-    it likely uses standard "RNR retransmissions" (RNR = receiver not ready)
-    which is generally slow in RDMA/RoCE networks. Here's a presentation about
-    it from ISPASS-2021 conference: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
-
-    ODP support is retained in the code just in case a good ODP implementation
-    appears one day.
-  info_ru: |
-    Использовать RDMA с On-Demand Paging. ODP - функция, доступная пока что
-    исключительно на адаптерах Mellanox ConnectX-4 и более новых. ODP позволяет
-    не регистрировать память для её использования RDMA-картой. Благодаря этому
-    можно не копировать данные при отправке их в сеть и, казалось бы, это должно
-    улучшать производительность - но **по факту** получается так, что
-    производительность только ухудшается, причём сильно. Пример - на 3-узловом
-    кластере с 8 NVMe в каждом узле и сетью 2*25 Гбит/с на чтение с RDMA без ODP
-    удаётся снять 3950000 iops, а с ODP - всего 239000 iops...
-
-    Это происходит из-за того, что реализация ODP у Mellanox неоптимальная и
-    основана на повторной передаче сообщений, когда карте не известен буфер -
-    вероятно, на стандартных "RNR retransmission" (RNR = receiver not ready).
-    А данные повторные передачи в RDMA/RoCE - всегда очень медленная штука.
-    Презентация на эту тему с конференции ISPASS-2021: https://tkygtr6.github.io/pub/ISPASS21_slides.pdf
-
-    Возможность использования ODP сохранена в коде на случай, если вдруг в один
-    прекрасный день появится хорошая реализация ODP.
 - name: peer_connect_interval
   type: sec
   min: 1

docs/config/src/osd.yml

@@ -253,21 +253,33 @@
   type: bool
   default: true
   info: |
-    This parameter makes Vitastor always keep metadata area of the block device
-    in memory. It's required for good performance because it allows to avoid
-    additional read-modify-write cycles during metadata modifications. Metadata
-    area size is currently roughly 224 MB per 1 TB of data. You can turn it off
-    to reduce memory usage by this value, but it will hurt performance. This
-    restriction is likely to be removed in the future along with the upgrade
-    of the metadata storage scheme.
+    Only for the old store ([meta_format](layout-osd.en.md#meta_format) 2).
+
+    This parameter makes Vitastor keep a copy of metadata area in memory as it is
+    on disk, in addition to the metadata database. When the option is enabled, every
+    metadata entry is effectively stored in RAM twice. It's required for good performance
+    because it allows to avoid additional read-modify-write cycles during metadata
+    modifications. Metadata area size with the old store is roughly 224 MB per 1 TB
+    of data. You can turn the option off to reduce memory usage by this value, but
+    it will reduce performance.
+
+    For the new store ([meta_format](layout-osd.en.md#meta_format) 3), the option
+    may be changed in the future to support operation without loading full metadata
+    database in memory.
   info_ru: |
-    Данный параметр заставляет Vitastor всегда держать область метаданных диска
-    в памяти. Это нужно, чтобы избегать дополнительных операций чтения с диска
-    при записи. Размер области метаданных на данный момент составляет примерно
-    224 МБ на 1 ТБ данных. При включении потребление памяти снизится примерно
-    на эту величину, но при этом также снизится и производительность. В будущем,
-    после обновления схемы хранения метаданных, это ограничение, скорее всего,
-    будет ликвидировано.
+    Только для старого хранилища ([meta_format](layout-osd.en.md#meta_format) 2).
+
+    Данный параметр заставляет Vitastor всегда держать копию области метаданных
+    в памяти в том же виде, как она лежит на диске, в дополнение к БД метаданных.
+    То есть, с включённой опцией каждая запись метаданных хранится в памяти дважды.
+    Это нужно, чтобы избегать дополнительных операций чтения с диска при записи.
+    Размер области метаданных в старом хранилище составляет примерно 224 МБ на
+    1 ТБ данных. Вы можете отключить опцию, чтобы снизить потребление памяти
+    примерно на эту величину, но при этом также снизится и производительность.
+
+    Для нового хранилища ([meta_format](layout-osd.en.md#meta_format) 3) опция,
+    возможно, будет переработана в будущем для поддержки работы без полной
+    загрузки метаданных в памяти.
 - name: inmemory_journal
   type: bool
   default: true
@@ -386,11 +398,15 @@
     blocks. The only situation when you should increase it to a larger value
     is when you enable journal_no_same_sector_overwrites. In this case set
     it to, for example, 1024.
+
+    Not applicable to the new store ([meta_format](layout-osd.en.md#meta_format) 3).
   info_ru: |
     Максимальное число буферов, разрешённых для использования под записываемые
     в журнал блоки метаданных. Единственная ситуация, в которой этот параметр
     нужно менять - это если вы включаете journal_no_same_sector_overwrites. В
     этом случае установите данный параметр, например, в 1024.
+
+    Неприменимо к новому хранилищу ([meta_format](layout-osd.en.md#meta_format) 3).
 - name: journal_no_same_sector_overwrites
   type: bool
   default: false
@@ -402,6 +418,8 @@
     journal after writing it instead of possibly overwriting it the second time.
 
     Most (99%) other SSDs don't need this option.
+
+    Not applicable to the new store ([meta_format](layout-osd.en.md#meta_format) 3).
   info_ru: |
     Включайте данную опцию для SSD вроде Intel D3-S4510 и D3-S4610, которые
     ОЧЕНЬ не любят, когда ПО перезаписывает один и тот же сектор несколько раз
@@ -412,6 +430,20 @@
     самого сектора.
 
     Почти все другие SSD (99% моделей) не требуют данной опции.
+
+    Неприменимо к новому хранилищу ([meta_format](layout-osd.en.md#meta_format) 3).
+- name: skip_corrupted_meta_entries
+  type: bool
+  default: false
+  info: |
+    Only for the new store ([meta_format](layout-osd.en.md#meta_format) 3).
+    Allow OSD to start when some metadata entries or blocks are corrupted by
+    skipping them. Should be only used as an emergency measure.
+  info_ru: |
+    Только для нового хранилища ([meta_format](layout-osd.en.md#meta_format) 3).
+    Разрешить OSD запускаться, даже если часть блоков или записей метаданных
+    повреждена, пропуская их. Опция предназначена для использования только в
+    целях аварийного восстановления.
 - name: throttle_small_writes
   type: bool
   default: false
@@ -801,3 +833,108 @@
   default: 100
   info: Retry interval for failed PG lock attempts.
   info_ru: Интервал повтора неудачных попыток блокировки PG.
+- name: atomic_write_size
+  type: int
+  default: 4096
+  info: |
+    Maximum data device atomic write size allowed for OSD to use.
+
+    Atomic writes allow to reduce the Write Amplification factor with the new store
+    ([meta_format](layout-osd.en.md#meta_format)=3) to almost 1 (i.e. almost no extra writes)
+    with replicated pools and reach the best possible write performance.
+
+    Default value is auto-detected during OSD initialization from
+    `/sys/block/xx/queue/atomic_write_max_bytes` or assumed to be 4096 bytes
+    because all known disks support 4 KB atomic writes. Auto-detection is only used for
+    NVMe disks because SAS disks require the explicit WRITE ATOMIC command which requires
+    RWF_ATOMIC (see below [#use_atomic_flag]) but that flag works incorrectly in current
+    Linux versions.
+
+    You can also check if your NVMe drives support atomic writes by running
+    the command `nvme id-ctrl /dev/nvme0n1 | grep awupf`. If the reported value,
+    plus 1, multiplied by the currently selected block size of the NVMe,
+    is more than 4 KB, then the new store can utilize it for better performance.
+    The only drives known to support it currently are [Micron and Kioxia](../intro/quickstart.en.md).
+
+    Atomic writes allow to skip double data writes in replicated pools, thus
+    reducing Write Amplification and improving write performance up to 2 times.
+  info_ru: |
+    Максимальный размер атомарной записи на диск данных, который OSD разрешено использовать.
+
+    Поддержка атомарной записи позволяет снизить мультипликатор записи (Write Amplification)
+    на диск с новым хранилищем ([meta_format](layout-osd.ru.md#meta_format)=3)
+    практически до 1 (то есть, почти до нулевого объёма лишней записи) в реплицированных
+    пулах и достигнуть наилучшей возможной производительности записи.
+
+    Значение по умолчанию авто-определяется во время инициализации OSD из
+    `/sys/block/xx/queue/atomic_write_max_bytes` либо принимается равным 4096,
+    так как все известные диски поддерживают атомарную запись 4 КБ блоков.
+    Автоопределение применяется только для NVMe-дисков, так как SAS диски требуют
+    использования отдельной команды WRITE ATOMIC, а для неё нужен флаг RWF_ATOMIC
+    (см. ниже [#use_atomic_flag]), а он в текущих версиях Linux работает некорректно.
+
+    Вы также можете проверить, поддерживают ли ваши NVMe-диски атомарную запись,
+    с помощью команды `nvme id-ctrl /dev/nvme0n1 | grep awupf`. Если значение awupf
+    плюс 1, умноженное на текущий выбранный размер блока NVMe-диска, больше 4 КБ,
+    то новое хранилище может использовать атомарные записи для достижения лучшей
+    производительности. Единственные известные диски, которые поддерживают это сейчас -
+    [Micron и Kioxia](../intro/quickstart.ru.md).
+
+    Атомарная запись позволяет не использовать двойную запись данных (в журнал и на
+    устройство данных) в реплицированных пулах и таким образом снижает амплификацию
+    записи (объём служебной записи на диск) и улучшает производительность записи
+    вплоть до 2-х кратного прироста.
+- name: use_atomic_flag
+  type: bool
+  info: |
+    This option controls whether Vitastor OSDs use RWF_ATOMIC write flag with atomic writes.
+    This flag is supported since Linux 6.11 and adds some safety to atomic writes - the kernel
+    guarantees to not fragment write requests with it and also to check them against the actual
+    device atomic write capabilities.
+
+    However, the option is disabled by default because the flag is currently UNUSABLE - Linux
+    incorrectly requires writes with that flag to be of power-of-2 length and length-aligned.
+    I.e., for example, 12 KB writes and not-8-KB aligned 8 KB writes are forbidden by the kernel,
+    even though the NVMe specification allows them.
+
+    For NVMe disks with `scheduler=none` writes aren't fragmented anyway so it's not a big deal.
+    However, you can rebuild your kernel with [this patch](../../patches/linux-fix-atomic-write-checks.diff)
+    and turn this option on. It will make your atomic writes a bit safer.
+  info_ru: |
+    Данная опция контролирует использование Vitastor OSD флага RWF_ATOMIC при атомарной записи
+    блоков. Этот флаг поддерживается, начиная с версии ядра Linux 6.11 и добавляет немного корректности
+    атомарным записям - ядро гарантирует отсутствие фрагментации запросов записи с этим флагом и
+    проверяет их на соответствие реальным возможностям устройства.
+
+    Однако, данная опция по умолчанию отключена, так как флаг в текущих версиях Linux работает
+    абсолютно НЕКОРРЕКТНО - при нём Linux требует, чтобы запросы записи имели длину, равную
+    степени двойки и были выровнены на эту длину. То есть, например, 12 КБ запросы записи, а также
+    8 КБ запросы записи по не-кратному 8 КБ смещению запрещаются ядром, хотя спецификация NVMe их
+    разрешает.
+
+    Для NVMe-дисков с `scheduler=none` запросы записи и так не фрагментируются, так что это не так
+    уж и важно, однако вы можете пересобрать своё ядро с [этим патчем](../../patches/linux-fix-atomic-write-checks.diff)
+    и включить данную опцию. Это сделает вашу атомарную запись капельку безопаснее.
+- name: pg_reshard_chunk_size
+  type: int
+  default: 100000
+  info: |
+    Pool PG count change is a CPU-intensive operation because OSDs store the full object database
+    in memory and have to move all entries between old and new PGs. Thus it's performed in chunks,
+    with pauses between chunks to prevent blocking OSD's event loop and other clients' operations.
+    This option sets the maximum number of object is a chunk. Moving 100k objects usually takes
+    50-100ms. Chunk size equal to 0 means unlimited.
+  info_ru: |
+    Изменение числа PG в пуле заметно загружает процессор, так как OSD хранят полную базу данных
+    объектов в памяти и им приходится перемещать все записи объектов между старыми и новыми PG.
+    Поэтому изменение применяется порциями, с паузами между порциями, чтобы не блокировать обработку
+    событий OSD и операции остальных клиентов. Данная опция задаёт максимальное число объектов
+    в порции. Перемещение 100 тысяч объектов (значение по умолчанию) обычно занимает порядка
+    50-100 миллисекунд. Значение опции 0 отключает лимит размера порции.
+- name: pg_reshard_chunk_pause_ms
+  type: ms
+  default: 100
+  info: |
+    This option sets the interval between handling two PG count change chunks.
+  info_ru: |
+    Данная опция задаёт интервал между обработкой двух порций изменения числа PG пулов.

docs/installation/docker.en.md

@@ -26,9 +26,9 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.
 
 1. Download a Docker image of the desired version: \
-   `docker pull vitalif/vitastor:v2.4.3`
+   `docker pull vitalif/vitastor:v3.0.4`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.4.3 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.4 install.sh`
 3. Reload udev rules: \
    `udevadm control --reload-rules`
 

docs/installation/docker.ru.md

@@ -25,9 +25,9 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.
 
 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitalif/vitastor:v2.4.3`
+   `docker pull vitalif/vitastor:v3.0.4`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.4.3 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.4 install.sh`
 3. Перезагрузите правила udev: \
    `udevadm control --reload-rules`
 

docs/installation/packages.en.md

@@ -33,15 +33,17 @@
   - CentOS 7: `yum install https://vitastor.io/rpms/centos/7/vitastor-release.rpm`
   - CentOS 8: `dnf install https://vitastor.io/rpms/centos/8/vitastor-release.rpm`
   - AlmaLinux 9 and other RHEL 9 clones (Rocky, Oracle...): `dnf install https://vitastor.io/rpms/centos/9/vitastor-release.rpm`
+  - AlmaLinux 10 and other RHEL 10 clones: `dnf install https://vitastor.io/rpms/centos/10/vitastor-release.rpm`
 - Enable EPEL: `yum/dnf install epel-release`
 - Enable additional CentOS repositories:
   - CentOS 7: `yum install centos-release-scl`
   - CentOS 8: `dnf install centos-release-advanced-virtualization`
-  - RHEL 9 clones: not required
+  - RHEL 9/10 clones: not required
 - Enable elrepo-kernel:
   - CentOS 7: `yum install https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm`
   - CentOS 8: `dnf install https://www.elrepo.org/elrepo-release-8.el8.elrepo.noarch.rpm`
   - RHEL 9 clones: `dnf install https://www.elrepo.org/elrepo-release-9.el9.elrepo.noarch.rpm`
+  - RHEL 10 clones: not required
 - Install packages: `yum/dnf install vitastor lpsolve etcd kernel-ml qemu-kvm`
 
 ## Installation requirements

docs/installation/packages.ru.md

@@ -33,15 +33,17 @@
   - CentOS 7: `yum install https://vitastor.io/rpms/centos/7/vitastor-release.rpm`
   - CentOS 8: `dnf install https://vitastor.io/rpms/centos/8/vitastor-release.rpm`
   - AlmaLinux 9 и другие клоны RHEL 9 (Rocky, Oracle...): `dnf install https://vitastor.io/rpms/centos/9/vitastor-release.rpm`
+  - AlmaLinux 10 и другие клоны RHEL 10: `dnf install https://vitastor.io/rpms/centos/10/vitastor-release.rpm`
 - Включите EPEL: `yum/dnf install epel-release`
 - Включите дополнительные репозитории CentOS:
   - CentOS 7: `yum install centos-release-scl`
   - CentOS 8: `dnf install centos-release-advanced-virtualization`
-  - Клоны RHEL 9: не нужно
+  - Клоны RHEL 9/10: не нужно
 - Включите elrepo-kernel:
   - CentOS 7: `yum install https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm`
   - CentOS 8: `dnf install https://www.elrepo.org/elrepo-release-8.el8.elrepo.noarch.rpm`
   - Клоны RHEL 9: `dnf install https://www.elrepo.org/elrepo-release-9.el9.elrepo.noarch.rpm`
+  - Клоны RHEL 10: не нужно
 - Установите пакеты: `yum/dnf install vitastor lpsolve etcd kernel-ml qemu-kvm`
 
 ## Установочные требования

docs/installation/proxmox.en.md

@@ -6,7 +6,7 @@
 
 # Proxmox VE
 
-To enable Vitastor support in Proxmox Virtual Environment (6.4-8.x are supported):
+To enable Vitastor support in Proxmox Virtual Environment (6.4-9.x are supported):
 
 - Add the corresponding Vitastor Debian repository into sources.list on Proxmox hosts:
   trixie for 9.0+, bookworm for 8.1+, pve8.0 for 8.0, bullseye for 7.4, pve7.3 for 7.3, pve7.2 for 7.2, pve7.1 for 7.1, buster for 6.4

docs/installation/proxmox.ru.md

@@ -6,7 +6,7 @@
 
 # Proxmox VE
 
-Чтобы подключить Vitastor к Proxmox Virtual Environment (поддерживаются версии 6.4-8.x):
+Чтобы подключить Vitastor к Proxmox Virtual Environment (поддерживаются версии 6.4-9.x):
 
 - Добавьте соответствующий Debian-репозиторий Vitastor в sources.list на хостах Proxmox:
   trixie для 9.0+, bookworm для 8.1+, pve8.0 для 8.0, bullseye для 7.4, pve7.3 для 7.3, pve7.2 для 7.2, pve7.1 для 7.1, buster для 6.4

docs/intro/features.en.md

@@ -14,6 +14,8 @@
 
 - Basic part: highly-available block storage with symmetric clustering and no SPOF
 - [Performance](../performance/bench2.en.md) ;-D
+  - [NVMe atomic write support](../config/osd.en.md#atomic_write_size) for reducing the amount
+    of "extra" disk writes to almost zero (Write Amplification = 1)
 - [Multiple redundancy schemes](../config/pool.en.md#scheme): Replication, XOR n+1, Reed-Solomon erasure codes
   based on jerasure and ISA-L libraries with any number of data and parity drives in a group
 - Configuration via simple JSON data structures in etcd (parameters, pools and images)

docs/intro/features.ru.md

@@ -14,6 +14,8 @@
 
 - Базовая часть - надёжное кластерное блочное хранилище без единой точки отказа
 - [Производительность](../performance/bench2.ru.md) ;-D
+  - [Поддержка атомарной записи NVMe](../config/osd.ru.md#atomic_write_size) для снижения объёма
+    служебной записи практически до нуля (Write Amplification = 1)
 - [Несколько схем отказоустойчивости](../config/pool.ru.md#scheme): репликация, XOR n+1 (1 диск чётности), коды коррекции ошибок
   Рида-Соломона на основе библиотек jerasure и ISA-L с любым числом дисков данных и чётности в группе
 - Конфигурация через простые человекочитаемые JSON-структуры в etcd

docs/intro/quickstart.en.md

@@ -18,9 +18,10 @@
 
 ## Preparation
 
-- Get some SATA or NVMe SSDs with capacitors (server-grade drives). You can use desktop SSDs
-  with lazy fsync, but prepare for inferior single-thread latency. Read more about capacitors
-  [here](../config/layout-cluster.en.md#immediate_commit).
+- Get some SATA or NVMe SSDs with capacitors (server-grade drives). The best performance
+  is achieved with Micron or Kioxia NVMes with atomic write support (see below). You can use desktop
+  SSDs with lazy fsync, but prepare for inferior single-thread latency. Read more about
+  capacitors [here](../config/layout-cluster.en.md#immediate_commit).
 - If you want to use HDDs, get modern HDDs with Media Cache or SSD Cache: HGST Ultrastar,
   Toshiba MG, Seagate EXOS or something similar. If your drives don't have such cache then
   you also need small SSDs for journal and metadata (even 2 GB per 1 TB of HDD space is enough).
@@ -30,9 +31,11 @@
 
 ## Recommended drives
 
-- SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel D3-S4510/4520/4610/4620, Kingston DC500M
-- NVMe: Micron 9100/9200/9300/9400, Micron 7300/7450, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
-  Intel DC-P3700/P4500/P4600, Intel D5-P4320/P5530, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
+- NVMe with atomic write support (ideal!): Micron 7450/7500/7550, Kioxia CD6/CD7/CD8/CD9
+- Other NVMe: Micron 9100/9200/9300/9400/9550, Micron 7300, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
+  Intel DC-P3700/P4500/P4600, Intel/Solidigm D5-P4320/P5530, Intel/Solidigm D7-P5500/P5600, Solidigm D7-PS1010/PS1030/P5810,
+  Intel Optane, Kingston DC1000B/DC1500M, Kioxia CD6/CD7/CD8/CD9
+- SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel/Solidigm D3-S4510/4520/4610/4620, Kingston DC500M
 - HDD: HGST Ultrastar, Toshiba MG, Seagate EXOS
 
 ## Configure monitors

docs/intro/quickstart.ru.md

@@ -18,8 +18,9 @@
 
 ## Подготовка
 
-- Возьмите серверы с SSD (SATA или NVMe), желательно с конденсаторами (серверные SSD). Можно
-  использовать и десктопные SSD, включив режим отложенного fsync, но производительность будет хуже.
+- Возьмите серверы с SSD (SATA или NVMe), желательно с конденсаторами (серверные SSD). Наилучшая
+  производительность достигается на дисках Micron и Kioxia с поддержкой атомарной записи (см. ниже).
+  Можно использовать и десктопные SSD, включив режим отложенного fsync, но производительность будет хуже.
   О конденсаторах читайте [здесь](../config/layout-cluster.ru.md#immediate_commit).
 - Если хотите использовать HDD, берите современные модели с Media или SSD кэшем - HGST Ultrastar,
   Toshiba MG, Seagate EXOS или что-то похожее. Если такого кэша у ваших дисков нет,
@@ -30,9 +31,11 @@
 
 ## Рекомендуемые диски
 
-- SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel D3-S4510/4520/4610/4620, Kingston DC500M
-- NVMe: Micron 9100/9200/9300/9400, Micron 7300/7450, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
-  Intel DC-P3700/P4500/P4600, Intel D5-P4320/P5530, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
+- NVMe с поддержкой атомарной записи (идеально!): Micron 7450/7500/7550, Kioxia CD6/CD7/CD8/CD9
+- Другие NVMe: Micron 9100/9200/9300/9400/9550, Micron 7300, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
+  Intel DC-P3700/P4500/P4600, Intel/Solidigm D5-P4320/P5530, Intel/Solidigm D7-P5500/P5600, Solidigm D7-PS1010/PS1030/P5810,
+  Intel Optane, Kingston DC1000B/DC1500M, Kioxia CD6/CD7/CD8/CD9
+- SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel/Solidigm D3-S4510/4520/4610/4620, Kingston DC500M
 - HDD: HGST Ultrastar, Toshiba MG, Seagate EXOS
 
 ## Настройте мониторы

docs/performance/theoretical.en.md

@@ -14,7 +14,7 @@ Replicated setups:
 - Linear read: `min(total network bandwidth, sum(disk read MB/s))`.
 - Linear write: `min(total network bandwidth, sum(disk write MB/s / number of replicas))`.
 - Saturated parallel read iops: `min(total network bandwidth, sum(disk read iops))`.
-- Saturated parallel write iops: `min(total network bandwidth / number of replicas, sum(disk write iops / number of replicas / (write amplification = 4)))`.
+- Saturated parallel write iops: `min(total network bandwidth / number of replicas, sum(disk write iops / number of replicas / write amplification))`.
 
 EC/XOR setups (EC N+K):
 - Single-threaded (T1Q1) read latency: 1.5 network roundtrips + 1 disk read.
@@ -26,28 +26,36 @@ EC/XOR setups (EC N+K):
 - Linear read: `min(total network bandwidth, sum(disk read MB/s))`.
 - Linear write: `min(total network bandwidth, sum(disk write MB/s * N/(N+K)))`.
 - Saturated parallel read iops: `min(total network bandwidth, sum(disk read iops))`.
-- Saturated parallel write iops: roughly `total iops / (N+K) / WA`. More exactly,
-  `min(total network bandwidth * N/(N+K), sum(disk randrw iops / (N*4 + K*5 + 1)))` with
-  random read/write mix corresponding to `(N-1)/(N*4 + K*5 + 1)*100 % reads`.
-  - For example, with EC 2+1 it is: `(7% randrw iops) / 14`.
-  - With EC 6+3 it is: `(12.5% randrw iops) / 40`.
+- Saturated parallel write iops: roughly `total iops / (N+K) / WA`. More exactly:
+  - With the new store: `min(total network bandwidth * N/(N+K), sum(disk randrw iops / (2 + N-1 + K*2)))`,
+    with random read/write mix corresponding to `(N-1)/(2 + N-1 + K*2)*100 % reads`.
+    - For example, with EC 2+1 it is: `(20% randrw iops) / 5`.
+    - With EC 6+3 it is: `(38% randrw iops) / 13`.
+  - With the old store: `min(total network bandwidth * N/(N+K), sum(disk randrw iops / (3 + N-1 + K*3)))`,
+    with random read/write mix corresponding to `(N-1)/(3 + N-1 + K*3)*100 % reads`.
+    - For example, with EC 2+1 it is: `(14% randrw iops) / 7`.
+    - With EC 6+3 it is: `(30% randrw iops) / 17`.
 
-Write amplification for 4 KB blocks is usually 3-5 in Vitastor:
-1. Journal block write
-2. Journal data write
-3. Metadata block write
-4. Another journal block write for EC/XOR setups
-5. Data block write
+Write Amplification factor:
+- For the new store and for 4 KB writes: WA is always 1 unless you set [atomic_write_size](../config/osd.en.md#atomic_write_size) to 0 manually.
+- For the new store and for 8-124 KB writes: WA is 1 if you use NVMe drives with atomic write support, or roughly 2 if you use other drives.
+- For the old store, WA is roughly `(2 * write size + 4 KB) / (write size)`. So, for 4 KB writes it's 3, and for 8-124 KB writes it's closer to 2.
+- For both the new and the old store and for writes of [block_size](../config/layout-cluster.en.md#block_size): WA is almost 1.
 
-If you manage to get an SSD which handles 512 byte blocks well (Optane?) you may
-lower 1, 3 and 4 to 512 bytes (1/8 of data size) and get WA as low as 2.375.
+Write Amplification consists of:
+- For the new store:
+  - Buffer block write if non-atomic
+  - Data block write
+  - Metadata write(s) (amortized)
+- For the old store:
+  - Journal block write (amortized)
+  - Journal data write
+  - Metadata block write
+  - Another journal block write for EC/XOR setups (amortized)
+  - Data block write
 
-Implemented NVDIMM support can basically eliminate WA at all - all extra writes will
-go to DRAM memory. But this requires a test cluster with NVDIMM - please contact me
-if you want to provide me with such cluster for tests.
-
-Lazy fsync also reduces WA for parallel workloads because journal blocks are only
-written when they fill up or fsync is requested.
+Other possibilities to reduce WA would be to use SSDs with internal 512-byte blocks
+or NVDIMM, but both options seem unavailable on the market at the moment.
 
 ## In Practice
 

docs/performance/theoretical.ru.md

@@ -27,29 +27,36 @@
 - Линейное чтение: сумма МБ/с чтения всех дисков, либо общая производительность сети, если в сеть упрётся раньше.
 - Линейная запись: сумма МБ/с записи всех дисков * N/(N+K), либо производительность сети * N / (N+K), если в сеть упрётся раньше.
 - Параллельное случайное мелкое чтение: сумма IOPS чтения всех дисков либо производительность сети, если в сеть упрётся раньше.
-- Параллельная случайная мелкая запись: грубо `(сумма IOPS / (N+K) / WA)`. Если точнее, то:
-  сумма смешанного IOPS всех дисков при `(N-1)/(N*4 + K*5 + 1)*100 %` чтения, делённая на `(N*4 + K*5 + 1)`.
-  Либо, производительность сети * N/(N+K), если в сеть упрётся раньше.
-  - Например, при EC 2+1 это: `(сумма IOPS при 7% чтения) / 14`.
-  - При EC 6+3 это: `(сумма IOPS при 12.5% чтения) / 40`.
+- Параллельная случайная мелкая запись: грубо `(сумма IOPS / (N+K) / WA)`.
+  Либо `производительность сети * N/(N+K)`, если в сеть упрётся раньше. Если точнее, то:
+  - С новым хранилищем: сумма смешанного IOPS всех дисков при `(N-1)/(2 + N-1 + K*2)*100 %` чтения, делённая на `(2 + N-1 + K*2)`.
+    - Например, при EC 2+1 это: `(сумма IOPS при 20% чтения) / 5`.
+    - При EC 6+3 это: `(сумма IOPS при 38% чтения) / 13`.
+  - Со старым хранилищем: сумма смешанного IOPS всех дисков при `(N-1)/(3 + N-1 + K*3)*100 %` чтения, делённая на `(3 + N-1 + K*3)`.
+    - Например, при EC 2+1 это: `(сумма IOPS при 14% чтения) / 7`.
+    - При EC 6+3 это: `(сумма IOPS при 30% чтения) / 17`.
 
-WA (мультипликатор записи) для 4 КБ блоков в Vitastor обычно составляет 3-5:
-1. Запись метаданных в журнал
-2. Запись блока данных в журнал
-3. Запись метаданных в БД
-4. Ещё одна запись метаданных в журнал при использовании EC
-5. Запись блока данных на диск данных
+WA (Write Amplification, мультипликатор записи):
+- С новым хранилищем для 4 КБ записи: WA всегда примерно 1, если только вы не установите [atomic_write_size](../config/osd.ru.md#atomic_write_size) вручную в 0.
+- С новым хранилищем и большими записями (8-124 КБ): WA примерно 1, если вы используете NVMe-диски с поддержкой атомарной записи,
+  или примерно 2, если вы используете другие диски.
+- Со старым хранилищем, WA примерно `(2 * размер записи + 4 КБ) / (размер записи)`. То есть, для 4 КБ записи WA=3, а для 8-124 КБ WA ближе к 2.
+- И с новым, и со старым хранилищем и для записи размером [block_size](../config/layout-cluster.ru.md#block_size): WA примерно равен 1.
 
-Если вы найдёте SSD, хорошо работающий с 512-байтными блоками данных (Optane?),
-то 1, 3 и 4 можно снизить до 512 байт (1/8 от размера данных) и получить WA всего 2.375.
+Мультипликатор записи состоит из:
+- С новым хранилищем:
+  - Запись блока буфера, если диски без поддержки атомарной записи
+  - Запись блока данных
+  - Запись(-и) блоков метаданных (амортизированные)
+- Со старым хранилищем:
+  - Запись блока журнала (амортизированная)
+  - Запись данных в журнал
+  - Запись блока метаданных
+  - Ещё одна запись блока журнала для EC/XOR пулов (амортизированная)
+  - Запись блока данных
 
-Если реализовать поддержку NVDIMM, то WA можно, условно говоря, ликвидировать вообще - все
-дополнительные операции записи смогут обслуживаться DRAM памятью. Но для этого необходим
-тестовый кластер с NVDIMM - пишите, если готовы предоставить такой для тестов.
-
-Кроме того, WA снижается при использовании отложенного/ленивого сброса при параллельной
-нагрузке, т.к. блоки журнала записываются на диск только когда они заполняются или явным
-образом запрашивается fsync.
+Другими потенциальными возможностями снижения WA могли бы быть SSD с внутренним 512-байтным блоком
+либо NVDIMM, но и то, и другое сейчас выглядит недоступным на рынке.
 
 ## На практике
 

docs/usage/admin.en.md

@@ -231,6 +231,18 @@ Upgrading from <= 0.5.x to >= 0.6.x is not supported.
 
 Downgrade are also allowed freely, except the following specific instructions:
 
+### 3.x -> 2.x
+
+Versions 3.0.0 and newer contain two store implementations - an old one and a new
+one, unsupported in 2.x and previous versions. So you should check your OSD store
+versions before downgrading to 2.x with the following command:
+
+`vitastor-disk read-sb /dev/vitastor/osdXX-data | jq -r .meta_format`
+
+If it prints 3 then OSD uses the new store and you can't downgrade it to 2.x.
+
+If it prints 2 or nothing then OSD uses the old store and the downgrade is allowed.
+
 ### 1.8.0 to 1.7.1
 
 Before downgrading from version >= 1.8.0 to version <= 1.7.1

docs/usage/admin.ru.md

@@ -228,6 +228,18 @@ done
 
 Откат (понижение версии) тоже свободно разрешён, кроме указанных ниже случаев:
 
+### 3.x -> 2.x
+
+Версии 3.0.0 и более новые содержат две реализации хранилища - старую и новую, не
+поддерживаемую в 2.x и предыдущих версиях. Таким образом, перед откатом на 2.x вам
+следует проверить, какая версия хранилища используется вашими OSD - командой:
+
+`vitastor-disk read-sb /dev/vitastor/osdXX-data | jq -r .meta_format`
+
+Если выводится 3, это новое хранилище и откатить такой OSD до 2.x нельзя.
+
+Если выводится 2 или не выводится ничего, это старое хранилище и откат разрешён.
+
 ### 1.8.0 -> 1.7.1
 
 Перед понижением версии с >= 1.8.0 до <= 1.7.1 вы должны скопировать ключ

docs/usage/disk.en.md

@@ -51,6 +51,9 @@ Options (automatic mode):
 ```
 --osd_per_disk <N>
   Create <N> OSDs on each disk (default 1)
+--meta_format 3
+  Metadata store version. 3 is the new log-structured store, 2 is the stable store
+  from Vitastor 0.9-2.x, 1 is the legacy store from Vitastor 0.6-0.8.
 --hybrid
   Prepare hybrid (HDD+SSD, NVMe+SATA or etc) OSDs using provided devices. By default,
   any passed SSDs will be used for journals and metadata, HDDs will be used for data,
@@ -92,6 +95,8 @@ Options (single-device mode):
 Options (both modes):
 
 ```
+--tags tag1,tag2           Set new OSD tag(s)
+--weight <number>          Set new OSD weight (between 0 to 1)
 --journal_size 1G/32M      Set journal size (area or partition size)
 --block_size 1M/128k       Set blockstore object size
 --bitmap_granularity 4k    Set bitmap granularity

docs/usage/disk.ru.md

@@ -50,6 +50,9 @@ vitastor-disk - инструмент командной строки для уп
 ```
 --osd_per_disk <N>
   Создавать по несколько (<N>) OSD на каждом диске (по умолчанию 1)
+--meta_format 3
+  Версия хранилища метаданных. 3 - новое лог-структурированное хранилище,
+  2 - стабильное хранилище из Vitastor 0.9-2.x, 1 - старое хранилище из Vitastor 0.6-0.8.
 --hybrid
   Инициализировать гибридные (HDD+SSD, NVMe+SATA и т.п.) OSD на указанных дисках.
   По умолчанию, SSD будут использованы для журналов и метаданных, а HDD - для данных,
@@ -93,6 +96,8 @@ vitastor-disk - инструмент командной строки для уп
 Опции для обоих режимов:
 
 ```
+--tags tag1,tag2           Задать теги для новых OSD
+--weight <number>          Задать вес для новых OSD (от 0 до 1)
 --journal_size 1G/32M      Задать размер журнала (области или раздела журнала)
 --block_size 1M/128k       Задать размер объекта хранилища
 --bitmap_granularity 4k    Задать гранулярность битовых карт

mon/lp_optimizer/lp_optimizer.js

@@ -10,16 +10,19 @@ const NO_OSD = 'Z';
 async function lp_solve(text)
 {
     const cp = child_process.spawn('lp_solve');
-    let stdout = '', stderr = '', finish_cb;
+    let stdout = '', stderr = '', finish_cb, finished = 0;
     cp.stdout.on('data', buf => stdout += buf.toString());
     cp.stderr.on('data', buf => stderr += buf.toString());
-    cp.on('exit', () => finish_cb && finish_cb());
+    cp.stdout.on('end', () => finish_cb());
+    cp.stderr.on('end', () => finish_cb());
     cp.stdin.write(text);
     cp.stdin.end();
-    if (cp.exitCode == null)
+    await new Promise(ok => (finish_cb = () =>
     {
-        await new Promise(ok => finish_cb = ok);
-    }
+        finished++;
+        if (finished == 2)
+            ok();
+    }));
     if (!stdout.trim())
     {
         return null;

mon/osd_tree.js

@@ -87,7 +87,7 @@ function make_hier_tree(global_config, tree)
     tree[''] = { children: [] };
     for (const node_id in tree)
     {
-        if (node_id === '' || !(tree[node_id].children||[]).length && (tree[node_id].size||0) <= 0)
+        if (node_id === '')
         {
             continue;
         }

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "2.4.3",
+  "version": "3.0.4",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {
@@ -9,7 +9,7 @@
   "author": "Vitaliy Filippov",
   "license": "UNLICENSED",
   "dependencies": {
-    "antietcd": "^1.1.3",
+    "antietcd": "^1.2.4",
     "sprintf-js": "^1.1.2",
     "ws": "^7.2.5"
   },

mon/pg_gen.js

@@ -9,7 +9,6 @@ const LPOptimizer = require('./lp_optimizer/lp_optimizer.js');
 const { scale_pg_count } = require('./pg_utils.js');
 const { make_hier_tree, filter_osds_by_root_node,
     filter_osds_by_tags, filter_osds_by_block_layout, get_affinity_osds } = require('./osd_tree.js');
-const { select_murmur3 } = require('./lp_optimizer/murmur3.js');
 
 function pick_primary(pool_id, pg_num, pool_config, osd_set, up_osds, aff_osds)
 {
@@ -39,7 +38,7 @@ function pick_primary(pool_id, pg_num, pool_config, osd_set, up_osds, aff_osds)
     {
         return 0;
     }
-    return alive_set[select_murmur3(alive_set.length, osd_num => pool_id+'/'+pg_num+'/'+osd_num)];
+    return alive_set[pg_num % alive_set.length];
 }
 
 function recheck_primary(state, global_config, up_osds, osd_tree)
@@ -53,6 +52,7 @@ function recheck_primary(state, global_config, up_osds, osd_tree)
             continue;
         }
         const aff_osds = get_affinity_osds(pool_cfg, up_osds, osd_tree);
+        let paused = false;
         for (let pg_num = 1; pg_num <= pool_cfg.pg_count; pg_num++)
         {
             if (!state.pg.config.items[pool_id])
@@ -75,6 +75,19 @@ function recheck_primary(state, global_config, up_osds, osd_tree)
                     );
                     new_pg_config.items[pool_id][pg_num].primary = new_primary;
                 }
+                paused = paused || !!pg_cfg.pause;
+            }
+        }
+        if (paused)
+        {
+            if (!new_pg_config)
+            {
+                new_pg_config = JSON.parse(JSON.stringify(state.pg.config));
+            }
+            console.log(`Resuming paused pool ${pool_id}`);
+            for (const pg in new_pg_config.items[pool_id])
+            {
+                delete new_pg_config.items[pool_id][pg].pause;
             }
         }
     }
@@ -179,10 +192,10 @@ async function generate_pool_pgs(state, global_config, pool_id, osd_tree, levels
     const rules = use_rules ? get_pg_rules(pool_id, pool_cfg, global_config.placement_levels) : null;
     const folded = fold_failure_domains(Object.values(pool_tree), use_rules ? rules : [ [ [ pool_cfg.failure_domain ] ] ]);
     // FIXME: Remove/merge make_hier_tree() step somewhere, however it's needed to remove empty nodes
-    const folded_tree = make_hier_tree(global_config, folded.nodes);
+    const folded_tree = make_hier_tree(global_config, folded.nodes.reduce((a, c) => { a[c.id] = c; return a; }, {}));
     const old_pg_count = prev_pgs.length;
     const optimize_cfg = {
-        osd_weights: folded.nodes.reduce((a, c) => { if (Number(c.id)) { a[c.id] = c.size; } return a; }, {}),
+        osd_weights: folded.nodes.reduce((a, c) => { if (/^\d+$/.exec(c.id) && c.size != null) { a[c.id] = c.size||0; } return a; }, {}),
         combinator: use_rules
             // new algorithm:
             ? new RuleCombinator(folded_tree, rules, pool_cfg.max_osd_combinations)

node-binding/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor",
-  "version": "2.4.3",
+  "version": "3.0.4",
   "description": "Low-level native bindings to Vitastor client library",
   "main": "index.js",
   "keywords": [

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '2.4.3'
+VITASTOR_VERSION = '3.0.4'
 
 LOG = logging.getLogger(__name__)
 

patches/linux-fix-atomic-write-checks.diff

@@ -0,0 +1,39 @@
+From 98d3f68a40130c438854f61db6025f9e9b099cb6 Mon Sep 17 00:00:00 2001
+From: Vitaliy Filippov <vitalifster@gmail.com>
+Date: Sat, 20 Dec 2025 14:44:35 +0300
+Subject: [PATCH] Do not require atomic writes to be power of 2 sized and
+ aligned on length boundary
+
+It contradicts NVMe specification where alignment is only required when atomic
+write boundary (NABSPF/NABO) is set and highly limits usage of NVMe atomic writes
+
+Signed-off-by: Vitaliy Filippov <vitalifster@gmail.com>
+---
+ fs/read_write.c | 8 --------
+ 1 file changed, 8 deletions(-)
+
+diff --git a/fs/read_write.c b/fs/read_write.c
+index 833bae068770..5467d710108d 100644
+--- a/fs/read_write.c
++++ b/fs/read_write.c
+@@ -1802,17 +1802,9 @@ int generic_file_rw_checks(struct file *file_in, struct file *file_out)
+ 
+ int generic_atomic_write_valid(struct kiocb *iocb, struct iov_iter *iter)
+ {
+-	size_t len = iov_iter_count(iter);
+-
+ 	if (!iter_is_ubuf(iter))
+ 		return -EINVAL;
+ 
+-	if (!is_power_of_2(len))
+-		return -EINVAL;
+-
+-	if (!IS_ALIGNED(iocb->ki_pos, len))
+-		return -EINVAL;
+-
+ 	if (!(iocb->ki_flags & IOCB_DIRECT))
+ 		return -EOPNOTSUPP;
+ 
+-- 
+2.51.0
+

rpm/build-tarball.sh

@@ -21,7 +21,7 @@ rpmbuild -bp fio.spec
 cd $VITASTOR
 VER=$(grep ^Version: rpm/vitastor-$REL.spec | awk '{print $2}')
 rm -rf fio
-ln -s ~/rpmbuild/BUILD/fio*/ fio
+ln -s $(ls -d ~/rpmbuild/BUILD/fio*/ | grep -v SPECPARTS) fio
 sh copy-fio-includes.sh
 rm fio
 mv fio-copy fio

rpm/vitastor-el10.Dockerfile

@@ -0,0 +1,17 @@
+# Build packages for AlmaLinux 10 inside a container
+# cd ..
+# docker pull --platform=linux/amd64/v2 quay.io/almalinuxorg/almalinux:10
+# docker build -t vitastor-buildenv:el10 -f rpm/vitastor-el10.Dockerfile .
+# docker run -i --rm -v ./:/root/vitastor vitastor-buildenv:el10 /root/vitastor/rpm/vitastor-build.sh
+
+FROM quay.io/almalinuxorg/almalinux:10
+
+WORKDIR /root
+
+RUN sed -i 's/enabled=0/enabled=1/' /etc/yum.repos.d/*.repo
+RUN dnf -y install epel-release dnf-plugins-core
+RUN dnf -y install https://vitastor.io/rpms/centos/10/vitastor-release-1.0-1.el10.noarch.rpm
+RUN dnf -y install gcc-c++ gperftools-devel fio nodejs rpm-build jerasure-devel isa-l-devel gf-complete-devel rdma-core-devel cmake libnl3-devel
+RUN dnf download --source fio
+RUN rpm --nomd5 -i fio*.src.rpm
+RUN cd ~/rpmbuild/SPECS && dnf builddep -y --spec fio.spec

rpm/vitastor-el10.spec

@@ -0,0 +1,198 @@
+Name:           vitastor
+Version:        3.0.4
+Release:        1%{?dist}
+Summary:        Vitastor, a fast software-defined clustered block storage
+
+License:        Vitastor Network Public License 1.1
+URL:            https://vitastor.io/
+Source0:        vitastor-3.0.4.el10.tar.gz
+
+BuildRequires:  gperftools-devel
+BuildRequires:  gcc-c++
+BuildRequires:  nodejs >= 10
+BuildRequires:  jerasure-devel
+BuildRequires:  isa-l-devel
+BuildRequires:  gf-complete-devel
+BuildRequires:  rdma-core-devel
+BuildRequires:  cmake
+BuildRequires:  libnl3-devel
+Requires:       vitastor-osd = %{version}-%{release}
+Requires:       vitastor-mon = %{version}-%{release}
+Requires:       vitastor-client = %{version}-%{release}
+Requires:       vitastor-client-devel = %{version}-%{release}
+Requires:       vitastor-fio = %{version}-%{release}
+
+%description
+Vitastor is a small, simple and fast clustered block storage (storage for VM drives),
+architecturally similar to Ceph which means strong consistency, primary-replication,
+symmetric clustering and automatic data distribution over any number of drives of any
+size with configurable redundancy (replication or erasure codes/XOR).
+
+
+%package -n vitastor-osd
+Summary:        Vitastor - OSD
+Requires:       vitastor-client = %{version}-%{release}
+Requires:       util-linux
+Requires:       parted
+
+
+%description -n vitastor-osd
+Vitastor object storage daemon, i.e. server program that stores data.
+
+
+%package -n vitastor-mon
+Summary:        Vitastor - monitor
+Requires:       nodejs >= 10
+Requires:       lpsolve
+
+
+%description -n vitastor-mon
+Vitastor monitor, i.e. server program responsible for watching cluster state and
+scheduling cluster-level operations.
+
+
+%package -n vitastor-client
+Summary:        Vitastor - client
+
+
+%description -n vitastor-client
+Vitastor client library and command-line interface.
+
+
+%package -n vitastor-client-devel
+Summary:        Vitastor - development files
+Group:          Development/Libraries
+Requires:       vitastor-client = %{version}-%{release}
+
+
+%description -n vitastor-client-devel
+Vitastor library headers for development.
+
+
+%package -n vitastor-fio
+Summary:        Vitastor - fio drivers
+Group:          Development/Libraries
+Requires:       vitastor-client = %{version}-%{release}
+Requires:       fio = 3.36-5.el10
+
+
+%description -n vitastor-fio
+Vitastor fio drivers for benchmarking.
+
+
+%package -n vitastor-opennebula
+Summary:        Vitastor for OpenNebula
+Group:          Development/Libraries
+Requires:       vitastor-client
+Requires:       jq
+Requires:       python3-lxml
+Requires:       patch
+Requires:       qemu-kvm-block-vitastor
+
+
+%description -n vitastor-opennebula
+Vitastor storage plugin for OpenNebula.
+
+
+%prep
+%setup -q
+
+
+%build
+%cmake
+%cmake_build
+
+
+%install
+rm -rf $RPM_BUILD_ROOT
+%cmake_install
+cd mon
+npm install --production
+cd ..
+mkdir -p %buildroot/usr/lib/vitastor
+cp -r mon %buildroot/usr/lib/vitastor
+mv %buildroot/usr/lib/vitastor/mon/scripts/make-etcd %buildroot/usr/lib/vitastor/mon/
+mkdir -p %buildroot/lib/systemd/system
+cp mon/scripts/vitastor.target mon/scripts/vitastor-mon.service mon/scripts/vitastor-osd@.service %buildroot/lib/systemd/system
+mkdir -p %buildroot/lib/udev/rules.d
+cp mon/scripts/90-vitastor.rules %buildroot/lib/udev/rules.d
+mkdir -p %buildroot/var/lib/one
+cp -r opennebula/remotes %buildroot/var/lib/one
+cp opennebula/install.sh %buildroot/var/lib/one/remotes/datastore/vitastor/
+mkdir -p %buildroot/etc/
+cp -r opennebula/sudoers.d %buildroot/etc/
+
+
+%files
+%doc GPL-2.0.txt VNPL-1.1.txt README.md README-ru.md
+
+
+%files -n vitastor-osd
+%_bindir/vitastor-osd
+%_bindir/vitastor-disk
+%_bindir/vitastor-dump-journal
+/lib/systemd/system/vitastor-osd@.service
+/lib/systemd/system/vitastor.target
+/lib/udev/rules.d/90-vitastor.rules
+
+
+%pre -n vitastor-osd
+groupadd -r -f vitastor 2>/dev/null ||:
+useradd -r -g vitastor -s /sbin/nologin -c "Vitastor daemons" -M -d /nonexistent vitastor 2>/dev/null ||:
+install -o vitastor -g vitastor -d /var/log/vitastor
+mkdir -p /etc/vitastor
+
+
+%files -n vitastor-mon
+/usr/lib/vitastor/mon
+/lib/systemd/system/vitastor-mon.service
+
+
+%pre -n vitastor-mon
+groupadd -r -f vitastor 2>/dev/null ||:
+useradd -r -g vitastor -s /sbin/nologin -c "Vitastor daemons" -M -d /nonexistent vitastor 2>/dev/null ||:
+mkdir -p /etc/vitastor
+mkdir -p /var/lib/vitastor
+chown vitastor:vitastor /var/lib/vitastor
+
+
+%files -n vitastor-client
+%_bindir/vitastor-nbd
+%_bindir/vitastor-ublk
+%_bindir/vitastor-nfs
+%_bindir/vitastor-cli
+%_bindir/vitastor-rm
+%_bindir/vitastor-kv
+%_bindir/vitastor-kv-stress
+%_bindir/vita
+%_libdir/libvitastor_client.so*
+%_libdir/libvitastor_kv.so*
+
+
+%files -n vitastor-client-devel
+%_includedir/vitastor_c.h
+%_includedir/vitastor_kv.h
+%_libdir/pkgconfig
+
+
+%files -n vitastor-fio
+%_libdir/libfio_vitastor.so
+%_libdir/libfio_vitastor_blk.so
+%_libdir/libfio_vitastor_sec.so
+
+
+%files -n vitastor-opennebula
+/var/lib/one
+/etc/sudoers.d/opennebula-vitastor
+
+
+%triggerin -n vitastor-opennebula -- opennebula
+[ $2 = 0 ] || exit 0
+/var/lib/one/remotes/datastore/vitastor/install.sh
+
+
+# Turn off the brp-python-bytecompile script
+%global __os_install_post %(echo '%{__os_install_post}' | sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g')
+
+
+%changelog

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.4.3
+Version:        3.0.4
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.4.3.el7.tar.gz
+Source0:        vitastor-3.0.4.el7.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  devtoolset-9-gcc-c++
@@ -171,7 +171,6 @@ chown vitastor:vitastor /var/lib/vitastor
 %_bindir/vitastor-kv
 %_bindir/vitastor-kv-stress
 %_bindir/vita
-%_libdir/libvitastor_blk.so*
 %_libdir/libvitastor_client.so*
 %_libdir/libvitastor_kv.so*
 

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.4.3
+Version:        3.0.4
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.4.3.el8.tar.gz
+Source0:        vitastor-3.0.4.el8.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-toolset-9-gcc-c++
@@ -168,7 +168,6 @@ chown vitastor:vitastor /var/lib/vitastor
 %_bindir/vitastor-kv
 %_bindir/vitastor-kv-stress
 %_bindir/vita
-%_libdir/libvitastor_blk.so*
 %_libdir/libvitastor_client.so*
 %_libdir/libvitastor_kv.so*
 

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.4.3
+Version:        3.0.4
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.4.3.el9.tar.gz
+Source0:        vitastor-3.0.4.el9.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++
@@ -165,7 +165,6 @@ chown vitastor:vitastor /var/lib/vitastor
 %_bindir/vitastor-kv
 %_bindir/vitastor-kv-stress
 %_bindir/vita
-%_libdir/libvitastor_blk.so*
 %_libdir/libvitastor_client.so*
 %_libdir/libvitastor_kv.so*
 

src/CMakeLists.txt

@@ -21,7 +21,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 endif()
 set(ENABLE_COVERAGE false CACHE BOOL "Enable code coverage")
 
-add_definitions(-DVITASTOR_VERSION="2.4.3")
+add_definitions(-DVITASTOR_VERSION="3.0.4")
 add_definitions(-D_GNU_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -fvisibility=hidden -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/blockstore/blockstore.h

@@ -183,6 +183,11 @@ public:
     // Update configuration
     virtual void parse_config(blockstore_config_t & config) = 0;
 
+    // Reshard database for a pool in chunks
+    // MUST be called only when nobody makes any modifications to the DB for this pool
+    virtual void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit) = 0;
+    virtual bool reshard_continue(void *reshard_state, uint64_t chunk_limit) = 0;
+
     // Event loop
     virtual void loop() = 0;
 

src/blockstore/blockstore_disk.cpp

@@ -48,11 +48,12 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     disk_alignment = parse_size(config["disk_alignment"]);
     journal_block_size = parse_size(config["journal_block_size"]);
     meta_block_size = parse_size(config["meta_block_size"]);
-    meta_block_target_free_space = parse_size(config["meta_block_target_free_space"]);
     bitmap_granularity = parse_size(config["bitmap_granularity"]);
     meta_format = stoull_full(config["meta_format"]);
     atomic_write_size = (config.find("atomic_write_size") != config.end()
         ? parse_size(config["atomic_write_size"]) : 4096);
+    use_atomic_flag = config.find("use_atomic_flag") != config.end() &&
+        (config["use_atomic_flag"] == "true" || config["use_atomic_flag"] == "1" || config["use_atomic_flag"] == "yes");
     if (config.find("data_io") == config.end() &&
         config.find("meta_io") == config.end() &&
         config.find("journal_io") == config.end())
@@ -154,14 +155,6 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     {
         throw std::runtime_error("meta_block_size must not exceed "+std::to_string(MAX_DATA_BLOCK_SIZE));
     }
-    if (!meta_block_target_free_space)
-    {
-        meta_block_target_free_space = 800;
-    }
-    if (meta_block_target_free_space >= meta_block_size)
-    {
-        throw std::runtime_error("meta_block_target_free_space must not exceed "+std::to_string(meta_block_size));
-    }
     if (data_offset % disk_alignment)
     {
         throw std::runtime_error("data_offset must be a multiple of disk_alignment = "+std::to_string(disk_alignment));
@@ -178,6 +171,10 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     {
         throw std::runtime_error("Data block size must be a multiple of sparse write tracking granularity");
     }
+    if (data_block_size / bitmap_granularity < 8)
+    {
+        throw std::runtime_error("Data block size must be at least bitmap_granularity*8");
+    }
     if (!data_csum_type)
     {
         csum_block_size = 0;
@@ -210,10 +207,6 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     {
         throw std::runtime_error("journal_offset must be a multiple of journal_block_size = "+std::to_string(journal_block_size));
     }
-    if (!meta_format)
-    {
-        meta_format = BLOCKSTORE_META_FORMAT_HEAP;
-    }
     if (meta_device == data_device)
     {
         disable_meta_fsync = disable_data_fsync;
@@ -224,7 +217,7 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     }
 }
 
-void blockstore_disk_t::calc_lengths()
+void blockstore_disk_t::calc_lengths(bool skip_meta_check)
 {
     // data
     data_len = data_device_size - data_offset;
@@ -270,13 +263,13 @@ void blockstore_disk_t::calc_lengths()
     }
     // required metadata size
     block_count = data_len / data_block_size;
-    clean_entry_bitmap_size = data_block_size / bitmap_granularity / 8;
+    clean_entry_bitmap_size = (data_block_size / bitmap_granularity + 7) / 8;
     clean_dyn_size = clean_entry_bitmap_size*2 + (csum_block_size
         ? data_block_size/csum_block_size*(data_csum_type & 0xFF) : 0);
+recalc:
     if (meta_format == BLOCKSTORE_META_FORMAT_HEAP)
     {
-        uint32_t entries_per_block = ((meta_block_size-meta_block_target_free_space) /
-            (sizeof(heap_object_t) + sizeof(heap_write_t) + clean_dyn_size));
+        uint32_t entries_per_block = meta_block_size / (sizeof(heap_big_write_t) + clean_dyn_size);
         min_meta_len = (block_count+entries_per_block-1) / entries_per_block * meta_block_size;
     }
     else if (meta_format == BLOCKSTORE_META_FORMAT_V1)
@@ -284,11 +277,32 @@ void blockstore_disk_t::calc_lengths()
         clean_entry_size = 24 /*sizeof(clean_disk_entry)*/ + 2*clean_entry_bitmap_size;
         min_meta_len = (1 + (block_count - 1 + meta_block_size / clean_entry_size)
             / (meta_block_size / clean_entry_size)) * meta_block_size;
+        if (!skip_meta_check && meta_area_size < min_meta_len)
+        {
+too_small:
+            throw std::runtime_error("Metadata area is too small, need at least "+std::to_string(min_meta_len)+
+                " bytes, have only "+std::to_string(meta_area_size)+" bytes");
+        }
     }
-    else if (meta_format == BLOCKSTORE_META_FORMAT_V2)
+    else if (meta_format == BLOCKSTORE_META_FORMAT_V2 || !meta_format)
     {
+        meta_format = BLOCKSTORE_META_FORMAT_V2;
         clean_entry_size = 24 /*sizeof(clean_disk_entry)*/ + clean_dyn_size + 4 /*entry_csum*/;
         min_meta_len = (1 + (block_count - 1 + meta_block_size / clean_entry_size) / (meta_block_size / clean_entry_size)) * meta_block_size;
+        if (!skip_meta_check && meta_area_size < min_meta_len)
+        {
+            if (!data_csum_type)
+            {
+                printf("Warning: Using old metadata format without checksums because the new format"
+                    " doesn't fit into provided area (%ju bytes required, %ju bytes available)\n", min_meta_len, meta_area_size);
+                meta_format = BLOCKSTORE_META_FORMAT_V1;
+                goto recalc;
+            }
+            else
+            {
+                goto too_small;
+            }
+        }
     }
     else
     {

src/blockstore/blockstore_disk.h

@@ -36,8 +36,8 @@ struct blockstore_disk_t
     uint32_t meta_block_size = 4096;
     // Atomic write size of the data block device
     uint32_t atomic_write_size = 4096;
-    // Target free space in metadata blocks
-    uint32_t meta_block_target_free_space = 800;
+    // Whether we should set RWF_ATOMIC on atomic writes
+    bool use_atomic_flag = false;
     // Sparse write tracking granularity. 4 KB is a good choice. Must be a multiple of disk_alignment
     uint32_t bitmap_granularity = 4096;
     // Data checksum type, BLOCKSTORE_CSUM_NONE or BLOCKSTORE_CSUM_CRC32C
@@ -61,9 +61,10 @@ struct blockstore_disk_t
     uint64_t discard_granularity = 0;
 
     int meta_fd = -1, data_fd = -1, journal_fd = -1;
-    uint64_t meta_offset, meta_device_sect, meta_device_size, meta_area_size, min_meta_len, meta_format = 0;
-    uint64_t data_offset, data_device_sect, data_device_size, data_len;
-    uint64_t journal_offset, journal_device_sect, journal_device_size, journal_len;
+    uint64_t meta_offset = 0, meta_device_sect = 0, meta_device_size = 0, meta_area_size = 0, min_meta_len = 0;
+    uint64_t data_offset = 0, data_device_sect = 0, data_device_size = 0, data_len = 0;
+    uint64_t journal_offset = 0, journal_device_sect = 0, journal_device_size = 0, journal_len = 0;
+    uint64_t meta_format = 0;
 
     uint64_t block_count = 0;
     uint32_t clean_entry_bitmap_size = 0;
@@ -75,7 +76,7 @@ struct blockstore_disk_t
     void open_data();
     void open_meta();
     void open_journal();
-    void calc_lengths();
+    void calc_lengths(bool skip_meta_check = false);
     void check_lengths();
     void close_all();
     int trim_data(std::function<bool(uint64_t)> is_free);

src/blockstore/blockstore_flush.cpp

@@ -18,12 +18,18 @@ journal_flusher_t::journal_flusher_t(blockstore_impl_t *bs)
     this->cur_flusher_count = bs->min_flusher_count;
     this->target_flusher_count = bs->min_flusher_count;
     active_flushers = 0;
-    advance_lsn_counter = 0;
     co = new journal_flusher_co[max_flusher_count];
     for (int i = 0; i < max_flusher_count; i++)
     {
         co[i].co_id = i;
         co[i].bs = bs;
+        co[i].new_bmp = (uint8_t*)malloc_or_die(3*bs->dsk.clean_entry_bitmap_size);
+        co[i].new_ext_bmp = co[i].new_bmp + bs->dsk.clean_entry_bitmap_size;
+        co[i].punch_bmp = co[i].new_bmp + 2*bs->dsk.clean_entry_bitmap_size;
+        if (bs->dsk.csum_block_size > 0)
+        {
+            co[i].new_csums = (uint8_t*)malloc_or_die(bs->dsk.data_block_size / bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF));
+        }
         co[i].flusher = this;
     }
 }
@@ -54,6 +60,18 @@ journal_flusher_t::~journal_flusher_t()
 
 journal_flusher_co::~journal_flusher_co()
 {
+    if (new_csums)
+    {
+        free(new_csums);
+        new_csums = NULL;
+    }
+    if (new_bmp)
+    {
+        free(new_bmp);
+        new_bmp = NULL;
+    }
+    new_ext_bmp = NULL;
+    punch_bmp = NULL;
     free_buffers();
 }
 
@@ -62,11 +80,6 @@ int journal_flusher_t::get_syncing_buffer()
     return syncing_buffer;
 }
 
-uint64_t journal_flusher_t::get_compact_counter()
-{
-    return compact_counter;
-}
-
 bool journal_flusher_t::is_active()
 {
     return active_flushers > 0;
@@ -86,9 +99,10 @@ void journal_flusher_t::release_trim()
 void journal_flusher_t::dump_diagnostics()
 {
     printf(
-        "Compaction queue: %u items, data: %ju/%ju blocks used, meta: %ju/%ju bytes used, %u/%ju blocks nearfull\n",
-        bs->heap->get_to_compact_count(),
+        "Compaction queue: %u items + %u future, data: %ju/%ju blocks used, buffer: %ju/%ju bytes used, meta: %ju/%ju bytes used, %u/%ju blocks nearfull\n",
+        bs->heap->get_compact_queue_size(), bs->heap->get_to_compact_count(),
         bs->heap->get_data_used_space()/bs->dsk.data_block_size, bs->dsk.block_count,
+        bs->heap->get_buffer_area_used_space(), bs->dsk.journal_len,
         bs->heap->get_meta_used_space(), bs->heap->get_meta_total_space(),
         bs->heap->get_meta_nearfull_blocks(), bs->dsk.meta_area_size/bs->dsk.meta_block_size-1
     );
@@ -113,8 +127,12 @@ void journal_flusher_t::loop()
         }
     }
     int prev_active = active_flushers;
-    for (int i = 0; (active_flushers > 0 || force_start > 0 || bs->heap->get_to_compact_count() > bs->flusher_start_threshold) && i < cur_flusher_count; i++)
+    for (int i = 0; (active_flushers > 0 || force_start > 0 ||
+        bs->heap->get_compact_queue_size() > bs->flusher_start_threshold ||
+        i == 0 && bs->intent_write_counter >= bs->journal_trim_interval) && i < cur_flusher_count; i++)
+    {
         co[i].loop();
+    }
     if (prev_active && !active_flushers && force_start > 0)
         bs->ringloop->wakeup();
 }
@@ -156,22 +174,33 @@ bool journal_flusher_co::loop()
     else if (wait_state == 19) goto resume_19;
     else if (wait_state == 20) goto resume_20;
     else if (wait_state == 21) goto resume_21;
-    else if (wait_state == 22) goto resume_22;
-    else if (wait_state == 23) goto resume_23;
-    else if (wait_state == 24) goto resume_24;
 resume_0:
     wait_state = 0;
     wait_count = 0;
     cur_oid = {};
     res = bs->heap->get_next_compact(cur_oid);
+    // Advance fsynced_lsn every <journal_trim_interval> intent writes
+    if ((bs->intent_write_counter >= bs->journal_trim_interval) && co_id == 0)
+    {
+        bs->intent_write_counter = 0;
+resume_17:
+resume_18:
+resume_19:
+        if (!fsync_buffer(17))
+            return false;
+resume_20:
+resume_21:
+        if (!trim_lsn(20))
+            return false;
+    }
     if (res == ENOENT && flusher->force_start > 0 && co_id == 0 &&
-        (!bs->dsk.disable_journal_fsync || !bs->dsk.disable_meta_fsync))
+        (!bs->dsk.disable_journal_fsync || !bs->dsk.disable_meta_fsync || !bs->dsk.disable_data_fsync))
     {
         flusher->active_flushers++;
-resume_21:
-resume_22:
-        res = fsync_buffer(21);
-        if (!res)
+resume_14:
+resume_15:
+resume_16:
+        if (!fsync_buffer(14))
         {
             return false;
         }
@@ -180,61 +209,94 @@ resume_22:
     }
     if (res == ENOENT)
     {
-        if (co_id == 0 && flusher->force_start > 0)
-        {
-            flusher->active_flushers++;
-resume_16:
-resume_17:
-resume_18:
-resume_19:
-resume_20:
-            if (!trim_lsn(16))
-                return false;
-            flusher->active_flushers--;
-        }
         cur_oid = {};
         wait_state = 0;
         return true;
     }
-    for (int i = 0; i < flusher->cur_flusher_count; i++)
+    if (flusher->flushing.find(cur_oid) != flusher->flushing.end())
     {
-        if (i != co_id && flusher->co[i].cur_oid == cur_oid)
+        for (int i = 0; i < flusher->cur_flusher_count; i++)
         {
-            // Already flushing it
-            flusher->co[i].should_repeat = true;
-            goto resume_0;
+            if (i != co_id && flusher->co[i].cur_oid == cur_oid)
+            {
+                // Already flushing it
+                flusher->co[i].should_repeat = true;
+                goto resume_0;
+            }
         }
+        assert(false);
     }
+    flusher->flushing.insert(cur_oid);
 resume_1:
+    wait_state = 1;
     should_repeat = false;
-    cur_obj = bs->heap->lock_and_read_entry(cur_oid, copy_id);
+    cur_obj = bs->heap->lock_and_read_entry(cur_oid);
     if (!cur_obj)
     {
         // Object does not exist
+        flusher->flushing.erase(cur_oid);
         goto resume_0;
     }
-    cur_version = cur_obj->get_writes()->version;
-    // Find the range to compact
-    compact_lsn = bs->heap->get_fsynced_lsn();
-    bs->heap->get_compact_range(cur_obj, compact_lsn, &begin_wr, &end_wr);
-    if (!begin_wr)
-    {
-        // Nothing to flush
-        bs->heap->unlock_entry(cur_oid, copy_id);
-        goto resume_0;
-    }
-    assert(!end_wr->next() && end_wr->entry_type == (BS_HEAP_BIG_WRITE|BS_HEAP_STABLE));
-    clean_loc = end_wr->big_location(bs->heap);
-    if (bs->log_level > 10)
-        printf("Compacting %jx:%jx l%ju .. l%ju (last l%ju)\n", cur_oid.inode, cur_oid.stripe, end_wr->lsn, begin_wr->lsn, compact_lsn);
-    flusher->active_flushers++;
     // Scan versions to flush
     free_buffers();
     copy_count = 0;
-    for (auto wr = begin_wr; wr != end_wr; wr = wr->next())
+    fsynced_lsn = bs->heap->get_fsynced_lsn();
+    bitmap_copied = false;
+    memset(new_bmp, 0, bs->dsk.clean_entry_bitmap_size);
+    csum_copy.clear();
+    compact_info = bs->heap->iterate_compaction(cur_obj, fsynced_lsn, flusher->force_start, [&](heap_entry_t *wr)
     {
-        bs->prepare_read(read_vec, cur_obj, wr, 0, bs->dsk.data_block_size);
-        copy_count++;
+        if (!bitmap_copied)
+        {
+            memcpy(new_ext_bmp, wr->get_ext_bitmap(bs->heap), bs->dsk.clean_entry_bitmap_size);
+            bitmap_copied = true;
+        }
+        bitmap_set(new_bmp, wr->small().offset, wr->small().len, bs->dsk.bitmap_granularity);
+        if (bs->dsk.csum_block_size && bs->dsk.csum_block_size <= bs->dsk.bitmap_granularity)
+        {
+            csum_copy.push_back(wr);
+        }
+        if (wr->type() == BS_HEAP_SMALL_WRITE ||
+            wr->type() == BS_HEAP_INTENT_WRITE && bs->dsk.csum_block_size > bs->dsk.bitmap_granularity)
+        {
+            auto res = bs->prepare_read(read_vec, cur_obj, wr, 0, bs->dsk.data_block_size,
+                wr->type() == BS_HEAP_INTENT_WRITE && bs->dsk.csum_block_size > bs->dsk.bitmap_granularity && !bs->perfect_csum_update
+                ? COPY_BUF_SKIP_CSUM : 0);
+            if (res > 0)
+                copy_count++;
+        }
+    });
+    if (!compact_info.compact_lsn)
+    {
+        // Flushing is aborted
+        flusher->flushing.erase(cur_oid);
+        bs->heap->unlock_entry(cur_oid);
+        goto resume_0;
+    }
+    mem_or(new_bmp, compact_info.clean_wr->get_int_bitmap(bs->heap), bs->dsk.clean_entry_bitmap_size);
+    if (!bitmap_copied)
+    {
+        memcpy(new_ext_bmp, compact_info.clean_wr->get_ext_bitmap(bs->heap), bs->dsk.clean_entry_bitmap_size);
+        bitmap_copied = true;
+    }
+    if (bs->dsk.csum_block_size && bs->dsk.csum_block_size <= bs->dsk.bitmap_granularity)
+    {
+        memcpy(new_csums, compact_info.clean_wr->get_checksums(bs->heap), bs->dsk.data_block_size/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF));
+        for (size_t i = csum_copy.size(); i > 0; i--)
+        {
+            auto wr = csum_copy[i-1];
+            memcpy(new_csums + wr->small().offset/bs->dsk.csum_block_size*(bs->dsk.data_csum_type & 0xFF),
+                wr->get_checksums(bs->heap), wr->small().len/bs->dsk.csum_block_size*(bs->dsk.data_csum_type & 0xFF));
+        }
+        csum_copy.clear();
+    }
+    clean_loc = compact_info.clean_wr->big_location(bs->heap);
+    flusher->active_flushers++;
+    if (bs->log_level > 10)
+    {
+        printf("Compacting %jx:%jx v%ju..v%ju / l%ju..l%ju (%d writes)\n", cur_oid.inode, cur_oid.stripe,
+            compact_info.clean_wr->version, compact_info.compact_version,
+            compact_info.clean_wr->lsn, compact_info.compact_lsn, copy_count);
     }
     overwrite_start = overwrite_end = 0;
     if (read_vec.size() > 0)
@@ -254,7 +316,6 @@ resume_1:
     }
     // Read buffered data
     cur_obj = NULL;
-    begin_wr = end_wr = NULL;
 resume_2:
 resume_3:
     if (!read_buffered(2))
@@ -272,29 +333,49 @@ resume_3:
         flusher->wanting_meta_fsync--;
     }
     res = check_and_punch_checksums();
+    if (res == ENOENT || res == EDOM)
+    {
+        // Abort compaction
+        flusher->flushing.erase(cur_oid);
+        bs->heap->unlock_entry(cur_oid);
+        flusher->active_flushers--;
+        goto resume_0;
+    }
     if (res == EBUSY)
     {
 resume_4:
+        modified_block = UINT32_MAX;
+        res = bs->heap->punch_holes(compact_info.clean_wr, punch_bmp, new_csums, &modified_block);
+        if (res == ENOENT)
+        {
+            // Abort compaction
+            flusher->flushing.erase(cur_oid);
+            bs->heap->unlock_entry(cur_oid);
+            flusher->active_flushers--;
+            goto resume_0;
+        }
+        if (res == EAGAIN)
+        {
+            // Retry, block is busy
+            wait_state = 4;
+            return false;
+        }
+        assert(res == 0);
 resume_5:
-        if (!write_meta_block(4))
+resume_6:
+        if (!write_meta_block(5))
         {
             return false;
         }
-resume_6:
 resume_7:
 resume_8:
-        if (!fsync_meta(6))
+resume_9:
+        if (!fsync_meta(7))
         {
             return false;
         }
         res = 0;
     }
-    else if (res == ENOENT || res == EDOM)
-    {
-        // Abort compaction
-        flusher->active_flushers--;
-        goto resume_0;
-    }
     assert(res == 0);
     // Submit data writes
     for (i = 0; i < read_vec.size(); i++)
@@ -304,7 +385,7 @@ resume_8:
             (read_vec[i].copy_flags & COPY_BUF_PADDED)) // FIXME Shit, simplify these flags
         {
             assert(read_vec[i].buf);
-            await_sqe(9);
+            await_sqe(10);
             data->iov = (struct iovec){ read_vec[i].buf + (read_vec[i].copy_flags & COPY_BUF_PADDED
                 ? read_vec[i].offset - read_vec[i].disk_offset : 0), (size_t)read_vec[i].len };
             data->callback = simple_callback_w;
@@ -312,67 +393,57 @@ resume_8:
             wait_count++;
         }
     }
-resume_10:
+resume_11:
     if (wait_count > 0)
     {
-        wait_state = 10;
+        wait_state = 11;
         return false;
     }
     // Lock is only needed to prevent freeing the big_write because we overwrite it...
-    bs->heap->unlock_entry(cur_oid, copy_id);
+    bs->heap->unlock_entry(cur_oid);
     // Mark the object compacted, but don't free and remove small_writes
     // We'll free and remove them only when trimming
     // The only thing we modify here are big_write block checksums if >4k block is used
-    cur_obj = bs->heap->read_entry(cur_oid, &modified_block);
+    cur_obj = bs->heap->read_entry(cur_oid);
     if (!cur_obj)
     {
         // Abort compaction
+        flusher->flushing.erase(cur_oid);
         goto resume_0;
     }
     if (!calc_block_checksums())
     {
         // Abort compaction
+        flusher->flushing.erase(cur_oid);
         goto resume_0;
     }
-    if (read_to_fill_incomplete)
+    res = bs->heap->add_compact(cur_obj, compact_info.compact_version, compact_info.compact_lsn, clean_loc,
+        compact_info.do_delete, &modified_block, new_bmp, new_ext_bmp, new_csums);
+    if (res == EBUSY)
     {
-resume_23:
-resume_24:
-        if (!write_meta_block(23))
-        {
-            return false;
-        }
-        // Recheck the object because it could be invalidated again
-        cur_obj = bs->heap->read_entry(cur_oid, &modified_block);
-        if (!cur_obj)
-        {
-            // Abort compaction
-            goto resume_0;
-        }
+        // Abort compaction, object is already overwritten by something else
+        flusher->flushing.erase(cur_oid);
+        goto resume_0;
     }
-    bs->heap->mark_object_compacted(cur_obj, compact_lsn);
-    // Done
-    if (bs->log_level > 10)
-        printf("Compacted %jx:%jx l%ju (%d writes)\n", cur_oid.inode, cur_oid.stripe, compact_lsn, copy_count);
-    flusher->compact_counter++;
-    flusher->active_flushers--;
-    // Advance compacted_lsn every <journal_trim_interval> objects
-    if (co_id == 0 && !((++flusher->advance_lsn_counter) % bs->journal_trim_interval))
-    {
-        flusher->advance_lsn_counter = 0;
-resume_11:
+    assert(res == 0);
 resume_12:
 resume_13:
-resume_14:
-resume_15:
-        if (!trim_lsn(11))
-            return false;
+    if (!write_meta_block(12))
+    {
+        return false;
     }
+    // Done
+    if (bs->log_level > 10)
+    {
+        printf("Compacted %jx:%jx l%ju (%d writes)\n", cur_oid.inode, cur_oid.stripe, compact_info.compact_lsn, copy_count);
+    }
+    flusher->active_flushers--;
     if (should_repeat)
     {
         // Flush the same object again
         goto resume_1;
     }
+    flusher->flushing.erase(cur_oid);
     // All done
     goto resume_0;
 }
@@ -409,11 +480,11 @@ void journal_flusher_co::fill_partial_checksum_blocks()
                 .copy_flags = COPY_BUF_DATA | copy_flags,
                 .offset = blk_begin,
                 .len = blk_end - blk_begin,
-                .disk_loc = end_wr->big_location(bs->heap),
+                .disk_loc = clean_loc,
                 .disk_offset = blk_begin,
                 .disk_len = blk_end - blk_begin,
                 .buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, blk_end - blk_begin),
-                .wr_lsn = end_wr->lsn,
+                .wr = compact_info.clean_wr,
             });
         }
         auto & vec = read_vec[read_vec.size()-1];
@@ -451,26 +522,28 @@ int journal_flusher_co::check_and_punch_checksums()
         return 0;
     }
     // Verify data checksums
-    cur_obj = bs->heap->read_locked_entry(cur_oid, copy_id);
+    cur_obj = bs->heap->read_entry(cur_oid);
+    if (!cur_obj)
+    {
+        // Object is deleted, abort compaction
+        return ENOENT;
+    }
     bool csum_ok = true;
     for (int i = 0; i < read_vec.size(); i++)
     {
         auto & vec = read_vec[i];
         if (!(vec.copy_flags & (COPY_BUF_COALESCED|COPY_BUF_ZERO|COPY_BUF_SKIP_CSUM)))
         {
-            heap_write_t *wr = cur_obj->get_writes();
-            while (wr && wr->lsn != vec.wr_lsn)
-                wr = wr->next();
-            assert(wr);
-            uint32_t *csums = (uint32_t*)(wr->get_checksums(bs->heap)
+            uint32_t *csums = (uint32_t*)(vec.wr->get_checksums(bs->heap)
                 + (vec.disk_offset/bs->dsk.csum_block_size)*(bs->dsk.data_csum_type & 0xFF)
-                - ((wr->type() == BS_HEAP_BIG_WRITE) ? 0 : (wr->small().offset/bs->dsk.csum_block_size)*(bs->dsk.data_csum_type & 0xFF)));
+                - ((vec.wr->type() == BS_HEAP_BIG_WRITE || vec.wr->type() == BS_HEAP_BIG_INTENT)
+                    ? 0 : (vec.wr->small().offset/bs->dsk.csum_block_size)*(bs->dsk.data_csum_type & 0xFF)));
             bs->heap->calc_block_checksums(
-                csums, vec.buf, wr->get_int_bitmap(bs->heap), vec.disk_offset, vec.disk_offset+vec.disk_len, false,
+                csums, vec.buf, vec.wr->get_int_bitmap(bs->heap), vec.disk_offset, vec.disk_offset+vec.disk_len, false,
                 [&](uint32_t mismatch_pos, uint32_t expected_csum, uint32_t real_csum)
                 {
                     printf("Checksum mismatch during compaction in object %jx:%jx v%ju, offset 0x%x in %s area at offset 0x%jx: got %08x, expected %08x\n",
-                        cur_oid.inode, cur_oid.stripe, wr->version, mismatch_pos,
+                        cur_oid.inode, cur_oid.stripe, vec.wr->version, mismatch_pos,
                         (vec.copy_flags & COPY_BUF_JOURNAL ? "buffer" : "data"),
                         vec.disk_loc+vec.disk_offset, real_csum, expected_csum);
                     csum_ok = false;
@@ -489,20 +562,8 @@ int journal_flusher_co::check_and_punch_checksums()
         // Nothing to do
         return 0;
     }
-    cur_obj = bs->heap->read_entry(cur_oid, &modified_block);
-    if (!cur_obj)
-    {
-        // Object is deleted, abort compaction
-        return ENOENT;
-    }
-    bs->heap->get_compact_range(cur_obj, compact_lsn, &begin_wr, &end_wr);
-    if (!begin_wr)
-    {
-        // Object is overwritten, abort compaction
-        return ENOENT;
-    }
-    uint8_t *bmp = end_wr->get_int_bitmap(bs->heap);
-    uint8_t *csums = end_wr->get_checksums(bs->heap);
+    memcpy(punch_bmp, compact_info.clean_wr->get_int_bitmap(bs->heap), bs->dsk.clean_entry_bitmap_size);
+    memcpy(new_csums, compact_info.clean_wr->get_checksums(bs->heap), bs->dsk.data_block_size/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF));
     // Clear bits
     for (auto & vec: read_vec)
     {
@@ -513,7 +574,7 @@ int journal_flusher_co::check_and_punch_checksums()
         if (!(vec.copy_flags & COPY_BUF_COALESCED) &&
             ((vec.offset % bs->dsk.csum_block_size) || (vec.len % bs->dsk.csum_block_size)))
         {
-            bitmap_clear(bmp, vec.offset, vec.len, bs->dsk.bitmap_granularity);
+            bitmap_clear(punch_bmp, vec.offset, vec.len, bs->dsk.bitmap_granularity);
         }
     }
     // Update partial block checksums
@@ -522,42 +583,20 @@ int journal_flusher_co::check_and_punch_checksums()
         if (vec.copy_flags & COPY_BUF_CSUM_FILL)
         {
             uint32_t csum_off = vec.offset/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF);
-            bs->heap->calc_block_checksums((uint32_t*)(csums+csum_off), vec.buf, bmp, vec.offset, vec.offset+vec.len, true, NULL);
+            bs->heap->calc_block_checksums((uint32_t*)(new_csums+csum_off), vec.buf, punch_bmp, vec.offset, vec.offset+vec.len, true, NULL);
         }
     }
-    cur_obj->crc32c = cur_obj->calc_crc32c();
-    if (res == ENOENT)
-    {
-        // Object is deleted, abort compaction
-        return ENOENT;
-    }
-    // Modified, we should write the block to disk
-    assert(!res);
+    // Modified, we should add_punch_holes and then write the block to disk
     return EBUSY;
 }
 
 bool journal_flusher_co::calc_block_checksums()
 {
-    if (bs->dsk.csum_block_size <= bs->dsk.bitmap_granularity || !read_vec.size())
+    if (bs->dsk.csum_block_size <= bs->dsk.bitmap_granularity)
     {
         return true;
     }
-    bs->heap->get_compact_range(cur_obj, compact_lsn, &begin_wr, &end_wr);
-    if (!begin_wr)
-    {
-        // Object is overwritten, abort compaction
-        return false;
-    }
-    uint8_t *bmp = end_wr->get_int_bitmap(bs->heap);
-    uint8_t *csums = end_wr->get_checksums(bs->heap);
-    // Set bits
-    for (auto & vec: read_vec)
-    {
-        if (!(vec.copy_flags & (COPY_BUF_COALESCED|COPY_BUF_CSUM_FILL)))
-        {
-            bitmap_set(bmp, vec.offset, vec.len, bs->dsk.bitmap_granularity);
-        }
-    }
+    memcpy(new_csums, compact_info.clean_wr->get_checksums(bs->heap), bs->dsk.data_block_size/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF));
     // Update block checksums
     size_t i = 0;
     while (i < read_vec.size() && !(read_vec[i].copy_flags & COPY_BUF_CSUM_FILL))
@@ -576,7 +615,7 @@ bool journal_flusher_co::calc_block_checksums()
         assert(!(end % bs->dsk.csum_block_size));
         uint32_t csum_off = start/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF);
         bs->heap->calc_block_checksums(
-            (uint32_t*)(csums+csum_off), bmp, start, end,
+            (uint32_t*)(new_csums+csum_off), new_bmp, start, end,
             [&](uint32_t start, uint32_t & len)
             {
                 // O(n^2) search, may be fixed later :-p
@@ -593,7 +632,6 @@ bool journal_flusher_co::calc_block_checksums()
             }, true, NULL
         );
     }
-    cur_obj->crc32c = cur_obj->calc_crc32c();
     return true;
 }
 
@@ -603,13 +641,15 @@ bool journal_flusher_co::write_meta_block(int wait_base)
         goto resume_0;
     else if (wait_state == wait_base+1)
         goto resume_1;
-    await_sqe(0);
-    data->iov = (struct iovec){ bs->heap->get_meta_block(modified_block), (size_t)bs->dsk.meta_block_size };
-    data->callback = simple_callback_w;
-    io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + (modified_block+1)*bs->dsk.meta_block_size);
-    wait_count++;
+resume_0:
+    if (bs->ringloop->space_left() < 1)
+    {
+        wait_state = wait_base+0;
+        return 0;
+    }
+    bs->prepare_meta_block_write(modified_block);
 resume_1:
-    if (wait_count > 0)
+    if (bs->meta_block_is_pending(modified_block))
     {
         wait_state = wait_base+1;
         return false;
@@ -664,11 +704,11 @@ bool journal_flusher_co::fsync_meta(int wait_base)
     if (wait_state == wait_base)        goto resume_0;
     else if (wait_state == wait_base+1) goto resume_1;
     else if (wait_state == wait_base+2) goto resume_2;
-resume_0:
     if (bs->dsk.disable_meta_fsync)
     {
         return true;
     }
+resume_0:
     if (flusher->wanting_meta_fsync || flusher->fsyncing_meta > 0)
     {
         wait_state = wait_base;
@@ -693,69 +733,30 @@ resume_2:
     return true;
 }
 
-int journal_flusher_co::fsync_buffer(int wait_base)
-{
-    if (wait_state == wait_base)        goto resume_0;
-    else if (wait_state == wait_base+1) goto resume_1;
-    if (bs->dsk.disable_journal_fsync && bs->dsk.disable_meta_fsync && bs->dsk.disable_data_fsync || !bs->unsynced_big_write_count && !bs->unsynced_small_write_count)
-    {
-        return 1;
-    }
-    if (flusher->syncing_buffer)
-    {
-        return 0;
-    }
-    flusher->active_flushers++;
-    flusher->syncing_buffer++;
-resume_0:
-    assert(!wait_count);
-    compact_lsn = bs->heap->get_completed_lsn();
-    if (!bs->submit_fsyncs(wait_count))
-    {
-        wait_state = wait_base+0;
-        return 0;
-    }
-resume_1:
-    if (wait_count > 0)
-    {
-        wait_state = wait_base+1;
-        return 0;
-    }
-    bs->heap->mark_lsn_fsynced(compact_lsn);
-    flusher->active_flushers--;
-    flusher->syncing_buffer--;
-    return 2;
-}
-
-bool journal_flusher_co::trim_lsn(int wait_base)
+bool journal_flusher_co::fsync_buffer(int wait_base)
 {
     if (wait_state == wait_base)        goto resume_0;
     else if (wait_state == wait_base+1) goto resume_1;
     else if (wait_state == wait_base+2) goto resume_2;
-    else if (wait_state == wait_base+3) goto resume_3;
-    else if (wait_state == wait_base+4) goto resume_4;
-    compact_lsn = bs->heap->get_compacted_lsn();
-    if (((blockstore_meta_header_v3_t*)bs->meta_superblock)->compacted_lsn == compact_lsn)
+    if (!bs->has_unsynced())
     {
         return true;
     }
-    flusher->active_flushers++;
-    assert(!wait_count);
-    if (!bs->dsk.disable_meta_fsync)
+resume_0:
+    if (flusher->syncing_buffer)
     {
-        await_sqe(0);
-        data->iov = { 0 };
-        data->callback = simple_callback_w;
-        io_uring_prep_fsync(sqe, bs->dsk.meta_fd, IORING_FSYNC_DATASYNC);
-        wait_count++;
+        wait_state = wait_base+0;
+        return false;
     }
-    if (!bs->dsk.disable_data_fsync && bs->dsk.data_fd != bs->dsk.meta_fd)
+    flusher->active_flushers++;
+    flusher->syncing_buffer++;
+resume_1:
+    assert(!wait_count);
+    fsynced_lsn = bs->heap->get_completed_lsn();
+    if (!bs->submit_fsyncs(wait_count))
     {
-        await_sqe(1);
-        data->iov = { 0 };
-        data->callback = simple_callback_w;
-        io_uring_prep_fsync(sqe, bs->dsk.data_fd, IORING_FSYNC_DATASYNC);
-        wait_count++;
+        wait_state = wait_base+1;
+        return false;
     }
 resume_2:
     if (wait_count > 0)
@@ -763,23 +764,37 @@ resume_2:
         wait_state = wait_base+2;
         return false;
     }
-    ((blockstore_meta_header_v3_t*)bs->meta_superblock)->compacted_lsn = compact_lsn;
+    bs->heap->mark_lsn_fsynced(fsynced_lsn);
+    flusher->active_flushers--;
+    flusher->syncing_buffer--;
+    return true;
+}
+
+bool journal_flusher_co::trim_lsn(int wait_base)
+{
+    if (wait_state == wait_base)        goto resume_0;
+    else if (wait_state == wait_base+1) goto resume_1;
+    fsynced_lsn = bs->heap->get_fsynced_lsn();
+    if (((blockstore_meta_header_v3_t*)bs->meta_superblock)->completed_lsn == fsynced_lsn)
+    {
+        return true;
+    }
+    flusher->active_flushers++;
+    ((blockstore_meta_header_v3_t*)bs->meta_superblock)->completed_lsn = fsynced_lsn;
     ((blockstore_meta_header_v3_t*)bs->meta_superblock)->set_crc32c();
-    await_sqe(3);
+    await_sqe(0);
     data->iov = (struct iovec){ bs->meta_superblock, (size_t)bs->dsk.meta_block_size };
     data->callback = simple_callback_w;
     io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset);
     // Update superblock with datasync
     sqe->rw_flags = RWF_DSYNC;
     wait_count++;
-resume_4:
+resume_1:
     if (wait_count > 0)
     {
-        wait_state = wait_base+4;
+        wait_state = wait_base+1;
         return false;
     }
-    bs->heap->mark_lsn_trimmed(compact_lsn);
-    flusher->compact_counter++;
     flusher->active_flushers--;
     return true;
 }

src/blockstore/blockstore_flush.h

@@ -6,7 +6,7 @@ struct copy_buffer_t
     uint32_t copy_flags;
     uint64_t offset, len, disk_loc, disk_offset, disk_len;
     uint8_t *buf;
-    uint64_t wr_lsn;
+    heap_entry_t *wr;
 };
 
 struct meta_sector_t
@@ -36,27 +36,28 @@ class journal_flusher_co
     int wait_state, wait_count;
     struct io_uring_sqe *sqe;
     struct ring_data_t *data;
+    uint8_t *new_csums = NULL;
+    uint8_t *new_bmp = NULL;
+    uint8_t *punch_bmp = NULL;
+    uint8_t *new_ext_bmp = NULL;
 
     std::function<void(ring_data_t*)> simple_callback_r, simple_callback_w;
 
     object_id cur_oid;
-    uint64_t copy_id;
-    uint64_t compact_lsn;
-    uint64_t cur_version;
-    heap_object_t *cur_obj;
-    heap_write_t *begin_wr, *end_wr;
+    heap_entry_t *cur_obj;
+    uint64_t fsynced_lsn;
+    heap_compact_t compact_info;
+    uint64_t clean_loc;
     uint32_t modified_block;
+    bool bitmap_copied;
     bool should_repeat;
 
     std::vector<copy_buffer_t> read_vec;
+    std::vector<heap_entry_t*> csum_copy;
     uint32_t overwrite_start, overwrite_end;
-    uint32_t big_start, big_end;
     int i, res;
     bool read_to_fill_incomplete;
     int copy_count;
-    uint64_t clean_loc;
-    flusher_meta_write_t meta_old, meta_new;
-    bool do_repeat = false;
 
     friend class journal_flusher_t;
 
@@ -68,7 +69,7 @@ class journal_flusher_co
     bool write_meta_block(int wait_base);
     bool read_buffered(int wait_base);
     bool fsync_meta(int wait_base);
-    int fsync_buffer(int wait_base);
+    bool fsync_buffer(int wait_base);
     bool trim_lsn(int wait_base);
 public:
     journal_flusher_co();
@@ -85,9 +86,7 @@ class journal_flusher_t
     blockstore_impl_t *bs;
     friend class journal_flusher_co;
 
-    int advance_lsn_counter = 0;
-    uint64_t compact_counter = 0;
-
+    robin_hood::unordered_flat_set<object_id> flushing;
     int active_flushers = 0;
     int wanting_meta_fsync = 0;
     bool fsyncing_meta = false;
@@ -98,7 +97,6 @@ public:
     ~journal_flusher_t();
     void loop();
     int get_syncing_buffer();
-    uint64_t get_compact_counter();
     bool is_active();
     void request_trim();
     void release_trim();

src/blockstore/blockstore_heap.h

@@ -1,4 +1,4 @@
-// Metadata storage version 3 ("heap")
+// Metadata storage version 3 ("lsm heap")
 // Copyright (c) Vitaliy Filippov, 2025+
 // License: VNPL-1.1 (see README.md for details)
 
@@ -19,326 +19,304 @@ struct pool_shard_settings_t
 {
     uint32_t pg_count;
     uint32_t pg_stripe_size;
+    uint32_t no_inode_stats;
 };
 
-#define BS_HEAP_TYPE 7
-#define BS_HEAP_OBJECT 1
+#define BS_HEAP_TYPE 0x07
+#define BS_HEAP_BIG_WRITE 1
 #define BS_HEAP_SMALL_WRITE 2
-#define BS_HEAP_BIG_WRITE 3
-#define BS_HEAP_TOMBSTONE 4
-#define BS_HEAP_INTENT_WRITE 5
-#define BS_HEAP_STABLE 8
+#define BS_HEAP_INTENT_WRITE 3
+#define BS_HEAP_BIG_INTENT 4
+#define BS_HEAP_DELETE 5
+#define BS_HEAP_COMMIT 6
+#define BS_HEAP_ROLLBACK 7
+#define BS_HEAP_STABLE 0x40
+#define BS_HEAP_GARBAGE 0x80
 
 class blockstore_heap_t;
 
-struct __attribute__((__packed__)) heap_small_write_t
+struct heap_small_write_t;
+struct heap_big_write_t;
+struct heap_big_intent_t;
+
+struct __attribute__((__packed__)) heap_entry_t
 {
     uint16_t size;
-    int16_t next_pos;
-    uint8_t flags;
+    uint16_t entry_type;
+    uint32_t crc32c;
     uint64_t lsn;
+    uint64_t inode;
+    uint64_t stripe;
     uint64_t version;
-    uint64_t location;
-    uint32_t offset;
-    uint32_t len;
-};
-
-struct __attribute__((__packed__)) heap_big_write_t
-{
-    uint16_t size;
-    int16_t next_pos;
-    uint8_t flags;
-    uint64_t lsn;
-    uint64_t version;
-    uint32_t block_num;
-};
-
-struct __attribute__((__packed__)) heap_tombstone_t
-{
-    uint16_t size;
-    int16_t next_pos;
-    uint8_t flags;
-    uint64_t lsn;
-    uint64_t version;
-};
-
-struct __attribute__((__packed__)) heap_write_t
-{
-    // size should have top bit cleared
-    uint16_t size = 0;
-    int16_t next_pos = 0;
-    uint8_t entry_type = 0; // BS_HEAP_*
-    uint64_t lsn = 0;
-    uint64_t version = 0;
 
     // uint8_t[] external_bitmap
     // uint8_t[] internal_bitmap
     // uint32_t[] checksums
 
-    heap_write_t *next();
     inline uint8_t type() const { return (entry_type & BS_HEAP_TYPE); }
     inline heap_small_write_t& small() { return *(heap_small_write_t*)this; }
     inline heap_big_write_t& big() { return *(heap_big_write_t*)this; }
+    inline heap_big_intent_t& big_intent() { return *(heap_big_intent_t*)this; }
+    bool is_garbage();
+    void set_garbage();
+    bool is_overwrite();
+    bool is_compactable();
+    bool is_before(heap_entry_t *other);
     uint32_t get_size(blockstore_heap_t *heap);
-    uint32_t get_csum_size(blockstore_heap_t *heap);
-    bool needs_recheck(blockstore_heap_t *heap);
-    bool needs_compact(blockstore_heap_t *heap);
-    bool is_compacted(uint64_t compacted_lsn);
-    bool can_be_collapsed(blockstore_heap_t *heap);
-    bool is_allowed_before_compacted(uint64_t compacted_lsn, bool is_last_entry);
     uint8_t *get_ext_bitmap(blockstore_heap_t *heap);
     uint8_t *get_int_bitmap(blockstore_heap_t *heap);
     uint8_t *get_checksums(blockstore_heap_t *heap);
     uint32_t *get_checksum(blockstore_heap_t *heap);
     uint64_t big_location(blockstore_heap_t *heap);
     void set_big_location(blockstore_heap_t *heap, uint64_t location);
-};
-
-struct __attribute__((__packed__)) heap_object_t
-{
-    // size should have top bit cleared
-    uint16_t size = 0;
-    // linked list of write entries...
-    // newest entries are stored first to simplify scanning
-    int16_t write_pos = 0;
-    uint8_t entry_type = 0; // BS_HEAP_*
-    uint32_t crc32c = 0;
-    uint64_t inode = 0;
-    uint64_t stripe = 0;
-
-    heap_write_t *get_writes();
     uint32_t calc_crc32c();
 };
 
-struct heap_object_lsn_t
+struct __attribute__((__packed__)) heap_small_write_t
 {
-    object_id oid;
-    uint64_t lsn;
+    heap_entry_t hdr;
+
+    uint64_t location;
+    uint32_t offset;
+    uint32_t len;
+
+    // Also includes 1 bitmap and 1 crc32c after the bitmap if checksums are disabled
 };
 
-inline bool operator < (const heap_object_lsn_t & a, const heap_object_lsn_t & b)
+struct __attribute__((__packed__)) heap_big_write_t
 {
-    return a.oid < b.oid || a.oid == b.oid && a.lsn < b.lsn;
-}
+    heap_entry_t hdr;
 
-struct tmp_compact_item_t
-{
-    object_id oid;
-    uint64_t lsn;
-    bool compact;
+    uint32_t block_num;
 };
 
-struct heap_mvcc_copy_id_t
+struct __attribute__((__packed__)) heap_big_intent_t
 {
-    object_id oid;
-    uint64_t copy_id;
+    heap_entry_t hdr;
+
+    uint32_t block_num;
+    uint32_t offset;
+    uint32_t len;
+
+    // Also includes 2 bitmaps and 1 crc32c if checksums are disabled
 };
 
-inline bool operator == (const heap_mvcc_copy_id_t & a, const heap_mvcc_copy_id_t & b)
+struct __attribute__((__packed__)) heap_list_item_t
 {
-    return a.oid.inode == b.oid.inode && a.oid.stripe == b.oid.stripe && a.copy_id == b.copy_id;
-}
-
-namespace std
-{
-    template<> struct hash<heap_mvcc_copy_id_t>
-    {
-        inline size_t operator()(const heap_mvcc_copy_id_t &s) const
-        {
-            size_t seed = std::hash<object_id>()(s.oid);
-            // Copy-pasted from spp::hash_combine()
-            seed ^= (s.copy_id + 0xc6a4a7935bd1e995 + (seed << 6) + (seed >> 2));
-            return seed;
-        }
-    };
+    heap_list_item_t *prev;
+    heap_list_item_t *next;
+    uint32_t block_num;
+    heap_entry_t entry;
 };
 
 struct heap_object_mvcc_t
 {
     uint32_t readers = 0;
-    heap_object_t *entry_copy = NULL;
+    heap_entry_t *garbage_entry = NULL;
 };
 
-struct __attribute__((__packed__)) heap_block_info_t
+struct heap_block_info_t
 {
     uint32_t used_space = 0;
-    uint32_t free_pos = 0;
-    uint8_t *data = NULL;
+    uint64_t mod_lsn = 0, mod_lsn_to = 0; // only 1 block write of LSN sequence is allowed at a moment
+    bool is_writing: 1;
+    bool has_garbage: 1;
+    std::vector<heap_list_item_t*> entries;
 };
 
 struct heap_inflight_lsn_t
 {
-    object_id oid;
     uint64_t flags;
+    heap_entry_t *wr;
 };
 
-struct heap_refqi_t
+struct heap_compact_t
 {
-    uint64_t lsn;
-    uint64_t inode;
-    uint64_t location;
-    uint32_t len;
-    bool is_data;
+    uint64_t compact_lsn, compact_version;
+    heap_entry_t *clean_wr;
+    bool do_delete;
+};
+
+struct heap_reshard_state_t;
+
+struct heap_li_hash
+{
+    size_t operator()(const heap_list_item_t* li) const noexcept
+    {
+        return robin_hood::hash_int(li->entry.stripe);
+    }
+};
+
+struct heap_li_equal
+{
+    constexpr bool operator()(const heap_list_item_t* a, const heap_list_item_t* b) const noexcept
+    {
+        return a->entry.stripe == b->entry.stripe;
+    }
 };
 
 using i64hash_t = robin_hood::hash<uint64_t>;
+using heap_inode_map_t = robin_hood::unordered_flat_set<heap_list_item_t*, heap_li_hash, heap_li_equal, 88>;
 using heap_block_index_t = robin_hood::unordered_flat_map<uint64_t,
-    robin_hood::unordered_flat_map<inode_t, robin_hood::unordered_flat_map<uint64_t, uint64_t, i64hash_t, std::equal_to<uint64_t>, 88>, i64hash_t>, i64hash_t>;
-using heap_mvcc_map_t = robin_hood::unordered_flat_map<heap_mvcc_copy_id_t, heap_object_mvcc_t>;
+    robin_hood::unordered_flat_map<inode_t, void*, i64hash_t>, i64hash_t>;
+using heap_mvcc_map_t = robin_hood::unordered_flat_map<object_id, heap_object_mvcc_t>;
 
 class blockstore_heap_t
 {
-    friend class heap_write_t;
-    friend class heap_object_t;
+    friend struct heap_entry_t;
 
     blockstore_disk_t *dsk = NULL;
     uint8_t* buffer_area = NULL;
-    bool abort_on_corruption = false;
-    bool abort_on_overlap = true;
     int log_level = 0;
-
     const uint32_t meta_block_count = 0;
-    uint32_t target_block_free_space = 800;
+    const uint32_t max_entry_size = 0;
 
-    uint64_t next_lsn = 0;
     robin_hood::unordered_flat_map<pool_id_t, pool_shard_settings_t> pool_shard_settings;
     // PG => inode => stripe => block number
     heap_block_index_t block_index;
     std::vector<heap_block_info_t> block_info;
     allocator_t *data_alloc = NULL;
     multilist_index_t *meta_alloc = NULL;
-    uint32_t meta_alloc_count = 0;
+    uint32_t meta_nearfull_blocks = 0;
     uint64_t meta_used_space = 0;
     multilist_alloc_t *buffer_alloc = NULL;
-    heap_mvcc_map_t object_mvcc;
-    std::unordered_map<uint64_t, uint32_t> mvcc_data_refs;
-    std::unordered_map<uint64_t, uint32_t> mvcc_buffer_refs;
     std::map<uint64_t, uint64_t> inode_space_stats;
     uint64_t buffer_area_used_space = 0;
     uint64_t data_used_space = 0;
 
-    // LSN queue: inflight (writing) -> completed [-> fsynced] -> compactable -> compacted [-> fsynced] -> trimmed and removed
+    uint64_t next_lsn = 0;
+    uint32_t last_allocated_block = UINT32_MAX;
+    heap_mvcc_map_t object_mvcc;
+
+    // LSN queue: inflight (writing) -> completed [-> fsynced]
     std::deque<heap_inflight_lsn_t> inflight_lsn;
     uint32_t to_compact_count = 0;
+    uint64_t compacted_count = 0;
+    uint32_t inflight_overwrite_count = 0;
     uint64_t first_inflight_lsn = 0;
     uint64_t completed_lsn = 0;
     uint64_t fsynced_lsn = 0;
-    uint64_t compacted_lsn = 0;
-    uint64_t next_compact_lsn = 0;
-    std::deque<heap_refqi_t> overwrite_ref_queue;
+    std::deque<object_id> compact_queue;
 
-    std::vector<tmp_compact_item_t> tmp_compact_queue;
-    std::deque<object_id> recheck_queue;
+    bool marked_used_blocks = false;
+    bool recheck_queue_filled = false;
+    std::vector<heap_list_item_t*> loaded_list_items;
+    std::set<uint32_t> recheck_modified_blocks;
+    std::deque<heap_entry_t*> recheck_queue;
     int recheck_in_progress = 0;
     bool in_recheck = false;
     std::function<void(bool is_data, uint64_t offset, uint64_t len, uint8_t* buf, std::function<void()>)> recheck_cb;
     int recheck_queue_depth = 0;
 
-    const uint32_t max_write_entry_size;
-
     uint64_t get_pg_id(inode_t inode, uint64_t stripe);
+    bool validate_object(heap_entry_t *obj);
+    void fill_recheck_queue();
+    int mark_used_blocks();
+    void recheck_buffer(heap_entry_t *cwr, uint8_t *buf);
     void defragment_block(uint32_t block_num);
-    uint32_t find_block_run(heap_block_info_t & block, uint32_t space);
-    uint32_t find_block_space(uint32_t block_num, uint32_t space);
-    uint32_t block_has_compactable(uint8_t *data);
-    uint32_t compact_object_to(heap_object_t *obj, uint64_t lsn, uint8_t *new_csums, bool do_free);
-    void copy_full_object(uint8_t *dst, heap_object_t *obj);
-    bool mvcc_save_copy(heap_object_t *obj);
-    bool mvcc_check_tracking(object_id oid);
-    void free_mvcc(heap_mvcc_map_t::iterator mvcc_it);
-    void allocate_block(heap_block_info_t & inf);
-    int allocate_new_object(object_id oid, uint32_t full_object_size, uint32_t *modified_block, heap_object_t **new_obj);
-    int add_object(object_id oid, heap_write_t *wr, uint32_t *modified_block);
-    void mark_overwritten(uint64_t over_lsn, uint64_t inode, heap_write_t *wr, heap_write_t *end_wr, bool tracking_active);
-    int update_object(uint32_t block_num, heap_object_t *obj, heap_write_t *wr, uint32_t *modified_block, uint32_t *moved_from_block);
-    void init_erase(uint32_t block_num, heap_object_t *obj);
-    void erase_object(uint32_t block_num, heap_object_t *obj, uint64_t lsn, bool tracking_active);
-    void reindex_block(uint32_t block_num, heap_object_t *from_obj);
-    void erase_block_index(inode_t inode, uint64_t stripe);
-    void deref_data(uint64_t inode, uint64_t location, bool free_at_0);
-    void deref_buffer(uint64_t inode, uint64_t location, uint32_t len, bool free_at_0);
-    void deref_overwrites(uint64_t lsn);
-    void free_object_space(inode_t inode, heap_write_t *from, heap_write_t *to, int mode = 0);
-    void add_used_space(uint32_t block_num, int32_t used_delta);
-    void push_inflight_lsn(object_id oid, uint64_t lsn, uint64_t flags);
+    void reshard_add(heap_reshard_state_t *st, heap_list_item_t *li);
 
+    void gc_block(heap_block_info_t & inf);
+    int allocate_entry(uint32_t entry_size, uint32_t *block_num, bool allow_last_free);
+    void insert_list_item(heap_list_item_t *li);
+    int add_entry(uint32_t wr_size, uint32_t *modified_block, bool allow_last_free,
+        bool explicit_complete, std::function<void(heap_entry_t *wr)> fill_entry);
+    int add_simple(heap_entry_t *obj, uint64_t version, uint32_t *modified_block, uint32_t entry_type);
+    uint32_t meta_alloc_pos(const heap_block_info_t & inf);
+    void modify_alloc(uint32_t block_num, std::function<void(heap_block_info_t &)> change_cb);
+    void mark_garbage_up_to(heap_entry_t *wr);
+    void mark_garbage(uint32_t block_num, heap_entry_t *prev_wr, uint32_t used_big);
+    void push_inflight_lsn(uint64_t lsn, heap_entry_t *wr, uint64_t flags);
+    void mark_completed_lsns(uint64_t mod_lsn);
+    void apply_inflight(heap_inflight_lsn_t & inflight);
 public:
     blockstore_heap_t(blockstore_disk_t *dsk, uint8_t *buffer_area, int log_level = 0);
     ~blockstore_heap_t();
-    // set initially compacted lsn - should be done before loading
-    void set_compacted_lsn(uint64_t compacted_lsn);
-    uint64_t get_compacted_lsn();
-    // load data from the disk, returns count of loaded write entries
-    void read_blocks(uint64_t disk_offset, uint64_t size, uint8_t *buf,
-        std::function<void(heap_object_t*)> handle_object, std::function<void(uint32_t, uint32_t, uint8_t*)> handle_block);
-    uint64_t load_blocks(uint64_t disk_offset, uint64_t size, uint8_t *buf);
-    // finish loading
+    void start_load(uint64_t completed_lsn);
+    // load data from the disk, returns EDOM on corruption
+    int read_blocks(uint64_t disk_offset, uint64_t size, uint8_t *buf, bool allow_corrupted,
+        std::function<void(uint32_t block_num, heap_entry_t* wr)> handle_write,
+        std::function<void(uint32_t, uint32_t, uint8_t*)> handle_block);
+    int load_blocks(uint64_t disk_offset, uint64_t size, uint8_t *buf,
+        bool allow_corrupted, uint64_t &entries_loaded);
+    // finish loading - should be called after load_blocks
     void finish_load();
+    // get blocks which are modified during loading and should be written to the disk
+    // before finishing initialization if not R/O
+    std::vector<uint32_t> get_recheck_modified_blocks();
     // recheck small write data after reading the database from disk
     bool recheck_small_writes(std::function<void(bool is_data, uint64_t offset, uint64_t len, uint8_t* buf, std::function<void()>)> read_buffer, int queue_depth);
-    // initialize metadata area (fill it with empty data)
-    // returns 0 when done, EAGAIN when the caller has to wait more
-    int initialize();
-    // read from the metadata area
-    // returns 0 when done, EAGAIN when the caller has to wait more
-    int read();
+    int finish_recheck();
     // reshard database according to the pool's PG count
-    void reshard(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size);
+    void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
+    bool reshard_continue(void* reshard_state, uint64_t chunk_limit);
+    bool reshard_check(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size);
+    void set_no_inode_stats(const std::vector<uint64_t> & pool_ids);
+    void recalc_inode_space_stats(uint64_t pool_id, bool per_inode);
     // read an object entry and lock it against removal
     // in the future, may become asynchronous
-    heap_object_t *lock_and_read_entry(object_id oid, uint64_t & copy_id);
-    // re-read a locked object entry with the given lsn (pointer may be invalidated)
-    heap_object_t *read_locked_entry(object_id oid, uint64_t copy_id);
+    heap_entry_t *lock_and_read_entry(object_id oid);
     // read an object entry without locking it
-    heap_object_t *read_entry(object_id oid, uint32_t *block_num_ptr, bool for_update = false);
+    heap_entry_t *read_entry(object_id oid);
     // unlock an entry
-    bool unlock_entry(object_id oid, uint64_t copy_id);
+    bool unlock_entry(object_id oid);
     // set or verify checksums in a write request
-    bool calc_checksums(heap_write_t *wr, uint8_t *data, bool set, uint32_t offset = 0, uint32_t len = 0);
+    bool calc_checksums(heap_entry_t *wr, uint8_t *data, bool set, uint32_t offset = UINT32_MAX, uint32_t len = UINT32_MAX);
     // set or verify raw block checksums
     bool calc_block_checksums(uint32_t *block_csums, uint8_t *data, uint8_t *bitmap, uint32_t start, uint32_t end,
         bool set, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);
     bool calc_block_checksums(uint32_t *block_csums, uint8_t *bitmap,
         uint32_t start, uint32_t end, std::function<uint8_t*(uint32_t start, uint32_t & len)> next,
         bool set, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);
-    // copy an object as is
-    int copy_object(heap_object_t *obj, uint32_t *modified_block);
-    // auto-compacts the object, then adds a write entry to it and to the compaction queue
+    // adds a small_write or intent_write entry to an object
     // return 0 if OK, or maybe ENOSPC
-    int post_write(object_id oid, heap_write_t *wr, uint32_t *modified_block, uint32_t *moved_from_block);
-    int post_write(uint32_t & block_num, object_id oid, heap_object_t *obj, heap_write_t *wr, uint32_t *moved_from_block);
+    int add_small_write(object_id oid, heap_entry_t **obj_ptr, uint16_t type, uint64_t version,
+        uint32_t offset, uint32_t len, uint64_t location, uint8_t *bitmap, uint8_t *data, uint32_t *modified_block);
+    // adds a big_write (overwrite) entry to an object
+    int add_big_write(object_id oid, heap_entry_t *old_head, bool stable, uint64_t version,
+        uint32_t offset, uint32_t len, uint64_t location, uint8_t *bitmap, uint8_t *data, uint32_t *modified_block);
+    // adds a "redirecting" big_intent entry to an object (same as big_write, used to avoid fsync on desktop SSDs)
+    int add_redirect_intent(object_id oid, heap_entry_t **obj_ptr, uint64_t version,
+        uint32_t offset, uint32_t len, uint64_t location, uint8_t *bitmap, uint8_t *data, uint32_t *modified_block);
+    // adds a big_intent (atomic partial modification) entry to an object
+    int add_big_intent(object_id oid, heap_entry_t **obj_ptr, uint64_t version,
+        uint32_t offset, uint32_t len, uint8_t *bitmap, uint8_t *data, uint8_t *checksums, uint32_t *modified_block);
+    // adds a compacted up to <version> entry to an object
+    int add_compact(heap_entry_t *obj, uint64_t compact_version, uint64_t compact_lsn, uint64_t compact_location,
+        bool do_delete, uint32_t *modified_block, uint8_t *new_int_bitmap, uint8_t *new_ext_bitmap, uint8_t *new_csums);
+    // "punch holes" in a big_entry
+    int punch_holes(heap_entry_t *wr, uint8_t *new_bitmap, uint8_t *new_csums, uint32_t *modified_block);
     // stabilize an unstable object version
     // return 0 if OK, ENOENT if not exists
-    int post_stabilize(object_id oid, uint64_t version, uint32_t *modified_block, uint64_t *new_lsn, uint64_t *new_to_lsn);
+    int add_commit(heap_entry_t *obj, uint64_t version, uint32_t *modified_block);
     // rollback an unstable object version
     // return 0 if OK, ENOENT if not exists, EBUSY if already stable
-    int post_rollback(object_id oid, uint64_t version, uint64_t *new_lsn, uint32_t *modified_block);
+    int add_rollback(heap_entry_t *obj, uint64_t version, uint32_t *modified_block);
     // forget an object
     // return error code
-    int post_delete(object_id oid, uint64_t *new_lsn, uint32_t *modified_block);
-    int post_delete(uint32_t block_num, heap_object_t *obj, uint64_t *new_lsn);
+    int add_delete(heap_entry_t *obj, uint32_t *modified_block);
     // get the next object to compact
     // guaranteed to return objects in min lsn order
     // returns 0 if OK, ENOENT if nothing to compact
     int get_next_compact(object_id & oid);
-    // get the range of an object eligible for compaction
-    void get_compact_range(heap_object_t *obj, uint64_t max_lsn, heap_write_t **begin_wr, heap_write_t **end_wr);
-    // mark an object as compacted up to the given lsn
-    int compact_object(object_id oid, uint64_t lsn, uint8_t *new_csums);
+    void iterate_with_stable(heap_entry_t *obj, uint64_t max_lsn, std::function<bool(heap_entry_t*, bool stable)> cb);
+    // iterate compactable entries
+    heap_compact_t iterate_compaction(heap_entry_t *obj, uint64_t fsynced_lsn, bool under_pressure,
+        std::function<void(heap_entry_t*)> small_wr_cb);
+    // iterate all objects
+    void iterate_objects(std::function<void(heap_entry_t*, uint32_t block_num)> cb);
     // retrieve object listing from a PG
     int list_objects(uint32_t pg_num, object_id min_oid, object_id max_oid,
         obj_ver_id **result_list, size_t *stable_count, size_t *unstable_count);
-    // set a block number for a new object and returns error status: 0, EAGAIN or ENOSPC
-    int get_block_for_new_object(uint32_t & out_block_num, uint32_t size = 0);
 
     // inflight write tracking
-    void mark_lsn_completed(uint64_t lsn);
-    void mark_lsn_fsynced(uint64_t lsn);
-    void mark_lsn_compacted(uint64_t lsn, bool allow_undone = false);
-    void mark_object_compacted(heap_object_t *obj, uint64_t max_lsn);
-    void mark_lsn_trimmed(uint64_t lsn);
+    void start_block_write(uint32_t block_num);
+    void complete_block_write(uint32_t block_num);
+    void complete_lsn_write(uint64_t lsn);
+    bool is_lsn_completed(uint64_t lsn);
     uint64_t get_completed_lsn();
     uint64_t get_fsynced_lsn();
+    void mark_lsn_fsynced(uint64_t lsn);
 
     // data device block allocator functions
     uint64_t find_free_data();
@@ -354,7 +332,8 @@ public:
     uint64_t get_buffer_area_used_space();
 
     // get metadata block data buffer and used space
-    uint8_t *get_meta_block(uint32_t block_num);
+    void get_meta_block(uint32_t block_num, uint8_t *buffer);
+    void fill_block_empty_space(uint8_t *buffer, uint32_t pos);
     uint32_t get_meta_block_used_space(uint32_t block_num);
 
     // get space usage statistics
@@ -363,14 +342,17 @@ public:
     uint64_t get_meta_total_space();
     uint64_t get_meta_used_space();
     uint32_t get_meta_nearfull_blocks();
-    uint32_t get_inflight_queue_size();
     uint32_t get_compact_queue_size();
     uint32_t get_to_compact_count();
+    uint64_t get_compacted_count();
 
-    // get maximum size for a temporary heap_write_t buffer
-    uint32_t get_max_write_entry_size();
-
-    // only for tests
-    void set_abort_on_corruption(bool fail);
-    void set_abort_on_overlap(bool fail);
+    uint64_t entry_pos(uint32_t block_num, uint32_t offset);
+    heap_entry_t *entry_from_pos(uint64_t entry_pos, bool allow_unallocated = false);
+    heap_entry_t *prev(heap_entry_t *wr);
+    uint32_t get_simple_entry_size();
+    uint32_t get_big_entry_size();
+    uint32_t get_big_intent_entry_size();
+    uint32_t get_small_entry_size(uint32_t offset, uint32_t len);
+    uint32_t get_csum_size(heap_entry_t *wr);
+    uint32_t get_csum_size(uint32_t entry_type, uint32_t offset = 0, uint32_t len = 0);
 };

src/blockstore/blockstore_impl.cpp

@@ -23,26 +23,22 @@ blockstore_impl_t::blockstore_impl_t(blockstore_config_t & config, ring_loop_i *
         dsk.open_meta();
         dsk.open_journal();
         dsk.calc_lengths();
-        zero_object = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.data_block_size);
+        dsk.check_lengths();
     }
     catch (std::exception & e)
     {
         dsk.close_all();
         throw;
     }
-    memset(zero_object, 0, dsk.data_block_size);
     meta_superblock = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size);
     memset(meta_superblock, 0, dsk.meta_block_size);
-}
-
-void blockstore_impl_t::init()
-{
     flusher = new journal_flusher_t(this);
     if (dsk.inmemory_journal)
     {
         buffer_area = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.journal_len);
     }
     heap = new blockstore_heap_t(&dsk, buffer_area, log_level);
+    ringloop->wakeup();
 }
 
 blockstore_impl_t::~blockstore_impl_t()
@@ -55,8 +51,6 @@ blockstore_impl_t::~blockstore_impl_t()
         free(buffer_area);
     if (meta_superblock)
         free(meta_superblock);
-    if (zero_object)
-        free(zero_object);
     ringloop->unregister_consumer(&ring_consumer);
     dsk.close_all();
 }
@@ -192,11 +186,19 @@ void blockstore_impl_t::loop()
         {
             flusher->loop();
         }
+        for (auto & block_num: pending_modified_blocks)
+        {
+            auto & mb = modified_blocks[block_num];
+            heap->get_meta_block(block_num, mb.buf);
+            heap->start_block_write(block_num);
+            mb.sent = true;
+        }
         int ret = ringloop->submit();
         if (ret < 0)
         {
             throw std::runtime_error(std::string("io_uring_submit: ") + strerror(-ret));
         }
+        pending_modified_blocks.clear();
         if ((initial_ring_space - ringloop->space_left()) > 0)
         {
             live = true;
@@ -214,7 +216,7 @@ bool blockstore_impl_t::is_safe_to_stop()
     {
         return false;
     }
-    if (unsynced_big_write_count > 0 || unsynced_small_write_count > 0)
+    if (has_unsynced())
     {
         if (!readonly && !stop_sync_submitted)
         {
@@ -250,7 +252,7 @@ void blockstore_impl_t::check_wait(blockstore_op_t *op)
     }
     else if (PRIV(op)->wait_for == WAIT_COMPACTION)
     {
-        if (flusher->get_compact_counter() <= PRIV(op)->wait_detail)
+        if (heap->get_compacted_count() <= PRIV(op)->wait_detail)
         {
             // do not submit
 #ifdef BLOCKSTORE_DEBUG
@@ -273,7 +275,8 @@ void blockstore_impl_t::enqueue_op(blockstore_op_t *op)
         ((op->opcode == BS_OP_READ || op->opcode == BS_OP_WRITE || op->opcode == BS_OP_WRITE_STABLE) && (
             op->offset >= dsk.data_block_size ||
             op->len > dsk.data_block_size-op->offset ||
-            (op->len % dsk.disk_alignment)
+            (op->offset % dsk.bitmap_granularity) ||
+            (op->len % dsk.bitmap_granularity)
         )) ||
         readonly && op->opcode != BS_OP_READ && op->opcode != BS_OP_LIST)
     {
@@ -320,9 +323,13 @@ void blockstore_impl_t::process_list(blockstore_op_t *op)
         FINISH_OP(op);
         return;
     }
-    // Check if the DB needs resharding
-    // (we don't know about PGs from the beginning, we only create "shards" here)
-    heap->reshard(INODE_POOL(min_inode), pg_count, pg_stripe_size);
+    // Check if the DB is sharded correctly
+    if (!heap->reshard_check(INODE_POOL(min_inode), pg_count, pg_stripe_size))
+    {
+        op->retval = -EAGAIN;
+        FINISH_OP(op);
+        return;
+    }
     obj_ver_id *result = NULL;
     size_t stable_count = 0, unstable_count = 0;
     int res = heap->list_objects(list_pg, op->min_oid, op->max_oid, &result, &stable_count, &unstable_count);
@@ -345,6 +352,7 @@ void blockstore_impl_t::process_list(blockstore_op_t *op)
 
 void blockstore_impl_t::set_no_inode_stats(const std::vector<uint64_t> & pool_ids)
 {
+    heap->set_no_inode_stats(pool_ids);
 }
 
 void blockstore_impl_t::dump_diagnostics()
@@ -388,3 +396,13 @@ std::string blockstore_impl_t::get_op_diag(blockstore_op_t *op)
         snprintf(buf, sizeof(buf), "state=%d", priv->op_state);
     return std::string(buf);
 }
+
+void* blockstore_impl_t::reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit)
+{
+    return heap->reshard_start(pool, pg_count, pg_stripe_size, chunk_limit);
+}
+
+bool blockstore_impl_t::reshard_continue(void *reshard_state, uint64_t chunk_limit)
+{
+    return heap->reshard_continue(reshard_state, chunk_limit);
+}

src/blockstore/blockstore_impl.h

@@ -29,18 +29,6 @@ class blockstore_impl_t;
 
 //#define BLOCKSTORE_DEBUG
 
-// - Sync must be submitted after previous writes/deletes (not before!)
-// - Reads may be submitted in parallel with writes/deletes because we use MVCC
-// - Writes may be submitted in any order, because they don't overlap. Each write
-//   goes into a new location - either on the journal device or on the data device
-// - Stable (stabilize) must be submitted after sync of that object is completed
-//   It's even OK to return an error to the caller if that object is not synced yet
-// - compacted_lsn should be moved forward only after all versions are moved to the main storage
-// - If an operation can not be submitted because the ring is full
-//   we should stop submission of other operations. Otherwise some "scatter" reads
-//   may end up blocked for a long time.
-// Otherwise, the submission order is free.
-
 #include "blockstore_init.h"
 
 #include "blockstore_flush.h"
@@ -53,27 +41,32 @@ struct blockstore_op_private_t
     int pending_ops;
     int op_state;
 
-    // Read, write, sync, stabilize
-    uint64_t lsn;
+    // Write, sync, stabilize
+    uint32_t modified_block, modified_block2;
 
     // Read
     std::vector<copy_buffer_t> read_vec;
 
+    // Read, write
+    uint64_t lsn;
+
     // Write
     uint64_t location;
-    bool is_big;
+    uint32_t write_type;
 
     // Stabilize, rollback
     int stab_pos;
 
-    // Stabilize
-    uint64_t to_lsn;
-
     // Write
-    struct iovec iov_zerofill[3];
     timespec tv_begin;
 };
 
+struct bs_modified_block_t
+{
+    bool sent;
+    uint8_t *buf;
+};
+
 class blockstore_impl_t: public blockstore_i
 {
 public:
@@ -85,6 +78,7 @@ public:
     // Suitable only for server SSDs with capacitors, requires disabled data and journal fsyncs
     int immediate_commit = IMMEDIATE_NONE;
     bool inmemory_meta = false;
+    bool skip_corrupted_meta_entries = false;
     uint32_t meta_write_recheck_parallelism = 0;
     // Maximum and minimum flusher count
     unsigned max_flusher_count = 0, min_flusher_count = 0;
@@ -115,13 +109,16 @@ public:
     uint8_t* meta_superblock = NULL;
     uint8_t *buffer_area = NULL;
     std::vector<blockstore_op_t*> submit_queue;
-    int unsynced_big_write_count = 0, unsynced_small_write_count = 0, unsynced_meta_write_count = 0;
+    int unsynced_data_write_count = 0, unsynced_buffer_write_count = 0, unsynced_meta_write_count = 0;
     int unsynced_queued_ops = 0;
-    uint8_t *zero_object = NULL;
+
+    std::vector<uint32_t> pending_modified_blocks;
+    robin_hood::unordered_flat_map<uint32_t, bs_modified_block_t> modified_blocks;
 
     journal_flusher_t *flusher;
     int write_iodepth = 0;
     int inflight_big = 0;
+    int intent_write_counter = 0;
     bool fsyncing_data = false;
 
     bool live = false, queue_stall = false;
@@ -146,18 +143,17 @@ public:
     int metadata_buf_size;
     blockstore_init_meta* metadata_init_reader;
 
-    void init();
     void check_wait(blockstore_op_t *op);
     void init_op(blockstore_op_t *op);
 
     // Read
     int dequeue_read(blockstore_op_t *op);
     int fulfill_read(blockstore_op_t *op);
-    uint32_t prepare_read(std::vector<copy_buffer_t> & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end);
-    uint32_t prepare_read_with_bitmaps(std::vector<copy_buffer_t> & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end);
+    uint32_t prepare_read(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end, uint32_t skip_csum);
+    uint32_t prepare_read_with_bitmaps(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end, uint32_t skip_csum);
     uint32_t prepare_read_zero(std::vector<copy_buffer_t> & read_vec, uint32_t start, uint32_t end);
-    uint32_t prepare_read_simple(std::vector<copy_buffer_t> & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end);
-    void prepare_disk_read(std::vector<copy_buffer_t> & read_vec, int pos, heap_object_t *obj, heap_write_t *wr,
+    uint32_t prepare_read_simple(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end, uint32_t skip_csum);
+    void prepare_disk_read(std::vector<copy_buffer_t> & read_vec, int pos, heap_entry_t *obj, heap_entry_t *wr,
         uint32_t blk_start, uint32_t blk_end, uint32_t start, uint32_t end, uint32_t copy_flags);
     void find_holes(std::vector<copy_buffer_t> & read_vec, uint32_t item_start, uint32_t item_end,
         std::function<void(int&, uint32_t, uint32_t)> callback);
@@ -167,9 +163,10 @@ public:
 
     // Write
     bool enqueue_write(blockstore_op_t *op);
-    void prepare_meta_block_write(blockstore_op_t *op, uint64_t modified_block, io_uring_sqe *sqe = NULL);
+    void prepare_meta_block_write(uint32_t modified_block);
+    bool meta_block_is_pending(uint32_t modified_block);
+    bool intent_write_allowed(blockstore_op_t *op, heap_entry_t *obj);
     int dequeue_write(blockstore_op_t *op);
-    int make_big_write(blockstore_op_t *op, uint32_t offset, uint32_t len, uint32_t *modified_block, uint32_t *moved_from_block);
     int continue_write(blockstore_op_t *op);
     void handle_write_event(ring_data_t *data, blockstore_op_t *op);
 
@@ -177,6 +174,7 @@ public:
     int continue_sync(blockstore_op_t *op);
     bool submit_fsyncs(int & wait_count);
     int do_sync(blockstore_op_t *op, int base_state);
+    bool has_unsynced();
 
     // Stabilize
     int dequeue_stable(blockstore_op_t *op);
@@ -192,6 +190,9 @@ public:
     void parse_config(blockstore_config_t & config);
     void parse_config(blockstore_config_t & config, bool init);
 
+    void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
+    bool reshard_continue(void *reshard_state, uint64_t chunk_limit);
+
     // Event loop
     void loop();
 
@@ -226,6 +227,6 @@ public:
     inline uint32_t get_block_size() { return dsk.data_block_size; }
     inline uint64_t get_block_count() { return dsk.block_count; }
     uint64_t get_free_block_count();
-    inline uint32_t get_bitmap_granularity() { return dsk.disk_alignment; }
+    inline uint32_t get_bitmap_granularity() { return dsk.bitmap_granularity; }
     inline uint64_t get_journal_size() { return dsk.journal_len; }
 };

src/blockstore/blockstore_init.cpp

@@ -51,6 +51,8 @@ int blockstore_init_meta::loop()
     else if (wait_state == 5) goto resume_5;
     else if (wait_state == 6) goto resume_6;
     else if (wait_state == 7) goto resume_7;
+    else if (wait_state == 8) goto resume_8;
+    else if (wait_state == 9) goto resume_9;
     metadata_buffer = memalign(MEM_ALIGNMENT, 2*bs->metadata_buf_size);
     if (!metadata_buffer)
         throw std::runtime_error("Failed to allocate metadata read buffer");
@@ -70,7 +72,6 @@ resume_1:
     }
     if (is_zero((uint64_t*)bs->meta_superblock, bs->dsk.meta_block_size))
     {
-        bs->dsk.check_lengths();
         {
             blockstore_meta_header_v3_t *hdr = (blockstore_meta_header_v3_t *)bs->meta_superblock;
             hdr->zero = 0;
@@ -79,12 +80,15 @@ resume_1:
             hdr->meta_block_size = bs->dsk.meta_block_size;
             hdr->data_block_size = bs->dsk.data_block_size;
             hdr->bitmap_granularity = bs->dsk.bitmap_granularity;
-            hdr->compacted_lsn = 0;
             if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_V2)
             {
                 hdr->data_csum_type = bs->dsk.data_csum_type;
                 hdr->csum_block_size = bs->dsk.csum_block_size;
             }
+            if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_HEAP)
+            {
+                hdr->meta_area_size = bs->dsk.meta_area_size;
+            }
             hdr->set_crc32c();
         }
         if (bs->readonly)
@@ -135,28 +139,22 @@ resume_1:
             hdr->data_block_size != bs->dsk.data_block_size ||
             hdr->bitmap_granularity != bs->dsk.bitmap_granularity ||
             hdr->data_csum_type != bs->dsk.data_csum_type ||
-            hdr->csum_block_size != bs->dsk.csum_block_size)
+            hdr->csum_block_size != bs->dsk.csum_block_size ||
+            hdr->meta_area_size != bs->dsk.meta_area_size)
         {
             printf(
                 "Configuration stored in metadata superblock"
-                " (meta_block_size=%u, data_block_size=%u, bitmap_granularity=%u, data_csum_type=%u, csum_block_size=%u)"
-                " differs from OSD configuration (%u/%u/%u, %u/%u).\n",
+                " (meta_block_size=%u, data_block_size=%u, bitmap_granularity=%u, data_csum_type=%u, csum_block_size=%u, meta_area_size=%ju)"
+                " differs from OSD configuration (%u/%u/%u, %u/%u, %ju).\n",
                 hdr->meta_block_size, hdr->data_block_size, hdr->bitmap_granularity,
-                hdr->data_csum_type, hdr->csum_block_size,
+                hdr->data_csum_type, hdr->csum_block_size, hdr->meta_area_size,
                 bs->dsk.meta_block_size, bs->dsk.data_block_size, bs->dsk.bitmap_granularity,
-                bs->dsk.data_csum_type, bs->dsk.csum_block_size
+                bs->dsk.data_csum_type, bs->dsk.csum_block_size, bs->dsk.meta_area_size
             );
             exit(1);
         }
-        if (bs->dsk.meta_format != hdr->version)
-        {
-            bs->dsk.meta_format = hdr->version;
-            bs->dsk.calc_lengths();
-        }
-        bs->dsk.check_lengths();
     }
-    bs->init();
-    bs->heap->set_compacted_lsn(((blockstore_meta_header_v3_t *)bs->meta_superblock)->compacted_lsn);
+    bs->heap->start_load(((blockstore_meta_header_v3_t *)bs->meta_superblock)->completed_lsn);
     if (bs->dsk.inmemory_journal)
     {
         // Read buffer area
@@ -210,8 +208,10 @@ resume_4:
                     io_uring_prep_readv(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bufs[i].offset);
                 else
                 {
-                    // Fill metadata with zeroes
-                    memset(data->iov.iov_base, 0, data->iov.iov_len);
+                    // Fill metadata with empty block pattern
+                    memset(bufs[i].buf, 0, bufs[i].size);
+                    for (uint64_t o = 0; o < bufs[i].size; o += bs->dsk.meta_block_size)
+                        bs->heap->fill_block_empty_space(bufs[i].buf + o, 0);
                     io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bufs[i].offset);
                 }
                 bs->ringloop->submit();
@@ -224,7 +224,11 @@ resume_4:
         if (bufs[i].state == INIT_META_READ_DONE)
         {
             // Handle result
-            entries_loaded += bs->heap->load_blocks(bufs[i].offset-bs->dsk.meta_block_size, bufs[i].size, bufs[i].buf);
+            uint64_t loaded = 0;
+            int r = bs->heap->load_blocks(bufs[i].offset-bs->dsk.meta_block_size, bufs[i].size, bufs[i].buf, bs->skip_corrupted_meta_entries, loaded);
+            if (r != 0)
+                exit(1);
+            entries_loaded += loaded;
             bufs[i].state = 0;
             bs->ringloop->wakeup();
         }
@@ -235,6 +239,7 @@ resume_4:
         return 1;
     }
     // metadata read finished
+    bs->heap->finish_load();
     printf("Metadata entries loaded: %ju, used blocks: %ju / %ju\n", entries_loaded, bs->heap->get_data_used_space() / bs->dsk.data_block_size, bs->dsk.block_count);
     if (zero_on_init && !bs->dsk.disable_meta_fsync)
     {
@@ -265,7 +270,7 @@ resume_6:
         }
         GET_SQE();
         data->iov = (iovec){ buf, len };
-        data->callback = [this, offset, cb](ring_data_t *data)
+        data->callback = [offset, cb](ring_data_t *data)
         {
             if (data->res < 0)
             {
@@ -280,7 +285,51 @@ resume_6:
     }, bs->meta_write_recheck_parallelism);
     return 1;
 resume_7:
-    bs->heap->finish_load();
+    if (bs->heap->finish_recheck() != 0)
+    {
+        exit(1);
+    }
+    recheck_mod = bs->heap->get_recheck_modified_blocks();
+    if (bs->readonly)
+    {
+        recheck_mod.clear();
+    }
+    for (i = 0; i < recheck_mod.size(); i++)
+    {
+resume_8:
+        if (wait_count >= bs->meta_write_recheck_parallelism || !(sqe = bs->get_sqe()))
+        {
+            bs->ringloop->submit();
+            wait_state = 8;
+            return 1;
+        }
+        uint32_t block_num = recheck_mod[i];
+        uint64_t block_offset = bs->dsk.meta_offset + (uint64_t)(block_num+1) * bs->dsk.meta_block_size;
+        data = ((ring_data_t*)sqe->user_data);
+        uint8_t *buf = (uint8_t*)malloc_or_die(bs->dsk.meta_block_size);
+        bs->heap->get_meta_block(block_num, buf);
+        data->iov = { buf, bs->dsk.meta_block_size };
+        data->callback = [this, buf, block_offset](ring_data_t *data)
+        {
+            wait_count--;
+            free(buf);
+            if (data->res != bs->dsk.meta_block_size)
+            {
+                throw std::runtime_error(
+                    "write metadata failed at offset " + std::to_string(block_offset) + ": " + strerror(-data->res)
+                );
+            }
+        };
+        io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, block_offset);
+        wait_count++;
+    }
+resume_9:
+    if (wait_count > 0)
+    {
+        bs->ringloop->submit();
+        wait_state = 9;
+        return 1;
+    }
     free(metadata_buffer);
     metadata_buffer = NULL;
     return 0;

src/blockstore/blockstore_init.h

@@ -15,6 +15,7 @@ class blockstore_init_meta
 {
     blockstore_impl_t *bs;
     int wait_state = 0;
+    int wait_count = 0;
     bool zero_on_init = false;
     void *metadata_buffer = NULL;
     blockstore_init_meta_buf bufs[2] = {};
@@ -25,6 +26,7 @@ class blockstore_init_meta
     uint64_t next_offset = 0;
     uint64_t last_read_offset = 0;
     uint64_t entries_loaded = 0;
+    std::vector<uint32_t> recheck_mod;
     int i = 0, j = 0;
     bool handle_meta_block(uint8_t *buf, uint64_t count, uint64_t done_cnt);
     void handle_event(ring_data_t *data, int buf_num);

src/blockstore/blockstore_internal.h

@@ -1,3 +1,6 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 (see README.md for details)
+
 #pragma once
 
 #define BS_SUBMIT_CHECK_SQES(n) \
@@ -48,3 +51,11 @@
 #define COPY_BUF_COALESCED  0x10
 #define COPY_BUF_PADDED     0x20
 #define COPY_BUF_SKIP_CSUM  0x40
+
+#ifndef RWF_ATOMIC
+#define RWF_ATOMIC 0x40
+#endif
+
+#ifndef RWF_DSYNC
+#define RWF_DSYNC 0x02
+#endif

src/blockstore/blockstore_open.cpp

@@ -28,6 +28,7 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init)
     throttle_target_parallelism = strtoull(config["throttle_target_parallelism"].c_str(), NULL, 10);
     throttle_threshold_us = strtoull(config["throttle_threshold_us"].c_str(), NULL, 10);
     perfect_csum_update = config["perfect_csum_update"] == "true" || config["perfect_csum_update"] == "1" || config["perfect_csum_update"] == "yes";
+    skip_corrupted_meta_entries = config["skip_corrupted_meta_entries"] == "true" || config["skip_corrupted_meta_entries"] == "1" || config["skip_corrupted_meta_entries"] == "yes";
     if (config["autosync_writes"] != "")
     {
         autosync_writes = strtoull(config["autosync_writes"].c_str(), NULL, 10);
@@ -42,7 +43,7 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init)
     }
     if (!journal_trim_interval)
     {
-        journal_trim_interval = 1024;
+        journal_trim_interval = 4096;
     }
     if (!flusher_start_threshold)
     {
@@ -96,6 +97,10 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init)
     {
         metadata_buf_size = 4*1024*1024;
     }
+    if (metadata_buf_size % dsk.meta_block_size)
+    {
+        throw std::runtime_error("metadata_buf_size should be a multiple of meta_block_size");
+    }
     if (!meta_write_recheck_parallelism)
     {
         meta_write_recheck_parallelism = 16;

src/blockstore/blockstore_read.cpp

@@ -7,7 +7,7 @@
 
 int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
 {
-    heap_object_t *obj = heap->lock_and_read_entry(op->oid, PRIV(op)->lsn);
+    heap_entry_t *obj = heap->lock_and_read_entry(op->oid);
     if (!obj)
     {
         op->version = 0;
@@ -17,16 +17,44 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
     }
     uint32_t fulfilled = 0;
     PRIV(op)->pending_ops = 0;
+    PRIV(op)->lsn = 0;
     auto & rv = PRIV(op)->read_vec;
     uint64_t result_version = 0;
     bool found = false;
-    for (auto wr = obj->get_writes(); wr; wr = wr->next())
+    uint32_t skip_csum = 0;
+    uint32_t blk_start = op->offset, blk_end = op->offset+op->len;
+    bool need_skip = dsk.csum_block_size > dsk.bitmap_granularity && !perfect_csum_update;
+    if (need_skip)
     {
-        if (op->version < wr->version)
+        PRIV(op)->lsn = obj->lsn;
+        blk_start = op->offset - op->offset%dsk.csum_block_size;
+        blk_end = op->offset + op->len;
+        if (blk_end % dsk.csum_block_size)
+            blk_end += dsk.csum_block_size - (blk_end % dsk.csum_block_size);
+    }
+    bool need_wait = false;
+    heap->iterate_with_stable(obj, obj->lsn, [&](heap_entry_t *wr, bool stable)
+    {
+        if (wr->type() == BS_HEAP_DELETE)
         {
-            continue;
+            return false;
         }
-        if (!found)
+        if (!heap->is_lsn_completed(wr->lsn))
+        {
+            if (wr->type() == BS_HEAP_BIG_INTENT && wr->big_intent().offset < blk_end && wr->big_intent().offset+wr->big_intent().len > blk_start ||
+                wr->type() == BS_HEAP_INTENT_WRITE && wr->small().offset < blk_end && wr->small().offset+wr->small().len > blk_start)
+            {
+                // Wait until intent write is completed
+                need_wait = true;
+                return false;
+            }
+            else if (wr->type() == BS_HEAP_SMALL_WRITE && wr->small().offset < blk_end && wr->small().offset+wr->small().len > blk_start)
+            {
+                // Skip entry and read the previous one
+                return true;
+            }
+        }
+        if (op->version >= wr->version && !found)
         {
             found = true;
             result_version = wr->version;
@@ -35,18 +63,39 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
                 memcpy(op->bitmap, wr->get_ext_bitmap(heap), dsk.clean_entry_bitmap_size);
             }
         }
-        fulfilled += prepare_read(PRIV(op)->read_vec, obj, wr, op->offset, op->offset+op->len);
-        if (fulfilled == op->len ||
-            wr->type() == BS_HEAP_BIG_WRITE ||
-            wr->type() == BS_HEAP_TOMBSTONE)
+        if (op->version >= wr->version)
         {
-            break;
+            fulfilled += prepare_read(PRIV(op)->read_vec, obj, wr, op->offset, op->offset+op->len,
+                wr->type() != BS_HEAP_SMALL_WRITE ? skip_csum : 0);
+            if (fulfilled == op->len ||
+                wr->type() == BS_HEAP_BIG_WRITE ||
+                wr->type() == BS_HEAP_BIG_INTENT ||
+                wr->type() == BS_HEAP_DELETE)
+            {
+                return false;
+            }
         }
+        if (need_skip && wr->type() == BS_HEAP_SMALL_WRITE &&
+            wr->small().offset < blk_end && wr->small().offset+wr->small().len > blk_start)
+        {
+            // Small write may mutate big write checksums during flush
+            skip_csum = COPY_BUF_SKIP_CSUM;
+        }
+        return true;
+    });
+    if (need_wait)
+    {
+undo_wait:
+        // Need to wait. undo added requests, unlock lsn
+        heap->unlock_entry(op->oid);
+        free_read_buffers(rv);
+        rv.clear();
+        return 0;
     }
     if (!found)
     {
         // May happen if there are entries but all of them are > requested version
-        heap->unlock_entry(op->oid, PRIV(op)->lsn);
+        heap->unlock_entry(op->oid);
         op->version = 0;
         op->retval = -ENOENT;
         FINISH_OP(op);
@@ -55,17 +104,13 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
     assert(fulfilled == op->len);
     if (!fulfill_read(op))
     {
-        // Need to wait. undo added requests, unlock lsn
-        heap->unlock_entry(op->oid, PRIV(op)->lsn);
-        free_read_buffers(rv);
-        rv.clear();
-        return 0;
+        goto undo_wait;
     }
     op->version = result_version;
     if (!PRIV(op)->pending_ops)
     {
         // everything is fulfilled from memory
-        heap->unlock_entry(op->oid, PRIV(op)->lsn);
+        heap->unlock_entry(op->oid);
         op->retval = op->len;
         free_read_buffers(rv);
         FINISH_OP(op);
@@ -108,20 +153,20 @@ int blockstore_impl_t::fulfill_read(blockstore_op_t *op)
     return 1;
 }
 
-uint32_t blockstore_impl_t::prepare_read(std::vector<copy_buffer_t> & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end)
+uint32_t blockstore_impl_t::prepare_read(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end, uint32_t skip_csum)
 {
-    if (wr->type() == BS_HEAP_BIG_WRITE)
+    if (wr->type() == BS_HEAP_BIG_WRITE || wr->type() == BS_HEAP_BIG_INTENT)
     {
-        return prepare_read_with_bitmaps(read_vec, obj, wr, start, end);
+        return prepare_read_with_bitmaps(read_vec, obj, wr, start, end, skip_csum);
     }
-    if (wr->type() == BS_HEAP_TOMBSTONE)
+    if (wr->type() == BS_HEAP_DELETE)
     {
         return prepare_read_zero(read_vec, start, end);
     }
-    return prepare_read_simple(read_vec, obj, wr, start, end);
+    return prepare_read_simple(read_vec, obj, wr, start, end, skip_csum);
 }
 
-uint32_t blockstore_impl_t::prepare_read_with_bitmaps(std::vector<copy_buffer_t> & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end)
+uint32_t blockstore_impl_t::prepare_read_with_bitmaps(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end, uint32_t skip_csum)
 {
     // BIG_WRITEs contain a bitmap and we have to handle its holes
     uint32_t res = 0;
@@ -144,7 +189,7 @@ uint32_t blockstore_impl_t::prepare_read_with_bitmaps(std::vector<copy_buffer_t>
         }
         if (bmp_end > bmp_start)
         {
-            res += prepare_read_simple(read_vec, obj, wr, bmp_start * dsk.bitmap_granularity, bmp_end * dsk.bitmap_granularity);
+            res += prepare_read_simple(read_vec, obj, wr, bmp_start * dsk.bitmap_granularity, bmp_end * dsk.bitmap_granularity, skip_csum);
             bmp_start = bmp_end;
         }
     }
@@ -166,7 +211,7 @@ uint32_t blockstore_impl_t::prepare_read_zero(std::vector<copy_buffer_t> & read_
     return res;
 }
 
-uint32_t blockstore_impl_t::prepare_read_simple(std::vector<copy_buffer_t> & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end)
+uint32_t blockstore_impl_t::prepare_read_simple(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end, uint32_t skip_csum)
 {
     uint32_t res = 0;
     if (wr->type() == BS_HEAP_SMALL_WRITE || wr->type() == BS_HEAP_INTENT_WRITE)
@@ -190,7 +235,7 @@ uint32_t blockstore_impl_t::prepare_read_simple(std::vector<copy_buffer_t> & rea
                 .disk_offset = start,
                 .disk_len = end-start,
                 .buf = buffer_area + wr->small().location + start - wr->small().offset,
-                .wr_lsn = wr->lsn,
+                .wr = wr,
             });
         }
         else if (dsk.csum_block_size <= dsk.bitmap_granularity)
@@ -209,14 +254,6 @@ uint32_t blockstore_impl_t::prepare_read_simple(std::vector<copy_buffer_t> & rea
                 blk_start = blk_start < wr->small().offset ? wr->small().offset : blk_start;
                 blk_end = blk_end > wr->small().offset+wr->small().len ? wr->small().offset+wr->small().len : blk_end;
             }
-            uint32_t skip_csum = 0;
-            if (!perfect_csum_update && wr->type() == BS_HEAP_BIG_WRITE)
-            {
-                for (auto owr = obj->get_writes(); owr && owr != wr; owr = owr->next())
-                    if ((owr->type() == BS_HEAP_INTENT_WRITE || owr->type() == BS_HEAP_SMALL_WRITE) &&
-                        owr->small().offset < blk_end && owr->small().offset+owr->small().len > blk_start)
-                        skip_csum = COPY_BUF_SKIP_CSUM;
-            }
             if ((blk_end-1)/dsk.csum_block_size == blk_start/dsk.csum_block_size ||
                 blk_end/dsk.csum_block_size == blk_start/dsk.csum_block_size+1 && blk_end != end && blk_start != start ||
                 blk_end == end && blk_start == start)
@@ -243,21 +280,37 @@ uint32_t blockstore_impl_t::prepare_read_simple(std::vector<copy_buffer_t> & rea
     return res;
 }
 
-void blockstore_impl_t::prepare_disk_read(std::vector<copy_buffer_t> & read_vec, int pos, heap_object_t *obj, heap_write_t *wr,
+void blockstore_impl_t::prepare_disk_read(std::vector<copy_buffer_t> & read_vec, int pos, heap_entry_t *obj, heap_entry_t *wr,
     uint32_t blk_start, uint32_t blk_end, uint32_t start, uint32_t end, uint32_t copy_flags)
 {
-    // Only one INTENT_WRITE is allowed at a time
-    assert(wr->type() != BS_HEAP_INTENT_WRITE || wr->next()->type() == BS_HEAP_BIG_WRITE);
+    uint64_t loc = 0;
+    if (wr->type() == BS_HEAP_INTENT_WRITE)
+    {
+        heap_entry_t *big_wr = wr;
+        while (big_wr && big_wr->type() == BS_HEAP_INTENT_WRITE)
+        {
+            big_wr = heap->prev(big_wr);
+        }
+        assert(big_wr->type() == BS_HEAP_BIG_WRITE || big_wr->type() == BS_HEAP_BIG_INTENT);
+        loc = big_wr->big_location(heap);
+    }
+    else if (wr->type() == BS_HEAP_SMALL_WRITE)
+    {
+        loc = wr->small().location-wr->small().offset;
+    }
+    else
+    {
+        assert(wr->type() == BS_HEAP_BIG_WRITE || wr->type() == BS_HEAP_BIG_INTENT);
+        loc = wr->big_location(heap);
+    }
     copy_buffer_t vec = {
         .copy_flags = (wr->type() == BS_HEAP_SMALL_WRITE ? COPY_BUF_JOURNAL : COPY_BUF_DATA) | copy_flags,
         .offset = start,
         .len = end-start,
-        .disk_loc = (wr->type() == BS_HEAP_INTENT_WRITE ? wr->next()->big_location(heap)
-            : (wr->type() == BS_HEAP_SMALL_WRITE ? wr->small().location-wr->small().offset
-            : wr->big_location(heap))),
+        .disk_loc = loc,
         .disk_offset = blk_start,
         .disk_len = blk_end - blk_start,
-        .wr_lsn = wr->lsn,
+        .wr = wr,
     };
     if (blk_start != start || blk_end != end)
     {
@@ -265,7 +318,7 @@ void blockstore_impl_t::prepare_disk_read(std::vector<copy_buffer_t> & read_vec,
         vec.copy_flags |= COPY_BUF_PADDED;
         if (pos > 0 && read_vec.size() >= pos &&
             read_vec[pos-1].copy_flags == vec.copy_flags &&
-            read_vec[pos-1].wr_lsn == vec.wr_lsn &&
+            read_vec[pos-1].wr == vec.wr &&
             read_vec[pos-1].disk_offset <= vec.disk_offset &&
             read_vec[pos-1].disk_offset+read_vec[pos-1].disk_len >= blk_end)
         {
@@ -349,7 +402,7 @@ void blockstore_impl_t::handle_read_event(ring_data_t *data, blockstore_op_t *op
             op->retval = -EDOM;
         else if (op->retval == 0)
             op->retval = op->len;
-        heap->unlock_entry(op->oid, PRIV(op)->lsn);
+        heap->unlock_entry(op->oid);
         free_read_buffers(PRIV(op)->read_vec);
         FINISH_OP(op);
     }
@@ -357,7 +410,15 @@ void blockstore_impl_t::handle_read_event(ring_data_t *data, blockstore_op_t *op
 
 bool blockstore_impl_t::verify_read_checksums(blockstore_op_t *op)
 {
-    heap_object_t *obj = heap->read_locked_entry(op->oid, PRIV(op)->lsn);
+    bool skip_all = false;
+    if (PRIV(op)->lsn)
+    {
+        heap_entry_t *obj = heap->read_entry(op->oid);
+        if (obj->lsn != PRIV(op)->lsn) // check top lsn
+        {
+            skip_all = true;
+        }
+    }
     auto & rv = PRIV(op)->read_vec;
     for (auto & vec: rv)
     {
@@ -365,23 +426,22 @@ bool blockstore_impl_t::verify_read_checksums(blockstore_op_t *op)
             continue;
         if (vec.copy_flags & COPY_BUF_PADDED)
             memcpy(op->buf + vec.offset - op->offset, vec.buf + vec.offset - vec.disk_offset, vec.len);
-        if (vec.copy_flags & (COPY_BUF_COALESCED|COPY_BUF_SKIP_CSUM))
+        if (vec.copy_flags & (COPY_BUF_COALESCED|COPY_BUF_SKIP_CSUM) ||
+            skip_all && vec.wr->type() != BS_HEAP_SMALL_WRITE)
             continue;
-        heap_write_t *wr = obj->get_writes();
-        while (wr && wr->lsn != vec.wr_lsn)
-            wr = wr->next();
-        assert(wr);
         uint8_t *buf = vec.buf ? vec.buf : (op->buf + vec.offset - op->offset);
-        uint32_t *csums = (uint32_t*)(wr->get_checksums(heap)
+        uint32_t *csums = (uint32_t*)(vec.wr->get_checksums(heap)
             + (vec.disk_offset/dsk.csum_block_size)*(dsk.data_csum_type & 0xFF)
-            - ((wr->type() == BS_HEAP_BIG_WRITE) ? 0 : (wr->small().offset/dsk.csum_block_size)*(dsk.data_csum_type & 0xFF)));
-        if (!heap->calc_block_checksums(csums, buf, wr->get_int_bitmap(heap),
+            - ((vec.wr->type() == BS_HEAP_BIG_WRITE || vec.wr->type() == BS_HEAP_BIG_INTENT)
+                ? 0 : (vec.wr->small().offset/dsk.csum_block_size)*(dsk.data_csum_type & 0xFF)));
+        if (!heap->calc_block_checksums(csums, buf, vec.wr->get_int_bitmap(heap),
             vec.disk_offset, vec.disk_offset+vec.disk_len, false, [&](uint32_t mismatch_pos, uint32_t expected_csum, uint32_t real_csum)
             {
                 printf(
-                    "Checksum mismatch in object %jx:%jx v%ju, offset 0x%x in %s area at offset 0x%jx: %08x expected vs %08x actual\n",
+                    "Checksum mismatch in object %jx:%jx v%ju, offset 0x%x in %s area at offset 0x%jx during read %x+%x: %08x expected vs %08x actual\n",
                     op->oid.inode, op->oid.stripe, op->version, mismatch_pos,
                     (vec.copy_flags & COPY_BUF_JOURNAL) ? "buffer" : "data", vec.disk_loc + vec.disk_offset,
+                    op->offset, op->len,
                     expected_csum, real_csum
                 );
             }))
@@ -394,23 +454,29 @@ bool blockstore_impl_t::verify_read_checksums(blockstore_op_t *op)
 
 int blockstore_impl_t::read_bitmap(object_id oid, uint64_t target_version, void *bitmap, uint64_t *result_version)
 {
-    heap_object_t *obj = heap->read_entry(oid, NULL);
+    heap_entry_t *obj = heap->read_entry(oid);
     if (obj)
     {
-        for (auto wr = obj->get_writes(); wr; wr = wr->next())
+        bool found = false;
+        heap->iterate_with_stable(obj, obj->lsn, [&](heap_entry_t *wr, bool stable)
         {
-            if (target_version < wr->version)
+            if (target_version >= wr->version)
             {
-                continue;
-            }
-            if (result_version)
-            {
-                *result_version = wr->version;
-            }
-            if (bitmap)
-            {
-                memcpy(bitmap, wr->get_ext_bitmap(heap), dsk.clean_entry_bitmap_size);
+                found = true;
+                if (result_version)
+                {
+                    *result_version = wr->version;
+                }
+                if (bitmap)
+                {
+                    memcpy(bitmap, wr->get_ext_bitmap(heap), dsk.clean_entry_bitmap_size);
+                }
+                return false;
             }
+            return true;
+        });
+        if (found)
+        {
             return 0;
         }
     }

src/blockstore/blockstore_stable.cpp

@@ -13,73 +13,75 @@ int blockstore_impl_t::dequeue_stable(blockstore_op_t *op)
     else if (priv->op_state == 2) goto resume_2;
     else if (priv->op_state == 3) goto resume_3;
     else if (priv->op_state == 4) goto resume_4;
+    else if (priv->op_state == 5) goto resume_5;
     assert(!priv->op_state);
-    // Modify in-memory state and assign contiguous LSNs
-    priv->stab_pos = 0;
-    priv->lsn = priv->to_lsn = 0;
     op->retval = 0;
-    while (priv->stab_pos < op->len)
+    priv->modified_block = priv->modified_block2 = UINT32_MAX;
+    for (priv->stab_pos = 0; priv->stab_pos < op->len; priv->stab_pos++)
     {
-        uint32_t modified_block = 0;
-        uint64_t new_lsn = 0;
-        uint64_t new_to_lsn = 0;
-        int res = op->opcode == BS_OP_STABLE
-            ? heap->post_stabilize(v[priv->stab_pos].oid, v[priv->stab_pos].version, &modified_block, &new_lsn, &new_to_lsn)
-            : heap->post_rollback(v[priv->stab_pos].oid, v[priv->stab_pos].version, &new_lsn, &modified_block);
-        if (res != 0)
         {
-            assert(res == ENOENT || res == EBUSY);
-            op->retval = -res;
-        }
-        if (new_lsn)
-        {
-            assert(priv->lsn == 0 || priv->to_lsn == new_lsn-1);
-            if (!priv->lsn)
-                priv->lsn = new_lsn;
-            priv->to_lsn = op->opcode == BS_OP_STABLE ? new_to_lsn : new_lsn;
-        }
-        priv->stab_pos++;
-    }
-    // Submit metadata writes
-    priv->stab_pos = 0;
-resume_1:
-    priv->op_state = 1;
-    while (priv->stab_pos < op->len)
-    {
-        uint32_t block_num = 0;
-        heap_object_t *obj = heap->read_entry(v[priv->stab_pos].oid, &block_num);
-        if (obj)
-        {
-            io_uring_sqe *sqe = get_sqe();
-            if (!sqe)
+            auto obj = heap->read_entry(v[priv->stab_pos].oid);
+            if (!obj)
             {
-                if (priv->pending_ops > 0)
-                    return 1;
-                priv->wait_detail = 1;
-                priv->wait_for = WAIT_SQE;
+                op->retval = -ENOENT;
+                FINISH_OP(op);
+                return 2;
+            }
+            int res = op->opcode == BS_OP_STABLE
+                ? heap->add_commit(obj, v[priv->stab_pos].version, &priv->modified_block2)
+                : heap->add_rollback(obj, v[priv->stab_pos].version, &priv->modified_block2);
+            if (res == EBUSY)
+            {
+                op->retval = -EBUSY;
+                FINISH_OP(op);
+                return 2;
+            }
+            if (res == ENOSPC)
+            {
+                if (!heap->get_to_compact_count())
+                {
+                    // no space
+                    op->retval = -ENOSPC;
+                    FINISH_OP(op);
+                    return 2;
+                }
+                if (priv->modified_block2 != UINT32_MAX)
+                {
+                    priv->stab_pos--;
+                    goto resume_1;
+                }
+                priv->wait_for = WAIT_COMPACTION;
+                priv->wait_detail = heap->get_compacted_count();
+                flusher->request_trim();
                 return 0;
             }
-            prepare_meta_block_write(op, block_num, sqe);
+            assert(res == 0);
         }
-        priv->stab_pos++;
-    }
-    if (priv->pending_ops > 0)
-    {
-        priv->op_state = 1;
-        return 1;
-    }
-    // Mark writes as completed to allow compaction
-    for (uint64_t lsn = priv->lsn; lsn <= priv->to_lsn; lsn++)
-    {
-        heap->mark_lsn_completed(lsn);
-    }
-    unsynced_meta_write_count++;
-    // Fsync, just because our semantics imply that commit (stabilize) is immediately fsynced
-    priv->op_state = 2;
+resume_1:
+        if (priv->modified_block != UINT32_MAX && priv->modified_block2 != priv->modified_block)
+        {
+            BS_SUBMIT_CHECK_SQES(1);
+            prepare_meta_block_write(priv->modified_block);
 resume_2:
+            if (meta_block_is_pending(priv->modified_block))
+            {
+                priv->op_state = 2;
+                return 1;
+            }
+        }
+        priv->modified_block = priv->modified_block2;
+        if (priv->stab_pos == op->len-1 && priv->modified_block2 != UINT32_MAX)
+        {
+            priv->modified_block2 = UINT32_MAX;
+            goto resume_1;
+        }
+    }
+    // Fsync, just because our semantics imply that commit (stabilize) is immediately fsynced
+    priv->op_state = 3;
 resume_3:
 resume_4:
-    int res = do_sync(op, 2);
+resume_5:
+    int res = do_sync(op, 3);
     if (res != 2)
     {
         return res;

src/blockstore/blockstore_sync.cpp

@@ -18,11 +18,19 @@ int blockstore_impl_t::continue_sync(blockstore_op_t *op)
     return res;
 }
 
+bool blockstore_impl_t::has_unsynced()
+{
+    bool data = (!dsk.disable_data_fsync && unsynced_data_write_count);
+    bool buffer = (!dsk.disable_journal_fsync && unsynced_buffer_write_count);
+    bool meta = (!dsk.disable_meta_fsync && unsynced_meta_write_count);
+    return data || buffer || meta;
+}
+
 bool blockstore_impl_t::submit_fsyncs(int & wait_count)
 {
-    int n = ((unsynced_small_write_count > 0 || unsynced_big_write_count > 0 || unsynced_meta_write_count > 0) && !dsk.disable_meta_fsync) +
-        (unsynced_small_write_count > 0 && !dsk.disable_journal_fsync && dsk.journal_fd != dsk.meta_fd) +
-        (unsynced_big_write_count > 0 && !dsk.disable_data_fsync && dsk.data_fd != dsk.meta_fd && dsk.data_fd != dsk.journal_fd);
+    int n = (unsynced_meta_write_count > 0 && !dsk.disable_meta_fsync) +
+        (unsynced_buffer_write_count > 0 && !dsk.disable_journal_fsync && dsk.journal_fd != dsk.meta_fd) +
+        (unsynced_data_write_count > 0 && !dsk.disable_data_fsync && dsk.data_fd != dsk.meta_fd && dsk.data_fd != dsk.journal_fd);
     if (ringloop->space_left() < n)
     {
         return false;
@@ -40,7 +48,7 @@ bool blockstore_impl_t::submit_fsyncs(int & wait_count)
         if (!wait_count)
             ringloop->wakeup();
     };
-    if ((unsynced_small_write_count > 0 || unsynced_big_write_count > 0 || unsynced_meta_write_count > 0) && !dsk.disable_meta_fsync)
+    if (unsynced_meta_write_count > 0 && !dsk.disable_meta_fsync)
     {
         // fsync meta
         io_uring_sqe *sqe = get_sqe();
@@ -51,7 +59,7 @@ bool blockstore_impl_t::submit_fsyncs(int & wait_count)
         data->callback = cb;
         wait_count++;
     }
-    if (unsynced_small_write_count > 0 && !dsk.disable_journal_fsync && dsk.meta_fd != dsk.journal_fd)
+    if (unsynced_buffer_write_count > 0 && !dsk.disable_journal_fsync && dsk.meta_fd != dsk.journal_fd)
     {
         // fsync buffer
         io_uring_sqe *sqe = get_sqe();
@@ -62,7 +70,7 @@ bool blockstore_impl_t::submit_fsyncs(int & wait_count)
         data->callback = cb;
         wait_count++;
     }
-    if (unsynced_big_write_count > 0 && !dsk.disable_data_fsync && dsk.data_fd != dsk.meta_fd && dsk.data_fd != dsk.journal_fd)
+    if (unsynced_data_write_count > 0 && !dsk.disable_data_fsync && dsk.data_fd != dsk.meta_fd && dsk.data_fd != dsk.journal_fd)
     {
         // fsync data
         io_uring_sqe *sqe = get_sqe();
@@ -73,8 +81,8 @@ bool blockstore_impl_t::submit_fsyncs(int & wait_count)
         data->callback = cb;
         wait_count++;
     }
-    unsynced_big_write_count = 0;
-    unsynced_small_write_count = 0;
+    unsynced_data_write_count = 0;
+    unsynced_buffer_write_count = 0;
     unsynced_meta_write_count = 0;
     return true;
 }
@@ -90,13 +98,13 @@ int blockstore_impl_t::do_sync(blockstore_op_t *op, int base_state)
         // Wait for flusher-initiated sync
         return 0;
     }
-    if (dsk.disable_journal_fsync && dsk.disable_meta_fsync && dsk.disable_data_fsync || !unsynced_big_write_count && !unsynced_small_write_count)
+    if (!has_unsynced())
     {
         // We can return immediately because sync only syncs previous writes
-        unsynced_big_write_count = unsynced_small_write_count = unsynced_meta_write_count = 0;
+        unsynced_data_write_count = unsynced_buffer_write_count = unsynced_meta_write_count = 0;
         return 2;
     }
-    PRIV(op)->lsn = heap->get_completed_lsn();
+    PRIV(op)->modified_block = heap->get_completed_lsn();
     if (!submit_fsyncs(PRIV(op)->pending_ops))
     {
         PRIV(op)->wait_detail = 1;
@@ -110,6 +118,6 @@ resume_1:
         return 1;
     }
 resume_2:
-    heap->mark_lsn_fsynced(PRIV(op)->lsn);
+    heap->mark_lsn_fsynced(PRIV(op)->modified_block);
     return 2;
 }

src/blockstore/blockstore_write.cpp

@@ -5,29 +5,107 @@
 #include "blockstore_internal.h"
 #include "allocator.h"
 
+#define _REDIRECT_INTENT 0x101
+
 bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
 {
     clock_gettime(CLOCK_REALTIME, &PRIV(op)->tv_begin);
     return true;
 }
 
-void blockstore_impl_t::prepare_meta_block_write(blockstore_op_t *op, uint64_t modified_block, io_uring_sqe *sqe)
+void blockstore_impl_t::prepare_meta_block_write(uint32_t modified_block)
 {
-    if (!sqe)
-    {
-        sqe = get_sqe();
-        assert(sqe != NULL);
-    }
+    if (modified_blocks.find(modified_block) != modified_blocks.end())
+        return;
+    io_uring_sqe *sqe = get_sqe();
+    assert(sqe != NULL);
     ring_data_t *data = ((ring_data_t*)sqe->user_data);
-    data->iov = (struct iovec){ heap->get_meta_block(modified_block), (size_t)dsk.meta_block_size };
-    data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
-    PRIV(op)->pending_ops++;
+    uint8_t *buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size);
+    data->iov = (struct iovec){ buf, (size_t)dsk.meta_block_size };
+    data->callback = [this, modified_block, buf](ring_data_t *data)
+    {
+        free(buf);
+        live = true;
+        if (data->res != data->iov.iov_len)
+        {
+            // FIXME: our state becomes corrupted after a write error. maybe do something better than just die
+            disk_error_abort("data write", data->res, data->iov.iov_len);
+        }
+        modified_blocks.erase(modified_block);
+        heap->complete_block_write(modified_block);
+        ringloop->wakeup();
+    };
     io_uring_prep_writev(
         sqe, dsk.meta_fd, &data->iov, 1, dsk.meta_offset + (modified_block+1)*dsk.meta_block_size
     );
+    unsynced_meta_write_count++;
+    pending_modified_blocks.push_back(modified_block);
+    modified_blocks[modified_block] = { .sent = false, .buf = buf };
+}
+
+bool blockstore_impl_t::meta_block_is_pending(uint32_t modified_block)
+{
+    auto mb_it = modified_blocks.find(modified_block);
+    return mb_it != modified_blocks.end();
+}
+
+bool blockstore_impl_t::intent_write_allowed(blockstore_op_t *op, heap_entry_t *obj)
+{
+    // Parallel writes to the same object are forbidden so "one intent at a time" is fulfilled automatically
+    // Intent writes are disabled when metadata fsync is enabled
+    if (!dsk.disable_meta_fsync)
+    {
+        return false;
+    }
+    // Intent writes are only for replication
+    if (op->opcode != BS_OP_WRITE_STABLE)
+    {
+        return false;
+    }
+    // Operation size should be less than or equal to atomic write size
+    if (!op->len || op->len > dsk.atomic_write_size)
+    {
+        return false;
+    }
+    // Intent-writes are disabled if "absolutely correct during compaction" checksum validation algorithm is enabled
+    // We could also do RMW here when perfect_csum_update is enabled, but it's unclear if we need it
+    if (perfect_csum_update && dsk.csum_block_size > dsk.bitmap_granularity &&
+        ((op->offset % dsk.csum_block_size) || (op->len % dsk.csum_block_size)))
+    {
+        return false;
+    }
+    bool ok = true;
+    heap->iterate_with_stable(obj, obj->lsn, [&](heap_entry_t *wr, bool stable)
+    {
+        // Intent writes are not allowed over buffered writes
+        auto t = wr->type();
+        if (t == BS_HEAP_SMALL_WRITE)
+        {
+            ok = false;
+            return false;
+        }
+        // Intent writes are not allowed over unstable writes
+        if (!stable)
+        {
+            ok = false;
+            return false;
+        }
+        // Intent writes are not allowed over unfinished intent writes
+        if ((t == BS_HEAP_INTENT_WRITE || t == BS_HEAP_BIG_INTENT) && wr->lsn > heap->get_fsynced_lsn())
+        {
+            ok = false;
+            return false;
+        }
+        // Intent writes are allowed over BIG_WRITEs even with fsyncs because BIG_WRITE is always counted as fsynced
+        if (t == BS_HEAP_BIG_WRITE || t == BS_HEAP_BIG_INTENT)
+        {
+            return false;
+        }
+        return true;
+    });
+    return ok;
 }
 
-// First step of the write algorithm: dequeue operation and submit initial write(s)
 int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
 {
     if (PRIV(op)->op_state)
@@ -38,40 +116,42 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
     {
         return 0;
     }
-    PRIV(op)->is_big = false;
-    uint32_t modified_block = UINT32_MAX, moved_from_block = UINT32_MAX;
-    heap_object_t *obj = heap->read_entry(op->oid, &modified_block);
+    PRIV(op)->modified_block = UINT32_MAX;
+    PRIV(op)->write_type = 0;
+    heap_entry_t *obj = heap->read_entry(op->oid);
     if (op->opcode == BS_OP_DELETE)
     {
         // Delete
-        if (!obj)
+        if (!obj || obj->type() == BS_HEAP_DELETE)
         {
             // Already deleted
             op->retval = 0;
             FINISH_OP(op);
             return 2;
         }
+        PRIV(op)->write_type = BS_HEAP_DELETE;
         BS_SUBMIT_CHECK_SQES(1);
-        int res = heap->post_delete(modified_block, obj, &PRIV(op)->lsn);
+        int res = heap->add_delete(obj, &PRIV(op)->modified_block);
+        if (res == ENOSPC)
+            goto enospc;
         assert(res == 0);
-        prepare_meta_block_write(op, modified_block);
+        prepare_meta_block_write(PRIV(op)->modified_block);
+        PRIV(op)->pending_ops++;
         PRIV(op)->op_state = 5;
         write_iodepth++;
     }
     // FIXME: Allow to do initial writes as buffered, not redirected
     // FIXME: Allow to do direct writes over holes
-    else if (!obj || obj->get_writes()->type() == BS_HEAP_TOMBSTONE ||
-        op->offset == 0 && op->len == dsk.data_block_size)
+    else if (!obj || obj->type() == BS_HEAP_DELETE || op->offset == 0 && op->len == dsk.data_block_size)
     {
         // Big (redirect) write
+        PRIV(op)->write_type = dsk.disable_data_fsync || op->opcode != BS_OP_WRITE_STABLE ? BS_HEAP_BIG_WRITE : _REDIRECT_INTENT;
         BS_SUBMIT_CHECK_SQES(1);
-        PRIV(op)->is_big = true;
-        uint32_t tmp_block;
-        uint64_t loc = heap->find_free_data();
-        if (loc == UINT64_MAX ||
-            !obj && heap->get_block_for_new_object(tmp_block) != 0)
+        PRIV(op)->location = heap->find_free_data();
+        if (PRIV(op)->location == UINT64_MAX)
         {
-            if (!heap->get_inflight_queue_size())
+enospc:
+            if (!heap->get_to_compact_count())
             {
                 // no space
                 op->retval = -ENOSPC;
@@ -79,11 +159,11 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
                 return 2;
             }
             PRIV(op)->wait_for = WAIT_COMPACTION;
-            PRIV(op)->wait_detail = flusher->get_compact_counter();
+            PRIV(op)->wait_detail = heap->get_compacted_count();
             flusher->request_trim();
             return 0;
         }
-        PRIV(op)->location = loc;
+        uint64_t loc = PRIV(op)->location;
 #ifdef BLOCKSTORE_DEBUG
         printf(
             "Allocate offset %ju for %jx:%jx v%ju\n",
@@ -91,103 +171,60 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
         );
 #endif
         heap->use_data(op->oid.inode, PRIV(op)->location);
-        if (!dsk.disable_data_fsync && dsk.disable_meta_fsync)
-        {
-            // Do big_write as an INTENT to avoid data fsync
-            int res = make_big_write(op, 0, 0, &modified_block, &moved_from_block);
-            assert(res == 0);
-            if (moved_from_block != UINT32_MAX)
-                prepare_meta_block_write(op, moved_from_block);
-            obj = heap->read_entry(op->oid, &modified_block);
-            heap->mark_lsn_completed(PRIV(op)->lsn);
-            goto process_intent;
-        }
         io_uring_sqe *sqe = get_sqe();
         ring_data_t *data = ((ring_data_t*)sqe->user_data);
-        uint64_t stripe_offset = (op->offset % dsk.bitmap_granularity);
-        uint64_t stripe_end = (op->offset + op->len) % dsk.bitmap_granularity;
-        // Zero fill up to dsk.bitmap_granularity
-        int vcnt = 0;
-        if (stripe_offset)
-        {
-            PRIV(op)->iov_zerofill[vcnt++] = (struct iovec){ zero_object, (size_t)stripe_offset };
-        }
-        PRIV(op)->iov_zerofill[vcnt++] = (struct iovec){ op->buf, op->len };
-        if (stripe_end)
-        {
-            stripe_end = dsk.bitmap_granularity - stripe_end;
-            PRIV(op)->iov_zerofill[vcnt++] = (struct iovec){ zero_object, (size_t)stripe_end };
-        }
-        data->iov.iov_len = op->len + stripe_offset + stripe_end; // to check it in the callback
+        data->iov = (struct iovec){ op->buf, op->len };
         data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
-        io_uring_prep_writev(
-            sqe, dsk.data_fd, PRIV(op)->iov_zerofill, vcnt, dsk.data_offset + loc + op->offset - stripe_offset
-        );
+        io_uring_prep_writev(sqe, dsk.data_fd, &data->iov, 1, dsk.data_offset + loc + op->offset);
         PRIV(op)->pending_ops++;
-        PRIV(op)->op_state = 1;
         write_iodepth++;
-        inflight_big++;
+        if (PRIV(op)->write_type == BS_HEAP_BIG_WRITE)
+        {
+            PRIV(op)->op_state = 1;
+            inflight_big++;
+        }
+        else
+            PRIV(op)->op_state = 3;
     }
-    // Only one INTENT_WRITE is allowed at a time, but in fact,
-    // parallel writes to the same object are forbidden anyway
-    else if (op->len > 0 && op->len <= dsk.atomic_write_size &&
-        // Intent-writes are disabled if "absolutely correct during compaction" checksum validation algorithm is enabled
-        // We could also do RMW here when perfect_csum_update is enabled, but it's unclear if we need it
-        (!perfect_csum_update || dsk.csum_block_size <= dsk.bitmap_granularity ||
-            !(op->offset % dsk.csum_block_size) &&
-            !(op->len % dsk.csum_block_size) &&
-            (obj->get_writes()->entry_type != (BS_HEAP_INTENT_WRITE|BS_HEAP_STABLE) ||
-            obj->get_writes()->can_be_collapsed(heap))) &&
-        // One intent-write is allowed even with fsyncs because BIG_WRITE is always counted as fsynced
-        dsk.disable_meta_fsync &&
-        (op->opcode == BS_OP_WRITE_STABLE &&
-            (obj->get_writes()->entry_type == (BS_HEAP_BIG_WRITE|BS_HEAP_STABLE) ||
-            obj->get_writes()->entry_type == (BS_HEAP_INTENT_WRITE|BS_HEAP_STABLE) && dsk.disable_data_fsync) ||
-        op->opcode == BS_OP_WRITE && obj->get_writes()->entry_type == BS_HEAP_BIG_WRITE))
+    else if (intent_write_allowed(op, obj))
     {
         // Direct intent-write
         BS_SUBMIT_CHECK_SQES(1);
-        if (obj->get_writes()->type() == BS_HEAP_BIG_WRITE)
+        int res = 0;
+        if (dsk.csum_block_size <= dsk.bitmap_granularity &&
+            (obj->entry_type == (BS_HEAP_BIG_INTENT|BS_HEAP_STABLE) ||
+            obj->entry_type == (BS_HEAP_BIG_WRITE|BS_HEAP_STABLE)))
         {
-            PRIV(op)->location = obj->get_writes()->big_location(heap);
+            // Even more simplified BIG_INTENT writes
+            // FIXME: Support RMW mode for csum_block_size > bitmap_granularity
+            PRIV(op)->write_type = BS_HEAP_BIG_INTENT;
+            PRIV(op)->location = obj->big_location(heap);
+            res = heap->add_big_intent(op->oid, &obj, op->version, op->offset, op->len, op->bitmap,
+                (uint8_t*)op->buf, NULL, &PRIV(op)->modified_block);
+            if (res == ENOSPC)
+                goto enospc;
+            assert(res == 0);
+            PRIV(op)->lsn = obj->lsn;
         }
         else
         {
-            assert(obj->get_writes()->next()->type() == BS_HEAP_BIG_WRITE);
-            PRIV(op)->location = obj->get_writes()->next()->big_location(heap);
+            PRIV(op)->write_type = BS_HEAP_INTENT_WRITE;
+            auto wr = obj;
+            while (wr && (wr->type() == BS_HEAP_INTENT_WRITE || wr->type() == BS_HEAP_COMMIT || wr->type() == BS_HEAP_ROLLBACK))
+            {
+                wr = heap->prev(wr);
+            }
+            assert(wr && (wr->type() == BS_HEAP_BIG_WRITE || wr->type() == BS_HEAP_BIG_INTENT));
+            PRIV(op)->location = wr->big_location(heap);
+            res = heap->add_small_write(op->oid, &obj, (BS_HEAP_INTENT_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0)),
+                op->version, op->offset, op->len, 0, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
+            if (res == ENOSPC)
+                goto enospc;
+            assert(res == 0);
+            PRIV(op)->lsn = obj->lsn;
         }
-process_intent:
-        uint8_t wr_buf[heap->get_max_write_entry_size()];
-        heap_write_t *wr = (heap_write_t*)wr_buf;
-        wr->version = op->version;
-        wr->entry_type = BS_HEAP_INTENT_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0);
-        wr->small().offset = op->offset;
-        wr->small().len = op->len;
-        wr->small().location = 0;
-        if (op->bitmap)
-            memcpy(wr->get_ext_bitmap(heap), op->bitmap, dsk.clean_entry_bitmap_size);
-        heap->calc_checksums(wr, (uint8_t*)op->buf, true);
-        int res = heap->post_write(modified_block, op->oid, obj, wr, &moved_from_block);
-        if (res == EAGAIN)
-        {
-            assert(heap->get_inflight_queue_size());
-            PRIV(op)->wait_for = WAIT_COMPACTION;
-            PRIV(op)->wait_detail = flusher->get_compact_counter();
-            flusher->request_trim();
-            return 0;
-        }
-        else if (res == ENOSPC)
-        {
-            // no space
-            op->retval = -ENOSPC;
-            FINISH_OP(op);
-            return 2;
-        }
-        assert(res == 0);
-        PRIV(op)->lsn = wr->lsn;
-        if (moved_from_block != UINT32_MAX)
-            prepare_meta_block_write(op, moved_from_block);
-        prepare_meta_block_write(op, modified_block);
+        prepare_meta_block_write(PRIV(op)->modified_block);
+        PRIV(op)->pending_ops++;
         PRIV(op)->op_state = 9;
         write_iodepth++;
     }
@@ -195,50 +232,27 @@ process_intent:
     {
         // Small (buffered) overwrite
         // First check if there is free buffer space
+        PRIV(op)->write_type = BS_HEAP_SMALL_WRITE;
         uint64_t loc = !op->len ? 0 : heap->find_free_buffer_area(op->len);
         if (loc == UINT64_MAX)
         {
             PRIV(op)->wait_for = WAIT_COMPACTION;
-            PRIV(op)->wait_detail = flusher->get_compact_counter();
+            PRIV(op)->wait_detail = heap->get_compacted_count();
             flusher->request_trim();
             return 0;
         }
         // There is sufficient space. Check SQE(s)
         BS_SUBMIT_CHECK_SQES(1 + (op->len > 0 ? 1 : 0));
-        uint8_t wr_buf[heap->get_max_write_entry_size()];
-        heap_write_t *wr = (heap_write_t*)wr_buf;
-        wr->version = op->version;
-        wr->entry_type = BS_HEAP_SMALL_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0);
-        wr->small().offset = op->offset;
-        wr->small().len = op->len;
-        wr->small().location = loc;
-        PRIV(op)->location = loc;
-        if (op->bitmap)
-            memcpy(wr->get_ext_bitmap(heap), op->bitmap, dsk.clean_entry_bitmap_size);
-        heap->calc_checksums(wr, (uint8_t*)op->buf, true);
-        int res = heap->post_write(modified_block, op->oid, obj, wr, &moved_from_block);
-        if (res == EAGAIN)
-        {
-            assert(heap->get_inflight_queue_size());
-            PRIV(op)->wait_for = WAIT_COMPACTION;
-            PRIV(op)->wait_detail = flusher->get_compact_counter();
-            flusher->request_trim();
-            return 0;
-        }
-        else if (res == ENOSPC)
-        {
-            // no space
-            op->retval = -ENOSPC;
-            FINISH_OP(op);
-            return 2;
-        }
+        int res = heap->add_small_write(op->oid, &obj, (BS_HEAP_SMALL_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0)),
+            op->version, op->offset, op->len, loc, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
+        if (res == ENOSPC)
+            goto enospc;
         assert(res == 0);
-        PRIV(op)->lsn = wr->lsn;
+        PRIV(op)->lsn = obj->lsn;
         if (op->len)
             heap->use_buffer_area(op->oid.inode, loc, op->len);
-        if (moved_from_block != UINT32_MAX)
-            prepare_meta_block_write(op, moved_from_block);
-        prepare_meta_block_write(op, modified_block);
+        prepare_meta_block_write(PRIV(op)->modified_block);
+        PRIV(op)->pending_ops++;
         if (op->len > 0)
         {
             // Prepare buffered data write
@@ -263,26 +277,6 @@ process_intent:
     return 1;
 }
 
-int blockstore_impl_t::make_big_write(blockstore_op_t *op, uint32_t offset, uint32_t len, uint32_t *modified_block, uint32_t *moved_from_block)
-{
-    uint8_t wr_buf[heap->get_max_write_entry_size()];
-    heap_write_t *wr = (heap_write_t*)wr_buf;
-    wr->entry_type = BS_HEAP_BIG_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0);
-    wr->version = op->version;
-    wr->set_big_location(heap, PRIV(op)->location);
-    if (op->bitmap)
-        memcpy(wr->get_ext_bitmap(heap), op->bitmap, dsk.clean_entry_bitmap_size);
-    memset(wr->get_int_bitmap(heap), 0, dsk.clean_entry_bitmap_size);
-    bitmap_set(wr->get_int_bitmap(heap), offset, len, dsk.bitmap_granularity);
-    heap->calc_checksums(wr, (uint8_t*)op->buf, true, offset, len);
-    int res = heap->post_write(op->oid, wr, modified_block, moved_from_block);
-    if (res != 0)
-        return res;
-    assert(res == 0);
-    PRIV(op)->lsn = wr->lsn;
-    return 0;
-}
-
 int blockstore_impl_t::continue_write(blockstore_op_t *op)
 {
     int op_state = PRIV(op)->op_state;
@@ -305,6 +299,12 @@ again:
     {
         // In progress
         assert(op_state < 10);
+        if (PRIV(op)->modified_block != UINT32_MAX &&
+            !meta_block_is_pending(PRIV(op)->modified_block))
+        {
+            PRIV(op)->pending_ops--;
+            PRIV(op)->modified_block = UINT32_MAX;
+        }
         if (PRIV(op)->pending_ops > 0)
             return 1;
         op_state++;
@@ -313,7 +313,6 @@ again:
 resume_2:
     // We must fsync all big writes to avoid complex write workflows
     // It's OK for all HDDs and for server SSDs, but slightly worse for desktop SSDs
-    // The other way is to add another type of MVCC to blockstore_heap: "forward" MVCC :)
     inflight_big--;
     if (!dsk.disable_data_fsync)
     {
@@ -349,33 +348,43 @@ resume_12:
     }
 resume_4:
     {
-        uint32_t modified_block = UINT32_MAX, moved_from_block = UINT32_MAX;
-        int res = make_big_write(op, op->offset, op->len, &modified_block, &moved_from_block);
-        if (res == EAGAIN)
+        auto obj = heap->read_entry(op->oid);
+        int res = 0;
+        if (PRIV(op)->write_type == _REDIRECT_INTENT)
         {
-            assert(heap->get_inflight_queue_size());
+            res = heap->add_redirect_intent(op->oid, &obj, op->version, op->offset, op->len,
+                PRIV(op)->location, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
+        }
+        else
+        {
+            res = heap->add_big_write(op->oid, obj, op->opcode == BS_OP_WRITE_STABLE,
+                op->version, op->offset, op->len, PRIV(op)->location, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
+        }
+        if (res == ENOSPC)
+        {
+            if (!heap->get_to_compact_count())
+            {
+                // no space
+                heap->free_data(op->oid.inode, PRIV(op)->location);
+                write_iodepth--;
+                op->retval = -ENOSPC;
+                FINISH_OP(op);
+                return 2;
+            }
             PRIV(op)->wait_for = WAIT_COMPACTION;
-            PRIV(op)->wait_detail = flusher->get_compact_counter();
+            PRIV(op)->wait_detail = heap->get_compacted_count();
             flusher->request_trim();
-            return 1;
+            return 0;
         }
-        else if (res == ENOSPC)
-        {
-            heap->free_data(op->oid.inode, PRIV(op)->location);
-            write_iodepth--;
-            op->retval = -ENOSPC;
-            FINISH_OP(op);
-            return 2;
-        }
-        if (moved_from_block != UINT32_MAX)
-            prepare_meta_block_write(op, moved_from_block);
-        prepare_meta_block_write(op, modified_block);
+        assert(res == 0);
+        prepare_meta_block_write(PRIV(op)->modified_block);
+        PRIV(op)->pending_ops++;
         PRIV(op)->op_state = 5;
         return 1;
     }
 resume_6:
-    // Apply throttling to not fill the journal too fast for the SSD+HDD case
-    if (!PRIV(op)->is_big && throttle_small_writes)
+    // Apply throttling to not fill the journal too quickly for the SSD+HDD case
+    if (PRIV(op)->write_type == BS_HEAP_SMALL_WRITE && throttle_small_writes)
     {
         // Apply throttling
         timespec tv_end;
@@ -395,11 +404,12 @@ resume_6:
         if (ref_us > exec_us + throttle_threshold_us)
         {
             // Pause reply
+            PRIV(op)->pending_ops++;
             PRIV(op)->op_state = 7;
             // Remember that the timer can in theory be called right here
             tfd->set_timer_us(ref_us-exec_us, false, [this, op](int timer_id)
             {
-                PRIV(op)->op_state = 8;
+                PRIV(op)->pending_ops--;
                 ringloop->wakeup();
             });
             return 1;
@@ -411,11 +421,27 @@ resume_8:
     printf("Ack write %jx:%jx v%ju\n", op->oid.inode, op->oid.stripe, op->version);
 #endif
     op->retval = op->len;
-    heap->mark_lsn_completed(PRIV(op)->lsn);
-    if (PRIV(op)->is_big)
-        unsynced_big_write_count++;
-    else
-        unsynced_small_write_count++;
+    if (PRIV(op)->write_type == BS_HEAP_BIG_WRITE)
+    {
+        unsynced_data_write_count++;
+    }
+    else if (PRIV(op)->write_type == BS_HEAP_SMALL_WRITE)
+    {
+        unsynced_buffer_write_count++;
+        heap->complete_lsn_write(PRIV(op)->lsn);
+    }
+    else if (PRIV(op)->write_type == BS_HEAP_BIG_INTENT ||
+        PRIV(op)->write_type == BS_HEAP_INTENT_WRITE)
+    {
+        unsynced_data_write_count++;
+        intent_write_counter++;
+        heap->complete_lsn_write(PRIV(op)->lsn);
+    }
+    else if (PRIV(op)->write_type == _REDIRECT_INTENT)
+    {
+        unsynced_data_write_count++;
+        intent_write_counter++;
+    }
     write_iodepth--;
     FINISH_OP(op);
     return 2;
@@ -426,6 +452,8 @@ resume_10:
     data->iov = (struct iovec){ op->buf, op->len };
     data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
     io_uring_prep_writev(sqe, dsk.data_fd, &data->iov, 1, dsk.data_offset + PRIV(op)->location + op->offset);
+    if (dsk.use_atomic_flag)
+        sqe->rw_flags = RWF_ATOMIC;
     PRIV(op)->pending_ops++;
     PRIV(op)->op_state = 7;
     return 1;

src/blockstore/fio_engine.cpp

@@ -28,6 +28,7 @@
 #include "epoll_manager.h"
 #include "malloc_or_die.h"
 #include "json11/json11.hpp"
+#include "../util/robin_hood.h"
 #include "fio_headers.h"
 
 struct bs_data
@@ -37,11 +38,15 @@ struct bs_data
     ring_loop_t *ringloop;
     /* The list of completed io_u structs. */
     std::vector<io_u*> completed;
+    robin_hood::unordered_flat_map<uint64_t, int> inflight_oids;
+    std::vector<io_u*> postponed;
     int op_n = 0, inflight = 0;
     bool ec = false;
     bool imm = true;
     bool last_sync = false;
     bool trace = false;
+    uint8_t *bitmap = NULL;
+    uint32_t block_size = 0;
 };
 
 struct bs_options
@@ -137,6 +142,7 @@ static void bs_cleanup(struct thread_data *td)
         delete bsd->bs;
         delete bsd->epmgr;
         delete bsd->ringloop;
+        free(bsd->bitmap);
         delete bsd;
     }
 }
@@ -160,9 +166,12 @@ static int bs_init(struct thread_data *td)
                 config[p.first] = p.second.dump();
         }
     }
+    bsd->bitmap = (uint8_t*)malloc_or_die(MAX_DATA_BLOCK_SIZE/512/8);
+    memset(bsd->bitmap, 0, MAX_DATA_BLOCK_SIZE/512/8);
     bsd->ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
     bsd->epmgr = new epoll_manager_t(bsd->ringloop);
     bsd->bs = blockstore_i::create(config, bsd->ringloop, bsd->epmgr->tfd);
+    bsd->block_size = bsd->bs->get_block_size();
     bsd->imm = config.find("immediate_commit") == config.end() ||
         config["immediate_commit"] == "all";
     while (1)
@@ -177,8 +186,33 @@ static int bs_init(struct thread_data *td)
     return 0;
 }
 
+static enum fio_q_status _bs_queue(struct thread_data *td, struct io_u *io, bool force);
+
+static void _bs_retry(struct bs_data *bsd, uint64_t offset)
+{
+    // Retry postponed ops
+    auto inflight_it = bsd->inflight_oids.find(offset / bsd->block_size);
+    assert(inflight_it != bsd->inflight_oids.end());
+    inflight_it->second--;
+    if (inflight_it->second > 0)
+    {
+        for (size_t i = 0; i < bsd->postponed.size(); i++)
+        {
+            auto oio = bsd->postponed[i];
+            if (oio->offset/bsd->block_size == offset/bsd->block_size)
+            {
+                bsd->postponed.erase(bsd->postponed.begin()+i);
+                _bs_queue((thread_data*)oio->engine_data, oio, true);
+                break;
+            }
+        }
+    }
+    else
+        bsd->inflight_oids.erase(inflight_it);
+}
+
 /* Begin read or write request. */
-static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
+static enum fio_q_status _bs_queue(struct thread_data *td, struct io_u *io, bool force)
 {
     bs_data *bsd = (bs_data*)td->io_ops_data;
     if (io->ddir == DDIR_SYNC && bsd->last_sync)
@@ -188,10 +222,22 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
 
     fio_ro_check(td, io);
 
-    io->engine_data = bsd;
+    io->engine_data = td;
 
     if (io->ddir == DDIR_WRITE || io->ddir == DDIR_READ)
-        assert(io->xfer_buflen <= bsd->bs->get_block_size());
+        assert(io->xfer_buflen <= bsd->block_size);
+
+    uint64_t stripe = io->offset / bsd->block_size;
+    if (!force && io->ddir == DDIR_WRITE)
+    {
+        auto & inflight = bsd->inflight_oids[stripe];
+        inflight++;
+        if (inflight > 1)
+        {
+            bsd->postponed.push_back(io);
+            return FIO_Q_QUEUED;
+        }
+    }
 
     blockstore_op_t *op = new blockstore_op_t;
     op->callback = NULL;
@@ -203,15 +249,16 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
         op->buf = (uint8_t*)io->xfer_buf;
         op->oid = {
             .inode = 1,
-            .stripe = io->offset / bsd->bs->get_block_size(),
+            .stripe = stripe,
         };
         op->version = UINT64_MAX; // last unstable
-        op->offset = io->offset % bsd->bs->get_block_size();
+        op->offset = io->offset % bsd->block_size;
         op->len = io->xfer_buflen;
+        op->bitmap = bsd->bitmap;
         op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
         {
             io->error = op->retval < 0 ? -op->retval : 0;
-            bs_data *bsd = (bs_data*)io->engine_data;
+            bs_data *bsd = ((bs_data*)((thread_data*)io->engine_data)->io_ops_data);
             bsd->inflight--;
             bsd->completed.push_back(io);
             if (bsd->trace)
@@ -224,16 +271,17 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
         op->buf = (uint8_t*)io->xfer_buf;
         op->oid = {
             .inode = 1,
-            .stripe = io->offset / bsd->bs->get_block_size(),
+            .stripe = stripe,
         };
         op->version = 0; // assign automatically
-        op->offset = io->offset % bsd->bs->get_block_size();
+        op->offset = io->offset % bsd->block_size;
         op->len = io->xfer_buflen;
+        op->bitmap = bsd->bitmap;
         if (bsd->ec)
         {
             op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
             {
-                bs_data *bsd = (bs_data*)io->engine_data;
+                bs_data *bsd = ((bs_data*)((thread_data*)io->engine_data)->io_ops_data);
                 if (bsd->trace)
                     printf("--- OP_WRITE %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
                 if (op->retval < 0)
@@ -241,6 +289,7 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
                     io->error = op->retval < 0 ? -op->retval : 0;
                     bsd->inflight--;
                     bsd->completed.push_back(io);
+                    _bs_retry(bsd, io->offset);
                     delete op;
                 }
                 else
@@ -254,12 +303,13 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
                     stab_op->len = 1;
                     stab_op->callback = [io, n](blockstore_op_t *op)
                     {
-                        bs_data *bsd = (bs_data*)io->engine_data;
+                        bs_data *bsd = ((bs_data*)((thread_data*)io->engine_data)->io_ops_data);
                         if (bsd->trace)
                             printf("--- OP_STABLE %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
                         io->error = op->retval < 0 ? -op->retval : 0;
                         bsd->inflight--;
                         bsd->completed.push_back(io);
+                        _bs_retry(bsd, io->offset);
                         delete op;
                     };
                     bsd->bs->enqueue_op(stab_op);
@@ -271,12 +321,13 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
         {
             op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
             {
-                bs_data *bsd = (bs_data*)io->engine_data;
+                bs_data *bsd = ((bs_data*)((thread_data*)io->engine_data)->io_ops_data);
                 if (bsd->trace)
                     printf("--- OP_WRITE_STABLE %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
                 io->error = op->retval < 0 ? -op->retval : 0;
                 bsd->inflight--;
                 bsd->completed.push_back(io);
+                _bs_retry(bsd, io->offset);
                 delete op;
             };
         }
@@ -286,7 +337,7 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
         op->opcode = BS_OP_SYNC;
         op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
         {
-            bs_data *bsd = (bs_data*)io->engine_data;
+            bs_data *bsd = ((bs_data*)((thread_data*)io->engine_data)->io_ops_data);
             io->error = op->retval < 0 ? -op->retval : 0;
             bsd->completed.push_back(io);
             bsd->inflight--;
@@ -314,6 +365,11 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
     return FIO_Q_QUEUED;
 }
 
+static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
+{
+    return _bs_queue(td, io, false);
+}
+
 static int bs_getevents(struct thread_data *td, unsigned int min, unsigned int max, const struct timespec *t)
 {
     bs_data *bsd = (bs_data*)td->io_ops_data;

src/blockstore/multilist.cpp

@@ -300,6 +300,11 @@ uint32_t multilist_index_t::find(uint32_t wanted_used)
     return heads[wanted_used];
 }
 
+uint32_t multilist_index_t::next(uint32_t pos)
+{
+    return nexts[pos];
+}
+
 void multilist_index_t::change(uint32_t pos, uint32_t old_used, uint32_t new_used)
 {
     if (new_used == old_used)

src/blockstore/multilist.h

@@ -32,6 +32,7 @@ struct multilist_index_t
     // used should be always < max_used
     multilist_index_t(uint32_t count, uint32_t max_used, uint32_t init_used);
     uint32_t find(uint32_t wanted_used);
+    uint32_t next(uint32_t pos);
     void change(uint32_t pos, uint32_t old_used, uint32_t new_used);
     void print();
 };

src/blockstore/ondisk_formats.h

@@ -185,7 +185,8 @@ struct __attribute__((__packed__)) blockstore_meta_header_v3_t
     uint32_t data_csum_type;
     uint32_t csum_block_size;
     uint32_t header_csum;
-    uint64_t compacted_lsn;
+    uint64_t meta_area_size;
+    uint64_t completed_lsn;
 
     void set_crc32c();
 };

src/blockstore/v1/impl.cpp

@@ -407,32 +407,77 @@ blockstore_clean_db_t& blockstore_impl_t::clean_db_shard(object_id oid)
     return clean_db_shards[(pool_id << (64-POOL_ID_BITS)) | pg_num];
 }
 
-void blockstore_impl_t::reshard_clean_db(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size)
+struct bs_reshard_state_t
 {
-    uint64_t pool_id = (uint64_t)pool;
+    int state = 0;
+    uint64_t pool_id = 0;
+    uint32_t pg_count = 0;
+    uint32_t pg_stripe_size = 0;
+    uint64_t chunk_size = 0;
+    std::map<pool_pg_id_t, blockstore_clean_db_t> old_shards;
     std::map<pool_pg_id_t, blockstore_clean_db_t> new_shards;
-    auto sh_it = clean_db_shards.lower_bound((pool_id << (64-POOL_ID_BITS)));
-    while (sh_it != clean_db_shards.end() &&
-        (sh_it->first >> (64-POOL_ID_BITS)) == pool_id)
+    std::map<pool_pg_id_t, blockstore_clean_db_t>::iterator sh_it;
+    blockstore_clean_db_t::iterator obj_it;
+};
+
+void* blockstore_impl_t::reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit)
+{
+    auto & settings = clean_db_settings[pool];
+    if (settings.pg_count == pg_count && settings.pg_stripe_size == pg_stripe_size)
     {
-        for (auto & pair: sh_it->second)
-        {
-            // like map_to_pg()
-            uint64_t pg_num = (pair.first.stripe / pg_stripe_size) % pg_count + 1;
-            uint64_t shard_id = (pool_id << (64-POOL_ID_BITS)) | pg_num;
-            new_shards[shard_id][pair.first] = pair.second;
-        }
+        return NULL;
+    }
+    bs_reshard_state_t *st = new bs_reshard_state_t;
+    st->state = 0;
+    st->pool_id = pool;
+    st->pg_count = pg_count;
+    st->pg_stripe_size = pg_stripe_size;
+    auto sh_it = clean_db_shards.lower_bound((st->pool_id << (64-POOL_ID_BITS)));
+    while (sh_it != clean_db_shards.end() &&
+        (sh_it->first >> (64-POOL_ID_BITS)) == st->pool_id)
+    {
+        st->old_shards[sh_it->first] = std::move(sh_it->second);
         clean_db_shards.erase(sh_it++);
     }
-    for (sh_it = new_shards.begin(); sh_it != new_shards.end(); sh_it++)
+    bool finished = reshard_continue(st, chunk_limit);
+    return finished ? NULL : st;
+}
+
+bool blockstore_impl_t::reshard_continue(void *reshard_state, uint64_t chunk_limit)
+{
+    bs_reshard_state_t *st = (bs_reshard_state_t*)reshard_state;
+    uint64_t chunk_size = 0;
+    if (st->state == 1)
+        goto resume_1;
+    for (st->sh_it = st->old_shards.begin(); st->sh_it != st->old_shards.end(); )
+    {
+        for (st->obj_it = st->sh_it->second.begin(); st->obj_it != st->sh_it->second.end(); st->obj_it++)
+        {
+            if (chunk_limit > 0 && chunk_size >= chunk_limit)
+            {
+                st->state = 1;
+                return false;
+            }
+resume_1:
+            // like map_to_pg()
+            uint64_t pg_num = (st->obj_it->first.stripe / st->pg_stripe_size) % st->pg_count + 1;
+            uint64_t shard_id = (st->pool_id << (64-POOL_ID_BITS)) | pg_num;
+            st->new_shards[shard_id][st->obj_it->first] = st->obj_it->second;
+            chunk_size++;
+        }
+        st->old_shards.erase(st->sh_it++);
+    }
+    for (auto sh_it = st->new_shards.begin(); sh_it != st->new_shards.end(); sh_it++)
     {
         auto & to = clean_db_shards[sh_it->first];
         to.swap(sh_it->second);
     }
-    clean_db_settings[pool_id] = (pool_shard_settings_t){
-        .pg_count = pg_count,
-        .pg_stripe_size = pg_stripe_size,
+    clean_db_settings[st->pool_id] = (pool_shard_settings_t){
+        .pg_count = st->pg_count,
+        .pg_stripe_size = st->pg_stripe_size,
     };
+    delete st;
+    return true;
 }
 
 void blockstore_impl_t::process_list(blockstore_op_t *op)
@@ -465,7 +510,10 @@ void blockstore_impl_t::process_list(blockstore_op_t *op)
                 sh_it->second.pg_count != pg_count ||
                 sh_it->second.pg_stripe_size != pg_stripe_size)
             {
-                reshard_clean_db(pool_id, pg_count, pg_stripe_size);
+                // Sharding mismatch
+                op->retval = -EAGAIN;
+                FINISH_OP(op);
+                return;
             }
             first_shard = last_shard = ((uint64_t)pool_id << (64-POOL_ID_BITS)) | list_pg;
         }

src/blockstore/v1/impl.h

@@ -202,7 +202,6 @@ class blockstore_impl_t: public blockstore_i
     uint8_t* get_clean_entry_bitmap(uint64_t block_loc, int offset);
 
     blockstore_clean_db_t& clean_db_shard(object_id oid);
-    void reshard_clean_db(pool_id_t pool_id, uint32_t pg_count, uint32_t pg_stripe_size);
     void recalc_inode_space_stats(uint64_t pool_id, bool per_inode);
 
     // Journaling
@@ -288,6 +287,10 @@ public:
     void parse_config(blockstore_config_t & config);
     void parse_config(blockstore_config_t & config, bool init);
 
+    // Reshard database for a pool
+    void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
+    bool reshard_continue(void *reshard_state, uint64_t chunk_limit);
+
     // Event loop
     void loop();
 

src/blockstore/v1/internal.h

@@ -1,3 +1,6 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 (see README.md for details)
+
 #pragma once
 
 // States are not stored on disk. Instead, they're deduced from the journal

src/blockstore/v1/write.cpp

@@ -183,7 +183,7 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
         uint32_t end = (op->offset+op->len-1) / dsk.csum_block_size;
         auto fn = state & BS_ST_BIG_WRITE ? crc32c_pad : crc32c_nopad;
         if (start == end)
-            data_csums[0] = fn(0, op->buf, op->len, op->offset - start*dsk.csum_block_size, end*dsk.csum_block_size - (op->offset+op->len));
+            data_csums[0] = fn(0, op->buf, op->len, op->offset - start*dsk.csum_block_size, (end+1)*dsk.csum_block_size - (op->offset+op->len));
         else
         {
             // First block

src/client/cluster_client.cpp

@@ -765,8 +765,13 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
     {
         return;
     }
-    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && enable_writeback && !(op->flags & OP_FLUSH_BUFFER) &&
-        !op->version /* no CAS writeback */)
+    // CAS writes are simplified: they're not cached, not resliced, not retried, and not part of the regular write queue at all
+    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && op->version)
+    {
+        execute_cas(op);
+        return;
+    }
+    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && enable_writeback && !(op->flags & OP_FLUSH_BUFFER))
     {
         if (wb->writebacks_active >= client_max_writeback_iodepth)
         {
@@ -788,7 +793,7 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
     }
     if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && !(op->flags & OP_IMMEDIATE_COMMIT))
     {
-        if (!(op->flags & OP_FLUSH_BUFFER) && !op->version /* no CAS write-repeat */)
+        if (!(op->flags & OP_FLUSH_BUFFER))
         {
             uint64_t flush_id = ++wb->last_flush_id;
             wb->copy_write(op, CACHE_REPEATING, flush_id);
@@ -847,6 +852,76 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
     }
 }
 
+void cluster_client_t::execute_cas(cluster_op_t *op)
+{
+    slice_rw(op);
+    op->needs_reslice = false;
+    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && op->version && op->parts.size() > 1)
+    {
+        // Atomic writes to multiple stripes are unsupported
+        op->retval = -EINVAL;
+        auto cb = std::move(op->callback);
+        cb(op);
+        return;
+    }
+    int res = try_send(op, 0, [this, op](osd_op_t *part)
+    {
+        int expected = part->req.hdr.opcode == OSD_OP_DELETE ? 0 : part->req.rw.len;
+        op->retval = part->reply.hdr.retval;
+        if (op->retval != expected && op->retval >= 0)
+            op->retval = -EIO;
+        op->retval = op->retval == -EPIPE ? -EINTR : op->retval;
+        auto peer_it = msgr.osd_peer_fds.find(op->parts[0].osd_num);
+        if (op->retval != 0 || (op->flags & OP_IMMEDIATE_COMMIT))
+        {
+            auto cb = std::move(op->callback);
+            cb(op);
+        }
+        else if (peer_it == msgr.osd_peer_fds.end())
+        {
+            // Care must be taken to make sure that the client doesn't reconnect to the OSD
+            // before executing the previously completed operation callback (!)
+            op->retval = -EINTR;
+            auto cb = std::move(op->callback);
+            cb(op);
+        }
+        else
+        {
+            // CAS writes have a built-in sync
+            auto peer_fd = peer_it->second;
+            *part = (osd_op_t){
+                .op_type = OSD_OP_OUT,
+                .peer_fd = peer_fd,
+                .req = {
+                    .hdr = {
+                        .magic = SECONDARY_OSD_OP_MAGIC,
+                        .opcode = OSD_OP_SYNC,
+                    },
+                },
+                .callback = [op](osd_op_t *part)
+                {
+                    if (part->reply.hdr.retval != 0)
+                    {
+                        op->retval = part->reply.hdr.retval;
+                        op->retval = op->retval == -EPIPE ? -EINTR : op->retval;
+                    }
+                    auto cb = std::move(op->callback);
+                    cb(op);
+                },
+            };
+            msgr.outbox_push(part);
+        }
+    });
+    if (res == TRY_SEND_CONNECTING || res == TRY_SEND_OFFLINE)
+    {
+        // In theory, CAS writes could wait for the PG to come up, but it's easier to just fail it
+        op->retval = -EINTR;
+        auto cb = std::move(op->callback);
+        cb(op);
+        return;
+    }
+}
+
 bool cluster_client_t::check_rw(cluster_op_t *op)
 {
     if (op->opcode == OSD_OP_SYNC)
@@ -956,13 +1031,6 @@ resume_0:
     // Slice the operation into parts
     slice_rw(op);
     op->needs_reslice = false;
-    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && op->version && op->parts.size() > 1)
-    {
-        // Atomic writes to multiple stripes are unsupported
-        op->retval = -EINVAL;
-        erase_op(op);
-        return 1;
-    }
 resume_1:
     // Send unsent parts, if they're not subject to change
     op->state = 2;
@@ -1189,9 +1257,9 @@ void cluster_client_t::slice_rw(cluster_op_t *op)
         {
             op->bitmap_buf = realloc_or_die(op->bitmap_buf, bitmap_mem);
             op->part_bitmaps = (uint8_t*)op->bitmap_buf + object_bitmap_size;
+            memset((uint8_t*)op->bitmap_buf+op->bitmap_buf_size, 0, bitmap_mem-op->bitmap_buf_size);
             op->bitmap_buf_size = bitmap_mem;
         }
-        memset(op->bitmap_buf, 0, bitmap_mem);
     }
     int iov_idx = 0;
     size_t iov_pos = 0;
@@ -1307,7 +1375,7 @@ bool cluster_client_t::affects_osd(uint64_t inode, uint64_t offset, uint64_t len
     return false;
 }
 
-int cluster_client_t::try_send(cluster_op_t *op, int i)
+int cluster_client_t::try_send(cluster_op_t *op, int i, std::function<void(osd_op_t *op_part)> cb)
 {
     if (!msgr_initialized)
     {
@@ -1337,7 +1405,7 @@ int cluster_client_t::try_send(cluster_op_t *op, int i)
         if (peer_it != msgr.osd_peer_fds.end())
         {
             int peer_fd = peer_it->second;
-            part->flags |= PART_SENT;
+            part->flags |= PART_SENT|PART_VALID;
             op->inflight_count++;
             uint64_t pg_bitmap_size = (pool_cfg.data_block_size / pool_cfg.bitmap_granularity / 8) * (
                 pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks
@@ -1367,7 +1435,7 @@ int cluster_client_t::try_send(cluster_op_t *op, int i)
                     ? (uint8_t*)op->part_bitmaps + pg_bitmap_size*i : NULL),
                 .bitmap_len = (unsigned)(op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP
                     ? pg_bitmap_size : 0),
-                .callback = [this, part](osd_op_t *op_part)
+                .callback = cb ? cb : [this, part](osd_op_t *op_part)
                 {
                     handle_op_part(part);
                 },
@@ -1550,14 +1618,11 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
             dirty_osds.insert(part->osd_num);
         part->flags |= PART_DONE;
         op->done_count++;
-        if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
+        if ((op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
+            && op->inode == op->cur_inode)
         {
-            copy_part_bitmap(op, part);
-            if (op->inode == op->cur_inode)
-            {
-                // Read only returns the version of the uppermost layer
-                op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0;
-            }
+            // Read only returns the version of the uppermost layer
+            op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0;
         }
         else if (op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE)
         {
@@ -1565,6 +1630,13 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
         }
         if (op->inflight_count == 0 && !op->retry_after)
         {
+            // Copy part bitmaps only after finishing all part reads
+            if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
+            {
+                for (auto & part: op->parts)
+                    if (part.flags == (PART_SENT|PART_VALID|PART_DONE))
+                        copy_part_bitmap(op, &part);
+            }
             if (op->opcode == OSD_OP_SYNC)
                 continue_sync(op);
             else

src/client/cluster_client.h

@@ -175,12 +175,13 @@ protected:
     void on_change_node_placement_hook();
 
     void execute_internal(cluster_op_t *op);
+    void execute_cas(cluster_op_t *op);
     void unshift_op(cluster_op_t *op);
     int continue_rw(cluster_op_t *op);
     bool check_rw(cluster_op_t *op);
     void slice_rw(cluster_op_t *op);
     void reset_retry_timer(int new_duration);
-    int try_send(cluster_op_t *op, int i);
+    int try_send(cluster_op_t *op, int i, std::function<void(osd_op_t *op_part)> cb = nullptr);
     int continue_sync(cluster_op_t *op);
     void send_sync(cluster_op_t *op, cluster_op_part_t *part);
     void handle_op_part(cluster_op_part_t *part);

src/client/cluster_client_impl.h

@@ -10,6 +10,7 @@
 #define PART_DONE 2
 #define PART_ERROR 4
 #define PART_RETRY 8
+#define PART_VALID 16
 #define CACHE_DIRTY 1
 #define CACHE_WRITTEN 2
 #define CACHE_FLUSHING 3

src/client/etcd_state_client.cpp

@@ -414,7 +414,10 @@ void etcd_state_client_t::start_etcd_watcher()
                 }
                 // Save revision only if it's present in the message - because sometimes etcd sends something without a header, like:
                 // {"error": {"grpc_code": 14, "http_code": 503, "http_status": "Service Unavailable", "message": "error reading from server: EOF"}}
-                if (etcd_watches_initialised == ETCD_TOTAL_WATCHES && !data["result"]["header"]["revision"].is_null())
+                // Also don't save revision from the initial created: true messages because they always contain the latest revision
+                if (etcd_watches_initialised == ETCD_TOTAL_WATCHES &&
+                    !data["result"]["header"]["revision"].is_null() &&
+                    !data["result"]["created"].bool_value())
                 {
                     // Restart watchers from the same revision number as in the last received message,
                     // not from the next one to protect against revision being split into multiple messages,
@@ -568,9 +571,19 @@ void etcd_state_client_t::start_ws_keepalive()
 
 void etcd_state_client_t::load_global_config()
 {
-    etcd_call("/kv/range", json11::Json::object {
-        { "key", base64_encode(etcd_prefix+"/config/global") }
-    }, etcd_quick_timeout, max_etcd_attempts, 0, [this](std::string err, json11::Json data)
+    json11::Json::object req = { { "success", json11::Json::array {
+        json11::Json::object {
+            { "request_range", json11::Json::object {
+                { "key", base64_encode(etcd_prefix+"/config/global") },
+            } }
+        },
+        json11::Json::object {
+            { "request_range", json11::Json::object {
+                { "key", base64_encode(etcd_prefix+"/config/pools") },
+            } }
+        },
+    } } };
+    etcd_txn(req, etcd_quick_timeout, max_etcd_attempts, 0, [this](std::string err, json11::Json data)
     {
         if (err != "")
         {
@@ -588,10 +601,12 @@ void etcd_state_client_t::load_global_config()
             }
             return;
         }
+        json11::Json config_kv = data["responses"][0]["response_range"]["kvs"][0];
+        json11::Json pools_kv = data["responses"][1]["response_range"]["kvs"][0];
         json11::Json::object global_config;
-        if (data["kvs"].array_items().size() > 0)
+        if (!config_kv.is_null())
         {
-            auto kv = parse_etcd_kv(data["kvs"][0]);
+            auto kv = parse_etcd_kv(config_kv);
             if (kv.value.is_object())
             {
                 global_config = kv.value.object_items();
@@ -608,6 +623,11 @@ void etcd_state_client_t::load_global_config()
             global_bitmap_granularity = DEFAULT_BITMAP_GRANULARITY;
         }
         global_immediate_commit = parse_immediate_commit(global_config["immediate_commit"].string_value(), IMMEDIATE_ALL);
+        if (!pools_kv.is_null())
+        {
+            auto kv = parse_etcd_kv(pools_kv);
+            parse_state(kv);
+        }
         on_load_config_hook(global_config);
     });
 }
@@ -945,8 +965,12 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
             if (pc.pg_stripe_size < min_stripe_size)
                 pc.pg_stripe_size = min_stripe_size;
             // Save
-            pc.real_pg_count = this->pool_config[pool_id].real_pg_count;
-            std::swap(pc.pg_config, this->pool_config[pool_id].pg_config);
+            auto & old_pc = this->pool_config[pool_id];
+            pc.real_pg_count = old_pc.real_pg_count;
+            pc.applied_pg_count = old_pc.applied_pg_count;
+            pc.applied_pg_stripe_size = old_pc.applied_pg_stripe_size;
+            pc.reshard_state = old_pc.reshard_state;
+            std::swap(pc.pg_config, old_pc.pg_config);
             std::swap(this->pool_config[pool_id], pc);
             auto & parsed_cfg = this->pool_config[pool_id];
             parsed_cfg.exists = true;

src/client/etcd_state_client.h

@@ -6,7 +6,7 @@
 #include <set>
 
 #include "json11/json11.hpp"
-#include "osd_id.h"
+#include "object_id.h"
 #include "timerfd_manager.h"
 
 #define ETCD_CONFIG_WATCH_ID 1
@@ -68,6 +68,11 @@ struct pool_config_t
     std::string used_for_app;
     int backfillfull = 0;
     int local_reads = 0;
+
+    // runtime data, used only by OSD:
+    uint64_t applied_pg_count = 0;
+    uint64_t applied_pg_stripe_size = 0;
+    void *reshard_state = NULL;
 };
 
 struct inode_config_t

src/client/http_client.cpp

@@ -23,7 +23,7 @@
 #define READ_BUFFER_SIZE 9000
 
 static std::string ws_format_frame(int type, uint64_t size);
-static bool ws_parse_frame(std::string & buf, int & type, std::string & res);
+static bool ws_parse_frame(std::string & buf, uint8_t & type, std::string & res);
 static void parse_http_headers(std::string & res, http_response_t *parsed);
 
 struct http_co_t
@@ -70,7 +70,7 @@ struct http_co_t
     void submit_read(bool check_timeout);
     void submit_send();
     bool handle_read();
-    void post_message(int type, const std::string & msg);
+    void post_message(uint8_t type, const std::string & msg);
     void send_request(const std::string & host, const std::string & request,
         const http_options_t & options, std::function<void(const http_response_t *response)> response_callback);
 };
@@ -199,12 +199,12 @@ void http_co_t::send_request(const std::string & host, const std::string & reque
     stackout();
 }
 
-void http_post_message(http_co_t *handler, int type, const std::string & msg)
+void http_post_message(http_co_t *handler, uint8_t type, const std::string & msg)
 {
     handler->post_message(type, msg);
 }
 
-void http_co_t::post_message(int type, const std::string & msg)
+void http_co_t::post_message(uint8_t type, const std::string & msg)
 {
     stackin();
     if (state == HTTP_CO_WEBSOCKET)
@@ -608,7 +608,13 @@ bool http_co_t::handle_read()
     {
         while (ws_parse_frame(response, parsed.ws_msg_type, parsed.body))
         {
-            response_callback(&parsed);
+            if (parsed.ws_msg_type == WS_PING)
+            {
+                // Reply with WS_PONG
+                post_message(WS_PONG, "");
+            }
+            else
+                response_callback(&parsed);
             parsed.body = "";
         }
     }
@@ -698,7 +704,7 @@ static std::string ws_format_frame(int type, uint64_t size)
     return res;
 }
 
-static bool ws_parse_frame(std::string & buf, int & type, std::string & res)
+static bool ws_parse_frame(std::string & buf, uint8_t & type, std::string & res)
 {
     uint64_t hdr = 2;
     if (buf.size() < hdr)

src/client/http_client.h

@@ -32,7 +32,7 @@ struct http_response_t
     int status_code = 0;
     std::string status_line;
     std::map<std::string, std::string> headers;
-    int ws_msg_type = -1;
+    uint8_t ws_msg_type = -1;
     std::string body;
 
     void parse_json_response(std::string & error, json11::Json & r) const;
@@ -46,5 +46,5 @@ http_co_t* open_websocket(timerfd_manager_t *tfd, const std::string & host, cons
     int timeout, std::function<void(const http_response_t *msg)> on_message);
 void http_request(http_co_t *handler, const std::string & host, const std::string & request,
     const http_options_t & options, std::function<void(const http_response_t *response)> response_callback);
-void http_post_message(http_co_t *handler, int type, const std::string & msg);
+void http_post_message(http_co_t *handler, uint8_t type, const std::string & msg);
 void http_close(http_co_t *co);

src/client/messenger.cpp

@@ -145,7 +145,7 @@ void osd_messenger_t::init()
         rdma_contexts = msgr_rdma_context_t::create_all(
             osd_num && osd_cluster_network_masks.size() ? osd_cluster_network_masks : osd_network_masks,
             rdma_device != "" ? rdma_device.c_str() : NULL,
-            rdma_port_num, rdma_gid_index, rdma_mtu, rdma_odp, log_level
+            rdma_port_num, rdma_gid_index, rdma_mtu, log_level
         );
         if (!rdma_contexts.size())
         {
@@ -322,7 +322,6 @@ void osd_messenger_t::parse_config(const json11::Json & config)
     this->rdma_max_msg = config["rdma_max_msg"].uint64_value();
     if (!this->rdma_max_msg || this->rdma_max_msg > 128*1024*1024)
         this->rdma_max_msg = 129*1024;
-    this->rdma_odp = config["rdma_odp"].bool_value();
 #endif
     if (!osd_num)
         this->iothread_count = (uint32_t)config["client_iothread_count"].uint64_value();
@@ -749,7 +748,7 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl)
                     }
                 });
                 // Add the initial receive request
-                try_recv_rdma(cl);
+                init_recv_rdma(cl);
             }
         }
 #endif

src/client/messenger.h

@@ -200,7 +200,6 @@ protected:
     std::vector<msgr_rdma_context_t *> rdma_contexts;
     uint64_t rdma_max_sge = 0, rdma_max_send = 0, rdma_max_recv = 0;
     uint64_t rdma_max_msg = 0;
-    bool rdma_odp = false;
     rdma_event_channel *rdmacm_evch = NULL;
     std::map<rdma_cm_id*, osd_client_t*> rdmacm_connections;
     std::map<rdma_cm_id*, rdmacm_connecting_t*> rdmacm_connecting;
@@ -210,7 +209,7 @@ protected:
     std::vector<int> read_ready_clients;
     std::vector<int> write_ready_clients;
     // We don't use ringloop->set_immediate here because we may have no ringloop in client :)
-    std::vector<osd_op_t*> set_immediate_ops;
+    std::deque<osd_op_t*> set_immediate_ops;
 
 public:
     timerfd_manager_t *tfd = NULL;
@@ -287,9 +286,7 @@ protected:
 
 #ifdef WITH_RDMA
     void try_send_rdma(osd_client_t *cl);
-    void try_send_rdma_odp(osd_client_t *cl);
-    void try_send_rdma_nodp(osd_client_t *cl);
-    bool try_recv_rdma(osd_client_t *cl);
+    bool init_recv_rdma(osd_client_t *cl);
     void handle_rdma_events(msgr_rdma_context_t *rdma_context);
     msgr_rdma_context_t* choose_rdma_context(osd_client_t *cl);
 #endif

src/client/msgr_rdma.cpp

@@ -59,14 +59,26 @@ msgr_rdma_context_t::~msgr_rdma_context_t()
         ibv_destroy_cq(cq);
     if (channel)
         ibv_destroy_comp_channel(channel);
-    if (mr)
-        ibv_dereg_mr(mr);
     if (pd)
         ibv_dealloc_pd(pd);
     if (context && !is_cm)
         ibv_close_device(context);
 }
 
+msgr_rdma_buf_t::~msgr_rdma_buf_t()
+{
+    if (buf)
+    {
+        free(buf);
+        buf = NULL;
+    }
+    if (mr)
+    {
+        ibv_dereg_mr(mr);
+        mr = NULL;
+    }
+}
+
 msgr_rdma_connection_t::~msgr_rdma_connection_t()
 {
     ctx->reserve_cqe(-max_send-max_recv);
@@ -84,26 +96,6 @@ msgr_rdma_connection_t::~msgr_rdma_connection_t()
     if (qp)
         ibv_destroy_qp(qp);
 #endif
-    if (recv_buffers.size())
-    {
-        for (auto b: recv_buffers)
-        {
-            if (b.mr)
-                ibv_dereg_mr(b.mr);
-            free(b.buf);
-        }
-        recv_buffers.clear();
-    }
-    if (send_out.mr)
-    {
-        ibv_dereg_mr(send_out.mr);
-        send_out.mr = NULL;
-    }
-    if (send_out.buf)
-    {
-        free(send_out.buf);
-        send_out.buf = NULL;
-    }
     send_out_size = 0;
 }
 
@@ -188,7 +180,7 @@ static int match_port_gid(const std::vector<addr_mask_t> & osd_network_masks, ib
 #endif
 
 std::vector<msgr_rdma_context_t*> msgr_rdma_context_t::create_all(const std::vector<addr_mask_t> & osd_network_masks,
-    const char *sel_dev_name, int sel_port_num, int sel_gid_index, uint32_t sel_mtu, bool odp, int log_level)
+    const char *sel_dev_name, int sel_port_num, int sel_gid_index, uint32_t sel_mtu, int log_level)
 {
     int res;
     std::vector<msgr_rdma_context_t*> ret;
@@ -277,7 +269,7 @@ std::vector<msgr_rdma_context_t*> msgr_rdma_context_t::create_all(const std::vec
                 {
                     if (log_level > 0)
                         log_rdma_dev_port_gid(dev, port_num, best_gid_idx, port_mtu, best_gidx);
-                    auto ctx = msgr_rdma_context_t::create(dev, portinfo, port_num, best_gid_idx, port_mtu, odp, log_level);
+                    auto ctx = msgr_rdma_context_t::create(dev, portinfo, port_num, best_gid_idx, port_mtu, log_level);
                     if (ctx)
                     {
                         ctx->net_mask = osd_network_masks[net_num];
@@ -297,7 +289,7 @@ std::vector<msgr_rdma_context_t*> msgr_rdma_context_t::create_all(const std::vec
                     log_rdma_dev_port_gid(dev, port_num, best_gid_idx, port_mtu, gidx);
                 }
 #endif
-                auto ctx = msgr_rdma_context_t::create(dev, portinfo, port_num, best_gid_idx, port_mtu, odp, log_level);
+                auto ctx = msgr_rdma_context_t::create(dev, portinfo, port_num, best_gid_idx, port_mtu, log_level);
                 if (ctx)
                     ret.push_back(ctx);
             }
@@ -312,7 +304,7 @@ cleanup:
     return ret;
 }
 
-msgr_rdma_context_t *msgr_rdma_context_t::create(ibv_device *dev, ibv_port_attr & portinfo, int ib_port, int gid_index, uint32_t mtu, bool odp, int log_level)
+msgr_rdma_context_t *msgr_rdma_context_t::create(ibv_device *dev, ibv_port_attr & portinfo, int ib_port, int gid_index, uint32_t mtu, int log_level)
 {
     msgr_rdma_context_t *ctx = new msgr_rdma_context_t();
     ibv_context *context = ibv_open_device(dev);
@@ -352,30 +344,6 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(ibv_device *dev, ibv_port_attr
         goto cleanup;
     }
 
-    ctx->odp = odp;
-    if (ctx->odp)
-    {
-        if (!(ctx->attrx.odp_caps.general_caps & IBV_ODP_SUPPORT) ||
-            !(ctx->attrx.odp_caps.general_caps & IBV_ODP_SUPPORT_IMPLICIT) ||
-            !(ctx->attrx.odp_caps.per_transport_caps.rc_odp_caps & IBV_ODP_SUPPORT_SEND) ||
-            !(ctx->attrx.odp_caps.per_transport_caps.rc_odp_caps & IBV_ODP_SUPPORT_RECV))
-        {
-            ctx->odp = false;
-            if (log_level > 0)
-                fprintf(stderr, "The RDMA device isn't implicit ODP (On-Demand Paging) capable, disabling it\n");
-        }
-    }
-
-    if (ctx->odp)
-    {
-        ctx->mr = ibv_reg_mr(ctx->pd, NULL, SIZE_MAX, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_ON_DEMAND);
-        if (!ctx->mr)
-        {
-            fprintf(stderr, "Couldn't register RDMA memory region\n");
-            goto cleanup;
-        }
-    }
-
     ctx->channel = ibv_create_comp_channel(ctx->context);
     if (!ctx->channel)
     {
@@ -394,8 +362,6 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(ibv_device *dev, ibv_port_attr
     return ctx;
 
 cleanup:
-    if (context)
-        ibv_close_device(context);
     delete ctx;
     return NULL;
 }
@@ -493,12 +459,12 @@ int msgr_rdma_connection_t::connect(msgr_rdma_address_t *dest)
             .grh        = {
                 .dgid = dest->gid,
                 .sgid_index = conn->ctx->gid_index,
-                .hop_limit = 1, // FIXME can it vary?
+                .hop_limit = 64, // FIXME can it vary?
             },
             .dlid       = dest->lid,
             .sl         = 0, // service level
             .src_path_bits = 0,
-            .is_global  = (uint8_t)(dest->gid.global.interface_id ? 1 : 0),
+            .is_global  = (uint8_t)(dest->gid.global.subnet_prefix || dest->gid.global.interface_id ? 1 : 0),
             .port_num   = conn->ctx->ib_port,
         },
         .max_rd_atomic  = 1,
@@ -611,67 +577,19 @@ static int try_send_rdma_copy(osd_client_t *cl, uint8_t *dst, int dst_len)
     return total_dst_len-dst_len;
 }
 
-void osd_messenger_t::try_send_rdma_odp(osd_client_t *cl)
-{
-    auto rc = cl->rdma_conn;
-    if (!cl->send_list.size() || rc->cur_send >= rc->max_send)
-    {
-        return;
-    }
-    uint64_t op_size = 0, op_sge = 0;
-    ibv_sge sge[rc->max_sge];
-    while (rc->send_pos < cl->send_list.size())
-    {
-        iovec & iov = cl->send_list[rc->send_pos];
-        if (op_size >= rc->max_msg || op_sge >= rc->max_sge)
-        {
-            rc->send_sizes.push_back(op_size);
-            try_send_rdma_wr(cl, sge, op_sge);
-            op_sge = 0;
-            op_size = 0;
-            if (rc->cur_send >= rc->max_send)
-            {
-                break;
-            }
-        }
-        uint32_t len = (uint32_t)(op_size+iov.iov_len-rc->send_buf_pos < rc->max_msg
-            ? iov.iov_len-rc->send_buf_pos : rc->max_msg-op_size);
-        sge[op_sge++] = {
-            .addr = (uintptr_t)((uint8_t*)iov.iov_base+rc->send_buf_pos),
-            .length = len,
-            .lkey = rc->ctx->mr->lkey,
-        };
-        op_size += len;
-        rc->send_buf_pos += len;
-        if (rc->send_buf_pos >= iov.iov_len)
-        {
-            rc->send_pos++;
-            rc->send_buf_pos = 0;
-        }
-    }
-    if (op_sge > 0)
-    {
-        rc->send_sizes.push_back(op_size);
-        try_send_rdma_wr(cl, sge, op_sge);
-    }
-}
-
-void osd_messenger_t::try_send_rdma_nodp(osd_client_t *cl)
+void osd_messenger_t::try_send_rdma(osd_client_t *cl)
 {
     auto rc = cl->rdma_conn;
     if (!rc->send_out_size)
     {
         // Allocate send ring buffer, if not yet
         rc->send_out_size = rc->max_msg*rdma_max_send;
-        rc->send_out.buf = malloc_or_die(rc->send_out_size);
-        if (!rc->ctx->odp)
+        rc->send_out.buf = (uint8_t*)malloc_or_die(rc->send_out_size);
+        rc->send_out.mr = ibv_reg_mr(rc->ctx->pd, rc->send_out.buf, rc->send_out_size, 0);
+        if (!rc->send_out.mr)
         {
-            rc->send_out.mr = ibv_reg_mr(rc->ctx->pd, rc->send_out.buf, rc->send_out_size, 0);
-            if (!rc->send_out.mr)
-            {
-                fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
-                exit(1);
-            }
+            fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
+            exit(1);
         }
     }
     // Copy data into the buffer and send it
@@ -696,7 +614,7 @@ void osd_messenger_t::try_send_rdma_nodp(osd_client_t *cl)
             ibv_sge sge = {
                 .addr = (uintptr_t)dst,
                 .length = (uint32_t)copied,
-                .lkey = rc->ctx->odp ? rc->ctx->mr->lkey : rc->send_out.mr->lkey,
+                .lkey = rc->send_out.mr->lkey,
             };
             try_send_rdma_wr(cl, &sge, 1);
             rc->send_sizes.push_back(copied);
@@ -704,20 +622,12 @@ void osd_messenger_t::try_send_rdma_nodp(osd_client_t *cl)
     }
 }
 
-void osd_messenger_t::try_send_rdma(osd_client_t *cl)
-{
-    if (cl->rdma_conn->ctx->odp)
-        try_send_rdma_odp(cl);
-    else
-        try_send_rdma_nodp(cl);
-}
-
-static void try_recv_rdma_wr(osd_client_t *cl, msgr_rdma_buf_t b)
+static void try_recv_rdma_wr(osd_client_t *cl, void *buf)
 {
     ibv_sge sge = {
-        .addr = (uintptr_t)b.buf,
+        .addr = (uintptr_t)buf,
         .length = (uint32_t)cl->rdma_conn->max_msg,
-        .lkey = cl->rdma_conn->ctx->odp ? cl->rdma_conn->ctx->mr->lkey : b.mr->lkey,
+        .lkey = cl->rdma_conn->recv_buf.mr->lkey,
     };
     ibv_recv_wr *bad_wr = NULL;
     ibv_recv_wr wr = {
@@ -734,22 +644,20 @@ static void try_recv_rdma_wr(osd_client_t *cl, msgr_rdma_buf_t b)
     cl->rdma_conn->cur_recv++;
 }
 
-bool osd_messenger_t::try_recv_rdma(osd_client_t *cl)
+bool osd_messenger_t::init_recv_rdma(osd_client_t *cl)
 {
     auto rc = cl->rdma_conn;
-    while (rc->cur_recv < rc->max_recv)
+    assert(!rc->recv_buf.buf);
+    rc->recv_buf.buf = (uint8_t*)malloc_or_die(rc->max_msg * rc->max_recv);
+    rc->recv_buf.mr = ibv_reg_mr(rc->ctx->pd, rc->recv_buf.buf, rc->max_msg * rc->max_recv, IBV_ACCESS_LOCAL_WRITE);
+    if (!rc->recv_buf.mr)
     {
-        msgr_rdma_buf_t b;
-        b.buf = malloc_or_die(rc->max_msg);
-        if (!rc->ctx->odp)
-        {
-            b.mr = ibv_reg_mr(rc->ctx->pd, b.buf, rc->max_msg, IBV_ACCESS_LOCAL_WRITE);
-            if (!b.mr)
-            {
-                fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
-                exit(1);
-            }
-        }
+        fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
+        exit(1);
+    }
+    for (uint32_t i = 0; i < rc->max_recv; i++)
+    {
+        uint8_t *b = rc->recv_buf.buf + i*rc->max_msg;
         rc->recv_buffers.push_back(b);
         try_recv_rdma_wr(cl, b);
     }
@@ -807,7 +715,7 @@ void osd_messenger_t::handle_rdma_events(msgr_rdma_context_t *rdma_context)
                 cl->ping_time_remaining = 0;
                 cl->idle_time_remaining = osd_idle_timeout;
                 rc->cur_recv--;
-                if (!handle_read_buffer(cl, rc->recv_buffers[rc->next_recv_buf].buf, wc[i].byte_len))
+                if (!handle_read_buffer(cl, rc->recv_buffers[rc->next_recv_buf], wc[i].byte_len))
                 {
                     // handle_read_buffer may stop the client
                     clear_immediate_ops(client_id);
@@ -821,14 +729,11 @@ void osd_messenger_t::handle_rdma_events(msgr_rdma_context_t *rdma_context)
                 rc->cur_send--;
                 uint64_t sent_size = rc->send_sizes.at(0);
                 rc->send_sizes.erase(rc->send_sizes.begin(), rc->send_sizes.begin()+1);
-                if (!rdma_context->odp)
-                {
-                    rc->send_done_pos += sent_size;
-                    rc->send_out_full = false;
-                    if (rc->send_done_pos == rc->send_out_size)
-                        rc->send_done_pos = 0;
-                    assert(rc->send_done_pos < rc->send_out_size);
-                }
+                rc->send_done_pos += sent_size;
+                rc->send_out_full = false;
+                if (rc->send_done_pos == rc->send_out_size)
+                    rc->send_done_pos = 0;
+                assert(rc->send_done_pos < rc->send_out_size);
                 int send_pos = 0, send_buf_pos = 0;
                 while (sent_size > 0)
                 {

src/client/msgr_rdma.h

@@ -26,8 +26,6 @@ struct msgr_rdma_context_t
     ibv_context *context = NULL;
     ibv_device_attr_ex attrx;
     ibv_pd *pd = NULL;
-    bool odp = false;
-    ibv_mr *mr = NULL;
     ibv_comp_channel *channel = NULL;
     ibv_cq *cq = NULL;
     ibv_port_attr portinfo;
@@ -43,9 +41,9 @@ struct msgr_rdma_context_t
     int cm_refs = 0;
 
     static std::vector<msgr_rdma_context_t*> create_all(const std::vector<addr_mask_t> & osd_network_masks,
-        const char *sel_dev_name, int sel_port_num, int sel_gid_index, uint32_t sel_mtu, bool odp, int log_level);
+        const char *sel_dev_name, int sel_port_num, int sel_gid_index, uint32_t sel_mtu, int log_level);
     static msgr_rdma_context_t *create(ibv_device *dev, ibv_port_attr & portinfo,
-        int ib_port, int gid_index, uint32_t mtu, bool odp, int log_level);
+        int ib_port, int gid_index, uint32_t mtu, int log_level);
     static msgr_rdma_context_t* create_cm(ibv_context *ctx);
     bool reserve_cqe(int n);
 
@@ -54,8 +52,10 @@ struct msgr_rdma_context_t
 
 struct msgr_rdma_buf_t
 {
-    void *buf = NULL;
+    uint8_t *buf = NULL;
     ibv_mr *mr = NULL;
+
+    ~msgr_rdma_buf_t();
 };
 
 struct msgr_rdma_connection_t
@@ -72,7 +72,8 @@ struct msgr_rdma_connection_t
     int cur_send = 0, cur_recv = 0;
     int send_pos = 0, send_buf_pos = 0;
     int next_recv_buf = 0;
-    std::vector<msgr_rdma_buf_t> recv_buffers;
+    std::vector<void*> recv_buffers;
+    msgr_rdma_buf_t recv_buf;
     std::vector<uint64_t> send_sizes;
     msgr_rdma_buf_t send_out;
     int send_out_pos = 0, send_done_pos = 0, send_out_size = 0;

src/client/msgr_rdmacm.cpp

@@ -178,7 +178,6 @@ msgr_rdma_context_t* msgr_rdma_context_t::create_cm(ibv_context *ctx)
         delete rdma_context;
         return NULL;
     }
-    rdma_context->odp = false;
     rdma_context->channel = ibv_create_comp_channel(rdma_context->context);
     if (!rdma_context->channel)
     {
@@ -531,7 +530,7 @@ void osd_messenger_t::rdmacm_established(rdma_cm_event *ev)
         fprintf(stderr, "Successfully connected with OSD %ju using RDMA-CM\n", peer_osd);
     }
     // Add initial receive request(s)
-    try_recv_rdma(cl);
+    init_recv_rdma(cl);
     if (peer_osd)
     {
         check_peer_config(cl);

src/client/msgr_receive.cpp

@@ -42,6 +42,7 @@ void osd_messenger_t::read_requests()
             }
             if (!sqe)
             {
+                cl->refs--;
                 cl->read_msg.msg_iovlen = 0;
                 read_ready_clients.erase(read_ready_clients.begin(), read_ready_clients.begin() + i);
                 return;
@@ -159,8 +160,10 @@ void osd_messenger_t::clear_immediate_ops(int peer_fd)
 
 void osd_messenger_t::handle_immediate_ops()
 {
-    for (auto op: set_immediate_ops)
+    while (set_immediate_ops.size())
     {
+        auto op = set_immediate_ops.front();
+        set_immediate_ops.pop_front();
         if (op->op_type == OSD_OP_IN)
         {
             exec_op(op);
@@ -171,7 +174,6 @@ void osd_messenger_t::handle_immediate_ops()
             std::function<void(osd_op_t*)>(op->callback)(op);
         }
     }
-    set_immediate_ops.clear();
 }
 
 bool osd_messenger_t::handle_read_buffer(osd_client_t *cl, void *curbuf, int remain)

src/client/msgr_send.cpp

@@ -326,8 +326,10 @@ void osd_messenger_t::handle_send(int result, bool prev, bool more, osd_client_t
             int expected = cl->send_list.size() < IOV_MAX ? cl->send_list.size() : IOV_MAX;
             if (done != expected)
             {
-                fprintf(stderr, "BUG (maybe kernel): Expected to send %d iovecs with MSG_WAITALL but sent %d\n", expected, done);
-                exit(1);
+                fprintf(stderr, "Client %d socket write error: expected to send "
+                    "%d iovecs with MSG_WAITALL but sent %d. Disconnecting client\n", cl->peer_fd, expected, done);
+                stop_client(cl->peer_fd);
+                return;
             }
             cl->zc_free_list.push_back(NULL); // end marker
         }
@@ -363,7 +365,7 @@ void osd_messenger_t::handle_send(int result, bool prev, bool more, osd_client_t
                 }
             });
             // Add the initial receive request
-            try_recv_rdma(cl);
+            init_recv_rdma(cl);
         }
 #endif
     }

src/client/msgr_stop.cpp

@@ -57,6 +57,7 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
     {
         return;
     }
+    clear_immediate_ops(peer_fd);
     if (log_level > 0)
     {
         if (cl->osd_num)
@@ -85,6 +86,16 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
             osd_peer_fds.erase(osd_it);
         }
     }
+#ifdef WITH_RDMA
+    if (cl->rdma_conn && cl->rdma_conn->cmid)
+    {
+        auto rdma_it = rdmacm_connections.find(cl->rdma_conn->cmid);
+        if (rdma_it != rdmacm_connections.end() && rdma_it->second == cl)
+        {
+            rdmacm_connections.erase(rdma_it);
+        }
+    }
+#endif
 #ifndef __MOCK__
     // Then remove FD from the eventloop so we don't accidentally read something
     tfd->set_fd_handler(peer_fd, false, NULL);
@@ -158,6 +169,13 @@ osd_client_t::~osd_client_t()
     }
     // Cancel outbound ops
     cancel_ops();
+    for (osd_op_t *op: zc_free_list)
+    {
+        if (op)
+        {
+            delete op;
+        }
+    }
 #ifndef __MOCK__
 #ifdef WITH_RDMA
     if (rdma_conn)

src/client/object_id.h

@@ -20,6 +20,15 @@ typedef uint64_t inode_t;
 // Pool ID is 16 bits long
 typedef uint32_t pool_id_t;
 
+typedef uint64_t osd_num_t;
+typedef uint32_t pg_num_t;
+
+struct pool_pg_num_t
+{
+    pool_id_t pool_id;
+    pg_num_t pg_num;
+};
+
 // 16 bytes per object/stripe id
 // stripe = (start of the parity stripe + peer role)
 // i.e. for example (256KB + one of 0,1,2)
@@ -61,6 +70,21 @@ inline bool operator < (const obj_ver_id & a, const obj_ver_id & b)
     return a.oid < b.oid || a.oid == b.oid && a.version < b.version;
 }
 
+inline bool operator < (const pool_pg_num_t & a, const pool_pg_num_t & b)
+{
+    return a.pool_id < b.pool_id || a.pool_id == b.pool_id && a.pg_num < b.pg_num;
+}
+
+inline bool operator == (const pool_pg_num_t & a, const pool_pg_num_t & b)
+{
+    return a.pool_id == b.pool_id && a.pg_num == b.pg_num;
+}
+
+inline bool operator != (const pool_pg_num_t & a, const pool_pg_num_t & b)
+{
+    return a.pool_id != b.pool_id || a.pg_num != b.pg_num;
+}
+
 namespace std
 {
     template<> struct hash<object_id>

src/client/osd_ops.h

@@ -4,7 +4,6 @@
 #pragma once
 
 #include "object_id.h"
-#include "osd_id.h"
 
 // Magic numbers
 #define SECONDARY_OSD_OP_MAGIC      0x2bd7b10325434553l
@@ -262,7 +261,10 @@ struct __attribute__((__packed__)) osd_reply_del_t
     uint32_t left_on_dead_count;
 };
 
-// sync to the primary OSD
+// sync to the primary OSD. semantics:
+// 1) any non-synced write may disappear on OSD restart. even if it's a CAS write.
+// 2) sync only guarantees to commit completed writes, not in-progress ones.
+// 3) sync is a no-op when immediate_commit is active.
 struct __attribute__((__packed__)) osd_op_sync_t
 {
     osd_op_header_t header;

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 2.4.3
+Version: 3.0.4
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}