WIP refactor dequeue_write/continue_write into a single "coroutine"

Test scenarios:
- 128KB aligned I/O (may work despite bug)
- 4KB random I/O (triggers assertion failure)
- 4KB mixed read/write (triggers crash)
- 4KB sequential write (triggers crash)

I accept Vitastor CLA agreement: https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/CLA-en.md

.gitea/workflows/test.yml

@@ -306,6 +306,78 @@ jobs:
           echo ""
         done
 
+  test_old_interrupted_rebalance:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: OLD=1 /root/vitastor/tests/test_interrupted_rebalance.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_interrupted_rebalance_imm:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: OLD=1 IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_interrupted_rebalance.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_interrupted_rebalance_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: OLD=1 SCHEME=ec /root/vitastor/tests/test_interrupted_rebalance.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_interrupted_rebalance_ec_imm:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: OLD=1 SCHEME=ec IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_interrupted_rebalance.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_create_halfhost:
     runs-on: ubuntu-latest
     needs: build
@@ -342,6 +414,24 @@ jobs:
           echo ""
         done
 
+  test_level_placement:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_level_placement.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_snapshot:
     runs-on: ubuntu-latest
     needs: build
@@ -378,6 +468,42 @@ jobs:
           echo ""
         done
 
+  test_old_snapshot:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_snapshot.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_snapshot_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 SCHEME=ec /root/vitastor/tests/test_snapshot.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_minsize_1:
     runs-on: ubuntu-latest
     needs: build
@@ -414,6 +540,24 @@ jobs:
           echo ""
         done
 
+  test_old_move_reappear:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_move_reappear.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_degraded:
     runs-on: ubuntu-latest
     needs: build
@@ -504,6 +648,42 @@ jobs:
           echo ""
         done
 
+  test_old_snapshot_chain:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_snapshot_chain.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_snapshot_chain_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 SCHEME=ec /root/vitastor/tests/test_snapshot_chain.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_snapshot_down:
     runs-on: ubuntu-latest
     needs: build
@@ -540,6 +720,78 @@ jobs:
           echo ""
         done
 
+  test_old_snapshot_down:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_snapshot_down.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_snapshot_down_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 SCHEME=ec /root/vitastor/tests/test_snapshot_down.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_kv_stress:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_kv_stress.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_kv_stress_imm:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_kv_stress.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_splitbrain:
     runs-on: ubuntu-latest
     needs: build
@@ -630,6 +882,78 @@ jobs:
           echo ""
         done
 
+  test_old_rebalance_verify:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: OLD=1 /root/vitastor/tests/test_rebalance_verify.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_rebalance_verify_imm:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: OLD=1 IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_rebalance_verify.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_rebalance_verify_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: OLD=1 SCHEME=ec /root/vitastor/tests/test_rebalance_verify.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_rebalance_verify_ec_imm:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: OLD=1 SCHEME=ec IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_rebalance_verify.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_dd:
     runs-on: ubuntu-latest
     needs: build
@@ -738,7 +1062,7 @@ jobs:
           echo ""
         done
 
-  test_write_no_same:
+  test_old_write:
     runs-on: ubuntu-latest
     needs: build
     container: ${{env.TEST_IMAGE}}:${{github.sha}}
@@ -746,7 +1070,61 @@ jobs:
     - name: Run test
       id: test
       timeout-minutes: 3
-      run: /root/vitastor/tests/test_write_no_same.sh
+      run: OLD=1 /root/vitastor/tests/test_write.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_write_xor:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 SCHEME=xor /root/vitastor/tests/test_write.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_write_old_iothreads:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: TEST_NAME=old_iothreads OLD=1 GLOBAL_CONFIG=',"client_iothread_count":4' /root/vitastor/tests/test_write.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_write_no_same:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_write_no_same.sh
     - name: Print logs
       if: always() && steps.test.outcome == 'failure'
       run: |
@@ -1026,6 +1404,42 @@ jobs:
           echo ""
         done
 
+  test_old_resize:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_resize.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_resize_auto:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_resize_auto.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_osd_tags:
     runs-on: ubuntu-latest
     needs: build
@@ -1116,6 +1530,78 @@ jobs:
           echo ""
         done
 
+  test_old_enospc:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_enospc.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_enospc_xor:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 SCHEME=xor /root/vitastor/tests/test_enospc.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_enospc_imm:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_enospc.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_enospc_imm_xor:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 IMMEDIATE_COMMIT=1 SCHEME=xor /root/vitastor/tests/test_enospc.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_scrub:
     runs-on: ubuntu-latest
     needs: build
@@ -1224,6 +1710,240 @@ jobs:
           echo ""
         done
 
+  test_old_scrub:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_scrub.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_scrub_zero_osd_2:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 ZERO_OSD=2 /root/vitastor/tests/test_scrub.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_scrub_xor:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 SCHEME=xor /root/vitastor/tests/test_scrub.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_scrub_pg_size_3:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 PG_SIZE=3 /root/vitastor/tests/test_scrub.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_scrub_pg_size_6_pg_minsize_4_osd_count_6_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 PG_SIZE=6 PG_MINSIZE=4 OSD_COUNT=6 SCHEME=ec /root/vitastor/tests/test_scrub.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_scrub_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 SCHEME=ec /root/vitastor/tests/test_scrub.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_old_partwr_csum:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: OLD=1 /root/vitastor/tests/test_partwr_csum.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_heal_old_csum_32k_dmj:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: TEST_NAME=old_csum_32k_dmj OLD=1 OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k --inmemory_metadata false --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_heal_old_csum_32k_dj:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: TEST_NAME=old_csum_32k_dj  OLD=1 OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_heal_old_csum_32k:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: TEST_NAME=old_csum_32k     OLD=1 OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_heal_old_csum_4k_dmj:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: TEST_NAME=old_csum_4k_dmj  OLD=1 OSD_ARGS="--data_csum_type crc32c --inmemory_metadata false --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_heal_old_csum_4k_dj:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: TEST_NAME=old_csum_4k_dj   OLD=1 OSD_ARGS="--data_csum_type crc32c --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_heal_old_csum_4k:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: TEST_NAME=old_csum_4k      OLD=1 OSD_ARGS="--data_csum_type crc32c" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_nfs:
     runs-on: ubuntu-latest
     needs: build
@@ -1242,3 +1962,21 @@ jobs:
           echo ""
         done
 
+  test_nfs_unaligned_append:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_nfs_unaligned_append.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+

.gitea/workflows/tests-to-yaml.pl

@@ -38,6 +38,10 @@ for my $line (<>)
             {
                 $test_name .= '_antietcd';
             }
+            elsif ($1 eq 'OLD')
+            {
+                $test_name =~ s/^test_/test_old_/s;
+            }
             else
             {
                 $test_name .= '_'.lc($1).'_'.$2;

CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "2.4.3")
+set(VITASTOR_VERSION "3.0.2")
 
 include(CTest)
 

README-ru.md

@@ -26,11 +26,15 @@ Vitastor поддерживает QEMU-драйвер, протоколы UBLK,
 
 ## Презентации и записи докладов
 
+- KuberConf'2025: [видео](https://vitastor.io/presentation/kuberconf.webm)
+- Highload'2025: [видео](https://vitastor.io/presentation/hl2025/hl2025.webm),
+  [на youtube](https://www.youtube.com/watch?v=0R8MLjFtz7g), презентация
+  ([на русском](https://vitastor.io/presentation/hl2025/), [на английском](https://vitastor.io/presentation/hl2025/en.html))
+- Highload'2022: презентация ([на русском](https://vitastor.io/presentation/highload/highload.html)),
+  [видео](https://vitastor.io/presentation/highload/talk.webm)
 - DevOpsConf'2021: презентация ([на русском](https://vitastor.io/presentation/devopsconf/devopsconf.html),
   [на английском](https://vitastor.io/presentation/devopsconf/devopsconf_en.html)),
   [видео](https://vitastor.io/presentation/devopsconf/talk.webm)
-- Highload'2022: презентация ([на русском](https://vitastor.io/presentation/highload/highload.html)),
-  [видео](https://vitastor.io/presentation/highload/talk.webm)
 
 ## Документация
 

README.md

@@ -26,11 +26,15 @@ Read more details in the documentation. You can start from here: [Quick Start](d
 
 ## Talks and presentations
 
+- KuberConf'2025: [video](https://vitastor.io/presentation/kuberconf.webm)
+- Highload'2025: [video](https://vitastor.io/presentation/hl2025/hl2025.webm),
+  [youtube](https://www.youtube.com/watch?v=0R8MLjFtz7g), presentation
+  ([in Russian](https://vitastor.io/presentation/hl2025/), [in English](https://vitastor.io/presentation/hl2025/en.html))
+- Highload'2022: presentation ([in Russian](https://vitastor.io/presentation/highload/highload.html)),
+  [video](https://vitastor.io/presentation/highload/talk.webm)
 - DevOpsConf'2021: presentation ([in Russian](https://vitastor.io/presentation/devopsconf/devopsconf.html),
   [in English](https://vitastor.io/presentation/devopsconf/devopsconf_en.html)),
   [video](https://vitastor.io/presentation/devopsconf/talk.webm)
-- Highload'2022: presentation ([in Russian](https://vitastor.io/presentation/highload/highload.html)),
-  [video](https://vitastor.io/presentation/highload/talk.webm)
 
 ## Documentation
 

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v2.4.3
+VITASTOR_VERSION ?= v3.0.2
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v2.4.3
+          image: vitalif/vitastor-csi:v3.0.2
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v2.4.3
+          image: vitalif/vitastor-csi:v3.0.2
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "2.4.3"
+    vitastorCSIDriverVersion = "3.0.2"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (2.4.3-1) unstable; urgency=medium
+vitastor (3.0.2-1) unstable; urgency=medium
 
   * Bugfixes
 

docker/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v2.4.3
+VITASTOR_VERSION ?= v3.0.2
 
 all: build push
 

docker/etc/vitastor/docker.conf

@@ -4,7 +4,7 @@
 #
 
 # Desired Vitastor version
-VITASTOR_VERSION=v2.4.3
+VITASTOR_VERSION=v3.0.2
 
 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here

docs/config/layout-osd.en.md

@@ -9,6 +9,7 @@
 These parameters apply to OSDs, are fixed at the moment of OSD drive
 initialization and can't be changed after it without losing data.
 
+- [meta_format](#meta_format)
 - [data_device](#data_device)
 - [meta_device](#meta_device)
 - [journal_device](#journal_device)
@@ -27,6 +28,21 @@ initialization and can't be changed after it without losing data.
 - [data_csum_type](#data_csum_type)
 - [csum_block_size](#csum_block_size)
 
+## meta_format
+
+- Type: integer
+- Default: 3
+
+OSD store implementation version and on-disk metadata format.
+
+Three versions are currently supported: 3, 2 and 1.
+- 3 the new log-structured store, it's overall faster, has lower Write
+  Amplification, which may be even close to 1 (i.e. almost no extra writes)
+  if your SSDs support atomic writes (see [atomic_write_size](osd.en.md#atomic_write_size)).
+- 2 is the old stable store from Vitastor 0.9-2.x.
+- 1 is the same old store but with a legacy metadata format from Vitastor
+  versions to up 0.8.x, without any support for checksums.
+
 ## data_device
 
 - Type: string

docs/config/layout-osd.ru.md

@@ -10,6 +10,7 @@
 дисковые параметры, задаются в момент инициализации дисков OSD и не могут быть
 изменены после этого без потери данных.
 
+- [meta_format](#meta_format)
 - [data_device](#data_device)
 - [meta_device](#meta_device)
 - [journal_device](#journal_device)
@@ -28,6 +29,23 @@
 - [data_csum_type](#data_csum_type)
 - [csum_block_size](#csum_block_size)
 
+## meta_format
+
+- Тип: целое число
+- Значение по умолчанию: 3
+
+Версия реализации дискового хранилища OSD и дискового формата метаданных.
+
+Поддерживаются три версии: 3, 2 и 1.
+- 3 - новое лог-структурированное хранилище, в целом более быстрое, со
+  сниженным фактором амплификации записи, который может составлять около 1
+  (то есть, практически без лишней служебной записи), если ваши SSD
+  поддерживают атомарную запись (см. [atomic_write_size](osd.ru.md#atomic_write_size)).
+- 2 - старое стабильное хранилище из версий Vitastor 0.9-2.x.
+- 1 - то же самое стабильное хранилище, но с ещё более старым форматом
+  метаданных из версий Vitastor до 0.8.x, без какой-либо поддержки
+  контрольных сумм.
+
 ## data_device
 
 - Тип: строка

docs/config/osd.en.md

@@ -65,6 +65,10 @@ with an OSD restart or, for some of them, even without restarting by updating co
 - [allow_net_split](#allow_net_split)
 - [enable_pg_locks](#enable_pg_locks)
 - [pg_lock_retry_interval_ms](#pg_lock_retry_interval_ms)
+- [atomic_write_size](#atomic_write_size)
+- [use_atomic_flag](#use_atomic_flag)
+- [pg_reshard_chunk_size](#pg_reshard_chunk_size)
+- [pg_reshard_chunk_pause_ms](#pg_reshard_chunk_pause_ms)
 
 ## bind_address
 
@@ -666,3 +670,66 @@ Use this parameter to enable or disable this function for all pools.
 - Default: 100
 
 Retry interval for failed PG lock attempts.
+
+## atomic_write_size
+
+- Type: integer
+- Default: 4096
+
+Maximum data device atomic write size allowed for OSD to use.
+
+Atomic writes allow to reduce the Write Amplification factor with the new store
+([meta_format](layout-osd.en.md#meta_format)=3) to almost 1 (i.e. almost no extra writes)
+with replicated pools and reach the best possible write performance.
+
+Default value is auto-detected during OSD initialization from
+`/sys/block/xx/queue/atomic_write_max_bytes` or assumed to be 4096 bytes
+because all known disks support 4 KB atomic writes. Auto-detection is only used for
+NVMe disks because SAS disks require the explicit WRITE ATOMIC command which requires
+RWF_ATOMIC (see below [#use_atomic_flag]) but that flag works incorrectly in current
+Linux versions.
+
+You can also check if your NVMe drives support atomic writes by running
+the command `nvme id-ctrl /dev/nvme0n1 | grep awupf`. If the reported value,
+plus 1, multiplied by the currently selected block size of the NVMe,
+is more than 4 KB, then the new store can utilize it for better performance.
+The only drives known to support it currently are [Micron and Kioxia](../intro/quickstart.en.md).
+
+Atomic writes allow to skip double data writes in replicated pools, thus
+reducing Write Amplification and improving write performance up to 2 times.
+
+## use_atomic_flag
+
+- Type: boolean
+
+This option controls whether Vitastor OSDs use RWF_ATOMIC write flag with atomic writes.
+This flag is supported since Linux 6.11 and adds some safety to atomic writes - the kernel
+guarantees to not fragment write requests with it and also to check them against the actual
+device atomic write capabilities.
+
+However, the option is disabled by default because the flag is currently UNUSABLE - Linux
+incorrectly requires writes with that flag to be of power-of-2 length and length-aligned.
+I.e., for example, 12 KB writes and not-8-KB aligned 8 KB writes are forbidden by the kernel,
+even though the NVMe specification allows them.
+
+For NVMe disks with `scheduler=none` writes aren't fragmented anyway so it's not a big deal.
+However, you can rebuild your kernel with [this patch](../../patches/linux-fix-atomic-write-checks.diff)
+and turn this option on. It will make your atomic writes a bit safer.
+
+## pg_reshard_chunk_size
+
+- Type: integer
+- Default: 100000
+
+Pool PG count change is a CPU-intensive operation because OSDs store the full object database
+in memory and have to move all entries between old and new PGs. Thus it's performed in chunks,
+with pauses between chunks to prevent blocking OSD's event loop and other clients' operations.
+This option sets the maximum number of object is a chunk. Moving 100k objects usually takes
+50-100ms. Chunk size equal to 0 means unlimited.
+
+## pg_reshard_chunk_pause_ms
+
+- Type: milliseconds
+- Default: 100
+
+This option sets the interval between handling two PG count change chunks.

docs/config/osd.ru.md

@@ -66,6 +66,10 @@
 - [allow_net_split](#allow_net_split)
 - [enable_pg_locks](#enable_pg_locks)
 - [pg_lock_retry_interval_ms](#pg_lock_retry_interval_ms)
+- [atomic_write_size](#atomic_write_size)
+- [use_atomic_flag](#use_atomic_flag)
+- [pg_reshard_chunk_size](#pg_reshard_chunk_size)
+- [pg_reshard_chunk_pause_ms](#pg_reshard_chunk_pause_ms)
 
 ## bind_address
 
@@ -699,3 +703,72 @@ pg_minsize OSD во время переключений, что может по
 - Значение по умолчанию: 100
 
 Интервал повтора неудачных попыток блокировки PG.
+
+## atomic_write_size
+
+- Тип: целое число
+- Значение по умолчанию: 4096
+
+Максимальный размер атомарной записи на диск данных, который OSD разрешено использовать.
+
+Поддержка атомарной записи позволяет снизить мультипликатор записи (Write Amplification)
+на диск с новым хранилищем ([meta_format](layout-osd.ru.md#meta_format)=3)
+практически до 1 (то есть, почти до нулевого объёма лишней записи) в реплицированных
+пулах и достигнуть наилучшей возможной производительности записи.
+
+Значение по умолчанию авто-определяется во время инициализации OSD из
+`/sys/block/xx/queue/atomic_write_max_bytes` либо принимается равным 4096,
+так как все известные диски поддерживают атомарную запись 4 КБ блоков.
+Автоопределение применяется только для NVMe-дисков, так как SAS диски требуют
+использования отдельной команды WRITE ATOMIC, а для неё нужен флаг RWF_ATOMIC
+(см. ниже [#use_atomic_flag]), а он в текущих версиях Linux работает некорректно.
+
+Вы также можете проверить, поддерживают ли ваши NVMe-диски атомарную запись,
+с помощью команды `nvme id-ctrl /dev/nvme0n1 | grep awupf`. Если значение awupf
+плюс 1, умноженное на текущий выбранный размер блока NVMe-диска, больше 4 КБ,
+то новое хранилище может использовать атомарные записи для достижения лучшей
+производительности. Единственные известные диски, которые поддерживают это сейчас -
+[Micron и Kioxia](../intro/quickstart.ru.md).
+
+Атомарная запись позволяет не использовать двойную запись данных (в журнал и на
+устройство данных) в реплицированных пулах и таким образом снижает амплификацию
+записи (объём служебной записи на диск) и улучшает производительность записи
+вплоть до 2-х кратного прироста.
+
+## use_atomic_flag
+
+- Тип: булево (да/нет)
+
+Данная опция контролирует использование Vitastor OSD флага RWF_ATOMIC при атомарной записи
+блоков. Этот флаг поддерживается, начиная с версии ядра Linux 6.11 и добавляет немного корректности
+атомарным записям - ядро гарантирует отсутствие фрагментации запросов записи с этим флагом и
+проверяет их на соответствие реальным возможностям устройства.
+
+Однако, данная опция по умолчанию отключена, так как флаг в текущих версиях Linux работает
+абсолютно НЕКОРРЕКТНО - при нём Linux требует, чтобы запросы записи имели длину, равную
+степени двойки и были выровнены на эту длину. То есть, например, 12 КБ запросы записи, а также
+8 КБ запросы записи по не-кратному 8 КБ смещению запрещаются ядром, хотя спецификация NVMe их
+разрешает.
+
+Для NVMe-дисков с `scheduler=none` запросы записи и так не фрагментируются, так что это не так
+уж и важно, однако вы можете пересобрать своё ядро с [этим патчем](../../patches/linux-fix-atomic-write-checks.diff)
+и включить данную опцию. Это сделает вашу атомарную запись капельку безопаснее.
+
+## pg_reshard_chunk_size
+
+- Тип: целое число
+- Значение по умолчанию: 100000
+
+Изменение числа PG в пуле заметно загружает процессор, так как OSD хранят полную базу данных
+объектов в памяти и им приходится перемещать все записи объектов между старыми и новыми PG.
+Поэтому изменение применяется порциями, с паузами между порциями, чтобы не блокировать обработку
+событий OSD и операции остальных клиентов. Данная опция задаёт максимальное число объектов
+в порции. Перемещение 100 тысяч объектов (значение по умолчанию) обычно занимает порядка
+50-100 миллисекунд. Значение опции 0 отключает лимит размера порции.
+
+## pg_reshard_chunk_pause_ms
+
+- Тип: миллисекунды
+- Значение по умолчанию: 100
+
+Данная опция задаёт интервал между обработкой двух порций изменения числа PG пулов.

docs/config/src/layout-osd.yml

@@ -1,3 +1,28 @@
+- name: meta_format
+  type: int
+  default: 3
+  info: |
+    OSD store implementation version and on-disk metadata format.
+
+    Three versions are currently supported: 3, 2 and 1.
+    - 3 the new log-structured store, it's overall faster, has lower Write
+      Amplification, which may be even close to 1 (i.e. almost no extra writes)
+      if your SSDs support atomic writes (see [atomic_write_size](osd.en.md#atomic_write_size)).
+    - 2 is the old stable store from Vitastor 0.9-2.x.
+    - 1 is the same old store but with a legacy metadata format from Vitastor
+      versions to up 0.8.x, without any support for checksums.
+  info_ru: |
+    Версия реализации дискового хранилища OSD и дискового формата метаданных.
+
+    Поддерживаются три версии: 3, 2 и 1.
+    - 3 - новое лог-структурированное хранилище, в целом более быстрое, со
+      сниженным фактором амплификации записи, который может составлять около 1
+      (то есть, практически без лишней служебной записи), если ваши SSD
+      поддерживают атомарную запись (см. [atomic_write_size](osd.ru.md#atomic_write_size)).
+    - 2 - старое стабильное хранилище из версий Vitastor 0.9-2.x.
+    - 1 - то же самое стабильное хранилище, но с ещё более старым форматом
+      метаданных из версий Vitastor до 0.8.x, без какой-либо поддержки
+      контрольных сумм.
 - name: data_device
   type: string
   info: |

docs/config/src/osd.yml

@@ -801,3 +801,108 @@
   default: 100
   info: Retry interval for failed PG lock attempts.
   info_ru: Интервал повтора неудачных попыток блокировки PG.
+- name: atomic_write_size
+  type: int
+  default: 4096
+  info: |
+    Maximum data device atomic write size allowed for OSD to use.
+
+    Atomic writes allow to reduce the Write Amplification factor with the new store
+    ([meta_format](layout-osd.en.md#meta_format)=3) to almost 1 (i.e. almost no extra writes)
+    with replicated pools and reach the best possible write performance.
+
+    Default value is auto-detected during OSD initialization from
+    `/sys/block/xx/queue/atomic_write_max_bytes` or assumed to be 4096 bytes
+    because all known disks support 4 KB atomic writes. Auto-detection is only used for
+    NVMe disks because SAS disks require the explicit WRITE ATOMIC command which requires
+    RWF_ATOMIC (see below [#use_atomic_flag]) but that flag works incorrectly in current
+    Linux versions.
+
+    You can also check if your NVMe drives support atomic writes by running
+    the command `nvme id-ctrl /dev/nvme0n1 | grep awupf`. If the reported value,
+    plus 1, multiplied by the currently selected block size of the NVMe,
+    is more than 4 KB, then the new store can utilize it for better performance.
+    The only drives known to support it currently are [Micron and Kioxia](../intro/quickstart.en.md).
+
+    Atomic writes allow to skip double data writes in replicated pools, thus
+    reducing Write Amplification and improving write performance up to 2 times.
+  info_ru: |
+    Максимальный размер атомарной записи на диск данных, который OSD разрешено использовать.
+
+    Поддержка атомарной записи позволяет снизить мультипликатор записи (Write Amplification)
+    на диск с новым хранилищем ([meta_format](layout-osd.ru.md#meta_format)=3)
+    практически до 1 (то есть, почти до нулевого объёма лишней записи) в реплицированных
+    пулах и достигнуть наилучшей возможной производительности записи.
+
+    Значение по умолчанию авто-определяется во время инициализации OSD из
+    `/sys/block/xx/queue/atomic_write_max_bytes` либо принимается равным 4096,
+    так как все известные диски поддерживают атомарную запись 4 КБ блоков.
+    Автоопределение применяется только для NVMe-дисков, так как SAS диски требуют
+    использования отдельной команды WRITE ATOMIC, а для неё нужен флаг RWF_ATOMIC
+    (см. ниже [#use_atomic_flag]), а он в текущих версиях Linux работает некорректно.
+
+    Вы также можете проверить, поддерживают ли ваши NVMe-диски атомарную запись,
+    с помощью команды `nvme id-ctrl /dev/nvme0n1 | grep awupf`. Если значение awupf
+    плюс 1, умноженное на текущий выбранный размер блока NVMe-диска, больше 4 КБ,
+    то новое хранилище может использовать атомарные записи для достижения лучшей
+    производительности. Единственные известные диски, которые поддерживают это сейчас -
+    [Micron и Kioxia](../intro/quickstart.ru.md).
+
+    Атомарная запись позволяет не использовать двойную запись данных (в журнал и на
+    устройство данных) в реплицированных пулах и таким образом снижает амплификацию
+    записи (объём служебной записи на диск) и улучшает производительность записи
+    вплоть до 2-х кратного прироста.
+- name: use_atomic_flag
+  type: bool
+  info: |
+    This option controls whether Vitastor OSDs use RWF_ATOMIC write flag with atomic writes.
+    This flag is supported since Linux 6.11 and adds some safety to atomic writes - the kernel
+    guarantees to not fragment write requests with it and also to check them against the actual
+    device atomic write capabilities.
+
+    However, the option is disabled by default because the flag is currently UNUSABLE - Linux
+    incorrectly requires writes with that flag to be of power-of-2 length and length-aligned.
+    I.e., for example, 12 KB writes and not-8-KB aligned 8 KB writes are forbidden by the kernel,
+    even though the NVMe specification allows them.
+
+    For NVMe disks with `scheduler=none` writes aren't fragmented anyway so it's not a big deal.
+    However, you can rebuild your kernel with [this patch](../../patches/linux-fix-atomic-write-checks.diff)
+    and turn this option on. It will make your atomic writes a bit safer.
+  info_ru: |
+    Данная опция контролирует использование Vitastor OSD флага RWF_ATOMIC при атомарной записи
+    блоков. Этот флаг поддерживается, начиная с версии ядра Linux 6.11 и добавляет немного корректности
+    атомарным записям - ядро гарантирует отсутствие фрагментации запросов записи с этим флагом и
+    проверяет их на соответствие реальным возможностям устройства.
+
+    Однако, данная опция по умолчанию отключена, так как флаг в текущих версиях Linux работает
+    абсолютно НЕКОРРЕКТНО - при нём Linux требует, чтобы запросы записи имели длину, равную
+    степени двойки и были выровнены на эту длину. То есть, например, 12 КБ запросы записи, а также
+    8 КБ запросы записи по не-кратному 8 КБ смещению запрещаются ядром, хотя спецификация NVMe их
+    разрешает.
+
+    Для NVMe-дисков с `scheduler=none` запросы записи и так не фрагментируются, так что это не так
+    уж и важно, однако вы можете пересобрать своё ядро с [этим патчем](../../patches/linux-fix-atomic-write-checks.diff)
+    и включить данную опцию. Это сделает вашу атомарную запись капельку безопаснее.
+- name: pg_reshard_chunk_size
+  type: int
+  default: 100000
+  info: |
+    Pool PG count change is a CPU-intensive operation because OSDs store the full object database
+    in memory and have to move all entries between old and new PGs. Thus it's performed in chunks,
+    with pauses between chunks to prevent blocking OSD's event loop and other clients' operations.
+    This option sets the maximum number of object is a chunk. Moving 100k objects usually takes
+    50-100ms. Chunk size equal to 0 means unlimited.
+  info_ru: |
+    Изменение числа PG в пуле заметно загружает процессор, так как OSD хранят полную базу данных
+    объектов в памяти и им приходится перемещать все записи объектов между старыми и новыми PG.
+    Поэтому изменение применяется порциями, с паузами между порциями, чтобы не блокировать обработку
+    событий OSD и операции остальных клиентов. Данная опция задаёт максимальное число объектов
+    в порции. Перемещение 100 тысяч объектов (значение по умолчанию) обычно занимает порядка
+    50-100 миллисекунд. Значение опции 0 отключает лимит размера порции.
+- name: pg_reshard_chunk_pause_ms
+  type: ms
+  default: 100
+  info: |
+    This option sets the interval between handling two PG count change chunks.
+  info_ru: |
+    Данная опция задаёт интервал между обработкой двух порций изменения числа PG пулов.

docs/installation/docker.en.md

@@ -26,9 +26,9 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.
 
 1. Download a Docker image of the desired version: \
-   `docker pull vitalif/vitastor:v2.4.3`
+   `docker pull vitalif/vitastor:v3.0.2`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.4.3 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.2 install.sh`
 3. Reload udev rules: \
    `udevadm control --reload-rules`
 

docs/installation/docker.ru.md

@@ -25,9 +25,9 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.
 
 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitalif/vitastor:v2.4.3`
+   `docker pull vitalif/vitastor:v3.0.2`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.4.3 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.2 install.sh`
 3. Перезагрузите правила udev: \
    `udevadm control --reload-rules`
 

docs/installation/proxmox.en.md

@@ -6,7 +6,7 @@
 
 # Proxmox VE
 
-To enable Vitastor support in Proxmox Virtual Environment (6.4-8.x are supported):
+To enable Vitastor support in Proxmox Virtual Environment (6.4-9.x are supported):
 
 - Add the corresponding Vitastor Debian repository into sources.list on Proxmox hosts:
   trixie for 9.0+, bookworm for 8.1+, pve8.0 for 8.0, bullseye for 7.4, pve7.3 for 7.3, pve7.2 for 7.2, pve7.1 for 7.1, buster for 6.4

docs/installation/proxmox.ru.md

@@ -6,7 +6,7 @@
 
 # Proxmox VE
 
-Чтобы подключить Vitastor к Proxmox Virtual Environment (поддерживаются версии 6.4-8.x):
+Чтобы подключить Vitastor к Proxmox Virtual Environment (поддерживаются версии 6.4-9.x):
 
 - Добавьте соответствующий Debian-репозиторий Vitastor в sources.list на хостах Proxmox:
   trixie для 9.0+, bookworm для 8.1+, pve8.0 для 8.0, bullseye для 7.4, pve7.3 для 7.3, pve7.2 для 7.2, pve7.1 для 7.1, buster для 6.4

docs/intro/features.en.md

@@ -14,6 +14,8 @@
 
 - Basic part: highly-available block storage with symmetric clustering and no SPOF
 - [Performance](../performance/bench2.en.md) ;-D
+  - [NVMe atomic write support](../config/osd.en.md#atomic_write_size) for reducing the amount
+    of "extra" disk writes to almost zero (Write Amplification = 1)
 - [Multiple redundancy schemes](../config/pool.en.md#scheme): Replication, XOR n+1, Reed-Solomon erasure codes
   based on jerasure and ISA-L libraries with any number of data and parity drives in a group
 - Configuration via simple JSON data structures in etcd (parameters, pools and images)

docs/intro/features.ru.md

@@ -14,6 +14,8 @@
 
 - Базовая часть - надёжное кластерное блочное хранилище без единой точки отказа
 - [Производительность](../performance/bench2.ru.md) ;-D
+  - [Поддержка атомарной записи NVMe](../config/osd.ru.md#atomic_write_size) для снижения объёма
+    служебной записи практически до нуля (Write Amplification = 1)
 - [Несколько схем отказоустойчивости](../config/pool.ru.md#scheme): репликация, XOR n+1 (1 диск чётности), коды коррекции ошибок
   Рида-Соломона на основе библиотек jerasure и ISA-L с любым числом дисков данных и чётности в группе
 - Конфигурация через простые человекочитаемые JSON-структуры в etcd

docs/intro/quickstart.en.md

@@ -18,9 +18,10 @@
 
 ## Preparation
 
-- Get some SATA or NVMe SSDs with capacitors (server-grade drives). You can use desktop SSDs
-  with lazy fsync, but prepare for inferior single-thread latency. Read more about capacitors
-  [here](../config/layout-cluster.en.md#immediate_commit).
+- Get some SATA or NVMe SSDs with capacitors (server-grade drives). The best performance
+  is achieved with Micron or Kioxia NVMes with atomic write support (see below). You can use desktop
+  SSDs with lazy fsync, but prepare for inferior single-thread latency. Read more about
+  capacitors [here](../config/layout-cluster.en.md#immediate_commit).
 - If you want to use HDDs, get modern HDDs with Media Cache or SSD Cache: HGST Ultrastar,
   Toshiba MG, Seagate EXOS or something similar. If your drives don't have such cache then
   you also need small SSDs for journal and metadata (even 2 GB per 1 TB of HDD space is enough).
@@ -30,9 +31,11 @@
 
 ## Recommended drives
 
-- SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel D3-S4510/4520/4610/4620, Kingston DC500M
-- NVMe: Micron 9100/9200/9300/9400, Micron 7300/7450, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
-  Intel DC-P3700/P4500/P4600, Intel D5-P4320/P5530, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
+- NVMe with atomic write support (ideal!): Micron 7450/7500/7550, Kioxia CD6/CD7/CD8/CD9
+- Other NVMe: Micron 9100/9200/9300/9400/9550, Micron 7300, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
+  Intel DC-P3700/P4500/P4600, Intel/Solidigm D5-P4320/P5530, Intel/Solidigm D7-P5500/P5600, Solidigm D7-PS1010/PS1030/P5810,
+  Intel Optane, Kingston DC1000B/DC1500M, Kioxia CD6/CD7/CD8/CD9
+- SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel/Solidigm D3-S4510/4520/4610/4620, Kingston DC500M
 - HDD: HGST Ultrastar, Toshiba MG, Seagate EXOS
 
 ## Configure monitors

docs/intro/quickstart.ru.md

@@ -18,8 +18,9 @@
 
 ## Подготовка
 
-- Возьмите серверы с SSD (SATA или NVMe), желательно с конденсаторами (серверные SSD). Можно
-  использовать и десктопные SSD, включив режим отложенного fsync, но производительность будет хуже.
+- Возьмите серверы с SSD (SATA или NVMe), желательно с конденсаторами (серверные SSD). Наилучшая
+  производительность достигается на дисках Micron и Kioxia с поддержкой атомарной записи (см. ниже).
+  Можно использовать и десктопные SSD, включив режим отложенного fsync, но производительность будет хуже.
   О конденсаторах читайте [здесь](../config/layout-cluster.ru.md#immediate_commit).
 - Если хотите использовать HDD, берите современные модели с Media или SSD кэшем - HGST Ultrastar,
   Toshiba MG, Seagate EXOS или что-то похожее. Если такого кэша у ваших дисков нет,
@@ -30,9 +31,11 @@
 
 ## Рекомендуемые диски
 
-- SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel D3-S4510/4520/4610/4620, Kingston DC500M
-- NVMe: Micron 9100/9200/9300/9400, Micron 7300/7450, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
-  Intel DC-P3700/P4500/P4600, Intel D5-P4320/P5530, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
+- NVMe с поддержкой атомарной записи (идеально!): Micron 7450/7500/7550, Kioxia CD6/CD7/CD8/CD9
+- Другие NVMe: Micron 9100/9200/9300/9400/9550, Micron 7300, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
+  Intel DC-P3700/P4500/P4600, Intel/Solidigm D5-P4320/P5530, Intel/Solidigm D7-P5500/P5600, Solidigm D7-PS1010/PS1030/P5810,
+  Intel Optane, Kingston DC1000B/DC1500M, Kioxia CD6/CD7/CD8/CD9
+- SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel/Solidigm D3-S4510/4520/4610/4620, Kingston DC500M
 - HDD: HGST Ultrastar, Toshiba MG, Seagate EXOS
 
 ## Настройте мониторы

docs/performance/theoretical.en.md

@@ -14,7 +14,7 @@ Replicated setups:
 - Linear read: `min(total network bandwidth, sum(disk read MB/s))`.
 - Linear write: `min(total network bandwidth, sum(disk write MB/s / number of replicas))`.
 - Saturated parallel read iops: `min(total network bandwidth, sum(disk read iops))`.
-- Saturated parallel write iops: `min(total network bandwidth / number of replicas, sum(disk write iops / number of replicas / (write amplification = 4)))`.
+- Saturated parallel write iops: `min(total network bandwidth / number of replicas, sum(disk write iops / number of replicas / write amplification))`.
 
 EC/XOR setups (EC N+K):
 - Single-threaded (T1Q1) read latency: 1.5 network roundtrips + 1 disk read.
@@ -26,28 +26,36 @@ EC/XOR setups (EC N+K):
 - Linear read: `min(total network bandwidth, sum(disk read MB/s))`.
 - Linear write: `min(total network bandwidth, sum(disk write MB/s * N/(N+K)))`.
 - Saturated parallel read iops: `min(total network bandwidth, sum(disk read iops))`.
-- Saturated parallel write iops: roughly `total iops / (N+K) / WA`. More exactly,
-  `min(total network bandwidth * N/(N+K), sum(disk randrw iops / (N*4 + K*5 + 1)))` with
-  random read/write mix corresponding to `(N-1)/(N*4 + K*5 + 1)*100 % reads`.
-  - For example, with EC 2+1 it is: `(7% randrw iops) / 14`.
-  - With EC 6+3 it is: `(12.5% randrw iops) / 40`.
+- Saturated parallel write iops: roughly `total iops / (N+K) / WA`. More exactly:
+  - With the new store: `min(total network bandwidth * N/(N+K), sum(disk randrw iops / (2 + N-1 + K*2)))`,
+    with random read/write mix corresponding to `(N-1)/(2 + N-1 + K*2)*100 % reads`.
+    - For example, with EC 2+1 it is: `(20% randrw iops) / 5`.
+    - With EC 6+3 it is: `(38% randrw iops) / 13`.
+  - With the old store: `min(total network bandwidth * N/(N+K), sum(disk randrw iops / (3 + N-1 + K*3)))`,
+    with random read/write mix corresponding to `(N-1)/(3 + N-1 + K*3)*100 % reads`.
+    - For example, with EC 2+1 it is: `(14% randrw iops) / 7`.
+    - With EC 6+3 it is: `(30% randrw iops) / 17`.
 
-Write amplification for 4 KB blocks is usually 3-5 in Vitastor:
-1. Journal block write
-2. Journal data write
-3. Metadata block write
-4. Another journal block write for EC/XOR setups
-5. Data block write
+Write Amplification factor:
+- For the new store and for 4 KB writes: WA is always 1 unless you set [atomic_write_size](../config/osd.en.md#atomic_write_size) to 0 manually.
+- For the new store and for 8-124 KB writes: WA is 1 if you use NVMe drives with atomic write support, or roughly 2 if you use other drives.
+- For the old store, WA is roughly `(2 * write size + 4 KB) / (write size)`. So, for 4 KB writes it's 3, and for 8-124 KB writes it's closer to 2.
+- For both the new and the old store and for writes of [block_size](../config/layout-cluster.en.md#block_size): WA is almost 1.
 
-If you manage to get an SSD which handles 512 byte blocks well (Optane?) you may
-lower 1, 3 and 4 to 512 bytes (1/8 of data size) and get WA as low as 2.375.
+Write Amplification consists of:
+- For the new store:
+  - Buffer block write if non-atomic
+  - Data block write
+  - Metadata write(s) (amortized)
+- For the old store:
+  - Journal block write (amortized)
+  - Journal data write
+  - Metadata block write
+  - Another journal block write for EC/XOR setups (amortized)
+  - Data block write
 
-Implemented NVDIMM support can basically eliminate WA at all - all extra writes will
-go to DRAM memory. But this requires a test cluster with NVDIMM - please contact me
-if you want to provide me with such cluster for tests.
-
-Lazy fsync also reduces WA for parallel workloads because journal blocks are only
-written when they fill up or fsync is requested.
+Other possibilities to reduce WA would be to use SSDs with internal 512-byte blocks
+or NVDIMM, but both options seem unavailable on the market at the moment.
 
 ## In Practice
 

docs/performance/theoretical.ru.md

@@ -27,29 +27,36 @@
 - Линейное чтение: сумма МБ/с чтения всех дисков, либо общая производительность сети, если в сеть упрётся раньше.
 - Линейная запись: сумма МБ/с записи всех дисков * N/(N+K), либо производительность сети * N / (N+K), если в сеть упрётся раньше.
 - Параллельное случайное мелкое чтение: сумма IOPS чтения всех дисков либо производительность сети, если в сеть упрётся раньше.
-- Параллельная случайная мелкая запись: грубо `(сумма IOPS / (N+K) / WA)`. Если точнее, то:
-  сумма смешанного IOPS всех дисков при `(N-1)/(N*4 + K*5 + 1)*100 %` чтения, делённая на `(N*4 + K*5 + 1)`.
-  Либо, производительность сети * N/(N+K), если в сеть упрётся раньше.
-  - Например, при EC 2+1 это: `(сумма IOPS при 7% чтения) / 14`.
-  - При EC 6+3 это: `(сумма IOPS при 12.5% чтения) / 40`.
+- Параллельная случайная мелкая запись: грубо `(сумма IOPS / (N+K) / WA)`.
+  Либо `производительность сети * N/(N+K)`, если в сеть упрётся раньше. Если точнее, то:
+  - С новым хранилищем: сумма смешанного IOPS всех дисков при `(N-1)/(2 + N-1 + K*2)*100 %` чтения, делённая на `(2 + N-1 + K*2)`.
+    - Например, при EC 2+1 это: `(сумма IOPS при 20% чтения) / 5`.
+    - При EC 6+3 это: `(сумма IOPS при 38% чтения) / 13`.
+  - Со старым хранилищем: сумма смешанного IOPS всех дисков при `(N-1)/(3 + N-1 + K*3)*100 %` чтения, делённая на `(3 + N-1 + K*3)`.
+    - Например, при EC 2+1 это: `(сумма IOPS при 14% чтения) / 7`.
+    - При EC 6+3 это: `(сумма IOPS при 30% чтения) / 17`.
 
-WA (мультипликатор записи) для 4 КБ блоков в Vitastor обычно составляет 3-5:
-1. Запись метаданных в журнал
-2. Запись блока данных в журнал
-3. Запись метаданных в БД
-4. Ещё одна запись метаданных в журнал при использовании EC
-5. Запись блока данных на диск данных
+WA (Write Amplification, мультипликатор записи):
+- С новым хранилищем для 4 КБ записи: WA всегда примерно 1, если только вы не установите [atomic_write_size](../config/osd.ru.md#atomic_write_size) вручную в 0.
+- С новым хранилищем и большими записями (8-124 КБ): WA примерно 1, если вы используете NVMe-диски с поддержкой атомарной записи,
+  или примерно 2, если вы используете другие диски.
+- Со старым хранилищем, WA примерно `(2 * размер записи + 4 КБ) / (размер записи)`. То есть, для 4 КБ записи WA=3, а для 8-124 КБ WA ближе к 2.
+- И с новым, и со старым хранилищем и для записи размером [block_size](../config/layout-cluster.ru.md#block_size): WA примерно равен 1.
 
-Если вы найдёте SSD, хорошо работающий с 512-байтными блоками данных (Optane?),
-то 1, 3 и 4 можно снизить до 512 байт (1/8 от размера данных) и получить WA всего 2.375.
+Мультипликатор записи состоит из:
+- С новым хранилищем:
+  - Запись блока буфера, если диски без поддержки атомарной записи
+  - Запись блока данных
+  - Запись(-и) блоков метаданных (амортизированные)
+- Со старым хранилищем:
+  - Запись блока журнала (амортизированная)
+  - Запись данных в журнал
+  - Запись блока метаданных
+  - Ещё одна запись блока журнала для EC/XOR пулов (амортизированная)
+  - Запись блока данных
 
-Если реализовать поддержку NVDIMM, то WA можно, условно говоря, ликвидировать вообще - все
-дополнительные операции записи смогут обслуживаться DRAM памятью. Но для этого необходим
-тестовый кластер с NVDIMM - пишите, если готовы предоставить такой для тестов.
-
-Кроме того, WA снижается при использовании отложенного/ленивого сброса при параллельной
-нагрузке, т.к. блоки журнала записываются на диск только когда они заполняются или явным
-образом запрашивается fsync.
+Другими потенциальными возможностями снижения WA могли бы быть SSD с внутренним 512-байтным блоком
+либо NVDIMM, но и то, и другое сейчас выглядит недоступным на рынке.
 
 ## На практике
 

docs/usage/admin.en.md

@@ -231,6 +231,18 @@ Upgrading from <= 0.5.x to >= 0.6.x is not supported.
 
 Downgrade are also allowed freely, except the following specific instructions:
 
+### 3.x -> 2.x
+
+Versions 3.0.0 and newer contain two store implementations - an old one and a new
+one, unsupported in 2.x and previous versions. So you should check your OSD store
+versions before downgrading to 2.x with the following command:
+
+`vitastor-disk read-sb /dev/vitastor/osdXX-data | jq -r .meta_format`
+
+If it prints 3 then OSD uses the new store and you can't downgrade it to 2.x.
+
+If it prints 2 or nothing then OSD uses the old store and the downgrade is allowed.
+
 ### 1.8.0 to 1.7.1
 
 Before downgrading from version >= 1.8.0 to version <= 1.7.1

docs/usage/admin.ru.md

@@ -228,6 +228,18 @@ done
 
 Откат (понижение версии) тоже свободно разрешён, кроме указанных ниже случаев:
 
+### 3.x -> 2.x
+
+Версии 3.0.0 и более новые содержат две реализации хранилища - старую и новую, не
+поддерживаемую в 2.x и предыдущих версиях. Таким образом, перед откатом на 2.x вам
+следует проверить, какая версия хранилища используется вашими OSD - командой:
+
+`vitastor-disk read-sb /dev/vitastor/osdXX-data | jq -r .meta_format`
+
+Если выводится 3, это новое хранилище и откатить такой OSD до 2.x нельзя.
+
+Если выводится 2 или не выводится ничего, это старое хранилище и откат разрешён.
+
 ### 1.8.0 -> 1.7.1
 
 Перед понижением версии с >= 1.8.0 до <= 1.7.1 вы должны скопировать ключ

docs/usage/disk.en.md

@@ -51,6 +51,9 @@ Options (automatic mode):
 ```
 --osd_per_disk <N>
   Create <N> OSDs on each disk (default 1)
+--meta_format 3
+  Metadata store version. 3 is the new log-structured store, 2 is the stable store
+  from Vitastor 0.9-2.x, 1 is the legacy store from Vitastor 0.6-0.8.
 --hybrid
   Prepare hybrid (HDD+SSD, NVMe+SATA or etc) OSDs using provided devices. By default,
   any passed SSDs will be used for journals and metadata, HDDs will be used for data,
@@ -92,6 +95,8 @@ Options (single-device mode):
 Options (both modes):
 
 ```
+--tags tag1,tag2           Set new OSD tag(s)
+--weight <number>          Set new OSD weight (between 0 to 1)
 --journal_size 1G/32M      Set journal size (area or partition size)
 --block_size 1M/128k       Set blockstore object size
 --bitmap_granularity 4k    Set bitmap granularity

docs/usage/disk.ru.md

@@ -50,6 +50,9 @@ vitastor-disk - инструмент командной строки для уп
 ```
 --osd_per_disk <N>
   Создавать по несколько (<N>) OSD на каждом диске (по умолчанию 1)
+--meta_format 3
+  Версия хранилища метаданных. 3 - новое лог-структурированное хранилище,
+  2 - стабильное хранилище из Vitastor 0.9-2.x, 1 - старое хранилище из Vitastor 0.6-0.8.
 --hybrid
   Инициализировать гибридные (HDD+SSD, NVMe+SATA и т.п.) OSD на указанных дисках.
   По умолчанию, SSD будут использованы для журналов и метаданных, а HDD - для данных,
@@ -93,6 +96,8 @@ vitastor-disk - инструмент командной строки для уп
 Опции для обоих режимов:
 
 ```
+--tags tag1,tag2           Задать теги для новых OSD
+--weight <number>          Задать вес для новых OSD (от 0 до 1)
 --journal_size 1G/32M      Задать размер журнала (области или раздела журнала)
 --block_size 1M/128k       Задать размер объекта хранилища
 --bitmap_granularity 4k    Задать гранулярность битовых карт

mon/osd_tree.js

@@ -87,7 +87,7 @@ function make_hier_tree(global_config, tree)
     tree[''] = { children: [] };
     for (const node_id in tree)
     {
-        if (node_id === '' || !(tree[node_id].children||[]).length && (tree[node_id].size||0) <= 0)
+        if (node_id === '')
         {
             continue;
         }

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "2.4.3",
+  "version": "3.0.2",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {
@@ -9,7 +9,7 @@
   "author": "Vitaliy Filippov",
   "license": "UNLICENSED",
   "dependencies": {
-    "antietcd": "^1.1.3",
+    "antietcd": "^1.2.2",
     "sprintf-js": "^1.1.2",
     "ws": "^7.2.5"
   },

mon/pg_gen.js

@@ -9,7 +9,6 @@ const LPOptimizer = require('./lp_optimizer/lp_optimizer.js');
 const { scale_pg_count } = require('./pg_utils.js');
 const { make_hier_tree, filter_osds_by_root_node,
     filter_osds_by_tags, filter_osds_by_block_layout, get_affinity_osds } = require('./osd_tree.js');
-const { select_murmur3 } = require('./lp_optimizer/murmur3.js');
 
 function pick_primary(pool_id, pg_num, pool_config, osd_set, up_osds, aff_osds)
 {
@@ -39,7 +38,7 @@ function pick_primary(pool_id, pg_num, pool_config, osd_set, up_osds, aff_osds)
     {
         return 0;
     }
-    return alive_set[select_murmur3(alive_set.length, osd_num => pool_id+'/'+pg_num+'/'+osd_num)];
+    return alive_set[pg_num % alive_set.length];
 }
 
 function recheck_primary(state, global_config, up_osds, osd_tree)
@@ -53,6 +52,7 @@ function recheck_primary(state, global_config, up_osds, osd_tree)
             continue;
         }
         const aff_osds = get_affinity_osds(pool_cfg, up_osds, osd_tree);
+        let paused = false;
         for (let pg_num = 1; pg_num <= pool_cfg.pg_count; pg_num++)
         {
             if (!state.pg.config.items[pool_id])
@@ -75,6 +75,19 @@ function recheck_primary(state, global_config, up_osds, osd_tree)
                     );
                     new_pg_config.items[pool_id][pg_num].primary = new_primary;
                 }
+                paused = paused || !!pg_cfg.pause;
+            }
+        }
+        if (paused)
+        {
+            if (!new_pg_config)
+            {
+                new_pg_config = JSON.parse(JSON.stringify(state.pg.config));
+            }
+            console.log(`Resuming paused pool ${pool_id}`);
+            for (const pg in new_pg_config.items[pool_id])
+            {
+                delete new_pg_config.items[pool_id][pg].pause;
             }
         }
     }
@@ -179,7 +192,7 @@ async function generate_pool_pgs(state, global_config, pool_id, osd_tree, levels
     const rules = use_rules ? get_pg_rules(pool_id, pool_cfg, global_config.placement_levels) : null;
     const folded = fold_failure_domains(Object.values(pool_tree), use_rules ? rules : [ [ [ pool_cfg.failure_domain ] ] ]);
     // FIXME: Remove/merge make_hier_tree() step somewhere, however it's needed to remove empty nodes
-    const folded_tree = make_hier_tree(global_config, folded.nodes);
+    const folded_tree = make_hier_tree(global_config, folded.nodes.reduce((a, c) => { a[c.id] = c; return a; }, {}));
     const old_pg_count = prev_pgs.length;
     const optimize_cfg = {
         osd_weights: folded.nodes.reduce((a, c) => { if (Number(c.id)) { a[c.id] = c.size; } return a; }, {}),

node-binding/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor",
-  "version": "2.4.3",
+  "version": "3.0.2",
   "description": "Low-level native bindings to Vitastor client library",
   "main": "index.js",
   "keywords": [

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '2.4.3'
+VITASTOR_VERSION = '3.0.2'
 
 LOG = logging.getLogger(__name__)
 

patches/linux-fix-atomic-write-checks.diff

@@ -0,0 +1,39 @@
+From 98d3f68a40130c438854f61db6025f9e9b099cb6 Mon Sep 17 00:00:00 2001
+From: Vitaliy Filippov <vitalifster@gmail.com>
+Date: Sat, 20 Dec 2025 14:44:35 +0300
+Subject: [PATCH] Do not require atomic writes to be power of 2 sized and
+ aligned on length boundary
+
+It contradicts NVMe specification where alignment is only required when atomic
+write boundary (NABSPF/NABO) is set and highly limits usage of NVMe atomic writes
+
+Signed-off-by: Vitaliy Filippov <vitalifster@gmail.com>
+---
+ fs/read_write.c | 8 --------
+ 1 file changed, 8 deletions(-)
+
+diff --git a/fs/read_write.c b/fs/read_write.c
+index 833bae068770..5467d710108d 100644
+--- a/fs/read_write.c
++++ b/fs/read_write.c
+@@ -1802,17 +1802,9 @@ int generic_file_rw_checks(struct file *file_in, struct file *file_out)
+ 
+ int generic_atomic_write_valid(struct kiocb *iocb, struct iov_iter *iter)
+ {
+-	size_t len = iov_iter_count(iter);
+-
+ 	if (!iter_is_ubuf(iter))
+ 		return -EINVAL;
+ 
+-	if (!is_power_of_2(len))
+-		return -EINVAL;
+-
+-	if (!IS_ALIGNED(iocb->ki_pos, len))
+-		return -EINVAL;
+-
+ 	if (!(iocb->ki_flags & IOCB_DIRECT))
+ 		return -EOPNOTSUPP;
+ 
+-- 
+2.51.0
+

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.4.3
+Version:        3.0.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.4.3.el7.tar.gz
+Source0:        vitastor-3.0.2.el7.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  devtoolset-9-gcc-c++
@@ -171,7 +171,6 @@ chown vitastor:vitastor /var/lib/vitastor
 %_bindir/vitastor-kv
 %_bindir/vitastor-kv-stress
 %_bindir/vita
-%_libdir/libvitastor_blk.so*
 %_libdir/libvitastor_client.so*
 %_libdir/libvitastor_kv.so*
 

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.4.3
+Version:        3.0.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.4.3.el8.tar.gz
+Source0:        vitastor-3.0.2.el8.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-toolset-9-gcc-c++
@@ -168,7 +168,6 @@ chown vitastor:vitastor /var/lib/vitastor
 %_bindir/vitastor-kv
 %_bindir/vitastor-kv-stress
 %_bindir/vita
-%_libdir/libvitastor_blk.so*
 %_libdir/libvitastor_client.so*
 %_libdir/libvitastor_kv.so*
 

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.4.3
+Version:        3.0.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.4.3.el9.tar.gz
+Source0:        vitastor-3.0.2.el9.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++
@@ -165,7 +165,6 @@ chown vitastor:vitastor /var/lib/vitastor
 %_bindir/vitastor-kv
 %_bindir/vitastor-kv-stress
 %_bindir/vita
-%_libdir/libvitastor_blk.so*
 %_libdir/libvitastor_client.so*
 %_libdir/libvitastor_kv.so*
 

src/CMakeLists.txt

@@ -21,7 +21,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 endif()
 set(ENABLE_COVERAGE false CACHE BOOL "Enable code coverage")
 
-add_definitions(-DVITASTOR_VERSION="2.4.3")
+add_definitions(-DVITASTOR_VERSION="3.0.2")
 add_definitions(-D_GNU_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -fvisibility=hidden -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/blockstore/blockstore.h

@@ -183,6 +183,12 @@ public:
     // Update configuration
     virtual void parse_config(blockstore_config_t & config) = 0;
 
+    // Reshard database for a pool in chunks
+    // MUST be called only when nobody makes any modifications to the DB for this pool
+    virtual void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit) = 0;
+    virtual bool reshard_continue(void *reshard_state, uint64_t chunk_limit) = 0;
+    virtual void reshard_abort(void *reshard_state) = 0;
+
     // Event loop
     virtual void loop() = 0;
 

src/blockstore/blockstore_disk.cpp

@@ -48,11 +48,12 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     disk_alignment = parse_size(config["disk_alignment"]);
     journal_block_size = parse_size(config["journal_block_size"]);
     meta_block_size = parse_size(config["meta_block_size"]);
-    meta_block_target_free_space = parse_size(config["meta_block_target_free_space"]);
     bitmap_granularity = parse_size(config["bitmap_granularity"]);
     meta_format = stoull_full(config["meta_format"]);
     atomic_write_size = (config.find("atomic_write_size") != config.end()
         ? parse_size(config["atomic_write_size"]) : 4096);
+    use_atomic_flag = config.find("use_atomic_flag") != config.end() &&
+        (config["use_atomic_flag"] == "true" || config["use_atomic_flag"] == "1" || config["use_atomic_flag"] == "yes");
     if (config.find("data_io") == config.end() &&
         config.find("meta_io") == config.end() &&
         config.find("journal_io") == config.end())
@@ -154,14 +155,6 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     {
         throw std::runtime_error("meta_block_size must not exceed "+std::to_string(MAX_DATA_BLOCK_SIZE));
     }
-    if (!meta_block_target_free_space)
-    {
-        meta_block_target_free_space = 800;
-    }
-    if (meta_block_target_free_space >= meta_block_size)
-    {
-        throw std::runtime_error("meta_block_target_free_space must not exceed "+std::to_string(meta_block_size));
-    }
     if (data_offset % disk_alignment)
     {
         throw std::runtime_error("data_offset must be a multiple of disk_alignment = "+std::to_string(disk_alignment));
@@ -178,6 +171,12 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     {
         throw std::runtime_error("Data block size must be a multiple of sparse write tracking granularity");
     }
+    if (data_block_size / bitmap_granularity < 8)
+    {
+        fprintf(stderr, "Warning: block_size (%u) / bitmap_granularity (%u) = %u bits. "
+            "Consider using larger block_size or bitmap_granularity for better performance.\n",
+            data_block_size, bitmap_granularity, data_block_size / bitmap_granularity);
+    }
     if (!data_csum_type)
     {
         csum_block_size = 0;
@@ -210,10 +209,6 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     {
         throw std::runtime_error("journal_offset must be a multiple of journal_block_size = "+std::to_string(journal_block_size));
     }
-    if (!meta_format)
-    {
-        meta_format = BLOCKSTORE_META_FORMAT_HEAP;
-    }
     if (meta_device == data_device)
     {
         disable_meta_fsync = disable_data_fsync;
@@ -224,7 +219,7 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     }
 }
 
-void blockstore_disk_t::calc_lengths()
+void blockstore_disk_t::calc_lengths(bool skip_meta_check)
 {
     // data
     data_len = data_device_size - data_offset;
@@ -270,13 +265,13 @@ void blockstore_disk_t::calc_lengths()
     }
     // required metadata size
     block_count = data_len / data_block_size;
-    clean_entry_bitmap_size = data_block_size / bitmap_granularity / 8;
+    clean_entry_bitmap_size = (data_block_size / bitmap_granularity + 7) / 8;
     clean_dyn_size = clean_entry_bitmap_size*2 + (csum_block_size
         ? data_block_size/csum_block_size*(data_csum_type & 0xFF) : 0);
+recalc:
     if (meta_format == BLOCKSTORE_META_FORMAT_HEAP)
     {
-        uint32_t entries_per_block = ((meta_block_size-meta_block_target_free_space) /
-            (sizeof(heap_object_t) + sizeof(heap_write_t) + clean_dyn_size));
+        uint32_t entries_per_block = meta_block_size / (sizeof(heap_big_write_t) + clean_dyn_size);
         min_meta_len = (block_count+entries_per_block-1) / entries_per_block * meta_block_size;
     }
     else if (meta_format == BLOCKSTORE_META_FORMAT_V1)
@@ -284,11 +279,32 @@ void blockstore_disk_t::calc_lengths()
         clean_entry_size = 24 /*sizeof(clean_disk_entry)*/ + 2*clean_entry_bitmap_size;
         min_meta_len = (1 + (block_count - 1 + meta_block_size / clean_entry_size)
             / (meta_block_size / clean_entry_size)) * meta_block_size;
+        if (!skip_meta_check && meta_area_size < min_meta_len)
+        {
+too_small:
+            throw std::runtime_error("Metadata area is too small, need at least "+std::to_string(min_meta_len)+
+                " bytes, have only "+std::to_string(meta_area_size)+" bytes");
+        }
     }
-    else if (meta_format == BLOCKSTORE_META_FORMAT_V2)
+    else if (meta_format == BLOCKSTORE_META_FORMAT_V2 || !meta_format)
     {
+        meta_format = BLOCKSTORE_META_FORMAT_V2;
         clean_entry_size = 24 /*sizeof(clean_disk_entry)*/ + clean_dyn_size + 4 /*entry_csum*/;
         min_meta_len = (1 + (block_count - 1 + meta_block_size / clean_entry_size) / (meta_block_size / clean_entry_size)) * meta_block_size;
+        if (!skip_meta_check && meta_area_size < min_meta_len)
+        {
+            if (!data_csum_type)
+            {
+                printf("Warning: Using old metadata format without checksums because the new format"
+                    " doesn't fit into provided area (%ju bytes required, %ju bytes available)\n", min_meta_len, meta_area_size);
+                meta_format = BLOCKSTORE_META_FORMAT_V1;
+                goto recalc;
+            }
+            else
+            {
+                goto too_small;
+            }
+        }
     }
     else
     {

src/blockstore/blockstore_disk.h

@@ -36,8 +36,8 @@ struct blockstore_disk_t
     uint32_t meta_block_size = 4096;
     // Atomic write size of the data block device
     uint32_t atomic_write_size = 4096;
-    // Target free space in metadata blocks
-    uint32_t meta_block_target_free_space = 800;
+    // Whether we should set RWF_ATOMIC on atomic writes
+    bool use_atomic_flag = false;
     // Sparse write tracking granularity. 4 KB is a good choice. Must be a multiple of disk_alignment
     uint32_t bitmap_granularity = 4096;
     // Data checksum type, BLOCKSTORE_CSUM_NONE or BLOCKSTORE_CSUM_CRC32C
@@ -61,9 +61,10 @@ struct blockstore_disk_t
     uint64_t discard_granularity = 0;
 
     int meta_fd = -1, data_fd = -1, journal_fd = -1;
-    uint64_t meta_offset, meta_device_sect, meta_device_size, meta_area_size, min_meta_len, meta_format = 0;
-    uint64_t data_offset, data_device_sect, data_device_size, data_len;
-    uint64_t journal_offset, journal_device_sect, journal_device_size, journal_len;
+    uint64_t meta_offset = 0, meta_device_sect = 0, meta_device_size = 0, meta_area_size = 0, min_meta_len = 0;
+    uint64_t data_offset = 0, data_device_sect = 0, data_device_size = 0, data_len = 0;
+    uint64_t journal_offset = 0, journal_device_sect = 0, journal_device_size = 0, journal_len = 0;
+    uint64_t meta_format = 0;
 
     uint64_t block_count = 0;
     uint32_t clean_entry_bitmap_size = 0;
@@ -75,7 +76,7 @@ struct blockstore_disk_t
     void open_data();
     void open_meta();
     void open_journal();
-    void calc_lengths();
+    void calc_lengths(bool skip_meta_check = false);
     void check_lengths();
     void close_all();
     int trim_data(std::function<bool(uint64_t)> is_free);

src/blockstore/blockstore_flush.cpp

@@ -18,12 +18,18 @@ journal_flusher_t::journal_flusher_t(blockstore_impl_t *bs)
     this->cur_flusher_count = bs->min_flusher_count;
     this->target_flusher_count = bs->min_flusher_count;
     active_flushers = 0;
-    advance_lsn_counter = 0;
     co = new journal_flusher_co[max_flusher_count];
     for (int i = 0; i < max_flusher_count; i++)
     {
         co[i].co_id = i;
         co[i].bs = bs;
+        co[i].new_bmp = (uint8_t*)malloc_or_die(3*bs->dsk.clean_entry_bitmap_size);
+        co[i].new_ext_bmp = co[i].new_bmp + bs->dsk.clean_entry_bitmap_size;
+        co[i].punch_bmp = co[i].new_bmp + 2*bs->dsk.clean_entry_bitmap_size;
+        if (bs->dsk.csum_block_size > 0)
+        {
+            co[i].new_csums = (uint8_t*)malloc_or_die(bs->dsk.data_block_size / bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF));
+        }
         co[i].flusher = this;
     }
 }
@@ -54,6 +60,18 @@ journal_flusher_t::~journal_flusher_t()
 
 journal_flusher_co::~journal_flusher_co()
 {
+    if (new_csums)
+    {
+        free(new_csums);
+        new_csums = NULL;
+    }
+    if (new_bmp)
+    {
+        free(new_bmp);
+        new_bmp = NULL;
+    }
+    new_ext_bmp = NULL;
+    punch_bmp = NULL;
     free_buffers();
 }
 
@@ -62,11 +80,6 @@ int journal_flusher_t::get_syncing_buffer()
     return syncing_buffer;
 }
 
-uint64_t journal_flusher_t::get_compact_counter()
-{
-    return compact_counter;
-}
-
 bool journal_flusher_t::is_active()
 {
     return active_flushers > 0;
@@ -86,9 +99,10 @@ void journal_flusher_t::release_trim()
 void journal_flusher_t::dump_diagnostics()
 {
     printf(
-        "Compaction queue: %u items, data: %ju/%ju blocks used, meta: %ju/%ju bytes used, %u/%ju blocks nearfull\n",
-        bs->heap->get_to_compact_count(),
+        "Compaction queue: %u items + %u future, data: %ju/%ju blocks used, buffer: %ju/%ju bytes used, meta: %ju/%ju bytes used, %u/%ju blocks nearfull\n",
+        bs->heap->get_compact_queue_size(), bs->heap->get_to_compact_count(),
         bs->heap->get_data_used_space()/bs->dsk.data_block_size, bs->dsk.block_count,
+        bs->heap->get_buffer_area_used_space(), bs->dsk.journal_len,
         bs->heap->get_meta_used_space(), bs->heap->get_meta_total_space(),
         bs->heap->get_meta_nearfull_blocks(), bs->dsk.meta_area_size/bs->dsk.meta_block_size-1
     );
@@ -113,8 +127,12 @@ void journal_flusher_t::loop()
         }
     }
     int prev_active = active_flushers;
-    for (int i = 0; (active_flushers > 0 || force_start > 0 || bs->heap->get_to_compact_count() > bs->flusher_start_threshold) && i < cur_flusher_count; i++)
+    for (int i = 0; (active_flushers > 0 || force_start > 0 ||
+        bs->heap->get_compact_queue_size() > bs->flusher_start_threshold ||
+        i == 0 && bs->intent_write_counter >= bs->journal_trim_interval) && i < cur_flusher_count; i++)
+    {
         co[i].loop();
+    }
     if (prev_active && !active_flushers && force_start > 0)
         bs->ringloop->wakeup();
 }
@@ -156,22 +174,33 @@ bool journal_flusher_co::loop()
     else if (wait_state == 19) goto resume_19;
     else if (wait_state == 20) goto resume_20;
     else if (wait_state == 21) goto resume_21;
-    else if (wait_state == 22) goto resume_22;
-    else if (wait_state == 23) goto resume_23;
-    else if (wait_state == 24) goto resume_24;
 resume_0:
     wait_state = 0;
     wait_count = 0;
     cur_oid = {};
     res = bs->heap->get_next_compact(cur_oid);
+    // Advance fsynced_lsn every <journal_trim_interval> intent writes
+    if ((bs->intent_write_counter >= bs->journal_trim_interval) && co_id == 0)
+    {
+        bs->intent_write_counter = 0;
+resume_17:
+resume_18:
+resume_19:
+        if (!fsync_buffer(17))
+            return false;
+resume_20:
+resume_21:
+        if (!trim_lsn(20))
+            return false;
+    }
     if (res == ENOENT && flusher->force_start > 0 && co_id == 0 &&
-        (!bs->dsk.disable_journal_fsync || !bs->dsk.disable_meta_fsync))
+        (!bs->dsk.disable_journal_fsync || !bs->dsk.disable_meta_fsync || !bs->dsk.disable_data_fsync))
     {
         flusher->active_flushers++;
-resume_21:
-resume_22:
-        res = fsync_buffer(21);
-        if (!res)
+resume_14:
+resume_15:
+resume_16:
+        if (!fsync_buffer(14))
         {
             return false;
         }
@@ -180,61 +209,94 @@ resume_22:
     }
     if (res == ENOENT)
     {
-        if (co_id == 0 && flusher->force_start > 0)
-        {
-            flusher->active_flushers++;
-resume_16:
-resume_17:
-resume_18:
-resume_19:
-resume_20:
-            if (!trim_lsn(16))
-                return false;
-            flusher->active_flushers--;
-        }
         cur_oid = {};
         wait_state = 0;
         return true;
     }
-    for (int i = 0; i < flusher->cur_flusher_count; i++)
+    if (flusher->flushing.find(cur_oid) != flusher->flushing.end())
     {
-        if (i != co_id && flusher->co[i].cur_oid == cur_oid)
+        for (int i = 0; i < flusher->cur_flusher_count; i++)
         {
-            // Already flushing it
-            flusher->co[i].should_repeat = true;
-            goto resume_0;
+            if (i != co_id && flusher->co[i].cur_oid == cur_oid)
+            {
+                // Already flushing it
+                flusher->co[i].should_repeat = true;
+                goto resume_0;
+            }
         }
+        assert(false);
     }
+    flusher->flushing.insert(cur_oid);
 resume_1:
+    wait_state = 1;
     should_repeat = false;
-    cur_obj = bs->heap->lock_and_read_entry(cur_oid, copy_id);
+    cur_obj = bs->heap->lock_and_read_entry(cur_oid);
     if (!cur_obj)
     {
         // Object does not exist
+        flusher->flushing.erase(cur_oid);
         goto resume_0;
     }
-    cur_version = cur_obj->get_writes()->version;
-    // Find the range to compact
-    compact_lsn = bs->heap->get_fsynced_lsn();
-    bs->heap->get_compact_range(cur_obj, compact_lsn, &begin_wr, &end_wr);
-    if (!begin_wr)
-    {
-        // Nothing to flush
-        bs->heap->unlock_entry(cur_oid, copy_id);
-        goto resume_0;
-    }
-    assert(!end_wr->next() && end_wr->entry_type == (BS_HEAP_BIG_WRITE|BS_HEAP_STABLE));
-    clean_loc = end_wr->big_location(bs->heap);
-    if (bs->log_level > 10)
-        printf("Compacting %jx:%jx l%ju .. l%ju (last l%ju)\n", cur_oid.inode, cur_oid.stripe, end_wr->lsn, begin_wr->lsn, compact_lsn);
-    flusher->active_flushers++;
     // Scan versions to flush
     free_buffers();
     copy_count = 0;
-    for (auto wr = begin_wr; wr != end_wr; wr = wr->next())
+    fsynced_lsn = bs->heap->get_fsynced_lsn();
+    bitmap_copied = false;
+    memset(new_bmp, 0, bs->dsk.clean_entry_bitmap_size);
+    csum_copy.clear();
+    compact_info = bs->heap->iterate_compaction(cur_obj, fsynced_lsn, flusher->force_start, [&](heap_entry_t *wr)
     {
-        bs->prepare_read(read_vec, cur_obj, wr, 0, bs->dsk.data_block_size);
-        copy_count++;
+        if (!bitmap_copied)
+        {
+            memcpy(new_ext_bmp, wr->get_ext_bitmap(bs->heap), bs->dsk.clean_entry_bitmap_size);
+            bitmap_copied = true;
+        }
+        bitmap_set(new_bmp, wr->small().offset, wr->small().len, bs->dsk.bitmap_granularity);
+        if (bs->dsk.csum_block_size && bs->dsk.csum_block_size <= bs->dsk.bitmap_granularity)
+        {
+            csum_copy.push_back(wr);
+        }
+        if (wr->type() == BS_HEAP_SMALL_WRITE ||
+            wr->type() == BS_HEAP_INTENT_WRITE && bs->dsk.csum_block_size > bs->dsk.bitmap_granularity)
+        {
+            auto res = bs->prepare_read(read_vec, cur_obj, wr, 0, bs->dsk.data_block_size,
+                wr->type() == BS_HEAP_INTENT_WRITE && bs->dsk.csum_block_size > bs->dsk.bitmap_granularity && !bs->perfect_csum_update
+                ? COPY_BUF_SKIP_CSUM : 0);
+            if (res > 0)
+                copy_count++;
+        }
+    });
+    if (!compact_info.compact_lsn)
+    {
+        // Flushing is aborted
+        flusher->flushing.erase(cur_oid);
+        bs->heap->unlock_entry(cur_oid);
+        goto resume_0;
+    }
+    mem_or(new_bmp, compact_info.clean_wr->get_int_bitmap(bs->heap), bs->dsk.clean_entry_bitmap_size);
+    if (!bitmap_copied)
+    {
+        memcpy(new_ext_bmp, compact_info.clean_wr->get_ext_bitmap(bs->heap), bs->dsk.clean_entry_bitmap_size);
+        bitmap_copied = true;
+    }
+    if (bs->dsk.csum_block_size && bs->dsk.csum_block_size <= bs->dsk.bitmap_granularity)
+    {
+        memcpy(new_csums, compact_info.clean_wr->get_checksums(bs->heap), bs->dsk.data_block_size/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF));
+        for (size_t i = csum_copy.size(); i > 0; i--)
+        {
+            auto wr = csum_copy[i-1];
+            memcpy(new_csums + wr->small().offset/bs->dsk.csum_block_size*(bs->dsk.data_csum_type & 0xFF),
+                wr->get_checksums(bs->heap), wr->small().len/bs->dsk.csum_block_size*(bs->dsk.data_csum_type & 0xFF));
+        }
+        csum_copy.clear();
+    }
+    clean_loc = compact_info.clean_wr->big_location(bs->heap);
+    flusher->active_flushers++;
+    if (bs->log_level > 10)
+    {
+        printf("Compacting %jx:%jx v%ju..v%ju / l%ju..l%ju (%d writes)\n", cur_oid.inode, cur_oid.stripe,
+            compact_info.clean_wr->version, compact_info.compact_version,
+            compact_info.clean_wr->lsn, compact_info.compact_lsn, copy_count);
     }
     overwrite_start = overwrite_end = 0;
     if (read_vec.size() > 0)
@@ -254,7 +316,6 @@ resume_1:
     }
     // Read buffered data
     cur_obj = NULL;
-    begin_wr = end_wr = NULL;
 resume_2:
 resume_3:
     if (!read_buffered(2))
@@ -272,29 +333,49 @@ resume_3:
         flusher->wanting_meta_fsync--;
     }
     res = check_and_punch_checksums();
+    if (res == ENOENT || res == EDOM)
+    {
+        // Abort compaction
+        flusher->flushing.erase(cur_oid);
+        bs->heap->unlock_entry(cur_oid);
+        flusher->active_flushers--;
+        goto resume_0;
+    }
     if (res == EBUSY)
     {
 resume_4:
+        modified_block = UINT32_MAX;
+        res = bs->heap->punch_holes(compact_info.clean_wr, punch_bmp, new_csums, &modified_block);
+        if (res == ENOENT)
+        {
+            // Abort compaction
+            flusher->flushing.erase(cur_oid);
+            bs->heap->unlock_entry(cur_oid);
+            flusher->active_flushers--;
+            goto resume_0;
+        }
+        if (res == EAGAIN)
+        {
+            // Retry, block is busy
+            wait_state = 4;
+            return false;
+        }
+        assert(res == 0);
 resume_5:
-        if (!write_meta_block(4))
+resume_6:
+        if (!write_meta_block(5))
         {
             return false;
         }
-resume_6:
 resume_7:
 resume_8:
-        if (!fsync_meta(6))
+resume_9:
+        if (!fsync_meta(7))
         {
             return false;
         }
         res = 0;
     }
-    else if (res == ENOENT || res == EDOM)
-    {
-        // Abort compaction
-        flusher->active_flushers--;
-        goto resume_0;
-    }
     assert(res == 0);
     // Submit data writes
     for (i = 0; i < read_vec.size(); i++)
@@ -304,7 +385,7 @@ resume_8:
             (read_vec[i].copy_flags & COPY_BUF_PADDED)) // FIXME Shit, simplify these flags
         {
             assert(read_vec[i].buf);
-            await_sqe(9);
+            await_sqe(10);
             data->iov = (struct iovec){ read_vec[i].buf + (read_vec[i].copy_flags & COPY_BUF_PADDED
                 ? read_vec[i].offset - read_vec[i].disk_offset : 0), (size_t)read_vec[i].len };
             data->callback = simple_callback_w;
@@ -312,67 +393,57 @@ resume_8:
             wait_count++;
         }
     }
-resume_10:
+resume_11:
     if (wait_count > 0)
     {
-        wait_state = 10;
+        wait_state = 11;
         return false;
     }
     // Lock is only needed to prevent freeing the big_write because we overwrite it...
-    bs->heap->unlock_entry(cur_oid, copy_id);
+    bs->heap->unlock_entry(cur_oid);
     // Mark the object compacted, but don't free and remove small_writes
     // We'll free and remove them only when trimming
     // The only thing we modify here are big_write block checksums if >4k block is used
-    cur_obj = bs->heap->read_entry(cur_oid, &modified_block);
+    cur_obj = bs->heap->read_entry(cur_oid);
     if (!cur_obj)
     {
         // Abort compaction
+        flusher->flushing.erase(cur_oid);
         goto resume_0;
     }
     if (!calc_block_checksums())
     {
         // Abort compaction
+        flusher->flushing.erase(cur_oid);
         goto resume_0;
     }
-    if (read_to_fill_incomplete)
+    res = bs->heap->add_compact(cur_obj, compact_info.compact_version, compact_info.compact_lsn, clean_loc,
+        compact_info.do_delete, &modified_block, new_bmp, new_ext_bmp, new_csums);
+    if (res == EBUSY)
     {
-resume_23:
-resume_24:
-        if (!write_meta_block(23))
-        {
-            return false;
-        }
-        // Recheck the object because it could be invalidated again
-        cur_obj = bs->heap->read_entry(cur_oid, &modified_block);
-        if (!cur_obj)
-        {
-            // Abort compaction
-            goto resume_0;
-        }
+        // Abort compaction, object is already overwritten by something else
+        flusher->flushing.erase(cur_oid);
+        goto resume_0;
     }
-    bs->heap->mark_object_compacted(cur_obj, compact_lsn);
-    // Done
-    if (bs->log_level > 10)
-        printf("Compacted %jx:%jx l%ju (%d writes)\n", cur_oid.inode, cur_oid.stripe, compact_lsn, copy_count);
-    flusher->compact_counter++;
-    flusher->active_flushers--;
-    // Advance compacted_lsn every <journal_trim_interval> objects
-    if (co_id == 0 && !((++flusher->advance_lsn_counter) % bs->journal_trim_interval))
-    {
-        flusher->advance_lsn_counter = 0;
-resume_11:
+    assert(res == 0);
 resume_12:
 resume_13:
-resume_14:
-resume_15:
-        if (!trim_lsn(11))
-            return false;
+    if (!write_meta_block(12))
+    {
+        return false;
     }
+    // Done
+    if (bs->log_level > 10)
+    {
+        printf("Compacted %jx:%jx l%ju (%d writes)\n", cur_oid.inode, cur_oid.stripe, compact_info.compact_lsn, copy_count);
+    }
+    flusher->active_flushers--;
     if (should_repeat)
     {
         // Flush the same object again
         goto resume_1;
     }
+    flusher->flushing.erase(cur_oid);
     // All done
     goto resume_0;
 }
@@ -409,11 +480,11 @@ void journal_flusher_co::fill_partial_checksum_blocks()
                 .copy_flags = COPY_BUF_DATA | copy_flags,
                 .offset = blk_begin,
                 .len = blk_end - blk_begin,
-                .disk_loc = end_wr->big_location(bs->heap),
+                .disk_loc = clean_loc,
                 .disk_offset = blk_begin,
                 .disk_len = blk_end - blk_begin,
                 .buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, blk_end - blk_begin),
-                .wr_lsn = end_wr->lsn,
+                .wr = compact_info.clean_wr,
             });
         }
         auto & vec = read_vec[read_vec.size()-1];
@@ -451,26 +522,28 @@ int journal_flusher_co::check_and_punch_checksums()
         return 0;
     }
     // Verify data checksums
-    cur_obj = bs->heap->read_locked_entry(cur_oid, copy_id);
+    cur_obj = bs->heap->read_entry(cur_oid);
+    if (!cur_obj)
+    {
+        // Object is deleted, abort compaction
+        return ENOENT;
+    }
     bool csum_ok = true;
     for (int i = 0; i < read_vec.size(); i++)
     {
         auto & vec = read_vec[i];
         if (!(vec.copy_flags & (COPY_BUF_COALESCED|COPY_BUF_ZERO|COPY_BUF_SKIP_CSUM)))
         {
-            heap_write_t *wr = cur_obj->get_writes();
-            while (wr && wr->lsn != vec.wr_lsn)
-                wr = wr->next();
-            assert(wr);
-            uint32_t *csums = (uint32_t*)(wr->get_checksums(bs->heap)
+            uint32_t *csums = (uint32_t*)(vec.wr->get_checksums(bs->heap)
                 + (vec.disk_offset/bs->dsk.csum_block_size)*(bs->dsk.data_csum_type & 0xFF)
-                - ((wr->type() == BS_HEAP_BIG_WRITE) ? 0 : (wr->small().offset/bs->dsk.csum_block_size)*(bs->dsk.data_csum_type & 0xFF)));
+                - ((vec.wr->type() == BS_HEAP_BIG_WRITE || vec.wr->type() == BS_HEAP_BIG_INTENT)
+                    ? 0 : (vec.wr->small().offset/bs->dsk.csum_block_size)*(bs->dsk.data_csum_type & 0xFF)));
             bs->heap->calc_block_checksums(
-                csums, vec.buf, wr->get_int_bitmap(bs->heap), vec.disk_offset, vec.disk_offset+vec.disk_len, false,
+                csums, vec.buf, vec.wr->get_int_bitmap(bs->heap), vec.disk_offset, vec.disk_offset+vec.disk_len, false,
                 [&](uint32_t mismatch_pos, uint32_t expected_csum, uint32_t real_csum)
                 {
                     printf("Checksum mismatch during compaction in object %jx:%jx v%ju, offset 0x%x in %s area at offset 0x%jx: got %08x, expected %08x\n",
-                        cur_oid.inode, cur_oid.stripe, wr->version, mismatch_pos,
+                        cur_oid.inode, cur_oid.stripe, vec.wr->version, mismatch_pos,
                         (vec.copy_flags & COPY_BUF_JOURNAL ? "buffer" : "data"),
                         vec.disk_loc+vec.disk_offset, real_csum, expected_csum);
                     csum_ok = false;
@@ -489,20 +562,8 @@ int journal_flusher_co::check_and_punch_checksums()
         // Nothing to do
         return 0;
     }
-    cur_obj = bs->heap->read_entry(cur_oid, &modified_block);
-    if (!cur_obj)
-    {
-        // Object is deleted, abort compaction
-        return ENOENT;
-    }
-    bs->heap->get_compact_range(cur_obj, compact_lsn, &begin_wr, &end_wr);
-    if (!begin_wr)
-    {
-        // Object is overwritten, abort compaction
-        return ENOENT;
-    }
-    uint8_t *bmp = end_wr->get_int_bitmap(bs->heap);
-    uint8_t *csums = end_wr->get_checksums(bs->heap);
+    memcpy(punch_bmp, compact_info.clean_wr->get_int_bitmap(bs->heap), bs->dsk.clean_entry_bitmap_size);
+    memcpy(new_csums, compact_info.clean_wr->get_checksums(bs->heap), bs->dsk.data_block_size/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF));
     // Clear bits
     for (auto & vec: read_vec)
     {
@@ -513,7 +574,7 @@ int journal_flusher_co::check_and_punch_checksums()
         if (!(vec.copy_flags & COPY_BUF_COALESCED) &&
             ((vec.offset % bs->dsk.csum_block_size) || (vec.len % bs->dsk.csum_block_size)))
         {
-            bitmap_clear(bmp, vec.offset, vec.len, bs->dsk.bitmap_granularity);
+            bitmap_clear(punch_bmp, vec.offset, vec.len, bs->dsk.bitmap_granularity);
         }
     }
     // Update partial block checksums
@@ -522,42 +583,20 @@ int journal_flusher_co::check_and_punch_checksums()
         if (vec.copy_flags & COPY_BUF_CSUM_FILL)
         {
             uint32_t csum_off = vec.offset/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF);
-            bs->heap->calc_block_checksums((uint32_t*)(csums+csum_off), vec.buf, bmp, vec.offset, vec.offset+vec.len, true, NULL);
+            bs->heap->calc_block_checksums((uint32_t*)(new_csums+csum_off), vec.buf, punch_bmp, vec.offset, vec.offset+vec.len, true, NULL);
         }
     }
-    cur_obj->crc32c = cur_obj->calc_crc32c();
-    if (res == ENOENT)
-    {
-        // Object is deleted, abort compaction
-        return ENOENT;
-    }
-    // Modified, we should write the block to disk
-    assert(!res);
+    // Modified, we should add_punch_holes and then write the block to disk
     return EBUSY;
 }
 
 bool journal_flusher_co::calc_block_checksums()
 {
-    if (bs->dsk.csum_block_size <= bs->dsk.bitmap_granularity || !read_vec.size())
+    if (bs->dsk.csum_block_size <= bs->dsk.bitmap_granularity)
     {
         return true;
     }
-    bs->heap->get_compact_range(cur_obj, compact_lsn, &begin_wr, &end_wr);
-    if (!begin_wr)
-    {
-        // Object is overwritten, abort compaction
-        return false;
-    }
-    uint8_t *bmp = end_wr->get_int_bitmap(bs->heap);
-    uint8_t *csums = end_wr->get_checksums(bs->heap);
-    // Set bits
-    for (auto & vec: read_vec)
-    {
-        if (!(vec.copy_flags & (COPY_BUF_COALESCED|COPY_BUF_CSUM_FILL)))
-        {
-            bitmap_set(bmp, vec.offset, vec.len, bs->dsk.bitmap_granularity);
-        }
-    }
+    memcpy(new_csums, compact_info.clean_wr->get_checksums(bs->heap), bs->dsk.data_block_size/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF));
     // Update block checksums
     size_t i = 0;
     while (i < read_vec.size() && !(read_vec[i].copy_flags & COPY_BUF_CSUM_FILL))
@@ -576,7 +615,7 @@ bool journal_flusher_co::calc_block_checksums()
         assert(!(end % bs->dsk.csum_block_size));
         uint32_t csum_off = start/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF);
         bs->heap->calc_block_checksums(
-            (uint32_t*)(csums+csum_off), bmp, start, end,
+            (uint32_t*)(new_csums+csum_off), new_bmp, start, end,
             [&](uint32_t start, uint32_t & len)
             {
                 // O(n^2) search, may be fixed later :-p
@@ -593,7 +632,6 @@ bool journal_flusher_co::calc_block_checksums()
             }, true, NULL
         );
     }
-    cur_obj->crc32c = cur_obj->calc_crc32c();
     return true;
 }
 
@@ -603,13 +641,15 @@ bool journal_flusher_co::write_meta_block(int wait_base)
         goto resume_0;
     else if (wait_state == wait_base+1)
         goto resume_1;
-    await_sqe(0);
-    data->iov = (struct iovec){ bs->heap->get_meta_block(modified_block), (size_t)bs->dsk.meta_block_size };
-    data->callback = simple_callback_w;
-    io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + (modified_block+1)*bs->dsk.meta_block_size);
-    wait_count++;
+resume_0:
+    if (bs->ringloop->space_left() < 1)
+    {
+        wait_state = wait_base+0;
+        return 0;
+    }
+    bs->prepare_meta_block_write(modified_block);
 resume_1:
-    if (wait_count > 0)
+    if (bs->meta_block_is_pending(modified_block))
     {
         wait_state = wait_base+1;
         return false;
@@ -664,11 +704,11 @@ bool journal_flusher_co::fsync_meta(int wait_base)
     if (wait_state == wait_base)        goto resume_0;
     else if (wait_state == wait_base+1) goto resume_1;
     else if (wait_state == wait_base+2) goto resume_2;
-resume_0:
     if (bs->dsk.disable_meta_fsync)
     {
         return true;
     }
+resume_0:
     if (flusher->wanting_meta_fsync || flusher->fsyncing_meta > 0)
     {
         wait_state = wait_base;
@@ -693,69 +733,30 @@ resume_2:
     return true;
 }
 
-int journal_flusher_co::fsync_buffer(int wait_base)
-{
-    if (wait_state == wait_base)        goto resume_0;
-    else if (wait_state == wait_base+1) goto resume_1;
-    if (bs->dsk.disable_journal_fsync && bs->dsk.disable_meta_fsync && bs->dsk.disable_data_fsync || !bs->unsynced_big_write_count && !bs->unsynced_small_write_count)
-    {
-        return 1;
-    }
-    if (flusher->syncing_buffer)
-    {
-        return 0;
-    }
-    flusher->active_flushers++;
-    flusher->syncing_buffer++;
-resume_0:
-    assert(!wait_count);
-    compact_lsn = bs->heap->get_completed_lsn();
-    if (!bs->submit_fsyncs(wait_count))
-    {
-        wait_state = wait_base+0;
-        return 0;
-    }
-resume_1:
-    if (wait_count > 0)
-    {
-        wait_state = wait_base+1;
-        return 0;
-    }
-    bs->heap->mark_lsn_fsynced(compact_lsn);
-    flusher->active_flushers--;
-    flusher->syncing_buffer--;
-    return 2;
-}
-
-bool journal_flusher_co::trim_lsn(int wait_base)
+bool journal_flusher_co::fsync_buffer(int wait_base)
 {
     if (wait_state == wait_base)        goto resume_0;
     else if (wait_state == wait_base+1) goto resume_1;
     else if (wait_state == wait_base+2) goto resume_2;
-    else if (wait_state == wait_base+3) goto resume_3;
-    else if (wait_state == wait_base+4) goto resume_4;
-    compact_lsn = bs->heap->get_compacted_lsn();
-    if (((blockstore_meta_header_v3_t*)bs->meta_superblock)->compacted_lsn == compact_lsn)
+    if (!bs->has_unsynced())
     {
         return true;
     }
-    flusher->active_flushers++;
-    assert(!wait_count);
-    if (!bs->dsk.disable_meta_fsync)
+resume_0:
+    if (flusher->syncing_buffer)
     {
-        await_sqe(0);
-        data->iov = { 0 };
-        data->callback = simple_callback_w;
-        io_uring_prep_fsync(sqe, bs->dsk.meta_fd, IORING_FSYNC_DATASYNC);
-        wait_count++;
+        wait_state = wait_base+0;
+        return false;
     }
-    if (!bs->dsk.disable_data_fsync && bs->dsk.data_fd != bs->dsk.meta_fd)
+    flusher->active_flushers++;
+    flusher->syncing_buffer++;
+resume_1:
+    assert(!wait_count);
+    fsynced_lsn = bs->heap->get_completed_lsn();
+    if (!bs->submit_fsyncs(wait_count))
     {
-        await_sqe(1);
-        data->iov = { 0 };
-        data->callback = simple_callback_w;
-        io_uring_prep_fsync(sqe, bs->dsk.data_fd, IORING_FSYNC_DATASYNC);
-        wait_count++;
+        wait_state = wait_base+1;
+        return false;
     }
 resume_2:
     if (wait_count > 0)
@@ -763,23 +764,37 @@ resume_2:
         wait_state = wait_base+2;
         return false;
     }
-    ((blockstore_meta_header_v3_t*)bs->meta_superblock)->compacted_lsn = compact_lsn;
+    bs->heap->mark_lsn_fsynced(fsynced_lsn);
+    flusher->active_flushers--;
+    flusher->syncing_buffer--;
+    return true;
+}
+
+bool journal_flusher_co::trim_lsn(int wait_base)
+{
+    if (wait_state == wait_base)        goto resume_0;
+    else if (wait_state == wait_base+1) goto resume_1;
+    fsynced_lsn = bs->heap->get_fsynced_lsn();
+    if (((blockstore_meta_header_v3_t*)bs->meta_superblock)->completed_lsn == fsynced_lsn)
+    {
+        return true;
+    }
+    flusher->active_flushers++;
+    ((blockstore_meta_header_v3_t*)bs->meta_superblock)->completed_lsn = fsynced_lsn;
     ((blockstore_meta_header_v3_t*)bs->meta_superblock)->set_crc32c();
-    await_sqe(3);
+    await_sqe(0);
     data->iov = (struct iovec){ bs->meta_superblock, (size_t)bs->dsk.meta_block_size };
     data->callback = simple_callback_w;
     io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset);
     // Update superblock with datasync
     sqe->rw_flags = RWF_DSYNC;
     wait_count++;
-resume_4:
+resume_1:
     if (wait_count > 0)
     {
-        wait_state = wait_base+4;
+        wait_state = wait_base+1;
         return false;
     }
-    bs->heap->mark_lsn_trimmed(compact_lsn);
-    flusher->compact_counter++;
     flusher->active_flushers--;
     return true;
 }

src/blockstore/blockstore_flush.h

@@ -6,7 +6,7 @@ struct copy_buffer_t
     uint32_t copy_flags;
     uint64_t offset, len, disk_loc, disk_offset, disk_len;
     uint8_t *buf;
-    uint64_t wr_lsn;
+    heap_entry_t *wr;
 };
 
 struct meta_sector_t
@@ -36,27 +36,28 @@ class journal_flusher_co
     int wait_state, wait_count;
     struct io_uring_sqe *sqe;
     struct ring_data_t *data;
+    uint8_t *new_csums = NULL;
+    uint8_t *new_bmp = NULL;
+    uint8_t *punch_bmp = NULL;
+    uint8_t *new_ext_bmp = NULL;
 
     std::function<void(ring_data_t*)> simple_callback_r, simple_callback_w;
 
     object_id cur_oid;
-    uint64_t copy_id;
-    uint64_t compact_lsn;
-    uint64_t cur_version;
-    heap_object_t *cur_obj;
-    heap_write_t *begin_wr, *end_wr;
+    heap_entry_t *cur_obj;
+    uint64_t fsynced_lsn;
+    heap_compact_t compact_info;
+    uint64_t clean_loc;
     uint32_t modified_block;
+    bool bitmap_copied;
     bool should_repeat;
 
     std::vector<copy_buffer_t> read_vec;
+    std::vector<heap_entry_t*> csum_copy;
     uint32_t overwrite_start, overwrite_end;
-    uint32_t big_start, big_end;
     int i, res;
     bool read_to_fill_incomplete;
     int copy_count;
-    uint64_t clean_loc;
-    flusher_meta_write_t meta_old, meta_new;
-    bool do_repeat = false;
 
     friend class journal_flusher_t;
 
@@ -68,7 +69,7 @@ class journal_flusher_co
     bool write_meta_block(int wait_base);
     bool read_buffered(int wait_base);
     bool fsync_meta(int wait_base);
-    int fsync_buffer(int wait_base);
+    bool fsync_buffer(int wait_base);
     bool trim_lsn(int wait_base);
 public:
     journal_flusher_co();
@@ -85,9 +86,7 @@ class journal_flusher_t
     blockstore_impl_t *bs;
     friend class journal_flusher_co;
 
-    int advance_lsn_counter = 0;
-    uint64_t compact_counter = 0;
-
+    robin_hood::unordered_flat_set<object_id> flushing;
     int active_flushers = 0;
     int wanting_meta_fsync = 0;
     bool fsyncing_meta = false;
@@ -98,7 +97,6 @@ public:
     ~journal_flusher_t();
     void loop();
     int get_syncing_buffer();
-    uint64_t get_compact_counter();
     bool is_active();
     void request_trim();
     void release_trim();

src/blockstore/blockstore_heap.h

@@ -1,4 +1,4 @@
-// Metadata storage version 3 ("heap")
+// Metadata storage version 3 ("lsm heap")
 // Copyright (c) Vitaliy Filippov, 2025+
 // License: VNPL-1.1 (see README.md for details)
 
@@ -19,326 +19,304 @@ struct pool_shard_settings_t
 {
     uint32_t pg_count;
     uint32_t pg_stripe_size;
+    uint32_t no_inode_stats;
 };
 
-#define BS_HEAP_TYPE 7
-#define BS_HEAP_OBJECT 1
+#define BS_HEAP_TYPE 0x07
+#define BS_HEAP_BIG_WRITE 1
 #define BS_HEAP_SMALL_WRITE 2
-#define BS_HEAP_BIG_WRITE 3
-#define BS_HEAP_TOMBSTONE 4
-#define BS_HEAP_INTENT_WRITE 5
-#define BS_HEAP_STABLE 8
+#define BS_HEAP_INTENT_WRITE 3
+#define BS_HEAP_BIG_INTENT 4
+#define BS_HEAP_DELETE 5
+#define BS_HEAP_COMMIT 6
+#define BS_HEAP_ROLLBACK 7
+#define BS_HEAP_STABLE 0x40
+#define BS_HEAP_GARBAGE 0x80
 
 class blockstore_heap_t;
 
-struct __attribute__((__packed__)) heap_small_write_t
+struct heap_small_write_t;
+struct heap_big_write_t;
+struct heap_big_intent_t;
+
+struct __attribute__((__packed__)) heap_entry_t
 {
     uint16_t size;
-    int16_t next_pos;
-    uint8_t flags;
+    uint16_t entry_type;
+    uint32_t crc32c;
     uint64_t lsn;
+    uint64_t inode;
+    uint64_t stripe;
     uint64_t version;
-    uint64_t location;
-    uint32_t offset;
-    uint32_t len;
-};
-
-struct __attribute__((__packed__)) heap_big_write_t
-{
-    uint16_t size;
-    int16_t next_pos;
-    uint8_t flags;
-    uint64_t lsn;
-    uint64_t version;
-    uint32_t block_num;
-};
-
-struct __attribute__((__packed__)) heap_tombstone_t
-{
-    uint16_t size;
-    int16_t next_pos;
-    uint8_t flags;
-    uint64_t lsn;
-    uint64_t version;
-};
-
-struct __attribute__((__packed__)) heap_write_t
-{
-    // size should have top bit cleared
-    uint16_t size = 0;
-    int16_t next_pos = 0;
-    uint8_t entry_type = 0; // BS_HEAP_*
-    uint64_t lsn = 0;
-    uint64_t version = 0;
 
     // uint8_t[] external_bitmap
     // uint8_t[] internal_bitmap
     // uint32_t[] checksums
 
-    heap_write_t *next();
     inline uint8_t type() const { return (entry_type & BS_HEAP_TYPE); }
     inline heap_small_write_t& small() { return *(heap_small_write_t*)this; }
     inline heap_big_write_t& big() { return *(heap_big_write_t*)this; }
+    inline heap_big_intent_t& big_intent() { return *(heap_big_intent_t*)this; }
+    bool is_garbage();
+    void set_garbage();
+    bool is_overwrite();
+    bool is_compactable();
+    bool is_before(heap_entry_t *other);
     uint32_t get_size(blockstore_heap_t *heap);
-    uint32_t get_csum_size(blockstore_heap_t *heap);
-    bool needs_recheck(blockstore_heap_t *heap);
-    bool needs_compact(blockstore_heap_t *heap);
-    bool is_compacted(uint64_t compacted_lsn);
-    bool can_be_collapsed(blockstore_heap_t *heap);
-    bool is_allowed_before_compacted(uint64_t compacted_lsn, bool is_last_entry);
     uint8_t *get_ext_bitmap(blockstore_heap_t *heap);
     uint8_t *get_int_bitmap(blockstore_heap_t *heap);
     uint8_t *get_checksums(blockstore_heap_t *heap);
     uint32_t *get_checksum(blockstore_heap_t *heap);
     uint64_t big_location(blockstore_heap_t *heap);
     void set_big_location(blockstore_heap_t *heap, uint64_t location);
-};
-
-struct __attribute__((__packed__)) heap_object_t
-{
-    // size should have top bit cleared
-    uint16_t size = 0;
-    // linked list of write entries...
-    // newest entries are stored first to simplify scanning
-    int16_t write_pos = 0;
-    uint8_t entry_type = 0; // BS_HEAP_*
-    uint32_t crc32c = 0;
-    uint64_t inode = 0;
-    uint64_t stripe = 0;
-
-    heap_write_t *get_writes();
     uint32_t calc_crc32c();
 };
 
-struct heap_object_lsn_t
+struct __attribute__((__packed__)) heap_small_write_t
 {
-    object_id oid;
-    uint64_t lsn;
+    heap_entry_t hdr;
+
+    uint64_t location;
+    uint32_t offset;
+    uint32_t len;
+
+    // Also includes 1 bitmap and 1 crc32c after the bitmap if checksums are disabled
 };
 
-inline bool operator < (const heap_object_lsn_t & a, const heap_object_lsn_t & b)
+struct __attribute__((__packed__)) heap_big_write_t
 {
-    return a.oid < b.oid || a.oid == b.oid && a.lsn < b.lsn;
-}
+    heap_entry_t hdr;
 
-struct tmp_compact_item_t
-{
-    object_id oid;
-    uint64_t lsn;
-    bool compact;
+    uint32_t block_num;
 };
 
-struct heap_mvcc_copy_id_t
+struct __attribute__((__packed__)) heap_big_intent_t
 {
-    object_id oid;
-    uint64_t copy_id;
+    heap_entry_t hdr;
+
+    uint32_t block_num;
+    uint32_t offset;
+    uint32_t len;
+
+    // Also includes 2 bitmaps and 1 crc32c if checksums are disabled
 };
 
-inline bool operator == (const heap_mvcc_copy_id_t & a, const heap_mvcc_copy_id_t & b)
+struct __attribute__((__packed__)) heap_list_item_t
 {
-    return a.oid.inode == b.oid.inode && a.oid.stripe == b.oid.stripe && a.copy_id == b.copy_id;
-}
-
-namespace std
-{
-    template<> struct hash<heap_mvcc_copy_id_t>
-    {
-        inline size_t operator()(const heap_mvcc_copy_id_t &s) const
-        {
-            size_t seed = std::hash<object_id>()(s.oid);
-            // Copy-pasted from spp::hash_combine()
-            seed ^= (s.copy_id + 0xc6a4a7935bd1e995 + (seed << 6) + (seed >> 2));
-            return seed;
-        }
-    };
+    heap_list_item_t *prev;
+    heap_list_item_t *next;
+    uint32_t block_num;
+    heap_entry_t entry;
 };
 
 struct heap_object_mvcc_t
 {
     uint32_t readers = 0;
-    heap_object_t *entry_copy = NULL;
+    heap_entry_t *garbage_entry = NULL;
 };
 
-struct __attribute__((__packed__)) heap_block_info_t
+struct heap_block_info_t
 {
     uint32_t used_space = 0;
-    uint32_t free_pos = 0;
-    uint8_t *data = NULL;
+    uint64_t mod_lsn = 0, mod_lsn_to = 0; // only 1 block write of LSN sequence is allowed at a moment
+    bool is_writing: 1;
+    bool has_garbage: 1;
+    std::vector<heap_list_item_t*> entries;
 };
 
 struct heap_inflight_lsn_t
 {
-    object_id oid;
     uint64_t flags;
+    heap_entry_t *wr;
 };
 
-struct heap_refqi_t
+struct heap_compact_t
 {
-    uint64_t lsn;
-    uint64_t inode;
-    uint64_t location;
-    uint32_t len;
-    bool is_data;
+    uint64_t compact_lsn, compact_version;
+    heap_entry_t *clean_wr;
+    bool do_delete;
+};
+
+struct heap_reshard_state_t;
+
+struct heap_li_hash
+{
+    size_t operator()(const heap_list_item_t* li) const noexcept
+    {
+        return robin_hood::hash_int(li->entry.stripe);
+    }
+};
+
+struct heap_li_equal
+{
+    constexpr bool operator()(const heap_list_item_t* a, const heap_list_item_t* b) const noexcept
+    {
+        return a->entry.stripe == b->entry.stripe;
+    }
 };
 
 using i64hash_t = robin_hood::hash<uint64_t>;
+using heap_inode_map_t = robin_hood::unordered_flat_set<heap_list_item_t*, heap_li_hash, heap_li_equal, 88>;
 using heap_block_index_t = robin_hood::unordered_flat_map<uint64_t,
-    robin_hood::unordered_flat_map<inode_t, robin_hood::unordered_flat_map<uint64_t, uint64_t, i64hash_t, std::equal_to<uint64_t>, 88>, i64hash_t>, i64hash_t>;
-using heap_mvcc_map_t = robin_hood::unordered_flat_map<heap_mvcc_copy_id_t, heap_object_mvcc_t>;
+    robin_hood::unordered_flat_map<inode_t, void*, i64hash_t>, i64hash_t>;
+using heap_mvcc_map_t = robin_hood::unordered_flat_map<object_id, heap_object_mvcc_t>;
 
 class blockstore_heap_t
 {
-    friend class heap_write_t;
-    friend class heap_object_t;
+    friend struct heap_entry_t;
 
     blockstore_disk_t *dsk = NULL;
     uint8_t* buffer_area = NULL;
-    bool abort_on_corruption = false;
-    bool abort_on_overlap = true;
     int log_level = 0;
-
     const uint32_t meta_block_count = 0;
-    uint32_t target_block_free_space = 800;
+    const uint32_t max_entry_size = 0;
 
-    uint64_t next_lsn = 0;
     robin_hood::unordered_flat_map<pool_id_t, pool_shard_settings_t> pool_shard_settings;
     // PG => inode => stripe => block number
     heap_block_index_t block_index;
     std::vector<heap_block_info_t> block_info;
     allocator_t *data_alloc = NULL;
     multilist_index_t *meta_alloc = NULL;
-    uint32_t meta_alloc_count = 0;
+    uint32_t meta_nearfull_blocks = 0;
     uint64_t meta_used_space = 0;
     multilist_alloc_t *buffer_alloc = NULL;
-    heap_mvcc_map_t object_mvcc;
-    std::unordered_map<uint64_t, uint32_t> mvcc_data_refs;
-    std::unordered_map<uint64_t, uint32_t> mvcc_buffer_refs;
     std::map<uint64_t, uint64_t> inode_space_stats;
     uint64_t buffer_area_used_space = 0;
     uint64_t data_used_space = 0;
 
-    // LSN queue: inflight (writing) -> completed [-> fsynced] -> compactable -> compacted [-> fsynced] -> trimmed and removed
+    uint64_t next_lsn = 0;
+    uint32_t last_allocated_block = UINT32_MAX;
+    heap_mvcc_map_t object_mvcc;
+
+    // LSN queue: inflight (writing) -> completed [-> fsynced]
     std::deque<heap_inflight_lsn_t> inflight_lsn;
     uint32_t to_compact_count = 0;
+    uint64_t compacted_count = 0;
+    uint32_t inflight_overwrite_count = 0;
     uint64_t first_inflight_lsn = 0;
     uint64_t completed_lsn = 0;
     uint64_t fsynced_lsn = 0;
-    uint64_t compacted_lsn = 0;
-    uint64_t next_compact_lsn = 0;
-    std::deque<heap_refqi_t> overwrite_ref_queue;
+    std::deque<object_id> compact_queue;
 
-    std::vector<tmp_compact_item_t> tmp_compact_queue;
-    std::deque<object_id> recheck_queue;
+    bool marked_used_blocks = false;
+    bool recheck_queue_filled = false;
+    std::set<uint32_t> recheck_modified_blocks;
+    std::deque<heap_entry_t*> recheck_queue;
     int recheck_in_progress = 0;
     bool in_recheck = false;
     std::function<void(bool is_data, uint64_t offset, uint64_t len, uint8_t* buf, std::function<void()>)> recheck_cb;
     int recheck_queue_depth = 0;
 
-    const uint32_t max_write_entry_size;
-
     uint64_t get_pg_id(inode_t inode, uint64_t stripe);
+    bool validate_object(heap_entry_t *obj);
+    void fill_recheck_queue();
+    int mark_used_blocks();
+    void recheck_buffer(heap_entry_t *cwr, uint8_t *buf);
     void defragment_block(uint32_t block_num);
-    uint32_t find_block_run(heap_block_info_t & block, uint32_t space);
-    uint32_t find_block_space(uint32_t block_num, uint32_t space);
-    uint32_t block_has_compactable(uint8_t *data);
-    uint32_t compact_object_to(heap_object_t *obj, uint64_t lsn, uint8_t *new_csums, bool do_free);
-    void copy_full_object(uint8_t *dst, heap_object_t *obj);
-    bool mvcc_save_copy(heap_object_t *obj);
-    bool mvcc_check_tracking(object_id oid);
-    void free_mvcc(heap_mvcc_map_t::iterator mvcc_it);
-    void allocate_block(heap_block_info_t & inf);
-    int allocate_new_object(object_id oid, uint32_t full_object_size, uint32_t *modified_block, heap_object_t **new_obj);
-    int add_object(object_id oid, heap_write_t *wr, uint32_t *modified_block);
-    void mark_overwritten(uint64_t over_lsn, uint64_t inode, heap_write_t *wr, heap_write_t *end_wr, bool tracking_active);
-    int update_object(uint32_t block_num, heap_object_t *obj, heap_write_t *wr, uint32_t *modified_block, uint32_t *moved_from_block);
-    void init_erase(uint32_t block_num, heap_object_t *obj);
-    void erase_object(uint32_t block_num, heap_object_t *obj, uint64_t lsn, bool tracking_active);
-    void reindex_block(uint32_t block_num, heap_object_t *from_obj);
-    void erase_block_index(inode_t inode, uint64_t stripe);
-    void deref_data(uint64_t inode, uint64_t location, bool free_at_0);
-    void deref_buffer(uint64_t inode, uint64_t location, uint32_t len, bool free_at_0);
-    void deref_overwrites(uint64_t lsn);
-    void free_object_space(inode_t inode, heap_write_t *from, heap_write_t *to, int mode = 0);
-    void add_used_space(uint32_t block_num, int32_t used_delta);
-    void push_inflight_lsn(object_id oid, uint64_t lsn, uint64_t flags);
+    void reshard_add(heap_reshard_state_t *st, heap_list_item_t *li);
 
+    int allocate_entry(uint32_t entry_size, uint32_t *block_num, bool allow_last_free);
+    void insert_list_item(heap_list_item_t *li);
+    int add_entry(uint32_t wr_size, uint32_t *modified_block, bool allow_last_free,
+        bool explicit_complete, std::function<void(heap_entry_t *wr)> fill_entry);
+    int add_simple(heap_entry_t *obj, uint64_t version, uint32_t *modified_block, uint32_t entry_type);
+    uint32_t meta_alloc_pos(const heap_block_info_t & inf);
+    void modify_alloc(uint32_t block_num, std::function<void(heap_block_info_t &)> change_cb);
+    void mark_garbage_up_to(heap_entry_t *wr);
+    void mark_garbage(uint32_t block_num, heap_entry_t *prev_wr, uint32_t used_big);
+    void push_inflight_lsn(uint64_t lsn, heap_entry_t *wr, uint64_t flags);
+    void mark_completed_lsns(uint64_t mod_lsn);
+    void apply_inflight(heap_inflight_lsn_t & inflight);
 public:
     blockstore_heap_t(blockstore_disk_t *dsk, uint8_t *buffer_area, int log_level = 0);
     ~blockstore_heap_t();
-    // set initially compacted lsn - should be done before loading
-    void set_compacted_lsn(uint64_t compacted_lsn);
-    uint64_t get_compacted_lsn();
-    // load data from the disk, returns count of loaded write entries
-    void read_blocks(uint64_t disk_offset, uint64_t size, uint8_t *buf,
-        std::function<void(heap_object_t*)> handle_object, std::function<void(uint32_t, uint32_t, uint8_t*)> handle_block);
-    uint64_t load_blocks(uint64_t disk_offset, uint64_t size, uint8_t *buf);
+    void start_load(uint64_t completed_lsn);
+    // load data from the disk, returns EDOM on corruption
+    int read_blocks(uint64_t disk_offset, uint64_t size, uint8_t *buf, bool allow_corrupted,
+        std::function<void(uint32_t block_num, heap_entry_t* wr)> handle_write,
+        std::function<void(uint32_t, uint32_t, uint8_t*)> handle_block);
+    int load_blocks(uint64_t disk_offset, uint64_t size, uint8_t *buf,
+        bool allow_corrupted, uint64_t &entries_loaded);
     // finish loading
-    void finish_load();
+    int finish_load(bool allow_corrupted = false);
+    // get blocks which are modified during loading and should be written to the disk
+    // before finishing initialization if not R/O
+    std::vector<uint32_t> get_recheck_modified_blocks();
     // recheck small write data after reading the database from disk
     bool recheck_small_writes(std::function<void(bool is_data, uint64_t offset, uint64_t len, uint8_t* buf, std::function<void()>)> read_buffer, int queue_depth);
-    // initialize metadata area (fill it with empty data)
-    // returns 0 when done, EAGAIN when the caller has to wait more
-    int initialize();
-    // read from the metadata area
-    // returns 0 when done, EAGAIN when the caller has to wait more
-    int read();
     // reshard database according to the pool's PG count
-    void reshard(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size);
+    void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
+    bool reshard_continue(void* reshard_state, uint64_t chunk_limit);
+    bool reshard_check(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size);
+    void reshard_abort(void* reshard_state);
+    void set_no_inode_stats(const std::vector<uint64_t> & pool_ids);
+    void recalc_inode_space_stats(uint64_t pool_id, bool per_inode);
     // read an object entry and lock it against removal
     // in the future, may become asynchronous
-    heap_object_t *lock_and_read_entry(object_id oid, uint64_t & copy_id);
+    heap_entry_t *lock_and_read_entry(object_id oid);
     // re-read a locked object entry with the given lsn (pointer may be invalidated)
-    heap_object_t *read_locked_entry(object_id oid, uint64_t copy_id);
+    heap_entry_t *read_locked_entry(object_id oid, uint64_t lsn);
     // read an object entry without locking it
-    heap_object_t *read_entry(object_id oid, uint32_t *block_num_ptr, bool for_update = false);
+    heap_entry_t *read_entry(object_id oid);
     // unlock an entry
-    bool unlock_entry(object_id oid, uint64_t copy_id);
+    bool unlock_entry(object_id oid);
     // set or verify checksums in a write request
-    bool calc_checksums(heap_write_t *wr, uint8_t *data, bool set, uint32_t offset = 0, uint32_t len = 0);
+    bool calc_checksums(heap_entry_t *wr, uint8_t *data, bool set, uint32_t offset = 0, uint32_t len = 0);
     // set or verify raw block checksums
     bool calc_block_checksums(uint32_t *block_csums, uint8_t *data, uint8_t *bitmap, uint32_t start, uint32_t end,
         bool set, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);
     bool calc_block_checksums(uint32_t *block_csums, uint8_t *bitmap,
         uint32_t start, uint32_t end, std::function<uint8_t*(uint32_t start, uint32_t & len)> next,
         bool set, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);
-    // copy an object as is
-    int copy_object(heap_object_t *obj, uint32_t *modified_block);
-    // auto-compacts the object, then adds a write entry to it and to the compaction queue
+    // adds a small_write or intent_write entry to an object
     // return 0 if OK, or maybe ENOSPC
-    int post_write(object_id oid, heap_write_t *wr, uint32_t *modified_block, uint32_t *moved_from_block);
-    int post_write(uint32_t & block_num, object_id oid, heap_object_t *obj, heap_write_t *wr, uint32_t *moved_from_block);
+    int add_small_write(object_id oid, heap_entry_t **obj_ptr, uint16_t type, uint64_t version,
+        uint32_t offset, uint32_t len, uint64_t location, uint8_t *bitmap, uint8_t *data, uint32_t *modified_block);
+    // adds a big_write (overwrite) entry to an object
+    int add_big_write(object_id oid, heap_entry_t *old_head, bool stable, uint64_t version,
+        uint32_t offset, uint32_t len, uint64_t location, uint8_t *bitmap, uint8_t *data, uint32_t *modified_block);
+    // adds a "redirecting" big_intent entry to an object (same as big_write, used to avoid fsync on desktop SSDs)
+    int add_redirect_intent(object_id oid, heap_entry_t **obj_ptr, uint64_t version,
+        uint32_t offset, uint32_t len, uint64_t location, uint8_t *bitmap, uint8_t *data, uint32_t *modified_block);
+    // adds a big_intent (atomic partial modification) entry to an object
+    int add_big_intent(object_id oid, heap_entry_t **obj_ptr, uint64_t version,
+        uint32_t offset, uint32_t len, uint8_t *bitmap, uint8_t *data, uint8_t *checksums, uint32_t *modified_block);
+    // adds a compacted up to <version> entry to an object
+    int add_compact(heap_entry_t *obj, uint64_t compact_version, uint64_t compact_lsn, uint64_t compact_location,
+        bool do_delete, uint32_t *modified_block, uint8_t *new_int_bitmap, uint8_t *new_ext_bitmap, uint8_t *new_csums);
+    // "punch holes" in a big_entry
+    int punch_holes(heap_entry_t *wr, uint8_t *new_bitmap, uint8_t *new_csums, uint32_t *modified_block);
     // stabilize an unstable object version
     // return 0 if OK, ENOENT if not exists
-    int post_stabilize(object_id oid, uint64_t version, uint32_t *modified_block, uint64_t *new_lsn, uint64_t *new_to_lsn);
+    int add_commit(heap_entry_t *obj, uint64_t version, uint32_t *modified_block);
     // rollback an unstable object version
     // return 0 if OK, ENOENT if not exists, EBUSY if already stable
-    int post_rollback(object_id oid, uint64_t version, uint64_t *new_lsn, uint32_t *modified_block);
+    int add_rollback(heap_entry_t *obj, uint64_t version, uint32_t *modified_block);
     // forget an object
     // return error code
-    int post_delete(object_id oid, uint64_t *new_lsn, uint32_t *modified_block);
-    int post_delete(uint32_t block_num, heap_object_t *obj, uint64_t *new_lsn);
+    int add_delete(heap_entry_t *obj, uint32_t *modified_block);
     // get the next object to compact
     // guaranteed to return objects in min lsn order
     // returns 0 if OK, ENOENT if nothing to compact
     int get_next_compact(object_id & oid);
-    // get the range of an object eligible for compaction
-    void get_compact_range(heap_object_t *obj, uint64_t max_lsn, heap_write_t **begin_wr, heap_write_t **end_wr);
-    // mark an object as compacted up to the given lsn
-    int compact_object(object_id oid, uint64_t lsn, uint8_t *new_csums);
+    void iterate_with_stable(heap_entry_t *obj, uint64_t max_lsn, std::function<bool(heap_entry_t*, bool stable)> cb);
+    // iterate compactable entries
+    heap_compact_t iterate_compaction(heap_entry_t *obj, uint64_t fsynced_lsn, bool under_pressure,
+        std::function<void(heap_entry_t*)> small_wr_cb);
+    // iterate all objects
+    void iterate_objects(std::function<void(heap_entry_t*, uint32_t block_num)> cb);
     // retrieve object listing from a PG
     int list_objects(uint32_t pg_num, object_id min_oid, object_id max_oid,
         obj_ver_id **result_list, size_t *stable_count, size_t *unstable_count);
-    // set a block number for a new object and returns error status: 0, EAGAIN or ENOSPC
-    int get_block_for_new_object(uint32_t & out_block_num, uint32_t size = 0);
 
     // inflight write tracking
-    void mark_lsn_completed(uint64_t lsn);
-    void mark_lsn_fsynced(uint64_t lsn);
-    void mark_lsn_compacted(uint64_t lsn, bool allow_undone = false);
-    void mark_object_compacted(heap_object_t *obj, uint64_t max_lsn);
-    void mark_lsn_trimmed(uint64_t lsn);
+    void start_block_write(uint32_t block_num);
+    void complete_block_write(uint32_t block_num);
+    void complete_lsn_write(uint64_t lsn);
+    bool is_lsn_completed(uint64_t lsn);
     uint64_t get_completed_lsn();
     uint64_t get_fsynced_lsn();
+    void mark_lsn_fsynced(uint64_t lsn);
 
     // data device block allocator functions
     uint64_t find_free_data();
@@ -354,7 +332,8 @@ public:
     uint64_t get_buffer_area_used_space();
 
     // get metadata block data buffer and used space
-    uint8_t *get_meta_block(uint32_t block_num);
+    void get_meta_block(uint32_t block_num, uint8_t *buffer);
+    void fill_block_empty_space(uint8_t *buffer, uint32_t pos);
     uint32_t get_meta_block_used_space(uint32_t block_num);
 
     // get space usage statistics
@@ -363,14 +342,17 @@ public:
     uint64_t get_meta_total_space();
     uint64_t get_meta_used_space();
     uint32_t get_meta_nearfull_blocks();
-    uint32_t get_inflight_queue_size();
     uint32_t get_compact_queue_size();
     uint32_t get_to_compact_count();
+    uint64_t get_compacted_count();
 
-    // get maximum size for a temporary heap_write_t buffer
-    uint32_t get_max_write_entry_size();
-
-    // only for tests
-    void set_abort_on_corruption(bool fail);
-    void set_abort_on_overlap(bool fail);
+    uint64_t entry_pos(uint32_t block_num, uint32_t offset);
+    heap_entry_t *entry_from_pos(uint64_t entry_pos, bool allow_unallocated = false);
+    heap_entry_t *prev(heap_entry_t *wr);
+    uint32_t get_simple_entry_size();
+    uint32_t get_big_entry_size();
+    uint32_t get_big_intent_entry_size();
+    uint32_t get_small_entry_size(uint32_t offset, uint32_t len);
+    uint32_t get_csum_size(heap_entry_t *wr);
+    uint32_t get_csum_size(uint32_t entry_type, uint32_t offset = 0, uint32_t len = 0);
 };

src/blockstore/blockstore_impl.cpp

@@ -23,26 +23,22 @@ blockstore_impl_t::blockstore_impl_t(blockstore_config_t & config, ring_loop_i *
         dsk.open_meta();
         dsk.open_journal();
         dsk.calc_lengths();
-        zero_object = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.data_block_size);
+        dsk.check_lengths();
     }
     catch (std::exception & e)
     {
         dsk.close_all();
         throw;
     }
-    memset(zero_object, 0, dsk.data_block_size);
     meta_superblock = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size);
     memset(meta_superblock, 0, dsk.meta_block_size);
-}
-
-void blockstore_impl_t::init()
-{
     flusher = new journal_flusher_t(this);
     if (dsk.inmemory_journal)
     {
         buffer_area = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.journal_len);
     }
     heap = new blockstore_heap_t(&dsk, buffer_area, log_level);
+    ringloop->wakeup();
 }
 
 blockstore_impl_t::~blockstore_impl_t()
@@ -55,8 +51,6 @@ blockstore_impl_t::~blockstore_impl_t()
         free(buffer_area);
     if (meta_superblock)
         free(meta_superblock);
-    if (zero_object)
-        free(zero_object);
     ringloop->unregister_consumer(&ring_consumer);
     dsk.close_all();
 }
@@ -192,11 +186,19 @@ void blockstore_impl_t::loop()
         {
             flusher->loop();
         }
+        for (auto & block_num: pending_modified_blocks)
+        {
+            auto & mb = modified_blocks[block_num];
+            heap->get_meta_block(block_num, mb.buf);
+            heap->start_block_write(block_num);
+            mb.sent = true;
+        }
         int ret = ringloop->submit();
         if (ret < 0)
         {
             throw std::runtime_error(std::string("io_uring_submit: ") + strerror(-ret));
         }
+        pending_modified_blocks.clear();
         if ((initial_ring_space - ringloop->space_left()) > 0)
         {
             live = true;
@@ -214,7 +216,7 @@ bool blockstore_impl_t::is_safe_to_stop()
     {
         return false;
     }
-    if (unsynced_big_write_count > 0 || unsynced_small_write_count > 0)
+    if (has_unsynced())
     {
         if (!readonly && !stop_sync_submitted)
         {
@@ -250,7 +252,7 @@ void blockstore_impl_t::check_wait(blockstore_op_t *op)
     }
     else if (PRIV(op)->wait_for == WAIT_COMPACTION)
     {
-        if (flusher->get_compact_counter() <= PRIV(op)->wait_detail)
+        if (heap->get_compacted_count() <= PRIV(op)->wait_detail)
         {
             // do not submit
 #ifdef BLOCKSTORE_DEBUG
@@ -273,7 +275,8 @@ void blockstore_impl_t::enqueue_op(blockstore_op_t *op)
         ((op->opcode == BS_OP_READ || op->opcode == BS_OP_WRITE || op->opcode == BS_OP_WRITE_STABLE) && (
             op->offset >= dsk.data_block_size ||
             op->len > dsk.data_block_size-op->offset ||
-            (op->len % dsk.disk_alignment)
+            (op->offset % dsk.bitmap_granularity) ||
+            (op->len % dsk.bitmap_granularity)
         )) ||
         readonly && op->opcode != BS_OP_READ && op->opcode != BS_OP_LIST)
     {
@@ -320,9 +323,13 @@ void blockstore_impl_t::process_list(blockstore_op_t *op)
         FINISH_OP(op);
         return;
     }
-    // Check if the DB needs resharding
-    // (we don't know about PGs from the beginning, we only create "shards" here)
-    heap->reshard(INODE_POOL(min_inode), pg_count, pg_stripe_size);
+    // Check if the DB is sharded correctly
+    if (!heap->reshard_check(INODE_POOL(min_inode), pg_count, pg_stripe_size))
+    {
+        op->retval = -EAGAIN;
+        FINISH_OP(op);
+        return;
+    }
     obj_ver_id *result = NULL;
     size_t stable_count = 0, unstable_count = 0;
     int res = heap->list_objects(list_pg, op->min_oid, op->max_oid, &result, &stable_count, &unstable_count);
@@ -345,6 +352,7 @@ void blockstore_impl_t::process_list(blockstore_op_t *op)
 
 void blockstore_impl_t::set_no_inode_stats(const std::vector<uint64_t> & pool_ids)
 {
+    heap->set_no_inode_stats(pool_ids);
 }
 
 void blockstore_impl_t::dump_diagnostics()
@@ -388,3 +396,18 @@ std::string blockstore_impl_t::get_op_diag(blockstore_op_t *op)
         snprintf(buf, sizeof(buf), "state=%d", priv->op_state);
     return std::string(buf);
 }
+
+void* blockstore_impl_t::reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit)
+{
+    return heap->reshard_start(pool, pg_count, pg_stripe_size, chunk_limit);
+}
+
+bool blockstore_impl_t::reshard_continue(void *reshard_state, uint64_t chunk_limit)
+{
+    return heap->reshard_continue(reshard_state, chunk_limit);
+}
+
+void blockstore_impl_t::reshard_abort(void *reshard_state)
+{
+    return heap->reshard_abort(reshard_state);
+}

src/blockstore/blockstore_impl.h

@@ -29,18 +29,6 @@ class blockstore_impl_t;
 
 //#define BLOCKSTORE_DEBUG
 
-// - Sync must be submitted after previous writes/deletes (not before!)
-// - Reads may be submitted in parallel with writes/deletes because we use MVCC
-// - Writes may be submitted in any order, because they don't overlap. Each write
-//   goes into a new location - either on the journal device or on the data device
-// - Stable (stabilize) must be submitted after sync of that object is completed
-//   It's even OK to return an error to the caller if that object is not synced yet
-// - compacted_lsn should be moved forward only after all versions are moved to the main storage
-// - If an operation can not be submitted because the ring is full
-//   we should stop submission of other operations. Otherwise some "scatter" reads
-//   may end up blocked for a long time.
-// Otherwise, the submission order is free.
-
 #include "blockstore_init.h"
 
 #include "blockstore_flush.h"
@@ -53,27 +41,32 @@ struct blockstore_op_private_t
     int pending_ops;
     int op_state;
 
-    // Read, write, sync, stabilize
-    uint64_t lsn;
+    // Write, sync, stabilize
+    uint32_t modified_block, modified_block2;
 
     // Read
     std::vector<copy_buffer_t> read_vec;
 
+    // Read, write
+    uint64_t lsn;
+
     // Write
     uint64_t location;
-    bool is_big;
+    uint32_t write_type;
 
     // Stabilize, rollback
     int stab_pos;
 
-    // Stabilize
-    uint64_t to_lsn;
-
     // Write
-    struct iovec iov_zerofill[3];
     timespec tv_begin;
 };
 
+struct bs_modified_block_t
+{
+    bool sent;
+    uint8_t *buf;
+};
+
 class blockstore_impl_t: public blockstore_i
 {
 public:
@@ -115,13 +108,16 @@ public:
     uint8_t* meta_superblock = NULL;
     uint8_t *buffer_area = NULL;
     std::vector<blockstore_op_t*> submit_queue;
-    int unsynced_big_write_count = 0, unsynced_small_write_count = 0, unsynced_meta_write_count = 0;
+    int unsynced_data_write_count = 0, unsynced_buffer_write_count = 0, unsynced_meta_write_count = 0;
     int unsynced_queued_ops = 0;
-    uint8_t *zero_object = NULL;
+
+    std::vector<uint32_t> pending_modified_blocks;
+    robin_hood::unordered_flat_map<uint32_t, bs_modified_block_t> modified_blocks;
 
     journal_flusher_t *flusher;
     int write_iodepth = 0;
     int inflight_big = 0;
+    int intent_write_counter = 0;
     bool fsyncing_data = false;
 
     bool live = false, queue_stall = false;
@@ -146,18 +142,17 @@ public:
     int metadata_buf_size;
     blockstore_init_meta* metadata_init_reader;
 
-    void init();
     void check_wait(blockstore_op_t *op);
     void init_op(blockstore_op_t *op);
 
     // Read
     int dequeue_read(blockstore_op_t *op);
     int fulfill_read(blockstore_op_t *op);
-    uint32_t prepare_read(std::vector<copy_buffer_t> & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end);
-    uint32_t prepare_read_with_bitmaps(std::vector<copy_buffer_t> & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end);
+    uint32_t prepare_read(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end, uint32_t skip_csum);
+    uint32_t prepare_read_with_bitmaps(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end, uint32_t skip_csum);
     uint32_t prepare_read_zero(std::vector<copy_buffer_t> & read_vec, uint32_t start, uint32_t end);
-    uint32_t prepare_read_simple(std::vector<copy_buffer_t> & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end);
-    void prepare_disk_read(std::vector<copy_buffer_t> & read_vec, int pos, heap_object_t *obj, heap_write_t *wr,
+    uint32_t prepare_read_simple(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end, uint32_t skip_csum);
+    void prepare_disk_read(std::vector<copy_buffer_t> & read_vec, int pos, heap_entry_t *obj, heap_entry_t *wr,
         uint32_t blk_start, uint32_t blk_end, uint32_t start, uint32_t end, uint32_t copy_flags);
     void find_holes(std::vector<copy_buffer_t> & read_vec, uint32_t item_start, uint32_t item_end,
         std::function<void(int&, uint32_t, uint32_t)> callback);
@@ -167,9 +162,10 @@ public:
 
     // Write
     bool enqueue_write(blockstore_op_t *op);
-    void prepare_meta_block_write(blockstore_op_t *op, uint64_t modified_block, io_uring_sqe *sqe = NULL);
+    void prepare_meta_block_write(uint32_t modified_block);
+    bool meta_block_is_pending(uint32_t modified_block);
+    bool intent_write_allowed(blockstore_op_t *op, heap_entry_t *obj);
     int dequeue_write(blockstore_op_t *op);
-    int make_big_write(blockstore_op_t *op, uint32_t offset, uint32_t len, uint32_t *modified_block, uint32_t *moved_from_block);
     int continue_write(blockstore_op_t *op);
     void handle_write_event(ring_data_t *data, blockstore_op_t *op);
 
@@ -177,6 +173,7 @@ public:
     int continue_sync(blockstore_op_t *op);
     bool submit_fsyncs(int & wait_count);
     int do_sync(blockstore_op_t *op, int base_state);
+    bool has_unsynced();
 
     // Stabilize
     int dequeue_stable(blockstore_op_t *op);
@@ -192,6 +189,10 @@ public:
     void parse_config(blockstore_config_t & config);
     void parse_config(blockstore_config_t & config, bool init);
 
+    void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
+    bool reshard_continue(void *reshard_state, uint64_t chunk_limit);
+    void reshard_abort(void *reshard_state);
+
     // Event loop
     void loop();
 
@@ -226,6 +227,6 @@ public:
     inline uint32_t get_block_size() { return dsk.data_block_size; }
     inline uint64_t get_block_count() { return dsk.block_count; }
     uint64_t get_free_block_count();
-    inline uint32_t get_bitmap_granularity() { return dsk.disk_alignment; }
+    inline uint32_t get_bitmap_granularity() { return dsk.bitmap_granularity; }
     inline uint64_t get_journal_size() { return dsk.journal_len; }
 };

src/blockstore/blockstore_init.cpp

@@ -51,6 +51,8 @@ int blockstore_init_meta::loop()
     else if (wait_state == 5) goto resume_5;
     else if (wait_state == 6) goto resume_6;
     else if (wait_state == 7) goto resume_7;
+    else if (wait_state == 8) goto resume_8;
+    else if (wait_state == 9) goto resume_9;
     metadata_buffer = memalign(MEM_ALIGNMENT, 2*bs->metadata_buf_size);
     if (!metadata_buffer)
         throw std::runtime_error("Failed to allocate metadata read buffer");
@@ -70,7 +72,6 @@ resume_1:
     }
     if (is_zero((uint64_t*)bs->meta_superblock, bs->dsk.meta_block_size))
     {
-        bs->dsk.check_lengths();
         {
             blockstore_meta_header_v3_t *hdr = (blockstore_meta_header_v3_t *)bs->meta_superblock;
             hdr->zero = 0;
@@ -79,12 +80,15 @@ resume_1:
             hdr->meta_block_size = bs->dsk.meta_block_size;
             hdr->data_block_size = bs->dsk.data_block_size;
             hdr->bitmap_granularity = bs->dsk.bitmap_granularity;
-            hdr->compacted_lsn = 0;
             if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_V2)
             {
                 hdr->data_csum_type = bs->dsk.data_csum_type;
                 hdr->csum_block_size = bs->dsk.csum_block_size;
             }
+            if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_HEAP)
+            {
+                hdr->meta_area_size = bs->dsk.meta_area_size;
+            }
             hdr->set_crc32c();
         }
         if (bs->readonly)
@@ -135,28 +139,22 @@ resume_1:
             hdr->data_block_size != bs->dsk.data_block_size ||
             hdr->bitmap_granularity != bs->dsk.bitmap_granularity ||
             hdr->data_csum_type != bs->dsk.data_csum_type ||
-            hdr->csum_block_size != bs->dsk.csum_block_size)
+            hdr->csum_block_size != bs->dsk.csum_block_size ||
+            hdr->meta_area_size != bs->dsk.meta_area_size)
         {
             printf(
                 "Configuration stored in metadata superblock"
-                " (meta_block_size=%u, data_block_size=%u, bitmap_granularity=%u, data_csum_type=%u, csum_block_size=%u)"
-                " differs from OSD configuration (%u/%u/%u, %u/%u).\n",
+                " (meta_block_size=%u, data_block_size=%u, bitmap_granularity=%u, data_csum_type=%u, csum_block_size=%u, meta_area_size=%ju)"
+                " differs from OSD configuration (%u/%u/%u, %u/%u, %ju).\n",
                 hdr->meta_block_size, hdr->data_block_size, hdr->bitmap_granularity,
-                hdr->data_csum_type, hdr->csum_block_size,
+                hdr->data_csum_type, hdr->csum_block_size, hdr->meta_area_size,
                 bs->dsk.meta_block_size, bs->dsk.data_block_size, bs->dsk.bitmap_granularity,
-                bs->dsk.data_csum_type, bs->dsk.csum_block_size
+                bs->dsk.data_csum_type, bs->dsk.csum_block_size, bs->dsk.meta_area_size
             );
             exit(1);
         }
-        if (bs->dsk.meta_format != hdr->version)
-        {
-            bs->dsk.meta_format = hdr->version;
-            bs->dsk.calc_lengths();
-        }
-        bs->dsk.check_lengths();
     }
-    bs->init();
-    bs->heap->set_compacted_lsn(((blockstore_meta_header_v3_t *)bs->meta_superblock)->compacted_lsn);
+    bs->heap->start_load(((blockstore_meta_header_v3_t *)bs->meta_superblock)->completed_lsn);
     if (bs->dsk.inmemory_journal)
     {
         // Read buffer area
@@ -210,8 +208,10 @@ resume_4:
                     io_uring_prep_readv(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bufs[i].offset);
                 else
                 {
-                    // Fill metadata with zeroes
-                    memset(data->iov.iov_base, 0, data->iov.iov_len);
+                    // Fill metadata with empty block pattern
+                    memset(bufs[i].buf, 0, bufs[i].size);
+                    for (uint64_t o = 0; o < bufs[i].size; o += bs->dsk.meta_block_size)
+                        bs->heap->fill_block_empty_space(bufs[i].buf + o, 0);
                     io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bufs[i].offset);
                 }
                 bs->ringloop->submit();
@@ -224,7 +224,11 @@ resume_4:
         if (bufs[i].state == INIT_META_READ_DONE)
         {
             // Handle result
-            entries_loaded += bs->heap->load_blocks(bufs[i].offset-bs->dsk.meta_block_size, bufs[i].size, bufs[i].buf);
+            uint64_t loaded = 0;
+            int r = bs->heap->load_blocks(bufs[i].offset-bs->dsk.meta_block_size, bufs[i].size, bufs[i].buf, false, loaded);
+            if (r != 0)
+                exit(1);
+            entries_loaded += loaded;
             bufs[i].state = 0;
             bs->ringloop->wakeup();
         }
@@ -265,7 +269,7 @@ resume_6:
         }
         GET_SQE();
         data->iov = (iovec){ buf, len };
-        data->callback = [this, offset, cb](ring_data_t *data)
+        data->callback = [offset, cb](ring_data_t *data)
         {
             if (data->res < 0)
             {
@@ -280,7 +284,51 @@ resume_6:
     }, bs->meta_write_recheck_parallelism);
     return 1;
 resume_7:
-    bs->heap->finish_load();
+    if (bs->heap->finish_load() != 0)
+    {
+        exit(1);
+    }
+    recheck_mod = bs->heap->get_recheck_modified_blocks();
+    if (bs->readonly)
+    {
+        recheck_mod.clear();
+    }
+    for (i = 0; i < recheck_mod.size(); i++)
+    {
+resume_8:
+        if (wait_count >= bs->meta_write_recheck_parallelism || !(sqe = bs->get_sqe()))
+        {
+            bs->ringloop->submit();
+            wait_state = 8;
+            return 1;
+        }
+        uint32_t block_num = recheck_mod[i];
+        uint64_t block_offset = bs->dsk.meta_offset + (uint64_t)(block_num+1) * bs->dsk.meta_block_size;
+        data = ((ring_data_t*)sqe->user_data);
+        uint8_t *buf = (uint8_t*)malloc_or_die(bs->dsk.meta_block_size);
+        bs->heap->get_meta_block(block_num, buf);
+        data->iov = { buf, bs->dsk.meta_block_size };
+        data->callback = [this, buf, block_offset](ring_data_t *data)
+        {
+            wait_count--;
+            free(buf);
+            if (data->res != bs->dsk.meta_block_size)
+            {
+                throw std::runtime_error(
+                    "write metadata failed at offset " + std::to_string(block_offset) + ": " + strerror(-data->res)
+                );
+            }
+        };
+        io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, block_offset);
+        wait_count++;
+    }
+resume_9:
+    if (wait_count > 0)
+    {
+        bs->ringloop->submit();
+        wait_state = 9;
+        return 1;
+    }
     free(metadata_buffer);
     metadata_buffer = NULL;
     return 0;

src/blockstore/blockstore_init.h

@@ -15,6 +15,7 @@ class blockstore_init_meta
 {
     blockstore_impl_t *bs;
     int wait_state = 0;
+    int wait_count = 0;
     bool zero_on_init = false;
     void *metadata_buffer = NULL;
     blockstore_init_meta_buf bufs[2] = {};
@@ -25,6 +26,7 @@ class blockstore_init_meta
     uint64_t next_offset = 0;
     uint64_t last_read_offset = 0;
     uint64_t entries_loaded = 0;
+    std::vector<uint32_t> recheck_mod;
     int i = 0, j = 0;
     bool handle_meta_block(uint8_t *buf, uint64_t count, uint64_t done_cnt);
     void handle_event(ring_data_t *data, int buf_num);

src/blockstore/blockstore_internal.h

@@ -1,3 +1,6 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 (see README.md for details)
+
 #pragma once
 
 #define BS_SUBMIT_CHECK_SQES(n) \
@@ -48,3 +51,11 @@
 #define COPY_BUF_COALESCED  0x10
 #define COPY_BUF_PADDED     0x20
 #define COPY_BUF_SKIP_CSUM  0x40
+
+#ifndef RWF_ATOMIC
+#define RWF_ATOMIC 0x40
+#endif
+
+#ifndef RWF_DSYNC
+#define RWF_DSYNC 0x02
+#endif

src/blockstore/blockstore_open.cpp

@@ -42,7 +42,7 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init)
     }
     if (!journal_trim_interval)
     {
-        journal_trim_interval = 1024;
+        journal_trim_interval = 4096;
     }
     if (!flusher_start_threshold)
     {
@@ -96,6 +96,10 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init)
     {
         metadata_buf_size = 4*1024*1024;
     }
+    if (metadata_buf_size % dsk.meta_block_size)
+    {
+        throw std::runtime_error("metadata_buf_size should be a multiple of meta_block_size");
+    }
     if (!meta_write_recheck_parallelism)
     {
         meta_write_recheck_parallelism = 16;

src/blockstore/blockstore_read.cpp

@@ -7,7 +7,7 @@
 
 int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
 {
-    heap_object_t *obj = heap->lock_and_read_entry(op->oid, PRIV(op)->lsn);
+    heap_entry_t *obj = heap->lock_and_read_entry(op->oid);
     if (!obj)
     {
         op->version = 0;
@@ -17,16 +17,44 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
     }
     uint32_t fulfilled = 0;
     PRIV(op)->pending_ops = 0;
+    PRIV(op)->lsn = 0;
     auto & rv = PRIV(op)->read_vec;
     uint64_t result_version = 0;
     bool found = false;
-    for (auto wr = obj->get_writes(); wr; wr = wr->next())
+    uint32_t skip_csum = 0;
+    uint32_t blk_start = op->offset, blk_end = op->offset+op->len;
+    bool need_skip = dsk.csum_block_size > dsk.bitmap_granularity && !perfect_csum_update;
+    if (need_skip)
     {
-        if (op->version < wr->version)
+        PRIV(op)->lsn = obj->lsn;
+        blk_start = op->offset - op->offset%dsk.csum_block_size;
+        blk_end = op->offset + op->len;
+        if (blk_end % dsk.csum_block_size)
+            blk_end += dsk.csum_block_size - (blk_end % dsk.csum_block_size);
+    }
+    bool need_wait = false;
+    heap->iterate_with_stable(obj, obj->lsn, [&](heap_entry_t *wr, bool stable)
+    {
+        if (wr->type() == BS_HEAP_DELETE)
         {
-            continue;
+            return false;
         }
-        if (!found)
+        if (!heap->is_lsn_completed(wr->lsn))
+        {
+            if (wr->type() == BS_HEAP_BIG_INTENT && wr->big_intent().offset < blk_end && wr->big_intent().offset+wr->big_intent().len > blk_start ||
+                wr->type() == BS_HEAP_INTENT_WRITE && wr->small().offset < blk_end && wr->small().offset+wr->small().len > blk_start)
+            {
+                // Wait until intent write is completed
+                need_wait = true;
+                return false;
+            }
+            else if (wr->type() == BS_HEAP_SMALL_WRITE && wr->small().offset < blk_end && wr->small().offset+wr->small().len > blk_start)
+            {
+                // Skip entry and read the previous one
+                return true;
+            }
+        }
+        if (op->version >= wr->version && !found)
         {
             found = true;
             result_version = wr->version;
@@ -35,18 +63,39 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
                 memcpy(op->bitmap, wr->get_ext_bitmap(heap), dsk.clean_entry_bitmap_size);
             }
         }
-        fulfilled += prepare_read(PRIV(op)->read_vec, obj, wr, op->offset, op->offset+op->len);
-        if (fulfilled == op->len ||
-            wr->type() == BS_HEAP_BIG_WRITE ||
-            wr->type() == BS_HEAP_TOMBSTONE)
+        if (op->version >= wr->version)
         {
-            break;
+            fulfilled += prepare_read(PRIV(op)->read_vec, obj, wr, op->offset, op->offset+op->len,
+                wr->type() != BS_HEAP_SMALL_WRITE ? skip_csum : 0);
+            if (fulfilled == op->len ||
+                wr->type() == BS_HEAP_BIG_WRITE ||
+                wr->type() == BS_HEAP_BIG_INTENT ||
+                wr->type() == BS_HEAP_DELETE)
+            {
+                return false;
+            }
         }
+        if (need_skip && wr->type() == BS_HEAP_SMALL_WRITE &&
+            wr->small().offset < blk_end && wr->small().offset+wr->small().len > blk_start)
+        {
+            // Small write may mutate big write checksums during flush
+            skip_csum = COPY_BUF_SKIP_CSUM;
+        }
+        return true;
+    });
+    if (need_wait)
+    {
+undo_wait:
+        // Need to wait. undo added requests, unlock lsn
+        heap->unlock_entry(op->oid);
+        free_read_buffers(rv);
+        rv.clear();
+        return 0;
     }
     if (!found)
     {
         // May happen if there are entries but all of them are > requested version
-        heap->unlock_entry(op->oid, PRIV(op)->lsn);
+        heap->unlock_entry(op->oid);
         op->version = 0;
         op->retval = -ENOENT;
         FINISH_OP(op);
@@ -55,17 +104,13 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
     assert(fulfilled == op->len);
     if (!fulfill_read(op))
     {
-        // Need to wait. undo added requests, unlock lsn
-        heap->unlock_entry(op->oid, PRIV(op)->lsn);
-        free_read_buffers(rv);
-        rv.clear();
-        return 0;
+        goto undo_wait;
     }
     op->version = result_version;
     if (!PRIV(op)->pending_ops)
     {
         // everything is fulfilled from memory
-        heap->unlock_entry(op->oid, PRIV(op)->lsn);
+        heap->unlock_entry(op->oid);
         op->retval = op->len;
         free_read_buffers(rv);
         FINISH_OP(op);
@@ -108,20 +153,20 @@ int blockstore_impl_t::fulfill_read(blockstore_op_t *op)
     return 1;
 }
 
-uint32_t blockstore_impl_t::prepare_read(std::vector<copy_buffer_t> & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end)
+uint32_t blockstore_impl_t::prepare_read(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end, uint32_t skip_csum)
 {
-    if (wr->type() == BS_HEAP_BIG_WRITE)
+    if (wr->type() == BS_HEAP_BIG_WRITE || wr->type() == BS_HEAP_BIG_INTENT)
     {
-        return prepare_read_with_bitmaps(read_vec, obj, wr, start, end);
+        return prepare_read_with_bitmaps(read_vec, obj, wr, start, end, skip_csum);
     }
-    if (wr->type() == BS_HEAP_TOMBSTONE)
+    if (wr->type() == BS_HEAP_DELETE)
     {
         return prepare_read_zero(read_vec, start, end);
     }
-    return prepare_read_simple(read_vec, obj, wr, start, end);
+    return prepare_read_simple(read_vec, obj, wr, start, end, skip_csum);
 }
 
-uint32_t blockstore_impl_t::prepare_read_with_bitmaps(std::vector<copy_buffer_t> & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end)
+uint32_t blockstore_impl_t::prepare_read_with_bitmaps(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end, uint32_t skip_csum)
 {
     // BIG_WRITEs contain a bitmap and we have to handle its holes
     uint32_t res = 0;
@@ -144,7 +189,7 @@ uint32_t blockstore_impl_t::prepare_read_with_bitmaps(std::vector<copy_buffer_t>
         }
         if (bmp_end > bmp_start)
         {
-            res += prepare_read_simple(read_vec, obj, wr, bmp_start * dsk.bitmap_granularity, bmp_end * dsk.bitmap_granularity);
+            res += prepare_read_simple(read_vec, obj, wr, bmp_start * dsk.bitmap_granularity, bmp_end * dsk.bitmap_granularity, skip_csum);
             bmp_start = bmp_end;
         }
     }
@@ -166,7 +211,7 @@ uint32_t blockstore_impl_t::prepare_read_zero(std::vector<copy_buffer_t> & read_
     return res;
 }
 
-uint32_t blockstore_impl_t::prepare_read_simple(std::vector<copy_buffer_t> & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end)
+uint32_t blockstore_impl_t::prepare_read_simple(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end, uint32_t skip_csum)
 {
     uint32_t res = 0;
     if (wr->type() == BS_HEAP_SMALL_WRITE || wr->type() == BS_HEAP_INTENT_WRITE)
@@ -190,7 +235,7 @@ uint32_t blockstore_impl_t::prepare_read_simple(std::vector<copy_buffer_t> & rea
                 .disk_offset = start,
                 .disk_len = end-start,
                 .buf = buffer_area + wr->small().location + start - wr->small().offset,
-                .wr_lsn = wr->lsn,
+                .wr = wr,
             });
         }
         else if (dsk.csum_block_size <= dsk.bitmap_granularity)
@@ -209,14 +254,6 @@ uint32_t blockstore_impl_t::prepare_read_simple(std::vector<copy_buffer_t> & rea
                 blk_start = blk_start < wr->small().offset ? wr->small().offset : blk_start;
                 blk_end = blk_end > wr->small().offset+wr->small().len ? wr->small().offset+wr->small().len : blk_end;
             }
-            uint32_t skip_csum = 0;
-            if (!perfect_csum_update && wr->type() == BS_HEAP_BIG_WRITE)
-            {
-                for (auto owr = obj->get_writes(); owr && owr != wr; owr = owr->next())
-                    if ((owr->type() == BS_HEAP_INTENT_WRITE || owr->type() == BS_HEAP_SMALL_WRITE) &&
-                        owr->small().offset < blk_end && owr->small().offset+owr->small().len > blk_start)
-                        skip_csum = COPY_BUF_SKIP_CSUM;
-            }
             if ((blk_end-1)/dsk.csum_block_size == blk_start/dsk.csum_block_size ||
                 blk_end/dsk.csum_block_size == blk_start/dsk.csum_block_size+1 && blk_end != end && blk_start != start ||
                 blk_end == end && blk_start == start)
@@ -243,21 +280,37 @@ uint32_t blockstore_impl_t::prepare_read_simple(std::vector<copy_buffer_t> & rea
     return res;
 }
 
-void blockstore_impl_t::prepare_disk_read(std::vector<copy_buffer_t> & read_vec, int pos, heap_object_t *obj, heap_write_t *wr,
+void blockstore_impl_t::prepare_disk_read(std::vector<copy_buffer_t> & read_vec, int pos, heap_entry_t *obj, heap_entry_t *wr,
     uint32_t blk_start, uint32_t blk_end, uint32_t start, uint32_t end, uint32_t copy_flags)
 {
-    // Only one INTENT_WRITE is allowed at a time
-    assert(wr->type() != BS_HEAP_INTENT_WRITE || wr->next()->type() == BS_HEAP_BIG_WRITE);
+    uint64_t loc = 0;
+    if (wr->type() == BS_HEAP_INTENT_WRITE)
+    {
+        heap_entry_t *big_wr = wr;
+        while (big_wr && big_wr->type() == BS_HEAP_INTENT_WRITE)
+        {
+            big_wr = heap->prev(big_wr);
+        }
+        assert(big_wr->type() == BS_HEAP_BIG_WRITE || big_wr->type() == BS_HEAP_BIG_INTENT);
+        loc = big_wr->big_location(heap);
+    }
+    else if (wr->type() == BS_HEAP_SMALL_WRITE)
+    {
+        loc = wr->small().location-wr->small().offset;
+    }
+    else
+    {
+        assert(wr->type() == BS_HEAP_BIG_WRITE || wr->type() == BS_HEAP_BIG_INTENT);
+        loc = wr->big_location(heap);
+    }
     copy_buffer_t vec = {
         .copy_flags = (wr->type() == BS_HEAP_SMALL_WRITE ? COPY_BUF_JOURNAL : COPY_BUF_DATA) | copy_flags,
         .offset = start,
         .len = end-start,
-        .disk_loc = (wr->type() == BS_HEAP_INTENT_WRITE ? wr->next()->big_location(heap)
-            : (wr->type() == BS_HEAP_SMALL_WRITE ? wr->small().location-wr->small().offset
-            : wr->big_location(heap))),
+        .disk_loc = loc,
         .disk_offset = blk_start,
         .disk_len = blk_end - blk_start,
-        .wr_lsn = wr->lsn,
+        .wr = wr,
     };
     if (blk_start != start || blk_end != end)
     {
@@ -265,7 +318,7 @@ void blockstore_impl_t::prepare_disk_read(std::vector<copy_buffer_t> & read_vec,
         vec.copy_flags |= COPY_BUF_PADDED;
         if (pos > 0 && read_vec.size() >= pos &&
             read_vec[pos-1].copy_flags == vec.copy_flags &&
-            read_vec[pos-1].wr_lsn == vec.wr_lsn &&
+            read_vec[pos-1].wr == vec.wr &&
             read_vec[pos-1].disk_offset <= vec.disk_offset &&
             read_vec[pos-1].disk_offset+read_vec[pos-1].disk_len >= blk_end)
         {
@@ -349,7 +402,7 @@ void blockstore_impl_t::handle_read_event(ring_data_t *data, blockstore_op_t *op
             op->retval = -EDOM;
         else if (op->retval == 0)
             op->retval = op->len;
-        heap->unlock_entry(op->oid, PRIV(op)->lsn);
+        heap->unlock_entry(op->oid);
         free_read_buffers(PRIV(op)->read_vec);
         FINISH_OP(op);
     }
@@ -357,7 +410,15 @@ void blockstore_impl_t::handle_read_event(ring_data_t *data, blockstore_op_t *op
 
 bool blockstore_impl_t::verify_read_checksums(blockstore_op_t *op)
 {
-    heap_object_t *obj = heap->read_locked_entry(op->oid, PRIV(op)->lsn);
+    bool skip_all = false;
+    if (PRIV(op)->lsn)
+    {
+        heap_entry_t *obj = heap->read_entry(op->oid);
+        if (obj->lsn != PRIV(op)->lsn) // check top lsn
+        {
+            skip_all = true;
+        }
+    }
     auto & rv = PRIV(op)->read_vec;
     for (auto & vec: rv)
     {
@@ -365,23 +426,22 @@ bool blockstore_impl_t::verify_read_checksums(blockstore_op_t *op)
             continue;
         if (vec.copy_flags & COPY_BUF_PADDED)
             memcpy(op->buf + vec.offset - op->offset, vec.buf + vec.offset - vec.disk_offset, vec.len);
-        if (vec.copy_flags & (COPY_BUF_COALESCED|COPY_BUF_SKIP_CSUM))
+        if (vec.copy_flags & (COPY_BUF_COALESCED|COPY_BUF_SKIP_CSUM) ||
+            skip_all && vec.wr->type() != BS_HEAP_SMALL_WRITE)
             continue;
-        heap_write_t *wr = obj->get_writes();
-        while (wr && wr->lsn != vec.wr_lsn)
-            wr = wr->next();
-        assert(wr);
         uint8_t *buf = vec.buf ? vec.buf : (op->buf + vec.offset - op->offset);
-        uint32_t *csums = (uint32_t*)(wr->get_checksums(heap)
+        uint32_t *csums = (uint32_t*)(vec.wr->get_checksums(heap)
             + (vec.disk_offset/dsk.csum_block_size)*(dsk.data_csum_type & 0xFF)
-            - ((wr->type() == BS_HEAP_BIG_WRITE) ? 0 : (wr->small().offset/dsk.csum_block_size)*(dsk.data_csum_type & 0xFF)));
-        if (!heap->calc_block_checksums(csums, buf, wr->get_int_bitmap(heap),
+            - ((vec.wr->type() == BS_HEAP_BIG_WRITE || vec.wr->type() == BS_HEAP_BIG_INTENT)
+                ? 0 : (vec.wr->small().offset/dsk.csum_block_size)*(dsk.data_csum_type & 0xFF)));
+        if (!heap->calc_block_checksums(csums, buf, vec.wr->get_int_bitmap(heap),
             vec.disk_offset, vec.disk_offset+vec.disk_len, false, [&](uint32_t mismatch_pos, uint32_t expected_csum, uint32_t real_csum)
             {
                 printf(
-                    "Checksum mismatch in object %jx:%jx v%ju, offset 0x%x in %s area at offset 0x%jx: %08x expected vs %08x actual\n",
+                    "Checksum mismatch in object %jx:%jx v%ju, offset 0x%x in %s area at offset 0x%jx during read %x+%x: %08x expected vs %08x actual\n",
                     op->oid.inode, op->oid.stripe, op->version, mismatch_pos,
                     (vec.copy_flags & COPY_BUF_JOURNAL) ? "buffer" : "data", vec.disk_loc + vec.disk_offset,
+                    op->offset, op->len,
                     expected_csum, real_csum
                 );
             }))
@@ -394,23 +454,29 @@ bool blockstore_impl_t::verify_read_checksums(blockstore_op_t *op)
 
 int blockstore_impl_t::read_bitmap(object_id oid, uint64_t target_version, void *bitmap, uint64_t *result_version)
 {
-    heap_object_t *obj = heap->read_entry(oid, NULL);
+    heap_entry_t *obj = heap->read_entry(oid);
     if (obj)
     {
-        for (auto wr = obj->get_writes(); wr; wr = wr->next())
+        bool found = false;
+        heap->iterate_with_stable(obj, obj->lsn, [&](heap_entry_t *wr, bool stable)
         {
-            if (target_version < wr->version)
+            if (target_version >= wr->version)
             {
-                continue;
-            }
-            if (result_version)
-            {
-                *result_version = wr->version;
-            }
-            if (bitmap)
-            {
-                memcpy(bitmap, wr->get_ext_bitmap(heap), dsk.clean_entry_bitmap_size);
+                found = true;
+                if (result_version)
+                {
+                    *result_version = wr->version;
+                }
+                if (bitmap)
+                {
+                    memcpy(bitmap, wr->get_ext_bitmap(heap), dsk.clean_entry_bitmap_size);
+                }
+                return false;
             }
+            return true;
+        });
+        if (found)
+        {
             return 0;
         }
     }

src/blockstore/blockstore_stable.cpp

@@ -13,73 +13,75 @@ int blockstore_impl_t::dequeue_stable(blockstore_op_t *op)
     else if (priv->op_state == 2) goto resume_2;
     else if (priv->op_state == 3) goto resume_3;
     else if (priv->op_state == 4) goto resume_4;
+    else if (priv->op_state == 5) goto resume_5;
     assert(!priv->op_state);
-    // Modify in-memory state and assign contiguous LSNs
-    priv->stab_pos = 0;
-    priv->lsn = priv->to_lsn = 0;
     op->retval = 0;
-    while (priv->stab_pos < op->len)
+    priv->modified_block = priv->modified_block2 = UINT32_MAX;
+    for (priv->stab_pos = 0; priv->stab_pos < op->len; priv->stab_pos++)
     {
-        uint32_t modified_block = 0;
-        uint64_t new_lsn = 0;
-        uint64_t new_to_lsn = 0;
-        int res = op->opcode == BS_OP_STABLE
-            ? heap->post_stabilize(v[priv->stab_pos].oid, v[priv->stab_pos].version, &modified_block, &new_lsn, &new_to_lsn)
-            : heap->post_rollback(v[priv->stab_pos].oid, v[priv->stab_pos].version, &new_lsn, &modified_block);
-        if (res != 0)
         {
-            assert(res == ENOENT || res == EBUSY);
-            op->retval = -res;
-        }
-        if (new_lsn)
-        {
-            assert(priv->lsn == 0 || priv->to_lsn == new_lsn-1);
-            if (!priv->lsn)
-                priv->lsn = new_lsn;
-            priv->to_lsn = op->opcode == BS_OP_STABLE ? new_to_lsn : new_lsn;
-        }
-        priv->stab_pos++;
-    }
-    // Submit metadata writes
-    priv->stab_pos = 0;
-resume_1:
-    priv->op_state = 1;
-    while (priv->stab_pos < op->len)
-    {
-        uint32_t block_num = 0;
-        heap_object_t *obj = heap->read_entry(v[priv->stab_pos].oid, &block_num);
-        if (obj)
-        {
-            io_uring_sqe *sqe = get_sqe();
-            if (!sqe)
+            auto obj = heap->read_entry(v[priv->stab_pos].oid);
+            if (!obj)
             {
-                if (priv->pending_ops > 0)
-                    return 1;
-                priv->wait_detail = 1;
-                priv->wait_for = WAIT_SQE;
+                op->retval = -ENOENT;
+                FINISH_OP(op);
+                return 2;
+            }
+            int res = op->opcode == BS_OP_STABLE
+                ? heap->add_commit(obj, v[priv->stab_pos].version, &priv->modified_block2)
+                : heap->add_rollback(obj, v[priv->stab_pos].version, &priv->modified_block2);
+            if (res == EBUSY)
+            {
+                op->retval = -EBUSY;
+                FINISH_OP(op);
+                return 2;
+            }
+            if (res == ENOSPC)
+            {
+                if (!heap->get_to_compact_count())
+                {
+                    // no space
+                    op->retval = -ENOSPC;
+                    FINISH_OP(op);
+                    return 2;
+                }
+                if (priv->modified_block2 != UINT32_MAX)
+                {
+                    priv->stab_pos--;
+                    goto resume_1;
+                }
+                priv->wait_for = WAIT_COMPACTION;
+                priv->wait_detail = heap->get_compacted_count();
+                flusher->request_trim();
                 return 0;
             }
-            prepare_meta_block_write(op, block_num, sqe);
+            assert(res == 0);
         }
-        priv->stab_pos++;
-    }
-    if (priv->pending_ops > 0)
-    {
-        priv->op_state = 1;
-        return 1;
-    }
-    // Mark writes as completed to allow compaction
-    for (uint64_t lsn = priv->lsn; lsn <= priv->to_lsn; lsn++)
-    {
-        heap->mark_lsn_completed(lsn);
-    }
-    unsynced_meta_write_count++;
-    // Fsync, just because our semantics imply that commit (stabilize) is immediately fsynced
-    priv->op_state = 2;
+        if (priv->modified_block != UINT32_MAX && priv->modified_block2 != priv->modified_block)
+        {
+resume_1:
+            BS_SUBMIT_CHECK_SQES(1);
+            prepare_meta_block_write(priv->modified_block);
 resume_2:
+            if (meta_block_is_pending(priv->modified_block))
+            {
+                priv->op_state = 2;
+                return 1;
+            }
+        }
+        priv->modified_block = priv->modified_block2;
+        if (priv->stab_pos == op->len-1 && priv->modified_block2 != UINT32_MAX)
+        {
+            priv->modified_block2 = UINT32_MAX;
+            goto resume_1;
+        }
+    }
+    // Fsync, just because our semantics imply that commit (stabilize) is immediately fsynced
+    priv->op_state = 3;
 resume_3:
 resume_4:
-    int res = do_sync(op, 2);
+resume_5:
+    int res = do_sync(op, 3);
     if (res != 2)
     {
         return res;

src/blockstore/blockstore_sync.cpp

@@ -18,11 +18,19 @@ int blockstore_impl_t::continue_sync(blockstore_op_t *op)
     return res;
 }
 
+bool blockstore_impl_t::has_unsynced()
+{
+    bool data = (!dsk.disable_data_fsync && unsynced_data_write_count);
+    bool buffer = (!dsk.disable_journal_fsync && unsynced_buffer_write_count);
+    bool meta = (!dsk.disable_meta_fsync && unsynced_meta_write_count);
+    return data || buffer || meta;
+}
+
 bool blockstore_impl_t::submit_fsyncs(int & wait_count)
 {
-    int n = ((unsynced_small_write_count > 0 || unsynced_big_write_count > 0 || unsynced_meta_write_count > 0) && !dsk.disable_meta_fsync) +
-        (unsynced_small_write_count > 0 && !dsk.disable_journal_fsync && dsk.journal_fd != dsk.meta_fd) +
-        (unsynced_big_write_count > 0 && !dsk.disable_data_fsync && dsk.data_fd != dsk.meta_fd && dsk.data_fd != dsk.journal_fd);
+    int n = (unsynced_meta_write_count > 0 && !dsk.disable_meta_fsync) +
+        (unsynced_buffer_write_count > 0 && !dsk.disable_journal_fsync && dsk.journal_fd != dsk.meta_fd) +
+        (unsynced_data_write_count > 0 && !dsk.disable_data_fsync && dsk.data_fd != dsk.meta_fd && dsk.data_fd != dsk.journal_fd);
     if (ringloop->space_left() < n)
     {
         return false;
@@ -40,7 +48,7 @@ bool blockstore_impl_t::submit_fsyncs(int & wait_count)
         if (!wait_count)
             ringloop->wakeup();
     };
-    if ((unsynced_small_write_count > 0 || unsynced_big_write_count > 0 || unsynced_meta_write_count > 0) && !dsk.disable_meta_fsync)
+    if (unsynced_meta_write_count > 0 && !dsk.disable_meta_fsync)
     {
         // fsync meta
         io_uring_sqe *sqe = get_sqe();
@@ -51,7 +59,7 @@ bool blockstore_impl_t::submit_fsyncs(int & wait_count)
         data->callback = cb;
         wait_count++;
     }
-    if (unsynced_small_write_count > 0 && !dsk.disable_journal_fsync && dsk.meta_fd != dsk.journal_fd)
+    if (unsynced_buffer_write_count > 0 && !dsk.disable_journal_fsync && dsk.meta_fd != dsk.journal_fd)
     {
         // fsync buffer
         io_uring_sqe *sqe = get_sqe();
@@ -62,7 +70,7 @@ bool blockstore_impl_t::submit_fsyncs(int & wait_count)
         data->callback = cb;
         wait_count++;
     }
-    if (unsynced_big_write_count > 0 && !dsk.disable_data_fsync && dsk.data_fd != dsk.meta_fd && dsk.data_fd != dsk.journal_fd)
+    if (unsynced_data_write_count > 0 && !dsk.disable_data_fsync && dsk.data_fd != dsk.meta_fd && dsk.data_fd != dsk.journal_fd)
     {
         // fsync data
         io_uring_sqe *sqe = get_sqe();
@@ -73,8 +81,8 @@ bool blockstore_impl_t::submit_fsyncs(int & wait_count)
         data->callback = cb;
         wait_count++;
     }
-    unsynced_big_write_count = 0;
-    unsynced_small_write_count = 0;
+    unsynced_data_write_count = 0;
+    unsynced_buffer_write_count = 0;
     unsynced_meta_write_count = 0;
     return true;
 }
@@ -90,13 +98,13 @@ int blockstore_impl_t::do_sync(blockstore_op_t *op, int base_state)
         // Wait for flusher-initiated sync
         return 0;
     }
-    if (dsk.disable_journal_fsync && dsk.disable_meta_fsync && dsk.disable_data_fsync || !unsynced_big_write_count && !unsynced_small_write_count)
+    if (!has_unsynced())
     {
         // We can return immediately because sync only syncs previous writes
-        unsynced_big_write_count = unsynced_small_write_count = unsynced_meta_write_count = 0;
+        unsynced_data_write_count = unsynced_buffer_write_count = unsynced_meta_write_count = 0;
         return 2;
     }
-    PRIV(op)->lsn = heap->get_completed_lsn();
+    PRIV(op)->modified_block = heap->get_completed_lsn();
     if (!submit_fsyncs(PRIV(op)->pending_ops))
     {
         PRIV(op)->wait_detail = 1;
@@ -110,6 +118,6 @@ resume_1:
         return 1;
     }
 resume_2:
-    heap->mark_lsn_fsynced(PRIV(op)->lsn);
+    heap->mark_lsn_fsynced(PRIV(op)->modified_block);
     return 2;
 }

src/blockstore/blockstore_write.cpp

@@ -5,29 +5,111 @@
 #include "blockstore_internal.h"
 #include "allocator.h"
 
+#define _REDIRECT_INTENT 0x101
+
 bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
 {
     clock_gettime(CLOCK_REALTIME, &PRIV(op)->tv_begin);
     return true;
 }
 
-void blockstore_impl_t::prepare_meta_block_write(blockstore_op_t *op, uint64_t modified_block, io_uring_sqe *sqe)
+bool blockstore_impl_t::prepare_meta_block_write(uint32_t modified_block)
 {
-    if (!sqe)
+    auto mod_it = modified_blocks.find(modified_block);
+    if (mod_it != modified_blocks.end())
     {
-        sqe = get_sqe();
-        assert(sqe != NULL);
+        return !mod_it->second.sent;
     }
+    io_uring_sqe *sqe = get_sqe();
+    assert(sqe != NULL);
     ring_data_t *data = ((ring_data_t*)sqe->user_data);
-    data->iov = (struct iovec){ heap->get_meta_block(modified_block), (size_t)dsk.meta_block_size };
-    data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
-    PRIV(op)->pending_ops++;
+    uint8_t *buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size);
+    data->iov = (struct iovec){ buf, (size_t)dsk.meta_block_size };
+    data->callback = [this, modified_block, buf](ring_data_t *data)
+    {
+        free(buf);
+        live = true;
+        if (data->res != data->iov.iov_len)
+        {
+            // FIXME: our state becomes corrupted after a write error. maybe do something better than just die
+            disk_error_abort("data write", data->res, data->iov.iov_len);
+        }
+        modified_blocks.erase(modified_block);
+        heap->complete_block_write(modified_block);
+        ringloop->wakeup();
+    };
     io_uring_prep_writev(
         sqe, dsk.meta_fd, &data->iov, 1, dsk.meta_offset + (modified_block+1)*dsk.meta_block_size
     );
+    unsynced_meta_write_count++;
+    pending_modified_blocks.push_back(modified_block);
+    modified_blocks[modified_block] = { .sent = false, .buf = buf };
+    return true;
+}
+
+bool blockstore_impl_t::meta_block_is_pending(uint32_t modified_block)
+{
+    auto mb_it = modified_blocks.find(modified_block);
+    return mb_it != modified_blocks.end();
+}
+
+bool blockstore_impl_t::intent_write_allowed(blockstore_op_t *op, heap_entry_t *obj)
+{
+    // Parallel writes to the same object are forbidden so "one intent at a time" is fulfilled automatically
+    // Intent writes are disabled when metadata fsync is enabled
+    if (!dsk.disable_meta_fsync)
+    {
+        return false;
+    }
+    // Intent writes are only for replication
+    if (op->opcode != BS_OP_WRITE_STABLE)
+    {
+        return false;
+    }
+    // Operation size should be less than or equal to atomic write size
+    if (!op->len || op->len > dsk.atomic_write_size)
+    {
+        return false;
+    }
+    // Intent-writes are disabled if "absolutely correct during compaction" checksum validation algorithm is enabled
+    // We could also do RMW here when perfect_csum_update is enabled, but it's unclear if we need it
+    if (perfect_csum_update && dsk.csum_block_size > dsk.bitmap_granularity &&
+        ((op->offset % dsk.csum_block_size) || (op->len % dsk.csum_block_size)))
+    {
+        return false;
+    }
+    bool ok = true;
+    heap->iterate_with_stable(obj, obj->lsn, [&](heap_entry_t *wr, bool stable)
+    {
+        // Intent writes are not allowed over buffered writes
+        auto t = wr->type();
+        if (t == BS_HEAP_SMALL_WRITE)
+        {
+            ok = false;
+            return false;
+        }
+        // Intent writes are not allowed over unstable writes
+        if (!stable)
+        {
+            ok = false;
+            return false;
+        }
+        // Intent writes are not allowed over unfinished intent writes
+        if ((t == BS_HEAP_INTENT_WRITE || t == BS_HEAP_BIG_INTENT) && wr->lsn > heap->get_fsynced_lsn())
+        {
+            ok = false;
+            return false;
+        }
+        // Intent writes are allowed over BIG_WRITEs even with fsyncs because BIG_WRITE is always counted as fsynced
+        if (t == BS_HEAP_BIG_WRITE || t == BS_HEAP_BIG_INTENT)
+        {
+            return false;
+        }
+        return true;
+    });
+    return ok;
 }
 
-// First step of the write algorithm: dequeue operation and submit initial write(s)
 int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
 {
     if (PRIV(op)->op_state)
@@ -38,40 +120,64 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
     {
         return 0;
     }
-    PRIV(op)->is_big = false;
-    uint32_t modified_block = UINT32_MAX, moved_from_block = UINT32_MAX;
-    heap_object_t *obj = heap->read_entry(op->oid, &modified_block);
+    PRIV(op)->modified_block = UINT32_MAX;
+    PRIV(op)->write_type = 0;
+    heap_entry_t *obj = heap->read_entry(op->oid);
     if (op->opcode == BS_OP_DELETE)
     {
+        return continue_delete(op, 0);
         // Delete
-        if (!obj)
+        if (!obj || obj->type() == BS_HEAP_DELETE)
         {
             // Already deleted
             op->retval = 0;
             FINISH_OP(op);
             return 2;
         }
+        PRIV(op)->write_type = BS_HEAP_DELETE;
         BS_SUBMIT_CHECK_SQES(1);
-        int res = heap->post_delete(modified_block, obj, &PRIV(op)->lsn);
+        int res = heap->add_delete(obj, &PRIV(op)->modified_block);
+        if (res == ENOSPC)
+        {
+            goto enospc;
+        }
         assert(res == 0);
-        prepare_meta_block_write(op, modified_block);
-        PRIV(op)->op_state = 5;
         write_iodepth++;
+resume_1:
+        while (!prepare_meta_block_write(PRIV(op)->modified_block))
+        {
+            PRIV(op)->op_state = 1;
+            return 1;
+        }
+rseume_2:
+        while (meta_block_is_pending(PRIV(op)->modified_block))
+        {
+            PRIV(op)->op_state = 2;
+            return 1;
+        }
+resume_3:
+resume_4:
+        if (!throttle_write(op, 3))
+        {
+            return 1;
+        }
+        write_iodepth--;
+        ack_write(op);
+        return 2;
     }
     // FIXME: Allow to do initial writes as buffered, not redirected
     // FIXME: Allow to do direct writes over holes
-    else if (!obj || obj->get_writes()->type() == BS_HEAP_TOMBSTONE ||
-        op->offset == 0 && op->len == dsk.data_block_size)
+    else if (!obj || obj->type() == BS_HEAP_DELETE || op->offset == 0 && op->len == dsk.data_block_size)
     {
+        return continue_big_write(op, 10);
         // Big (redirect) write
+        PRIV(op)->write_type = dsk.disable_data_fsync || op->opcode != BS_OP_WRITE_STABLE ? BS_HEAP_BIG_WRITE : _REDIRECT_INTENT;
         BS_SUBMIT_CHECK_SQES(1);
-        PRIV(op)->is_big = true;
-        uint32_t tmp_block;
-        uint64_t loc = heap->find_free_data();
-        if (loc == UINT64_MAX ||
-            !obj && heap->get_block_for_new_object(tmp_block) != 0)
+        PRIV(op)->location = heap->find_free_data();
+        if (PRIV(op)->location == UINT64_MAX)
         {
-            if (!heap->get_inflight_queue_size())
+enospc:
+            if (!heap->get_to_compact_count())
             {
                 // no space
                 op->retval = -ENOSPC;
@@ -79,11 +185,12 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
                 return 2;
             }
             PRIV(op)->wait_for = WAIT_COMPACTION;
-            PRIV(op)->wait_detail = flusher->get_compact_counter();
+            PRIV(op)->wait_detail = heap->get_compacted_count();
             flusher->request_trim();
             return 0;
         }
-        PRIV(op)->location = loc;
+        write_iodepth++;
+        uint64_t loc = PRIV(op)->location;
 #ifdef BLOCKSTORE_DEBUG
         printf(
             "Allocate offset %ju for %jx:%jx v%ju\n",
@@ -91,154 +198,170 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
         );
 #endif
         heap->use_data(op->oid.inode, PRIV(op)->location);
-        if (!dsk.disable_data_fsync && dsk.disable_meta_fsync)
-        {
-            // Do big_write as an INTENT to avoid data fsync
-            int res = make_big_write(op, 0, 0, &modified_block, &moved_from_block);
-            assert(res == 0);
-            if (moved_from_block != UINT32_MAX)
-                prepare_meta_block_write(op, moved_from_block);
-            obj = heap->read_entry(op->oid, &modified_block);
-            heap->mark_lsn_completed(PRIV(op)->lsn);
-            goto process_intent;
-        }
         io_uring_sqe *sqe = get_sqe();
         ring_data_t *data = ((ring_data_t*)sqe->user_data);
-        uint64_t stripe_offset = (op->offset % dsk.bitmap_granularity);
-        uint64_t stripe_end = (op->offset + op->len) % dsk.bitmap_granularity;
-        // Zero fill up to dsk.bitmap_granularity
-        int vcnt = 0;
-        if (stripe_offset)
-        {
-            PRIV(op)->iov_zerofill[vcnt++] = (struct iovec){ zero_object, (size_t)stripe_offset };
-        }
-        PRIV(op)->iov_zerofill[vcnt++] = (struct iovec){ op->buf, op->len };
-        if (stripe_end)
-        {
-            stripe_end = dsk.bitmap_granularity - stripe_end;
-            PRIV(op)->iov_zerofill[vcnt++] = (struct iovec){ zero_object, (size_t)stripe_end };
-        }
-        data->iov.iov_len = op->len + stripe_offset + stripe_end; // to check it in the callback
+        data->iov = (struct iovec){ op->buf, op->len };
         data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
-        io_uring_prep_writev(
-            sqe, dsk.data_fd, PRIV(op)->iov_zerofill, vcnt, dsk.data_offset + loc + op->offset - stripe_offset
-        );
+        io_uring_prep_writev(sqe, dsk.data_fd, &data->iov, 1, dsk.data_offset + loc + op->offset);
+        if (PRIV(op)->write_type == BS_HEAP_BIG_WRITE)
+            inflight_big++;
         PRIV(op)->pending_ops++;
-        PRIV(op)->op_state = 1;
-        write_iodepth++;
-        inflight_big++;
-    }
-    // Only one INTENT_WRITE is allowed at a time, but in fact,
-    // parallel writes to the same object are forbidden anyway
-    else if (op->len > 0 && op->len <= dsk.atomic_write_size &&
-        // Intent-writes are disabled if "absolutely correct during compaction" checksum validation algorithm is enabled
-        // We could also do RMW here when perfect_csum_update is enabled, but it's unclear if we need it
-        (!perfect_csum_update || dsk.csum_block_size <= dsk.bitmap_granularity ||
-            !(op->offset % dsk.csum_block_size) &&
-            !(op->len % dsk.csum_block_size) &&
-            (obj->get_writes()->entry_type != (BS_HEAP_INTENT_WRITE|BS_HEAP_STABLE) ||
-            obj->get_writes()->can_be_collapsed(heap))) &&
-        // One intent-write is allowed even with fsyncs because BIG_WRITE is always counted as fsynced
-        dsk.disable_meta_fsync &&
-        (op->opcode == BS_OP_WRITE_STABLE &&
-            (obj->get_writes()->entry_type == (BS_HEAP_BIG_WRITE|BS_HEAP_STABLE) ||
-            obj->get_writes()->entry_type == (BS_HEAP_INTENT_WRITE|BS_HEAP_STABLE) && dsk.disable_data_fsync) ||
-        op->opcode == BS_OP_WRITE && obj->get_writes()->entry_type == BS_HEAP_BIG_WRITE))
-    {
-        // Direct intent-write
-        BS_SUBMIT_CHECK_SQES(1);
-        if (obj->get_writes()->type() == BS_HEAP_BIG_WRITE)
+resume_10:
+        if (PRIV(op)->pending_ops > 0)
         {
-            PRIV(op)->location = obj->get_writes()->big_location(heap);
+            PRIV(op)->op_state = 10;
+            return 1;
+        }
+        if (PRIV(op)->write_type == BS_HEAP_BIG_WRITE)
+        {
+            inflight_big--;
+resume_11:
+resume_12:
+resume_13:
+            if (!fsync_big_write(op, 11))
+                return 1;
+        }
+        heap_entry_t *obj = heap->read_entry(op->oid);
+        int res = 0;
+        if (PRIV(op)->write_type == _REDIRECT_INTENT)
+        {
+            res = heap->add_redirect_intent(op->oid, &obj, op->version, op->offset, op->len,
+                PRIV(op)->location, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
         }
         else
         {
-            assert(obj->get_writes()->next()->type() == BS_HEAP_BIG_WRITE);
-            PRIV(op)->location = obj->get_writes()->next()->big_location(heap);
+            res = heap->add_big_write(op->oid, obj, op->opcode == BS_OP_WRITE_STABLE,
+                op->version, op->offset, op->len, PRIV(op)->location, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
         }
-process_intent:
-        uint8_t wr_buf[heap->get_max_write_entry_size()];
-        heap_write_t *wr = (heap_write_t*)wr_buf;
-        wr->version = op->version;
-        wr->entry_type = BS_HEAP_INTENT_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0);
-        wr->small().offset = op->offset;
-        wr->small().len = op->len;
-        wr->small().location = 0;
-        if (op->bitmap)
-            memcpy(wr->get_ext_bitmap(heap), op->bitmap, dsk.clean_entry_bitmap_size);
-        heap->calc_checksums(wr, (uint8_t*)op->buf, true);
-        int res = heap->post_write(modified_block, op->oid, obj, wr, &moved_from_block);
-        if (res == EAGAIN)
+        if (res == ENOSPC)
         {
-            assert(heap->get_inflight_queue_size());
+            if (!heap->get_to_compact_count())
+            {
+                // no space
+                heap->free_data(op->oid.inode, PRIV(op)->location);
+                write_iodepth--;
+                op->retval = -ENOSPC;
+                FINISH_OP(op);
+                return 2;
+            }
             PRIV(op)->wait_for = WAIT_COMPACTION;
-            PRIV(op)->wait_detail = flusher->get_compact_counter();
+            PRIV(op)->wait_detail = heap->get_compacted_count();
             flusher->request_trim();
             return 0;
         }
-        else if (res == ENOSPC)
-        {
-            // no space
-            op->retval = -ENOSPC;
-            FINISH_OP(op);
-            return 2;
-        }
         assert(res == 0);
-        PRIV(op)->lsn = wr->lsn;
-        if (moved_from_block != UINT32_MAX)
-            prepare_meta_block_write(op, moved_from_block);
-        prepare_meta_block_write(op, modified_block);
-        PRIV(op)->op_state = 9;
+resume_14:
+        while (!prepare_meta_block_write(PRIV(op)->modified_block))
+        {
+            PRIV(op)->op_state = 14;
+            return 1;
+        }
+resume_15:
+        while (meta_block_is_pending(PRIV(op)->modified_block))
+        {
+            PRIV(op)->op_state = 15;
+            return 1;
+        }
+        write_iodepth--;
+        ack_write(op);
+        return 2;
+    }
+    else if (intent_write_allowed(op, obj))
+    {
+        return continue_intent_write(op, 20);
+        // Direct intent-write
+        BS_SUBMIT_CHECK_SQES(1);
+        int res = 0;
+        if (dsk.csum_block_size <= dsk.bitmap_granularity &&
+            (obj->entry_type == (BS_HEAP_BIG_INTENT|BS_HEAP_STABLE) ||
+            obj->entry_type == (BS_HEAP_BIG_WRITE|BS_HEAP_STABLE)))
+        {
+            // Even more simplified BIG_INTENT writes
+            // FIXME: Support RMW mode for csum_block_size > bitmap_granularity
+            PRIV(op)->write_type = BS_HEAP_BIG_INTENT;
+            PRIV(op)->location = obj->big_location(heap);
+            res = heap->add_big_intent(op->oid, &obj, op->version, op->offset, op->len, op->bitmap,
+                (uint8_t*)op->buf, NULL, &PRIV(op)->modified_block);
+            if (res == ENOSPC)
+                goto enospc;
+            assert(res == 0);
+            PRIV(op)->lsn = obj->lsn;
+        }
+        else
+        {
+            PRIV(op)->write_type = BS_HEAP_INTENT_WRITE;
+            auto wr = obj;
+            while (wr && (wr->type() == BS_HEAP_INTENT_WRITE || wr->type() == BS_HEAP_COMMIT || wr->type() == BS_HEAP_ROLLBACK))
+            {
+                wr = heap->prev(wr);
+            }
+            assert(wr && (wr->type() == BS_HEAP_BIG_WRITE || wr->type() == BS_HEAP_BIG_INTENT));
+            PRIV(op)->location = wr->big_location(heap);
+            res = heap->add_small_write(op->oid, &obj, (BS_HEAP_INTENT_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0)),
+                op->version, op->offset, op->len, 0, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
+            if (res == ENOSPC)
+                goto enospc;
+            assert(res == 0);
+            PRIV(op)->lsn = obj->lsn;
+        }
         write_iodepth++;
+resume_20:
+        while (!prepare_meta_block_write(PRIV(op)->modified_block))
+        {
+            PRIV(op)->op_state = 20;
+            return 1;
+        }
+resume_21:
+        while (meta_block_is_pending(PRIV(op)->modified_block))
+        {
+            PRIV(op)->op_state = 21;
+            return 1;
+        }
+        // Direct intent-write
+        // LSN is not marked as completed so big_write won't be freed
+        BS_SUBMIT_GET_SQE(sqe, data);
+        data->iov = (struct iovec){ op->buf, op->len };
+        data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
+        io_uring_prep_writev(sqe, dsk.data_fd, &data->iov, 1, dsk.data_offset + PRIV(op)->location + op->offset);
+        if (dsk.use_atomic_flag)
+            sqe->rw_flags = RWF_ATOMIC;
+        PRIV(op)->pending_ops++;
+resume_22:
+        if (PRIV(op)->pending_ops > 0)
+        {
+            PRIV(op)->op_state = 22;
+            return 1;
+        }
+        write_iodepth--;
+        ack_write(op);
+        return 2;
     }
     else
     {
+        return continue_small_write(op, 30);
         // Small (buffered) overwrite
         // First check if there is free buffer space
+        PRIV(op)->write_type = BS_HEAP_SMALL_WRITE;
         uint64_t loc = !op->len ? 0 : heap->find_free_buffer_area(op->len);
         if (loc == UINT64_MAX)
         {
             PRIV(op)->wait_for = WAIT_COMPACTION;
-            PRIV(op)->wait_detail = flusher->get_compact_counter();
+            PRIV(op)->wait_detail = heap->get_compacted_count();
             flusher->request_trim();
             return 0;
         }
         // There is sufficient space. Check SQE(s)
-        BS_SUBMIT_CHECK_SQES(1 + (op->len > 0 ? 1 : 0));
-        uint8_t wr_buf[heap->get_max_write_entry_size()];
-        heap_write_t *wr = (heap_write_t*)wr_buf;
-        wr->version = op->version;
-        wr->entry_type = BS_HEAP_SMALL_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0);
-        wr->small().offset = op->offset;
-        wr->small().len = op->len;
-        wr->small().location = loc;
-        PRIV(op)->location = loc;
-        if (op->bitmap)
-            memcpy(wr->get_ext_bitmap(heap), op->bitmap, dsk.clean_entry_bitmap_size);
-        heap->calc_checksums(wr, (uint8_t*)op->buf, true);
-        int res = heap->post_write(modified_block, op->oid, obj, wr, &moved_from_block);
-        if (res == EAGAIN)
-        {
-            assert(heap->get_inflight_queue_size());
-            PRIV(op)->wait_for = WAIT_COMPACTION;
-            PRIV(op)->wait_detail = flusher->get_compact_counter();
-            flusher->request_trim();
-            return 0;
-        }
-        else if (res == ENOSPC)
-        {
-            // no space
-            op->retval = -ENOSPC;
-            FINISH_OP(op);
-            return 2;
-        }
+        BS_SUBMIT_CHECK_SQES(1 + (op->len > 0 ? 1 : 0)); ---> refactor too
+        int res = heap->add_small_write(op->oid, &obj, (BS_HEAP_SMALL_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0)),
+            op->version, op->offset, op->len, loc, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
+        if (res == ENOSPC)
+            goto enospc;
         assert(res == 0);
-        PRIV(op)->lsn = wr->lsn;
+        PRIV(op)->lsn = obj->lsn;
         if (op->len)
             heap->use_buffer_area(op->oid.inode, loc, op->len);
-        if (moved_from_block != UINT32_MAX)
-            prepare_meta_block_write(op, moved_from_block);
-        prepare_meta_block_write(op, modified_block);
+        prepare_meta_block_write(PRIV(op)->modified_block);
+        PRIV(op)->pending_ops++;
         if (op->len > 0)
         {
             // Prepare buffered data write
@@ -263,26 +386,6 @@ process_intent:
     return 1;
 }
 
-int blockstore_impl_t::make_big_write(blockstore_op_t *op, uint32_t offset, uint32_t len, uint32_t *modified_block, uint32_t *moved_from_block)
-{
-    uint8_t wr_buf[heap->get_max_write_entry_size()];
-    heap_write_t *wr = (heap_write_t*)wr_buf;
-    wr->entry_type = BS_HEAP_BIG_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0);
-    wr->version = op->version;
-    wr->set_big_location(heap, PRIV(op)->location);
-    if (op->bitmap)
-        memcpy(wr->get_ext_bitmap(heap), op->bitmap, dsk.clean_entry_bitmap_size);
-    memset(wr->get_int_bitmap(heap), 0, dsk.clean_entry_bitmap_size);
-    bitmap_set(wr->get_int_bitmap(heap), offset, len, dsk.bitmap_granularity);
-    heap->calc_checksums(wr, (uint8_t*)op->buf, true, offset, len);
-    int res = heap->post_write(op->oid, wr, modified_block, moved_from_block);
-    if (res != 0)
-        return res;
-    assert(res == 0);
-    PRIV(op)->lsn = wr->lsn;
-    return 0;
-}
-
 int blockstore_impl_t::continue_write(blockstore_op_t *op)
 {
     int op_state = PRIV(op)->op_state;
@@ -305,34 +408,58 @@ again:
     {
         // In progress
         assert(op_state < 10);
+        if (PRIV(op)->modified_block != UINT32_MAX &&
+            !meta_block_is_pending(PRIV(op)->modified_block))
+        {
+            PRIV(op)->pending_ops--;
+            PRIV(op)->modified_block = UINT32_MAX;
+        }
         if (PRIV(op)->pending_ops > 0)
             return 1;
         op_state++;
         goto again;
     }
 resume_2:
+
+
+resume_4:
+resume_6:
+
+resume_8:
+    ack
+    return 2;
+resume_10:
+    return 1;
+}
+
+bool blockstore_impl_t::fsync_big_write(blockstore_op_t *op, int base_state)
+{
+    if (PRIV(op)->state == base_state)
+        goto resume_0;
+    else if (PRIV(op)->state == base_state+1)
+        goto resume_1;
+    else if (PRIV(op)->state == base_state+2)
+        goto resume_2;
     // We must fsync all big writes to avoid complex write workflows
     // It's OK for all HDDs and for server SSDs, but slightly worse for desktop SSDs
-    // The other way is to add another type of MVCC to blockstore_heap: "forward" MVCC :)
-    inflight_big--;
     if (!dsk.disable_data_fsync)
     {
         // fsync data in a batch
-resume_11:
+resume_0:
         if (inflight_big > 0)
         {
-            PRIV(op)->op_state = 11;
-            return 1;
+            PRIV(op)->op_state = base_state;
+            return false;
         }
         if (fsyncing_data)
         {
-resume_12:
+resume_1:
             if (fsyncing_data)
             {
-                PRIV(op)->op_state = 12;
-                return 1;
+                PRIV(op)->op_state = base_state+1;
+                return false;
             }
-            goto resume_4;
+            return true;
         }
         fsyncing_data = true;
         BS_SUBMIT_GET_SQE(sqe, data);
@@ -344,38 +471,24 @@ resume_12:
             handle_write_event(data, op);
         };
         PRIV(op)->pending_ops++;
-        PRIV(op)->op_state = 3;
-        return 1;
+resume_2:
+        if (PRIV(op)->pending_ops > 0)
+        {
+            PRIV(op)->op_state = base_state+2;
+            return false;
+        }
     }
-resume_4:
+    return true;
+}
+
+bool blockstore_impl_t::throttle_write(blockstore_op_t *op, int base_state)
+{
+    // Apply throttling to not fill the journal too quickly for the SSD+HDD case
+    if (PRIV(op)->op_state >= base_state+1)
     {
-        uint32_t modified_block = UINT32_MAX, moved_from_block = UINT32_MAX;
-        int res = make_big_write(op, op->offset, op->len, &modified_block, &moved_from_block);
-        if (res == EAGAIN)
-        {
-            assert(heap->get_inflight_queue_size());
-            PRIV(op)->wait_for = WAIT_COMPACTION;
-            PRIV(op)->wait_detail = flusher->get_compact_counter();
-            flusher->request_trim();
-            return 1;
-        }
-        else if (res == ENOSPC)
-        {
-            heap->free_data(op->oid.inode, PRIV(op)->location);
-            write_iodepth--;
-            op->retval = -ENOSPC;
-            FINISH_OP(op);
-            return 2;
-        }
-        if (moved_from_block != UINT32_MAX)
-            prepare_meta_block_write(op, moved_from_block);
-        prepare_meta_block_write(op, modified_block);
-        PRIV(op)->op_state = 5;
-        return 1;
+        return true;
     }
-resume_6:
-    // Apply throttling to not fill the journal too fast for the SSD+HDD case
-    if (!PRIV(op)->is_big && throttle_small_writes)
+    if (PRIV(op)->write_type == BS_HEAP_SMALL_WRITE && throttle_small_writes)
     {
         // Apply throttling
         timespec tv_end;
@@ -395,40 +508,48 @@ resume_6:
         if (ref_us > exec_us + throttle_threshold_us)
         {
             // Pause reply
-            PRIV(op)->op_state = 7;
+            PRIV(op)->op_state = base_state;
             // Remember that the timer can in theory be called right here
             tfd->set_timer_us(ref_us-exec_us, false, [this, op](int timer_id)
             {
-                PRIV(op)->op_state = 8;
+                PRIV(op)->op_state++;
                 ringloop->wakeup();
             });
-            return 1;
+            return false;
         }
     }
-resume_8:
+    return true;
+}
+
+void blockstore_impl_t::ack_write(blockstore_op_t *op)
+{
     // Acknowledge write
 #ifdef BLOCKSTORE_DEBUG
     printf("Ack write %jx:%jx v%ju\n", op->oid.inode, op->oid.stripe, op->version);
 #endif
     op->retval = op->len;
-    heap->mark_lsn_completed(PRIV(op)->lsn);
-    if (PRIV(op)->is_big)
-        unsynced_big_write_count++;
-    else
-        unsynced_small_write_count++;
-    write_iodepth--;
+    if (PRIV(op)->write_type == BS_HEAP_BIG_WRITE)
+    {
+        unsynced_data_write_count++;
+    }
+    else if (PRIV(op)->write_type == BS_HEAP_SMALL_WRITE)
+    {
+        unsynced_buffer_write_count++;
+        heap->complete_lsn_write(PRIV(op)->lsn);
+    }
+    else if (PRIV(op)->write_type == BS_HEAP_BIG_INTENT ||
+        PRIV(op)->write_type == BS_HEAP_INTENT_WRITE)
+    {
+        unsynced_data_write_count++;
+        intent_write_counter++;
+        heap->complete_lsn_write(PRIV(op)->lsn);
+    }
+    else if (PRIV(op)->write_type == _REDIRECT_INTENT)
+    {
+        unsynced_data_write_count++;
+        intent_write_counter++;
+    }
     FINISH_OP(op);
-    return 2;
-resume_10:
-    // Direct intent-write
-    // LSN is not marked as completed so big_write won't be freed
-    BS_SUBMIT_GET_SQE(sqe, data);
-    data->iov = (struct iovec){ op->buf, op->len };
-    data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
-    io_uring_prep_writev(sqe, dsk.data_fd, &data->iov, 1, dsk.data_offset + PRIV(op)->location + op->offset);
-    PRIV(op)->pending_ops++;
-    PRIV(op)->op_state = 7;
-    return 1;
 }
 
 void blockstore_impl_t::handle_write_event(ring_data_t *data, blockstore_op_t *op)

src/blockstore/fio_engine.cpp

@@ -28,6 +28,7 @@
 #include "epoll_manager.h"
 #include "malloc_or_die.h"
 #include "json11/json11.hpp"
+#include "../util/robin_hood.h"
 #include "fio_headers.h"
 
 struct bs_data
@@ -37,11 +38,15 @@ struct bs_data
     ring_loop_t *ringloop;
     /* The list of completed io_u structs. */
     std::vector<io_u*> completed;
+    robin_hood::unordered_flat_map<uint64_t, int> inflight_oids;
+    std::vector<io_u*> postponed;
     int op_n = 0, inflight = 0;
     bool ec = false;
     bool imm = true;
     bool last_sync = false;
     bool trace = false;
+    uint8_t *bitmap = NULL;
+    uint32_t block_size = 0;
 };
 
 struct bs_options
@@ -137,6 +142,7 @@ static void bs_cleanup(struct thread_data *td)
         delete bsd->bs;
         delete bsd->epmgr;
         delete bsd->ringloop;
+        free(bsd->bitmap);
         delete bsd;
     }
 }
@@ -160,9 +166,12 @@ static int bs_init(struct thread_data *td)
                 config[p.first] = p.second.dump();
         }
     }
+    bsd->bitmap = (uint8_t*)malloc_or_die(MAX_DATA_BLOCK_SIZE/512/8);
+    memset(bsd->bitmap, 0, MAX_DATA_BLOCK_SIZE/512/8);
     bsd->ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
     bsd->epmgr = new epoll_manager_t(bsd->ringloop);
     bsd->bs = blockstore_i::create(config, bsd->ringloop, bsd->epmgr->tfd);
+    bsd->block_size = bsd->bs->get_block_size();
     bsd->imm = config.find("immediate_commit") == config.end() ||
         config["immediate_commit"] == "all";
     while (1)
@@ -177,8 +186,33 @@ static int bs_init(struct thread_data *td)
     return 0;
 }
 
+static enum fio_q_status _bs_queue(struct thread_data *td, struct io_u *io, bool force);
+
+static void _bs_retry(struct bs_data *bsd, uint64_t offset)
+{
+    // Retry postponed ops
+    auto inflight_it = bsd->inflight_oids.find(offset / bsd->block_size);
+    assert(inflight_it != bsd->inflight_oids.end());
+    inflight_it->second--;
+    if (inflight_it->second > 0)
+    {
+        for (size_t i = 0; i < bsd->postponed.size(); i++)
+        {
+            auto oio = bsd->postponed[i];
+            if (oio->offset/bsd->block_size == offset/bsd->block_size)
+            {
+                bsd->postponed.erase(bsd->postponed.begin()+i);
+                _bs_queue((thread_data*)oio->engine_data, oio, true);
+                break;
+            }
+        }
+    }
+    else
+        bsd->inflight_oids.erase(inflight_it);
+}
+
 /* Begin read or write request. */
-static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
+static enum fio_q_status _bs_queue(struct thread_data *td, struct io_u *io, bool force)
 {
     bs_data *bsd = (bs_data*)td->io_ops_data;
     if (io->ddir == DDIR_SYNC && bsd->last_sync)
@@ -188,10 +222,22 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
 
     fio_ro_check(td, io);
 
-    io->engine_data = bsd;
+    io->engine_data = td;
 
     if (io->ddir == DDIR_WRITE || io->ddir == DDIR_READ)
-        assert(io->xfer_buflen <= bsd->bs->get_block_size());
+        assert(io->xfer_buflen <= bsd->block_size);
+
+    uint64_t stripe = io->offset / bsd->block_size;
+    if (!force && io->ddir == DDIR_WRITE)
+    {
+        auto & inflight = bsd->inflight_oids[stripe];
+        inflight++;
+        if (inflight > 1)
+        {
+            bsd->postponed.push_back(io);
+            return FIO_Q_QUEUED;
+        }
+    }
 
     blockstore_op_t *op = new blockstore_op_t;
     op->callback = NULL;
@@ -203,15 +249,16 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
         op->buf = (uint8_t*)io->xfer_buf;
         op->oid = {
             .inode = 1,
-            .stripe = io->offset / bsd->bs->get_block_size(),
+            .stripe = stripe,
         };
         op->version = UINT64_MAX; // last unstable
-        op->offset = io->offset % bsd->bs->get_block_size();
+        op->offset = io->offset % bsd->block_size;
         op->len = io->xfer_buflen;
+        op->bitmap = bsd->bitmap;
         op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
         {
             io->error = op->retval < 0 ? -op->retval : 0;
-            bs_data *bsd = (bs_data*)io->engine_data;
+            bs_data *bsd = ((bs_data*)((thread_data*)io->engine_data)->io_ops_data);
             bsd->inflight--;
             bsd->completed.push_back(io);
             if (bsd->trace)
@@ -224,16 +271,17 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
         op->buf = (uint8_t*)io->xfer_buf;
         op->oid = {
             .inode = 1,
-            .stripe = io->offset / bsd->bs->get_block_size(),
+            .stripe = stripe,
         };
         op->version = 0; // assign automatically
-        op->offset = io->offset % bsd->bs->get_block_size();
+        op->offset = io->offset % bsd->block_size;
         op->len = io->xfer_buflen;
+        op->bitmap = bsd->bitmap;
         if (bsd->ec)
         {
             op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
             {
-                bs_data *bsd = (bs_data*)io->engine_data;
+                bs_data *bsd = ((bs_data*)((thread_data*)io->engine_data)->io_ops_data);
                 if (bsd->trace)
                     printf("--- OP_WRITE %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
                 if (op->retval < 0)
@@ -241,6 +289,7 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
                     io->error = op->retval < 0 ? -op->retval : 0;
                     bsd->inflight--;
                     bsd->completed.push_back(io);
+                    _bs_retry(bsd, io->offset);
                     delete op;
                 }
                 else
@@ -254,12 +303,13 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
                     stab_op->len = 1;
                     stab_op->callback = [io, n](blockstore_op_t *op)
                     {
-                        bs_data *bsd = (bs_data*)io->engine_data;
+                        bs_data *bsd = ((bs_data*)((thread_data*)io->engine_data)->io_ops_data);
                         if (bsd->trace)
                             printf("--- OP_STABLE %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
                         io->error = op->retval < 0 ? -op->retval : 0;
                         bsd->inflight--;
                         bsd->completed.push_back(io);
+                        _bs_retry(bsd, io->offset);
                         delete op;
                     };
                     bsd->bs->enqueue_op(stab_op);
@@ -271,12 +321,13 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
         {
             op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
             {
-                bs_data *bsd = (bs_data*)io->engine_data;
+                bs_data *bsd = ((bs_data*)((thread_data*)io->engine_data)->io_ops_data);
                 if (bsd->trace)
                     printf("--- OP_WRITE_STABLE %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
                 io->error = op->retval < 0 ? -op->retval : 0;
                 bsd->inflight--;
                 bsd->completed.push_back(io);
+                _bs_retry(bsd, io->offset);
                 delete op;
             };
         }
@@ -286,7 +337,7 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
         op->opcode = BS_OP_SYNC;
         op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
         {
-            bs_data *bsd = (bs_data*)io->engine_data;
+            bs_data *bsd = ((bs_data*)((thread_data*)io->engine_data)->io_ops_data);
             io->error = op->retval < 0 ? -op->retval : 0;
             bsd->completed.push_back(io);
             bsd->inflight--;
@@ -314,6 +365,11 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
     return FIO_Q_QUEUED;
 }
 
+static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
+{
+    return _bs_queue(td, io, false);
+}
+
 static int bs_getevents(struct thread_data *td, unsigned int min, unsigned int max, const struct timespec *t)
 {
     bs_data *bsd = (bs_data*)td->io_ops_data;

src/blockstore/multilist.cpp

@@ -300,6 +300,11 @@ uint32_t multilist_index_t::find(uint32_t wanted_used)
     return heads[wanted_used];
 }
 
+uint32_t multilist_index_t::next(uint32_t pos)
+{
+    return nexts[pos];
+}
+
 void multilist_index_t::change(uint32_t pos, uint32_t old_used, uint32_t new_used)
 {
     if (new_used == old_used)

src/blockstore/multilist.h

@@ -32,6 +32,7 @@ struct multilist_index_t
     // used should be always < max_used
     multilist_index_t(uint32_t count, uint32_t max_used, uint32_t init_used);
     uint32_t find(uint32_t wanted_used);
+    uint32_t next(uint32_t pos);
     void change(uint32_t pos, uint32_t old_used, uint32_t new_used);
     void print();
 };

src/blockstore/ondisk_formats.h

@@ -185,7 +185,8 @@ struct __attribute__((__packed__)) blockstore_meta_header_v3_t
     uint32_t data_csum_type;
     uint32_t csum_block_size;
     uint32_t header_csum;
-    uint64_t compacted_lsn;
+    uint64_t meta_area_size;
+    uint64_t completed_lsn;
 
     void set_crc32c();
 };

src/blockstore/v1/impl.cpp

@@ -407,32 +407,88 @@ blockstore_clean_db_t& blockstore_impl_t::clean_db_shard(object_id oid)
     return clean_db_shards[(pool_id << (64-POOL_ID_BITS)) | pg_num];
 }
 
-void blockstore_impl_t::reshard_clean_db(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size)
+struct bs_reshard_state_t
 {
-    uint64_t pool_id = (uint64_t)pool;
+    int state = 0;
+    uint64_t pool_id = 0;
+    uint32_t pg_count = 0;
+    uint32_t pg_stripe_size = 0;
+    uint64_t chunk_size = 0;
+    std::map<pool_pg_id_t, blockstore_clean_db_t> old_shards;
     std::map<pool_pg_id_t, blockstore_clean_db_t> new_shards;
-    auto sh_it = clean_db_shards.lower_bound((pool_id << (64-POOL_ID_BITS)));
-    while (sh_it != clean_db_shards.end() &&
-        (sh_it->first >> (64-POOL_ID_BITS)) == pool_id)
+    std::map<pool_pg_id_t, blockstore_clean_db_t>::iterator sh_it;
+    blockstore_clean_db_t::iterator obj_it;
+};
+
+void* blockstore_impl_t::reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit)
+{
+    auto & settings = clean_db_settings[pool];
+    if (settings.pg_count == pg_count && settings.pg_stripe_size == pg_stripe_size)
     {
-        for (auto & pair: sh_it->second)
-        {
-            // like map_to_pg()
-            uint64_t pg_num = (pair.first.stripe / pg_stripe_size) % pg_count + 1;
-            uint64_t shard_id = (pool_id << (64-POOL_ID_BITS)) | pg_num;
-            new_shards[shard_id][pair.first] = pair.second;
-        }
+        return NULL;
+    }
+    bs_reshard_state_t *st = new bs_reshard_state_t;
+    st->state = 0;
+    st->pool_id = pool;
+    st->pg_count = pg_count;
+    st->pg_stripe_size = pg_stripe_size;
+    auto sh_it = clean_db_shards.lower_bound((st->pool_id << (64-POOL_ID_BITS)));
+    while (sh_it != clean_db_shards.end() &&
+        (sh_it->first >> (64-POOL_ID_BITS)) == st->pool_id)
+    {
+        st->old_shards[sh_it->first] = std::move(sh_it->second);
         clean_db_shards.erase(sh_it++);
     }
-    for (sh_it = new_shards.begin(); sh_it != new_shards.end(); sh_it++)
+    bool finished = reshard_continue(st, chunk_limit);
+    return finished ? NULL : st;
+}
+
+bool blockstore_impl_t::reshard_continue(void *reshard_state, uint64_t chunk_limit)
+{
+    bs_reshard_state_t *st = (bs_reshard_state_t*)reshard_state;
+    uint64_t chunk_size = 0;
+    if (st->state == 1)
+        goto resume_1;
+    for (st->sh_it = st->old_shards.begin(); st->sh_it != st->old_shards.end(); )
+    {
+        for (st->obj_it = st->sh_it->second.begin(); st->obj_it != st->sh_it->second.end(); st->obj_it++)
+        {
+            if (chunk_limit > 0 && chunk_size >= chunk_limit)
+            {
+                st->state = 1;
+                return false;
+            }
+resume_1:
+            // like map_to_pg()
+            uint64_t pg_num = (st->obj_it->first.stripe / st->pg_stripe_size) % st->pg_count + 1;
+            uint64_t shard_id = (st->pool_id << (64-POOL_ID_BITS)) | pg_num;
+            st->new_shards[shard_id][st->obj_it->first] = st->obj_it->second;
+            chunk_size++;
+        }
+        st->old_shards.erase(st->sh_it++);
+    }
+    for (auto sh_it = st->new_shards.begin(); sh_it != st->new_shards.end(); sh_it++)
     {
         auto & to = clean_db_shards[sh_it->first];
         to.swap(sh_it->second);
     }
-    clean_db_settings[pool_id] = (pool_shard_settings_t){
-        .pg_count = pg_count,
-        .pg_stripe_size = pg_stripe_size,
+    clean_db_settings[st->pool_id] = (pool_shard_settings_t){
+        .pg_count = st->pg_count,
+        .pg_stripe_size = st->pg_stripe_size,
     };
+    delete st;
+    return true;
+}
+
+void blockstore_impl_t::reshard_abort(void *reshard_state)
+{
+    bs_reshard_state_t *st = (bs_reshard_state_t*)reshard_state;
+    for (auto sh_it = st->old_shards.begin(); sh_it != st->old_shards.end(); sh_it++)
+    {
+        auto & to = clean_db_shards[sh_it->first];
+        to.swap(sh_it->second);
+    }
+    delete st;
 }
 
 void blockstore_impl_t::process_list(blockstore_op_t *op)
@@ -465,7 +521,10 @@ void blockstore_impl_t::process_list(blockstore_op_t *op)
                 sh_it->second.pg_count != pg_count ||
                 sh_it->second.pg_stripe_size != pg_stripe_size)
             {
-                reshard_clean_db(pool_id, pg_count, pg_stripe_size);
+                // Sharding mismatch
+                op->retval = -EAGAIN;
+                FINISH_OP(op);
+                return;
             }
             first_shard = last_shard = ((uint64_t)pool_id << (64-POOL_ID_BITS)) | list_pg;
         }

src/blockstore/v1/impl.h

@@ -202,7 +202,6 @@ class blockstore_impl_t: public blockstore_i
     uint8_t* get_clean_entry_bitmap(uint64_t block_loc, int offset);
 
     blockstore_clean_db_t& clean_db_shard(object_id oid);
-    void reshard_clean_db(pool_id_t pool_id, uint32_t pg_count, uint32_t pg_stripe_size);
     void recalc_inode_space_stats(uint64_t pool_id, bool per_inode);
 
     // Journaling
@@ -288,6 +287,11 @@ public:
     void parse_config(blockstore_config_t & config);
     void parse_config(blockstore_config_t & config, bool init);
 
+    // Reshard database for a pool
+    void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
+    bool reshard_continue(void *reshard_state, uint64_t chunk_limit);
+    void reshard_abort(void *reshard_state);
+
     // Event loop
     void loop();
 

src/blockstore/v1/internal.h

@@ -1,3 +1,6 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 (see README.md for details)
+
 #pragma once
 
 // States are not stored on disk. Instead, they're deduced from the journal

src/blockstore/v1/write.cpp

@@ -183,7 +183,7 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
         uint32_t end = (op->offset+op->len-1) / dsk.csum_block_size;
         auto fn = state & BS_ST_BIG_WRITE ? crc32c_pad : crc32c_nopad;
         if (start == end)
-            data_csums[0] = fn(0, op->buf, op->len, op->offset - start*dsk.csum_block_size, end*dsk.csum_block_size - (op->offset+op->len));
+            data_csums[0] = fn(0, op->buf, op->len, op->offset - start*dsk.csum_block_size, (end+1)*dsk.csum_block_size - (op->offset+op->len));
         else
         {
             // First block

src/client/cluster_client.cpp

@@ -765,8 +765,13 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
     {
         return;
     }
-    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && enable_writeback && !(op->flags & OP_FLUSH_BUFFER) &&
-        !op->version /* no CAS writeback */)
+    // CAS writes are simplified: they're not cached, not resliced, not retried, and not part of the regular write queue at all
+    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && op->version)
+    {
+        execute_cas(op);
+        return;
+    }
+    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && enable_writeback && !(op->flags & OP_FLUSH_BUFFER))
     {
         if (wb->writebacks_active >= client_max_writeback_iodepth)
         {
@@ -788,7 +793,7 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
     }
     if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && !(op->flags & OP_IMMEDIATE_COMMIT))
     {
-        if (!(op->flags & OP_FLUSH_BUFFER) && !op->version /* no CAS write-repeat */)
+        if (!(op->flags & OP_FLUSH_BUFFER))
         {
             uint64_t flush_id = ++wb->last_flush_id;
             wb->copy_write(op, CACHE_REPEATING, flush_id);
@@ -847,6 +852,76 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
     }
 }
 
+void cluster_client_t::execute_cas(cluster_op_t *op)
+{
+    slice_rw(op);
+    op->needs_reslice = false;
+    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && op->version && op->parts.size() > 1)
+    {
+        // Atomic writes to multiple stripes are unsupported
+        op->retval = -EINVAL;
+        auto cb = std::move(op->callback);
+        cb(op);
+        return;
+    }
+    int res = try_send(op, 0, [this, op](osd_op_t *part)
+    {
+        int expected = part->req.hdr.opcode == OSD_OP_DELETE ? 0 : part->req.rw.len;
+        op->retval = part->reply.hdr.retval;
+        if (op->retval != expected && op->retval >= 0)
+            op->retval = -EIO;
+        op->retval = op->retval == -EPIPE ? -EINTR : op->retval;
+        auto peer_it = msgr.osd_peer_fds.find(op->parts[0].osd_num);
+        if (op->retval != 0 || (op->flags & OP_IMMEDIATE_COMMIT))
+        {
+            auto cb = std::move(op->callback);
+            cb(op);
+        }
+        else if (peer_it == msgr.osd_peer_fds.end())
+        {
+            // Care must be taken to make sure that the client doesn't reconnect to the OSD
+            // before executing the previously completed operation callback (!)
+            op->retval = -EINTR;
+            auto cb = std::move(op->callback);
+            cb(op);
+        }
+        else
+        {
+            // CAS writes have a built-in sync
+            auto peer_fd = peer_it->second;
+            *part = (osd_op_t){
+                .op_type = OSD_OP_OUT,
+                .peer_fd = peer_fd,
+                .req = {
+                    .hdr = {
+                        .magic = SECONDARY_OSD_OP_MAGIC,
+                        .opcode = OSD_OP_SYNC,
+                    },
+                },
+                .callback = [op](osd_op_t *part)
+                {
+                    if (part->reply.hdr.retval != 0)
+                    {
+                        op->retval = part->reply.hdr.retval;
+                        op->retval = op->retval == -EPIPE ? -EINTR : op->retval;
+                    }
+                    auto cb = std::move(op->callback);
+                    cb(op);
+                },
+            };
+            msgr.outbox_push(part);
+        }
+    });
+    if (res == TRY_SEND_CONNECTING || res == TRY_SEND_OFFLINE)
+    {
+        // In theory, CAS writes could wait for the PG to come up, but it's easier to just fail it
+        op->retval = -EINTR;
+        auto cb = std::move(op->callback);
+        cb(op);
+        return;
+    }
+}
+
 bool cluster_client_t::check_rw(cluster_op_t *op)
 {
     if (op->opcode == OSD_OP_SYNC)
@@ -956,13 +1031,6 @@ resume_0:
     // Slice the operation into parts
     slice_rw(op);
     op->needs_reslice = false;
-    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && op->version && op->parts.size() > 1)
-    {
-        // Atomic writes to multiple stripes are unsupported
-        op->retval = -EINVAL;
-        erase_op(op);
-        return 1;
-    }
 resume_1:
     // Send unsent parts, if they're not subject to change
     op->state = 2;
@@ -1189,9 +1257,9 @@ void cluster_client_t::slice_rw(cluster_op_t *op)
         {
             op->bitmap_buf = realloc_or_die(op->bitmap_buf, bitmap_mem);
             op->part_bitmaps = (uint8_t*)op->bitmap_buf + object_bitmap_size;
+            memset((uint8_t*)op->bitmap_buf+op->bitmap_buf_size, 0, bitmap_mem-op->bitmap_buf_size);
             op->bitmap_buf_size = bitmap_mem;
         }
-        memset(op->bitmap_buf, 0, bitmap_mem);
     }
     int iov_idx = 0;
     size_t iov_pos = 0;
@@ -1307,7 +1375,7 @@ bool cluster_client_t::affects_osd(uint64_t inode, uint64_t offset, uint64_t len
     return false;
 }
 
-int cluster_client_t::try_send(cluster_op_t *op, int i)
+int cluster_client_t::try_send(cluster_op_t *op, int i, std::function<void(osd_op_t *op_part)> cb)
 {
     if (!msgr_initialized)
     {
@@ -1337,7 +1405,7 @@ int cluster_client_t::try_send(cluster_op_t *op, int i)
         if (peer_it != msgr.osd_peer_fds.end())
         {
             int peer_fd = peer_it->second;
-            part->flags |= PART_SENT;
+            part->flags |= PART_SENT|PART_VALID;
             op->inflight_count++;
             uint64_t pg_bitmap_size = (pool_cfg.data_block_size / pool_cfg.bitmap_granularity / 8) * (
                 pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks
@@ -1367,7 +1435,7 @@ int cluster_client_t::try_send(cluster_op_t *op, int i)
                     ? (uint8_t*)op->part_bitmaps + pg_bitmap_size*i : NULL),
                 .bitmap_len = (unsigned)(op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP
                     ? pg_bitmap_size : 0),
-                .callback = [this, part](osd_op_t *op_part)
+                .callback = cb ? cb : [this, part](osd_op_t *op_part)
                 {
                     handle_op_part(part);
                 },
@@ -1550,14 +1618,11 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
             dirty_osds.insert(part->osd_num);
         part->flags |= PART_DONE;
         op->done_count++;
-        if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
+        if ((op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
+            && op->inode == op->cur_inode)
         {
-            copy_part_bitmap(op, part);
-            if (op->inode == op->cur_inode)
-            {
-                // Read only returns the version of the uppermost layer
-                op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0;
-            }
+            // Read only returns the version of the uppermost layer
+            op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0;
         }
         else if (op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE)
         {
@@ -1565,6 +1630,13 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
         }
         if (op->inflight_count == 0 && !op->retry_after)
         {
+            // Copy part bitmaps only after finishing all part reads
+            if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
+            {
+                for (auto & part: op->parts)
+                    if (part.flags == (PART_SENT|PART_VALID|PART_DONE))
+                        copy_part_bitmap(op, &part);
+            }
             if (op->opcode == OSD_OP_SYNC)
                 continue_sync(op);
             else

src/client/cluster_client.h

@@ -175,12 +175,13 @@ protected:
     void on_change_node_placement_hook();
 
     void execute_internal(cluster_op_t *op);
+    void execute_cas(cluster_op_t *op);
     void unshift_op(cluster_op_t *op);
     int continue_rw(cluster_op_t *op);
     bool check_rw(cluster_op_t *op);
     void slice_rw(cluster_op_t *op);
     void reset_retry_timer(int new_duration);
-    int try_send(cluster_op_t *op, int i);
+    int try_send(cluster_op_t *op, int i, std::function<void(osd_op_t *op_part)> cb = nullptr);
     int continue_sync(cluster_op_t *op);
     void send_sync(cluster_op_t *op, cluster_op_part_t *part);
     void handle_op_part(cluster_op_part_t *part);

src/client/cluster_client_impl.h

@@ -10,6 +10,7 @@
 #define PART_DONE 2
 #define PART_ERROR 4
 #define PART_RETRY 8
+#define PART_VALID 16
 #define CACHE_DIRTY 1
 #define CACHE_WRITTEN 2
 #define CACHE_FLUSHING 3

src/client/etcd_state_client.cpp

@@ -568,9 +568,19 @@ void etcd_state_client_t::start_ws_keepalive()
 
 void etcd_state_client_t::load_global_config()
 {
-    etcd_call("/kv/range", json11::Json::object {
-        { "key", base64_encode(etcd_prefix+"/config/global") }
-    }, etcd_quick_timeout, max_etcd_attempts, 0, [this](std::string err, json11::Json data)
+    json11::Json::object req = { { "success", json11::Json::array {
+        json11::Json::object {
+            { "request_range", json11::Json::object {
+                { "key", base64_encode(etcd_prefix+"/config/global") },
+            } }
+        },
+        json11::Json::object {
+            { "request_range", json11::Json::object {
+                { "key", base64_encode(etcd_prefix+"/config/pools") },
+            } }
+        },
+    } } };
+    etcd_txn(req, etcd_quick_timeout, max_etcd_attempts, 0, [this](std::string err, json11::Json data)
     {
         if (err != "")
         {
@@ -588,10 +598,12 @@ void etcd_state_client_t::load_global_config()
             }
             return;
         }
+        json11::Json config_kv = data["responses"][0]["response_range"]["kvs"][0];
+        json11::Json pools_kv = data["responses"][1]["response_range"]["kvs"][0];
         json11::Json::object global_config;
-        if (data["kvs"].array_items().size() > 0)
+        if (!config_kv.is_null())
         {
-            auto kv = parse_etcd_kv(data["kvs"][0]);
+            auto kv = parse_etcd_kv(config_kv);
             if (kv.value.is_object())
             {
                 global_config = kv.value.object_items();
@@ -608,6 +620,11 @@ void etcd_state_client_t::load_global_config()
             global_bitmap_granularity = DEFAULT_BITMAP_GRANULARITY;
         }
         global_immediate_commit = parse_immediate_commit(global_config["immediate_commit"].string_value(), IMMEDIATE_ALL);
+        if (!pools_kv.is_null())
+        {
+            auto kv = parse_etcd_kv(pools_kv);
+            parse_state(kv);
+        }
         on_load_config_hook(global_config);
     });
 }
@@ -945,8 +962,12 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
             if (pc.pg_stripe_size < min_stripe_size)
                 pc.pg_stripe_size = min_stripe_size;
             // Save
-            pc.real_pg_count = this->pool_config[pool_id].real_pg_count;
-            std::swap(pc.pg_config, this->pool_config[pool_id].pg_config);
+            auto & old_pc = this->pool_config[pool_id];
+            pc.real_pg_count = old_pc.real_pg_count;
+            pc.applied_pg_count = old_pc.applied_pg_count;
+            pc.applied_pg_stripe_size = old_pc.applied_pg_stripe_size;
+            pc.reshard_state = old_pc.reshard_state;
+            std::swap(pc.pg_config, old_pc.pg_config);
             std::swap(this->pool_config[pool_id], pc);
             auto & parsed_cfg = this->pool_config[pool_id];
             parsed_cfg.exists = true;

src/client/etcd_state_client.h

@@ -68,6 +68,11 @@ struct pool_config_t
     std::string used_for_app;
     int backfillfull = 0;
     int local_reads = 0;
+
+    // runtime data, used only by OSD:
+    uint64_t applied_pg_count = 0;
+    uint64_t applied_pg_stripe_size = 0;
+    void *reshard_state = NULL;
 };
 
 struct inode_config_t

src/client/http_client.cpp

@@ -23,7 +23,7 @@
 #define READ_BUFFER_SIZE 9000
 
 static std::string ws_format_frame(int type, uint64_t size);
-static bool ws_parse_frame(std::string & buf, int & type, std::string & res);
+static bool ws_parse_frame(std::string & buf, uint8_t & type, std::string & res);
 static void parse_http_headers(std::string & res, http_response_t *parsed);
 
 struct http_co_t
@@ -70,7 +70,7 @@ struct http_co_t
     void submit_read(bool check_timeout);
     void submit_send();
     bool handle_read();
-    void post_message(int type, const std::string & msg);
+    void post_message(uint8_t type, const std::string & msg);
     void send_request(const std::string & host, const std::string & request,
         const http_options_t & options, std::function<void(const http_response_t *response)> response_callback);
 };
@@ -199,12 +199,12 @@ void http_co_t::send_request(const std::string & host, const std::string & reque
     stackout();
 }
 
-void http_post_message(http_co_t *handler, int type, const std::string & msg)
+void http_post_message(http_co_t *handler, uint8_t type, const std::string & msg)
 {
     handler->post_message(type, msg);
 }
 
-void http_co_t::post_message(int type, const std::string & msg)
+void http_co_t::post_message(uint8_t type, const std::string & msg)
 {
     stackin();
     if (state == HTTP_CO_WEBSOCKET)
@@ -608,7 +608,13 @@ bool http_co_t::handle_read()
     {
         while (ws_parse_frame(response, parsed.ws_msg_type, parsed.body))
         {
-            response_callback(&parsed);
+            if (parsed.ws_msg_type == WS_PING)
+            {
+                // Reply with WS_PONG
+                post_message(WS_PONG, "");
+            }
+            else
+                response_callback(&parsed);
             parsed.body = "";
         }
     }
@@ -698,7 +704,7 @@ static std::string ws_format_frame(int type, uint64_t size)
     return res;
 }
 
-static bool ws_parse_frame(std::string & buf, int & type, std::string & res)
+static bool ws_parse_frame(std::string & buf, uint8_t & type, std::string & res)
 {
     uint64_t hdr = 2;
     if (buf.size() < hdr)

src/client/http_client.h

@@ -32,7 +32,7 @@ struct http_response_t
     int status_code = 0;
     std::string status_line;
     std::map<std::string, std::string> headers;
-    int ws_msg_type = -1;
+    uint8_t ws_msg_type = -1;
     std::string body;
 
     void parse_json_response(std::string & error, json11::Json & r) const;
@@ -46,5 +46,5 @@ http_co_t* open_websocket(timerfd_manager_t *tfd, const std::string & host, cons
     int timeout, std::function<void(const http_response_t *msg)> on_message);
 void http_request(http_co_t *handler, const std::string & host, const std::string & request,
     const http_options_t & options, std::function<void(const http_response_t *response)> response_callback);
-void http_post_message(http_co_t *handler, int type, const std::string & msg);
+void http_post_message(http_co_t *handler, uint8_t type, const std::string & msg);
 void http_close(http_co_t *co);

src/client/messenger.cpp

@@ -749,7 +749,7 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl)
                     }
                 });
                 // Add the initial receive request
-                try_recv_rdma(cl);
+                init_recv_rdma(cl);
             }
         }
 #endif

src/client/messenger.h

@@ -289,7 +289,7 @@ protected:
     void try_send_rdma(osd_client_t *cl);
     void try_send_rdma_odp(osd_client_t *cl);
     void try_send_rdma_nodp(osd_client_t *cl);
-    bool try_recv_rdma(osd_client_t *cl);
+    bool init_recv_rdma(osd_client_t *cl);
     void handle_rdma_events(msgr_rdma_context_t *rdma_context);
     msgr_rdma_context_t* choose_rdma_context(osd_client_t *cl);
 #endif

src/client/msgr_rdma.cpp

@@ -67,6 +67,20 @@ msgr_rdma_context_t::~msgr_rdma_context_t()
         ibv_close_device(context);
 }
 
+msgr_rdma_buf_t::~msgr_rdma_buf_t()
+{
+    if (buf)
+    {
+        free(buf);
+        buf = NULL;
+    }
+    if (mr)
+    {
+        ibv_dereg_mr(mr);
+        mr = NULL;
+    }
+}
+
 msgr_rdma_connection_t::~msgr_rdma_connection_t()
 {
     ctx->reserve_cqe(-max_send-max_recv);
@@ -84,26 +98,6 @@ msgr_rdma_connection_t::~msgr_rdma_connection_t()
     if (qp)
         ibv_destroy_qp(qp);
 #endif
-    if (recv_buffers.size())
-    {
-        for (auto b: recv_buffers)
-        {
-            if (b.mr)
-                ibv_dereg_mr(b.mr);
-            free(b.buf);
-        }
-        recv_buffers.clear();
-    }
-    if (send_out.mr)
-    {
-        ibv_dereg_mr(send_out.mr);
-        send_out.mr = NULL;
-    }
-    if (send_out.buf)
-    {
-        free(send_out.buf);
-        send_out.buf = NULL;
-    }
     send_out_size = 0;
 }
 
@@ -493,12 +487,12 @@ int msgr_rdma_connection_t::connect(msgr_rdma_address_t *dest)
             .grh        = {
                 .dgid = dest->gid,
                 .sgid_index = conn->ctx->gid_index,
-                .hop_limit = 1, // FIXME can it vary?
+                .hop_limit = 64, // FIXME can it vary?
             },
             .dlid       = dest->lid,
             .sl         = 0, // service level
             .src_path_bits = 0,
-            .is_global  = (uint8_t)(dest->gid.global.interface_id ? 1 : 0),
+            .is_global  = (uint8_t)(dest->gid.global.subnet_prefix || dest->gid.global.interface_id ? 1 : 0),
             .port_num   = conn->ctx->ib_port,
         },
         .max_rd_atomic  = 1,
@@ -663,7 +657,7 @@ void osd_messenger_t::try_send_rdma_nodp(osd_client_t *cl)
     {
         // Allocate send ring buffer, if not yet
         rc->send_out_size = rc->max_msg*rdma_max_send;
-        rc->send_out.buf = malloc_or_die(rc->send_out_size);
+        rc->send_out.buf = (uint8_t*)malloc_or_die(rc->send_out_size);
         if (!rc->ctx->odp)
         {
             rc->send_out.mr = ibv_reg_mr(rc->ctx->pd, rc->send_out.buf, rc->send_out_size, 0);
@@ -712,12 +706,12 @@ void osd_messenger_t::try_send_rdma(osd_client_t *cl)
         try_send_rdma_nodp(cl);
 }
 
-static void try_recv_rdma_wr(osd_client_t *cl, msgr_rdma_buf_t b)
+static void try_recv_rdma_wr(osd_client_t *cl, void *buf)
 {
     ibv_sge sge = {
-        .addr = (uintptr_t)b.buf,
+        .addr = (uintptr_t)buf,
         .length = (uint32_t)cl->rdma_conn->max_msg,
-        .lkey = cl->rdma_conn->ctx->odp ? cl->rdma_conn->ctx->mr->lkey : b.mr->lkey,
+        .lkey = cl->rdma_conn->ctx->odp ? cl->rdma_conn->ctx->mr->lkey : cl->rdma_conn->recv_buf.mr->lkey,
     };
     ibv_recv_wr *bad_wr = NULL;
     ibv_recv_wr wr = {
@@ -734,22 +728,23 @@ static void try_recv_rdma_wr(osd_client_t *cl, msgr_rdma_buf_t b)
     cl->rdma_conn->cur_recv++;
 }
 
-bool osd_messenger_t::try_recv_rdma(osd_client_t *cl)
+bool osd_messenger_t::init_recv_rdma(osd_client_t *cl)
 {
     auto rc = cl->rdma_conn;
-    while (rc->cur_recv < rc->max_recv)
+    assert(!rc->recv_buf.buf);
+    rc->recv_buf.buf = (uint8_t*)malloc_or_die(rc->max_msg * rc->max_recv);
+    if (!rc->ctx->odp)
     {
-        msgr_rdma_buf_t b;
-        b.buf = malloc_or_die(rc->max_msg);
-        if (!rc->ctx->odp)
+        rc->recv_buf.mr = ibv_reg_mr(rc->ctx->pd, rc->recv_buf.buf, rc->max_msg * rc->max_recv, IBV_ACCESS_LOCAL_WRITE);
+        if (!rc->recv_buf.mr)
         {
-            b.mr = ibv_reg_mr(rc->ctx->pd, b.buf, rc->max_msg, IBV_ACCESS_LOCAL_WRITE);
-            if (!b.mr)
-            {
-                fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
-                exit(1);
-            }
+            fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
+            exit(1);
         }
+    }
+    for (uint32_t i = 0; i < rc->max_recv; i++)
+    {
+        uint8_t *b = rc->recv_buf.buf + i*rc->max_msg;
         rc->recv_buffers.push_back(b);
         try_recv_rdma_wr(cl, b);
     }
@@ -807,7 +802,7 @@ void osd_messenger_t::handle_rdma_events(msgr_rdma_context_t *rdma_context)
                 cl->ping_time_remaining = 0;
                 cl->idle_time_remaining = osd_idle_timeout;
                 rc->cur_recv--;
-                if (!handle_read_buffer(cl, rc->recv_buffers[rc->next_recv_buf].buf, wc[i].byte_len))
+                if (!handle_read_buffer(cl, rc->recv_buffers[rc->next_recv_buf], wc[i].byte_len))
                 {
                     // handle_read_buffer may stop the client
                     clear_immediate_ops(client_id);

src/client/msgr_rdma.h

@@ -54,8 +54,10 @@ struct msgr_rdma_context_t
 
 struct msgr_rdma_buf_t
 {
-    void *buf = NULL;
+    uint8_t *buf = NULL;
     ibv_mr *mr = NULL;
+
+    ~msgr_rdma_buf_t();
 };
 
 struct msgr_rdma_connection_t
@@ -72,7 +74,8 @@ struct msgr_rdma_connection_t
     int cur_send = 0, cur_recv = 0;
     int send_pos = 0, send_buf_pos = 0;
     int next_recv_buf = 0;
-    std::vector<msgr_rdma_buf_t> recv_buffers;
+    std::vector<void*> recv_buffers;
+    msgr_rdma_buf_t recv_buf;
     std::vector<uint64_t> send_sizes;
     msgr_rdma_buf_t send_out;
     int send_out_pos = 0, send_done_pos = 0, send_out_size = 0;

src/client/msgr_rdmacm.cpp

@@ -531,7 +531,7 @@ void osd_messenger_t::rdmacm_established(rdma_cm_event *ev)
         fprintf(stderr, "Successfully connected with OSD %ju using RDMA-CM\n", peer_osd);
     }
     // Add initial receive request(s)
-    try_recv_rdma(cl);
+    init_recv_rdma(cl);
     if (peer_osd)
     {
         check_peer_config(cl);

src/client/msgr_receive.cpp

@@ -42,6 +42,7 @@ void osd_messenger_t::read_requests()
             }
             if (!sqe)
             {
+                cl->refs--;
                 cl->read_msg.msg_iovlen = 0;
                 read_ready_clients.erase(read_ready_clients.begin(), read_ready_clients.begin() + i);
                 return;

src/client/msgr_send.cpp

@@ -326,8 +326,10 @@ void osd_messenger_t::handle_send(int result, bool prev, bool more, osd_client_t
             int expected = cl->send_list.size() < IOV_MAX ? cl->send_list.size() : IOV_MAX;
             if (done != expected)
             {
-                fprintf(stderr, "BUG (maybe kernel): Expected to send %d iovecs with MSG_WAITALL but sent %d\n", expected, done);
-                exit(1);
+                fprintf(stderr, "Client %d socket write error: expected to send "
+                    "%d iovecs with MSG_WAITALL but sent %d. Disconnecting client\n", cl->peer_fd, expected, done);
+                stop_client(cl->peer_fd);
+                return;
             }
             cl->zc_free_list.push_back(NULL); // end marker
         }
@@ -363,7 +365,7 @@ void osd_messenger_t::handle_send(int result, bool prev, bool more, osd_client_t
                 }
             });
             // Add the initial receive request
-            try_recv_rdma(cl);
+            init_recv_rdma(cl);
         }
 #endif
     }

src/client/msgr_stop.cpp

@@ -85,6 +85,16 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
             osd_peer_fds.erase(osd_it);
         }
     }
+#ifdef WITH_RDMA
+    if (cl->rdma_conn && cl->rdma_conn->cmid)
+    {
+        auto rdma_it = rdmacm_connections.find(cl->rdma_conn->cmid);
+        if (rdma_it != rdmacm_connections.end() && rdma_it->second == cl)
+        {
+            rdmacm_connections.erase(rdma_it);
+        }
+    }
+#endif
 #ifndef __MOCK__
     // Then remove FD from the eventloop so we don't accidentally read something
     tfd->set_fd_handler(peer_fd, false, NULL);
@@ -158,6 +168,13 @@ osd_client_t::~osd_client_t()
     }
     // Cancel outbound ops
     cancel_ops();
+    for (osd_op_t *op: zc_free_list)
+    {
+        if (op)
+        {
+            delete op;
+        }
+    }
 #ifndef __MOCK__
 #ifdef WITH_RDMA
     if (rdma_conn)

src/client/osd_ops.h

@@ -262,7 +262,10 @@ struct __attribute__((__packed__)) osd_reply_del_t
     uint32_t left_on_dead_count;
 };
 
-// sync to the primary OSD
+// sync to the primary OSD. semantics:
+// 1) any non-synced write may disappear on OSD restart. even if it's a CAS write.
+// 2) sync only guarantees to commit completed writes, not in-progress ones.
+// 3) sync is a no-op when immediate_commit is active.
 struct __attribute__((__packed__)) osd_op_sync_t
 {
     osd_op_header_t header;

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 2.4.3
+Version: 3.0.2
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/cmd/cli.cpp

@@ -245,7 +245,7 @@ static json11::Json::object parse_args(int narg, const char *args[])
     cfg["progress"] = "1";
     for (int i = 1; i < narg; i++)
     {
-        bool argHasValue = (!(i == narg-1) && (args[i+1][0] != '-'));
+        bool argHasValue = (i < narg-1);
         if (args[i][0] == '-' && args[i][1] == 'h' && args[i][2] == 0)
         {
             cfg["help"] = "1";

src/cmd/cli_dd.cpp

@@ -111,7 +111,7 @@ struct dd_in_info_t
             {
                 in_granularity = 512;
             }
-            if (lseek(ifd, 1, SEEK_SET) == (off_t)-1)
+            if (lseek(ifd, 1, SEEK_SET) != (off_t)1)
             {
                 in_seekable = false;
             }
@@ -389,7 +389,7 @@ struct cli_dd_t
     int state = 0;
     int copy_error = 0;
     int in_waiting = 0, out_waiting = 0;
-    cli_result_t result;
+    cli_result_t result = {};
 
     bool is_done()
     {
@@ -890,12 +890,12 @@ resume_2:
         clock_gettime(CLOCK_REALTIME, &tv_begin);
         tv_progress = tv_begin;
 resume_3:
-        while ((ignore_errors || !copy_error) && (!in_eof || read_buffers.size() || in_waiting > 0 || out_waiting > 0))
+        while ((ignore_errors || !copy_error) && (!in_eof || read_buffers.size() || in_waiting > 0 || out_waiting > 0 || short_writes.size()))
         {
             print_progress(false);
             while ((ignore_errors || !copy_error) &&
                 (!in_eof && in_waiting < in_iodepth && read_buffers.size() < out_iodepth ||
-                read_buffers.size() && out_waiting < out_iodepth))
+                (read_buffers.size() || short_writes.size()) && out_waiting < out_iodepth))
             {
                 if (!in_eof && in_waiting < in_iodepth && read_buffers.size() < out_iodepth)
                 {
@@ -904,7 +904,7 @@ resume_3:
                         break;
                     }
                 }
-                if (read_buffers.size() && out_waiting < out_iodepth)
+                if ((read_buffers.size() || short_writes.size()) && out_waiting < out_iodepth)
                 {
                     if (!add_write_op())
                     {
@@ -931,7 +931,8 @@ close_end:
         oinfo.close_output(parent);
         iinfo.close_input(parent);
         // Done
-        result.err = copy_error;
+        if (copy_error)
+            result.err = copy_error;
         state = 100;
     }
 };

src/cmd/cli_modify_osd.cpp

@@ -120,12 +120,23 @@ resume_1:
                 else
                     osd_cfg.erase("noout");
             }
-            compare.push_back(json11::Json::object {
-                { "target", "MOD" },
-                { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/osd/"+std::to_string(osd_num)) },
-                { "result", "LESS" },
-                { "mod_revision", osd_cfg_mod_rev+1 },
-            });
+            if (osd_cfg_mod_rev)
+            {
+                compare.push_back(json11::Json::object {
+                    { "target", "MOD" },
+                    { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/osd/"+std::to_string(osd_num)) },
+                    { "result", "LESS" },
+                    { "mod_revision", osd_cfg_mod_rev+1 },
+                });
+            }
+            else
+            {
+                compare.push_back(json11::Json::object {
+                    { "target", "VERSION" },
+                    { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/osd/"+std::to_string(osd_num)) },
+                    { "version", 0 },
+                });
+            }
             if (!osd_cfg.size())
             {
                 success.push_back(json11::Json::object {

src/disk_tool/disk_simple_offsets.cpp

@@ -132,16 +132,13 @@ void disk_tool_simple_offsets(json11::Json cfg, bool json_output)
     uint64_t meta_size;
     if (meta_format == BLOCKSTORE_META_FORMAT_HEAP)
     {
-        uint32_t min_object_size = sizeof(heap_object_t)+sizeof(heap_write_t)+data_csum_size+2*clean_entry_bitmap_size;
-        uint32_t meta_block_target_free_space = cfg["meta_block_target_free_space"].uint64_value();
-        if (!meta_block_target_free_space || meta_block_target_free_space > device_block_size-min_object_size)
-            meta_block_target_free_space = 800;
+        uint32_t min_object_size = sizeof(heap_big_intent_t) + (data_csum_size ? data_csum_size : 4) + 2*clean_entry_bitmap_size;
         double meta_reserve = cfg["meta_reserve"].number_value();
         if (!meta_reserve)
             meta_reserve = 1.5;
         else if (meta_reserve < 1)
             meta_reserve = 1;
-        uint32_t entries_per_block = (device_block_size-meta_block_target_free_space) / min_object_size;
+        uint32_t entries_per_block = device_block_size / min_object_size;
         meta_size = device_block_size * (uint64_t)((object_count+entries_per_block-1) / entries_per_block * meta_reserve);
     }
     else if (meta_format == BLOCKSTORE_META_FORMAT_V2)

src/disk_tool/disk_tool.cpp

@@ -26,6 +26,9 @@ static const char *help_text =
     "  Options (automatic mode):\n"
     "    --osd_per_disk <N>\n"
     "      Create <N> OSDs on each disk (default 1)\n"
+    "    --meta_format 3\n"
+    "      Metadata store version. 3 is the new log-structured store, 2 is the stable store\n"
+    "      from Vitastor 0.9-2.x, 1 is the legacy store from Vitastor 0.6-0.8.\n"
     "    --hybrid\n"
     "      Prepare hybrid (HDD+SSD, NVMe+SATA or etc) OSDs using provided devices. By default,\n"
     "      any passed SSDs will be used for journals and metadata, HDDs will be used for data,\n"
@@ -62,6 +65,8 @@ static const char *help_text =
     "    --force                    Bypass partition safety checks (for emptiness and so on)\n"
     "  \n"
     "  Options (both modes):\n"
+    "    --tags tag1,tag2           Set new OSD tag(s)\n"
+    "    --weight <number>          Set new OSD weight (between 0 and 1)\n"
     "    --journal_size 32M/1G      Set journal size (area or partition size)\n"
     "    --block_size 128k/1M       Set blockstore object size\n"
     "    --bitmap_granularity 4k    Set bitmap granularity\n"
@@ -87,7 +92,8 @@ static const char *help_text =
     "  inmemory_metadata, inmemory_journal, max_write_iodepth,\n"
     "  min_flusher_count, max_flusher_count, journal_sector_buffer_count,\n"
     "  journal_no_same_sector_overwrites, throttle_small_writes, throttle_target_iops,\n"
-    "  throttle_target_mbs, throttle_target_parallelism, throttle_threshold_us.\n"
+    "  throttle_target_mbs, throttle_target_parallelism, throttle_threshold_us,\n"
+    "  atomic_write_size, use_atomic_flag.\n"
     "\n"
     "vitastor-disk upgrade-simple <UNIT_FILE|OSD_NUMBER>\n"
     "  Upgrade an OSD created by old (0.7.1 and older) make-osd.sh or make-osd-hybrid.js scripts.\n"
@@ -370,7 +376,7 @@ int main(int argc, char *argv[])
             self.dsk.meta_device = cmd[1];
             self.dsk.meta_block_size = strtoul(cmd[2], NULL, 10);
             self.dsk.meta_offset = strtoull(cmd[3], NULL, 10);
-            self.dsk.meta_area_size = strtoull(cmd[4], NULL, 10);
+            self.dsk.min_meta_len = self.dsk.meta_area_size = strtoull(cmd[4], NULL, 10);
         }
         else if (cmd.size() == 2)
         {

src/disk_tool/disk_tool.h

@@ -27,6 +27,9 @@
 #define VITASTOR_PART_TYPE "e7009fac-a5a1-4d72-af72-53de13059903"
 #define DEFAULT_HYBRID_JOURNAL "1G"
 #define DEFAULT_HYBRID_SSD_JOURNAL "128M"
+#define VITASTOR_META_FORMAT_NAME_V1 "0.6"
+#define VITASTOR_META_FORMAT_NAME_V2 "0.9"
+#define VITASTOR_META_FORMAT_NAME_HEAP "3.0"
 
 struct resizer_data_moving_t;
 
@@ -49,20 +52,22 @@ struct disk_tool_t
     bool all = false, json = false, now = false;
     bool dump_with_blocks = false, dump_with_data = false;
     bool dump_as_old = false;
+    bool skip_obsolete = false;
     int log_level = 1;
+    double meta_reserve_multiple = 2;
+    uint64_t meta_reserve_min_size = (uint64_t)1024*1024*1024;
     blockstore_disk_t dsk;
 
     // resize data and/or move metadata and journal
     int iodepth;
     std::string new_meta_device, new_journal_device;
-    uint64_t new_data_offset, new_data_len;
-    uint64_t new_journal_offset, new_journal_len;
-    uint64_t new_meta_offset, new_meta_len;
+    uint64_t new_data_offset = 0, new_data_len = 0;
+    uint64_t new_journal_offset = 0, new_journal_len = 0;
+    uint64_t new_meta_offset = 0, new_meta_len = 0;
 
     /**** State ****/
 
-    uint64_t meta_pos;
-    uint64_t journal_pos, journal_calc_data_pos;
+    uint64_t journal_pos = 0, journal_calc_data_pos = 0;
 
     uint8_t *buffer_area = NULL;
     bool first_block, first_entry;
@@ -72,26 +77,25 @@ struct disk_tool_t
     std::map<uint64_t, uint64_t>::iterator remap_it;
     ring_loop_t *ringloop = NULL;
     ring_consumer_t ring_consumer;
-    int remap_active;
+    int remap_active = 0;
     journal_entry_start je_start;
     uint8_t *new_journal_buf = NULL, *new_meta_buf = NULL, *new_journal_ptr = NULL, *new_journal_data = NULL;
     blockstore_meta_header_v3_t *new_meta_hdr = NULL;
-    blockstore_disk_t new_dsk;
-    blockstore_heap_t *new_heap = NULL;
-    uint64_t new_journal_in_pos;
-    int64_t data_idx_diff;
-    uint64_t total_blocks, free_first, free_last;
-    uint64_t new_clean_entry_bitmap_size, new_data_csum_size, new_clean_entry_size, new_entries_per_block;
+    uint64_t new_journal_in_pos = 0;
+    int64_t data_idx_diff = 0;
+    uint64_t total_blocks = 0, free_first = 0, free_last = 0;
+    uint64_t new_clean_entry_bitmap_size = 0, new_data_csum_size = 0, new_clean_entry_size = 0, new_entries_per_block = 0;
     uint32_t new_meta_format = 0;
     int new_journal_fd = -1, new_meta_fd = -1;
     resizer_data_moving_t *moving_blocks = NULL;
+    bool atomic_warned = false;
 
-    bool started;
+    bool started = false;
     void *small_write_data = NULL;
-    uint32_t data_crc32;
-    bool data_csum_valid;
-    uint32_t crc32_last;
-    uint32_t new_crc32_prev;
+    uint32_t data_crc32 = 0;
+    bool data_csum_valid = false;
+    uint32_t crc32_last = 0;
+    uint32_t new_crc32_prev = 0;
 
     ~disk_tool_t();
 
@@ -100,15 +104,15 @@ struct disk_tool_t
     int process_journal(std::function<int(void*)> block_fn, bool do_open = true);
     int process_journal_block(void *buf, std::function<void(int, journal_entry*)> iter_fn);
     int process_meta(std::function<void(blockstore_meta_header_v3_t *)> hdr_fn,
-        std::function<void(blockstore_heap_t *heap, heap_object_t *obj, uint32_t meta_block_num)> obj_fn,
+        std::function<void(blockstore_heap_t *heap, heap_entry_t *obj, uint32_t meta_block_num)> obj_fn,
         std::function<void(uint64_t block_num, clean_disk_entry *entry_v1, uint8_t *bitmap)> record_fn,
         bool with_data, bool do_open);
 
     int dump_meta();
     void dump_meta_header(blockstore_meta_header_v3_t *hdr);
     void dump_meta_entry(uint64_t block_num, clean_disk_entry *entry, uint8_t *bitmap);
-    void dump_heap_entry_as_old(blockstore_heap_t *heap, heap_object_t *obj);
-    void dump_heap_entry(blockstore_heap_t *heap, heap_object_t *obj);
+    void dump_heap_entry_as_old(blockstore_heap_t *heap, heap_entry_t *obj);
+    void dump_heap_entry(blockstore_heap_t *heap, heap_entry_t *obj);
 
     int dump_load_check_superblock(const std::string & device);
 
@@ -132,6 +136,10 @@ struct disk_tool_t
     void choose_journal_block(uint32_t je_size);
     int resize_rebuild_journal();
     int resize_write_new_journal();
+    void remap_big_write(heap_entry_t *wr);
+    void remap_small_write(heap_entry_t *wr);
+    void fill_old_clean_entry(blockstore_heap_t *heap, heap_entry_t *big_wr);
+    void fill_old_journal_entry(blockstore_heap_t *heap, heap_entry_t *wr);
     int resize_rebuild_meta();
     int resize_write_new_meta();
     void free_new_meta();
@@ -150,6 +158,7 @@ struct disk_tool_t
     json11::Json read_osd_superblock(std::string device, bool expect_exist = true, bool ignore_nonref = false);
     uint32_t write_osd_superblock(std::string device, json11::Json params);
 
+    void parse_meta_reserve();
     int prepare_one(std::map<std::string, std::string> options, int is_hdd, json11::Json::object & result);
     int check_existing_partition(std::string & dev_by_uuid);
     int fix_partition_type(std::string & dev_by_uuid);
@@ -167,7 +176,8 @@ void disk_tool_simple_offsets(json11::Json cfg, bool json_output);
 
 uint64_t sscanf_json(const char *fmt, const json11::Json & str);
 void fromhexstr(const std::string & from, int bytes, uint8_t *to);
-int disable_cache(std::string dev);
+int disable_cache(const std::string & dev);
+uint64_t get_atomic_write_size(const std::string & dev);
 uint64_t get_device_size(const std::string & dev, bool should_exist = false);
 std::string get_parent_device(std::string dev);
 int shell_exec(const std::vector<std::string> & cmd, const std::string & in, std::string *out, std::string *err);

src/disk_tool/disk_tool_discard.cpp

@@ -54,12 +54,13 @@ int disk_tool_t::trim_data(std::string device)
     fprintf(stderr, "Reading metadata\n");
     data_alloc = new allocator_t(dsk.block_count);
     r = process_meta(
-        [this](blockstore_meta_header_v3_t *hdr) {},
-        [this](blockstore_heap_t *heap, heap_object_t *obj, uint32_t meta_block_num)
+        [](blockstore_meta_header_v3_t *hdr) {},
+        [this](blockstore_heap_t *heap, heap_entry_t *obj, uint32_t meta_block_num)
         {
-            for (auto wr = obj->get_writes(); wr; wr = wr->next())
+            for (auto wr = obj; wr; wr = heap->prev(wr))
             {
-                if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE)
+                if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE ||
+                    (wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_INTENT)
                 {
                     data_alloc->set(wr->big_location(heap) / dsk.data_block_size, true);
                 }

src/disk_tool/disk_tool_meta.cpp

@@ -7,10 +7,8 @@
 #include "json_util.h"
 #include "malloc_or_die.h"
 
-#define FREE_SPACE_BIT 0x8000
-
 int disk_tool_t::process_meta(std::function<void(blockstore_meta_header_v3_t *)> hdr_fn,
-    std::function<void(blockstore_heap_t *heap, heap_object_t *obj, uint32_t meta_block_num)> obj_fn,
+    std::function<void(blockstore_heap_t *heap, heap_entry_t *obj, uint32_t meta_block_num)> obj_fn,
     std::function<void(uint64_t block_num, clean_disk_entry *entry_v1, uint8_t *bitmap)> record_fn,
     bool with_data, bool do_open)
 {
@@ -25,7 +23,7 @@ int disk_tool_t::process_meta(std::function<void(blockstore_meta_header_v3_t *)>
         buf_size = 8*dsk.meta_block_size;
     uint8_t *data = NULL;
     data = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, buf_size);
-    blockstore_meta_header_v3_t *hdr = (blockstore_meta_header_v3_t *)data;
+    blockstore_meta_header_v3_t *hdr = (blockstore_meta_header_v3_t *)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size);
     if (do_open)
     {
         if (dsk.meta_fd >= 0)
@@ -101,26 +99,25 @@ close_error:
         blockstore_heap_t *heap = new blockstore_heap_t(&dsk, buffer_area, log_level);
         // Load heap and just iterate it in memory
         hdr_fn(hdr);
-        hdr = NULL;
-        meta_pos = dsk.meta_block_size;
+        uint64_t meta_pos = dsk.meta_block_size;
         lseek64(dsk.meta_fd, dsk.meta_offset+meta_pos, 0);
-        while (meta_pos < dsk.meta_area_size)
+        uint64_t entries_loaded = 0;
+        while (meta_pos < hdr->meta_area_size)
         {
-            uint64_t read_len = buf_size < dsk.meta_area_size-meta_pos ? buf_size : dsk.meta_area_size-meta_pos;
+            uint64_t read_len = buf_size < hdr->meta_area_size-meta_pos ? buf_size : hdr->meta_area_size-meta_pos;
             read_blocking(dsk.meta_fd, data, read_len);
-            heap->read_blocks(meta_pos-dsk.meta_block_size, read_len, data, [&](heap_object_t *obj)
-            {
-                obj_fn(heap, obj, ((uint8_t*)obj-data+meta_pos)/dsk.meta_block_size);
-            }, [](uint32_t, uint32_t, uint8_t*){});
+            r = heap->load_blocks(meta_pos-dsk.meta_block_size, read_len, data, true, entries_loaded);
             meta_pos += read_len;
         }
+        heap->iterate_objects([&](heap_entry_t* obj, uint32_t meta_block_num)
+        {
+            obj_fn(heap, obj, meta_block_num);
+        });
+        obj_fn(heap, NULL, 0);
         delete heap;
     }
     else if (hdr->zero == 0 && hdr->magic == BLOCKSTORE_META_MAGIC_V1)
     {
-        dsk.meta_format = hdr->version;
-        dsk.calc_lengths();
-        dsk.check_lengths();
         if (hdr->version == BLOCKSTORE_META_FORMAT_V1)
         {
             // Vitastor 0.6-0.8 - static array of clean_disk_entry with bitmaps
@@ -165,8 +162,7 @@ csum_unknown:
         // Read
         uint64_t block_num = 0;
         hdr_fn(hdr);
-        hdr = NULL;
-        meta_pos = dsk.meta_block_size;
+        uint64_t meta_pos = dsk.meta_block_size;
         lseek64(dsk.meta_fd, dsk.meta_offset+meta_pos, 0);
         while (meta_pos < dsk.min_meta_len)
         {
@@ -201,11 +197,12 @@ csum_unknown:
         lseek64(dsk.meta_fd, dsk.meta_offset, 0);
         dsk.clean_entry_bitmap_size = 0;
         dsk.clean_entry_size = sizeof(clean_disk_entry);
+        uint64_t meta_pos = 0;
         uint64_t block_num = 0;
         hdr_fn(NULL);
-        while (meta_pos < dsk.meta_area_size)
+        while (meta_pos < dsk.min_meta_len)
         {
-            uint64_t read_len = buf_size < dsk.meta_area_size-meta_pos ? buf_size : dsk.meta_area_size-meta_pos;
+            uint64_t read_len = buf_size < dsk.min_meta_len-meta_pos ? buf_size : dsk.min_meta_len-meta_pos;
             read_blocking(dsk.meta_fd, data, read_len);
             meta_pos += read_len;
             for (uint64_t blk = 0; blk < read_len; blk += dsk.meta_block_size)
@@ -223,6 +220,7 @@ csum_unknown:
     }
 close_free:
     free(data);
+    free(hdr);
     if (buffer_area)
     {
         free(buffer_area);
@@ -269,20 +267,24 @@ int disk_tool_t::dump_load_check_superblock(const std::string & device)
 
 int disk_tool_t::dump_meta()
 {
+    dump_with_data = options["format"] == "data";
+    dump_as_old = options["format"] == "old";
     int r = process_meta(
         [this](blockstore_meta_header_v3_t *hdr)
         {
             if (dump_as_old)
             {
                 hdr->version = BLOCKSTORE_META_FORMAT_V2;
-                hdr->compacted_lsn = 0;
+                hdr->completed_lsn = 0;
                 hdr->header_csum = 0;
                 hdr->header_csum = crc32c(0, hdr, sizeof(blockstore_meta_header_v2_t));
             }
             dump_meta_header(hdr);
         },
-        [this](blockstore_heap_t *heap, heap_object_t *obj, uint32_t meta_block_num)
+        [this](blockstore_heap_t *heap, heap_entry_t *obj, uint32_t meta_block_num)
         {
+            if (!obj)
+                return;
             if (dump_as_old)
                 dump_heap_entry_as_old(heap, obj);
             else
@@ -303,7 +305,7 @@ void disk_tool_t::dump_meta_header(blockstore_meta_header_v3_t *hdr)
         if (hdr->version == BLOCKSTORE_META_FORMAT_V1)
         {
             printf(
-                "{\"version\":\"0.6\",\"meta_block_size\":%u,\"data_block_size\":%u,\"bitmap_granularity\":%u,"
+                "{\"version\":\"" VITASTOR_META_FORMAT_NAME_V1 "\",\"meta_block_size\":%u,\"data_block_size\":%u,\"bitmap_granularity\":%u,"
                 "\"entries\":[\n",
                 hdr->meta_block_size, hdr->data_block_size, hdr->bitmap_granularity
             );
@@ -311,7 +313,7 @@ void disk_tool_t::dump_meta_header(blockstore_meta_header_v3_t *hdr)
         else if (hdr->version == BLOCKSTORE_META_FORMAT_V2)
         {
             printf(
-                "{\"version\":\"0.9\",\"meta_block_size\":%u,\"data_block_size\":%u,\"bitmap_granularity\":%u,"
+                "{\"version\":\"" VITASTOR_META_FORMAT_NAME_V2 "\",\"meta_block_size\":%u,\"data_block_size\":%u,\"bitmap_granularity\":%u,"
                 "\"data_csum_type\":\"%s\",\"csum_block_size\":%u,\"entries\":[\n",
                 hdr->meta_block_size, hdr->data_block_size, hdr->bitmap_granularity,
                 csum_type_str(hdr->data_csum_type).c_str(), hdr->csum_block_size
@@ -320,7 +322,7 @@ void disk_tool_t::dump_meta_header(blockstore_meta_header_v3_t *hdr)
         else if (hdr->version == BLOCKSTORE_META_FORMAT_HEAP)
         {
             printf(
-                "{\"version\":\"3.0\",\"meta_block_size\":%u,\"data_block_size\":%u,\"bitmap_granularity\":%u,"
+                "{\"version\":\"" VITASTOR_META_FORMAT_NAME_HEAP "\",\"meta_block_size\":%u,\"data_block_size\":%u,\"bitmap_granularity\":%u,"
                 "\"data_csum_type\":\"%s\",\"csum_block_size\":%u,\"entries\":[\n",
                 hdr->meta_block_size, hdr->data_block_size, hdr->bitmap_granularity,
                 csum_type_str(hdr->data_csum_type).c_str(), hdr->csum_block_size
@@ -334,14 +336,14 @@ void disk_tool_t::dump_meta_header(blockstore_meta_header_v3_t *hdr)
     first_entry = true;
 }
 
-void disk_tool_t::dump_heap_entry_as_old(blockstore_heap_t *heap, heap_object_t *obj)
+void disk_tool_t::dump_heap_entry_as_old(blockstore_heap_t *heap, heap_entry_t *obj)
 {
-    heap_write_t *wr = NULL;
-    for (wr = obj->get_writes(); wr && wr->entry_type != (BS_HEAP_BIG_WRITE|BS_HEAP_STABLE) &&
-        wr->entry_type != (BS_HEAP_TOMBSTONE|BS_HEAP_STABLE); wr = wr->next())
+    heap_entry_t *wr = NULL;
+    for (wr = obj; wr && !wr->is_overwrite(); wr = heap->prev(wr))
     {
     }
-    if (!wr || wr->entry_type != (BS_HEAP_BIG_WRITE|BS_HEAP_STABLE))
+    if (!wr || wr->entry_type != (BS_HEAP_BIG_WRITE|BS_HEAP_STABLE) &&
+        wr->entry_type != (BS_HEAP_BIG_INTENT|BS_HEAP_STABLE))
     {
         return;
     }
@@ -365,7 +367,7 @@ void disk_tool_t::dump_heap_entry_as_old(blockstore_heap_t *heap, heap_object_t
         printf("%02x", bitmap[i]);
     }
     uint8_t *csums = wr->get_checksums(heap);
-    uint32_t csum_size = wr->get_csum_size(heap);
+    uint32_t csum_size = heap->get_csum_size(wr);
     if (csums)
     {
         printf("\",\"block_csums\":\"");
@@ -382,7 +384,7 @@ void disk_tool_t::dump_heap_entry_as_old(blockstore_heap_t *heap, heap_object_t
     first_entry = false;
 }
 
-void disk_tool_t::dump_heap_entry(blockstore_heap_t *heap, heap_object_t *obj)
+void disk_tool_t::dump_heap_entry(blockstore_heap_t *heap, heap_entry_t *obj)
 {
     printf(
 #define ENTRY_FMT "{\"pool\":%u,\"inode\":\"0x%jx\",\"stripe\":\"0x%jx\",\"writes\":["
@@ -390,29 +392,38 @@ void disk_tool_t::dump_heap_entry(blockstore_heap_t *heap, heap_object_t *obj)
 #undef ENTRY_FMT
         INODE_POOL(obj->inode), INODE_NO_POOL(obj->inode), obj->stripe
     );
-    heap_write_t *wr = NULL;
+    heap_entry_t *wr = NULL;
     bool first_wr = true;
-    for (wr = obj->get_writes(); wr; wr = wr->next())
+    for (wr = obj; wr; wr = heap->prev(wr))
     {
+        auto t = wr->type();
         printf(
 #define ENTRY_FMT "{\"lsn\":%ju,\"version\":%ju,\"type\":\"%s\",\"stable\":%s"
             (first_wr ? ENTRY_FMT : ("," ENTRY_FMT)),
 #undef ENTRY_FMT
-            wr->lsn, wr->version, (wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_SMALL_WRITE ? "small" : (
-                (wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE ? "big" : (
-                (wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_INTENT_WRITE ? "intent" : (
-                (wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_TOMBSTONE ? "tombstone" : "unknown"))),
+            wr->lsn, wr->version, (t == BS_HEAP_BIG_WRITE ? "big" : (
+                t == BS_HEAP_BIG_INTENT ? "big_intent" : (
+                t == BS_HEAP_SMALL_WRITE ? "small" : (
+                t == BS_HEAP_INTENT_WRITE ? "intent" : (
+                t == BS_HEAP_DELETE ? "delete" : (
+                t == BS_HEAP_COMMIT ? "commit" : (
+                t == BS_HEAP_ROLLBACK ? "rollback" : (
+                "unknown")))))))),
             (wr->entry_type & BS_HEAP_STABLE) ? "true" : "false"
         );
-        if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE)
+        if (t == BS_HEAP_BIG_WRITE)
         {
             printf(",\"location\":%ju", wr->big_location(heap));
         }
-        else if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_INTENT_WRITE)
+        else if (t == BS_HEAP_BIG_INTENT)
+        {
+            printf(",\"location\":%ju,\"offset\":%u,\"len\":%u", wr->big_location(heap), wr->big_intent().offset, wr->big_intent().len);
+        }
+        else if (t == BS_HEAP_INTENT_WRITE)
         {
             printf(",\"offset\":%u,\"len\":%u", wr->small().offset, wr->small().len);
         }
-        else if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_SMALL_WRITE)
+        else if (t == BS_HEAP_SMALL_WRITE)
         {
             if (!dump_with_data)
             {
@@ -446,7 +457,7 @@ void disk_tool_t::dump_heap_entry(blockstore_heap_t *heap, heap_object_t *obj)
         if (csums)
         {
             printf(",\"block_csums\":\"");
-            uint32_t csum_size = wr->get_csum_size(heap);
+            uint32_t csum_size = heap->get_csum_size(wr);
             for (uint32_t i = 0; i < csum_size; i++)
                 printf("%02x", csums[i]);
             printf("\"");
@@ -503,12 +514,16 @@ void disk_tool_t::dump_meta_entry(uint64_t block_num, clean_disk_entry *entry, u
 
 int disk_tool_t::write_json_meta(json11::Json meta)
 {
+    if (meta["version"].string_value() == VITASTOR_META_FORMAT_NAME_HEAP)
+    {
+        return write_json_heap(meta, meta["journal"]);
+    }
     new_meta_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_meta_len);
     memset(new_meta_buf, 0, new_meta_len);
     blockstore_meta_header_v2_t *new_hdr = (blockstore_meta_header_v2_t *)new_meta_buf;
     new_hdr->zero = 0;
     new_hdr->magic = BLOCKSTORE_META_MAGIC_V1;
-    new_hdr->version = meta["version"].uint64_value() == BLOCKSTORE_META_FORMAT_V1
+    new_hdr->version = meta["version"].string_value() == VITASTOR_META_FORMAT_NAME_V1
         ? BLOCKSTORE_META_FORMAT_V1 : BLOCKSTORE_META_FORMAT_V2;
     new_hdr->meta_block_size = meta["meta_block_size"].uint64_value()
         ? meta["meta_block_size"].uint64_value() : 4096;
@@ -574,6 +589,9 @@ int disk_tool_t::write_json_meta(json11::Json meta)
 
 int disk_tool_t::write_json_heap(json11::Json meta, json11::Json journal)
 {
+    new_meta_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_meta_len);
+    memset(new_meta_buf, 0, new_meta_len);
+    new_meta_hdr = (blockstore_meta_header_v3_t *)new_meta_buf;
     new_meta_hdr->zero = 0;
     new_meta_hdr->magic = BLOCKSTORE_META_MAGIC_V1;
     new_meta_hdr->version = BLOCKSTORE_META_FORMAT_HEAP;
@@ -590,71 +608,98 @@ int disk_tool_t::write_json_heap(json11::Json meta, json11::Json journal)
             : BLOCKSTORE_CSUM_NONE);
     new_meta_hdr->csum_block_size = meta["csum_block_size"].uint64_value();
     new_meta_hdr->header_csum = crc32c(0, new_meta_hdr, sizeof(blockstore_meta_header_v3_t));
+    new_meta_hdr->meta_area_size = new_meta_len;
     new_clean_entry_bitmap_size = (new_meta_hdr->data_block_size / new_meta_hdr->bitmap_granularity + 7) / 8;
     new_clean_entry_size = 0;
     new_entries_per_block = 0;
     new_data_csum_size = (new_meta_hdr->data_csum_type
         ? ((new_meta_hdr->data_block_size+new_meta_hdr->csum_block_size-1)/new_meta_hdr->csum_block_size*(new_meta_hdr->data_csum_type & 0xFF))
         : 0);
-    new_journal_buf = new_journal_len ? (uint8_t*)memalign(MEM_ALIGNMENT, new_journal_len) : NULL;
+    new_journal_buf = NULL;
     if (new_journal_len)
     {
+        new_journal_buf = (uint8_t*)memalign(MEM_ALIGNMENT, new_journal_len);
         memset(new_journal_buf, 0, new_journal_len);
     }
     uint64_t total_used_space = 0;
     uint32_t used_space = 0;
+    uint64_t meta_offset = dsk.meta_block_size;
+    blockstore_heap_t heap(&dsk, NULL, 0);
+    heap_entry_t *wr = NULL;
+    auto get_wr = [&](uint32_t entry_size)
+    {
+        if (used_space > new_meta_hdr->meta_block_size-entry_size)
+        {
+            heap.fill_block_empty_space(new_meta_buf + meta_offset, used_space);
+            meta_offset += new_meta_hdr->meta_block_size;
+            used_space = 0;
+            if (meta_offset >= new_meta_len)
+            {
+                fprintf(stderr, "Metadata doesn't fit into the new area (total used space: %ju)\n", total_used_space);
+                return (heap_entry_t*)NULL;
+            }
+        }
+        auto wr = (heap_entry_t*)(new_meta_buf + meta_offset + used_space);
+        used_space += entry_size;
+        return wr;
+    };
     // FIXME: Use a streaming json parser
-    if (meta["version"] == "3.0")
+    if (meta["version"].string_value() == VITASTOR_META_FORMAT_NAME_HEAP)
     {
         // New format
-        std::vector<uint8_t> object_buf;
-        new_heap = new blockstore_heap_t(&dsk, new_journal_buf, 0);
         for (const auto & meta_entry: meta["entries"].array_items())
         {
-            bool invalid = false;
             object_id oid = {
                 .inode = (sscanf_json(NULL, meta_entry["pool"]) << (64-POOL_ID_BITS)) | sscanf_json(NULL, meta_entry["inode"]),
                 .stripe = sscanf_json(NULL, meta_entry["stripe"]),
             };
-            object_buf.clear();
-            object_buf.resize(sizeof(heap_object_t));
-            heap_object_t *obj = (heap_object_t*)object_buf.data();
-            obj->size = sizeof(heap_object_t);
-            obj->write_pos = meta_entry["writes"].array_items().size() ? sizeof(heap_object_t) : 0;
-            obj->entry_type = BS_HEAP_OBJECT;
-            obj->inode = oid.inode;
-            obj->stripe = oid.stripe;
-            size_t pos = sizeof(heap_object_t);
-            heap_write_t *last_wr = NULL;
-            for (auto & write_entry: meta_entry["writes"].array_items())
+            for (const auto & write_entry: meta_entry["writes"].array_items())
             {
-                object_buf.resize(object_buf.size() + new_heap->get_max_write_entry_size());
-                heap_write_t *wr = (heap_write_t*)(object_buf.data() + pos);
-                last_wr = wr;
-                uint8_t wr_type = 0;
+                uint32_t wr_type = 0;
                 if (write_entry["type"] == "small")
                     wr_type = BS_HEAP_SMALL_WRITE;
                 else if (write_entry["type"] == "intent")
                     wr_type = BS_HEAP_INTENT_WRITE;
                 else if (write_entry["type"] == "big")
                     wr_type = BS_HEAP_BIG_WRITE;
-                else if (write_entry["type"] == "tombstone")
-                    wr_type = BS_HEAP_TOMBSTONE;
+                else if (write_entry["type"] == "big_intent")
+                    wr_type = BS_HEAP_BIG_INTENT;
+                else if (write_entry["type"] == "delete")
+                    wr_type = BS_HEAP_DELETE;
+                else if (write_entry["type"] == "commit")
+                    wr_type = BS_HEAP_COMMIT;
+                else if (write_entry["type"] == "rollback")
+                    wr_type = BS_HEAP_ROLLBACK;
                 else
                 {
-                    fprintf(stderr, "Write entry in %s has invalid type: %s, skipping object\n", meta_entry.dump().c_str(), write_entry["type"].dump().c_str());
-                    invalid = true;
-                    break;
+                    fprintf(stderr, "Write entry %s has invalid type: %s, aborting\n",
+                        write_entry.dump().c_str(), write_entry["type"].dump().c_str());
+close_err0:
+                    free(new_meta_buf);
+                    new_meta_buf = NULL;
+                    return 1;
                 }
+                uint64_t wr_offset = write_entry["offset"].uint64_value();
+                uint64_t wr_len = write_entry["len"].uint64_value();
+                uint32_t wr_size = (wr_type == BS_HEAP_SMALL_WRITE || wr_type == BS_HEAP_INTENT_WRITE
+                    ? heap.get_small_entry_size(wr_offset, wr_len)
+                    : (wr_type == BS_HEAP_BIG_WRITE
+                        ? heap.get_big_entry_size()
+                        : (wr_type == BS_HEAP_BIG_INTENT
+                            ? heap.get_big_intent_entry_size()
+                            : heap.get_simple_entry_size())));
+                if (!(wr = get_wr(wr_size)))
+                    goto close_err0;
+                wr->inode = oid.inode;
+                wr->stripe = oid.stripe;
                 wr->entry_type = wr_type | (write_entry["stable"].bool_value() ? BS_HEAP_STABLE : 0);
                 wr->lsn = write_entry["lsn"].uint64_value();
                 wr->version = write_entry["version"].uint64_value();
-                wr->size = wr->get_size(new_heap);
-                wr->next_pos = wr->size;
+                wr->size = wr->get_size(&heap);
                 if (wr_type == BS_HEAP_SMALL_WRITE || wr_type == BS_HEAP_INTENT_WRITE)
                 {
-                    wr->small().offset = write_entry["offset"].uint64_value();
-                    wr->small().len = write_entry["len"].uint64_value();
+                    wr->small().offset = wr_offset;
+                    wr->small().len = wr_len;
                     wr->small().location = write_entry["location"].uint64_value();
                     if (wr_type == BS_HEAP_SMALL_WRITE && write_entry["data"].is_string() && wr->small().len > 0)
                     {
@@ -664,40 +709,51 @@ int disk_tool_t::write_json_heap(json11::Json meta, json11::Json journal)
                             free_new_meta();
                             return 1;
                         }
-                        wr->small().location = new_heap->find_free_buffer_area(wr->small().len);
+                        wr->small().location = heap.find_free_buffer_area(wr->small().len);
                         fromhexstr(write_entry["data"].string_value(), wr->small().len, new_journal_buf + wr->small().location);
                     }
                 }
                 else if (wr_type == BS_HEAP_BIG_WRITE)
+                {
+                    uint64_t loc = write_entry["location"].uint64_value();
+                    if ((loc % dsk.data_block_size) || (loc / dsk.data_block_size) >= 0xFFFF0000)
+                    {
+                        fprintf(stderr, "Write entry %s has invalid location: 0x%jx, aborting\n",
+                            write_entry.dump().c_str(), loc);
+                        free_new_meta();
+                        return 1;
+                    }
+                    wr->set_big_location(&heap, loc);
+                }
+                else if (wr_type == BS_HEAP_BIG_INTENT)
                 {
                     uint64_t loc = write_entry["location"].uint64_value();
                     assert(!(loc % dsk.data_block_size));
                     assert((loc / dsk.data_block_size) < 0xFFFF0000);
-                    wr->set_big_location(new_heap, loc);
+                    auto & bi = wr->big_intent();
+                    bi.block_num = loc / dsk.data_block_size;
+                    bi.offset = wr_offset;
+                    bi.len = wr_len;
                 }
-                if (write_entry["bitmap"].is_string() && wr->get_int_bitmap(new_heap))
+                if (write_entry["bitmap"].is_string() && wr->get_int_bitmap(&heap))
                 {
-                    fromhexstr(write_entry["bitmap"].string_value(), new_clean_entry_bitmap_size, wr->get_int_bitmap(new_heap));
+                    fromhexstr(write_entry["bitmap"].string_value(), new_clean_entry_bitmap_size, wr->get_int_bitmap(&heap));
                 }
-                if (write_entry["ext_bitmap"].is_string() && wr->get_ext_bitmap(new_heap))
+                if (write_entry["ext_bitmap"].is_string() && wr->get_ext_bitmap(&heap))
                 {
-                    fromhexstr(write_entry["ext_bitmap"].string_value(), new_clean_entry_bitmap_size, wr->get_ext_bitmap(new_heap));
+                    fromhexstr(write_entry["ext_bitmap"].string_value(), new_clean_entry_bitmap_size, wr->get_ext_bitmap(&heap));
                 }
-                if (write_entry["block_csums"].is_string() && wr->get_checksums(new_heap))
+                if (write_entry["block_csums"].is_string() && wr->get_checksums(&heap))
                 {
-                    fromhexstr(write_entry["block_csums"].string_value(), wr->get_csum_size(new_heap), wr->get_ext_bitmap(new_heap));
+                    fromhexstr(write_entry["block_csums"].string_value(), heap.get_csum_size(wr), wr->get_ext_bitmap(&heap));
                 }
-                if (write_entry["data_crc32c"].is_string() && wr->get_checksum(new_heap))
+                if (write_entry["data_crc32c"].is_string() && wr->get_checksum(&heap))
                 {
-                    *wr->get_checksum(new_heap) = sscanf_json("%jx", write_entry["data_crc32c"]);
+                    *wr->get_checksum(&heap) = sscanf_json("%jx", write_entry["data_crc32c"]);
                 }
+                wr->crc32c = wr->calc_crc32c();
+                assert((uint8_t*)wr + wr->size == new_meta_buf + meta_offset + used_space);
             }
-            if (invalid)
-            {
-                continue;
-            }
-            last_wr->next_pos = 0;
-            new_heap->copy_object(obj, NULL);
         }
     }
     else
@@ -718,82 +774,17 @@ close_err:
         journal = json11::Json();
         // Convert old format to the new format
         uint64_t next_lsn = 0;
-        uint64_t meta_offset = 0;
-        const uint32_t space_per_object = sizeof(heap_object_t) + sizeof(heap_write_t) +
-            new_clean_entry_bitmap_size*2 + new_data_csum_size;
         uint64_t buffer_pos = 0;
-        // FIXME: Rather ugly. Remove the dependency on dsk from heap?
-        blockstore_disk_t dsk;
-        dsk.bitmap_granularity = new_meta_hdr->bitmap_granularity;
-        dsk.block_count = 16;
-        dsk.data_block_size = new_meta_hdr->data_block_size;
-        dsk.clean_entry_bitmap_size = new_clean_entry_bitmap_size;
-        dsk.csum_block_size = new_meta_hdr->csum_block_size;
-        dsk.data_csum_type = new_meta_hdr->data_csum_type;
-        dsk.journal_len = 4096;
-        dsk.meta_area_size = new_meta_len;
-        dsk.meta_block_size = new_meta_hdr->meta_block_size;
-        dsk.meta_block_target_free_space = 800;
-        blockstore_heap_t heap(&dsk, NULL, 0);
         for (const auto & meta_entry: meta["entries"].array_items())
         {
             object_id oid = {
                 .inode = (sscanf_json(NULL, meta_entry["pool"]) << (64-POOL_ID_BITS)) | sscanf_json(NULL, meta_entry["inode"]),
                 .stripe = sscanf_json(NULL, meta_entry["stripe"]),
             };
-            uint32_t space_for_this = space_per_object;
-            auto j_it = journal_by_object.find(oid);
-            if (j_it != journal_by_object.end())
-            {
-                for (auto & rec: j_it->second)
-                {
-                    if (rec["type"] == "small_write" || rec["type"] == "small_write_instant")
-                    {
-                        uint64_t off = rec["offset"].uint64_value();
-                        uint64_t len = rec["len"].uint64_value();
-                        if (off+len > new_meta_hdr->data_block_size)
-                        {
-                            fprintf(stderr, "Journal entry has too large offset or length: %s\n", json11::Json(rec).dump().c_str());
-                            goto close_err;
-                        }
-                        space_for_this += sizeof(heap_write_t) + new_clean_entry_bitmap_size +
-                            ((off+len+new_meta_hdr->csum_block_size-1)/new_meta_hdr->csum_block_size - off/new_meta_hdr->csum_block_size) * (new_meta_hdr->data_csum_type & 0xFF);
-                    }
-                    else /*if (rec["type"] == "big_write" || rec["type"] == "big_write_instant")*/
-                    {
-                        space_for_this += sizeof(heap_write_t) + 2*new_clean_entry_bitmap_size + new_data_csum_size;
-                    }
-                }
-            }
-            if (space_for_this > new_meta_hdr->meta_block_size)
-            {
-                fprintf(stderr, "Object doesn't fit in a single metadata block. Object meta: %s, object journal: %s\n",
-                    meta_entry.dump().c_str(), json11::Json(j_it->second).dump().c_str());
+            if (!(wr = get_wr(heap.get_big_entry_size())))
                 goto close_err;
-            }
-            if (used_space + space_for_this > new_meta_hdr->meta_block_size-dsk.meta_block_target_free_space)
-            {
-                if (used_space < new_meta_hdr->meta_block_size-2)
-                {
-                    *((uint16_t*)(new_meta_buf + meta_offset + used_space)) = FREE_SPACE_BIT | (uint16_t)(new_meta_hdr->meta_block_size-used_space);
-                }
-                meta_offset += new_meta_hdr->meta_block_size;
-                used_space = 0;
-                if (meta_offset >= new_meta_len)
-                {
-                    fprintf(stderr, "Metadata doesn't fit into the new area (total used space: %ju, minimum free space in block: %u/%u)\n",
-                        total_used_space, dsk.meta_block_target_free_space, new_meta_hdr->meta_block_size);
-                    goto close_err;
-                }
-            }
-            heap_object_t *obj = (heap_object_t*)(new_meta_buf + meta_offset + used_space);
-            obj->size = sizeof(heap_object_t);
-            obj->write_pos = sizeof(heap_object_t);
-            obj->entry_type = BS_HEAP_OBJECT;
-            obj->inode = oid.inode;
-            obj->stripe = oid.stripe;
-            heap_write_t *wr = obj->get_writes();
-            wr->next_pos = 0;
+            wr->inode = oid.inode;
+            wr->stripe = oid.stripe;
             wr->entry_type = BS_HEAP_BIG_WRITE|BS_HEAP_STABLE;
             wr->lsn = ++next_lsn;
             wr->version = sscanf_json(NULL, meta_entry["version"]);
@@ -803,17 +794,24 @@ close_err:
             fromhexstr(meta_entry["ext_bitmap"].string_value(), new_clean_entry_bitmap_size, wr->get_ext_bitmap(&heap));
             if (new_meta_hdr->data_csum_type != 0)
                 fromhexstr(meta_entry["data_csum"].string_value(), new_data_csum_size, wr->get_checksums(&heap));
+            wr->crc32c = wr->calc_crc32c();
+            assert((uint8_t*)wr + wr->size == new_meta_buf + meta_offset + used_space);
+            auto j_it = journal_by_object.find(oid);
             if (j_it != journal_by_object.end())
             {
                 for (auto & rec: j_it->second)
                 {
-                    wr->next_pos = wr->get_size(&heap);
-                    wr = wr->next();
-                    wr->next_pos = 0;
-                    wr->lsn = ++next_lsn;
-                    wr->version = rec["ver"].uint64_value();
                     uint64_t wr_offset = rec["offset"].uint64_value();
                     uint64_t wr_len = rec["len"].uint64_value();
+                    if (!(wr = get_wr(rec["type"] == "small_write" || rec["type"] == "small_write_instant"
+                        ? heap.get_small_entry_size(wr_offset, wr_len) : heap.get_big_entry_size())))
+                    {
+                        goto close_err;
+                    }
+                    wr->inode = oid.inode;
+                    wr->stripe = oid.stripe;
+                    wr->lsn = ++next_lsn;
+                    wr->version = rec["ver"].uint64_value();
                     if (rec["type"] == "small_write" || rec["type"] == "small_write_instant")
                     {
                         if (wr_len > 0 && !rec["data"].is_string())
@@ -862,20 +860,20 @@ close_err:
                         assert(0);
                     }
                     wr->size = wr->get_size(&heap);
+                    wr->crc32c = wr->calc_crc32c();
+                    assert((uint8_t*)wr + wr->size == new_meta_buf + meta_offset + used_space);
                 }
             }
-            obj->crc32c = obj->calc_crc32c();
-            assert(((uint8_t*)wr + wr->size - (uint8_t*)obj) == space_for_this);
-            used_space += space_for_this;
-            total_used_space += space_for_this;
-        }
-        if (used_space > 0 && used_space < new_meta_hdr->meta_block_size-2)
-        {
-            *((uint16_t*)(new_meta_buf + meta_offset + used_space)) = FREE_SPACE_BIT | (uint16_t)(new_meta_hdr->meta_block_size-used_space);
         }
     }
+    while (meta_offset < new_meta_len)
+    {
+        heap.fill_block_empty_space(new_meta_buf + meta_offset, used_space);
+        meta_offset += dsk.meta_block_size;
+        used_space = 0;
+    }
     int r = resize_write_new_meta();
-    if (r == 0)
+    if (new_journal_buf && r == 0)
     {
         r = resize_write_new_journal();
     }

src/disk_tool/disk_tool_prepare.cpp

@@ -6,6 +6,25 @@
 #include "json_util.h"
 #include "osd_id.h"
 
+void disk_tool_t::parse_meta_reserve()
+{
+    meta_reserve_multiple = 2;
+    meta_reserve_min_size = (uint64_t)1024*1024*1024;
+    if (options.find("meta_reserve") != options.end())
+    {
+        int p1 = options["meta_reserve"].find("x"), p2 = options["meta_reserve"].find(",");
+        if (p1 >= 0 && p2 >= 0)
+        {
+            sscanf(options["meta_reserve"].c_str()+(p1 < p2 ? 0 : p2), "%lf", &meta_reserve_multiple);
+            meta_reserve_min_size = parse_size(options["meta_reserve"].substr(p1 < p2 ? p2 : 0, p1 < p2 ? options["meta_reserve"].size()-p2 : p2));
+        }
+        else if (p1 >= 0)
+            sscanf(options["meta_reserve"].c_str(), "%lf", &meta_reserve_multiple);
+        else
+            meta_reserve_min_size = parse_size(options["meta_reserve"]);
+    }
+}
+
 int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_hdd, json11::Json::object & result)
 {
     static const char *allow_additional_params[] = {
@@ -32,6 +51,8 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
         "discard_on_start",
         "min_discard_size",
         "discard_granularity",
+        "atomic_write_size",
+        "use_atomic_flag",
     };
     if (options.find("force") == options.end())
     {
@@ -57,6 +78,36 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
             if (check_existing_partition(dev) != 0)
                 return 1;
         }
+        if (options.find("weight") != options.end())
+        {
+            double reweight = json11::Json(options["weight"]).number_value();
+            if (reweight < 0 || reweight > 1)
+            {
+                fprintf(stderr, "OSD weight must be between 0 and 1\n");
+                return 1;
+            }
+        }
+    }
+    if (options.find("atomic_write_size") == options.end())
+    {
+        auto data_dev = realpath_str(options["data_device"], false);
+        if (data_dev.substr(0, 9) == "/dev/nvme")
+        {
+            uint64_t atomic_write_size = get_atomic_write_size(data_dev);
+            if (atomic_write_size > 4096)
+            {
+                // FIXME: Enable use_atomic_flag when the kernel does checks correctly
+                fprintf(stderr, "Data device %s supports atomic writes up to %ju bytes, enabling. Enjoy faster writes!\n",
+                    data_dev.c_str(), atomic_write_size);
+                options["atomic_write_size"] = std::to_string(atomic_write_size);
+                if (!atomic_warned)
+                {
+                    fprintf(stderr, "WARNING: RWF_ATOMIC can't be used because Linux checks atomic writes incorrectly.\n"
+                        " Please don't change scheduler from default 'none' and check use_atomic_flag documentation for more details.\n");
+                    atomic_warned = true;
+                }
+            }
+        }
     }
     for (auto dev: std::vector<std::string>{"data", "meta", "journal"})
     {
@@ -128,9 +179,19 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
         dsk.open_data();
         dsk.open_meta();
         dsk.open_journal();
-        dsk.calc_lengths();
+        dsk.calc_lengths(true);
         if (dsk.data_device == dsk.meta_device && !new_meta_len)
-            dsk.data_offset += (dsk.meta_format == BLOCKSTORE_META_FORMAT_HEAP ? dsk.min_meta_len*2 : dsk.min_meta_len);
+        {
+            uint64_t new_meta_len = dsk.min_meta_len;
+            if (dsk.meta_format == BLOCKSTORE_META_FORMAT_HEAP)
+            {
+                new_meta_len = dsk.min_meta_len*meta_reserve_multiple;
+                if (new_meta_len < meta_reserve_min_size)
+                    new_meta_len = meta_reserve_min_size;
+                new_meta_len = (new_meta_len + dsk.meta_block_size-1) & ~((uint64_t)dsk.meta_block_size-1);
+            }
+            dsk.data_offset += new_meta_len;
+        }
         dsk.meta_area_size = (dsk.data_device == dsk.meta_device ? dsk.data_offset : dsk.meta_device_size) - dsk.meta_offset;
         sb = json11::Json::object {
             { "meta_format", options["meta_format"] },
@@ -189,9 +250,28 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
         return 1;
     }
     sb["osd_num"] = osd_num;
-    // Zero out metadata and journal
-    if (write_zero(dsk.meta_fd, sb["meta_offset"].uint64_value(), dsk.meta_area_size) != 0 ||
-        write_zero(dsk.journal_fd, sb["journal_offset"].uint64_value(), dsk.journal_len) != 0)
+    if (options.find("weight") != options.end() || options.find("tags") != options.end())
+    {
+        std::vector<std::string> cmd = { "vitastor-cli", "modify-osd", std::to_string(osd_num) };
+        if (options.find("weight") != options.end())
+        {
+            cmd.push_back("--reweight");
+            cmd.push_back(options["weight"]);
+        }
+        if (options.find("tags") != options.end())
+        {
+            cmd.push_back("--tags");
+            cmd.push_back(options["tags"]);
+        }
+        if (shell_exec(cmd, "", NULL, NULL) != 0)
+        {
+            fprintf(stderr, "Failed to modify OSD %ju tags and/or reweight\n", osd_num);
+            return 1;
+        }
+    }
+    // Zero out the first block of metadata and journal - OSD will zero the rest on the first run
+    if (write_zero(dsk.meta_fd, sb["meta_offset"].uint64_value(), 4096) != 0 ||
+        write_zero(dsk.journal_fd, sb["journal_offset"].uint64_value(), 4096) != 0)
     {
         fprintf(stderr, "Failed to zero out metadata or journal: %s\n", strerror(errno));
         dsk.close_all();
@@ -537,7 +617,7 @@ int disk_tool_t::get_meta_partition(std::vector<vitastor_dev_info_t> & ssds, std
         dsk.open_data();
         dsk.open_meta();
         dsk.open_journal();
-        dsk.calc_lengths();
+        dsk.calc_lengths(true);
         dsk.close_all();
         meta_size = dsk.min_meta_len;
     }
@@ -548,24 +628,10 @@ int disk_tool_t::get_meta_partition(std::vector<vitastor_dev_info_t> & ssds, std
         return 1;
     }
     // Leave some extra space for future metadata formats and round metadata area size to multiples of 1 MB
-    uint64_t meta_reserve_multiple = 2, min_meta_size = (uint64_t)1024*1024*1024;
-    if (options.find("meta_reserve") != options.end())
-    {
-        int p1 = options["meta_reserve"].find("x"), p2 = options["meta_reserve"].find(",");
-        if (p1 >= 0 && p2 >= 0)
-        {
-            meta_reserve_multiple = stoull_full(options["meta_reserve"].substr(p1 < p2 ? 0 : p2, p1 - (p1 < p2 ? 0 : p2)));
-            min_meta_size = parse_size(options["meta_reserve"].substr(p1 < p2 ? p2 : 0, p1 < p2 ? options["meta_reserve"].size()-p2 : p2));
-        }
-        else if (p1 >= 0)
-            meta_reserve_multiple = stoull_full(options["meta_reserve"].substr(0, p1));
-        else
-            min_meta_size = parse_size(options["meta_reserve"]);
-    }
     meta_size = ((meta_size+1024*1024-1)/1024/1024)*1024*1024;
     meta_size *= meta_reserve_multiple;
-    if (meta_size < min_meta_size)
-        meta_size = min_meta_size;
+    if (meta_size < meta_reserve_min_size)
+        meta_size = meta_reserve_min_size;
     // Pick an SSD for journal&meta, balancing the number of serviced OSDs across SSDs
     int sel = -1;
     for (int i = 0; i < ssds.size(); i++)
@@ -596,6 +662,7 @@ int disk_tool_t::get_meta_partition(std::vector<vitastor_dev_info_t> & ssds, std
 
 int disk_tool_t::prepare(std::vector<std::string> devices)
 {
+    parse_meta_reserve();
     if (options.find("data_device") != options.end() && options["data_device"] != "")
     {
         if (options.find("hybrid") != options.end() ||
@@ -643,6 +710,8 @@ int disk_tool_t::prepare(std::vector<std::string> devices)
     std::vector<vitastor_dev_info_t> ssds;
     if (options.find("disable_data_fsync") == options.end())
         options["disable_data_fsync"] = "auto";
+    if (options["meta_format"] == "")
+        options["meta_format"] = std::to_string(BLOCKSTORE_META_FORMAT_HEAP);
     if (hybrid)
     {
         if (options.find("disable_meta_fsync") == options.end())

src/disk_tool/disk_tool_resize.cpp

@@ -25,6 +25,7 @@ struct resizer_data_moving_t
 int disk_tool_t::raw_resize()
 {
     int r;
+    parse_meta_reserve();
     // Parse parameters
     r = resize_parse_params();
     if (r != 0)
@@ -37,11 +38,12 @@ int disk_tool_t::raw_resize()
         {
             resize_init(hdr);
         },
-        [this](blockstore_heap_t *heap, heap_object_t *obj, uint32_t meta_block_num)
+        [this](blockstore_heap_t *heap, heap_entry_t *obj, uint32_t meta_block_num)
         {
-            for (auto wr = obj->get_writes(); wr; wr = wr->next())
+            for (auto wr = obj; wr; wr = heap->prev(wr))
             {
-                if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE)
+                if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE ||
+                    (wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_INTENT)
                 {
                     data_alloc->set(wr->big().block_num, true);
                 }
@@ -154,6 +156,7 @@ int disk_tool_t::resize_parse_params()
         ? parse_size(options["new_journal_len"]) : dsk.journal_len;
     new_meta_format = options.find("new_meta_format") != options.end()
         ? stoull_full(options["new_meta_format"]) : 0;
+    skip_obsolete = options.find("skip_obsolete") != options.end();
     if (new_data_len+new_data_offset > dsk.data_device_size)
         new_data_len = dsk.data_device_size-new_data_offset;
     if (new_meta_device == dsk.data_device && new_data_offset < new_meta_offset &&
@@ -223,17 +226,28 @@ void disk_tool_t::resize_init(blockstore_meta_header_v3_t *hdr)
     new_data_csum_size = (dsk.data_csum_type
         ? ((dsk.data_block_size+dsk.csum_block_size-1)/dsk.csum_block_size*(dsk.data_csum_type & 0xFF))
         : 0);
-    new_clean_entry_size = new_clean_entry_header_size + 2*new_clean_entry_bitmap_size + new_data_csum_size;
-    new_entries_per_block = dsk.meta_block_size/new_clean_entry_size;
-    uint64_t new_meta_blocks = 1 + (new_data_len/dsk.data_block_size + new_entries_per_block-1) / new_entries_per_block;
-    if (!new_meta_len)
+    if (new_meta_format != BLOCKSTORE_META_FORMAT_HEAP)
     {
-        new_meta_len = dsk.meta_block_size*new_meta_blocks;
+        new_clean_entry_size = new_clean_entry_header_size + 2*new_clean_entry_bitmap_size + new_data_csum_size;
+        new_entries_per_block = dsk.meta_block_size/new_clean_entry_size;
+        uint64_t new_meta_blocks = 1 + (new_data_len/dsk.data_block_size + new_entries_per_block-1) / new_entries_per_block;
+        if (!new_meta_len)
+        {
+            new_meta_len = dsk.meta_block_size*new_meta_blocks;
+        }
+        if (new_meta_len < dsk.meta_block_size*new_meta_blocks)
+        {
+            fprintf(stderr, "New metadata area size is too small, should be at least %ju bytes\n", dsk.meta_block_size*new_meta_blocks);
+            exit(1);
+        }
     }
-    if (new_meta_len < dsk.meta_block_size*new_meta_blocks)
+    else
     {
-        fprintf(stderr, "New metadata area size is too small, should be at least %ju bytes\n", dsk.meta_block_size*new_meta_blocks);
-        exit(1);
+        new_clean_entry_size = new_entries_per_block = 0;
+        if (!new_meta_len)
+        {
+            new_meta_len = dsk.meta_area_size;
+        }
     }
     // Check that new metadata, journal and data areas don't overlap
     if (new_meta_device == dsk.data_device && new_meta_offset < new_data_offset+new_data_len &&
@@ -517,32 +531,106 @@ int disk_tool_t::resize_write_new_journal()
     return 0;
 }
 
-int disk_tool_t::resize_rebuild_meta()
+void disk_tool_t::remap_big_write(heap_entry_t *wr)
 {
-    if (new_meta_format == BLOCKSTORE_META_FORMAT_HEAP)
+    uint64_t block_num = wr->big().block_num;
+    auto remap_it = data_remap.find(block_num);
+    if (remap_it != data_remap.end())
+        block_num = remap_it->second;
+    if (block_num < free_first || block_num >= total_blocks-free_last)
     {
-        new_dsk = dsk;
-        new_dsk.data_offset = new_data_offset;
-        new_dsk.data_len = new_data_len;
-        new_dsk.block_count = new_data_len / dsk.data_block_size;
-        new_dsk.journal_device = new_journal_device;
-        new_dsk.journal_offset = new_journal_offset;
-        new_dsk.journal_len = new_journal_len;
-        new_dsk.meta_device = new_meta_device;
-        new_dsk.meta_offset = new_meta_offset;
-        new_dsk.meta_area_size = new_meta_len;
-        new_dsk.meta_format = new_meta_format;
-        new_heap = new blockstore_heap_t(&new_dsk, NULL, 0);
-        new_meta_hdr = (blockstore_meta_header_v3_t *)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size);
-        memset(new_meta_hdr, 0, dsk.meta_block_size);
+        fprintf(stderr, "BUG: remapped block %ju not in range %ju..%ju\n", block_num, free_first, total_blocks-free_last);
+        exit(1);
+    }
+    block_num += data_idx_diff;
+    wr->big().block_num = block_num;
+    wr->crc32c = wr->calc_crc32c();
+}
+
+void disk_tool_t::remap_small_write(heap_entry_t *wr)
+{
+    if (new_meta_format == BLOCKSTORE_META_FORMAT_HEAP && wr->small().len > 0)
+    {
+        if (new_journal_ptr-new_journal_buf+wr->small().len > new_journal_len)
+        {
+            fprintf(stderr, "Small write data doesn't fit into the new buffer area\n");
+            exit(1);
+        }
+        memcpy(new_journal_ptr, buffer_area+wr->small().location, wr->small().len);
+        wr->small().location = new_journal_ptr-new_journal_buf;
+        new_journal_ptr += wr->small().len;
+        wr->crc32c = wr->calc_crc32c();
+    }
+}
+
+void disk_tool_t::fill_old_clean_entry(blockstore_heap_t *heap, heap_entry_t *big_wr)
+{
+    uint64_t block_num = big_wr->big().block_num;
+    clean_disk_entry *new_entry = (clean_disk_entry*)(new_meta_buf + dsk.meta_block_size +
+        dsk.meta_block_size*(block_num / new_entries_per_block) +
+        new_clean_entry_size*(block_num % new_entries_per_block));
+    new_entry->oid = (object_id){ .inode = big_wr->inode, .stripe = big_wr->stripe };
+    new_entry->version = big_wr->version;
+    memcpy(new_entry->bitmap, big_wr->get_ext_bitmap(heap), new_clean_entry_bitmap_size);
+    memcpy(new_entry->bitmap + new_clean_entry_bitmap_size, big_wr->get_int_bitmap(heap), new_clean_entry_bitmap_size);
+    memcpy(new_entry->bitmap + 2*new_clean_entry_bitmap_size, big_wr->get_checksums(heap), new_data_csum_size);
+    uint32_t *new_entry_csum = (uint32_t*)(((uint8_t*)new_entry) + new_clean_entry_size - 4);
+    *new_entry_csum = crc32c(0, new_entry, new_clean_entry_size - 4);
+}
+
+void disk_tool_t::fill_old_journal_entry(blockstore_heap_t *heap, heap_entry_t *wr)
+{
+    assert(wr->type() == BS_HEAP_SMALL_WRITE ||
+        wr->type() == BS_HEAP_BIG_WRITE ||
+        wr->type() == BS_HEAP_BIG_INTENT);
+    uint32_t je_size = ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_SMALL_WRITE
+        ? sizeof(journal_entry_small_write) + dsk.dirty_dyn_size(wr->small().offset, wr->small().len)
+        : sizeof(journal_entry_big_write) + dsk.dirty_dyn_size(0, dsk.data_block_size));
+    choose_journal_block(je_size);
+    journal_entry *je = (journal_entry*)(new_journal_ptr + new_journal_in_pos);
+    je->magic = JOURNAL_MAGIC;
+    je->type = (wr->entry_type & BS_HEAP_STABLE) ? JE_SMALL_WRITE_INSTANT : JE_SMALL_WRITE;
+    je->size = je_size;
+    je->crc32_prev = new_crc32_prev;
+    je->small_write.oid = (object_id){ .inode = wr->inode, .stripe = wr->stripe };
+    je->small_write.version = wr->version;
+    if (wr->type() == BS_HEAP_SMALL_WRITE)
+    {
+        je->small_write.offset = wr->small().offset;
+        je->small_write.len = wr->small().len;
+        je->small_write.data_offset = new_journal_data-new_journal_buf;
+        if (je->small_write.data_offset + je->small_write.len > new_journal_len)
+        {
+            fprintf(stderr, "Error: live entries don't fit to the new journal\n");
+            exit(1);
+        }
+        memcpy(new_journal_data, buffer_area+wr->small().location, je->small_write.len);
+        new_journal_data += je->small_write.len;
+        if (dsk.data_csum_type == 0 && wr->get_checksum(heap))
+            je->small_write.crc32_data = *wr->get_checksum(heap);
     }
     else
     {
-        new_meta_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_meta_len);
-        memset(new_meta_buf, 0, new_meta_len);
-        new_meta_hdr = (blockstore_meta_header_v3_t *)new_meta_buf;
+        je->big_write.location = wr->big_location(heap);
     }
-    std::vector<heap_write_t*> writes;
+    memcpy((uint8_t*)je + je->size, wr->get_ext_bitmap(heap), new_clean_entry_bitmap_size);
+    if (dsk.data_csum_type != 0 && wr->get_checksums(heap))
+    {
+        memcpy((uint8_t*)je + je->size + new_clean_entry_bitmap_size, wr->get_checksums(heap), heap->get_csum_size(wr));
+    }
+    je->crc32 = je_crc32(je);
+    new_journal_in_pos += je->size;
+    new_crc32_prev = je->crc32;
+}
+
+int disk_tool_t::resize_rebuild_meta()
+{
+    new_meta_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_meta_len);
+    memset(new_meta_buf, 0, new_meta_len);
+    new_meta_hdr = (blockstore_meta_header_v3_t *)new_meta_buf;
+    uint64_t new_meta_pos = dsk.meta_block_size;
+    uint64_t next_lsn = 0;
+    std::vector<heap_entry_t*> writes;
     int r = process_meta(
         [&](blockstore_meta_header_v3_t *hdr)
         {
@@ -554,7 +642,8 @@ int disk_tool_t::resize_rebuild_meta()
             new_meta_hdr->bitmap_granularity = dsk.bitmap_granularity ? dsk.bitmap_granularity : 4096;
             new_meta_hdr->data_csum_type = dsk.data_csum_type;
             new_meta_hdr->csum_block_size = dsk.csum_block_size;
-            new_meta_hdr->compacted_lsn = hdr->compacted_lsn;
+            new_meta_hdr->completed_lsn = hdr->completed_lsn;
+            new_meta_hdr->meta_area_size = new_meta_len;
             new_meta_hdr->header_csum = 0;
             new_meta_hdr->header_csum = crc32c(0, new_meta_hdr, new_meta_hdr->version == BLOCKSTORE_META_FORMAT_HEAP
                 ? sizeof(blockstore_meta_header_v3_t) : sizeof(blockstore_meta_header_v2_t));
@@ -563,119 +652,100 @@ int disk_tool_t::resize_rebuild_meta()
                 build_journal_start();
             }
         },
-        [&](blockstore_heap_t *heap, heap_object_t *obj, uint32_t meta_block_num)
+        [&](blockstore_heap_t *heap, heap_entry_t *obj, uint32_t meta_block_num)
         {
-            for (auto wr = obj->get_writes(); wr; wr = wr->next())
+            if (!obj)
             {
-                if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE)
+                // Finish
+                if (new_meta_format == BLOCKSTORE_META_FORMAT_HEAP)
                 {
-                    uint64_t block_num = wr->big().block_num;
-                    auto remap_it = data_remap.find(block_num);
-                    if (remap_it != data_remap.end())
-                        block_num = remap_it->second;
-                    if (block_num < free_first || block_num >= total_blocks-free_last)
+                    heap->fill_block_empty_space(new_meta_buf, new_meta_pos);
+                    new_meta_pos = (new_meta_pos/dsk.meta_block_size + 1) * dsk.meta_block_size;
+                    while (new_meta_pos < new_meta_len)
                     {
-                        fprintf(stderr, "BUG: remapped block %ju not in range %ju..%ju\n", block_num, free_first, total_blocks-free_last);
-                        exit(1);
-                    }
-                    block_num += data_idx_diff;
-                    wr->big().block_num = block_num;
-                }
-                else if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_SMALL_WRITE)
-                {
-                    if (new_heap && wr->small().len > 0)
-                    {
-                        if (new_journal_ptr-new_journal_buf+wr->small().len > new_journal_len)
-                        {
-                            fprintf(stderr, "Small write data doesn't fit into the new buffer area\n");
-                            exit(1);
-                        }
-                        memcpy(new_journal_ptr, buffer_area+wr->small().location, wr->small().len);
-                        wr->small().location = new_journal_ptr-new_journal_buf;
-                        new_journal_ptr += wr->small().len;
+                        heap->fill_block_empty_space(new_meta_buf, new_meta_pos);
+                        new_meta_pos += dsk.meta_block_size;
                     }
                 }
-                else if (!new_heap)
+                return;
+            }
+            auto handle_write = [&](heap_entry_t *wr, bool stable)
+            {
+                if (wr->type() == BS_HEAP_BIG_WRITE || wr->type() == BS_HEAP_BIG_INTENT)
                 {
-                    fprintf(stderr, "Object %jx:%jx can't be converted to the old format because it contains %s\n",
-                        obj->inode, obj->stripe, (wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_TOMBSTONE
-                            ? "a tombstone" : ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_INTENT_WRITE ? "an intent_write entry" : "an unknown entry"));
+                    remap_big_write(wr);
+                }
+                else if (wr->type() == BS_HEAP_SMALL_WRITE)
+                {
+                    remap_small_write(wr);
+                }
+                else if (wr->type() != BS_HEAP_DELETE && new_meta_format != BLOCKSTORE_META_FORMAT_HEAP)
+                {
+                    fprintf(stderr, "Object %jx:%jx can't be converted to the old format because it contains an entry of type 0x%x%s\n",
+                        wr->inode, wr->stripe, wr->entry_type,
+                        (wr->type() == BS_HEAP_INTENT_WRITE ? " (intent_write)" : ""));
                     exit(1);
                 }
-            }
-            if (new_heap)
-            {
-                // New -> New
-                new_heap->copy_object(obj, NULL);
-            }
-            else
-            {
-                // Fill journal
-                writes.clear();
-                for (auto wr = obj->get_writes(); wr; wr = wr->next())
+                if (new_meta_format == BLOCKSTORE_META_FORMAT_HEAP)
                 {
-                    writes.push_back(wr);
-                }
-                for (ssize_t i = writes.size()-2; i >= 0; i--)
-                {
-                    auto wr = writes[i];
-                    assert((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_SMALL_WRITE || wr->entry_type == BS_HEAP_BIG_WRITE);
-                    uint32_t je_size = ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_SMALL_WRITE
-                        ? sizeof(journal_entry_small_write) + dsk.dirty_dyn_size(wr->small().offset, wr->small().len)
-                        : sizeof(journal_entry_big_write) + dsk.dirty_dyn_size(0, dsk.data_block_size));
-                    choose_journal_block(je_size);
-                    journal_entry *je = (journal_entry*)(new_journal_ptr + new_journal_in_pos);
-                    je->magic = JOURNAL_MAGIC;
-                    je->type = (wr->entry_type & BS_HEAP_STABLE) ? JE_SMALL_WRITE_INSTANT : JE_SMALL_WRITE;
-                    je->size = je_size;
-                    je->crc32_prev = new_crc32_prev;
-                    je->small_write.oid = (object_id){ .inode = obj->inode, .stripe = obj->stripe };
-                    je->small_write.version = wr->version;
-                    if (wr->type() == BS_HEAP_SMALL_WRITE)
+                    // New -> New
+                    if ((new_meta_pos % dsk.meta_block_size) + wr->size > dsk.meta_block_size)
                     {
-                        je->small_write.offset = wr->small().offset;
-                        je->small_write.len = wr->small().len;
-                        je->small_write.data_offset = new_journal_data-new_journal_buf;
-                        if (je->small_write.data_offset + je->small_write.len > new_journal_len)
+                        heap->fill_block_empty_space(new_meta_buf, new_meta_pos);
+                        new_meta_pos = (new_meta_pos/dsk.meta_block_size + 1) * dsk.meta_block_size;
+                        if (new_meta_pos >= new_meta_len)
                         {
-                            fprintf(stderr, "Error: live entries don't fit to the new journal\n");
+                            fprintf(stderr, "New metadata doesn't fit into the provided area\n");
                             exit(1);
                         }
-                        memcpy(new_journal_data, buffer_area+wr->small().location, je->small_write.len);
-                        new_journal_data += je->small_write.len;
-                        if (dsk.data_csum_type == 0 && wr->get_checksum(heap))
-                            je->small_write.crc32_data = *wr->get_checksum(heap);
+                    }
+                    memcpy(new_meta_buf + new_meta_pos, wr, wr->size);
+                    new_meta_pos += wr->size;
+                    if (skip_obsolete && wr->type() == BS_HEAP_BIG_WRITE && stable)
+                    {
+                        // Skip older writes
+                        return false;
+                    }
+                }
+                else
+                {
+                    // New -> Old
+                    if (wr->type() == BS_HEAP_DELETE && stable)
+                    {
+                        // Object is deleted, skip it
+                        return false;
+                    }
+                    if (wr->type() == BS_HEAP_BIG_WRITE && stable)
+                    {
+                        fill_old_clean_entry(heap, wr);
+                        return false;
                     }
                     else
                     {
-                        je->big_write.location = wr->big_location(heap);
+                        writes.push_back(wr);
                     }
-                    memcpy((uint8_t*)je + je->size, wr->get_ext_bitmap(heap), new_clean_entry_bitmap_size);
-                    if (dsk.data_csum_type != 0 && wr->get_checksums(heap))
-                    {
-                        memcpy((uint8_t*)je + je->size + new_clean_entry_bitmap_size, wr->get_checksums(heap), wr->get_csum_size(heap));
-                    }
-                    je->crc32 = je_crc32(je);
-                    new_journal_in_pos += je->size;
-                    new_crc32_prev = je->crc32;
                 }
-                // New -> Old
-                if (writes[writes.size()-1]->entry_type == BS_HEAP_BIG_WRITE|BS_HEAP_STABLE)
+                return true;
+            };
+            if (new_meta_format != BLOCKSTORE_META_FORMAT_HEAP || skip_obsolete)
+            {
+                heap->iterate_with_stable(obj, obj->lsn, handle_write);
+            }
+            else
+            {
+                for (auto wr = obj; wr; wr = heap->prev(wr))
                 {
-                    auto big_wr = writes[writes.size()-1];
-                    uint64_t block_num = big_wr->big().block_num;
-                    clean_disk_entry *new_entry = (clean_disk_entry*)(new_meta_buf + dsk.meta_block_size +
-                        dsk.meta_block_size*(block_num / new_entries_per_block) +
-                        new_clean_entry_size*(block_num % new_entries_per_block));
-                    new_entry->oid = (object_id){ .inode = obj->inode, .stripe = obj->stripe };
-                    new_entry->version = big_wr->version;
-                    memcpy(new_entry->bitmap, big_wr->get_ext_bitmap(heap), new_clean_entry_bitmap_size);
-                    memcpy(new_entry->bitmap + new_clean_entry_bitmap_size, big_wr->get_int_bitmap(heap), new_clean_entry_bitmap_size);
-                    memcpy(new_entry->bitmap + 2*new_clean_entry_bitmap_size, big_wr->get_checksums(heap), new_data_csum_size);
-                    uint32_t *new_entry_csum = (uint32_t*)(((uint8_t*)new_entry) + new_clean_entry_size - 4);
-                    *new_entry_csum = crc32c(0, new_entry, new_clean_entry_size - 4);
+                    handle_write(wr, false);
                 }
             }
+            if (writes.size())
+            {
+                for (size_t i = writes.size(); i > 0; i--)
+                {
+                    fill_old_journal_entry(heap, writes[i-1]);
+                }
+                writes.clear();
+            }
         },
         [&](uint64_t block_num, clean_disk_entry *entry, uint8_t *bitmap)
         {
@@ -688,22 +758,36 @@ int disk_tool_t::resize_rebuild_meta()
                 exit(1);
             }
             block_num += data_idx_diff;
-            if (new_heap)
+            if (new_meta_format == BLOCKSTORE_META_FORMAT_HEAP)
             {
                 // Old -> New
-                uint8_t wr_buf[new_heap->get_max_write_entry_size()];
-                heap_write_t *wr = (heap_write_t*)wr_buf;
+                auto big_entry_size = sizeof(heap_big_write_t) + dsk.clean_entry_bitmap_size*2 +
+                    (!dsk.data_csum_type ? 0 : dsk.data_block_size/dsk.csum_block_size * (dsk.data_csum_type & 0xFF));
+                if ((new_meta_pos % dsk.meta_block_size) + big_entry_size > dsk.meta_block_size)
+                {
+                    new_meta_pos = (new_meta_pos % dsk.meta_block_size) + dsk.meta_block_size;
+                    if (new_meta_pos >= new_meta_len)
+                    {
+                        fprintf(stderr, "New metadata doesn't fit into the provided area\n");
+                        exit(1);
+                    }
+                }
+                heap_entry_t *wr = (heap_entry_t*)(new_meta_buf + new_meta_pos);
+                wr->size = big_entry_size;
                 wr->entry_type = BS_HEAP_BIG_WRITE|BS_HEAP_STABLE;
+                wr->inode = entry->oid.inode;
+                wr->stripe = entry->oid.stripe;
+                wr->version = entry->version;
                 wr->big().block_num = block_num;
-                wr->next_pos = 0;
-                wr->size = wr->get_size(new_heap);
+                wr->lsn = ++next_lsn;
                 if (bitmap)
                 {
-                    memcpy(wr->get_ext_bitmap(new_heap), bitmap, new_clean_entry_bitmap_size);
-                    memcpy(wr->get_int_bitmap(new_heap), bitmap+new_clean_entry_bitmap_size, new_clean_entry_bitmap_size);
-                    memcpy(wr->get_checksums(new_heap), bitmap+2*new_clean_entry_bitmap_size, new_data_csum_size);
+                    memcpy(((uint8_t*)wr) + sizeof(heap_big_write_t), bitmap, new_clean_entry_bitmap_size);
+                    memcpy(((uint8_t*)wr) + sizeof(heap_big_write_t) + new_clean_entry_bitmap_size, bitmap+new_clean_entry_bitmap_size, new_clean_entry_bitmap_size);
+                    memcpy(((uint8_t*)wr) + sizeof(heap_big_write_t) + 2*new_clean_entry_bitmap_size, bitmap+2*new_clean_entry_bitmap_size, new_data_csum_size);
                 }
-                new_heap->post_write(entry->oid, wr, NULL, NULL);
+                wr->crc32c = wr->calc_crc32c();
+                new_meta_pos += wr->size;
             }
             else
             {
@@ -735,34 +819,7 @@ int disk_tool_t::resize_write_new_meta()
         return 1;
     }
     lseek64(new_meta_fd, new_meta_offset, 0);
-    if (new_meta_buf)
-    {
-        write_blocking(new_meta_fd, new_meta_buf, new_meta_len);
-    }
-    else
-    {
-        assert(new_heap);
-        uint32_t new_meta_blocks = new_meta_len / dsk.meta_block_size - 1;
-        uint8_t *zero_block = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size);
-        memset(zero_block, 0, dsk.meta_block_size);
-        std::vector<iovec> iov;
-        iov.reserve(IOV_MAX);
-        iov.push_back((iovec){ .iov_base = new_meta_hdr, .iov_len = dsk.meta_block_size });
-        for (uint32_t i = 0; i < new_meta_blocks; i++)
-        {
-            uint8_t *data = new_heap->get_meta_block(i);
-            iov.push_back((iovec){ .iov_base = data ? data : zero_block, .iov_len = dsk.meta_block_size });
-            if (iov.size() >= IOV_MAX)
-            {
-                writev_blocking(new_meta_fd, iov.data(), iov.size());
-                iov.clear();
-            }
-        }
-        if (iov.size() > 0)
-            writev_blocking(new_meta_fd, iov.data(), iov.size());
-        free(zero_block);
-        zero_block = NULL;
-    }
+    write_blocking(new_meta_fd, new_meta_buf, new_meta_len);
     fsync(new_meta_fd);
     close(new_meta_fd);
     new_meta_fd = -1;
@@ -771,11 +828,6 @@ int disk_tool_t::resize_write_new_meta()
 
 void disk_tool_t::free_new_meta()
 {
-    if (new_heap)
-    {
-        delete new_heap;
-        new_heap = NULL;
-    }
     if ((uint8_t*)new_meta_hdr != new_meta_buf)
     {
         free(new_meta_hdr);

src/disk_tool/disk_tool_resize_auto.cpp

@@ -8,6 +8,7 @@
 
 int disk_tool_t::resize_data(std::string device)
 {
+    parse_meta_reserve();
     if (options.find("move_journal") == options.end() &&
         options.find("move_data") == options.end() &&
         options.find("journal_size") == options.end() &&
@@ -61,7 +62,7 @@ int disk_tool_t::resize_data(std::string device)
         dsk.journal_fd = old_journal_fd;
         dsk.meta_fd = old_meta_fd;
         dsk.data_fd = old_data_fd;
-        dsk.calc_lengths();
+        dsk.calc_lengths(true);
         dsk.journal_fd = -1;
         dsk.meta_fd = -1;
         dsk.data_fd = -1;
@@ -83,7 +84,8 @@ int disk_tool_t::resize_data(std::string device)
         ? move_options["new_meta_device"] : dsk.meta_device;
     // Calculate new data & meta offsets
     if (!new_meta_len)
-        new_meta_len = (dsk.meta_format == BLOCKSTORE_META_FORMAT_HEAP ? dsk.min_meta_len*2 : dsk.min_meta_len);
+        new_meta_len = (dsk.meta_format == BLOCKSTORE_META_FORMAT_HEAP ? dsk.meta_area_size : dsk.min_meta_len);
+    move_options["new_meta_len"] = std::to_string(new_meta_len);
     new_data_offset = 4096 + (new_journal_device == dsk.data_device ? new_journal_len : 0) +
         (new_meta_device == dsk.data_device ? new_meta_len : 0);
     new_data_offset += ((dsk.data_offset-new_data_offset) % dsk.data_block_size);

src/disk_tool/disk_tool_udev.cpp

@@ -367,6 +367,19 @@ int disk_tool_t::pre_exec_osd(std::string device)
         {
             return 1;
         }
+        if (sb["params"]["atomic_write_size"].uint64_value() > 4096 &&
+            sb["params"]["use_atomic_flag"].bool_value())
+        {
+            uint64_t atomic_write_size = get_atomic_write_size(sb["real_data_device"].string_value());
+            if (atomic_write_size < sb["params"]["atomic_write_size"].uint64_value())
+            {
+                fprintf(stderr, "Atomic write size is set to %ju in the OSD superblock but data device %s only supports %ju."
+                    " Did you enable IOMMU? Linux has a hardcoded max_hw_sectors_kb value for NVMe drives.\n",
+                    sb["params"]["atomic_write_size"].uint64_value(),
+                    sb["real_data_device"].string_value().c_str(), atomic_write_size);
+                return 1;
+            }
+        }
         if (json_is_true(sb["params"]["disable_meta_fsync"]) &&
             sb["real_meta_device"].string_value() != "" && sb["real_meta_device"] != sb["real_data_device"] &&
             check_disabled_cache(sb["real_meta_device"].string_value()) != 0)

src/disk_tool/disk_tool_upgrade.cpp

@@ -107,7 +107,7 @@ int disk_tool_t::upgrade_simple_unit(std::string unit)
                 dsk.open_data();
                 dsk.open_meta();
                 dsk.open_journal();
-                dsk.calc_lengths();
+                dsk.calc_lengths(true);
                 dsk.close_all();
             }
             catch (std::exception & e)

src/disk_tool/disk_tool_utils.cpp

@@ -56,8 +56,16 @@ static int check_queue_cache(std::string dev, std::string parent_dev)
     return trim(r) == "write through" ? 0 : -1;
 }
 
+uint64_t get_atomic_write_size(const std::string & dev)
+{
+    auto parent_dev = get_parent_device(dev);
+    if (parent_dev == "")
+        return 0;
+    return stoull_full(trim(read_file("/sys/block/"+parent_dev.substr(5)+"/queue/atomic_write_max_bytes")));
+}
+
 // returns 1 = warning, -1 = error, 0 = success
-int disable_cache(std::string dev)
+int disable_cache(const std::string & dev)
 {
     auto parent_dev = get_parent_device(dev);
     if (parent_dev == "")

src/kv/kv_cli.cpp

@@ -97,6 +97,7 @@ void kv_cli_t::parse_args(int narg, const char *args[])
                 "  dump [<start> [end]]\n"
                 "  dumpjson [<start> [end]]\n"
                 "  loadjson\n"
+                "  rescue\n"
                 "\n"
                 "<IMAGE> should be the name of Vitastor image with the DB.\n"
                 "Without <COMMAND>, you get an interactive DB shell.\n"
@@ -298,6 +299,50 @@ struct kv_cli_list_t
     int n = 0;
     std::function<void(int)> cb;
 
+    void handle_key(int res, const std::string & key, const std::string & value)
+    {
+        if (res < 0)
+        {
+            if (res != -ENOENT)
+                fprintf(stderr, "Error: %s (code %d)\n", strerror(-res), res);
+            if (format == 2)
+                write("\n}\n");
+            if (handle)
+                db->list_close(handle);
+            flush();
+            cb(res == -ENOENT ? 0 : res);
+            delete this;
+        }
+        else
+        {
+            if (format == 2)
+            {
+                write(n ? ",\n  " : "{\n  ");
+                write(addslashes(key));
+                write(": ");
+                write(addslashes(value));
+            }
+            else if (format == 1)
+            {
+                write("set ");
+                write(auto_addslashes(key));
+                write(" ");
+                write(value);
+                write("\n");
+            }
+            else
+            {
+                write(key);
+                write(" = ");
+                write(value);
+                write("\n");
+            }
+            n++;
+            if (handle)
+                db->list_next(handle, NULL);
+        }
+    }
+
     void write(const std::string & str)
     {
         if (buf.capacity() < KV_LIST_BUF_SIZE)
@@ -316,6 +361,7 @@ struct kv_cli_list_t
             if (res > 0)
                 done += res;
         }
+        buf.clear();
     }
 };
 
@@ -624,44 +670,18 @@ void kv_cli_t::handle_cmd(const std::vector<std::string> & cmd, std::function<vo
         lst->cb = std::move(cb);
         db->list_next(lst->handle, [lst](int res, const std::string & key, const std::string & value)
         {
-            if (res < 0)
-            {
-                if (res != -ENOENT)
-                    fprintf(stderr, "Error: %s (code %d)\n", strerror(-res), res);
-                if (lst->format == 2)
-                    lst->write("\n}\n");
-                lst->flush();
-                lst->db->list_close(lst->handle);
-                lst->cb(res == -ENOENT ? 0 : res);
-                delete lst;
-            }
-            else
-            {
-                if (lst->format == 2)
-                {
-                    lst->write(lst->n ? ",\n  " : "{\n  ");
-                    lst->write(addslashes(key));
-                    lst->write(": ");
-                    lst->write(addslashes(value));
-                }
-                else if (lst->format == 1)
-                {
-                    lst->write("set ");
-                    lst->write(auto_addslashes(key));
-                    lst->write(" ");
-                    lst->write(value);
-                    lst->write("\n");
-                }
-                else
-                {
-                    lst->write(key);
-                    lst->write(" = ");
-                    lst->write(value);
-                    lst->write("\n");
-                }
-                lst->n++;
-                lst->db->list_next(lst->handle, NULL);
-            }
+            lst->handle_key(res, key, value);
+        });
+    }
+    else if (opname == "rescue")
+    {
+        kv_cli_list_t *lst = new kv_cli_list_t;
+        lst->db = db;
+        lst->format = 2;
+        lst->cb = std::move(cb);
+        db->rescue([lst](int res, const std::string & key, const std::string & value)
+        {
+            lst->handle_key(res, key, value);
         });
     }
     else if (opname == "loadjson")

src/kv/kv_db.cpp

@@ -96,7 +96,7 @@ struct kv_block_t
 
     void set_data_size();
     static int kv_size(const std::string & key, const std::string & value);
-    int parse(uint64_t offset, uint8_t *data, int size);
+    int parse(uint64_t offset, uint8_t *data, int size, bool allow_empty = false);
     bool serialize(uint8_t *data, int size);
     void apply_change();
     void cancel_change();
@@ -139,7 +139,6 @@ struct kv_db_t
     uint64_t next_free = 0;
     uint32_t kv_block_size = 0;
     uint32_t ino_block_size = 0;
-    bool immediate_commit = false;
     uint64_t memory_limit = 128*1024*1024;
     uint64_t evict_unused_age = 1000;
     uint64_t evict_max_misses = 10;
@@ -173,6 +172,7 @@ struct kv_db_t
     void open(inode_t inode_id, json11::Json cfg, std::function<void(int)> cb);
     void set_config(json11::Json cfg);
     void close(std::function<void()> cb);
+    void rescue(std::function<void(int res, const std::string & key, const std::string & value)> cb);
 
     void find_size(uint64_t min, uint64_t max, int phase, std::function<void(int, uint64_t)> cb);
     void run_continue_update(uint64_t offset);
@@ -243,13 +243,13 @@ static std::string read_string(uint8_t *data, int size, int *pos)
     return key;
 }
 
-int kv_block_t::parse(uint64_t offset, uint8_t *data, int size)
+int kv_block_t::parse(uint64_t offset, uint8_t *data, int size, bool allow_empty)
 {
     kv_stored_block_t *blk = (kv_stored_block_t *)data;
     if (blk->magic == 0 || blk->type == KV_EMPTY)
     {
         // empty block
-        if (offset != 0)
+        if (!allow_empty)
             fprintf(stderr, "K/V: Block %ju is %s\n", offset, blk->magic == 0 ? "empty" : "cleared");
         return -ENOTBLK;
     }
@@ -389,6 +389,10 @@ bool kv_block_t::serialize(uint8_t *buf, int size)
             return false;
         blk->items++;
     }
+    if (pos < size)
+    {
+        memset(buf+pos, 0, size-pos);
+    }
     return true;
 }
 
@@ -524,7 +528,6 @@ void kv_db_t::open(inode_t inode_id, json11::Json cfg, std::function<void(int)>
         return;
     }
     this->inode_id = inode_id;
-    this->immediate_commit = cli->get_immediate_commit(inode_id);
     this->ino_block_size = pool_cfg.data_block_size * pg_data_size;
     this->kv_block_size = kv_block_size;
     this->next_free = 0;
@@ -542,6 +545,127 @@ void kv_db_t::open(inode_t inode_id, json11::Json cfg, std::function<void(int)>
     });
 }
 
+struct kv_rescue_t
+{
+    kv_db_t *db = NULL;
+    uint64_t size = 0;
+    uint64_t pos = 0;
+    uint64_t cur_size = 0;
+    uint64_t cur_offset = 0;
+    int state = 0;
+    kv_block_t blk;
+    std::vector<uint8_t> buf;
+    std::function<void(int res, const std::string & key, const std::string & value)> cb;
+
+    void finish(int retval)
+    {
+        auto cb = std::move(this->cb);
+        cb(retval, "", "");
+        delete this;
+    }
+
+    void send_read()
+    {
+        if (pos >= size)
+        {
+            finish(-ENOENT);
+            return;
+        }
+        if (!buf.size())
+        {
+            buf.resize(1048576);
+        }
+        cluster_op_t *op = new cluster_op_t;
+        op->opcode = OSD_OP_READ;
+        op->inode = db->inode_id;
+        op->offset = pos;
+        cur_size = op->len = pos+buf.size() < size ? buf.size() : size-pos;
+        op->iov.push_back(buf.data(), cur_size);
+        op->callback = [=](cluster_op_t *op)
+        {
+            if (op->retval != op->len)
+            {
+                // error
+                finish(op->retval >= 0 ? -EIO : op->retval);
+                return;
+            }
+            state = 2;
+            cur_offset = 0;
+            run();
+            delete op;
+        };
+        db->cli->execute(op);
+        state = 1;
+    }
+
+    void parse_block()
+    {
+        if (cur_offset < cur_size)
+        {
+            blk = {};
+            int err = blk.parse(pos+cur_offset, buf.data()+cur_offset, db->kv_block_size, true);
+            if (err != 0)
+            {
+            }
+            else if (blk.type == KV_LEAF || blk.type == KV_LEAF_SPLIT)
+            {
+                for (auto it = blk.data.begin(); it != blk.data.end(); it++)
+                {
+                    cb(0, it->first, it->second);
+                }
+            }
+            cur_offset += db->kv_block_size;
+        }
+        else
+        {
+            pos += cur_size;
+            state = 0;
+        }
+    }
+
+    void run()
+    {
+        while (true)
+        {
+            if (state == 0)
+            {
+                send_read();
+                return;
+            }
+            else if (state == 1)
+            {
+                return;
+            }
+            else if (state == 2)
+            {
+                parse_block();
+            }
+        }
+    }
+};
+
+void kv_db_t::rescue(std::function<void(int res, const std::string & key, const std::string & value)> cb)
+{
+    if (!inode_id || closing)
+    {
+        cb(-EINVAL, "", "");
+        return;
+    }
+    find_size(0, 0, 1, [=](int res, uint64_t size)
+    {
+        if (res < 0)
+        {
+            cb(res, "", "");
+            return;
+        }
+        kv_rescue_t *st = new kv_rescue_t();
+        st->db = this;
+        st->size = size;
+        st->cb = cb;
+        st->run();
+    });
+}
+
 void kv_db_t::set_config(json11::Json cfg)
 {
     this->memory_limit = cfg["kv_memory_limit"].is_null() ? 128*1024*1024 : cfg["kv_memory_limit"].uint64_value();
@@ -563,7 +687,6 @@ void kv_db_t::close(std::function<void()> cb)
         next_free = 0;
         kv_block_size = 0;
         ino_block_size = 0;
-        immediate_commit = false;
         block_cache.clear();
         known_versions.clear();
         cb();
@@ -939,7 +1062,7 @@ static void get_block(kv_db_t *db, uint64_t offset, int cur_level, int recheck_p
                 del_block_level(db, blk);
                 *blk = {};
             }
-            int err = blk->parse(op->offset, (uint8_t*)op->iov.buf[0].iov_base, op->len);
+            int err = blk->parse(op->offset, (uint8_t*)op->iov.buf[0].iov_base, op->len, op->offset == 0);
             if (err == 0)
             {
                 blk->level = cur_level;
@@ -1002,6 +1125,8 @@ kv_op_t::~kv_op_t()
     {
         done = true;
         db->active_ops--;
+        if (!db->active_ops && db->closing)
+            db->close(db->on_close);
     }
 }
 
@@ -1272,22 +1397,7 @@ static void write_block(kv_db_t *db, kv_block_t *blk, std::function<void(int)> c
             }
         }
         delete op;
-        if (res < 0 || db->immediate_commit)
-        {
-            cb(res);
-        }
-        else
-        {
-            op = new cluster_op_t;
-            op->opcode = OSD_OP_SYNC;
-            op->callback = [cb](cluster_op_t *op)
-            {
-                auto res = op->retval;
-                delete op;
-                cb(res);
-            };
-            db->cli->execute(op);
-        }
+        cb(res);
     };
     db->cli->execute(op);
 }
@@ -1662,6 +1772,7 @@ void kv_op_t::update_block(int path_pos, bool is_delete, const std::string & key
         {
             if (res < 0)
             {
+                blk->cancel_change();
                 auto blk_offset = blk->offset;
                 del_block_level(db, blk);
                 db->block_cache.erase(blk_offset);
@@ -1794,6 +1905,7 @@ void kv_op_t::update_block(int path_pos, bool is_delete, const std::string & key
             {
                 if (write_res < 0)
                 {
+                    blk->cancel_change();
                     auto blk_offset = blk->offset;
                     del_block_level(db, blk);
                     db->block_cache.erase(blk_offset);
@@ -1981,6 +2093,11 @@ void vitastorkv_dbw_t::close(std::function<void()> cb)
     db->close(cb);
 }
 
+void vitastorkv_dbw_t::rescue(std::function<void(int res, const std::string & key, const std::string & value)> cb)
+{
+    db->rescue(cb);
+}
+
 void vitastorkv_dbw_t::get(const std::string & key, std::function<void(int res, const std::string & value)> cb, bool cached)
 {
     auto *op = new kv_op_t;