Use 32-bit big write location (OK for up to 512 TB OSDs)

This is required to:
1) later inline the last "big_write" entry into the object to slightly reduce memory usage
2) eliminate an ugly hack where entry type is determined by its size
3) make the storage scheme extensible i.e. when adding new entry types

.gitea/workflows/test.yml

@@ -306,78 +306,6 @@ jobs:
           echo ""
         done
 
-  test_old_interrupted_rebalance:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: OLD=1 /root/vitastor/tests/test_interrupted_rebalance.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_old_interrupted_rebalance_imm:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: OLD=1 IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_interrupted_rebalance.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_old_interrupted_rebalance_ec:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: OLD=1 SCHEME=ec /root/vitastor/tests/test_interrupted_rebalance.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_old_interrupted_rebalance_ec_imm:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: OLD=1 SCHEME=ec IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_interrupted_rebalance.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
   test_create_halfhost:
     runs-on: ubuntu-latest
     needs: build
@@ -414,24 +342,6 @@ jobs:
           echo ""
         done
 
-  test_level_placement:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_level_placement.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
   test_snapshot:
     runs-on: ubuntu-latest
     needs: build
@@ -468,42 +378,6 @@ jobs:
           echo ""
         done
 
-  test_old_snapshot:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 /root/vitastor/tests/test_snapshot.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_old_snapshot_ec:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 SCHEME=ec /root/vitastor/tests/test_snapshot.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
   test_minsize_1:
     runs-on: ubuntu-latest
     needs: build
@@ -540,24 +414,6 @@ jobs:
           echo ""
         done
 
-  test_old_move_reappear:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 /root/vitastor/tests/test_move_reappear.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
   test_degraded:
     runs-on: ubuntu-latest
     needs: build
@@ -648,42 +504,6 @@ jobs:
           echo ""
         done
 
-  test_old_snapshot_chain:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 /root/vitastor/tests/test_snapshot_chain.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_old_snapshot_chain_ec:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 SCHEME=ec /root/vitastor/tests/test_snapshot_chain.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
   test_snapshot_down:
     runs-on: ubuntu-latest
     needs: build
@@ -720,78 +540,6 @@ jobs:
           echo ""
         done
 
-  test_old_snapshot_down:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 /root/vitastor/tests/test_snapshot_down.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_old_snapshot_down_ec:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 SCHEME=ec /root/vitastor/tests/test_snapshot_down.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_kv_stress:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_kv_stress.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_kv_stress_imm:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_kv_stress.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
   test_splitbrain:
     runs-on: ubuntu-latest
     needs: build
@@ -882,78 +630,6 @@ jobs:
           echo ""
         done
 
-  test_old_rebalance_verify:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: OLD=1 /root/vitastor/tests/test_rebalance_verify.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_old_rebalance_verify_imm:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: OLD=1 IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_rebalance_verify.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_old_rebalance_verify_ec:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: OLD=1 SCHEME=ec /root/vitastor/tests/test_rebalance_verify.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_old_rebalance_verify_ec_imm:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: OLD=1 SCHEME=ec IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_rebalance_verify.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
   test_dd:
     runs-on: ubuntu-latest
     needs: build
@@ -1062,7 +738,7 @@ jobs:
           echo ""
         done
 
-  test_old_write:
+  test_write_no_same:
     runs-on: ubuntu-latest
     needs: build
     container: ${{env.TEST_IMAGE}}:${{github.sha}}
@@ -1070,61 +746,7 @@ jobs:
     - name: Run test
       id: test
       timeout-minutes: 3
-      run: OLD=1 /root/vitastor/tests/test_write.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_old_write_xor:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 SCHEME=xor /root/vitastor/tests/test_write.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_write_old_iothreads:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: TEST_NAME=old_iothreads OLD=1 GLOBAL_CONFIG=',"client_iothread_count":4' /root/vitastor/tests/test_write.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_old_write_no_same:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 /root/vitastor/tests/test_write_no_same.sh
+      run: /root/vitastor/tests/test_write_no_same.sh
     - name: Print logs
       if: always() && steps.test.outcome == 'failure'
       run: |
@@ -1404,42 +1026,6 @@ jobs:
           echo ""
         done
 
-  test_old_resize:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 /root/vitastor/tests/test_resize.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_old_resize_auto:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 /root/vitastor/tests/test_resize_auto.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
   test_osd_tags:
     runs-on: ubuntu-latest
     needs: build
@@ -1530,78 +1116,6 @@ jobs:
           echo ""
         done
 
-  test_old_enospc:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 /root/vitastor/tests/test_enospc.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_old_enospc_xor:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 SCHEME=xor /root/vitastor/tests/test_enospc.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_old_enospc_imm:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_enospc.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_old_enospc_imm_xor:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 IMMEDIATE_COMMIT=1 SCHEME=xor /root/vitastor/tests/test_enospc.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
   test_scrub:
     runs-on: ubuntu-latest
     needs: build
@@ -1710,240 +1224,6 @@ jobs:
           echo ""
         done
 
-  test_old_scrub:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 /root/vitastor/tests/test_scrub.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_old_scrub_zero_osd_2:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 ZERO_OSD=2 /root/vitastor/tests/test_scrub.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_old_scrub_xor:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 SCHEME=xor /root/vitastor/tests/test_scrub.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_old_scrub_pg_size_3:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 PG_SIZE=3 /root/vitastor/tests/test_scrub.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_old_scrub_pg_size_6_pg_minsize_4_osd_count_6_ec:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 PG_SIZE=6 PG_MINSIZE=4 OSD_COUNT=6 SCHEME=ec /root/vitastor/tests/test_scrub.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_old_scrub_ec:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 SCHEME=ec /root/vitastor/tests/test_scrub.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_old_partwr_csum:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: OLD=1 /root/vitastor/tests/test_partwr_csum.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_heal_old_csum_32k_dmj:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: TEST_NAME=old_csum_32k_dmj OLD=1 OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k --inmemory_metadata false --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_heal_old_csum_32k_dj:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: TEST_NAME=old_csum_32k_dj  OLD=1 OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_heal_old_csum_32k:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: TEST_NAME=old_csum_32k     OLD=1 OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_heal_old_csum_4k_dmj:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: TEST_NAME=old_csum_4k_dmj  OLD=1 OSD_ARGS="--data_csum_type crc32c --inmemory_metadata false --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_heal_old_csum_4k_dj:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: TEST_NAME=old_csum_4k_dj   OLD=1 OSD_ARGS="--data_csum_type crc32c --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_heal_old_csum_4k:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: TEST_NAME=old_csum_4k      OLD=1 OSD_ARGS="--data_csum_type crc32c" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
   test_nfs:
     runs-on: ubuntu-latest
     needs: build
@@ -1962,21 +1242,3 @@ jobs:
           echo ""
         done
 
-  test_nfs_unaligned_append:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_nfs_unaligned_append.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-

.gitea/workflows/tests-to-yaml.pl

@@ -38,10 +38,6 @@ for my $line (<>)
             {
                 $test_name .= '_antietcd';
             }
-            elsif ($1 eq 'OLD')
-            {
-                $test_name =~ s/^test_/test_old_/s;
-            }
             else
             {
                 $test_name .= '_'.lc($1).'_'.$2;

CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "3.0.3")
+set(VITASTOR_VERSION "2.4.3")
 
 include(CTest)
 

README-ru.md

@@ -26,15 +26,11 @@ Vitastor поддерживает QEMU-драйвер, протоколы UBLK,
 
 ## Презентации и записи докладов
 
-- KuberConf'2025: [видео](https://vitastor.io/presentation/kuberconf.webm)
-- Highload'2025: [видео](https://vitastor.io/presentation/hl2025/hl2025.webm),
-  [на youtube](https://www.youtube.com/watch?v=0R8MLjFtz7g), презентация
-  ([на русском](https://vitastor.io/presentation/hl2025/), [на английском](https://vitastor.io/presentation/hl2025/en.html))
-- Highload'2022: презентация ([на русском](https://vitastor.io/presentation/highload/highload.html)),
-  [видео](https://vitastor.io/presentation/highload/talk.webm)
 - DevOpsConf'2021: презентация ([на русском](https://vitastor.io/presentation/devopsconf/devopsconf.html),
   [на английском](https://vitastor.io/presentation/devopsconf/devopsconf_en.html)),
   [видео](https://vitastor.io/presentation/devopsconf/talk.webm)
+- Highload'2022: презентация ([на русском](https://vitastor.io/presentation/highload/highload.html)),
+  [видео](https://vitastor.io/presentation/highload/talk.webm)
 
 ## Документация
 

README.md

@@ -26,15 +26,11 @@ Read more details in the documentation. You can start from here: [Quick Start](d
 
 ## Talks and presentations
 
-- KuberConf'2025: [video](https://vitastor.io/presentation/kuberconf.webm)
-- Highload'2025: [video](https://vitastor.io/presentation/hl2025/hl2025.webm),
-  [youtube](https://www.youtube.com/watch?v=0R8MLjFtz7g), presentation
-  ([in Russian](https://vitastor.io/presentation/hl2025/), [in English](https://vitastor.io/presentation/hl2025/en.html))
-- Highload'2022: presentation ([in Russian](https://vitastor.io/presentation/highload/highload.html)),
-  [video](https://vitastor.io/presentation/highload/talk.webm)
 - DevOpsConf'2021: presentation ([in Russian](https://vitastor.io/presentation/devopsconf/devopsconf.html),
   [in English](https://vitastor.io/presentation/devopsconf/devopsconf_en.html)),
   [video](https://vitastor.io/presentation/devopsconf/talk.webm)
+- Highload'2022: presentation ([in Russian](https://vitastor.io/presentation/highload/highload.html)),
+  [video](https://vitastor.io/presentation/highload/talk.webm)
 
 ## Documentation
 

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.3
+VITASTOR_VERSION ?= v2.4.3
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v3.0.3
+          image: vitalif/vitastor-csi:v2.4.3
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v3.0.3
+          image: vitalif/vitastor-csi:v2.4.3
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "3.0.3"
+    vitastorCSIDriverVersion = "2.4.3"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (3.0.3-1) unstable; urgency=medium
+vitastor (2.4.3-1) unstable; urgency=medium
 
   * Bugfixes
 

docker/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.3
+VITASTOR_VERSION ?= v2.4.3
 
 all: build push
 

docker/etc/vitastor/docker.conf

@@ -4,7 +4,7 @@
 #
 
 # Desired Vitastor version
-VITASTOR_VERSION=v3.0.3
+VITASTOR_VERSION=v2.4.3
 
 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here

docs/config/layout-osd.en.md

@@ -9,7 +9,6 @@
 These parameters apply to OSDs, are fixed at the moment of OSD drive
 initialization and can't be changed after it without losing data.
 
-- [meta_format](#meta_format)
 - [data_device](#data_device)
 - [meta_device](#meta_device)
 - [journal_device](#journal_device)
@@ -28,21 +27,6 @@ initialization and can't be changed after it without losing data.
 - [data_csum_type](#data_csum_type)
 - [csum_block_size](#csum_block_size)
 
-## meta_format
-
-- Type: integer
-- Default: 3
-
-OSD store implementation version and on-disk metadata format.
-
-Three versions are currently supported: 3, 2 and 1.
-- 3 the new log-structured store, it's overall faster, has lower Write
-  Amplification, which may be even close to 1 (i.e. almost no extra writes)
-  if your SSDs support atomic writes (see [atomic_write_size](osd.en.md#atomic_write_size)).
-- 2 is the old stable store from Vitastor 0.9-2.x.
-- 1 is the same old store but with a legacy metadata format from Vitastor
-  versions to up 0.8.x, without any support for checksums.
-
 ## data_device
 
 - Type: string

docs/config/layout-osd.ru.md

@@ -10,7 +10,6 @@
 дисковые параметры, задаются в момент инициализации дисков OSD и не могут быть
 изменены после этого без потери данных.
 
-- [meta_format](#meta_format)
 - [data_device](#data_device)
 - [meta_device](#meta_device)
 - [journal_device](#journal_device)
@@ -29,23 +28,6 @@
 - [data_csum_type](#data_csum_type)
 - [csum_block_size](#csum_block_size)
 
-## meta_format
-
-- Тип: целое число
-- Значение по умолчанию: 3
-
-Версия реализации дискового хранилища OSD и дискового формата метаданных.
-
-Поддерживаются три версии: 3, 2 и 1.
-- 3 - новое лог-структурированное хранилище, в целом более быстрое, со
-  сниженным фактором амплификации записи, который может составлять около 1
-  (то есть, практически без лишней служебной записи), если ваши SSD
-  поддерживают атомарную запись (см. [atomic_write_size](osd.ru.md#atomic_write_size)).
-- 2 - старое стабильное хранилище из версий Vitastor 0.9-2.x.
-- 1 - то же самое стабильное хранилище, но с ещё более старым форматом
-  метаданных из версий Vitastor до 0.8.x, без какой-либо поддержки
-  контрольных сумм.
-
 ## data_device
 
 - Тип: строка

docs/config/osd.en.md

@@ -65,10 +65,6 @@ with an OSD restart or, for some of them, even without restarting by updating co
 - [allow_net_split](#allow_net_split)
 - [enable_pg_locks](#enable_pg_locks)
 - [pg_lock_retry_interval_ms](#pg_lock_retry_interval_ms)
-- [atomic_write_size](#atomic_write_size)
-- [use_atomic_flag](#use_atomic_flag)
-- [pg_reshard_chunk_size](#pg_reshard_chunk_size)
-- [pg_reshard_chunk_pause_ms](#pg_reshard_chunk_pause_ms)
 
 ## bind_address
 
@@ -670,66 +666,3 @@ Use this parameter to enable or disable this function for all pools.
 - Default: 100
 
 Retry interval for failed PG lock attempts.
-
-## atomic_write_size
-
-- Type: integer
-- Default: 4096
-
-Maximum data device atomic write size allowed for OSD to use.
-
-Atomic writes allow to reduce the Write Amplification factor with the new store
-([meta_format](layout-osd.en.md#meta_format)=3) to almost 1 (i.e. almost no extra writes)
-with replicated pools and reach the best possible write performance.
-
-Default value is auto-detected during OSD initialization from
-`/sys/block/xx/queue/atomic_write_max_bytes` or assumed to be 4096 bytes
-because all known disks support 4 KB atomic writes. Auto-detection is only used for
-NVMe disks because SAS disks require the explicit WRITE ATOMIC command which requires
-RWF_ATOMIC (see below [#use_atomic_flag]) but that flag works incorrectly in current
-Linux versions.
-
-You can also check if your NVMe drives support atomic writes by running
-the command `nvme id-ctrl /dev/nvme0n1 | grep awupf`. If the reported value,
-plus 1, multiplied by the currently selected block size of the NVMe,
-is more than 4 KB, then the new store can utilize it for better performance.
-The only drives known to support it currently are [Micron and Kioxia](../intro/quickstart.en.md).
-
-Atomic writes allow to skip double data writes in replicated pools, thus
-reducing Write Amplification and improving write performance up to 2 times.
-
-## use_atomic_flag
-
-- Type: boolean
-
-This option controls whether Vitastor OSDs use RWF_ATOMIC write flag with atomic writes.
-This flag is supported since Linux 6.11 and adds some safety to atomic writes - the kernel
-guarantees to not fragment write requests with it and also to check them against the actual
-device atomic write capabilities.
-
-However, the option is disabled by default because the flag is currently UNUSABLE - Linux
-incorrectly requires writes with that flag to be of power-of-2 length and length-aligned.
-I.e., for example, 12 KB writes and not-8-KB aligned 8 KB writes are forbidden by the kernel,
-even though the NVMe specification allows them.
-
-For NVMe disks with `scheduler=none` writes aren't fragmented anyway so it's not a big deal.
-However, you can rebuild your kernel with [this patch](../../patches/linux-fix-atomic-write-checks.diff)
-and turn this option on. It will make your atomic writes a bit safer.
-
-## pg_reshard_chunk_size
-
-- Type: integer
-- Default: 100000
-
-Pool PG count change is a CPU-intensive operation because OSDs store the full object database
-in memory and have to move all entries between old and new PGs. Thus it's performed in chunks,
-with pauses between chunks to prevent blocking OSD's event loop and other clients' operations.
-This option sets the maximum number of object is a chunk. Moving 100k objects usually takes
-50-100ms. Chunk size equal to 0 means unlimited.
-
-## pg_reshard_chunk_pause_ms
-
-- Type: milliseconds
-- Default: 100
-
-This option sets the interval between handling two PG count change chunks.

docs/config/osd.ru.md

@@ -66,10 +66,6 @@
 - [allow_net_split](#allow_net_split)
 - [enable_pg_locks](#enable_pg_locks)
 - [pg_lock_retry_interval_ms](#pg_lock_retry_interval_ms)
-- [atomic_write_size](#atomic_write_size)
-- [use_atomic_flag](#use_atomic_flag)
-- [pg_reshard_chunk_size](#pg_reshard_chunk_size)
-- [pg_reshard_chunk_pause_ms](#pg_reshard_chunk_pause_ms)
 
 ## bind_address
 
@@ -703,72 +699,3 @@ pg_minsize OSD во время переключений, что может по
 - Значение по умолчанию: 100
 
 Интервал повтора неудачных попыток блокировки PG.
-
-## atomic_write_size
-
-- Тип: целое число
-- Значение по умолчанию: 4096
-
-Максимальный размер атомарной записи на диск данных, который OSD разрешено использовать.
-
-Поддержка атомарной записи позволяет снизить мультипликатор записи (Write Amplification)
-на диск с новым хранилищем ([meta_format](layout-osd.ru.md#meta_format)=3)
-практически до 1 (то есть, почти до нулевого объёма лишней записи) в реплицированных
-пулах и достигнуть наилучшей возможной производительности записи.
-
-Значение по умолчанию авто-определяется во время инициализации OSD из
-`/sys/block/xx/queue/atomic_write_max_bytes` либо принимается равным 4096,
-так как все известные диски поддерживают атомарную запись 4 КБ блоков.
-Автоопределение применяется только для NVMe-дисков, так как SAS диски требуют
-использования отдельной команды WRITE ATOMIC, а для неё нужен флаг RWF_ATOMIC
-(см. ниже [#use_atomic_flag]), а он в текущих версиях Linux работает некорректно.
-
-Вы также можете проверить, поддерживают ли ваши NVMe-диски атомарную запись,
-с помощью команды `nvme id-ctrl /dev/nvme0n1 | grep awupf`. Если значение awupf
-плюс 1, умноженное на текущий выбранный размер блока NVMe-диска, больше 4 КБ,
-то новое хранилище может использовать атомарные записи для достижения лучшей
-производительности. Единственные известные диски, которые поддерживают это сейчас -
-[Micron и Kioxia](../intro/quickstart.ru.md).
-
-Атомарная запись позволяет не использовать двойную запись данных (в журнал и на
-устройство данных) в реплицированных пулах и таким образом снижает амплификацию
-записи (объём служебной записи на диск) и улучшает производительность записи
-вплоть до 2-х кратного прироста.
-
-## use_atomic_flag
-
-- Тип: булево (да/нет)
-
-Данная опция контролирует использование Vitastor OSD флага RWF_ATOMIC при атомарной записи
-блоков. Этот флаг поддерживается, начиная с версии ядра Linux 6.11 и добавляет немного корректности
-атомарным записям - ядро гарантирует отсутствие фрагментации запросов записи с этим флагом и
-проверяет их на соответствие реальным возможностям устройства.
-
-Однако, данная опция по умолчанию отключена, так как флаг в текущих версиях Linux работает
-абсолютно НЕКОРРЕКТНО - при нём Linux требует, чтобы запросы записи имели длину, равную
-степени двойки и были выровнены на эту длину. То есть, например, 12 КБ запросы записи, а также
-8 КБ запросы записи по не-кратному 8 КБ смещению запрещаются ядром, хотя спецификация NVMe их
-разрешает.
-
-Для NVMe-дисков с `scheduler=none` запросы записи и так не фрагментируются, так что это не так
-уж и важно, однако вы можете пересобрать своё ядро с [этим патчем](../../patches/linux-fix-atomic-write-checks.diff)
-и включить данную опцию. Это сделает вашу атомарную запись капельку безопаснее.
-
-## pg_reshard_chunk_size
-
-- Тип: целое число
-- Значение по умолчанию: 100000
-
-Изменение числа PG в пуле заметно загружает процессор, так как OSD хранят полную базу данных
-объектов в памяти и им приходится перемещать все записи объектов между старыми и новыми PG.
-Поэтому изменение применяется порциями, с паузами между порциями, чтобы не блокировать обработку
-событий OSD и операции остальных клиентов. Данная опция задаёт максимальное число объектов
-в порции. Перемещение 100 тысяч объектов (значение по умолчанию) обычно занимает порядка
-50-100 миллисекунд. Значение опции 0 отключает лимит размера порции.
-
-## pg_reshard_chunk_pause_ms
-
-- Тип: миллисекунды
-- Значение по умолчанию: 100
-
-Данная опция задаёт интервал между обработкой двух порций изменения числа PG пулов.

docs/config/src/layout-osd.yml

@@ -1,28 +1,3 @@
-- name: meta_format
-  type: int
-  default: 3
-  info: |
-    OSD store implementation version and on-disk metadata format.
-
-    Three versions are currently supported: 3, 2 and 1.
-    - 3 the new log-structured store, it's overall faster, has lower Write
-      Amplification, which may be even close to 1 (i.e. almost no extra writes)
-      if your SSDs support atomic writes (see [atomic_write_size](osd.en.md#atomic_write_size)).
-    - 2 is the old stable store from Vitastor 0.9-2.x.
-    - 1 is the same old store but with a legacy metadata format from Vitastor
-      versions to up 0.8.x, without any support for checksums.
-  info_ru: |
-    Версия реализации дискового хранилища OSD и дискового формата метаданных.
-
-    Поддерживаются три версии: 3, 2 и 1.
-    - 3 - новое лог-структурированное хранилище, в целом более быстрое, со
-      сниженным фактором амплификации записи, который может составлять около 1
-      (то есть, практически без лишней служебной записи), если ваши SSD
-      поддерживают атомарную запись (см. [atomic_write_size](osd.ru.md#atomic_write_size)).
-    - 2 - старое стабильное хранилище из версий Vitastor 0.9-2.x.
-    - 1 - то же самое стабильное хранилище, но с ещё более старым форматом
-      метаданных из версий Vitastor до 0.8.x, без какой-либо поддержки
-      контрольных сумм.
 - name: data_device
   type: string
   info: |

docs/config/src/osd.yml

@@ -801,108 +801,3 @@
   default: 100
   info: Retry interval for failed PG lock attempts.
   info_ru: Интервал повтора неудачных попыток блокировки PG.
-- name: atomic_write_size
-  type: int
-  default: 4096
-  info: |
-    Maximum data device atomic write size allowed for OSD to use.
-
-    Atomic writes allow to reduce the Write Amplification factor with the new store
-    ([meta_format](layout-osd.en.md#meta_format)=3) to almost 1 (i.e. almost no extra writes)
-    with replicated pools and reach the best possible write performance.
-
-    Default value is auto-detected during OSD initialization from
-    `/sys/block/xx/queue/atomic_write_max_bytes` or assumed to be 4096 bytes
-    because all known disks support 4 KB atomic writes. Auto-detection is only used for
-    NVMe disks because SAS disks require the explicit WRITE ATOMIC command which requires
-    RWF_ATOMIC (see below [#use_atomic_flag]) but that flag works incorrectly in current
-    Linux versions.
-
-    You can also check if your NVMe drives support atomic writes by running
-    the command `nvme id-ctrl /dev/nvme0n1 | grep awupf`. If the reported value,
-    plus 1, multiplied by the currently selected block size of the NVMe,
-    is more than 4 KB, then the new store can utilize it for better performance.
-    The only drives known to support it currently are [Micron and Kioxia](../intro/quickstart.en.md).
-
-    Atomic writes allow to skip double data writes in replicated pools, thus
-    reducing Write Amplification and improving write performance up to 2 times.
-  info_ru: |
-    Максимальный размер атомарной записи на диск данных, который OSD разрешено использовать.
-
-    Поддержка атомарной записи позволяет снизить мультипликатор записи (Write Amplification)
-    на диск с новым хранилищем ([meta_format](layout-osd.ru.md#meta_format)=3)
-    практически до 1 (то есть, почти до нулевого объёма лишней записи) в реплицированных
-    пулах и достигнуть наилучшей возможной производительности записи.
-
-    Значение по умолчанию авто-определяется во время инициализации OSD из
-    `/sys/block/xx/queue/atomic_write_max_bytes` либо принимается равным 4096,
-    так как все известные диски поддерживают атомарную запись 4 КБ блоков.
-    Автоопределение применяется только для NVMe-дисков, так как SAS диски требуют
-    использования отдельной команды WRITE ATOMIC, а для неё нужен флаг RWF_ATOMIC
-    (см. ниже [#use_atomic_flag]), а он в текущих версиях Linux работает некорректно.
-
-    Вы также можете проверить, поддерживают ли ваши NVMe-диски атомарную запись,
-    с помощью команды `nvme id-ctrl /dev/nvme0n1 | grep awupf`. Если значение awupf
-    плюс 1, умноженное на текущий выбранный размер блока NVMe-диска, больше 4 КБ,
-    то новое хранилище может использовать атомарные записи для достижения лучшей
-    производительности. Единственные известные диски, которые поддерживают это сейчас -
-    [Micron и Kioxia](../intro/quickstart.ru.md).
-
-    Атомарная запись позволяет не использовать двойную запись данных (в журнал и на
-    устройство данных) в реплицированных пулах и таким образом снижает амплификацию
-    записи (объём служебной записи на диск) и улучшает производительность записи
-    вплоть до 2-х кратного прироста.
-- name: use_atomic_flag
-  type: bool
-  info: |
-    This option controls whether Vitastor OSDs use RWF_ATOMIC write flag with atomic writes.
-    This flag is supported since Linux 6.11 and adds some safety to atomic writes - the kernel
-    guarantees to not fragment write requests with it and also to check them against the actual
-    device atomic write capabilities.
-
-    However, the option is disabled by default because the flag is currently UNUSABLE - Linux
-    incorrectly requires writes with that flag to be of power-of-2 length and length-aligned.
-    I.e., for example, 12 KB writes and not-8-KB aligned 8 KB writes are forbidden by the kernel,
-    even though the NVMe specification allows them.
-
-    For NVMe disks with `scheduler=none` writes aren't fragmented anyway so it's not a big deal.
-    However, you can rebuild your kernel with [this patch](../../patches/linux-fix-atomic-write-checks.diff)
-    and turn this option on. It will make your atomic writes a bit safer.
-  info_ru: |
-    Данная опция контролирует использование Vitastor OSD флага RWF_ATOMIC при атомарной записи
-    блоков. Этот флаг поддерживается, начиная с версии ядра Linux 6.11 и добавляет немного корректности
-    атомарным записям - ядро гарантирует отсутствие фрагментации запросов записи с этим флагом и
-    проверяет их на соответствие реальным возможностям устройства.
-
-    Однако, данная опция по умолчанию отключена, так как флаг в текущих версиях Linux работает
-    абсолютно НЕКОРРЕКТНО - при нём Linux требует, чтобы запросы записи имели длину, равную
-    степени двойки и были выровнены на эту длину. То есть, например, 12 КБ запросы записи, а также
-    8 КБ запросы записи по не-кратному 8 КБ смещению запрещаются ядром, хотя спецификация NVMe их
-    разрешает.
-
-    Для NVMe-дисков с `scheduler=none` запросы записи и так не фрагментируются, так что это не так
-    уж и важно, однако вы можете пересобрать своё ядро с [этим патчем](../../patches/linux-fix-atomic-write-checks.diff)
-    и включить данную опцию. Это сделает вашу атомарную запись капельку безопаснее.
-- name: pg_reshard_chunk_size
-  type: int
-  default: 100000
-  info: |
-    Pool PG count change is a CPU-intensive operation because OSDs store the full object database
-    in memory and have to move all entries between old and new PGs. Thus it's performed in chunks,
-    with pauses between chunks to prevent blocking OSD's event loop and other clients' operations.
-    This option sets the maximum number of object is a chunk. Moving 100k objects usually takes
-    50-100ms. Chunk size equal to 0 means unlimited.
-  info_ru: |
-    Изменение числа PG в пуле заметно загружает процессор, так как OSD хранят полную базу данных
-    объектов в памяти и им приходится перемещать все записи объектов между старыми и новыми PG.
-    Поэтому изменение применяется порциями, с паузами между порциями, чтобы не блокировать обработку
-    событий OSD и операции остальных клиентов. Данная опция задаёт максимальное число объектов
-    в порции. Перемещение 100 тысяч объектов (значение по умолчанию) обычно занимает порядка
-    50-100 миллисекунд. Значение опции 0 отключает лимит размера порции.
-- name: pg_reshard_chunk_pause_ms
-  type: ms
-  default: 100
-  info: |
-    This option sets the interval between handling two PG count change chunks.
-  info_ru: |
-    Данная опция задаёт интервал между обработкой двух порций изменения числа PG пулов.

docs/installation/docker.en.md

@@ -26,9 +26,9 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.
 
 1. Download a Docker image of the desired version: \
-   `docker pull vitalif/vitastor:v3.0.3`
+   `docker pull vitalif/vitastor:v2.4.3`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.3 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.4.3 install.sh`
 3. Reload udev rules: \
    `udevadm control --reload-rules`
 

docs/installation/docker.ru.md

@@ -25,9 +25,9 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.
 
 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitalif/vitastor:v3.0.3`
+   `docker pull vitalif/vitastor:v2.4.3`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.3 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.4.3 install.sh`
 3. Перезагрузите правила udev: \
    `udevadm control --reload-rules`
 

docs/installation/proxmox.en.md

@@ -6,7 +6,7 @@
 
 # Proxmox VE
 
-To enable Vitastor support in Proxmox Virtual Environment (6.4-9.x are supported):
+To enable Vitastor support in Proxmox Virtual Environment (6.4-8.x are supported):
 
 - Add the corresponding Vitastor Debian repository into sources.list on Proxmox hosts:
   trixie for 9.0+, bookworm for 8.1+, pve8.0 for 8.0, bullseye for 7.4, pve7.3 for 7.3, pve7.2 for 7.2, pve7.1 for 7.1, buster for 6.4

docs/installation/proxmox.ru.md

@@ -6,7 +6,7 @@
 
 # Proxmox VE
 
-Чтобы подключить Vitastor к Proxmox Virtual Environment (поддерживаются версии 6.4-9.x):
+Чтобы подключить Vitastor к Proxmox Virtual Environment (поддерживаются версии 6.4-8.x):
 
 - Добавьте соответствующий Debian-репозиторий Vitastor в sources.list на хостах Proxmox:
   trixie для 9.0+, bookworm для 8.1+, pve8.0 для 8.0, bullseye для 7.4, pve7.3 для 7.3, pve7.2 для 7.2, pve7.1 для 7.1, buster для 6.4

docs/intro/features.en.md

@@ -14,8 +14,6 @@
 
 - Basic part: highly-available block storage with symmetric clustering and no SPOF
 - [Performance](../performance/bench2.en.md) ;-D
-  - [NVMe atomic write support](../config/osd.en.md#atomic_write_size) for reducing the amount
-    of "extra" disk writes to almost zero (Write Amplification = 1)
 - [Multiple redundancy schemes](../config/pool.en.md#scheme): Replication, XOR n+1, Reed-Solomon erasure codes
   based on jerasure and ISA-L libraries with any number of data and parity drives in a group
 - Configuration via simple JSON data structures in etcd (parameters, pools and images)

docs/intro/features.ru.md

@@ -14,8 +14,6 @@
 
 - Базовая часть - надёжное кластерное блочное хранилище без единой точки отказа
 - [Производительность](../performance/bench2.ru.md) ;-D
-  - [Поддержка атомарной записи NVMe](../config/osd.ru.md#atomic_write_size) для снижения объёма
-    служебной записи практически до нуля (Write Amplification = 1)
 - [Несколько схем отказоустойчивости](../config/pool.ru.md#scheme): репликация, XOR n+1 (1 диск чётности), коды коррекции ошибок
   Рида-Соломона на основе библиотек jerasure и ISA-L с любым числом дисков данных и чётности в группе
 - Конфигурация через простые человекочитаемые JSON-структуры в etcd

docs/intro/quickstart.en.md

@@ -18,10 +18,9 @@
 
 ## Preparation
 
-- Get some SATA or NVMe SSDs with capacitors (server-grade drives). The best performance
-  is achieved with Micron or Kioxia NVMes with atomic write support (see below). You can use desktop
-  SSDs with lazy fsync, but prepare for inferior single-thread latency. Read more about
-  capacitors [here](../config/layout-cluster.en.md#immediate_commit).
+- Get some SATA or NVMe SSDs with capacitors (server-grade drives). You can use desktop SSDs
+  with lazy fsync, but prepare for inferior single-thread latency. Read more about capacitors
+  [here](../config/layout-cluster.en.md#immediate_commit).
 - If you want to use HDDs, get modern HDDs with Media Cache or SSD Cache: HGST Ultrastar,
   Toshiba MG, Seagate EXOS or something similar. If your drives don't have such cache then
   you also need small SSDs for journal and metadata (even 2 GB per 1 TB of HDD space is enough).
@@ -31,11 +30,9 @@
 
 ## Recommended drives
 
-- NVMe with atomic write support (ideal!): Micron 7450/7500/7550, Kioxia CD6/CD7/CD8/CD9
-- Other NVMe: Micron 9100/9200/9300/9400/9550, Micron 7300, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
-  Intel DC-P3700/P4500/P4600, Intel/Solidigm D5-P4320/P5530, Intel/Solidigm D7-P5500/P5600, Solidigm D7-PS1010/PS1030/P5810,
-  Intel Optane, Kingston DC1000B/DC1500M, Kioxia CD6/CD7/CD8/CD9
-- SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel/Solidigm D3-S4510/4520/4610/4620, Kingston DC500M
+- SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel D3-S4510/4520/4610/4620, Kingston DC500M
+- NVMe: Micron 9100/9200/9300/9400, Micron 7300/7450, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
+  Intel DC-P3700/P4500/P4600, Intel D5-P4320/P5530, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
 - HDD: HGST Ultrastar, Toshiba MG, Seagate EXOS
 
 ## Configure monitors

docs/intro/quickstart.ru.md

@@ -18,9 +18,8 @@
 
 ## Подготовка
 
-- Возьмите серверы с SSD (SATA или NVMe), желательно с конденсаторами (серверные SSD). Наилучшая
-  производительность достигается на дисках Micron и Kioxia с поддержкой атомарной записи (см. ниже).
-  Можно использовать и десктопные SSD, включив режим отложенного fsync, но производительность будет хуже.
+- Возьмите серверы с SSD (SATA или NVMe), желательно с конденсаторами (серверные SSD). Можно
+  использовать и десктопные SSD, включив режим отложенного fsync, но производительность будет хуже.
   О конденсаторах читайте [здесь](../config/layout-cluster.ru.md#immediate_commit).
 - Если хотите использовать HDD, берите современные модели с Media или SSD кэшем - HGST Ultrastar,
   Toshiba MG, Seagate EXOS или что-то похожее. Если такого кэша у ваших дисков нет,
@@ -31,11 +30,9 @@
 
 ## Рекомендуемые диски
 
-- NVMe с поддержкой атомарной записи (идеально!): Micron 7450/7500/7550, Kioxia CD6/CD7/CD8/CD9
-- Другие NVMe: Micron 9100/9200/9300/9400/9550, Micron 7300, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
-  Intel DC-P3700/P4500/P4600, Intel/Solidigm D5-P4320/P5530, Intel/Solidigm D7-P5500/P5600, Solidigm D7-PS1010/PS1030/P5810,
-  Intel Optane, Kingston DC1000B/DC1500M, Kioxia CD6/CD7/CD8/CD9
-- SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel/Solidigm D3-S4510/4520/4610/4620, Kingston DC500M
+- SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel D3-S4510/4520/4610/4620, Kingston DC500M
+- NVMe: Micron 9100/9200/9300/9400, Micron 7300/7450, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
+  Intel DC-P3700/P4500/P4600, Intel D5-P4320/P5530, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
 - HDD: HGST Ultrastar, Toshiba MG, Seagate EXOS
 
 ## Настройте мониторы

docs/performance/theoretical.en.md

@@ -14,7 +14,7 @@ Replicated setups:
 - Linear read: `min(total network bandwidth, sum(disk read MB/s))`.
 - Linear write: `min(total network bandwidth, sum(disk write MB/s / number of replicas))`.
 - Saturated parallel read iops: `min(total network bandwidth, sum(disk read iops))`.
-- Saturated parallel write iops: `min(total network bandwidth / number of replicas, sum(disk write iops / number of replicas / write amplification))`.
+- Saturated parallel write iops: `min(total network bandwidth / number of replicas, sum(disk write iops / number of replicas / (write amplification = 4)))`.
 
 EC/XOR setups (EC N+K):
 - Single-threaded (T1Q1) read latency: 1.5 network roundtrips + 1 disk read.
@@ -26,36 +26,28 @@ EC/XOR setups (EC N+K):
 - Linear read: `min(total network bandwidth, sum(disk read MB/s))`.
 - Linear write: `min(total network bandwidth, sum(disk write MB/s * N/(N+K)))`.
 - Saturated parallel read iops: `min(total network bandwidth, sum(disk read iops))`.
-- Saturated parallel write iops: roughly `total iops / (N+K) / WA`. More exactly:
-  - With the new store: `min(total network bandwidth * N/(N+K), sum(disk randrw iops / (2 + N-1 + K*2)))`,
-    with random read/write mix corresponding to `(N-1)/(2 + N-1 + K*2)*100 % reads`.
-    - For example, with EC 2+1 it is: `(20% randrw iops) / 5`.
-    - With EC 6+3 it is: `(38% randrw iops) / 13`.
-  - With the old store: `min(total network bandwidth * N/(N+K), sum(disk randrw iops / (3 + N-1 + K*3)))`,
-    with random read/write mix corresponding to `(N-1)/(3 + N-1 + K*3)*100 % reads`.
-    - For example, with EC 2+1 it is: `(14% randrw iops) / 7`.
-    - With EC 6+3 it is: `(30% randrw iops) / 17`.
+- Saturated parallel write iops: roughly `total iops / (N+K) / WA`. More exactly,
+  `min(total network bandwidth * N/(N+K), sum(disk randrw iops / (N*4 + K*5 + 1)))` with
+  random read/write mix corresponding to `(N-1)/(N*4 + K*5 + 1)*100 % reads`.
+  - For example, with EC 2+1 it is: `(7% randrw iops) / 14`.
+  - With EC 6+3 it is: `(12.5% randrw iops) / 40`.
 
-Write Amplification factor:
-- For the new store and for 4 KB writes: WA is always 1 unless you set [atomic_write_size](../config/osd.en.md#atomic_write_size) to 0 manually.
-- For the new store and for 8-124 KB writes: WA is 1 if you use NVMe drives with atomic write support, or roughly 2 if you use other drives.
-- For the old store, WA is roughly `(2 * write size + 4 KB) / (write size)`. So, for 4 KB writes it's 3, and for 8-124 KB writes it's closer to 2.
-- For both the new and the old store and for writes of [block_size](../config/layout-cluster.en.md#block_size): WA is almost 1.
+Write amplification for 4 KB blocks is usually 3-5 in Vitastor:
+1. Journal block write
+2. Journal data write
+3. Metadata block write
+4. Another journal block write for EC/XOR setups
+5. Data block write
 
-Write Amplification consists of:
-- For the new store:
-  - Buffer block write if non-atomic
-  - Data block write
-  - Metadata write(s) (amortized)
-- For the old store:
-  - Journal block write (amortized)
-  - Journal data write
-  - Metadata block write
-  - Another journal block write for EC/XOR setups (amortized)
-  - Data block write
+If you manage to get an SSD which handles 512 byte blocks well (Optane?) you may
+lower 1, 3 and 4 to 512 bytes (1/8 of data size) and get WA as low as 2.375.
 
-Other possibilities to reduce WA would be to use SSDs with internal 512-byte blocks
-or NVDIMM, but both options seem unavailable on the market at the moment.
+Implemented NVDIMM support can basically eliminate WA at all - all extra writes will
+go to DRAM memory. But this requires a test cluster with NVDIMM - please contact me
+if you want to provide me with such cluster for tests.
+
+Lazy fsync also reduces WA for parallel workloads because journal blocks are only
+written when they fill up or fsync is requested.
 
 ## In Practice
 

docs/performance/theoretical.ru.md

@@ -27,36 +27,29 @@
 - Линейное чтение: сумма МБ/с чтения всех дисков, либо общая производительность сети, если в сеть упрётся раньше.
 - Линейная запись: сумма МБ/с записи всех дисков * N/(N+K), либо производительность сети * N / (N+K), если в сеть упрётся раньше.
 - Параллельное случайное мелкое чтение: сумма IOPS чтения всех дисков либо производительность сети, если в сеть упрётся раньше.
-- Параллельная случайная мелкая запись: грубо `(сумма IOPS / (N+K) / WA)`.
-  Либо `производительность сети * N/(N+K)`, если в сеть упрётся раньше. Если точнее, то:
-  - С новым хранилищем: сумма смешанного IOPS всех дисков при `(N-1)/(2 + N-1 + K*2)*100 %` чтения, делённая на `(2 + N-1 + K*2)`.
-    - Например, при EC 2+1 это: `(сумма IOPS при 20% чтения) / 5`.
-    - При EC 6+3 это: `(сумма IOPS при 38% чтения) / 13`.
-  - Со старым хранилищем: сумма смешанного IOPS всех дисков при `(N-1)/(3 + N-1 + K*3)*100 %` чтения, делённая на `(3 + N-1 + K*3)`.
-    - Например, при EC 2+1 это: `(сумма IOPS при 14% чтения) / 7`.
-    - При EC 6+3 это: `(сумма IOPS при 30% чтения) / 17`.
+- Параллельная случайная мелкая запись: грубо `(сумма IOPS / (N+K) / WA)`. Если точнее, то:
+  сумма смешанного IOPS всех дисков при `(N-1)/(N*4 + K*5 + 1)*100 %` чтения, делённая на `(N*4 + K*5 + 1)`.
+  Либо, производительность сети * N/(N+K), если в сеть упрётся раньше.
+  - Например, при EC 2+1 это: `(сумма IOPS при 7% чтения) / 14`.
+  - При EC 6+3 это: `(сумма IOPS при 12.5% чтения) / 40`.
 
-WA (Write Amplification, мультипликатор записи):
-- С новым хранилищем для 4 КБ записи: WA всегда примерно 1, если только вы не установите [atomic_write_size](../config/osd.ru.md#atomic_write_size) вручную в 0.
-- С новым хранилищем и большими записями (8-124 КБ): WA примерно 1, если вы используете NVMe-диски с поддержкой атомарной записи,
-  или примерно 2, если вы используете другие диски.
-- Со старым хранилищем, WA примерно `(2 * размер записи + 4 КБ) / (размер записи)`. То есть, для 4 КБ записи WA=3, а для 8-124 КБ WA ближе к 2.
-- И с новым, и со старым хранилищем и для записи размером [block_size](../config/layout-cluster.ru.md#block_size): WA примерно равен 1.
+WA (мультипликатор записи) для 4 КБ блоков в Vitastor обычно составляет 3-5:
+1. Запись метаданных в журнал
+2. Запись блока данных в журнал
+3. Запись метаданных в БД
+4. Ещё одна запись метаданных в журнал при использовании EC
+5. Запись блока данных на диск данных
 
-Мультипликатор записи состоит из:
-- С новым хранилищем:
-  - Запись блока буфера, если диски без поддержки атомарной записи
-  - Запись блока данных
-  - Запись(-и) блоков метаданных (амортизированные)
-- Со старым хранилищем:
-  - Запись блока журнала (амортизированная)
-  - Запись данных в журнал
-  - Запись блока метаданных
-  - Ещё одна запись блока журнала для EC/XOR пулов (амортизированная)
-  - Запись блока данных
+Если вы найдёте SSD, хорошо работающий с 512-байтными блоками данных (Optane?),
+то 1, 3 и 4 можно снизить до 512 байт (1/8 от размера данных) и получить WA всего 2.375.
 
-Другими потенциальными возможностями снижения WA могли бы быть SSD с внутренним 512-байтным блоком
-либо NVDIMM, но и то, и другое сейчас выглядит недоступным на рынке.
+Если реализовать поддержку NVDIMM, то WA можно, условно говоря, ликвидировать вообще - все
+дополнительные операции записи смогут обслуживаться DRAM памятью. Но для этого необходим
+тестовый кластер с NVDIMM - пишите, если готовы предоставить такой для тестов.
+
+Кроме того, WA снижается при использовании отложенного/ленивого сброса при параллельной
+нагрузке, т.к. блоки журнала записываются на диск только когда они заполняются или явным
+образом запрашивается fsync.
 
 ## На практике
 

docs/usage/admin.en.md

@@ -231,18 +231,6 @@ Upgrading from <= 0.5.x to >= 0.6.x is not supported.
 
 Downgrade are also allowed freely, except the following specific instructions:
 
-### 3.x -> 2.x
-
-Versions 3.0.0 and newer contain two store implementations - an old one and a new
-one, unsupported in 2.x and previous versions. So you should check your OSD store
-versions before downgrading to 2.x with the following command:
-
-`vitastor-disk read-sb /dev/vitastor/osdXX-data | jq -r .meta_format`
-
-If it prints 3 then OSD uses the new store and you can't downgrade it to 2.x.
-
-If it prints 2 or nothing then OSD uses the old store and the downgrade is allowed.
-
 ### 1.8.0 to 1.7.1
 
 Before downgrading from version >= 1.8.0 to version <= 1.7.1

docs/usage/admin.ru.md

@@ -228,18 +228,6 @@ done
 
 Откат (понижение версии) тоже свободно разрешён, кроме указанных ниже случаев:
 
-### 3.x -> 2.x
-
-Версии 3.0.0 и более новые содержат две реализации хранилища - старую и новую, не
-поддерживаемую в 2.x и предыдущих версиях. Таким образом, перед откатом на 2.x вам
-следует проверить, какая версия хранилища используется вашими OSD - командой:
-
-`vitastor-disk read-sb /dev/vitastor/osdXX-data | jq -r .meta_format`
-
-Если выводится 3, это новое хранилище и откатить такой OSD до 2.x нельзя.
-
-Если выводится 2 или не выводится ничего, это старое хранилище и откат разрешён.
-
 ### 1.8.0 -> 1.7.1
 
 Перед понижением версии с >= 1.8.0 до <= 1.7.1 вы должны скопировать ключ

docs/usage/disk.en.md

@@ -51,9 +51,6 @@ Options (automatic mode):
 ```
 --osd_per_disk <N>
   Create <N> OSDs on each disk (default 1)
---meta_format 3
-  Metadata store version. 3 is the new log-structured store, 2 is the stable store
-  from Vitastor 0.9-2.x, 1 is the legacy store from Vitastor 0.6-0.8.
 --hybrid
   Prepare hybrid (HDD+SSD, NVMe+SATA or etc) OSDs using provided devices. By default,
   any passed SSDs will be used for journals and metadata, HDDs will be used for data,
@@ -95,8 +92,6 @@ Options (single-device mode):
 Options (both modes):
 
 ```
---tags tag1,tag2           Set new OSD tag(s)
---weight <number>          Set new OSD weight (between 0 to 1)
 --journal_size 1G/32M      Set journal size (area or partition size)
 --block_size 1M/128k       Set blockstore object size
 --bitmap_granularity 4k    Set bitmap granularity

docs/usage/disk.ru.md

@@ -50,9 +50,6 @@ vitastor-disk - инструмент командной строки для уп
 ```
 --osd_per_disk <N>
   Создавать по несколько (<N>) OSD на каждом диске (по умолчанию 1)
---meta_format 3
-  Версия хранилища метаданных. 3 - новое лог-структурированное хранилище,
-  2 - стабильное хранилище из Vitastor 0.9-2.x, 1 - старое хранилище из Vitastor 0.6-0.8.
 --hybrid
   Инициализировать гибридные (HDD+SSD, NVMe+SATA и т.п.) OSD на указанных дисках.
   По умолчанию, SSD будут использованы для журналов и метаданных, а HDD - для данных,
@@ -96,8 +93,6 @@ vitastor-disk - инструмент командной строки для уп
 Опции для обоих режимов:
 
 ```
---tags tag1,tag2           Задать теги для новых OSD
---weight <number>          Задать вес для новых OSD (от 0 до 1)
 --journal_size 1G/32M      Задать размер журнала (области или раздела журнала)
 --block_size 1M/128k       Задать размер объекта хранилища
 --bitmap_granularity 4k    Задать гранулярность битовых карт

mon/lp_optimizer/lp_optimizer.js

@@ -10,19 +10,16 @@ const NO_OSD = 'Z';
 async function lp_solve(text)
 {
     const cp = child_process.spawn('lp_solve');
-    let stdout = '', stderr = '', finish_cb, finished = 0;
+    let stdout = '', stderr = '', finish_cb;
     cp.stdout.on('data', buf => stdout += buf.toString());
     cp.stderr.on('data', buf => stderr += buf.toString());
-    cp.stdout.on('end', () => finish_cb());
-    cp.stderr.on('end', () => finish_cb());
+    cp.on('exit', () => finish_cb && finish_cb());
     cp.stdin.write(text);
     cp.stdin.end();
-    await new Promise(ok => (finish_cb = () =>
+    if (cp.exitCode == null)
     {
-        finished++;
-        if (finished == 2)
-            ok();
-    }));
+        await new Promise(ok => finish_cb = ok);
+    }
     if (!stdout.trim())
     {
         return null;

mon/osd_tree.js

@@ -87,7 +87,7 @@ function make_hier_tree(global_config, tree)
     tree[''] = { children: [] };
     for (const node_id in tree)
     {
-        if (node_id === '')
+        if (node_id === '' || !(tree[node_id].children||[]).length && (tree[node_id].size||0) <= 0)
         {
             continue;
         }

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "3.0.3",
+  "version": "2.4.3",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {
@@ -9,7 +9,7 @@
   "author": "Vitaliy Filippov",
   "license": "UNLICENSED",
   "dependencies": {
-    "antietcd": "^1.2.2",
+    "antietcd": "^1.1.3",
     "sprintf-js": "^1.1.2",
     "ws": "^7.2.5"
   },

mon/pg_gen.js

@@ -9,6 +9,7 @@ const LPOptimizer = require('./lp_optimizer/lp_optimizer.js');
 const { scale_pg_count } = require('./pg_utils.js');
 const { make_hier_tree, filter_osds_by_root_node,
     filter_osds_by_tags, filter_osds_by_block_layout, get_affinity_osds } = require('./osd_tree.js');
+const { select_murmur3 } = require('./lp_optimizer/murmur3.js');
 
 function pick_primary(pool_id, pg_num, pool_config, osd_set, up_osds, aff_osds)
 {
@@ -38,7 +39,7 @@ function pick_primary(pool_id, pg_num, pool_config, osd_set, up_osds, aff_osds)
     {
         return 0;
     }
-    return alive_set[pg_num % alive_set.length];
+    return alive_set[select_murmur3(alive_set.length, osd_num => pool_id+'/'+pg_num+'/'+osd_num)];
 }
 
 function recheck_primary(state, global_config, up_osds, osd_tree)
@@ -52,7 +53,6 @@ function recheck_primary(state, global_config, up_osds, osd_tree)
             continue;
         }
         const aff_osds = get_affinity_osds(pool_cfg, up_osds, osd_tree);
-        let paused = false;
         for (let pg_num = 1; pg_num <= pool_cfg.pg_count; pg_num++)
         {
             if (!state.pg.config.items[pool_id])
@@ -75,19 +75,6 @@ function recheck_primary(state, global_config, up_osds, osd_tree)
                     );
                     new_pg_config.items[pool_id][pg_num].primary = new_primary;
                 }
-                paused = paused || !!pg_cfg.pause;
-            }
-        }
-        if (paused)
-        {
-            if (!new_pg_config)
-            {
-                new_pg_config = JSON.parse(JSON.stringify(state.pg.config));
-            }
-            console.log(`Resuming paused pool ${pool_id}`);
-            for (const pg in new_pg_config.items[pool_id])
-            {
-                delete new_pg_config.items[pool_id][pg].pause;
             }
         }
     }
@@ -192,10 +179,10 @@ async function generate_pool_pgs(state, global_config, pool_id, osd_tree, levels
     const rules = use_rules ? get_pg_rules(pool_id, pool_cfg, global_config.placement_levels) : null;
     const folded = fold_failure_domains(Object.values(pool_tree), use_rules ? rules : [ [ [ pool_cfg.failure_domain ] ] ]);
     // FIXME: Remove/merge make_hier_tree() step somewhere, however it's needed to remove empty nodes
-    const folded_tree = make_hier_tree(global_config, folded.nodes.reduce((a, c) => { a[c.id] = c; return a; }, {}));
+    const folded_tree = make_hier_tree(global_config, folded.nodes);
     const old_pg_count = prev_pgs.length;
     const optimize_cfg = {
-        osd_weights: folded.nodes.reduce((a, c) => { if (/^\d+$/.exec(c.id) && c.size != null) { a[c.id] = c.size||0; } return a; }, {}),
+        osd_weights: folded.nodes.reduce((a, c) => { if (Number(c.id)) { a[c.id] = c.size; } return a; }, {}),
         combinator: use_rules
             // new algorithm:
             ? new RuleCombinator(folded_tree, rules, pool_cfg.max_osd_combinations)

node-binding/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor",
-  "version": "3.0.3",
+  "version": "2.4.3",
   "description": "Low-level native bindings to Vitastor client library",
   "main": "index.js",
   "keywords": [

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '3.0.3'
+VITASTOR_VERSION = '2.4.3'
 
 LOG = logging.getLogger(__name__)
 

patches/linux-fix-atomic-write-checks.diff

@@ -1,39 +0,0 @@
-From 98d3f68a40130c438854f61db6025f9e9b099cb6 Mon Sep 17 00:00:00 2001
-From: Vitaliy Filippov <vitalifster@gmail.com>
-Date: Sat, 20 Dec 2025 14:44:35 +0300
-Subject: [PATCH] Do not require atomic writes to be power of 2 sized and
- aligned on length boundary
-
-It contradicts NVMe specification where alignment is only required when atomic
-write boundary (NABSPF/NABO) is set and highly limits usage of NVMe atomic writes
-
-Signed-off-by: Vitaliy Filippov <vitalifster@gmail.com>
----
- fs/read_write.c | 8 --------
- 1 file changed, 8 deletions(-)
-
-diff --git a/fs/read_write.c b/fs/read_write.c
-index 833bae068770..5467d710108d 100644
---- a/fs/read_write.c
-+++ b/fs/read_write.c
-@@ -1802,17 +1802,9 @@ int generic_file_rw_checks(struct file *file_in, struct file *file_out)
- 
- int generic_atomic_write_valid(struct kiocb *iocb, struct iov_iter *iter)
- {
--	size_t len = iov_iter_count(iter);
--
- 	if (!iter_is_ubuf(iter))
- 		return -EINVAL;
- 
--	if (!is_power_of_2(len))
--		return -EINVAL;
--
--	if (!IS_ALIGNED(iocb->ki_pos, len))
--		return -EINVAL;
--
- 	if (!(iocb->ki_flags & IOCB_DIRECT))
- 		return -EOPNOTSUPP;
- 
--- 
-2.51.0
-

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.3
+Version:        2.4.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.3.el7.tar.gz
+Source0:        vitastor-2.4.3.el7.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  devtoolset-9-gcc-c++
@@ -171,6 +171,7 @@ chown vitastor:vitastor /var/lib/vitastor
 %_bindir/vitastor-kv
 %_bindir/vitastor-kv-stress
 %_bindir/vita
+%_libdir/libvitastor_blk.so*
 %_libdir/libvitastor_client.so*
 %_libdir/libvitastor_kv.so*
 

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.3
+Version:        2.4.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.3.el8.tar.gz
+Source0:        vitastor-2.4.3.el8.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-toolset-9-gcc-c++
@@ -168,6 +168,7 @@ chown vitastor:vitastor /var/lib/vitastor
 %_bindir/vitastor-kv
 %_bindir/vitastor-kv-stress
 %_bindir/vita
+%_libdir/libvitastor_blk.so*
 %_libdir/libvitastor_client.so*
 %_libdir/libvitastor_kv.so*
 

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.3
+Version:        2.4.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.3.el9.tar.gz
+Source0:        vitastor-2.4.3.el9.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++
@@ -165,6 +165,7 @@ chown vitastor:vitastor /var/lib/vitastor
 %_bindir/vitastor-kv
 %_bindir/vitastor-kv-stress
 %_bindir/vita
+%_libdir/libvitastor_blk.so*
 %_libdir/libvitastor_client.so*
 %_libdir/libvitastor_kv.so*
 

src/CMakeLists.txt

@@ -21,7 +21,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 endif()
 set(ENABLE_COVERAGE false CACHE BOOL "Enable code coverage")
 
-add_definitions(-DVITASTOR_VERSION="3.0.3")
+add_definitions(-DVITASTOR_VERSION="2.4.3")
 add_definitions(-D_GNU_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -fvisibility=hidden -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/blockstore/blockstore.h

@@ -183,11 +183,6 @@ public:
     // Update configuration
     virtual void parse_config(blockstore_config_t & config) = 0;
 
-    // Reshard database for a pool in chunks
-    // MUST be called only when nobody makes any modifications to the DB for this pool
-    virtual void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit) = 0;
-    virtual bool reshard_continue(void *reshard_state, uint64_t chunk_limit) = 0;
-
     // Event loop
     virtual void loop() = 0;
 

src/blockstore/blockstore_disk.cpp

@@ -48,12 +48,11 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     disk_alignment = parse_size(config["disk_alignment"]);
     journal_block_size = parse_size(config["journal_block_size"]);
     meta_block_size = parse_size(config["meta_block_size"]);
+    meta_block_target_free_space = parse_size(config["meta_block_target_free_space"]);
     bitmap_granularity = parse_size(config["bitmap_granularity"]);
     meta_format = stoull_full(config["meta_format"]);
     atomic_write_size = (config.find("atomic_write_size") != config.end()
         ? parse_size(config["atomic_write_size"]) : 4096);
-    use_atomic_flag = config.find("use_atomic_flag") != config.end() &&
-        (config["use_atomic_flag"] == "true" || config["use_atomic_flag"] == "1" || config["use_atomic_flag"] == "yes");
     if (config.find("data_io") == config.end() &&
         config.find("meta_io") == config.end() &&
         config.find("journal_io") == config.end())
@@ -155,6 +154,14 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     {
         throw std::runtime_error("meta_block_size must not exceed "+std::to_string(MAX_DATA_BLOCK_SIZE));
     }
+    if (!meta_block_target_free_space)
+    {
+        meta_block_target_free_space = 800;
+    }
+    if (meta_block_target_free_space >= meta_block_size)
+    {
+        throw std::runtime_error("meta_block_target_free_space must not exceed "+std::to_string(meta_block_size));
+    }
     if (data_offset % disk_alignment)
     {
         throw std::runtime_error("data_offset must be a multiple of disk_alignment = "+std::to_string(disk_alignment));
@@ -171,12 +178,6 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     {
         throw std::runtime_error("Data block size must be a multiple of sparse write tracking granularity");
     }
-    if (data_block_size / bitmap_granularity < 8)
-    {
-        fprintf(stderr, "Warning: block_size (%u) / bitmap_granularity (%u) = %u bits. "
-            "Consider using larger block_size or bitmap_granularity for better performance.\n",
-            data_block_size, bitmap_granularity, data_block_size / bitmap_granularity);
-    }
     if (!data_csum_type)
     {
         csum_block_size = 0;
@@ -209,6 +210,10 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     {
         throw std::runtime_error("journal_offset must be a multiple of journal_block_size = "+std::to_string(journal_block_size));
     }
+    if (!meta_format)
+    {
+        meta_format = BLOCKSTORE_META_FORMAT_HEAP;
+    }
     if (meta_device == data_device)
     {
         disable_meta_fsync = disable_data_fsync;
@@ -219,7 +224,7 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     }
 }
 
-void blockstore_disk_t::calc_lengths(bool skip_meta_check)
+void blockstore_disk_t::calc_lengths()
 {
     // data
     data_len = data_device_size - data_offset;
@@ -265,13 +270,13 @@ void blockstore_disk_t::calc_lengths(bool skip_meta_check)
     }
     // required metadata size
     block_count = data_len / data_block_size;
-    clean_entry_bitmap_size = (data_block_size / bitmap_granularity + 7) / 8;
+    clean_entry_bitmap_size = data_block_size / bitmap_granularity / 8;
     clean_dyn_size = clean_entry_bitmap_size*2 + (csum_block_size
         ? data_block_size/csum_block_size*(data_csum_type & 0xFF) : 0);
-recalc:
     if (meta_format == BLOCKSTORE_META_FORMAT_HEAP)
     {
-        uint32_t entries_per_block = meta_block_size / (sizeof(heap_big_write_t) + clean_dyn_size);
+        uint32_t entries_per_block = ((meta_block_size-meta_block_target_free_space) /
+            (sizeof(heap_object_t) + sizeof(heap_write_t) + clean_dyn_size));
         min_meta_len = (block_count+entries_per_block-1) / entries_per_block * meta_block_size;
     }
     else if (meta_format == BLOCKSTORE_META_FORMAT_V1)
@@ -279,32 +284,11 @@ recalc:
         clean_entry_size = 24 /*sizeof(clean_disk_entry)*/ + 2*clean_entry_bitmap_size;
         min_meta_len = (1 + (block_count - 1 + meta_block_size / clean_entry_size)
             / (meta_block_size / clean_entry_size)) * meta_block_size;
-        if (!skip_meta_check && meta_area_size < min_meta_len)
-        {
-too_small:
-            throw std::runtime_error("Metadata area is too small, need at least "+std::to_string(min_meta_len)+
-                " bytes, have only "+std::to_string(meta_area_size)+" bytes");
-        }
     }
-    else if (meta_format == BLOCKSTORE_META_FORMAT_V2 || !meta_format)
+    else if (meta_format == BLOCKSTORE_META_FORMAT_V2)
     {
-        meta_format = BLOCKSTORE_META_FORMAT_V2;
         clean_entry_size = 24 /*sizeof(clean_disk_entry)*/ + clean_dyn_size + 4 /*entry_csum*/;
         min_meta_len = (1 + (block_count - 1 + meta_block_size / clean_entry_size) / (meta_block_size / clean_entry_size)) * meta_block_size;
-        if (!skip_meta_check && meta_area_size < min_meta_len)
-        {
-            if (!data_csum_type)
-            {
-                printf("Warning: Using old metadata format without checksums because the new format"
-                    " doesn't fit into provided area (%ju bytes required, %ju bytes available)\n", min_meta_len, meta_area_size);
-                meta_format = BLOCKSTORE_META_FORMAT_V1;
-                goto recalc;
-            }
-            else
-            {
-                goto too_small;
-            }
-        }
     }
     else
     {

src/blockstore/blockstore_disk.h

@@ -36,8 +36,8 @@ struct blockstore_disk_t
     uint32_t meta_block_size = 4096;
     // Atomic write size of the data block device
     uint32_t atomic_write_size = 4096;
-    // Whether we should set RWF_ATOMIC on atomic writes
-    bool use_atomic_flag = false;
+    // Target free space in metadata blocks
+    uint32_t meta_block_target_free_space = 800;
     // Sparse write tracking granularity. 4 KB is a good choice. Must be a multiple of disk_alignment
     uint32_t bitmap_granularity = 4096;
     // Data checksum type, BLOCKSTORE_CSUM_NONE or BLOCKSTORE_CSUM_CRC32C
@@ -61,10 +61,9 @@ struct blockstore_disk_t
     uint64_t discard_granularity = 0;
 
     int meta_fd = -1, data_fd = -1, journal_fd = -1;
-    uint64_t meta_offset = 0, meta_device_sect = 0, meta_device_size = 0, meta_area_size = 0, min_meta_len = 0;
-    uint64_t data_offset = 0, data_device_sect = 0, data_device_size = 0, data_len = 0;
-    uint64_t journal_offset = 0, journal_device_sect = 0, journal_device_size = 0, journal_len = 0;
-    uint64_t meta_format = 0;
+    uint64_t meta_offset, meta_device_sect, meta_device_size, meta_area_size, min_meta_len, meta_format = 0;
+    uint64_t data_offset, data_device_sect, data_device_size, data_len;
+    uint64_t journal_offset, journal_device_sect, journal_device_size, journal_len;
 
     uint64_t block_count = 0;
     uint32_t clean_entry_bitmap_size = 0;
@@ -76,7 +75,7 @@ struct blockstore_disk_t
     void open_data();
     void open_meta();
     void open_journal();
-    void calc_lengths(bool skip_meta_check = false);
+    void calc_lengths();
     void check_lengths();
     void close_all();
     int trim_data(std::function<bool(uint64_t)> is_free);

src/blockstore/blockstore_flush.cpp

@@ -18,18 +18,12 @@ journal_flusher_t::journal_flusher_t(blockstore_impl_t *bs)
     this->cur_flusher_count = bs->min_flusher_count;
     this->target_flusher_count = bs->min_flusher_count;
     active_flushers = 0;
+    advance_lsn_counter = 0;
     co = new journal_flusher_co[max_flusher_count];
     for (int i = 0; i < max_flusher_count; i++)
     {
         co[i].co_id = i;
         co[i].bs = bs;
-        co[i].new_bmp = (uint8_t*)malloc_or_die(3*bs->dsk.clean_entry_bitmap_size);
-        co[i].new_ext_bmp = co[i].new_bmp + bs->dsk.clean_entry_bitmap_size;
-        co[i].punch_bmp = co[i].new_bmp + 2*bs->dsk.clean_entry_bitmap_size;
-        if (bs->dsk.csum_block_size > 0)
-        {
-            co[i].new_csums = (uint8_t*)malloc_or_die(bs->dsk.data_block_size / bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF));
-        }
         co[i].flusher = this;
     }
 }
@@ -60,18 +54,6 @@ journal_flusher_t::~journal_flusher_t()
 
 journal_flusher_co::~journal_flusher_co()
 {
-    if (new_csums)
-    {
-        free(new_csums);
-        new_csums = NULL;
-    }
-    if (new_bmp)
-    {
-        free(new_bmp);
-        new_bmp = NULL;
-    }
-    new_ext_bmp = NULL;
-    punch_bmp = NULL;
     free_buffers();
 }
 
@@ -80,6 +62,11 @@ int journal_flusher_t::get_syncing_buffer()
     return syncing_buffer;
 }
 
+uint64_t journal_flusher_t::get_compact_counter()
+{
+    return compact_counter;
+}
+
 bool journal_flusher_t::is_active()
 {
     return active_flushers > 0;
@@ -99,10 +86,9 @@ void journal_flusher_t::release_trim()
 void journal_flusher_t::dump_diagnostics()
 {
     printf(
-        "Compaction queue: %u items + %u future, data: %ju/%ju blocks used, buffer: %ju/%ju bytes used, meta: %ju/%ju bytes used, %u/%ju blocks nearfull\n",
-        bs->heap->get_compact_queue_size(), bs->heap->get_to_compact_count(),
+        "Compaction queue: %u items, data: %ju/%ju blocks used, meta: %ju/%ju bytes used, %u/%ju blocks nearfull\n",
+        bs->heap->get_to_compact_count(),
         bs->heap->get_data_used_space()/bs->dsk.data_block_size, bs->dsk.block_count,
-        bs->heap->get_buffer_area_used_space(), bs->dsk.journal_len,
         bs->heap->get_meta_used_space(), bs->heap->get_meta_total_space(),
         bs->heap->get_meta_nearfull_blocks(), bs->dsk.meta_area_size/bs->dsk.meta_block_size-1
     );
@@ -127,12 +113,8 @@ void journal_flusher_t::loop()
         }
     }
     int prev_active = active_flushers;
-    for (int i = 0; (active_flushers > 0 || force_start > 0 ||
-        bs->heap->get_compact_queue_size() > bs->flusher_start_threshold ||
-        i == 0 && bs->intent_write_counter >= bs->journal_trim_interval) && i < cur_flusher_count; i++)
-    {
+    for (int i = 0; (active_flushers > 0 || force_start > 0 || bs->heap->get_to_compact_count() > bs->flusher_start_threshold) && i < cur_flusher_count; i++)
         co[i].loop();
-    }
     if (prev_active && !active_flushers && force_start > 0)
         bs->ringloop->wakeup();
 }
@@ -174,33 +156,22 @@ bool journal_flusher_co::loop()
     else if (wait_state == 19) goto resume_19;
     else if (wait_state == 20) goto resume_20;
     else if (wait_state == 21) goto resume_21;
+    else if (wait_state == 22) goto resume_22;
+    else if (wait_state == 23) goto resume_23;
+    else if (wait_state == 24) goto resume_24;
 resume_0:
     wait_state = 0;
     wait_count = 0;
     cur_oid = {};
     res = bs->heap->get_next_compact(cur_oid);
-    // Advance fsynced_lsn every <journal_trim_interval> intent writes
-    if ((bs->intent_write_counter >= bs->journal_trim_interval) && co_id == 0)
-    {
-        bs->intent_write_counter = 0;
-resume_17:
-resume_18:
-resume_19:
-        if (!fsync_buffer(17))
-            return false;
-resume_20:
-resume_21:
-        if (!trim_lsn(20))
-            return false;
-    }
     if (res == ENOENT && flusher->force_start > 0 && co_id == 0 &&
-        (!bs->dsk.disable_journal_fsync || !bs->dsk.disable_meta_fsync || !bs->dsk.disable_data_fsync))
+        (!bs->dsk.disable_journal_fsync || !bs->dsk.disable_meta_fsync))
     {
         flusher->active_flushers++;
-resume_14:
-resume_15:
-resume_16:
-        if (!fsync_buffer(14))
+resume_21:
+resume_22:
+        res = fsync_buffer(21);
+        if (!res)
         {
             return false;
         }
@@ -209,94 +180,61 @@ resume_16:
     }
     if (res == ENOENT)
     {
+        if (co_id == 0 && flusher->force_start > 0)
+        {
+            flusher->active_flushers++;
+resume_16:
+resume_17:
+resume_18:
+resume_19:
+resume_20:
+            if (!trim_lsn(16))
+                return false;
+            flusher->active_flushers--;
+        }
         cur_oid = {};
         wait_state = 0;
         return true;
     }
-    if (flusher->flushing.find(cur_oid) != flusher->flushing.end())
+    for (int i = 0; i < flusher->cur_flusher_count; i++)
     {
-        for (int i = 0; i < flusher->cur_flusher_count; i++)
+        if (i != co_id && flusher->co[i].cur_oid == cur_oid)
         {
-            if (i != co_id && flusher->co[i].cur_oid == cur_oid)
-            {
-                // Already flushing it
-                flusher->co[i].should_repeat = true;
-                goto resume_0;
-            }
+            // Already flushing it
+            flusher->co[i].should_repeat = true;
+            goto resume_0;
         }
-        assert(false);
     }
-    flusher->flushing.insert(cur_oid);
 resume_1:
-    wait_state = 1;
     should_repeat = false;
-    cur_obj = bs->heap->lock_and_read_entry(cur_oid);
+    cur_obj = bs->heap->lock_and_read_entry(cur_oid, copy_id);
     if (!cur_obj)
     {
         // Object does not exist
-        flusher->flushing.erase(cur_oid);
         goto resume_0;
     }
+    cur_version = cur_obj->get_writes()->version;
+    // Find the range to compact
+    compact_lsn = bs->heap->get_fsynced_lsn();
+    bs->heap->get_compact_range(cur_obj, compact_lsn, &begin_wr, &end_wr);
+    if (!begin_wr)
+    {
+        // Nothing to flush
+        bs->heap->unlock_entry(cur_oid, copy_id);
+        goto resume_0;
+    }
+    assert(!end_wr->next() && end_wr->entry_type == (BS_HEAP_BIG_WRITE|BS_HEAP_STABLE));
+    clean_loc = end_wr->big_location(bs->heap);
+    if (bs->log_level > 10)
+        printf("Compacting %jx:%jx l%ju .. l%ju (last l%ju)\n", cur_oid.inode, cur_oid.stripe, end_wr->lsn, begin_wr->lsn, compact_lsn);
+    flusher->active_flushers++;
     // Scan versions to flush
     free_buffers();
     copy_count = 0;
-    fsynced_lsn = bs->heap->get_fsynced_lsn();
-    bitmap_copied = false;
-    memset(new_bmp, 0, bs->dsk.clean_entry_bitmap_size);
-    csum_copy.clear();
-    compact_info = bs->heap->iterate_compaction(cur_obj, fsynced_lsn, flusher->force_start, [&](heap_entry_t *wr)
+    for (auto wr = begin_wr; wr != end_wr; wr = wr->next())
     {
-        if (!bitmap_copied)
-        {
-            memcpy(new_ext_bmp, wr->get_ext_bitmap(bs->heap), bs->dsk.clean_entry_bitmap_size);
-            bitmap_copied = true;
-        }
-        bitmap_set(new_bmp, wr->small().offset, wr->small().len, bs->dsk.bitmap_granularity);
-        if (bs->dsk.csum_block_size && bs->dsk.csum_block_size <= bs->dsk.bitmap_granularity)
-        {
-            csum_copy.push_back(wr);
-        }
-        if (wr->type() == BS_HEAP_SMALL_WRITE ||
-            wr->type() == BS_HEAP_INTENT_WRITE && bs->dsk.csum_block_size > bs->dsk.bitmap_granularity)
-        {
-            auto res = bs->prepare_read(read_vec, cur_obj, wr, 0, bs->dsk.data_block_size,
-                wr->type() == BS_HEAP_INTENT_WRITE && bs->dsk.csum_block_size > bs->dsk.bitmap_granularity && !bs->perfect_csum_update
-                ? COPY_BUF_SKIP_CSUM : 0);
-            if (res > 0)
-                copy_count++;
-        }
-    });
-    if (!compact_info.compact_lsn)
-    {
-        // Flushing is aborted
-        flusher->flushing.erase(cur_oid);
-        bs->heap->unlock_entry(cur_oid);
-        goto resume_0;
-    }
-    mem_or(new_bmp, compact_info.clean_wr->get_int_bitmap(bs->heap), bs->dsk.clean_entry_bitmap_size);
-    if (!bitmap_copied)
-    {
-        memcpy(new_ext_bmp, compact_info.clean_wr->get_ext_bitmap(bs->heap), bs->dsk.clean_entry_bitmap_size);
-        bitmap_copied = true;
-    }
-    if (bs->dsk.csum_block_size && bs->dsk.csum_block_size <= bs->dsk.bitmap_granularity)
-    {
-        memcpy(new_csums, compact_info.clean_wr->get_checksums(bs->heap), bs->dsk.data_block_size/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF));
-        for (size_t i = csum_copy.size(); i > 0; i--)
-        {
-            auto wr = csum_copy[i-1];
-            memcpy(new_csums + wr->small().offset/bs->dsk.csum_block_size*(bs->dsk.data_csum_type & 0xFF),
-                wr->get_checksums(bs->heap), wr->small().len/bs->dsk.csum_block_size*(bs->dsk.data_csum_type & 0xFF));
-        }
-        csum_copy.clear();
-    }
-    clean_loc = compact_info.clean_wr->big_location(bs->heap);
-    flusher->active_flushers++;
-    if (bs->log_level > 10)
-    {
-        printf("Compacting %jx:%jx v%ju..v%ju / l%ju..l%ju (%d writes)\n", cur_oid.inode, cur_oid.stripe,
-            compact_info.clean_wr->version, compact_info.compact_version,
-            compact_info.clean_wr->lsn, compact_info.compact_lsn, copy_count);
+        bs->prepare_read(read_vec, cur_obj, wr, 0, bs->dsk.data_block_size);
+        copy_count++;
     }
     overwrite_start = overwrite_end = 0;
     if (read_vec.size() > 0)
@@ -316,6 +254,7 @@ resume_1:
     }
     // Read buffered data
     cur_obj = NULL;
+    begin_wr = end_wr = NULL;
 resume_2:
 resume_3:
     if (!read_buffered(2))
@@ -333,49 +272,29 @@ resume_3:
         flusher->wanting_meta_fsync--;
     }
     res = check_and_punch_checksums();
-    if (res == ENOENT || res == EDOM)
-    {
-        // Abort compaction
-        flusher->flushing.erase(cur_oid);
-        bs->heap->unlock_entry(cur_oid);
-        flusher->active_flushers--;
-        goto resume_0;
-    }
     if (res == EBUSY)
     {
 resume_4:
-        modified_block = UINT32_MAX;
-        res = bs->heap->punch_holes(compact_info.clean_wr, punch_bmp, new_csums, &modified_block);
-        if (res == ENOENT)
-        {
-            // Abort compaction
-            flusher->flushing.erase(cur_oid);
-            bs->heap->unlock_entry(cur_oid);
-            flusher->active_flushers--;
-            goto resume_0;
-        }
-        if (res == EAGAIN)
-        {
-            // Retry, block is busy
-            wait_state = 4;
-            return false;
-        }
-        assert(res == 0);
 resume_5:
-resume_6:
-        if (!write_meta_block(5))
+        if (!write_meta_block(4))
         {
             return false;
         }
+resume_6:
 resume_7:
 resume_8:
-resume_9:
-        if (!fsync_meta(7))
+        if (!fsync_meta(6))
         {
             return false;
         }
         res = 0;
     }
+    else if (res == ENOENT || res == EDOM)
+    {
+        // Abort compaction
+        flusher->active_flushers--;
+        goto resume_0;
+    }
     assert(res == 0);
     // Submit data writes
     for (i = 0; i < read_vec.size(); i++)
@@ -385,7 +304,7 @@ resume_9:
             (read_vec[i].copy_flags & COPY_BUF_PADDED)) // FIXME Shit, simplify these flags
         {
             assert(read_vec[i].buf);
-            await_sqe(10);
+            await_sqe(9);
             data->iov = (struct iovec){ read_vec[i].buf + (read_vec[i].copy_flags & COPY_BUF_PADDED
                 ? read_vec[i].offset - read_vec[i].disk_offset : 0), (size_t)read_vec[i].len };
             data->callback = simple_callback_w;
@@ -393,57 +312,67 @@ resume_9:
             wait_count++;
         }
     }
-resume_11:
+resume_10:
     if (wait_count > 0)
     {
-        wait_state = 11;
+        wait_state = 10;
         return false;
     }
     // Lock is only needed to prevent freeing the big_write because we overwrite it...
-    bs->heap->unlock_entry(cur_oid);
+    bs->heap->unlock_entry(cur_oid, copy_id);
     // Mark the object compacted, but don't free and remove small_writes
     // We'll free and remove them only when trimming
     // The only thing we modify here are big_write block checksums if >4k block is used
-    cur_obj = bs->heap->read_entry(cur_oid);
+    cur_obj = bs->heap->read_entry(cur_oid, &modified_block);
     if (!cur_obj)
     {
         // Abort compaction
-        flusher->flushing.erase(cur_oid);
         goto resume_0;
     }
     if (!calc_block_checksums())
     {
         // Abort compaction
-        flusher->flushing.erase(cur_oid);
         goto resume_0;
     }
-    res = bs->heap->add_compact(cur_obj, compact_info.compact_version, compact_info.compact_lsn, clean_loc,
-        compact_info.do_delete, &modified_block, new_bmp, new_ext_bmp, new_csums);
-    if (res == EBUSY)
+    if (read_to_fill_incomplete)
     {
-        // Abort compaction, object is already overwritten by something else
-        flusher->flushing.erase(cur_oid);
-        goto resume_0;
-    }
-    assert(res == 0);
-resume_12:
-resume_13:
-    if (!write_meta_block(12))
-    {
-        return false;
+resume_23:
+resume_24:
+        if (!write_meta_block(23))
+        {
+            return false;
+        }
+        // Recheck the object because it could be invalidated again
+        cur_obj = bs->heap->read_entry(cur_oid, &modified_block);
+        if (!cur_obj)
+        {
+            // Abort compaction
+            goto resume_0;
+        }
     }
+    bs->heap->mark_object_compacted(cur_obj, compact_lsn);
     // Done
     if (bs->log_level > 10)
-    {
-        printf("Compacted %jx:%jx l%ju (%d writes)\n", cur_oid.inode, cur_oid.stripe, compact_info.compact_lsn, copy_count);
-    }
+        printf("Compacted %jx:%jx l%ju (%d writes)\n", cur_oid.inode, cur_oid.stripe, compact_lsn, copy_count);
+    flusher->compact_counter++;
     flusher->active_flushers--;
+    // Advance compacted_lsn every <journal_trim_interval> objects
+    if (co_id == 0 && !((++flusher->advance_lsn_counter) % bs->journal_trim_interval))
+    {
+        flusher->advance_lsn_counter = 0;
+resume_11:
+resume_12:
+resume_13:
+resume_14:
+resume_15:
+        if (!trim_lsn(11))
+            return false;
+    }
     if (should_repeat)
     {
         // Flush the same object again
         goto resume_1;
     }
-    flusher->flushing.erase(cur_oid);
     // All done
     goto resume_0;
 }
@@ -480,11 +409,11 @@ void journal_flusher_co::fill_partial_checksum_blocks()
                 .copy_flags = COPY_BUF_DATA | copy_flags,
                 .offset = blk_begin,
                 .len = blk_end - blk_begin,
-                .disk_loc = clean_loc,
+                .disk_loc = end_wr->big_location(bs->heap),
                 .disk_offset = blk_begin,
                 .disk_len = blk_end - blk_begin,
                 .buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, blk_end - blk_begin),
-                .wr = compact_info.clean_wr,
+                .wr_lsn = end_wr->lsn,
             });
         }
         auto & vec = read_vec[read_vec.size()-1];
@@ -522,28 +451,26 @@ int journal_flusher_co::check_and_punch_checksums()
         return 0;
     }
     // Verify data checksums
-    cur_obj = bs->heap->read_entry(cur_oid);
-    if (!cur_obj)
-    {
-        // Object is deleted, abort compaction
-        return ENOENT;
-    }
+    cur_obj = bs->heap->read_locked_entry(cur_oid, copy_id);
     bool csum_ok = true;
     for (int i = 0; i < read_vec.size(); i++)
     {
         auto & vec = read_vec[i];
         if (!(vec.copy_flags & (COPY_BUF_COALESCED|COPY_BUF_ZERO|COPY_BUF_SKIP_CSUM)))
         {
-            uint32_t *csums = (uint32_t*)(vec.wr->get_checksums(bs->heap)
+            heap_write_t *wr = cur_obj->get_writes();
+            while (wr && wr->lsn != vec.wr_lsn)
+                wr = wr->next();
+            assert(wr);
+            uint32_t *csums = (uint32_t*)(wr->get_checksums(bs->heap)
                 + (vec.disk_offset/bs->dsk.csum_block_size)*(bs->dsk.data_csum_type & 0xFF)
-                - ((vec.wr->type() == BS_HEAP_BIG_WRITE || vec.wr->type() == BS_HEAP_BIG_INTENT)
-                    ? 0 : (vec.wr->small().offset/bs->dsk.csum_block_size)*(bs->dsk.data_csum_type & 0xFF)));
+                - ((wr->type() == BS_HEAP_BIG_WRITE) ? 0 : (wr->small().offset/bs->dsk.csum_block_size)*(bs->dsk.data_csum_type & 0xFF)));
             bs->heap->calc_block_checksums(
-                csums, vec.buf, vec.wr->get_int_bitmap(bs->heap), vec.disk_offset, vec.disk_offset+vec.disk_len, false,
+                csums, vec.buf, wr->get_int_bitmap(bs->heap), vec.disk_offset, vec.disk_offset+vec.disk_len, false,
                 [&](uint32_t mismatch_pos, uint32_t expected_csum, uint32_t real_csum)
                 {
                     printf("Checksum mismatch during compaction in object %jx:%jx v%ju, offset 0x%x in %s area at offset 0x%jx: got %08x, expected %08x\n",
-                        cur_oid.inode, cur_oid.stripe, vec.wr->version, mismatch_pos,
+                        cur_oid.inode, cur_oid.stripe, wr->version, mismatch_pos,
                         (vec.copy_flags & COPY_BUF_JOURNAL ? "buffer" : "data"),
                         vec.disk_loc+vec.disk_offset, real_csum, expected_csum);
                     csum_ok = false;
@@ -562,8 +489,20 @@ int journal_flusher_co::check_and_punch_checksums()
         // Nothing to do
         return 0;
     }
-    memcpy(punch_bmp, compact_info.clean_wr->get_int_bitmap(bs->heap), bs->dsk.clean_entry_bitmap_size);
-    memcpy(new_csums, compact_info.clean_wr->get_checksums(bs->heap), bs->dsk.data_block_size/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF));
+    cur_obj = bs->heap->read_entry(cur_oid, &modified_block);
+    if (!cur_obj)
+    {
+        // Object is deleted, abort compaction
+        return ENOENT;
+    }
+    bs->heap->get_compact_range(cur_obj, compact_lsn, &begin_wr, &end_wr);
+    if (!begin_wr)
+    {
+        // Object is overwritten, abort compaction
+        return ENOENT;
+    }
+    uint8_t *bmp = end_wr->get_int_bitmap(bs->heap);
+    uint8_t *csums = end_wr->get_checksums(bs->heap);
     // Clear bits
     for (auto & vec: read_vec)
     {
@@ -574,7 +513,7 @@ int journal_flusher_co::check_and_punch_checksums()
         if (!(vec.copy_flags & COPY_BUF_COALESCED) &&
             ((vec.offset % bs->dsk.csum_block_size) || (vec.len % bs->dsk.csum_block_size)))
         {
-            bitmap_clear(punch_bmp, vec.offset, vec.len, bs->dsk.bitmap_granularity);
+            bitmap_clear(bmp, vec.offset, vec.len, bs->dsk.bitmap_granularity);
         }
     }
     // Update partial block checksums
@@ -583,20 +522,42 @@ int journal_flusher_co::check_and_punch_checksums()
         if (vec.copy_flags & COPY_BUF_CSUM_FILL)
         {
             uint32_t csum_off = vec.offset/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF);
-            bs->heap->calc_block_checksums((uint32_t*)(new_csums+csum_off), vec.buf, punch_bmp, vec.offset, vec.offset+vec.len, true, NULL);
+            bs->heap->calc_block_checksums((uint32_t*)(csums+csum_off), vec.buf, bmp, vec.offset, vec.offset+vec.len, true, NULL);
         }
     }
-    // Modified, we should add_punch_holes and then write the block to disk
+    cur_obj->crc32c = cur_obj->calc_crc32c();
+    if (res == ENOENT)
+    {
+        // Object is deleted, abort compaction
+        return ENOENT;
+    }
+    // Modified, we should write the block to disk
+    assert(!res);
     return EBUSY;
 }
 
 bool journal_flusher_co::calc_block_checksums()
 {
-    if (bs->dsk.csum_block_size <= bs->dsk.bitmap_granularity)
+    if (bs->dsk.csum_block_size <= bs->dsk.bitmap_granularity || !read_vec.size())
     {
         return true;
     }
-    memcpy(new_csums, compact_info.clean_wr->get_checksums(bs->heap), bs->dsk.data_block_size/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF));
+    bs->heap->get_compact_range(cur_obj, compact_lsn, &begin_wr, &end_wr);
+    if (!begin_wr)
+    {
+        // Object is overwritten, abort compaction
+        return false;
+    }
+    uint8_t *bmp = end_wr->get_int_bitmap(bs->heap);
+    uint8_t *csums = end_wr->get_checksums(bs->heap);
+    // Set bits
+    for (auto & vec: read_vec)
+    {
+        if (!(vec.copy_flags & (COPY_BUF_COALESCED|COPY_BUF_CSUM_FILL)))
+        {
+            bitmap_set(bmp, vec.offset, vec.len, bs->dsk.bitmap_granularity);
+        }
+    }
     // Update block checksums
     size_t i = 0;
     while (i < read_vec.size() && !(read_vec[i].copy_flags & COPY_BUF_CSUM_FILL))
@@ -615,7 +576,7 @@ bool journal_flusher_co::calc_block_checksums()
         assert(!(end % bs->dsk.csum_block_size));
         uint32_t csum_off = start/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF);
         bs->heap->calc_block_checksums(
-            (uint32_t*)(new_csums+csum_off), new_bmp, start, end,
+            (uint32_t*)(csums+csum_off), bmp, start, end,
             [&](uint32_t start, uint32_t & len)
             {
                 // O(n^2) search, may be fixed later :-p
@@ -632,6 +593,7 @@ bool journal_flusher_co::calc_block_checksums()
             }, true, NULL
         );
     }
+    cur_obj->crc32c = cur_obj->calc_crc32c();
     return true;
 }
 
@@ -641,15 +603,13 @@ bool journal_flusher_co::write_meta_block(int wait_base)
         goto resume_0;
     else if (wait_state == wait_base+1)
         goto resume_1;
-resume_0:
-    if (bs->ringloop->space_left() < 1)
-    {
-        wait_state = wait_base+0;
-        return 0;
-    }
-    bs->prepare_meta_block_write(modified_block);
+    await_sqe(0);
+    data->iov = (struct iovec){ bs->heap->get_meta_block(modified_block), (size_t)bs->dsk.meta_block_size };
+    data->callback = simple_callback_w;
+    io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + (modified_block+1)*bs->dsk.meta_block_size);
+    wait_count++;
 resume_1:
-    if (bs->meta_block_is_pending(modified_block))
+    if (wait_count > 0)
     {
         wait_state = wait_base+1;
         return false;
@@ -704,11 +664,11 @@ bool journal_flusher_co::fsync_meta(int wait_base)
     if (wait_state == wait_base)        goto resume_0;
     else if (wait_state == wait_base+1) goto resume_1;
     else if (wait_state == wait_base+2) goto resume_2;
+resume_0:
     if (bs->dsk.disable_meta_fsync)
     {
         return true;
     }
-resume_0:
     if (flusher->wanting_meta_fsync || flusher->fsyncing_meta > 0)
     {
         wait_state = wait_base;
@@ -733,30 +693,69 @@ resume_2:
     return true;
 }
 
-bool journal_flusher_co::fsync_buffer(int wait_base)
+int journal_flusher_co::fsync_buffer(int wait_base)
+{
+    if (wait_state == wait_base)        goto resume_0;
+    else if (wait_state == wait_base+1) goto resume_1;
+    if (bs->dsk.disable_journal_fsync && bs->dsk.disable_meta_fsync && bs->dsk.disable_data_fsync || !bs->unsynced_big_write_count && !bs->unsynced_small_write_count)
+    {
+        return 1;
+    }
+    if (flusher->syncing_buffer)
+    {
+        return 0;
+    }
+    flusher->active_flushers++;
+    flusher->syncing_buffer++;
+resume_0:
+    assert(!wait_count);
+    compact_lsn = bs->heap->get_completed_lsn();
+    if (!bs->submit_fsyncs(wait_count))
+    {
+        wait_state = wait_base+0;
+        return 0;
+    }
+resume_1:
+    if (wait_count > 0)
+    {
+        wait_state = wait_base+1;
+        return 0;
+    }
+    bs->heap->mark_lsn_fsynced(compact_lsn);
+    flusher->active_flushers--;
+    flusher->syncing_buffer--;
+    return 2;
+}
+
+bool journal_flusher_co::trim_lsn(int wait_base)
 {
     if (wait_state == wait_base)        goto resume_0;
     else if (wait_state == wait_base+1) goto resume_1;
     else if (wait_state == wait_base+2) goto resume_2;
-    if (!bs->has_unsynced())
+    else if (wait_state == wait_base+3) goto resume_3;
+    else if (wait_state == wait_base+4) goto resume_4;
+    compact_lsn = bs->heap->get_compacted_lsn();
+    if (((blockstore_meta_header_v3_t*)bs->meta_superblock)->compacted_lsn == compact_lsn)
     {
         return true;
     }
-resume_0:
-    if (flusher->syncing_buffer)
-    {
-        wait_state = wait_base+0;
-        return false;
-    }
     flusher->active_flushers++;
-    flusher->syncing_buffer++;
-resume_1:
     assert(!wait_count);
-    fsynced_lsn = bs->heap->get_completed_lsn();
-    if (!bs->submit_fsyncs(wait_count))
+    if (!bs->dsk.disable_meta_fsync)
     {
-        wait_state = wait_base+1;
-        return false;
+        await_sqe(0);
+        data->iov = { 0 };
+        data->callback = simple_callback_w;
+        io_uring_prep_fsync(sqe, bs->dsk.meta_fd, IORING_FSYNC_DATASYNC);
+        wait_count++;
+    }
+    if (!bs->dsk.disable_data_fsync && bs->dsk.data_fd != bs->dsk.meta_fd)
+    {
+        await_sqe(1);
+        data->iov = { 0 };
+        data->callback = simple_callback_w;
+        io_uring_prep_fsync(sqe, bs->dsk.data_fd, IORING_FSYNC_DATASYNC);
+        wait_count++;
     }
 resume_2:
     if (wait_count > 0)
@@ -764,37 +763,23 @@ resume_2:
         wait_state = wait_base+2;
         return false;
     }
-    bs->heap->mark_lsn_fsynced(fsynced_lsn);
-    flusher->active_flushers--;
-    flusher->syncing_buffer--;
-    return true;
-}
-
-bool journal_flusher_co::trim_lsn(int wait_base)
-{
-    if (wait_state == wait_base)        goto resume_0;
-    else if (wait_state == wait_base+1) goto resume_1;
-    fsynced_lsn = bs->heap->get_fsynced_lsn();
-    if (((blockstore_meta_header_v3_t*)bs->meta_superblock)->completed_lsn == fsynced_lsn)
-    {
-        return true;
-    }
-    flusher->active_flushers++;
-    ((blockstore_meta_header_v3_t*)bs->meta_superblock)->completed_lsn = fsynced_lsn;
+    ((blockstore_meta_header_v3_t*)bs->meta_superblock)->compacted_lsn = compact_lsn;
     ((blockstore_meta_header_v3_t*)bs->meta_superblock)->set_crc32c();
-    await_sqe(0);
+    await_sqe(3);
     data->iov = (struct iovec){ bs->meta_superblock, (size_t)bs->dsk.meta_block_size };
     data->callback = simple_callback_w;
     io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset);
     // Update superblock with datasync
     sqe->rw_flags = RWF_DSYNC;
     wait_count++;
-resume_1:
+resume_4:
     if (wait_count > 0)
     {
-        wait_state = wait_base+1;
+        wait_state = wait_base+4;
         return false;
     }
+    bs->heap->mark_lsn_trimmed(compact_lsn);
+    flusher->compact_counter++;
     flusher->active_flushers--;
     return true;
 }

src/blockstore/blockstore_flush.h

@@ -6,7 +6,7 @@ struct copy_buffer_t
     uint32_t copy_flags;
     uint64_t offset, len, disk_loc, disk_offset, disk_len;
     uint8_t *buf;
-    heap_entry_t *wr;
+    uint64_t wr_lsn;
 };
 
 struct meta_sector_t
@@ -36,28 +36,27 @@ class journal_flusher_co
     int wait_state, wait_count;
     struct io_uring_sqe *sqe;
     struct ring_data_t *data;
-    uint8_t *new_csums = NULL;
-    uint8_t *new_bmp = NULL;
-    uint8_t *punch_bmp = NULL;
-    uint8_t *new_ext_bmp = NULL;
 
     std::function<void(ring_data_t*)> simple_callback_r, simple_callback_w;
 
     object_id cur_oid;
-    heap_entry_t *cur_obj;
-    uint64_t fsynced_lsn;
-    heap_compact_t compact_info;
-    uint64_t clean_loc;
+    uint64_t copy_id;
+    uint64_t compact_lsn;
+    uint64_t cur_version;
+    heap_object_t *cur_obj;
+    heap_write_t *begin_wr, *end_wr;
     uint32_t modified_block;
-    bool bitmap_copied;
     bool should_repeat;
 
     std::vector<copy_buffer_t> read_vec;
-    std::vector<heap_entry_t*> csum_copy;
     uint32_t overwrite_start, overwrite_end;
+    uint32_t big_start, big_end;
     int i, res;
     bool read_to_fill_incomplete;
     int copy_count;
+    uint64_t clean_loc;
+    flusher_meta_write_t meta_old, meta_new;
+    bool do_repeat = false;
 
     friend class journal_flusher_t;
 
@@ -69,7 +68,7 @@ class journal_flusher_co
     bool write_meta_block(int wait_base);
     bool read_buffered(int wait_base);
     bool fsync_meta(int wait_base);
-    bool fsync_buffer(int wait_base);
+    int fsync_buffer(int wait_base);
     bool trim_lsn(int wait_base);
 public:
     journal_flusher_co();
@@ -86,7 +85,9 @@ class journal_flusher_t
     blockstore_impl_t *bs;
     friend class journal_flusher_co;
 
-    robin_hood::unordered_flat_set<object_id> flushing;
+    int advance_lsn_counter = 0;
+    uint64_t compact_counter = 0;
+
     int active_flushers = 0;
     int wanting_meta_fsync = 0;
     bool fsyncing_meta = false;
@@ -97,6 +98,7 @@ public:
     ~journal_flusher_t();
     void loop();
     int get_syncing_buffer();
+    uint64_t get_compact_counter();
     bool is_active();
     void request_trim();
     void release_trim();

src/blockstore/blockstore_heap.h

@@ -1,4 +1,4 @@
-// Metadata storage version 3 ("lsm heap")
+// Metadata storage version 3 ("heap")
 // Copyright (c) Vitaliy Filippov, 2025+
 // License: VNPL-1.1 (see README.md for details)
 
@@ -19,301 +19,326 @@ struct pool_shard_settings_t
 {
     uint32_t pg_count;
     uint32_t pg_stripe_size;
-    uint32_t no_inode_stats;
 };
 
-#define BS_HEAP_TYPE 0x07
-#define BS_HEAP_BIG_WRITE 1
+#define BS_HEAP_TYPE 7
+#define BS_HEAP_OBJECT 1
 #define BS_HEAP_SMALL_WRITE 2
-#define BS_HEAP_INTENT_WRITE 3
-#define BS_HEAP_BIG_INTENT 4
-#define BS_HEAP_DELETE 5
-#define BS_HEAP_COMMIT 6
-#define BS_HEAP_ROLLBACK 7
-#define BS_HEAP_STABLE 0x40
-#define BS_HEAP_GARBAGE 0x80
+#define BS_HEAP_BIG_WRITE 3
+#define BS_HEAP_TOMBSTONE 4
+#define BS_HEAP_INTENT_WRITE 5
+#define BS_HEAP_STABLE 8
 
 class blockstore_heap_t;
 
-struct heap_small_write_t;
-struct heap_big_write_t;
-struct heap_big_intent_t;
-
-struct __attribute__((__packed__)) heap_entry_t
+struct __attribute__((__packed__)) heap_small_write_t
 {
     uint16_t size;
-    uint16_t entry_type;
-    uint32_t crc32c;
+    int16_t next_pos;
+    uint8_t flags;
     uint64_t lsn;
-    uint64_t inode;
-    uint64_t stripe;
     uint64_t version;
+    uint64_t location;
+    uint32_t offset;
+    uint32_t len;
+};
+
+struct __attribute__((__packed__)) heap_big_write_t
+{
+    uint16_t size;
+    int16_t next_pos;
+    uint8_t flags;
+    uint64_t lsn;
+    uint64_t version;
+    uint32_t block_num;
+};
+
+struct __attribute__((__packed__)) heap_tombstone_t
+{
+    uint16_t size;
+    int16_t next_pos;
+    uint8_t flags;
+    uint64_t lsn;
+    uint64_t version;
+};
+
+struct __attribute__((__packed__)) heap_write_t
+{
+    // size should have top bit cleared
+    uint16_t size = 0;
+    int16_t next_pos = 0;
+    uint8_t entry_type = 0; // BS_HEAP_*
+    uint64_t lsn = 0;
+    uint64_t version = 0;
 
     // uint8_t[] external_bitmap
     // uint8_t[] internal_bitmap
     // uint32_t[] checksums
 
+    heap_write_t *next();
     inline uint8_t type() const { return (entry_type & BS_HEAP_TYPE); }
     inline heap_small_write_t& small() { return *(heap_small_write_t*)this; }
     inline heap_big_write_t& big() { return *(heap_big_write_t*)this; }
-    inline heap_big_intent_t& big_intent() { return *(heap_big_intent_t*)this; }
-    bool is_garbage();
-    void set_garbage();
-    bool is_overwrite();
-    bool is_compactable();
-    bool is_before(heap_entry_t *other);
     uint32_t get_size(blockstore_heap_t *heap);
+    uint32_t get_csum_size(blockstore_heap_t *heap);
+    bool needs_recheck(blockstore_heap_t *heap);
+    bool needs_compact(blockstore_heap_t *heap);
+    bool is_compacted(uint64_t compacted_lsn);
+    bool can_be_collapsed(blockstore_heap_t *heap);
+    bool is_allowed_before_compacted(uint64_t compacted_lsn, bool is_last_entry);
     uint8_t *get_ext_bitmap(blockstore_heap_t *heap);
     uint8_t *get_int_bitmap(blockstore_heap_t *heap);
     uint8_t *get_checksums(blockstore_heap_t *heap);
     uint32_t *get_checksum(blockstore_heap_t *heap);
     uint64_t big_location(blockstore_heap_t *heap);
     void set_big_location(blockstore_heap_t *heap, uint64_t location);
+};
+
+struct __attribute__((__packed__)) heap_object_t
+{
+    // size should have top bit cleared
+    uint16_t size = 0;
+    // linked list of write entries...
+    // newest entries are stored first to simplify scanning
+    int16_t write_pos = 0;
+    uint8_t entry_type = 0; // BS_HEAP_*
+    uint32_t crc32c = 0;
+    uint64_t inode = 0;
+    uint64_t stripe = 0;
+
+    heap_write_t *get_writes();
     uint32_t calc_crc32c();
 };
 
-struct __attribute__((__packed__)) heap_small_write_t
+struct heap_object_lsn_t
 {
-    heap_entry_t hdr;
-
-    uint64_t location;
-    uint32_t offset;
-    uint32_t len;
-
-    // Also includes 1 bitmap and 1 crc32c after the bitmap if checksums are disabled
+    object_id oid;
+    uint64_t lsn;
 };
 
-struct __attribute__((__packed__)) heap_big_write_t
+inline bool operator < (const heap_object_lsn_t & a, const heap_object_lsn_t & b)
 {
-    heap_entry_t hdr;
+    return a.oid < b.oid || a.oid == b.oid && a.lsn < b.lsn;
+}
 
-    uint32_t block_num;
+struct tmp_compact_item_t
+{
+    object_id oid;
+    uint64_t lsn;
+    bool compact;
 };
 
-struct __attribute__((__packed__)) heap_big_intent_t
+struct heap_mvcc_copy_id_t
 {
-    heap_entry_t hdr;
-
-    uint32_t block_num;
-    uint32_t offset;
-    uint32_t len;
-
-    // Also includes 2 bitmaps and 1 crc32c if checksums are disabled
+    object_id oid;
+    uint64_t copy_id;
 };
 
-struct __attribute__((__packed__)) heap_list_item_t
+inline bool operator == (const heap_mvcc_copy_id_t & a, const heap_mvcc_copy_id_t & b)
 {
-    heap_list_item_t *prev;
-    heap_list_item_t *next;
-    uint32_t block_num;
-    heap_entry_t entry;
+    return a.oid.inode == b.oid.inode && a.oid.stripe == b.oid.stripe && a.copy_id == b.copy_id;
+}
+
+namespace std
+{
+    template<> struct hash<heap_mvcc_copy_id_t>
+    {
+        inline size_t operator()(const heap_mvcc_copy_id_t &s) const
+        {
+            size_t seed = std::hash<object_id>()(s.oid);
+            // Copy-pasted from spp::hash_combine()
+            seed ^= (s.copy_id + 0xc6a4a7935bd1e995 + (seed << 6) + (seed >> 2));
+            return seed;
+        }
+    };
 };
 
 struct heap_object_mvcc_t
 {
     uint32_t readers = 0;
-    heap_entry_t *garbage_entry = NULL;
+    heap_object_t *entry_copy = NULL;
 };
 
-struct heap_block_info_t
+struct __attribute__((__packed__)) heap_block_info_t
 {
     uint32_t used_space = 0;
-    uint64_t mod_lsn = 0, mod_lsn_to = 0; // only 1 block write of LSN sequence is allowed at a moment
-    bool is_writing: 1;
-    bool has_garbage: 1;
-    std::vector<heap_list_item_t*> entries;
+    uint32_t free_pos = 0;
+    uint8_t *data = NULL;
 };
 
 struct heap_inflight_lsn_t
 {
+    object_id oid;
     uint64_t flags;
-    heap_entry_t *wr;
 };
 
-struct heap_compact_t
+struct heap_refqi_t
 {
-    uint64_t compact_lsn, compact_version;
-    heap_entry_t *clean_wr;
-    bool do_delete;
-};
-
-struct heap_reshard_state_t;
-
-struct heap_li_hash
-{
-    size_t operator()(const heap_list_item_t* li) const noexcept
-    {
-        return robin_hood::hash_int(li->entry.stripe);
-    }
-};
-
-struct heap_li_equal
-{
-    constexpr bool operator()(const heap_list_item_t* a, const heap_list_item_t* b) const noexcept
-    {
-        return a->entry.stripe == b->entry.stripe;
-    }
+    uint64_t lsn;
+    uint64_t inode;
+    uint64_t location;
+    uint32_t len;
+    bool is_data;
 };
 
 using i64hash_t = robin_hood::hash<uint64_t>;
-using heap_inode_map_t = robin_hood::unordered_flat_set<heap_list_item_t*, heap_li_hash, heap_li_equal, 88>;
 using heap_block_index_t = robin_hood::unordered_flat_map<uint64_t,
-    robin_hood::unordered_flat_map<inode_t, void*, i64hash_t>, i64hash_t>;
-using heap_mvcc_map_t = robin_hood::unordered_flat_map<object_id, heap_object_mvcc_t>;
+    robin_hood::unordered_flat_map<inode_t, robin_hood::unordered_flat_map<uint64_t, uint64_t, i64hash_t, std::equal_to<uint64_t>, 88>, i64hash_t>, i64hash_t>;
+using heap_mvcc_map_t = robin_hood::unordered_flat_map<heap_mvcc_copy_id_t, heap_object_mvcc_t>;
 
 class blockstore_heap_t
 {
-    friend struct heap_entry_t;
+    friend class heap_write_t;
+    friend class heap_object_t;
 
     blockstore_disk_t *dsk = NULL;
     uint8_t* buffer_area = NULL;
+    bool abort_on_corruption = false;
+    bool abort_on_overlap = true;
     int log_level = 0;
-    const uint32_t meta_block_count = 0;
-    const uint32_t max_entry_size = 0;
 
+    const uint32_t meta_block_count = 0;
+    uint32_t target_block_free_space = 800;
+
+    uint64_t next_lsn = 0;
     robin_hood::unordered_flat_map<pool_id_t, pool_shard_settings_t> pool_shard_settings;
     // PG => inode => stripe => block number
     heap_block_index_t block_index;
     std::vector<heap_block_info_t> block_info;
     allocator_t *data_alloc = NULL;
     multilist_index_t *meta_alloc = NULL;
-    uint32_t meta_nearfull_blocks = 0;
+    uint32_t meta_alloc_count = 0;
     uint64_t meta_used_space = 0;
     multilist_alloc_t *buffer_alloc = NULL;
+    heap_mvcc_map_t object_mvcc;
+    std::unordered_map<uint64_t, uint32_t> mvcc_data_refs;
+    std::unordered_map<uint64_t, uint32_t> mvcc_buffer_refs;
     std::map<uint64_t, uint64_t> inode_space_stats;
     uint64_t buffer_area_used_space = 0;
     uint64_t data_used_space = 0;
 
-    uint64_t next_lsn = 0;
-    uint32_t last_allocated_block = UINT32_MAX;
-    heap_mvcc_map_t object_mvcc;
-
-    // LSN queue: inflight (writing) -> completed [-> fsynced]
+    // LSN queue: inflight (writing) -> completed [-> fsynced] -> compactable -> compacted [-> fsynced] -> trimmed and removed
     std::deque<heap_inflight_lsn_t> inflight_lsn;
     uint32_t to_compact_count = 0;
-    uint64_t compacted_count = 0;
-    uint32_t inflight_overwrite_count = 0;
     uint64_t first_inflight_lsn = 0;
     uint64_t completed_lsn = 0;
     uint64_t fsynced_lsn = 0;
-    std::deque<object_id> compact_queue;
+    uint64_t compacted_lsn = 0;
+    uint64_t next_compact_lsn = 0;
+    std::deque<heap_refqi_t> overwrite_ref_queue;
 
-    bool marked_used_blocks = false;
-    bool recheck_queue_filled = false;
-    std::set<uint32_t> recheck_modified_blocks;
-    std::deque<heap_entry_t*> recheck_queue;
+    std::vector<tmp_compact_item_t> tmp_compact_queue;
+    std::deque<object_id> recheck_queue;
     int recheck_in_progress = 0;
     bool in_recheck = false;
     std::function<void(bool is_data, uint64_t offset, uint64_t len, uint8_t* buf, std::function<void()>)> recheck_cb;
     int recheck_queue_depth = 0;
 
-    uint64_t get_pg_id(inode_t inode, uint64_t stripe);
-    bool validate_object(heap_entry_t *obj);
-    void fill_recheck_queue();
-    int mark_used_blocks();
-    void recheck_buffer(heap_entry_t *cwr, uint8_t *buf);
-    void defragment_block(uint32_t block_num);
-    void reshard_add(heap_reshard_state_t *st, heap_list_item_t *li);
+    const uint32_t max_write_entry_size;
+
+    uint64_t get_pg_id(inode_t inode, uint64_t stripe);
+    void defragment_block(uint32_t block_num);
+    uint32_t find_block_run(heap_block_info_t & block, uint32_t space);
+    uint32_t find_block_space(uint32_t block_num, uint32_t space);
+    uint32_t block_has_compactable(uint8_t *data);
+    uint32_t compact_object_to(heap_object_t *obj, uint64_t lsn, uint8_t *new_csums, bool do_free);
+    void copy_full_object(uint8_t *dst, heap_object_t *obj);
+    bool mvcc_save_copy(heap_object_t *obj);
+    bool mvcc_check_tracking(object_id oid);
+    void free_mvcc(heap_mvcc_map_t::iterator mvcc_it);
+    void allocate_block(heap_block_info_t & inf);
+    int allocate_new_object(object_id oid, uint32_t full_object_size, uint32_t *modified_block, heap_object_t **new_obj);
+    int add_object(object_id oid, heap_write_t *wr, uint32_t *modified_block);
+    void mark_overwritten(uint64_t over_lsn, uint64_t inode, heap_write_t *wr, heap_write_t *end_wr, bool tracking_active);
+    int update_object(uint32_t block_num, heap_object_t *obj, heap_write_t *wr, uint32_t *modified_block, uint32_t *moved_from_block);
+    void init_erase(uint32_t block_num, heap_object_t *obj);
+    void erase_object(uint32_t block_num, heap_object_t *obj, uint64_t lsn, bool tracking_active);
+    void reindex_block(uint32_t block_num, heap_object_t *from_obj);
+    void erase_block_index(inode_t inode, uint64_t stripe);
+    void deref_data(uint64_t inode, uint64_t location, bool free_at_0);
+    void deref_buffer(uint64_t inode, uint64_t location, uint32_t len, bool free_at_0);
+    void deref_overwrites(uint64_t lsn);
+    void free_object_space(inode_t inode, heap_write_t *from, heap_write_t *to, int mode = 0);
+    void add_used_space(uint32_t block_num, int32_t used_delta);
+    void push_inflight_lsn(object_id oid, uint64_t lsn, uint64_t flags);
 
-    int allocate_entry(uint32_t entry_size, uint32_t *block_num, bool allow_last_free);
-    void insert_list_item(heap_list_item_t *li);
-    int add_entry(uint32_t wr_size, uint32_t *modified_block, bool allow_last_free,
-        bool explicit_complete, std::function<void(heap_entry_t *wr)> fill_entry);
-    int add_simple(heap_entry_t *obj, uint64_t version, uint32_t *modified_block, uint32_t entry_type);
-    uint32_t meta_alloc_pos(const heap_block_info_t & inf);
-    void modify_alloc(uint32_t block_num, std::function<void(heap_block_info_t &)> change_cb);
-    void mark_garbage_up_to(heap_entry_t *wr);
-    void mark_garbage(uint32_t block_num, heap_entry_t *prev_wr, uint32_t used_big);
-    void push_inflight_lsn(uint64_t lsn, heap_entry_t *wr, uint64_t flags);
-    void mark_completed_lsns(uint64_t mod_lsn);
-    void apply_inflight(heap_inflight_lsn_t & inflight);
 public:
     blockstore_heap_t(blockstore_disk_t *dsk, uint8_t *buffer_area, int log_level = 0);
     ~blockstore_heap_t();
-    void start_load(uint64_t completed_lsn);
-    // load data from the disk, returns EDOM on corruption
-    int read_blocks(uint64_t disk_offset, uint64_t size, uint8_t *buf, bool allow_corrupted,
-        std::function<void(uint32_t block_num, heap_entry_t* wr)> handle_write,
-        std::function<void(uint32_t, uint32_t, uint8_t*)> handle_block);
-    int load_blocks(uint64_t disk_offset, uint64_t size, uint8_t *buf,
-        bool allow_corrupted, uint64_t &entries_loaded);
+    // set initially compacted lsn - should be done before loading
+    void set_compacted_lsn(uint64_t compacted_lsn);
+    uint64_t get_compacted_lsn();
+    // load data from the disk, returns count of loaded write entries
+    void read_blocks(uint64_t disk_offset, uint64_t size, uint8_t *buf,
+        std::function<void(heap_object_t*)> handle_object, std::function<void(uint32_t, uint32_t, uint8_t*)> handle_block);
+    uint64_t load_blocks(uint64_t disk_offset, uint64_t size, uint8_t *buf);
     // finish loading
-    int finish_load(bool allow_corrupted = false);
-    // get blocks which are modified during loading and should be written to the disk
-    // before finishing initialization if not R/O
-    std::vector<uint32_t> get_recheck_modified_blocks();
+    void finish_load();
     // recheck small write data after reading the database from disk
     bool recheck_small_writes(std::function<void(bool is_data, uint64_t offset, uint64_t len, uint8_t* buf, std::function<void()>)> read_buffer, int queue_depth);
+    // initialize metadata area (fill it with empty data)
+    // returns 0 when done, EAGAIN when the caller has to wait more
+    int initialize();
+    // read from the metadata area
+    // returns 0 when done, EAGAIN when the caller has to wait more
+    int read();
     // reshard database according to the pool's PG count
-    void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
-    bool reshard_continue(void* reshard_state, uint64_t chunk_limit);
-    bool reshard_check(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size);
-    void set_no_inode_stats(const std::vector<uint64_t> & pool_ids);
-    void recalc_inode_space_stats(uint64_t pool_id, bool per_inode);
+    void reshard(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size);
     // read an object entry and lock it against removal
     // in the future, may become asynchronous
-    heap_entry_t *lock_and_read_entry(object_id oid);
+    heap_object_t *lock_and_read_entry(object_id oid, uint64_t & copy_id);
+    // re-read a locked object entry with the given lsn (pointer may be invalidated)
+    heap_object_t *read_locked_entry(object_id oid, uint64_t copy_id);
     // read an object entry without locking it
-    heap_entry_t *read_entry(object_id oid);
+    heap_object_t *read_entry(object_id oid, uint32_t *block_num_ptr, bool for_update = false);
     // unlock an entry
-    bool unlock_entry(object_id oid);
+    bool unlock_entry(object_id oid, uint64_t copy_id);
     // set or verify checksums in a write request
-    bool calc_checksums(heap_entry_t *wr, uint8_t *data, bool set, uint32_t offset = UINT32_MAX, uint32_t len = UINT32_MAX);
+    bool calc_checksums(heap_write_t *wr, uint8_t *data, bool set, uint32_t offset = 0, uint32_t len = 0);
     // set or verify raw block checksums
     bool calc_block_checksums(uint32_t *block_csums, uint8_t *data, uint8_t *bitmap, uint32_t start, uint32_t end,
         bool set, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);
     bool calc_block_checksums(uint32_t *block_csums, uint8_t *bitmap,
         uint32_t start, uint32_t end, std::function<uint8_t*(uint32_t start, uint32_t & len)> next,
         bool set, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);
-    // adds a small_write or intent_write entry to an object
+    // copy an object as is
+    int copy_object(heap_object_t *obj, uint32_t *modified_block);
+    // auto-compacts the object, then adds a write entry to it and to the compaction queue
     // return 0 if OK, or maybe ENOSPC
-    int add_small_write(object_id oid, heap_entry_t **obj_ptr, uint16_t type, uint64_t version,
-        uint32_t offset, uint32_t len, uint64_t location, uint8_t *bitmap, uint8_t *data, uint32_t *modified_block);
-    // adds a big_write (overwrite) entry to an object
-    int add_big_write(object_id oid, heap_entry_t *old_head, bool stable, uint64_t version,
-        uint32_t offset, uint32_t len, uint64_t location, uint8_t *bitmap, uint8_t *data, uint32_t *modified_block);
-    // adds a "redirecting" big_intent entry to an object (same as big_write, used to avoid fsync on desktop SSDs)
-    int add_redirect_intent(object_id oid, heap_entry_t **obj_ptr, uint64_t version,
-        uint32_t offset, uint32_t len, uint64_t location, uint8_t *bitmap, uint8_t *data, uint32_t *modified_block);
-    // adds a big_intent (atomic partial modification) entry to an object
-    int add_big_intent(object_id oid, heap_entry_t **obj_ptr, uint64_t version,
-        uint32_t offset, uint32_t len, uint8_t *bitmap, uint8_t *data, uint8_t *checksums, uint32_t *modified_block);
-    // adds a compacted up to <version> entry to an object
-    int add_compact(heap_entry_t *obj, uint64_t compact_version, uint64_t compact_lsn, uint64_t compact_location,
-        bool do_delete, uint32_t *modified_block, uint8_t *new_int_bitmap, uint8_t *new_ext_bitmap, uint8_t *new_csums);
-    // "punch holes" in a big_entry
-    int punch_holes(heap_entry_t *wr, uint8_t *new_bitmap, uint8_t *new_csums, uint32_t *modified_block);
+    int post_write(object_id oid, heap_write_t *wr, uint32_t *modified_block, uint32_t *moved_from_block);
+    int post_write(uint32_t & block_num, object_id oid, heap_object_t *obj, heap_write_t *wr, uint32_t *moved_from_block);
     // stabilize an unstable object version
     // return 0 if OK, ENOENT if not exists
-    int add_commit(heap_entry_t *obj, uint64_t version, uint32_t *modified_block);
+    int post_stabilize(object_id oid, uint64_t version, uint32_t *modified_block, uint64_t *new_lsn, uint64_t *new_to_lsn);
     // rollback an unstable object version
     // return 0 if OK, ENOENT if not exists, EBUSY if already stable
-    int add_rollback(heap_entry_t *obj, uint64_t version, uint32_t *modified_block);
+    int post_rollback(object_id oid, uint64_t version, uint64_t *new_lsn, uint32_t *modified_block);
     // forget an object
     // return error code
-    int add_delete(heap_entry_t *obj, uint32_t *modified_block);
+    int post_delete(object_id oid, uint64_t *new_lsn, uint32_t *modified_block);
+    int post_delete(uint32_t block_num, heap_object_t *obj, uint64_t *new_lsn);
     // get the next object to compact
     // guaranteed to return objects in min lsn order
     // returns 0 if OK, ENOENT if nothing to compact
     int get_next_compact(object_id & oid);
-    void iterate_with_stable(heap_entry_t *obj, uint64_t max_lsn, std::function<bool(heap_entry_t*, bool stable)> cb);
-    // iterate compactable entries
-    heap_compact_t iterate_compaction(heap_entry_t *obj, uint64_t fsynced_lsn, bool under_pressure,
-        std::function<void(heap_entry_t*)> small_wr_cb);
-    // iterate all objects
-    void iterate_objects(std::function<void(heap_entry_t*, uint32_t block_num)> cb);
+    // get the range of an object eligible for compaction
+    void get_compact_range(heap_object_t *obj, uint64_t max_lsn, heap_write_t **begin_wr, heap_write_t **end_wr);
+    // mark an object as compacted up to the given lsn
+    int compact_object(object_id oid, uint64_t lsn, uint8_t *new_csums);
     // retrieve object listing from a PG
     int list_objects(uint32_t pg_num, object_id min_oid, object_id max_oid,
         obj_ver_id **result_list, size_t *stable_count, size_t *unstable_count);
+    // set a block number for a new object and returns error status: 0, EAGAIN or ENOSPC
+    int get_block_for_new_object(uint32_t & out_block_num, uint32_t size = 0);
 
     // inflight write tracking
-    void start_block_write(uint32_t block_num);
-    void complete_block_write(uint32_t block_num);
-    void complete_lsn_write(uint64_t lsn);
-    bool is_lsn_completed(uint64_t lsn);
+    void mark_lsn_completed(uint64_t lsn);
+    void mark_lsn_fsynced(uint64_t lsn);
+    void mark_lsn_compacted(uint64_t lsn, bool allow_undone = false);
+    void mark_object_compacted(heap_object_t *obj, uint64_t max_lsn);
+    void mark_lsn_trimmed(uint64_t lsn);
     uint64_t get_completed_lsn();
     uint64_t get_fsynced_lsn();
-    void mark_lsn_fsynced(uint64_t lsn);
 
     // data device block allocator functions
     uint64_t find_free_data();
@@ -329,8 +354,7 @@ public:
     uint64_t get_buffer_area_used_space();
 
     // get metadata block data buffer and used space
-    void get_meta_block(uint32_t block_num, uint8_t *buffer);
-    void fill_block_empty_space(uint8_t *buffer, uint32_t pos);
+    uint8_t *get_meta_block(uint32_t block_num);
     uint32_t get_meta_block_used_space(uint32_t block_num);
 
     // get space usage statistics
@@ -339,17 +363,14 @@ public:
     uint64_t get_meta_total_space();
     uint64_t get_meta_used_space();
     uint32_t get_meta_nearfull_blocks();
+    uint32_t get_inflight_queue_size();
     uint32_t get_compact_queue_size();
     uint32_t get_to_compact_count();
-    uint64_t get_compacted_count();
 
-    uint64_t entry_pos(uint32_t block_num, uint32_t offset);
-    heap_entry_t *entry_from_pos(uint64_t entry_pos, bool allow_unallocated = false);
-    heap_entry_t *prev(heap_entry_t *wr);
-    uint32_t get_simple_entry_size();
-    uint32_t get_big_entry_size();
-    uint32_t get_big_intent_entry_size();
-    uint32_t get_small_entry_size(uint32_t offset, uint32_t len);
-    uint32_t get_csum_size(heap_entry_t *wr);
-    uint32_t get_csum_size(uint32_t entry_type, uint32_t offset = 0, uint32_t len = 0);
+    // get maximum size for a temporary heap_write_t buffer
+    uint32_t get_max_write_entry_size();
+
+    // only for tests
+    void set_abort_on_corruption(bool fail);
+    void set_abort_on_overlap(bool fail);
 };

src/blockstore/blockstore_impl.cpp

@@ -23,22 +23,26 @@ blockstore_impl_t::blockstore_impl_t(blockstore_config_t & config, ring_loop_i *
         dsk.open_meta();
         dsk.open_journal();
         dsk.calc_lengths();
-        dsk.check_lengths();
+        zero_object = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.data_block_size);
     }
     catch (std::exception & e)
     {
         dsk.close_all();
         throw;
     }
+    memset(zero_object, 0, dsk.data_block_size);
     meta_superblock = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size);
     memset(meta_superblock, 0, dsk.meta_block_size);
+}
+
+void blockstore_impl_t::init()
+{
     flusher = new journal_flusher_t(this);
     if (dsk.inmemory_journal)
     {
         buffer_area = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.journal_len);
     }
     heap = new blockstore_heap_t(&dsk, buffer_area, log_level);
-    ringloop->wakeup();
 }
 
 blockstore_impl_t::~blockstore_impl_t()
@@ -51,6 +55,8 @@ blockstore_impl_t::~blockstore_impl_t()
         free(buffer_area);
     if (meta_superblock)
         free(meta_superblock);
+    if (zero_object)
+        free(zero_object);
     ringloop->unregister_consumer(&ring_consumer);
     dsk.close_all();
 }
@@ -186,19 +192,11 @@ void blockstore_impl_t::loop()
         {
             flusher->loop();
         }
-        for (auto & block_num: pending_modified_blocks)
-        {
-            auto & mb = modified_blocks[block_num];
-            heap->get_meta_block(block_num, mb.buf);
-            heap->start_block_write(block_num);
-            mb.sent = true;
-        }
         int ret = ringloop->submit();
         if (ret < 0)
         {
             throw std::runtime_error(std::string("io_uring_submit: ") + strerror(-ret));
         }
-        pending_modified_blocks.clear();
         if ((initial_ring_space - ringloop->space_left()) > 0)
         {
             live = true;
@@ -216,7 +214,7 @@ bool blockstore_impl_t::is_safe_to_stop()
     {
         return false;
     }
-    if (has_unsynced())
+    if (unsynced_big_write_count > 0 || unsynced_small_write_count > 0)
     {
         if (!readonly && !stop_sync_submitted)
         {
@@ -252,7 +250,7 @@ void blockstore_impl_t::check_wait(blockstore_op_t *op)
     }
     else if (PRIV(op)->wait_for == WAIT_COMPACTION)
     {
-        if (heap->get_compacted_count() <= PRIV(op)->wait_detail)
+        if (flusher->get_compact_counter() <= PRIV(op)->wait_detail)
         {
             // do not submit
 #ifdef BLOCKSTORE_DEBUG
@@ -275,8 +273,7 @@ void blockstore_impl_t::enqueue_op(blockstore_op_t *op)
         ((op->opcode == BS_OP_READ || op->opcode == BS_OP_WRITE || op->opcode == BS_OP_WRITE_STABLE) && (
             op->offset >= dsk.data_block_size ||
             op->len > dsk.data_block_size-op->offset ||
-            (op->offset % dsk.bitmap_granularity) ||
-            (op->len % dsk.bitmap_granularity)
+            (op->len % dsk.disk_alignment)
         )) ||
         readonly && op->opcode != BS_OP_READ && op->opcode != BS_OP_LIST)
     {
@@ -323,13 +320,9 @@ void blockstore_impl_t::process_list(blockstore_op_t *op)
         FINISH_OP(op);
         return;
     }
-    // Check if the DB is sharded correctly
-    if (!heap->reshard_check(INODE_POOL(min_inode), pg_count, pg_stripe_size))
-    {
-        op->retval = -EAGAIN;
-        FINISH_OP(op);
-        return;
-    }
+    // Check if the DB needs resharding
+    // (we don't know about PGs from the beginning, we only create "shards" here)
+    heap->reshard(INODE_POOL(min_inode), pg_count, pg_stripe_size);
     obj_ver_id *result = NULL;
     size_t stable_count = 0, unstable_count = 0;
     int res = heap->list_objects(list_pg, op->min_oid, op->max_oid, &result, &stable_count, &unstable_count);
@@ -352,7 +345,6 @@ void blockstore_impl_t::process_list(blockstore_op_t *op)
 
 void blockstore_impl_t::set_no_inode_stats(const std::vector<uint64_t> & pool_ids)
 {
-    heap->set_no_inode_stats(pool_ids);
 }
 
 void blockstore_impl_t::dump_diagnostics()
@@ -396,13 +388,3 @@ std::string blockstore_impl_t::get_op_diag(blockstore_op_t *op)
         snprintf(buf, sizeof(buf), "state=%d", priv->op_state);
     return std::string(buf);
 }
-
-void* blockstore_impl_t::reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit)
-{
-    return heap->reshard_start(pool, pg_count, pg_stripe_size, chunk_limit);
-}
-
-bool blockstore_impl_t::reshard_continue(void *reshard_state, uint64_t chunk_limit)
-{
-    return heap->reshard_continue(reshard_state, chunk_limit);
-}

src/blockstore/blockstore_impl.h

@@ -29,6 +29,18 @@ class blockstore_impl_t;
 
 //#define BLOCKSTORE_DEBUG
 
+// - Sync must be submitted after previous writes/deletes (not before!)
+// - Reads may be submitted in parallel with writes/deletes because we use MVCC
+// - Writes may be submitted in any order, because they don't overlap. Each write
+//   goes into a new location - either on the journal device or on the data device
+// - Stable (stabilize) must be submitted after sync of that object is completed
+//   It's even OK to return an error to the caller if that object is not synced yet
+// - compacted_lsn should be moved forward only after all versions are moved to the main storage
+// - If an operation can not be submitted because the ring is full
+//   we should stop submission of other operations. Otherwise some "scatter" reads
+//   may end up blocked for a long time.
+// Otherwise, the submission order is free.
+
 #include "blockstore_init.h"
 
 #include "blockstore_flush.h"
@@ -41,30 +53,25 @@ struct blockstore_op_private_t
     int pending_ops;
     int op_state;
 
-    // Write, sync, stabilize
-    uint32_t modified_block, modified_block2;
+    // Read, write, sync, stabilize
+    uint64_t lsn;
 
     // Read
     std::vector<copy_buffer_t> read_vec;
 
-    // Read, write
-    uint64_t lsn;
-
     // Write
     uint64_t location;
-    uint32_t write_type;
+    bool is_big;
 
     // Stabilize, rollback
     int stab_pos;
 
-    // Write
-    timespec tv_begin;
-};
+    // Stabilize
+    uint64_t to_lsn;
 
-struct bs_modified_block_t
-{
-    bool sent;
-    uint8_t *buf;
+    // Write
+    struct iovec iov_zerofill[3];
+    timespec tv_begin;
 };
 
 class blockstore_impl_t: public blockstore_i
@@ -108,16 +115,13 @@ public:
     uint8_t* meta_superblock = NULL;
     uint8_t *buffer_area = NULL;
     std::vector<blockstore_op_t*> submit_queue;
-    int unsynced_data_write_count = 0, unsynced_buffer_write_count = 0, unsynced_meta_write_count = 0;
+    int unsynced_big_write_count = 0, unsynced_small_write_count = 0, unsynced_meta_write_count = 0;
     int unsynced_queued_ops = 0;
-
-    std::vector<uint32_t> pending_modified_blocks;
-    robin_hood::unordered_flat_map<uint32_t, bs_modified_block_t> modified_blocks;
+    uint8_t *zero_object = NULL;
 
     journal_flusher_t *flusher;
     int write_iodepth = 0;
     int inflight_big = 0;
-    int intent_write_counter = 0;
     bool fsyncing_data = false;
 
     bool live = false, queue_stall = false;
@@ -142,17 +146,18 @@ public:
     int metadata_buf_size;
     blockstore_init_meta* metadata_init_reader;
 
+    void init();
     void check_wait(blockstore_op_t *op);
     void init_op(blockstore_op_t *op);
 
     // Read
     int dequeue_read(blockstore_op_t *op);
     int fulfill_read(blockstore_op_t *op);
-    uint32_t prepare_read(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end, uint32_t skip_csum);
-    uint32_t prepare_read_with_bitmaps(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end, uint32_t skip_csum);
+    uint32_t prepare_read(std::vector<copy_buffer_t> & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end);
+    uint32_t prepare_read_with_bitmaps(std::vector<copy_buffer_t> & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end);
     uint32_t prepare_read_zero(std::vector<copy_buffer_t> & read_vec, uint32_t start, uint32_t end);
-    uint32_t prepare_read_simple(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end, uint32_t skip_csum);
-    void prepare_disk_read(std::vector<copy_buffer_t> & read_vec, int pos, heap_entry_t *obj, heap_entry_t *wr,
+    uint32_t prepare_read_simple(std::vector<copy_buffer_t> & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end);
+    void prepare_disk_read(std::vector<copy_buffer_t> & read_vec, int pos, heap_object_t *obj, heap_write_t *wr,
         uint32_t blk_start, uint32_t blk_end, uint32_t start, uint32_t end, uint32_t copy_flags);
     void find_holes(std::vector<copy_buffer_t> & read_vec, uint32_t item_start, uint32_t item_end,
         std::function<void(int&, uint32_t, uint32_t)> callback);
@@ -162,10 +167,9 @@ public:
 
     // Write
     bool enqueue_write(blockstore_op_t *op);
-    void prepare_meta_block_write(uint32_t modified_block);
-    bool meta_block_is_pending(uint32_t modified_block);
-    bool intent_write_allowed(blockstore_op_t *op, heap_entry_t *obj);
+    void prepare_meta_block_write(blockstore_op_t *op, uint64_t modified_block, io_uring_sqe *sqe = NULL);
     int dequeue_write(blockstore_op_t *op);
+    int make_big_write(blockstore_op_t *op, uint32_t offset, uint32_t len, uint32_t *modified_block, uint32_t *moved_from_block);
     int continue_write(blockstore_op_t *op);
     void handle_write_event(ring_data_t *data, blockstore_op_t *op);
 
@@ -173,7 +177,6 @@ public:
     int continue_sync(blockstore_op_t *op);
     bool submit_fsyncs(int & wait_count);
     int do_sync(blockstore_op_t *op, int base_state);
-    bool has_unsynced();
 
     // Stabilize
     int dequeue_stable(blockstore_op_t *op);
@@ -189,9 +192,6 @@ public:
     void parse_config(blockstore_config_t & config);
     void parse_config(blockstore_config_t & config, bool init);
 
-    void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
-    bool reshard_continue(void *reshard_state, uint64_t chunk_limit);
-
     // Event loop
     void loop();
 
@@ -226,6 +226,6 @@ public:
     inline uint32_t get_block_size() { return dsk.data_block_size; }
     inline uint64_t get_block_count() { return dsk.block_count; }
     uint64_t get_free_block_count();
-    inline uint32_t get_bitmap_granularity() { return dsk.bitmap_granularity; }
+    inline uint32_t get_bitmap_granularity() { return dsk.disk_alignment; }
     inline uint64_t get_journal_size() { return dsk.journal_len; }
 };

src/blockstore/blockstore_init.cpp

@@ -51,8 +51,6 @@ int blockstore_init_meta::loop()
     else if (wait_state == 5) goto resume_5;
     else if (wait_state == 6) goto resume_6;
     else if (wait_state == 7) goto resume_7;
-    else if (wait_state == 8) goto resume_8;
-    else if (wait_state == 9) goto resume_9;
     metadata_buffer = memalign(MEM_ALIGNMENT, 2*bs->metadata_buf_size);
     if (!metadata_buffer)
         throw std::runtime_error("Failed to allocate metadata read buffer");
@@ -72,6 +70,7 @@ resume_1:
     }
     if (is_zero((uint64_t*)bs->meta_superblock, bs->dsk.meta_block_size))
     {
+        bs->dsk.check_lengths();
         {
             blockstore_meta_header_v3_t *hdr = (blockstore_meta_header_v3_t *)bs->meta_superblock;
             hdr->zero = 0;
@@ -80,15 +79,12 @@ resume_1:
             hdr->meta_block_size = bs->dsk.meta_block_size;
             hdr->data_block_size = bs->dsk.data_block_size;
             hdr->bitmap_granularity = bs->dsk.bitmap_granularity;
+            hdr->compacted_lsn = 0;
             if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_V2)
             {
                 hdr->data_csum_type = bs->dsk.data_csum_type;
                 hdr->csum_block_size = bs->dsk.csum_block_size;
             }
-            if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_HEAP)
-            {
-                hdr->meta_area_size = bs->dsk.meta_area_size;
-            }
             hdr->set_crc32c();
         }
         if (bs->readonly)
@@ -139,22 +135,28 @@ resume_1:
             hdr->data_block_size != bs->dsk.data_block_size ||
             hdr->bitmap_granularity != bs->dsk.bitmap_granularity ||
             hdr->data_csum_type != bs->dsk.data_csum_type ||
-            hdr->csum_block_size != bs->dsk.csum_block_size ||
-            hdr->meta_area_size != bs->dsk.meta_area_size)
+            hdr->csum_block_size != bs->dsk.csum_block_size)
         {
             printf(
                 "Configuration stored in metadata superblock"
-                " (meta_block_size=%u, data_block_size=%u, bitmap_granularity=%u, data_csum_type=%u, csum_block_size=%u, meta_area_size=%ju)"
-                " differs from OSD configuration (%u/%u/%u, %u/%u, %ju).\n",
+                " (meta_block_size=%u, data_block_size=%u, bitmap_granularity=%u, data_csum_type=%u, csum_block_size=%u)"
+                " differs from OSD configuration (%u/%u/%u, %u/%u).\n",
                 hdr->meta_block_size, hdr->data_block_size, hdr->bitmap_granularity,
-                hdr->data_csum_type, hdr->csum_block_size, hdr->meta_area_size,
+                hdr->data_csum_type, hdr->csum_block_size,
                 bs->dsk.meta_block_size, bs->dsk.data_block_size, bs->dsk.bitmap_granularity,
-                bs->dsk.data_csum_type, bs->dsk.csum_block_size, bs->dsk.meta_area_size
+                bs->dsk.data_csum_type, bs->dsk.csum_block_size
             );
             exit(1);
         }
+        if (bs->dsk.meta_format != hdr->version)
+        {
+            bs->dsk.meta_format = hdr->version;
+            bs->dsk.calc_lengths();
+        }
+        bs->dsk.check_lengths();
     }
-    bs->heap->start_load(((blockstore_meta_header_v3_t *)bs->meta_superblock)->completed_lsn);
+    bs->init();
+    bs->heap->set_compacted_lsn(((blockstore_meta_header_v3_t *)bs->meta_superblock)->compacted_lsn);
     if (bs->dsk.inmemory_journal)
     {
         // Read buffer area
@@ -208,10 +210,8 @@ resume_4:
                     io_uring_prep_readv(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bufs[i].offset);
                 else
                 {
-                    // Fill metadata with empty block pattern
-                    memset(bufs[i].buf, 0, bufs[i].size);
-                    for (uint64_t o = 0; o < bufs[i].size; o += bs->dsk.meta_block_size)
-                        bs->heap->fill_block_empty_space(bufs[i].buf + o, 0);
+                    // Fill metadata with zeroes
+                    memset(data->iov.iov_base, 0, data->iov.iov_len);
                     io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bufs[i].offset);
                 }
                 bs->ringloop->submit();
@@ -224,11 +224,7 @@ resume_4:
         if (bufs[i].state == INIT_META_READ_DONE)
         {
             // Handle result
-            uint64_t loaded = 0;
-            int r = bs->heap->load_blocks(bufs[i].offset-bs->dsk.meta_block_size, bufs[i].size, bufs[i].buf, false, loaded);
-            if (r != 0)
-                exit(1);
-            entries_loaded += loaded;
+            entries_loaded += bs->heap->load_blocks(bufs[i].offset-bs->dsk.meta_block_size, bufs[i].size, bufs[i].buf);
             bufs[i].state = 0;
             bs->ringloop->wakeup();
         }
@@ -269,7 +265,7 @@ resume_6:
         }
         GET_SQE();
         data->iov = (iovec){ buf, len };
-        data->callback = [offset, cb](ring_data_t *data)
+        data->callback = [this, offset, cb](ring_data_t *data)
         {
             if (data->res < 0)
             {
@@ -284,51 +280,7 @@ resume_6:
     }, bs->meta_write_recheck_parallelism);
     return 1;
 resume_7:
-    if (bs->heap->finish_load() != 0)
-    {
-        exit(1);
-    }
-    recheck_mod = bs->heap->get_recheck_modified_blocks();
-    if (bs->readonly)
-    {
-        recheck_mod.clear();
-    }
-    for (i = 0; i < recheck_mod.size(); i++)
-    {
-resume_8:
-        if (wait_count >= bs->meta_write_recheck_parallelism || !(sqe = bs->get_sqe()))
-        {
-            bs->ringloop->submit();
-            wait_state = 8;
-            return 1;
-        }
-        uint32_t block_num = recheck_mod[i];
-        uint64_t block_offset = bs->dsk.meta_offset + (uint64_t)(block_num+1) * bs->dsk.meta_block_size;
-        data = ((ring_data_t*)sqe->user_data);
-        uint8_t *buf = (uint8_t*)malloc_or_die(bs->dsk.meta_block_size);
-        bs->heap->get_meta_block(block_num, buf);
-        data->iov = { buf, bs->dsk.meta_block_size };
-        data->callback = [this, buf, block_offset](ring_data_t *data)
-        {
-            wait_count--;
-            free(buf);
-            if (data->res != bs->dsk.meta_block_size)
-            {
-                throw std::runtime_error(
-                    "write metadata failed at offset " + std::to_string(block_offset) + ": " + strerror(-data->res)
-                );
-            }
-        };
-        io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, block_offset);
-        wait_count++;
-    }
-resume_9:
-    if (wait_count > 0)
-    {
-        bs->ringloop->submit();
-        wait_state = 9;
-        return 1;
-    }
+    bs->heap->finish_load();
     free(metadata_buffer);
     metadata_buffer = NULL;
     return 0;

src/blockstore/blockstore_init.h

@@ -15,7 +15,6 @@ class blockstore_init_meta
 {
     blockstore_impl_t *bs;
     int wait_state = 0;
-    int wait_count = 0;
     bool zero_on_init = false;
     void *metadata_buffer = NULL;
     blockstore_init_meta_buf bufs[2] = {};
@@ -26,7 +25,6 @@ class blockstore_init_meta
     uint64_t next_offset = 0;
     uint64_t last_read_offset = 0;
     uint64_t entries_loaded = 0;
-    std::vector<uint32_t> recheck_mod;
     int i = 0, j = 0;
     bool handle_meta_block(uint8_t *buf, uint64_t count, uint64_t done_cnt);
     void handle_event(ring_data_t *data, int buf_num);

src/blockstore/blockstore_internal.h

@@ -1,6 +1,3 @@
-// Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.1 (see README.md for details)
-
 #pragma once
 
 #define BS_SUBMIT_CHECK_SQES(n) \
@@ -51,11 +48,3 @@
 #define COPY_BUF_COALESCED  0x10
 #define COPY_BUF_PADDED     0x20
 #define COPY_BUF_SKIP_CSUM  0x40
-
-#ifndef RWF_ATOMIC
-#define RWF_ATOMIC 0x40
-#endif
-
-#ifndef RWF_DSYNC
-#define RWF_DSYNC 0x02
-#endif

src/blockstore/blockstore_open.cpp

@@ -42,7 +42,7 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init)
     }
     if (!journal_trim_interval)
     {
-        journal_trim_interval = 4096;
+        journal_trim_interval = 1024;
     }
     if (!flusher_start_threshold)
     {
@@ -96,10 +96,6 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init)
     {
         metadata_buf_size = 4*1024*1024;
     }
-    if (metadata_buf_size % dsk.meta_block_size)
-    {
-        throw std::runtime_error("metadata_buf_size should be a multiple of meta_block_size");
-    }
     if (!meta_write_recheck_parallelism)
     {
         meta_write_recheck_parallelism = 16;

src/blockstore/blockstore_read.cpp

@@ -7,7 +7,7 @@
 
 int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
 {
-    heap_entry_t *obj = heap->lock_and_read_entry(op->oid);
+    heap_object_t *obj = heap->lock_and_read_entry(op->oid, PRIV(op)->lsn);
     if (!obj)
     {
         op->version = 0;
@@ -17,44 +17,16 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
     }
     uint32_t fulfilled = 0;
     PRIV(op)->pending_ops = 0;
-    PRIV(op)->lsn = 0;
     auto & rv = PRIV(op)->read_vec;
     uint64_t result_version = 0;
     bool found = false;
-    uint32_t skip_csum = 0;
-    uint32_t blk_start = op->offset, blk_end = op->offset+op->len;
-    bool need_skip = dsk.csum_block_size > dsk.bitmap_granularity && !perfect_csum_update;
-    if (need_skip)
+    for (auto wr = obj->get_writes(); wr; wr = wr->next())
     {
-        PRIV(op)->lsn = obj->lsn;
-        blk_start = op->offset - op->offset%dsk.csum_block_size;
-        blk_end = op->offset + op->len;
-        if (blk_end % dsk.csum_block_size)
-            blk_end += dsk.csum_block_size - (blk_end % dsk.csum_block_size);
-    }
-    bool need_wait = false;
-    heap->iterate_with_stable(obj, obj->lsn, [&](heap_entry_t *wr, bool stable)
-    {
-        if (wr->type() == BS_HEAP_DELETE)
+        if (op->version < wr->version)
         {
-            return false;
+            continue;
         }
-        if (!heap->is_lsn_completed(wr->lsn))
-        {
-            if (wr->type() == BS_HEAP_BIG_INTENT && wr->big_intent().offset < blk_end && wr->big_intent().offset+wr->big_intent().len > blk_start ||
-                wr->type() == BS_HEAP_INTENT_WRITE && wr->small().offset < blk_end && wr->small().offset+wr->small().len > blk_start)
-            {
-                // Wait until intent write is completed
-                need_wait = true;
-                return false;
-            }
-            else if (wr->type() == BS_HEAP_SMALL_WRITE && wr->small().offset < blk_end && wr->small().offset+wr->small().len > blk_start)
-            {
-                // Skip entry and read the previous one
-                return true;
-            }
-        }
-        if (op->version >= wr->version && !found)
+        if (!found)
         {
             found = true;
             result_version = wr->version;
@@ -63,39 +35,18 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
                 memcpy(op->bitmap, wr->get_ext_bitmap(heap), dsk.clean_entry_bitmap_size);
             }
         }
-        if (op->version >= wr->version)
+        fulfilled += prepare_read(PRIV(op)->read_vec, obj, wr, op->offset, op->offset+op->len);
+        if (fulfilled == op->len ||
+            wr->type() == BS_HEAP_BIG_WRITE ||
+            wr->type() == BS_HEAP_TOMBSTONE)
         {
-            fulfilled += prepare_read(PRIV(op)->read_vec, obj, wr, op->offset, op->offset+op->len,
-                wr->type() != BS_HEAP_SMALL_WRITE ? skip_csum : 0);
-            if (fulfilled == op->len ||
-                wr->type() == BS_HEAP_BIG_WRITE ||
-                wr->type() == BS_HEAP_BIG_INTENT ||
-                wr->type() == BS_HEAP_DELETE)
-            {
-                return false;
-            }
+            break;
         }
-        if (need_skip && wr->type() == BS_HEAP_SMALL_WRITE &&
-            wr->small().offset < blk_end && wr->small().offset+wr->small().len > blk_start)
-        {
-            // Small write may mutate big write checksums during flush
-            skip_csum = COPY_BUF_SKIP_CSUM;
-        }
-        return true;
-    });
-    if (need_wait)
-    {
-undo_wait:
-        // Need to wait. undo added requests, unlock lsn
-        heap->unlock_entry(op->oid);
-        free_read_buffers(rv);
-        rv.clear();
-        return 0;
     }
     if (!found)
     {
         // May happen if there are entries but all of them are > requested version
-        heap->unlock_entry(op->oid);
+        heap->unlock_entry(op->oid, PRIV(op)->lsn);
         op->version = 0;
         op->retval = -ENOENT;
         FINISH_OP(op);
@@ -104,13 +55,17 @@ undo_wait:
     assert(fulfilled == op->len);
     if (!fulfill_read(op))
     {
-        goto undo_wait;
+        // Need to wait. undo added requests, unlock lsn
+        heap->unlock_entry(op->oid, PRIV(op)->lsn);
+        free_read_buffers(rv);
+        rv.clear();
+        return 0;
     }
     op->version = result_version;
     if (!PRIV(op)->pending_ops)
     {
         // everything is fulfilled from memory
-        heap->unlock_entry(op->oid);
+        heap->unlock_entry(op->oid, PRIV(op)->lsn);
         op->retval = op->len;
         free_read_buffers(rv);
         FINISH_OP(op);
@@ -153,20 +108,20 @@ int blockstore_impl_t::fulfill_read(blockstore_op_t *op)
     return 1;
 }
 
-uint32_t blockstore_impl_t::prepare_read(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end, uint32_t skip_csum)
+uint32_t blockstore_impl_t::prepare_read(std::vector<copy_buffer_t> & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end)
 {
-    if (wr->type() == BS_HEAP_BIG_WRITE || wr->type() == BS_HEAP_BIG_INTENT)
+    if (wr->type() == BS_HEAP_BIG_WRITE)
     {
-        return prepare_read_with_bitmaps(read_vec, obj, wr, start, end, skip_csum);
+        return prepare_read_with_bitmaps(read_vec, obj, wr, start, end);
     }
-    if (wr->type() == BS_HEAP_DELETE)
+    if (wr->type() == BS_HEAP_TOMBSTONE)
     {
         return prepare_read_zero(read_vec, start, end);
     }
-    return prepare_read_simple(read_vec, obj, wr, start, end, skip_csum);
+    return prepare_read_simple(read_vec, obj, wr, start, end);
 }
 
-uint32_t blockstore_impl_t::prepare_read_with_bitmaps(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end, uint32_t skip_csum)
+uint32_t blockstore_impl_t::prepare_read_with_bitmaps(std::vector<copy_buffer_t> & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end)
 {
     // BIG_WRITEs contain a bitmap and we have to handle its holes
     uint32_t res = 0;
@@ -189,7 +144,7 @@ uint32_t blockstore_impl_t::prepare_read_with_bitmaps(std::vector<copy_buffer_t>
         }
         if (bmp_end > bmp_start)
         {
-            res += prepare_read_simple(read_vec, obj, wr, bmp_start * dsk.bitmap_granularity, bmp_end * dsk.bitmap_granularity, skip_csum);
+            res += prepare_read_simple(read_vec, obj, wr, bmp_start * dsk.bitmap_granularity, bmp_end * dsk.bitmap_granularity);
             bmp_start = bmp_end;
         }
     }
@@ -211,7 +166,7 @@ uint32_t blockstore_impl_t::prepare_read_zero(std::vector<copy_buffer_t> & read_
     return res;
 }
 
-uint32_t blockstore_impl_t::prepare_read_simple(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end, uint32_t skip_csum)
+uint32_t blockstore_impl_t::prepare_read_simple(std::vector<copy_buffer_t> & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end)
 {
     uint32_t res = 0;
     if (wr->type() == BS_HEAP_SMALL_WRITE || wr->type() == BS_HEAP_INTENT_WRITE)
@@ -235,7 +190,7 @@ uint32_t blockstore_impl_t::prepare_read_simple(std::vector<copy_buffer_t> & rea
                 .disk_offset = start,
                 .disk_len = end-start,
                 .buf = buffer_area + wr->small().location + start - wr->small().offset,
-                .wr = wr,
+                .wr_lsn = wr->lsn,
             });
         }
         else if (dsk.csum_block_size <= dsk.bitmap_granularity)
@@ -254,6 +209,14 @@ uint32_t blockstore_impl_t::prepare_read_simple(std::vector<copy_buffer_t> & rea
                 blk_start = blk_start < wr->small().offset ? wr->small().offset : blk_start;
                 blk_end = blk_end > wr->small().offset+wr->small().len ? wr->small().offset+wr->small().len : blk_end;
             }
+            uint32_t skip_csum = 0;
+            if (!perfect_csum_update && wr->type() == BS_HEAP_BIG_WRITE)
+            {
+                for (auto owr = obj->get_writes(); owr && owr != wr; owr = owr->next())
+                    if ((owr->type() == BS_HEAP_INTENT_WRITE || owr->type() == BS_HEAP_SMALL_WRITE) &&
+                        owr->small().offset < blk_end && owr->small().offset+owr->small().len > blk_start)
+                        skip_csum = COPY_BUF_SKIP_CSUM;
+            }
             if ((blk_end-1)/dsk.csum_block_size == blk_start/dsk.csum_block_size ||
                 blk_end/dsk.csum_block_size == blk_start/dsk.csum_block_size+1 && blk_end != end && blk_start != start ||
                 blk_end == end && blk_start == start)
@@ -280,37 +243,21 @@ uint32_t blockstore_impl_t::prepare_read_simple(std::vector<copy_buffer_t> & rea
     return res;
 }
 
-void blockstore_impl_t::prepare_disk_read(std::vector<copy_buffer_t> & read_vec, int pos, heap_entry_t *obj, heap_entry_t *wr,
+void blockstore_impl_t::prepare_disk_read(std::vector<copy_buffer_t> & read_vec, int pos, heap_object_t *obj, heap_write_t *wr,
     uint32_t blk_start, uint32_t blk_end, uint32_t start, uint32_t end, uint32_t copy_flags)
 {
-    uint64_t loc = 0;
-    if (wr->type() == BS_HEAP_INTENT_WRITE)
-    {
-        heap_entry_t *big_wr = wr;
-        while (big_wr && big_wr->type() == BS_HEAP_INTENT_WRITE)
-        {
-            big_wr = heap->prev(big_wr);
-        }
-        assert(big_wr->type() == BS_HEAP_BIG_WRITE || big_wr->type() == BS_HEAP_BIG_INTENT);
-        loc = big_wr->big_location(heap);
-    }
-    else if (wr->type() == BS_HEAP_SMALL_WRITE)
-    {
-        loc = wr->small().location-wr->small().offset;
-    }
-    else
-    {
-        assert(wr->type() == BS_HEAP_BIG_WRITE || wr->type() == BS_HEAP_BIG_INTENT);
-        loc = wr->big_location(heap);
-    }
+    // Only one INTENT_WRITE is allowed at a time
+    assert(wr->type() != BS_HEAP_INTENT_WRITE || wr->next()->type() == BS_HEAP_BIG_WRITE);
     copy_buffer_t vec = {
         .copy_flags = (wr->type() == BS_HEAP_SMALL_WRITE ? COPY_BUF_JOURNAL : COPY_BUF_DATA) | copy_flags,
         .offset = start,
         .len = end-start,
-        .disk_loc = loc,
+        .disk_loc = (wr->type() == BS_HEAP_INTENT_WRITE ? wr->next()->big_location(heap)
+            : (wr->type() == BS_HEAP_SMALL_WRITE ? wr->small().location-wr->small().offset
+            : wr->big_location(heap))),
         .disk_offset = blk_start,
         .disk_len = blk_end - blk_start,
-        .wr = wr,
+        .wr_lsn = wr->lsn,
     };
     if (blk_start != start || blk_end != end)
     {
@@ -318,7 +265,7 @@ void blockstore_impl_t::prepare_disk_read(std::vector<copy_buffer_t> & read_vec,
         vec.copy_flags |= COPY_BUF_PADDED;
         if (pos > 0 && read_vec.size() >= pos &&
             read_vec[pos-1].copy_flags == vec.copy_flags &&
-            read_vec[pos-1].wr == vec.wr &&
+            read_vec[pos-1].wr_lsn == vec.wr_lsn &&
             read_vec[pos-1].disk_offset <= vec.disk_offset &&
             read_vec[pos-1].disk_offset+read_vec[pos-1].disk_len >= blk_end)
         {
@@ -402,7 +349,7 @@ void blockstore_impl_t::handle_read_event(ring_data_t *data, blockstore_op_t *op
             op->retval = -EDOM;
         else if (op->retval == 0)
             op->retval = op->len;
-        heap->unlock_entry(op->oid);
+        heap->unlock_entry(op->oid, PRIV(op)->lsn);
         free_read_buffers(PRIV(op)->read_vec);
         FINISH_OP(op);
     }
@@ -410,15 +357,7 @@ void blockstore_impl_t::handle_read_event(ring_data_t *data, blockstore_op_t *op
 
 bool blockstore_impl_t::verify_read_checksums(blockstore_op_t *op)
 {
-    bool skip_all = false;
-    if (PRIV(op)->lsn)
-    {
-        heap_entry_t *obj = heap->read_entry(op->oid);
-        if (obj->lsn != PRIV(op)->lsn) // check top lsn
-        {
-            skip_all = true;
-        }
-    }
+    heap_object_t *obj = heap->read_locked_entry(op->oid, PRIV(op)->lsn);
     auto & rv = PRIV(op)->read_vec;
     for (auto & vec: rv)
     {
@@ -426,22 +365,23 @@ bool blockstore_impl_t::verify_read_checksums(blockstore_op_t *op)
             continue;
         if (vec.copy_flags & COPY_BUF_PADDED)
             memcpy(op->buf + vec.offset - op->offset, vec.buf + vec.offset - vec.disk_offset, vec.len);
-        if (vec.copy_flags & (COPY_BUF_COALESCED|COPY_BUF_SKIP_CSUM) ||
-            skip_all && vec.wr->type() != BS_HEAP_SMALL_WRITE)
+        if (vec.copy_flags & (COPY_BUF_COALESCED|COPY_BUF_SKIP_CSUM))
             continue;
+        heap_write_t *wr = obj->get_writes();
+        while (wr && wr->lsn != vec.wr_lsn)
+            wr = wr->next();
+        assert(wr);
         uint8_t *buf = vec.buf ? vec.buf : (op->buf + vec.offset - op->offset);
-        uint32_t *csums = (uint32_t*)(vec.wr->get_checksums(heap)
+        uint32_t *csums = (uint32_t*)(wr->get_checksums(heap)
             + (vec.disk_offset/dsk.csum_block_size)*(dsk.data_csum_type & 0xFF)
-            - ((vec.wr->type() == BS_HEAP_BIG_WRITE || vec.wr->type() == BS_HEAP_BIG_INTENT)
-                ? 0 : (vec.wr->small().offset/dsk.csum_block_size)*(dsk.data_csum_type & 0xFF)));
-        if (!heap->calc_block_checksums(csums, buf, vec.wr->get_int_bitmap(heap),
+            - ((wr->type() == BS_HEAP_BIG_WRITE) ? 0 : (wr->small().offset/dsk.csum_block_size)*(dsk.data_csum_type & 0xFF)));
+        if (!heap->calc_block_checksums(csums, buf, wr->get_int_bitmap(heap),
             vec.disk_offset, vec.disk_offset+vec.disk_len, false, [&](uint32_t mismatch_pos, uint32_t expected_csum, uint32_t real_csum)
             {
                 printf(
-                    "Checksum mismatch in object %jx:%jx v%ju, offset 0x%x in %s area at offset 0x%jx during read %x+%x: %08x expected vs %08x actual\n",
+                    "Checksum mismatch in object %jx:%jx v%ju, offset 0x%x in %s area at offset 0x%jx: %08x expected vs %08x actual\n",
                     op->oid.inode, op->oid.stripe, op->version, mismatch_pos,
                     (vec.copy_flags & COPY_BUF_JOURNAL) ? "buffer" : "data", vec.disk_loc + vec.disk_offset,
-                    op->offset, op->len,
                     expected_csum, real_csum
                 );
             }))
@@ -454,29 +394,23 @@ bool blockstore_impl_t::verify_read_checksums(blockstore_op_t *op)
 
 int blockstore_impl_t::read_bitmap(object_id oid, uint64_t target_version, void *bitmap, uint64_t *result_version)
 {
-    heap_entry_t *obj = heap->read_entry(oid);
+    heap_object_t *obj = heap->read_entry(oid, NULL);
     if (obj)
     {
-        bool found = false;
-        heap->iterate_with_stable(obj, obj->lsn, [&](heap_entry_t *wr, bool stable)
+        for (auto wr = obj->get_writes(); wr; wr = wr->next())
         {
-            if (target_version >= wr->version)
+            if (target_version < wr->version)
             {
-                found = true;
-                if (result_version)
-                {
-                    *result_version = wr->version;
-                }
-                if (bitmap)
-                {
-                    memcpy(bitmap, wr->get_ext_bitmap(heap), dsk.clean_entry_bitmap_size);
-                }
-                return false;
+                continue;
+            }
+            if (result_version)
+            {
+                *result_version = wr->version;
+            }
+            if (bitmap)
+            {
+                memcpy(bitmap, wr->get_ext_bitmap(heap), dsk.clean_entry_bitmap_size);
             }
-            return true;
-        });
-        if (found)
-        {
             return 0;
         }
     }

src/blockstore/blockstore_stable.cpp

@@ -13,75 +13,73 @@ int blockstore_impl_t::dequeue_stable(blockstore_op_t *op)
     else if (priv->op_state == 2) goto resume_2;
     else if (priv->op_state == 3) goto resume_3;
     else if (priv->op_state == 4) goto resume_4;
-    else if (priv->op_state == 5) goto resume_5;
     assert(!priv->op_state);
+    // Modify in-memory state and assign contiguous LSNs
+    priv->stab_pos = 0;
+    priv->lsn = priv->to_lsn = 0;
     op->retval = 0;
-    priv->modified_block = priv->modified_block2 = UINT32_MAX;
-    for (priv->stab_pos = 0; priv->stab_pos < op->len; priv->stab_pos++)
+    while (priv->stab_pos < op->len)
     {
+        uint32_t modified_block = 0;
+        uint64_t new_lsn = 0;
+        uint64_t new_to_lsn = 0;
+        int res = op->opcode == BS_OP_STABLE
+            ? heap->post_stabilize(v[priv->stab_pos].oid, v[priv->stab_pos].version, &modified_block, &new_lsn, &new_to_lsn)
+            : heap->post_rollback(v[priv->stab_pos].oid, v[priv->stab_pos].version, &new_lsn, &modified_block);
+        if (res != 0)
         {
-            auto obj = heap->read_entry(v[priv->stab_pos].oid);
-            if (!obj)
+            assert(res == ENOENT || res == EBUSY);
+            op->retval = -res;
+        }
+        if (new_lsn)
+        {
+            assert(priv->lsn == 0 || priv->to_lsn == new_lsn-1);
+            if (!priv->lsn)
+                priv->lsn = new_lsn;
+            priv->to_lsn = op->opcode == BS_OP_STABLE ? new_to_lsn : new_lsn;
+        }
+        priv->stab_pos++;
+    }
+    // Submit metadata writes
+    priv->stab_pos = 0;
+resume_1:
+    priv->op_state = 1;
+    while (priv->stab_pos < op->len)
+    {
+        uint32_t block_num = 0;
+        heap_object_t *obj = heap->read_entry(v[priv->stab_pos].oid, &block_num);
+        if (obj)
+        {
+            io_uring_sqe *sqe = get_sqe();
+            if (!sqe)
             {
-                op->retval = -ENOENT;
-                FINISH_OP(op);
-                return 2;
-            }
-            int res = op->opcode == BS_OP_STABLE
-                ? heap->add_commit(obj, v[priv->stab_pos].version, &priv->modified_block2)
-                : heap->add_rollback(obj, v[priv->stab_pos].version, &priv->modified_block2);
-            if (res == EBUSY)
-            {
-                op->retval = -EBUSY;
-                FINISH_OP(op);
-                return 2;
-            }
-            if (res == ENOSPC)
-            {
-                if (!heap->get_to_compact_count())
-                {
-                    // no space
-                    op->retval = -ENOSPC;
-                    FINISH_OP(op);
-                    return 2;
-                }
-                if (priv->modified_block2 != UINT32_MAX)
-                {
-                    priv->stab_pos--;
-                    goto resume_1;
-                }
-                priv->wait_for = WAIT_COMPACTION;
-                priv->wait_detail = heap->get_compacted_count();
-                flusher->request_trim();
+                if (priv->pending_ops > 0)
+                    return 1;
+                priv->wait_detail = 1;
+                priv->wait_for = WAIT_SQE;
                 return 0;
             }
-            assert(res == 0);
-        }
-resume_1:
-        if (priv->modified_block != UINT32_MAX && priv->modified_block2 != priv->modified_block)
-        {
-            BS_SUBMIT_CHECK_SQES(1);
-            prepare_meta_block_write(priv->modified_block);
-resume_2:
-            if (meta_block_is_pending(priv->modified_block))
-            {
-                priv->op_state = 2;
-                return 1;
-            }
-        }
-        priv->modified_block = priv->modified_block2;
-        if (priv->stab_pos == op->len-1 && priv->modified_block2 != UINT32_MAX)
-        {
-            priv->modified_block2 = UINT32_MAX;
-            goto resume_1;
+            prepare_meta_block_write(op, block_num, sqe);
         }
+        priv->stab_pos++;
     }
+    if (priv->pending_ops > 0)
+    {
+        priv->op_state = 1;
+        return 1;
+    }
+    // Mark writes as completed to allow compaction
+    for (uint64_t lsn = priv->lsn; lsn <= priv->to_lsn; lsn++)
+    {
+        heap->mark_lsn_completed(lsn);
+    }
+    unsynced_meta_write_count++;
     // Fsync, just because our semantics imply that commit (stabilize) is immediately fsynced
-    priv->op_state = 3;
+    priv->op_state = 2;
+resume_2:
 resume_3:
 resume_4:
-resume_5:
-    int res = do_sync(op, 3);
+    int res = do_sync(op, 2);
     if (res != 2)
     {
         return res;

src/blockstore/blockstore_sync.cpp

@@ -18,19 +18,11 @@ int blockstore_impl_t::continue_sync(blockstore_op_t *op)
     return res;
 }
 
-bool blockstore_impl_t::has_unsynced()
-{
-    bool data = (!dsk.disable_data_fsync && unsynced_data_write_count);
-    bool buffer = (!dsk.disable_journal_fsync && unsynced_buffer_write_count);
-    bool meta = (!dsk.disable_meta_fsync && unsynced_meta_write_count);
-    return data || buffer || meta;
-}
-
 bool blockstore_impl_t::submit_fsyncs(int & wait_count)
 {
-    int n = (unsynced_meta_write_count > 0 && !dsk.disable_meta_fsync) +
-        (unsynced_buffer_write_count > 0 && !dsk.disable_journal_fsync && dsk.journal_fd != dsk.meta_fd) +
-        (unsynced_data_write_count > 0 && !dsk.disable_data_fsync && dsk.data_fd != dsk.meta_fd && dsk.data_fd != dsk.journal_fd);
+    int n = ((unsynced_small_write_count > 0 || unsynced_big_write_count > 0 || unsynced_meta_write_count > 0) && !dsk.disable_meta_fsync) +
+        (unsynced_small_write_count > 0 && !dsk.disable_journal_fsync && dsk.journal_fd != dsk.meta_fd) +
+        (unsynced_big_write_count > 0 && !dsk.disable_data_fsync && dsk.data_fd != dsk.meta_fd && dsk.data_fd != dsk.journal_fd);
     if (ringloop->space_left() < n)
     {
         return false;
@@ -48,7 +40,7 @@ bool blockstore_impl_t::submit_fsyncs(int & wait_count)
         if (!wait_count)
             ringloop->wakeup();
     };
-    if (unsynced_meta_write_count > 0 && !dsk.disable_meta_fsync)
+    if ((unsynced_small_write_count > 0 || unsynced_big_write_count > 0 || unsynced_meta_write_count > 0) && !dsk.disable_meta_fsync)
     {
         // fsync meta
         io_uring_sqe *sqe = get_sqe();
@@ -59,7 +51,7 @@ bool blockstore_impl_t::submit_fsyncs(int & wait_count)
         data->callback = cb;
         wait_count++;
     }
-    if (unsynced_buffer_write_count > 0 && !dsk.disable_journal_fsync && dsk.meta_fd != dsk.journal_fd)
+    if (unsynced_small_write_count > 0 && !dsk.disable_journal_fsync && dsk.meta_fd != dsk.journal_fd)
     {
         // fsync buffer
         io_uring_sqe *sqe = get_sqe();
@@ -70,7 +62,7 @@ bool blockstore_impl_t::submit_fsyncs(int & wait_count)
         data->callback = cb;
         wait_count++;
     }
-    if (unsynced_data_write_count > 0 && !dsk.disable_data_fsync && dsk.data_fd != dsk.meta_fd && dsk.data_fd != dsk.journal_fd)
+    if (unsynced_big_write_count > 0 && !dsk.disable_data_fsync && dsk.data_fd != dsk.meta_fd && dsk.data_fd != dsk.journal_fd)
     {
         // fsync data
         io_uring_sqe *sqe = get_sqe();
@@ -81,8 +73,8 @@ bool blockstore_impl_t::submit_fsyncs(int & wait_count)
         data->callback = cb;
         wait_count++;
     }
-    unsynced_data_write_count = 0;
-    unsynced_buffer_write_count = 0;
+    unsynced_big_write_count = 0;
+    unsynced_small_write_count = 0;
     unsynced_meta_write_count = 0;
     return true;
 }
@@ -98,13 +90,13 @@ int blockstore_impl_t::do_sync(blockstore_op_t *op, int base_state)
         // Wait for flusher-initiated sync
         return 0;
     }
-    if (!has_unsynced())
+    if (dsk.disable_journal_fsync && dsk.disable_meta_fsync && dsk.disable_data_fsync || !unsynced_big_write_count && !unsynced_small_write_count)
     {
         // We can return immediately because sync only syncs previous writes
-        unsynced_data_write_count = unsynced_buffer_write_count = unsynced_meta_write_count = 0;
+        unsynced_big_write_count = unsynced_small_write_count = unsynced_meta_write_count = 0;
         return 2;
     }
-    PRIV(op)->modified_block = heap->get_completed_lsn();
+    PRIV(op)->lsn = heap->get_completed_lsn();
     if (!submit_fsyncs(PRIV(op)->pending_ops))
     {
         PRIV(op)->wait_detail = 1;
@@ -118,6 +110,6 @@ resume_1:
         return 1;
     }
 resume_2:
-    heap->mark_lsn_fsynced(PRIV(op)->modified_block);
+    heap->mark_lsn_fsynced(PRIV(op)->lsn);
     return 2;
 }

src/blockstore/blockstore_write.cpp

@@ -5,107 +5,29 @@
 #include "blockstore_internal.h"
 #include "allocator.h"
 
-#define _REDIRECT_INTENT 0x101
-
 bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
 {
     clock_gettime(CLOCK_REALTIME, &PRIV(op)->tv_begin);
     return true;
 }
 
-void blockstore_impl_t::prepare_meta_block_write(uint32_t modified_block)
+void blockstore_impl_t::prepare_meta_block_write(blockstore_op_t *op, uint64_t modified_block, io_uring_sqe *sqe)
 {
-    if (modified_blocks.find(modified_block) != modified_blocks.end())
-        return;
-    io_uring_sqe *sqe = get_sqe();
-    assert(sqe != NULL);
-    ring_data_t *data = ((ring_data_t*)sqe->user_data);
-    uint8_t *buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size);
-    data->iov = (struct iovec){ buf, (size_t)dsk.meta_block_size };
-    data->callback = [this, modified_block, buf](ring_data_t *data)
+    if (!sqe)
     {
-        free(buf);
-        live = true;
-        if (data->res != data->iov.iov_len)
-        {
-            // FIXME: our state becomes corrupted after a write error. maybe do something better than just die
-            disk_error_abort("data write", data->res, data->iov.iov_len);
-        }
-        modified_blocks.erase(modified_block);
-        heap->complete_block_write(modified_block);
-        ringloop->wakeup();
-    };
+        sqe = get_sqe();
+        assert(sqe != NULL);
+    }
+    ring_data_t *data = ((ring_data_t*)sqe->user_data);
+    data->iov = (struct iovec){ heap->get_meta_block(modified_block), (size_t)dsk.meta_block_size };
+    data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
+    PRIV(op)->pending_ops++;
     io_uring_prep_writev(
         sqe, dsk.meta_fd, &data->iov, 1, dsk.meta_offset + (modified_block+1)*dsk.meta_block_size
     );
-    unsynced_meta_write_count++;
-    pending_modified_blocks.push_back(modified_block);
-    modified_blocks[modified_block] = { .sent = false, .buf = buf };
-}
-
-bool blockstore_impl_t::meta_block_is_pending(uint32_t modified_block)
-{
-    auto mb_it = modified_blocks.find(modified_block);
-    return mb_it != modified_blocks.end();
-}
-
-bool blockstore_impl_t::intent_write_allowed(blockstore_op_t *op, heap_entry_t *obj)
-{
-    // Parallel writes to the same object are forbidden so "one intent at a time" is fulfilled automatically
-    // Intent writes are disabled when metadata fsync is enabled
-    if (!dsk.disable_meta_fsync)
-    {
-        return false;
-    }
-    // Intent writes are only for replication
-    if (op->opcode != BS_OP_WRITE_STABLE)
-    {
-        return false;
-    }
-    // Operation size should be less than or equal to atomic write size
-    if (!op->len || op->len > dsk.atomic_write_size)
-    {
-        return false;
-    }
-    // Intent-writes are disabled if "absolutely correct during compaction" checksum validation algorithm is enabled
-    // We could also do RMW here when perfect_csum_update is enabled, but it's unclear if we need it
-    if (perfect_csum_update && dsk.csum_block_size > dsk.bitmap_granularity &&
-        ((op->offset % dsk.csum_block_size) || (op->len % dsk.csum_block_size)))
-    {
-        return false;
-    }
-    bool ok = true;
-    heap->iterate_with_stable(obj, obj->lsn, [&](heap_entry_t *wr, bool stable)
-    {
-        // Intent writes are not allowed over buffered writes
-        auto t = wr->type();
-        if (t == BS_HEAP_SMALL_WRITE)
-        {
-            ok = false;
-            return false;
-        }
-        // Intent writes are not allowed over unstable writes
-        if (!stable)
-        {
-            ok = false;
-            return false;
-        }
-        // Intent writes are not allowed over unfinished intent writes
-        if ((t == BS_HEAP_INTENT_WRITE || t == BS_HEAP_BIG_INTENT) && wr->lsn > heap->get_fsynced_lsn())
-        {
-            ok = false;
-            return false;
-        }
-        // Intent writes are allowed over BIG_WRITEs even with fsyncs because BIG_WRITE is always counted as fsynced
-        if (t == BS_HEAP_BIG_WRITE || t == BS_HEAP_BIG_INTENT)
-        {
-            return false;
-        }
-        return true;
-    });
-    return ok;
 }
 
+// First step of the write algorithm: dequeue operation and submit initial write(s)
 int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
 {
     if (PRIV(op)->op_state)
@@ -116,42 +38,40 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
     {
         return 0;
     }
-    PRIV(op)->modified_block = UINT32_MAX;
-    PRIV(op)->write_type = 0;
-    heap_entry_t *obj = heap->read_entry(op->oid);
+    PRIV(op)->is_big = false;
+    uint32_t modified_block = UINT32_MAX, moved_from_block = UINT32_MAX;
+    heap_object_t *obj = heap->read_entry(op->oid, &modified_block);
     if (op->opcode == BS_OP_DELETE)
     {
         // Delete
-        if (!obj || obj->type() == BS_HEAP_DELETE)
+        if (!obj)
         {
             // Already deleted
             op->retval = 0;
             FINISH_OP(op);
             return 2;
         }
-        PRIV(op)->write_type = BS_HEAP_DELETE;
         BS_SUBMIT_CHECK_SQES(1);
-        int res = heap->add_delete(obj, &PRIV(op)->modified_block);
-        if (res == ENOSPC)
-            goto enospc;
+        int res = heap->post_delete(modified_block, obj, &PRIV(op)->lsn);
         assert(res == 0);
-        prepare_meta_block_write(PRIV(op)->modified_block);
-        PRIV(op)->pending_ops++;
+        prepare_meta_block_write(op, modified_block);
         PRIV(op)->op_state = 5;
         write_iodepth++;
     }
     // FIXME: Allow to do initial writes as buffered, not redirected
     // FIXME: Allow to do direct writes over holes
-    else if (!obj || obj->type() == BS_HEAP_DELETE || op->offset == 0 && op->len == dsk.data_block_size)
+    else if (!obj || obj->get_writes()->type() == BS_HEAP_TOMBSTONE ||
+        op->offset == 0 && op->len == dsk.data_block_size)
     {
         // Big (redirect) write
-        PRIV(op)->write_type = dsk.disable_data_fsync || op->opcode != BS_OP_WRITE_STABLE ? BS_HEAP_BIG_WRITE : _REDIRECT_INTENT;
         BS_SUBMIT_CHECK_SQES(1);
-        PRIV(op)->location = heap->find_free_data();
-        if (PRIV(op)->location == UINT64_MAX)
+        PRIV(op)->is_big = true;
+        uint32_t tmp_block;
+        uint64_t loc = heap->find_free_data();
+        if (loc == UINT64_MAX ||
+            !obj && heap->get_block_for_new_object(tmp_block) != 0)
         {
-enospc:
-            if (!heap->get_to_compact_count())
+            if (!heap->get_inflight_queue_size())
             {
                 // no space
                 op->retval = -ENOSPC;
@@ -159,11 +79,11 @@ enospc:
                 return 2;
             }
             PRIV(op)->wait_for = WAIT_COMPACTION;
-            PRIV(op)->wait_detail = heap->get_compacted_count();
+            PRIV(op)->wait_detail = flusher->get_compact_counter();
             flusher->request_trim();
             return 0;
         }
-        uint64_t loc = PRIV(op)->location;
+        PRIV(op)->location = loc;
 #ifdef BLOCKSTORE_DEBUG
         printf(
             "Allocate offset %ju for %jx:%jx v%ju\n",
@@ -171,60 +91,103 @@ enospc:
         );
 #endif
         heap->use_data(op->oid.inode, PRIV(op)->location);
+        if (!dsk.disable_data_fsync && dsk.disable_meta_fsync)
+        {
+            // Do big_write as an INTENT to avoid data fsync
+            int res = make_big_write(op, 0, 0, &modified_block, &moved_from_block);
+            assert(res == 0);
+            if (moved_from_block != UINT32_MAX)
+                prepare_meta_block_write(op, moved_from_block);
+            obj = heap->read_entry(op->oid, &modified_block);
+            heap->mark_lsn_completed(PRIV(op)->lsn);
+            goto process_intent;
+        }
         io_uring_sqe *sqe = get_sqe();
         ring_data_t *data = ((ring_data_t*)sqe->user_data);
-        data->iov = (struct iovec){ op->buf, op->len };
-        data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
-        io_uring_prep_writev(sqe, dsk.data_fd, &data->iov, 1, dsk.data_offset + loc + op->offset);
-        PRIV(op)->pending_ops++;
-        write_iodepth++;
-        if (PRIV(op)->write_type == BS_HEAP_BIG_WRITE)
+        uint64_t stripe_offset = (op->offset % dsk.bitmap_granularity);
+        uint64_t stripe_end = (op->offset + op->len) % dsk.bitmap_granularity;
+        // Zero fill up to dsk.bitmap_granularity
+        int vcnt = 0;
+        if (stripe_offset)
         {
-            PRIV(op)->op_state = 1;
-            inflight_big++;
+            PRIV(op)->iov_zerofill[vcnt++] = (struct iovec){ zero_object, (size_t)stripe_offset };
         }
-        else
-            PRIV(op)->op_state = 3;
+        PRIV(op)->iov_zerofill[vcnt++] = (struct iovec){ op->buf, op->len };
+        if (stripe_end)
+        {
+            stripe_end = dsk.bitmap_granularity - stripe_end;
+            PRIV(op)->iov_zerofill[vcnt++] = (struct iovec){ zero_object, (size_t)stripe_end };
+        }
+        data->iov.iov_len = op->len + stripe_offset + stripe_end; // to check it in the callback
+        data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
+        io_uring_prep_writev(
+            sqe, dsk.data_fd, PRIV(op)->iov_zerofill, vcnt, dsk.data_offset + loc + op->offset - stripe_offset
+        );
+        PRIV(op)->pending_ops++;
+        PRIV(op)->op_state = 1;
+        write_iodepth++;
+        inflight_big++;
     }
-    else if (intent_write_allowed(op, obj))
+    // Only one INTENT_WRITE is allowed at a time, but in fact,
+    // parallel writes to the same object are forbidden anyway
+    else if (op->len > 0 && op->len <= dsk.atomic_write_size &&
+        // Intent-writes are disabled if "absolutely correct during compaction" checksum validation algorithm is enabled
+        // We could also do RMW here when perfect_csum_update is enabled, but it's unclear if we need it
+        (!perfect_csum_update || dsk.csum_block_size <= dsk.bitmap_granularity ||
+            !(op->offset % dsk.csum_block_size) &&
+            !(op->len % dsk.csum_block_size) &&
+            (obj->get_writes()->entry_type != (BS_HEAP_INTENT_WRITE|BS_HEAP_STABLE) ||
+            obj->get_writes()->can_be_collapsed(heap))) &&
+        // One intent-write is allowed even with fsyncs because BIG_WRITE is always counted as fsynced
+        dsk.disable_meta_fsync &&
+        (op->opcode == BS_OP_WRITE_STABLE &&
+            (obj->get_writes()->entry_type == (BS_HEAP_BIG_WRITE|BS_HEAP_STABLE) ||
+            obj->get_writes()->entry_type == (BS_HEAP_INTENT_WRITE|BS_HEAP_STABLE) && dsk.disable_data_fsync) ||
+        op->opcode == BS_OP_WRITE && obj->get_writes()->entry_type == BS_HEAP_BIG_WRITE))
     {
         // Direct intent-write
         BS_SUBMIT_CHECK_SQES(1);
-        int res = 0;
-        if (dsk.csum_block_size <= dsk.bitmap_granularity &&
-            (obj->entry_type == (BS_HEAP_BIG_INTENT|BS_HEAP_STABLE) ||
-            obj->entry_type == (BS_HEAP_BIG_WRITE|BS_HEAP_STABLE)))
+        if (obj->get_writes()->type() == BS_HEAP_BIG_WRITE)
         {
-            // Even more simplified BIG_INTENT writes
-            // FIXME: Support RMW mode for csum_block_size > bitmap_granularity
-            PRIV(op)->write_type = BS_HEAP_BIG_INTENT;
-            PRIV(op)->location = obj->big_location(heap);
-            res = heap->add_big_intent(op->oid, &obj, op->version, op->offset, op->len, op->bitmap,
-                (uint8_t*)op->buf, NULL, &PRIV(op)->modified_block);
-            if (res == ENOSPC)
-                goto enospc;
-            assert(res == 0);
-            PRIV(op)->lsn = obj->lsn;
+            PRIV(op)->location = obj->get_writes()->big_location(heap);
         }
         else
         {
-            PRIV(op)->write_type = BS_HEAP_INTENT_WRITE;
-            auto wr = obj;
-            while (wr && (wr->type() == BS_HEAP_INTENT_WRITE || wr->type() == BS_HEAP_COMMIT || wr->type() == BS_HEAP_ROLLBACK))
-            {
-                wr = heap->prev(wr);
-            }
-            assert(wr && (wr->type() == BS_HEAP_BIG_WRITE || wr->type() == BS_HEAP_BIG_INTENT));
-            PRIV(op)->location = wr->big_location(heap);
-            res = heap->add_small_write(op->oid, &obj, (BS_HEAP_INTENT_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0)),
-                op->version, op->offset, op->len, 0, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
-            if (res == ENOSPC)
-                goto enospc;
-            assert(res == 0);
-            PRIV(op)->lsn = obj->lsn;
+            assert(obj->get_writes()->next()->type() == BS_HEAP_BIG_WRITE);
+            PRIV(op)->location = obj->get_writes()->next()->big_location(heap);
         }
-        prepare_meta_block_write(PRIV(op)->modified_block);
-        PRIV(op)->pending_ops++;
+process_intent:
+        uint8_t wr_buf[heap->get_max_write_entry_size()];
+        heap_write_t *wr = (heap_write_t*)wr_buf;
+        wr->version = op->version;
+        wr->entry_type = BS_HEAP_INTENT_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0);
+        wr->small().offset = op->offset;
+        wr->small().len = op->len;
+        wr->small().location = 0;
+        if (op->bitmap)
+            memcpy(wr->get_ext_bitmap(heap), op->bitmap, dsk.clean_entry_bitmap_size);
+        heap->calc_checksums(wr, (uint8_t*)op->buf, true);
+        int res = heap->post_write(modified_block, op->oid, obj, wr, &moved_from_block);
+        if (res == EAGAIN)
+        {
+            assert(heap->get_inflight_queue_size());
+            PRIV(op)->wait_for = WAIT_COMPACTION;
+            PRIV(op)->wait_detail = flusher->get_compact_counter();
+            flusher->request_trim();
+            return 0;
+        }
+        else if (res == ENOSPC)
+        {
+            // no space
+            op->retval = -ENOSPC;
+            FINISH_OP(op);
+            return 2;
+        }
+        assert(res == 0);
+        PRIV(op)->lsn = wr->lsn;
+        if (moved_from_block != UINT32_MAX)
+            prepare_meta_block_write(op, moved_from_block);
+        prepare_meta_block_write(op, modified_block);
         PRIV(op)->op_state = 9;
         write_iodepth++;
     }
@@ -232,27 +195,50 @@ enospc:
     {
         // Small (buffered) overwrite
         // First check if there is free buffer space
-        PRIV(op)->write_type = BS_HEAP_SMALL_WRITE;
         uint64_t loc = !op->len ? 0 : heap->find_free_buffer_area(op->len);
         if (loc == UINT64_MAX)
         {
             PRIV(op)->wait_for = WAIT_COMPACTION;
-            PRIV(op)->wait_detail = heap->get_compacted_count();
+            PRIV(op)->wait_detail = flusher->get_compact_counter();
             flusher->request_trim();
             return 0;
         }
         // There is sufficient space. Check SQE(s)
         BS_SUBMIT_CHECK_SQES(1 + (op->len > 0 ? 1 : 0));
-        int res = heap->add_small_write(op->oid, &obj, (BS_HEAP_SMALL_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0)),
-            op->version, op->offset, op->len, loc, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
-        if (res == ENOSPC)
-            goto enospc;
+        uint8_t wr_buf[heap->get_max_write_entry_size()];
+        heap_write_t *wr = (heap_write_t*)wr_buf;
+        wr->version = op->version;
+        wr->entry_type = BS_HEAP_SMALL_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0);
+        wr->small().offset = op->offset;
+        wr->small().len = op->len;
+        wr->small().location = loc;
+        PRIV(op)->location = loc;
+        if (op->bitmap)
+            memcpy(wr->get_ext_bitmap(heap), op->bitmap, dsk.clean_entry_bitmap_size);
+        heap->calc_checksums(wr, (uint8_t*)op->buf, true);
+        int res = heap->post_write(modified_block, op->oid, obj, wr, &moved_from_block);
+        if (res == EAGAIN)
+        {
+            assert(heap->get_inflight_queue_size());
+            PRIV(op)->wait_for = WAIT_COMPACTION;
+            PRIV(op)->wait_detail = flusher->get_compact_counter();
+            flusher->request_trim();
+            return 0;
+        }
+        else if (res == ENOSPC)
+        {
+            // no space
+            op->retval = -ENOSPC;
+            FINISH_OP(op);
+            return 2;
+        }
         assert(res == 0);
-        PRIV(op)->lsn = obj->lsn;
+        PRIV(op)->lsn = wr->lsn;
         if (op->len)
             heap->use_buffer_area(op->oid.inode, loc, op->len);
-        prepare_meta_block_write(PRIV(op)->modified_block);
-        PRIV(op)->pending_ops++;
+        if (moved_from_block != UINT32_MAX)
+            prepare_meta_block_write(op, moved_from_block);
+        prepare_meta_block_write(op, modified_block);
         if (op->len > 0)
         {
             // Prepare buffered data write
@@ -277,6 +263,26 @@ enospc:
     return 1;
 }
 
+int blockstore_impl_t::make_big_write(blockstore_op_t *op, uint32_t offset, uint32_t len, uint32_t *modified_block, uint32_t *moved_from_block)
+{
+    uint8_t wr_buf[heap->get_max_write_entry_size()];
+    heap_write_t *wr = (heap_write_t*)wr_buf;
+    wr->entry_type = BS_HEAP_BIG_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0);
+    wr->version = op->version;
+    wr->set_big_location(heap, PRIV(op)->location);
+    if (op->bitmap)
+        memcpy(wr->get_ext_bitmap(heap), op->bitmap, dsk.clean_entry_bitmap_size);
+    memset(wr->get_int_bitmap(heap), 0, dsk.clean_entry_bitmap_size);
+    bitmap_set(wr->get_int_bitmap(heap), offset, len, dsk.bitmap_granularity);
+    heap->calc_checksums(wr, (uint8_t*)op->buf, true, offset, len);
+    int res = heap->post_write(op->oid, wr, modified_block, moved_from_block);
+    if (res != 0)
+        return res;
+    assert(res == 0);
+    PRIV(op)->lsn = wr->lsn;
+    return 0;
+}
+
 int blockstore_impl_t::continue_write(blockstore_op_t *op)
 {
     int op_state = PRIV(op)->op_state;
@@ -299,12 +305,6 @@ again:
     {
         // In progress
         assert(op_state < 10);
-        if (PRIV(op)->modified_block != UINT32_MAX &&
-            !meta_block_is_pending(PRIV(op)->modified_block))
-        {
-            PRIV(op)->pending_ops--;
-            PRIV(op)->modified_block = UINT32_MAX;
-        }
         if (PRIV(op)->pending_ops > 0)
             return 1;
         op_state++;
@@ -313,6 +313,7 @@ again:
 resume_2:
     // We must fsync all big writes to avoid complex write workflows
     // It's OK for all HDDs and for server SSDs, but slightly worse for desktop SSDs
+    // The other way is to add another type of MVCC to blockstore_heap: "forward" MVCC :)
     inflight_big--;
     if (!dsk.disable_data_fsync)
     {
@@ -348,43 +349,33 @@ resume_12:
     }
 resume_4:
     {
-        auto obj = heap->read_entry(op->oid);
-        int res = 0;
-        if (PRIV(op)->write_type == _REDIRECT_INTENT)
+        uint32_t modified_block = UINT32_MAX, moved_from_block = UINT32_MAX;
+        int res = make_big_write(op, op->offset, op->len, &modified_block, &moved_from_block);
+        if (res == EAGAIN)
         {
-            res = heap->add_redirect_intent(op->oid, &obj, op->version, op->offset, op->len,
-                PRIV(op)->location, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
-        }
-        else
-        {
-            res = heap->add_big_write(op->oid, obj, op->opcode == BS_OP_WRITE_STABLE,
-                op->version, op->offset, op->len, PRIV(op)->location, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
-        }
-        if (res == ENOSPC)
-        {
-            if (!heap->get_to_compact_count())
-            {
-                // no space
-                heap->free_data(op->oid.inode, PRIV(op)->location);
-                write_iodepth--;
-                op->retval = -ENOSPC;
-                FINISH_OP(op);
-                return 2;
-            }
+            assert(heap->get_inflight_queue_size());
             PRIV(op)->wait_for = WAIT_COMPACTION;
-            PRIV(op)->wait_detail = heap->get_compacted_count();
+            PRIV(op)->wait_detail = flusher->get_compact_counter();
             flusher->request_trim();
-            return 0;
+            return 1;
         }
-        assert(res == 0);
-        prepare_meta_block_write(PRIV(op)->modified_block);
-        PRIV(op)->pending_ops++;
+        else if (res == ENOSPC)
+        {
+            heap->free_data(op->oid.inode, PRIV(op)->location);
+            write_iodepth--;
+            op->retval = -ENOSPC;
+            FINISH_OP(op);
+            return 2;
+        }
+        if (moved_from_block != UINT32_MAX)
+            prepare_meta_block_write(op, moved_from_block);
+        prepare_meta_block_write(op, modified_block);
         PRIV(op)->op_state = 5;
         return 1;
     }
 resume_6:
-    // Apply throttling to not fill the journal too quickly for the SSD+HDD case
-    if (PRIV(op)->write_type == BS_HEAP_SMALL_WRITE && throttle_small_writes)
+    // Apply throttling to not fill the journal too fast for the SSD+HDD case
+    if (!PRIV(op)->is_big && throttle_small_writes)
     {
         // Apply throttling
         timespec tv_end;
@@ -420,27 +411,11 @@ resume_8:
     printf("Ack write %jx:%jx v%ju\n", op->oid.inode, op->oid.stripe, op->version);
 #endif
     op->retval = op->len;
-    if (PRIV(op)->write_type == BS_HEAP_BIG_WRITE)
-    {
-        unsynced_data_write_count++;
-    }
-    else if (PRIV(op)->write_type == BS_HEAP_SMALL_WRITE)
-    {
-        unsynced_buffer_write_count++;
-        heap->complete_lsn_write(PRIV(op)->lsn);
-    }
-    else if (PRIV(op)->write_type == BS_HEAP_BIG_INTENT ||
-        PRIV(op)->write_type == BS_HEAP_INTENT_WRITE)
-    {
-        unsynced_data_write_count++;
-        intent_write_counter++;
-        heap->complete_lsn_write(PRIV(op)->lsn);
-    }
-    else if (PRIV(op)->write_type == _REDIRECT_INTENT)
-    {
-        unsynced_data_write_count++;
-        intent_write_counter++;
-    }
+    heap->mark_lsn_completed(PRIV(op)->lsn);
+    if (PRIV(op)->is_big)
+        unsynced_big_write_count++;
+    else
+        unsynced_small_write_count++;
     write_iodepth--;
     FINISH_OP(op);
     return 2;
@@ -451,8 +426,6 @@ resume_10:
     data->iov = (struct iovec){ op->buf, op->len };
     data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
     io_uring_prep_writev(sqe, dsk.data_fd, &data->iov, 1, dsk.data_offset + PRIV(op)->location + op->offset);
-    if (dsk.use_atomic_flag)
-        sqe->rw_flags = RWF_ATOMIC;
     PRIV(op)->pending_ops++;
     PRIV(op)->op_state = 7;
     return 1;

src/blockstore/fio_engine.cpp

@@ -28,7 +28,6 @@
 #include "epoll_manager.h"
 #include "malloc_or_die.h"
 #include "json11/json11.hpp"
-#include "../util/robin_hood.h"
 #include "fio_headers.h"
 
 struct bs_data
@@ -38,15 +37,11 @@ struct bs_data
     ring_loop_t *ringloop;
     /* The list of completed io_u structs. */
     std::vector<io_u*> completed;
-    robin_hood::unordered_flat_map<uint64_t, int> inflight_oids;
-    std::vector<io_u*> postponed;
     int op_n = 0, inflight = 0;
     bool ec = false;
     bool imm = true;
     bool last_sync = false;
     bool trace = false;
-    uint8_t *bitmap = NULL;
-    uint32_t block_size = 0;
 };
 
 struct bs_options
@@ -142,7 +137,6 @@ static void bs_cleanup(struct thread_data *td)
         delete bsd->bs;
         delete bsd->epmgr;
         delete bsd->ringloop;
-        free(bsd->bitmap);
         delete bsd;
     }
 }
@@ -166,12 +160,9 @@ static int bs_init(struct thread_data *td)
                 config[p.first] = p.second.dump();
         }
     }
-    bsd->bitmap = (uint8_t*)malloc_or_die(MAX_DATA_BLOCK_SIZE/512/8);
-    memset(bsd->bitmap, 0, MAX_DATA_BLOCK_SIZE/512/8);
     bsd->ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
     bsd->epmgr = new epoll_manager_t(bsd->ringloop);
     bsd->bs = blockstore_i::create(config, bsd->ringloop, bsd->epmgr->tfd);
-    bsd->block_size = bsd->bs->get_block_size();
     bsd->imm = config.find("immediate_commit") == config.end() ||
         config["immediate_commit"] == "all";
     while (1)
@@ -186,33 +177,8 @@ static int bs_init(struct thread_data *td)
     return 0;
 }
 
-static enum fio_q_status _bs_queue(struct thread_data *td, struct io_u *io, bool force);
-
-static void _bs_retry(struct bs_data *bsd, uint64_t offset)
-{
-    // Retry postponed ops
-    auto inflight_it = bsd->inflight_oids.find(offset / bsd->block_size);
-    assert(inflight_it != bsd->inflight_oids.end());
-    inflight_it->second--;
-    if (inflight_it->second > 0)
-    {
-        for (size_t i = 0; i < bsd->postponed.size(); i++)
-        {
-            auto oio = bsd->postponed[i];
-            if (oio->offset/bsd->block_size == offset/bsd->block_size)
-            {
-                bsd->postponed.erase(bsd->postponed.begin()+i);
-                _bs_queue((thread_data*)oio->engine_data, oio, true);
-                break;
-            }
-        }
-    }
-    else
-        bsd->inflight_oids.erase(inflight_it);
-}
-
 /* Begin read or write request. */
-static enum fio_q_status _bs_queue(struct thread_data *td, struct io_u *io, bool force)
+static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
 {
     bs_data *bsd = (bs_data*)td->io_ops_data;
     if (io->ddir == DDIR_SYNC && bsd->last_sync)
@@ -222,22 +188,10 @@ static enum fio_q_status _bs_queue(struct thread_data *td, struct io_u *io, bool
 
     fio_ro_check(td, io);
 
-    io->engine_data = td;
+    io->engine_data = bsd;
 
     if (io->ddir == DDIR_WRITE || io->ddir == DDIR_READ)
-        assert(io->xfer_buflen <= bsd->block_size);
-
-    uint64_t stripe = io->offset / bsd->block_size;
-    if (!force && io->ddir == DDIR_WRITE)
-    {
-        auto & inflight = bsd->inflight_oids[stripe];
-        inflight++;
-        if (inflight > 1)
-        {
-            bsd->postponed.push_back(io);
-            return FIO_Q_QUEUED;
-        }
-    }
+        assert(io->xfer_buflen <= bsd->bs->get_block_size());
 
     blockstore_op_t *op = new blockstore_op_t;
     op->callback = NULL;
@@ -249,16 +203,15 @@ static enum fio_q_status _bs_queue(struct thread_data *td, struct io_u *io, bool
         op->buf = (uint8_t*)io->xfer_buf;
         op->oid = {
             .inode = 1,
-            .stripe = stripe,
+            .stripe = io->offset / bsd->bs->get_block_size(),
         };
         op->version = UINT64_MAX; // last unstable
-        op->offset = io->offset % bsd->block_size;
+        op->offset = io->offset % bsd->bs->get_block_size();
         op->len = io->xfer_buflen;
-        op->bitmap = bsd->bitmap;
         op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
         {
             io->error = op->retval < 0 ? -op->retval : 0;
-            bs_data *bsd = ((bs_data*)((thread_data*)io->engine_data)->io_ops_data);
+            bs_data *bsd = (bs_data*)io->engine_data;
             bsd->inflight--;
             bsd->completed.push_back(io);
             if (bsd->trace)
@@ -271,17 +224,16 @@ static enum fio_q_status _bs_queue(struct thread_data *td, struct io_u *io, bool
         op->buf = (uint8_t*)io->xfer_buf;
         op->oid = {
             .inode = 1,
-            .stripe = stripe,
+            .stripe = io->offset / bsd->bs->get_block_size(),
         };
         op->version = 0; // assign automatically
-        op->offset = io->offset % bsd->block_size;
+        op->offset = io->offset % bsd->bs->get_block_size();
         op->len = io->xfer_buflen;
-        op->bitmap = bsd->bitmap;
         if (bsd->ec)
         {
             op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
             {
-                bs_data *bsd = ((bs_data*)((thread_data*)io->engine_data)->io_ops_data);
+                bs_data *bsd = (bs_data*)io->engine_data;
                 if (bsd->trace)
                     printf("--- OP_WRITE %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
                 if (op->retval < 0)
@@ -289,7 +241,6 @@ static enum fio_q_status _bs_queue(struct thread_data *td, struct io_u *io, bool
                     io->error = op->retval < 0 ? -op->retval : 0;
                     bsd->inflight--;
                     bsd->completed.push_back(io);
-                    _bs_retry(bsd, io->offset);
                     delete op;
                 }
                 else
@@ -303,13 +254,12 @@ static enum fio_q_status _bs_queue(struct thread_data *td, struct io_u *io, bool
                     stab_op->len = 1;
                     stab_op->callback = [io, n](blockstore_op_t *op)
                     {
-                        bs_data *bsd = ((bs_data*)((thread_data*)io->engine_data)->io_ops_data);
+                        bs_data *bsd = (bs_data*)io->engine_data;
                         if (bsd->trace)
                             printf("--- OP_STABLE %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
                         io->error = op->retval < 0 ? -op->retval : 0;
                         bsd->inflight--;
                         bsd->completed.push_back(io);
-                        _bs_retry(bsd, io->offset);
                         delete op;
                     };
                     bsd->bs->enqueue_op(stab_op);
@@ -321,13 +271,12 @@ static enum fio_q_status _bs_queue(struct thread_data *td, struct io_u *io, bool
         {
             op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
             {
-                bs_data *bsd = ((bs_data*)((thread_data*)io->engine_data)->io_ops_data);
+                bs_data *bsd = (bs_data*)io->engine_data;
                 if (bsd->trace)
                     printf("--- OP_WRITE_STABLE %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
                 io->error = op->retval < 0 ? -op->retval : 0;
                 bsd->inflight--;
                 bsd->completed.push_back(io);
-                _bs_retry(bsd, io->offset);
                 delete op;
             };
         }
@@ -337,7 +286,7 @@ static enum fio_q_status _bs_queue(struct thread_data *td, struct io_u *io, bool
         op->opcode = BS_OP_SYNC;
         op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
         {
-            bs_data *bsd = ((bs_data*)((thread_data*)io->engine_data)->io_ops_data);
+            bs_data *bsd = (bs_data*)io->engine_data;
             io->error = op->retval < 0 ? -op->retval : 0;
             bsd->completed.push_back(io);
             bsd->inflight--;
@@ -365,11 +314,6 @@ static enum fio_q_status _bs_queue(struct thread_data *td, struct io_u *io, bool
     return FIO_Q_QUEUED;
 }
 
-static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
-{
-    return _bs_queue(td, io, false);
-}
-
 static int bs_getevents(struct thread_data *td, unsigned int min, unsigned int max, const struct timespec *t)
 {
     bs_data *bsd = (bs_data*)td->io_ops_data;

src/blockstore/multilist.cpp

@@ -300,11 +300,6 @@ uint32_t multilist_index_t::find(uint32_t wanted_used)
     return heads[wanted_used];
 }
 
-uint32_t multilist_index_t::next(uint32_t pos)
-{
-    return nexts[pos];
-}
-
 void multilist_index_t::change(uint32_t pos, uint32_t old_used, uint32_t new_used)
 {
     if (new_used == old_used)

src/blockstore/multilist.h

@@ -32,7 +32,6 @@ struct multilist_index_t
     // used should be always < max_used
     multilist_index_t(uint32_t count, uint32_t max_used, uint32_t init_used);
     uint32_t find(uint32_t wanted_used);
-    uint32_t next(uint32_t pos);
     void change(uint32_t pos, uint32_t old_used, uint32_t new_used);
     void print();
 };

src/blockstore/ondisk_formats.h

@@ -185,8 +185,7 @@ struct __attribute__((__packed__)) blockstore_meta_header_v3_t
     uint32_t data_csum_type;
     uint32_t csum_block_size;
     uint32_t header_csum;
-    uint64_t meta_area_size;
-    uint64_t completed_lsn;
+    uint64_t compacted_lsn;
 
     void set_crc32c();
 };

src/blockstore/v1/impl.cpp

@@ -407,77 +407,32 @@ blockstore_clean_db_t& blockstore_impl_t::clean_db_shard(object_id oid)
     return clean_db_shards[(pool_id << (64-POOL_ID_BITS)) | pg_num];
 }
 
-struct bs_reshard_state_t
+void blockstore_impl_t::reshard_clean_db(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size)
 {
-    int state = 0;
-    uint64_t pool_id = 0;
-    uint32_t pg_count = 0;
-    uint32_t pg_stripe_size = 0;
-    uint64_t chunk_size = 0;
-    std::map<pool_pg_id_t, blockstore_clean_db_t> old_shards;
+    uint64_t pool_id = (uint64_t)pool;
     std::map<pool_pg_id_t, blockstore_clean_db_t> new_shards;
-    std::map<pool_pg_id_t, blockstore_clean_db_t>::iterator sh_it;
-    blockstore_clean_db_t::iterator obj_it;
-};
-
-void* blockstore_impl_t::reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit)
-{
-    auto & settings = clean_db_settings[pool];
-    if (settings.pg_count == pg_count && settings.pg_stripe_size == pg_stripe_size)
-    {
-        return NULL;
-    }
-    bs_reshard_state_t *st = new bs_reshard_state_t;
-    st->state = 0;
-    st->pool_id = pool;
-    st->pg_count = pg_count;
-    st->pg_stripe_size = pg_stripe_size;
-    auto sh_it = clean_db_shards.lower_bound((st->pool_id << (64-POOL_ID_BITS)));
+    auto sh_it = clean_db_shards.lower_bound((pool_id << (64-POOL_ID_BITS)));
     while (sh_it != clean_db_shards.end() &&
-        (sh_it->first >> (64-POOL_ID_BITS)) == st->pool_id)
+        (sh_it->first >> (64-POOL_ID_BITS)) == pool_id)
     {
-        st->old_shards[sh_it->first] = std::move(sh_it->second);
+        for (auto & pair: sh_it->second)
+        {
+            // like map_to_pg()
+            uint64_t pg_num = (pair.first.stripe / pg_stripe_size) % pg_count + 1;
+            uint64_t shard_id = (pool_id << (64-POOL_ID_BITS)) | pg_num;
+            new_shards[shard_id][pair.first] = pair.second;
+        }
         clean_db_shards.erase(sh_it++);
     }
-    bool finished = reshard_continue(st, chunk_limit);
-    return finished ? NULL : st;
-}
-
-bool blockstore_impl_t::reshard_continue(void *reshard_state, uint64_t chunk_limit)
-{
-    bs_reshard_state_t *st = (bs_reshard_state_t*)reshard_state;
-    uint64_t chunk_size = 0;
-    if (st->state == 1)
-        goto resume_1;
-    for (st->sh_it = st->old_shards.begin(); st->sh_it != st->old_shards.end(); )
-    {
-        for (st->obj_it = st->sh_it->second.begin(); st->obj_it != st->sh_it->second.end(); st->obj_it++)
-        {
-            if (chunk_limit > 0 && chunk_size >= chunk_limit)
-            {
-                st->state = 1;
-                return false;
-            }
-resume_1:
-            // like map_to_pg()
-            uint64_t pg_num = (st->obj_it->first.stripe / st->pg_stripe_size) % st->pg_count + 1;
-            uint64_t shard_id = (st->pool_id << (64-POOL_ID_BITS)) | pg_num;
-            st->new_shards[shard_id][st->obj_it->first] = st->obj_it->second;
-            chunk_size++;
-        }
-        st->old_shards.erase(st->sh_it++);
-    }
-    for (auto sh_it = st->new_shards.begin(); sh_it != st->new_shards.end(); sh_it++)
+    for (sh_it = new_shards.begin(); sh_it != new_shards.end(); sh_it++)
     {
         auto & to = clean_db_shards[sh_it->first];
         to.swap(sh_it->second);
     }
-    clean_db_settings[st->pool_id] = (pool_shard_settings_t){
-        .pg_count = st->pg_count,
-        .pg_stripe_size = st->pg_stripe_size,
+    clean_db_settings[pool_id] = (pool_shard_settings_t){
+        .pg_count = pg_count,
+        .pg_stripe_size = pg_stripe_size,
     };
-    delete st;
-    return true;
 }
 
 void blockstore_impl_t::process_list(blockstore_op_t *op)
@@ -510,10 +465,7 @@ void blockstore_impl_t::process_list(blockstore_op_t *op)
                 sh_it->second.pg_count != pg_count ||
                 sh_it->second.pg_stripe_size != pg_stripe_size)
             {
-                // Sharding mismatch
-                op->retval = -EAGAIN;
-                FINISH_OP(op);
-                return;
+                reshard_clean_db(pool_id, pg_count, pg_stripe_size);
             }
             first_shard = last_shard = ((uint64_t)pool_id << (64-POOL_ID_BITS)) | list_pg;
         }

src/blockstore/v1/impl.h

@@ -202,6 +202,7 @@ class blockstore_impl_t: public blockstore_i
     uint8_t* get_clean_entry_bitmap(uint64_t block_loc, int offset);
 
     blockstore_clean_db_t& clean_db_shard(object_id oid);
+    void reshard_clean_db(pool_id_t pool_id, uint32_t pg_count, uint32_t pg_stripe_size);
     void recalc_inode_space_stats(uint64_t pool_id, bool per_inode);
 
     // Journaling
@@ -287,10 +288,6 @@ public:
     void parse_config(blockstore_config_t & config);
     void parse_config(blockstore_config_t & config, bool init);
 
-    // Reshard database for a pool
-    void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit);
-    bool reshard_continue(void *reshard_state, uint64_t chunk_limit);
-
     // Event loop
     void loop();
 

src/blockstore/v1/internal.h

@@ -1,6 +1,3 @@
-// Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.1 (see README.md for details)
-
 #pragma once
 
 // States are not stored on disk. Instead, they're deduced from the journal

src/blockstore/v1/write.cpp

@@ -183,7 +183,7 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
         uint32_t end = (op->offset+op->len-1) / dsk.csum_block_size;
         auto fn = state & BS_ST_BIG_WRITE ? crc32c_pad : crc32c_nopad;
         if (start == end)
-            data_csums[0] = fn(0, op->buf, op->len, op->offset - start*dsk.csum_block_size, (end+1)*dsk.csum_block_size - (op->offset+op->len));
+            data_csums[0] = fn(0, op->buf, op->len, op->offset - start*dsk.csum_block_size, end*dsk.csum_block_size - (op->offset+op->len));
         else
         {
             // First block

src/client/cluster_client.cpp

@@ -765,13 +765,8 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
     {
         return;
     }
-    // CAS writes are simplified: they're not cached, not resliced, not retried, and not part of the regular write queue at all
-    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && op->version)
-    {
-        execute_cas(op);
-        return;
-    }
-    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && enable_writeback && !(op->flags & OP_FLUSH_BUFFER))
+    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && enable_writeback && !(op->flags & OP_FLUSH_BUFFER) &&
+        !op->version /* no CAS writeback */)
     {
         if (wb->writebacks_active >= client_max_writeback_iodepth)
         {
@@ -793,7 +788,7 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
     }
     if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && !(op->flags & OP_IMMEDIATE_COMMIT))
     {
-        if (!(op->flags & OP_FLUSH_BUFFER))
+        if (!(op->flags & OP_FLUSH_BUFFER) && !op->version /* no CAS write-repeat */)
         {
             uint64_t flush_id = ++wb->last_flush_id;
             wb->copy_write(op, CACHE_REPEATING, flush_id);
@@ -852,76 +847,6 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
     }
 }
 
-void cluster_client_t::execute_cas(cluster_op_t *op)
-{
-    slice_rw(op);
-    op->needs_reslice = false;
-    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && op->version && op->parts.size() > 1)
-    {
-        // Atomic writes to multiple stripes are unsupported
-        op->retval = -EINVAL;
-        auto cb = std::move(op->callback);
-        cb(op);
-        return;
-    }
-    int res = try_send(op, 0, [this, op](osd_op_t *part)
-    {
-        int expected = part->req.hdr.opcode == OSD_OP_DELETE ? 0 : part->req.rw.len;
-        op->retval = part->reply.hdr.retval;
-        if (op->retval != expected && op->retval >= 0)
-            op->retval = -EIO;
-        op->retval = op->retval == -EPIPE ? -EINTR : op->retval;
-        auto peer_it = msgr.osd_peer_fds.find(op->parts[0].osd_num);
-        if (op->retval != 0 || (op->flags & OP_IMMEDIATE_COMMIT))
-        {
-            auto cb = std::move(op->callback);
-            cb(op);
-        }
-        else if (peer_it == msgr.osd_peer_fds.end())
-        {
-            // Care must be taken to make sure that the client doesn't reconnect to the OSD
-            // before executing the previously completed operation callback (!)
-            op->retval = -EINTR;
-            auto cb = std::move(op->callback);
-            cb(op);
-        }
-        else
-        {
-            // CAS writes have a built-in sync
-            auto peer_fd = peer_it->second;
-            *part = (osd_op_t){
-                .op_type = OSD_OP_OUT,
-                .peer_fd = peer_fd,
-                .req = {
-                    .hdr = {
-                        .magic = SECONDARY_OSD_OP_MAGIC,
-                        .opcode = OSD_OP_SYNC,
-                    },
-                },
-                .callback = [op](osd_op_t *part)
-                {
-                    if (part->reply.hdr.retval != 0)
-                    {
-                        op->retval = part->reply.hdr.retval;
-                        op->retval = op->retval == -EPIPE ? -EINTR : op->retval;
-                    }
-                    auto cb = std::move(op->callback);
-                    cb(op);
-                },
-            };
-            msgr.outbox_push(part);
-        }
-    });
-    if (res == TRY_SEND_CONNECTING || res == TRY_SEND_OFFLINE)
-    {
-        // In theory, CAS writes could wait for the PG to come up, but it's easier to just fail it
-        op->retval = -EINTR;
-        auto cb = std::move(op->callback);
-        cb(op);
-        return;
-    }
-}
-
 bool cluster_client_t::check_rw(cluster_op_t *op)
 {
     if (op->opcode == OSD_OP_SYNC)
@@ -1031,6 +956,13 @@ resume_0:
     // Slice the operation into parts
     slice_rw(op);
     op->needs_reslice = false;
+    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && op->version && op->parts.size() > 1)
+    {
+        // Atomic writes to multiple stripes are unsupported
+        op->retval = -EINVAL;
+        erase_op(op);
+        return 1;
+    }
 resume_1:
     // Send unsent parts, if they're not subject to change
     op->state = 2;
@@ -1257,9 +1189,9 @@ void cluster_client_t::slice_rw(cluster_op_t *op)
         {
             op->bitmap_buf = realloc_or_die(op->bitmap_buf, bitmap_mem);
             op->part_bitmaps = (uint8_t*)op->bitmap_buf + object_bitmap_size;
-            memset((uint8_t*)op->bitmap_buf+op->bitmap_buf_size, 0, bitmap_mem-op->bitmap_buf_size);
             op->bitmap_buf_size = bitmap_mem;
         }
+        memset(op->bitmap_buf, 0, bitmap_mem);
     }
     int iov_idx = 0;
     size_t iov_pos = 0;
@@ -1375,7 +1307,7 @@ bool cluster_client_t::affects_osd(uint64_t inode, uint64_t offset, uint64_t len
     return false;
 }
 
-int cluster_client_t::try_send(cluster_op_t *op, int i, std::function<void(osd_op_t *op_part)> cb)
+int cluster_client_t::try_send(cluster_op_t *op, int i)
 {
     if (!msgr_initialized)
     {
@@ -1405,7 +1337,7 @@ int cluster_client_t::try_send(cluster_op_t *op, int i, std::function<void(osd_o
         if (peer_it != msgr.osd_peer_fds.end())
         {
             int peer_fd = peer_it->second;
-            part->flags |= PART_SENT|PART_VALID;
+            part->flags |= PART_SENT;
             op->inflight_count++;
             uint64_t pg_bitmap_size = (pool_cfg.data_block_size / pool_cfg.bitmap_granularity / 8) * (
                 pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks
@@ -1435,7 +1367,7 @@ int cluster_client_t::try_send(cluster_op_t *op, int i, std::function<void(osd_o
                     ? (uint8_t*)op->part_bitmaps + pg_bitmap_size*i : NULL),
                 .bitmap_len = (unsigned)(op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP
                     ? pg_bitmap_size : 0),
-                .callback = cb ? cb : [this, part](osd_op_t *op_part)
+                .callback = [this, part](osd_op_t *op_part)
                 {
                     handle_op_part(part);
                 },
@@ -1618,11 +1550,14 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
             dirty_osds.insert(part->osd_num);
         part->flags |= PART_DONE;
         op->done_count++;
-        if ((op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
-            && op->inode == op->cur_inode)
+        if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
         {
-            // Read only returns the version of the uppermost layer
-            op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0;
+            copy_part_bitmap(op, part);
+            if (op->inode == op->cur_inode)
+            {
+                // Read only returns the version of the uppermost layer
+                op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0;
+            }
         }
         else if (op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE)
         {
@@ -1630,13 +1565,6 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
         }
         if (op->inflight_count == 0 && !op->retry_after)
         {
-            // Copy part bitmaps only after finishing all part reads
-            if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
-            {
-                for (auto & part: op->parts)
-                    if (part.flags == (PART_SENT|PART_VALID|PART_DONE))
-                        copy_part_bitmap(op, &part);
-            }
             if (op->opcode == OSD_OP_SYNC)
                 continue_sync(op);
             else

src/client/cluster_client.h

@@ -175,13 +175,12 @@ protected:
     void on_change_node_placement_hook();
 
     void execute_internal(cluster_op_t *op);
-    void execute_cas(cluster_op_t *op);
     void unshift_op(cluster_op_t *op);
     int continue_rw(cluster_op_t *op);
     bool check_rw(cluster_op_t *op);
     void slice_rw(cluster_op_t *op);
     void reset_retry_timer(int new_duration);
-    int try_send(cluster_op_t *op, int i, std::function<void(osd_op_t *op_part)> cb = nullptr);
+    int try_send(cluster_op_t *op, int i);
     int continue_sync(cluster_op_t *op);
     void send_sync(cluster_op_t *op, cluster_op_part_t *part);
     void handle_op_part(cluster_op_part_t *part);

src/client/cluster_client_impl.h

@@ -10,7 +10,6 @@
 #define PART_DONE 2
 #define PART_ERROR 4
 #define PART_RETRY 8
-#define PART_VALID 16
 #define CACHE_DIRTY 1
 #define CACHE_WRITTEN 2
 #define CACHE_FLUSHING 3

src/client/etcd_state_client.cpp

@@ -568,19 +568,9 @@ void etcd_state_client_t::start_ws_keepalive()
 
 void etcd_state_client_t::load_global_config()
 {
-    json11::Json::object req = { { "success", json11::Json::array {
-        json11::Json::object {
-            { "request_range", json11::Json::object {
-                { "key", base64_encode(etcd_prefix+"/config/global") },
-            } }
-        },
-        json11::Json::object {
-            { "request_range", json11::Json::object {
-                { "key", base64_encode(etcd_prefix+"/config/pools") },
-            } }
-        },
-    } } };
-    etcd_txn(req, etcd_quick_timeout, max_etcd_attempts, 0, [this](std::string err, json11::Json data)
+    etcd_call("/kv/range", json11::Json::object {
+        { "key", base64_encode(etcd_prefix+"/config/global") }
+    }, etcd_quick_timeout, max_etcd_attempts, 0, [this](std::string err, json11::Json data)
     {
         if (err != "")
         {
@@ -598,12 +588,10 @@ void etcd_state_client_t::load_global_config()
             }
             return;
         }
-        json11::Json config_kv = data["responses"][0]["response_range"]["kvs"][0];
-        json11::Json pools_kv = data["responses"][1]["response_range"]["kvs"][0];
         json11::Json::object global_config;
-        if (!config_kv.is_null())
+        if (data["kvs"].array_items().size() > 0)
         {
-            auto kv = parse_etcd_kv(config_kv);
+            auto kv = parse_etcd_kv(data["kvs"][0]);
             if (kv.value.is_object())
             {
                 global_config = kv.value.object_items();
@@ -620,11 +608,6 @@ void etcd_state_client_t::load_global_config()
             global_bitmap_granularity = DEFAULT_BITMAP_GRANULARITY;
         }
         global_immediate_commit = parse_immediate_commit(global_config["immediate_commit"].string_value(), IMMEDIATE_ALL);
-        if (!pools_kv.is_null())
-        {
-            auto kv = parse_etcd_kv(pools_kv);
-            parse_state(kv);
-        }
         on_load_config_hook(global_config);
     });
 }
@@ -962,12 +945,8 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
             if (pc.pg_stripe_size < min_stripe_size)
                 pc.pg_stripe_size = min_stripe_size;
             // Save
-            auto & old_pc = this->pool_config[pool_id];
-            pc.real_pg_count = old_pc.real_pg_count;
-            pc.applied_pg_count = old_pc.applied_pg_count;
-            pc.applied_pg_stripe_size = old_pc.applied_pg_stripe_size;
-            pc.reshard_state = old_pc.reshard_state;
-            std::swap(pc.pg_config, old_pc.pg_config);
+            pc.real_pg_count = this->pool_config[pool_id].real_pg_count;
+            std::swap(pc.pg_config, this->pool_config[pool_id].pg_config);
             std::swap(this->pool_config[pool_id], pc);
             auto & parsed_cfg = this->pool_config[pool_id];
             parsed_cfg.exists = true;

src/client/etcd_state_client.h

@@ -68,11 +68,6 @@ struct pool_config_t
     std::string used_for_app;
     int backfillfull = 0;
     int local_reads = 0;
-
-    // runtime data, used only by OSD:
-    uint64_t applied_pg_count = 0;
-    uint64_t applied_pg_stripe_size = 0;
-    void *reshard_state = NULL;
 };
 
 struct inode_config_t

src/client/http_client.cpp

@@ -23,7 +23,7 @@
 #define READ_BUFFER_SIZE 9000
 
 static std::string ws_format_frame(int type, uint64_t size);
-static bool ws_parse_frame(std::string & buf, uint8_t & type, std::string & res);
+static bool ws_parse_frame(std::string & buf, int & type, std::string & res);
 static void parse_http_headers(std::string & res, http_response_t *parsed);
 
 struct http_co_t
@@ -70,7 +70,7 @@ struct http_co_t
     void submit_read(bool check_timeout);
     void submit_send();
     bool handle_read();
-    void post_message(uint8_t type, const std::string & msg);
+    void post_message(int type, const std::string & msg);
     void send_request(const std::string & host, const std::string & request,
         const http_options_t & options, std::function<void(const http_response_t *response)> response_callback);
 };
@@ -199,12 +199,12 @@ void http_co_t::send_request(const std::string & host, const std::string & reque
     stackout();
 }
 
-void http_post_message(http_co_t *handler, uint8_t type, const std::string & msg)
+void http_post_message(http_co_t *handler, int type, const std::string & msg)
 {
     handler->post_message(type, msg);
 }
 
-void http_co_t::post_message(uint8_t type, const std::string & msg)
+void http_co_t::post_message(int type, const std::string & msg)
 {
     stackin();
     if (state == HTTP_CO_WEBSOCKET)
@@ -608,13 +608,7 @@ bool http_co_t::handle_read()
     {
         while (ws_parse_frame(response, parsed.ws_msg_type, parsed.body))
         {
-            if (parsed.ws_msg_type == WS_PING)
-            {
-                // Reply with WS_PONG
-                post_message(WS_PONG, "");
-            }
-            else
-                response_callback(&parsed);
+            response_callback(&parsed);
             parsed.body = "";
         }
     }
@@ -704,7 +698,7 @@ static std::string ws_format_frame(int type, uint64_t size)
     return res;
 }
 
-static bool ws_parse_frame(std::string & buf, uint8_t & type, std::string & res)
+static bool ws_parse_frame(std::string & buf, int & type, std::string & res)
 {
     uint64_t hdr = 2;
     if (buf.size() < hdr)

src/client/http_client.h

@@ -32,7 +32,7 @@ struct http_response_t
     int status_code = 0;
     std::string status_line;
     std::map<std::string, std::string> headers;
-    uint8_t ws_msg_type = -1;
+    int ws_msg_type = -1;
     std::string body;
 
     void parse_json_response(std::string & error, json11::Json & r) const;
@@ -46,5 +46,5 @@ http_co_t* open_websocket(timerfd_manager_t *tfd, const std::string & host, cons
     int timeout, std::function<void(const http_response_t *msg)> on_message);
 void http_request(http_co_t *handler, const std::string & host, const std::string & request,
     const http_options_t & options, std::function<void(const http_response_t *response)> response_callback);
-void http_post_message(http_co_t *handler, uint8_t type, const std::string & msg);
+void http_post_message(http_co_t *handler, int type, const std::string & msg);
 void http_close(http_co_t *co);

src/client/messenger.cpp

@@ -749,7 +749,7 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl)
                     }
                 });
                 // Add the initial receive request
-                init_recv_rdma(cl);
+                try_recv_rdma(cl);
             }
         }
 #endif

src/client/messenger.h

@@ -289,7 +289,7 @@ protected:
     void try_send_rdma(osd_client_t *cl);
     void try_send_rdma_odp(osd_client_t *cl);
     void try_send_rdma_nodp(osd_client_t *cl);
-    bool init_recv_rdma(osd_client_t *cl);
+    bool try_recv_rdma(osd_client_t *cl);
     void handle_rdma_events(msgr_rdma_context_t *rdma_context);
     msgr_rdma_context_t* choose_rdma_context(osd_client_t *cl);
 #endif

src/client/msgr_rdma.cpp

@@ -67,20 +67,6 @@ msgr_rdma_context_t::~msgr_rdma_context_t()
         ibv_close_device(context);
 }
 
-msgr_rdma_buf_t::~msgr_rdma_buf_t()
-{
-    if (buf)
-    {
-        free(buf);
-        buf = NULL;
-    }
-    if (mr)
-    {
-        ibv_dereg_mr(mr);
-        mr = NULL;
-    }
-}
-
 msgr_rdma_connection_t::~msgr_rdma_connection_t()
 {
     ctx->reserve_cqe(-max_send-max_recv);
@@ -98,6 +84,26 @@ msgr_rdma_connection_t::~msgr_rdma_connection_t()
     if (qp)
         ibv_destroy_qp(qp);
 #endif
+    if (recv_buffers.size())
+    {
+        for (auto b: recv_buffers)
+        {
+            if (b.mr)
+                ibv_dereg_mr(b.mr);
+            free(b.buf);
+        }
+        recv_buffers.clear();
+    }
+    if (send_out.mr)
+    {
+        ibv_dereg_mr(send_out.mr);
+        send_out.mr = NULL;
+    }
+    if (send_out.buf)
+    {
+        free(send_out.buf);
+        send_out.buf = NULL;
+    }
     send_out_size = 0;
 }
 
@@ -487,12 +493,12 @@ int msgr_rdma_connection_t::connect(msgr_rdma_address_t *dest)
             .grh        = {
                 .dgid = dest->gid,
                 .sgid_index = conn->ctx->gid_index,
-                .hop_limit = 64, // FIXME can it vary?
+                .hop_limit = 1, // FIXME can it vary?
             },
             .dlid       = dest->lid,
             .sl         = 0, // service level
             .src_path_bits = 0,
-            .is_global  = (uint8_t)(dest->gid.global.subnet_prefix || dest->gid.global.interface_id ? 1 : 0),
+            .is_global  = (uint8_t)(dest->gid.global.interface_id ? 1 : 0),
             .port_num   = conn->ctx->ib_port,
         },
         .max_rd_atomic  = 1,
@@ -657,7 +663,7 @@ void osd_messenger_t::try_send_rdma_nodp(osd_client_t *cl)
     {
         // Allocate send ring buffer, if not yet
         rc->send_out_size = rc->max_msg*rdma_max_send;
-        rc->send_out.buf = (uint8_t*)malloc_or_die(rc->send_out_size);
+        rc->send_out.buf = malloc_or_die(rc->send_out_size);
         if (!rc->ctx->odp)
         {
             rc->send_out.mr = ibv_reg_mr(rc->ctx->pd, rc->send_out.buf, rc->send_out_size, 0);
@@ -706,12 +712,12 @@ void osd_messenger_t::try_send_rdma(osd_client_t *cl)
         try_send_rdma_nodp(cl);
 }
 
-static void try_recv_rdma_wr(osd_client_t *cl, void *buf)
+static void try_recv_rdma_wr(osd_client_t *cl, msgr_rdma_buf_t b)
 {
     ibv_sge sge = {
-        .addr = (uintptr_t)buf,
+        .addr = (uintptr_t)b.buf,
         .length = (uint32_t)cl->rdma_conn->max_msg,
-        .lkey = cl->rdma_conn->ctx->odp ? cl->rdma_conn->ctx->mr->lkey : cl->rdma_conn->recv_buf.mr->lkey,
+        .lkey = cl->rdma_conn->ctx->odp ? cl->rdma_conn->ctx->mr->lkey : b.mr->lkey,
     };
     ibv_recv_wr *bad_wr = NULL;
     ibv_recv_wr wr = {
@@ -728,23 +734,22 @@ static void try_recv_rdma_wr(osd_client_t *cl, void *buf)
     cl->rdma_conn->cur_recv++;
 }
 
-bool osd_messenger_t::init_recv_rdma(osd_client_t *cl)
+bool osd_messenger_t::try_recv_rdma(osd_client_t *cl)
 {
     auto rc = cl->rdma_conn;
-    assert(!rc->recv_buf.buf);
-    rc->recv_buf.buf = (uint8_t*)malloc_or_die(rc->max_msg * rc->max_recv);
-    if (!rc->ctx->odp)
+    while (rc->cur_recv < rc->max_recv)
     {
-        rc->recv_buf.mr = ibv_reg_mr(rc->ctx->pd, rc->recv_buf.buf, rc->max_msg * rc->max_recv, IBV_ACCESS_LOCAL_WRITE);
-        if (!rc->recv_buf.mr)
+        msgr_rdma_buf_t b;
+        b.buf = malloc_or_die(rc->max_msg);
+        if (!rc->ctx->odp)
         {
-            fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
-            exit(1);
+            b.mr = ibv_reg_mr(rc->ctx->pd, b.buf, rc->max_msg, IBV_ACCESS_LOCAL_WRITE);
+            if (!b.mr)
+            {
+                fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
+                exit(1);
+            }
         }
-    }
-    for (uint32_t i = 0; i < rc->max_recv; i++)
-    {
-        uint8_t *b = rc->recv_buf.buf + i*rc->max_msg;
         rc->recv_buffers.push_back(b);
         try_recv_rdma_wr(cl, b);
     }
@@ -802,7 +807,7 @@ void osd_messenger_t::handle_rdma_events(msgr_rdma_context_t *rdma_context)
                 cl->ping_time_remaining = 0;
                 cl->idle_time_remaining = osd_idle_timeout;
                 rc->cur_recv--;
-                if (!handle_read_buffer(cl, rc->recv_buffers[rc->next_recv_buf], wc[i].byte_len))
+                if (!handle_read_buffer(cl, rc->recv_buffers[rc->next_recv_buf].buf, wc[i].byte_len))
                 {
                     // handle_read_buffer may stop the client
                     clear_immediate_ops(client_id);

src/client/msgr_rdma.h

@@ -54,10 +54,8 @@ struct msgr_rdma_context_t
 
 struct msgr_rdma_buf_t
 {
-    uint8_t *buf = NULL;
+    void *buf = NULL;
     ibv_mr *mr = NULL;
-
-    ~msgr_rdma_buf_t();
 };
 
 struct msgr_rdma_connection_t
@@ -74,8 +72,7 @@ struct msgr_rdma_connection_t
     int cur_send = 0, cur_recv = 0;
     int send_pos = 0, send_buf_pos = 0;
     int next_recv_buf = 0;
-    std::vector<void*> recv_buffers;
-    msgr_rdma_buf_t recv_buf;
+    std::vector<msgr_rdma_buf_t> recv_buffers;
     std::vector<uint64_t> send_sizes;
     msgr_rdma_buf_t send_out;
     int send_out_pos = 0, send_done_pos = 0, send_out_size = 0;

src/client/msgr_rdmacm.cpp

@@ -531,7 +531,7 @@ void osd_messenger_t::rdmacm_established(rdma_cm_event *ev)
         fprintf(stderr, "Successfully connected with OSD %ju using RDMA-CM\n", peer_osd);
     }
     // Add initial receive request(s)
-    init_recv_rdma(cl);
+    try_recv_rdma(cl);
     if (peer_osd)
     {
         check_peer_config(cl);

src/client/msgr_receive.cpp

@@ -42,7 +42,6 @@ void osd_messenger_t::read_requests()
             }
             if (!sqe)
             {
-                cl->refs--;
                 cl->read_msg.msg_iovlen = 0;
                 read_ready_clients.erase(read_ready_clients.begin(), read_ready_clients.begin() + i);
                 return;

src/client/msgr_send.cpp

@@ -326,10 +326,8 @@ void osd_messenger_t::handle_send(int result, bool prev, bool more, osd_client_t
             int expected = cl->send_list.size() < IOV_MAX ? cl->send_list.size() : IOV_MAX;
             if (done != expected)
             {
-                fprintf(stderr, "Client %d socket write error: expected to send "
-                    "%d iovecs with MSG_WAITALL but sent %d. Disconnecting client\n", cl->peer_fd, expected, done);
-                stop_client(cl->peer_fd);
-                return;
+                fprintf(stderr, "BUG (maybe kernel): Expected to send %d iovecs with MSG_WAITALL but sent %d\n", expected, done);
+                exit(1);
             }
             cl->zc_free_list.push_back(NULL); // end marker
         }
@@ -365,7 +363,7 @@ void osd_messenger_t::handle_send(int result, bool prev, bool more, osd_client_t
                 }
             });
             // Add the initial receive request
-            init_recv_rdma(cl);
+            try_recv_rdma(cl);
         }
 #endif
     }

src/client/msgr_stop.cpp

@@ -85,16 +85,6 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
             osd_peer_fds.erase(osd_it);
         }
     }
-#ifdef WITH_RDMA
-    if (cl->rdma_conn && cl->rdma_conn->cmid)
-    {
-        auto rdma_it = rdmacm_connections.find(cl->rdma_conn->cmid);
-        if (rdma_it != rdmacm_connections.end() && rdma_it->second == cl)
-        {
-            rdmacm_connections.erase(rdma_it);
-        }
-    }
-#endif
 #ifndef __MOCK__
     // Then remove FD from the eventloop so we don't accidentally read something
     tfd->set_fd_handler(peer_fd, false, NULL);
@@ -168,13 +158,6 @@ osd_client_t::~osd_client_t()
     }
     // Cancel outbound ops
     cancel_ops();
-    for (osd_op_t *op: zc_free_list)
-    {
-        if (op)
-        {
-            delete op;
-        }
-    }
 #ifndef __MOCK__
 #ifdef WITH_RDMA
     if (rdma_conn)

src/client/osd_ops.h

@@ -262,10 +262,7 @@ struct __attribute__((__packed__)) osd_reply_del_t
     uint32_t left_on_dead_count;
 };
 
-// sync to the primary OSD. semantics:
-// 1) any non-synced write may disappear on OSD restart. even if it's a CAS write.
-// 2) sync only guarantees to commit completed writes, not in-progress ones.
-// 3) sync is a no-op when immediate_commit is active.
+// sync to the primary OSD
 struct __attribute__((__packed__)) osd_op_sync_t
 {
     osd_op_header_t header;

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 3.0.3
+Version: 2.4.3
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/cmd/cli.cpp

@@ -245,7 +245,7 @@ static json11::Json::object parse_args(int narg, const char *args[])
     cfg["progress"] = "1";
     for (int i = 1; i < narg; i++)
     {
-        bool argHasValue = (i < narg-1);
+        bool argHasValue = (!(i == narg-1) && (args[i+1][0] != '-'));
         if (args[i][0] == '-' && args[i][1] == 'h' && args[i][2] == 0)
         {
             cfg["help"] = "1";

src/cmd/cli_dd.cpp

@@ -111,7 +111,7 @@ struct dd_in_info_t
             {
                 in_granularity = 512;
             }
-            if (lseek(ifd, 1, SEEK_SET) != (off_t)1)
+            if (lseek(ifd, 1, SEEK_SET) == (off_t)-1)
             {
                 in_seekable = false;
             }
@@ -389,7 +389,7 @@ struct cli_dd_t
     int state = 0;
     int copy_error = 0;
     int in_waiting = 0, out_waiting = 0;
-    cli_result_t result = {};
+    cli_result_t result;
 
     bool is_done()
     {
@@ -890,12 +890,12 @@ resume_2:
         clock_gettime(CLOCK_REALTIME, &tv_begin);
         tv_progress = tv_begin;
 resume_3:
-        while ((ignore_errors || !copy_error) && (!in_eof || read_buffers.size() || in_waiting > 0 || out_waiting > 0 || short_writes.size()))
+        while ((ignore_errors || !copy_error) && (!in_eof || read_buffers.size() || in_waiting > 0 || out_waiting > 0))
         {
             print_progress(false);
             while ((ignore_errors || !copy_error) &&
                 (!in_eof && in_waiting < in_iodepth && read_buffers.size() < out_iodepth ||
-                (read_buffers.size() || short_writes.size()) && out_waiting < out_iodepth))
+                read_buffers.size() && out_waiting < out_iodepth))
             {
                 if (!in_eof && in_waiting < in_iodepth && read_buffers.size() < out_iodepth)
                 {
@@ -904,7 +904,7 @@ resume_3:
                         break;
                     }
                 }
-                if ((read_buffers.size() || short_writes.size()) && out_waiting < out_iodepth)
+                if (read_buffers.size() && out_waiting < out_iodepth)
                 {
                     if (!add_write_op())
                     {
@@ -931,8 +931,7 @@ close_end:
         oinfo.close_output(parent);
         iinfo.close_input(parent);
         // Done
-        if (copy_error)
-            result.err = copy_error;
+        result.err = copy_error;
         state = 100;
     }
 };

src/cmd/cli_modify_osd.cpp

@@ -120,23 +120,12 @@ resume_1:
                 else
                     osd_cfg.erase("noout");
             }
-            if (osd_cfg_mod_rev)
-            {
-                compare.push_back(json11::Json::object {
-                    { "target", "MOD" },
-                    { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/osd/"+std::to_string(osd_num)) },
-                    { "result", "LESS" },
-                    { "mod_revision", osd_cfg_mod_rev+1 },
-                });
-            }
-            else
-            {
-                compare.push_back(json11::Json::object {
-                    { "target", "VERSION" },
-                    { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/osd/"+std::to_string(osd_num)) },
-                    { "version", 0 },
-                });
-            }
+            compare.push_back(json11::Json::object {
+                { "target", "MOD" },
+                { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/osd/"+std::to_string(osd_num)) },
+                { "result", "LESS" },
+                { "mod_revision", osd_cfg_mod_rev+1 },
+            });
             if (!osd_cfg.size())
             {
                 success.push_back(json11::Json::object {

src/disk_tool/disk_simple_offsets.cpp

@@ -132,13 +132,16 @@ void disk_tool_simple_offsets(json11::Json cfg, bool json_output)
     uint64_t meta_size;
     if (meta_format == BLOCKSTORE_META_FORMAT_HEAP)
     {
-        uint32_t min_object_size = sizeof(heap_big_intent_t) + (data_csum_size ? data_csum_size : 4) + 2*clean_entry_bitmap_size;
+        uint32_t min_object_size = sizeof(heap_object_t)+sizeof(heap_write_t)+data_csum_size+2*clean_entry_bitmap_size;
+        uint32_t meta_block_target_free_space = cfg["meta_block_target_free_space"].uint64_value();
+        if (!meta_block_target_free_space || meta_block_target_free_space > device_block_size-min_object_size)
+            meta_block_target_free_space = 800;
         double meta_reserve = cfg["meta_reserve"].number_value();
         if (!meta_reserve)
             meta_reserve = 1.5;
         else if (meta_reserve < 1)
             meta_reserve = 1;
-        uint32_t entries_per_block = device_block_size / min_object_size;
+        uint32_t entries_per_block = (device_block_size-meta_block_target_free_space) / min_object_size;
         meta_size = device_block_size * (uint64_t)((object_count+entries_per_block-1) / entries_per_block * meta_reserve);
     }
     else if (meta_format == BLOCKSTORE_META_FORMAT_V2)

src/disk_tool/disk_tool.cpp

@@ -26,9 +26,6 @@ static const char *help_text =
     "  Options (automatic mode):\n"
     "    --osd_per_disk <N>\n"
     "      Create <N> OSDs on each disk (default 1)\n"
-    "    --meta_format 3\n"
-    "      Metadata store version. 3 is the new log-structured store, 2 is the stable store\n"
-    "      from Vitastor 0.9-2.x, 1 is the legacy store from Vitastor 0.6-0.8.\n"
     "    --hybrid\n"
     "      Prepare hybrid (HDD+SSD, NVMe+SATA or etc) OSDs using provided devices. By default,\n"
     "      any passed SSDs will be used for journals and metadata, HDDs will be used for data,\n"
@@ -65,8 +62,6 @@ static const char *help_text =
     "    --force                    Bypass partition safety checks (for emptiness and so on)\n"
     "  \n"
     "  Options (both modes):\n"
-    "    --tags tag1,tag2           Set new OSD tag(s)\n"
-    "    --weight <number>          Set new OSD weight (between 0 and 1)\n"
     "    --journal_size 32M/1G      Set journal size (area or partition size)\n"
     "    --block_size 128k/1M       Set blockstore object size\n"
     "    --bitmap_granularity 4k    Set bitmap granularity\n"
@@ -92,8 +87,7 @@ static const char *help_text =
     "  inmemory_metadata, inmemory_journal, max_write_iodepth,\n"
     "  min_flusher_count, max_flusher_count, journal_sector_buffer_count,\n"
     "  journal_no_same_sector_overwrites, throttle_small_writes, throttle_target_iops,\n"
-    "  throttle_target_mbs, throttle_target_parallelism, throttle_threshold_us,\n"
-    "  atomic_write_size, use_atomic_flag.\n"
+    "  throttle_target_mbs, throttle_target_parallelism, throttle_threshold_us.\n"
     "\n"
     "vitastor-disk upgrade-simple <UNIT_FILE|OSD_NUMBER>\n"
     "  Upgrade an OSD created by old (0.7.1 and older) make-osd.sh or make-osd-hybrid.js scripts.\n"
@@ -376,7 +370,7 @@ int main(int argc, char *argv[])
             self.dsk.meta_device = cmd[1];
             self.dsk.meta_block_size = strtoul(cmd[2], NULL, 10);
             self.dsk.meta_offset = strtoull(cmd[3], NULL, 10);
-            self.dsk.min_meta_len = self.dsk.meta_area_size = strtoull(cmd[4], NULL, 10);
+            self.dsk.meta_area_size = strtoull(cmd[4], NULL, 10);
         }
         else if (cmd.size() == 2)
         {

src/disk_tool/disk_tool.h

@@ -27,9 +27,6 @@
 #define VITASTOR_PART_TYPE "e7009fac-a5a1-4d72-af72-53de13059903"
 #define DEFAULT_HYBRID_JOURNAL "1G"
 #define DEFAULT_HYBRID_SSD_JOURNAL "128M"
-#define VITASTOR_META_FORMAT_NAME_V1 "0.6"
-#define VITASTOR_META_FORMAT_NAME_V2 "0.9"
-#define VITASTOR_META_FORMAT_NAME_HEAP "3.0"
 
 struct resizer_data_moving_t;
 
@@ -52,22 +49,20 @@ struct disk_tool_t
     bool all = false, json = false, now = false;
     bool dump_with_blocks = false, dump_with_data = false;
     bool dump_as_old = false;
-    bool skip_obsolete = false;
     int log_level = 1;
-    double meta_reserve_multiple = 2;
-    uint64_t meta_reserve_min_size = (uint64_t)1024*1024*1024;
     blockstore_disk_t dsk;
 
     // resize data and/or move metadata and journal
     int iodepth;
     std::string new_meta_device, new_journal_device;
-    uint64_t new_data_offset = 0, new_data_len = 0;
-    uint64_t new_journal_offset = 0, new_journal_len = 0;
-    uint64_t new_meta_offset = 0, new_meta_len = 0;
+    uint64_t new_data_offset, new_data_len;
+    uint64_t new_journal_offset, new_journal_len;
+    uint64_t new_meta_offset, new_meta_len;
 
     /**** State ****/
 
-    uint64_t journal_pos = 0, journal_calc_data_pos = 0;
+    uint64_t meta_pos;
+    uint64_t journal_pos, journal_calc_data_pos;
 
     uint8_t *buffer_area = NULL;
     bool first_block, first_entry;
@@ -77,25 +72,26 @@ struct disk_tool_t
     std::map<uint64_t, uint64_t>::iterator remap_it;
     ring_loop_t *ringloop = NULL;
     ring_consumer_t ring_consumer;
-    int remap_active = 0;
+    int remap_active;
     journal_entry_start je_start;
     uint8_t *new_journal_buf = NULL, *new_meta_buf = NULL, *new_journal_ptr = NULL, *new_journal_data = NULL;
     blockstore_meta_header_v3_t *new_meta_hdr = NULL;
-    uint64_t new_journal_in_pos = 0;
-    int64_t data_idx_diff = 0;
-    uint64_t total_blocks = 0, free_first = 0, free_last = 0;
-    uint64_t new_clean_entry_bitmap_size = 0, new_data_csum_size = 0, new_clean_entry_size = 0, new_entries_per_block = 0;
+    blockstore_disk_t new_dsk;
+    blockstore_heap_t *new_heap = NULL;
+    uint64_t new_journal_in_pos;
+    int64_t data_idx_diff;
+    uint64_t total_blocks, free_first, free_last;
+    uint64_t new_clean_entry_bitmap_size, new_data_csum_size, new_clean_entry_size, new_entries_per_block;
     uint32_t new_meta_format = 0;
     int new_journal_fd = -1, new_meta_fd = -1;
     resizer_data_moving_t *moving_blocks = NULL;
-    bool atomic_warned = false;
 
-    bool started = false;
+    bool started;
     void *small_write_data = NULL;
-    uint32_t data_crc32 = 0;
-    bool data_csum_valid = false;
-    uint32_t crc32_last = 0;
-    uint32_t new_crc32_prev = 0;
+    uint32_t data_crc32;
+    bool data_csum_valid;
+    uint32_t crc32_last;
+    uint32_t new_crc32_prev;
 
     ~disk_tool_t();
 
@@ -104,15 +100,15 @@ struct disk_tool_t
     int process_journal(std::function<int(void*)> block_fn, bool do_open = true);
     int process_journal_block(void *buf, std::function<void(int, journal_entry*)> iter_fn);
     int process_meta(std::function<void(blockstore_meta_header_v3_t *)> hdr_fn,
-        std::function<void(blockstore_heap_t *heap, heap_entry_t *obj, uint32_t meta_block_num)> obj_fn,
+        std::function<void(blockstore_heap_t *heap, heap_object_t *obj, uint32_t meta_block_num)> obj_fn,
         std::function<void(uint64_t block_num, clean_disk_entry *entry_v1, uint8_t *bitmap)> record_fn,
         bool with_data, bool do_open);
 
     int dump_meta();
     void dump_meta_header(blockstore_meta_header_v3_t *hdr);
     void dump_meta_entry(uint64_t block_num, clean_disk_entry *entry, uint8_t *bitmap);
-    void dump_heap_entry_as_old(blockstore_heap_t *heap, heap_entry_t *obj);
-    void dump_heap_entry(blockstore_heap_t *heap, heap_entry_t *obj);
+    void dump_heap_entry_as_old(blockstore_heap_t *heap, heap_object_t *obj);
+    void dump_heap_entry(blockstore_heap_t *heap, heap_object_t *obj);
 
     int dump_load_check_superblock(const std::string & device);
 
@@ -136,10 +132,6 @@ struct disk_tool_t
     void choose_journal_block(uint32_t je_size);
     int resize_rebuild_journal();
     int resize_write_new_journal();
-    void remap_big_write(heap_entry_t *wr);
-    void remap_small_write(heap_entry_t *wr);
-    void fill_old_clean_entry(blockstore_heap_t *heap, heap_entry_t *big_wr);
-    void fill_old_journal_entry(blockstore_heap_t *heap, heap_entry_t *wr);
     int resize_rebuild_meta();
     int resize_write_new_meta();
     void free_new_meta();
@@ -158,7 +150,6 @@ struct disk_tool_t
     json11::Json read_osd_superblock(std::string device, bool expect_exist = true, bool ignore_nonref = false);
     uint32_t write_osd_superblock(std::string device, json11::Json params);
 
-    void parse_meta_reserve();
     int prepare_one(std::map<std::string, std::string> options, int is_hdd, json11::Json::object & result);
     int check_existing_partition(std::string & dev_by_uuid);
     int fix_partition_type(std::string & dev_by_uuid);
@@ -176,8 +167,7 @@ void disk_tool_simple_offsets(json11::Json cfg, bool json_output);
 
 uint64_t sscanf_json(const char *fmt, const json11::Json & str);
 void fromhexstr(const std::string & from, int bytes, uint8_t *to);
-int disable_cache(const std::string & dev);
-uint64_t get_atomic_write_size(const std::string & dev);
+int disable_cache(std::string dev);
 uint64_t get_device_size(const std::string & dev, bool should_exist = false);
 std::string get_parent_device(std::string dev);
 int shell_exec(const std::vector<std::string> & cmd, const std::string & in, std::string *out, std::string *err);

src/disk_tool/disk_tool_discard.cpp

@@ -54,13 +54,12 @@ int disk_tool_t::trim_data(std::string device)
     fprintf(stderr, "Reading metadata\n");
     data_alloc = new allocator_t(dsk.block_count);
     r = process_meta(
-        [](blockstore_meta_header_v3_t *hdr) {},
-        [this](blockstore_heap_t *heap, heap_entry_t *obj, uint32_t meta_block_num)
+        [this](blockstore_meta_header_v3_t *hdr) {},
+        [this](blockstore_heap_t *heap, heap_object_t *obj, uint32_t meta_block_num)
         {
-            for (auto wr = obj; wr; wr = heap->prev(wr))
+            for (auto wr = obj->get_writes(); wr; wr = wr->next())
             {
-                if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE ||
-                    (wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_INTENT)
+                if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE)
                 {
                     data_alloc->set(wr->big_location(heap) / dsk.data_block_size, true);
                 }

src/disk_tool/disk_tool_meta.cpp

@@ -7,8 +7,10 @@
 #include "json_util.h"
 #include "malloc_or_die.h"
 
+#define FREE_SPACE_BIT 0x8000
+
 int disk_tool_t::process_meta(std::function<void(blockstore_meta_header_v3_t *)> hdr_fn,
-    std::function<void(blockstore_heap_t *heap, heap_entry_t *obj, uint32_t meta_block_num)> obj_fn,
+    std::function<void(blockstore_heap_t *heap, heap_object_t *obj, uint32_t meta_block_num)> obj_fn,
     std::function<void(uint64_t block_num, clean_disk_entry *entry_v1, uint8_t *bitmap)> record_fn,
     bool with_data, bool do_open)
 {
@@ -23,7 +25,7 @@ int disk_tool_t::process_meta(std::function<void(blockstore_meta_header_v3_t *)>
         buf_size = 8*dsk.meta_block_size;
     uint8_t *data = NULL;
     data = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, buf_size);
-    blockstore_meta_header_v3_t *hdr = (blockstore_meta_header_v3_t *)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size);
+    blockstore_meta_header_v3_t *hdr = (blockstore_meta_header_v3_t *)data;
     if (do_open)
     {
         if (dsk.meta_fd >= 0)
@@ -99,25 +101,26 @@ close_error:
         blockstore_heap_t *heap = new blockstore_heap_t(&dsk, buffer_area, log_level);
         // Load heap and just iterate it in memory
         hdr_fn(hdr);
-        uint64_t meta_pos = dsk.meta_block_size;
+        hdr = NULL;
+        meta_pos = dsk.meta_block_size;
         lseek64(dsk.meta_fd, dsk.meta_offset+meta_pos, 0);
-        uint64_t entries_loaded = 0;
-        while (meta_pos < hdr->meta_area_size)
+        while (meta_pos < dsk.meta_area_size)
         {
-            uint64_t read_len = buf_size < hdr->meta_area_size-meta_pos ? buf_size : hdr->meta_area_size-meta_pos;
+            uint64_t read_len = buf_size < dsk.meta_area_size-meta_pos ? buf_size : dsk.meta_area_size-meta_pos;
             read_blocking(dsk.meta_fd, data, read_len);
-            r = heap->load_blocks(meta_pos-dsk.meta_block_size, read_len, data, true, entries_loaded);
+            heap->read_blocks(meta_pos-dsk.meta_block_size, read_len, data, [&](heap_object_t *obj)
+            {
+                obj_fn(heap, obj, ((uint8_t*)obj-data+meta_pos)/dsk.meta_block_size);
+            }, [](uint32_t, uint32_t, uint8_t*){});
             meta_pos += read_len;
         }
-        heap->iterate_objects([&](heap_entry_t* obj, uint32_t meta_block_num)
-        {
-            obj_fn(heap, obj, meta_block_num);
-        });
-        obj_fn(heap, NULL, 0);
         delete heap;
     }
     else if (hdr->zero == 0 && hdr->magic == BLOCKSTORE_META_MAGIC_V1)
     {
+        dsk.meta_format = hdr->version;
+        dsk.calc_lengths();
+        dsk.check_lengths();
         if (hdr->version == BLOCKSTORE_META_FORMAT_V1)
         {
             // Vitastor 0.6-0.8 - static array of clean_disk_entry with bitmaps
@@ -162,7 +165,8 @@ csum_unknown:
         // Read
         uint64_t block_num = 0;
         hdr_fn(hdr);
-        uint64_t meta_pos = dsk.meta_block_size;
+        hdr = NULL;
+        meta_pos = dsk.meta_block_size;
         lseek64(dsk.meta_fd, dsk.meta_offset+meta_pos, 0);
         while (meta_pos < dsk.min_meta_len)
         {
@@ -197,12 +201,11 @@ csum_unknown:
         lseek64(dsk.meta_fd, dsk.meta_offset, 0);
         dsk.clean_entry_bitmap_size = 0;
         dsk.clean_entry_size = sizeof(clean_disk_entry);
-        uint64_t meta_pos = 0;
         uint64_t block_num = 0;
         hdr_fn(NULL);
-        while (meta_pos < dsk.min_meta_len)
+        while (meta_pos < dsk.meta_area_size)
         {
-            uint64_t read_len = buf_size < dsk.min_meta_len-meta_pos ? buf_size : dsk.min_meta_len-meta_pos;
+            uint64_t read_len = buf_size < dsk.meta_area_size-meta_pos ? buf_size : dsk.meta_area_size-meta_pos;
             read_blocking(dsk.meta_fd, data, read_len);
             meta_pos += read_len;
             for (uint64_t blk = 0; blk < read_len; blk += dsk.meta_block_size)
@@ -220,7 +223,6 @@ csum_unknown:
     }
 close_free:
     free(data);
-    free(hdr);
     if (buffer_area)
     {
         free(buffer_area);
@@ -267,24 +269,20 @@ int disk_tool_t::dump_load_check_superblock(const std::string & device)
 
 int disk_tool_t::dump_meta()
 {
-    dump_with_data = options["format"] == "data";
-    dump_as_old = options["format"] == "old";
     int r = process_meta(
         [this](blockstore_meta_header_v3_t *hdr)
         {
             if (dump_as_old)
             {
                 hdr->version = BLOCKSTORE_META_FORMAT_V2;
-                hdr->completed_lsn = 0;
+                hdr->compacted_lsn = 0;
                 hdr->header_csum = 0;
                 hdr->header_csum = crc32c(0, hdr, sizeof(blockstore_meta_header_v2_t));
             }
             dump_meta_header(hdr);
         },
-        [this](blockstore_heap_t *heap, heap_entry_t *obj, uint32_t meta_block_num)
+        [this](blockstore_heap_t *heap, heap_object_t *obj, uint32_t meta_block_num)
         {
-            if (!obj)
-                return;
             if (dump_as_old)
                 dump_heap_entry_as_old(heap, obj);
             else
@@ -305,7 +303,7 @@ void disk_tool_t::dump_meta_header(blockstore_meta_header_v3_t *hdr)
         if (hdr->version == BLOCKSTORE_META_FORMAT_V1)
         {
             printf(
-                "{\"version\":\"" VITASTOR_META_FORMAT_NAME_V1 "\",\"meta_block_size\":%u,\"data_block_size\":%u,\"bitmap_granularity\":%u,"
+                "{\"version\":\"0.6\",\"meta_block_size\":%u,\"data_block_size\":%u,\"bitmap_granularity\":%u,"
                 "\"entries\":[\n",
                 hdr->meta_block_size, hdr->data_block_size, hdr->bitmap_granularity
             );
@@ -313,7 +311,7 @@ void disk_tool_t::dump_meta_header(blockstore_meta_header_v3_t *hdr)
         else if (hdr->version == BLOCKSTORE_META_FORMAT_V2)
         {
             printf(
-                "{\"version\":\"" VITASTOR_META_FORMAT_NAME_V2 "\",\"meta_block_size\":%u,\"data_block_size\":%u,\"bitmap_granularity\":%u,"
+                "{\"version\":\"0.9\",\"meta_block_size\":%u,\"data_block_size\":%u,\"bitmap_granularity\":%u,"
                 "\"data_csum_type\":\"%s\",\"csum_block_size\":%u,\"entries\":[\n",
                 hdr->meta_block_size, hdr->data_block_size, hdr->bitmap_granularity,
                 csum_type_str(hdr->data_csum_type).c_str(), hdr->csum_block_size
@@ -322,7 +320,7 @@ void disk_tool_t::dump_meta_header(blockstore_meta_header_v3_t *hdr)
         else if (hdr->version == BLOCKSTORE_META_FORMAT_HEAP)
         {
             printf(
-                "{\"version\":\"" VITASTOR_META_FORMAT_NAME_HEAP "\",\"meta_block_size\":%u,\"data_block_size\":%u,\"bitmap_granularity\":%u,"
+                "{\"version\":\"3.0\",\"meta_block_size\":%u,\"data_block_size\":%u,\"bitmap_granularity\":%u,"
                 "\"data_csum_type\":\"%s\",\"csum_block_size\":%u,\"entries\":[\n",
                 hdr->meta_block_size, hdr->data_block_size, hdr->bitmap_granularity,
                 csum_type_str(hdr->data_csum_type).c_str(), hdr->csum_block_size
@@ -336,14 +334,14 @@ void disk_tool_t::dump_meta_header(blockstore_meta_header_v3_t *hdr)
     first_entry = true;
 }
 
-void disk_tool_t::dump_heap_entry_as_old(blockstore_heap_t *heap, heap_entry_t *obj)
+void disk_tool_t::dump_heap_entry_as_old(blockstore_heap_t *heap, heap_object_t *obj)
 {
-    heap_entry_t *wr = NULL;
-    for (wr = obj; wr && !wr->is_overwrite(); wr = heap->prev(wr))
+    heap_write_t *wr = NULL;
+    for (wr = obj->get_writes(); wr && wr->entry_type != (BS_HEAP_BIG_WRITE|BS_HEAP_STABLE) &&
+        wr->entry_type != (BS_HEAP_TOMBSTONE|BS_HEAP_STABLE); wr = wr->next())
     {
     }
-    if (!wr || wr->entry_type != (BS_HEAP_BIG_WRITE|BS_HEAP_STABLE) &&
-        wr->entry_type != (BS_HEAP_BIG_INTENT|BS_HEAP_STABLE))
+    if (!wr || wr->entry_type != (BS_HEAP_BIG_WRITE|BS_HEAP_STABLE))
     {
         return;
     }
@@ -367,7 +365,7 @@ void disk_tool_t::dump_heap_entry_as_old(blockstore_heap_t *heap, heap_entry_t *
         printf("%02x", bitmap[i]);
     }
     uint8_t *csums = wr->get_checksums(heap);
-    uint32_t csum_size = heap->get_csum_size(wr);
+    uint32_t csum_size = wr->get_csum_size(heap);
     if (csums)
     {
         printf("\",\"block_csums\":\"");
@@ -384,7 +382,7 @@ void disk_tool_t::dump_heap_entry_as_old(blockstore_heap_t *heap, heap_entry_t *
     first_entry = false;
 }
 
-void disk_tool_t::dump_heap_entry(blockstore_heap_t *heap, heap_entry_t *obj)
+void disk_tool_t::dump_heap_entry(blockstore_heap_t *heap, heap_object_t *obj)
 {
     printf(
 #define ENTRY_FMT "{\"pool\":%u,\"inode\":\"0x%jx\",\"stripe\":\"0x%jx\",\"writes\":["
@@ -392,38 +390,29 @@ void disk_tool_t::dump_heap_entry(blockstore_heap_t *heap, heap_entry_t *obj)
 #undef ENTRY_FMT
         INODE_POOL(obj->inode), INODE_NO_POOL(obj->inode), obj->stripe
     );
-    heap_entry_t *wr = NULL;
+    heap_write_t *wr = NULL;
     bool first_wr = true;
-    for (wr = obj; wr; wr = heap->prev(wr))
+    for (wr = obj->get_writes(); wr; wr = wr->next())
     {
-        auto t = wr->type();
         printf(
 #define ENTRY_FMT "{\"lsn\":%ju,\"version\":%ju,\"type\":\"%s\",\"stable\":%s"
             (first_wr ? ENTRY_FMT : ("," ENTRY_FMT)),
 #undef ENTRY_FMT
-            wr->lsn, wr->version, (t == BS_HEAP_BIG_WRITE ? "big" : (
-                t == BS_HEAP_BIG_INTENT ? "big_intent" : (
-                t == BS_HEAP_SMALL_WRITE ? "small" : (
-                t == BS_HEAP_INTENT_WRITE ? "intent" : (
-                t == BS_HEAP_DELETE ? "delete" : (
-                t == BS_HEAP_COMMIT ? "commit" : (
-                t == BS_HEAP_ROLLBACK ? "rollback" : (
-                "unknown")))))))),
+            wr->lsn, wr->version, (wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_SMALL_WRITE ? "small" : (
+                (wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE ? "big" : (
+                (wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_INTENT_WRITE ? "intent" : (
+                (wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_TOMBSTONE ? "tombstone" : "unknown"))),
             (wr->entry_type & BS_HEAP_STABLE) ? "true" : "false"
         );
-        if (t == BS_HEAP_BIG_WRITE)
+        if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE)
         {
             printf(",\"location\":%ju", wr->big_location(heap));
         }
-        else if (t == BS_HEAP_BIG_INTENT)
-        {
-            printf(",\"location\":%ju,\"offset\":%u,\"len\":%u", wr->big_location(heap), wr->big_intent().offset, wr->big_intent().len);
-        }
-        else if (t == BS_HEAP_INTENT_WRITE)
+        else if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_INTENT_WRITE)
         {
             printf(",\"offset\":%u,\"len\":%u", wr->small().offset, wr->small().len);
         }
-        else if (t == BS_HEAP_SMALL_WRITE)
+        else if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_SMALL_WRITE)
         {
             if (!dump_with_data)
             {
@@ -457,7 +446,7 @@ void disk_tool_t::dump_heap_entry(blockstore_heap_t *heap, heap_entry_t *obj)
         if (csums)
         {
             printf(",\"block_csums\":\"");
-            uint32_t csum_size = heap->get_csum_size(wr);
+            uint32_t csum_size = wr->get_csum_size(heap);
             for (uint32_t i = 0; i < csum_size; i++)
                 printf("%02x", csums[i]);
             printf("\"");
@@ -514,16 +503,12 @@ void disk_tool_t::dump_meta_entry(uint64_t block_num, clean_disk_entry *entry, u
 
 int disk_tool_t::write_json_meta(json11::Json meta)
 {
-    if (meta["version"].string_value() == VITASTOR_META_FORMAT_NAME_HEAP)
-    {
-        return write_json_heap(meta, meta["journal"]);
-    }
     new_meta_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_meta_len);
     memset(new_meta_buf, 0, new_meta_len);
     blockstore_meta_header_v2_t *new_hdr = (blockstore_meta_header_v2_t *)new_meta_buf;
     new_hdr->zero = 0;
     new_hdr->magic = BLOCKSTORE_META_MAGIC_V1;
-    new_hdr->version = meta["version"].string_value() == VITASTOR_META_FORMAT_NAME_V1
+    new_hdr->version = meta["version"].uint64_value() == BLOCKSTORE_META_FORMAT_V1
         ? BLOCKSTORE_META_FORMAT_V1 : BLOCKSTORE_META_FORMAT_V2;
     new_hdr->meta_block_size = meta["meta_block_size"].uint64_value()
         ? meta["meta_block_size"].uint64_value() : 4096;
@@ -589,9 +574,6 @@ int disk_tool_t::write_json_meta(json11::Json meta)
 
 int disk_tool_t::write_json_heap(json11::Json meta, json11::Json journal)
 {
-    new_meta_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_meta_len);
-    memset(new_meta_buf, 0, new_meta_len);
-    new_meta_hdr = (blockstore_meta_header_v3_t *)new_meta_buf;
     new_meta_hdr->zero = 0;
     new_meta_hdr->magic = BLOCKSTORE_META_MAGIC_V1;
     new_meta_hdr->version = BLOCKSTORE_META_FORMAT_HEAP;
@@ -608,98 +590,71 @@ int disk_tool_t::write_json_heap(json11::Json meta, json11::Json journal)
             : BLOCKSTORE_CSUM_NONE);
     new_meta_hdr->csum_block_size = meta["csum_block_size"].uint64_value();
     new_meta_hdr->header_csum = crc32c(0, new_meta_hdr, sizeof(blockstore_meta_header_v3_t));
-    new_meta_hdr->meta_area_size = new_meta_len;
     new_clean_entry_bitmap_size = (new_meta_hdr->data_block_size / new_meta_hdr->bitmap_granularity + 7) / 8;
     new_clean_entry_size = 0;
     new_entries_per_block = 0;
     new_data_csum_size = (new_meta_hdr->data_csum_type
         ? ((new_meta_hdr->data_block_size+new_meta_hdr->csum_block_size-1)/new_meta_hdr->csum_block_size*(new_meta_hdr->data_csum_type & 0xFF))
         : 0);
-    new_journal_buf = NULL;
+    new_journal_buf = new_journal_len ? (uint8_t*)memalign(MEM_ALIGNMENT, new_journal_len) : NULL;
     if (new_journal_len)
     {
-        new_journal_buf = (uint8_t*)memalign(MEM_ALIGNMENT, new_journal_len);
         memset(new_journal_buf, 0, new_journal_len);
     }
     uint64_t total_used_space = 0;
     uint32_t used_space = 0;
-    uint64_t meta_offset = dsk.meta_block_size;
-    blockstore_heap_t heap(&dsk, NULL, 0);
-    heap_entry_t *wr = NULL;
-    auto get_wr = [&](uint32_t entry_size)
-    {
-        if (used_space > new_meta_hdr->meta_block_size-entry_size)
-        {
-            heap.fill_block_empty_space(new_meta_buf + meta_offset, used_space);
-            meta_offset += new_meta_hdr->meta_block_size;
-            used_space = 0;
-            if (meta_offset >= new_meta_len)
-            {
-                fprintf(stderr, "Metadata doesn't fit into the new area (total used space: %ju)\n", total_used_space);
-                return (heap_entry_t*)NULL;
-            }
-        }
-        auto wr = (heap_entry_t*)(new_meta_buf + meta_offset + used_space);
-        used_space += entry_size;
-        return wr;
-    };
     // FIXME: Use a streaming json parser
-    if (meta["version"].string_value() == VITASTOR_META_FORMAT_NAME_HEAP)
+    if (meta["version"] == "3.0")
     {
         // New format
+        std::vector<uint8_t> object_buf;
+        new_heap = new blockstore_heap_t(&dsk, new_journal_buf, 0);
         for (const auto & meta_entry: meta["entries"].array_items())
         {
+            bool invalid = false;
             object_id oid = {
                 .inode = (sscanf_json(NULL, meta_entry["pool"]) << (64-POOL_ID_BITS)) | sscanf_json(NULL, meta_entry["inode"]),
                 .stripe = sscanf_json(NULL, meta_entry["stripe"]),
             };
-            for (const auto & write_entry: meta_entry["writes"].array_items())
+            object_buf.clear();
+            object_buf.resize(sizeof(heap_object_t));
+            heap_object_t *obj = (heap_object_t*)object_buf.data();
+            obj->size = sizeof(heap_object_t);
+            obj->write_pos = meta_entry["writes"].array_items().size() ? sizeof(heap_object_t) : 0;
+            obj->entry_type = BS_HEAP_OBJECT;
+            obj->inode = oid.inode;
+            obj->stripe = oid.stripe;
+            size_t pos = sizeof(heap_object_t);
+            heap_write_t *last_wr = NULL;
+            for (auto & write_entry: meta_entry["writes"].array_items())
             {
-                uint32_t wr_type = 0;
+                object_buf.resize(object_buf.size() + new_heap->get_max_write_entry_size());
+                heap_write_t *wr = (heap_write_t*)(object_buf.data() + pos);
+                last_wr = wr;
+                uint8_t wr_type = 0;
                 if (write_entry["type"] == "small")
                     wr_type = BS_HEAP_SMALL_WRITE;
                 else if (write_entry["type"] == "intent")
                     wr_type = BS_HEAP_INTENT_WRITE;
                 else if (write_entry["type"] == "big")
                     wr_type = BS_HEAP_BIG_WRITE;
-                else if (write_entry["type"] == "big_intent")
-                    wr_type = BS_HEAP_BIG_INTENT;
-                else if (write_entry["type"] == "delete")
-                    wr_type = BS_HEAP_DELETE;
-                else if (write_entry["type"] == "commit")
-                    wr_type = BS_HEAP_COMMIT;
-                else if (write_entry["type"] == "rollback")
-                    wr_type = BS_HEAP_ROLLBACK;
+                else if (write_entry["type"] == "tombstone")
+                    wr_type = BS_HEAP_TOMBSTONE;
                 else
                 {
-                    fprintf(stderr, "Write entry %s has invalid type: %s, aborting\n",
-                        write_entry.dump().c_str(), write_entry["type"].dump().c_str());
-close_err0:
-                    free(new_meta_buf);
-                    new_meta_buf = NULL;
-                    return 1;
+                    fprintf(stderr, "Write entry in %s has invalid type: %s, skipping object\n", meta_entry.dump().c_str(), write_entry["type"].dump().c_str());
+                    invalid = true;
+                    break;
                 }
-                uint64_t wr_offset = write_entry["offset"].uint64_value();
-                uint64_t wr_len = write_entry["len"].uint64_value();
-                uint32_t wr_size = (wr_type == BS_HEAP_SMALL_WRITE || wr_type == BS_HEAP_INTENT_WRITE
-                    ? heap.get_small_entry_size(wr_offset, wr_len)
-                    : (wr_type == BS_HEAP_BIG_WRITE
-                        ? heap.get_big_entry_size()
-                        : (wr_type == BS_HEAP_BIG_INTENT
-                            ? heap.get_big_intent_entry_size()
-                            : heap.get_simple_entry_size())));
-                if (!(wr = get_wr(wr_size)))
-                    goto close_err0;
-                wr->inode = oid.inode;
-                wr->stripe = oid.stripe;
                 wr->entry_type = wr_type | (write_entry["stable"].bool_value() ? BS_HEAP_STABLE : 0);
                 wr->lsn = write_entry["lsn"].uint64_value();
                 wr->version = write_entry["version"].uint64_value();
-                wr->size = wr->get_size(&heap);
+                wr->size = wr->get_size(new_heap);
+                wr->next_pos = wr->size;
                 if (wr_type == BS_HEAP_SMALL_WRITE || wr_type == BS_HEAP_INTENT_WRITE)
                 {
-                    wr->small().offset = wr_offset;
-                    wr->small().len = wr_len;
+                    wr->small().offset = write_entry["offset"].uint64_value();
+                    wr->small().len = write_entry["len"].uint64_value();
                     wr->small().location = write_entry["location"].uint64_value();
                     if (wr_type == BS_HEAP_SMALL_WRITE && write_entry["data"].is_string() && wr->small().len > 0)
                     {
@@ -709,51 +664,40 @@ close_err0:
                             free_new_meta();
                             return 1;
                         }
-                        wr->small().location = heap.find_free_buffer_area(wr->small().len);
+                        wr->small().location = new_heap->find_free_buffer_area(wr->small().len);
                         fromhexstr(write_entry["data"].string_value(), wr->small().len, new_journal_buf + wr->small().location);
                     }
                 }
                 else if (wr_type == BS_HEAP_BIG_WRITE)
-                {
-                    uint64_t loc = write_entry["location"].uint64_value();
-                    if ((loc % dsk.data_block_size) || (loc / dsk.data_block_size) >= 0xFFFF0000)
-                    {
-                        fprintf(stderr, "Write entry %s has invalid location: 0x%jx, aborting\n",
-                            write_entry.dump().c_str(), loc);
-                        free_new_meta();
-                        return 1;
-                    }
-                    wr->set_big_location(&heap, loc);
-                }
-                else if (wr_type == BS_HEAP_BIG_INTENT)
                 {
                     uint64_t loc = write_entry["location"].uint64_value();
                     assert(!(loc % dsk.data_block_size));
                     assert((loc / dsk.data_block_size) < 0xFFFF0000);
-                    auto & bi = wr->big_intent();
-                    bi.block_num = loc / dsk.data_block_size;
-                    bi.offset = wr_offset;
-                    bi.len = wr_len;
+                    wr->set_big_location(new_heap, loc);
                 }
-                if (write_entry["bitmap"].is_string() && wr->get_int_bitmap(&heap))
+                if (write_entry["bitmap"].is_string() && wr->get_int_bitmap(new_heap))
                 {
-                    fromhexstr(write_entry["bitmap"].string_value(), new_clean_entry_bitmap_size, wr->get_int_bitmap(&heap));
+                    fromhexstr(write_entry["bitmap"].string_value(), new_clean_entry_bitmap_size, wr->get_int_bitmap(new_heap));
                 }
-                if (write_entry["ext_bitmap"].is_string() && wr->get_ext_bitmap(&heap))
+                if (write_entry["ext_bitmap"].is_string() && wr->get_ext_bitmap(new_heap))
                 {
-                    fromhexstr(write_entry["ext_bitmap"].string_value(), new_clean_entry_bitmap_size, wr->get_ext_bitmap(&heap));
+                    fromhexstr(write_entry["ext_bitmap"].string_value(), new_clean_entry_bitmap_size, wr->get_ext_bitmap(new_heap));
                 }
-                if (write_entry["block_csums"].is_string() && wr->get_checksums(&heap))
+                if (write_entry["block_csums"].is_string() && wr->get_checksums(new_heap))
                 {
-                    fromhexstr(write_entry["block_csums"].string_value(), heap.get_csum_size(wr), wr->get_ext_bitmap(&heap));
+                    fromhexstr(write_entry["block_csums"].string_value(), wr->get_csum_size(new_heap), wr->get_ext_bitmap(new_heap));
                 }
-                if (write_entry["data_crc32c"].is_string() && wr->get_checksum(&heap))
+                if (write_entry["data_crc32c"].is_string() && wr->get_checksum(new_heap))
                 {
-                    *wr->get_checksum(&heap) = sscanf_json("%jx", write_entry["data_crc32c"]);
+                    *wr->get_checksum(new_heap) = sscanf_json("%jx", write_entry["data_crc32c"]);
                 }
-                wr->crc32c = wr->calc_crc32c();
-                assert((uint8_t*)wr + wr->size == new_meta_buf + meta_offset + used_space);
             }
+            if (invalid)
+            {
+                continue;
+            }
+            last_wr->next_pos = 0;
+            new_heap->copy_object(obj, NULL);
         }
     }
     else
@@ -774,17 +718,82 @@ close_err:
         journal = json11::Json();
         // Convert old format to the new format
         uint64_t next_lsn = 0;
+        uint64_t meta_offset = 0;
+        const uint32_t space_per_object = sizeof(heap_object_t) + sizeof(heap_write_t) +
+            new_clean_entry_bitmap_size*2 + new_data_csum_size;
         uint64_t buffer_pos = 0;
+        // FIXME: Rather ugly. Remove the dependency on dsk from heap?
+        blockstore_disk_t dsk;
+        dsk.bitmap_granularity = new_meta_hdr->bitmap_granularity;
+        dsk.block_count = 16;
+        dsk.data_block_size = new_meta_hdr->data_block_size;
+        dsk.clean_entry_bitmap_size = new_clean_entry_bitmap_size;
+        dsk.csum_block_size = new_meta_hdr->csum_block_size;
+        dsk.data_csum_type = new_meta_hdr->data_csum_type;
+        dsk.journal_len = 4096;
+        dsk.meta_area_size = new_meta_len;
+        dsk.meta_block_size = new_meta_hdr->meta_block_size;
+        dsk.meta_block_target_free_space = 800;
+        blockstore_heap_t heap(&dsk, NULL, 0);
         for (const auto & meta_entry: meta["entries"].array_items())
         {
             object_id oid = {
                 .inode = (sscanf_json(NULL, meta_entry["pool"]) << (64-POOL_ID_BITS)) | sscanf_json(NULL, meta_entry["inode"]),
                 .stripe = sscanf_json(NULL, meta_entry["stripe"]),
             };
-            if (!(wr = get_wr(heap.get_big_entry_size())))
+            uint32_t space_for_this = space_per_object;
+            auto j_it = journal_by_object.find(oid);
+            if (j_it != journal_by_object.end())
+            {
+                for (auto & rec: j_it->second)
+                {
+                    if (rec["type"] == "small_write" || rec["type"] == "small_write_instant")
+                    {
+                        uint64_t off = rec["offset"].uint64_value();
+                        uint64_t len = rec["len"].uint64_value();
+                        if (off+len > new_meta_hdr->data_block_size)
+                        {
+                            fprintf(stderr, "Journal entry has too large offset or length: %s\n", json11::Json(rec).dump().c_str());
+                            goto close_err;
+                        }
+                        space_for_this += sizeof(heap_write_t) + new_clean_entry_bitmap_size +
+                            ((off+len+new_meta_hdr->csum_block_size-1)/new_meta_hdr->csum_block_size - off/new_meta_hdr->csum_block_size) * (new_meta_hdr->data_csum_type & 0xFF);
+                    }
+                    else /*if (rec["type"] == "big_write" || rec["type"] == "big_write_instant")*/
+                    {
+                        space_for_this += sizeof(heap_write_t) + 2*new_clean_entry_bitmap_size + new_data_csum_size;
+                    }
+                }
+            }
+            if (space_for_this > new_meta_hdr->meta_block_size)
+            {
+                fprintf(stderr, "Object doesn't fit in a single metadata block. Object meta: %s, object journal: %s\n",
+                    meta_entry.dump().c_str(), json11::Json(j_it->second).dump().c_str());
                 goto close_err;
-            wr->inode = oid.inode;
-            wr->stripe = oid.stripe;
+            }
+            if (used_space + space_for_this > new_meta_hdr->meta_block_size-dsk.meta_block_target_free_space)
+            {
+                if (used_space < new_meta_hdr->meta_block_size-2)
+                {
+                    *((uint16_t*)(new_meta_buf + meta_offset + used_space)) = FREE_SPACE_BIT | (uint16_t)(new_meta_hdr->meta_block_size-used_space);
+                }
+                meta_offset += new_meta_hdr->meta_block_size;
+                used_space = 0;
+                if (meta_offset >= new_meta_len)
+                {
+                    fprintf(stderr, "Metadata doesn't fit into the new area (total used space: %ju, minimum free space in block: %u/%u)\n",
+                        total_used_space, dsk.meta_block_target_free_space, new_meta_hdr->meta_block_size);
+                    goto close_err;
+                }
+            }
+            heap_object_t *obj = (heap_object_t*)(new_meta_buf + meta_offset + used_space);
+            obj->size = sizeof(heap_object_t);
+            obj->write_pos = sizeof(heap_object_t);
+            obj->entry_type = BS_HEAP_OBJECT;
+            obj->inode = oid.inode;
+            obj->stripe = oid.stripe;
+            heap_write_t *wr = obj->get_writes();
+            wr->next_pos = 0;
             wr->entry_type = BS_HEAP_BIG_WRITE|BS_HEAP_STABLE;
             wr->lsn = ++next_lsn;
             wr->version = sscanf_json(NULL, meta_entry["version"]);
@@ -794,24 +803,17 @@ close_err:
             fromhexstr(meta_entry["ext_bitmap"].string_value(), new_clean_entry_bitmap_size, wr->get_ext_bitmap(&heap));
             if (new_meta_hdr->data_csum_type != 0)
                 fromhexstr(meta_entry["data_csum"].string_value(), new_data_csum_size, wr->get_checksums(&heap));
-            wr->crc32c = wr->calc_crc32c();
-            assert((uint8_t*)wr + wr->size == new_meta_buf + meta_offset + used_space);
-            auto j_it = journal_by_object.find(oid);
             if (j_it != journal_by_object.end())
             {
                 for (auto & rec: j_it->second)
                 {
-                    uint64_t wr_offset = rec["offset"].uint64_value();
-                    uint64_t wr_len = rec["len"].uint64_value();
-                    if (!(wr = get_wr(rec["type"] == "small_write" || rec["type"] == "small_write_instant"
-                        ? heap.get_small_entry_size(wr_offset, wr_len) : heap.get_big_entry_size())))
-                    {
-                        goto close_err;
-                    }
-                    wr->inode = oid.inode;
-                    wr->stripe = oid.stripe;
+                    wr->next_pos = wr->get_size(&heap);
+                    wr = wr->next();
+                    wr->next_pos = 0;
                     wr->lsn = ++next_lsn;
                     wr->version = rec["ver"].uint64_value();
+                    uint64_t wr_offset = rec["offset"].uint64_value();
+                    uint64_t wr_len = rec["len"].uint64_value();
                     if (rec["type"] == "small_write" || rec["type"] == "small_write_instant")
                     {
                         if (wr_len > 0 && !rec["data"].is_string())
@@ -860,20 +862,20 @@ close_err:
                         assert(0);
                     }
                     wr->size = wr->get_size(&heap);
-                    wr->crc32c = wr->calc_crc32c();
-                    assert((uint8_t*)wr + wr->size == new_meta_buf + meta_offset + used_space);
                 }
             }
+            obj->crc32c = obj->calc_crc32c();
+            assert(((uint8_t*)wr + wr->size - (uint8_t*)obj) == space_for_this);
+            used_space += space_for_this;
+            total_used_space += space_for_this;
+        }
+        if (used_space > 0 && used_space < new_meta_hdr->meta_block_size-2)
+        {
+            *((uint16_t*)(new_meta_buf + meta_offset + used_space)) = FREE_SPACE_BIT | (uint16_t)(new_meta_hdr->meta_block_size-used_space);
         }
     }
-    while (meta_offset < new_meta_len)
-    {
-        heap.fill_block_empty_space(new_meta_buf + meta_offset, used_space);
-        meta_offset += dsk.meta_block_size;
-        used_space = 0;
-    }
     int r = resize_write_new_meta();
-    if (new_journal_buf && r == 0)
+    if (r == 0)
     {
         r = resize_write_new_journal();
     }

src/disk_tool/disk_tool_prepare.cpp

@@ -6,25 +6,6 @@
 #include "json_util.h"
 #include "osd_id.h"
 
-void disk_tool_t::parse_meta_reserve()
-{
-    meta_reserve_multiple = 2;
-    meta_reserve_min_size = (uint64_t)1024*1024*1024;
-    if (options.find("meta_reserve") != options.end())
-    {
-        int p1 = options["meta_reserve"].find("x"), p2 = options["meta_reserve"].find(",");
-        if (p1 >= 0 && p2 >= 0)
-        {
-            sscanf(options["meta_reserve"].c_str()+(p1 < p2 ? 0 : p2), "%lf", &meta_reserve_multiple);
-            meta_reserve_min_size = parse_size(options["meta_reserve"].substr(p1 < p2 ? p2 : 0, p1 < p2 ? options["meta_reserve"].size()-p2 : p2));
-        }
-        else if (p1 >= 0)
-            sscanf(options["meta_reserve"].c_str(), "%lf", &meta_reserve_multiple);
-        else
-            meta_reserve_min_size = parse_size(options["meta_reserve"]);
-    }
-}
-
 int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_hdd, json11::Json::object & result)
 {
     static const char *allow_additional_params[] = {
@@ -51,8 +32,6 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
         "discard_on_start",
         "min_discard_size",
         "discard_granularity",
-        "atomic_write_size",
-        "use_atomic_flag",
     };
     if (options.find("force") == options.end())
     {
@@ -78,36 +57,6 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
             if (check_existing_partition(dev) != 0)
                 return 1;
         }
-        if (options.find("weight") != options.end())
-        {
-            double reweight = json11::Json(options["weight"]).number_value();
-            if (reweight < 0 || reweight > 1)
-            {
-                fprintf(stderr, "OSD weight must be between 0 and 1\n");
-                return 1;
-            }
-        }
-    }
-    if (options.find("atomic_write_size") == options.end())
-    {
-        auto data_dev = realpath_str(options["data_device"], false);
-        if (data_dev.substr(0, 9) == "/dev/nvme")
-        {
-            uint64_t atomic_write_size = get_atomic_write_size(data_dev);
-            if (atomic_write_size > 4096)
-            {
-                // FIXME: Enable use_atomic_flag when the kernel does checks correctly
-                fprintf(stderr, "Data device %s supports atomic writes up to %ju bytes, enabling. Enjoy faster writes!\n",
-                    data_dev.c_str(), atomic_write_size);
-                options["atomic_write_size"] = std::to_string(atomic_write_size);
-                if (!atomic_warned)
-                {
-                    fprintf(stderr, "WARNING: RWF_ATOMIC can't be used because Linux checks atomic writes incorrectly.\n"
-                        " Please don't change scheduler from default 'none' and check use_atomic_flag documentation for more details.\n");
-                    atomic_warned = true;
-                }
-            }
-        }
     }
     for (auto dev: std::vector<std::string>{"data", "meta", "journal"})
     {
@@ -179,19 +128,9 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
         dsk.open_data();
         dsk.open_meta();
         dsk.open_journal();
-        dsk.calc_lengths(true);
+        dsk.calc_lengths();
         if (dsk.data_device == dsk.meta_device && !new_meta_len)
-        {
-            uint64_t new_meta_len = dsk.min_meta_len;
-            if (dsk.meta_format == BLOCKSTORE_META_FORMAT_HEAP)
-            {
-                new_meta_len = dsk.min_meta_len*meta_reserve_multiple;
-                if (new_meta_len < meta_reserve_min_size)
-                    new_meta_len = meta_reserve_min_size;
-                new_meta_len = (new_meta_len + dsk.meta_block_size-1) & ~((uint64_t)dsk.meta_block_size-1);
-            }
-            dsk.data_offset += new_meta_len;
-        }
+            dsk.data_offset += (dsk.meta_format == BLOCKSTORE_META_FORMAT_HEAP ? dsk.min_meta_len*2 : dsk.min_meta_len);
         dsk.meta_area_size = (dsk.data_device == dsk.meta_device ? dsk.data_offset : dsk.meta_device_size) - dsk.meta_offset;
         sb = json11::Json::object {
             { "meta_format", options["meta_format"] },
@@ -250,28 +189,9 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
         return 1;
     }
     sb["osd_num"] = osd_num;
-    if (options.find("weight") != options.end() || options.find("tags") != options.end())
-    {
-        std::vector<std::string> cmd = { "vitastor-cli", "modify-osd", std::to_string(osd_num) };
-        if (options.find("weight") != options.end())
-        {
-            cmd.push_back("--reweight");
-            cmd.push_back(options["weight"]);
-        }
-        if (options.find("tags") != options.end())
-        {
-            cmd.push_back("--tags");
-            cmd.push_back(options["tags"]);
-        }
-        if (shell_exec(cmd, "", NULL, NULL) != 0)
-        {
-            fprintf(stderr, "Failed to modify OSD %ju tags and/or reweight\n", osd_num);
-            return 1;
-        }
-    }
-    // Zero out the first block of metadata and journal - OSD will zero the rest on the first run
-    if (write_zero(dsk.meta_fd, sb["meta_offset"].uint64_value(), 4096) != 0 ||
-        write_zero(dsk.journal_fd, sb["journal_offset"].uint64_value(), 4096) != 0)
+    // Zero out metadata and journal
+    if (write_zero(dsk.meta_fd, sb["meta_offset"].uint64_value(), dsk.meta_area_size) != 0 ||
+        write_zero(dsk.journal_fd, sb["journal_offset"].uint64_value(), dsk.journal_len) != 0)
     {
         fprintf(stderr, "Failed to zero out metadata or journal: %s\n", strerror(errno));
         dsk.close_all();
@@ -617,7 +537,7 @@ int disk_tool_t::get_meta_partition(std::vector<vitastor_dev_info_t> & ssds, std
         dsk.open_data();
         dsk.open_meta();
         dsk.open_journal();
-        dsk.calc_lengths(true);
+        dsk.calc_lengths();
         dsk.close_all();
         meta_size = dsk.min_meta_len;
     }
@@ -628,10 +548,24 @@ int disk_tool_t::get_meta_partition(std::vector<vitastor_dev_info_t> & ssds, std
         return 1;
     }
     // Leave some extra space for future metadata formats and round metadata area size to multiples of 1 MB
+    uint64_t meta_reserve_multiple = 2, min_meta_size = (uint64_t)1024*1024*1024;
+    if (options.find("meta_reserve") != options.end())
+    {
+        int p1 = options["meta_reserve"].find("x"), p2 = options["meta_reserve"].find(",");
+        if (p1 >= 0 && p2 >= 0)
+        {
+            meta_reserve_multiple = stoull_full(options["meta_reserve"].substr(p1 < p2 ? 0 : p2, p1 - (p1 < p2 ? 0 : p2)));
+            min_meta_size = parse_size(options["meta_reserve"].substr(p1 < p2 ? p2 : 0, p1 < p2 ? options["meta_reserve"].size()-p2 : p2));
+        }
+        else if (p1 >= 0)
+            meta_reserve_multiple = stoull_full(options["meta_reserve"].substr(0, p1));
+        else
+            min_meta_size = parse_size(options["meta_reserve"]);
+    }
     meta_size = ((meta_size+1024*1024-1)/1024/1024)*1024*1024;
     meta_size *= meta_reserve_multiple;
-    if (meta_size < meta_reserve_min_size)
-        meta_size = meta_reserve_min_size;
+    if (meta_size < min_meta_size)
+        meta_size = min_meta_size;
     // Pick an SSD for journal&meta, balancing the number of serviced OSDs across SSDs
     int sel = -1;
     for (int i = 0; i < ssds.size(); i++)
@@ -662,7 +596,6 @@ int disk_tool_t::get_meta_partition(std::vector<vitastor_dev_info_t> & ssds, std
 
 int disk_tool_t::prepare(std::vector<std::string> devices)
 {
-    parse_meta_reserve();
     if (options.find("data_device") != options.end() && options["data_device"] != "")
     {
         if (options.find("hybrid") != options.end() ||
@@ -710,8 +643,6 @@ int disk_tool_t::prepare(std::vector<std::string> devices)
     std::vector<vitastor_dev_info_t> ssds;
     if (options.find("disable_data_fsync") == options.end())
         options["disable_data_fsync"] = "auto";
-    if (options["meta_format"] == "")
-        options["meta_format"] = std::to_string(BLOCKSTORE_META_FORMAT_HEAP);
     if (hybrid)
     {
         if (options.find("disable_meta_fsync") == options.end())

src/disk_tool/disk_tool_resize.cpp

@@ -25,7 +25,6 @@ struct resizer_data_moving_t
 int disk_tool_t::raw_resize()
 {
     int r;
-    parse_meta_reserve();
     // Parse parameters
     r = resize_parse_params();
     if (r != 0)
@@ -38,12 +37,11 @@ int disk_tool_t::raw_resize()
         {
             resize_init(hdr);
         },
-        [this](blockstore_heap_t *heap, heap_entry_t *obj, uint32_t meta_block_num)
+        [this](blockstore_heap_t *heap, heap_object_t *obj, uint32_t meta_block_num)
         {
-            for (auto wr = obj; wr; wr = heap->prev(wr))
+            for (auto wr = obj->get_writes(); wr; wr = wr->next())
             {
-                if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE ||
-                    (wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_INTENT)
+                if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE)
                 {
                     data_alloc->set(wr->big().block_num, true);
                 }
@@ -156,7 +154,6 @@ int disk_tool_t::resize_parse_params()
         ? parse_size(options["new_journal_len"]) : dsk.journal_len;
     new_meta_format = options.find("new_meta_format") != options.end()
         ? stoull_full(options["new_meta_format"]) : 0;
-    skip_obsolete = options.find("skip_obsolete") != options.end();
     if (new_data_len+new_data_offset > dsk.data_device_size)
         new_data_len = dsk.data_device_size-new_data_offset;
     if (new_meta_device == dsk.data_device && new_data_offset < new_meta_offset &&
@@ -226,28 +223,17 @@ void disk_tool_t::resize_init(blockstore_meta_header_v3_t *hdr)
     new_data_csum_size = (dsk.data_csum_type
         ? ((dsk.data_block_size+dsk.csum_block_size-1)/dsk.csum_block_size*(dsk.data_csum_type & 0xFF))
         : 0);
-    if (new_meta_format != BLOCKSTORE_META_FORMAT_HEAP)
+    new_clean_entry_size = new_clean_entry_header_size + 2*new_clean_entry_bitmap_size + new_data_csum_size;
+    new_entries_per_block = dsk.meta_block_size/new_clean_entry_size;
+    uint64_t new_meta_blocks = 1 + (new_data_len/dsk.data_block_size + new_entries_per_block-1) / new_entries_per_block;
+    if (!new_meta_len)
     {
-        new_clean_entry_size = new_clean_entry_header_size + 2*new_clean_entry_bitmap_size + new_data_csum_size;
-        new_entries_per_block = dsk.meta_block_size/new_clean_entry_size;
-        uint64_t new_meta_blocks = 1 + (new_data_len/dsk.data_block_size + new_entries_per_block-1) / new_entries_per_block;
-        if (!new_meta_len)
-        {
-            new_meta_len = dsk.meta_block_size*new_meta_blocks;
-        }
-        if (new_meta_len < dsk.meta_block_size*new_meta_blocks)
-        {
-            fprintf(stderr, "New metadata area size is too small, should be at least %ju bytes\n", dsk.meta_block_size*new_meta_blocks);
-            exit(1);
-        }
+        new_meta_len = dsk.meta_block_size*new_meta_blocks;
     }
-    else
+    if (new_meta_len < dsk.meta_block_size*new_meta_blocks)
     {
-        new_clean_entry_size = new_entries_per_block = 0;
-        if (!new_meta_len)
-        {
-            new_meta_len = dsk.meta_area_size;
-        }
+        fprintf(stderr, "New metadata area size is too small, should be at least %ju bytes\n", dsk.meta_block_size*new_meta_blocks);
+        exit(1);
     }
     // Check that new metadata, journal and data areas don't overlap
     if (new_meta_device == dsk.data_device && new_meta_offset < new_data_offset+new_data_len &&
@@ -531,106 +517,32 @@ int disk_tool_t::resize_write_new_journal()
     return 0;
 }
 
-void disk_tool_t::remap_big_write(heap_entry_t *wr)
+int disk_tool_t::resize_rebuild_meta()
 {
-    uint64_t block_num = wr->big().block_num;
-    auto remap_it = data_remap.find(block_num);
-    if (remap_it != data_remap.end())
-        block_num = remap_it->second;
-    if (block_num < free_first || block_num >= total_blocks-free_last)
+    if (new_meta_format == BLOCKSTORE_META_FORMAT_HEAP)
     {
-        fprintf(stderr, "BUG: remapped block %ju not in range %ju..%ju\n", block_num, free_first, total_blocks-free_last);
-        exit(1);
-    }
-    block_num += data_idx_diff;
-    wr->big().block_num = block_num;
-    wr->crc32c = wr->calc_crc32c();
-}
-
-void disk_tool_t::remap_small_write(heap_entry_t *wr)
-{
-    if (new_meta_format == BLOCKSTORE_META_FORMAT_HEAP && wr->small().len > 0)
-    {
-        if (new_journal_ptr-new_journal_buf+wr->small().len > new_journal_len)
-        {
-            fprintf(stderr, "Small write data doesn't fit into the new buffer area\n");
-            exit(1);
-        }
-        memcpy(new_journal_ptr, buffer_area+wr->small().location, wr->small().len);
-        wr->small().location = new_journal_ptr-new_journal_buf;
-        new_journal_ptr += wr->small().len;
-        wr->crc32c = wr->calc_crc32c();
-    }
-}
-
-void disk_tool_t::fill_old_clean_entry(blockstore_heap_t *heap, heap_entry_t *big_wr)
-{
-    uint64_t block_num = big_wr->big().block_num;
-    clean_disk_entry *new_entry = (clean_disk_entry*)(new_meta_buf + dsk.meta_block_size +
-        dsk.meta_block_size*(block_num / new_entries_per_block) +
-        new_clean_entry_size*(block_num % new_entries_per_block));
-    new_entry->oid = (object_id){ .inode = big_wr->inode, .stripe = big_wr->stripe };
-    new_entry->version = big_wr->version;
-    memcpy(new_entry->bitmap, big_wr->get_ext_bitmap(heap), new_clean_entry_bitmap_size);
-    memcpy(new_entry->bitmap + new_clean_entry_bitmap_size, big_wr->get_int_bitmap(heap), new_clean_entry_bitmap_size);
-    memcpy(new_entry->bitmap + 2*new_clean_entry_bitmap_size, big_wr->get_checksums(heap), new_data_csum_size);
-    uint32_t *new_entry_csum = (uint32_t*)(((uint8_t*)new_entry) + new_clean_entry_size - 4);
-    *new_entry_csum = crc32c(0, new_entry, new_clean_entry_size - 4);
-}
-
-void disk_tool_t::fill_old_journal_entry(blockstore_heap_t *heap, heap_entry_t *wr)
-{
-    assert(wr->type() == BS_HEAP_SMALL_WRITE ||
-        wr->type() == BS_HEAP_BIG_WRITE ||
-        wr->type() == BS_HEAP_BIG_INTENT);
-    uint32_t je_size = ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_SMALL_WRITE
-        ? sizeof(journal_entry_small_write) + dsk.dirty_dyn_size(wr->small().offset, wr->small().len)
-        : sizeof(journal_entry_big_write) + dsk.dirty_dyn_size(0, dsk.data_block_size));
-    choose_journal_block(je_size);
-    journal_entry *je = (journal_entry*)(new_journal_ptr + new_journal_in_pos);
-    je->magic = JOURNAL_MAGIC;
-    je->type = (wr->entry_type & BS_HEAP_STABLE) ? JE_SMALL_WRITE_INSTANT : JE_SMALL_WRITE;
-    je->size = je_size;
-    je->crc32_prev = new_crc32_prev;
-    je->small_write.oid = (object_id){ .inode = wr->inode, .stripe = wr->stripe };
-    je->small_write.version = wr->version;
-    if (wr->type() == BS_HEAP_SMALL_WRITE)
-    {
-        je->small_write.offset = wr->small().offset;
-        je->small_write.len = wr->small().len;
-        je->small_write.data_offset = new_journal_data-new_journal_buf;
-        if (je->small_write.data_offset + je->small_write.len > new_journal_len)
-        {
-            fprintf(stderr, "Error: live entries don't fit to the new journal\n");
-            exit(1);
-        }
-        memcpy(new_journal_data, buffer_area+wr->small().location, je->small_write.len);
-        new_journal_data += je->small_write.len;
-        if (dsk.data_csum_type == 0 && wr->get_checksum(heap))
-            je->small_write.crc32_data = *wr->get_checksum(heap);
+        new_dsk = dsk;
+        new_dsk.data_offset = new_data_offset;
+        new_dsk.data_len = new_data_len;
+        new_dsk.block_count = new_data_len / dsk.data_block_size;
+        new_dsk.journal_device = new_journal_device;
+        new_dsk.journal_offset = new_journal_offset;
+        new_dsk.journal_len = new_journal_len;
+        new_dsk.meta_device = new_meta_device;
+        new_dsk.meta_offset = new_meta_offset;
+        new_dsk.meta_area_size = new_meta_len;
+        new_dsk.meta_format = new_meta_format;
+        new_heap = new blockstore_heap_t(&new_dsk, NULL, 0);
+        new_meta_hdr = (blockstore_meta_header_v3_t *)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size);
+        memset(new_meta_hdr, 0, dsk.meta_block_size);
     }
     else
     {
-        je->big_write.location = wr->big_location(heap);
+        new_meta_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_meta_len);
+        memset(new_meta_buf, 0, new_meta_len);
+        new_meta_hdr = (blockstore_meta_header_v3_t *)new_meta_buf;
     }
-    memcpy((uint8_t*)je + je->size, wr->get_ext_bitmap(heap), new_clean_entry_bitmap_size);
-    if (dsk.data_csum_type != 0 && wr->get_checksums(heap))
-    {
-        memcpy((uint8_t*)je + je->size + new_clean_entry_bitmap_size, wr->get_checksums(heap), heap->get_csum_size(wr));
-    }
-    je->crc32 = je_crc32(je);
-    new_journal_in_pos += je->size;
-    new_crc32_prev = je->crc32;
-}
-
-int disk_tool_t::resize_rebuild_meta()
-{
-    new_meta_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_meta_len);
-    memset(new_meta_buf, 0, new_meta_len);
-    new_meta_hdr = (blockstore_meta_header_v3_t *)new_meta_buf;
-    uint64_t new_meta_pos = dsk.meta_block_size;
-    uint64_t next_lsn = 0;
-    std::vector<heap_entry_t*> writes;
+    std::vector<heap_write_t*> writes;
     int r = process_meta(
         [&](blockstore_meta_header_v3_t *hdr)
         {
@@ -642,8 +554,7 @@ int disk_tool_t::resize_rebuild_meta()
             new_meta_hdr->bitmap_granularity = dsk.bitmap_granularity ? dsk.bitmap_granularity : 4096;
             new_meta_hdr->data_csum_type = dsk.data_csum_type;
             new_meta_hdr->csum_block_size = dsk.csum_block_size;
-            new_meta_hdr->completed_lsn = hdr->completed_lsn;
-            new_meta_hdr->meta_area_size = new_meta_len;
+            new_meta_hdr->compacted_lsn = hdr->compacted_lsn;
             new_meta_hdr->header_csum = 0;
             new_meta_hdr->header_csum = crc32c(0, new_meta_hdr, new_meta_hdr->version == BLOCKSTORE_META_FORMAT_HEAP
                 ? sizeof(blockstore_meta_header_v3_t) : sizeof(blockstore_meta_header_v2_t));
@@ -652,99 +563,118 @@ int disk_tool_t::resize_rebuild_meta()
                 build_journal_start();
             }
         },
-        [&](blockstore_heap_t *heap, heap_entry_t *obj, uint32_t meta_block_num)
+        [&](blockstore_heap_t *heap, heap_object_t *obj, uint32_t meta_block_num)
         {
-            if (!obj)
+            for (auto wr = obj->get_writes(); wr; wr = wr->next())
             {
-                // Finish
-                if (new_meta_format == BLOCKSTORE_META_FORMAT_HEAP)
+                if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE)
                 {
-                    heap->fill_block_empty_space(new_meta_buf, new_meta_pos);
-                    new_meta_pos = (new_meta_pos/dsk.meta_block_size + 1) * dsk.meta_block_size;
-                    while (new_meta_pos < new_meta_len)
+                    uint64_t block_num = wr->big().block_num;
+                    auto remap_it = data_remap.find(block_num);
+                    if (remap_it != data_remap.end())
+                        block_num = remap_it->second;
+                    if (block_num < free_first || block_num >= total_blocks-free_last)
                     {
-                        heap->fill_block_empty_space(new_meta_buf, new_meta_pos);
-                        new_meta_pos += dsk.meta_block_size;
+                        fprintf(stderr, "BUG: remapped block %ju not in range %ju..%ju\n", block_num, free_first, total_blocks-free_last);
+                        exit(1);
                     }
+                    block_num += data_idx_diff;
+                    wr->big().block_num = block_num;
                 }
-                return;
-            }
-            auto handle_write = [&](heap_entry_t *wr, bool stable)
-            {
-                if (wr->type() == BS_HEAP_BIG_WRITE || wr->type() == BS_HEAP_BIG_INTENT)
+                else if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_SMALL_WRITE)
                 {
-                    remap_big_write(wr);
-                }
-                else if (wr->type() == BS_HEAP_SMALL_WRITE)
-                {
-                    remap_small_write(wr);
-                }
-                else if (wr->type() != BS_HEAP_DELETE && new_meta_format != BLOCKSTORE_META_FORMAT_HEAP)
-                {
-                    fprintf(stderr, "Object %jx:%jx can't be converted to the old format because it contains an entry of type 0x%x%s\n",
-                        wr->inode, wr->stripe, wr->entry_type,
-                        (wr->type() == BS_HEAP_INTENT_WRITE ? " (intent_write)" : ""));
-                    exit(1);
-                }
-                if (new_meta_format == BLOCKSTORE_META_FORMAT_HEAP)
-                {
-                    // New -> New
-                    if ((new_meta_pos % dsk.meta_block_size) + wr->size > dsk.meta_block_size)
+                    if (new_heap && wr->small().len > 0)
                     {
-                        heap->fill_block_empty_space(new_meta_buf, new_meta_pos);
-                        new_meta_pos = (new_meta_pos/dsk.meta_block_size + 1) * dsk.meta_block_size;
-                        if (new_meta_pos >= new_meta_len)
+                        if (new_journal_ptr-new_journal_buf+wr->small().len > new_journal_len)
                         {
-                            fprintf(stderr, "New metadata doesn't fit into the provided area\n");
+                            fprintf(stderr, "Small write data doesn't fit into the new buffer area\n");
                             exit(1);
                         }
-                    }
-                    memcpy(new_meta_buf + new_meta_pos, wr, wr->size);
-                    new_meta_pos += wr->size;
-                    if (skip_obsolete && wr->type() == BS_HEAP_BIG_WRITE && stable)
-                    {
-                        // Skip older writes
-                        return false;
+                        memcpy(new_journal_ptr, buffer_area+wr->small().location, wr->small().len);
+                        wr->small().location = new_journal_ptr-new_journal_buf;
+                        new_journal_ptr += wr->small().len;
                     }
                 }
-                else
+                else if (!new_heap)
                 {
-                    // New -> Old
-                    if (wr->type() == BS_HEAP_DELETE && stable)
-                    {
-                        // Object is deleted, skip it
-                        return false;
-                    }
-                    if (wr->type() == BS_HEAP_BIG_WRITE && stable)
-                    {
-                        fill_old_clean_entry(heap, wr);
-                        return false;
-                    }
-                    else
-                    {
-                        writes.push_back(wr);
-                    }
+                    fprintf(stderr, "Object %jx:%jx can't be converted to the old format because it contains %s\n",
+                        obj->inode, obj->stripe, (wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_TOMBSTONE
+                            ? "a tombstone" : ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_INTENT_WRITE ? "an intent_write entry" : "an unknown entry"));
+                    exit(1);
                 }
-                return true;
-            };
-            if (new_meta_format != BLOCKSTORE_META_FORMAT_HEAP || skip_obsolete)
+            }
+            if (new_heap)
             {
-                heap->iterate_with_stable(obj, obj->lsn, handle_write);
+                // New -> New
+                new_heap->copy_object(obj, NULL);
             }
             else
             {
-                for (auto wr = obj; wr; wr = heap->prev(wr))
-                {
-                    handle_write(wr, false);
-                }
-            }
-            if (writes.size())
-            {
-                for (size_t i = writes.size(); i > 0; i--)
-                {
-                    fill_old_journal_entry(heap, writes[i-1]);
-                }
+                // Fill journal
                 writes.clear();
+                for (auto wr = obj->get_writes(); wr; wr = wr->next())
+                {
+                    writes.push_back(wr);
+                }
+                for (ssize_t i = writes.size()-2; i >= 0; i--)
+                {
+                    auto wr = writes[i];
+                    assert((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_SMALL_WRITE || wr->entry_type == BS_HEAP_BIG_WRITE);
+                    uint32_t je_size = ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_SMALL_WRITE
+                        ? sizeof(journal_entry_small_write) + dsk.dirty_dyn_size(wr->small().offset, wr->small().len)
+                        : sizeof(journal_entry_big_write) + dsk.dirty_dyn_size(0, dsk.data_block_size));
+                    choose_journal_block(je_size);
+                    journal_entry *je = (journal_entry*)(new_journal_ptr + new_journal_in_pos);
+                    je->magic = JOURNAL_MAGIC;
+                    je->type = (wr->entry_type & BS_HEAP_STABLE) ? JE_SMALL_WRITE_INSTANT : JE_SMALL_WRITE;
+                    je->size = je_size;
+                    je->crc32_prev = new_crc32_prev;
+                    je->small_write.oid = (object_id){ .inode = obj->inode, .stripe = obj->stripe };
+                    je->small_write.version = wr->version;
+                    if (wr->type() == BS_HEAP_SMALL_WRITE)
+                    {
+                        je->small_write.offset = wr->small().offset;
+                        je->small_write.len = wr->small().len;
+                        je->small_write.data_offset = new_journal_data-new_journal_buf;
+                        if (je->small_write.data_offset + je->small_write.len > new_journal_len)
+                        {
+                            fprintf(stderr, "Error: live entries don't fit to the new journal\n");
+                            exit(1);
+                        }
+                        memcpy(new_journal_data, buffer_area+wr->small().location, je->small_write.len);
+                        new_journal_data += je->small_write.len;
+                        if (dsk.data_csum_type == 0 && wr->get_checksum(heap))
+                            je->small_write.crc32_data = *wr->get_checksum(heap);
+                    }
+                    else
+                    {
+                        je->big_write.location = wr->big_location(heap);
+                    }
+                    memcpy((uint8_t*)je + je->size, wr->get_ext_bitmap(heap), new_clean_entry_bitmap_size);
+                    if (dsk.data_csum_type != 0 && wr->get_checksums(heap))
+                    {
+                        memcpy((uint8_t*)je + je->size + new_clean_entry_bitmap_size, wr->get_checksums(heap), wr->get_csum_size(heap));
+                    }
+                    je->crc32 = je_crc32(je);
+                    new_journal_in_pos += je->size;
+                    new_crc32_prev = je->crc32;
+                }
+                // New -> Old
+                if (writes[writes.size()-1]->entry_type == BS_HEAP_BIG_WRITE|BS_HEAP_STABLE)
+                {
+                    auto big_wr = writes[writes.size()-1];
+                    uint64_t block_num = big_wr->big().block_num;
+                    clean_disk_entry *new_entry = (clean_disk_entry*)(new_meta_buf + dsk.meta_block_size +
+                        dsk.meta_block_size*(block_num / new_entries_per_block) +
+                        new_clean_entry_size*(block_num % new_entries_per_block));
+                    new_entry->oid = (object_id){ .inode = obj->inode, .stripe = obj->stripe };
+                    new_entry->version = big_wr->version;
+                    memcpy(new_entry->bitmap, big_wr->get_ext_bitmap(heap), new_clean_entry_bitmap_size);
+                    memcpy(new_entry->bitmap + new_clean_entry_bitmap_size, big_wr->get_int_bitmap(heap), new_clean_entry_bitmap_size);
+                    memcpy(new_entry->bitmap + 2*new_clean_entry_bitmap_size, big_wr->get_checksums(heap), new_data_csum_size);
+                    uint32_t *new_entry_csum = (uint32_t*)(((uint8_t*)new_entry) + new_clean_entry_size - 4);
+                    *new_entry_csum = crc32c(0, new_entry, new_clean_entry_size - 4);
+                }
             }
         },
         [&](uint64_t block_num, clean_disk_entry *entry, uint8_t *bitmap)
@@ -758,36 +688,22 @@ int disk_tool_t::resize_rebuild_meta()
                 exit(1);
             }
             block_num += data_idx_diff;
-            if (new_meta_format == BLOCKSTORE_META_FORMAT_HEAP)
+            if (new_heap)
             {
                 // Old -> New
-                auto big_entry_size = sizeof(heap_big_write_t) + dsk.clean_entry_bitmap_size*2 +
-                    (!dsk.data_csum_type ? 0 : dsk.data_block_size/dsk.csum_block_size * (dsk.data_csum_type & 0xFF));
-                if ((new_meta_pos % dsk.meta_block_size) + big_entry_size > dsk.meta_block_size)
-                {
-                    new_meta_pos = (new_meta_pos % dsk.meta_block_size) + dsk.meta_block_size;
-                    if (new_meta_pos >= new_meta_len)
-                    {
-                        fprintf(stderr, "New metadata doesn't fit into the provided area\n");
-                        exit(1);
-                    }
-                }
-                heap_entry_t *wr = (heap_entry_t*)(new_meta_buf + new_meta_pos);
-                wr->size = big_entry_size;
+                uint8_t wr_buf[new_heap->get_max_write_entry_size()];
+                heap_write_t *wr = (heap_write_t*)wr_buf;
                 wr->entry_type = BS_HEAP_BIG_WRITE|BS_HEAP_STABLE;
-                wr->inode = entry->oid.inode;
-                wr->stripe = entry->oid.stripe;
-                wr->version = entry->version;
                 wr->big().block_num = block_num;
-                wr->lsn = ++next_lsn;
+                wr->next_pos = 0;
+                wr->size = wr->get_size(new_heap);
                 if (bitmap)
                 {
-                    memcpy(((uint8_t*)wr) + sizeof(heap_big_write_t), bitmap, new_clean_entry_bitmap_size);
-                    memcpy(((uint8_t*)wr) + sizeof(heap_big_write_t) + new_clean_entry_bitmap_size, bitmap+new_clean_entry_bitmap_size, new_clean_entry_bitmap_size);
-                    memcpy(((uint8_t*)wr) + sizeof(heap_big_write_t) + 2*new_clean_entry_bitmap_size, bitmap+2*new_clean_entry_bitmap_size, new_data_csum_size);
+                    memcpy(wr->get_ext_bitmap(new_heap), bitmap, new_clean_entry_bitmap_size);
+                    memcpy(wr->get_int_bitmap(new_heap), bitmap+new_clean_entry_bitmap_size, new_clean_entry_bitmap_size);
+                    memcpy(wr->get_checksums(new_heap), bitmap+2*new_clean_entry_bitmap_size, new_data_csum_size);
                 }
-                wr->crc32c = wr->calc_crc32c();
-                new_meta_pos += wr->size;
+                new_heap->post_write(entry->oid, wr, NULL, NULL);
             }
             else
             {
@@ -819,7 +735,34 @@ int disk_tool_t::resize_write_new_meta()
         return 1;
     }
     lseek64(new_meta_fd, new_meta_offset, 0);
-    write_blocking(new_meta_fd, new_meta_buf, new_meta_len);
+    if (new_meta_buf)
+    {
+        write_blocking(new_meta_fd, new_meta_buf, new_meta_len);
+    }
+    else
+    {
+        assert(new_heap);
+        uint32_t new_meta_blocks = new_meta_len / dsk.meta_block_size - 1;
+        uint8_t *zero_block = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size);
+        memset(zero_block, 0, dsk.meta_block_size);
+        std::vector<iovec> iov;
+        iov.reserve(IOV_MAX);
+        iov.push_back((iovec){ .iov_base = new_meta_hdr, .iov_len = dsk.meta_block_size });
+        for (uint32_t i = 0; i < new_meta_blocks; i++)
+        {
+            uint8_t *data = new_heap->get_meta_block(i);
+            iov.push_back((iovec){ .iov_base = data ? data : zero_block, .iov_len = dsk.meta_block_size });
+            if (iov.size() >= IOV_MAX)
+            {
+                writev_blocking(new_meta_fd, iov.data(), iov.size());
+                iov.clear();
+            }
+        }
+        if (iov.size() > 0)
+            writev_blocking(new_meta_fd, iov.data(), iov.size());
+        free(zero_block);
+        zero_block = NULL;
+    }
     fsync(new_meta_fd);
     close(new_meta_fd);
     new_meta_fd = -1;
@@ -828,6 +771,11 @@ int disk_tool_t::resize_write_new_meta()
 
 void disk_tool_t::free_new_meta()
 {
+    if (new_heap)
+    {
+        delete new_heap;
+        new_heap = NULL;
+    }
     if ((uint8_t*)new_meta_hdr != new_meta_buf)
     {
         free(new_meta_hdr);

src/disk_tool/disk_tool_resize_auto.cpp

@@ -8,7 +8,6 @@
 
 int disk_tool_t::resize_data(std::string device)
 {
-    parse_meta_reserve();
     if (options.find("move_journal") == options.end() &&
         options.find("move_data") == options.end() &&
         options.find("journal_size") == options.end() &&
@@ -62,7 +61,7 @@ int disk_tool_t::resize_data(std::string device)
         dsk.journal_fd = old_journal_fd;
         dsk.meta_fd = old_meta_fd;
         dsk.data_fd = old_data_fd;
-        dsk.calc_lengths(true);
+        dsk.calc_lengths();
         dsk.journal_fd = -1;
         dsk.meta_fd = -1;
         dsk.data_fd = -1;
@@ -84,8 +83,7 @@ int disk_tool_t::resize_data(std::string device)
         ? move_options["new_meta_device"] : dsk.meta_device;
     // Calculate new data & meta offsets
     if (!new_meta_len)
-        new_meta_len = (dsk.meta_format == BLOCKSTORE_META_FORMAT_HEAP ? dsk.meta_area_size : dsk.min_meta_len);
-    move_options["new_meta_len"] = std::to_string(new_meta_len);
+        new_meta_len = (dsk.meta_format == BLOCKSTORE_META_FORMAT_HEAP ? dsk.min_meta_len*2 : dsk.min_meta_len);
     new_data_offset = 4096 + (new_journal_device == dsk.data_device ? new_journal_len : 0) +
         (new_meta_device == dsk.data_device ? new_meta_len : 0);
     new_data_offset += ((dsk.data_offset-new_data_offset) % dsk.data_block_size);

src/disk_tool/disk_tool_udev.cpp

@@ -367,19 +367,6 @@ int disk_tool_t::pre_exec_osd(std::string device)
         {
             return 1;
         }
-        if (sb["params"]["atomic_write_size"].uint64_value() > 4096 &&
-            sb["params"]["use_atomic_flag"].bool_value())
-        {
-            uint64_t atomic_write_size = get_atomic_write_size(sb["real_data_device"].string_value());
-            if (atomic_write_size < sb["params"]["atomic_write_size"].uint64_value())
-            {
-                fprintf(stderr, "Atomic write size is set to %ju in the OSD superblock but data device %s only supports %ju."
-                    " Did you enable IOMMU? Linux has a hardcoded max_hw_sectors_kb value for NVMe drives.\n",
-                    sb["params"]["atomic_write_size"].uint64_value(),
-                    sb["real_data_device"].string_value().c_str(), atomic_write_size);
-                return 1;
-            }
-        }
         if (json_is_true(sb["params"]["disable_meta_fsync"]) &&
             sb["real_meta_device"].string_value() != "" && sb["real_meta_device"] != sb["real_data_device"] &&
             check_disabled_cache(sb["real_meta_device"].string_value()) != 0)

src/disk_tool/disk_tool_upgrade.cpp

@@ -107,7 +107,7 @@ int disk_tool_t::upgrade_simple_unit(std::string unit)
                 dsk.open_data();
                 dsk.open_meta();
                 dsk.open_journal();
-                dsk.calc_lengths(true);
+                dsk.calc_lengths();
                 dsk.close_all();
             }
             catch (std::exception & e)

src/disk_tool/disk_tool_utils.cpp

@@ -56,16 +56,8 @@ static int check_queue_cache(std::string dev, std::string parent_dev)
     return trim(r) == "write through" ? 0 : -1;
 }
 
-uint64_t get_atomic_write_size(const std::string & dev)
-{
-    auto parent_dev = get_parent_device(dev);
-    if (parent_dev == "")
-        return 0;
-    return stoull_full(trim(read_file("/sys/block/"+parent_dev.substr(5)+"/queue/atomic_write_max_bytes")));
-}
-
 // returns 1 = warning, -1 = error, 0 = success
-int disable_cache(const std::string & dev)
+int disable_cache(std::string dev)
 {
     auto parent_dev = get_parent_device(dev);
     if (parent_dev == "")

src/kv/kv_cli.cpp

@@ -97,7 +97,6 @@ void kv_cli_t::parse_args(int narg, const char *args[])
                 "  dump [<start> [end]]\n"
                 "  dumpjson [<start> [end]]\n"
                 "  loadjson\n"
-                "  rescue\n"
                 "\n"
                 "<IMAGE> should be the name of Vitastor image with the DB.\n"
                 "Without <COMMAND>, you get an interactive DB shell.\n"
@@ -299,50 +298,6 @@ struct kv_cli_list_t
     int n = 0;
     std::function<void(int)> cb;
 
-    void handle_key(int res, const std::string & key, const std::string & value)
-    {
-        if (res < 0)
-        {
-            if (res != -ENOENT)
-                fprintf(stderr, "Error: %s (code %d)\n", strerror(-res), res);
-            if (format == 2)
-                write("\n}\n");
-            if (handle)
-                db->list_close(handle);
-            flush();
-            cb(res == -ENOENT ? 0 : res);
-            delete this;
-        }
-        else
-        {
-            if (format == 2)
-            {
-                write(n ? ",\n  " : "{\n  ");
-                write(addslashes(key));
-                write(": ");
-                write(addslashes(value));
-            }
-            else if (format == 1)
-            {
-                write("set ");
-                write(auto_addslashes(key));
-                write(" ");
-                write(value);
-                write("\n");
-            }
-            else
-            {
-                write(key);
-                write(" = ");
-                write(value);
-                write("\n");
-            }
-            n++;
-            if (handle)
-                db->list_next(handle, NULL);
-        }
-    }
-
     void write(const std::string & str)
     {
         if (buf.capacity() < KV_LIST_BUF_SIZE)
@@ -361,7 +316,6 @@ struct kv_cli_list_t
             if (res > 0)
                 done += res;
         }
-        buf.clear();
     }
 };
 
@@ -670,18 +624,44 @@ void kv_cli_t::handle_cmd(const std::vector<std::string> & cmd, std::function<vo
         lst->cb = std::move(cb);
         db->list_next(lst->handle, [lst](int res, const std::string & key, const std::string & value)
         {
-            lst->handle_key(res, key, value);
-        });
-    }
-    else if (opname == "rescue")
-    {
-        kv_cli_list_t *lst = new kv_cli_list_t;
-        lst->db = db;
-        lst->format = 2;
-        lst->cb = std::move(cb);
-        db->rescue([lst](int res, const std::string & key, const std::string & value)
-        {
-            lst->handle_key(res, key, value);
+            if (res < 0)
+            {
+                if (res != -ENOENT)
+                    fprintf(stderr, "Error: %s (code %d)\n", strerror(-res), res);
+                if (lst->format == 2)
+                    lst->write("\n}\n");
+                lst->flush();
+                lst->db->list_close(lst->handle);
+                lst->cb(res == -ENOENT ? 0 : res);
+                delete lst;
+            }
+            else
+            {
+                if (lst->format == 2)
+                {
+                    lst->write(lst->n ? ",\n  " : "{\n  ");
+                    lst->write(addslashes(key));
+                    lst->write(": ");
+                    lst->write(addslashes(value));
+                }
+                else if (lst->format == 1)
+                {
+                    lst->write("set ");
+                    lst->write(auto_addslashes(key));
+                    lst->write(" ");
+                    lst->write(value);
+                    lst->write("\n");
+                }
+                else
+                {
+                    lst->write(key);
+                    lst->write(" = ");
+                    lst->write(value);
+                    lst->write("\n");
+                }
+                lst->n++;
+                lst->db->list_next(lst->handle, NULL);
+            }
         });
     }
     else if (opname == "loadjson")