Do not expect second notification when cqe returns error

2025-11-16 02:19:48 +03:00
131 changed files with 6614 additions and 20617 deletions
@@ -306,78 +306,6 @@ jobs:
          echo ""
        done
  test_old_interrupted_rebalance:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 10
      run: OLD=1 /root/vitastor/tests/test_interrupted_rebalance.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_old_interrupted_rebalance_imm:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 10
      run: OLD=1 IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_interrupted_rebalance.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_old_interrupted_rebalance_ec:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 10
      run: OLD=1 SCHEME=ec /root/vitastor/tests/test_interrupted_rebalance.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_old_interrupted_rebalance_ec_imm:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 10
      run: OLD=1 SCHEME=ec IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_interrupted_rebalance.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_create_halfhost:
    runs-on: ubuntu-latest
    needs: build
@@ -450,42 +378,6 @@ jobs:
          echo ""
        done
  test_old_snapshot:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 3
      run: OLD=1 /root/vitastor/tests/test_snapshot.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_old_snapshot_ec:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 3
      run: OLD=1 SCHEME=ec /root/vitastor/tests/test_snapshot.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_minsize_1:
    runs-on: ubuntu-latest
    needs: build
@@ -522,24 +414,6 @@ jobs:
          echo ""
        done
  test_old_move_reappear:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 3
      run: OLD=1 /root/vitastor/tests/test_move_reappear.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_degraded:
    runs-on: ubuntu-latest
    needs: build
@@ -630,42 +504,6 @@ jobs:
          echo ""
        done
  test_old_snapshot_chain:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 3
      run: OLD=1 /root/vitastor/tests/test_snapshot_chain.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_old_snapshot_chain_ec:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 3
      run: OLD=1 SCHEME=ec /root/vitastor/tests/test_snapshot_chain.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_snapshot_down:
    runs-on: ubuntu-latest
    needs: build
@@ -702,78 +540,6 @@ jobs:
          echo ""
        done
  test_old_snapshot_down:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 3
      run: OLD=1 /root/vitastor/tests/test_snapshot_down.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_old_snapshot_down_ec:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 3
      run: OLD=1 SCHEME=ec /root/vitastor/tests/test_snapshot_down.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_kv_stress:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 3
      run: /root/vitastor/tests/test_kv_stress.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_kv_stress_imm:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 3
      run: IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_kv_stress.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_splitbrain:
    runs-on: ubuntu-latest
    needs: build
@@ -864,78 +630,6 @@ jobs:
          echo ""
        done
  test_old_rebalance_verify:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 10
      run: OLD=1 /root/vitastor/tests/test_rebalance_verify.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_old_rebalance_verify_imm:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 10
      run: OLD=1 IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_rebalance_verify.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_old_rebalance_verify_ec:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 10
      run: OLD=1 SCHEME=ec /root/vitastor/tests/test_rebalance_verify.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_old_rebalance_verify_ec_imm:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 10
      run: OLD=1 SCHEME=ec IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_rebalance_verify.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_dd:
    runs-on: ubuntu-latest
    needs: build
@@ -1044,7 +738,7 @@ jobs:
          echo ""
        done
-  test_old_write:
+  test_write_no_same:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
@@ -1052,61 +746,7 @@ jobs:
    - name: Run test
      id: test
      timeout-minutes: 3
-      run: OLD=1 /root/vitastor/tests/test_write.sh
+      run: /root/vitastor/tests/test_write_no_same.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_old_write_xor:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 3
      run: OLD=1 SCHEME=xor /root/vitastor/tests/test_write.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_write_old_iothreads:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 3
      run: TEST_NAME=old_iothreads OLD=1 GLOBAL_CONFIG=',"client_iothread_count":4' /root/vitastor/tests/test_write.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_old_write_no_same:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 3
      run: OLD=1 /root/vitastor/tests/test_write_no_same.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
@@ -1386,42 +1026,6 @@ jobs:
          echo ""
        done
  test_old_resize:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 3
      run: OLD=1 /root/vitastor/tests/test_resize.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_old_resize_auto:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 3
      run: OLD=1 /root/vitastor/tests/test_resize_auto.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_osd_tags:
    runs-on: ubuntu-latest
    needs: build
@@ -1512,78 +1116,6 @@ jobs:
          echo ""
        done
  test_old_enospc:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 3
      run: OLD=1 /root/vitastor/tests/test_enospc.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_old_enospc_xor:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 3
      run: OLD=1 SCHEME=xor /root/vitastor/tests/test_enospc.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_old_enospc_imm:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 3
      run: OLD=1 IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_enospc.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_old_enospc_imm_xor:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 3
      run: OLD=1 IMMEDIATE_COMMIT=1 SCHEME=xor /root/vitastor/tests/test_enospc.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_scrub:
    runs-on: ubuntu-latest
    needs: build
@@ -1692,222 +1224,6 @@ jobs:
          echo ""
        done
  test_old_scrub:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 3
      run: OLD=1 /root/vitastor/tests/test_scrub.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_old_scrub_zero_osd_2:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 3
      run: OLD=1 ZERO_OSD=2 /root/vitastor/tests/test_scrub.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_old_scrub_xor:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 3
      run: OLD=1 SCHEME=xor /root/vitastor/tests/test_scrub.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_old_scrub_pg_size_3:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 3
      run: OLD=1 PG_SIZE=3 /root/vitastor/tests/test_scrub.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_old_scrub_pg_size_6_pg_minsize_4_osd_count_6_ec:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 3
      run: OLD=1 PG_SIZE=6 PG_MINSIZE=4 OSD_COUNT=6 SCHEME=ec /root/vitastor/tests/test_scrub.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_old_scrub_ec:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 3
      run: OLD=1 SCHEME=ec /root/vitastor/tests/test_scrub.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_heal_old_csum_32k_dmj:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 10
      run: TEST_NAME=old_csum_32k_dmj OLD=1 OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k --inmemory_metadata false --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_heal_old_csum_32k_dj:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 10
      run: TEST_NAME=old_csum_32k_dj  OLD=1 OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_heal_old_csum_32k:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 10
      run: TEST_NAME=old_csum_32k     OLD=1 OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_heal_old_csum_4k_dmj:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 10
      run: TEST_NAME=old_csum_4k_dmj  OLD=1 OSD_ARGS="--data_csum_type crc32c --inmemory_metadata false --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_heal_old_csum_4k_dj:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 10
      run: TEST_NAME=old_csum_4k_dj   OLD=1 OSD_ARGS="--data_csum_type crc32c --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_heal_old_csum_4k:
    runs-on: ubuntu-latest
    needs: build
    container: ${{env.TEST_IMAGE}}:${{github.sha}}
    steps:
    - name: Run test
      id: test
      timeout-minutes: 10
      run: TEST_NAME=old_csum_4k      OLD=1 OSD_ARGS="--data_csum_type crc32c" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
    - name: Print logs
      if: always() && steps.test.outcome == 'failure'
      run: |
        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
          echo "-------- $i --------"
          cat $i
          echo ""
        done
  test_nfs:
    runs-on: ubuntu-latest
    needs: build
@@ -38,10 +38,6 @@ for my $line (<>)
            {
                $test_name .= '_antietcd';
            }
            elsif ($1 eq 'OLD')
            {
                $test_name =~ s/^test_/test_old_/s;
            }
            else
            {
                $test_name .= '_'.lc($1).'_'.$2;
@@ -2,19 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 project(vitastor)
-set(VITASTOR_VERSION "3.0.0")
+set(VITASTOR_VERSION "2.4.3")
 include(CTest)
 add_custom_target(build_tests)
 add_custom_target(test
 	COMMAND
 	echo leak:tcmalloc > ${CMAKE_CURRENT_BINARY_DIR}/lsan-suppress.txt &&
 	env LSAN_OPTIONS=suppressions=${CMAKE_CURRENT_BINARY_DIR}/lsan-suppress.txt ${CMAKE_CTEST_COMMAND}
 )
 # make -j16 -C ../../build test_heap && ../../build/src/test/test_heap
 # make -j16 -C ../../build test_heap && rm -f $(find ../../build -name '*.gcda') && ctest -V -T test -T coverage -R heap --test-dir ../../build && (cd ../../build; gcovr -f ../src --html --html-nested -o coverage/index.html; cd ../src/test)
 # make -j16 -C ../../build test_blockstore && rm -f $(find ../../build -name '*.gcda') && ctest -V -T test -T coverage -R blockstore --test-dir ../../build && (cd ../../build; gcovr -f ../src --html --html-nested -o coverage/index.html; cd ../src/test)
 # kcov --include-path=../../../src ../../kcov ./test_blockstore
 add_dependencies(test build_tests)
 add_subdirectory(src)
@@ -26,15 +26,11 @@ Vitastor поддерживает QEMU-драйвер, протоколы UBLK,
 ## Презентации и записи докладов
 - KuberConf'2025: [видео](https://vitastor.io/presentation/kuberconf.webm)
 - Highload'2025: [видео](https://vitastor.io/presentation/hl2025/hl2025.webm),
  [на youtube](https://www.youtube.com/watch?v=0R8MLjFtz7g), презентация
  ([на русском](https://vitastor.io/presentation/hl2025/), [на английском](https://vitastor.io/presentation/hl2025/en.html))
 - Highload'2022: презентация ([на русском](https://vitastor.io/presentation/highload/highload.html)),
  [видео](https://vitastor.io/presentation/highload/talk.webm)
 - DevOpsConf'2021: презентация ([на русском](https://vitastor.io/presentation/devopsconf/devopsconf.html),
  [на английском](https://vitastor.io/presentation/devopsconf/devopsconf_en.html)),
  [видео](https://vitastor.io/presentation/devopsconf/talk.webm)
 - Highload'2022: презентация ([на русском](https://vitastor.io/presentation/highload/highload.html)),
  [видео](https://vitastor.io/presentation/highload/talk.webm)
 ## Документация
@@ -26,15 +26,11 @@ Read more details in the documentation. You can start from here: [Quick Start](d
 ## Talks and presentations
 - KuberConf'2025: [video](https://vitastor.io/presentation/kuberconf.webm)
 - Highload'2025: [video](https://vitastor.io/presentation/hl2025/hl2025.webm),
  [youtube](https://www.youtube.com/watch?v=0R8MLjFtz7g), presentation
  ([in Russian](https://vitastor.io/presentation/hl2025/), [in English](https://vitastor.io/presentation/hl2025/en.html))
 - Highload'2022: presentation ([in Russian](https://vitastor.io/presentation/highload/highload.html)),
  [video](https://vitastor.io/presentation/highload/talk.webm)
 - DevOpsConf'2021: presentation ([in Russian](https://vitastor.io/presentation/devopsconf/devopsconf.html),
  [in English](https://vitastor.io/presentation/devopsconf/devopsconf_en.html)),
  [video](https://vitastor.io/presentation/devopsconf/talk.webm)
 - Highload'2022: presentation ([in Russian](https://vitastor.io/presentation/highload/highload.html)),
  [video](https://vitastor.io/presentation/highload/talk.webm)
 ## Documentation
@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.0
+VITASTOR_VERSION ?= v2.4.3
 all: build push
@@ -49,7 +49,7 @@ spec:
            capabilities:
              add: ["SYS_ADMIN"]
            allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v3.0.0
+          image: vitalif/vitastor-csi:v2.4.3
          args:
            - "--node=$(NODE_ID)"
            - "--endpoint=$(CSI_ENDPOINT)"
@@ -121,7 +121,7 @@ spec:
            privileged: true
            capabilities:
              add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v3.0.0
+          image: vitalif/vitastor-csi:v2.4.3
          args:
            - "--node=$(NODE_ID)"
            - "--endpoint=$(CSI_ENDPOINT)"
@@ -5,7 +5,7 @@ package vitastor
 const (
    vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "3.0.0"
+    vitastorCSIDriverVersion = "2.4.3"
 )
 // Config struct fills the parameters of request or user input
@@ -1,4 +1,4 @@
-vitastor (3.0.0-1) unstable; urgency=medium
+vitastor (2.4.3-1) unstable; urgency=medium
  * Bugfixes
@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v3.0.0
+VITASTOR_VERSION ?= v2.4.3
 all: build push
@@ -4,7 +4,7 @@
 #
 # Desired Vitastor version
-VITASTOR_VERSION=v3.0.0
+VITASTOR_VERSION=v2.4.3
 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here
@@ -9,7 +9,6 @@
 These parameters apply to OSDs, are fixed at the moment of OSD drive
 initialization and can't be changed after it without losing data.
 - [meta_format](#meta_format)
 - [data_device](#data_device)
 - [meta_device](#meta_device)
 - [journal_device](#journal_device)
@@ -28,21 +27,6 @@ initialization and can't be changed after it without losing data.
 - [data_csum_type](#data_csum_type)
 - [csum_block_size](#csum_block_size)
 ## meta_format
 - Type: integer
 - Default: 3
 OSD store implementation version and on-disk metadata format.
 Three versions are currently supported: 3, 2 and 1.
 - 3 the new log-structured store, it's overall faster, has lower Write
  Amplification, which may be even close to 1 (i.e. almost no extra writes)
  if your SSDs support atomic writes (see [atomic_write_size](osd.en.md#atomic_write_size)).
 - 2 is the old stable store from Vitastor 0.9-2.x.
 - 1 is the same old store but with a legacy metadata format from Vitastor
  versions to up 0.8.x, without any support for checksums.
 ## data_device
 - Type: string
@@ -10,7 +10,6 @@
 дисковые параметры, задаются в момент инициализации дисков OSD и не могут быть
 изменены после этого без потери данных.
 - [meta_format](#meta_format)
 - [data_device](#data_device)
 - [meta_device](#meta_device)
 - [journal_device](#journal_device)
@@ -29,23 +28,6 @@
 - [data_csum_type](#data_csum_type)
 - [csum_block_size](#csum_block_size)
 ## meta_format
 - Тип: целое число
 - Значение по умолчанию: 3
 Версия реализации дискового хранилища OSD и дискового формата метаданных.
 Поддерживаются три версии: 3, 2 и 1.
 - 3 - новое лог-структурированное хранилище, в целом более быстрое, со
  сниженным фактором амплификации записи, который может составлять около 1
  (то есть, практически без лишней служебной записи), если ваши SSD
  поддерживают атомарную запись (см. [atomic_write_size](osd.ru.md#atomic_write_size)).
 - 2 - старое стабильное хранилище из версий Vitastor 0.9-2.x.
 - 1 - то же самое стабильное хранилище, но с ещё более старым форматом
  метаданных из версий Vitastor до 0.8.x, без какой-либо поддержки
  контрольных сумм.
 ## data_device
 - Тип: строка
@@ -65,8 +65,6 @@ with an OSD restart or, for some of them, even without restarting by updating co
 - [allow_net_split](#allow_net_split)
 - [enable_pg_locks](#enable_pg_locks)
 - [pg_lock_retry_interval_ms](#pg_lock_retry_interval_ms)
 - [atomic_write_size](#atomic_write_size)
 - [use_atomic_flag](#use_atomic_flag)
 ## bind_address
@@ -493,7 +491,7 @@ Can be used to slow down scrubbing if it affects user load too much.
 ## scrub_list_limit
 - Type: integer
- Default: 262144
+- Default: 1000
 - Can be changed online: yes
 Number of objects to list in one listing operation during scrub.
@@ -668,41 +666,3 @@ Use this parameter to enable or disable this function for all pools.
 - Default: 100
 Retry interval for failed PG lock attempts.
 ## atomic_write_size
 - Type: integer
 - Default: 4096
 Maximum data device atomic write size allowed for OSD to use.
 Atomic writes allow to reduce the Write Amplification factor with the new store
 ([meta_format](layout-osd.en.md#meta_format)=3) to almost 1 (i.e. almost no extra writes)
 with replicated pools and reach the best possible write performance.
 Default value is auto-detected during OSD initialization from
 `/sys/block/xx/queue/atomic_write_max_bytes` or assumed to be 4096 bytes
 because all known disks support 4 KB atomic writes.
 You can also check if your NVMe drives support atomic writes by running
 the command `nvme id-ctrl /dev/nvme0n1 | grep awupf`. If the reported value,
 plus 1, multiplied by the currently selected block size of the NVMe,
 is more than 4 KB, then the new store can utilize it for better performance.
 The only drives known to support it currently are [Micron and Kioxia](../intro/quickstart.en.md).
 Atomic writes allow to skip double data writes in replicated pools, thus
 reducing Write Amplification and improving write performance up to 2 times.
 ## use_atomic_flag
 - Type: boolean
 This option controls whether the Vitastor OSD uses RWF_ATOMIC write flag with atomic
 writes. This flag is only supported on Linux kernel since 6.11. Atomic writes are
 generally only safe to use with this flag because it tells the kernel to never fragment
 write requests and also to check the write against the actual atomic write capabilities
 of the device.
 This option is enabled by default when atomic_write_size is set to a value larger than 4 KB.
 You can disable it if you're sure that your disks support atomic writes and you want to
 bypass the Linux atomic write checks.
@@ -66,8 +66,6 @@
 - [allow_net_split](#allow_net_split)
 - [enable_pg_locks](#enable_pg_locks)
 - [pg_lock_retry_interval_ms](#pg_lock_retry_interval_ms)
 - [atomic_write_size](#atomic_write_size)
 - [use_atomic_flag](#use_atomic_flag)
 ## bind_address
@@ -516,7 +514,7 @@ fsync небезопасным даже с режимом "directsync".
 ## scrub_list_limit
 - Тип: целое число
- Значение по умолчанию: 262144
+- Значение по умолчанию: 1000
 - Можно менять на лету: да
 Размер загружаемых за одну операцию списков объектов в процессе фоновой
@@ -701,45 +699,3 @@ pg_minsize OSD во время переключений, что может по
 - Значение по умолчанию: 100
 Интервал повтора неудачных попыток блокировки PG.
 ## atomic_write_size
 - Тип: целое число
 - Значение по умолчанию: 4096
 Максимальный размер атомарной записи на диск данных, который OSD разрешено использовать.
 Поддержка атомарной записи позволяет снизить мультипликатор записи (Write Amplification)
 на диск с новым хранилищем ([meta_format](layout-osd.ru.md#meta_format)=3)
 практически до 1 (то есть, почти до нулевого объёма лишней записи) в реплицированных
 пулах и достигнуть наилучшей возможной производительности записи.
 Значение по умолчанию авто-определяется во время инициализации OSD из
 `/sys/block/xx/queue/atomic_write_max_bytes` либо принимается равным 4096,
 так как все известные диски поддерживают атомарную запись 4 КБ блоков.
 Вы также можете проверить, поддерживают ли ваши NVMe-диски атомарную запись,
 с помощью команды `nvme id-ctrl /dev/nvme0n1 | grep awupf`. Если значение awupf
 плюс 1, умноженное на текущий выбранный размер блока NVMe-диска, больше 4 КБ,
 то новое хранилище может использовать атомарные записи для достижения лучшей
 производительности. Единственные известные диски, которые поддерживают это сейчас -
 [Micron и Kioxia](../intro/quickstart.ru.md).
 Атомарная запись позволяет не использовать двойную запись данных (в журнал и на
 устройство данных) в реплицированных пулах и таким образом снижает амплификацию
 записи (объём служебной записи на диск) и улучшает производительность записи
 вплоть до 2-х кратного прироста.
 ## use_atomic_flag
 - Тип: булево (да/нет)
 Данная опция контролирует использование Vitastor OSD флага RWF_ATOMIC при атомарной записи
 блоков. Этот флаг поддерживается только в ядрах Linux начиная с 6.11. Атомарная запись
 является безопасной только при использовании этого флага, так как он сообщает ядру о том,
 что запрос записи нельзя фрагментировать и о том, что запрос нужно проверить на соответствие
 реальным возможностям атомарной записи устройства.
 Опция включается по умолчанию, когда atomic_write_size устанавливается в значение больше 4 КБ.
 Вы можете явно отключить её, если уверены, что ваши диски поддерживают атомарную запись и
 хотите обойти проверки уровня ядра.
@@ -1,28 +1,3 @@
 - name: meta_format
  type: int
  default: 3
  info: |
    OSD store implementation version and on-disk metadata format.
    Three versions are currently supported: 3, 2 and 1.
    - 3 the new log-structured store, it's overall faster, has lower Write
      Amplification, which may be even close to 1 (i.e. almost no extra writes)
      if your SSDs support atomic writes (see [atomic_write_size](osd.en.md#atomic_write_size)).
    - 2 is the old stable store from Vitastor 0.9-2.x.
    - 1 is the same old store but with a legacy metadata format from Vitastor
      versions to up 0.8.x, without any support for checksums.
  info_ru: |
    Версия реализации дискового хранилища OSD и дискового формата метаданных.
    Поддерживаются три версии: 3, 2 и 1.
    - 3 - новое лог-структурированное хранилище, в целом более быстрое, со
      сниженным фактором амплификации записи, который может составлять около 1
      (то есть, практически без лишней служебной записи), если ваши SSD
      поддерживают атомарную запись (см. [atomic_write_size](osd.ru.md#atomic_write_size)).
    - 2 - старое стабильное хранилище из версий Vitastor 0.9-2.x.
    - 1 - то же самое стабильное хранилище, но с ещё более старым форматом
      метаданных из версий Vitastor до 0.8.x, без какой-либо поддержки
      контрольных сумм.
 - name: data_device
  type: string
  info: |
@@ -566,7 +566,7 @@
    сильно влияет на пользовательскую нагрузку.
 - name: scrub_list_limit
  type: int
-  default: 262144
+  default: 1000
  online: true
  info: |
    Number of objects to list in one listing operation during scrub.
@@ -801,70 +801,3 @@
  default: 100
  info: Retry interval for failed PG lock attempts.
  info_ru: Интервал повтора неудачных попыток блокировки PG.
 - name: atomic_write_size
  type: int
  default: 4096
  info: |
    Maximum data device atomic write size allowed for OSD to use.
    Atomic writes allow to reduce the Write Amplification factor with the new store
    ([meta_format](layout-osd.en.md#meta_format)=3) to almost 1 (i.e. almost no extra writes)
    with replicated pools and reach the best possible write performance.
    Default value is auto-detected during OSD initialization from
    `/sys/block/xx/queue/atomic_write_max_bytes` or assumed to be 4096 bytes
    because all known disks support 4 KB atomic writes.
    You can also check if your NVMe drives support atomic writes by running
    the command `nvme id-ctrl /dev/nvme0n1 | grep awupf`. If the reported value,
    plus 1, multiplied by the currently selected block size of the NVMe,
    is more than 4 KB, then the new store can utilize it for better performance.
    The only drives known to support it currently are [Micron and Kioxia](../intro/quickstart.en.md).
    Atomic writes allow to skip double data writes in replicated pools, thus
    reducing Write Amplification and improving write performance up to 2 times.
  info_ru: |
    Максимальный размер атомарной записи на диск данных, который OSD разрешено использовать.
    Поддержка атомарной записи позволяет снизить мультипликатор записи (Write Amplification)
    на диск с новым хранилищем ([meta_format](layout-osd.ru.md#meta_format)=3)
    практически до 1 (то есть, почти до нулевого объёма лишней записи) в реплицированных
    пулах и достигнуть наилучшей возможной производительности записи.
    Значение по умолчанию авто-определяется во время инициализации OSD из
    `/sys/block/xx/queue/atomic_write_max_bytes` либо принимается равным 4096,
    так как все известные диски поддерживают атомарную запись 4 КБ блоков.
    Вы также можете проверить, поддерживают ли ваши NVMe-диски атомарную запись,
    с помощью команды `nvme id-ctrl /dev/nvme0n1 | grep awupf`. Если значение awupf
    плюс 1, умноженное на текущий выбранный размер блока NVMe-диска, больше 4 КБ,
    то новое хранилище может использовать атомарные записи для достижения лучшей
    производительности. Единственные известные диски, которые поддерживают это сейчас -
    [Micron и Kioxia](../intro/quickstart.ru.md).
    Атомарная запись позволяет не использовать двойную запись данных (в журнал и на
    устройство данных) в реплицированных пулах и таким образом снижает амплификацию
    записи (объём служебной записи на диск) и улучшает производительность записи
    вплоть до 2-х кратного прироста.
 - name: use_atomic_flag
  type: bool
  info: |
    This option controls whether the Vitastor OSD uses RWF_ATOMIC write flag with atomic
    writes. This flag is only supported on Linux kernel since 6.11. Atomic writes are
    generally only safe to use with this flag because it tells the kernel to never fragment
    write requests and also to check the write against the actual atomic write capabilities
    of the device.
    This option is enabled by default when atomic_write_size is set to a value larger than 4 KB.
    You can disable it if you're sure that your disks support atomic writes and you want to
    bypass the Linux atomic write checks.
  info_ru: |
    Данная опция контролирует использование Vitastor OSD флага RWF_ATOMIC при атомарной записи
    блоков. Этот флаг поддерживается только в ядрах Linux начиная с 6.11. Атомарная запись
    является безопасной только при использовании этого флага, так как он сообщает ядру о том,
    что запрос записи нельзя фрагментировать и о том, что запрос нужно проверить на соответствие
    реальным возможностям атомарной записи устройства.
    Опция включается по умолчанию, когда atomic_write_size устанавливается в значение больше 4 КБ.
    Вы можете явно отключить её, если уверены, что ваши диски поддерживают атомарную запись и
    хотите обойти проверки уровня ядра.
@@ -26,9 +26,9 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.
 1. Download a Docker image of the desired version: \
-   `docker pull vitalif/vitastor:v3.0.0`
+   `docker pull vitalif/vitastor:v2.4.3`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.0 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.4.3 install.sh`
 3. Reload udev rules: \
   `udevadm control --reload-rules`
@@ -25,9 +25,9 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.
 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitalif/vitastor:v3.0.0`
+   `docker pull vitalif/vitastor:v2.4.3`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v3.0.0 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.4.3 install.sh`
 3. Перезагрузите правила udev: \
   `udevadm control --reload-rules`
@@ -14,8 +14,6 @@
 - Basic part: highly-available block storage with symmetric clustering and no SPOF
 - [Performance](../performance/bench2.en.md) ;-D
  - [NVMe atomic write support](../config/osd.en.md#atomic_write_size) for reducing the amount
    of "extra" disk writes to almost zero (Write Amplification = 1)
 - [Multiple redundancy schemes](../config/pool.en.md#scheme): Replication, XOR n+1, Reed-Solomon erasure codes
  based on jerasure and ISA-L libraries with any number of data and parity drives in a group
 - Configuration via simple JSON data structures in etcd (parameters, pools and images)
@@ -14,8 +14,6 @@
 - Базовая часть - надёжное кластерное блочное хранилище без единой точки отказа
 - [Производительность](../performance/bench2.ru.md) ;-D
  - [Поддержка атомарной записи NVMe](../config/osd.ru.md#atomic_write_size) для снижения объёма
    служебной записи практически до нуля (Write Amplification = 1)
 - [Несколько схем отказоустойчивости](../config/pool.ru.md#scheme): репликация, XOR n+1 (1 диск чётности), коды коррекции ошибок
  Рида-Соломона на основе библиотек jerasure и ISA-L с любым числом дисков данных и чётности в группе
 - Конфигурация через простые человекочитаемые JSON-структуры в etcd
@@ -18,10 +18,9 @@
 ## Preparation
- Get some SATA or NVMe SSDs with capacitors (server-grade drives). The best performance
+- Get some SATA or NVMe SSDs with capacitors (server-grade drives). You can use desktop SSDs
-  is achieved with Micron or Kioxia NVMes with atomic write support (see below). You can use desktop
+  with lazy fsync, but prepare for inferior single-thread latency. Read more about capacitors
-  SSDs with lazy fsync, but prepare for inferior single-thread latency. Read more about
+  [here](../config/layout-cluster.en.md#immediate_commit).
  capacitors [here](../config/layout-cluster.en.md#immediate_commit).
 - If you want to use HDDs, get modern HDDs with Media Cache or SSD Cache: HGST Ultrastar,
  Toshiba MG, Seagate EXOS or something similar. If your drives don't have such cache then
  you also need small SSDs for journal and metadata (even 2 GB per 1 TB of HDD space is enough).
@@ -31,11 +30,9 @@
 ## Recommended drives
- NVMe with atomic write support (ideal!): Micron 7450/7500/7550, Kioxia CD6/CD7/CD8/CD9
+- SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel D3-S4510/4520/4610/4620, Kingston DC500M
- Other NVMe: Micron 9100/9200/9300/9400/9550, Micron 7300, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
+- NVMe: Micron 9100/9200/9300/9400, Micron 7300/7450, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
-  Intel DC-P3700/P4500/P4600, Intel/Solidigm D5-P4320/P5530, Intel/Solidigm D7-P5500/P5600, Solidigm D7-PS1010/PS1030/P5810,
+  Intel DC-P3700/P4500/P4600, Intel D5-P4320/P5530, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
  Intel Optane, Kingston DC1000B/DC1500M, Kioxia CD6/CD7/CD8/CD9
 - SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel/Solidigm D3-S4510/4520/4610/4620, Kingston DC500M
 - HDD: HGST Ultrastar, Toshiba MG, Seagate EXOS
 ## Configure monitors
@@ -18,9 +18,8 @@
 ## Подготовка
- Возьмите серверы с SSD (SATA или NVMe), желательно с конденсаторами (серверные SSD). Наилучшая
+- Возьмите серверы с SSD (SATA или NVMe), желательно с конденсаторами (серверные SSD). Можно
-  производительность достигается на дисках Micron и Kioxia с поддержкой атомарной записи (см. ниже).
+  использовать и десктопные SSD, включив режим отложенного fsync, но производительность будет хуже.
  Можно использовать и десктопные SSD, включив режим отложенного fsync, но производительность будет хуже.
  О конденсаторах читайте [здесь](../config/layout-cluster.ru.md#immediate_commit).
 - Если хотите использовать HDD, берите современные модели с Media или SSD кэшем - HGST Ultrastar,
  Toshiba MG, Seagate EXOS или что-то похожее. Если такого кэша у ваших дисков нет,
@@ -31,11 +30,9 @@
 ## Рекомендуемые диски
- NVMe с поддержкой атомарной записи (идеально!): Micron 7450/7500/7550, Kioxia CD6/CD7/CD8/CD9
+- SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel D3-S4510/4520/4610/4620, Kingston DC500M
- Другие NVMe: Micron 9100/9200/9300/9400/9550, Micron 7300, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
+- NVMe: Micron 9100/9200/9300/9400, Micron 7300/7450, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
-  Intel DC-P3700/P4500/P4600, Intel/Solidigm D5-P4320/P5530, Intel/Solidigm D7-P5500/P5600, Solidigm D7-PS1010/PS1030/P5810,
+  Intel DC-P3700/P4500/P4600, Intel D5-P4320/P5530, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
  Intel Optane, Kingston DC1000B/DC1500M, Kioxia CD6/CD7/CD8/CD9
 - SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel/Solidigm D3-S4510/4520/4610/4620, Kingston DC500M
 - HDD: HGST Ultrastar, Toshiba MG, Seagate EXOS
 ## Настройте мониторы
@@ -14,7 +14,7 @@ Replicated setups:
 - Linear read: `min(total network bandwidth, sum(disk read MB/s))`.
 - Linear write: `min(total network bandwidth, sum(disk write MB/s / number of replicas))`.
 - Saturated parallel read iops: `min(total network bandwidth, sum(disk read iops))`.
- Saturated parallel write iops: `min(total network bandwidth / number of replicas, sum(disk write iops / number of replicas / write amplification))`.
+- Saturated parallel write iops: `min(total network bandwidth / number of replicas, sum(disk write iops / number of replicas / (write amplification = 4)))`.
 EC/XOR setups (EC N+K):
 - Single-threaded (T1Q1) read latency: 1.5 network roundtrips + 1 disk read.
@@ -26,36 +26,28 @@ EC/XOR setups (EC N+K):
 - Linear read: `min(total network bandwidth, sum(disk read MB/s))`.
 - Linear write: `min(total network bandwidth, sum(disk write MB/s * N/(N+K)))`.
 - Saturated parallel read iops: `min(total network bandwidth, sum(disk read iops))`.
- Saturated parallel write iops: roughly `total iops / (N+K) / WA`. More exactly:
+- Saturated parallel write iops: roughly `total iops / (N+K) / WA`. More exactly,
-  - With the new store: `min(total network bandwidth * N/(N+K), sum(disk randrw iops / (2 + N-1 + K*2)))`,
+  `min(total network bandwidth * N/(N+K), sum(disk randrw iops / (N*4 + K*5 + 1)))` with
-    with random read/write mix corresponding to `(N-1)/(2 + N-1 + K*2)*100 % reads`.
+  random read/write mix corresponding to `(N-1)/(N*4 + K*5 + 1)*100 % reads`.
-    - For example, with EC 2+1 it is: `(20% randrw iops) / 5`.
+  - For example, with EC 2+1 it is: `(7% randrw iops) / 14`.
-    - With EC 6+3 it is: `(38% randrw iops) / 13`.
+  - With EC 6+3 it is: `(12.5% randrw iops) / 40`.
  - With the old store: `min(total network bandwidth * N/(N+K), sum(disk randrw iops / (3 + N-1 + K*3)))`,
    with random read/write mix corresponding to `(N-1)/(3 + N-1 + K*3)*100 % reads`.
    - For example, with EC 2+1 it is: `(14% randrw iops) / 7`.
    - With EC 6+3 it is: `(30% randrw iops) / 17`.
-Write Amplification factor:
+Write amplification for 4 KB blocks is usually 3-5 in Vitastor:
- For the new store and for 4 KB writes: WA is always 1 unless you set [atomic_write_size](../config/osd.en.md#atomic_write_size) to 0 manually.
+1. Journal block write
- For the new store and for 8-124 KB writes: WA is 1 if you use NVMe drives with atomic write support, or roughly 2 if you use other drives.
+2. Journal data write
- For the old store, WA is roughly `(2 * write size + 4 KB) / (write size)`. So, for 4 KB writes it's 3, and for 8-124 KB writes it's closer to 2.
+3. Metadata block write
- For both the new and the old store and for writes of [block_size](../config/layout-cluster.en.md#block_size): WA is almost 1.
+4. Another journal block write for EC/XOR setups
 5. Data block write
-Write Amplification consists of:
+If you manage to get an SSD which handles 512 byte blocks well (Optane?) you may
- For the new store:
+lower 1, 3 and 4 to 512 bytes (1/8 of data size) and get WA as low as 2.375.
  - Buffer block write if non-atomic
  - Data block write
  - Metadata write(s) (amortized)
 - For the old store:
  - Journal block write (amortized)
  - Journal data write
  - Metadata block write
  - Another journal block write for EC/XOR setups (amortized)
  - Data block write
-Other possibilities to reduce WA would be to use SSDs with internal 512-byte blocks
+Implemented NVDIMM support can basically eliminate WA at all - all extra writes will
-or NVDIMM, but both options seem unavailable on the market at the moment.
+go to DRAM memory. But this requires a test cluster with NVDIMM - please contact me
 if you want to provide me with such cluster for tests.
 Lazy fsync also reduces WA for parallel workloads because journal blocks are only
 written when they fill up or fsync is requested.
 ## In Practice
@@ -27,36 +27,29 @@
 - Линейное чтение: сумма МБ/с чтения всех дисков, либо общая производительность сети, если в сеть упрётся раньше.
 - Линейная запись: сумма МБ/с записи всех дисков * N/(N+K), либо производительность сети * N / (N+K), если в сеть упрётся раньше.
 - Параллельное случайное мелкое чтение: сумма IOPS чтения всех дисков либо производительность сети, если в сеть упрётся раньше.
- Параллельная случайная мелкая запись: грубо `(сумма IOPS / (N+K) / WA)`.
+- Параллельная случайная мелкая запись: грубо `(сумма IOPS / (N+K) / WA)`. Если точнее, то:
-  Либо `производительность сети * N/(N+K)`, если в сеть упрётся раньше. Если точнее, то:
+  сумма смешанного IOPS всех дисков при `(N-1)/(N*4 + K*5 + 1)*100 %` чтения, делённая на `(N*4 + K*5 + 1)`.
-  - С новым хранилищем: сумма смешанного IOPS всех дисков при `(N-1)/(2 + N-1 + K*2)*100 %` чтения, делённая на `(2 + N-1 + K*2)`.
+  Либо, производительность сети * N/(N+K), если в сеть упрётся раньше.
-    - Например, при EC 2+1 это: `(сумма IOPS при 20% чтения) / 5`.
+  - Например, при EC 2+1 это: `(сумма IOPS при 7% чтения) / 14`.
-    - При EC 6+3 это: `(сумма IOPS при 38% чтения) / 13`.
+  - При EC 6+3 это: `(сумма IOPS при 12.5% чтения) / 40`.
  - Со старым хранилищем: сумма смешанного IOPS всех дисков при `(N-1)/(3 + N-1 + K*3)*100 %` чтения, делённая на `(3 + N-1 + K*3)`.
    - Например, при EC 2+1 это: `(сумма IOPS при 14% чтения) / 7`.
    - При EC 6+3 это: `(сумма IOPS при 30% чтения) / 17`.
-WA (Write Amplification, мультипликатор записи):
+WA (мультипликатор записи) для 4 КБ блоков в Vitastor обычно составляет 3-5:
- С новым хранилищем для 4 КБ записи: WA всегда примерно 1, если только вы не установите [atomic_write_size](../config/osd.ru.md#atomic_write_size) вручную в 0.
+1. Запись метаданных в журнал
- С новым хранилищем и большими записями (8-124 КБ): WA примерно 1, если вы используете NVMe-диски с поддержкой атомарной записи,
+2. Запись блока данных в журнал
-  или примерно 2, если вы используете другие диски.
+3. Запись метаданных в БД
- Со старым хранилищем, WA примерно `(2 * размер записи + 4 КБ) / (размер записи)`. То есть, для 4 КБ записи WA=3, а для 8-124 КБ WA ближе к 2.
+4. Ещё одна запись метаданных в журнал при использовании EC
- И с новым, и со старым хранилищем и для записи размером [block_size](../config/layout-cluster.ru.md#block_size): WA примерно равен 1.
+5. Запись блока данных на диск данных
-Мультипликатор записи состоит из:
+Если вы найдёте SSD, хорошо работающий с 512-байтными блоками данных (Optane?),
- С новым хранилищем:
+то 1, 3 и 4 можно снизить до 512 байт (1/8 от размера данных) и получить WA всего 2.375.
  - Запись блока буфера, если диски без поддержки атомарной записи
  - Запись блока данных
  - Запись(-и) блоков метаданных (амортизированные)
 - Со старым хранилищем:
  - Запись блока журнала (амортизированная)
  - Запись данных в журнал
  - Запись блока метаданных
  - Ещё одна запись блока журнала для EC/XOR пулов (амортизированная)
  - Запись блока данных
-Другими потенциальными возможностями снижения WA могли бы быть SSD с внутренним 512-байтным блоком
+Если реализовать поддержку NVDIMM, то WA можно, условно говоря, ликвидировать вообще - все
-либо NVDIMM, но и то, и другое сейчас выглядит недоступным на рынке.
+дополнительные операции записи смогут обслуживаться DRAM памятью. Но для этого необходим
 тестовый кластер с NVDIMM - пишите, если готовы предоставить такой для тестов.
 Кроме того, WA снижается при использовании отложенного/ленивого сброса при параллельной
 нагрузке, т.к. блоки журнала записываются на диск только когда они заполняются или явным
 образом запрашивается fsync.
 ## На практике
@@ -231,18 +231,6 @@ Upgrading from <= 0.5.x to >= 0.6.x is not supported.
 Downgrade are also allowed freely, except the following specific instructions:
 ### 3.x -> 2.x
 Versions 3.0.0 and newer contain two store implementations - an old one and a new
 one, unsupported in 2.x and previous versions. So you should check your OSD store
 versions before downgrading to 2.x with the following command:
 `vitastor-disk read-sb /dev/vitastor/osdXX-data | jq -r .meta_format`
 If it prints 3 then OSD uses the new store and you can't downgrade it to 2.x.
 If it prints 2 or nothing then OSD uses the old store and the downgrade is allowed.
 ### 1.8.0 to 1.7.1
 Before downgrading from version >= 1.8.0 to version <= 1.7.1
@@ -228,18 +228,6 @@ done
 Откат (понижение версии) тоже свободно разрешён, кроме указанных ниже случаев:
 ### 3.x -> 2.x
 Версии 3.0.0 и более новые содержат две реализации хранилища - старую и новую, не
 поддерживаемую в 2.x и предыдущих версиях. Таким образом, перед откатом на 2.x вам
 следует проверить, какая версия хранилища используется вашими OSD - командой:
 `vitastor-disk read-sb /dev/vitastor/osdXX-data | jq -r .meta_format`
 Если выводится 3, это новое хранилище и откатить такой OSD до 2.x нельзя.
 Если выводится 2 или не выводится ничего, это старое хранилище и откат разрешён.
 ### 1.8.0 -> 1.7.1
 Перед понижением версии с >= 1.8.0 до <= 1.7.1 вы должны скопировать ключ
@@ -51,9 +51,6 @@ Options (automatic mode):
 ```
 --osd_per_disk <N>
  Create <N> OSDs on each disk (default 1)
 --meta_format 3
  Metadata store version. 3 is the new log-structured store, 2 is the stable store
  from Vitastor 0.9-2.x, 1 is the legacy store from Vitastor 0.6-0.8.
 --hybrid
  Prepare hybrid (HDD+SSD, NVMe+SATA or etc) OSDs using provided devices. By default,
  any passed SSDs will be used for journals and metadata, HDDs will be used for data,
@@ -50,9 +50,6 @@ vitastor-disk - инструмент командной строки для уп
 ```
 --osd_per_disk <N>
  Создавать по несколько (<N>) OSD на каждом диске (по умолчанию 1)
 --meta_format 3
  Версия хранилища метаданных. 3 - новое лог-структурированное хранилище,
  2 - стабильное хранилище из Vitastor 0.9-2.x, 1 - старое хранилище из Vitastor 0.6-0.8.
 --hybrid
  Инициализировать гибридные (HDD+SSD, NVMe+SATA и т.п.) OSD на указанных дисках.
  По умолчанию, SSD будут использованы для журналов и метаданных, а HDD - для данных,
@@ -1,6 +1,6 @@
 {
  "name": "vitastor-mon",
-  "version": "3.0.0",
+  "version": "2.4.3",
  "description": "Vitastor SDS monitor service",
  "main": "mon-main.js",
  "scripts": {
@@ -9,6 +9,7 @@ const LPOptimizer = require('./lp_optimizer/lp_optimizer.js');
 const { scale_pg_count } = require('./pg_utils.js');
 const { make_hier_tree, filter_osds_by_root_node,
    filter_osds_by_tags, filter_osds_by_block_layout, get_affinity_osds } = require('./osd_tree.js');
 const { select_murmur3 } = require('./lp_optimizer/murmur3.js');
 function pick_primary(pool_id, pg_num, pool_config, osd_set, up_osds, aff_osds)
 {
@@ -38,7 +39,7 @@ function pick_primary(pool_id, pg_num, pool_config, osd_set, up_osds, aff_osds)
    {
        return 0;
    }
-    return alive_set[pg_num % alive_set.length];
+    return alive_set[select_murmur3(alive_set.length, osd_num => pool_id+'/'+pg_num+'/'+osd_num)];
 }
 function recheck_primary(state, global_config, up_osds, osd_tree)
@@ -1,6 +1,6 @@
 {
  "name": "vitastor",
-  "version": "3.0.0",
+  "version": "2.4.3",
  "description": "Low-level native bindings to Vitastor client library",
  "main": "index.js",
  "keywords": [
@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
-VITASTOR_VERSION = '3.0.0'
+VITASTOR_VERSION = '2.4.3'
 LOG = logging.getLogger(__name__)
@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.0
+Version:        2.4.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.0.el7.tar.gz
+Source0:        vitastor-2.4.3.el7.tar.gz
 BuildRequires:  gperftools-devel
 BuildRequires:  devtoolset-9-gcc-c++
@@ -171,6 +171,7 @@ chown vitastor:vitastor /var/lib/vitastor
 %_bindir/vitastor-kv
 %_bindir/vitastor-kv-stress
 %_bindir/vita
 %_libdir/libvitastor_blk.so*
 %_libdir/libvitastor_client.so*
 %_libdir/libvitastor_kv.so*
@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.0
+Version:        2.4.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.0.el8.tar.gz
+Source0:        vitastor-2.4.3.el8.tar.gz
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-toolset-9-gcc-c++
@@ -168,6 +168,7 @@ chown vitastor:vitastor /var/lib/vitastor
 %_bindir/vitastor-kv
 %_bindir/vitastor-kv-stress
 %_bindir/vita
 %_libdir/libvitastor_blk.so*
 %_libdir/libvitastor_client.so*
 %_libdir/libvitastor_kv.so*
@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        3.0.0
+Version:        2.4.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-3.0.0.el9.tar.gz
+Source0:        vitastor-2.4.3.el9.tar.gz
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++
@@ -165,6 +165,7 @@ chown vitastor:vitastor /var/lib/vitastor
 %_bindir/vitastor-kv
 %_bindir/vitastor-kv-stress
 %_bindir/vita
 %_libdir/libvitastor_blk.so*
 %_libdir/libvitastor_client.so*
 %_libdir/libvitastor_kv.so*
@@ -19,9 +19,8 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	endif()
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 set(ENABLE_COVERAGE false CACHE BOOL "Enable code coverage")
-add_definitions(-DVITASTOR_VERSION="3.0.0")
+add_definitions(-DVITASTOR_VERSION="2.4.3")
 add_definitions(-D_GNU_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -fvisibility=hidden -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})
@@ -32,11 +31,6 @@ set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fvisibility-inlines-hid
 set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL} -fvisibility-inlines-hidden")
 set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fvisibility-inlines-hidden")
 if (${ENABLE_COVERAGE})
 	add_definitions(-coverage)
 	add_link_options(-coverage)
 endif()
 set(CMAKE_BUILD_TYPE RelWithDebInfo)
 string(REGEX REPLACE "([\\/\\-]O)[^ \t\r\n]*" "\\13" CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
 string(REGEX REPLACE "([\\/\\-]O)[^ \t\r\n]*" "\\13" CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL}")
@@ -84,6 +78,14 @@ else()
 	set(LIBURING_LIBRARIES uring)
 endif (${WITH_SYSTEM_LIBURING})
 add_custom_target(build_tests)
 add_custom_target(test
 	COMMAND
 	echo leak:tcmalloc > ${CMAKE_CURRENT_BINARY_DIR}/lsan-suppress.txt &&
 	env LSAN_OPTIONS=suppressions=${CMAKE_CURRENT_BINARY_DIR}/lsan-suppress.txt ${CMAKE_CTEST_COMMAND}
 )
 add_dependencies(test build_tests)
 include_directories(
 	../
 	${CMAKE_SOURCE_DIR}/src/blockstore
@@ -115,7 +117,7 @@ install_symlink(vitastor-disk ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/vi
 install_symlink(vitastor-cli ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/vitastor-rm)
 install_symlink(vitastor-cli ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/vita)
 install(
-	TARGETS vitastor_client vitastor_kv
+	TARGETS vitastor_blk vitastor_client vitastor_kv
 	LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
 	PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
 )
@@ -2,18 +2,15 @@ cmake_minimum_required(VERSION 2.8.12)
 project(vitastor)
-# libvitastor_blk.a
+# libvitastor_blk.so
-add_library(vitastor_blk STATIC
+add_library(vitastor_blk SHARED
-	../util/allocator.cpp ../util/crc32c.c ../util/ringloop.cpp
+	../util/allocator.cpp blockstore.cpp blockstore_impl.cpp blockstore_disk.cpp blockstore_init.cpp blockstore_open.cpp blockstore_journal.cpp blockstore_read.cpp
-	multilist.cpp blockstore_heap.cpp blockstore_disk.cpp
+	blockstore_write.cpp blockstore_sync.cpp blockstore_stable.cpp blockstore_rollback.cpp blockstore_flush.cpp ../util/crc32c.c ../util/ringloop.cpp
 	blockstore.cpp blockstore_impl.cpp blockstore_init.cpp blockstore_open.cpp
 	blockstore_flush.cpp blockstore_read.cpp blockstore_stable.cpp blockstore_sync.cpp blockstore_write.cpp
 	v1/flush.cpp v1/impl.cpp v1/init.cpp v1/journal.cpp v1/open.cpp v1/read.cpp v1/rollback.cpp v1/stable.cpp v1/sync.cpp v1/write.cpp
 )
 target_compile_options(vitastor_blk PUBLIC -fPIC)
 target_link_libraries(vitastor_blk
 	${LIBURING_LIBRARIES}
 	${ISAL_LIBRARIES}
 	tcmalloc_minimal
 	# for timerfd_manager
 	vitastor_common
 )
@@ -1,16 +1,89 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 #include "str_util.h"
 #include "blockstore_impl.h"
 #include "v1/impl.h"
-blockstore_i* blockstore_i::create(blockstore_config_t & config, ring_loop_i *ringloop, timerfd_manager_t *tfd)
+blockstore_t::blockstore_t(blockstore_config_t & config, ring_loop_t *ringloop, timerfd_manager_t *tfd)
 {
-    auto meta_format = stoull_full(config["meta_format"]);
+    impl = new blockstore_impl_t(config, ringloop, tfd);
-    if (meta_format == BLOCKSTORE_META_FORMAT_HEAP)
+}
-        return new blockstore_impl_t(config, ringloop, tfd);
+
-    else
+blockstore_t::~blockstore_t()
-        return new v1::blockstore_impl_t(config, ringloop, tfd);
+{
    delete impl;
 }
 void blockstore_t::parse_config(blockstore_config_t & config)
 {
    impl->parse_config(config, false);
 }
 void blockstore_t::loop()
 {
    impl->loop();
 }
 bool blockstore_t::is_started()
 {
    return impl->is_started();
 }
 bool blockstore_t::is_stalled()
 {
    return impl->is_stalled();
 }
 bool blockstore_t::is_safe_to_stop()
 {
    return impl->is_safe_to_stop();
 }
 void blockstore_t::enqueue_op(blockstore_op_t *op)
 {
    impl->enqueue_op(op);
 }
 int blockstore_t::read_bitmap(object_id oid, uint64_t target_version, void *bitmap, uint64_t *result_version)
 {
    return impl->read_bitmap(oid, target_version, bitmap, result_version);
 }
 std::map<uint64_t, uint64_t> & blockstore_t::get_inode_space_stats()
 {
    return impl->inode_space_stats;
 }
 void blockstore_t::dump_diagnostics()
 {
    return impl->dump_diagnostics();
 }
 uint32_t blockstore_t::get_block_size()
 {
    return impl->get_block_size();
 }
 uint64_t blockstore_t::get_block_count()
 {
    return impl->get_block_count();
 }
 uint64_t blockstore_t::get_free_block_count()
 {
    return impl->get_free_block_count();
 }
 uint64_t blockstore_t::get_journal_size()
 {
    return impl->get_journal_size();
 }
 uint32_t blockstore_t::get_bitmap_granularity()
 {
    return impl->get_bitmap_granularity();
 }
 void blockstore_t::set_no_inode_stats(const std::vector<uint64_t> & pool_ids)
 {
    impl->set_no_inode_stats(pool_ids);
 }
@@ -17,14 +17,22 @@
 #include "ringloop.h"
 #include "timerfd_manager.h"
 // Memory alignment for direct I/O (usually 512 bytes)
 #ifndef DIRECT_IO_ALIGNMENT
 #define DIRECT_IO_ALIGNMENT 512
 #endif
 // Memory allocation alignment (page size is usually optimal)
 #ifndef MEM_ALIGNMENT
 #define MEM_ALIGNMENT 4096
 #endif
 // Default block size is 128 KB, current allowed range is 4K - 128M
 #define DEFAULT_DATA_BLOCK_ORDER 17
 #define MIN_DATA_BLOCK_SIZE 4*1024
 #define MAX_DATA_BLOCK_SIZE 128*1024*1024
 #define DEFAULT_BITMAP_GRANULARITY 4096
 #define MIN_JOURNAL_SIZE 1024*1024
 #define BS_OP_MIN 1
 #define BS_OP_READ 1
 #define BS_OP_WRITE 2
@@ -38,18 +46,8 @@
 #define BS_OP_PRIVATE_DATA_SIZE 256
 #define IMMEDIATE_NONE 0
 #define IMMEDIATE_SMALL 1
 #define IMMEDIATE_ALL 2
 /*
 All operations may be submitted in any order, because reads only see completed writes,
 syncs only sync completed writes and writes don't depend on each other.
 The only restriction is that the external code MUST NOT submit multiple writes for one
 object in parallel. This is a natural restriction because `version` numbers are used though.
 Blockstore opcode documentation:
 ## BS_OP_READ / BS_OP_WRITE / BS_OP_WRITE_STABLE
@@ -164,8 +162,8 @@ struct __attribute__ ((visibility("default"))) blockstore_op_t
            uint32_t list_stable_limit;
        };
    };
-    uint8_t *buf = NULL;
+    void *buf = NULL;
-    uint8_t *bitmap = NULL;
+    void *bitmap = NULL;
    int retval = 0;
    uint8_t private_data[BS_OP_PRIVATE_DATA_SIZE];
@@ -173,54 +171,53 @@ struct __attribute__ ((visibility("default"))) blockstore_op_t
 typedef std::map<std::string, std::string> blockstore_config_t;
-class __attribute__((visibility("default"))) blockstore_i
+class blockstore_impl_t;
 {
 public:
    static blockstore_i* create(blockstore_config_t & config, ring_loop_i *ringloop, timerfd_manager_t *tfd);
-    virtual ~blockstore_i() = default;
+class __attribute__((visibility("default"))) blockstore_t
 {
    blockstore_impl_t *impl;
 public:
    blockstore_t(blockstore_config_t & config, ring_loop_t *ringloop, timerfd_manager_t *tfd);
    ~blockstore_t();
    // Update configuration
-    virtual void parse_config(blockstore_config_t & config) = 0;
+    void parse_config(blockstore_config_t & config);
    // Event loop
-    virtual void loop() = 0;
+    void loop();
    // Returns true when blockstore is ready to process operations
    // (Although you're free to enqueue them before that)
-    virtual bool is_started() = 0;
+    bool is_started();
    // Returns true when blockstore is stalled
-    virtual bool is_stalled() = 0;
+    bool is_stalled();
    // Returns true when it's safe to destroy the instance. If destroying the instance
    // requires to purge some queues, starts that process. Should be called in the event
    // loop until it returns true.
-    virtual bool is_safe_to_stop() = 0;
+    bool is_safe_to_stop();
    // Submission
-    virtual void enqueue_op(blockstore_op_t *op) = 0;
+    void enqueue_op(blockstore_op_t *op);
    // Simplified synchronous operation: get object bitmap & current version
-    virtual int read_bitmap(object_id oid, uint64_t target_version, void *bitmap, uint64_t *result_version = NULL) = 0;
+    int read_bitmap(object_id oid, uint64_t target_version, void *bitmap, uint64_t *result_version = NULL);
    // Get per-inode space usage statistics
-    virtual const std::map<uint64_t, uint64_t> & get_inode_space_stats() = 0;
+    std::map<uint64_t, uint64_t> & get_inode_space_stats();
    // Set per-pool no_inode_stats
-    virtual void set_no_inode_stats(const std::vector<uint64_t> & pool_ids) = 0;
+    void set_no_inode_stats(const std::vector<uint64_t> & pool_ids);
    // Print diagnostics to stdout
-    virtual void dump_diagnostics() = 0;
+    void dump_diagnostics();
-    // Get diagnostic string for an operation
+    uint32_t get_block_size();
-    virtual std::string get_op_diag(blockstore_op_t *op) = 0;
+    uint64_t get_block_count();
    uint64_t get_free_block_count();
-    virtual uint32_t get_block_size() = 0;
+    uint64_t get_journal_size();
    virtual uint64_t get_block_count() = 0;
    virtual uint64_t get_free_block_count() = 0;
-    virtual uint64_t get_journal_size() = 0;
+    uint32_t get_bitmap_granularity();
    virtual uint32_t get_bitmap_granularity() = 0;
 };
@@ -2,15 +2,11 @@
 // License: VNPL-1.1 (see README.md for details)
 #include <sys/file.h>
 #include <sys/ioctl.h>
 #include <unistd.h>
 #include <stdexcept>
-#include "blockstore.h"
+#include "blockstore_impl.h"
 #include "ondisk_formats.h"
 #include "blockstore_disk.h"
 #include "blockstore_heap.h"
 #include "str_util.h"
 #include "allocator.h"
@@ -50,10 +46,6 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
    meta_block_size = parse_size(config["meta_block_size"]);
    bitmap_granularity = parse_size(config["bitmap_granularity"]);
    meta_format = stoull_full(config["meta_format"]);
    atomic_write_size = (config.find("atomic_write_size") != config.end()
        ? parse_size(config["atomic_write_size"]) : 4096);
    use_atomic_flag = config.find("use_atomic_flag") != config.end() &&
        (config["use_atomic_flag"] == "true" || config["use_atomic_flag"] == "1" || config["use_atomic_flag"] == "yes");
    if (config.find("data_io") == config.end() &&
        config.find("meta_io") == config.end() &&
        config.find("journal_io") == config.end())
@@ -98,28 +90,12 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
    if (!min_discard_size)
        min_discard_size = 1024*1024;
    discard_granularity = parse_size(config["discard_granularity"]);
    inmemory_meta = config["inmemory_metadata"] != "false" && config["inmemory_metadata"] != "0" &&
        config["inmemory_metadata"] != "no";
    inmemory_journal = config["inmemory_journal"] != "false" && config["inmemory_journal"] != "0" &&
        config["inmemory_journal"] != "no";
    disable_data_fsync = config["disable_data_fsync"] == "true" || config["disable_data_fsync"] == "1" || config["disable_data_fsync"] == "yes";
    disable_meta_fsync = config["disable_meta_fsync"] == "true" || config["disable_meta_fsync"] == "1" || config["disable_meta_fsync"] == "yes";
    disable_journal_fsync = config["disable_journal_fsync"] == "true" || config["disable_journal_fsync"] == "1" || config["disable_journal_fsync"] == "yes";
    if (mock_mode)
    {
        data_device_size = parse_size(config["data_device_size"]);
        data_device_sect = parse_size(config["data_device_sect"]);
        meta_device_size = parse_size(config["meta_device_size"]);
        meta_device_sect = parse_size(config["meta_device_sect"]);
        journal_device_size = parse_size(config["journal_device_size"]);
        journal_device_sect = parse_size(config["journal_device_sect"]);
    }
    // Validate
    if (!data_block_size)
    {
        data_block_size = (1 << DEFAULT_DATA_BLOCK_ORDER);
    }
-    if (is_power_of_two(data_block_size) >= 64 || data_block_size < MIN_DATA_BLOCK_SIZE || data_block_size >= MAX_DATA_BLOCK_SIZE)
+    if ((block_order = is_power_of_two(data_block_size)) >= 64 || data_block_size < MIN_DATA_BLOCK_SIZE || data_block_size >= MAX_DATA_BLOCK_SIZE)
    {
        throw std::runtime_error("Bad block size");
    }
@@ -203,25 +179,17 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
    {
        throw std::runtime_error("journal_offset must be a multiple of journal_block_size = "+std::to_string(journal_block_size));
    }
    if (meta_device == data_device)
    {
        disable_meta_fsync = disable_data_fsync;
    }
    if (journal_device == meta_device)
    {
        disable_journal_fsync = disable_meta_fsync;
    }
 }
 void blockstore_disk_t::calc_lengths(bool skip_meta_check)
 {
    // data
    data_len = data_device_size - data_offset;
-    if (data_device == meta_device && data_offset < meta_offset)
+    if (data_fd == meta_fd && data_offset < meta_offset)
    {
        data_len = meta_offset - data_offset;
    }
-    if (data_device == journal_device && data_offset < journal_offset)
+    if (data_fd == journal_fd && data_offset < journal_offset)
    {
        data_len = data_len < journal_offset-data_offset
            ? data_len : journal_offset-data_offset;
@@ -236,23 +204,23 @@ void blockstore_disk_t::calc_lengths(bool skip_meta_check)
        data_len = cfg_data_size;
    }
    // meta
-    meta_area_size = (meta_device == data_device ? data_device_size : meta_device_size) - meta_offset;
+    uint64_t meta_area_size = (meta_fd == data_fd ? data_device_size : meta_device_size) - meta_offset;
-    if (meta_device == data_device && meta_offset <= data_offset)
+    if (meta_fd == data_fd && meta_offset <= data_offset)
    {
        meta_area_size = data_offset - meta_offset;
    }
-    if (meta_device == journal_device && meta_offset <= journal_offset)
+    if (meta_fd == journal_fd && meta_offset <= journal_offset)
    {
        meta_area_size = meta_area_size < journal_offset-meta_offset
            ? meta_area_size : journal_offset-meta_offset;
    }
    // journal
-    journal_len = (journal_device == data_device ? data_device_size : (journal_device == meta_device ? meta_device_size : journal_device_size)) - journal_offset;
+    journal_len = (journal_fd == data_fd ? data_device_size : (journal_fd == meta_fd ? meta_device_size : journal_device_size)) - journal_offset;
-    if (journal_device == data_device && journal_offset <= data_offset)
+    if (journal_fd == data_fd && journal_offset <= data_offset)
    {
        journal_len = data_offset - journal_offset;
    }
-    if (journal_device == meta_device && journal_offset <= meta_offset)
+    if (journal_fd == meta_fd && journal_offset <= meta_offset)
    {
        journal_len = journal_len < meta_offset-journal_offset
            ? journal_len : meta_offset-journal_offset;
@@ -262,58 +230,37 @@ void blockstore_disk_t::calc_lengths(bool skip_meta_check)
    clean_entry_bitmap_size = data_block_size / bitmap_granularity / 8;
    clean_dyn_size = clean_entry_bitmap_size*2 + (csum_block_size
        ? data_block_size/csum_block_size*(data_csum_type & 0xFF) : 0);
-recalc:
+    clean_entry_size = sizeof(clean_disk_entry) + clean_dyn_size + 4 /*entry_csum*/;
-    if (meta_format == BLOCKSTORE_META_FORMAT_HEAP)
+    meta_len = (1 + (block_count - 1 + meta_block_size / clean_entry_size) / (meta_block_size / clean_entry_size)) * meta_block_size;
    bool new_doesnt_fit = (!meta_format && !skip_meta_check && meta_area_size < meta_len && !data_csum_type);
    if (meta_format == BLOCKSTORE_META_FORMAT_V1 || new_doesnt_fit)
    {
-        uint32_t entries_per_block = meta_block_size / (sizeof(heap_big_write_t) + clean_dyn_size);
+        uint64_t clean_entry_v0_size = sizeof(clean_disk_entry) + 2*clean_entry_bitmap_size;
-        min_meta_len = (block_count+entries_per_block-1) / entries_per_block * meta_block_size;
+        uint64_t meta_v0_len = (1 + (block_count - 1 + meta_block_size / clean_entry_v0_size)
-    }
+            / (meta_block_size / clean_entry_v0_size)) * meta_block_size;
-    else if (meta_format == BLOCKSTORE_META_FORMAT_V1)
+        if (meta_format == BLOCKSTORE_META_FORMAT_V1 || meta_area_size >= meta_v0_len)
    {
        clean_entry_size = 24 /*sizeof(clean_disk_entry)*/ + 2*clean_entry_bitmap_size;
        min_meta_len = (1 + (block_count - 1 + meta_block_size / clean_entry_size)
            / (meta_block_size / clean_entry_size)) * meta_block_size;
        if (!skip_meta_check && meta_area_size < min_meta_len)
        {
-too_small:
+            // Old metadata fits.
-            throw std::runtime_error("Metadata area is too small, need at least "+std::to_string(min_meta_len)+
+            if (new_doesnt_fit)
                " bytes, have only "+std::to_string(meta_area_size)+" bytes");
        }
    }
    else if (meta_format == BLOCKSTORE_META_FORMAT_V2 || !meta_format)
    {
        meta_format = BLOCKSTORE_META_FORMAT_V2;
        clean_entry_size = 24 /*sizeof(clean_disk_entry)*/ + clean_dyn_size + 4 /*entry_csum*/;
        min_meta_len = (1 + (block_count - 1 + meta_block_size / clean_entry_size) / (meta_block_size / clean_entry_size)) * meta_block_size;
        if (!skip_meta_check && meta_area_size < min_meta_len)
        {
            if (!data_csum_type)
            {
                printf("Warning: Using old metadata format without checksums because the new format"
-                    " doesn't fit into provided area (%ju bytes required, %ju bytes available)\n", min_meta_len, meta_area_size);
+                    " doesn't fit into provided area (%ju bytes required, %ju bytes available)\n", meta_len, meta_area_size);
                meta_format = BLOCKSTORE_META_FORMAT_V1;
                goto recalc;
            }
            else
            {
                goto too_small;
            }
            clean_entry_size = clean_entry_v0_size;
            meta_len = meta_v0_len;
            meta_format = BLOCKSTORE_META_FORMAT_V1;
        }
        else
            meta_format = BLOCKSTORE_META_FORMAT_V2;
    }
    else
        meta_format = BLOCKSTORE_META_FORMAT_V2;
    if (!skip_meta_check && meta_area_size < meta_len)
    {
-        throw std::runtime_error("meta_format = "+std::to_string(meta_format)+" is not supported");
+        throw std::runtime_error("Metadata area is too small, need at least "+std::to_string(meta_len)+" bytes, have only "+std::to_string(meta_area_size)+" bytes");
    }
 }
 void blockstore_disk_t::check_lengths()
 {
    if (meta_area_size < min_meta_len)
    {
        throw std::runtime_error("Metadata area is too small, need at least "+std::to_string(min_meta_len)+" bytes, have only "+std::to_string(meta_area_size)+" bytes");
    }
    // requested journal size
-    if (cfg_journal_size > journal_len)
+    if (!skip_meta_check && cfg_journal_size > journal_len)
    {
        throw std::runtime_error("Requested journal_size is too large");
    }
@@ -374,19 +321,12 @@ static int bs_openmode(const std::string & mode)
 void blockstore_disk_t::open_data()
 {
-    if (data_fd >= 0)
+    data_fd = open(data_device.c_str(), bs_openmode(data_io) | O_RDWR);
    {
        throw std::runtime_error("data device is already opened");
    }
    data_fd = mock_mode ? MOCK_DATA_FD : open(data_device.c_str(), bs_openmode(data_io) | O_RDWR);
    if (data_fd == -1)
    {
        throw std::runtime_error("Failed to open data device "+data_device+": "+std::string(strerror(errno)));
    }
-    if (!mock_mode)
+    check_size(data_fd, &data_device_size, &data_device_sect, "data device");
    {
        check_size(data_fd, &data_device_size, &data_device_sect, "data device");
    }
    if (disk_alignment % data_device_sect)
    {
        throw std::runtime_error(
@@ -398,7 +338,7 @@ void blockstore_disk_t::open_data()
    {
        throw std::runtime_error("data_offset exceeds device size = "+std::to_string(data_device_size));
    }
-    if (!mock_mode && !disable_flock && flock(data_fd, LOCK_EX|LOCK_NB) != 0)
+    if (!disable_flock && flock(data_fd, LOCK_EX|LOCK_NB) != 0)
    {
        throw std::runtime_error(std::string("Failed to lock data device: ") + strerror(errno));
    }
@@ -406,26 +346,19 @@ void blockstore_disk_t::open_data()
 void blockstore_disk_t::open_meta()
 {
    if (meta_fd >= 0)
    {
        throw std::runtime_error("metadata device is already opened");
    }
    if (meta_device != data_device || meta_io != data_io)
    {
-        meta_fd = mock_mode ? MOCK_META_FD : open(meta_device.c_str(), bs_openmode(meta_io) | O_RDWR);
+        meta_fd = open(meta_device.c_str(), bs_openmode(meta_io) | O_RDWR);
        if (meta_fd == -1)
        {
            throw std::runtime_error("Failed to open metadata device "+meta_device+": "+std::string(strerror(errno)));
        }
-        if (!mock_mode)
+        check_size(meta_fd, &meta_device_size, &meta_device_sect, "metadata device");
        {
            check_size(meta_fd, &meta_device_size, &meta_device_sect, "metadata device");
        }
        if (meta_offset >= meta_device_size)
        {
            throw std::runtime_error("meta_offset exceeds device size = "+std::to_string(meta_device_size));
        }
-        if (!mock_mode && !disable_flock && meta_device != data_device && flock(meta_fd, LOCK_EX|LOCK_NB) != 0)
+        if (!disable_flock && meta_device != data_device && flock(meta_fd, LOCK_EX|LOCK_NB) != 0)
        {
            throw std::runtime_error(std::string("Failed to lock metadata device: ") + strerror(errno));
        }
@@ -451,26 +384,15 @@ void blockstore_disk_t::open_meta()
 void blockstore_disk_t::open_journal()
 {
    if (journal_fd >= 0)
    {
        throw std::runtime_error("journal device is already opened");
    }
    if (journal_device != meta_device || journal_io != meta_io)
    {
-        journal_fd = mock_mode ? MOCK_JOURNAL_FD : open(journal_device.c_str(), bs_openmode(journal_io) | O_RDWR);
+        journal_fd = open(journal_device.c_str(), bs_openmode(journal_io) | O_RDWR);
        if (journal_fd == -1)
        {
            throw std::runtime_error("Failed to open journal device "+journal_device+": "+std::string(strerror(errno)));
        }
-        if (!mock_mode)
+        check_size(journal_fd, &journal_device_size, &journal_device_sect, "journal device");
-        {
+        if (!disable_flock && journal_device != meta_device && flock(journal_fd, LOCK_EX|LOCK_NB) != 0)
            check_size(journal_fd, &journal_device_size, &journal_device_sect, "journal device");
        }
        if (journal_offset >= journal_device_size)
        {
            throw std::runtime_error("journal_offset exceeds device size = "+std::to_string(journal_device_size));
        }
        if (!mock_mode && !disable_flock && journal_device != meta_device && flock(journal_fd, LOCK_EX|LOCK_NB) != 0)
        {
            throw std::runtime_error(std::string("Failed to lock journal device: ") + strerror(errno));
        }
@@ -496,32 +418,25 @@ void blockstore_disk_t::open_journal()
 void blockstore_disk_t::close_all()
 {
-    if (!mock_mode)
+    if (data_fd >= 0)
-    {
+        close(data_fd);
-        if (data_fd >= 0)
+    if (meta_fd >= 0 && meta_fd != data_fd)
-            close(data_fd);
+        close(meta_fd);
-        if (meta_fd >= 0 && meta_fd != data_fd)
+    if (journal_fd >= 0 && journal_fd != meta_fd)
-            close(meta_fd);
+        close(journal_fd);
        if (journal_fd >= 0 && journal_fd != meta_fd)
            close(journal_fd);
    }
    data_fd = meta_fd = journal_fd = -1;
 }
 // Sadly DISCARD only works through ioctl(), but it seems to always block the device queue,
 // so it's not a big deal that we can only run it synchronously.
-int blockstore_disk_t::trim_data(std::function<bool(uint64_t)> is_free)
+int blockstore_disk_t::trim_data(allocator_t *alloc)
 {
    if (mock_mode)
    {
        return -EINVAL;
    }
    int r = 0;
    uint64_t j = 0, i = 0;
    uint64_t discarded = 0;
    for (; i <= block_count; i++)
    {
-        if (i >= block_count || is_free(i))
+        if (i >= block_count || alloc->get(i))
        {
            if (i > j && (i-j)*data_block_size >= min_discard_size)
            {
@@ -8,19 +8,10 @@
 #include <string>
 #include <map>
 // Memory alignment for direct I/O (usually 512 bytes)
 #ifndef DIRECT_IO_ALIGNMENT
 #define DIRECT_IO_ALIGNMENT 512
 #endif
 #define BLOCKSTORE_CSUM_NONE 0
 // Lower byte of checksum type is its length
 #define BLOCKSTORE_CSUM_CRC32C 0x104
 #define MOCK_DATA_FD 1000
 #define MOCK_META_FD 1001
 #define MOCK_JOURNAL_FD 1002
 class allocator_t;
 struct blockstore_disk_t
@@ -31,15 +22,11 @@ struct blockstore_disk_t
    // Required write alignment and journal/metadata/data areas' location alignment
    uint32_t disk_alignment = 4096;
    // Journal block size - minimum_io_size of the journal device is the best choice
-    uint32_t journal_block_size = 4096;
+    uint64_t journal_block_size = 4096;
    // Metadata block size - minimum_io_size of the metadata device is the best choice
-    uint32_t meta_block_size = 4096;
+    uint64_t meta_block_size = 4096;
    // Atomic write size of the data block device
    uint32_t atomic_write_size = 4096;
    // Whether we should set RWF_ATOMIC on atomic writes
    bool use_atomic_flag = false;
    // Sparse write tracking granularity. 4 KB is a good choice. Must be a multiple of disk_alignment
-    uint32_t bitmap_granularity = 4096;
+    uint64_t bitmap_granularity = 4096;
    // Data checksum type, BLOCKSTORE_CSUM_NONE or BLOCKSTORE_CSUM_CRC32C
    uint32_t data_csum_type = BLOCKSTORE_CSUM_NONE;
    // Checksum block size, must be a multiple of bitmap_granularity
@@ -49,37 +36,27 @@ struct blockstore_disk_t
    // I/O modes for data, metadata and journal: direct or "" = O_DIRECT, cached = O_SYNC, directsync = O_DIRECT|O_SYNC
    // O_SYNC without O_DIRECT = use Linux page cache for reads and writes
    std::string data_io, meta_io, journal_io;
    // It is safe to disable fsync() if drive write cache is writethrough
    bool disable_data_fsync = false, disable_meta_fsync = false, disable_journal_fsync = false;
    // Keep journal (buffered data) in memory?
    bool inmemory_meta = true;
    // Keep metadata in memory?
    bool inmemory_journal = true;
    // Data discard granularity and minimum size (for the sake of performance)
    bool discard_on_start = false;
    uint64_t min_discard_size = 1024*1024;
    uint64_t discard_granularity = 0;
    int meta_fd = -1, data_fd = -1, journal_fd = -1;
-    uint64_t meta_offset = 0, meta_device_sect = 0, meta_device_size = 0, meta_area_size = 0, min_meta_len = 0;
+    uint64_t meta_offset, meta_device_sect, meta_device_size, meta_len, meta_format = 0;
-    uint64_t data_offset = 0, data_device_sect = 0, data_device_size = 0, data_len = 0;
+    uint64_t data_offset, data_device_sect, data_device_size, data_len;
-    uint64_t journal_offset = 0, journal_device_sect = 0, journal_device_size = 0, journal_len = 0;
+    uint64_t journal_offset, journal_device_sect, journal_device_size, journal_len;
    uint64_t meta_format = 0;
    uint32_t block_order = 0;
    uint64_t block_count = 0;
-    uint32_t clean_entry_bitmap_size = 0;
+    uint32_t clean_entry_bitmap_size = 0, clean_entry_size = 0, clean_dyn_size = 0;
    uint32_t clean_entry_size = 0, clean_dyn_size = 0; // for meta_v1/2
    bool mock_mode = false;
    void parse_config(std::map<std::string, std::string> & config);
    void open_data();
    void open_meta();
    void open_journal();
    void calc_lengths(bool skip_meta_check = false);
    void check_lengths();
    void close_all();
-    int trim_data(std::function<bool(uint64_t)> is_free);
+    int trim_data(allocator_t *alloc);
    inline uint64_t dirty_dyn_size(uint64_t offset, uint64_t len)
    {
@@ -1,12 +1,22 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 #define COPY_BUF_JOURNAL 1
 #define COPY_BUF_DATA 2
 #define COPY_BUF_ZERO 4
 #define COPY_BUF_CSUM_FILL 8
 #define COPY_BUF_COALESCED 16
 #define COPY_BUF_META_BLOCK 32
 #define COPY_BUF_JOURNALED_BIG 64
 struct copy_buffer_t
 {
-    uint32_t copy_flags;
+    int copy_flags;
-    uint64_t offset, len, disk_loc, disk_offset, disk_len;
+    uint64_t offset, len, disk_offset;
-    uint8_t *buf;
+    uint64_t journal_sector; // only for reads: sector+1 if used and !journal.inmemory, otherwise 0
-    heap_entry_t *wr;
+    void *buf;
    uint8_t *csum_buf;
    int *dyn_data;
 };
 struct meta_sector_t
@@ -17,6 +27,13 @@ struct meta_sector_t
    int usage_count;
 };
 struct flusher_sync_t
 {
    bool fsync_meta;
    int ready_count;
    int state;
 };
 struct flusher_meta_write_t
 {
    uint64_t sector, pos;
@@ -32,74 +49,94 @@ class journal_flusher_co
 {
    blockstore_impl_t *bs;
    journal_flusher_t *flusher;
-    int co_id;
+    int wait_state, wait_count, wait_journal_count;
    int wait_state, wait_count;
    struct io_uring_sqe *sqe;
    struct ring_data_t *data;
    uint8_t *new_csums = NULL;
    uint8_t *new_bmp = NULL;
    uint8_t *punch_bmp = NULL;
    uint8_t *new_ext_bmp = NULL;
-    std::function<void(ring_data_t*)> simple_callback_r, simple_callback_w;
+    std::list<flusher_sync_t>::iterator cur_sync;
-    object_id cur_oid;
+    obj_ver_id cur;
-    heap_entry_t *cur_obj;
+    std::map<obj_ver_id, dirty_entry>::iterator dirty_it, dirty_start, dirty_end;
-    uint64_t fsynced_lsn;
+    std::map<object_id, uint64_t>::iterator repeat_it;
-    heap_compact_t compact_info;
+    std::function<void(ring_data_t*)> simple_callback_r, simple_callback_rj, simple_callback_w;
    uint64_t clean_loc;
    uint32_t modified_block;
    bool bitmap_copied;
    bool should_repeat;
-    std::vector<copy_buffer_t> read_vec;
+    bool try_trim = false;
-    std::vector<heap_entry_t*> csum_copy;
+    bool skip_copy, has_delete, has_writes;
-    uint32_t overwrite_start, overwrite_end;
+    std::vector<copy_buffer_t> v;
-    int i, res;
+    std::vector<copy_buffer_t>::iterator it;
-    bool read_to_fill_incomplete;
+    int i;
    bool fill_incomplete, cleared_incomplete;
    int read_to_fill_incomplete;
    int copy_count;
-    bool do_repeat = false;
+    uint64_t clean_loc, clean_ver, old_clean_loc, old_clean_ver;
    flusher_meta_write_t meta_old, meta_new;
    bool clean_init_bitmap;
    uint64_t clean_bitmap_offset, clean_bitmap_len;
    uint8_t *clean_init_dyn_ptr;
    uint8_t *new_clean_bitmap;
    uint64_t new_trim_pos;
    friend class journal_flusher_t;
-
+    void scan_dirty();
-    void iterate_checksum_holes(std::function<void(int & pos, uint32_t hole_start, uint32_t hole_end)> cb);
+    bool read_dirty(int wait_base);
-    void fill_partial_checksum_blocks();
+    bool modify_meta_do_reads(int wait_base);
    bool wait_meta_reads(int wait_base);
    bool modify_meta_read(uint64_t meta_loc, flusher_meta_write_t &wr, int wait_base);
    bool clear_incomplete_csum_block_bits(int wait_base);
    void calc_block_checksums(uint32_t *new_data_csums, bool skip_overwrites);
    void update_metadata_entry();
    bool write_meta_block(flusher_meta_write_t & meta_block, int wait_base);
    void update_clean_db();
    void free_data_blocks();
    bool fsync_batch(bool fsync_meta, int wait_base);
    bool trim_journal(int wait_base);
    void free_buffers();
    int check_and_punch_checksums();
    bool calc_block_checksums();
    bool write_meta_block(int wait_base);
    bool read_buffered(int wait_base);
    bool fsync_meta(int wait_base);
    bool fsync_buffer(int wait_base);
    bool trim_lsn(int wait_base);
 public:
    journal_flusher_co();
    ~journal_flusher_co();
    bool loop();
 };
 // Journal flusher itself
 class journal_flusher_t
 {
-    int force_start = 0;
+    int trim_wanted = 0;
-    int min_flusher_count = 0, max_flusher_count = 0, cur_flusher_count = 0, target_flusher_count = 0;
+    bool dequeuing;
    int min_flusher_count, max_flusher_count, cur_flusher_count, target_flusher_count;
    int flusher_start_threshold;
    journal_flusher_co *co;
    blockstore_impl_t *bs;
    friend class journal_flusher_co;
-    robin_hood::unordered_flat_set<object_id> flushing;
+    int journal_trim_counter;
-    int active_flushers = 0;
+    bool trimming;
-    int wanting_meta_fsync = 0;
+    void* journal_superblock;
-    bool fsyncing_meta = false;
+
-    int syncing_buffer = 0;
+    int active_flushers;
    int syncing_flushers;
    std::list<flusher_sync_t> syncs;
    std::map<object_id, uint64_t> sync_to_repeat;
    std::map<uint64_t, meta_sector_t> meta_sectors;
    std::deque<object_id> flush_queue;
    std::unordered_map<object_id, uint64_t> flush_versions;
    std::unordered_set<uint64_t> inflight_meta_sectors;
    bool try_find_older(std::map<obj_ver_id, dirty_entry>::iterator & dirty_end, obj_ver_id & cur);
    bool try_find_other(std::map<obj_ver_id, dirty_entry>::iterator & dirty_end, obj_ver_id & cur);
 public:
    journal_flusher_t(blockstore_impl_t *bs);
    ~journal_flusher_t();
    void loop();
-    int get_syncing_buffer();
+    bool is_trim_wanted() { return trim_wanted; }
    bool is_active();
    void mark_trim_possible();
    void request_trim();
    void release_trim();
    void enqueue_flush(obj_ver_id oid);
    void unshift_flush(obj_ver_id oid, bool force);
    void remove_flush(object_id oid);
    void dump_diagnostics();
    bool is_mutated(uint64_t clean_loc);
 };
@@ -1,358 +0,0 @@
 // Metadata storage version 3 ("lsm heap")
 // Copyright (c) Vitaliy Filippov, 2025+
 // License: VNPL-1.1 (see README.md for details)
 #pragma once
 #include <map>
 #include <unordered_map>
 #include <set>
 #include <deque>
 #include <vector>
 #include "../client/object_id.h"
 #include "../util/robin_hood.h"
 #include "blockstore_disk.h"
 #include "multilist.h"
 struct pool_shard_settings_t
 {
    uint32_t pg_count;
    uint32_t pg_stripe_size;
    uint32_t no_inode_stats;
 };
 #define BS_HEAP_TYPE 0x07
 #define BS_HEAP_BIG_WRITE 1
 #define BS_HEAP_SMALL_WRITE 2
 #define BS_HEAP_INTENT_WRITE 3
 #define BS_HEAP_BIG_INTENT 4
 #define BS_HEAP_DELETE 5
 #define BS_HEAP_COMMIT 6
 #define BS_HEAP_ROLLBACK 7
 #define BS_HEAP_STABLE 0x40
 #define BS_HEAP_GARBAGE 0x80
 class blockstore_heap_t;
 struct heap_small_write_t;
 struct heap_big_write_t;
 struct heap_big_intent_t;
 struct __attribute__((__packed__)) heap_entry_t
 {
    uint16_t size;
    uint16_t entry_type;
    uint32_t crc32c;
    uint64_t lsn;
    uint64_t inode;
    uint64_t stripe;
    uint64_t version;
    // uint8_t[] external_bitmap
    // uint8_t[] internal_bitmap
    // uint32_t[] checksums
    inline uint8_t type() const { return (entry_type & BS_HEAP_TYPE); }
    inline heap_small_write_t& small() { return *(heap_small_write_t*)this; }
    inline heap_big_write_t& big() { return *(heap_big_write_t*)this; }
    inline heap_big_intent_t& big_intent() { return *(heap_big_intent_t*)this; }
    bool is_garbage();
    void set_garbage();
    bool is_overwrite();
    bool is_compactable();
    bool is_before(heap_entry_t *other);
    uint32_t get_size(blockstore_heap_t *heap);
    uint8_t *get_ext_bitmap(blockstore_heap_t *heap);
    uint8_t *get_int_bitmap(blockstore_heap_t *heap);
    uint8_t *get_checksums(blockstore_heap_t *heap);
    uint32_t *get_checksum(blockstore_heap_t *heap);
    uint64_t big_location(blockstore_heap_t *heap);
    void set_big_location(blockstore_heap_t *heap, uint64_t location);
    uint32_t calc_crc32c();
 };
 struct __attribute__((__packed__)) heap_small_write_t
 {
    heap_entry_t hdr;
    uint64_t location;
    uint32_t offset;
    uint32_t len;
    // Also includes 1 bitmap and 1 crc32c after the bitmap if checksums are disabled
 };
 struct __attribute__((__packed__)) heap_big_write_t
 {
    heap_entry_t hdr;
    uint32_t block_num;
 };
 struct __attribute__((__packed__)) heap_big_intent_t
 {
    heap_entry_t hdr;
    uint32_t block_num;
    uint32_t offset;
    uint32_t len;
    // Also includes 2 bitmaps and 1 crc32c if checksums are disabled
 };
 struct __attribute__((__packed__)) heap_list_item_t
 {
    heap_list_item_t *prev;
    heap_list_item_t *next;
    uint32_t block_num;
    heap_entry_t entry;
 };
 struct heap_object_mvcc_t
 {
    uint32_t readers = 0;
    heap_entry_t *garbage_entry = NULL;
 };
 struct heap_block_info_t
 {
    uint32_t used_space = 0;
    uint64_t mod_lsn = 0, mod_lsn_to = 0; // only 1 block write of LSN sequence is allowed at a moment
    bool is_writing: 1;
    bool has_garbage: 1;
    std::vector<heap_list_item_t*> entries;
 };
 struct heap_inflight_lsn_t
 {
    uint64_t flags;
    heap_entry_t *wr;
 };
 struct heap_compact_t
 {
    uint64_t compact_lsn, compact_version;
    heap_entry_t *clean_wr;
    bool do_delete;
 };
 struct heap_li_hash
 {
    size_t operator()(const heap_list_item_t* li) const noexcept
    {
        return robin_hood::hash_int(li->entry.stripe);
    }
 };
 struct heap_li_equal
 {
    constexpr bool operator()(const heap_list_item_t* a, const heap_list_item_t* b) const noexcept
    {
        return a->entry.stripe == b->entry.stripe;
    }
 };
 using i64hash_t = robin_hood::hash<uint64_t>;
 using heap_inode_map_t = robin_hood::unordered_flat_set<heap_list_item_t*, heap_li_hash, heap_li_equal, 88>;
 using heap_block_index_t = robin_hood::unordered_flat_map<uint64_t,
    robin_hood::unordered_flat_map<inode_t, void*, i64hash_t>, i64hash_t>;
 using heap_mvcc_map_t = robin_hood::unordered_flat_map<object_id, heap_object_mvcc_t>;
 class blockstore_heap_t
 {
    friend class heap_entry_t;
    blockstore_disk_t *dsk = NULL;
    uint8_t* buffer_area = NULL;
    int log_level = 0;
    const uint32_t meta_block_count = 0;
    const uint32_t max_entry_size = 0;
    robin_hood::unordered_flat_map<pool_id_t, pool_shard_settings_t> pool_shard_settings;
    // PG => inode => stripe => block number
    heap_block_index_t block_index;
    std::vector<heap_block_info_t> block_info;
    allocator_t *data_alloc = NULL;
    multilist_index_t *meta_alloc = NULL;
    uint32_t meta_nearfull_blocks = 0;
    uint64_t meta_used_space = 0;
    multilist_alloc_t *buffer_alloc = NULL;
    std::map<uint64_t, uint64_t> inode_space_stats;
    uint64_t buffer_area_used_space = 0;
    uint64_t data_used_space = 0;
    uint64_t next_lsn = 0;
    uint32_t last_allocated_block = UINT32_MAX;
    heap_mvcc_map_t object_mvcc;
    // LSN queue: inflight (writing) -> completed [-> fsynced]
    std::deque<heap_inflight_lsn_t> inflight_lsn;
    uint32_t to_compact_count = 0;
    uint64_t compacted_count = 0;
    uint32_t inflight_overwrite_count = 0;
    uint64_t first_inflight_lsn = 0;
    uint64_t completed_lsn = 0;
    uint64_t fsynced_lsn = 0;
    std::deque<object_id> compact_queue;
    bool marked_used_blocks = false;
    bool recheck_queue_filled = false;
    std::set<uint32_t> recheck_modified_blocks;
    std::deque<heap_entry_t*> recheck_queue;
    int recheck_in_progress = 0;
    bool in_recheck = false;
    std::function<void(bool is_data, uint64_t offset, uint64_t len, uint8_t* buf, std::function<void()>)> recheck_cb;
    int recheck_queue_depth = 0;
    void inode_map_put(void* & inode_idx, heap_list_item_t* li);
    void inode_map_get(void *inode_idx, heap_inode_map_t::iterator & li_it, heap_list_item_t* & li, uint64_t stripe);
    void inode_map_free(void* inode_idx);
    void inode_map_iterate(void* & inode_idx, std::function<void(heap_list_item_t*)> cb);
    void inode_map_replace(void* & inode_idx, const heap_inode_map_t::iterator & li_it, heap_list_item_t* new_li);
    void inode_map_erase(void* & inode_idx, const heap_inode_map_t::iterator & li_it, heap_list_item_t* li);
    uint64_t get_pg_id(inode_t inode, uint64_t stripe);
    bool validate_object(heap_entry_t *obj);
    void fill_recheck_queue();
    int mark_used_blocks();
    void recheck_buffer(heap_entry_t *cwr, uint8_t *buf);
    void defragment_block(uint32_t block_num);
    int allocate_entry(uint32_t entry_size, uint32_t *block_num, bool allow_last_free);
    void insert_list_item(heap_list_item_t *li);
    int add_entry(uint32_t wr_size, uint32_t *modified_block, bool allow_last_free,
        bool explicit_complete, std::function<void(heap_entry_t *wr)> fill_entry);
    int add_simple(heap_entry_t *obj, uint64_t version, uint32_t *modified_block, uint32_t entry_type);
    uint32_t meta_alloc_pos(const heap_block_info_t & inf);
    void modify_alloc(uint32_t block_num, std::function<void(heap_block_info_t &)> change_cb);
    void mark_garbage_up_to(heap_entry_t *wr);
    void mark_garbage(uint32_t block_num, heap_entry_t *prev_wr, uint32_t used_big);
    void push_inflight_lsn(uint64_t lsn, heap_entry_t *wr, uint64_t flags);
    void mark_completed_lsns(uint64_t mod_lsn);
    void apply_inflight(heap_inflight_lsn_t & inflight);
 public:
    blockstore_heap_t(blockstore_disk_t *dsk, uint8_t *buffer_area, int log_level = 0);
    ~blockstore_heap_t();
    void start_load(uint64_t completed_lsn);
    // load data from the disk, returns EDOM on corruption
    int read_blocks(uint64_t disk_offset, uint64_t size, uint8_t *buf, bool allow_corrupted,
        std::function<void(uint32_t block_num, heap_entry_t* wr)> handle_write,
        std::function<void(uint32_t, uint32_t, uint8_t*)> handle_block);
    int load_blocks(uint64_t disk_offset, uint64_t size, uint8_t *buf,
        bool allow_corrupted, uint64_t &entries_loaded);
    // finish loading
    int finish_load(bool allow_corrupted = false);
    // get blocks which are modified during loading and should be written to the disk
    // before finishing initialization if not R/O
    std::vector<uint32_t> get_recheck_modified_blocks();
    // recheck small write data after reading the database from disk
    bool recheck_small_writes(std::function<void(bool is_data, uint64_t offset, uint64_t len, uint8_t* buf, std::function<void()>)> read_buffer, int queue_depth);
    // reshard database according to the pool's PG count
    void reshard(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size);
    void set_no_inode_stats(const std::vector<uint64_t> & pool_ids);
    void recalc_inode_space_stats(uint64_t pool_id, bool per_inode);
    // read an object entry and lock it against removal
    // in the future, may become asynchronous
    heap_entry_t *lock_and_read_entry(object_id oid);
    // re-read a locked object entry with the given lsn (pointer may be invalidated)
    heap_entry_t *read_locked_entry(object_id oid, uint64_t lsn);
    // read an object entry without locking it
    heap_entry_t *read_entry(object_id oid);
    // unlock an entry
    bool unlock_entry(object_id oid);
    // set or verify checksums in a write request
    bool calc_checksums(heap_entry_t *wr, uint8_t *data, bool set, uint32_t offset = 0, uint32_t len = 0);
    // set or verify raw block checksums
    bool calc_block_checksums(uint32_t *block_csums, uint8_t *data, uint8_t *bitmap, uint32_t start, uint32_t end,
        bool set, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);
    bool calc_block_checksums(uint32_t *block_csums, uint8_t *bitmap,
        uint32_t start, uint32_t end, std::function<uint8_t*(uint32_t start, uint32_t & len)> next,
        bool set, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);
    // adds a small_write or intent_write entry to an object
    // return 0 if OK, or maybe ENOSPC
    int add_small_write(object_id oid, heap_entry_t **obj_ptr, uint16_t type, uint64_t version,
        uint32_t offset, uint32_t len, uint64_t location, uint8_t *bitmap, uint8_t *data, uint32_t *modified_block);
    // adds a big_write (overwrite) entry to an object
    int add_big_write(object_id oid, heap_entry_t *old_head, bool stable, uint64_t version,
        uint32_t offset, uint32_t len, uint64_t location, uint8_t *bitmap, uint8_t *data, uint32_t *modified_block);
    // adds a "redirecting" big_intent entry to an object (same as big_write, used to avoid fsync on desktop SSDs)
    int add_redirect_intent(object_id oid, heap_entry_t **obj_ptr, uint64_t version,
        uint32_t offset, uint32_t len, uint64_t location, uint8_t *bitmap, uint8_t *data, uint32_t *modified_block);
    // adds a big_intent (atomic partial modification) entry to an object
    int add_big_intent(object_id oid, heap_entry_t **obj_ptr, uint64_t version,
        uint32_t offset, uint32_t len, uint8_t *bitmap, uint8_t *data, uint8_t *checksums, uint32_t *modified_block);
    // adds a compacted up to <version> entry to an object
    int add_compact(heap_entry_t *obj, uint64_t compact_version, uint64_t compact_lsn, uint64_t compact_location,
        bool do_delete, uint32_t *modified_block, uint8_t *new_int_bitmap, uint8_t *new_ext_bitmap, uint8_t *new_csums);
    // "punch holes" in a big_entry
    int punch_holes(heap_entry_t *wr, uint8_t *new_bitmap, uint8_t *new_csums, uint32_t *modified_block);
    // stabilize an unstable object version
    // return 0 if OK, ENOENT if not exists
    int add_commit(heap_entry_t *obj, uint64_t version, uint32_t *modified_block);
    // rollback an unstable object version
    // return 0 if OK, ENOENT if not exists, EBUSY if already stable
    int add_rollback(heap_entry_t *obj, uint64_t version, uint32_t *modified_block);
    // forget an object
    // return error code
    int add_delete(heap_entry_t *obj, uint32_t *modified_block);
    // get the next object to compact
    // guaranteed to return objects in min lsn order
    // returns 0 if OK, ENOENT if nothing to compact
    int get_next_compact(object_id & oid);
    void iterate_with_stable(heap_entry_t *obj, uint64_t max_lsn, std::function<bool(heap_entry_t*, bool stable)> cb);
    // iterate compactable entries
    heap_compact_t iterate_compaction(heap_entry_t *obj, uint64_t fsynced_lsn, bool under_pressure,
        std::function<void(heap_entry_t*)> small_wr_cb);
    // iterate all objects
    void iterate_objects(std::function<void(heap_entry_t*, uint32_t block_num)> cb);
    // retrieve object listing from a PG
    int list_objects(uint32_t pg_num, object_id min_oid, object_id max_oid,
        obj_ver_id **result_list, size_t *stable_count, size_t *unstable_count);
    // inflight write tracking
    void start_block_write(uint32_t block_num);
    void complete_block_write(uint32_t block_num);
    void complete_lsn_write(uint64_t lsn);
    uint64_t get_completed_lsn();
    uint64_t get_fsynced_lsn();
    void mark_lsn_fsynced(uint64_t lsn);
    // data device block allocator functions
    uint64_t find_free_data();
    bool is_data_used(uint64_t location);
    void use_data(inode_t inode, uint64_t location);
    void free_data(inode_t inode, uint64_t location);
    // buffer device allocator functions
    uint64_t find_free_buffer_area(uint64_t size);
    bool is_buffer_area_free(uint64_t location, uint64_t size);
    void use_buffer_area(inode_t inode, uint64_t location, uint64_t size);
    void free_buffer_area(inode_t inode, uint64_t location, uint64_t size);
    uint64_t get_buffer_area_used_space();
    // get metadata block data buffer and used space
    void get_meta_block(uint32_t block_num, uint8_t *buffer);
    void fill_block_empty_space(uint8_t *buffer, uint32_t pos);
    uint32_t get_meta_block_used_space(uint32_t block_num);
    // get space usage statistics
    uint64_t get_data_used_space();
    const std::map<uint64_t, uint64_t> & get_inode_space_stats();
    uint64_t get_meta_total_space();
    uint64_t get_meta_used_space();
    uint32_t get_meta_nearfull_blocks();
    uint32_t get_compact_queue_size();
    uint32_t get_to_compact_count();
    uint64_t get_compacted_count();
    uint64_t entry_pos(uint32_t block_num, uint32_t offset);
    heap_entry_t *entry_from_pos(uint64_t entry_pos, bool allow_unallocated = false);
    heap_entry_t *prev(heap_entry_t *wr);
    uint32_t get_simple_entry_size();
    uint32_t get_big_entry_size();
    uint32_t get_big_intent_entry_size();
    uint32_t get_small_entry_size(uint32_t offset, uint32_t len);
    uint32_t get_csum_size(heap_entry_t *wr);
    uint32_t get_csum_size(uint32_t entry_type, uint32_t offset = 0, uint32_t len = 0);
 };
@@ -1,18 +1,13 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 #include <stdexcept>
 #include "blockstore_impl.h"
 #include "blockstore_internal.h"
 #include "crc32c.h"
-blockstore_impl_t::blockstore_impl_t(blockstore_config_t & config, ring_loop_i *ringloop, timerfd_manager_t *tfd, bool mock_mode)
+blockstore_impl_t::blockstore_impl_t(blockstore_config_t & config, ring_loop_t *ringloop, timerfd_manager_t *tfd)
 {
    assert(sizeof(blockstore_op_private_t) <= BS_OP_PRIVATE_DATA_SIZE);
    this->tfd = tfd;
    this->ringloop = ringloop;
    dsk.mock_mode = mock_mode;
    ring_consumer.loop = [this]() { loop(); };
    ringloop->register_consumer(&ring_consumer);
    initialized = 0;
@@ -22,39 +17,31 @@ blockstore_impl_t::blockstore_impl_t(blockstore_config_t & config, ring_loop_i *
        dsk.open_data();
        dsk.open_meta();
        dsk.open_journal();
-        dsk.calc_lengths();
+        calc_lengths();
        alloc_dyn_data = dsk.clean_dyn_size > sizeof(void*) || dsk.csum_block_size > 0;
        zero_object = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.data_block_size);
        data_alloc = new allocator_t(dsk.block_count);
    }
    catch (std::exception & e)
    {
        dsk.close_all();
        throw;
    }
    meta_superblock = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size);
    memset(meta_superblock, 0, dsk.meta_block_size);
 }
 void blockstore_impl_t::init()
 {
    flusher = new journal_flusher_t(this);
    if (dsk.inmemory_journal)
    {
        buffer_area = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.journal_len);
    }
    heap = new blockstore_heap_t(&dsk, buffer_area, log_level);
 }
 blockstore_impl_t::~blockstore_impl_t()
 {
-    if (flusher)
+    delete data_alloc;
-        delete flusher;
+    delete flusher;
-    if (heap)
+    if (zero_object)
-        delete heap;
+        free(zero_object);
    if (buffer_area)
        free(buffer_area);
    if (meta_superblock)
        free(meta_superblock);
    ringloop->unregister_consumer(&ring_consumer);
    dsk.close_all();
    if (metadata_buffer)
        free(metadata_buffer);
    if (clean_bitmaps)
        free(clean_bitmaps);
 }
 bool blockstore_impl_t::is_started()
@@ -70,9 +57,10 @@ bool blockstore_impl_t::is_stalled()
 // main event loop - produce requests
 void blockstore_impl_t::loop()
 {
    // FIXME: initialized == 10 is ugly
    if (initialized != 10)
    {
-        // read metadata
+        // read metadata, then journal
        if (initialized == 0)
        {
            metadata_init_reader = new blockstore_init_meta(this);
@@ -85,41 +73,69 @@ void blockstore_impl_t::loop()
            {
                delete metadata_init_reader;
                metadata_init_reader = NULL;
                journal_init_reader = new blockstore_init_journal(this);
                initialized = 2;
            }
        }
        if (initialized == 2)
        {
            int res = journal_init_reader->loop();
            if (!res)
            {
                delete journal_init_reader;
                journal_init_reader = NULL;
                initialized = 3;
                ringloop->wakeup();
            }
        }
        if (initialized == 3)
        {
            if (!readonly && dsk.discard_on_start)
                dsk.trim_data(data_alloc);
            if (journal.flush_journal)
                initialized = 4;
            else
                initialized = 10;
        }
        if (initialized == 4)
        {
            if (readonly)
            {
-                dsk.trim_data([this](uint64_t block_num){ return heap->is_data_used(block_num * dsk.data_block_size); });
+                printf("Can't flush the journal in readonly mode\n");
                exit(1);
            }
-            initialized = 10;
+            flusher->loop();
            ringloop->submit();
        }
    }
    else
    {
        // try to submit ops
        unsigned initial_ring_space = ringloop->space_left();
-        int op_idx = 0, new_idx = 0;
+        // has_writes == 0 - no writes before the current queue item
-        bool has_unfinished_writes = false;
+        // has_writes == 1 - some writes in progress
        // has_writes == 2 - tried to submit some writes, but failed
        int has_writes = 0, op_idx = 0, new_idx = 0;
        for (; op_idx < submit_queue.size(); op_idx++, new_idx++)
        {
            auto op = submit_queue[op_idx];
            submit_queue[new_idx] = op;
            // FIXME: This needs some simplification
            // Writes should not block reads if the ring is not full and reads don't depend on them
            // In all other cases we should stop submission
            if (PRIV(op)->wait_for)
            {
                check_wait(op);
                if (PRIV(op)->wait_for == WAIT_SQE)
                {
                    // ring is full, stop submission
                    break;
                }
                else if (PRIV(op)->wait_for)
                {
-                    has_unfinished_writes = has_unfinished_writes || op->opcode == BS_OP_WRITE ||
+                    if (op->opcode == BS_OP_WRITE || op->opcode == BS_OP_WRITE_STABLE || op->opcode == BS_OP_DELETE)
-                        op->opcode == BS_OP_WRITE_STABLE || op->opcode == BS_OP_DELETE ||
+                    {
-                        op->opcode == BS_OP_STABLE || op->opcode == BS_OP_ROLLBACK;
+                        has_writes = 2;
                    }
                    continue;
                }
            }
@@ -132,33 +148,46 @@ void blockstore_impl_t::loop()
            {
                wr_st = dequeue_read(op);
            }
-            else if (op->opcode == BS_OP_WRITE || op->opcode == BS_OP_WRITE_STABLE || op->opcode == BS_OP_DELETE)
+            else if (op->opcode == BS_OP_WRITE || op->opcode == BS_OP_WRITE_STABLE)
            {
                if (has_writes == 2)
                {
                    // Some writes already could not be submitted
                    continue;
                }
                wr_st = dequeue_write(op);
-                has_unfinished_writes = has_unfinished_writes || (wr_st != 2);
+                has_writes = wr_st > 0 ? 1 : 2;
            }
            else if (op->opcode == BS_OP_DELETE)
            {
                if (has_writes == 2)
                {
                    // Some writes already could not be submitted
                    continue;
                }
                wr_st = dequeue_del(op);
                has_writes = wr_st > 0 ? 1 : 2;
            }
            else if (op->opcode == BS_OP_SYNC)
            {
-                // syncs only completed writes, so doesn't have to be blocked by anything
+                // sync only completed writes?
                // wait for the data device fsync to complete, then submit journal writes for big writes
                // then submit an fsync operation
                wr_st = continue_sync(op);
            }
-            else if (op->opcode == BS_OP_STABLE || op->opcode == BS_OP_ROLLBACK)
+            else if (op->opcode == BS_OP_STABLE)
            {
                wr_st = dequeue_stable(op);
-                has_unfinished_writes = has_unfinished_writes || (wr_st != 2);
+            }
            else if (op->opcode == BS_OP_ROLLBACK)
            {
                wr_st = dequeue_rollback(op);
            }
            else if (op->opcode == BS_OP_LIST)
            {
-                // LIST has to be blocked by previous writes and commits/rollbacks
+                // LIST doesn't have to be blocked by previous modifications
-                if (!has_unfinished_writes)
+                process_list(op);
-                {
+                wr_st = 2;
                    process_list(op);
                    wr_st = 2;
                }
                else
                {
                    wr_st = 0;
                }
            }
            if (wr_st == 2)
            {
@@ -167,13 +196,16 @@ void blockstore_impl_t::loop()
            }
            if (wr_st == 0)
            {
                PRIV(op)->pending_ops = 0;
                ringloop->restore(prev_sqe_pos);
                if (PRIV(op)->wait_for == WAIT_SQE)
                {
                    // ring is full, stop submission
                    break;
                }
                else if (PRIV(op)->wait_for == WAIT_JOURNAL)
                {
                    PRIV(op)->wait_detail2 = (unstable_writes.size()+unstable_unsynced);
                }
            }
        }
        if (op_idx != new_idx)
@@ -188,19 +220,19 @@ void blockstore_impl_t::loop()
        {
            flusher->loop();
        }
        for (auto & block_num: pending_modified_blocks)
        {
            auto & mb = modified_blocks[block_num];
            heap->get_meta_block(block_num, mb.buf);
            heap->start_block_write(block_num);
            mb.sent = true;
        }
        int ret = ringloop->submit();
        if (ret < 0)
        {
            throw std::runtime_error(std::string("io_uring_submit: ") + strerror(-ret));
        }
-        pending_modified_blocks.clear();
+        for (auto s: journal.submitting_sectors)
        {
            // Mark journal sector writes as submitted
            if (journal.sector_info[s].submit_id)
                journal.sector_info[s].written = true;
            journal.sector_info[s].submit_id = 0;
        }
        journal.submitting_sectors.clear();
        if ((initial_ring_space - ringloop->space_left()) > 0)
        {
            live = true;
@@ -218,7 +250,7 @@ bool blockstore_impl_t::is_safe_to_stop()
    {
        return false;
    }
-    if (has_unsynced())
+    if (unsynced_big_writes.size() > 0 || unsynced_small_writes.size() > 0)
    {
        if (!readonly && !stop_sync_submitted)
        {
@@ -242,7 +274,7 @@ void blockstore_impl_t::check_wait(blockstore_op_t *op)
 {
    if (PRIV(op)->wait_for == WAIT_SQE)
    {
-        if (ringloop->space_left() < PRIV(op)->wait_detail)
+        if (ringloop->sqes_left() < PRIV(op)->wait_detail)
        {
            // stop submission if there's still no free space
 #ifdef BLOCKSTORE_DEBUG
@@ -252,13 +284,40 @@ void blockstore_impl_t::check_wait(blockstore_op_t *op)
        }
        PRIV(op)->wait_for = 0;
    }
-    else if (PRIV(op)->wait_for == WAIT_COMPACTION)
+    else if (PRIV(op)->wait_for == WAIT_JOURNAL)
    {
-        if (heap->get_compacted_count() <= PRIV(op)->wait_detail)
+        if (journal.used_start == PRIV(op)->wait_detail &&
            (unstable_writes.size()+unstable_unsynced) == PRIV(op)->wait_detail2)
        {
            // do not submit
 #ifdef BLOCKSTORE_DEBUG
-            printf("Still waiting for more flushes\n");
+            printf("Still waiting to flush journal offset %08jx\n", PRIV(op)->wait_detail);
 #endif
            return;
        }
        flusher->release_trim();
        PRIV(op)->wait_for = 0;
    }
    else if (PRIV(op)->wait_for == WAIT_JOURNAL_BUFFER)
    {
        int next = ((journal.cur_sector + 1) % journal.sector_count);
        if (journal.sector_info[next].flush_count > 0 ||
            journal.sector_info[next].dirty)
        {
            // do not submit
 #ifdef BLOCKSTORE_DEBUG
            printf("Still waiting for a journal buffer\n");
 #endif
            return;
        }
        PRIV(op)->wait_for = 0;
    }
    else if (PRIV(op)->wait_for == WAIT_FREE)
    {
        if (!data_alloc->get_free_count() && big_to_flush > 0)
        {
 #ifdef BLOCKSTORE_DEBUG
            printf("Still waiting for free space on the data device\n");
 #endif
            return;
        }
@@ -277,8 +336,7 @@ void blockstore_impl_t::enqueue_op(blockstore_op_t *op)
        ((op->opcode == BS_OP_READ || op->opcode == BS_OP_WRITE || op->opcode == BS_OP_WRITE_STABLE) && (
            op->offset >= dsk.data_block_size ||
            op->len > dsk.data_block_size-op->offset ||
-            (op->offset % dsk.bitmap_granularity) ||
+            (op->len % dsk.disk_alignment)
            (op->len % dsk.bitmap_granularity)
        )) ||
        readonly && op->opcode != BS_OP_READ && op->opcode != BS_OP_LIST)
    {
@@ -305,11 +363,75 @@ void blockstore_impl_t::init_op(blockstore_op_t *op)
 {
    // Call constructor without allocating memory. We'll call destructor before returning op back
    new ((void*)op->private_data) blockstore_op_private_t;
    PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 0;
    PRIV(op)->wait_for = 0;
    PRIV(op)->op_state = 0;
    PRIV(op)->pending_ops = 0;
 }
 static bool replace_stable(object_id oid, uint64_t version, int search_start, int search_end, obj_ver_id* list)
 {
    while (search_start < search_end)
    {
        int pos = search_start+(search_end-search_start)/2;
        if (oid < list[pos].oid)
        {
            search_end = pos;
        }
        else if (list[pos].oid < oid)
        {
            search_start = pos+1;
        }
        else
        {
            list[pos].version = version;
            return true;
        }
    }
    return false;
 }
 blockstore_clean_db_t& blockstore_impl_t::clean_db_shard(object_id oid)
 {
    uint64_t pg_num = 0;
    uint64_t pool_id = (oid.inode >> (64-POOL_ID_BITS));
    auto sh_it = clean_db_settings.find(pool_id);
    if (sh_it != clean_db_settings.end())
    {
        // like map_to_pg()
        pg_num = (oid.stripe / sh_it->second.pg_stripe_size) % sh_it->second.pg_count + 1;
    }
    return clean_db_shards[(pool_id << (64-POOL_ID_BITS)) | pg_num];
 }
 void blockstore_impl_t::reshard_clean_db(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size)
 {
    uint64_t pool_id = (uint64_t)pool;
    std::map<pool_pg_id_t, blockstore_clean_db_t> new_shards;
    auto sh_it = clean_db_shards.lower_bound((pool_id << (64-POOL_ID_BITS)));
    while (sh_it != clean_db_shards.end() &&
        (sh_it->first >> (64-POOL_ID_BITS)) == pool_id)
    {
        for (auto & pair: sh_it->second)
        {
            // like map_to_pg()
            uint64_t pg_num = (pair.first.stripe / pg_stripe_size) % pg_count + 1;
            uint64_t shard_id = (pool_id << (64-POOL_ID_BITS)) | pg_num;
            new_shards[shard_id][pair.first] = pair.second;
        }
        clean_db_shards.erase(sh_it++);
    }
    for (sh_it = new_shards.begin(); sh_it != new_shards.end(); sh_it++)
    {
        auto & to = clean_db_shards[sh_it->first];
        to.swap(sh_it->second);
    }
    clean_db_settings[pool_id] = (pool_shard_settings_t){
        .pg_count = pg_count,
        .pg_stripe_size = pg_stripe_size,
    };
 }
 void blockstore_impl_t::process_list(blockstore_op_t *op)
 {
    uint32_t list_pg = op->pg_number+1;
@@ -318,8 +440,7 @@ void blockstore_impl_t::process_list(blockstore_op_t *op)
    uint64_t min_inode = op->min_oid.inode;
    uint64_t max_inode = op->max_oid.inode;
    // Check PG
-    if (!pg_count || (pg_stripe_size < MIN_DATA_BLOCK_SIZE || list_pg > pg_count) ||
+    if (pg_count != 0 && (pg_stripe_size < MIN_DATA_BLOCK_SIZE || list_pg > pg_count))
        !INODE_POOL(min_inode) || INODE_POOL(min_inode) != INODE_POOL(max_inode))
    {
        op->retval = -EINVAL;
        FINISH_OP(op);
@@ -327,45 +448,250 @@ void blockstore_impl_t::process_list(blockstore_op_t *op)
    }
    // Check if the DB needs resharding
    // (we don't know about PGs from the beginning, we only create "shards" here)
-    heap->reshard(INODE_POOL(min_inode), pg_count, pg_stripe_size);
+    uint64_t first_shard = 0, last_shard = UINT64_MAX;
-    obj_ver_id *result = NULL;
+    if (min_inode != 0 &&
-    size_t stable_count = 0, unstable_count = 0;
+        // Check if min_inode == max_inode == pool_id<<N, i.e. this is a pool listing
-    int res = heap->list_objects(list_pg, op->min_oid, op->max_oid, &result, &stable_count, &unstable_count);
+        (min_inode >> (64-POOL_ID_BITS)) == (max_inode >> (64-POOL_ID_BITS)))
    if (op->list_stable_limit)
    {
-        // Ordered result is expected - used by scrub
+        pool_id_t pool_id = (min_inode >> (64-POOL_ID_BITS));
-        // We use an unordered map
+        if (pg_count > 1)
        std::sort(result, result + stable_count);
        if (stable_count > op->list_stable_limit)
        {
-            memmove(result + op->list_stable_limit, result + stable_count, unstable_count);
+            // Per-pg listing
-            stable_count = op->list_stable_limit;
+            auto sh_it = clean_db_settings.find(pool_id);
            if (sh_it == clean_db_settings.end() ||
                sh_it->second.pg_count != pg_count ||
                sh_it->second.pg_stripe_size != pg_stripe_size)
            {
                reshard_clean_db(pool_id, pg_count, pg_stripe_size);
            }
            first_shard = last_shard = ((uint64_t)pool_id << (64-POOL_ID_BITS)) | list_pg;
        }
        else
        {
            // Per-pool listing
            first_shard = ((uint64_t)pool_id << (64-POOL_ID_BITS));
            last_shard = ((uint64_t)(pool_id+1) << (64-POOL_ID_BITS)) - 1;
        }
    }
    // Copy clean_db entries
    int stable_count = 0, stable_alloc = 0;
    if (min_inode != max_inode)
    {
        for (auto shard_it = clean_db_shards.lower_bound(first_shard);
            shard_it != clean_db_shards.end() && shard_it->first <= last_shard;
            shard_it++)
        {
            auto & clean_db = shard_it->second;
            stable_alloc += clean_db.size();
        }
    }
    if (op->list_stable_limit > 0)
    {
        stable_alloc = op->list_stable_limit;
        if (stable_alloc > 1024*1024)
            stable_alloc = 1024*1024;
    }
    if (stable_alloc < 32768)
    {
        stable_alloc = 32768;
    }
    obj_ver_id *stable = (obj_ver_id*)malloc(sizeof(obj_ver_id) * stable_alloc);
    if (!stable)
    {
        op->retval = -ENOMEM;
        FINISH_OP(op);
        return;
    }
    auto max_oid = op->max_oid;
    bool limited = false;
    pool_pg_id_t last_shard_id = 0;
    for (auto shard_it = clean_db_shards.lower_bound(first_shard);
        shard_it != clean_db_shards.end() && shard_it->first <= last_shard;
        shard_it++)
    {
        auto & clean_db = shard_it->second;
        auto clean_it = clean_db.begin(), clean_end = clean_db.end();
        if (op->min_oid.inode != 0 || op->min_oid.stripe != 0)
        {
            clean_it = clean_db.lower_bound(op->min_oid);
        }
        if ((max_oid.inode != 0 || max_oid.stripe != 0) && !(max_oid < op->min_oid))
        {
            clean_end = clean_db.upper_bound(max_oid);
        }
        for (; clean_it != clean_end; clean_it++)
        {
            if (stable_count >= stable_alloc)
            {
                stable_alloc *= 2;
                obj_ver_id* nst = (obj_ver_id*)realloc(stable, sizeof(obj_ver_id) * stable_alloc);
                if (!nst)
                {
                    op->retval = -ENOMEM;
                    FINISH_OP(op);
                    return;
                }
                stable = nst;
            }
            stable[stable_count++] = {
                .oid = clean_it->first,
                .version = clean_it->second.version,
            };
            if (op->list_stable_limit > 0 && stable_count >= op->list_stable_limit)
            {
                if (!limited)
                {
                    limited = true;
                    max_oid = stable[stable_count-1].oid;
                }
                break;
            }
        }
        if (op->list_stable_limit > 0)
        {
            // To maintain the order, we have to include objects in the same range from other shards
            if (last_shard_id != 0 && last_shard_id != shard_it->first)
                std::sort(stable, stable+stable_count);
            if (stable_count > op->list_stable_limit)
                stable_count = op->list_stable_limit;
        }
        last_shard_id = shard_it->first;
    }
    if (op->list_stable_limit == 0 && first_shard != last_shard)
    {
        // If that's not a per-PG listing, sort clean entries (already sorted if list_stable_limit != 0)
        std::sort(stable, stable+stable_count);
    }
    int clean_stable_count = stable_count;
    // Copy dirty_db entries (sorted, too)
    int unstable_count = 0, unstable_alloc = 0;
    obj_ver_id *unstable = NULL;
    {
        auto dirty_it = dirty_db.begin(), dirty_end = dirty_db.end();
        if (op->min_oid.inode != 0 || op->min_oid.stripe != 0)
        {
            dirty_it = dirty_db.lower_bound({
                .oid = op->min_oid,
                .version = 0,
            });
        }
        if ((max_oid.inode != 0 || max_oid.stripe != 0) && !(max_oid < op->min_oid))
        {
            dirty_end = dirty_db.upper_bound({
                .oid = max_oid,
                .version = UINT64_MAX,
            });
        }
        for (; dirty_it != dirty_end; dirty_it++)
        {
            if (!pg_count || ((dirty_it->first.oid.stripe / pg_stripe_size) % pg_count + 1) == list_pg) // like map_to_pg()
            {
                if (IS_DELETE(dirty_it->second.state))
                {
                    // Deletions are always stable, so try to zero out two possible entries
                    if (!replace_stable(dirty_it->first.oid, 0, 0, clean_stable_count, stable))
                    {
                        replace_stable(dirty_it->first.oid, 0, clean_stable_count, stable_count, stable);
                    }
                }
                else if (IS_STABLE(dirty_it->second.state) || (dirty_it->second.state & BS_ST_INSTANT))
                {
                    // First try to replace a clean stable version in the first part of the list
                    if (!replace_stable(dirty_it->first.oid, dirty_it->first.version, 0, clean_stable_count, stable))
                    {
                        // Then try to replace the last dirty stable version in the second part of the list
                        if (stable_count > 0 && stable[stable_count-1].oid == dirty_it->first.oid)
                        {
                            stable[stable_count-1].version = dirty_it->first.version;
                        }
                        else
                        {
                            if (stable_count >= stable_alloc)
                            {
                                stable_alloc += 32768;
                                obj_ver_id *nst = (obj_ver_id*)realloc(stable, sizeof(obj_ver_id) * stable_alloc);
                                if (!nst)
                                {
                                    if (unstable)
                                        free(unstable);
                                    op->retval = -ENOMEM;
                                    FINISH_OP(op);
                                    return;
                                }
                                stable = nst;
                            }
                            stable[stable_count++] = dirty_it->first;
                        }
                    }
                    if (op->list_stable_limit > 0 && stable_count >= op->list_stable_limit)
                    {
                        // Stop here
                        break;
                    }
                }
                else
                {
                    if (unstable_count >= unstable_alloc)
                    {
                        unstable_alloc += 32768;
                        obj_ver_id *nst = (obj_ver_id*)realloc(unstable, sizeof(obj_ver_id) * unstable_alloc);
                        if (!nst)
                        {
                            if (stable)
                                free(stable);
                            op->retval = -ENOMEM;
                            FINISH_OP(op);
                            return;
                        }
                        unstable = nst;
                    }
                    unstable[unstable_count++] = dirty_it->first;
                }
            }
        }
    }
    // Remove zeroed out stable entries
    int j = 0;
    for (int i = 0; i < stable_count; i++)
    {
        if (stable[i].version != 0)
        {
            stable[j++] = stable[i];
        }
    }
    stable_count = j;
    if (stable_count+unstable_count > stable_alloc)
    {
        stable_alloc = stable_count+unstable_count;
        obj_ver_id *nst = (obj_ver_id*)realloc(stable, sizeof(obj_ver_id) * stable_alloc);
        if (!nst)
        {
            if (unstable)
                free(unstable);
            op->retval = -ENOMEM;
            FINISH_OP(op);
            return;
        }
        stable = nst;
    }
    // Copy unstable entries
    for (int i = 0; i < unstable_count; i++)
    {
        stable[j++] = unstable[i];
    }
    free(unstable);
    op->version = stable_count;
-    op->retval = res == 0 ? stable_count+unstable_count : -res;
+    op->retval = stable_count+unstable_count;
-    op->buf = (uint8_t*)result;
+    op->buf = stable;
    FINISH_OP(op);
 }
 void blockstore_impl_t::set_no_inode_stats(const std::vector<uint64_t> & pool_ids)
 {
    heap->set_no_inode_stats(pool_ids);
 }
 void blockstore_impl_t::dump_diagnostics()
 {
    journal.dump_diagnostics();
    flusher->dump_diagnostics();
 }
 void blockstore_meta_header_v3_t::set_crc32c()
 {
    header_csum = 0;
    uint32_t calc = crc32c(0, this, version == BLOCKSTORE_META_FORMAT_HEAP
        ? sizeof(blockstore_meta_header_v3_t) : sizeof(blockstore_meta_header_v2_t));
    header_csum = calc;
 }
 void blockstore_impl_t::disk_error_abort(const char *op, int retval, int expected)
 {
    if (retval == -EAGAIN)
@@ -379,18 +705,85 @@ void blockstore_impl_t::disk_error_abort(const char *op, int retval, int expecte
    exit(1);
 }
-uint64_t blockstore_impl_t::get_free_block_count()
+void blockstore_impl_t::set_no_inode_stats(const std::vector<uint64_t> & pool_ids)
 {
-    return dsk.block_count - heap->get_data_used_space()/dsk.data_block_size;
+    for (auto & np: no_inode_stats)
    {
        np.second = 2;
    }
    for (auto pool_id: pool_ids)
    {
        if (!no_inode_stats[pool_id])
            recalc_inode_space_stats(pool_id, false);
        no_inode_stats[pool_id] = 1;
    }
    for (auto np_it = no_inode_stats.begin(); np_it != no_inode_stats.end(); )
    {
        if (np_it->second == 2)
        {
            recalc_inode_space_stats(np_it->first, true);
            no_inode_stats.erase(np_it++);
        }
        else
            np_it++;
    }
 }
-std::string blockstore_impl_t::get_op_diag(blockstore_op_t *op)
+void blockstore_impl_t::recalc_inode_space_stats(uint64_t pool_id, bool per_inode)
 {
-    char buf[256];
+    auto sp_begin = inode_space_stats.lower_bound((pool_id << (64-POOL_ID_BITS)));
-    auto priv = PRIV(op);
+    auto sp_end = inode_space_stats.lower_bound(((pool_id+1) << (64-POOL_ID_BITS)));
-    if (priv->wait_for)
+    inode_space_stats.erase(sp_begin, sp_end);
-        snprintf(buf, sizeof(buf), "state=%d wait=%d (detail=%ju)", priv->op_state, priv->wait_for, priv->wait_detail);
+    auto sh_it = clean_db_shards.lower_bound((pool_id << (64-POOL_ID_BITS)));
-    else
+    while (sh_it != clean_db_shards.end() &&
-        snprintf(buf, sizeof(buf), "state=%d", priv->op_state);
+        (sh_it->first >> (64-POOL_ID_BITS)) == pool_id)
-    return std::string(buf);
+    {
        for (auto & pair: sh_it->second)
        {
            uint64_t space_id = per_inode ? pair.first.inode : (pool_id << (64-POOL_ID_BITS));
            inode_space_stats[space_id] += dsk.data_block_size;
        }
        sh_it++;
    }
    object_id last_oid = {};
    bool last_exists = false;
    auto dirty_it = dirty_db.lower_bound((obj_ver_id){ .oid = { .inode = (pool_id << (64-POOL_ID_BITS)) } });
    while (dirty_it != dirty_db.end() && (dirty_it->first.oid.inode >> (64-POOL_ID_BITS)) == pool_id)
    {
        if (IS_STABLE(dirty_it->second.state) && (IS_BIG_WRITE(dirty_it->second.state) || IS_DELETE(dirty_it->second.state)))
        {
            bool exists = false;
            if (last_oid == dirty_it->first.oid)
            {
                exists = last_exists;
            }
            else
            {
                auto & clean_db = clean_db_shard(dirty_it->first.oid);
                auto clean_it = clean_db.find(dirty_it->first.oid);
                exists = clean_it != clean_db.end();
            }
            uint64_t space_id = per_inode ? dirty_it->first.oid.inode : (pool_id << (64-POOL_ID_BITS));
            if (IS_BIG_WRITE(dirty_it->second.state))
            {
                if (!exists)
                    inode_space_stats[space_id] += dsk.data_block_size;
                last_exists = true;
            }
            else
            {
                if (exists)
                {
                    auto & sp = inode_space_stats[space_id];
                    if (sp > dsk.data_block_size)
                        sp -= dsk.data_block_size;
                    else
                        inode_space_stats.erase(space_id);
                }
                last_exists = false;
            }
            last_oid = dirty_it->first.oid;
        }
        dirty_it++;
    }
 }
@@ -5,8 +5,6 @@
 #include "blockstore.h"
 #include "blockstore_disk.h"
 #include "blockstore_heap.h"
 #include "ondisk_formats.h"
 #include <sys/types.h>
 #include <sys/ioctl.h>
@@ -23,66 +21,240 @@
 #include <unordered_map>
 #include <unordered_set>
-#include "malloc_or_die.h"
+#include "cpp-btree/btree_map.h"
-class blockstore_impl_t;
+#include "malloc_or_die.h"
 #include "allocator.h"
 //#define BLOCKSTORE_DEBUG
 // States are not stored on disk. Instead, they're deduced from the journal
 #define BS_ST_SMALL_WRITE 0x01
 #define BS_ST_BIG_WRITE 0x02
 #define BS_ST_DELETE 0x03
 #define BS_ST_WAIT_DEL 0x10
 #define BS_ST_WAIT_BIG 0x20
 #define BS_ST_IN_FLIGHT 0x30
 #define BS_ST_SUBMITTED 0x40
 #define BS_ST_WRITTEN 0x50
 #define BS_ST_SYNCED 0x60
 #define BS_ST_STABLE 0x70
 #define BS_ST_INSTANT 0x100
 #define IMMEDIATE_NONE 0
 #define IMMEDIATE_SMALL 1
 #define IMMEDIATE_ALL 2
 #define BS_ST_TYPE_MASK 0x0F
 #define BS_ST_WORKFLOW_MASK 0xF0
 #define IS_IN_FLIGHT(st) (((st) & 0xF0) <= BS_ST_SUBMITTED)
 #define IS_STABLE(st) (((st) & 0xF0) == BS_ST_STABLE)
 #define IS_SYNCED(st) (((st) & 0xF0) >= BS_ST_SYNCED)
 #define IS_JOURNAL(st) (((st) & 0x0F) == BS_ST_SMALL_WRITE)
 #define IS_BIG_WRITE(st) (((st) & 0x0F) == BS_ST_BIG_WRITE)
 #define IS_DELETE(st) (((st) & 0x0F) == BS_ST_DELETE)
 #define IS_INSTANT(st) (((st) & BS_ST_TYPE_MASK) == BS_ST_DELETE || ((st) & BS_ST_INSTANT))
 #define BS_SUBMIT_CHECK_SQES(n) \
    if (ringloop->sqes_left() < (n))\
    {\
        /* Pause until there are more requests available */\
        PRIV(op)->wait_detail = (n);\
        PRIV(op)->wait_for = WAIT_SQE;\
        return 0;\
    }
 #define BS_SUBMIT_GET_SQE(sqe, data) \
    BS_SUBMIT_GET_ONLY_SQE(sqe); \
    struct ring_data_t *data = ((ring_data_t*)sqe->user_data)
 #define BS_SUBMIT_GET_ONLY_SQE(sqe) \
    struct io_uring_sqe *sqe = get_sqe();\
    if (!sqe)\
    {\
        /* Pause until there are more requests available */\
        PRIV(op)->wait_detail = 1;\
        PRIV(op)->wait_for = WAIT_SQE;\
        return 0;\
    }
 #define BS_SUBMIT_GET_SQE_DECL(sqe) \
    sqe = get_sqe();\
    if (!sqe)\
    {\
        /* Pause until there are more requests available */\
        PRIV(op)->wait_detail = 1;\
        PRIV(op)->wait_for = WAIT_SQE;\
        return 0;\
    }
 #include "blockstore_journal.h"
 // "VITAstor"
 #define BLOCKSTORE_META_MAGIC_V1 0x726F747341544956l
 #define BLOCKSTORE_META_FORMAT_V1 1
 #define BLOCKSTORE_META_FORMAT_V2 2
 // metadata header (superblock)
 struct __attribute__((__packed__)) blockstore_meta_header_v1_t
 {
    uint64_t zero;
    uint64_t magic;
    uint64_t version;
    uint32_t meta_block_size;
    uint32_t data_block_size;
    uint32_t bitmap_granularity;
 };
 struct __attribute__((__packed__)) blockstore_meta_header_v2_t
 {
    uint64_t zero;
    uint64_t magic;
    uint64_t version;
    uint32_t meta_block_size;
    uint32_t data_block_size;
    uint32_t bitmap_granularity;
    uint32_t data_csum_type;
    uint32_t csum_block_size;
    uint32_t header_csum;
 };
 // 32 bytes = 24 bytes + block bitmap (4 bytes by default) + external attributes (also bitmap, 4 bytes by default)
 // per "clean" entry on disk with fixed metadata tables
 struct __attribute__((__packed__)) clean_disk_entry
 {
    object_id oid;
    uint64_t version;
    uint8_t bitmap[];
    // Two more fields come after bitmap in metadata version 2:
    // uint32_t data_csum[];
    // uint32_t entry_csum;
 };
 // 32 = 16 + 16 bytes per "clean" entry in memory (object_id => clean_entry)
 struct __attribute__((__packed__)) clean_entry
 {
    uint64_t version;
    uint64_t location;
 };
 // 64 = 24 + 40 bytes per dirty entry in memory (obj_ver_id => dirty_entry). Plus checksums
 struct __attribute__((__packed__)) dirty_entry
 {
    uint32_t state;
    uint32_t flags;    // unneeded, but present for alignment
    uint64_t location; // location in either journal or data -> in BYTES
    uint32_t offset;   // data offset within object (stripe)
    uint32_t len;      // data length
    uint64_t journal_sector; // journal sector used for this entry
    void* dyn_data;    // dynamic data: external bitmap and data block checksums. may be a pointer to the in-memory journal
 };
 // - Sync must be submitted after previous writes/deletes (not before!)
 // - Reads to the same object must be submitted after previous writes/deletes
 //   are written (not necessarily synced) in their location. This is because we
 //   rely on read-modify-write for erasure coding and we must return new data
 //   to calculate parity for subsequent writes
 // - Writes may be submitted in any order, because they don't overlap. Each write
 //   goes into a new location - either on the journal device or on the data device
 // - Stable (stabilize) must be submitted after sync of that object is completed
 //   It's even OK to return an error to the caller if that object is not synced yet
 // - Journal trim may be processed only after all versions are moved to
 //   the main storage AND after all read operations for older versions complete
 // - If an operation can not be submitted because the ring is full
 //   we should stop submission of other operations. Otherwise some "scatter" reads
 //   may end up blocked for a long time.
 // Otherwise, the submit order is free, that is all operations may be submitted immediately
 // In fact, adding a write operation must immediately result in dirty_db being populated
 // Suspend operation until there are more free SQEs
 #define WAIT_SQE 1
 // Suspend operation until there are <wait_detail> bytes of free space in the journal on disk
 #define WAIT_JOURNAL 3
 // Suspend operation until the next journal sector buffer is free
 #define WAIT_JOURNAL_BUFFER 4
 // Suspend operation until there is some free space on the data device
 #define WAIT_FREE 5
 struct used_clean_obj_t
 {
    int refs;
    bool was_freed; // was freed by a parallel flush?
    bool was_changed; // was changed by a parallel flush?
 };
 // https://github.com/algorithm-ninja/cpp-btree
 // https://github.com/greg7mdp/sparsepp/ was used previously, but it was TERRIBLY slow after resizing
 // with sparsepp, random reads dropped to ~700 iops very fast with just as much as ~32k objects in the DB
 typedef btree::btree_map<object_id, clean_entry> blockstore_clean_db_t;
 typedef std::map<obj_ver_id, dirty_entry> blockstore_dirty_db_t;
 #include "blockstore_init.h"
 #include "blockstore_flush.h"
 #define PRIV(op) ((blockstore_op_private_t*)(op)->private_data)
 #define FINISH_OP(op) PRIV(op)->~blockstore_op_private_t(); std::function<void (blockstore_op_t*)>(op->callback)(op)
 struct blockstore_op_private_t
 {
    // Wait status
    int wait_for;
-    uint64_t wait_detail;
+    uint64_t wait_detail, wait_detail2;
    int pending_ops;
    int op_state;
    // Write, sync, stabilize
    uint32_t modified_block, modified_block2;
    // Read
    uint64_t clean_block_used;
    std::vector<copy_buffer_t> read_vec;
-    // Read, write
+    // Sync, write
-    uint64_t lsn;
+    uint64_t min_flushed_journal_sector, max_flushed_journal_sector;
    // Write
    uint64_t location;
    uint32_t write_type;
    // Stabilize, rollback
    int stab_pos;
    // Write
    struct iovec iov_zerofill[3];
    // Warning: must not have a default value here because it's written to before calling constructor in blockstore_write.cpp O_o
    uint64_t real_version;
    timespec tv_begin;
    // Sync
    std::vector<obj_ver_id> sync_big_writes, sync_small_writes;
 };
-struct bs_modified_block_t
+typedef uint32_t pool_id_t;
 typedef uint64_t pool_pg_id_t;
 #define POOL_ID_BITS 16
 struct pool_shard_settings_t
 {
-    bool sent;
+    uint32_t pg_count;
-    uint8_t *buf;
+    uint32_t pg_stripe_size;
 };
-class blockstore_impl_t: public blockstore_i
+#define STAB_SPLIT_DONE 1
 #define STAB_SPLIT_WAIT 2
 #define STAB_SPLIT_SYNC 3
 #define STAB_SPLIT_TODO 4
 class blockstore_impl_t
 {
 public:
    blockstore_disk_t dsk;
    /******* OPTIONS *******/
    bool readonly = false;
    // It is safe to disable fsync() if drive write cache is writethrough
    bool disable_data_fsync = false, disable_meta_fsync = false, disable_journal_fsync = false;
    // Enable if you want every operation to be executed with an "implicit fsync"
    // Suitable only for server SSDs with capacitors, requires disabled data and journal fsyncs
    int immediate_commit = IMMEDIATE_NONE;
    bool inmemory_meta = false;
    uint32_t meta_write_recheck_parallelism = 0;
    // Maximum and minimum flusher count
-    unsigned max_flusher_count = 0, min_flusher_count = 0;
+    unsigned max_flusher_count, min_flusher_count;
-    unsigned journal_trim_interval = 0;
+    unsigned journal_trim_interval;
    unsigned flusher_start_threshold = 0;
    // Maximum queue depth
    unsigned max_write_iodepth = 128;
    // Enable small (journaled) write throttling, useful for the SSD+HDD case
@@ -97,97 +269,141 @@ public:
    uint64_t autosync_writes = 128;
    // Log level (0-10)
    int log_level = 0;
    // Enable correct block checksum validation on objects updated with small writes when checksum block
    // is larger than bitmap_granularity, at the expense of extra metadata fsyncs during compaction
    bool perfect_csum_update = false;
    /******* END OF OPTIONS *******/
    struct ring_consumer_t ring_consumer;
-    blockstore_heap_t *heap = NULL;
+    std::map<pool_id_t, pool_shard_settings_t> clean_db_settings;
-    uint8_t* meta_superblock = NULL;
+    std::map<pool_pg_id_t, blockstore_clean_db_t> clean_db_shards;
-    uint8_t *buffer_area = NULL;
+    std::map<uint64_t, int> no_inode_stats;
    uint8_t *clean_bitmaps = NULL;
    blockstore_dirty_db_t dirty_db;
    std::vector<blockstore_op_t*> submit_queue;
-    int unsynced_data_write_count = 0, unsynced_buffer_write_count = 0, unsynced_meta_write_count = 0;
+    std::vector<obj_ver_id> unsynced_big_writes, unsynced_small_writes;
    int unsynced_big_write_count = 0, unstable_unsynced = 0;
    int unsynced_queued_ops = 0;
    allocator_t *data_alloc = NULL;
    uint64_t used_blocks = 0;
    uint8_t *zero_object = NULL;
-    std::vector<uint32_t> pending_modified_blocks;
+    void *metadata_buffer = NULL;
    robin_hood::unordered_flat_map<uint32_t, bs_modified_block_t> modified_blocks;
    struct journal_t journal;
    journal_flusher_t *flusher;
    int big_to_flush = 0;
    int write_iodepth = 0;
-    int inflight_big = 0;
+    bool alloc_dyn_data = false;
-    int intent_write_counter = 0;
+
-    bool fsyncing_data = false;
+    // clean data blocks referenced by read operations
    std::map<uint64_t, used_clean_obj_t> used_clean_objects;
    bool live = false, queue_stall = false;
-    ring_loop_i *ringloop = NULL;
+    ring_loop_t *ringloop;
-    timerfd_manager_t *tfd = NULL;
+    timerfd_manager_t *tfd;
-    bool stop_sync_submitted = false;
+    bool stop_sync_submitted;
    inline struct io_uring_sqe* get_sqe()
    {
        return ringloop->get_sqe();
    }
    friend class blockstore_init_meta;
    friend class blockstore_init_journal;
    friend struct blockstore_journal_check_t;
    friend class journal_flusher_t;
    friend class journal_flusher_co;
    void calc_lengths();
    void open_data();
    void open_meta();
    void open_journal();
    uint8_t* get_clean_entry_bitmap(uint64_t block_loc, int offset);
    blockstore_clean_db_t& clean_db_shard(object_id oid);
    void reshard_clean_db(pool_id_t pool_id, uint32_t pg_count, uint32_t pg_stripe_size);
    void recalc_inode_space_stats(uint64_t pool_id, bool per_inode);
    // Journaling
    void prepare_journal_sector_write(int sector, blockstore_op_t *op);
    void handle_journal_write(ring_data_t *data, uint64_t flush_id);
    void disk_error_abort(const char *op, int retval, int expected);
    // Asynchronous init
    int initialized;
    int metadata_buf_size;
    blockstore_init_meta* metadata_init_reader;
    blockstore_init_journal* journal_init_reader;
    void init();
    void check_wait(blockstore_op_t *op);
    void init_op(blockstore_op_t *op);
    // Read
-    int dequeue_read(blockstore_op_t *op);
+    int dequeue_read(blockstore_op_t *read_op);
    int fulfill_read(blockstore_op_t *op);
    uint32_t prepare_read(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end, uint32_t skip_csum);
    uint32_t prepare_read_with_bitmaps(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end, uint32_t skip_csum);
    uint32_t prepare_read_zero(std::vector<copy_buffer_t> & read_vec, uint32_t start, uint32_t end);
    uint32_t prepare_read_simple(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end, uint32_t skip_csum);
    void prepare_disk_read(std::vector<copy_buffer_t> & read_vec, int pos, heap_entry_t *obj, heap_entry_t *wr,
        uint32_t blk_start, uint32_t blk_end, uint32_t start, uint32_t end, uint32_t copy_flags);
    void find_holes(std::vector<copy_buffer_t> & read_vec, uint32_t item_start, uint32_t item_end,
-        std::function<void(int&, uint32_t, uint32_t)> callback);
+        std::function<int(int, bool, uint32_t, uint32_t)> callback);
-    void free_read_buffers(std::vector<copy_buffer_t> & rv);
+    int fulfill_read(blockstore_op_t *read_op,
        uint64_t &fulfilled, uint32_t item_start, uint32_t item_end,
        uint32_t item_state, uint64_t item_version, uint64_t item_location,
        uint64_t journal_sector, uint8_t *csum, int *dyn_data);
    bool fulfill_clean_read(blockstore_op_t *read_op, uint64_t & fulfilled,
        uint8_t *clean_entry_bitmap, int *dyn_data,
        uint32_t item_start, uint32_t item_end, uint64_t clean_loc, uint64_t clean_ver);
    int fill_partial_checksum_blocks(std::vector<copy_buffer_t> & rv, uint64_t & fulfilled,
        uint8_t *clean_entry_bitmap, int *dyn_data, bool from_journal, uint8_t *read_buf, uint64_t read_offset, uint64_t read_end);
    int pad_journal_read(std::vector<copy_buffer_t> & rv, copy_buffer_t & cp,
        uint64_t dirty_offset, uint64_t dirty_end, uint64_t dirty_loc, uint8_t *csum_ptr, int *dyn_data,
        uint64_t offset, uint64_t submit_len, uint64_t & blk_begin, uint64_t & blk_end, uint8_t* & blk_buf);
    bool read_range_fulfilled(std::vector<copy_buffer_t> & rv, uint64_t & fulfilled, uint8_t *read_buf,
        uint8_t *clean_entry_bitmap, uint32_t item_start, uint32_t item_end);
    bool read_checksum_block(blockstore_op_t *op, int rv_pos, uint64_t &fulfilled, uint64_t clean_loc);
    uint8_t* read_clean_meta_block(blockstore_op_t *read_op, uint64_t clean_loc, int rv_pos);
    bool verify_padded_checksums(uint8_t *clean_entry_bitmap, uint8_t *csum_buf, uint32_t offset,
        iovec *iov, int n_iov, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);
    bool verify_journal_checksums(uint8_t *csums, uint32_t offset,
        iovec *iov, int n_iov, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);
    bool verify_clean_padded_checksums(blockstore_op_t *op, uint64_t clean_loc, uint8_t *dyn_data, bool from_journal,
        iovec *iov, int n_iov, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);
    int fulfill_read_push(blockstore_op_t *op, void *buf, uint64_t offset, uint64_t len,
        uint32_t item_state, uint64_t item_version);
    void handle_read_event(ring_data_t *data, blockstore_op_t *op);
    bool verify_read_checksums(blockstore_op_t *op);
    // Write
    bool enqueue_write(blockstore_op_t *op);
-    void prepare_meta_block_write(uint32_t modified_block);
+    void cancel_all_writes(blockstore_op_t *op, blockstore_dirty_db_t::iterator dirty_it, int retval);
    bool meta_block_is_pending(uint32_t modified_block);
    bool intent_write_allowed(blockstore_op_t *op, heap_entry_t *obj);
    int dequeue_write(blockstore_op_t *op);
    int dequeue_del(blockstore_op_t *op);
    int continue_write(blockstore_op_t *op);
    void release_journal_sectors(blockstore_op_t *op);
    void handle_write_event(ring_data_t *data, blockstore_op_t *op);
    // Sync
    int continue_sync(blockstore_op_t *op);
-    bool submit_fsyncs(int & wait_count);
+    void ack_sync(blockstore_op_t *op);
    int do_sync(blockstore_op_t *op, int base_state);
    bool has_unsynced();
    // Stabilize
    int dequeue_stable(blockstore_op_t *op);
    int continue_stable(blockstore_op_t *op);
    void mark_stable(obj_ver_id ov, bool forget_dirty = false);
    void stabilize_object(object_id oid, uint64_t max_ver);
    blockstore_op_t* selective_sync(blockstore_op_t *op);
    int split_stab_op(blockstore_op_t *op, std::function<int(obj_ver_id v)> decider);
    // Rollback
    int dequeue_rollback(blockstore_op_t *op);
    int continue_rollback(blockstore_op_t *op);
    void mark_rolled_back(const obj_ver_id & ov);
    void erase_dirty(blockstore_dirty_db_t::iterator dirty_start, blockstore_dirty_db_t::iterator dirty_end, uint64_t clean_loc);
    void free_dirty_dyn_data(dirty_entry & e);
    // List
    void process_list(blockstore_op_t *op);
-/*public:*/
+public:
-    blockstore_impl_t(blockstore_config_t & config, ring_loop_i *ringloop, timerfd_manager_t *tfd, bool mock_mode = false);
+    blockstore_impl_t(blockstore_config_t & config, ring_loop_t *ringloop, timerfd_manager_t *tfd);
    ~blockstore_impl_t();
    void parse_config(blockstore_config_t & config);
    void parse_config(blockstore_config_t & config, bool init);
    // Event loop
@@ -211,19 +427,21 @@ public:
    // Simplified synchronous operation: get object bitmap & current version
    int read_bitmap(object_id oid, uint64_t target_version, void *bitmap, uint64_t *result_version = NULL);
    // Unstable writes are added here (map of object_id -> version)
    std::unordered_map<object_id, uint64_t> unstable_writes;
    // Space usage statistics
    std::map<uint64_t, uint64_t> inode_space_stats;
    // Set per-pool no_inode_stats
    void set_no_inode_stats(const std::vector<uint64_t> & pool_ids);
    // Print diagnostics to stdout
    void dump_diagnostics();
    // Get diagnostic string for an operation
    std::string get_op_diag(blockstore_op_t *op);
    const std::map<uint64_t, uint64_t> & get_inode_space_stats() { return heap->get_inode_space_stats(); }
    inline uint32_t get_block_size() { return dsk.data_block_size; }
    inline uint64_t get_block_count() { return dsk.block_count; }
-    uint64_t get_free_block_count();
+    inline uint64_t get_free_block_count() { return dsk.block_count - used_blocks; }
-    inline uint32_t get_bitmap_granularity() { return dsk.bitmap_granularity; }
+    inline uint32_t get_bitmap_granularity() { return dsk.disk_alignment; }
    inline uint64_t get_journal_size() { return dsk.journal_len; }
 };
@@ -15,7 +15,6 @@ class blockstore_init_meta
 {
    blockstore_impl_t *bs;
    int wait_state = 0;
    int wait_count = 0;
    bool zero_on_init = false;
    void *metadata_buffer = NULL;
    blockstore_init_meta_buf bufs[2] = {};
@@ -26,11 +25,47 @@ class blockstore_init_meta
    uint64_t next_offset = 0;
    uint64_t last_read_offset = 0;
    uint64_t entries_loaded = 0;
-    std::vector<uint32_t> recheck_mod;
+    unsigned entries_per_block = 0;
    int i = 0, j = 0;
    std::vector<uint64_t> entries_to_zero;
    bool handle_meta_block(uint8_t *buf, uint64_t count, uint64_t done_cnt);
    void handle_event(ring_data_t *data, int buf_num);
 public:
    blockstore_init_meta(blockstore_impl_t *bs);
    int loop();
 };
 struct bs_init_journal_done
 {
    void *buf;
    uint64_t pos, len;
 };
 class blockstore_init_journal
 {
    blockstore_impl_t *bs;
    int wait_state = 0, wait_count = 0, handle_res = 0;
    uint64_t entries_loaded = 0;
    uint32_t crc32_last = 0;
    bool started = false;
    uint64_t next_free;
    std::vector<bs_init_journal_done> done;
    std::vector<obj_ver_id> double_allocs;
    std::vector<iovec> small_write_data;
    uint64_t journal_pos = 0;
    uint64_t continue_pos = 0;
    void *init_write_buf = NULL;
    uint64_t init_write_sector = 0;
    bool wrapped = false;
    void *submitted_buf;
    struct io_uring_sqe *sqe;
    struct ring_data_t *data;
    journal_entry_start *je_start;
    std::function<void(ring_data_t*)> simple_callback;
    int handle_journal_part(void *buf, uint64_t done_pos, uint64_t len);
    void handle_event(ring_data_t *data);
    void erase_dirty_object(blockstore_dirty_db_t::iterator dirty_it);
 public:
    blockstore_init_journal(blockstore_impl_t* bs);
    int loop();
 };
@@ -1,61 +0,0 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 #pragma once
 #define BS_SUBMIT_CHECK_SQES(n) \
    if (ringloop->space_left() < (n))\
    {\
        /* Pause until there are more requests available */\
        PRIV(op)->wait_detail = (n);\
        PRIV(op)->wait_for = WAIT_SQE;\
        return 0;\
    }
 #define BS_SUBMIT_GET_SQE(sqe, data) \
    BS_SUBMIT_GET_ONLY_SQE(sqe); \
    struct ring_data_t *data = ((ring_data_t*)sqe->user_data)
 #define BS_SUBMIT_GET_ONLY_SQE(sqe) \
    struct io_uring_sqe *sqe = get_sqe();\
    if (!sqe)\
    {\
        /* Pause until there are more requests available */\
        PRIV(op)->wait_detail = 1;\
        PRIV(op)->wait_for = WAIT_SQE;\
        return 0;\
    }
 #define BS_SUBMIT_GET_SQE_DECL(sqe) \
    sqe = get_sqe();\
    if (!sqe)\
    {\
        /* Pause until there are more requests available */\
        PRIV(op)->wait_detail = 1;\
        PRIV(op)->wait_for = WAIT_SQE;\
        return 0;\
    }
 #define PRIV(op) ((blockstore_op_private_t*)(op)->private_data)
 #define FINISH_OP(op) PRIV(op)->~blockstore_op_private_t(); std::function<void (blockstore_op_t*)>(op->callback)(op)
 // Suspend operation until there are more free SQEs
 #define WAIT_SQE 1
 // Suspend operation until there are <wait_detail> bytes of free space in the journal on disk
 #define WAIT_COMPACTION 2
 #define COPY_BUF_JOURNAL    0x01
 #define COPY_BUF_DATA       0x02
 #define COPY_BUF_ZERO       0x04
 #define COPY_BUF_CSUM_FILL  0x08
 #define COPY_BUF_COALESCED  0x10
 #define COPY_BUF_PADDED     0x20
 #define COPY_BUF_SKIP_CSUM  0x40
 #ifndef RWF_ATOMIC
 #define RWF_ATOMIC 0x40
 #endif
 #ifndef RWF_DSYNC
 #define RWF_DSYNC 0x02
 #endif
@@ -1,10 +1,7 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
-#include "impl.h"
+#include "blockstore_impl.h"
 #include "internal.h"
 namespace v1 {
 blockstore_journal_check_t::blockstore_journal_check_t(blockstore_impl_t *bs)
 {
@@ -329,5 +326,3 @@ void journal_t::dump_diagnostics()
        journal_used_it == used_sectors.end() ? 0 : journal_used_it->second
    );
 }
 } // namespace v1
@@ -1,11 +1,12 @@
 // Metadata on-disk structures
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 #pragma once
 #include "crc32c.h"
 #include <set>
 #define MIN_JOURNAL_SIZE 4*1024*1024
 #define JOURNAL_MAGIC 0x4A33
 #define JOURNAL_VERSION_V1 1
 #define JOURNAL_VERSION_V2 2
@@ -144,61 +145,74 @@ inline uint32_t je_crc32(journal_entry *je)
    return crc32c(0x48674bc7, ((uint8_t*)je)+4, je->size-4);
 }
-// "VITAstor"
+struct journal_sector_info_t
 #define BLOCKSTORE_META_MAGIC_V1 0x726F747341544956l
 #define BLOCKSTORE_META_FORMAT_V1 1
 #define BLOCKSTORE_META_FORMAT_V2 2
 #define BLOCKSTORE_META_FORMAT_HEAP 3
 // metadata header (superblock)
 struct __attribute__((__packed__)) blockstore_meta_header_v1_t
 {
-    uint64_t zero;
+    uint64_t offset;
-    uint64_t magic;
+    uint64_t flush_count;
-    uint64_t version;
+    bool written;
-    uint32_t meta_block_size;
+    bool dirty;
-    uint32_t data_block_size;
+    uint64_t submit_id;
    uint32_t bitmap_granularity;
 };
-struct __attribute__((__packed__)) blockstore_meta_header_v2_t
+struct pending_journaling_t
 {
-    uint64_t zero;
+    int pending;
-    uint64_t magic;
+    int sector;
-    uint64_t version;
+    blockstore_op_t *op;
    uint32_t meta_block_size;
    uint32_t data_block_size;
    uint32_t bitmap_granularity;
    uint32_t data_csum_type;
    uint32_t csum_block_size;
    uint32_t header_csum;
 };
-struct __attribute__((__packed__)) blockstore_meta_header_v3_t
+struct journal_t
 {
-    uint64_t zero;
+    int fd;
-    uint64_t magic;
+    bool inmemory = false;
-    uint64_t version;
+    bool flush_journal = false;
-    uint32_t meta_block_size;
+    void *buffer = NULL;
    uint32_t data_block_size;
    uint32_t bitmap_granularity;
    uint32_t data_csum_type;
    uint32_t csum_block_size;
    uint32_t header_csum;
    uint64_t meta_area_size;
    uint64_t completed_lsn;
-    void set_crc32c();
+    uint64_t block_size;
    uint64_t offset, len;
    // Next free block offset
    uint64_t next_free = 0;
    // First occupied block offset
    uint64_t used_start = 0;
    // End of the last block not used for writing anymore
    uint64_t dirty_start = 0;
    uint32_t crc32_last = 0;
    // Current sector(s) used for writing
    void *sector_buf = NULL;
    journal_sector_info_t *sector_info = NULL;
    uint64_t sector_count;
    bool no_same_sector_overwrites = false;
    int cur_sector = 0;
    int in_sector_pos = 0;
    std::vector<int> submitting_sectors;
    std::multimap<uint64_t, pending_journaling_t> flushing_ops;
    uint64_t submit_id = 0;
    // Used sector map
    // May use ~ 80 MB per 1 GB of used journal space in the worst case
    std::map<uint64_t, uint64_t> used_sectors;
    ~journal_t();
    bool trim();
    uint64_t get_trim_pos();
    void dump_diagnostics();
    inline bool entry_fits(int size)
    {
        return !(block_size - in_sector_pos < size ||
            no_same_sector_overwrites && sector_info[cur_sector].written);
    }
 };
-// 32 bytes = 24 bytes + block bitmap (4 bytes by default) + external attributes (also bitmap, 4 bytes by default)
+struct blockstore_journal_check_t
 // per "clean" entry on disk with fixed metadata tables
 struct __attribute__((__packed__)) clean_disk_entry
 {
-    object_id oid;
+    blockstore_impl_t *bs;
-    uint64_t version;
+    uint64_t next_pos, next_sector, next_in_pos;
-    uint8_t bitmap[];
+    int sectors_to_write, first_sector;
-    // Two more fields come after bitmap in metadata version 2:
+    bool right_dir; // writing to the end or the beginning of the ring buffer
-    // uint32_t data_csum[];
+
-    // uint32_t entry_csum;
+    blockstore_journal_check_t(blockstore_impl_t *bs);
    int check_available(blockstore_op_t *op, int required, int size, int data_after);
 };
 journal_entry* prefill_single_journal_entry(journal_t & journal, uint16_t type, uint32_t size);
@@ -2,14 +2,8 @@
 // License: VNPL-1.1 (see README.md for details)
 #include <sys/file.h>
 #include <stdexcept>
 #include "blockstore_impl.h"
 void blockstore_impl_t::parse_config(blockstore_config_t & config)
 {
    return parse_config(config, false);
 }
 void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init)
 {
    // Online-configurable options:
@@ -20,14 +14,12 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init)
    }
    min_flusher_count = strtoull(config["min_flusher_count"].c_str(), NULL, 10);
    journal_trim_interval = strtoull(config["journal_trim_interval"].c_str(), NULL, 10);
    flusher_start_threshold = strtoull(config["flusher_start_threshold"].c_str(), NULL, 10);
    max_write_iodepth = strtoull(config["max_write_iodepth"].c_str(), NULL, 10);
    throttle_small_writes = config["throttle_small_writes"] == "true" || config["throttle_small_writes"] == "1" || config["throttle_small_writes"] == "yes";
    throttle_target_iops = strtoull(config["throttle_target_iops"].c_str(), NULL, 10);
    throttle_target_mbs = strtoull(config["throttle_target_mbs"].c_str(), NULL, 10);
    throttle_target_parallelism = strtoull(config["throttle_target_parallelism"].c_str(), NULL, 10);
    throttle_threshold_us = strtoull(config["throttle_threshold_us"].c_str(), NULL, 10);
    perfect_csum_update = config["perfect_csum_update"] == "true" || config["perfect_csum_update"] == "1" || config["perfect_csum_update"] == "yes";
    if (config["autosync_writes"] != "")
    {
        autosync_writes = strtoull(config["autosync_writes"].c_str(), NULL, 10);
@@ -36,17 +28,13 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init)
    {
        max_flusher_count = 256;
    }
-    if (!min_flusher_count)
+    if (!min_flusher_count || journal.flush_journal)
    {
        min_flusher_count = 1;
    }
    if (!journal_trim_interval)
    {
-        journal_trim_interval = 4096;
+        journal_trim_interval = 512;
    }
    if (!flusher_start_threshold)
    {
        flusher_start_threshold = 32;
    }
    if (!max_write_iodepth)
    {
@@ -80,6 +68,23 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init)
    {
        readonly = true;
    }
    if (config["disable_data_fsync"] == "true" || config["disable_data_fsync"] == "1" || config["disable_data_fsync"] == "yes")
    {
        disable_data_fsync = true;
    }
    if (config["disable_meta_fsync"] == "true" || config["disable_meta_fsync"] == "1" || config["disable_meta_fsync"] == "yes")
    {
        disable_meta_fsync = true;
    }
    if (config["disable_journal_fsync"] == "true" || config["disable_journal_fsync"] == "1" || config["disable_journal_fsync"] == "yes")
    {
        disable_journal_fsync = true;
    }
    if (config["flush_journal"] == "true" || config["flush_journal"] == "1" || config["flush_journal"] == "yes")
    {
        // Only flush journal and exit
        journal.flush_journal = true;
    }
    if (config["immediate_commit"] == "all")
    {
        immediate_commit = IMMEDIATE_ALL;
@@ -89,27 +94,85 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init)
        immediate_commit = IMMEDIATE_SMALL;
    }
    metadata_buf_size = strtoull(config["meta_buf_size"].c_str(), NULL, 10);
-    meta_write_recheck_parallelism = strtoull(config["meta_write_recheck_parallelism"].c_str(), NULL, 10);
+    inmemory_meta = config["inmemory_metadata"] != "false" && config["inmemory_metadata"] != "0" &&
        config["inmemory_metadata"] != "no";
    journal.sector_count = strtoull(config["journal_sector_buffer_count"].c_str(), NULL, 10);
    journal.no_same_sector_overwrites = config["journal_no_same_sector_overwrites"] == "true" ||
        config["journal_no_same_sector_overwrites"] == "1" || config["journal_no_same_sector_overwrites"] == "yes";
    journal.inmemory = config["inmemory_journal"] != "false" && config["inmemory_journal"] != "0" &&
        config["inmemory_journal"] != "no";
    log_level = strtoull(config["log_level"].c_str(), NULL, 10);
    // Validate
    if (journal.sector_count < 2)
    {
        journal.sector_count = 32;
    }
    if (metadata_buf_size < 65536)
    {
        metadata_buf_size = 4*1024*1024;
    }
-    if (metadata_buf_size % dsk.meta_block_size)
+    if (dsk.meta_device == dsk.data_device)
    {
-        throw std::runtime_error("metadata_buf_size should be a multiple of meta_block_size");
+        disable_meta_fsync = disable_data_fsync;
    }
-    if (!meta_write_recheck_parallelism)
+    if (dsk.journal_device == dsk.meta_device)
    {
-        meta_write_recheck_parallelism = 16;
+        disable_journal_fsync = disable_meta_fsync;
    }
-    if (immediate_commit != IMMEDIATE_NONE && !dsk.disable_journal_fsync)
+    if (immediate_commit != IMMEDIATE_NONE && !disable_journal_fsync)
    {
        throw std::runtime_error("immediate_commit requires disable_journal_fsync");
    }
-    if (immediate_commit == IMMEDIATE_ALL && !dsk.disable_data_fsync)
+    if (immediate_commit == IMMEDIATE_ALL && !disable_data_fsync)
    {
        throw std::runtime_error("immediate_commit=all requires disable_journal_fsync and disable_data_fsync");
    }
    // init some fields
    journal.block_size = dsk.journal_block_size;
    journal.next_free = dsk.journal_block_size;
    journal.used_start = dsk.journal_block_size;
    // no free space because sector is initially unmapped
    journal.in_sector_pos = dsk.journal_block_size;
 }
 void blockstore_impl_t::calc_lengths()
 {
    dsk.calc_lengths();
    journal.len = dsk.journal_len;
    journal.block_size = dsk.journal_block_size;
    journal.offset = dsk.journal_offset;
    if (inmemory_meta)
    {
        metadata_buffer = memalign(MEM_ALIGNMENT, dsk.meta_len);
        if (!metadata_buffer)
            throw std::runtime_error("Failed to allocate memory for the metadata ("+std::to_string(dsk.meta_len/1024/1024)+" MB)");
    }
    else if (dsk.clean_entry_bitmap_size || dsk.data_csum_type)
    {
        clean_bitmaps = (uint8_t*)malloc(dsk.block_count * 2 * dsk.clean_entry_bitmap_size);
        if (!clean_bitmaps)
        {
            throw std::runtime_error(
                "Failed to allocate memory for the metadata sparse write bitmap ("+
                std::to_string(dsk.block_count * 2 * dsk.clean_entry_bitmap_size / 1024 / 1024)+" MB)"
            );
        }
    }
    if (journal.inmemory)
    {
        journal.buffer = memalign(MEM_ALIGNMENT, journal.len);
        if (!journal.buffer)
            throw std::runtime_error("Failed to allocate memory for journal ("+std::to_string(journal.len/1024/1024)+" MB)");
    }
    else
    {
        journal.sector_buf = (uint8_t*)memalign(MEM_ALIGNMENT, journal.sector_count * dsk.journal_block_size);
        if (!journal.sector_buf)
            throw std::bad_alloc();
    }
    journal.sector_info = (journal_sector_info_t*)calloc(journal.sector_count, sizeof(journal_sector_info_t));
    if (!journal.sector_info)
    {
        throw std::bad_alloc();
    }
 }
@@ -1,10 +1,7 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
-#include "impl.h"
+#include "blockstore_impl.h"
 #include "internal.h"
 namespace v1 {
 int blockstore_impl_t::dequeue_rollback(blockstore_op_t *op)
 {
@@ -213,10 +210,10 @@ void blockstore_impl_t::erase_dirty(blockstore_dirty_db_t::iterator dirty_start,
            dirty_it->second.location != UINT64_MAX)
        {
 #ifdef BLOCKSTORE_DEBUG
-            printf("Free block %ju from %jx:%jx v%ju\n", dirty_it->second.location / dsk.data_block_size,
+            printf("Free block %ju from %jx:%jx v%ju\n", dirty_it->second.location >> dsk.block_order,
                dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version);
 #endif
-            data_alloc->set(dirty_it->second.location / dsk.data_block_size, false);
+            data_alloc->set(dirty_it->second.location >> dsk.block_order, false);
        }
        auto used = --journal.used_sectors.at(dirty_it->second.journal_sector);
 #ifdef BLOCKSTORE_DEBUG
@@ -259,5 +256,3 @@ void blockstore_impl_t::free_dirty_dyn_data(dirty_entry & e)
        e.dyn_data = NULL;
    }
 }
 } // namespace v1
@@ -2,91 +2,560 @@
 // License: VNPL-1.1 (see README.md for details)
 #include "blockstore_impl.h"
 #include "blockstore_internal.h"
-// Handles both stabilize (commit) and rollback
+// Stabilize small write:
 // 1) Copy data from the journal to the data device
 // 2) Increase version on the metadata device and sync it
 // 3) Advance clean_db entry's version, clear previous journal entries
 //
 // This makes 1 4K small write+sync look like:
 // 512b+4K (journal) + sync + 512b (journal) + sync + 4K (data) [+ sync?] + 512b (metadata) + sync.
 // WA = 2.375. It's not the best, SSD FTL-like redirect-write could probably be lower
 // even with defragmentation. But it's fixed and it's still better than in Ceph. :)
 // except for HDD-only clusters, because each write results in 3 seeks.
 // Stabilize big write:
 // 1) Copy metadata from the journal to the metadata device
 // 2) Move dirty_db entry to clean_db and clear previous journal entries
 //
 // This makes 1 128K big write+sync look like:
 // 128K (data) + sync + 512b (journal) + sync + 512b (journal) + sync + 512b (metadata) + sync.
 // WA = 1.012. Very good :)
 // Stabilize delete:
 // 1) Remove metadata entry and sync it
 // 2) Remove dirty_db entry and clear previous journal entries
 // We have 2 problems here:
 // - In the cluster environment, we must store the "tombstones" of deleted objects until
 //   all replicas (not just quorum) agrees about their deletion. That is, "stabilize" is
 //   not possible for deletes in degraded placement groups
 // - With simple "fixed" metadata tables we can't just clear the metadata entry of the latest
 //   object version. We must clear all previous entries, too.
 // FIXME Fix both problems - probably, by switching from "fixed" metadata tables to "dynamic"
 // AND We must do it in batches, for the sake of reduced fsync call count
 // AND We must know what we stabilize. Basic workflow is like:
 // 1) primary OSD receives sync request
 // 2) it submits syncs to blockstore and peers
 // 3) after everyone acks sync it acks sync to the client
 // 4) after a while it takes his synced object list and sends stabilize requests
 //    to peers and to its own blockstore, thus freeing the old version
 struct ver_vector_t
 {
    obj_ver_id *items = NULL;
    uint64_t alloc = 0, size = 0;
 };
 static void init_versions(ver_vector_t & vec, obj_ver_id *start, obj_ver_id *end, uint64_t len)
 {
    if (!vec.items)
    {
        vec.alloc = len;
        vec.items = (obj_ver_id*)malloc_or_die(sizeof(obj_ver_id) * vec.alloc);
        for (auto sv = start; sv < end; sv++)
        {
            vec.items[vec.size++] = *sv;
        }
    }
 }
 static void append_version(ver_vector_t & vec, obj_ver_id ov)
 {
    if (vec.size >= vec.alloc)
    {
        vec.alloc = !vec.alloc ? 4 : vec.alloc*2;
        vec.items = (obj_ver_id*)realloc_or_die(vec.items, sizeof(obj_ver_id) * vec.alloc);
    }
    vec.items[vec.size++] = ov;
 }
 static bool check_unsynced(std::vector<obj_ver_id> & check, obj_ver_id ov, std::vector<obj_ver_id> & to, int *count)
 {
    bool found = false;
    int j = 0, k = 0;
    while (j < check.size())
    {
        if (check[j] == ov)
            found = true;
        if (check[j].oid == ov.oid && check[j].version <= ov.version)
        {
            to.push_back(check[j++]);
            if (count)
                (*count)--;
        }
        else
            check[k++] = check[j++];
    }
    check.resize(k);
    return found;
 }
 blockstore_op_t* blockstore_impl_t::selective_sync(blockstore_op_t *op)
 {
    unsynced_big_write_count -= unsynced_big_writes.size();
    unsynced_big_writes.swap(PRIV(op)->sync_big_writes);
    unsynced_big_write_count += unsynced_big_writes.size();
    unsynced_small_writes.swap(PRIV(op)->sync_small_writes);
    // Create a sync operation, insert into the end of the queue
    // And move ourselves into the end too!
    // Rather hacky but that's what we need...
    blockstore_op_t *sync_op = new blockstore_op_t;
    sync_op->opcode = BS_OP_SYNC;
    sync_op->buf = NULL;
    sync_op->callback = [](blockstore_op_t *sync_op)
    {
        delete sync_op;
    };
    init_op(sync_op);
    int sync_res = continue_sync(sync_op);
    if (sync_res != 2)
    {
        // Put SYNC into the queue if it's not finished yet
        submit_queue.push_back(sync_op);
    }
    // Restore unsynced_writes
    unsynced_small_writes.swap(PRIV(op)->sync_small_writes);
    unsynced_big_write_count -= unsynced_big_writes.size();
    unsynced_big_writes.swap(PRIV(op)->sync_big_writes);
    unsynced_big_write_count += unsynced_big_writes.size();
    if (sync_res == 2)
    {
        // Sync is immediately completed
        return NULL;
    }
    return sync_op;
 }
 // Returns: 2 = stop processing and dequeue, 0 = stop processing and do not dequeue, 1 = proceed with op itself
 int blockstore_impl_t::split_stab_op(blockstore_op_t *op, std::function<int(obj_ver_id v)> decider)
 {
    bool add_sync = false;
    ver_vector_t good_vers, bad_vers;
    obj_ver_id* v;
    int i, todo = 0;
    for (i = 0, v = (obj_ver_id*)op->buf; i < op->len; i++, v++)
    {
        int action = decider(*v);
        if (action < 0)
        {
            // Rollback changes
            for (auto & ov: PRIV(op)->sync_big_writes)
            {
                unsynced_big_writes.push_back(ov);
                unsynced_big_write_count++;
            }
            for (auto & ov: PRIV(op)->sync_small_writes)
            {
                unsynced_small_writes.push_back(ov);
            }
            free(good_vers.items);
            good_vers.items = NULL;
            free(bad_vers.items);
            bad_vers.items = NULL;
            // Error
            op->retval = action;
            FINISH_OP(op);
            return 2;
        }
        else if (action == STAB_SPLIT_DONE)
        {
            // Already done
            init_versions(good_vers, (obj_ver_id*)op->buf, v, op->len);
        }
        else if (action == STAB_SPLIT_WAIT)
        {
            // Already in progress, we just have to wait until it finishes
            init_versions(good_vers, (obj_ver_id*)op->buf, v, op->len);
            append_version(bad_vers, *v);
        }
        else if (action == STAB_SPLIT_SYNC)
        {
            // Needs a SYNC, we have to send a SYNC if not already in progress
            //
            // If the object is not present in unsynced_(big|small)_writes then
            // it's currently being synced. If it's present then we can initiate
            // its sync ourselves.
            init_versions(good_vers, (obj_ver_id*)op->buf, v, op->len);
            append_version(bad_vers, *v);
            if (!add_sync)
            {
                PRIV(op)->sync_big_writes.clear();
                PRIV(op)->sync_small_writes.clear();
                add_sync = true;
            }
            check_unsynced(unsynced_small_writes, *v, PRIV(op)->sync_small_writes, NULL);
            check_unsynced(unsynced_big_writes, *v, PRIV(op)->sync_big_writes, &unsynced_big_write_count);
        }
        else /* if (action == STAB_SPLIT_TODO) */
        {
            if (good_vers.items)
            {
                // If we're selecting versions then append it
                // Main idea is that 99% of the time all versions passed to BS_OP_STABLE are synced
                // And we don't want to select/allocate anything in that optimistic case
                append_version(good_vers, *v);
            }
            todo++;
        }
    }
    // In a pessimistic scenario, an operation may be split into 3:
    // - Stabilize synced entries
    // - Sync unsynced entries
    // - Continue for unsynced entries after sync
    add_sync = add_sync && (PRIV(op)->sync_big_writes.size() || PRIV(op)->sync_small_writes.size());
    if (!todo && !bad_vers.size)
    {
        // Already stable
        op->retval = 0;
        FINISH_OP(op);
        return 2;
    }
    op->retval = 0;
    if (!todo && !add_sync)
    {
        // Only wait for inflight writes or current in-progress syncs
        return 0;
    }
    blockstore_op_t *sync_op = NULL, *split_stab_op = NULL;
    if (add_sync)
    {
        // Initiate a selective sync for PRIV(op)->sync_(big|small)_writes
        sync_op = selective_sync(op);
    }
    if (bad_vers.size)
    {
        // Split part of the request into a separate operation
        split_stab_op = new blockstore_op_t;
        split_stab_op->opcode = op->opcode;
        split_stab_op->buf = bad_vers.items;
        split_stab_op->len = bad_vers.size;
        init_op(split_stab_op);
        submit_queue.push_back(split_stab_op);
    }
    if (sync_op || split_stab_op || good_vers.items)
    {
        void *orig_buf = op->buf;
        if (good_vers.items)
        {
            op->buf = good_vers.items;
            op->len = good_vers.size;
        }
        // Make a wrapped callback
        int *split_op_counter = (int*)malloc_or_die(sizeof(int));
        *split_op_counter = (sync_op ? 1 : 0) + (split_stab_op ? 1 : 0) + (todo ? 1 : 0);
        auto cb = [op, good_items = good_vers.items,
            bad_items = bad_vers.items, split_op_counter,
            orig_buf, real_cb = op->callback](blockstore_op_t *split_op)
        {
            if (split_op->retval != 0)
                op->retval = split_op->retval;
            (*split_op_counter)--;
            assert((*split_op_counter) >= 0);
            if (op != split_op)
                delete split_op;
            if (!*split_op_counter)
            {
                free(good_items);
                free(bad_items);
                free(split_op_counter);
                op->buf = orig_buf;
                real_cb(op);
            }
        };
        if (sync_op)
        {
            sync_op->callback = cb;
        }
        if (split_stab_op)
        {
            split_stab_op->callback = cb;
        }
        op->callback = cb;
    }
    if (!todo)
    {
        // All work is postponed
        op->callback = NULL;
        return 2;
    }
    return 1;
 }
 int blockstore_impl_t::dequeue_stable(blockstore_op_t *op)
 {
-    obj_ver_id *v = (obj_ver_id*)op->buf;
+    if (PRIV(op)->op_state)
    auto priv = PRIV(op);
    if (priv->op_state == 1)      goto resume_1;
    else if (priv->op_state == 2) goto resume_2;
    else if (priv->op_state == 3) goto resume_3;
    else if (priv->op_state == 4) goto resume_4;
    else if (priv->op_state == 5) goto resume_5;
    assert(!priv->op_state);
    op->retval = 0;
    priv->modified_block = priv->modified_block2 = UINT32_MAX;
    for (priv->stab_pos = 0; priv->stab_pos < op->len; priv->stab_pos++)
    {
        return continue_stable(op);
    }
    int r = split_stab_op(op, [this](obj_ver_id ov)
    {
        auto dirty_it = dirty_db.find(ov);
        if (dirty_it == dirty_db.end())
        {
-            auto obj = heap->read_entry(v[priv->stab_pos].oid);
+            auto & clean_db = clean_db_shard(ov.oid);
-            if (!obj)
+            auto clean_it = clean_db.find(ov.oid);
            if (clean_it == clean_db.end() || clean_it->second.version < ov.version)
            {
-                op->retval = -ENOENT;
+                // No such object version
-                FINISH_OP(op);
+                printf("Error: %jx:%jx v%ju not found while stabilizing\n", ov.oid.inode, ov.oid.stripe, ov.version);
-                return 2;
+                return -ENOENT;
            }
-            int res = op->opcode == BS_OP_STABLE
+            else
                ? heap->add_commit(obj, v[priv->stab_pos].version, &priv->modified_block2)
                : heap->add_rollback(obj, v[priv->stab_pos].version, &priv->modified_block2);
            if (res == EBUSY)
            {
-                op->retval = -EBUSY;
+                // Already stable
-                FINISH_OP(op);
+                return STAB_SPLIT_DONE;
                return 2;
            }
            if (res == ENOSPC)
            {
                if (!heap->get_to_compact_count())
                {
                    // no space
                    op->retval = -ENOSPC;
                    FINISH_OP(op);
                    return 2;
                }
                if (priv->modified_block2 != UINT32_MAX)
                {
                    priv->stab_pos--;
                    goto resume_1;
                }
                priv->wait_for = WAIT_COMPACTION;
                priv->wait_detail = heap->get_compacted_count();
                flusher->request_trim();
                return 0;
            }
            assert(res == 0);
        }
-        if (priv->modified_block != UINT32_MAX && priv->modified_block2 != priv->modified_block)
+        else if (IS_STABLE(dirty_it->second.state))
        {
-resume_1:
+            // Already stable
-            BS_SUBMIT_CHECK_SQES(1);
+            return STAB_SPLIT_DONE;
-            prepare_meta_block_write(priv->modified_block);
+        }
        while (true)
        {
            if (IS_IN_FLIGHT(dirty_it->second.state))
            {
                // Object write is still in progress. Wait until the write request completes
                return STAB_SPLIT_WAIT;
            }
            else if (!IS_SYNCED(dirty_it->second.state))
            {
                // Object not synced yet - sync it
                // In previous versions we returned EBUSY here and required
                // the caller (OSD) to issue a global sync first. But a global sync
                // waits for all writes in the queue including inflight writes. And
                // inflight writes may themselves be blocked by unstable writes being
                // still present in the journal and not flushed away from it.
                // So we must sync specific objects here.
                //
                // Even more, we have to process "stabilize" request in parts. That is,
                // we must stabilize all objects which are already synced. Otherwise
                // they may block objects which are NOT synced yet.
                return STAB_SPLIT_SYNC;
            }
            else if (IS_STABLE(dirty_it->second.state))
            {
                break;
            }
            // Check previous versions too
            if (dirty_it == dirty_db.begin())
            {
                break;
            }
            dirty_it--;
            if (dirty_it->first.oid != ov.oid)
            {
                break;
            }
        }
        return STAB_SPLIT_TODO;
    });
    if (r != 1)
    {
        return r;
    }
    // Check journal space
    blockstore_journal_check_t space_check(this);
    if (!space_check.check_available(op, op->len, sizeof(journal_entry_stable), 0))
    {
        return 0;
    }
    // There is sufficient space. Check SQEs
    BS_SUBMIT_CHECK_SQES(space_check.sectors_to_write);
    // Prepare and submit journal entries
    int s = 0;
    auto v = (obj_ver_id*)op->buf;
    for (int i = 0; i < op->len; i++, v++)
    {
        if (!journal.entry_fits(sizeof(journal_entry_stable)) &&
            journal.sector_info[journal.cur_sector].dirty)
        {
            prepare_journal_sector_write(journal.cur_sector, op);
            s++;
        }
        journal_entry_stable *je = (journal_entry_stable*)
            prefill_single_journal_entry(journal, JE_STABLE, sizeof(journal_entry_stable));
        je->oid = v->oid;
        je->version = v->version;
        je->crc32 = je_crc32((journal_entry*)je);
        journal.crc32_last = je->crc32;
    }
    prepare_journal_sector_write(journal.cur_sector, op);
    s++;
    assert(s == space_check.sectors_to_write);
    PRIV(op)->op_state = 1;
    return 1;
 }
 int blockstore_impl_t::continue_stable(blockstore_op_t *op)
 {
    if (PRIV(op)->op_state == 2)
        goto resume_2;
    else if (PRIV(op)->op_state == 4)
        goto resume_4;
    else
        return 1;
 resume_2:
-            if (meta_block_is_pending(priv->modified_block))
+    if (!disable_journal_fsync)
            {
                priv->op_state = 2;
                return 1;
            }
        }
        priv->modified_block = priv->modified_block2;
        if (priv->stab_pos == op->len-1 && priv->modified_block2 != UINT32_MAX)
        {
            priv->modified_block2 = UINT32_MAX;
            goto resume_1;
        }
    }
    // Fsync, just because our semantics imply that commit (stabilize) is immediately fsynced
    priv->op_state = 3;
 resume_3:
 resume_4:
 resume_5:
    int res = do_sync(op, 3);
    if (res != 2)
    {
-        return res;
+        BS_SUBMIT_GET_SQE(sqe, data);
        io_uring_prep_fsync(sqe, dsk.journal_fd, IORING_FSYNC_DATASYNC);
        data->iov = { 0 };
        data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
        PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 0;
        PRIV(op)->pending_ops = 1;
        PRIV(op)->op_state = 3;
        return 1;
    }
-    // Done. Don't touch op->retval - if anything resulted in ENOENT, return it as is
+resume_4:
    // Mark dirty_db entries as stable, acknowledge op completion
    obj_ver_id* v;
    int i;
    for (i = 0, v = (obj_ver_id*)op->buf; i < op->len; i++, v++)
    {
        // Mark all dirty_db entries up to op->version as stable
 #ifdef BLOCKSTORE_DEBUG
        printf("Stabilize %jx:%jx v%ju\n", v->oid.inode, v->oid.stripe, v->version);
 #endif
        mark_stable(*v);
    }
    // Acknowledge op
    op->retval = 0;
    FINISH_OP(op);
    return 2;
 }
 void blockstore_impl_t::mark_stable(obj_ver_id v, bool forget_dirty)
 {
    auto dirty_it = dirty_db.find(v);
    if (dirty_it != dirty_db.end())
    {
        if (IS_INSTANT(dirty_it->second.state))
        {
            // 'Instant' (non-EC) operations may complete and try to become stable out of order. Prevent it.
            auto back_it = dirty_it;
            while (back_it != dirty_db.begin())
            {
                back_it--;
                if (back_it->first.oid != v.oid)
                {
                    break;
                }
                if (!IS_STABLE(back_it->second.state))
                {
                    // There are preceding unstable versions, can't flush <v>
                    return;
                }
            }
            while (true)
            {
                dirty_it++;
                if (dirty_it == dirty_db.end() || dirty_it->first.oid != v.oid ||
                    !IS_SYNCED(dirty_it->second.state))
                {
                    dirty_it--;
                    break;
                }
                v.version = dirty_it->first.version;
            }
        }
        while (1)
        {
            bool was_stable = IS_STABLE(dirty_it->second.state);
            if ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_SYNCED)
            {
                dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | BS_ST_STABLE;
                // Allocations and deletions are counted when they're stabilized
                if (IS_BIG_WRITE(dirty_it->second.state))
                {
                    int exists = -1;
                    if (dirty_it != dirty_db.begin())
                    {
                        auto prev_it = dirty_it;
                        prev_it--;
                        if (prev_it->first.oid == v.oid)
                        {
                            exists = IS_DELETE(prev_it->second.state) ? 0 : 1;
                        }
                    }
                    if (exists == -1)
                    {
                        auto & clean_db = clean_db_shard(v.oid);
                        auto clean_it = clean_db.find(v.oid);
                        exists = clean_it != clean_db.end() ? 1 : 0;
                    }
                    if (!exists)
                    {
                        uint64_t space_id = dirty_it->first.oid.inode;
                        if (no_inode_stats[dirty_it->first.oid.inode >> (64-POOL_ID_BITS)])
                            space_id = space_id & ~(((uint64_t)1 << (64-POOL_ID_BITS)) - 1);
                        inode_space_stats[space_id] += dsk.data_block_size;
                        used_blocks++;
                    }
                    big_to_flush++;
                }
                else if (IS_DELETE(dirty_it->second.state))
                {
                    uint64_t space_id = dirty_it->first.oid.inode;
                    if (no_inode_stats[dirty_it->first.oid.inode >> (64-POOL_ID_BITS)])
                        space_id = space_id & ~(((uint64_t)1 << (64-POOL_ID_BITS)) - 1);
                    auto & sp = inode_space_stats[space_id];
                    if (sp > dsk.data_block_size)
                        sp -= dsk.data_block_size;
                    else
                        inode_space_stats.erase(space_id);
                    used_blocks--;
                    big_to_flush++;
                }
            }
            else if (IS_IN_FLIGHT(dirty_it->second.state))
            {
                // mark_stable should never be called for in-flight or submitted writes
                printf(
                    "BUG: Attempt to mark_stable object %jx:%jx v%ju state of which is %x\n",
                    dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version,
                    dirty_it->second.state
                );
                exit(1);
            }
            if (forget_dirty && (IS_BIG_WRITE(dirty_it->second.state) ||
                IS_DELETE(dirty_it->second.state)))
            {
                // Big write overrides all previous dirty entries
                auto erase_end = dirty_it;
                while (dirty_it != dirty_db.begin())
                {
                    dirty_it--;
                    if (dirty_it->first.oid != v.oid)
                    {
                        dirty_it++;
                        break;
                    }
                }
                auto & clean_db = clean_db_shard(v.oid);
                auto clean_it = clean_db.find(v.oid);
                uint64_t clean_loc = clean_it != clean_db.end()
                    ? clean_it->second.location : UINT64_MAX;
                erase_dirty(dirty_it, erase_end, clean_loc);
                break;
            }
            if (was_stable || dirty_it == dirty_db.begin())
            {
                break;
            }
            dirty_it--;
            if (dirty_it->first.oid != v.oid)
            {
                break;
            }
        }
        flusher->enqueue_flush(v);
    }
    auto unstab_it = unstable_writes.find(v.oid);
    if (unstab_it != unstable_writes.end() &&
        unstab_it->second <= v.version)
    {
        unstable_writes.erase(unstab_it);
    }
 }
@@ -2,122 +2,232 @@
 // License: VNPL-1.1 (see README.md for details)
 #include "blockstore_impl.h"
-#include "blockstore_internal.h"
+
 #define SYNC_HAS_SMALL 1
 #define SYNC_HAS_BIG 2
 #define SYNC_DATA_SYNC_SENT 3
 #define SYNC_DATA_SYNC_DONE 4
 #define SYNC_JOURNAL_WRITE_SENT 5
 #define SYNC_JOURNAL_WRITE_DONE 6
 #define SYNC_JOURNAL_SYNC_SENT 7
 #define SYNC_DONE 8
 int blockstore_impl_t::continue_sync(blockstore_op_t *op)
 {
-    if (!PRIV(op)->op_state)
+    if (immediate_commit == IMMEDIATE_ALL)
    {
        // We can return immediately because sync is only dequeued after all previous writes
        op->retval = 0;
    }
    int res = do_sync(op, 0);
    if (res == 2)
    {
        FINISH_OP(op);
    }
    return res;
 }
 bool blockstore_impl_t::has_unsynced()
 {
    bool data = (!dsk.disable_data_fsync && unsynced_data_write_count);
    bool buffer = (!dsk.disable_journal_fsync && unsynced_buffer_write_count);
    bool meta = (!dsk.disable_meta_fsync && unsynced_meta_write_count);
    return data || buffer || meta;
 }
 bool blockstore_impl_t::submit_fsyncs(int & wait_count)
 {
    int n = (unsynced_meta_write_count > 0 && !dsk.disable_meta_fsync) +
        (unsynced_buffer_write_count > 0 && !dsk.disable_journal_fsync && dsk.journal_fd != dsk.meta_fd) +
        (unsynced_data_write_count > 0 && !dsk.disable_data_fsync && dsk.data_fd != dsk.meta_fd && dsk.data_fd != dsk.journal_fd);
    if (ringloop->space_left() < n)
    {
        return false;
    }
    if (!n)
    {
        return true;
    }
    auto cb = [this, & wait_count](ring_data_t *data)
    {
        if (data->res != 0)
            disk_error_abort("sync meta", data->res, 0);
        wait_count--;
        assert(wait_count >= 0);
        if (!wait_count)
            ringloop->wakeup();
    };
    if (unsynced_meta_write_count > 0 && !dsk.disable_meta_fsync)
    {
        // fsync meta
        io_uring_sqe *sqe = get_sqe();
        assert(sqe);
        ring_data_t *data = ((ring_data_t*)sqe->user_data);
        io_uring_prep_fsync(sqe, dsk.meta_fd, IORING_FSYNC_DATASYNC);
        data->iov = { 0 };
        data->callback = cb;
        wait_count++;
    }
    if (unsynced_buffer_write_count > 0 && !dsk.disable_journal_fsync && dsk.meta_fd != dsk.journal_fd)
    {
        // fsync buffer
        io_uring_sqe *sqe = get_sqe();
        assert(sqe);
        ring_data_t *data = ((ring_data_t*)sqe->user_data);
        io_uring_prep_fsync(sqe, dsk.journal_fd, IORING_FSYNC_DATASYNC);
        data->iov = { 0 };
        data->callback = cb;
        wait_count++;
    }
    if (unsynced_data_write_count > 0 && !dsk.disable_data_fsync && dsk.data_fd != dsk.meta_fd && dsk.data_fd != dsk.journal_fd)
    {
        // fsync data
        io_uring_sqe *sqe = get_sqe();
        assert(sqe);
        ring_data_t *data = ((ring_data_t*)sqe->user_data);
        io_uring_prep_fsync(sqe, dsk.data_fd, IORING_FSYNC_DATASYNC);
        data->iov = { 0 };
        data->callback = cb;
        wait_count++;
    }
    unsynced_data_write_count = 0;
    unsynced_buffer_write_count = 0;
    unsynced_meta_write_count = 0;
    return true;
 }
 int blockstore_impl_t::do_sync(blockstore_op_t *op, int base_state)
 {
    int op_state = PRIV(op)->op_state - base_state;
    if (op_state == 1) goto resume_1;
    if (op_state == 2) goto resume_2;
    assert(!op_state);
    if (flusher->get_syncing_buffer())
    {
        // Wait for flusher-initiated sync
        return 0;
    }
    if (!has_unsynced())
    {
        // We can return immediately because sync only syncs previous writes
        unsynced_data_write_count = unsynced_buffer_write_count = unsynced_meta_write_count = 0;
        return 2;
    }
-    PRIV(op)->modified_block = heap->get_completed_lsn();
+    if (PRIV(op)->op_state == 0)
    if (!submit_fsyncs(PRIV(op)->pending_ops))
    {
-        PRIV(op)->wait_detail = 1;
+        stop_sync_submitted = false;
-        PRIV(op)->wait_for = WAIT_SQE;
+        unsynced_big_write_count -= unsynced_big_writes.size();
-        return 0;
+        PRIV(op)->sync_big_writes.swap(unsynced_big_writes);
        PRIV(op)->sync_small_writes.swap(unsynced_small_writes);
        unsynced_big_writes.clear();
        unsynced_small_writes.clear();
        if (PRIV(op)->sync_big_writes.size() > 0)
            PRIV(op)->op_state = SYNC_HAS_BIG;
        else if (PRIV(op)->sync_small_writes.size() > 0)
            PRIV(op)->op_state = SYNC_HAS_SMALL;
        else
            PRIV(op)->op_state = SYNC_DONE;
    }
-resume_1:
+    if (PRIV(op)->op_state == SYNC_HAS_SMALL)
    if (PRIV(op)->pending_ops > 0)
    {
-        PRIV(op)->op_state = base_state+1;
+        // No big writes, just fsync the journal
        if (journal.sector_info[journal.cur_sector].dirty)
        {
            // Write out the last journal sector if it happens to be dirty
            BS_SUBMIT_CHECK_SQES(1);
            prepare_journal_sector_write(journal.cur_sector, op);
            PRIV(op)->op_state = SYNC_JOURNAL_WRITE_SENT;
            return 1;
        }
        else
        {
            PRIV(op)->op_state = SYNC_JOURNAL_WRITE_DONE;
        }
    }
    if (PRIV(op)->op_state == SYNC_HAS_BIG)
    {
        // 1st step: fsync data
        if (!disable_data_fsync)
        {
            BS_SUBMIT_GET_SQE(sqe, data);
            io_uring_prep_fsync(sqe, dsk.data_fd, IORING_FSYNC_DATASYNC);
            data->iov = { 0 };
            data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
            PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 0;
            PRIV(op)->pending_ops = 1;
            PRIV(op)->op_state = SYNC_DATA_SYNC_SENT;
            return 1;
        }
        else
        {
            PRIV(op)->op_state = SYNC_DATA_SYNC_DONE;
        }
    }
    if (PRIV(op)->op_state == SYNC_DATA_SYNC_DONE)
    {
        // 2nd step: Data device is synced, prepare & write journal entries
        // Check space in the journal and journal memory buffers
        blockstore_journal_check_t space_check(this);
        if (dsk.csum_block_size)
        {
            // More complex check because all journal entries have different lengths
            int left = PRIV(op)->sync_big_writes.size();
            for (auto & sbw: PRIV(op)->sync_big_writes)
            {
                left--;
                auto & dirty_entry = dirty_db.at(sbw);
                uint64_t dyn_size = dsk.dirty_dyn_size(dirty_entry.offset, dirty_entry.len);
                if (!space_check.check_available(op, 1, sizeof(journal_entry_big_write) + dyn_size, 0))
                {
                    return 0;
                }
            }
        }
        else if (!space_check.check_available(op, PRIV(op)->sync_big_writes.size(),
            sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size, 0))
        {
            return 0;
        }
        // Check SQEs. Don't bother about merging, submit each journal sector as a separate request
        BS_SUBMIT_CHECK_SQES(space_check.sectors_to_write);
        // Prepare and submit journal entries
        auto it = PRIV(op)->sync_big_writes.begin();
        int s = 0;
        while (it != PRIV(op)->sync_big_writes.end())
        {
            auto & dirty_entry = dirty_db.at(*it);
            uint64_t dyn_size = dsk.dirty_dyn_size(dirty_entry.offset, dirty_entry.len);
            if (!journal.entry_fits(sizeof(journal_entry_big_write) + dyn_size) &&
                journal.sector_info[journal.cur_sector].dirty)
            {
                prepare_journal_sector_write(journal.cur_sector, op);
                s++;
            }
            journal_entry_big_write *je = (journal_entry_big_write*)prefill_single_journal_entry(
                journal, (dirty_entry.state & BS_ST_INSTANT) ? JE_BIG_WRITE_INSTANT : JE_BIG_WRITE,
                sizeof(journal_entry_big_write) + dyn_size
            );
            auto jsec = dirty_entry.journal_sector = journal.sector_info[journal.cur_sector].offset;
            assert(journal.next_free >= journal.used_start
                ? (jsec >= journal.used_start && jsec < journal.next_free)
                : (jsec >= journal.used_start || jsec < journal.next_free));
            journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++;
 #ifdef BLOCKSTORE_DEBUG
            printf(
                "journal offset %08jx is used by %jx:%jx v%ju (%ju refs)\n",
                dirty_entry.journal_sector, it->oid.inode, it->oid.stripe, it->version,
                journal.used_sectors[journal.sector_info[journal.cur_sector].offset]
            );
 #endif
            je->oid = it->oid;
            je->version = it->version;
            je->offset = dirty_entry.offset;
            je->len = dirty_entry.len;
            je->location = dirty_entry.location;
            memcpy((void*)(je+1), (alloc_dyn_data
                ? (uint8_t*)dirty_entry.dyn_data+sizeof(int) : (uint8_t*)&dirty_entry.dyn_data), dyn_size);
            je->crc32 = je_crc32((journal_entry*)je);
            journal.crc32_last = je->crc32;
            it++;
        }
        prepare_journal_sector_write(journal.cur_sector, op);
        s++;
        assert(s == space_check.sectors_to_write);
        PRIV(op)->op_state = SYNC_JOURNAL_WRITE_SENT;
        return 1;
    }
-resume_2:
+    if (PRIV(op)->op_state == SYNC_JOURNAL_WRITE_DONE)
-    heap->mark_lsn_fsynced(PRIV(op)->modified_block);
+    {
-    return 2;
+        if (!disable_journal_fsync)
        {
            BS_SUBMIT_GET_SQE(sqe, data);
            io_uring_prep_fsync(sqe, dsk.journal_fd, IORING_FSYNC_DATASYNC);
            data->iov = { 0 };
            data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
            PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 0;
            PRIV(op)->pending_ops = 1;
            PRIV(op)->op_state = SYNC_JOURNAL_SYNC_SENT;
            return 1;
        }
        else
        {
            PRIV(op)->op_state = SYNC_DONE;
        }
    }
    if (PRIV(op)->op_state == SYNC_DONE)
    {
        ack_sync(op);
        return 2;
    }
    return 1;
 }
 void blockstore_impl_t::ack_sync(blockstore_op_t *op)
 {
    // Handle states
    for (auto it = PRIV(op)->sync_big_writes.begin(); it != PRIV(op)->sync_big_writes.end(); it++)
    {
 #ifdef BLOCKSTORE_DEBUG
        printf("Ack sync big %jx:%jx v%ju\n", it->oid.inode, it->oid.stripe, it->version);
 #endif
        auto & unstab = unstable_writes[it->oid];
        unstab = unstab < it->version ? it->version : unstab;
        auto dirty_it = dirty_db.find(*it);
        dirty_it->second.state = ((dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | BS_ST_SYNCED);
        if (dirty_it->second.state & BS_ST_INSTANT)
        {
            mark_stable(dirty_it->first);
        }
        else
        {
            unstable_unsynced--;
            assert(unstable_unsynced >= 0);
        }
        dirty_it++;
        while (dirty_it != dirty_db.end() && dirty_it->first.oid == it->oid)
        {
            if ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_WAIT_BIG)
            {
                dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | BS_ST_IN_FLIGHT;
            }
            dirty_it++;
        }
    }
    for (auto it = PRIV(op)->sync_small_writes.begin(); it != PRIV(op)->sync_small_writes.end(); it++)
    {
 #ifdef BLOCKSTORE_DEBUG
        printf("Ack sync small %jx:%jx v%ju\n", it->oid.inode, it->oid.stripe, it->version);
 #endif
        auto & unstab = unstable_writes[it->oid];
        unstab = unstab < it->version ? it->version : unstab;
        if (dirty_db[*it].state == (BS_ST_DELETE | BS_ST_WRITTEN))
        {
            dirty_db[*it].state = (BS_ST_DELETE | BS_ST_SYNCED);
            // Deletions are treated as immediately stable
            mark_stable(*it);
        }
        else /* (BS_ST_INSTANT?) | BS_ST_SMALL_WRITE | BS_ST_WRITTEN */
        {
            dirty_db[*it].state = (dirty_db[*it].state & ~BS_ST_WORKFLOW_MASK) | BS_ST_SYNCED;
            if (dirty_db[*it].state & BS_ST_INSTANT)
            {
                mark_stable(*it);
            }
            else
            {
                unstable_unsynced--;
                assert(unstable_unsynced >= 0);
            }
        }
    }
    op->retval = 0;
    FINISH_OP(op);
 }
@@ -12,7 +12,7 @@
 // [LD_PRELOAD=libasan.so.8] \
 // fio -name=test -thread -ioengine=../build/src/blockstore/libfio_vitastor_blk.so \
 //     -bs=4k -direct=1 -rw=randwrite -iodepth=16 -size=900M -loops=10 \
-//     -bs_config='{"data_device":"./test_data.bin","meta_offset":0,"journal_offset":16777216,"data_offset":33554432,"disable_data_fsync":true,"meta_format":3,"immediate_commit":"all","log_level":100,"journal_no_same_sector_overwrites":true,"journal_sector_buffer_count":1024}'
+//     -bs_config='{"data_device":"./test_data.bin","meta_offset":0,"journal_offset":16777216,"data_offset":33554432,"disable_data_fsync":true,"immediate_commit":"all","journal_no_same_sector_overwrites":true}'
 //
 // Linear write:
 //
@@ -28,25 +28,20 @@
 #include "epoll_manager.h"
 #include "malloc_or_die.h"
 #include "json11/json11.hpp"
 #include "../util/robin_hood.h"
 #include "fio_headers.h"
 struct bs_data
 {
-    blockstore_i *bs;
+    blockstore_t *bs;
    epoll_manager_t *epmgr;
    ring_loop_t *ringloop;
    /* The list of completed io_u structs. */
    std::vector<io_u*> completed;
    robin_hood::unordered_flat_map<uint64_t, int> inflight_oids;
    std::vector<io_u*> postponed;
    int op_n = 0, inflight = 0;
    bool ec = false;
    bool imm = true;
    bool last_sync = false;
    bool trace = false;
    uint8_t *bitmap = NULL;
    uint32_t block_size = 0;
 };
 struct bs_options
@@ -142,7 +137,6 @@ static void bs_cleanup(struct thread_data *td)
        delete bsd->bs;
        delete bsd->epmgr;
        delete bsd->ringloop;
        free(bsd->bitmap);
        delete bsd;
    }
 }
@@ -166,12 +160,9 @@ static int bs_init(struct thread_data *td)
                config[p.first] = p.second.dump();
        }
    }
    bsd->bitmap = (uint8_t*)malloc_or_die(MAX_DATA_BLOCK_SIZE/512/8);
    memset(bsd->bitmap, 0, MAX_DATA_BLOCK_SIZE/512/8);
    bsd->ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
    bsd->epmgr = new epoll_manager_t(bsd->ringloop);
-    bsd->bs = blockstore_i::create(config, bsd->ringloop, bsd->epmgr->tfd);
+    bsd->bs = new blockstore_t(config, bsd->ringloop, bsd->epmgr->tfd);
    bsd->block_size = bsd->bs->get_block_size();
    bsd->imm = config.find("immediate_commit") == config.end() ||
        config["immediate_commit"] == "all";
    while (1)
@@ -186,33 +177,8 @@ static int bs_init(struct thread_data *td)
    return 0;
 }
 static enum fio_q_status _bs_queue(struct thread_data *td, struct io_u *io, bool force);
 static void _bs_retry(struct bs_data *bsd, uint64_t offset)
 {
    // Retry postponed ops
    auto inflight_it = bsd->inflight_oids.find(offset / bsd->block_size);
    assert(inflight_it != bsd->inflight_oids.end());
    inflight_it->second--;
    if (inflight_it->second > 0)
    {
        for (size_t i = 0; i < bsd->postponed.size(); i++)
        {
            auto oio = bsd->postponed[i];
            if (oio->offset/bsd->block_size == offset/bsd->block_size)
            {
                bsd->postponed.erase(bsd->postponed.begin()+i);
                _bs_queue((thread_data*)oio->engine_data, oio, true);
                break;
            }
        }
    }
    else
        bsd->inflight_oids.erase(inflight_it);
 }
 /* Begin read or write request. */
-static enum fio_q_status _bs_queue(struct thread_data *td, struct io_u *io, bool force)
+static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
 {
    bs_data *bsd = (bs_data*)td->io_ops_data;
    if (io->ddir == DDIR_SYNC && bsd->last_sync)
@@ -222,22 +188,10 @@ static enum fio_q_status _bs_queue(struct thread_data *td, struct io_u *io, bool
    fio_ro_check(td, io);
-    io->engine_data = td;
+    io->engine_data = bsd;
    if (io->ddir == DDIR_WRITE || io->ddir == DDIR_READ)
-        assert(io->xfer_buflen <= bsd->block_size);
+        assert(io->xfer_buflen <= bsd->bs->get_block_size());
    uint64_t stripe = io->offset / bsd->block_size;
    if (!force && io->ddir == DDIR_WRITE)
    {
        auto & inflight = bsd->inflight_oids[stripe];
        inflight++;
        if (inflight > 1)
        {
            bsd->postponed.push_back(io);
            return FIO_Q_QUEUED;
        }
    }
    blockstore_op_t *op = new blockstore_op_t;
    op->callback = NULL;
@@ -246,19 +200,18 @@ static enum fio_q_status _bs_queue(struct thread_data *td, struct io_u *io, bool
    {
    case DDIR_READ:
        op->opcode = BS_OP_READ;
-        op->buf = (uint8_t*)io->xfer_buf;
+        op->buf = io->xfer_buf;
        op->oid = {
            .inode = 1,
-            .stripe = stripe,
+            .stripe = io->offset / bsd->bs->get_block_size(),
        };
        op->version = UINT64_MAX; // last unstable
-        op->offset = io->offset % bsd->block_size;
+        op->offset = io->offset % bsd->bs->get_block_size();
        op->len = io->xfer_buflen;
        op->bitmap = bsd->bitmap;
        op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
        {
            io->error = op->retval < 0 ? -op->retval : 0;
-            bs_data *bsd = ((bs_data*)((thread_data*)io->engine_data)->io_ops_data);
+            bs_data *bsd = (bs_data*)io->engine_data;
            bsd->inflight--;
            bsd->completed.push_back(io);
            if (bsd->trace)
@@ -268,20 +221,19 @@ static enum fio_q_status _bs_queue(struct thread_data *td, struct io_u *io, bool
        break;
    case DDIR_WRITE:
        op->opcode = bsd->ec ? BS_OP_WRITE : BS_OP_WRITE_STABLE;
-        op->buf = (uint8_t*)io->xfer_buf;
+        op->buf = io->xfer_buf;
        op->oid = {
            .inode = 1,
-            .stripe = stripe,
+            .stripe = io->offset / bsd->bs->get_block_size(),
        };
        op->version = 0; // assign automatically
-        op->offset = io->offset % bsd->block_size;
+        op->offset = io->offset % bsd->bs->get_block_size();
        op->len = io->xfer_buflen;
        op->bitmap = bsd->bitmap;
        if (bsd->ec)
        {
            op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
            {
-                bs_data *bsd = ((bs_data*)((thread_data*)io->engine_data)->io_ops_data);
+                bs_data *bsd = (bs_data*)io->engine_data;
                if (bsd->trace)
                    printf("--- OP_WRITE %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
                if (op->retval < 0)
@@ -289,27 +241,25 @@ static enum fio_q_status _bs_queue(struct thread_data *td, struct io_u *io, bool
                    io->error = op->retval < 0 ? -op->retval : 0;
                    bsd->inflight--;
                    bsd->completed.push_back(io);
                    _bs_retry(bsd, io->offset);
                    delete op;
                }
                else
                {
                    auto stab_op = new blockstore_op_t;
                    stab_op->opcode = BS_OP_STABLE;
-                    stab_op->buf = (uint8_t*)malloc_or_die(sizeof(obj_ver_id));
+                    stab_op->buf = malloc_or_die(sizeof(obj_ver_id));
                    obj_ver_id *ver = (obj_ver_id *)stab_op->buf;
                    ver[0].oid = op->oid;
                    ver[0].version = op->version;
                    stab_op->len = 1;
                    stab_op->callback = [io, n](blockstore_op_t *op)
                    {
-                        bs_data *bsd = ((bs_data*)((thread_data*)io->engine_data)->io_ops_data);
+                        bs_data *bsd = (bs_data*)io->engine_data;
                        if (bsd->trace)
                            printf("--- OP_STABLE %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
                        io->error = op->retval < 0 ? -op->retval : 0;
                        bsd->inflight--;
                        bsd->completed.push_back(io);
                        _bs_retry(bsd, io->offset);
                        delete op;
                    };
                    bsd->bs->enqueue_op(stab_op);
@@ -321,13 +271,12 @@ static enum fio_q_status _bs_queue(struct thread_data *td, struct io_u *io, bool
        {
            op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
            {
-                bs_data *bsd = ((bs_data*)((thread_data*)io->engine_data)->io_ops_data);
+                bs_data *bsd = (bs_data*)io->engine_data;
                if (bsd->trace)
                    printf("--- OP_WRITE_STABLE %zx n=%d retval=%d\n", (size_t)op, n, op->retval);
                io->error = op->retval < 0 ? -op->retval : 0;
                bsd->inflight--;
                bsd->completed.push_back(io);
                _bs_retry(bsd, io->offset);
                delete op;
            };
        }
@@ -337,7 +286,7 @@ static enum fio_q_status _bs_queue(struct thread_data *td, struct io_u *io, bool
        op->opcode = BS_OP_SYNC;
        op->callback = [io, n = bsd->op_n](blockstore_op_t *op)
        {
-            bs_data *bsd = ((bs_data*)((thread_data*)io->engine_data)->io_ops_data);
+            bs_data *bsd = (bs_data*)io->engine_data;
            io->error = op->retval < 0 ? -op->retval : 0;
            bsd->completed.push_back(io);
            bsd->inflight--;
@@ -365,11 +314,6 @@ static enum fio_q_status _bs_queue(struct thread_data *td, struct io_u *io, bool
    return FIO_Q_QUEUED;
 }
 static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
 {
    return _bs_queue(td, io, false);
 }
 static int bs_getevents(struct thread_data *td, unsigned int min, unsigned int max, const struct timespec *t)
 {
    bs_data *bsd = (bs_data*)td->io_ops_data;
@@ -1,343 +0,0 @@
 // Variable-length O(1) disk space allocator
 // Copyright (c) Vitaliy Filippov, 2025+
 // License: VNPL-1.1 (see README.md for details)
 #include <assert.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <set>
 #include "multilist.h"
 multilist_alloc_t::multilist_alloc_t(uint32_t count, uint32_t maxn):
    count(count), maxn(maxn)
 {
    // not-so-memory-efficient: 16 MB memory per 1 GB buffer space, but buffer spaces are small, so OK
    assert(count > 1 && count < 0x80000000);
    sizes.resize(count);
    nexts.resize(count); // nexts[i] = 0 -> area is used; nexts[i] = 1 -> no next; nexts[i] >= 2 -> next item
    prevs.resize(count);
    heads.resize(maxn); // heads[i] = 0 -> empty list; heads[i] >= 1 -> list head
    sizes[0] = count;
    sizes[count-1] = -count; // end
    nexts[0] = 1;
    heads[maxn-1] = 1;
 #ifdef MULTILIST_TRACE
    print();
 #endif
 }
 bool multilist_alloc_t::is_free(uint32_t pos)
 {
    assert(pos < count);
    if (sizes[pos] < 0)
        pos += sizes[pos]+1;
    while (pos > 0 && !sizes[pos])
        pos--;
    return nexts[pos] > 0;
 }
 uint32_t multilist_alloc_t::find(uint32_t size)
 {
    assert(size > 0);
    assert(size <= maxn);
    for (uint32_t i = size-1; i < maxn; i++)
    {
        if (heads[i])
        {
            return heads[i]-1;
        }
    }
    return UINT32_MAX;
 }
 void multilist_alloc_t::verify()
 {
    std::set<uint32_t> reachable;
    for (int i = 0; i < maxn; i++)
    {
        uint32_t cur = heads[i];
        while (cur)
        {
            if (!nexts[cur-1])
            {
                fprintf(stderr, "ERROR: item %d from freelist %d is not free\n", cur-1, i);
                print();
                abort();
            }
            if (nexts[cur-1] >= count+2)
            {
                fprintf(stderr, "ERROR: next out of range at %d: %d\n", cur-1, nexts[cur-1]);
                print();
                abort();
            }
            if (!(i < maxn-1 ? sizes[cur-1] == i+1 : (sizes[cur-1] >= i+1)))
            {
                fprintf(stderr, "ERROR: item %d is in wrong freelist: expected size %d, but actual size is %d\n", cur-1, i+1, sizes[cur-1]);
                print();
                abort();
            }
            if (reachable.find(cur-1) != reachable.end())
            {
                fprintf(stderr, "ERROR: doubly-claimed item %d\n", cur-1);
                print();
                abort();
            }
            reachable.insert(cur-1);
            cur = nexts[cur-1]-1;
        }
    }
    for (int i = 0; i < count; )
    {
        if (sizes[i])
        {
            assert(i+sizes[i] <= count);
            if (sizes[i] > 1 && sizes[i+sizes[i]-1] != -sizes[i])
            {
                fprintf(stderr, "ERROR: start/end mismatch at %d: sizes[%d] should be %d, but is %d\n", i, i+sizes[i]-1, -sizes[i], sizes[i+sizes[i]-1]);
                print();
                abort();
            }
            for (int j = i+1; j < i+sizes[i]-1; j++)
            {
                if (sizes[j])
                {
                    fprintf(stderr, "ERROR: internal non-zero at %d: %d\n", j, sizes[j]);
                    print();
                    abort();
                }
            }
            if (nexts[i] && reachable.find(i) == reachable.end())
            {
                fprintf(stderr, "ERROR: %d is unreachable from heads\n", i);
                print();
                abort();
            }
            if (nexts[i] >= 2)
            {
                if (nexts[i] >= 2+count)
                {
                    fprintf(stderr, "ERROR: next out of range at %d: %d\n", i, nexts[i]);
                    print();
                    abort();
                }
                if (prevs[nexts[i]-2] != i+1)
                {
                    fprintf(stderr, "ERROR: prev[next] (%d) != this (%d) at %d", prevs[nexts[i]-2], i+1, i);
                    print();
                    abort();
                }
            }
            i += (sizes[i] > 1 ? sizes[i] : 1);
        }
        else
            i++;
    }
 }
 void multilist_alloc_t::print()
 {
    printf("heads:");
    for (int i = 0; i < maxn; i++)
        if (heads[i])
            printf(" %u=%u", i, heads[i]);
    printf("\n");
    printf("sizes:");
    for (int i = 0; i < count; i++)
        if (sizes[i])
            printf(" %d=%d", i, sizes[i]);
    printf("\n");
    printf("prevs:");
    for (int i = 0; i < count; i++)
        if (prevs[i])
            printf(" %d=%d", i, prevs[i]);
    printf("\n");
    printf("nexts:");
    for (int i = 0; i < count; i++)
        if (nexts[i])
            printf(" %d=%d", i, nexts[i]);
    printf("\n");
    printf("items:");
    for (int i = 0; i < count; )
    {
        if (sizes[i])
        {
            printf(" %u=(s:%d,n:%u,p:%u)", i, sizes[i], nexts[i], prevs[i]);
            assert(i+sizes[i] <= count);
            i += (sizes[i] > 1 ? sizes[i] : 1);
        }
        else
            i++;
    }
    printf("\n");
 }
 void multilist_alloc_t::use(uint32_t pos, uint32_t size)
 {
    assert(pos+size <= count && size > 0);
    if (sizes[pos] <= 0)
    {
        uint32_t start = pos;
        if (sizes[start] < 0)
            start += sizes[start]+1;
        else
            while (start > 0 && !sizes[start])
                start--;
        assert(sizes[start] >= size);
        use_full(start);
        uint32_t full = sizes[start];
        sizes[pos-1] = -pos+start;
        sizes[start] = pos-start;
        free(start);
        sizes[pos+size-1] = -size;
        sizes[pos] = size;
        if (pos+size < start+full)
        {
            sizes[start+full-1] = -(start+full-pos-size);
            sizes[pos+size] = start+full-pos-size;
            free(pos+size);
        }
    }
    else
    {
        assert(sizes[pos] >= size);
        use_full(pos);
        if (sizes[pos] > size)
        {
            uint32_t full = sizes[pos];
            sizes[pos+size-1] = -size;
            sizes[pos] = size;
            sizes[pos+full-1] = -full+size;
            sizes[pos+size] = full-size;
            free(pos+size);
        }
    }
 #ifdef MULTILIST_TRACE
    print();
 #endif
 }
 void multilist_alloc_t::use_full(uint32_t pos)
 {
    uint32_t prevsize = sizes[pos];
    assert(prevsize);
    assert(nexts[pos]);
    uint32_t pi = (prevsize < maxn ? prevsize : maxn)-1;
    if (heads[pi] == pos+1)
        heads[pi] = nexts[pos]-1;
    if (prevs[pos])
        nexts[prevs[pos]-1] = nexts[pos];
    if (nexts[pos] >= 2)
        prevs[nexts[pos]-2] = prevs[pos];
    prevs[pos] = 0;
    nexts[pos] = 0;
 }
 void multilist_alloc_t::free(uint32_t pos)
 {
    do_free(pos);
 #ifdef MULTILIST_TRACE
    print();
 #endif
 }
 void multilist_alloc_t::do_free(uint32_t pos)
 {
    assert(!nexts[pos]);
    uint32_t size = sizes[pos];
    assert(size > 0);
    // merge with previous?
    if (pos > 0 && nexts[pos+(sizes[pos-1] == 1 ? -1 : sizes[pos-1])] > 0)
    {
        assert(sizes[pos-1] < 0 || sizes[pos-1] == 1);
        uint32_t prevsize = sizes[pos-1] < 0 ? -sizes[pos-1] : 1;
        use_full(pos-prevsize);
        sizes[pos] = 0;
        sizes[pos-1] = 0;
        size += prevsize;
        pos -= prevsize;
        sizes[pos+size-1] = -size;
        sizes[pos] = size;
    }
    // merge with next?
    if (pos+size < count && nexts[pos+size] >= 1)
    {
        uint32_t nextsize = sizes[pos+size];
        use_full(pos+size);
        sizes[pos+size] = 0;
        sizes[pos+size-1] = 0;
        size += nextsize;
        sizes[pos+size-1] = -size;
        sizes[pos] = size;
    }
    uint32_t ni = (size < maxn ? size : maxn)-1;
    nexts[pos] = heads[ni]+1;
    prevs[pos] = 0;
    if (heads[ni])
        prevs[heads[ni]-1] = pos+1;
    heads[ni] = pos+1;
 }
 multilist_index_t::multilist_index_t(uint32_t count, uint32_t max_used, uint32_t init_used):
    count(count), max_used(max_used)
 {
    assert(init_used < max_used);
    nexts.resize(count, UINT32_MAX);
    prevs.resize(count, UINT32_MAX);
    heads.resize(max_used, UINT32_MAX);
    for (size_t i = 0; i < count-1; i++)
    {
        nexts[i] = i+1;
        prevs[i+1] = i;
    }
    prevs[0] = UINT32_MAX;
    nexts[count-1] = UINT32_MAX;
    heads[init_used] = 0;
 }
 uint32_t multilist_index_t::find(uint32_t wanted_used)
 {
    assert(wanted_used < max_used);
    return heads[wanted_used];
 }
 uint32_t multilist_index_t::next(uint32_t pos)
 {
    return nexts[pos];
 }
 void multilist_index_t::change(uint32_t pos, uint32_t old_used, uint32_t new_used)
 {
    if (new_used == old_used)
        return;
    assert(old_used < max_used && new_used < max_used);
    if (prevs[pos] != UINT32_MAX)
        nexts[prevs[pos]] = nexts[pos];
    if (nexts[pos] != UINT32_MAX)
        prevs[nexts[pos]] = prevs[pos];
    if (heads[old_used] == pos)
        heads[old_used] = nexts[pos];
    prevs[pos] = UINT32_MAX;
    if (heads[new_used] != UINT32_MAX)
        prevs[heads[new_used]] = pos;
    nexts[pos] = heads[new_used];
    heads[new_used] = pos;
 }
 void multilist_index_t::print()
 {
    printf("heads:");
    for (int i = 0; i < max_used; i++)
        if (heads[i] != UINT32_MAX)
            printf(" %u=%u", i, heads[i]);
    printf("\n");
    printf("prevs:");
    for (int i = 0; i < count; i++)
        if (prevs[i] != UINT32_MAX)
            printf(" %d=%d", i, prevs[i]);
    printf("\n");
    printf("nexts:");
    for (int i = 0; i < count; i++)
        if (nexts[i] != UINT32_MAX)
            printf(" %d=%d", i, nexts[i]);
    printf("\n");
 }
@@ -1,38 +0,0 @@
 // Variable-length O(1) disk space allocator
 // Copyright (c) Vitaliy Filippov, 2025+
 // License: VNPL-1.1 (see README.md for details)
 #pragma once
 #include <stdint.h>
 #include <vector>
 struct multilist_alloc_t
 {
    const uint32_t count, maxn;
    std::vector<int32_t> sizes;
    std::vector<uint32_t> nexts, prevs, heads;
    multilist_alloc_t(uint32_t count, uint32_t maxn);
    bool is_free(uint32_t pos);
    uint32_t find(uint32_t size);
    void use_full(uint32_t pos);
    void use(uint32_t pos, uint32_t size);
    void do_free(uint32_t pos);
    void free(uint32_t pos);
    void verify();
    void print();
 };
 struct multilist_index_t
 {
    const uint32_t count, max_used;
    std::vector<uint32_t> nexts, prevs, heads;
    // used should be always < max_used
    multilist_index_t(uint32_t count, uint32_t max_used, uint32_t init_used);
    uint32_t find(uint32_t wanted_used);
    uint32_t next(uint32_t pos);
    void change(uint32_t pos, uint32_t old_used, uint32_t new_used);
    void print();
 };
@@ -1,134 +0,0 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 struct copy_buffer_t
 {
    int copy_flags;
    uint64_t offset, len, disk_offset;
    uint64_t journal_sector; // only for reads: sector+1 if used and !journal.inmemory, otherwise 0
    void *buf;
    uint8_t *csum_buf;
    int *dyn_data;
 };
 struct meta_sector_t
 {
    uint64_t offset, len;
    int state;
    void *buf;
    int usage_count;
 };
 struct flusher_sync_t
 {
    bool fsync_meta;
    int ready_count;
    int state;
 };
 struct flusher_meta_write_t
 {
    uint64_t sector, pos;
    bool submitted;
    void *buf;
    std::map<uint64_t, meta_sector_t>::iterator it;
 };
 class journal_flusher_t;
 // Journal flusher coroutine
 class journal_flusher_co
 {
    blockstore_impl_t *bs;
    journal_flusher_t *flusher;
    int wait_state, wait_count, wait_journal_count;
    struct io_uring_sqe *sqe;
    struct ring_data_t *data;
    std::list<flusher_sync_t>::iterator cur_sync;
    obj_ver_id cur;
    std::map<obj_ver_id, dirty_entry>::iterator dirty_it, dirty_start, dirty_end;
    std::map<object_id, uint64_t>::iterator repeat_it;
    std::function<void(ring_data_t*)> simple_callback_r, simple_callback_rj, simple_callback_w;
    bool try_trim = false;
    bool skip_copy, has_delete, has_writes;
    std::vector<copy_buffer_t> v;
    std::vector<copy_buffer_t>::iterator it;
    int i;
    bool fill_incomplete, cleared_incomplete;
    int read_to_fill_incomplete;
    int copy_count;
    uint64_t clean_loc, clean_ver, old_clean_loc, old_clean_ver;
    flusher_meta_write_t meta_old, meta_new;
    bool clean_init_bitmap;
    uint64_t clean_bitmap_offset, clean_bitmap_len;
    uint8_t *clean_init_dyn_ptr;
    uint8_t *new_clean_bitmap;
    uint64_t new_trim_pos;
    friend class journal_flusher_t;
    void scan_dirty();
    bool read_dirty(int wait_base);
    bool modify_meta_do_reads(int wait_base);
    bool wait_meta_reads(int wait_base);
    bool modify_meta_read(uint64_t meta_loc, flusher_meta_write_t &wr, int wait_base);
    bool clear_incomplete_csum_block_bits(int wait_base);
    void calc_block_checksums(uint32_t *new_data_csums, bool skip_overwrites);
    void update_metadata_entry();
    bool write_meta_block(flusher_meta_write_t & meta_block, int wait_base);
    void update_clean_db();
    void free_data_blocks();
    bool fsync_batch(bool fsync_meta, int wait_base);
    bool trim_journal(int wait_base);
    void free_buffers();
 public:
    journal_flusher_co();
    bool loop();
 };
 // Journal flusher itself
 class journal_flusher_t
 {
    int trim_wanted = 0;
    bool dequeuing;
    int min_flusher_count, max_flusher_count, cur_flusher_count, target_flusher_count;
    int flusher_start_threshold;
    journal_flusher_co *co;
    blockstore_impl_t *bs;
    friend class journal_flusher_co;
    int journal_trim_counter;
    bool trimming;
    void* journal_superblock;
    int active_flushers;
    int syncing_flushers;
    std::list<flusher_sync_t> syncs;
    std::map<object_id, uint64_t> sync_to_repeat;
    std::map<uint64_t, meta_sector_t> meta_sectors;
    std::deque<object_id> flush_queue;
    std::unordered_map<object_id, uint64_t> flush_versions;
    std::unordered_set<uint64_t> inflight_meta_sectors;
    bool try_find_older(std::map<obj_ver_id, dirty_entry>::iterator & dirty_end, obj_ver_id & cur);
    bool try_find_other(std::map<obj_ver_id, dirty_entry>::iterator & dirty_end, obj_ver_id & cur);
 public:
    journal_flusher_t(blockstore_impl_t *bs);
    ~journal_flusher_t();
    void loop();
    bool is_trim_wanted() { return trim_wanted; }
    bool is_active();
    void mark_trim_possible();
    void request_trim();
    void release_trim();
    void enqueue_flush(obj_ver_id oid);
    void unshift_flush(obj_ver_id oid, bool force);
    void remove_flush(object_id oid);
    void dump_diagnostics();
    bool is_mutated(uint64_t clean_loc);
 };
@@ -1,810 +0,0 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 #include "impl.h"
 #include "internal.h"
 namespace v1 {
 blockstore_impl_t::blockstore_impl_t(blockstore_config_t & config, ring_loop_i *ringloop, timerfd_manager_t *tfd)
 {
    assert(sizeof(blockstore_op_private_t) <= BS_OP_PRIVATE_DATA_SIZE);
    this->tfd = tfd;
    this->ringloop = ringloop;
    ring_consumer.loop = [this]() { loop(); };
    ringloop->register_consumer(&ring_consumer);
    initialized = 0;
    parse_config(config, true);
    try
    {
        dsk.open_data();
        dsk.open_meta();
        dsk.open_journal();
        calc_lengths();
        alloc_dyn_data = dsk.clean_dyn_size > sizeof(void*) || dsk.csum_block_size > 0;
        zero_object = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.data_block_size);
        data_alloc = new allocator_t(dsk.block_count);
    }
    catch (std::exception & e)
    {
        dsk.close_all();
        throw;
    }
    flusher = new journal_flusher_t(this);
 }
 blockstore_impl_t::~blockstore_impl_t()
 {
    delete data_alloc;
    delete flusher;
    if (zero_object)
        free(zero_object);
    ringloop->unregister_consumer(&ring_consumer);
    dsk.close_all();
    if (metadata_buffer)
        free(metadata_buffer);
    if (clean_bitmaps)
        free(clean_bitmaps);
 }
 bool blockstore_impl_t::is_started()
 {
    return initialized == 10;
 }
 bool blockstore_impl_t::is_stalled()
 {
    return queue_stall;
 }
 // main event loop - produce requests
 void blockstore_impl_t::loop()
 {
    // FIXME: initialized == 10 is ugly
    if (initialized != 10)
    {
        // read metadata, then journal
        if (initialized == 0)
        {
            metadata_init_reader = new blockstore_init_meta(this);
            initialized = 1;
        }
        if (initialized == 1)
        {
            int res = metadata_init_reader->loop();
            if (!res)
            {
                delete metadata_init_reader;
                metadata_init_reader = NULL;
                journal_init_reader = new blockstore_init_journal(this);
                initialized = 2;
            }
        }
        if (initialized == 2)
        {
            int res = journal_init_reader->loop();
            if (!res)
            {
                delete journal_init_reader;
                journal_init_reader = NULL;
                initialized = 3;
                ringloop->wakeup();
            }
        }
        if (initialized == 3)
        {
            if (!readonly && dsk.discard_on_start)
                dsk.trim_data([this](uint64_t block_num){ return data_alloc->get(block_num); });
            if (journal.flush_journal)
                initialized = 4;
            else
                initialized = 10;
        }
        if (initialized == 4)
        {
            if (readonly)
            {
                printf("Can't flush the journal in readonly mode\n");
                exit(1);
            }
            flusher->loop();
            ringloop->submit();
        }
    }
    else
    {
        // try to submit ops
        unsigned initial_ring_space = ringloop->space_left();
        // has_writes == 0 - no writes before the current queue item
        // has_writes == 1 - some writes in progress
        // has_writes == 2 - tried to submit some writes, but failed
        int has_writes = 0, op_idx = 0, new_idx = 0;
        for (; op_idx < submit_queue.size(); op_idx++, new_idx++)
        {
            auto op = submit_queue[op_idx];
            submit_queue[new_idx] = op;
            // FIXME: This needs some simplification
            // Writes should not block reads if the ring is not full and reads don't depend on them
            // In all other cases we should stop submission
            if (PRIV(op)->wait_for)
            {
                check_wait(op);
                if (PRIV(op)->wait_for == WAIT_SQE)
                {
                    break;
                }
                else if (PRIV(op)->wait_for)
                {
                    if (op->opcode == BS_OP_WRITE || op->opcode == BS_OP_WRITE_STABLE || op->opcode == BS_OP_DELETE)
                    {
                        has_writes = 2;
                    }
                    continue;
                }
            }
            unsigned prev_sqe_pos = ringloop->save();
            // 0 = can't submit
            // 1 = in progress
            // 2 = can be removed from queue
            int wr_st = 0;
            if (op->opcode == BS_OP_READ)
            {
                wr_st = dequeue_read(op);
            }
            else if (op->opcode == BS_OP_WRITE || op->opcode == BS_OP_WRITE_STABLE)
            {
                if (has_writes == 2)
                {
                    // Some writes already could not be submitted
                    continue;
                }
                wr_st = dequeue_write(op);
                has_writes = wr_st > 0 ? 1 : 2;
            }
            else if (op->opcode == BS_OP_DELETE)
            {
                if (has_writes == 2)
                {
                    // Some writes already could not be submitted
                    continue;
                }
                wr_st = dequeue_del(op);
                has_writes = wr_st > 0 ? 1 : 2;
            }
            else if (op->opcode == BS_OP_SYNC)
            {
                // sync only completed writes?
                // wait for the data device fsync to complete, then submit journal writes for big writes
                // then submit an fsync operation
                wr_st = continue_sync(op);
            }
            else if (op->opcode == BS_OP_STABLE)
            {
                wr_st = dequeue_stable(op);
            }
            else if (op->opcode == BS_OP_ROLLBACK)
            {
                wr_st = dequeue_rollback(op);
            }
            else if (op->opcode == BS_OP_LIST)
            {
                // LIST doesn't have to be blocked by previous modifications
                process_list(op);
                wr_st = 2;
            }
            if (wr_st == 2)
            {
                submit_queue[op_idx] = NULL;
                new_idx--;
            }
            if (wr_st == 0)
            {
                ringloop->restore(prev_sqe_pos);
                if (PRIV(op)->wait_for == WAIT_SQE)
                {
                    // ring is full, stop submission
                    break;
                }
                else if (PRIV(op)->wait_for == WAIT_JOURNAL)
                {
                    PRIV(op)->wait_detail2 = (unstable_writes.size()+unstable_unsynced);
                }
            }
        }
        if (op_idx != new_idx)
        {
            while (op_idx < submit_queue.size())
            {
                submit_queue[new_idx++] = submit_queue[op_idx++];
            }
            submit_queue.resize(new_idx);
        }
        if (!readonly)
        {
            flusher->loop();
        }
        int ret = ringloop->submit();
        if (ret < 0)
        {
            throw std::runtime_error(std::string("io_uring_submit: ") + strerror(-ret));
        }
        for (auto s: journal.submitting_sectors)
        {
            // Mark journal sector writes as submitted
            if (journal.sector_info[s].submit_id)
                journal.sector_info[s].written = true;
            journal.sector_info[s].submit_id = 0;
        }
        journal.submitting_sectors.clear();
        if ((initial_ring_space - ringloop->space_left()) > 0)
        {
            live = true;
        }
        queue_stall = !live && !ringloop->has_work();
        live = false;
    }
 }
 bool blockstore_impl_t::is_safe_to_stop()
 {
    // It's safe to stop blockstore when there are no in-flight operations,
    // no in-progress syncs and flusher isn't doing anything
    if (submit_queue.size() > 0 || !readonly && flusher->is_active())
    {
        return false;
    }
    if (unsynced_big_writes.size() > 0 || unsynced_small_writes.size() > 0)
    {
        if (!readonly && !stop_sync_submitted)
        {
            // We should sync the blockstore before unmounting
            blockstore_op_t *op = new blockstore_op_t;
            op->opcode = BS_OP_SYNC;
            op->buf = NULL;
            op->callback = [](blockstore_op_t *op)
            {
                delete op;
            };
            enqueue_op(op);
            stop_sync_submitted = true;
        }
        return false;
    }
    return true;
 }
 void blockstore_impl_t::check_wait(blockstore_op_t *op)
 {
    if (PRIV(op)->wait_for == WAIT_SQE)
    {
        if (ringloop->space_left() < PRIV(op)->wait_detail)
        {
            // stop submission if there's still no free space
 #ifdef BLOCKSTORE_DEBUG
            printf("Still waiting for %ju SQE(s)\n", PRIV(op)->wait_detail);
 #endif
            return;
        }
        PRIV(op)->wait_for = 0;
    }
    else if (PRIV(op)->wait_for == WAIT_JOURNAL)
    {
        if (journal.used_start == PRIV(op)->wait_detail &&
            (unstable_writes.size()+unstable_unsynced) == PRIV(op)->wait_detail2)
        {
            // do not submit
 #ifdef BLOCKSTORE_DEBUG
            printf("Still waiting to flush journal offset %08jx\n", PRIV(op)->wait_detail);
 #endif
            return;
        }
        flusher->release_trim();
        PRIV(op)->wait_for = 0;
    }
    else if (PRIV(op)->wait_for == WAIT_JOURNAL_BUFFER)
    {
        int next = ((journal.cur_sector + 1) % journal.sector_count);
        if (journal.sector_info[next].flush_count > 0 ||
            journal.sector_info[next].dirty)
        {
            // do not submit
 #ifdef BLOCKSTORE_DEBUG
            printf("Still waiting for a journal buffer\n");
 #endif
            return;
        }
        PRIV(op)->wait_for = 0;
    }
    else if (PRIV(op)->wait_for == WAIT_FREE)
    {
        if (!data_alloc->get_free_count() && big_to_flush > 0)
        {
 #ifdef BLOCKSTORE_DEBUG
            printf("Still waiting for free space on the data device\n");
 #endif
            return;
        }
        flusher->release_trim();
        PRIV(op)->wait_for = 0;
    }
    else
    {
        throw std::runtime_error("BUG: op->wait_for value is unexpected");
    }
 }
 void blockstore_impl_t::enqueue_op(blockstore_op_t *op)
 {
    if (op->opcode < BS_OP_MIN || op->opcode > BS_OP_MAX ||
        ((op->opcode == BS_OP_READ || op->opcode == BS_OP_WRITE || op->opcode == BS_OP_WRITE_STABLE) && (
            op->offset >= dsk.data_block_size ||
            op->len > dsk.data_block_size-op->offset ||
            (op->len % dsk.disk_alignment)
        )) ||
        readonly && op->opcode != BS_OP_READ && op->opcode != BS_OP_LIST)
    {
        // Basic verification not passed
        op->retval = -EINVAL;
        ringloop->set_immediate([op]() { std::function<void (blockstore_op_t*)>(op->callback)(op); });
        return;
    }
    if ((op->opcode == BS_OP_WRITE || op->opcode == BS_OP_WRITE_STABLE || op->opcode == BS_OP_DELETE) && !enqueue_write(op))
    {
        ringloop->set_immediate([op]() { std::function<void (blockstore_op_t*)>(op->callback)(op); });
        return;
    }
    if (op->opcode == BS_OP_SYNC)
    {
        unsynced_queued_ops = 0;
    }
    init_op(op);
    submit_queue.push_back(op);
    ringloop->wakeup();
 }
 void blockstore_impl_t::init_op(blockstore_op_t *op)
 {
    // Call constructor without allocating memory. We'll call destructor before returning op back
    new ((void*)op->private_data) blockstore_op_private_t;
    PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 0;
    PRIV(op)->wait_for = 0;
    PRIV(op)->op_state = 0;
    PRIV(op)->pending_ops = 0;
 }
 static bool replace_stable(object_id oid, uint64_t version, int search_start, int search_end, obj_ver_id* list)
 {
    while (search_start < search_end)
    {
        int pos = search_start+(search_end-search_start)/2;
        if (oid < list[pos].oid)
        {
            search_end = pos;
        }
        else if (list[pos].oid < oid)
        {
            search_start = pos+1;
        }
        else
        {
            list[pos].version = version;
            return true;
        }
    }
    return false;
 }
 blockstore_clean_db_t& blockstore_impl_t::clean_db_shard(object_id oid)
 {
    uint64_t pg_num = 0;
    uint64_t pool_id = (oid.inode >> (64-POOL_ID_BITS));
    auto sh_it = clean_db_settings.find(pool_id);
    if (sh_it != clean_db_settings.end())
    {
        // like map_to_pg()
        pg_num = (oid.stripe / sh_it->second.pg_stripe_size) % sh_it->second.pg_count + 1;
    }
    return clean_db_shards[(pool_id << (64-POOL_ID_BITS)) | pg_num];
 }
 void blockstore_impl_t::reshard_clean_db(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size)
 {
    uint64_t pool_id = (uint64_t)pool;
    std::map<pool_pg_id_t, blockstore_clean_db_t> new_shards;
    auto sh_it = clean_db_shards.lower_bound((pool_id << (64-POOL_ID_BITS)));
    while (sh_it != clean_db_shards.end() &&
        (sh_it->first >> (64-POOL_ID_BITS)) == pool_id)
    {
        for (auto & pair: sh_it->second)
        {
            // like map_to_pg()
            uint64_t pg_num = (pair.first.stripe / pg_stripe_size) % pg_count + 1;
            uint64_t shard_id = (pool_id << (64-POOL_ID_BITS)) | pg_num;
            new_shards[shard_id][pair.first] = pair.second;
        }
        clean_db_shards.erase(sh_it++);
    }
    for (sh_it = new_shards.begin(); sh_it != new_shards.end(); sh_it++)
    {
        auto & to = clean_db_shards[sh_it->first];
        to.swap(sh_it->second);
    }
    clean_db_settings[pool_id] = (pool_shard_settings_t){
        .pg_count = pg_count,
        .pg_stripe_size = pg_stripe_size,
    };
 }
 void blockstore_impl_t::process_list(blockstore_op_t *op)
 {
    uint32_t list_pg = op->pg_number+1;
    uint32_t pg_count = op->pg_count;
    uint64_t pg_stripe_size = op->pg_alignment;
    uint64_t min_inode = op->min_oid.inode;
    uint64_t max_inode = op->max_oid.inode;
    // Check PG
    if (pg_count != 0 && (pg_stripe_size < MIN_DATA_BLOCK_SIZE || list_pg > pg_count))
    {
        op->retval = -EINVAL;
        FINISH_OP(op);
        return;
    }
    // Check if the DB needs resharding
    // (we don't know about PGs from the beginning, we only create "shards" here)
    uint64_t first_shard = 0, last_shard = UINT64_MAX;
    if (min_inode != 0 &&
        // Check if min_inode == max_inode == pool_id<<N, i.e. this is a pool listing
        (min_inode >> (64-POOL_ID_BITS)) == (max_inode >> (64-POOL_ID_BITS)))
    {
        pool_id_t pool_id = (min_inode >> (64-POOL_ID_BITS));
        if (pg_count > 1)
        {
            // Per-pg listing
            auto sh_it = clean_db_settings.find(pool_id);
            if (sh_it == clean_db_settings.end() ||
                sh_it->second.pg_count != pg_count ||
                sh_it->second.pg_stripe_size != pg_stripe_size)
            {
                reshard_clean_db(pool_id, pg_count, pg_stripe_size);
            }
            first_shard = last_shard = ((uint64_t)pool_id << (64-POOL_ID_BITS)) | list_pg;
        }
        else
        {
            // Per-pool listing
            first_shard = ((uint64_t)pool_id << (64-POOL_ID_BITS));
            last_shard = ((uint64_t)(pool_id+1) << (64-POOL_ID_BITS)) - 1;
        }
    }
    // Copy clean_db entries
    int stable_count = 0, stable_alloc = 0;
    if (min_inode != max_inode)
    {
        for (auto shard_it = clean_db_shards.lower_bound(first_shard);
            shard_it != clean_db_shards.end() && shard_it->first <= last_shard;
            shard_it++)
        {
            auto & clean_db = shard_it->second;
            stable_alloc += clean_db.size();
        }
    }
    if (op->list_stable_limit > 0)
    {
        stable_alloc = op->list_stable_limit;
        if (stable_alloc > 1024*1024)
            stable_alloc = 1024*1024;
    }
    if (stable_alloc < 32768)
    {
        stable_alloc = 32768;
    }
    obj_ver_id *stable = (obj_ver_id*)malloc(sizeof(obj_ver_id) * stable_alloc);
    if (!stable)
    {
        op->retval = -ENOMEM;
        FINISH_OP(op);
        return;
    }
    auto max_oid = op->max_oid;
    bool limited = false;
    pool_pg_id_t last_shard_id = 0;
    for (auto shard_it = clean_db_shards.lower_bound(first_shard);
        shard_it != clean_db_shards.end() && shard_it->first <= last_shard;
        shard_it++)
    {
        auto & clean_db = shard_it->second;
        auto clean_it = clean_db.begin(), clean_end = clean_db.end();
        if (op->min_oid.inode != 0 || op->min_oid.stripe != 0)
        {
            clean_it = clean_db.lower_bound(op->min_oid);
        }
        if ((max_oid.inode != 0 || max_oid.stripe != 0) && !(max_oid < op->min_oid))
        {
            clean_end = clean_db.upper_bound(max_oid);
        }
        for (; clean_it != clean_end; clean_it++)
        {
            if (stable_count >= stable_alloc)
            {
                stable_alloc *= 2;
                obj_ver_id* nst = (obj_ver_id*)realloc(stable, sizeof(obj_ver_id) * stable_alloc);
                if (!nst)
                {
                    op->retval = -ENOMEM;
                    FINISH_OP(op);
                    return;
                }
                stable = nst;
            }
            stable[stable_count++] = {
                .oid = clean_it->first,
                .version = clean_it->second.version,
            };
            if (op->list_stable_limit > 0 && stable_count >= op->list_stable_limit)
            {
                if (!limited)
                {
                    limited = true;
                    max_oid = stable[stable_count-1].oid;
                }
                break;
            }
        }
        if (op->list_stable_limit > 0)
        {
            // To maintain the order, we have to include objects in the same range from other shards
            if (last_shard_id != 0 && last_shard_id != shard_it->first)
                std::sort(stable, stable+stable_count);
            if (stable_count > op->list_stable_limit)
                stable_count = op->list_stable_limit;
        }
        last_shard_id = shard_it->first;
    }
    if (op->list_stable_limit == 0 && first_shard != last_shard)
    {
        // If that's not a per-PG listing, sort clean entries (already sorted if list_stable_limit != 0)
        std::sort(stable, stable+stable_count);
    }
    int clean_stable_count = stable_count;
    // Copy dirty_db entries (sorted, too)
    int unstable_count = 0, unstable_alloc = 0;
    obj_ver_id *unstable = NULL;
    {
        auto dirty_it = dirty_db.begin(), dirty_end = dirty_db.end();
        if (op->min_oid.inode != 0 || op->min_oid.stripe != 0)
        {
            dirty_it = dirty_db.lower_bound({
                .oid = op->min_oid,
                .version = 0,
            });
        }
        if ((max_oid.inode != 0 || max_oid.stripe != 0) && !(max_oid < op->min_oid))
        {
            dirty_end = dirty_db.upper_bound({
                .oid = max_oid,
                .version = UINT64_MAX,
            });
        }
        for (; dirty_it != dirty_end; dirty_it++)
        {
            if (!pg_count || ((dirty_it->first.oid.stripe / pg_stripe_size) % pg_count + 1) == list_pg) // like map_to_pg()
            {
                if (IS_DELETE(dirty_it->second.state))
                {
                    // Deletions are always stable, so try to zero out two possible entries
                    if (!replace_stable(dirty_it->first.oid, 0, 0, clean_stable_count, stable))
                    {
                        replace_stable(dirty_it->first.oid, 0, clean_stable_count, stable_count, stable);
                    }
                }
                else if (IS_STABLE(dirty_it->second.state) || (dirty_it->second.state & BS_ST_INSTANT))
                {
                    // First try to replace a clean stable version in the first part of the list
                    if (!replace_stable(dirty_it->first.oid, dirty_it->first.version, 0, clean_stable_count, stable))
                    {
                        // Then try to replace the last dirty stable version in the second part of the list
                        if (stable_count > 0 && stable[stable_count-1].oid == dirty_it->first.oid)
                        {
                            stable[stable_count-1].version = dirty_it->first.version;
                        }
                        else
                        {
                            if (stable_count >= stable_alloc)
                            {
                                stable_alloc += 32768;
                                obj_ver_id *nst = (obj_ver_id*)realloc(stable, sizeof(obj_ver_id) * stable_alloc);
                                if (!nst)
                                {
                                    if (unstable)
                                        free(unstable);
                                    op->retval = -ENOMEM;
                                    FINISH_OP(op);
                                    return;
                                }
                                stable = nst;
                            }
                            stable[stable_count++] = dirty_it->first;
                        }
                    }
                    if (op->list_stable_limit > 0 && stable_count >= op->list_stable_limit)
                    {
                        // Stop here
                        break;
                    }
                }
                else
                {
                    if (unstable_count >= unstable_alloc)
                    {
                        unstable_alloc += 32768;
                        obj_ver_id *nst = (obj_ver_id*)realloc(unstable, sizeof(obj_ver_id) * unstable_alloc);
                        if (!nst)
                        {
                            if (stable)
                                free(stable);
                            op->retval = -ENOMEM;
                            FINISH_OP(op);
                            return;
                        }
                        unstable = nst;
                    }
                    unstable[unstable_count++] = dirty_it->first;
                }
            }
        }
    }
    // Remove zeroed out stable entries
    int j = 0;
    for (int i = 0; i < stable_count; i++)
    {
        if (stable[i].version != 0)
        {
            stable[j++] = stable[i];
        }
    }
    stable_count = j;
    if (stable_count+unstable_count > stable_alloc)
    {
        stable_alloc = stable_count+unstable_count;
        obj_ver_id *nst = (obj_ver_id*)realloc(stable, sizeof(obj_ver_id) * stable_alloc);
        if (!nst)
        {
            if (unstable)
                free(unstable);
            op->retval = -ENOMEM;
            FINISH_OP(op);
            return;
        }
        stable = nst;
    }
    // Copy unstable entries
    for (int i = 0; i < unstable_count; i++)
    {
        stable[j++] = unstable[i];
    }
    free(unstable);
    op->version = stable_count;
    op->retval = stable_count+unstable_count;
    op->buf = (uint8_t*)stable;
    FINISH_OP(op);
 }
 void blockstore_impl_t::dump_diagnostics()
 {
    journal.dump_diagnostics();
    flusher->dump_diagnostics();
 }
 void blockstore_impl_t::disk_error_abort(const char *op, int retval, int expected)
 {
    if (retval == -EAGAIN)
    {
        fprintf(stderr, "EAGAIN error received from a disk %s during flush."
            " It must never happen with io_uring and indicates a kernel bug."
            " Please upgrade your kernel. Aborting.\n", op);
        exit(1);
    }
    fprintf(stderr, "Disk %s failed: result is %d, expected %d. Can't continue, sorry :-(\n", op, retval, expected);
    exit(1);
 }
 const std::map<uint64_t, uint64_t> & blockstore_impl_t::get_inode_space_stats()
 {
    return inode_space_stats;
 }
 void blockstore_impl_t::set_no_inode_stats(const std::vector<uint64_t> & pool_ids)
 {
    for (auto & np: no_inode_stats)
    {
        np.second = 2;
    }
    for (auto pool_id: pool_ids)
    {
        if (!no_inode_stats[pool_id])
            recalc_inode_space_stats(pool_id, false);
        no_inode_stats[pool_id] = 1;
    }
    for (auto np_it = no_inode_stats.begin(); np_it != no_inode_stats.end(); )
    {
        if (np_it->second == 2)
        {
            recalc_inode_space_stats(np_it->first, true);
            no_inode_stats.erase(np_it++);
        }
        else
            np_it++;
    }
 }
 void blockstore_impl_t::recalc_inode_space_stats(uint64_t pool_id, bool per_inode)
 {
    auto sp_begin = inode_space_stats.lower_bound((pool_id << (64-POOL_ID_BITS)));
    auto sp_end = inode_space_stats.lower_bound(((pool_id+1) << (64-POOL_ID_BITS)));
    inode_space_stats.erase(sp_begin, sp_end);
    auto sh_it = clean_db_shards.lower_bound((pool_id << (64-POOL_ID_BITS)));
    while (sh_it != clean_db_shards.end() &&
        (sh_it->first >> (64-POOL_ID_BITS)) == pool_id)
    {
        for (auto & pair: sh_it->second)
        {
            uint64_t space_id = per_inode ? pair.first.inode : (pool_id << (64-POOL_ID_BITS));
            inode_space_stats[space_id] += dsk.data_block_size;
        }
        sh_it++;
    }
    object_id last_oid = {};
    bool last_exists = false;
    auto dirty_it = dirty_db.lower_bound((obj_ver_id){ .oid = { .inode = (pool_id << (64-POOL_ID_BITS)) } });
    while (dirty_it != dirty_db.end() && (dirty_it->first.oid.inode >> (64-POOL_ID_BITS)) == pool_id)
    {
        if (IS_STABLE(dirty_it->second.state) && (IS_BIG_WRITE(dirty_it->second.state) || IS_DELETE(dirty_it->second.state)))
        {
            bool exists = false;
            if (last_oid == dirty_it->first.oid)
            {
                exists = last_exists;
            }
            else
            {
                auto & clean_db = clean_db_shard(dirty_it->first.oid);
                auto clean_it = clean_db.find(dirty_it->first.oid);
                exists = clean_it != clean_db.end();
            }
            uint64_t space_id = per_inode ? dirty_it->first.oid.inode : (pool_id << (64-POOL_ID_BITS));
            if (IS_BIG_WRITE(dirty_it->second.state))
            {
                if (!exists)
                    inode_space_stats[space_id] += dsk.data_block_size;
                last_exists = true;
            }
            else
            {
                if (exists)
                {
                    auto & sp = inode_space_stats[space_id];
                    if (sp > dsk.data_block_size)
                        sp -= dsk.data_block_size;
                    else
                        inode_space_stats.erase(space_id);
                }
                last_exists = false;
            }
            last_oid = dirty_it->first.oid;
        }
        dirty_it++;
    }
 }
 std::string blockstore_impl_t::get_op_diag(blockstore_op_t *op)
 {
    char buf[256];
    auto priv = PRIV(op);
    if (priv->wait_for)
        snprintf(buf, sizeof(buf), "state=%d wait=%d (detail=%ju)", priv->op_state, priv->wait_for, priv->wait_detail);
    else
        snprintf(buf, sizeof(buf), "state=%d", priv->op_state);
    return std::string(buf);
 }
 } // namespace v1
@@ -1,334 +0,0 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 #pragma once
 #include "blockstore.h"
 #include "blockstore_disk.h"
 #include "ondisk_formats.h"
 #include <sys/types.h>
 #include <sys/ioctl.h>
 #include <sys/stat.h>
 #include <fcntl.h>
 #include <time.h>
 #include <unistd.h>
 #include <linux/fs.h>
 #include <vector>
 #include <list>
 #include <deque>
 #include <new>
 #include <unordered_map>
 #include <unordered_set>
 #include "cpp-btree/btree_map.h"
 #include "malloc_or_die.h"
 #include "allocator.h"
 #include "crc32c.h"
 //#define BLOCKSTORE_DEBUG
 namespace v1 {
 #include "journal.h"
 // 32 = 16 + 16 bytes per "clean" entry in memory (object_id => clean_entry)
 struct __attribute__((__packed__)) clean_entry
 {
    uint64_t version;
    uint64_t location;
 };
 // 64 = 24 + 40 bytes per dirty entry in memory (obj_ver_id => dirty_entry). Plus checksums
 struct __attribute__((__packed__)) dirty_entry
 {
    uint32_t state;
    uint32_t flags;    // unneeded, but present for alignment
    uint64_t location; // location in either journal or data -> in BYTES
    uint32_t offset;   // data offset within object (stripe)
    uint32_t len;      // data length
    uint64_t journal_sector; // journal sector used for this entry
    void* dyn_data;    // dynamic data: external bitmap and data block checksums. may be a pointer to the in-memory journal
 };
 // - Sync must be submitted after previous writes/deletes (not before!)
 // - Reads to the same object must be submitted after previous writes/deletes
 //   are written (not necessarily synced) in their location. This is because we
 //   rely on read-modify-write for erasure coding and we must return new data
 //   to calculate parity for subsequent writes
 // - Writes may be submitted in any order, because they don't overlap. Each write
 //   goes into a new location - either on the journal device or on the data device
 // - Stable (stabilize) must be submitted after sync of that object is completed
 //   It's even OK to return an error to the caller if that object is not synced yet
 // - Journal trim may be processed only after all versions are moved to
 //   the main storage AND after all read operations for older versions complete
 // - If an operation can not be submitted because the ring is full
 //   we should stop submission of other operations. Otherwise some "scatter" reads
 //   may end up blocked for a long time.
 // Otherwise, the submit order is free, that is all operations may be submitted immediately
 // In fact, adding a write operation must immediately result in dirty_db being populated
 struct used_clean_obj_t
 {
    int refs;
    bool was_freed; // was freed by a parallel flush?
    bool was_changed; // was changed by a parallel flush?
 };
 // https://github.com/algorithm-ninja/cpp-btree
 // https://github.com/greg7mdp/sparsepp/ was used previously, but it was TERRIBLY slow after resizing
 // with sparsepp, random reads dropped to ~700 iops very fast with just as much as ~32k objects in the DB
 typedef btree::btree_map<object_id, clean_entry> blockstore_clean_db_t;
 typedef std::map<obj_ver_id, dirty_entry> blockstore_dirty_db_t;
 #include "init.h"
 #include "flush.h"
 struct blockstore_op_private_t
 {
    // Wait status
    int wait_for;
    uint64_t wait_detail, wait_detail2;
    int pending_ops;
    int op_state;
    // Read
    uint64_t clean_block_used;
    std::vector<copy_buffer_t> read_vec;
    // Sync, write
    uint64_t min_flushed_journal_sector, max_flushed_journal_sector;
    // Write
    struct iovec iov_zerofill[3];
    // Warning: must not have a default value here because it's written to before calling constructor in blockstore_write.cpp O_o
    uint64_t real_version;
    timespec tv_begin;
    // Sync
    std::vector<obj_ver_id> sync_big_writes, sync_small_writes;
 };
 struct pool_shard_settings_t
 {
    uint32_t pg_count;
    uint32_t pg_stripe_size;
 };
 typedef uint64_t pool_pg_id_t;
 class blockstore_impl_t: public blockstore_i
 {
    blockstore_disk_t dsk;
    /******* OPTIONS *******/
    bool readonly = false;
    // It is safe to disable fsync() if drive write cache is writethrough
    bool disable_data_fsync = false, disable_meta_fsync = false, disable_journal_fsync = false;
    // Enable if you want every operation to be executed with an "implicit fsync"
    // Suitable only for server SSDs with capacitors, requires disabled data and journal fsyncs
    int immediate_commit = IMMEDIATE_NONE;
    bool inmemory_meta = false;
    // Maximum and minimum flusher count
    unsigned max_flusher_count, min_flusher_count;
    unsigned journal_trim_interval;
    // Maximum queue depth
    unsigned max_write_iodepth = 128;
    // Enable small (journaled) write throttling, useful for the SSD+HDD case
    bool throttle_small_writes = false;
    // Target data device iops, bandwidth and parallelism for throttling (100/100/1 is the default for HDD)
    int throttle_target_iops = 100;
    int throttle_target_mbs = 100;
    int throttle_target_parallelism = 1;
    // Minimum difference in microseconds between target and real execution times to throttle the response
    int throttle_threshold_us = 50;
    // Maximum writes between automatically added fsync operations
    uint64_t autosync_writes = 128;
    // Log level (0-10)
    int log_level = 0;
    /******* END OF OPTIONS *******/
    struct ring_consumer_t ring_consumer;
    std::map<pool_id_t, pool_shard_settings_t> clean_db_settings;
    std::map<pool_pg_id_t, blockstore_clean_db_t> clean_db_shards;
    std::map<uint64_t, int> no_inode_stats;
    std::map<uint64_t, uint64_t> inode_space_stats;
    uint8_t *clean_bitmaps = NULL;
    blockstore_dirty_db_t dirty_db;
    std::vector<blockstore_op_t*> submit_queue;
    std::vector<obj_ver_id> unsynced_big_writes, unsynced_small_writes;
    int unsynced_big_write_count = 0, unstable_unsynced = 0;
    int unsynced_queued_ops = 0;
    allocator_t *data_alloc = NULL;
    uint64_t used_blocks = 0;
    uint8_t *zero_object = NULL;
    void *metadata_buffer = NULL;
    struct journal_t journal;
    journal_flusher_t *flusher;
    int big_to_flush = 0;
    int write_iodepth = 0;
    bool alloc_dyn_data = false;
    // clean data blocks referenced by read operations
    std::map<uint64_t, used_clean_obj_t> used_clean_objects;
    bool live = false, queue_stall = false;
    ring_loop_i *ringloop;
    timerfd_manager_t *tfd;
    bool stop_sync_submitted;
    inline struct io_uring_sqe* get_sqe()
    {
        return ringloop->get_sqe();
    }
    friend class blockstore_init_meta;
    friend class blockstore_init_journal;
    friend struct blockstore_journal_check_t;
    friend class journal_flusher_t;
    friend class journal_flusher_co;
    void calc_lengths();
    void open_data();
    void open_meta();
    void open_journal();
    uint8_t* get_clean_entry_bitmap(uint64_t block_loc, int offset);
    blockstore_clean_db_t& clean_db_shard(object_id oid);
    void reshard_clean_db(pool_id_t pool_id, uint32_t pg_count, uint32_t pg_stripe_size);
    void recalc_inode_space_stats(uint64_t pool_id, bool per_inode);
    // Journaling
    void prepare_journal_sector_write(int sector, blockstore_op_t *op);
    void handle_journal_write(ring_data_t *data, uint64_t flush_id);
    void disk_error_abort(const char *op, int retval, int expected);
    // Asynchronous init
    int initialized;
    int metadata_buf_size;
    blockstore_init_meta* metadata_init_reader;
    blockstore_init_journal* journal_init_reader;
    void check_wait(blockstore_op_t *op);
    void init_op(blockstore_op_t *op);
    // Read
    int dequeue_read(blockstore_op_t *read_op);
    void find_holes(std::vector<copy_buffer_t> & read_vec, uint32_t item_start, uint32_t item_end,
        std::function<int(int, bool, uint32_t, uint32_t)> callback);
    int fulfill_read(blockstore_op_t *read_op,
        uint64_t &fulfilled, uint32_t item_start, uint32_t item_end,
        uint32_t item_state, uint64_t item_version, uint64_t item_location,
        uint64_t journal_sector, uint8_t *csum, int *dyn_data);
    bool fulfill_clean_read(blockstore_op_t *read_op, uint64_t & fulfilled,
        uint8_t *clean_entry_bitmap, int *dyn_data,
        uint32_t item_start, uint32_t item_end, uint64_t clean_loc, uint64_t clean_ver);
    int fill_partial_checksum_blocks(std::vector<copy_buffer_t> & rv, uint64_t & fulfilled,
        uint8_t *clean_entry_bitmap, int *dyn_data, bool from_journal, uint8_t *read_buf, uint64_t read_offset, uint64_t read_end);
    int pad_journal_read(std::vector<copy_buffer_t> & rv, copy_buffer_t & cp,
        uint64_t dirty_offset, uint64_t dirty_end, uint64_t dirty_loc, uint8_t *csum_ptr, int *dyn_data,
        uint64_t offset, uint64_t submit_len, uint64_t & blk_begin, uint64_t & blk_end, uint8_t* & blk_buf);
    bool read_range_fulfilled(std::vector<copy_buffer_t> & rv, uint64_t & fulfilled, uint8_t *read_buf,
        uint8_t *clean_entry_bitmap, uint32_t item_start, uint32_t item_end);
    bool read_checksum_block(blockstore_op_t *op, int rv_pos, uint64_t &fulfilled, uint64_t clean_loc);
    uint8_t* read_clean_meta_block(blockstore_op_t *read_op, uint64_t clean_loc, int rv_pos);
    bool verify_padded_checksums(uint8_t *clean_entry_bitmap, uint8_t *csum_buf, uint32_t offset,
        iovec *iov, int n_iov, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);
    bool verify_journal_checksums(uint8_t *csums, uint32_t offset,
        iovec *iov, int n_iov, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);
    bool verify_clean_padded_checksums(blockstore_op_t *op, uint64_t clean_loc, uint8_t *dyn_data, bool from_journal,
        iovec *iov, int n_iov, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);
    int fulfill_read_push(blockstore_op_t *op, void *buf, uint64_t offset, uint64_t len,
        uint32_t item_state, uint64_t item_version);
    void handle_read_event(ring_data_t *data, blockstore_op_t *op);
    // Write
    bool enqueue_write(blockstore_op_t *op);
    void cancel_all_writes(blockstore_op_t *op, blockstore_dirty_db_t::iterator dirty_it, int retval);
    int dequeue_write(blockstore_op_t *op);
    int dequeue_del(blockstore_op_t *op);
    int continue_write(blockstore_op_t *op);
    void release_journal_sectors(blockstore_op_t *op);
    void handle_write_event(ring_data_t *data, blockstore_op_t *op);
    // Sync
    int continue_sync(blockstore_op_t *op);
    void ack_sync(blockstore_op_t *op);
    // Stabilize
    int dequeue_stable(blockstore_op_t *op);
    int continue_stable(blockstore_op_t *op);
    void mark_stable(obj_ver_id ov, bool forget_dirty = false);
    void stabilize_object(object_id oid, uint64_t max_ver);
    blockstore_op_t* selective_sync(blockstore_op_t *op);
    int split_stab_op(blockstore_op_t *op, std::function<int(obj_ver_id v)> decider);
    // Rollback
    int dequeue_rollback(blockstore_op_t *op);
    int continue_rollback(blockstore_op_t *op);
    void mark_rolled_back(const obj_ver_id & ov);
    void erase_dirty(blockstore_dirty_db_t::iterator dirty_start, blockstore_dirty_db_t::iterator dirty_end, uint64_t clean_loc);
    void free_dirty_dyn_data(dirty_entry & e);
    // List
    void process_list(blockstore_op_t *op);
 public:
    blockstore_impl_t(blockstore_config_t & config, ring_loop_i *ringloop, timerfd_manager_t *tfd);
    ~blockstore_impl_t();
    void parse_config(blockstore_config_t & config);
    void parse_config(blockstore_config_t & config, bool init);
    // Event loop
    void loop();
    // Returns true when blockstore is ready to process operations
    // (Although you're free to enqueue them before that)
    bool is_started();
    // Returns true when it's safe to destroy the instance. If destroying the instance
    // requires to purge some queues, starts that process. Should be called in the event
    // loop until it returns true.
    bool is_safe_to_stop();
    // Returns true if stalled
    bool is_stalled();
    // Submission
    void enqueue_op(blockstore_op_t *op);
    // Simplified synchronous operation: get object bitmap & current version
    int read_bitmap(object_id oid, uint64_t target_version, void *bitmap, uint64_t *result_version = NULL);
    // Unstable writes are added here (map of object_id -> version)
    std::unordered_map<object_id, uint64_t> unstable_writes;
    // Get space usage statistics
    const std::map<uint64_t, uint64_t> & get_inode_space_stats();
    // Set per-pool no_inode_stats
    void set_no_inode_stats(const std::vector<uint64_t> & pool_ids);
    // Print diagnostics to stdout
    void dump_diagnostics();
    // Get diagnostic string for an operation
    std::string get_op_diag(blockstore_op_t *op);
    inline uint32_t get_block_size() { return dsk.data_block_size; }
    inline uint64_t get_block_count() { return dsk.block_count; }
    inline uint64_t get_free_block_count() { return dsk.block_count - used_blocks; }
    inline uint32_t get_bitmap_granularity() { return dsk.disk_alignment; }
    inline uint64_t get_journal_size() { return dsk.journal_len; }
 };
 } // namespace v1
@@ -1,71 +0,0 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 #pragma once
 struct blockstore_init_meta_buf
 {
    uint8_t *buf = NULL;
    uint64_t size = 0;
    uint64_t offset = 0;
    int state = 0;
 };
 class blockstore_init_meta
 {
    blockstore_impl_t *bs;
    int wait_state = 0;
    bool zero_on_init = false;
    void *metadata_buffer = NULL;
    blockstore_init_meta_buf bufs[2] = {};
    int submitted = 0;
    struct io_uring_sqe *sqe;
    struct ring_data_t *data;
    uint64_t md_offset = 0;
    uint64_t next_offset = 0;
    uint64_t last_read_offset = 0;
    uint64_t entries_loaded = 0;
    unsigned entries_per_block = 0;
    int i = 0, j = 0;
    std::vector<uint64_t> entries_to_zero;
    bool handle_meta_block(uint8_t *buf, uint64_t count, uint64_t done_cnt);
    void handle_event(ring_data_t *data, int buf_num);
 public:
    blockstore_init_meta(blockstore_impl_t *bs);
    int loop();
 };
 struct bs_init_journal_done
 {
    void *buf;
    uint64_t pos, len;
 };
 class blockstore_init_journal
 {
    blockstore_impl_t *bs;
    int wait_state = 0, wait_count = 0, handle_res = 0;
    uint64_t entries_loaded = 0;
    uint32_t crc32_last = 0;
    bool started = false;
    uint64_t next_free;
    std::vector<bs_init_journal_done> done;
    std::vector<obj_ver_id> double_allocs;
    std::vector<iovec> small_write_data;
    uint64_t journal_pos = 0;
    uint64_t continue_pos = 0;
    void *init_write_buf = NULL;
    uint64_t init_write_sector = 0;
    bool wrapped = false;
    void *submitted_buf;
    struct io_uring_sqe *sqe;
    struct ring_data_t *data;
    journal_entry_start *je_start;
    std::function<void(ring_data_t*)> simple_callback;
    int handle_journal_part(void *buf, uint64_t done_pos, uint64_t len);
    void handle_event(ring_data_t *data);
    void erase_dirty_object(blockstore_dirty_db_t::iterator dirty_it);
 public:
    blockstore_init_journal(blockstore_impl_t* bs);
    int loop();
 };
@@ -1,88 +0,0 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 #pragma once
 // States are not stored on disk. Instead, they're deduced from the journal
 #define BS_ST_SMALL_WRITE 0x01
 #define BS_ST_BIG_WRITE 0x02
 #define BS_ST_DELETE 0x03
 #define BS_ST_WAIT_DEL 0x10
 #define BS_ST_WAIT_BIG 0x20
 #define BS_ST_IN_FLIGHT 0x30
 #define BS_ST_SUBMITTED 0x40
 #define BS_ST_WRITTEN 0x50
 #define BS_ST_SYNCED 0x60
 #define BS_ST_STABLE 0x70
 #define BS_ST_INSTANT 0x100
 #define BS_ST_TYPE_MASK 0x0F
 #define BS_ST_WORKFLOW_MASK 0xF0
 #define IS_IN_FLIGHT(st) (((st) & 0xF0) <= BS_ST_SUBMITTED)
 #define IS_STABLE(st) (((st) & 0xF0) == BS_ST_STABLE)
 #define IS_SYNCED(st) (((st) & 0xF0) >= BS_ST_SYNCED)
 #define IS_JOURNAL(st) (((st) & 0x0F) == BS_ST_SMALL_WRITE)
 #define IS_BIG_WRITE(st) (((st) & 0x0F) == BS_ST_BIG_WRITE)
 #define IS_DELETE(st) (((st) & 0x0F) == BS_ST_DELETE)
 #define IS_INSTANT(st) (((st) & BS_ST_TYPE_MASK) == BS_ST_DELETE || ((st) & BS_ST_INSTANT))
 #define BS_SUBMIT_CHECK_SQES(n) \
    if (ringloop->space_left() < (n))\
    {\
        /* Pause until there are more requests available */\
        PRIV(op)->wait_detail = (n);\
        PRIV(op)->wait_for = WAIT_SQE;\
        return 0;\
    }
 #define BS_SUBMIT_GET_SQE(sqe, data) \
    BS_SUBMIT_GET_ONLY_SQE(sqe); \
    struct ring_data_t *data = ((ring_data_t*)sqe->user_data)
 #define BS_SUBMIT_GET_ONLY_SQE(sqe) \
    struct io_uring_sqe *sqe = get_sqe();\
    if (!sqe)\
    {\
        /* Pause until there are more requests available */\
        PRIV(op)->wait_detail = 1;\
        PRIV(op)->wait_for = WAIT_SQE;\
        return 0;\
    }
 #define BS_SUBMIT_GET_SQE_DECL(sqe) \
    sqe = get_sqe();\
    if (!sqe)\
    {\
        /* Pause until there are more requests available */\
        PRIV(op)->wait_detail = 1;\
        PRIV(op)->wait_for = WAIT_SQE;\
        return 0;\
    }
 #define PRIV(op) ((blockstore_op_private_t*)(op)->private_data)
 #define FINISH_OP(op) PRIV(op)->~blockstore_op_private_t(); std::function<void (blockstore_op_t*)>(op->callback)(op)
 // Suspend operation until there are more free SQEs
 #define WAIT_SQE 1
 // Suspend operation until there are <wait_detail> bytes of free space in the journal on disk
 #define WAIT_JOURNAL 3
 // Suspend operation until the next journal sector buffer is free
 #define WAIT_JOURNAL_BUFFER 4
 // Suspend operation until there is some free space on the data device
 #define WAIT_FREE 5
 #define COPY_BUF_JOURNAL 1
 #define COPY_BUF_DATA 2
 #define COPY_BUF_ZERO 4
 #define COPY_BUF_CSUM_FILL 8
 #define COPY_BUF_COALESCED 16
 #define COPY_BUF_META_BLOCK 32
 #define COPY_BUF_JOURNALED_BIG 64
 #define STAB_SPLIT_DONE 1
 #define STAB_SPLIT_WAIT 2
 #define STAB_SPLIT_SYNC 3
 #define STAB_SPLIT_TODO 4
@@ -1,78 +0,0 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 #pragma once
 class blockstore_impl_t;
 struct journal_sector_info_t
 {
    uint64_t offset;
    uint64_t flush_count;
    bool written;
    bool dirty;
    uint64_t submit_id;
 };
 struct pending_journaling_t
 {
    int pending;
    int sector;
    blockstore_op_t *op;
 };
 struct journal_t
 {
    int fd;
    bool inmemory = false;
    bool flush_journal = false;
    void *buffer = NULL;
    uint64_t block_size;
    uint64_t offset, len;
    // Next free block offset
    uint64_t next_free = 0;
    // First occupied block offset
    uint64_t used_start = 0;
    // End of the last block not used for writing anymore
    uint64_t dirty_start = 0;
    uint32_t crc32_last = 0;
    // Current sector(s) used for writing
    void *sector_buf = NULL;
    journal_sector_info_t *sector_info = NULL;
    uint64_t sector_count;
    bool no_same_sector_overwrites = false;
    int cur_sector = 0;
    int in_sector_pos = 0;
    std::vector<int> submitting_sectors;
    std::multimap<uint64_t, pending_journaling_t> flushing_ops;
    uint64_t submit_id = 0;
    // Used sector map
    // May use ~ 80 MB per 1 GB of used journal space in the worst case
    std::map<uint64_t, uint64_t> used_sectors;
    ~journal_t();
    bool trim();
    uint64_t get_trim_pos();
    void dump_diagnostics();
    inline bool entry_fits(int size)
    {
        return !(block_size - in_sector_pos < size ||
            no_same_sector_overwrites && sector_info[cur_sector].written);
    }
 };
 struct blockstore_journal_check_t
 {
    blockstore_impl_t *bs;
    uint64_t next_pos, next_sector, next_in_pos;
    int sectors_to_write, first_sector;
    bool right_dir; // writing to the end or the beginning of the ring buffer
    blockstore_journal_check_t(blockstore_impl_t *bs);
    int check_available(blockstore_op_t *op, int required, int size, int data_after);
 };
 journal_entry* prefill_single_journal_entry(journal_t & journal, uint16_t type, uint32_t size);
@@ -1,187 +0,0 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 #include <sys/file.h>
 #include "impl.h"
 namespace v1 {
 void blockstore_impl_t::parse_config(blockstore_config_t & config)
 {
    return parse_config(config, false);
 }
 void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init)
 {
    // Online-configurable options:
    max_flusher_count = strtoull(config["max_flusher_count"].c_str(), NULL, 10);
    if (!max_flusher_count)
    {
        max_flusher_count = strtoull(config["flusher_count"].c_str(), NULL, 10);
    }
    min_flusher_count = strtoull(config["min_flusher_count"].c_str(), NULL, 10);
    journal_trim_interval = strtoull(config["journal_trim_interval"].c_str(), NULL, 10);
    max_write_iodepth = strtoull(config["max_write_iodepth"].c_str(), NULL, 10);
    throttle_small_writes = config["throttle_small_writes"] == "true" || config["throttle_small_writes"] == "1" || config["throttle_small_writes"] == "yes";
    throttle_target_iops = strtoull(config["throttle_target_iops"].c_str(), NULL, 10);
    throttle_target_mbs = strtoull(config["throttle_target_mbs"].c_str(), NULL, 10);
    throttle_target_parallelism = strtoull(config["throttle_target_parallelism"].c_str(), NULL, 10);
    throttle_threshold_us = strtoull(config["throttle_threshold_us"].c_str(), NULL, 10);
    if (config["autosync_writes"] != "")
    {
        autosync_writes = strtoull(config["autosync_writes"].c_str(), NULL, 10);
    }
    if (!max_flusher_count)
    {
        max_flusher_count = 256;
    }
    if (!min_flusher_count || journal.flush_journal)
    {
        min_flusher_count = 1;
    }
    if (!journal_trim_interval)
    {
        journal_trim_interval = 512;
    }
    if (!max_write_iodepth)
    {
        max_write_iodepth = 128;
    }
    if (!throttle_target_iops)
    {
        throttle_target_iops = 100;
    }
    if (!throttle_target_mbs)
    {
        throttle_target_mbs = 100;
    }
    if (!throttle_target_parallelism)
    {
        throttle_target_parallelism = 1;
    }
    if (!throttle_threshold_us)
    {
        throttle_threshold_us = 50;
    }
    if (!init)
    {
        return;
    }
    // Offline-configurable options:
    // Common disk options
    dsk.parse_config(config);
    // Parse
    if (config["readonly"] == "true" || config["readonly"] == "1" || config["readonly"] == "yes")
    {
        readonly = true;
    }
    if (config["disable_data_fsync"] == "true" || config["disable_data_fsync"] == "1" || config["disable_data_fsync"] == "yes")
    {
        disable_data_fsync = true;
    }
    if (config["disable_meta_fsync"] == "true" || config["disable_meta_fsync"] == "1" || config["disable_meta_fsync"] == "yes")
    {
        disable_meta_fsync = true;
    }
    if (config["disable_journal_fsync"] == "true" || config["disable_journal_fsync"] == "1" || config["disable_journal_fsync"] == "yes")
    {
        disable_journal_fsync = true;
    }
    if (config["flush_journal"] == "true" || config["flush_journal"] == "1" || config["flush_journal"] == "yes")
    {
        // Only flush journal and exit
        journal.flush_journal = true;
    }
    if (config["immediate_commit"] == "all")
    {
        immediate_commit = IMMEDIATE_ALL;
    }
    else if (config["immediate_commit"] == "small")
    {
        immediate_commit = IMMEDIATE_SMALL;
    }
    metadata_buf_size = strtoull(config["meta_buf_size"].c_str(), NULL, 10);
    inmemory_meta = config["inmemory_metadata"] != "false" && config["inmemory_metadata"] != "0" &&
        config["inmemory_metadata"] != "no";
    journal.sector_count = strtoull(config["journal_sector_buffer_count"].c_str(), NULL, 10);
    journal.no_same_sector_overwrites = config["journal_no_same_sector_overwrites"] == "true" ||
        config["journal_no_same_sector_overwrites"] == "1" || config["journal_no_same_sector_overwrites"] == "yes";
    journal.inmemory = config["inmemory_journal"] != "false" && config["inmemory_journal"] != "0" &&
        config["inmemory_journal"] != "no";
    log_level = strtoull(config["log_level"].c_str(), NULL, 10);
    // Validate
    if (journal.sector_count < 2)
    {
        journal.sector_count = 32;
    }
    if (metadata_buf_size < 65536)
    {
        metadata_buf_size = 4*1024*1024;
    }
    if (dsk.meta_device == dsk.data_device)
    {
        disable_meta_fsync = disable_data_fsync;
    }
    if (dsk.journal_device == dsk.meta_device)
    {
        disable_journal_fsync = disable_meta_fsync;
    }
    if (immediate_commit != IMMEDIATE_NONE && !disable_journal_fsync)
    {
        throw std::runtime_error("immediate_commit requires disable_journal_fsync");
    }
    if (immediate_commit == IMMEDIATE_ALL && !disable_data_fsync)
    {
        throw std::runtime_error("immediate_commit=all requires disable_journal_fsync and disable_data_fsync");
    }
    // init some fields
    journal.block_size = dsk.journal_block_size;
    journal.next_free = dsk.journal_block_size;
    journal.used_start = dsk.journal_block_size;
    // no free space because sector is initially unmapped
    journal.in_sector_pos = dsk.journal_block_size;
 }
 void blockstore_impl_t::calc_lengths()
 {
    dsk.calc_lengths();
    journal.len = dsk.journal_len;
    journal.block_size = dsk.journal_block_size;
    journal.offset = dsk.journal_offset;
    if (inmemory_meta)
    {
        metadata_buffer = memalign(MEM_ALIGNMENT, dsk.meta_area_size);
        if (!metadata_buffer)
            throw std::runtime_error("Failed to allocate memory for the metadata ("+std::to_string(dsk.meta_area_size/1024/1024)+" MB)");
    }
    else if (dsk.clean_entry_bitmap_size || dsk.data_csum_type)
    {
        clean_bitmaps = (uint8_t*)malloc(dsk.block_count * 2 * dsk.clean_entry_bitmap_size);
        if (!clean_bitmaps)
        {
            throw std::runtime_error(
                "Failed to allocate memory for the metadata sparse write bitmap ("+
                std::to_string(dsk.block_count * 2 * dsk.clean_entry_bitmap_size / 1024 / 1024)+" MB)"
            );
        }
    }
    if (journal.inmemory)
    {
        journal.buffer = memalign(MEM_ALIGNMENT, journal.len);
        if (!journal.buffer)
            throw std::runtime_error("Failed to allocate memory for journal ("+std::to_string(journal.len/1024/1024)+" MB)");
    }
    else
    {
        journal.sector_buf = (uint8_t*)memalign(MEM_ALIGNMENT, journal.sector_count * dsk.journal_block_size);
        if (!journal.sector_buf)
            throw std::bad_alloc();
    }
    journal.sector_info = (journal_sector_info_t*)calloc(journal.sector_count, sizeof(journal_sector_info_t));
    if (!journal.sector_info)
    {
        throw std::bad_alloc();
    }
 }
 }
@@ -1,566 +0,0 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 #include "impl.h"
 #include "internal.h"
 namespace v1 {
 // Stabilize small write:
 // 1) Copy data from the journal to the data device
 // 2) Increase version on the metadata device and sync it
 // 3) Advance clean_db entry's version, clear previous journal entries
 //
 // This makes 1 4K small write+sync look like:
 // 512b+4K (journal) + sync + 512b (journal) + sync + 4K (data) [+ sync?] + 512b (metadata) + sync.
 // WA = 2.375. It's not the best, SSD FTL-like redirect-write could probably be lower
 // even with defragmentation. But it's fixed and it's still better than in Ceph. :)
 // except for HDD-only clusters, because each write results in 3 seeks.
 // Stabilize big write:
 // 1) Copy metadata from the journal to the metadata device
 // 2) Move dirty_db entry to clean_db and clear previous journal entries
 //
 // This makes 1 128K big write+sync look like:
 // 128K (data) + sync + 512b (journal) + sync + 512b (journal) + sync + 512b (metadata) + sync.
 // WA = 1.012. Very good :)
 // Stabilize delete:
 // 1) Remove metadata entry and sync it
 // 2) Remove dirty_db entry and clear previous journal entries
 // We have 2 problems here:
 // - In the cluster environment, we must store the "tombstones" of deleted objects until
 //   all replicas (not just quorum) agrees about their deletion. That is, "stabilize" is
 //   not possible for deletes in degraded placement groups
 // - With simple "fixed" metadata tables we can't just clear the metadata entry of the latest
 //   object version. We must clear all previous entries, too.
 // FIXME Fix both problems - probably, by switching from "fixed" metadata tables to "dynamic"
 // AND We must do it in batches, for the sake of reduced fsync call count
 // AND We must know what we stabilize. Basic workflow is like:
 // 1) primary OSD receives sync request
 // 2) it submits syncs to blockstore and peers
 // 3) after everyone acks sync it acks sync to the client
 // 4) after a while it takes his synced object list and sends stabilize requests
 //    to peers and to its own blockstore, thus freeing the old version
 struct ver_vector_t
 {
    obj_ver_id *items = NULL;
    uint64_t alloc = 0, size = 0;
 };
 static void init_versions(ver_vector_t & vec, obj_ver_id *start, obj_ver_id *end, uint64_t len)
 {
    if (!vec.items)
    {
        vec.alloc = len;
        vec.items = (obj_ver_id*)malloc_or_die(sizeof(obj_ver_id) * vec.alloc);
        for (auto sv = start; sv < end; sv++)
        {
            vec.items[vec.size++] = *sv;
        }
    }
 }
 static void append_version(ver_vector_t & vec, obj_ver_id ov)
 {
    if (vec.size >= vec.alloc)
    {
        vec.alloc = !vec.alloc ? 4 : vec.alloc*2;
        vec.items = (obj_ver_id*)realloc_or_die(vec.items, sizeof(obj_ver_id) * vec.alloc);
    }
    vec.items[vec.size++] = ov;
 }
 static bool check_unsynced(std::vector<obj_ver_id> & check, obj_ver_id ov, std::vector<obj_ver_id> & to, int *count)
 {
    bool found = false;
    int j = 0, k = 0;
    while (j < check.size())
    {
        if (check[j] == ov)
            found = true;
        if (check[j].oid == ov.oid && check[j].version <= ov.version)
        {
            to.push_back(check[j++]);
            if (count)
                (*count)--;
        }
        else
            check[k++] = check[j++];
    }
    check.resize(k);
    return found;
 }
 blockstore_op_t* blockstore_impl_t::selective_sync(blockstore_op_t *op)
 {
    unsynced_big_write_count -= unsynced_big_writes.size();
    unsynced_big_writes.swap(PRIV(op)->sync_big_writes);
    unsynced_big_write_count += unsynced_big_writes.size();
    unsynced_small_writes.swap(PRIV(op)->sync_small_writes);
    // Create a sync operation, insert into the end of the queue
    // And move ourselves into the end too!
    // Rather hacky but that's what we need...
    blockstore_op_t *sync_op = new blockstore_op_t;
    sync_op->opcode = BS_OP_SYNC;
    sync_op->buf = NULL;
    sync_op->callback = [](blockstore_op_t *sync_op)
    {
        delete sync_op;
    };
    init_op(sync_op);
    int sync_res = continue_sync(sync_op);
    if (sync_res != 2)
    {
        // Put SYNC into the queue if it's not finished yet
        submit_queue.push_back(sync_op);
    }
    // Restore unsynced_writes
    unsynced_small_writes.swap(PRIV(op)->sync_small_writes);
    unsynced_big_write_count -= unsynced_big_writes.size();
    unsynced_big_writes.swap(PRIV(op)->sync_big_writes);
    unsynced_big_write_count += unsynced_big_writes.size();
    if (sync_res == 2)
    {
        // Sync is immediately completed
        return NULL;
    }
    return sync_op;
 }
 // Returns: 2 = stop processing and dequeue, 0 = stop processing and do not dequeue, 1 = proceed with op itself
 int blockstore_impl_t::split_stab_op(blockstore_op_t *op, std::function<int(obj_ver_id v)> decider)
 {
    bool add_sync = false;
    ver_vector_t good_vers, bad_vers;
    obj_ver_id* v;
    int i, todo = 0;
    for (i = 0, v = (obj_ver_id*)op->buf; i < op->len; i++, v++)
    {
        int action = decider(*v);
        if (action < 0)
        {
            // Rollback changes
            for (auto & ov: PRIV(op)->sync_big_writes)
            {
                unsynced_big_writes.push_back(ov);
                unsynced_big_write_count++;
            }
            for (auto & ov: PRIV(op)->sync_small_writes)
            {
                unsynced_small_writes.push_back(ov);
            }
            free(good_vers.items);
            good_vers.items = NULL;
            free(bad_vers.items);
            bad_vers.items = NULL;
            // Error
            op->retval = action;
            FINISH_OP(op);
            return 2;
        }
        else if (action == STAB_SPLIT_DONE)
        {
            // Already done
            init_versions(good_vers, (obj_ver_id*)op->buf, v, op->len);
        }
        else if (action == STAB_SPLIT_WAIT)
        {
            // Already in progress, we just have to wait until it finishes
            init_versions(good_vers, (obj_ver_id*)op->buf, v, op->len);
            append_version(bad_vers, *v);
        }
        else if (action == STAB_SPLIT_SYNC)
        {
            // Needs a SYNC, we have to send a SYNC if not already in progress
            //
            // If the object is not present in unsynced_(big|small)_writes then
            // it's currently being synced. If it's present then we can initiate
            // its sync ourselves.
            init_versions(good_vers, (obj_ver_id*)op->buf, v, op->len);
            append_version(bad_vers, *v);
            if (!add_sync)
            {
                PRIV(op)->sync_big_writes.clear();
                PRIV(op)->sync_small_writes.clear();
                add_sync = true;
            }
            check_unsynced(unsynced_small_writes, *v, PRIV(op)->sync_small_writes, NULL);
            check_unsynced(unsynced_big_writes, *v, PRIV(op)->sync_big_writes, &unsynced_big_write_count);
        }
        else /* if (action == STAB_SPLIT_TODO) */
        {
            if (good_vers.items)
            {
                // If we're selecting versions then append it
                // Main idea is that 99% of the time all versions passed to BS_OP_STABLE are synced
                // And we don't want to select/allocate anything in that optimistic case
                append_version(good_vers, *v);
            }
            todo++;
        }
    }
    // In a pessimistic scenario, an operation may be split into 3:
    // - Stabilize synced entries
    // - Sync unsynced entries
    // - Continue for unsynced entries after sync
    add_sync = add_sync && (PRIV(op)->sync_big_writes.size() || PRIV(op)->sync_small_writes.size());
    if (!todo && !bad_vers.size)
    {
        // Already stable
        op->retval = 0;
        FINISH_OP(op);
        return 2;
    }
    op->retval = 0;
    if (!todo && !add_sync)
    {
        // Only wait for inflight writes or current in-progress syncs
        return 0;
    }
    blockstore_op_t *sync_op = NULL, *split_stab_op = NULL;
    if (add_sync)
    {
        // Initiate a selective sync for PRIV(op)->sync_(big|small)_writes
        sync_op = selective_sync(op);
    }
    if (bad_vers.size)
    {
        // Split part of the request into a separate operation
        split_stab_op = new blockstore_op_t;
        split_stab_op->opcode = op->opcode;
        split_stab_op->buf = (uint8_t*)bad_vers.items;
        split_stab_op->len = bad_vers.size;
        init_op(split_stab_op);
        submit_queue.push_back(split_stab_op);
    }
    if (sync_op || split_stab_op || good_vers.items)
    {
        uint8_t *orig_buf = op->buf;
        if (good_vers.items)
        {
            op->buf = (uint8_t*)good_vers.items;
            op->len = good_vers.size;
        }
        // Make a wrapped callback
        int *split_op_counter = (int*)malloc_or_die(sizeof(int));
        *split_op_counter = (sync_op ? 1 : 0) + (split_stab_op ? 1 : 0) + (todo ? 1 : 0);
        auto cb = [op, good_items = good_vers.items,
            bad_items = bad_vers.items, split_op_counter,
            orig_buf, real_cb = op->callback](blockstore_op_t *split_op)
        {
            if (split_op->retval != 0)
                op->retval = split_op->retval;
            (*split_op_counter)--;
            assert((*split_op_counter) >= 0);
            if (op != split_op)
                delete split_op;
            if (!*split_op_counter)
            {
                free(good_items);
                free(bad_items);
                free(split_op_counter);
                op->buf = orig_buf;
                real_cb(op);
            }
        };
        if (sync_op)
        {
            sync_op->callback = cb;
        }
        if (split_stab_op)
        {
            split_stab_op->callback = cb;
        }
        op->callback = cb;
    }
    if (!todo)
    {
        // All work is postponed
        op->callback = NULL;
        return 2;
    }
    return 1;
 }
 int blockstore_impl_t::dequeue_stable(blockstore_op_t *op)
 {
    if (PRIV(op)->op_state)
    {
        return continue_stable(op);
    }
    int r = split_stab_op(op, [this](obj_ver_id ov)
    {
        auto dirty_it = dirty_db.find(ov);
        if (dirty_it == dirty_db.end())
        {
            auto & clean_db = clean_db_shard(ov.oid);
            auto clean_it = clean_db.find(ov.oid);
            if (clean_it == clean_db.end() || clean_it->second.version < ov.version)
            {
                // No such object version
                printf("Error: %jx:%jx v%ju not found while stabilizing\n", ov.oid.inode, ov.oid.stripe, ov.version);
                return -ENOENT;
            }
            else
            {
                // Already stable
                return STAB_SPLIT_DONE;
            }
        }
        else if (IS_STABLE(dirty_it->second.state))
        {
            // Already stable
            return STAB_SPLIT_DONE;
        }
        while (true)
        {
            if (IS_IN_FLIGHT(dirty_it->second.state))
            {
                // Object write is still in progress. Wait until the write request completes
                return STAB_SPLIT_WAIT;
            }
            else if (!IS_SYNCED(dirty_it->second.state))
            {
                // Object not synced yet - sync it
                // In previous versions we returned EBUSY here and required
                // the caller (OSD) to issue a global sync first. But a global sync
                // waits for all writes in the queue including inflight writes. And
                // inflight writes may themselves be blocked by unstable writes being
                // still present in the journal and not flushed away from it.
                // So we must sync specific objects here.
                //
                // Even more, we have to process "stabilize" request in parts. That is,
                // we must stabilize all objects which are already synced. Otherwise
                // they may block objects which are NOT synced yet.
                return STAB_SPLIT_SYNC;
            }
            else if (IS_STABLE(dirty_it->second.state))
            {
                break;
            }
            // Check previous versions too
            if (dirty_it == dirty_db.begin())
            {
                break;
            }
            dirty_it--;
            if (dirty_it->first.oid != ov.oid)
            {
                break;
            }
        }
        return STAB_SPLIT_TODO;
    });
    if (r != 1)
    {
        return r;
    }
    // Check journal space
    blockstore_journal_check_t space_check(this);
    if (!space_check.check_available(op, op->len, sizeof(journal_entry_stable), 0))
    {
        return 0;
    }
    // There is sufficient space. Check SQEs
    BS_SUBMIT_CHECK_SQES(space_check.sectors_to_write);
    // Prepare and submit journal entries
    int s = 0;
    auto v = (obj_ver_id*)op->buf;
    for (int i = 0; i < op->len; i++, v++)
    {
        if (!journal.entry_fits(sizeof(journal_entry_stable)) &&
            journal.sector_info[journal.cur_sector].dirty)
        {
            prepare_journal_sector_write(journal.cur_sector, op);
            s++;
        }
        journal_entry_stable *je = (journal_entry_stable*)
            prefill_single_journal_entry(journal, JE_STABLE, sizeof(journal_entry_stable));
        je->oid = v->oid;
        je->version = v->version;
        je->crc32 = je_crc32((journal_entry*)je);
        journal.crc32_last = je->crc32;
    }
    prepare_journal_sector_write(journal.cur_sector, op);
    s++;
    assert(s == space_check.sectors_to_write);
    PRIV(op)->op_state = 1;
    return 1;
 }
 int blockstore_impl_t::continue_stable(blockstore_op_t *op)
 {
    if (PRIV(op)->op_state == 2)
        goto resume_2;
    else if (PRIV(op)->op_state == 4)
        goto resume_4;
    else
        return 1;
 resume_2:
    if (!disable_journal_fsync)
    {
        BS_SUBMIT_GET_SQE(sqe, data);
        io_uring_prep_fsync(sqe, dsk.journal_fd, IORING_FSYNC_DATASYNC);
        data->iov = { 0 };
        data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
        PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 0;
        PRIV(op)->pending_ops = 1;
        PRIV(op)->op_state = 3;
        return 1;
    }
 resume_4:
    // Mark dirty_db entries as stable, acknowledge op completion
    obj_ver_id* v;
    int i;
    for (i = 0, v = (obj_ver_id*)op->buf; i < op->len; i++, v++)
    {
        // Mark all dirty_db entries up to op->version as stable
 #ifdef BLOCKSTORE_DEBUG
        printf("Stabilize %jx:%jx v%ju\n", v->oid.inode, v->oid.stripe, v->version);
 #endif
        mark_stable(*v);
    }
    // Acknowledge op
    op->retval = 0;
    FINISH_OP(op);
    return 2;
 }
 void blockstore_impl_t::mark_stable(obj_ver_id v, bool forget_dirty)
 {
    auto dirty_it = dirty_db.find(v);
    if (dirty_it != dirty_db.end())
    {
        if (IS_INSTANT(dirty_it->second.state))
        {
            // 'Instant' (non-EC) operations may complete and try to become stable out of order. Prevent it.
            auto back_it = dirty_it;
            while (back_it != dirty_db.begin())
            {
                back_it--;
                if (back_it->first.oid != v.oid)
                {
                    break;
                }
                if (!IS_STABLE(back_it->second.state))
                {
                    // There are preceding unstable versions, can't flush <v>
                    return;
                }
            }
            while (true)
            {
                dirty_it++;
                if (dirty_it == dirty_db.end() || dirty_it->first.oid != v.oid ||
                    !IS_SYNCED(dirty_it->second.state))
                {
                    dirty_it--;
                    break;
                }
                v.version = dirty_it->first.version;
            }
        }
        while (1)
        {
            bool was_stable = IS_STABLE(dirty_it->second.state);
            if ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_SYNCED)
            {
                dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | BS_ST_STABLE;
                // Allocations and deletions are counted when they're stabilized
                if (IS_BIG_WRITE(dirty_it->second.state))
                {
                    int exists = -1;
                    if (dirty_it != dirty_db.begin())
                    {
                        auto prev_it = dirty_it;
                        prev_it--;
                        if (prev_it->first.oid == v.oid)
                        {
                            exists = IS_DELETE(prev_it->second.state) ? 0 : 1;
                        }
                    }
                    if (exists == -1)
                    {
                        auto & clean_db = clean_db_shard(v.oid);
                        auto clean_it = clean_db.find(v.oid);
                        exists = clean_it != clean_db.end() ? 1 : 0;
                    }
                    if (!exists)
                    {
                        uint64_t space_id = dirty_it->first.oid.inode;
                        if (no_inode_stats[dirty_it->first.oid.inode >> (64-POOL_ID_BITS)])
                            space_id = space_id & ~(((uint64_t)1 << (64-POOL_ID_BITS)) - 1);
                        inode_space_stats[space_id] += dsk.data_block_size;
                        used_blocks++;
                    }
                    big_to_flush++;
                }
                else if (IS_DELETE(dirty_it->second.state))
                {
                    uint64_t space_id = dirty_it->first.oid.inode;
                    if (no_inode_stats[dirty_it->first.oid.inode >> (64-POOL_ID_BITS)])
                        space_id = space_id & ~(((uint64_t)1 << (64-POOL_ID_BITS)) - 1);
                    auto & sp = inode_space_stats[space_id];
                    if (sp > dsk.data_block_size)
                        sp -= dsk.data_block_size;
                    else
                        inode_space_stats.erase(space_id);
                    used_blocks--;
                    big_to_flush++;
                }
            }
            else if (IS_IN_FLIGHT(dirty_it->second.state))
            {
                // mark_stable should never be called for in-flight or submitted writes
                printf(
                    "BUG: Attempt to mark_stable object %jx:%jx v%ju state of which is %x\n",
                    dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version,
                    dirty_it->second.state
                );
                exit(1);
            }
            if (forget_dirty && (IS_BIG_WRITE(dirty_it->second.state) ||
                IS_DELETE(dirty_it->second.state)))
            {
                // Big write overrides all previous dirty entries
                auto erase_end = dirty_it;
                while (dirty_it != dirty_db.begin())
                {
                    dirty_it--;
                    if (dirty_it->first.oid != v.oid)
                    {
                        dirty_it++;
                        break;
                    }
                }
                auto & clean_db = clean_db_shard(v.oid);
                auto clean_it = clean_db.find(v.oid);
                uint64_t clean_loc = clean_it != clean_db.end()
                    ? clean_it->second.location : UINT64_MAX;
                erase_dirty(dirty_it, erase_end, clean_loc);
                break;
            }
            if (was_stable || dirty_it == dirty_db.begin())
            {
                break;
            }
            dirty_it--;
            if (dirty_it->first.oid != v.oid)
            {
                break;
            }
        }
        flusher->enqueue_flush(v);
    }
    auto unstab_it = unstable_writes.find(v.oid);
    if (unstab_it != unstable_writes.end() &&
        unstab_it->second <= v.version)
    {
        unstable_writes.erase(unstab_it);
    }
 }
 } // namespace v1
@@ -1,238 +0,0 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 #include "impl.h"
 #include "internal.h"
 namespace v1 {
 #define SYNC_HAS_SMALL 1
 #define SYNC_HAS_BIG 2
 #define SYNC_DATA_SYNC_SENT 3
 #define SYNC_DATA_SYNC_DONE 4
 #define SYNC_JOURNAL_WRITE_SENT 5
 #define SYNC_JOURNAL_WRITE_DONE 6
 #define SYNC_JOURNAL_SYNC_SENT 7
 #define SYNC_DONE 8
 int blockstore_impl_t::continue_sync(blockstore_op_t *op)
 {
    if (immediate_commit == IMMEDIATE_ALL)
    {
        // We can return immediately because sync is only dequeued after all previous writes
        op->retval = 0;
        FINISH_OP(op);
        return 2;
    }
    if (PRIV(op)->op_state == 0)
    {
        stop_sync_submitted = false;
        unsynced_big_write_count -= unsynced_big_writes.size();
        PRIV(op)->sync_big_writes.swap(unsynced_big_writes);
        PRIV(op)->sync_small_writes.swap(unsynced_small_writes);
        unsynced_big_writes.clear();
        unsynced_small_writes.clear();
        if (PRIV(op)->sync_big_writes.size() > 0)
            PRIV(op)->op_state = SYNC_HAS_BIG;
        else if (PRIV(op)->sync_small_writes.size() > 0)
            PRIV(op)->op_state = SYNC_HAS_SMALL;
        else
            PRIV(op)->op_state = SYNC_DONE;
    }
    if (PRIV(op)->op_state == SYNC_HAS_SMALL)
    {
        // No big writes, just fsync the journal
        if (journal.sector_info[journal.cur_sector].dirty)
        {
            // Write out the last journal sector if it happens to be dirty
            BS_SUBMIT_CHECK_SQES(1);
            prepare_journal_sector_write(journal.cur_sector, op);
            PRIV(op)->op_state = SYNC_JOURNAL_WRITE_SENT;
            return 1;
        }
        else
        {
            PRIV(op)->op_state = SYNC_JOURNAL_WRITE_DONE;
        }
    }
    if (PRIV(op)->op_state == SYNC_HAS_BIG)
    {
        // 1st step: fsync data
        if (!disable_data_fsync)
        {
            BS_SUBMIT_GET_SQE(sqe, data);
            io_uring_prep_fsync(sqe, dsk.data_fd, IORING_FSYNC_DATASYNC);
            data->iov = { 0 };
            data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
            PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 0;
            PRIV(op)->pending_ops = 1;
            PRIV(op)->op_state = SYNC_DATA_SYNC_SENT;
            return 1;
        }
        else
        {
            PRIV(op)->op_state = SYNC_DATA_SYNC_DONE;
        }
    }
    if (PRIV(op)->op_state == SYNC_DATA_SYNC_DONE)
    {
        // 2nd step: Data device is synced, prepare & write journal entries
        // Check space in the journal and journal memory buffers
        blockstore_journal_check_t space_check(this);
        if (dsk.csum_block_size)
        {
            // More complex check because all journal entries have different lengths
            int left = PRIV(op)->sync_big_writes.size();
            for (auto & sbw: PRIV(op)->sync_big_writes)
            {
                left--;
                auto & dirty_entry = dirty_db.at(sbw);
                uint64_t dyn_size = dsk.dirty_dyn_size(dirty_entry.offset, dirty_entry.len);
                if (!space_check.check_available(op, 1, sizeof(journal_entry_big_write) + dyn_size, 0))
                {
                    return 0;
                }
            }
        }
        else if (!space_check.check_available(op, PRIV(op)->sync_big_writes.size(),
            sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size, 0))
        {
            return 0;
        }
        // Check SQEs. Don't bother about merging, submit each journal sector as a separate request
        BS_SUBMIT_CHECK_SQES(space_check.sectors_to_write);
        // Prepare and submit journal entries
        auto it = PRIV(op)->sync_big_writes.begin();
        int s = 0;
        while (it != PRIV(op)->sync_big_writes.end())
        {
            auto & dirty_entry = dirty_db.at(*it);
            uint64_t dyn_size = dsk.dirty_dyn_size(dirty_entry.offset, dirty_entry.len);
            if (!journal.entry_fits(sizeof(journal_entry_big_write) + dyn_size) &&
                journal.sector_info[journal.cur_sector].dirty)
            {
                prepare_journal_sector_write(journal.cur_sector, op);
                s++;
            }
            journal_entry_big_write *je = (journal_entry_big_write*)prefill_single_journal_entry(
                journal, (dirty_entry.state & BS_ST_INSTANT) ? JE_BIG_WRITE_INSTANT : JE_BIG_WRITE,
                sizeof(journal_entry_big_write) + dyn_size
            );
            auto jsec = dirty_entry.journal_sector = journal.sector_info[journal.cur_sector].offset;
            assert(journal.next_free >= journal.used_start
                ? (jsec >= journal.used_start && jsec < journal.next_free)
                : (jsec >= journal.used_start || jsec < journal.next_free));
            journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++;
 #ifdef BLOCKSTORE_DEBUG
            printf(
                "journal offset %08jx is used by %jx:%jx v%ju (%ju refs)\n",
                dirty_entry.journal_sector, it->oid.inode, it->oid.stripe, it->version,
                journal.used_sectors[journal.sector_info[journal.cur_sector].offset]
            );
 #endif
            je->oid = it->oid;
            je->version = it->version;
            je->offset = dirty_entry.offset;
            je->len = dirty_entry.len;
            je->location = dirty_entry.location;
            memcpy((void*)(je+1), (alloc_dyn_data
                ? (uint8_t*)dirty_entry.dyn_data+sizeof(int) : (uint8_t*)&dirty_entry.dyn_data), dyn_size);
            je->crc32 = je_crc32((journal_entry*)je);
            journal.crc32_last = je->crc32;
            it++;
        }
        prepare_journal_sector_write(journal.cur_sector, op);
        s++;
        assert(s == space_check.sectors_to_write);
        PRIV(op)->op_state = SYNC_JOURNAL_WRITE_SENT;
        return 1;
    }
    if (PRIV(op)->op_state == SYNC_JOURNAL_WRITE_DONE)
    {
        if (!disable_journal_fsync)
        {
            BS_SUBMIT_GET_SQE(sqe, data);
            io_uring_prep_fsync(sqe, dsk.journal_fd, IORING_FSYNC_DATASYNC);
            data->iov = { 0 };
            data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
            PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 0;
            PRIV(op)->pending_ops = 1;
            PRIV(op)->op_state = SYNC_JOURNAL_SYNC_SENT;
            return 1;
        }
        else
        {
            PRIV(op)->op_state = SYNC_DONE;
        }
    }
    if (PRIV(op)->op_state == SYNC_DONE)
    {
        ack_sync(op);
        return 2;
    }
    return 1;
 }
 void blockstore_impl_t::ack_sync(blockstore_op_t *op)
 {
    // Handle states
    for (auto it = PRIV(op)->sync_big_writes.begin(); it != PRIV(op)->sync_big_writes.end(); it++)
    {
 #ifdef BLOCKSTORE_DEBUG
        printf("Ack sync big %jx:%jx v%ju\n", it->oid.inode, it->oid.stripe, it->version);
 #endif
        auto & unstab = unstable_writes[it->oid];
        unstab = unstab < it->version ? it->version : unstab;
        auto dirty_it = dirty_db.find(*it);
        dirty_it->second.state = ((dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | BS_ST_SYNCED);
        if (dirty_it->second.state & BS_ST_INSTANT)
        {
            mark_stable(dirty_it->first);
        }
        else
        {
            unstable_unsynced--;
            assert(unstable_unsynced >= 0);
        }
        dirty_it++;
        while (dirty_it != dirty_db.end() && dirty_it->first.oid == it->oid)
        {
            if ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_WAIT_BIG)
            {
                dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | BS_ST_IN_FLIGHT;
            }
            dirty_it++;
        }
    }
    for (auto it = PRIV(op)->sync_small_writes.begin(); it != PRIV(op)->sync_small_writes.end(); it++)
    {
 #ifdef BLOCKSTORE_DEBUG
        printf("Ack sync small %jx:%jx v%ju\n", it->oid.inode, it->oid.stripe, it->version);
 #endif
        auto & unstab = unstable_writes[it->oid];
        unstab = unstab < it->version ? it->version : unstab;
        if (dirty_db[*it].state == (BS_ST_DELETE | BS_ST_WRITTEN))
        {
            dirty_db[*it].state = (BS_ST_DELETE | BS_ST_SYNCED);
            // Deletions are treated as immediately stable
            mark_stable(*it);
        }
        else /* (BS_ST_INSTANT?) | BS_ST_SMALL_WRITE | BS_ST_WRITTEN */
        {
            dirty_db[*it].state = (dirty_db[*it].state & ~BS_ST_WORKFLOW_MASK) | BS_ST_SYNCED;
            if (dirty_db[*it].state & BS_ST_INSTANT)
            {
                mark_stable(*it);
            }
            else
            {
                unstable_unsynced--;
                assert(unstable_unsynced >= 0);
            }
        }
    }
    op->retval = 0;
    FINISH_OP(op);
 }
 } // namespace v1
@@ -1,828 +0,0 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 #include "impl.h"
 #include "internal.h"
 namespace v1 {
 bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
 {
    // Check or assign version number
    bool found = false, deleted = false, unsynced = false, is_del = (op->opcode == BS_OP_DELETE);
    bool wait_big = false, wait_del = false;
    void *dyn = NULL;
    if (is_del)
    {
        op->len = 0;
    }
    size_t dyn_size = dsk.dirty_dyn_size(op->offset, op->len);
    if (!is_del && alloc_dyn_data)
    {
        // FIXME: Working with `dyn_data` has to be refactored somehow but I first have to decide how :)
        // +sizeof(int) = refcount
        dyn = calloc_or_die(1, dyn_size+sizeof(int));
        *((int*)dyn) = 1;
    }
    uint8_t *dyn_ptr = (alloc_dyn_data ? (uint8_t*)dyn+sizeof(int) : (uint8_t*)&dyn);
    uint64_t version = 1;
    if (dirty_db.size() > 0)
    {
        auto dirty_it = dirty_db.upper_bound((obj_ver_id){
            .oid = op->oid,
            .version = UINT64_MAX,
        });
        dirty_it--; // segfaults when dirty_db is empty
        if (dirty_it != dirty_db.end() && dirty_it->first.oid == op->oid)
        {
            found = true;
            version = dirty_it->first.version + 1;
            deleted = IS_DELETE(dirty_it->second.state);
            unsynced = !IS_SYNCED(dirty_it->second.state);
            wait_del = ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_WAIT_DEL);
            wait_big = (dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_BIG_WRITE
                ? !IS_SYNCED(dirty_it->second.state)
                : ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_WAIT_BIG);
            if (!is_del && !deleted)
            {
                void *dyn_from = alloc_dyn_data
                    ? (uint8_t*)dirty_it->second.dyn_data + sizeof(int) : (uint8_t*)&dirty_it->second.dyn_data;
                memcpy(dyn_ptr, dyn_from, dsk.clean_entry_bitmap_size);
            }
        }
    }
    if (!found)
    {
        auto & clean_db = clean_db_shard(op->oid);
        auto clean_it = clean_db.find(op->oid);
        if (clean_it != clean_db.end())
        {
            version = clean_it->second.version + 1;
            if (!is_del)
            {
                void *bmp_ptr = get_clean_entry_bitmap(clean_it->second.location, dsk.clean_entry_bitmap_size);
                memcpy(dyn_ptr, bmp_ptr, dsk.clean_entry_bitmap_size);
            }
        }
        else
        {
            deleted = true;
        }
    }
    if (deleted && is_del)
    {
        // Already deleted
        op->retval = 0;
        return false;
    }
    PRIV(op)->real_version = 0;
    if (op->version == 0)
    {
        op->version = version;
    }
    else if (op->version < version)
    {
        // Implicit operations must be added like that: DEL [FLUSH] BIG [SYNC] SMALL SMALL
        if (deleted || wait_del)
        {
            // It's allowed to write versions with low numbers over deletes
            // However, we have to flush those deletes first as we use version number for ordering
 #ifdef BLOCKSTORE_DEBUG
            printf("Write %jx:%jx v%ju over delete (real v%ju) offset=%u len=%u\n", op->oid.inode, op->oid.stripe, version, op->version, op->offset, op->len);
 #endif
            wait_del = true;
            PRIV(op)->real_version = op->version;
            op->version = version;
            if (unsynced)
            {
                // Issue an additional sync so the delete reaches the journal
                blockstore_op_t *sync_op = new blockstore_op_t;
                sync_op->opcode = BS_OP_SYNC;
                sync_op->oid = op->oid;
                sync_op->version = op->version;
                sync_op->callback = [this](blockstore_op_t *sync_op)
                {
                    flusher->unshift_flush((obj_ver_id){
                        .oid = sync_op->oid,
                        .version = sync_op->version-1,
                    }, true);
                    delete sync_op;
                };
                enqueue_op(sync_op);
            }
            else
            {
                flusher->unshift_flush((obj_ver_id){
                    .oid = op->oid,
                    .version = version-1,
                }, true);
            }
        }
        else
        {
            // Invalid version requested
 #ifdef BLOCKSTORE_DEBUG
            printf("Write %jx:%jx v%ju requested, but we already have v%ju\n", op->oid.inode, op->oid.stripe, op->version, version);
 #endif
            op->retval = -EEXIST;
            if (!is_del && alloc_dyn_data)
            {
                free(dyn);
            }
            return false;
        }
    }
    bool imm = (op->len < dsk.data_block_size ? (immediate_commit != IMMEDIATE_NONE) : (immediate_commit == IMMEDIATE_ALL));
    if (wait_big && !is_del && !deleted && op->len < dsk.data_block_size && !imm ||
        !imm && autosync_writes && unsynced_queued_ops >= autosync_writes)
    {
        // Issue an additional sync so that the previous big write can reach the journal
        blockstore_op_t *sync_op = new blockstore_op_t;
        sync_op->opcode = BS_OP_SYNC;
        sync_op->callback = [](blockstore_op_t *sync_op)
        {
            delete sync_op;
        };
        enqueue_op(sync_op);
    }
    else if (!imm)
        unsynced_queued_ops++;
 #ifdef BLOCKSTORE_DEBUG
    if (is_del)
        printf("Delete %jx:%jx v%ju\n", op->oid.inode, op->oid.stripe, op->version);
    else if (!wait_del)
        printf("Write %jx:%jx v%ju offset=%u len=%u\n", op->oid.inode, op->oid.stripe, op->version, op->offset, op->len);
 #endif
    // No strict need to add it into dirty_db here except maybe for listings to return
    // correct data when there are inflight operations in the queue
    uint32_t state;
    if (is_del)
        state = BS_ST_DELETE | BS_ST_IN_FLIGHT;
    else
    {
        state = (op->len == dsk.data_block_size || deleted ? BS_ST_BIG_WRITE : BS_ST_SMALL_WRITE);
        if (state == BS_ST_SMALL_WRITE && throttle_small_writes)
            clock_gettime(CLOCK_REALTIME, &PRIV(op)->tv_begin);
        if (wait_del)
            state |= BS_ST_WAIT_DEL;
        else if (state == BS_ST_SMALL_WRITE && wait_big)
            state |= BS_ST_WAIT_BIG;
        else
            state |= BS_ST_IN_FLIGHT;
        if (op->opcode == BS_OP_WRITE_STABLE)
            state |= BS_ST_INSTANT;
        if (op->bitmap)
            memcpy(dyn_ptr, op->bitmap, dsk.clean_entry_bitmap_size);
    }
    // Calculate checksums
    // FIXME: Allow to receive checksums from outside?
    if (!is_del && dsk.data_csum_type && op->len > 0)
    {
        uint32_t *data_csums = (uint32_t*)(dyn_ptr + dsk.clean_entry_bitmap_size);
        uint32_t start = op->offset / dsk.csum_block_size;
        uint32_t end = (op->offset+op->len-1) / dsk.csum_block_size;
        auto fn = state & BS_ST_BIG_WRITE ? crc32c_pad : crc32c_nopad;
        if (start == end)
            data_csums[0] = fn(0, op->buf, op->len, op->offset - start*dsk.csum_block_size, end*dsk.csum_block_size - (op->offset+op->len));
        else
        {
            // First block
            data_csums[0] = fn(0, op->buf, dsk.csum_block_size*(start+1)-op->offset, op->offset - start*dsk.csum_block_size, 0);
            // Intermediate blocks
            for (uint32_t i = start+1; i < end; i++)
                data_csums[i-start] = crc32c(0, (uint8_t*)op->buf + dsk.csum_block_size*i-op->offset, dsk.csum_block_size);
            // Last block
            data_csums[end-start] = fn(
                0, (uint8_t*)op->buf + end*dsk.csum_block_size - op->offset,
                op->offset+op->len - end*dsk.csum_block_size,
                0, (end+1)*dsk.csum_block_size - (op->offset+op->len)
            );
        }
    }
    dirty_db.emplace((obj_ver_id){
        .oid = op->oid,
        .version = op->version,
    }, (dirty_entry){
        .state = state,
        .flags = 0,
        .location = 0,
        .offset = is_del ? 0 : op->offset,
        .len = is_del ? 0 : op->len,
        .journal_sector = 0,
        .dyn_data = dyn,
    });
    return true;
 }
 void blockstore_impl_t::cancel_all_writes(blockstore_op_t *op, blockstore_dirty_db_t::iterator dirty_it, int retval)
 {
    while (dirty_it != dirty_db.end() && dirty_it->first.oid == op->oid)
    {
        free_dirty_dyn_data(dirty_it->second);
        dirty_db.erase(dirty_it++);
    }
    bool found = false;
    for (auto other_op: submit_queue)
    {
        if (!other_op)
        {
            // freed operations during submitting are zeroed
        }
        else if (other_op == op)
        {
            // <op> may be present in queue multiple times due to moving operations in submit_queue
            found = true;
        }
        else if (found && other_op->oid == op->oid &&
            (other_op->opcode == BS_OP_WRITE || other_op->opcode == BS_OP_WRITE_STABLE))
        {
            // Mark operations to cancel them
            PRIV(other_op)->real_version = UINT64_MAX;
            other_op->retval = retval;
        }
    }
    op->retval = retval;
    FINISH_OP(op);
 }
 // First step of the write algorithm: dequeue operation and submit initial write(s)
 int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
 {
    if (PRIV(op)->op_state)
    {
        return continue_write(op);
    }
    auto dirty_it = dirty_db.find((obj_ver_id){
        .oid = op->oid,
        .version = op->version,
    });
    assert(dirty_it != dirty_db.end());
    if ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) < BS_ST_IN_FLIGHT)
    {
        // Don't dequeue
        return 0;
    }
    if (PRIV(op)->real_version != 0)
    {
        if (PRIV(op)->real_version == UINT64_MAX)
        {
            // This is the flag value used to cancel operations
            FINISH_OP(op);
            return 2;
        }
        // Restore original low version number for unblocked operations
 #ifdef BLOCKSTORE_DEBUG
        printf("Restoring %jx:%jx version: v%ju -> v%ju\n", op->oid.inode, op->oid.stripe, op->version, PRIV(op)->real_version);
 #endif
        auto prev_it = dirty_it;
        if (prev_it != dirty_db.begin())
        {
            prev_it--;
            if (prev_it->first.oid == op->oid && prev_it->first.version >= PRIV(op)->real_version)
            {
                // Original version is still invalid
                // All subsequent writes to the same object must be canceled too
                printf("Tried to write %jx:%jx v%ju after delete (old version v%ju), but already have v%ju\n",
                    op->oid.inode, op->oid.stripe, PRIV(op)->real_version, op->version, prev_it->first.version);
                cancel_all_writes(op, dirty_it, -EEXIST);
                return 2;
            }
        }
        op->version = PRIV(op)->real_version;
        PRIV(op)->real_version = 0;
        dirty_entry e = dirty_it->second;
        dirty_db.erase(dirty_it);
        dirty_it = dirty_db.emplace((obj_ver_id){
            .oid = op->oid,
            .version = op->version,
        }, e).first;
    }
    if (write_iodepth >= max_write_iodepth)
    {
        return 0;
    }
    if ((dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_BIG_WRITE)
    {
        blockstore_journal_check_t space_check(this);
        if (!space_check.check_available(op, unsynced_big_write_count + 1,
            sizeof(journal_entry_big_write) + dsk.clean_dyn_size,
            (unstable_writes.size()+unstable_unsynced+((dirty_it->second.state & BS_ST_INSTANT) ? 0 : 1))*journal.block_size))
        {
            return 0;
        }
        // Big (redirect) write
        uint64_t loc = data_alloc->find_free();
        if (loc == UINT64_MAX)
        {
            // no space
            if (big_to_flush > 0)
            {
                // hope that some space will be available after flush
                flusher->request_trim();
                PRIV(op)->wait_for = WAIT_FREE;
                return 0;
            }
            cancel_all_writes(op, dirty_it, -ENOSPC);
            return 2;
        }
        if (inmemory_meta)
        {
            // Check once more that metadata entry is zeroed (the reverse means a bug or corruption)
            uint64_t sector = (loc / (dsk.meta_block_size / dsk.clean_entry_size)) * dsk.meta_block_size;
            uint64_t pos = (loc % (dsk.meta_block_size / dsk.clean_entry_size));
            clean_disk_entry *entry = (clean_disk_entry*)((uint8_t*)metadata_buffer + sector + pos*dsk.clean_entry_size);
            if (entry->oid.inode || entry->oid.stripe || entry->version)
            {
                printf(
                    "Fatal error (metadata corruption or bug): tried to write object %jx:%jx v%ju"
                    " over a non-zero metadata entry %ju with %jx:%jx v%ju\n", op->oid.inode,
                    op->oid.stripe, op->version, loc, entry->oid.inode, entry->oid.stripe, entry->version
                );
                exit(1);
            }
        }
        BS_SUBMIT_GET_SQE(sqe, data);
        write_iodepth++;
        dirty_it->second.location = loc * dsk.data_block_size;
        dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | BS_ST_SUBMITTED;
 #ifdef BLOCKSTORE_DEBUG
        printf(
            "Allocate block %ju for %jx:%jx v%ju\n",
            loc, op->oid.inode, op->oid.stripe, op->version
        );
 #endif
        data_alloc->set(loc, true);
        uint64_t stripe_offset = (op->offset % dsk.bitmap_granularity);
        uint64_t stripe_end = (op->offset + op->len) % dsk.bitmap_granularity;
        // Zero fill up to dsk.bitmap_granularity
        int vcnt = 0;
        if (stripe_offset)
        {
            PRIV(op)->iov_zerofill[vcnt++] = (struct iovec){ zero_object, (size_t)stripe_offset };
        }
        PRIV(op)->iov_zerofill[vcnt++] = (struct iovec){ op->buf, op->len };
        if (stripe_end)
        {
            stripe_end = dsk.bitmap_granularity - stripe_end;
            PRIV(op)->iov_zerofill[vcnt++] = (struct iovec){ zero_object, (size_t)stripe_end };
        }
        data->iov.iov_len = op->len + stripe_offset + stripe_end; // to check it in the callback
        data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
        io_uring_prep_writev(
            sqe, dsk.data_fd, PRIV(op)->iov_zerofill, vcnt, dsk.data_offset + (loc * dsk.data_block_size) + op->offset - stripe_offset
        );
        PRIV(op)->pending_ops = 1;
        if (!(dirty_it->second.state & BS_ST_INSTANT))
        {
            unstable_unsynced++;
        }
        if (immediate_commit != IMMEDIATE_ALL)
        {
            // Increase the counter, but don't save into unsynced_writes yet (can't sync until the write is finished)
            unsynced_big_write_count++;
            PRIV(op)->op_state = 3;
        }
        else
        {
            PRIV(op)->op_state = 1;
        }
    }
    else /* if ((dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_SMALL_WRITE) */
    {
        // Small (journaled) write
        // First check if the journal has sufficient space
        uint64_t dyn_size = dsk.dirty_dyn_size(op->offset, op->len);
        blockstore_journal_check_t space_check(this);
        if (unsynced_big_write_count &&
            !space_check.check_available(op, unsynced_big_write_count,
                sizeof(journal_entry_big_write) + dsk.clean_dyn_size, 0)
            || !space_check.check_available(op, 1,
                sizeof(journal_entry_small_write) + dyn_size,
                op->len + (unstable_writes.size()+unstable_unsynced+((dirty_it->second.state & BS_ST_INSTANT) ? 0 : 1))*journal.block_size))
        {
            return 0;
        }
        // There is sufficient space. Check SQE(s)
        BS_SUBMIT_CHECK_SQES(
            // Write current journal sector only if it's dirty and full, or in the immediate_commit mode
            (immediate_commit != IMMEDIATE_NONE ||
                !journal.entry_fits(sizeof(journal_entry_small_write) + dyn_size) ? 1 : 0) +
            (op->len > 0 ? 1 : 0)
        );
        write_iodepth++;
        // Got SQEs. Prepare previous journal sector write if required
        if (immediate_commit == IMMEDIATE_NONE &&
            !journal.entry_fits(sizeof(journal_entry_small_write) + dyn_size))
        {
            prepare_journal_sector_write(journal.cur_sector, op);
        }
        // Then pre-fill journal entry
        journal_entry_small_write *je = (journal_entry_small_write*)prefill_single_journal_entry(
            journal, op->opcode == BS_OP_WRITE_STABLE ? JE_SMALL_WRITE_INSTANT : JE_SMALL_WRITE,
            sizeof(journal_entry_small_write) + dyn_size
        );
        auto jsec = dirty_it->second.journal_sector = journal.sector_info[journal.cur_sector].offset;
        if (!(journal.next_free >= journal.used_start
            ? (jsec >= journal.used_start && jsec < journal.next_free)
            : (jsec >= journal.used_start || jsec < journal.next_free)))
        {
            printf(
                "BUG: journal offset %08jx is used by %jx:%jx v%ju (%ju refs) BUT used_start=%jx next_free=%jx\n",
                dirty_it->second.journal_sector, dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version,
                journal.used_sectors[journal.sector_info[journal.cur_sector].offset],
                journal.used_start, journal.next_free
            );
            abort();
        }
        journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++;
 #ifdef BLOCKSTORE_DEBUG
        printf(
            "journal offset %08jx is used by %jx:%jx v%ju (%ju refs)\n",
            dirty_it->second.journal_sector, dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version,
            journal.used_sectors[journal.sector_info[journal.cur_sector].offset]
        );
 #endif
        // Figure out where data will be
        auto next_next_free = (journal.next_free + op->len) <= journal.len ? journal.next_free : dsk.journal_block_size;
        if (op->len > 0)
        {
            auto journal_used_it = journal.used_sectors.lower_bound(next_next_free);
            if (journal_used_it != journal.used_sectors.end() &&
                journal_used_it->first < next_next_free + op->len)
            {
                printf(
                    "BUG: Attempt to overwrite used offset (%jx, %ju refs) of the journal with the object %jx:%jx v%ju: data at %jx, len %x!"
                    " Journal used_start=%08jx (%ju refs), next_free=%08jx, dirty_start=%08jx\n",
                    journal_used_it->first, journal_used_it->second, op->oid.inode, op->oid.stripe, op->version, next_next_free, op->len,
                    journal.used_start, journal.used_sectors[journal.used_start], journal.next_free, journal.dirty_start
                );
                exit(1);
            }
        }
        // double check that next_free doesn't cross used_start from the left
        assert(journal.next_free >= journal.used_start && next_next_free >= journal.next_free || next_next_free < journal.used_start);
        journal.next_free = next_next_free;
        je->oid = op->oid;
        je->version = op->version;
        je->offset = op->offset;
        je->len = op->len;
        je->data_offset = journal.next_free;
        je->crc32_data = dsk.csum_block_size ? 0 : crc32c(0, op->buf, op->len);
        memcpy((void*)(je+1), (alloc_dyn_data
            ? (uint8_t*)dirty_it->second.dyn_data+sizeof(int) : (uint8_t*)&dirty_it->second.dyn_data), dyn_size);
        je->crc32 = je_crc32((journal_entry*)je);
        journal.crc32_last = je->crc32;
        if (immediate_commit != IMMEDIATE_NONE)
        {
            prepare_journal_sector_write(journal.cur_sector, op);
        }
        if (op->len > 0)
        {
            // Prepare journal data write
            if (journal.inmemory)
            {
                // Copy data
                memcpy((uint8_t*)journal.buffer + journal.next_free, op->buf, op->len);
            }
            BS_SUBMIT_GET_SQE(sqe2, data2);
            data2->iov = (struct iovec){ op->buf, op->len };
            ++journal.submit_id;
            assert(journal.submit_id != 0); // check overflow
            // Make subsequent journal writes wait for our data write
            journal.flushing_ops.emplace(journal.submit_id, (pending_journaling_t){
                .pending = 1,
                .sector = -1,
                .op = op,
            });
            data2->callback = [this, flush_id = journal.submit_id](ring_data_t *data) { handle_journal_write(data, flush_id); };
            io_uring_prep_writev(
                sqe2, dsk.journal_fd, &data2->iov, 1, journal.offset + journal.next_free
            );
            PRIV(op)->pending_ops++;
        }
        else
        {
            // Zero-length overwrite. Allowed to bump object version in EC placement groups without actually writing data
        }
        dirty_it->second.location = journal.next_free;
        dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | BS_ST_SUBMITTED;
        next_next_free = journal.next_free + op->len;
        if (next_next_free >= journal.len)
            next_next_free = dsk.journal_block_size;
        // double check that next_free doesn't cross used_start from the left
        assert(journal.next_free >= journal.used_start && next_next_free >= journal.next_free || next_next_free < journal.used_start);
        journal.next_free = next_next_free;
        if (!(dirty_it->second.state & BS_ST_INSTANT))
        {
            unstable_unsynced++;
        }
        if (!PRIV(op)->pending_ops)
        {
            PRIV(op)->op_state = 4;
            return continue_write(op);
        }
        else
        {
            PRIV(op)->op_state = 3;
        }
    }
    return 1;
 }
 int blockstore_impl_t::continue_write(blockstore_op_t *op)
 {
    int op_state = PRIV(op)->op_state;
    if (op_state == 2)
        goto resume_2;
    else if (op_state == 4)
        goto resume_4;
    else if (op_state == 6)
        goto resume_6;
    else
    {
        // In progress
        return 1;
    }
 resume_2:
    // Only for the immediate_commit mode: prepare and submit big_write journal entry
    {
        auto dirty_it = dirty_db.find((obj_ver_id){
            .oid = op->oid,
            .version = op->version,
        });
        assert(dirty_it != dirty_db.end());
        uint64_t dyn_size = dsk.dirty_dyn_size(op->offset, op->len);
        blockstore_journal_check_t space_check(this);
        if (!space_check.check_available(op, 1, sizeof(journal_entry_big_write) + dyn_size,
            (unstable_writes.size()+unstable_unsynced+((dirty_it->second.state & BS_ST_INSTANT) ? 0 : 1))*journal.block_size))
        {
            return 0;
        }
        BS_SUBMIT_CHECK_SQES(1);
        journal_entry_big_write *je = (journal_entry_big_write*)prefill_single_journal_entry(
            journal, op->opcode == BS_OP_WRITE_STABLE ? JE_BIG_WRITE_INSTANT : JE_BIG_WRITE,
            sizeof(journal_entry_big_write) + dyn_size
        );
        auto jsec = dirty_it->second.journal_sector = journal.sector_info[journal.cur_sector].offset;
        if (!(journal.next_free >= journal.used_start
            ? (jsec >= journal.used_start && jsec < journal.next_free)
            : (jsec >= journal.used_start || jsec < journal.next_free)))
        {
            printf(
                "BUG: journal offset %08jx is used by %jx:%jx v%ju (%ju refs) BUT used_start=%jx next_free=%jx\n",
                dirty_it->second.journal_sector, dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version,
                journal.used_sectors[journal.sector_info[journal.cur_sector].offset],
                journal.used_start, journal.next_free
            );
            abort();
        }
        journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++;
 #ifdef BLOCKSTORE_DEBUG
        printf(
            "journal offset %08jx is used by %jx:%jx v%ju (%ju refs)\n",
            journal.sector_info[journal.cur_sector].offset, op->oid.inode, op->oid.stripe, op->version,
            journal.used_sectors[journal.sector_info[journal.cur_sector].offset]
        );
 #endif
        je->oid = op->oid;
        je->version = op->version;
        je->offset = op->offset;
        je->len = op->len;
        je->location = dirty_it->second.location;
        memcpy((void*)(je+1), (alloc_dyn_data
            ? (uint8_t*)dirty_it->second.dyn_data+sizeof(int) : (uint8_t*)&dirty_it->second.dyn_data), dyn_size);
        je->crc32 = je_crc32((journal_entry*)je);
        journal.crc32_last = je->crc32;
        prepare_journal_sector_write(journal.cur_sector, op);
        PRIV(op)->op_state = 3;
        return 1;
    }
 resume_4:
    // Switch object state
    {
        auto dirty_it = dirty_db.find((obj_ver_id){
            .oid = op->oid,
            .version = op->version,
        });
        assert(dirty_it != dirty_db.end());
 #ifdef BLOCKSTORE_DEBUG
        printf("Ack write %jx:%jx v%ju = state 0x%x\n", op->oid.inode, op->oid.stripe, op->version, dirty_it->second.state);
 #endif
        bool is_big = (dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_BIG_WRITE;
        bool imm = is_big ? (immediate_commit == IMMEDIATE_ALL) : (immediate_commit != IMMEDIATE_NONE);
        bool is_instant = IS_INSTANT(dirty_it->second.state);
        if (imm)
        {
            auto & unstab = unstable_writes[op->oid];
            unstab = unstab < op->version ? op->version : unstab;
            if (!is_instant)
            {
                unstable_unsynced--;
                assert(unstable_unsynced >= 0);
            }
        }
        dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK)
            | (imm ? BS_ST_SYNCED : BS_ST_WRITTEN);
        if (imm && is_instant)
        {
            // Deletions and 'instant' operations are treated as immediately stable
            mark_stable(dirty_it->first);
        }
        if (!imm)
        {
            if (is_big)
            {
                // Remember big write as unsynced
                unsynced_big_writes.push_back((obj_ver_id){
                    .oid = op->oid,
                    .version = op->version,
                });
            }
            else
            {
                // Remember small write as unsynced
                unsynced_small_writes.push_back((obj_ver_id){
                    .oid = op->oid,
                    .version = op->version,
                });
            }
        }
        if (imm && (dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_BIG_WRITE)
        {
            // Unblock small writes
            dirty_it++;
            while (dirty_it != dirty_db.end() && dirty_it->first.oid == op->oid)
            {
                if ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_WAIT_BIG)
                {
                    dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | BS_ST_IN_FLIGHT;
                }
                dirty_it++;
            }
        }
        // Apply throttling to not fill the journal too fast for the SSD+HDD case
        if (!is_big && throttle_small_writes)
        {
            // Apply throttling
            timespec tv_end;
            clock_gettime(CLOCK_REALTIME, &tv_end);
            uint64_t exec_us =
                (tv_end.tv_sec - PRIV(op)->tv_begin.tv_sec)*1000000 +
                (tv_end.tv_nsec - PRIV(op)->tv_begin.tv_nsec)/1000;
            // Compare with target execution time
            // 100% free -> target time = 0
            // 0% free -> target time = iodepth/parallelism * (iops + size/bw) / write per second
            uint64_t used_start = journal.get_trim_pos();
            uint64_t journal_free_space = journal.next_free < used_start
                ? (used_start - journal.next_free)
                : (journal.len - journal.next_free + used_start - journal.block_size);
            uint64_t ref_us =
                (write_iodepth <= throttle_target_parallelism ? 100 : 100*write_iodepth/throttle_target_parallelism)
                * (1000000/throttle_target_iops + op->len*1000000/throttle_target_mbs/1024/1024)
                / 100;
            ref_us -= ref_us * journal_free_space / journal.len;
            if (ref_us > exec_us + throttle_threshold_us)
            {
                // Pause reply
                PRIV(op)->op_state = 5;
                // Remember that the timer can in theory be called right here
                tfd->set_timer_us(ref_us-exec_us, false, [this, op](int timer_id)
                {
                    PRIV(op)->op_state++;
                    ringloop->wakeup();
                });
                return 1;
            }
        }
    }
 resume_6:
    // Acknowledge write
    op->retval = op->len;
    write_iodepth--;
    FINISH_OP(op);
    return 2;
 }
 void blockstore_impl_t::handle_write_event(ring_data_t *data, blockstore_op_t *op)
 {
    live = true;
    if (data->res != data->iov.iov_len)
    {
        // FIXME: our state becomes corrupted after a write error. maybe do something better than just die
        disk_error_abort("data write", data->res, data->iov.iov_len);
    }
    PRIV(op)->pending_ops--;
    assert(PRIV(op)->pending_ops >= 0);
    if (PRIV(op)->pending_ops == 0)
    {
        release_journal_sectors(op);
        PRIV(op)->op_state++;
        ringloop->wakeup();
    }
 }
 void blockstore_impl_t::release_journal_sectors(blockstore_op_t *op)
 {
    // Release flushed journal sectors
    if (PRIV(op)->min_flushed_journal_sector > 0 &&
        PRIV(op)->max_flushed_journal_sector > 0)
    {
        uint64_t s = PRIV(op)->min_flushed_journal_sector;
        while (1)
        {
            if (!journal.sector_info[s-1].dirty && journal.sector_info[s-1].flush_count == 0)
            {
                if (s == (1+journal.cur_sector))
                {
                    // Forcibly move to the next sector and move dirty position
                    journal.in_sector_pos = journal.block_size;
                }
                // We know for sure that we won't write into this sector anymore
                uint64_t new_ds = journal.sector_info[s-1].offset + journal.block_size;
                if (new_ds >= journal.len)
                {
                    new_ds = journal.block_size;
                }
                if ((journal.dirty_start + (journal.dirty_start >= journal.used_start ? 0 : journal.len)) <
                    (new_ds + (new_ds >= journal.used_start ? 0 : journal.len)))
                {
                    journal.dirty_start = new_ds;
                }
            }
            if (s == PRIV(op)->max_flushed_journal_sector)
                break;
            s = 1 + s % journal.sector_count;
        }
        PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 0;
    }
 }
 int blockstore_impl_t::dequeue_del(blockstore_op_t *op)
 {
    if (PRIV(op)->op_state)
    {
        return continue_write(op);
    }
    auto dirty_it = dirty_db.find((obj_ver_id){
        .oid = op->oid,
        .version = op->version,
    });
    assert(dirty_it != dirty_db.end());
    blockstore_journal_check_t space_check(this);
    if (!space_check.check_available(op, 1, sizeof(journal_entry_del), (unstable_writes.size()+unstable_unsynced)*journal.block_size))
    {
        return 0;
    }
    // Write current journal sector only if it's dirty and full, or in the immediate_commit mode
    BS_SUBMIT_CHECK_SQES(
        (immediate_commit != IMMEDIATE_NONE ||
            (dsk.journal_block_size - journal.in_sector_pos) < sizeof(journal_entry_del) &&
            journal.sector_info[journal.cur_sector].dirty) ? 1 : 0
    );
    if (write_iodepth >= max_write_iodepth)
    {
        return 0;
    }
    write_iodepth++;
    // Prepare journal sector write
    if (immediate_commit == IMMEDIATE_NONE &&
        (dsk.journal_block_size - journal.in_sector_pos) < sizeof(journal_entry_del) &&
        journal.sector_info[journal.cur_sector].dirty)
    {
        prepare_journal_sector_write(journal.cur_sector, op);
    }
    // Pre-fill journal entry
    journal_entry_del *je = (journal_entry_del*)prefill_single_journal_entry(
        journal, JE_DELETE, sizeof(struct journal_entry_del)
    );
    dirty_it->second.journal_sector = journal.sector_info[journal.cur_sector].offset;
    journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++;
 #ifdef BLOCKSTORE_DEBUG
    printf(
        "journal offset %08jx is used by %jx:%jx v%ju (%ju refs)\n",
        dirty_it->second.journal_sector, dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version,
        journal.used_sectors[journal.sector_info[journal.cur_sector].offset]
    );
 #endif
    je->oid = op->oid;
    je->version = op->version;
    je->crc32 = je_crc32((journal_entry*)je);
    journal.crc32_last = je->crc32;
    dirty_it->second.state = BS_ST_DELETE | BS_ST_SUBMITTED;
    if (immediate_commit != IMMEDIATE_NONE)
    {
        prepare_journal_sector_write(journal.cur_sector, op);
    }
    if (!PRIV(op)->pending_ops)
    {
        PRIV(op)->op_state = 4;
        return continue_write(op);
    }
    else
    {
        PRIV(op)->op_state = 3;
    }
    return 1;
 }
 } // namespace v1
@@ -765,13 +765,8 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
    {
        return;
    }
-    // CAS writes are simplified: they're not cached, not resliced, not retried, and not part of the regular write queue at all
+    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && enable_writeback && !(op->flags & OP_FLUSH_BUFFER) &&
-    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && op->version)
+        !op->version /* no CAS writeback */)
    {
        execute_cas(op);
        return;
    }
    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && enable_writeback && !(op->flags & OP_FLUSH_BUFFER))
    {
        if (wb->writebacks_active >= client_max_writeback_iodepth)
        {
@@ -793,7 +788,7 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
    }
    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && !(op->flags & OP_IMMEDIATE_COMMIT))
    {
-        if (!(op->flags & OP_FLUSH_BUFFER))
+        if (!(op->flags & OP_FLUSH_BUFFER) && !op->version /* no CAS write-repeat */)
        {
            uint64_t flush_id = ++wb->last_flush_id;
            wb->copy_write(op, CACHE_REPEATING, flush_id);
@@ -852,72 +847,6 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
    }
 }
 void cluster_client_t::execute_cas(cluster_op_t *op)
 {
    slice_rw(op);
    op->needs_reslice = false;
    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && op->version && op->parts.size() > 1)
    {
        // Atomic writes to multiple stripes are unsupported
        op->retval = -EINVAL;
        auto cb = std::move(op->callback);
        cb(op);
        return;
    }
    int res = try_send(op, 0, [this, op](osd_op_t *part)
    {
        int expected = part->req.hdr.opcode == OSD_OP_DELETE ? 0 : part->req.rw.len;
        op->retval = part->reply.hdr.retval;
        op->retval = op->retval == expected ? 0 : (op->retval >= 0 ? -EIO : op->retval);
        op->retval = op->retval == -EPIPE ? -EINTR : op->retval;
        auto peer_it = msgr.osd_peer_fds.find(op->parts[0].osd_num);
        if (op->retval != 0 || (op->flags & OP_IMMEDIATE_COMMIT))
        {
            auto cb = std::move(op->callback);
            cb(op);
        }
        else if (peer_it == msgr.osd_peer_fds.end())
        {
            // Care must be taken to make sure that the client doesn't reconnect to the OSD
            // before executing the previously completed operation callback (!)
            op->retval = -EINTR;
            auto cb = std::move(op->callback);
            cb(op);
        }
        else
        {
            // CAS writes have a built-in sync
            auto peer_fd = peer_it->second;
            *part = (osd_op_t){
                .op_type = OSD_OP_OUT,
                .peer_fd = peer_fd,
                .req = {
                    .hdr = {
                        .magic = SECONDARY_OSD_OP_MAGIC,
                        .opcode = OSD_OP_SYNC,
                    },
                },
                .callback = [this, op](osd_op_t *part)
                {
                    op->retval = part->reply.hdr.retval;
                    op->retval = op->retval == -EPIPE ? -EINTR : op->retval;
                    auto cb = std::move(op->callback);
                    cb(op);
                },
            };
            msgr.outbox_push(part);
        }
    });
    if (res == TRY_SEND_CONNECTING || res == TRY_SEND_OFFLINE)
    {
        // In theory, CAS writes could wait for the PG to come up, but it's easier to just fail it
        op->retval = -EINTR;
        auto cb = std::move(op->callback);
        cb(op);
        return;
    }
 }
 bool cluster_client_t::check_rw(cluster_op_t *op)
 {
    if (op->opcode == OSD_OP_SYNC)
@@ -1027,6 +956,13 @@ resume_0:
    // Slice the operation into parts
    slice_rw(op);
    op->needs_reslice = false;
    if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && op->version && op->parts.size() > 1)
    {
        // Atomic writes to multiple stripes are unsupported
        op->retval = -EINVAL;
        erase_op(op);
        return 1;
    }
 resume_1:
    // Send unsent parts, if they're not subject to change
    op->state = 2;
@@ -1371,7 +1307,7 @@ bool cluster_client_t::affects_osd(uint64_t inode, uint64_t offset, uint64_t len
    return false;
 }
-int cluster_client_t::try_send(cluster_op_t *op, int i, std::function<void(osd_op_t *op_part)> cb)
+int cluster_client_t::try_send(cluster_op_t *op, int i)
 {
    if (!msgr_initialized)
    {
@@ -1431,7 +1367,7 @@ int cluster_client_t::try_send(cluster_op_t *op, int i, std::function<void(osd_o
                    ? (uint8_t*)op->part_bitmaps + pg_bitmap_size*i : NULL),
                .bitmap_len = (unsigned)(op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP
                    ? pg_bitmap_size : 0),
-                .callback = cb ? cb : [this, part](osd_op_t *op_part)
+                .callback = [this, part](osd_op_t *op_part)
                {
                    handle_op_part(part);
                },
@@ -175,13 +175,12 @@ protected:
    void on_change_node_placement_hook();
    void execute_internal(cluster_op_t *op);
    void execute_cas(cluster_op_t *op);
    void unshift_op(cluster_op_t *op);
    int continue_rw(cluster_op_t *op);
    bool check_rw(cluster_op_t *op);
    void slice_rw(cluster_op_t *op);
    void reset_retry_timer(int new_duration);
-    int try_send(cluster_op_t *op, int i, std::function<void(osd_op_t *op_part)> cb = nullptr);
+    int try_send(cluster_op_t *op, int i);
    int continue_sync(cluster_op_t *op);
    void send_sync(cluster_op_t *op, cluster_op_part_t *part);
    void handle_op_part(cluster_op_part_t *part);
@@ -323,12 +323,8 @@ void osd_messenger_t::handle_send(int result, bool prev, bool more, osd_client_t
        }
        if (more)
        {
-            int expected = cl->send_list.size() < IOV_MAX ? cl->send_list.size() : IOV_MAX;
+            auto expected = cl->send_list.size() < IOV_MAX ? cl->send_list.size() : IOV_MAX;
-            if (done != expected)
+            assert(done == expected);
            {
                fprintf(stderr, "BUG (maybe kernel): Expected to send %d iovecs with MSG_WAITALL but sent %d\n", expected, done);
                exit(1);
            }
            cl->zc_free_list.push_back(NULL); // end marker
        }
        if (done > 0)
@@ -6,20 +6,8 @@
 #include <stdint.h>
 #include <functional>
 #define POOL_SCHEME_REPLICATED 1
 #define POOL_SCHEME_XOR 2
 #define POOL_SCHEME_EC 3
 #define POOL_ID_MAX 0x10000
 #define POOL_ID_BITS 16
 #define INODE_POOL(inode) (pool_id_t)((inode) >> (64 - POOL_ID_BITS))
 #define INODE_NO_POOL(inode) (inode_t)((inode) & (((uint64_t)1 << (64-POOL_ID_BITS)) - 1))
 #define INODE_WITH_POOL(pool_id, inode) (((inode_t)(pool_id) << (64-POOL_ID_BITS)) | INODE_NO_POOL(inode))
 typedef uint64_t inode_t;
 // Pool ID is 16 bits long
 typedef uint32_t pool_id_t;
 // 16 bytes per object/stripe id
 // stripe = (start of the parity stripe + peer role)
 // i.e. for example (256KB + one of 0,1,2)
@@ -44,6 +44,11 @@
 #define DIRECT_IO_ALIGNMENT 512
 #endif
 // Memory allocation alignment (page size is usually optimal)
 #ifndef MEM_ALIGNMENT
 #define MEM_ALIGNMENT 4096
 #endif
 // Constants for osd_reply_describe_item_t.loc_bad
 #define LOC_OUTDATED 1
 #define LOC_CORRUPTED 2
@@ -262,10 +267,7 @@ struct __attribute__((__packed__)) osd_reply_del_t
    uint32_t left_on_dead_count;
 };
-// sync to the primary OSD. semantics:
+// sync to the primary OSD
 // 1) any non-synced write may disappear on OSD restart. even if it's a CAS write.
 // 2) sync only guarantees to commit completed writes, not in-progress ones.
 // 3) sync is a no-op when immediate_commit is active.
 struct __attribute__((__packed__)) osd_op_sync_t
 {
    osd_op_header_t header;
@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 Name: Vitastor
 Description: Vitastor client library
-Version: 3.0.0
+Version: 2.4.3
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
@@ -7,8 +7,7 @@ add_executable(vitastor-disk
 	disk_tool.cpp disk_simple_offsets.cpp
 	disk_tool_discard.cpp disk_tool_journal.cpp disk_tool_meta.cpp disk_tool_prepare.cpp disk_tool_resize.cpp
 	disk_tool_resize_auto.cpp disk_tool_udev.cpp disk_tool_utils.cpp disk_tool_upgrade.cpp
-	../util/crc32c.c ../util/str_util.cpp ../util/json_util.cpp ../../json11/json11.cpp ../util/rw_blocking.cpp ../util/allocator.cpp ../util/ringloop.cpp
+	../util/crc32c.c ../util/str_util.cpp ../util/json_util.cpp ../../json11/json11.cpp ../util/rw_blocking.cpp ../util/allocator.cpp ../util/ringloop.cpp ../blockstore/blockstore_disk.cpp
 	../blockstore/blockstore_disk.cpp ../blockstore/blockstore_heap.cpp ../blockstore/multilist.cpp
 )
 target_link_libraries(vitastor-disk
 	tcmalloc_minimal
@@ -11,8 +11,6 @@
 #include "str_util.h"
 #include "blockstore.h"
 #include "blockstore_disk.h"
 #include "blockstore_heap.h"
 #include "ondisk_formats.h"
 // Calculate offsets for a block device and print OSD command line parameters
 void disk_tool_simple_offsets(json11::Json cfg, bool json_output)
@@ -25,9 +23,6 @@ void disk_tool_simple_offsets(json11::Json cfg, bool json_output)
    uint64_t journal_offset = parse_size(cfg["journal_offset"].string_value());
    uint64_t device_size = parse_size(cfg["device_size"].string_value());
    uint32_t csum_block_size = parse_size(cfg["csum_block_size"].string_value());
    uint32_t meta_format = cfg["meta_format"].uint64_value();
    if (!meta_format)
        meta_format = BLOCKSTORE_META_FORMAT_HEAP;
    uint32_t data_csum_type = BLOCKSTORE_CSUM_NONE;
    if (cfg["data_csum_type"] == "crc32c")
        data_csum_type = BLOCKSTORE_CSUM_CRC32C;
@@ -128,42 +123,15 @@ void disk_tool_simple_offsets(json11::Json cfg, bool json_output)
    uint64_t meta_offset = journal_offset + ((journal_size+device_block_size-1)/device_block_size)*device_block_size;
    uint64_t data_csum_size = (data_csum_type ? data_block_size/csum_block_size*(data_csum_type & 0xFF) : 0);
    uint64_t clean_entry_bitmap_size = data_block_size/bitmap_granularity/8;
    uint64_t clean_entry_size = 24 /*sizeof(clean_disk_entry)*/ + 2*clean_entry_bitmap_size + data_csum_size + 4 /*entry_csum*/;
    uint64_t entries_per_block = device_block_size / clean_entry_size;
    uint64_t object_count = ((device_size-meta_offset)/data_block_size);
-    uint64_t meta_size;
+    uint64_t meta_size = (1 + (object_count+entries_per_block-1)/entries_per_block) * device_block_size;
    if (meta_format == BLOCKSTORE_META_FORMAT_HEAP)
    {
        uint32_t min_object_size = sizeof(heap_big_intent_t) + (data_csum_size ? data_csum_size : 4) + 2*clean_entry_bitmap_size;
        double meta_reserve = cfg["meta_reserve"].number_value();
        if (!meta_reserve)
            meta_reserve = 1.5;
        else if (meta_reserve < 1)
            meta_reserve = 1;
        uint32_t entries_per_block = device_block_size / min_object_size;
        meta_size = device_block_size * (uint64_t)((object_count+entries_per_block-1) / entries_per_block * meta_reserve);
    }
    else if (meta_format == BLOCKSTORE_META_FORMAT_V2)
    {
        uint64_t clean_entry_size = 24 /*sizeof(clean_disk_entry)*/ + 2*clean_entry_bitmap_size + data_csum_size + 4 /*entry_csum*/;
        uint64_t entries_per_block = device_block_size / clean_entry_size;
        meta_size = (1 + (object_count+entries_per_block-1)/entries_per_block) * device_block_size;
    }
    else if (meta_format == BLOCKSTORE_META_FORMAT_V1)
    {
        uint64_t clean_entry_size = 24 /*sizeof(clean_disk_entry)*/ + 2*clean_entry_bitmap_size;
        uint64_t entries_per_block = device_block_size / clean_entry_size;
        meta_size = (1 + (object_count+entries_per_block-1)/entries_per_block) * device_block_size;
    }
    else
    {
        fprintf(stderr, "meta_format %u is not supported\n", meta_format);
        exit(1);
    }
    uint64_t data_offset = meta_offset + meta_size;
    if (format == "json")
    {
        // JSON
        printf("%s\n", json11::Json(json11::Json::object {
            { "meta_format", (uint64_t)meta_format },
            { "meta_block_size", device_block_size },
            { "journal_block_size", device_block_size },
            { "data_size", device_size-data_offset },
@@ -177,9 +145,9 @@ void disk_tool_simple_offsets(json11::Json cfg, bool json_output)
    {
        // Env
        printf(
-            "meta_format=%u\nmeta_block_size=%ju\njournal_block_size=%ju\ndata_size=%ju\n"
+            "meta_block_size=%ju\njournal_block_size=%ju\ndata_size=%ju\n"
            "data_device=%s\njournal_offset=%ju\nmeta_offset=%ju\ndata_offset=%ju\n",
-            meta_format, device_block_size, device_block_size, device_size-data_offset,
+            device_block_size, device_block_size, device_size-data_offset,
            device.c_str(), journal_offset, meta_offset, data_offset
        );
    }
@@ -199,8 +167,8 @@ void disk_tool_simple_offsets(json11::Json cfg, bool json_output)
            printf("--data_size %ju\n", device_size-data_offset);
        }
        printf(
-            "--meta_format %u\n--data_device %s\n--journal_offset %ju\n--meta_offset %ju\n--data_offset %ju\n",
+            "--data_device %s\n--journal_offset %ju\n--meta_offset %ju\n--data_offset %ju\n",
-            meta_format, device.c_str(), journal_offset, meta_offset, data_offset
+            device.c_str(), journal_offset, meta_offset, data_offset
        );
    }
 }
@@ -26,9 +26,6 @@ static const char *help_text =
    "  Options (automatic mode):\n"
    "    --osd_per_disk <N>\n"
    "      Create <N> OSDs on each disk (default 1)\n"
    "    --meta_format 3\n"
    "      Metadata store version. 3 is the new log-structured store, 2 is the stable store\n"
    "      from Vitastor 0.9-2.x, 1 is the legacy store from Vitastor 0.6-0.8.\n"
    "    --hybrid\n"
    "      Prepare hybrid (HDD+SSD, NVMe+SATA or etc) OSDs using provided devices. By default,\n"
    "      any passed SSDs will be used for journals and metadata, HDDs will be used for data,\n"
@@ -90,8 +87,7 @@ static const char *help_text =
    "  inmemory_metadata, inmemory_journal, max_write_iodepth,\n"
    "  min_flusher_count, max_flusher_count, journal_sector_buffer_count,\n"
    "  journal_no_same_sector_overwrites, throttle_small_writes, throttle_target_iops,\n"
-    "  throttle_target_mbs, throttle_target_parallelism, throttle_threshold_us,\n"
+    "  throttle_target_mbs, throttle_target_parallelism, throttle_threshold_us.\n"
    "  atomic_write_size, use_atomic_flag.\n"
    "\n"
    "vitastor-disk upgrade-simple <UNIT_FILE|OSD_NUMBER>\n"
    "  Upgrade an OSD created by old (0.7.1 and older) make-osd.sh or make-osd-hybrid.js scripts.\n"
@@ -185,8 +181,8 @@ static const char *help_text =
    "  Options:\n"
    "  --all             Scan the whole journal area for entries and dump them, even outdated ones\n"
    "  --json            Dump journal in JSON format\n"
-    "  --format data     (Default) Dump journal entries as an array, with small write data\n"
+    "  --format entries  (Default) Dump actual journal entries as an array, without data\n"
-    "  --format entries  Dump actual journal entries as an array, without data\n"
+    "  --format data     Same as \"entries\", but also include small write data\n"
    "  --format blocks   Dump as an array of journal blocks each containing array of entries\n"
    "\n"
    "vitastor-disk write-journal <osd_device>\n"
@@ -196,12 +192,12 @@ static const char *help_text =
    "  You can specify any OSD device (data, metadata or journal), or the layout manually.\n"
    "\n"
    "vitastor-disk dump-meta <osd_device>\n"
-    "vitastor-disk dump-meta [osd_options...]\n"
+    "vitastor-disk dump-meta <meta_file> <meta_block_size> <offset> <size>\n"
    "  Dump metadata in JSON format.\n"
    "  You can specify any OSD device (data, metadata or journal), or the layout manually.\n"
    "\n"
    "vitastor-disk write-meta <osd_device>\n"
-    "vitastor-disk write-meta [osd_options...]\n"
+    "vitastor-disk write-meta <meta_file> <offset> <size>\n"
    "  Write metadata from JSON taken from standard input in the same format as produced by `dump-meta`.\n"
    "  You can specify any OSD device (data, metadata or journal), or the layout manually.\n"
    "\n"
@@ -368,63 +364,47 @@ int main(int argc, char *argv[])
    }
    else if (!strcmp(cmd[0], "dump-meta"))
    {
-        if (cmd.size() == 5)
+        if (cmd.size() != 2 && cmd.size() < 5)
        {
            print_help(help_text, "vitastor-disk", cmd[0], false);
            return 1;
        }
        self.dsk.meta_device = cmd[1];
        if (cmd.size() > 2)
        {
            // Old format
            self.dsk.meta_device = cmd[1];
            self.dsk.meta_block_size = strtoul(cmd[2], NULL, 10);
            self.dsk.meta_offset = strtoull(cmd[3], NULL, 10);
-            self.dsk.min_meta_len = self.dsk.meta_area_size = strtoull(cmd[4], NULL, 10);
+            self.dsk.meta_len = strtoull(cmd[4], NULL, 10);
        }
        else if (cmd.size() == 2)
        {
            // First argument is an OSD device - take metadata layout parameters from it
            self.dsk.meta_device = cmd[1];
            if (self.dump_load_check_superblock(self.dsk.meta_device))
                return 1;
        }
        else
        {
-            // Parse all OSD options from cmdline
+            // First argument is an OSD device - take metadata layout parameters from it
-            self.dsk.parse_config(self.options);
+            if (self.dump_load_check_superblock(self.dsk.meta_device))
-            if (self.options["io"] != "")
+                return 1;
                self.dsk.data_io = self.dsk.meta_io = self.dsk.journal_io = self.options["io"];
            // FIXME: This is a really repeated pattern, move it somewhere
            self.dsk.open_data();
            self.dsk.open_meta();
            self.dsk.open_journal();
            self.dsk.calc_lengths();
            self.dsk.close_all();
        }
        return self.dump_meta();
    }
    else if (!strcmp(cmd[0], "write-meta"))
    {
-        if (cmd.size() == 4)
+        if (cmd.size() != 2 && cmd.size() < 4)
        {
            print_help(help_text, "vitastor-disk", cmd[0], false);
            return 1;
        }
        self.new_meta_device = cmd[1];
        if (cmd.size() > 2)
        {
            self.new_meta_device = cmd[1];
            self.new_meta_offset = strtoull(cmd[2], NULL, 10);
            self.new_meta_len = strtoull(cmd[3], NULL, 10);
        }
-        else if (cmd.size() == 2)
+        else
        {
            // First argument is an OSD device - take metadata layout parameters from it
            self.new_meta_device = cmd[1];
            if (self.dump_load_check_superblock(self.new_meta_device))
                return 1;
            self.new_meta_device = self.dsk.meta_device;
            self.new_meta_offset = self.dsk.meta_offset;
-            self.new_meta_len = self.dsk.meta_area_size;
+            self.new_meta_len = self.dsk.meta_len;
        }
        else
        {
            // Parse all OSD options from cmdline
            self.dsk.parse_config(self.options);
            self.dsk.open_data();
            self.dsk.open_meta();
            self.dsk.open_journal();
            self.dsk.calc_lengths();
            self.dsk.close_all();
        }
        std::string json_err;
        json11::Json meta = json11::Json::parse(read_all_fd(0), json_err);
@@ -8,18 +8,14 @@
 #endif
 #include <map>
 #include <set>
 #include <vector>
 #include <string>
 #include <functional>
 #include "json11/json11.hpp"
 #include "blockstore_disk.h"
-#include "blockstore.h"
+#include "blockstore_impl.h"
 #include "blockstore_heap.h"
 #include "ondisk_formats.h"
 #include "crc32c.h"
 #include "allocator.h"
 // vITADisk
 #define VITASTOR_DISK_MAGIC 0x6b73694441544976
@@ -27,9 +23,6 @@
 #define VITASTOR_PART_TYPE "e7009fac-a5a1-4d72-af72-53de13059903"
 #define DEFAULT_HYBRID_JOURNAL "1G"
 #define DEFAULT_HYBRID_SSD_JOURNAL "128M"
 #define VITASTOR_META_FORMAT_NAME_V1 "0.6"
 #define VITASTOR_META_FORMAT_NAME_V2 "0.9"
 #define VITASTOR_META_FORMAT_NAME_HEAP "3.0"
 struct resizer_data_moving_t;
@@ -50,51 +43,44 @@ struct disk_tool_t
    std::map<std::string, std::string> options;
    bool test_mode = false;
    bool all = false, json = false, now = false;
-    bool dump_with_blocks = false, dump_with_data = false;
+    bool dump_with_blocks, dump_with_data;
    bool dump_as_old = false;
    bool skip_obsolete = false;
    int log_level = 1;
    double meta_reserve_multiple = 2;
    uint64_t meta_reserve_min_size = (uint64_t)1024*1024*1024;
    blockstore_disk_t dsk;
    // resize data and/or move metadata and journal
    int iodepth;
    std::string new_meta_device, new_journal_device;
-    uint64_t new_data_offset = 0, new_data_len = 0;
+    uint64_t new_data_offset, new_data_len;
-    uint64_t new_journal_offset = 0, new_journal_len = 0;
+    uint64_t new_journal_offset, new_journal_len;
-    uint64_t new_meta_offset = 0, new_meta_len = 0;
+    uint64_t new_meta_offset, new_meta_len;
    /**** State ****/
-    uint64_t journal_pos = 0, journal_calc_data_pos = 0;
+    uint64_t meta_pos;
    uint64_t journal_pos, journal_calc_data_pos;
    uint8_t *buffer_area = NULL;
    bool first_block, first_entry;
-    allocator_t *data_alloc = NULL;
+    allocator_t *data_alloc;
    std::map<uint64_t, uint64_t> data_remap;
    std::map<uint64_t, uint64_t>::iterator remap_it;
-    ring_loop_t *ringloop = NULL;
+    ring_loop_t *ringloop;
    ring_consumer_t ring_consumer;
-    int remap_active = 0;
+    int remap_active;
    journal_entry_start je_start;
-    uint8_t *new_journal_buf = NULL, *new_meta_buf = NULL, *new_journal_ptr = NULL, *new_journal_data = NULL;
+    uint8_t *new_journal_buf, *new_meta_buf, *new_journal_ptr, *new_journal_data;
-    blockstore_meta_header_v3_t *new_meta_hdr = NULL;
+    uint64_t new_journal_in_pos;
-    uint64_t new_journal_in_pos = 0;
+    int64_t data_idx_diff;
-    int64_t data_idx_diff = 0;
+    uint64_t total_blocks, free_first, free_last;
-    uint64_t total_blocks = 0, free_first = 0, free_last = 0;
+    uint64_t new_clean_entry_bitmap_size, new_data_csum_size, new_clean_entry_size, new_entries_per_block;
-    uint64_t new_clean_entry_bitmap_size = 0, new_data_csum_size = 0, new_clean_entry_size = 0, new_entries_per_block = 0;
+    int new_journal_fd, new_meta_fd;
-    uint32_t new_meta_format = 0;
+    resizer_data_moving_t *moving_blocks;
    int new_journal_fd = -1, new_meta_fd = -1;
    resizer_data_moving_t *moving_blocks = NULL;
-    bool started = false;
+    bool started;
-    void *small_write_data = NULL;
+    void *small_write_data;
-    uint32_t data_crc32 = 0;
+    uint32_t data_crc32;
-    bool data_csum_valid = false;
+    bool data_csum_valid;
-    uint32_t crc32_last = 0;
+    uint32_t crc32_last;
-    uint32_t new_crc32_prev = 0;
+    uint32_t new_crc32_prev;
    ~disk_tool_t();
@@ -102,24 +88,17 @@ struct disk_tool_t
    void dump_journal_entry(int num, journal_entry *je, bool json);
    int process_journal(std::function<int(void*)> block_fn, bool do_open = true);
    int process_journal_block(void *buf, std::function<void(int, journal_entry*)> iter_fn);
-    int process_meta(std::function<void(blockstore_meta_header_v3_t *)> hdr_fn,
+    int process_meta(std::function<void(blockstore_meta_header_v2_t *)> hdr_fn,
-        std::function<void(blockstore_heap_t *heap, heap_entry_t *obj, uint32_t meta_block_num)> obj_fn,
+        std::function<void(uint64_t, clean_disk_entry*, uint8_t*)> record_fn, bool do_open = true);
        std::function<void(uint64_t block_num, clean_disk_entry *entry_v1, uint8_t *bitmap)> record_fn,
        bool with_data, bool do_open);
    int dump_meta();
-    void dump_meta_header(blockstore_meta_header_v3_t *hdr);
+    void dump_meta_header(blockstore_meta_header_v2_t *hdr);
    void dump_meta_entry(uint64_t block_num, clean_disk_entry *entry, uint8_t *bitmap);
    void dump_heap_entry_as_old(blockstore_heap_t *heap, heap_entry_t *obj);
    void dump_heap_entry(blockstore_heap_t *heap, heap_entry_t *obj);
    int dump_load_check_superblock(const std::string & device);
    int write_json_journal(json11::Json entries);
    int write_json_meta(json11::Json meta);
    int write_json_heap(json11::Json meta, json11::Json journal);
    int index_journal_by_object(json11::Json journal,
        std::map<object_id, std::vector<json11::Json::object>> & journal_by_object);
    int resize_data(std::string device);
    int resize_parse_move_journal(std::map<std::string, std::string> & move_options, bool dry_run);
@@ -127,21 +106,13 @@ struct disk_tool_t
    int raw_resize();
    int resize_parse_params();
-    void resize_init(blockstore_meta_header_v3_t *hdr);
+    void resize_init(blockstore_meta_header_v2_t *hdr);
    int resize_remap_blocks();
    int resize_copy_data();
-    void resize_alloc_journal();
+    int resize_rewrite_journal();
    void build_journal_start();
    void choose_journal_block(uint32_t je_size);
    int resize_rebuild_journal();
    int resize_write_new_journal();
-    void remap_big_write(heap_entry_t *wr);
+    int resize_rewrite_meta();
    void remap_small_write(heap_entry_t *wr);
    void fill_old_clean_entry(blockstore_heap_t *heap, heap_entry_t *big_wr);
    void fill_old_journal_entry(blockstore_heap_t *heap, heap_entry_t *wr);
    int resize_rebuild_meta();
    int resize_write_new_meta();
    void free_new_meta();
    int udev_import(std::string device);
    int read_sb(std::string device);
@@ -157,7 +128,6 @@ struct disk_tool_t
    json11::Json read_osd_superblock(std::string device, bool expect_exist = true, bool ignore_nonref = false);
    uint32_t write_osd_superblock(std::string device, json11::Json params);
    void parse_meta_reserve();
    int prepare_one(std::map<std::string, std::string> options, int is_hdd, json11::Json::object & result);
    int check_existing_partition(std::string & dev_by_uuid);
    int fix_partition_type(std::string & dev_by_uuid);
@@ -175,8 +145,7 @@ void disk_tool_simple_offsets(json11::Json cfg, bool json_output);
 uint64_t sscanf_json(const char *fmt, const json11::Json & str);
 void fromhexstr(const std::string & from, int bytes, uint8_t *to);
-int disable_cache(const std::string & dev);
+int disable_cache(std::string dev);
 uint64_t get_atomic_write_size(const std::string & dev);
 uint64_t get_device_size(const std::string & dev, bool should_exist = false);
 std::string get_parent_device(std::string dev);
 int shell_exec(const std::vector<std::string> & cmd, const std::string & in, std::string *out, std::string *err);
@@ -54,23 +54,12 @@ int disk_tool_t::trim_data(std::string device)
    fprintf(stderr, "Reading metadata\n");
    data_alloc = new allocator_t(dsk.block_count);
    r = process_meta(
-        [this](blockstore_meta_header_v3_t *hdr) {},
+        [this](blockstore_meta_header_v2_t *hdr) {},
        [this](blockstore_heap_t *heap, heap_entry_t *obj, uint32_t meta_block_num)
        {
            for (auto wr = obj; wr; wr = heap->prev(wr))
            {
                if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE ||
                    (wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_INTENT)
                {
                    data_alloc->set(wr->big_location(heap) / dsk.data_block_size, true);
                }
            }
        },
        [this](uint64_t block_num, clean_disk_entry *entry, uint8_t *bitmap)
        {
            data_alloc->set(block_num, true);
        },
-        false, false
+        false
    );
    if (r != 0)
    {
@@ -94,7 +83,7 @@ int disk_tool_t::trim_data(std::string device)
        return r;
    }
    // Trim
-    r = dsk.trim_data([&](uint64_t block_num) { return data_alloc->get(block_num); });
+    r = dsk.trim_data(data_alloc);
    dsk.close_all();
    return r == 0;
 }
@@ -1,15 +1,12 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 #include <unistd.h>
 #include "disk_tool.h"
 #include "malloc_or_die.h"
 int disk_tool_t::dump_journal()
 {
-    dump_with_blocks = options["format"] == "blocks" || options["format"] == "blocks,data";
+    dump_with_blocks = options["format"] == "blocks";
-    dump_with_data = options["format"] == "data" || options["format"] == "blocks,data" || options["format"] == "";
+    dump_with_data = options["format"] == "data" || options["format"] == "blocks,data";
    if (dsk.journal_block_size < DIRECT_IO_ALIGNMENT || (dsk.journal_block_size % DIRECT_IO_ALIGNMENT) ||
        dsk.journal_block_size > 128*1024)
    {
@@ -528,9 +525,7 @@ int disk_tool_t::write_json_journal(json11::Json entries)
                .data_offset = (uint64_t)(new_journal_data-new_journal_buf),
                .crc32_data = !dsk.data_csum_type ? 0 : (uint32_t)sscanf_json("%x", rec["data_crc32"]),
            };
-            uint32_t data_csum_blocks = !dsk.data_csum_type ? 0 :
+            uint32_t data_csum_size = !dsk.data_csum_type ? 0 : ne->small_write.len/dsk.csum_block_size*(dsk.data_csum_type & 0xFF);
                (((ne->small_write.offset+ne->small_write.len)/dsk.csum_block_size - ne->small_write.len/dsk.csum_block_size));
            uint32_t data_csum_size = data_csum_blocks*(dsk.data_csum_type & 0xFF);
            fromhexstr(rec["bitmap"].string_value(), dsk.clean_entry_bitmap_size, ((uint8_t*)ne) + sizeof(journal_entry_small_write) + data_csum_size);
            fromhexstr(rec["data"].string_value(), ne->small_write.len, new_journal_data);
            if (ne->small_write.len > 0 && !rec["data"].is_string())
@@ -539,21 +534,17 @@ int disk_tool_t::write_json_journal(json11::Json entries)
                free(new_journal_buf);
                return 1;
            }
-            if (ne->small_write.len > 0)
+            if (dsk.data_csum_type)
                fromhexstr(rec["block_csums"].string_value(), data_csum_size, ((uint8_t*)ne) + sizeof(journal_entry_small_write));
            if (rec["data"].is_string())
            {
                if (!dsk.data_csum_type)
                    ne->small_write.crc32_data = crc32c(0, new_journal_data, ne->small_write.len);
                else if (dsk.data_csum_type == BLOCKSTORE_CSUM_CRC32C)
                {
                    uint32_t *block_csums = (uint32_t*)(((uint8_t*)ne) + sizeof(journal_entry_small_write));
-                    for (uint32_t i = 0; i < data_csum_blocks; i++)
+                    for (uint32_t i = 0; i < ne->small_write.len; i += dsk.csum_block_size, block_csums++)
-                    {
+                        *block_csums = crc32c(0, new_journal_data+i, dsk.csum_block_size);
                        uint32_t block_begin = (ne->small_write.offset/dsk.csum_block_size + i) * dsk.csum_block_size;
                        uint32_t block_end = (ne->small_write.offset/dsk.csum_block_size + (i+1)) * dsk.csum_block_size;
                        block_begin = block_begin < ne->small_write.offset ? ne->small_write.offset : block_begin;
                        block_end = block_end > ne->small_write.offset+ne->small_write.len ? ne->small_write.offset+ne->small_write.len : block_end;
                        block_csums[i] = crc32c(0, new_journal_data+block_begin-ne->small_write.offset, block_end-block_begin);
                    }
                }
            }
            new_journal_data += ne->small_write.len;
@@ -574,9 +565,7 @@ int disk_tool_t::write_json_journal(json11::Json entries)
                .len = (uint32_t)rec["len"].uint64_value(),
                .location = sscanf_json(NULL, rec["loc"]),
            };
-            uint32_t data_csum_blocks = !dsk.data_csum_type ? 0 :
+            uint32_t data_csum_size = !dsk.data_csum_type ? 0 : ne->big_write.len/dsk.csum_block_size*(dsk.data_csum_type & 0xFF);
                (((ne->small_write.offset+ne->small_write.len)/dsk.csum_block_size - ne->small_write.len/dsk.csum_block_size));
            uint32_t data_csum_size = data_csum_blocks*(dsk.data_csum_type & 0xFF);
            fromhexstr(rec["bitmap"].string_value(), dsk.clean_entry_bitmap_size, ((uint8_t*)ne) + sizeof(journal_entry_big_write) + data_csum_size);
            if (dsk.data_csum_type)
                fromhexstr(rec["block_csums"].string_value(), data_csum_size, ((uint8_t*)ne) + sizeof(journal_entry_big_write));
@@ -5,118 +5,40 @@
 #include "rw_blocking.h"
 #include "osd_id.h"
 #include "json_util.h"
 #include "malloc_or_die.h"
-int disk_tool_t::process_meta(std::function<void(blockstore_meta_header_v3_t *)> hdr_fn,
+int disk_tool_t::process_meta(std::function<void(blockstore_meta_header_v2_t *)> hdr_fn,
-    std::function<void(blockstore_heap_t *heap, heap_entry_t *obj, uint32_t meta_block_num)> obj_fn,
+    std::function<void(uint64_t, clean_disk_entry*, uint8_t*)> record_fn, bool do_open)
    std::function<void(uint64_t block_num, clean_disk_entry *entry_v1, uint8_t *bitmap)> record_fn,
    bool with_data, bool do_open)
 {
    int r = 0;
    if (dsk.meta_block_size % DIRECT_IO_ALIGNMENT)
    {
        fprintf(stderr, "Invalid metadata block size: is not a multiple of %d\n", DIRECT_IO_ALIGNMENT);
        return 1;
    }
    int buf_size = 1024*1024;
    if (buf_size % dsk.meta_block_size)
        buf_size = 8*dsk.meta_block_size;
    uint8_t *data = NULL;
    data = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, buf_size);
    blockstore_meta_header_v3_t *hdr = (blockstore_meta_header_v3_t *)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size);
    if (do_open)
    {
        if (dsk.meta_fd >= 0)
        {
            fprintf(stderr, "Bug: Metadata device is already opened\n");
-close_error:
+            return 1;
            r = 1;
            goto close_free;
        }
        dsk.meta_fd = open(dsk.meta_device.c_str(), (options["io"] == "cached" ? 0 : O_DIRECT) | O_RDONLY);
        if (dsk.meta_fd < 0)
        {
            fprintf(stderr, "Failed to open metadata device %s: %s\n", dsk.meta_device.c_str(), strerror(errno));
-            goto close_error;
+            return 1;
        }
    }
-    else if (dsk.meta_fd < 0)
+    int buf_size = 1024*1024;
-    {
+    if (buf_size % dsk.meta_block_size)
-        fprintf(stderr, "Bug: Metadata device is not opened\n");
+        buf_size = 8*dsk.meta_block_size;
-        goto close_error;
+    if (buf_size > dsk.meta_len)
-    }
+        buf_size = dsk.meta_len;
-    // Check superblock
+    void *data = memalign_or_die(MEM_ALIGNMENT, buf_size);
    lseek64(dsk.meta_fd, dsk.meta_offset, 0);
-    read_blocking(dsk.meta_fd, hdr, dsk.meta_block_size);
+    read_blocking(dsk.meta_fd, data, dsk.meta_block_size);
-    if (hdr->zero == 0 && hdr->magic == BLOCKSTORE_META_MAGIC_V1 && hdr->version == BLOCKSTORE_META_FORMAT_HEAP)
+    // Check superblock
-    {
+    blockstore_meta_header_v2_t *hdr = (blockstore_meta_header_v2_t *)data;
-        if (hdr->data_csum_type != 0 &&
+    if (hdr->zero == 0 && hdr->magic == BLOCKSTORE_META_MAGIC_V1)
            hdr->data_csum_type != BLOCKSTORE_CSUM_CRC32C)
        {
            goto csum_unknown;
        }
        if (!dsk.journal_len && !with_data)
        {
            fprintf(stderr, "Buffer area (former journal) location must be specified to dump \"heap\" with data\n");
            goto close_error;
        }
        // Load buffer_area
        if (with_data)
        {
            buffer_area = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.journal_len);
            if (dsk.journal_device == dsk.meta_device || dsk.journal_device == "")
            {
                dsk.journal_fd = dsk.meta_fd;
            }
            else if (do_open)
            {
                if (dsk.journal_fd >= 0)
                {
                    fprintf(stderr, "Bug: Metadata device is already opened\n");
                    goto close_error;
                }
                dsk.journal_fd = open(dsk.journal_device.c_str(), (options["io"] == "cached" ? 0 : O_DIRECT) | O_RDONLY);
                if (dsk.journal_fd < 0)
                {
                    fprintf(stderr, "Failed to open journal device %s: %s\n", dsk.journal_device.c_str(), strerror(errno));
                    goto close_error;
                }
            }
            else if (dsk.journal_fd < 0)
            {
                fprintf(stderr, "Bug: journal device is not opened\n");
                goto close_error;
            }
            uint64_t journal_pos = 0;
            lseek64(dsk.journal_fd, dsk.journal_offset+journal_pos, 0);
            while (journal_pos < dsk.journal_len)
            {
                uint64_t read_len = buf_size < dsk.journal_len-journal_pos ? buf_size : dsk.journal_len-journal_pos;
                read_blocking(dsk.journal_fd, buffer_area+journal_pos, read_len);
                journal_pos += read_len;
            }
        }
        blockstore_heap_t *heap = new blockstore_heap_t(&dsk, buffer_area, log_level);
        // Load heap and just iterate it in memory
        hdr_fn(hdr);
        uint64_t meta_pos = dsk.meta_block_size;
        lseek64(dsk.meta_fd, dsk.meta_offset+meta_pos, 0);
        uint64_t entries_loaded = 0;
        while (meta_pos < hdr->meta_area_size)
        {
            uint64_t read_len = buf_size < hdr->meta_area_size-meta_pos ? buf_size : hdr->meta_area_size-meta_pos;
            read_blocking(dsk.meta_fd, data, read_len);
            r = heap->load_blocks(meta_pos-dsk.meta_block_size, read_len, data, true, entries_loaded);
            meta_pos += read_len;
        }
        heap->iterate_objects([&](heap_entry_t* obj, uint32_t meta_block_num)
        {
            obj_fn(heap, obj, meta_block_num);
        });
        obj_fn(heap, NULL, 0);
        delete heap;
    }
    else if (hdr->zero == 0 && hdr->magic == BLOCKSTORE_META_MAGIC_V1)
    {
        if (hdr->version == BLOCKSTORE_META_FORMAT_V1)
        {
@@ -131,21 +53,41 @@ close_error:
            if (hdr->data_csum_type != 0 &&
                hdr->data_csum_type != BLOCKSTORE_CSUM_CRC32C)
            {
 csum_unknown:
                fprintf(stderr, "I don't know checksum format %u, the only supported format is crc32c = %u.\n", hdr->data_csum_type, BLOCKSTORE_CSUM_CRC32C);
-                goto close_error;
+                free(data);
                if (do_open)
                {
                    close(dsk.meta_fd);
                    dsk.meta_fd = -1;
                }
                return 1;
            }
        }
        else
        {
            // Unsupported version
            fprintf(stderr, "Metadata format is too new for me (stored version is %ju, max supported %u).\n", hdr->version, BLOCKSTORE_META_FORMAT_V2);
-            goto close_error;
+            free(data);
            if (do_open)
            {
                close(dsk.meta_fd);
                dsk.meta_fd = -1;
            }
            return 1;
        }
        if (hdr->meta_block_size != dsk.meta_block_size)
        {
            fprintf(stderr, "Using block size of %u bytes based on information from the superblock\n", hdr->meta_block_size);
            dsk.meta_block_size = hdr->meta_block_size;
            if (buf_size % dsk.meta_block_size)
            {
                buf_size = 8*dsk.meta_block_size;
                void *new_data = memalign_or_die(MEM_ALIGNMENT, buf_size);
                memcpy(new_data, data, dsk.meta_block_size);
                free(data);
                data = new_data;
                hdr = (blockstore_meta_header_v2_t *)data;
            }
        }
        dsk.meta_format = hdr->version;
        dsk.data_block_size = hdr->data_block_size;
@@ -159,14 +101,14 @@ csum_unknown:
                    *(hdr->data_csum_type & 0xff))
                : 0)
            + (dsk.meta_format == BLOCKSTORE_META_FORMAT_V2 ? 4 /*entry_csum*/ : 0);
        // Read
        uint64_t block_num = 0;
        hdr_fn(hdr);
-        uint64_t meta_pos = dsk.meta_block_size;
+        hdr = NULL;
        meta_pos = dsk.meta_block_size;
        lseek64(dsk.meta_fd, dsk.meta_offset+meta_pos, 0);
-        while (meta_pos < dsk.min_meta_len)
+        while (meta_pos < dsk.meta_len)
        {
-            uint64_t read_len = buf_size < dsk.min_meta_len-meta_pos ? buf_size : dsk.min_meta_len-meta_pos;
+            uint64_t read_len = buf_size < dsk.meta_len-meta_pos ? buf_size : dsk.meta_len-meta_pos;
            read_blocking(dsk.meta_fd, data, read_len);
            meta_pos += read_len;
            for (uint64_t blk = 0; blk < read_len; blk += dsk.meta_block_size)
@@ -181,7 +123,7 @@ csum_unknown:
                            uint32_t *entry_csum = (uint32_t*)((uint8_t*)entry + dsk.clean_entry_size - 4);
                            if (*entry_csum != crc32c(0, entry, dsk.clean_entry_size - 4))
                            {
-                                fprintf(stderr, "Metadata entry %lu is corrupt (checksum mismatch), skipping\n", block_num);
+                                fprintf(stderr, "Metadata entry %ju is corrupt (checksum mismatch), skipping\n", block_num);
                                continue;
                            }
                        }
@@ -193,16 +135,14 @@ csum_unknown:
    }
    else
    {
-        // Vitastor 0.4-0.5 - static array of clean_disk_entry without header
+        // Vitastor 0.4-0.5 - static array of clean_disk_entry
        lseek64(dsk.meta_fd, dsk.meta_offset, 0);
        dsk.clean_entry_bitmap_size = 0;
        dsk.clean_entry_size = sizeof(clean_disk_entry);
        uint64_t meta_pos = 0;
        uint64_t block_num = 0;
        hdr_fn(NULL);
-        while (meta_pos < dsk.min_meta_len)
+        while (meta_pos < dsk.meta_len)
        {
-            uint64_t read_len = buf_size < dsk.min_meta_len-meta_pos ? buf_size : dsk.min_meta_len-meta_pos;
+            uint64_t read_len = buf_size < dsk.meta_len-meta_pos ? buf_size : dsk.meta_len-meta_pos;
            read_blocking(dsk.meta_fd, data, read_len);
            meta_pos += read_len;
            for (uint64_t blk = 0; blk < read_len; blk += dsk.meta_block_size)
@@ -218,26 +158,13 @@ csum_unknown:
            }
        }
    }
 close_free:
    free(data);
    free(hdr);
    if (buffer_area)
    {
        free(buffer_area);
        buffer_area = NULL;
    }
    if (do_open)
    {
        close(dsk.meta_fd);
        dsk.meta_fd = -1;
        if (dsk.journal_fd >= 0)
        {
            if (dsk.journal_fd != dsk.meta_fd)
                close(dsk.journal_fd);
            dsk.journal_fd = -1;
        }
    }
-    return r;
+    return 0;
 }
 int disk_tool_t::dump_load_check_superblock(const std::string & device)
@@ -253,7 +180,7 @@ int disk_tool_t::dump_load_check_superblock(const std::string & device)
        dsk.open_data();
        dsk.open_meta();
        dsk.open_journal();
-        dsk.calc_lengths();
+        dsk.calc_lengths(true);
    }
    catch (std::exception & e)
    {
@@ -267,45 +194,23 @@ int disk_tool_t::dump_load_check_superblock(const std::string & device)
 int disk_tool_t::dump_meta()
 {
    dump_with_data = options["format"] == "data";
    dump_as_old = options["format"] == "old";
    int r = process_meta(
-        [this](blockstore_meta_header_v3_t *hdr)
+        [this](blockstore_meta_header_v2_t *hdr) { dump_meta_header(hdr); },
-        {
+        [this](uint64_t block_num, clean_disk_entry *entry, uint8_t *bitmap) { dump_meta_entry(block_num, entry, bitmap); }
            if (dump_as_old)
            {
                hdr->version = BLOCKSTORE_META_FORMAT_V2;
                hdr->completed_lsn = 0;
                hdr->header_csum = 0;
                hdr->header_csum = crc32c(0, hdr, sizeof(blockstore_meta_header_v2_t));
            }
            dump_meta_header(hdr);
        },
        [this](blockstore_heap_t *heap, heap_entry_t *obj, uint32_t meta_block_num)
        {
            if (!obj)
                return;
            if (dump_as_old)
                dump_heap_entry_as_old(heap, obj);
            else
                dump_heap_entry(heap, obj);
        },
        [this](uint64_t block_num, clean_disk_entry *entry, uint8_t *bitmap) { dump_meta_entry(block_num, entry, bitmap); },
        true, true
    );
    if (r == 0)
        printf("\n]}\n");
    return r;
 }
-void disk_tool_t::dump_meta_header(blockstore_meta_header_v3_t *hdr)
+void disk_tool_t::dump_meta_header(blockstore_meta_header_v2_t *hdr)
 {
    if (hdr)
    {
        if (hdr->version == BLOCKSTORE_META_FORMAT_V1)
        {
            printf(
-                "{\"version\":\"" VITASTOR_META_FORMAT_NAME_V1 "\",\"meta_block_size\":%u,\"data_block_size\":%u,\"bitmap_granularity\":%u,"
+                "{\"version\":\"0.6\",\"meta_block_size\":%u,\"data_block_size\":%u,\"bitmap_granularity\":%u,"
                "\"entries\":[\n",
                hdr->meta_block_size, hdr->data_block_size, hdr->bitmap_granularity
            );
@@ -313,16 +218,7 @@ void disk_tool_t::dump_meta_header(blockstore_meta_header_v3_t *hdr)
        else if (hdr->version == BLOCKSTORE_META_FORMAT_V2)
        {
            printf(
-                "{\"version\":\"" VITASTOR_META_FORMAT_NAME_V2 "\",\"meta_block_size\":%u,\"data_block_size\":%u,\"bitmap_granularity\":%u,"
+                "{\"version\":\"0.9\",\"meta_block_size\":%u,\"data_block_size\":%u,\"bitmap_granularity\":%u,"
                "\"data_csum_type\":\"%s\",\"csum_block_size\":%u,\"entries\":[\n",
                hdr->meta_block_size, hdr->data_block_size, hdr->bitmap_granularity,
                csum_type_str(hdr->data_csum_type).c_str(), hdr->csum_block_size
            );
        }
        else if (hdr->version == BLOCKSTORE_META_FORMAT_HEAP)
        {
            printf(
                "{\"version\":\"" VITASTOR_META_FORMAT_NAME_HEAP "\",\"meta_block_size\":%u,\"data_block_size\":%u,\"bitmap_granularity\":%u,"
                "\"data_csum_type\":\"%s\",\"csum_block_size\":%u,\"entries\":[\n",
                hdr->meta_block_size, hdr->data_block_size, hdr->bitmap_granularity,
                csum_type_str(hdr->data_csum_type).c_str(), hdr->csum_block_size
@@ -331,148 +227,11 @@ void disk_tool_t::dump_meta_header(blockstore_meta_header_v3_t *hdr)
    }
    else
    {
-        printf("{\"version\":\"0.5\",\"meta_block_size\":%u,\"entries\":[\n", dsk.meta_block_size);
+        printf("{\"version\":\"0.5\",\"meta_block_size\":%ju,\"entries\":[\n", dsk.meta_block_size);
    }
    first_entry = true;
 }
 void disk_tool_t::dump_heap_entry_as_old(blockstore_heap_t *heap, heap_entry_t *obj)
 {
    heap_entry_t *wr = NULL;
    for (wr = obj; wr && !wr->is_overwrite(); wr = heap->prev(wr))
    {
    }
    if (!wr || wr->entry_type != (BS_HEAP_BIG_WRITE|BS_HEAP_STABLE) &&
        wr->entry_type != (BS_HEAP_BIG_INTENT|BS_HEAP_STABLE))
    {
        return;
    }
    printf(
 #define ENTRY_FMT "{\"block\":%u,\"pool\":%u,\"inode\":\"0x%jx\",\"stripe\":\"0x%jx\",\"version\":%ju"
        (first_entry ? ENTRY_FMT : (",\n" ENTRY_FMT)),
 #undef ENTRY_FMT
        wr->big().block_num, INODE_POOL(obj->inode), INODE_NO_POOL(obj->inode),
        obj->stripe, wr->version
    );
    printf(",\"bitmap\":\"");
    uint8_t* bitmap = wr->get_int_bitmap(heap);
    for (uint64_t i = 0; i < dsk.clean_entry_bitmap_size; i++)
    {
        printf("%02x", bitmap[i]);
    }
    bitmap = wr->get_ext_bitmap(heap);
    printf("\",\"ext_bitmap\":\"");
    for (uint64_t i = 0; i < dsk.clean_entry_bitmap_size; i++)
    {
        printf("%02x", bitmap[i]);
    }
    uint8_t *csums = wr->get_checksums(heap);
    uint32_t csum_size = heap->get_csum_size(wr);
    if (csums)
    {
        printf("\",\"block_csums\":\"");
        for (uint32_t i = 0; i < csum_size; i++)
        {
            printf("%02x", csums[i]);
        }
    }
    if (wr->get_checksum(heap))
    {
        printf("\",\"crc32c\":\"%08x", *wr->get_checksum(heap));
    }
    printf("\"}");
    first_entry = false;
 }
 void disk_tool_t::dump_heap_entry(blockstore_heap_t *heap, heap_entry_t *obj)
 {
    printf(
 #define ENTRY_FMT "{\"pool\":%u,\"inode\":\"0x%jx\",\"stripe\":\"0x%jx\",\"writes\":["
        (first_entry ? ENTRY_FMT : (",\n" ENTRY_FMT)),
 #undef ENTRY_FMT
        INODE_POOL(obj->inode), INODE_NO_POOL(obj->inode), obj->stripe
    );
    heap_entry_t *wr = NULL;
    bool first_wr = true;
    for (wr = obj; wr; wr = heap->prev(wr))
    {
        auto t = wr->type();
        printf(
 #define ENTRY_FMT "{\"lsn\":%ju,\"version\":%ju,\"type\":\"%s\",\"stable\":%s"
            (first_wr ? ENTRY_FMT : ("," ENTRY_FMT)),
 #undef ENTRY_FMT
            wr->lsn, wr->version, (t == BS_HEAP_BIG_WRITE ? "big" : (
                t == BS_HEAP_BIG_INTENT ? "big_intent" : (
                t == BS_HEAP_SMALL_WRITE ? "small" : (
                t == BS_HEAP_INTENT_WRITE ? "intent" : (
                t == BS_HEAP_DELETE ? "delete" : (
                t == BS_HEAP_COMMIT ? "commit" : (
                t == BS_HEAP_ROLLBACK ? "rollback" : (
                "unknown")))))))),
            (wr->entry_type & BS_HEAP_STABLE) ? "true" : "false"
        );
        if (t == BS_HEAP_BIG_WRITE)
        {
            printf(",\"location\":%ju", wr->big_location(heap));
        }
        else if (t == BS_HEAP_BIG_INTENT)
        {
            printf(",\"location\":%ju,\"offset\":%u,\"len\":%u", wr->big_location(heap), wr->big_intent().offset, wr->big_intent().len);
        }
        else if (t == BS_HEAP_INTENT_WRITE)
        {
            printf(",\"offset\":%u,\"len\":%u", wr->small().offset, wr->small().len);
        }
        else if (t == BS_HEAP_SMALL_WRITE)
        {
            if (!dump_with_data)
            {
                printf(",\"offset\":%u,\"len\":%u,\"location\":%ju", wr->small().offset, wr->small().len, wr->small().location);
            }
            else
            {
                printf(",\"data\":\"");
                for (uint32_t i = 0; i < wr->small().len; i++)
                    printf("%02x", buffer_area[wr->small().location + i]);
                printf("\"");
            }
        }
        uint8_t* bitmap = wr->get_int_bitmap(heap);
        if (bitmap)
        {
            printf(",\"bitmap\":\"");
            for (uint64_t i = 0; i < dsk.clean_entry_bitmap_size; i++)
                printf("%02x", bitmap[i]);
            printf("\"");
        }
        bitmap = wr->get_ext_bitmap(heap);
        if (bitmap)
        {
            printf(",\"ext_bitmap\":\"");
            for (uint64_t i = 0; i < dsk.clean_entry_bitmap_size; i++)
                printf("%02x", bitmap[i]);
            printf("\"");
        }
        uint8_t *csums = wr->get_checksums(heap);
        if (csums)
        {
            printf(",\"block_csums\":\"");
            uint32_t csum_size = heap->get_csum_size(wr);
            for (uint32_t i = 0; i < csum_size; i++)
                printf("%02x", csums[i]);
            printf("\"");
        }
        if (wr->get_checksum(heap))
        {
            printf(",\"data_crc32c\":\"%08x\"", *wr->get_checksum(heap));
        }
        printf("}");
        first_wr = false;
    }
    printf("]}");
    first_entry = false;
 }
 void disk_tool_t::dump_meta_entry(uint64_t block_num, clean_disk_entry *entry, uint8_t *bitmap)
 {
    printf(
@@ -514,16 +273,12 @@ void disk_tool_t::dump_meta_entry(uint64_t block_num, clean_disk_entry *entry, u
 int disk_tool_t::write_json_meta(json11::Json meta)
 {
    if (meta["version"].string_value() == VITASTOR_META_FORMAT_NAME_HEAP)
    {
        return write_json_heap(meta, meta["journal"]);
    }
    new_meta_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_meta_len);
    memset(new_meta_buf, 0, new_meta_len);
    blockstore_meta_header_v2_t *new_hdr = (blockstore_meta_header_v2_t *)new_meta_buf;
    new_hdr->zero = 0;
    new_hdr->magic = BLOCKSTORE_META_MAGIC_V1;
-    new_hdr->version = meta["version"].string_value() == VITASTOR_META_FORMAT_NAME_V1
+    new_hdr->version = meta["version"].uint64_value() == BLOCKSTORE_META_FORMAT_V1
        ? BLOCKSTORE_META_FORMAT_V1 : BLOCKSTORE_META_FORMAT_V2;
    new_hdr->meta_block_size = meta["meta_block_size"].uint64_value()
        ? meta["meta_block_size"].uint64_value() : 4096;
@@ -539,7 +294,7 @@ int disk_tool_t::write_json_meta(json11::Json meta)
                ? BLOCKSTORE_CSUM_CRC32C
                : BLOCKSTORE_CSUM_NONE);
        new_hdr->csum_block_size = meta["csum_block_size"].uint64_value();
-        new_hdr->header_csum = crc32c(0, new_hdr, sizeof(blockstore_meta_header_v2_t));
+        new_hdr->header_csum = crc32c(0, new_hdr, sizeof(*new_hdr));
    }
    uint32_t new_clean_entry_header_size = (new_hdr->version == BLOCKSTORE_META_FORMAT_V1
        ? sizeof(clean_disk_entry) : sizeof(clean_disk_entry) + 4 /*entry_csum*/);
@@ -549,7 +304,6 @@ int disk_tool_t::write_json_meta(json11::Json meta)
        : 0);
    new_clean_entry_size = new_clean_entry_header_size + 2*new_clean_entry_bitmap_size + new_data_csum_size;
    new_entries_per_block = new_hdr->meta_block_size / new_clean_entry_size;
    // FIXME: Use a streaming json parser
    for (const auto & e: meta["entries"].array_items())
    {
        uint64_t data_block = e["block"].uint64_value();
@@ -583,362 +337,7 @@ int disk_tool_t::write_json_meta(json11::Json meta)
        }
    }
    int r = resize_write_new_meta();
-    free_new_meta();
+    free(new_meta_buf);
    new_meta_buf = NULL;
    return r;
 }
 int disk_tool_t::write_json_heap(json11::Json meta, json11::Json journal)
 {
    new_meta_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_meta_len);
    memset(new_meta_buf, 0, new_meta_len);
    new_meta_hdr = (blockstore_meta_header_v3_t *)new_meta_buf;
    new_meta_hdr->zero = 0;
    new_meta_hdr->magic = BLOCKSTORE_META_MAGIC_V1;
    new_meta_hdr->version = BLOCKSTORE_META_FORMAT_HEAP;
    new_meta_hdr->meta_block_size = meta["meta_block_size"].uint64_value()
        ? meta["meta_block_size"].uint64_value() : 4096;
    new_meta_hdr->data_block_size = meta["data_block_size"].uint64_value()
        ? meta["data_block_size"].uint64_value() : 131072;
    new_meta_hdr->bitmap_granularity = meta["bitmap_granularity"].uint64_value()
        ? meta["bitmap_granularity"].uint64_value() : 4096;
    new_meta_hdr->data_csum_type = meta["data_csum_type"].is_number()
        ? meta["data_csum_type"].uint64_value()
        : (meta["data_csum_type"].string_value() == "crc32c"
            ? BLOCKSTORE_CSUM_CRC32C
            : BLOCKSTORE_CSUM_NONE);
    new_meta_hdr->csum_block_size = meta["csum_block_size"].uint64_value();
    new_meta_hdr->header_csum = crc32c(0, new_meta_hdr, sizeof(blockstore_meta_header_v3_t));
    new_meta_hdr->meta_area_size = new_meta_len;
    new_clean_entry_bitmap_size = (new_meta_hdr->data_block_size / new_meta_hdr->bitmap_granularity + 7) / 8;
    new_clean_entry_size = 0;
    new_entries_per_block = 0;
    new_data_csum_size = (new_meta_hdr->data_csum_type
        ? ((new_meta_hdr->data_block_size+new_meta_hdr->csum_block_size-1)/new_meta_hdr->csum_block_size*(new_meta_hdr->data_csum_type & 0xFF))
        : 0);
    new_journal_buf = NULL;
    if (new_journal_len)
    {
        new_journal_buf = (uint8_t*)memalign(MEM_ALIGNMENT, new_journal_len);
        memset(new_journal_buf, 0, new_journal_len);
    }
    uint64_t total_used_space = 0;
    uint32_t used_space = 0;
    uint64_t meta_offset = dsk.meta_block_size;
    blockstore_heap_t heap(&dsk, NULL, 0);
    heap_entry_t *wr = NULL;
    auto get_wr = [&](uint32_t entry_size)
    {
        if (used_space > new_meta_hdr->meta_block_size-entry_size)
        {
            heap.fill_block_empty_space(new_meta_buf + meta_offset, used_space);
            meta_offset += new_meta_hdr->meta_block_size;
            used_space = 0;
            if (meta_offset >= new_meta_len)
            {
                fprintf(stderr, "Metadata doesn't fit into the new area (total used space: %ju)\n", total_used_space);
                return (heap_entry_t*)NULL;
            }
        }
        auto wr = (heap_entry_t*)(new_meta_buf + meta_offset + used_space);
        used_space += entry_size;
        return wr;
    };
    // FIXME: Use a streaming json parser
    if (meta["version"].string_value() == VITASTOR_META_FORMAT_NAME_HEAP)
    {
        // New format
        for (const auto & meta_entry: meta["entries"].array_items())
        {
            object_id oid = {
                .inode = (sscanf_json(NULL, meta_entry["pool"]) << (64-POOL_ID_BITS)) | sscanf_json(NULL, meta_entry["inode"]),
                .stripe = sscanf_json(NULL, meta_entry["stripe"]),
            };
            for (const auto & write_entry: meta_entry["writes"].array_items())
            {
                uint32_t wr_type = 0;
                if (write_entry["type"] == "small")
                    wr_type = BS_HEAP_SMALL_WRITE;
                else if (write_entry["type"] == "intent")
                    wr_type = BS_HEAP_INTENT_WRITE;
                else if (write_entry["type"] == "big")
                    wr_type = BS_HEAP_BIG_WRITE;
                else if (write_entry["type"] == "big_intent")
                    wr_type = BS_HEAP_BIG_INTENT;
                else if (write_entry["type"] == "delete")
                    wr_type = BS_HEAP_DELETE;
                else if (write_entry["type"] == "commit")
                    wr_type = BS_HEAP_COMMIT;
                else if (write_entry["type"] == "rollback")
                    wr_type = BS_HEAP_ROLLBACK;
                else
                {
                    fprintf(stderr, "Write entry %s has invalid type: %s, aborting\n",
                        write_entry.dump().c_str(), write_entry["type"].dump().c_str());
 close_err0:
                    free(new_meta_buf);
                    new_meta_buf = NULL;
                    return 1;
                }
                uint64_t wr_offset = write_entry["offset"].uint64_value();
                uint64_t wr_len = write_entry["len"].uint64_value();
                uint32_t wr_size = (wr_type == BS_HEAP_SMALL_WRITE || wr_type == BS_HEAP_INTENT_WRITE
                    ? heap.get_small_entry_size(wr_offset, wr_len)
                    : (wr_type == BS_HEAP_BIG_WRITE
                        ? heap.get_big_entry_size()
                        : (wr_type == BS_HEAP_BIG_INTENT
                            ? heap.get_big_intent_entry_size()
                            : heap.get_simple_entry_size())));
                if (!(wr = get_wr(wr_size)))
                    goto close_err0;
                wr->inode = oid.inode;
                wr->stripe = oid.stripe;
                wr->entry_type = wr_type | (write_entry["stable"].bool_value() ? BS_HEAP_STABLE : 0);
                wr->lsn = write_entry["lsn"].uint64_value();
                wr->version = write_entry["version"].uint64_value();
                wr->size = wr->get_size(&heap);
                if (wr_type == BS_HEAP_SMALL_WRITE || wr_type == BS_HEAP_INTENT_WRITE)
                {
                    wr->small().offset = wr_offset;
                    wr->small().len = wr_len;
                    wr->small().location = write_entry["location"].uint64_value();
                    if (wr_type == BS_HEAP_SMALL_WRITE && write_entry["data"].is_string() && wr->small().len > 0)
                    {
                        if (!new_journal_buf)
                        {
                            fprintf(stderr, "Loading small write data requires overwriting buffer area\n");
                            free_new_meta();
                            return 1;
                        }
                        wr->small().location = heap.find_free_buffer_area(wr->small().len);
                        fromhexstr(write_entry["data"].string_value(), wr->small().len, new_journal_buf + wr->small().location);
                    }
                }
                else if (wr_type == BS_HEAP_BIG_WRITE)
                {
                    uint64_t loc = write_entry["location"].uint64_value();
                    if ((loc % dsk.data_block_size) || (loc / dsk.data_block_size) >= 0xFFFF0000)
                    {
                        fprintf(stderr, "Write entry %s has invalid location: 0x%jx, aborting\n",
                            write_entry.dump().c_str(), loc);
                        free_new_meta();
                        return 1;
                    }
                    wr->set_big_location(&heap, loc);
                }
                else if (wr_type == BS_HEAP_BIG_INTENT)
                {
                    uint64_t loc = write_entry["location"].uint64_value();
                    assert(!(loc % dsk.data_block_size));
                    assert((loc / dsk.data_block_size) < 0xFFFF0000);
                    auto & bi = wr->big_intent();
                    bi.block_num = loc / dsk.data_block_size;
                    bi.offset = wr_offset;
                    bi.len = wr_len;
                }
                if (write_entry["bitmap"].is_string() && wr->get_int_bitmap(&heap))
                {
                    fromhexstr(write_entry["bitmap"].string_value(), new_clean_entry_bitmap_size, wr->get_int_bitmap(&heap));
                }
                if (write_entry["ext_bitmap"].is_string() && wr->get_ext_bitmap(&heap))
                {
                    fromhexstr(write_entry["ext_bitmap"].string_value(), new_clean_entry_bitmap_size, wr->get_ext_bitmap(&heap));
                }
                if (write_entry["block_csums"].is_string() && wr->get_checksums(&heap))
                {
                    fromhexstr(write_entry["block_csums"].string_value(), heap.get_csum_size(wr), wr->get_ext_bitmap(&heap));
                }
                if (write_entry["data_crc32c"].is_string() && wr->get_checksum(&heap))
                {
                    *wr->get_checksum(&heap) = sscanf_json("%jx", write_entry["data_crc32c"]);
                }
                wr->crc32c = wr->calc_crc32c();
                assert((uint8_t*)wr + wr->size == new_meta_buf + meta_offset + used_space);
            }
        }
    }
    else
    {
        if (!journal.is_array())
        {
            fprintf(stderr, "Metadata should include journal in you want to convert it to the \"heap\" format\n");
 close_err:
            free(new_meta_buf);
            new_meta_buf = NULL;
            return 1;
        }
        std::map<object_id, std::vector<json11::Json::object>> journal_by_object;
        if (index_journal_by_object(journal, journal_by_object) != 0)
        {
            goto close_err;
        }
        journal = json11::Json();
        // Convert old format to the new format
        uint64_t next_lsn = 0;
        uint64_t buffer_pos = 0;
        for (const auto & meta_entry: meta["entries"].array_items())
        {
            object_id oid = {
                .inode = (sscanf_json(NULL, meta_entry["pool"]) << (64-POOL_ID_BITS)) | sscanf_json(NULL, meta_entry["inode"]),
                .stripe = sscanf_json(NULL, meta_entry["stripe"]),
            };
            if (!(wr = get_wr(heap.get_big_entry_size())))
                goto close_err;
            wr->inode = oid.inode;
            wr->stripe = oid.stripe;
            wr->entry_type = BS_HEAP_BIG_WRITE|BS_HEAP_STABLE;
            wr->lsn = ++next_lsn;
            wr->version = sscanf_json(NULL, meta_entry["version"]);
            wr->set_big_location(&heap, meta_entry["block"].uint64_value() * new_meta_hdr->data_block_size);
            wr->size = wr->get_size(&heap);
            fromhexstr(meta_entry["bitmap"].string_value(), new_clean_entry_bitmap_size, wr->get_int_bitmap(&heap));
            fromhexstr(meta_entry["ext_bitmap"].string_value(), new_clean_entry_bitmap_size, wr->get_ext_bitmap(&heap));
            if (new_meta_hdr->data_csum_type != 0)
                fromhexstr(meta_entry["data_csum"].string_value(), new_data_csum_size, wr->get_checksums(&heap));
            wr->crc32c = wr->calc_crc32c();
            assert((uint8_t*)wr + wr->size == new_meta_buf + meta_offset + used_space);
            auto j_it = journal_by_object.find(oid);
            if (j_it != journal_by_object.end())
            {
                for (auto & rec: j_it->second)
                {
                    uint64_t wr_offset = rec["offset"].uint64_value();
                    uint64_t wr_len = rec["len"].uint64_value();
                    if (!(wr = get_wr(rec["type"] == "small_write" || rec["type"] == "small_write_instant"
                        ? heap.get_small_entry_size(wr_offset, wr_len) : heap.get_big_entry_size())))
                    {
                        goto close_err;
                    }
                    wr->inode = oid.inode;
                    wr->stripe = oid.stripe;
                    wr->lsn = ++next_lsn;
                    wr->version = rec["ver"].uint64_value();
                    if (rec["type"] == "small_write" || rec["type"] == "small_write_instant")
                    {
                        if (wr_len > 0 && !rec["data"].is_string())
                        {
                            fprintf(stderr, "Error: entry data is missing, please generate the dump with --json --format data\n");
                            goto close_err;
                        }
                        wr->entry_type = BS_HEAP_SMALL_WRITE | (rec["type"] == "small_write_instant" ? BS_HEAP_STABLE : 0);
                        wr->small().offset = wr_offset;
                        wr->small().len = wr_len;
                        wr->small().location = buffer_pos;
                        fromhexstr(rec["bitmap"].string_value(), new_clean_entry_bitmap_size, wr->get_ext_bitmap(&heap));
                        fromhexstr(rec["data"].string_value(), wr_len, new_journal_buf+buffer_pos);
                        if (wr_len > 0)
                        {
                            if (!new_meta_hdr->data_csum_type)
                                *wr->get_checksum(&heap) = crc32c(0, new_journal_buf+buffer_pos, wr_len);
                            else
                                heap.calc_block_checksums((uint32_t*)wr->get_checksums(&heap), new_journal_buf+buffer_pos, NULL, wr_offset, wr_offset+wr_len, true, NULL);
                        }
                        buffer_pos += wr_len;
                    }
                    else if (rec["type"] == "big_write" || rec["type"] == "big_write_instant")
                    {
                        wr->entry_type = BS_HEAP_BIG_WRITE | (rec["type"] == "big_write_instant" ? BS_HEAP_STABLE : 0);
                        wr->set_big_location(&heap, sscanf_json(NULL, rec["loc"]));
                        bitmap_set(wr->get_int_bitmap(&heap), wr_offset, wr_len, new_meta_hdr->bitmap_granularity);
                        fromhexstr(rec["bitmap"].string_value(), new_clean_entry_bitmap_size, wr->get_ext_bitmap(&heap));
                        if (new_meta_hdr->data_csum_type != 0)
                        {
                            if ((wr_offset % new_meta_hdr->csum_block_size) || (wr_len % new_meta_hdr->csum_block_size))
                            {
                                fprintf(stderr,
                                    "Error: big_write journal entries not aligned to csum_block_size can't be converted between v0.9 and v3.0 metadata\n"
                                    "Stop writes and flush the journal or convert OSDs one by one without the journal if you still want to do it.\n");
                                goto close_err;
                            }
                            fromhexstr(rec["block_csums"].string_value(),
                                ((wr_offset+wr_len+new_meta_hdr->csum_block_size-1)/new_meta_hdr->csum_block_size
                                    - wr_offset/new_meta_hdr->csum_block_size) * (new_meta_hdr->data_csum_type & 0xFF),
                                wr->get_checksums(&heap));
                        }
                    }
                    else
                    {
                        assert(0);
                    }
                    wr->size = wr->get_size(&heap);
                    wr->crc32c = wr->calc_crc32c();
                    assert((uint8_t*)wr + wr->size == new_meta_buf + meta_offset + used_space);
                }
            }
        }
    }
    while (meta_offset < new_meta_len)
    {
        heap.fill_block_empty_space(new_meta_buf + meta_offset, used_space);
        meta_offset += dsk.meta_block_size;
        used_space = 0;
    }
    int r = resize_write_new_meta();
    if (new_journal_buf && r == 0)
    {
        r = resize_write_new_journal();
    }
    free_new_meta();
    return r;
 }
 int disk_tool_t::index_journal_by_object(json11::Json journal,
    std::map<object_id, std::vector<json11::Json::object>> & journal_by_object)
 {
    for (const auto & rec: journal.array_items())
    {
        object_id oid = {
            .inode = sscanf_json(NULL, rec["inode"]),
            .stripe = sscanf_json(NULL, rec["stripe"]),
        };
        auto & jbo = journal_by_object[oid];
        if (rec["type"] == "small_write" || rec["type"] == "small_write_instant")
        {
            jbo.push_back(rec.object_items());
        }
        else if (rec["type"] == "big_write" || rec["type"] == "big_write_instant")
        {
            if (rec["type"] == "big_write_instant")
                jbo.clear();
            jbo.push_back(rec.object_items());
        }
        else if (rec["type"] == "delete")
        {
            jbo.clear();
        }
        else if (rec["type"] == "stable")
        {
            uint64_t commit_to = rec["version"].uint64_value();
            for (size_t i = 0; i < jbo.size(); i++)
            {
                if (jbo[i]["version"].uint64_value() <= commit_to)
                {
                    if (jbo[i]["type"] == "big_write")
                    {
                        jbo.erase(jbo.begin(), jbo.begin()+i);
                        i = 0;
                        jbo[i]["type"] = "big_write_instant";
                    }
                    else if (jbo[i]["type"] == "small_write")
                    {
                        jbo[i]["type"] = "small_write_instant";
                    }
                }
            }
        }
        else if (rec["type"] == "rollback")
        {
            uint64_t rollback_to = rec["version"].uint64_value();
            for (size_t i = jbo.size()-1; i >= 0; i--)
            {
                if (jbo[i]["version"].uint64_value() > rollback_to)
                    jbo.erase(jbo.begin()+i, jbo.begin()+i+1);
            }
        }
        else
        {
            fprintf(stderr, "Unknown journal entry type: %s\n", rec.dump().c_str());
            return -1;
        }
    }
    return 0;
 }
@@ -6,29 +6,9 @@
 #include "json_util.h"
 #include "osd_id.h"
 void disk_tool_t::parse_meta_reserve()
 {
    meta_reserve_multiple = 2;
    meta_reserve_min_size = (uint64_t)1024*1024*1024;
    if (options.find("meta_reserve") != options.end())
    {
        int p1 = options["meta_reserve"].find("x"), p2 = options["meta_reserve"].find(",");
        if (p1 >= 0 && p2 >= 0)
        {
            sscanf(options["meta_reserve"].c_str()+(p1 < p2 ? 0 : p2), "%lf", &meta_reserve_multiple);
            meta_reserve_min_size = parse_size(options["meta_reserve"].substr(p1 < p2 ? p2 : 0, p1 < p2 ? options["meta_reserve"].size()-p2 : p2));
        }
        else if (p1 >= 0)
            sscanf(options["meta_reserve"].c_str(), "%lf", &meta_reserve_multiple);
        else
            meta_reserve_min_size = parse_size(options["meta_reserve"]);
    }
 }
 int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_hdd, json11::Json::object & result)
 {
    static const char *allow_additional_params[] = {
        "meta_format",
        "data_csum_type",
        "csum_block_size",
        "autosync_writes",
@@ -51,8 +31,6 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
        "discard_on_start",
        "min_discard_size",
        "discard_granularity",
        "atomic_write_size",
        "use_atomic_flag",
    };
    if (options.find("force") == options.end())
    {
@@ -79,24 +57,6 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
                return 1;
        }
    }
    if (options.find("atomic_write_size") == options.end())
    {
        auto data_dev = realpath_str(options["data_device"], false);
        uint64_t atomic_write_size = get_atomic_write_size(data_dev);
        if (atomic_write_size > 4096)
        {
            fprintf(stderr, "Data device %s supports atomic writes up to %ju bytes, enabling. Enjoy faster writes!\n",
                data_dev.c_str(), atomic_write_size);
            options["atomic_write_size"] = std::to_string(atomic_write_size);
            options["use_atomic_flag"] = "1";
        }
    }
    else if (options.find("use_atomic_flag") == options.end() &&
        parse_size(options["atomic_write_size"]) > 4096)
    {
        fprintf(stderr, "Atomic writes larger than 4 KB are enabled manually, enabling use_atomic_flag too.\n");
        options["use_atomic_flag"] = "1";
    }
    for (auto dev: std::vector<std::string>{"data", "meta", "journal"})
    {
        if (options[dev+"_device"] != "" && options["disable_"+dev+"_fsync"] == "auto")
@@ -112,10 +72,6 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
                options["disable_"+dev+"_fsync"] = "1";
        }
    }
    if (options["meta_format"] == "")
    {
        options["meta_format"] = std::to_string(BLOCKSTORE_META_FORMAT_HEAP);
    }
    if (options["meta_device"] == "" || options["meta_device"] == options["data_device"])
    {
        options["disable_meta_fsync"] = options["disable_data_fsync"];
@@ -152,50 +108,35 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
        if (options.find("autosync_writes") == options.end())
            options["autosync_writes"] = "512";
    }
    uint64_t new_meta_len = parse_size(options["meta_len"]);
    json11::Json::object sb;
    blockstore_disk_t dsk;
    try
    {
        dsk.parse_config(options);
-        // Calculate metadata sizes (with excess)
+        // Set all offsets to 4096 to calculate metadata size with excess
        dsk.journal_offset = 4096;
-        dsk.meta_offset = 4096 + (dsk.meta_device == dsk.journal_device ? dsk.cfg_journal_size : 0);
+        dsk.meta_offset = 4096;
-        dsk.data_offset = 4096 + (dsk.data_device == dsk.meta_device && new_meta_len ? new_meta_len : 0) +
+        dsk.data_offset = 4096;
            (dsk.data_device == dsk.journal_device ? dsk.cfg_journal_size : 0);
        dsk.data_io = dsk.meta_io = dsk.journal_io = (options["io"] == "cached" ? "cached" : "direct");
        dsk.open_data();
        dsk.open_meta();
        dsk.open_journal();
        dsk.calc_lengths(true);
        if (dsk.data_device == dsk.meta_device && !new_meta_len)
        {
            uint64_t new_meta_len = dsk.min_meta_len;
            if (dsk.meta_format == BLOCKSTORE_META_FORMAT_HEAP)
            {
                new_meta_len = dsk.min_meta_len*meta_reserve_multiple;
                if (new_meta_len < meta_reserve_min_size)
                    new_meta_len = meta_reserve_min_size;
                new_meta_len = (new_meta_len + dsk.meta_block_size-1) & ~((uint64_t)dsk.meta_block_size-1);
            }
            dsk.data_offset += new_meta_len;
        }
        dsk.meta_area_size = (dsk.data_device == dsk.meta_device ? dsk.data_offset : dsk.meta_device_size) - dsk.meta_offset;
        sb = json11::Json::object {
            { "meta_format", options["meta_format"] },
            { "data_device", options["data_device"] },
            { "meta_device", options["meta_device"] },
            { "journal_device", options["journal_device"] },
            { "block_size", (uint64_t)dsk.data_block_size },
-            { "meta_block_size", (uint64_t)dsk.meta_block_size },
+            { "meta_block_size", dsk.meta_block_size },
-            { "journal_block_size", (uint64_t)dsk.journal_block_size },
+            { "journal_block_size", dsk.journal_block_size },
            { "data_size", dsk.cfg_data_size },
            { "disk_alignment", (uint64_t)dsk.disk_alignment },
-            { "bitmap_granularity", (uint64_t)dsk.bitmap_granularity },
+            { "bitmap_granularity", dsk.bitmap_granularity },
            { "disable_device_lock", dsk.disable_flock },
            { "journal_offset", 4096 },
-            { "meta_offset", dsk.meta_offset },
+            { "meta_offset", 4096 + (dsk.meta_device == dsk.journal_device ? dsk.journal_len : 0) },
-            { "data_offset", dsk.data_offset },
+            { "data_offset", 4096 + (dsk.data_device == dsk.meta_device ? dsk.meta_len : 0) +
                (dsk.data_device == dsk.journal_device ? dsk.journal_len : 0) },
            { "journal_no_same_sector_overwrites", !is_hdd || is_hybrid },
            { "journal_sector_buffer_count", 1024 },
            { "disable_data_fsync", json_is_true(options["disable_data_fsync"]) },
@@ -239,7 +180,7 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
    }
    sb["osd_num"] = osd_num;
    // Zero out metadata and journal
-    if (write_zero(dsk.meta_fd, sb["meta_offset"].uint64_value(), dsk.meta_area_size) != 0 ||
+    if (write_zero(dsk.meta_fd, sb["meta_offset"].uint64_value(), dsk.meta_len) != 0 ||
        write_zero(dsk.journal_fd, sb["journal_offset"].uint64_value(), dsk.journal_len) != 0)
    {
        fprintf(stderr, "Failed to zero out metadata or journal: %s\n", strerror(errno));
@@ -588,7 +529,7 @@ int disk_tool_t::get_meta_partition(std::vector<vitastor_dev_info_t> & ssds, std
        dsk.open_journal();
        dsk.calc_lengths(true);
        dsk.close_all();
-        meta_size = dsk.min_meta_len;
+        meta_size = dsk.meta_len;
    }
    catch (std::exception & e)
    {
@@ -597,10 +538,24 @@ int disk_tool_t::get_meta_partition(std::vector<vitastor_dev_info_t> & ssds, std
        return 1;
    }
    // Leave some extra space for future metadata formats and round metadata area size to multiples of 1 MB
    uint64_t meta_reserve_multiple = 2, min_meta_size = (uint64_t)1024*1024*1024;
    if (options.find("meta_reserve") != options.end())
    {
        int p1 = options["meta_reserve"].find("x"), p2 = options["meta_reserve"].find(",");
        if (p1 >= 0 && p2 >= 0)
        {
            meta_reserve_multiple = stoull_full(options["meta_reserve"].substr(p1 < p2 ? 0 : p2, p1 - (p1 < p2 ? 0 : p2)));
            min_meta_size = parse_size(options["meta_reserve"].substr(p1 < p2 ? p2 : 0, p1 < p2 ? options["meta_reserve"].size()-p2 : p2));
        }
        else if (p1 >= 0)
            meta_reserve_multiple = stoull_full(options["meta_reserve"].substr(0, p1));
        else
            min_meta_size = parse_size(options["meta_reserve"]);
    }
    meta_size = ((meta_size+1024*1024-1)/1024/1024)*1024*1024;
    meta_size *= meta_reserve_multiple;
-    if (meta_size < meta_reserve_min_size)
+    if (meta_size < min_meta_size)
-        meta_size = meta_reserve_min_size;
+        meta_size = min_meta_size;
    // Pick an SSD for journal&meta, balancing the number of serviced OSDs across SSDs
    int sel = -1;
    for (int i = 0; i < ssds.size(); i++)
@@ -631,7 +586,6 @@ int disk_tool_t::get_meta_partition(std::vector<vitastor_dev_info_t> & ssds, std
 int disk_tool_t::prepare(std::vector<std::string> devices)
 {
    parse_meta_reserve();
    if (options.find("data_device") != options.end() && options["data_device"] != "")
    {
        if (options.find("hybrid") != options.end() ||
@@ -679,8 +633,6 @@ int disk_tool_t::prepare(std::vector<std::string> devices)
    std::vector<vitastor_dev_info_t> ssds;
    if (options.find("disable_data_fsync") == options.end())
        options["disable_data_fsync"] = "auto";
    if (options["meta_format"] == "")
        options["meta_format"] = std::to_string(BLOCKSTORE_META_FORMAT_HEAP);
    if (hybrid)
    {
        if (options.find("disable_meta_fsync") == options.end())
@@ -1,13 +1,9 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 #define _XOPEN_SOURCE
 #include <limits.h>
 #include "disk_tool.h"
 #include "rw_blocking.h"
 #include "str_util.h"
 #include "malloc_or_die.h"
 #define DM_ST_EMPTY 0
 #define DM_ST_TO_READ 1
@@ -25,92 +21,68 @@ struct resizer_data_moving_t
 int disk_tool_t::raw_resize()
 {
    int r;
    parse_meta_reserve();
    // Parse parameters
    r = resize_parse_params();
    if (r != 0)
-        goto ret;
+        return r;
    // Fill allocator
    fprintf(stderr, "Reading metadata\n");
    data_alloc = new allocator_t((new_data_len < dsk.data_len ? dsk.data_len : new_data_len) / dsk.data_block_size);
    r = process_meta(
-        [this](blockstore_meta_header_v3_t *hdr)
+        [this](blockstore_meta_header_v2_t *hdr)
        {
            resize_init(hdr);
        },
        [this](blockstore_heap_t *heap, heap_entry_t *obj, uint32_t meta_block_num)
        {
            for (auto wr = obj; wr; wr = heap->prev(wr))
            {
                if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE ||
                    (wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_INTENT)
                {
                    data_alloc->set(wr->big().block_num, true);
                }
            }
        },
        [this](uint64_t block_num, clean_disk_entry *entry, uint8_t *bitmap)
        {
            data_alloc->set(block_num, true);
-        },
+        }
        true, true
    );
    if (r != 0)
-        goto ret;
+        return r;
-    if (dsk.meta_format != BLOCKSTORE_META_FORMAT_HEAP)
+    fprintf(stderr, "Reading journal\n");
    r = process_journal([this](void *buf)
    {
-        fprintf(stderr, "Reading journal\n");
+        return process_journal_block(buf, [this](int num, journal_entry *je)
        r = process_journal([this](void *buf)
        {
-            return process_journal_block(buf, [this](int num, journal_entry *je)
+            if (je->type == JE_BIG_WRITE || je->type == JE_BIG_WRITE_INSTANT)
            {
-                if (je->type == JE_BIG_WRITE || je->type == JE_BIG_WRITE_INSTANT)
+                data_alloc->set(je->big_write.location / dsk.data_block_size, true);
-                {
+            }
                    data_alloc->set(je->big_write.location / dsk.data_block_size, true);
                }
            });
        });
-        if (r != 0)
+    });
-            goto ret;
+    if (r != 0)
-    }
+        return r;
    // Remap blocks
    r = resize_remap_blocks();
    if (r != 0)
-        goto ret;
+        return r;
    // Copy data blocks into new places
    fprintf(stderr, "Moving data blocks\n");
    r = resize_copy_data();
    if (r != 0)
-        goto ret;
+        return r;
-    // Rewrite metadata
+    // Rewrite journal
-    resize_alloc_journal();
+    fprintf(stderr, "Rebuilding journal\n");
-    fprintf(stderr, "Rebuilding metadata\n");
+    r = resize_rewrite_journal();
    r = resize_rebuild_meta();
    if (r != 0)
-        goto ret;
+        return r;
-    if (new_meta_format != BLOCKSTORE_META_FORMAT_HEAP)
+    // Rewrite metadata
-    {
+    fprintf(stderr, "Rebuilding metadata\n");
-        // Rewrite journal
+    r = resize_rewrite_meta();
-        fprintf(stderr, "Rebuilding journal\n");
+    if (r != 0)
-        r = resize_rebuild_journal();
+        return r;
        if (r != 0)
            goto ret;
        fprintf(stderr, "Writing new journal\n");
    }
    else
        fprintf(stderr, "Writing new buffer area\n");
    // Write new journal
    fprintf(stderr, "Writing new journal\n");
    r = resize_write_new_journal();
    if (r != 0)
-        goto ret;
+        return r;
    // Write new metadata
    fprintf(stderr, "Writing new metadata\n");
    r = resize_write_new_meta();
    if (r != 0)
-        goto ret;
+        return r;
    fprintf(stderr, "Done\n");
 ret:
    free_new_meta();
    return 0;
 }
@@ -154,9 +126,6 @@ int disk_tool_t::resize_parse_params()
        ? parse_size(options["new_journal_offset"]) : dsk.journal_offset;
    new_journal_len = options.find("new_journal_len") != options.end()
        ? parse_size(options["new_journal_len"]) : dsk.journal_len;
    new_meta_format = options.find("new_meta_format") != options.end()
        ? stoull_full(options["new_meta_format"]) : 0;
    skip_obsolete = options.find("skip_obsolete") != options.end();
    if (new_data_len+new_data_offset > dsk.data_device_size)
        new_data_len = dsk.data_device_size-new_data_offset;
    if (new_meta_device == dsk.data_device && new_data_offset < new_meta_offset &&
@@ -170,7 +139,7 @@ int disk_tool_t::resize_parse_params()
        new_data_offset == dsk.data_offset &&
        new_data_len == dsk.data_len &&
        new_meta_offset == dsk.meta_offset &&
-        (new_meta_len == dsk.meta_area_size || new_meta_len == 0) &&
+        (new_meta_len == dsk.meta_len || new_meta_len == 0) &&
        new_journal_offset == dsk.journal_offset &&
        new_journal_len == dsk.journal_len &&
        options.find("force") == options.end())
@@ -182,7 +151,7 @@ int disk_tool_t::resize_parse_params()
    return 0;
 }
-void disk_tool_t::resize_init(blockstore_meta_header_v3_t *hdr)
+void disk_tool_t::resize_init(blockstore_meta_header_v2_t *hdr)
 {
    if (hdr && dsk.data_block_size != hdr->data_block_size)
    {
@@ -201,15 +170,6 @@ void disk_tool_t::resize_init(blockstore_meta_header_v3_t *hdr)
        dsk.data_csum_type = hdr->data_csum_type;
        dsk.csum_block_size = hdr->csum_block_size;
    }
    if (hdr && dsk.meta_format != hdr->version)
    {
        dsk.meta_format = hdr->version;
    }
    if (new_meta_format == 0)
    {
        new_meta_format = hdr && hdr->version == BLOCKSTORE_META_FORMAT_HEAP ? BLOCKSTORE_META_FORMAT_HEAP : BLOCKSTORE_META_FORMAT_V2;
    }
    dsk.calc_lengths();
    if (((new_data_offset-dsk.data_offset) % dsk.data_block_size))
    {
        fprintf(stderr, "Data alignment mismatch: old data offset is 0x%jx, new is 0x%jx, but alignment on %x should be equal\n",
@@ -226,28 +186,17 @@ void disk_tool_t::resize_init(blockstore_meta_header_v3_t *hdr)
    new_data_csum_size = (dsk.data_csum_type
        ? ((dsk.data_block_size+dsk.csum_block_size-1)/dsk.csum_block_size*(dsk.data_csum_type & 0xFF))
        : 0);
-    if (new_meta_format != BLOCKSTORE_META_FORMAT_HEAP)
+    new_clean_entry_size = new_clean_entry_header_size + 2*new_clean_entry_bitmap_size + new_data_csum_size;
    new_entries_per_block = dsk.meta_block_size/new_clean_entry_size;
    uint64_t new_meta_blocks = 1 + (new_data_len/dsk.data_block_size + new_entries_per_block-1) / new_entries_per_block;
    if (!new_meta_len)
    {
-        new_clean_entry_size = new_clean_entry_header_size + 2*new_clean_entry_bitmap_size + new_data_csum_size;
+        new_meta_len = dsk.meta_block_size*new_meta_blocks;
        new_entries_per_block = dsk.meta_block_size/new_clean_entry_size;
        uint64_t new_meta_blocks = 1 + (new_data_len/dsk.data_block_size + new_entries_per_block-1) / new_entries_per_block;
        if (!new_meta_len)
        {
            new_meta_len = dsk.meta_block_size*new_meta_blocks;
        }
        if (new_meta_len < dsk.meta_block_size*new_meta_blocks)
        {
            fprintf(stderr, "New metadata area size is too small, should be at least %ju bytes\n", dsk.meta_block_size*new_meta_blocks);
            exit(1);
        }
    }
-    else
+    if (new_meta_len < dsk.meta_block_size*new_meta_blocks)
    {
-        new_clean_entry_size = new_entries_per_block = 0;
+        fprintf(stderr, "New metadata area size is too small, should be at least %ju bytes\n", dsk.meta_block_size*new_meta_blocks);
-        if (!new_meta_len)
+        exit(1);
        {
            new_meta_len = dsk.meta_area_size;
        }
    }
    // Check that new metadata, journal and data areas don't overlap
    if (new_meta_device == dsk.data_device && new_meta_offset < new_data_offset+new_data_len &&
@@ -410,57 +359,15 @@ int disk_tool_t::resize_copy_data()
    return 0;
 }
-void disk_tool_t::resize_alloc_journal()
+int disk_tool_t::resize_rewrite_journal()
 {
    new_journal_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_journal_len);
    memset(new_journal_buf, 0, new_journal_len);
    new_journal_ptr = new_journal_buf;
    new_journal_data = new_journal_ptr + dsk.journal_block_size;
    new_journal_in_pos = 0;
 }
 void disk_tool_t::build_journal_start()
 {
    journal_entry *ne = (journal_entry*)(new_journal_ptr + new_journal_in_pos);
    *((journal_entry_start*)ne) = (journal_entry_start){
        .magic = JOURNAL_MAGIC,
        .type = JE_START,
        .size = sizeof(journal_entry_start),
        .journal_start = dsk.journal_block_size,
        .version = JOURNAL_VERSION_V2,
        .data_csum_type = dsk.data_csum_type,
        .csum_block_size = dsk.csum_block_size,
    };
    ne->crc32 = je_crc32(ne);
    new_journal_ptr += dsk.journal_block_size;
    new_journal_data = new_journal_ptr+dsk.journal_block_size;
    new_journal_in_pos = 0;
 }
 void disk_tool_t::choose_journal_block(uint32_t je_size)
 {
    if (dsk.journal_block_size < new_journal_in_pos+je_size)
    {
        new_journal_ptr = new_journal_data;
        if (new_journal_ptr-new_journal_buf >= new_journal_len)
        {
            fprintf(stderr, "Error: live entries don't fit to the new journal\n");
            exit(1);
        }
        new_journal_data = new_journal_ptr+dsk.journal_block_size;
        new_journal_in_pos = 0;
        if (dsk.journal_block_size < je_size)
        {
            fprintf(stderr, "Error: journal entry too large (%u bytes)\n", je_size);
            exit(1);
        }
    }
 }
 int disk_tool_t::resize_rebuild_journal()
 {
    // Simply overwriting on the fly may be impossible because old and new areas may overlap
    // For now, just build new journal data in memory
    new_journal_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_journal_len);
    new_journal_ptr = new_journal_buf;
    new_journal_data = new_journal_ptr + dsk.journal_block_size;
    new_journal_in_pos = 0;
    memset(new_journal_buf, 0, new_journal_len);
    process_journal([this](void *buf)
    {
        return process_journal_block(buf, [this](int num, journal_entry *je)
@@ -477,11 +384,39 @@ int disk_tool_t::resize_rebuild_journal()
                    );
                    exit(1);
                }
-                build_journal_start();
+                journal_entry *ne = (journal_entry*)(new_journal_ptr + new_journal_in_pos);
                *((journal_entry_start*)ne) = (journal_entry_start){
                    .magic = JOURNAL_MAGIC,
                    .type = JE_START,
                    .size = sizeof(journal_entry_start),
                    .journal_start = dsk.journal_block_size,
                    .version = JOURNAL_VERSION_V2,
                    .data_csum_type = dsk.data_csum_type,
                    .csum_block_size = dsk.csum_block_size,
                };
                ne->crc32 = je_crc32(ne);
                new_journal_ptr += dsk.journal_block_size;
                new_journal_data = new_journal_ptr+dsk.journal_block_size;
                new_journal_in_pos = 0;
            }
            else
            {
-                choose_journal_block(je->size);
+                if (dsk.journal_block_size < new_journal_in_pos+je->size)
                {
                    new_journal_ptr = new_journal_data;
                    if (new_journal_ptr-new_journal_buf >= new_journal_len)
                    {
                        fprintf(stderr, "Error: live entries don't fit to the new journal\n");
                        exit(1);
                    }
                    new_journal_data = new_journal_ptr+dsk.journal_block_size;
                    new_journal_in_pos = 0;
                    if (dsk.journal_block_size < je->size)
                    {
                        fprintf(stderr, "Error: journal entry too large (%u bytes)\n", je->size);
                        exit(1);
                    }
                }
                journal_entry *ne = (journal_entry*)(new_journal_ptr + new_journal_in_pos);
                memcpy(ne, je, je->size);
                ne->crc32_prev = new_crc32_prev;
@@ -528,226 +463,30 @@ int disk_tool_t::resize_write_new_journal()
    fsync(new_journal_fd);
    close(new_journal_fd);
    new_journal_fd = -1;
    free(new_journal_buf);
    new_journal_buf = NULL;
    return 0;
 }
-void disk_tool_t::remap_big_write(heap_entry_t *wr)
+int disk_tool_t::resize_rewrite_meta()
 {
    uint64_t block_num = wr->big().block_num;
    auto remap_it = data_remap.find(block_num);
    if (remap_it != data_remap.end())
        block_num = remap_it->second;
    if (block_num < free_first || block_num >= total_blocks-free_last)
    {
        fprintf(stderr, "BUG: remapped block %ju not in range %ju..%ju\n", block_num, free_first, total_blocks-free_last);
        exit(1);
    }
    block_num += data_idx_diff;
    wr->big().block_num = block_num;
    wr->crc32c = wr->calc_crc32c();
 }
 void disk_tool_t::remap_small_write(heap_entry_t *wr)
 {
    if (new_meta_format == BLOCKSTORE_META_FORMAT_HEAP && wr->small().len > 0)
    {
        if (new_journal_ptr-new_journal_buf+wr->small().len > new_journal_len)
        {
            fprintf(stderr, "Small write data doesn't fit into the new buffer area\n");
            exit(1);
        }
        memcpy(new_journal_ptr, buffer_area+wr->small().location, wr->small().len);
        wr->small().location = new_journal_ptr-new_journal_buf;
        new_journal_ptr += wr->small().len;
        wr->crc32c = wr->calc_crc32c();
    }
 }
 void disk_tool_t::fill_old_clean_entry(blockstore_heap_t *heap, heap_entry_t *big_wr)
 {
    uint64_t block_num = big_wr->big().block_num;
    clean_disk_entry *new_entry = (clean_disk_entry*)(new_meta_buf + dsk.meta_block_size +
        dsk.meta_block_size*(block_num / new_entries_per_block) +
        new_clean_entry_size*(block_num % new_entries_per_block));
    new_entry->oid = (object_id){ .inode = big_wr->inode, .stripe = big_wr->stripe };
    new_entry->version = big_wr->version;
    memcpy(new_entry->bitmap, big_wr->get_ext_bitmap(heap), new_clean_entry_bitmap_size);
    memcpy(new_entry->bitmap + new_clean_entry_bitmap_size, big_wr->get_int_bitmap(heap), new_clean_entry_bitmap_size);
    memcpy(new_entry->bitmap + 2*new_clean_entry_bitmap_size, big_wr->get_checksums(heap), new_data_csum_size);
    uint32_t *new_entry_csum = (uint32_t*)(((uint8_t*)new_entry) + new_clean_entry_size - 4);
    *new_entry_csum = crc32c(0, new_entry, new_clean_entry_size - 4);
 }
 void disk_tool_t::fill_old_journal_entry(blockstore_heap_t *heap, heap_entry_t *wr)
 {
    assert(wr->type() == BS_HEAP_SMALL_WRITE ||
        wr->type() == BS_HEAP_BIG_WRITE ||
        wr->type() == BS_HEAP_BIG_INTENT);
    uint32_t je_size = ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_SMALL_WRITE
        ? sizeof(journal_entry_small_write) + dsk.dirty_dyn_size(wr->small().offset, wr->small().len)
        : sizeof(journal_entry_big_write) + dsk.dirty_dyn_size(0, dsk.data_block_size));
    choose_journal_block(je_size);
    journal_entry *je = (journal_entry*)(new_journal_ptr + new_journal_in_pos);
    je->magic = JOURNAL_MAGIC;
    je->type = (wr->entry_type & BS_HEAP_STABLE) ? JE_SMALL_WRITE_INSTANT : JE_SMALL_WRITE;
    je->size = je_size;
    je->crc32_prev = new_crc32_prev;
    je->small_write.oid = (object_id){ .inode = wr->inode, .stripe = wr->stripe };
    je->small_write.version = wr->version;
    if (wr->type() == BS_HEAP_SMALL_WRITE)
    {
        je->small_write.offset = wr->small().offset;
        je->small_write.len = wr->small().len;
        je->small_write.data_offset = new_journal_data-new_journal_buf;
        if (je->small_write.data_offset + je->small_write.len > new_journal_len)
        {
            fprintf(stderr, "Error: live entries don't fit to the new journal\n");
            exit(1);
        }
        memcpy(new_journal_data, buffer_area+wr->small().location, je->small_write.len);
        new_journal_data += je->small_write.len;
        if (dsk.data_csum_type == 0 && wr->get_checksum(heap))
            je->small_write.crc32_data = *wr->get_checksum(heap);
    }
    else
    {
        je->big_write.location = wr->big_location(heap);
    }
    memcpy((uint8_t*)je + je->size, wr->get_ext_bitmap(heap), new_clean_entry_bitmap_size);
    if (dsk.data_csum_type != 0 && wr->get_checksums(heap))
    {
        memcpy((uint8_t*)je + je->size + new_clean_entry_bitmap_size, wr->get_checksums(heap), heap->get_csum_size(wr));
    }
    je->crc32 = je_crc32(je);
    new_journal_in_pos += je->size;
    new_crc32_prev = je->crc32;
 }
 int disk_tool_t::resize_rebuild_meta()
 {
    new_meta_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, new_meta_len);
    memset(new_meta_buf, 0, new_meta_len);
    new_meta_hdr = (blockstore_meta_header_v3_t *)new_meta_buf;
    uint64_t new_meta_pos = dsk.meta_block_size;
    uint64_t next_lsn = 0;
    std::vector<heap_entry_t*> writes;
    int r = process_meta(
-        [&](blockstore_meta_header_v3_t *hdr)
+        [this](blockstore_meta_header_v2_t *hdr)
        {
-            new_meta_hdr->zero = 0;
+            blockstore_meta_header_v2_t *new_hdr = (blockstore_meta_header_v2_t *)new_meta_buf;
-            new_meta_hdr->magic = BLOCKSTORE_META_MAGIC_V1;
+            new_hdr->zero = 0;
-            new_meta_hdr->version = new_meta_format == 0 ? BLOCKSTORE_META_FORMAT_HEAP : new_meta_format;
+            new_hdr->magic = BLOCKSTORE_META_MAGIC_V1;
-            new_meta_hdr->meta_block_size = dsk.meta_block_size;
+            new_hdr->version = BLOCKSTORE_META_FORMAT_V2;
-            new_meta_hdr->data_block_size = dsk.data_block_size;
+            new_hdr->meta_block_size = dsk.meta_block_size;
-            new_meta_hdr->bitmap_granularity = dsk.bitmap_granularity ? dsk.bitmap_granularity : 4096;
+            new_hdr->data_block_size = dsk.data_block_size;
-            new_meta_hdr->data_csum_type = dsk.data_csum_type;
+            new_hdr->bitmap_granularity = dsk.bitmap_granularity ? dsk.bitmap_granularity : 4096;
-            new_meta_hdr->csum_block_size = dsk.csum_block_size;
+            new_hdr->data_csum_type = dsk.data_csum_type;
-            new_meta_hdr->completed_lsn = hdr->completed_lsn;
+            new_hdr->csum_block_size = dsk.csum_block_size;
-            new_meta_hdr->meta_area_size = new_meta_len;
+            new_hdr->header_csum = crc32c(0, new_hdr, sizeof(*new_hdr));
            new_meta_hdr->header_csum = 0;
            new_meta_hdr->header_csum = crc32c(0, new_meta_hdr, new_meta_hdr->version == BLOCKSTORE_META_FORMAT_HEAP
                ? sizeof(blockstore_meta_header_v3_t) : sizeof(blockstore_meta_header_v2_t));
            if (hdr->version == BLOCKSTORE_META_FORMAT_HEAP && new_meta_format != BLOCKSTORE_META_FORMAT_HEAP)
            {
                build_journal_start();
            }
        },
-        [&](blockstore_heap_t *heap, heap_entry_t *obj, uint32_t meta_block_num)
+        [this](uint64_t block_num, clean_disk_entry *entry, uint8_t *bitmap)
        {
            if (!obj)
            {
                // Finish
                if (new_meta_format == BLOCKSTORE_META_FORMAT_HEAP)
                {
                    heap->fill_block_empty_space(new_meta_buf, new_meta_pos);
                    new_meta_pos = (new_meta_pos/dsk.meta_block_size + 1) * dsk.meta_block_size;
                    while (new_meta_pos < new_meta_len)
                    {
                        heap->fill_block_empty_space(new_meta_buf, new_meta_pos);
                        new_meta_pos += dsk.meta_block_size;
                    }
                }
                return;
            }
            auto handle_write = [&](heap_entry_t *wr, bool stable)
            {
                if (wr->type() == BS_HEAP_BIG_WRITE || wr->type() == BS_HEAP_BIG_INTENT)
                {
                    remap_big_write(wr);
                }
                else if (wr->type() == BS_HEAP_SMALL_WRITE)
                {
                    remap_small_write(wr);
                }
                else if (wr->type() != BS_HEAP_DELETE && new_meta_format != BLOCKSTORE_META_FORMAT_HEAP)
                {
                    fprintf(stderr, "Object %jx:%jx can't be converted to the old format because it contains an entry of type 0x%x%s\n",
                        wr->inode, wr->stripe, wr->entry_type,
                        (wr->type() == BS_HEAP_INTENT_WRITE ? " (intent_write)" : ""));
                    exit(1);
                }
                if (new_meta_format == BLOCKSTORE_META_FORMAT_HEAP)
                {
                    // New -> New
                    if ((new_meta_pos % dsk.meta_block_size) + wr->size > dsk.meta_block_size)
                    {
                        heap->fill_block_empty_space(new_meta_buf, new_meta_pos);
                        new_meta_pos = (new_meta_pos/dsk.meta_block_size + 1) * dsk.meta_block_size;
                        if (new_meta_pos >= new_meta_len)
                        {
                            fprintf(stderr, "New metadata doesn't fit into the provided area\n");
                            exit(1);
                        }
                    }
                    memcpy(new_meta_buf + new_meta_pos, wr, wr->size);
                    new_meta_pos += wr->size;
                    if (skip_obsolete && wr->type() == BS_HEAP_BIG_WRITE && stable)
                    {
                        // Skip older writes
                        return false;
                    }
                }
                else
                {
                    // New -> Old
                    if (wr->type() == BS_HEAP_DELETE && stable)
                    {
                        // Object is deleted, skip it
                        return false;
                    }
                    if (wr->type() == BS_HEAP_BIG_WRITE && stable)
                    {
                        fill_old_clean_entry(heap, wr);
                        return false;
                    }
                    else
                    {
                        writes.push_back(wr);
                    }
                }
                return true;
            };
            if (new_meta_format != BLOCKSTORE_META_FORMAT_HEAP || skip_obsolete)
            {
                heap->iterate_with_stable(obj, obj->lsn, handle_write);
            }
            else
            {
                for (auto wr = obj; wr; wr = heap->prev(wr))
                {
                    handle_write(wr, false);
                }
            }
            if (writes.size())
            {
                for (size_t i = writes.size(); i > 0; i--)
                {
                    fill_old_journal_entry(heap, writes[i-1]);
                }
                writes.clear();
            }
        },
        [&](uint64_t block_num, clean_disk_entry *entry, uint8_t *bitmap)
        {
            auto remap_it = data_remap.find(block_num);
            if (remap_it != data_remap.end())
@@ -758,56 +497,26 @@ int disk_tool_t::resize_rebuild_meta()
                exit(1);
            }
            block_num += data_idx_diff;
-            if (new_meta_format == BLOCKSTORE_META_FORMAT_HEAP)
+            clean_disk_entry *new_entry = (clean_disk_entry*)(new_meta_buf + dsk.meta_block_size +
-            {
+                dsk.meta_block_size*(block_num / new_entries_per_block) +
-                // Old -> New
+                new_clean_entry_size*(block_num % new_entries_per_block));
-                auto big_entry_size = sizeof(heap_big_write_t) + dsk.clean_entry_bitmap_size*2 +
+            new_entry->oid = entry->oid;
-                    (!dsk.data_csum_type ? 0 : dsk.data_block_size/dsk.csum_block_size * (dsk.data_csum_type & 0xFF));
+            new_entry->version = entry->version;
-                if ((new_meta_pos % dsk.meta_block_size) + big_entry_size > dsk.meta_block_size)
+            if (bitmap)
-                {
+                memcpy(new_entry->bitmap, bitmap, 2*new_clean_entry_bitmap_size + new_data_csum_size);
                    new_meta_pos = (new_meta_pos % dsk.meta_block_size) + dsk.meta_block_size;
                    if (new_meta_pos >= new_meta_len)
                    {
                        fprintf(stderr, "New metadata doesn't fit into the provided area\n");
                        exit(1);
                    }
                }
                heap_entry_t *wr = (heap_entry_t*)(new_meta_buf + new_meta_pos);
                wr->size = big_entry_size;
                wr->entry_type = BS_HEAP_BIG_WRITE|BS_HEAP_STABLE;
                wr->inode = entry->oid.inode;
                wr->stripe = entry->oid.stripe;
                wr->version = entry->version;
                wr->big().block_num = block_num;
                wr->lsn = ++next_lsn;
                if (bitmap)
                {
                    memcpy(((uint8_t*)wr) + sizeof(heap_big_write_t), bitmap, new_clean_entry_bitmap_size);
                    memcpy(((uint8_t*)wr) + sizeof(heap_big_write_t) + new_clean_entry_bitmap_size, bitmap+new_clean_entry_bitmap_size, new_clean_entry_bitmap_size);
                    memcpy(((uint8_t*)wr) + sizeof(heap_big_write_t) + 2*new_clean_entry_bitmap_size, bitmap+2*new_clean_entry_bitmap_size, new_data_csum_size);
                }
                wr->crc32c = wr->calc_crc32c();
                new_meta_pos += wr->size;
            }
            else
-            {
+                memset(new_entry->bitmap, 0xff, 2*new_clean_entry_bitmap_size);
-                // Old -> Old
+            uint32_t *new_entry_csum = (uint32_t*)(((uint8_t*)new_entry) + new_clean_entry_size - 4);
-                clean_disk_entry *new_entry = (clean_disk_entry*)(new_meta_buf + dsk.meta_block_size +
+            *new_entry_csum = crc32c(0, new_entry, new_clean_entry_size - 4);
-                    dsk.meta_block_size*(block_num / new_entries_per_block) +
+        }
                    new_clean_entry_size*(block_num % new_entries_per_block));
                new_entry->oid = entry->oid;
                new_entry->version = entry->version;
                if (bitmap)
                    memcpy(new_entry->bitmap, bitmap, 2*new_clean_entry_bitmap_size + new_data_csum_size);
                else
                    memset(new_entry->bitmap, 0xff, 2*new_clean_entry_bitmap_size);
                uint32_t *new_entry_csum = (uint32_t*)(((uint8_t*)new_entry) + new_clean_entry_size - 4);
                *new_entry_csum = crc32c(0, new_entry, new_clean_entry_size - 4);
            }
        },
        true, true
    );
-    return r;
+    if (r != 0)
    {
        free(new_meta_buf);
        new_meta_buf = NULL;
        return r;
    }
    return 0;
 }
 int disk_tool_t::resize_write_new_meta()
@@ -823,24 +532,7 @@ int disk_tool_t::resize_write_new_meta()
    fsync(new_meta_fd);
    close(new_meta_fd);
    new_meta_fd = -1;
    free(new_meta_buf);
    new_meta_buf = NULL;
    return 0;
 }
 void disk_tool_t::free_new_meta()
 {
    if ((uint8_t*)new_meta_hdr != new_meta_buf)
    {
        free(new_meta_hdr);
        new_meta_hdr = NULL;
    }
    if (new_meta_buf)
    {
        free(new_meta_buf);
        new_meta_buf = NULL;
    }
    if (new_journal_buf)
    {
        free(new_journal_buf);
        new_journal_buf = NULL;
    }
 }
@@ -8,7 +8,6 @@
 int disk_tool_t::resize_data(std::string device)
 {
    parse_meta_reserve();
    if (options.find("move_journal") == options.end() &&
        options.find("move_data") == options.end() &&
        options.find("journal_size") == options.end() &&
@@ -30,7 +29,7 @@ int disk_tool_t::resize_data(std::string device)
        dsk.open_data();
        dsk.open_meta();
        dsk.open_journal();
-        dsk.calc_lengths();
+        dsk.calc_lengths(true);
    }
    catch (std::exception & e)
    {
@@ -83,11 +82,8 @@ int disk_tool_t::resize_data(std::string device)
    auto new_meta_device = move_options.find("new_meta_device") != move_options.end()
        ? move_options["new_meta_device"] : dsk.meta_device;
    // Calculate new data & meta offsets
    if (!new_meta_len)
        new_meta_len = (dsk.meta_format == BLOCKSTORE_META_FORMAT_HEAP ? dsk.meta_area_size : dsk.min_meta_len);
    move_options["new_meta_len"] = std::to_string(new_meta_len);
    new_data_offset = 4096 + (new_journal_device == dsk.data_device ? new_journal_len : 0) +
-        (new_meta_device == dsk.data_device ? new_meta_len : 0);
+        (new_meta_device == dsk.data_device ? dsk.meta_len : 0);
    new_data_offset += ((dsk.data_offset-new_data_offset) % dsk.data_block_size);
    if (new_data_offset != dsk.data_offset)
        move_options["new_data_offset"] = std::to_string(new_data_offset);
@@ -240,7 +236,7 @@ int disk_tool_t::resize_parse_move_meta(std::map<std::string, std::string> & mov
        auto new_journal_device = move_options.find("new_journal_device") != move_options.end()
            ? move_options["new_journal_device"] : dsk.journal_device;
        move_options["new_meta_device"] = dsk.data_device;
-        move_options["new_meta_len"] = std::to_string(new_meta_len);
+        move_options["new_meta_len"] = std::to_string(dsk.meta_len);
    }
    else
    {
@@ -250,6 +246,7 @@ int disk_tool_t::resize_parse_move_meta(std::map<std::string, std::string> & mov
        std::string parent_dev = get_parent_device(real_dev);
        if (parent_dev == "")
            return 1;
        uint64_t new_meta_len = 0;
        if (parent_dev == real_dev)
        {
            // whole disk - create partition
@@ -263,7 +260,7 @@ int disk_tool_t::resize_parse_move_meta(std::map<std::string, std::string> & mov
                fprintf(stderr, "metadata is already on a partition of %s\n", options["move_meta"].c_str());
                return 0;
            }
-            new_meta_len = ((dsk.meta_area_size+1024*1024-1)/1024/1024)*1024*1024;
+            new_meta_len = ((dsk.meta_len+1024*1024-1)/1024/1024)*1024*1024;
            if (!dry_run)
            {
                auto devinfos = collect_devices({ real_dev });
@@ -7,7 +7,6 @@
 #include "rw_blocking.h"
 #include "str_util.h"
 #include "json_util.h"
 #include "malloc_or_die.h"
 struct __attribute__((__packed__)) vitastor_disk_superblock_t
 {
@@ -367,19 +366,6 @@ int disk_tool_t::pre_exec_osd(std::string device)
        {
            return 1;
        }
        if (sb["params"]["atomic_write_size"].uint64_value() > 4096 &&
            sb["params"]["use_atomic_flag"].bool_value())
        {
            uint64_t atomic_write_size = get_atomic_write_size(sb["real_data_device"].string_value());
            if (atomic_write_size < sb["params"]["atomic_write_size"].uint64_value())
            {
                fprintf(stderr, "Atomic write size is set to %ju in the OSD superblock but data device %s only supports %ju."
                    " Did you enable IOMMU? Linux has a hardcoded max_hw_sectors_kb value for NVMe drives.\n",
                    sb["params"]["atomic_write_size"].uint64_value(),
                    sb["real_data_device"].string_value().c_str(), atomic_write_size);
                return 1;
            }
        }
        if (json_is_true(sb["params"]["disable_meta_fsync"]) &&
            sb["real_meta_device"].string_value() != "" && sb["real_meta_device"] != sb["real_data_device"] &&
            check_disabled_cache(sb["real_meta_device"].string_value()) != 0)
@@ -116,8 +116,9 @@ int disk_tool_t::upgrade_simple_unit(std::string unit)
                fprintf(stderr, "Error: %s\n", e.what());
                return 1;
            }
-            if (m_is_d && m_o < d_o && d_o-m_o < dsk.min_meta_len)
+            options.erase("meta_format");
-                d_o += ((dsk.min_meta_len - (d_o-m_o) + blk-1) / blk) * blk;
+            if (m_is_d && m_o < d_o && d_o-m_o < dsk.meta_len)
                d_o += ((dsk.meta_len - (d_o-m_o) + blk-1) / blk) * blk;
        }
        resize["new_data_offset"] = d_o;
        resize["new_meta_offset"] = m_o;
@@ -2,13 +2,11 @@
 // License: VNPL-1.1 (see README.md for details)
 #include <sys/wait.h>
 #include <sys/ioctl.h>
 #include <dirent.h>
 #include "disk_tool.h"
 #include "rw_blocking.h"
 #include "str_util.h"
 #include "malloc_or_die.h"
 uint64_t sscanf_json(const char *fmt, const json11::Json & str)
 {
@@ -56,16 +54,8 @@ static int check_queue_cache(std::string dev, std::string parent_dev)
    return trim(r) == "write through" ? 0 : -1;
 }
 uint64_t get_atomic_write_size(const std::string & dev)
 {
    auto parent_dev = get_parent_device(dev);
    if (parent_dev == "")
        return 0;
    return stoull_full(trim(read_file("/sys/block/"+parent_dev.substr(5)+"/queue/atomic_write_max_bytes")));
 }
 // returns 1 = warning, -1 = error, 0 = success
-int disable_cache(const std::string & dev)
+int disable_cache(std::string dev)
 {
    auto parent_dev = get_parent_device(dev);
    if (parent_dev == "")
@@ -97,7 +97,6 @@ void kv_cli_t::parse_args(int narg, const char *args[])
                "  dump [<start> [end]]\n"
                "  dumpjson [<start> [end]]\n"
                "  loadjson\n"
                "  rescue\n"
                "\n"
                "<IMAGE> should be the name of Vitastor image with the DB.\n"
                "Without <COMMAND>, you get an interactive DB shell.\n"
@@ -299,50 +298,6 @@ struct kv_cli_list_t
    int n = 0;
    std::function<void(int)> cb;
    void handle_key(int res, const std::string & key, const std::string & value)
    {
        if (res < 0)
        {
            if (res != -ENOENT)
                fprintf(stderr, "Error: %s (code %d)\n", strerror(-res), res);
            if (format == 2)
                write("\n}\n");
            if (handle)
                db->list_close(handle);
            flush();
            cb(res == -ENOENT ? 0 : res);
            delete this;
        }
        else
        {
            if (format == 2)
            {
                write(n ? ",\n  " : "{\n  ");
                write(addslashes(key));
                write(": ");
                write(addslashes(value));
            }
            else if (format == 1)
            {
                write("set ");
                write(auto_addslashes(key));
                write(" ");
                write(value);
                write("\n");
            }
            else
            {
                write(key);
                write(" = ");
                write(value);
                write("\n");
            }
            n++;
            if (handle)
                db->list_next(handle, NULL);
        }
    }
    void write(const std::string & str)
    {
        if (buf.capacity() < KV_LIST_BUF_SIZE)
@@ -361,7 +316,6 @@ struct kv_cli_list_t
            if (res > 0)
                done += res;
        }
        buf.clear();
    }
 };
@@ -670,18 +624,44 @@ void kv_cli_t::handle_cmd(const std::vector<std::string> & cmd, std::function<vo
        lst->cb = std::move(cb);
        db->list_next(lst->handle, [lst](int res, const std::string & key, const std::string & value)
        {
-            lst->handle_key(res, key, value);
+            if (res < 0)
-        });
+            {
-    }
+                if (res != -ENOENT)
-    else if (opname == "rescue")
+                    fprintf(stderr, "Error: %s (code %d)\n", strerror(-res), res);
-    {
+                if (lst->format == 2)
-        kv_cli_list_t *lst = new kv_cli_list_t;
+                    lst->write("\n}\n");
-        lst->db = db;
+                lst->flush();
-        lst->format = 2;
+                lst->db->list_close(lst->handle);
-        lst->cb = std::move(cb);
+                lst->cb(res == -ENOENT ? 0 : res);
-        db->rescue([lst](int res, const std::string & key, const std::string & value)
+                delete lst;
-        {
+            }
-            lst->handle_key(res, key, value);
+            else
            {
                if (lst->format == 2)
                {
                    lst->write(lst->n ? ",\n  " : "{\n  ");
                    lst->write(addslashes(key));
                    lst->write(": ");
                    lst->write(addslashes(value));
                }
                else if (lst->format == 1)
                {
                    lst->write("set ");
                    lst->write(auto_addslashes(key));
                    lst->write(" ");
                    lst->write(value);
                    lst->write("\n");
                }
                else
                {
                    lst->write(key);
                    lst->write(" = ");
                    lst->write(value);
                    lst->write("\n");
                }
                lst->n++;
                lst->db->list_next(lst->handle, NULL);
            }
        });
    }
    else if (opname == "loadjson")
@@ -96,7 +96,7 @@ struct kv_block_t
    void set_data_size();
    static int kv_size(const std::string & key, const std::string & value);
-    int parse(uint64_t offset, uint8_t *data, int size, bool allow_empty = false);
+    int parse(uint64_t offset, uint8_t *data, int size);
    bool serialize(uint8_t *data, int size);
    void apply_change();
    void cancel_change();
@@ -139,6 +139,7 @@ struct kv_db_t
    uint64_t next_free = 0;
    uint32_t kv_block_size = 0;
    uint32_t ino_block_size = 0;
    bool immediate_commit = false;
    uint64_t memory_limit = 128*1024*1024;
    uint64_t evict_unused_age = 1000;
    uint64_t evict_max_misses = 10;
@@ -172,7 +173,6 @@ struct kv_db_t
    void open(inode_t inode_id, json11::Json cfg, std::function<void(int)> cb);
    void set_config(json11::Json cfg);
    void close(std::function<void()> cb);
    void rescue(std::function<void(int res, const std::string & key, const std::string & value)> cb);
    void find_size(uint64_t min, uint64_t max, int phase, std::function<void(int, uint64_t)> cb);
    void run_continue_update(uint64_t offset);
@@ -243,13 +243,13 @@ static std::string read_string(uint8_t *data, int size, int *pos)
    return key;
 }
-int kv_block_t::parse(uint64_t offset, uint8_t *data, int size, bool allow_empty)
+int kv_block_t::parse(uint64_t offset, uint8_t *data, int size)
 {
    kv_stored_block_t *blk = (kv_stored_block_t *)data;
    if (blk->magic == 0 || blk->type == KV_EMPTY)
    {
        // empty block
-        if (!allow_empty)
+        if (offset != 0)
            fprintf(stderr, "K/V: Block %ju is %s\n", offset, blk->magic == 0 ? "empty" : "cleared");
        return -ENOTBLK;
    }
@@ -389,10 +389,6 @@ bool kv_block_t::serialize(uint8_t *buf, int size)
            return false;
        blk->items++;
    }
    if (pos < size)
    {
        memset(buf+pos, 0, size-pos);
    }
    return true;
 }
@@ -528,6 +524,7 @@ void kv_db_t::open(inode_t inode_id, json11::Json cfg, std::function<void(int)>
        return;
    }
    this->inode_id = inode_id;
    this->immediate_commit = cli->get_immediate_commit(inode_id);
    this->ino_block_size = pool_cfg.data_block_size * pg_data_size;
    this->kv_block_size = kv_block_size;
    this->next_free = 0;
@@ -545,127 +542,6 @@ void kv_db_t::open(inode_t inode_id, json11::Json cfg, std::function<void(int)>
    });
 }
 struct kv_rescue_t
 {
    kv_db_t *db = NULL;
    uint64_t size = 0;
    uint64_t pos = 0;
    uint64_t cur_size = 0;
    uint64_t cur_offset = 0;
    int state = 0;
    kv_block_t blk;
    std::vector<uint8_t> buf;
    std::function<void(int res, const std::string & key, const std::string & value)> cb;
    void finish(int retval)
    {
        auto cb = std::move(this->cb);
        cb(retval, "", "");
        delete this;
    }
    void send_read()
    {
        if (pos >= size)
        {
            finish(-ENOENT);
            return;
        }
        if (!buf.size())
        {
            buf.resize(1048576);
        }
        cluster_op_t *op = new cluster_op_t;
        op->opcode = OSD_OP_READ;
        op->inode = db->inode_id;
        op->offset = pos;
        cur_size = op->len = pos+buf.size() < size ? buf.size() : size-pos;
        op->iov.push_back(buf.data(), cur_size);
        op->callback = [=](cluster_op_t *op)
        {
            if (op->retval != op->len)
            {
                // error
                finish(op->retval >= 0 ? -EIO : op->retval);
                return;
            }
            state = 2;
            cur_offset = 0;
            run();
            delete op;
        };
        db->cli->execute(op);
        state = 1;
    }
    void parse_block()
    {
        if (cur_offset < cur_size)
        {
            blk = {};
            int err = blk.parse(pos+cur_offset, buf.data()+cur_offset, db->kv_block_size, true);
            if (err != 0)
            {
            }
            else if (blk.type == KV_LEAF || blk.type == KV_LEAF_SPLIT)
            {
                for (auto it = blk.data.begin(); it != blk.data.end(); it++)
                {
                    cb(0, it->first, it->second);
                }
            }
            cur_offset += db->kv_block_size;
        }
        else
        {
            pos += cur_size;
            state = 0;
        }
    }
    void run()
    {
        while (true)
        {
            if (state == 0)
            {
                send_read();
                return;
            }
            else if (state == 1)
            {
                return;
            }
            else if (state == 2)
            {
                parse_block();
            }
        }
    }
 };
 void kv_db_t::rescue(std::function<void(int res, const std::string & key, const std::string & value)> cb)
 {
    if (!inode_id || closing)
    {
        cb(-EINVAL, "", "");
        return;
    }
    find_size(0, 0, 1, [=](int res, uint64_t size)
    {
        if (res < 0)
        {
            cb(res, "", "");
            return;
        }
        kv_rescue_t *st = new kv_rescue_t();
        st->db = this;
        st->size = size;
        st->cb = cb;
        st->run();
    });
 }
 void kv_db_t::set_config(json11::Json cfg)
 {
    this->memory_limit = cfg["kv_memory_limit"].is_null() ? 128*1024*1024 : cfg["kv_memory_limit"].uint64_value();
@@ -687,6 +563,7 @@ void kv_db_t::close(std::function<void()> cb)
        next_free = 0;
        kv_block_size = 0;
        ino_block_size = 0;
        immediate_commit = false;
        block_cache.clear();
        known_versions.clear();
        cb();
@@ -1062,7 +939,7 @@ static void get_block(kv_db_t *db, uint64_t offset, int cur_level, int recheck_p
                del_block_level(db, blk);
                *blk = {};
            }
-            int err = blk->parse(op->offset, (uint8_t*)op->iov.buf[0].iov_base, op->len, op->offset == 0);
+            int err = blk->parse(op->offset, (uint8_t*)op->iov.buf[0].iov_base, op->len);
            if (err == 0)
            {
                blk->level = cur_level;
@@ -1125,8 +1002,6 @@ kv_op_t::~kv_op_t()
    {
        done = true;
        db->active_ops--;
        if (!db->active_ops && db->closing)
            db->close(db->on_close);
    }
 }
@@ -1397,7 +1272,22 @@ static void write_block(kv_db_t *db, kv_block_t *blk, std::function<void(int)> c
            }
        }
        delete op;
-        cb(res);
+        if (res < 0 || db->immediate_commit)
        {
            cb(res);
        }
        else
        {
            op = new cluster_op_t;
            op->opcode = OSD_OP_SYNC;
            op->callback = [cb](cluster_op_t *op)
            {
                auto res = op->retval;
                delete op;
                cb(res);
            };
            db->cli->execute(op);
        }
    };
    db->cli->execute(op);
 }
@@ -1772,7 +1662,6 @@ void kv_op_t::update_block(int path_pos, bool is_delete, const std::string & key
        {
            if (res < 0)
            {
                blk->cancel_change();
                auto blk_offset = blk->offset;
                del_block_level(db, blk);
                db->block_cache.erase(blk_offset);
@@ -1905,7 +1794,6 @@ void kv_op_t::update_block(int path_pos, bool is_delete, const std::string & key
            {
                if (write_res < 0)
                {
                    blk->cancel_change();
                    auto blk_offset = blk->offset;
                    del_block_level(db, blk);
                    db->block_cache.erase(blk_offset);
@@ -2093,11 +1981,6 @@ void vitastorkv_dbw_t::close(std::function<void()> cb)
    db->close(cb);
 }
 void vitastorkv_dbw_t::rescue(std::function<void(int res, const std::string & key, const std::string & value)> cb)
 {
    db->rescue(cb);
 }
 void vitastorkv_dbw_t::get(const std::string & key, std::function<void(int res, const std::string & value)> cb, bool cached)
 {
    auto *op = new kv_op_t;
@@ -82,7 +82,6 @@ public:
    uint64_t total_prob = 0;
    uint64_t ops_sent = 0, ops_done = 0;
    int stat_timer_id = -1;
    int run_timer_id = -1;
    int in_progress = 0;
    bool reopening = false;
    std::set<kv_test_listing_t*> listings;
@@ -300,11 +299,9 @@ void kv_test_t::run(json11::Json cfg)
    ringloop->register_consumer(&consumer);
    if (print_stats_interval)
        stat_timer_id = epmgr->tfd->set_timer(print_stats_interval*1000, true, [this](int) { print_stats(prev_stat, prev_stat_time); });
    if (runtime_sec)
        run_timer_id = epmgr->tfd->set_timer(runtime_sec*1000, false, [this](int) { run_timer_id = -1; op_count = 0; });
    clock_gettime(CLOCK_REALTIME, &start_stat_time);
    prev_stat_time = start_stat_time;
-    while (!finished || in_progress > 0)
+    while (!finished)
    {
        ringloop->loop();
        if (!finished)
@@ -312,8 +309,6 @@ void kv_test_t::run(json11::Json cfg)
    }
    if (stat_timer_id >= 0)
        epmgr->tfd->clear_timer(stat_timer_id);
    if (run_timer_id >= 0)
        epmgr->tfd->clear_timer(run_timer_id);
    ringloop->unregister_consumer(&consumer);
    // Print total stats
    print_total_stats();
@@ -28,7 +28,6 @@ struct __attribute__((visibility("default"))) vitastorkv_dbw_t
    void open(uint64_t inode_id, std::map<std::string, std::string> cfg, std::function<void(int)> cb);
    void set_config(std::map<std::string, std::string> cfg);
    void close(std::function<void()> cb);
    void rescue(std::function<void(int res, const std::string & key, const std::string & value)> cb);
    uint64_t get_size();
@@ -166,20 +166,7 @@ int kv_nfs3_access_proc(void *opaque, rpc_op_t *rop)
            fprintf(stderr, "[%d] ACCESS %ju -> %s\n", self->nfs_fd, ino, value.c_str());
        if (res < 0)
        {
-            // Broken inode (non-existing), allow only root access
+            *reply = (ACCESS3res){ .status = vitastor_nfs_map_err(-res) };
            if (!rop->auth_sys.uid)
            {
                *reply = (ACCESS3res){
                    .status = NFS3_OK,
                    .resok = (ACCESS3resok){
                        .access = args->access,
                    },
                };
            }
            else
            {
                *reply = (ACCESS3res){ .status = vitastor_nfs_map_err(-res) };
            }
        }
        else
        {
--- a/Show More
+++ b/Show More
`@@ -1,4 +1,4 @@`
	`VITASTOR_VERSION ?= v3.0.0`	`VITASTOR_VERSION ?= v2.4.3`

	`all: build push`	`all: build push`
`@@ -1,4 +1,4 @@`
	`vitastor (3.0.0-1) unstable; urgency=medium`	`vitastor (2.4.3-1) unstable; urgency=medium`

	`* Bugfixes`	`* Bugfixes`