diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index eeb30c26..0a78a62d 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -1206,6 +1206,78 @@ jobs: echo "" done + test_checksum: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_checksum.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_old_checksum: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: OLD=1 /root/vitastor/tests/test_checksum.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_corrupt_all: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_corrupt_all.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_old_corrupt_all: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: OLD=1 /root/vitastor/tests/test_corrupt_all.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + test_reweight_half: runs-on: ubuntu-latest needs: build diff --git a/src/osd/osd_peering_pg.cpp b/src/osd/osd_peering_pg.cpp index fc6a4c25..30d79e76 100644 --- a/src/osd/osd_peering_pg.cpp +++ b/src/osd/osd_peering_pg.cpp @@ -334,36 +334,44 @@ pg_osd_set_state_t* pg_t::add_object_to_state(const object_id oid, const uint64_ if (it == state_dict.end()) { std::vector read_target; + bool found = false; + uint32_t bad_mask = (LOC_OUTDATED | LOC_CORRUPTED); +retry: if (scheme == POOL_SCHEME_REPLICATED) { for (auto & o: osd_set) { - if (!(o.loc_bad & (LOC_OUTDATED | LOC_CORRUPTED))) + if (!(o.loc_bad & bad_mask)) { read_target.push_back(o.osd_num); + found = true; } } - while (read_target.size() < pg_size) + if (read_target.size() < pg_size) { // FIXME: This is because we then use .data() and assume it's at least long - read_target.push_back(0); + read_target.resize(pg_size); } } else { read_target.resize(pg_size); - for (int i = 0; i < pg_size; i++) - { - read_target[i] = 0; - } for (auto & o: osd_set) { - if (!(o.loc_bad & (LOC_OUTDATED | LOC_CORRUPTED))) + if (!(o.loc_bad & bad_mask)) { read_target[o.role] = o.osd_num; + found = true; } } } + if (!found && (bad_mask & LOC_CORRUPTED)) + { + // Allow to try reading corrupted copies in rare cases when the object is corrupted on all OSDs + bad_mask = LOC_OUTDATED; + read_target.clear(); + goto retry; + } state_dict[osd_set] = { .read_target = read_target, .osd_set = osd_set, diff --git a/src/osd/osd_primary.cpp b/src/osd/osd_primary.cpp index f3ed5d52..cff36938 100644 --- a/src/osd/osd_primary.cpp +++ b/src/osd/osd_primary.cpp @@ -271,7 +271,6 @@ resume_0: } cur_op->buf = alloc_read_buffer(op_data->stripes, pg ? pg->pg_data_size : 1, 0); submit_primary_subops(SUBMIT_RMW_READ, op_data->target_ver, op_data->prev_set, cur_op); - op_data->st = 1; } else { @@ -284,11 +283,14 @@ resume_0: op_data->degraded = 1; cur_op->buf = alloc_read_buffer(op_data->stripes, pg->pg_size, 0); submit_primary_subops(SUBMIT_RMW_READ, op_data->target_ver, op_data->prev_set, cur_op); - op_data->st = 1; } } resume_1: - return; + if (op_data->n_subops > 0) + { + op_data->st = 1; + return; + } resume_2: if (op_data->errors > 0) { @@ -296,8 +298,9 @@ resume_2: { // I/O or checksum error // FIXME: ref = true ideally... because new_state != state is not necessarily true if it's freed and recreated - op_data->object_state = mark_object_corrupted(*pg, op_data->oid, op_data->object_state, op_data->stripes, false); - goto resume_0; + auto new_object_state = mark_object_corrupted(*pg, op_data->oid, op_data->object_state, op_data->stripes, false); + if (new_object_state != op_data->object_state) + goto resume_0; } finish_op(cur_op, op_data->errcode); return; @@ -736,8 +739,11 @@ resume_1: submit_primary_subops(SUBMIT_RMW_READ, UINT64_MAX, op_data->prev_set, cur_op); op_data->prev_set = NULL; resume_2: - op_data->st = 2; - return; + if (op_data->n_subops > 0) + { + op_data->st = 2; + return; + } resume_3: if (op_data->errors > 0) { diff --git a/src/osd/osd_primary_subops.cpp b/src/osd/osd_primary_subops.cpp index eb7f4e83..e83876dd 100644 --- a/src/osd/osd_primary_subops.cpp +++ b/src/osd/osd_primary_subops.cpp @@ -134,11 +134,17 @@ void osd_t::submit_primary_subops(int submit_type, uint64_t op_version, const ui n_subops = 1; else zero_read = -1; - osd_op_t *subops = new osd_op_t[n_subops]; op_data->fact_ver = 0; op_data->done = op_data->errors = op_data->drops = op_data->errcode = 0; op_data->n_subops = n_subops; - op_data->subops = subops; + if (!n_subops) + { + op_data->errcode = -EIO; + op_data->subops = NULL; + op_data->errors = 1; + return; + } + op_data->subops = new osd_op_t[n_subops]; int sent = submit_primary_subop_batch(submit_type, op_data->oid.inode, op_version, op_data->stripes, osd_set, cur_op, 0, zero_read); assert(sent == n_subops); } diff --git a/src/osd/osd_primary_write.cpp b/src/osd/osd_primary_write.cpp index b75a2188..a71cbfa1 100644 --- a/src/osd/osd_primary_write.cpp +++ b/src/osd/osd_primary_write.cpp @@ -108,57 +108,31 @@ retry_1: } } // Read required blocks - { - if (op_data->object_state && (op_data->object_state->state & OBJ_INCOMPLETE)) - { - // Allow to read version number (just version number!) from corrupted chunks - // to allow full overwrite of a corrupted object - bool found = false; - for (int role = 0; role < pg.pg_size; role++) - { - if (op_data->prev_set[role] != 0 || op_data->stripes[role].read_end > op_data->stripes[role].read_start) - { - found = true; - break; - } - } - if (!found) - { - osd_num_t corrupted_target[pg.pg_size]; - for (int role = 0; role < pg.pg_size; role++) - { - corrupted_target[role] = 0; - } - for (auto & loc: op_data->object_state->osd_set) - { - if (!(loc.loc_bad & LOC_OUTDATED) && !corrupted_target[loc.role]) - { - corrupted_target[loc.role] = loc.osd_num; - } - } - submit_primary_subops(SUBMIT_RMW_READ, UINT64_MAX, corrupted_target, cur_op); - goto resume_2; - } - } - submit_primary_subops(SUBMIT_RMW_READ, UINT64_MAX, op_data->prev_set, cur_op); - } + submit_primary_subops(SUBMIT_RMW_READ, UINT64_MAX, op_data->prev_set, cur_op); resume_2: - op_data->st = 2; - return; + if (op_data->n_subops > 0) + { + op_data->st = 2; + return; + } resume_3: if (op_data->errors > 0) { if (op_data->errcode == -EIO || op_data->errcode == -EDOM) { // Mark object corrupted and retry - op_data->object_state = mark_object_corrupted(pg, op_data->oid, op_data->object_state, op_data->stripes, true); - op_data->prev_set = op_data->object_state ? op_data->object_state->read_target.data() : pg.cur_set.data(); - if (cur_op->rmw_buf) + pg_osd_set_state_t *new_object_state = mark_object_corrupted(pg, op_data->oid, op_data->object_state, op_data->stripes, true); + if (new_object_state != op_data->object_state) { - free(cur_op->rmw_buf); - cur_op->rmw_buf = NULL; + op_data->object_state = new_object_state; + op_data->prev_set = op_data->object_state ? op_data->object_state->read_target.data() : pg.cur_set.data(); + if (cur_op->rmw_buf) + { + free(cur_op->rmw_buf); + cur_op->rmw_buf = NULL; + } + goto retry_1; } - goto retry_1; } deref_object_state(pg, &op_data->object_state, true); pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->errcode); diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 1b87463e..1109aa31 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -96,6 +96,11 @@ TEST_NAME=local_read POOLCFG='"local_reads":"random",' ./test_heal.sh SCHEME=ec ./test_heal.sh ANTIETCD=1 ./test_heal.sh +./test_checksum.sh +OLD=1 ./test_checksum.sh +./test_corrupt_all.sh +OLD=1 ./test_corrupt_all.sh + ./test_reweight_half.sh ./test_snapshot_pool2.sh ./test_snapshot_read_bitmap.sh diff --git a/tests/test_corrupt_all.sh b/tests/test_corrupt_all.sh new file mode 100755 index 00000000..4968a729 --- /dev/null +++ b/tests/test_corrupt_all.sh @@ -0,0 +1,46 @@ +#!/bin/bash -ex + +SCHEME=replicated +PG_SIZE=2 +OSD_COUNT=2 +IMG_SIZE=128 +OSD_ARGS="--data_csum_type crc32c --csum_block_size 4k --inmemory_journal false --journal_trim_interval $((IMG_SIZE*8)) $OSD_ARGS" +OFFSET_ARGS="--data_csum_type crc32c --csum_block_size 4k $OFFSET_ARGS" +GLOBAL_CONFIG=',"client_eio_retry_interval":0' +. `dirname $0`/run_3osds.sh +check_qemu + +$ETCDCTL put /vitastor/config/inode/1/1 '{"name":"testimg","size":'$((IMG_SIZE*1024*1024))'}' + +# Write +$VITASTOR_FIO -bs=1M -direct=1 -iodepth=4 -end_fsync=1 -rw=write -image=testimg -runtime=10 +#$VITASTOR_FIO -bs=4k -direct=1 -iodepth=16 -end_fsync=1 -rw=randwrite -image=testimg -number_ios=10000 + +# Intentionally corrupt OSD data and restart both of them +kill $OSD1_PID $OSD2_PID +data_offset=$(build/src/disk_tool/vitastor-disk simple-offsets ./testdata/bin/test_osd1.bin $OFFSET_ARGS | grep data_offset | awk '{print $2}') +truncate -s $data_offset ./testdata/bin/test_osd1.bin +dd if=/dev/zero of=./testdata/bin/test_osd1.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1)) +truncate -s $data_offset ./testdata/bin/test_osd2.bin +dd if=/dev/zero of=./testdata/bin/test_osd2.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1)) +start_osd 1 +start_osd 2 + +# Wait until start +wait_up 10 + +# Trigger scrub +$ETCDCTL put /vitastor/pg/history/1/1 `$ETCDCTL get --print-value-only /vitastor/pg/history/1/1 | jq -s -c '(.[0] // {}) + {"next_scrub":1}'` + +# Wait for scrub to finish +wait_condition 300 "$ETCDCTL get --prefix /vitastor/pg/history/ --print-value-only | jq -s -e '([ .[] | select(.next_scrub == 0 or .next_scrub == null) ] | length) == $PG_COUNT'" Scrubbing + +# Verify that ALL objects are now corrupted+incomplete +$VITASTOR_CLI describe --json &>./testdata/describe.json +$VITASTOR_CLI describe --json | jq -e '[ .[] | select(.corrupted) ] | length == '$((IMG_SIZE * 8 * PG_SIZE)) + +# Check that we can remove or overwrite them +$VITASTOR_FIO -bs=4M -direct=1 -iodepth=1 -end_fsync=1 -rw=write -offset=$((IMG_SIZE/2))M -image=testimg -runtime=10 +$VITASTOR_CLI rm-data --pool 1 --inode 1 + +format_green OK