thebestvibedev/tromcho.net
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix removal and overwriting of totaled objects (objects with all corrupted copies)

Browse Source
This commit is contained in:
Vitaliy Filippov
2026-02-11 01:24:58 +03:00
parent 9ec18f0aa1
commit 4d75c6c8f9
7 changed files with 176 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitea/workflows/test.yml
+72
View File
@@ -1206,6 +1206,78 @@ jobs:
echo ""
done
test_checksum:
runs-on: ubuntu-latest
needs: build
container: ${{env.TEST_IMAGE}}:${{github.sha}}
steps:
- name: Run test
id: test
timeout-minutes: 3
run: /root/vitastor/tests/test_checksum.sh
- name: Print logs
if: always() && steps.test.outcome == 'failure'
run: |
for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
echo "-------- $i --------"
cat $i
echo ""
done
test_old_checksum:
runs-on: ubuntu-latest
needs: build
container: ${{env.TEST_IMAGE}}:${{github.sha}}
steps:
- name: Run test
id: test
timeout-minutes: 3
run: OLD=1 /root/vitastor/tests/test_checksum.sh
- name: Print logs
if: always() && steps.test.outcome == 'failure'
run: |
for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
echo "-------- $i --------"
cat $i
echo ""
done
test_corrupt_all:
runs-on: ubuntu-latest
needs: build
container: ${{env.TEST_IMAGE}}:${{github.sha}}
steps:
- name: Run test
id: test
timeout-minutes: 3
run: /root/vitastor/tests/test_corrupt_all.sh
- name: Print logs
if: always() && steps.test.outcome == 'failure'
run: |
for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
echo "-------- $i --------"
cat $i
echo ""
done
test_old_corrupt_all:
runs-on: ubuntu-latest
needs: build
container: ${{env.TEST_IMAGE}}:${{github.sha}}
steps:
- name: Run test
id: test
timeout-minutes: 3
run: OLD=1 /root/vitastor/tests/test_corrupt_all.sh
- name: Print logs
if: always() && steps.test.outcome == 'failure'
run: |
for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
echo "-------- $i --------"
cat $i
echo ""
done
test_reweight_half:
runs-on: ubuntu-latest
needs: build
src/osd/osd_peering_pg.cpp
+16 -8
View File
@@ -334,36 +334,44 @@ pg_osd_set_state_t* pg_t::add_object_to_state(const object_id oid, const uint64_
if (it == state_dict.end())
{
std::vector<osd_num_t> read_target;
bool found = false;
uint32_t bad_mask = (LOC_OUTDATED | LOC_CORRUPTED);
retry:
if (scheme == POOL_SCHEME_REPLICATED)
{
for (auto & o: osd_set)
{
if (!(o.loc_bad & (LOC_OUTDATED | LOC_CORRUPTED)))
if (!(o.loc_bad & bad_mask))
{
read_target.push_back(o.osd_num);
found = true;
}
}
while (read_target.size() < pg_size)
if (read_target.size() < pg_size)
{
// FIXME: This is because we then use .data() and assume it's at least <pg_size> long
read_target.push_back(0);
read_target.resize(pg_size);
}
}
else
{
read_target.resize(pg_size);
for (int i = 0; i < pg_size; i++)
{
read_target[i] = 0;
}
for (auto & o: osd_set)
{
if (!(o.loc_bad & (LOC_OUTDATED | LOC_CORRUPTED)))
if (!(o.loc_bad & bad_mask))
{
read_target[o.role] = o.osd_num;
found = true;
}
}
}
if (!found && (bad_mask & LOC_CORRUPTED))
{
// Allow to try reading corrupted copies in rare cases when the object is corrupted on all OSDs
bad_mask = LOC_OUTDATED;
read_target.clear();
goto retry;
}
state_dict[osd_set] = {
.read_target = read_target,
.osd_set = osd_set,
src/osd/osd_primary.cpp
+13 -7
View File
@@ -271,7 +271,6 @@ resume_0:
}
cur_op->buf = alloc_read_buffer(op_data->stripes, pg ? pg->pg_data_size : 1, 0);
submit_primary_subops(SUBMIT_RMW_READ, op_data->target_ver, op_data->prev_set, cur_op);
op_data->st = 1;
}
else
{
@@ -284,11 +283,14 @@ resume_0:
op_data->degraded = 1;
cur_op->buf = alloc_read_buffer(op_data->stripes, pg->pg_size, 0);
submit_primary_subops(SUBMIT_RMW_READ, op_data->target_ver, op_data->prev_set, cur_op);
op_data->st = 1;
}
}
resume_1:
return;
if (op_data->n_subops > 0)
{
op_data->st = 1;
return;
}
resume_2:
if (op_data->errors > 0)
{
@@ -296,8 +298,9 @@ resume_2:
{
// I/O or checksum error
// FIXME: ref = true ideally... because new_state != state is not necessarily true if it's freed and recreated
op_data->object_state = mark_object_corrupted(*pg, op_data->oid, op_data->object_state, op_data->stripes, false);
goto resume_0;
auto new_object_state = mark_object_corrupted(*pg, op_data->oid, op_data->object_state, op_data->stripes, false);
if (new_object_state != op_data->object_state)
goto resume_0;
}
finish_op(cur_op, op_data->errcode);
return;
@@ -736,8 +739,11 @@ resume_1:
submit_primary_subops(SUBMIT_RMW_READ, UINT64_MAX, op_data->prev_set, cur_op);
op_data->prev_set = NULL;
resume_2:
op_data->st = 2;
return;
if (op_data->n_subops > 0)
{
op_data->st = 2;
return;
}
resume_3:
if (op_data->errors > 0)
{
src/osd/osd_primary_subops.cpp
+8 -2
View File
@@ -134,11 +134,17 @@ void osd_t::submit_primary_subops(int submit_type, uint64_t op_version, const ui
n_subops = 1;
else
zero_read = -1;
osd_op_t *subops = new osd_op_t[n_subops];
op_data->fact_ver = 0;
op_data->done = op_data->errors = op_data->drops = op_data->errcode = 0;
op_data->n_subops = n_subops;
op_data->subops = subops;
if (!n_subops)
{
op_data->errcode = -EIO;
op_data->subops = NULL;
op_data->errors = 1;
return;
}
op_data->subops = new osd_op_t[n_subops];
int sent = submit_primary_subop_batch(submit_type, op_data->oid.inode, op_version, op_data->stripes, osd_set, cur_op, 0, zero_read);
assert(sent == n_subops);
}
src/osd/osd_primary_write.cpp
+16 -42
View File
@@ -108,57 +108,31 @@ retry_1:
}
}
// Read required blocks
{
if (op_data->object_state && (op_data->object_state->state & OBJ_INCOMPLETE))
{
// Allow to read version number (just version number!) from corrupted chunks
// to allow full overwrite of a corrupted object
bool found = false;
for (int role = 0; role < pg.pg_size; role++)
{
if (op_data->prev_set[role] != 0 || op_data->stripes[role].read_end > op_data->stripes[role].read_start)
{
found = true;
break;
}
}
if (!found)
{
osd_num_t corrupted_target[pg.pg_size];
for (int role = 0; role < pg.pg_size; role++)
{
corrupted_target[role] = 0;
}
for (auto & loc: op_data->object_state->osd_set)
{
if (!(loc.loc_bad & LOC_OUTDATED) && !corrupted_target[loc.role])
{
corrupted_target[loc.role] = loc.osd_num;
}
}
submit_primary_subops(SUBMIT_RMW_READ, UINT64_MAX, corrupted_target, cur_op);
goto resume_2;
}
}
submit_primary_subops(SUBMIT_RMW_READ, UINT64_MAX, op_data->prev_set, cur_op);
}
submit_primary_subops(SUBMIT_RMW_READ, UINT64_MAX, op_data->prev_set, cur_op);
resume_2:
op_data->st = 2;
return;
if (op_data->n_subops > 0)
{
op_data->st = 2;
return;
}
resume_3:
if (op_data->errors > 0)
{
if (op_data->errcode == -EIO || op_data->errcode == -EDOM)
{
// Mark object corrupted and retry
op_data->object_state = mark_object_corrupted(pg, op_data->oid, op_data->object_state, op_data->stripes, true);
op_data->prev_set = op_data->object_state ? op_data->object_state->read_target.data() : pg.cur_set.data();
if (cur_op->rmw_buf)
pg_osd_set_state_t *new_object_state = mark_object_corrupted(pg, op_data->oid, op_data->object_state, op_data->stripes, true);
if (new_object_state != op_data->object_state)
{
free(cur_op->rmw_buf);
cur_op->rmw_buf = NULL;
op_data->object_state = new_object_state;
op_data->prev_set = op_data->object_state ? op_data->object_state->read_target.data() : pg.cur_set.data();
if (cur_op->rmw_buf)
{
free(cur_op->rmw_buf);
cur_op->rmw_buf = NULL;
}
goto retry_1;
}
goto retry_1;
}
deref_object_state(pg, &op_data->object_state, true);
pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->errcode);
tests/run_tests.sh
+5
View File
@@ -96,6 +96,11 @@ TEST_NAME=local_read POOLCFG='"local_reads":"random",' ./test_heal.sh
SCHEME=ec ./test_heal.sh
ANTIETCD=1 ./test_heal.sh
./test_checksum.sh
OLD=1 ./test_checksum.sh
./test_corrupt_all.sh
OLD=1 ./test_corrupt_all.sh
./test_reweight_half.sh
./test_snapshot_pool2.sh
./test_snapshot_read_bitmap.sh
tests/test_corrupt_all.sh
Executable
+46
View File
@@ -0,0 +1,46 @@
#!/bin/bash -ex
SCHEME=replicated
PG_SIZE=2
OSD_COUNT=2
IMG_SIZE=128
OSD_ARGS="--data_csum_type crc32c --csum_block_size 4k --inmemory_journal false --journal_trim_interval $((IMG_SIZE*8)) $OSD_ARGS"
OFFSET_ARGS="--data_csum_type crc32c --csum_block_size 4k $OFFSET_ARGS"
GLOBAL_CONFIG=',"client_eio_retry_interval":0'
. `dirname $0`/run_3osds.sh
check_qemu
$ETCDCTL put /vitastor/config/inode/1/1 '{"name":"testimg","size":'$((IMG_SIZE*1024*1024))'}'
# Write
$VITASTOR_FIO -bs=1M -direct=1 -iodepth=4 -end_fsync=1 -rw=write -image=testimg -runtime=10
#$VITASTOR_FIO -bs=4k -direct=1 -iodepth=16 -end_fsync=1 -rw=randwrite -image=testimg -number_ios=10000
# Intentionally corrupt OSD data and restart both of them
kill $OSD1_PID $OSD2_PID
data_offset=$(build/src/disk_tool/vitastor-disk simple-offsets ./testdata/bin/test_osd1.bin $OFFSET_ARGS | grep data_offset | awk '{print $2}')
truncate -s $data_offset ./testdata/bin/test_osd1.bin
dd if=/dev/zero of=./testdata/bin/test_osd1.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
truncate -s $data_offset ./testdata/bin/test_osd2.bin
dd if=/dev/zero of=./testdata/bin/test_osd2.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
start_osd 1
start_osd 2
# Wait until start
wait_up 10
# Trigger scrub
$ETCDCTL put /vitastor/pg/history/1/1 `$ETCDCTL get --print-value-only /vitastor/pg/history/1/1 | jq -s -c '(.[0] // {}) + {"next_scrub":1}'`
# Wait for scrub to finish
wait_condition 300 "$ETCDCTL get --prefix /vitastor/pg/history/ --print-value-only | jq -s -e '([ .[] | select(.next_scrub == 0 or .next_scrub == null) ] | length) == $PG_COUNT'" Scrubbing
# Verify that ALL objects are now corrupted+incomplete
$VITASTOR_CLI describe --json &>./testdata/describe.json
$VITASTOR_CLI describe --json | jq -e '[ .[] | select(.corrupted) ] | length == '$((IMG_SIZE * 8 * PG_SIZE))
# Check that we can remove or overwrite them
$VITASTOR_FIO -bs=4M -direct=1 -iodepth=1 -end_fsync=1 -rw=write -offset=$((IMG_SIZE/2))M -image=testimg -runtime=10
$VITASTOR_CLI rm-data --pool 1 --inode 1
format_green OK