Experimental: Handle degraded deletions by comparing object versions with epochs

CAUTION! This version is not fool proof yet. If you purge data of an OSD by overwriting the disk with zeroes and restart it then the same data will also be removed from other replicas :-). I plan to add protection from this situation before merging it into master. The idea is to make each OSD store a random "cookie" on disk and remove itself from history automatically if the cookie doesn't match.
2023-04-29 00:21:22 +03:00
parent 5e295e346e
commit 4c9bf6727b
20 changed files with 553 additions and 105 deletions
@@ -14,6 +14,8 @@ SCHEME=ec ./test_change_pg_count.sh

 ./test_create_nomaxid.sh

+./test_degraded_delete.sh
+
 ./test_etcd_fail.sh

 ./test_failure_domain.sh
@@ -0,0 +1,131 @@
+#!/bin/bash -ex
+
+# Run 3 OSDs
+
+. `dirname $0`/run_3osds.sh
+
+# Write inodes 1 and 2
+
+LD_PRELOAD="build/src/libfio_vitastor.so" \
+    fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=1M -direct=1 -iodepth=4 \
+        -rw=write -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -runtime=10
+
+LD_PRELOAD="build/src/libfio_vitastor.so" \
+    fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=1M -direct=1 -iodepth=4 \
+        -rw=write -etcd=$ETCD_URL -pool=1 -inode=2 -size=128M -runtime=10
+
+LD_PRELOAD="build/src/libfio_vitastor.so" \
+    fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -iodepth=16 \
+        -rw=randwrite -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -runtime=10 &>/dev/null &
+
+sleep 5
+
+# Stop OSD 1
+
+kill -INT $OSD1_PID
+sleep 2
+
+# Remove inode 2
+
+build/src/vitastor-cli rm-data --etcd_address $ETCD_URL --pool 1 --inode 2
+
+# Run 3 more OSDs and move PG to 4,5,6
+
+for i in $(seq 4 6); do
+    dd if=/dev/zero of=./testdata/test_osd$i.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
+    build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) &>./testdata/osd$i.log &
+    eval OSD${i}_PID=$!
+done
+
+$ETCDCTL put /vitastor/config/osd/1 '{"reweight":0}'
+$ETCDCTL put /vitastor/config/osd/2 '{"reweight":0}'
+$ETCDCTL put /vitastor/config/osd/3 '{"reweight":0}'
+
+# Wait for rebalance to finish
+
+wait_finish_rebalance()
+{
+    local sec=$1
+    local st=$2
+    local i=0
+    while [[ $i -lt $sec ]]; do
+        if $ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e \
+            '([ .[] | select(.state == ['$st'] and (.peers | contains([1]) | not) and (.peers | contains([2,3]) | not)) ] | length) == '$PG_COUNT; then
+            break
+        fi
+        sleep 1
+        i=$((i+1))
+        if [ $i -eq $sec ]; then
+            format_error "Rebalance couldn't finish in $sec seconds"
+        fi
+    done
+}
+
+wait_finish_rebalance 60 '"active","left_on_dead"'
+
+# Stop OSD 2,3
+
+kill -INT $OSD2_PID
+kill -INT $OSD3_PID
+sleep 2
+
+# Verify that PGs are still active
+
+if ! ($ETCDCTL get /vitastor/pg/state/1/ --prefix --print-value-only | jq -s -e '[ .[] | select(.state == ["active","left_on_dead"]) ] | length == '$PG_COUNT); then
+    format_error "FAILED: $PG_COUNT PG(s) NOT UP"
+fi
+
+# Start OSD 1
+
+build/src/vitastor-osd --osd_num 1 --bind_address 127.0.0.1 $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd1.bin 2>/dev/null) &>./testdata/osd1.log &
+OSD1_PID=$!
+
+# Verify that inode 2 is removed and inode 1 is in place
+
+wait_repeer_1()
+{
+    local sec=$1
+    local i=0
+    while [[ $i -lt $sec ]]; do
+        if grep -q 'Repeer because of OSD 1' testdata/osd4.log testdata/osd5.log testdata/osd6.log; then
+            break
+        fi
+        sleep 1
+        i=$((i+1))
+        if [ $i -eq $sec ]; then
+            format_error "OSD 4/5/6 do not peer with older OSD 1"
+        fi
+    done
+}
+
+wait_repeer_1 15
+
+wait_finish_rebalance 15 '"active"'
+
+if [ "$SCHEME" = "replicated" ]; then
+    NOBJ=1024
+else
+    NOBJ=$((1024/(PG_SIZE-1)))
+fi
+
+if ! ($ETCDCTL get /vitastor/pg/stats/1/1 --print-value-only | jq -s -e '.[0].object_count == '$NOBJ); then
+    format_error "FAILED: PG SHOULD CONTAIN EXACTLY 128 MB OF DATA, BUT IT DOESN'T"
+fi
+
+qemu-img convert -S 4096 -p \
+    -f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:pool=1:inode=1:size=4096" \
+    -O raw ./testdata/inode1.bin
+
+qemu-img convert -S 4096 -p \
+    -f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:pool=1:inode=2:size="$((128*1024*1024)) \
+    -O raw ./testdata/inode2.bin
+
+if (dd if=/dev/zero bs=4096 count=1 | diff - ./testdata/inode1.bin); then
+    format_error "FAILED: INODE 1 SEEMS LOST"
+fi
+
+if ! (dd if=/dev/zero bs=1M count=128 | diff - ./testdata/inode2.bin); then
+    format_error "FAILED: INODE 2 SEEMS RESTORED"
+fi
+
+format_green OK