diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index cf1dd7c9..0a095273 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -197,6 +197,24 @@ jobs: echo "" done + test_etcd_fail_antietcd: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 10 + run: ANTIETCD=1 /root/vitastor/tests/test_etcd_fail.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + test_interrupted_rebalance: runs-on: ubuntu-latest needs: build @@ -665,6 +683,24 @@ jobs: echo "" done + test_heal_antietcd: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 10 + run: ANTIETCD=1 /root/vitastor/tests/test_heal.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + test_heal_csum_32k_dmj: runs-on: ubuntu-latest needs: build diff --git a/.gitea/workflows/tests-to-yaml.pl b/.gitea/workflows/tests-to-yaml.pl index 4a820331..d1a0942d 100755 --- a/.gitea/workflows/tests-to-yaml.pl +++ b/.gitea/workflows/tests-to-yaml.pl @@ -34,6 +34,10 @@ for my $line (<>) { $test_name .= '_imm'; } + elsif ($1 eq 'ANTIETCD') + { + $test_name .= '_antietcd'; + } else { $test_name .= '_'.lc($1).'_'.$2; diff --git a/tests/common.sh b/tests/common.sh index cace1f1a..7775142d 100644 --- a/tests/common.sh +++ b/tests/common.sh @@ -24,6 +24,7 @@ ETCD=${ETCD:-etcd} ETCD_IP=${ETCD_IP:-127.0.0.1} ETCD_PORT=${ETCD_PORT:-12379} ETCD_COUNT=${ETCD_COUNT:-1} +ANTIETCD=${ANTIETCD} if [ "$KEEP_DATA" = "" ]; then rm -rf ./testdata @@ -32,36 +33,59 @@ if [ "$KEEP_DATA" = "" ]; then fi ETCD_URL="http://$ETCD_IP:$ETCD_PORT" -ETCD_CLUSTER="etcd1=http://$ETCD_IP:$((ETCD_PORT+1))" for i in $(seq 2 $ETCD_COUNT); do ETCD_URL="$ETCD_URL,http://$ETCD_IP:$((ETCD_PORT+2*i-2))" - ETCD_CLUSTER="$ETCD_CLUSTER,etcd$i=http://$ETCD_IP:$((ETCD_PORT+2*i-1))" done -ETCDCTL="${ETCD}ctl --endpoints=$ETCD_URL --dial-timeout=5s --command-timeout=10s" start_etcd() { local i=$1 - local t=/run/user/$(id -u) - findmnt $t >/dev/null || (sudo mkdir -p $t && sudo mount -t tmpfs tmpfs $t) - ionice -c2 -n0 $ETCD -name etcd$i --data-dir /run/user/$(id -u)/testdata_etcd$i \ - --advertise-client-urls http://$ETCD_IP:$((ETCD_PORT+2*i-2)) --listen-client-urls http://$ETCD_IP:$((ETCD_PORT+2*i-2)) \ - --initial-advertise-peer-urls http://$ETCD_IP:$((ETCD_PORT+2*i-1)) --listen-peer-urls http://$ETCD_IP:$((ETCD_PORT+2*i-1)) \ - --initial-cluster-token vitastor-tests-etcd --initial-cluster-state new \ - --initial-cluster "$ETCD_CLUSTER" --max-request-bytes=104857600 \ - --max-txn-ops=100000 --auto-compaction-retention=10 --auto-compaction-mode=revision &>./testdata/etcd$i.log & - eval ETCD${i}_PID=$! + if [[ -z "$ANTIETCD" ]]; then + local t=/run/user/$(id -u) + findmnt $t >/dev/null || (sudo mkdir -p $t && sudo mount -t tmpfs tmpfs $t) + ionice -c2 -n0 $ETCD -name etcd$i --data-dir /run/user/$(id -u)/testdata_etcd$i \ + --advertise-client-urls http://$ETCD_IP:$((ETCD_PORT+2*i-2)) --listen-client-urls http://$ETCD_IP:$((ETCD_PORT+2*i-2)) \ + --initial-advertise-peer-urls http://$ETCD_IP:$((ETCD_PORT+2*i-1)) --listen-peer-urls http://$ETCD_IP:$((ETCD_PORT+2*i-1)) \ + --initial-cluster-token vitastor-tests-etcd --initial-cluster-state new \ + --initial-cluster "$ETCD_CLUSTER" --max-request-bytes=104857600 \ + --max-txn-ops=100000 --auto-compaction-retention=10 --auto-compaction-mode=revision &>./testdata/etcd$i.log & + eval ETCD${i}_PID=$! + else + node mon/mon-main.js $MON_PARAMS --antietcd_port $((ETCD_PORT+2*i-2)) --etcd_address $ETCD_URL --etcd_prefix "/vitastor" --verbose 1 >>./testdata/mon$i.log 2>&1 & + eval ETCD${i}_PID=$! + fi } -for i in $(seq 1 $ETCD_COUNT); do - start_etcd $i -done -for i in {1..30}; do - ${ETCD}ctl --endpoints=$ETCD_URL --dial-timeout=1s --command-timeout=1s member list >/dev/null && break - if [[ $i = 30 ]]; then - format_error "Failed to start etcd" - fi -done +start_etcd_cluster() +{ + ETCD_CLUSTER="etcd1=http://$ETCD_IP:$((ETCD_PORT+1))" + for i in $(seq 2 $ETCD_COUNT); do + ETCD_CLUSTER="$ETCD_CLUSTER,etcd$i=http://$ETCD_IP:$((ETCD_PORT+2*i-1))" + done + for i in $(seq 1 $ETCD_COUNT); do + start_etcd $i + done +} + +wait_etcd() +{ + for i in {1..30}; do + $ETCDCTL --dial-timeout=1s --command-timeout=1s get --prefix / && break + if [[ $i = 30 ]]; then + format_error "Failed to start etcd" + fi + sleep 1 + done +} + +if [[ -n "$ANTIETCD" ]]; then + ETCDCTL="node mon/node_modules/.bin/anticli -e $ETCD_URL" + MON_PARAMS="--use_antietcd 1 --antietcd_data_dir ./testdata --antietcd_persist_interval 500 $MON_PARAMS" +else + ETCDCTL="${ETCD}ctl --endpoints=$ETCD_URL --dial-timeout=5s --command-timeout=10s" + MON_PARAMS="$MON_PARAMS" + start_etcd_cluster +fi echo leak:fio >> testdata/lsan-suppress.txt echo leak:tcmalloc >> testdata/lsan-suppress.txt diff --git a/tests/run_3osds.sh b/tests/run_3osds.sh index d0d65422..d588596a 100644 --- a/tests/run_3osds.sh +++ b/tests/run_3osds.sh @@ -18,6 +18,16 @@ else OSD_COUNT=${OSD_COUNT:-3} fi +if [[ -n "$ANTIETCD" ]]; then + for i in $(seq 1 $ETCD_COUNT); do + start_etcd $i + done +else + node mon/mon-main.js $MON_PARAMS --etcd_address $ETCD_URL --etcd_prefix "/vitastor" --verbose 1 >>./testdata/mon.log 2>&1 & + MON_PID=$! +fi +wait_etcd + if [ "$IMMEDIATE_COMMIT" != "" ]; then NO_SAME="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all --log_level 10 --etcd_stats_interval 5" $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"immediate_commit":"all","client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}' @@ -54,9 +64,6 @@ for i in $(seq 1 $OSD_COUNT); do start_osd $i done -node mon/mon-main.js --etcd_address $ETCD_URL --etcd_prefix "/vitastor" --verbose 1 >>./testdata/mon.log 2>&1 & -MON_PID=$! - if [ "$SCHEME" = "ec" ]; then PG_SIZE=${PG_SIZE:-5} PG_MINSIZE=${PG_MINSIZE:-4} diff --git a/tests/run_tests.sh b/tests/run_tests.sh index a0da98dc..97d730e5 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -15,6 +15,7 @@ SCHEME=ec ./test_change_pg_count.sh ./test_create_nomaxid.sh ./test_etcd_fail.sh +ANTIETCD=1 ./test_etcd_fail.sh ./test_interrupted_rebalance.sh IMMEDIATE_COMMIT=1 ./test_interrupted_rebalance.sh @@ -56,6 +57,7 @@ SCHEME=xor ./test_write.sh PG_SIZE=2 ./test_heal.sh SCHEME=ec ./test_heal.sh +ANTIETCD=1 ./test_heal.sh TEST_NAME=csum_32k_dmj OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k --inmemory_metadata false --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS ./test_heal.sh TEST_NAME=csum_32k_dj OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS ./test_heal.sh diff --git a/tests/test_failure_domain.sh b/tests/test_failure_domain.sh index d028b8fe..a1e23d2c 100755 --- a/tests/test_failure_domain.sh +++ b/tests/test_failure_domain.sh @@ -2,6 +2,10 @@ . `dirname $0`/common.sh +node mon/mon-main.js $MON_PARAMS --etcd_address $ETCD_URL --etcd_prefix "/vitastor" >>./testdata/mon.log 2>&1 & +MON_PID=$! +wait_etcd + TIME=$(date '+%s') $ETCDCTL put /vitastor/config/global '{"placement_levels":{"rack":1,"host":2,"osd":3},"immediate_commit":"none"}' $ETCDCTL put /vitastor/config/node_placement '{"rack1":{"level":"rack"},"rack2":{"level":"rack"},"host1":{"level":"host","parent":"rack1"},"host2":{"level":"host","parent":"rack1"},"host3":{"level":"host","parent":"rack2"},"host4":{"level":"host","parent":"rack2"}}' @@ -22,12 +26,9 @@ $ETCDCTL get --print-value-only /vitastor/config/pools | jq -s -e '. == [{}]' build/src/cmd/vitastor-cli --etcd_address $ETCD_URL create-pool testpool -s 2 -n 4 --failure_domain rack --force $ETCDCTL get --print-value-only /vitastor/config/pools | jq -s -e '. == [{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":4,"failure_domain":"rack"}}]' -node mon/mon-main.js --etcd_address $ETCD_URL --etcd_prefix "/vitastor" >>./testdata/mon.log 2>&1 & -MON_PID=$! - sleep 2 -etcdctl --endpoints=http://localhost:12379 get --prefix /vitastor/config/pgs --print-value-only | \ +$ETCDCTL get --prefix /vitastor/config/pgs --print-value-only | \ jq -s -e '([ .[0].items["1"] | .[].osd_set | map_values(. | tonumber) | select((.[0] <= 4) != (.[1] <= 4)) ] | length) == 4' format_green OK diff --git a/tests/test_move_reappear.sh b/tests/test_move_reappear.sh index e647aa9f..af331b30 100755 --- a/tests/test_move_reappear.sh +++ b/tests/test_move_reappear.sh @@ -1,5 +1,7 @@ #!/bin/bash -ex +USE_ANTIETCD="" + . `dirname $0`/common.sh OSD_SIZE=1024 diff --git a/tests/test_osd_tags.sh b/tests/test_osd_tags.sh index 7327f4e8..40e678af 100755 --- a/tests/test_osd_tags.sh +++ b/tests/test_osd_tags.sh @@ -2,6 +2,10 @@ . `dirname $0`/common.sh +node mon/mon-main.js $MON_PARAMS --etcd_address $ETCD_URL --etcd_prefix "/vitastor" >>./testdata/mon.log 2>&1 & +MON_PID=$! +wait_etcd + TIME=$(date '+%s') $ETCDCTL put /vitastor/config/osd/1 '{"tags":["a"]}' $ETCDCTL put /vitastor/config/osd/2 '{"tags":["a"]}' @@ -21,15 +25,12 @@ $ETCDCTL put /vitastor/osd/stats/7 '{"host":"stor4","size":1073741824,"time":"'$ $ETCDCTL put /vitastor/osd/stats/8 '{"host":"stor4","size":1073741824,"time":"'$TIME'"}' $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":2,"pg_count":16,"failure_domain":"host","osd_tags":["a"],"immediate_commit":"none"}}' -node mon/mon-main.js --etcd_address $ETCD_URL --etcd_prefix "/vitastor" >>./testdata/mon.log 2>&1 & -MON_PID=$! - sleep 2 -etcdctl --endpoints=http://localhost:12379 get --prefix /vitastor/config/pgs --print-value-only +$ETCDCTL get --prefix /vitastor/config/pgs --print-value-only -if ! (etcdctl --endpoints=http://localhost:12379 get --prefix /vitastor/config/pgs --print-value-only | \ - jq -s -e '[ [ .[0].items["1"] | .[].osd_set | map(. | select(. != "" and (.|tonumber) < 5)) ][] | select((. | length) == 2) ] | length == 16'); then +if ! ($ETCDCTL get --prefix /vitastor/config/pgs --print-value-only | \ + jq -s -e '[ [ .[] | select(has("items")) | .items["1"] | .[].osd_set | map(. | select(. != "" and (.|tonumber) < 5)) ][] | select((. | length) == 2) ] | length == 16'); then format_error "Some PGs missing replicas" fi diff --git a/tests/test_root_node.sh b/tests/test_root_node.sh index c8118336..288d4ad1 100755 --- a/tests/test_root_node.sh +++ b/tests/test_root_node.sh @@ -2,6 +2,10 @@ . `dirname $0`/common.sh +node mon/mon-main.js $MON_PARAMS --etcd_address $ETCD_URL --etcd_prefix "/vitastor" >>./testdata/mon.log 2>&1 & +MON_PID=$! +wait_etcd + TIME=$(date '+%s') $ETCDCTL put /vitastor/config/global '{"placement_levels":{"rack":100,"host":101,"osd":102},"immediate_commit":"none"}' $ETCDCTL put /vitastor/config/node_placement '{"rack1":{"level":"rack"},"rack2":{"level":"rack"},"stor1":{"level":"host","parent":"rack1"},"stor2":{"level":"host","parent":"rack1"},"stor3":{"level":"host","parent":"rack2"},"stor4":{"level":"host","parent":"rack2"}}' @@ -15,14 +19,11 @@ $ETCDCTL put /vitastor/osd/stats/7 '{"host":"stor4","size":1073741824,"time":"'$ $ETCDCTL put /vitastor/osd/stats/8 '{"host":"stor4","size":1073741824,"time":"'$TIME'"}' $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":2,"pg_count":16,"failure_domain":"host","root_node":"rack1"}}' -node mon/mon-main.js --etcd_address $ETCD_URL --etcd_prefix "/vitastor" >>./testdata/mon.log 2>&1 & -MON_PID=$! - sleep 2 -etcdctl --endpoints=http://localhost:12379 get --prefix /vitastor/config/pgs --print-value-only +$ETCDCTL get --prefix /vitastor/config/pgs --print-value-only -if ! (etcdctl --endpoints=http://localhost:12379 get --prefix /vitastor/config/pgs --print-value-only | \ +if ! ($ETCDCTL get --prefix /vitastor/config/pgs --print-value-only | \ jq -s -e '[ [ .[0].items["1"] | .[].osd_set | map(. | select(. != "" and (.|tonumber) < 5)) ][] | select((. | length) == 2) ] | length == 16'); then format_error "Some PGs missing replicas" fi diff --git a/tests/test_vm_cont.sh b/tests/test_vm_cont.sh index 48b6d469..aab6aa12 100755 --- a/tests/test_vm_cont.sh +++ b/tests/test_vm_cont.sh @@ -3,9 +3,13 @@ export KEEP_DATA=1 . `dirname $0`/common.sh -etcdctl --endpoints=http://127.0.0.1:12379/v3 del --prefix /vitastor/mon/master -etcdctl --endpoints=http://127.0.0.1:12379/v3 del --prefix /vitastor/pg/state -etcdctl --endpoints=http://127.0.0.1:12379/v3 del --prefix /vitastor/osd/state +node mon/mon-main.js $MON_PARAMS --etcd_address $ETCD_URL --etcd_prefix "/vitastor" --verbose 1 >>./testdata/mon.log 2>&1 & +MON_PID=$! +wait_etcd + +$ETCDCTL del --prefix /vitastor/mon/master +$ETCDCTL del --prefix /vitastor/pg/state +$ETCDCTL del --prefix /vitastor/osd/state OSD_COUNT=3 OSD_ARGS="$OSD_ARGS" @@ -15,9 +19,6 @@ for i in $(seq 1 $OSD_COUNT); do eval OSD${i}_PID=$! done -node mon/mon-main.js --etcd_address $ETCD_URL --etcd_prefix "/vitastor" --verbose 1 >>./testdata/mon.log 2>&1 & -MON_PID=$! - sleep 3 if ! ($ETCDCTL get /vitastor/pg/state/1/1 --print-value-only | jq -s -e '(. | length) != 0 and .[0].state == ["active"]'); then