From 9eb858fb5ff60dca7171d183c0de7514c1712782 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Fri, 1 May 2026 15:43:54 +0300 Subject: [PATCH] Add a CI test with RDMA --- .gitea/workflows/test.yml | 18 ++++++++++++++++++ tests/common.sh | 3 +++ tests/run_3osds.sh | 10 +++++++++- tests/run_tests.sh | 1 + tests/test_move_reappear.sh | 2 +- tests/test_resize.sh | 2 +- tests/test_splitbrain.sh | 2 +- tests/test_vm_cont.sh | 2 +- 8 files changed, 35 insertions(+), 5 deletions(-) diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index 4b5825de..a349f21e 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -1350,6 +1350,24 @@ jobs: echo "" done + test_heal_ec_rdma: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 10 + run: TEST_NAME=ec_rdma RDMA=1 SCHEME=ec /root/vitastor/tests/test_heal.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + test_checksum: runs-on: ubuntu-latest needs: build diff --git a/tests/common.sh b/tests/common.sh index 0ed521ac..be4c7b08 100644 --- a/tests/common.sh +++ b/tests/common.sh @@ -28,6 +28,7 @@ ANTIETCD=${ANTIETCD} USE_RAMDISK=${USE_RAMDISK} ETCD_SCHEME=${ETCD_SCHEME:-http} OSD_TLS=${OSD_TLS:-1} +RDMA=${RDMA:-0} RAMDISK=/run/user/$(id -u) findmnt $RAMDISK >/dev/null || (sudo mkdir -p $RAMDISK && sudo mount -t tmpfs tmpfs $RAMDISK) @@ -125,6 +126,8 @@ VITASTOR_CFG='"etcd_address":"'$ETCD_URL'"'"$VITASTOR_CFG" if [[ "$ETCD_SCHEME" = "https" ]]; then VITASTOR_CFG="$VITASTOR_CFG"',"etcd_ca":"'$(pwd)'/testdata/etcd.crt"' fi +VITASTOR_CFG="$VITASTOR_CFG"',"osd_network":"'$ETCD_IP'/32"' +VITASTOR_CFG="$VITASTOR_CFG"',"use_rdma":"'$RDMA'"' if [[ "$OSD_TLS" = "1" ]]; then cd ./testdata openssl req -days 3650 -x509 -new -newkey rsa:4096 -nodes -keyout client_ca.key -out client_ca.crt \ diff --git a/tests/run_3osds.sh b/tests/run_3osds.sh index a223dc9e..59cc9a3f 100644 --- a/tests/run_3osds.sh +++ b/tests/run_3osds.sh @@ -40,7 +40,7 @@ start_osd_on() { local i=$1 local dev=$2 - build/src/osd/vitastor-osd --osd_num $i --bind_address $ETCD_IP $NO_SAME $OSD_ARGS \ + build/src/osd/vitastor-osd --osd_num $i $NO_SAME $OSD_ARGS \ $(build/src/disk_tool/vitastor-disk simple-offsets --format options $OFFSET_ARGS $dev $OFFSET_ARGS 2>/dev/null) \ >>./testdata/osd$i.log 2>&1 & eval OSD${i}_PID=$! @@ -119,6 +119,14 @@ if [[ $OSD_COUNT -gt 0 ]]; then wait_up 120 fi +if [[ "$RDMA" = 1 ]]; then + for i in $(seq 1 $OSD_COUNT); do + if ! grep 'RDMA initialized successfully' ./testdata/osd$i.log; then + format_error "OSD $i failed to initialize RDMA, please try to enable rdma-rxe on the host" + fi + done +fi + try_reweight() { osd=$1 diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 1f14f354..afa2d6a6 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -105,6 +105,7 @@ PG_SIZE=2 ./test_heal.sh TEST_NAME=local_read POOLCFG='"local_reads":"random",' ./test_heal.sh SCHEME=ec ./test_heal.sh ANTIETCD=1 ./test_heal.sh +TEST_NAME=ec_rdma RDMA=1 SCHEME=ec ./test_heal.sh ./test_checksum.sh TEST_NAME=xxhash OSD_ARGS="--data_csum_type xxh3_32" ./test_checksum.sh diff --git a/tests/test_move_reappear.sh b/tests/test_move_reappear.sh index 03baaf19..baab5ace 100755 --- a/tests/test_move_reappear.sh +++ b/tests/test_move_reappear.sh @@ -9,7 +9,7 @@ OSD_COUNT=5 OSD_ARGS="$OSD_ARGS" for i in $(seq 1 $OSD_COUNT); do dd if=/dev/zero of=./testdata/bin/test_osd$i.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1)) - build/src/osd/vitastor-osd --log_level 10 --osd_num $i --bind_address 127.0.0.1 --etcd_stats_interval 5 $OSD_ARGS \ + build/src/osd/vitastor-osd --log_level 10 --osd_num $i --etcd_stats_interval 5 $OSD_ARGS \ $(build/src/disk_tool/vitastor-disk simple-offsets --format options ./testdata/bin/test_osd$i.bin $OFFSET_ARGS 2>/dev/null) >>./testdata/osd$i.log 2>&1 & eval OSD${i}_PID=$! done diff --git a/tests/test_resize.sh b/tests/test_resize.sh index e0af7c8d..d00498ae 100755 --- a/tests/test_resize.sh +++ b/tests/test_resize.sh @@ -69,7 +69,7 @@ done $ETCDCTL del --prefix /vitastor/osd/state/ for i in $(seq 1 $OSD_COUNT); do - build/src/osd/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $NO_SAME $OSD_ARGS \ + build/src/osd/vitastor-osd --osd_num $i $NO_SAME $OSD_ARGS \ --meta_format $meta_format \ --data_device ./testdata/bin/test_osd$i.bin \ --meta_offset 0 \ diff --git a/tests/test_splitbrain.sh b/tests/test_splitbrain.sh index 0b845c8a..bb6c819f 100755 --- a/tests/test_splitbrain.sh +++ b/tests/test_splitbrain.sh @@ -21,7 +21,7 @@ $VITASTOR_FIO -bs=4k -direct=1 -iodepth=1 -fsync=1 \ # Kill OSD 2, start OSD 1 kill $OSD2_PID -build/src/osd/vitastor-osd --osd_num 1 --bind_address 127.0.0.1 $NO_SAME $OSD_ARGS \ +build/src/osd/vitastor-osd --osd_num 1 $NO_SAME $OSD_ARGS \ $(build/src/disk_tool/vitastor-disk simple-offsets --format options --device ./testdata/bin/test_osd2.bin $OFFSET_ARGS 2>/dev/null) >>./testdata/osd2.log 2>&1 & sleep 2 diff --git a/tests/test_vm_cont.sh b/tests/test_vm_cont.sh index 785ecb9c..0eab82f5 100755 --- a/tests/test_vm_cont.sh +++ b/tests/test_vm_cont.sh @@ -15,7 +15,7 @@ OSD_COUNT=3 OSD_ARGS="$OSD_ARGS" OFFSET_ARGS="$OFFSET_ARGS" for i in $(seq 1 $OSD_COUNT); do - build/src/osd/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $OSD_ARGS \ + build/src/osd/vitastor-osd --osd_num $i $OSD_ARGS \ $(build/src/disk_tool/vitastor-disk simple-offsets --format options ./testdata/bin/test_osd$i.bin $OFFSET_ARGS 2>/dev/null) >>./testdata/osd$i.log 2>&1 & eval OSD${i}_PID=$! done