Remove peer_fd from the event loop only after clearing cl->refs

- Daemonize before forking in NFS proxy to fix OSD RDMA support (#107)
- Fix possible PG_INCOMPLETE on node outage when allow_net_split is false
- Fix build under Ubuntu 25.10
- Fix journal_no_same_sector_overwrites mode after optimizing WA (#109)

.gitea/workflows/test.yml

@@ -414,6 +414,24 @@ jobs:
           echo ""
         done
 
+  test_degraded:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_degraded.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_rm:
     runs-on: ubuntu-latest
     needs: build

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "2.4.2")
+set(VITASTOR_VERSION "2.4.3")
 
 add_subdirectory(src)

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v2.4.2
+VITASTOR_VERSION ?= v2.4.3
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v2.4.2
+          image: vitalif/vitastor-csi:v2.4.3
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v2.4.2
+          image: vitalif/vitastor-csi:v2.4.3
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "2.4.2"
+    vitastorCSIDriverVersion = "2.4.3"
 )
 
 // Config struct fills the parameters of request or user input

debian/build-vitastor-ubuntu-questing.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+# 25.10 Questing quokka
+
+docker build --build-arg DISTRO=ubuntu --build-arg REL=questing -t vitastor-buildenv:questing -f vitastor-buildenv.Dockerfile .
+docker run -it --rm -e REL=questing -v `dirname $0`/../:/root/vitastor vitastor-buildenv:questing /root/vitastor/debian/vitastor-build.sh

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (2.4.2-1) unstable; urgency=medium
+vitastor (2.4.3-1) unstable; urgency=medium
 
   * Bugfixes
 

docker/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v2.4.2
+VITASTOR_VERSION ?= v2.4.3
 
 all: build push
 

docker/etc/vitastor/docker.conf

@@ -4,7 +4,7 @@
 #
 
 # Desired Vitastor version
-VITASTOR_VERSION=v2.4.2
+VITASTOR_VERSION=v2.4.3
 
 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here

docs/installation/docker.en.md

@@ -26,9 +26,9 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.
 
 1. Download a Docker image of the desired version: \
-   `docker pull vitalif/vitastor:v2.4.2`
+   `docker pull vitalif/vitastor:v2.4.3`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.4.2 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.4.3 install.sh`
 3. Reload udev rules: \
    `udevadm control --reload-rules`
 

docs/installation/docker.ru.md

@@ -25,9 +25,9 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.
 
 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitalif/vitastor:v2.4.2`
+   `docker pull vitalif/vitastor:v2.4.3`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.4.2 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitalif/vitastor:v2.4.3 install.sh`
 3. Перезагрузите правила udev: \
    `udevadm control --reload-rules`
 

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "2.4.2",
+  "version": "2.4.3",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {

node-binding/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor",
-  "version": "2.4.2",
+  "version": "2.4.3",
   "description": "Low-level native bindings to Vitastor client library",
   "main": "index.js",
   "keywords": [

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '2.4.2'
+VITASTOR_VERSION = '2.4.3'
 
 LOG = logging.getLogger(__name__)
 

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.4.2
+Version:        2.4.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.4.2.el7.tar.gz
+Source0:        vitastor-2.4.3.el7.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  devtoolset-9-gcc-c++

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.4.2
+Version:        2.4.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.4.2.el8.tar.gz
+Source0:        vitastor-2.4.3.el8.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-toolset-9-gcc-c++

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.4.2
+Version:        2.4.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.4.2.el9.tar.gz
+Source0:        vitastor-2.4.3.el9.tar.gz
 
 BuildRequires:  gperftools-devel
 BuildRequires:  gcc-c++

src/CMakeLists.txt

@@ -20,7 +20,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVITASTOR_VERSION="2.4.2")
+add_definitions(-DVITASTOR_VERSION="2.4.3")
 add_definitions(-D_GNU_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -fvisibility=hidden -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/blockstore/blockstore_impl.cpp

@@ -228,9 +228,9 @@ void blockstore_impl_t::loop()
         for (auto s: journal.submitting_sectors)
         {
             // Mark journal sector writes as submitted
-            journal.sector_info[s].submit_id = 0;
             if (journal.sector_info[s].submit_id)
                 journal.sector_info[s].written = true;
+            journal.sector_info[s].submit_id = 0;
         }
         journal.submitting_sectors.clear();
         if ((initial_ring_space - ringloop->space_left()) > 0)

src/client/messenger.cpp

@@ -276,6 +276,7 @@ osd_messenger_t::~osd_messenger_t()
 #ifdef WITH_RDMA
     for (auto rdma_context: rdma_contexts)
     {
+        tfd->set_fd_handler(rdma_context->channel->fd, false, NULL);
         delete rdma_context;
     }
     rdma_contexts.clear();

src/client/messenger.h

@@ -234,6 +234,7 @@ public:
     void parse_config(const json11::Json & config);
     void connect_peer(uint64_t osd_num, json11::Json peer_state);
     void stop_client(int peer_fd, bool force = false, bool force_delete = false);
+    void destroy_client(osd_client_t *cl);
     void outbox_push(osd_op_t *cur_op);
     std::function<void(osd_op_t*)> exec_op;
     std::function<void(osd_num_t)> repeer_pgs;

src/client/msgr_receive.cpp

@@ -78,7 +78,7 @@ bool osd_messenger_t::handle_read(int result, osd_client_t *cl)
     {
         if (cl->refs <= 0)
         {
-            delete cl;
+            destroy_client(cl);
         }
         return false;
     }

src/client/msgr_send.cpp

@@ -274,7 +274,7 @@ void osd_messenger_t::handle_send(int result, bool prev, bool more, osd_client_t
     {
         if (cl->refs <= 0)
         {
-            delete cl;
+            destroy_client(cl);
         }
         return;
     }

src/client/msgr_stop.cpp

@@ -85,30 +85,13 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
             osd_peer_fds.erase(osd_it);
         }
     }
+    // Do not remove socket from the event loop as it may have refs > 0 and we want to clear them
 #ifndef __MOCK__
-    // Then remove FD from the eventloop so we don't accidentally read something
-    tfd->set_fd_handler(peer_fd, false, NULL);
     if (cl->connect_timeout_id >= 0)
     {
         tfd->clear_timer(cl->connect_timeout_id);
         cl->connect_timeout_id = -1;
     }
-    for (auto rit = read_ready_clients.begin(); rit != read_ready_clients.end(); rit++)
-    {
-        if (*rit == peer_fd)
-        {
-            read_ready_clients.erase(rit);
-            break;
-        }
-    }
-    for (auto wit = write_ready_clients.begin(); wit != write_ready_clients.end(); wit++)
-    {
-        if (*wit == peer_fd)
-        {
-            write_ready_clients.erase(wit);
-            break;
-        }
-    }
 #endif
     if (cl->in_osd_num && break_pg_locks)
     {
@@ -123,19 +106,42 @@ void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
         // so do not repeer on it.
         repeer_pgs(cl->osd_num);
     }
-    // Find the item again because it can be invalidated at this point
-    it = clients.find(peer_fd);
-    if (it != clients.end())
-    {
-        clients.erase(it);
-    }
     cl->refs--;
     if (cl->refs <= 0 || force_delete)
     {
-        delete cl;
+        destroy_client(cl);
     }
 }
 
+void osd_messenger_t::destroy_client(osd_client_t *cl)
+{
+#ifndef __MOCK__
+    if (cl->peer_fd >= 0)
+    {
+        // Remove FD from the eventloop
+        tfd->set_fd_handler(cl->peer_fd, false, NULL);
+        for (auto rit = read_ready_clients.begin(); rit != read_ready_clients.end(); rit++)
+        {
+            if (*rit == cl->peer_fd)
+            {
+                read_ready_clients.erase(rit);
+                break;
+            }
+        }
+        for (auto wit = write_ready_clients.begin(); wit != write_ready_clients.end(); wit++)
+        {
+            if (*wit == cl->peer_fd)
+            {
+                write_ready_clients.erase(wit);
+                break;
+            }
+        }
+    }
+#endif
+    clients.erase(cl->peer_fd);
+    delete cl;
+}
+
 osd_client_t::~osd_client_t()
 {
     free(in_buf);

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 2.4.2
+Version: 2.4.3
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/nfs/nfs_proxy.cpp

@@ -276,6 +276,19 @@ void nfs_proxy_t::run(json11::Json cfg)
     }
     // Check default pool
     check_default_pool();
+    // Daemonize before initializing messenger and RDMA because otherwise RDMA doesn't survive fork()
+    bool bg = cfg["foreground"].is_null() && cfg["cmd"].is_null();
+    int notifyfd[2] = { -1, -1 };
+    if (bg)
+    {
+        if (socketpair(AF_UNIX, SOCK_STREAM, 0, notifyfd) < 0)
+        {
+            perror("socketpair");
+            exit(1);
+        }
+        daemonize_fork(notifyfd);
+        close(notifyfd[0]);
+    }
     // Init VitastorFS after starting client because it depends on loaded inode configuration
     if (fsname != "")
     {
@@ -294,6 +307,13 @@ void nfs_proxy_t::run(json11::Json cfg)
     {
         kvfs->upgrade_db([this](int res) { finished = true; });
     }
+    if (bg)
+    {
+        daemonize_reopen_stdio();
+        int ok = 0;
+        (void)write(notifyfd[1], &ok, sizeof(ok));
+        close(notifyfd[1]);
+    }
     while (!finished)
     {
         ringloop->loop();
@@ -409,10 +429,6 @@ void nfs_proxy_t::run_server(json11::Json cfg)
     {
         mount_fs();
     }
-    if (cfg["foreground"].is_null())
-    {
-        daemonize();
-    }
     if (pidfile != "")
     {
         write_pid();
@@ -699,7 +715,8 @@ void nfs_client_t::handle_read(int result)
         return;
     if (result <= 0 && result != -EAGAIN && result != -EINTR && result != -ECANCELED)
     {
-        printf("Failed read from client %d: %d (%s)\n", nfs_fd, result, strerror(-result));
+        if (result != 0)
+            printf("Failed read from client %d: %d (%s)\n", nfs_fd, result, strerror(-result));
         stop();
         return;
     }
@@ -1224,26 +1241,31 @@ void nfs_client_t::free_or_rdma(rpc_op_t *rop, void *buf)
 #endif
 }
 
-void nfs_proxy_t::daemonize()
+void nfs_proxy_t::daemonize_fork(int *notifyfd)
 {
-    // Stop all clients because client I/O sometimes breaks during daemonize
-    // I.e. the new process stops receiving events on the old FD
-    // It doesn't happen if we call sleep(1) here, but we don't want to call sleep(1)...
-    for (auto & cli: rpc_clients)
-        cli->stop();
     if (fork())
-        exit(0);
+    {
+        // Parent - check status
+        close(notifyfd[1]);
+        int child_errno = 1;
+        (void)read(notifyfd[0], &child_errno, sizeof(child_errno));
+        exit(child_errno);
+    }
     setsid();
     if (fork())
         exit(0);
-    if (chdir("/") != 0)
-        fprintf(stderr, "Warning: Failed to chdir into /\n");
+}
+
+void nfs_proxy_t::daemonize_reopen_stdio()
+{
     close(0);
     close(1);
     close(2);
     open("/dev/null", O_RDONLY);
     open(logfile.c_str(), O_WRONLY|O_APPEND|O_CREAT, 0666);
     open(logfile.c_str(), O_WRONLY|O_APPEND|O_CREAT, 0666);
+    if (chdir("/") != 0)
+        fprintf(stderr, "Warning: Failed to chdir into /\n");
 }
 
 void nfs_proxy_t::write_pid()

src/nfs/nfs_proxy.h

@@ -83,7 +83,8 @@ public:
     void check_default_pool();
     nfs_client_t* create_client();
     void do_accept(int listen_fd);
-    void daemonize();
+    void daemonize_fork(int *notifyfd);
+    void daemonize_reopen_stdio();
     void write_pid();
     void mount_fs();
     void check_already_mounted();

src/osd/osd_cluster.cpp

@@ -410,7 +410,11 @@ void osd_t::report_statistics()
 
 void osd_t::on_change_osd_state_hook(osd_num_t peer_osd)
 {
-    if (msgr.wanted_peers.find(peer_osd) != msgr.wanted_peers.end())
+    if (st_cli.peer_states[peer_osd].is_null())
+    {
+        repeer_pgs(peer_osd);
+    }
+    else if (msgr.wanted_peers.find(peer_osd) != msgr.wanted_peers.end())
     {
         msgr.connect_peer(peer_osd, st_cli.peer_states[peer_osd]);
     }

src/osd/osd_peering_pg.h

@@ -4,6 +4,7 @@
 #include <map>
 #include <vector>
 #include <algorithm>
+#include <stdint.h>
 
 #include "cpp-btree/btree_map.h"
 

tests/run_tests.sh

@@ -36,6 +36,8 @@ SCHEME=ec ./test_snapshot.sh
 
 ./test_move_reappear.sh
 
+./test_degraded.sh
+
 ./test_rm.sh
 
 ./test_rm_degraded.sh

tests/test_degraded.sh

@@ -0,0 +1,28 @@
+#!/bin/bash -ex
+
+OSD_COUNT=0
+PG_COUNT=32
+PG_SIZE=3
+PG_MINSIZE=2
+POOLCFG='"failure_domain":"host",'
+. `dirname $0`/run_3osds.sh
+
+$ETCDCTL put /vitastor/config/node_placement '{"1":{"parent":"host1"},"2":{"parent":"host2"},"3":{"parent":"host3"},"4":{"parent":"host1"},"5":{"parent":"host2"},"6":{"parent":"host3"},"host1":{"level":"host"},"host2":{"level":"host"}}'
+
+OSD_COUNT=6
+for i in $(seq 1 $OSD_COUNT); do
+    start_osd $i
+done
+wait_up 120
+
+LD_PRELOAD="build/src/client/libfio_vitastor.so" \
+fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4k -direct=1 -iodepth=4 -rw=randwrite -loops=1000 \
+    -etcd=$ETCD_URL -pool=1 -inode=2 -size=256M -cluster_log_level=10 -runtime=5 &
+FIO_PID=$!
+
+sleep 15
+kill -9 $OSD1_PID $OSD4_PID
+
+wait $FIO_PID
+
+format_green OK