Support DNS resolving via libc-ares

2026-05-19 17:19:35 +03:00
parent eb1dbd7459
commit 26be916e3f
18 changed files with 132 additions and 40 deletions
@@ -3,7 +3,7 @@ Section: admin
 Priority: optional
 Maintainer: Vitaliy Filippov <vitalif@yourcmc.ru>
 Build-Depends: debhelper, g++ (>= 8), libstdc++6 (>= 8),
-  linux-libc-dev, libgoogle-perftools-dev, libjerasure-dev, libgf-complete-dev,
+  linux-libc-dev, libgoogle-perftools-dev, libjerasure-dev, libgf-complete-dev, libc-ares-dev,
  libibverbs-dev, librdmacm-dev, libisal-dev, cmake, pkg-config, libnl-3-dev, libnl-genl-3-dev,
  node-bindings <!nocheck>, node-gyp, node-nan
 Standards-Version: 4.5.0
@@ -25,7 +25,7 @@ RUN set -e -x; \
    echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf

 RUN apt-get update && \
-    apt-get -y install fio libgoogle-perftools-dev devscripts libjerasure-dev cmake \
+    apt-get -y install fio libgoogle-perftools-dev devscripts libjerasure-dev cmake libc-ares-dev \
        libibverbs-dev librdmacm-dev libisal-dev libnl-3-dev libnl-genl-3-dev curl nodejs npm node-nan node-bindings && \
    apt-get -y build-dep fio && \
    apt-get --download-only source fio
@@ -15,7 +15,7 @@
 - gcc and g++ 8 or newer, clang 10 or newer, or other compiler with C++11 plus
  designated initializers support from C++20
 - CMake
- jerasure headers and libraries
+- jerasure, c-ares headers and libraries
 - ISA-L, libibverbs, librdmacm, libnl3 headers and libraries (optional)

 ## Basic instructions
@@ -15,7 +15,7 @@
 - gcc и g++ >= 8, либо clang >= 10, либо другой компилятор с поддержкой C++11 плюс
  назначенных инициализаторов (designated initializers) из C++20
 - CMake
- Заголовки и библиотеки jerasure
+- Заголовки и библиотеки jerasure, c-ares
 - Опционально - заголовки и библиотеки ISA-L, libibverbs, librdmacm, libnl3

 ## Базовая инструкция
@@ -11,7 +11,7 @@ WORKDIR /root
 RUN sed -i 's/enabled=0/enabled=1/' /etc/yum.repos.d/*.repo
 RUN dnf -y install epel-release dnf-plugins-core
 RUN dnf -y install https://vitastor.io/rpms/centos/10/vitastor-release-1.0-1.el10.noarch.rpm
-RUN dnf -y install gcc-c++ gperftools-devel fio nodejs rpm-build jerasure-devel isa-l-devel gf-complete-devel rdma-core-devel cmake libnl3-devel
+RUN dnf -y install gcc-c++ gperftools-devel fio nodejs rpm-build jerasure-devel isa-l-devel gf-complete-devel rdma-core-devel cmake libnl3-devel c-ares-devel
 RUN dnf download --source fio
 RUN rpm --nomd5 -i fio*.src.rpm
 RUN cd ~/rpmbuild/SPECS && dnf builddep -y --spec fio.spec
@@ -16,6 +16,7 @@ BuildRequires:  gf-complete-devel
 BuildRequires:  rdma-core-devel
 BuildRequires:  cmake
 BuildRequires:  libnl3-devel
+BuildRequires:  c-ares-devel
 Requires:       vitastor-osd = %{version}-%{release}
 Requires:       vitastor-mon = %{version}-%{release}
 Requires:       vitastor-client = %{version}-%{release}
@@ -15,7 +15,7 @@ RUN yum -y --enablerepo=extras install centos-release-scl epel-release yum-utils
 RUN perl -i -pe 's!mirrorlist=!#mirrorlist=!s; s!#\s*baseurl=http://mirror.centos.org!baseurl=http://vault.centos.org!' /etc/yum.repos.d/CentOS-SCLo-scl*.repo
 RUN yum -y install https://vitastor.io/rpms/centos/7/vitastor-release-1.0-1.el7.noarch.rpm
 RUN yum -y install devtoolset-9-gcc-c++ devtoolset-9-libatomic-devel gcc make cmake gperftools-devel \
-    fio rh-nodejs12 jerasure-devel libisa-l-devel gf-complete-devel rdma-core-devel libnl3-devel
+    fio rh-nodejs12 jerasure-devel libisa-l-devel gf-complete-devel rdma-core-devel libnl3-devel c-ares-devel
 RUN yumdownloader --disablerepo=centos-sclo-rh --source fio
 RUN rpm --nomd5 -i fio*.src.rpm
 RUN rm -f /etc/yum.repos.d/CentOS-Media.repo
@@ -17,6 +17,7 @@ BuildRequires:  gf-complete-devel
 BuildRequires:  rdma-core-devel
 BuildRequires:  cmake3
 BuildRequires:  libnl3-devel
+BuildRequires:  c-ares-devel
 Requires:       vitastor-osd = %{version}-%{release}
 Requires:       vitastor-mon = %{version}-%{release}
 Requires:       vitastor-client = %{version}-%{release}
@@ -13,7 +13,7 @@ RUN dnf -y install centos-release-advanced-virtualization epel-release dnf-plugi
 RUN sed -i 's/^mirrorlist=/#mirrorlist=/; s!#baseurl=.*!baseurl=http://vault.centos.org/centos/8.4.2105/virt/$basearch/$avdir/!; s!^baseurl=.*Source/.*!baseurl=http://vault.centos.org/centos/8.4.2105/virt/Source/advanced-virtualization/!' /etc/yum.repos.d/CentOS-Advanced-Virtualization.repo
 RUN yum -y install https://vitastor.io/rpms/centos/8/vitastor-release-1.0-1.el8.noarch.rpm
 RUN dnf -y install gcc-toolset-9 gcc-toolset-9-gcc-c++ gperftools-devel \
-    fio nodejs rpm-build jerasure-devel libisa-l-devel gf-complete-devel libibverbs-devel libarchive cmake libnl3-devel
+    fio nodejs rpm-build jerasure-devel libisa-l-devel gf-complete-devel libibverbs-devel libarchive cmake libnl3-devel c-ares-devel
 RUN dnf download --source fio
 RUN rpm --nomd5 -i fio*.src.rpm
 RUN cd ~/rpmbuild/SPECS && dnf builddep -y --enablerepo=powertools --spec fio.spec
@@ -16,6 +16,7 @@ BuildRequires:  gf-complete-devel
 BuildRequires:  rdma-core-devel
 BuildRequires:  cmake
 BuildRequires:  libnl3-devel
+BuildRequires:  c-ares-devel
 Requires:       vitastor-osd = %{version}-%{release}
 Requires:       vitastor-mon = %{version}-%{release}
 Requires:       vitastor-client = %{version}-%{release}
@@ -10,7 +10,7 @@ WORKDIR /root
 RUN sed -i 's/enabled=0/enabled=1/' /etc/yum.repos.d/*.repo
 RUN dnf -y install epel-release dnf-plugins-core
 RUN dnf -y install https://vitastor.io/rpms/centos/9/vitastor-release-1.0-1.el9.noarch.rpm
-RUN dnf -y install gcc-c++ gperftools-devel fio nodejs rpm-build jerasure-devel libisa-l-devel gf-complete-devel rdma-core-devel libarchive cmake libnl3-devel
+RUN dnf -y install gcc-c++ gperftools-devel fio nodejs rpm-build jerasure-devel libisa-l-devel gf-complete-devel rdma-core-devel libarchive cmake libnl3-devel c-ares-devel
 RUN dnf download --source fio
 RUN rpm --nomd5 -i fio*.src.rpm
 RUN cd ~/rpmbuild/SPECS && dnf builddep -y --spec fio.spec
@@ -16,6 +16,7 @@ BuildRequires:  gf-complete-devel
 BuildRequires:  rdma-core-devel
 BuildRequires:  cmake
 BuildRequires:  libnl3-devel
+BuildRequires:  c-ares-devel
 Requires:       vitastor-osd = %{version}-%{release}
 Requires:       vitastor-mon = %{version}-%{release}
 Requires:       vitastor-client = %{version}-%{release}
@@ -79,6 +79,9 @@ if (OPENSSL_FOUND)
 	add_definitions(-DWITH_OPENSSL)
 endif (OPENSSL_FOUND)

+pkg_check_modules(CARES REQUIRED libcares)
+include_directories(${CARES_INCLUDE_DIRS})
+
 if (${WITH_SYSTEM_LIBURING})
 	pkg_check_modules(LIBURING REQUIRED liburing>=2.10)
 	include_directories(${LIBURING_INCLUDE_DIRS})
@@ -16,7 +16,7 @@ add_library(vitastor_common STATIC
 	msgr_encrypt.cpp msgr_stop.cpp msgr_op.cpp msgr_send.cpp msgr_receive.cpp ../util/ringloop.cpp ../../json11/json11.cpp
 	http_client.cpp osd_ops.cpp pg_states.cpp ../util/timerfd_manager.cpp ../util/str_util.cpp ../util/json_util.cpp ${MSGR_RDMA} ${MSGR_RDMACM}
 )
-target_link_libraries(vitastor_common pthread ${OPENSSL_LIBRARIES})
+target_link_libraries(vitastor_common pthread ${OPENSSL_LIBRARIES} ${CARES_LIBRARIES})
 target_compile_options(vitastor_common PUBLIC -fPIC)

 # libvitastor_client.so
@@ -84,7 +84,7 @@ http_context_t *etcd_state_client_t::get_http_ctx()
    if (!http_ctx)
    {
        std::string error;
-        http_ctx = http_context_init(etcd_client_cert, etcd_client_key, etcd_ca, true, error);
+        http_ctx = http_context_init(tfd, etcd_client_cert, etcd_client_key, etcd_ca, true, error);
        if (!http_ctx)
        {
            fprintf(stderr, "Failed to initialize HTTP context: %s\n", error.c_str());
@@ -113,7 +113,7 @@ void etcd_state_client_t::etcd_call_oneshot(std::string etcd_address, std::strin
        "Content-Length: "+std::to_string(req.size())+"\r\n"
        "Connection: close\r\n"
        "\r\n"+req;
-    auto http_cli = http_init(tfd, get_http_ctx());
+    auto http_cli = http_init(get_http_ctx());
    auto cb = [http_cli, callback](http_message_t *response)
    {
        std::string err;
@@ -190,7 +190,7 @@ void etcd_state_client_t::etcd_call(std::string api, json11::Json payload, int t
            callback(err, data);
    };
    if (!keepalive_client)
-        keepalive_client = http_init(tfd, get_http_ctx());
+        keepalive_client = http_init(get_http_ctx());
    http_request(keepalive_client, etcd_address, req, { .timeout = timeout, .keepalive = true, .ssl = ssl }, cb);
 }

@@ -384,7 +384,7 @@ void etcd_state_client_t::start_etcd_watcher()
            etcd_watch_revision_config, etcd_watch_revision_osd, etcd_watch_revision_pg);
    }
    if (!etcd_watch_ws)
-        etcd_watch_ws = http_init(tfd, get_http_ctx());
+        etcd_watch_ws = http_init(get_http_ctx());
    else
        http_close(etcd_watch_ws);
    open_websocket(etcd_watch_ws, etcd_address, etcd_api_path+"/watch", { .timeout = etcd_slow_timeout, .ssl = ssl },
@@ -21,6 +21,9 @@
 #include <openssl/ssl.h>
 #endif

+// libc-ares
+#include <ares.h>
+
 #include "addr_util.h"
 #include "str_util.h"
 #include "json_util.h"
@@ -36,6 +39,9 @@ static void parse_http_headers(std::string & res, http_message_t *parsed, bool i

 struct http_context_t
 {
+    timerfd_manager_t *tfd = NULL;
+    ares_channel ares = NULL;
+
    std::string ssl_cert;
    std::string ssl_key;
    std::string ssl_ca;
@@ -46,6 +52,8 @@ struct http_context_t

    ~http_context_t()
    {
+        ares_destroy(ares);
+        ares = NULL;
 #ifdef WITH_OPENSSL
        if (ssl_ctx)
        {
@@ -72,12 +80,14 @@ struct http_co_t
    BIO *ssl_bio = NULL;
 #endif

-    timerfd_manager_t *tfd;
+    timerfd_manager_t *tfd = NULL;
+
    std::function<void(http_message_t*)> response_callback;

    int request_timeout = 0;
    bool ssl = false;
    std::string host;
+    std::string host_port;
    std::string request;
    std::string ws_outbox;
    std::string response;
@@ -106,7 +116,8 @@ struct http_co_t
    inline void stackout() { onstack--; if (!onstack && ended) end(); }
    inline void end() { ended = true; if (!onstack) { delete this; } }
    void run_cb_and_clear();
-    void start_connection();
+    void resolve_and_connect();
+    void start_connection(sockaddr *addr, size_t addr_size);
    void start_ws_connection();
    void close_connection();
    void next_request();
@@ -123,26 +134,44 @@ struct http_co_t
    void reply(const std::string & msg);
    void send_request(const std::string & host, const std::string & request,
        const http_options_t & options, std::function<void(http_message_t *response)> response_callback);
+
+    void ares_addrinfo_cb(int status, int timeouts, struct ares_addrinfo *result);
 };

 #define HTTP_CO_CLOSED 0
-#define HTTP_CO_CONNECTING 1
-#define HTTP_CO_SENDING_REQUEST 2
-#define HTTP_CO_REQUEST_SENT 3
-#define HTTP_CO_HEADERS_RECEIVED 4
-#define HTTP_CO_WEBSOCKET 5
-#define HTTP_CO_CHUNKED 6
-#define HTTP_CO_KEEPALIVE 7
-#define HTTP_CO_SERVER 8
-#define HTTP_CO_REQ_HDR_RECEIVED 9
-#define HTTP_CO_REQUEST_RECEIVED 10
+#define HTTP_CO_RESOLVING 1
+#define HTTP_CO_CONNECTING 2
+#define HTTP_CO_SENDING_REQUEST 3
+#define HTTP_CO_REQUEST_SENT 4
+#define HTTP_CO_HEADERS_RECEIVED 5
+#define HTTP_CO_WEBSOCKET 6
+#define HTTP_CO_CHUNKED 7
+#define HTTP_CO_KEEPALIVE 8
+#define HTTP_CO_SERVER 9
+#define HTTP_CO_REQ_HDR_RECEIVED 10
+#define HTTP_CO_REQUEST_RECEIVED 11

 #define DEFAULT_TIMEOUT 5000

-http_context_t* http_context_init(const std::string & ssl_cert, const std::string & ssl_key,
+void http_ares_cb(void *data, ares_socket_t socket_fd, int readable, int writable)
+{
+    http_context_t *ctx = (http_context_t *)data;
+    ctx->tfd->set_fd_handler(socket_fd, writable, [ctx](int fd, int epoll_events)
+    {
+        ares_process_fd(ctx->ares, (epoll_events & (EPOLLIN|EPOLLRDHUP)) ? fd : 0, (epoll_events & EPOLLOUT) ? fd : 0);
+    });
+}
+
+http_context_t* http_context_init(timerfd_manager_t *tfd, const std::string & ssl_cert, const std::string & ssl_key,
    const std::string & ssl_ca, bool verify_peer, std::string & error)
 {
    http_context_t *ctx = new http_context_t;
+    ctx->tfd = tfd;
+    ares_options opts = {
+        .sock_state_cb = http_ares_cb,
+        .sock_state_cb_data = ctx,
+    };
+    ares_init_options(&ctx->ares, &opts, ARES_OPT_SOCK_STATE_CB);
 #ifdef WITH_OPENSSL
    SSL_CTX *ssl_ctx = SSL_CTX_new(TLS_method());
    ctx->ssl_cert = ssl_cert;
@@ -175,10 +204,10 @@ void http_context_destroy(http_context_t *ctx)
    delete ctx;
 }

-http_co_t *http_init(timerfd_manager_t *tfd, http_context_t *ctx)
+http_co_t *http_init(http_context_t *ctx)
 {
    http_co_t *handler = new http_co_t();
-    handler->tfd = tfd;
+    handler->tfd = ctx->tfd;
    handler->state = HTTP_CO_CLOSED;
    handler->ctx = ctx;
    return handler;
@@ -199,6 +228,7 @@ void open_websocket(http_co_t *handler, const std::string & host, const std::str
        "Sec-WebSocket-Version: 13\r\n"
        "\r\n";
    handler->host = host;
+    handler->host_port = "";
    handler->request_timeout = options.timeout < 0 ? -1 : (options.timeout == 0 ? DEFAULT_TIMEOUT : options.timeout);
    handler->want_streaming = false;
    handler->keepalive = false;
@@ -223,6 +253,7 @@ void http_serve(http_co_t *handler, int peer_fd, const http_options_t & options,
    if (handler->state != HTTP_CO_SERVER || handler->peer_fd != peer_fd)
        handler->close_connection();
    handler->host = "";
+    handler->host_port = "";
    handler->request_timeout = options.timeout < 0 ? -1 : (options.timeout == 0 ? DEFAULT_TIMEOUT : options.timeout);
    handler->want_streaming = false;
    handler->keepalive = false;
@@ -277,6 +308,7 @@ void http_co_t::send_request(const std::string & host, const std::string & reque
    this->keepalive = options.keepalive;
    this->ssl = options.ssl;
    this->host = host;
+    this->host_port = "";
    this->request = request;
    this->response = "";
    this->sent = 0;
@@ -289,7 +321,7 @@ void http_co_t::send_request(const std::string & host, const std::string & reque
    }
    else
    {
-        start_connection();
+        resolve_and_connect();
    }
    // Do it _after_ state assignment because set_timer() can actually trigger
    // other timers and requests (reenterability is our friend)
@@ -435,7 +467,7 @@ void http_co_t::close_connection()
 void http_co_t::start_ws_connection()
 {
    stackin();
-    start_connection();
+    resolve_and_connect();
    if (request_timeout > 0)
    {
        timeout_id = tfd->set_timer(request_timeout, false, [this](int timer_id)
@@ -453,19 +485,72 @@ void http_co_t::start_ws_connection()
    stackout();
 }

-void http_co_t::start_connection()
+void http_ares_addrinfo_cb(void *data, int status, int timeouts, struct ares_addrinfo *result)
+{
+    ((http_co_t*)data)->ares_addrinfo_cb(status, timeouts, result);
+}
+
+void http_co_t::ares_addrinfo_cb(int status, int timeouts, struct ares_addrinfo *result)
 {
    stackin();
-    struct sockaddr_storage addr;
-    if (!string_to_addr(host.c_str(), 1, 80, &addr))
+    if (status != ARES_SUCCESS)
    {
        close_connection();
-        parsed = { .error = "Invalid address: "+host, .status_code = EINVAL };
+        parsed = { .error = ares_strerror(status), .status_code = EINVAL };
        run_cb_and_clear();
        stackout();
        return;
    }
-    peer_fd = socket(addr.ss_family, SOCK_STREAM, 0);
+    ares_addrinfo_node *node = NULL;
+    int count = 0;
+    for (node = result->nodes; node; node = node->ai_next)
+    {
+        count++;
+    }
+    assert(count > 0);
+    int pos = lrand48() % count;
+    for (node = result->nodes; node && pos > 0; node = node->ai_next)
+    {
+        pos--;
+    }
+    assert(node);
+    start_connection(node->ai_addr, node->ai_addrlen);
+    stackout();
+}
+
+void http_co_t::resolve_and_connect()
+{
+    stackin();
+    struct sockaddr_storage addr;
+    if (!string_to_addr(host.c_str(), 1, ssl ? 443 : 80, &addr))
+    {
+        // Try to resolve host via c-ares
+        state = HTTP_CO_RESOLVING;
+        ares_addrinfo_hints hints = { .ai_flags = ARES_AI_NOSORT|ARES_AI_NUMERICSERV };
+        if (host_port.empty())
+        {
+            auto pos = host.rfind(':');
+            if (pos != std::string::npos)
+            {
+                host_port = host.substr(pos+1);
+                host = host.substr(0, pos);
+            }
+            else
+                host_port = ssl ? "443" : "80";
+        }
+        ares_getaddrinfo(ctx->ares, host.c_str(), host_port.c_str(), &hints, http_ares_addrinfo_cb, this);
+    }
+    else
+    {
+        start_connection((sockaddr*)&addr, sizeof(addr));
+    }
+    stackout();
+}
+
+void http_co_t::start_connection(sockaddr *addr, size_t addr_size)
+{
+    stackin();
+    peer_fd = socket(addr->sa_family, SOCK_STREAM, 0);
    if (peer_fd < 0)
    {
        close_connection();
@@ -514,7 +599,7 @@ init_err:
    }
 #endif
    // Finally call connect
-    int r = ::connect(peer_fd, (sockaddr*)&addr, sizeof(addr));
+    int r = ::connect(peer_fd, addr, addr_size);
    if (r < 0 && errno != EINPROGRESS)
    {
        close_connection();
@@ -46,10 +46,10 @@ struct http_message_t
 // Opened websocket or keepalive HTTP connection
 struct http_co_t;

-http_context_t* http_context_init(const std::string & ssl_cert, const std::string & ssl_key,
+http_context_t* http_context_init(timerfd_manager_t *tfd, const std::string & ssl_cert, const std::string & ssl_key,
    const std::string & ssl_ca, bool verify_peer, std::string & error);
 void http_context_destroy(http_context_t *ctx);
-http_co_t* http_init(timerfd_manager_t *tfd, http_context_t *ctx = NULL);
+http_co_t* http_init(http_context_t *ctx = NULL);
 void open_websocket(http_co_t *handler, const std::string & host, const std::string & path,
    const http_options_t & options, std::function<void(http_message_t *msg)> on_message);
 void http_request(http_co_t *handler, const std::string & host, const std::string & request,
@@ -114,7 +114,7 @@ struct cli_serve_t
            std::string ssl_key = options["ssl_key"].string_value();
            std::string ssl_ca = options["ssl_ca"].string_value();
            std::string error;
-            http_ctx = http_context_init(ssl_cert, ssl_key, ssl_ca, ssl_ca != "", error);
+            http_ctx = http_context_init(parent->epmgr->tfd, ssl_cert, ssl_key, ssl_ca, ssl_ca != "", error);
            if (error != "")
            {
                result = (cli_result_t){ .err = EINVAL, .text = error };
@@ -176,7 +176,7 @@ struct cli_serve_t
            cli_serve_conn_t *conn = new cli_serve_conn_t;
            conn->peer_fd = peer_fd;
            conn->peer_addr = peer_addr_str;
-            conn->co = http_init(parent->epmgr->tfd, http_ctx);
+            conn->co = http_init(http_ctx);
            http_serve(conn->co, peer_fd, (http_options_t){ .ssl = ssl }, [this, conn](http_message_t *msg)
            {
                process_request(conn, msg);