Support DNS resolving via libc-ares

debian/control

@@ -3,7 +3,7 @@ Section: admin
 Priority: optional
 Maintainer: Vitaliy Filippov <vitalif@yourcmc.ru>
 Build-Depends: debhelper, g++ (>= 8), libstdc++6 (>= 8),
-  linux-libc-dev, libgoogle-perftools-dev, libjerasure-dev, libgf-complete-dev,
+  linux-libc-dev, libgoogle-perftools-dev, libjerasure-dev, libgf-complete-dev, libc-ares-dev,
   libibverbs-dev, librdmacm-dev, libisal-dev, cmake, pkg-config, libnl-3-dev, libnl-genl-3-dev,
   node-bindings <!nocheck>, node-gyp, node-nan
 Standards-Version: 4.5.0

debian/vitastor-buildenv.Dockerfile

@@ -25,7 +25,7 @@ RUN set -e -x; \
     echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
 
 RUN apt-get update && \
-    apt-get -y install fio libgoogle-perftools-dev devscripts libjerasure-dev cmake \
+    apt-get -y install fio libgoogle-perftools-dev devscripts libjerasure-dev cmake libc-ares-dev \
         libibverbs-dev librdmacm-dev libisal-dev libnl-3-dev libnl-genl-3-dev curl nodejs npm node-nan node-bindings && \
     apt-get -y build-dep fio && \
     apt-get --download-only source fio

docs/installation/source.en.md

@@ -15,8 +15,8 @@
 - gcc and g++ 8 or newer, clang 10 or newer, or other compiler with C++11 plus
   designated initializers support from C++20
 - CMake
-- jerasure headers and libraries
+- jerasure, c-ares headers and libraries
-- ISA-L, libibverbs and librdmacm headers and libraries (optional)
+- ISA-L, libibverbs, librdmacm, libnl3 headers and libraries (optional)
 - tcmalloc (google-perftools-dev)
 
 ## Basic instructions

docs/installation/source.ru.md

@@ -15,8 +15,8 @@
 - gcc и g++ >= 8, либо clang >= 10, либо другой компилятор с поддержкой C++11 плюс
   назначенных инициализаторов (designated initializers) из C++20
 - CMake
-- Заголовки и библиотеки jerasure
+- Заголовки и библиотеки jerasure, c-ares
-- Опционально - заголовки и библиотеки ISA-L, libibverbs, librdmacm
+- Опционально - заголовки и библиотеки ISA-L, libibverbs, librdmacm, libnl3
 - tcmalloc (google-perftools-dev)
 
 ## Базовая инструкция

rpm/vitastor-el10.Dockerfile

@@ -11,7 +11,7 @@ WORKDIR /root
 RUN sed -i 's/enabled=0/enabled=1/' /etc/yum.repos.d/*.repo
 RUN dnf -y install epel-release dnf-plugins-core
 RUN dnf -y install https://vitastor.io/rpms/centos/10/vitastor-release-1.0-1.el10.noarch.rpm
-RUN dnf -y install gcc-c++ gperftools-devel fio nodejs rpm-build jerasure-devel isa-l-devel gf-complete-devel rdma-core-devel cmake libnl3-devel
+RUN dnf -y install gcc-c++ gperftools-devel fio nodejs rpm-build jerasure-devel isa-l-devel gf-complete-devel rdma-core-devel cmake libnl3-devel c-ares-devel
 RUN dnf download --source fio
 RUN rpm --nomd5 -i fio*.src.rpm
 RUN cd ~/rpmbuild/SPECS && dnf builddep -y --spec fio.spec

rpm/vitastor-el10.spec

@@ -16,6 +16,7 @@ BuildRequires:  gf-complete-devel
 BuildRequires:  rdma-core-devel
 BuildRequires:  cmake
 BuildRequires:  libnl3-devel
+BuildRequires:  c-ares-devel
 Requires:       vitastor-osd = %{version}-%{release}
 Requires:       vitastor-mon = %{version}-%{release}
 Requires:       vitastor-client = %{version}-%{release}

rpm/vitastor-el7.Dockerfile

@@ -15,7 +15,7 @@ RUN yum -y --enablerepo=extras install centos-release-scl epel-release yum-utils
 RUN perl -i -pe 's!mirrorlist=!#mirrorlist=!s; s!#\s*baseurl=http://mirror.centos.org!baseurl=http://vault.centos.org!' /etc/yum.repos.d/CentOS-SCLo-scl*.repo
 RUN yum -y install https://vitastor.io/rpms/centos/7/vitastor-release-1.0-1.el7.noarch.rpm
 RUN yum -y install devtoolset-9-gcc-c++ devtoolset-9-libatomic-devel gcc make cmake gperftools-devel \
-    fio rh-nodejs12 jerasure-devel libisa-l-devel gf-complete-devel rdma-core-devel libnl3-devel
+    fio rh-nodejs12 jerasure-devel libisa-l-devel gf-complete-devel rdma-core-devel libnl3-devel c-ares-devel
 RUN yumdownloader --disablerepo=centos-sclo-rh --source fio
 RUN rpm --nomd5 -i fio*.src.rpm
 RUN rm -f /etc/yum.repos.d/CentOS-Media.repo

rpm/vitastor-el7.spec

@@ -17,6 +17,7 @@ BuildRequires:  gf-complete-devel
 BuildRequires:  rdma-core-devel
 BuildRequires:  cmake3
 BuildRequires:  libnl3-devel
+BuildRequires:  c-ares-devel
 Requires:       vitastor-osd = %{version}-%{release}
 Requires:       vitastor-mon = %{version}-%{release}
 Requires:       vitastor-client = %{version}-%{release}

rpm/vitastor-el8.Dockerfile

@@ -13,7 +13,7 @@ RUN dnf -y install centos-release-advanced-virtualization epel-release dnf-plugi
 RUN sed -i 's/^mirrorlist=/#mirrorlist=/; s!#baseurl=.*!baseurl=http://vault.centos.org/centos/8.4.2105/virt/$basearch/$avdir/!; s!^baseurl=.*Source/.*!baseurl=http://vault.centos.org/centos/8.4.2105/virt/Source/advanced-virtualization/!' /etc/yum.repos.d/CentOS-Advanced-Virtualization.repo
 RUN yum -y install https://vitastor.io/rpms/centos/8/vitastor-release-1.0-1.el8.noarch.rpm
 RUN dnf -y install gcc-toolset-9 gcc-toolset-9-gcc-c++ gperftools-devel \
-    fio nodejs rpm-build jerasure-devel libisa-l-devel gf-complete-devel libibverbs-devel libarchive cmake libnl3-devel
+    fio nodejs rpm-build jerasure-devel libisa-l-devel gf-complete-devel libibverbs-devel libarchive cmake libnl3-devel c-ares-devel
 RUN dnf download --source fio
 RUN rpm --nomd5 -i fio*.src.rpm
 RUN cd ~/rpmbuild/SPECS && dnf builddep -y --enablerepo=powertools --spec fio.spec

rpm/vitastor-el8.spec

@@ -16,6 +16,7 @@ BuildRequires:  gf-complete-devel
 BuildRequires:  rdma-core-devel
 BuildRequires:  cmake
 BuildRequires:  libnl3-devel
+BuildRequires:  c-ares-devel
 Requires:       vitastor-osd = %{version}-%{release}
 Requires:       vitastor-mon = %{version}-%{release}
 Requires:       vitastor-client = %{version}-%{release}

rpm/vitastor-el9.Dockerfile

@@ -10,7 +10,7 @@ WORKDIR /root
 RUN sed -i 's/enabled=0/enabled=1/' /etc/yum.repos.d/*.repo
 RUN dnf -y install epel-release dnf-plugins-core
 RUN dnf -y install https://vitastor.io/rpms/centos/9/vitastor-release-1.0-1.el9.noarch.rpm
-RUN dnf -y install gcc-c++ gperftools-devel fio nodejs rpm-build jerasure-devel libisa-l-devel gf-complete-devel rdma-core-devel libarchive cmake libnl3-devel
+RUN dnf -y install gcc-c++ gperftools-devel fio nodejs rpm-build jerasure-devel libisa-l-devel gf-complete-devel rdma-core-devel libarchive cmake libnl3-devel c-ares-devel
 RUN dnf download --source fio
 RUN rpm --nomd5 -i fio*.src.rpm
 RUN cd ~/rpmbuild/SPECS && dnf builddep -y --spec fio.spec

rpm/vitastor-el9.spec

@@ -16,6 +16,7 @@ BuildRequires:  gf-complete-devel
 BuildRequires:  rdma-core-devel
 BuildRequires:  cmake
 BuildRequires:  libnl3-devel
+BuildRequires:  c-ares-devel
 Requires:       vitastor-osd = %{version}-%{release}
 Requires:       vitastor-mon = %{version}-%{release}
 Requires:       vitastor-client = %{version}-%{release}

src/CMakeLists.txt

@@ -79,6 +79,9 @@ if (OPENSSL_FOUND)
 	add_definitions(-DWITH_OPENSSL)
 endif (OPENSSL_FOUND)
 
+pkg_check_modules(CARES REQUIRED libcares)
+include_directories(${CARES_INCLUDE_DIRS})
+
 if (${WITH_SYSTEM_LIBURING})
 	pkg_check_modules(LIBURING REQUIRED liburing>=2.10)
 	include_directories(${LIBURING_INCLUDE_DIRS})

src/client/CMakeLists.txt

@@ -16,7 +16,7 @@ add_library(vitastor_common STATIC
 	msgr_encrypt.cpp msgr_stop.cpp msgr_op.cpp msgr_send.cpp msgr_receive.cpp ../util/ringloop.cpp ../../json11/json11.cpp
 	http_client.cpp osd_ops.cpp pg_states.cpp ../util/timerfd_manager.cpp ../util/str_util.cpp ../util/json_util.cpp ${MSGR_RDMA} ${MSGR_RDMACM}
 )
-target_link_libraries(vitastor_common pthread ${OPENSSL_LIBRARIES})
+target_link_libraries(vitastor_common pthread ${OPENSSL_LIBRARIES} ${CARES_LIBRARIES})
 target_compile_options(vitastor_common PUBLIC -fPIC)
 
 # libvitastor_client.so

src/client/etcd_state_client.cpp

@@ -84,7 +84,7 @@ http_context_t *etcd_state_client_t::get_http_ctx()
     if (!http_ctx)
     {
         std::string error;
-        http_ctx = http_context_init(etcd_client_cert, etcd_client_key, etcd_ca, true, error);
+        http_ctx = http_context_init(tfd, etcd_client_cert, etcd_client_key, etcd_ca, true, error);
         if (!http_ctx)
         {
             fprintf(stderr, "Failed to initialize HTTP context: %s\n", error.c_str());
@@ -113,7 +113,7 @@ void etcd_state_client_t::etcd_call_oneshot(std::string etcd_address, std::strin
         "Content-Length: "+std::to_string(req.size())+"\r\n"
         "Connection: close\r\n"
         "\r\n"+req;
-    auto http_cli = http_init(tfd, get_http_ctx());
+    auto http_cli = http_init(get_http_ctx());
     auto cb = [http_cli, callback](http_message_t *response)
     {
         std::string err;
@@ -190,7 +190,7 @@ void etcd_state_client_t::etcd_call(std::string api, json11::Json payload, int t
             callback(err, data);
     };
     if (!keepalive_client)
-        keepalive_client = http_init(tfd, get_http_ctx());
+        keepalive_client = http_init(get_http_ctx());
     http_request(keepalive_client, etcd_address, req, { .timeout = timeout, .keepalive = true, .ssl = ssl }, cb);
 }
 
@@ -384,7 +384,7 @@ void etcd_state_client_t::start_etcd_watcher()
             etcd_watch_revision_config, etcd_watch_revision_osd, etcd_watch_revision_pg);
     }
     if (!etcd_watch_ws)
-        etcd_watch_ws = http_init(tfd, get_http_ctx());
+        etcd_watch_ws = http_init(get_http_ctx());
     else
         http_close(etcd_watch_ws);
     open_websocket(etcd_watch_ws, etcd_address, etcd_api_path+"/watch", { .timeout = etcd_slow_timeout, .ssl = ssl },

src/client/http_client.cpp

@@ -21,6 +21,9 @@
 #include <openssl/ssl.h>
 #endif
 
+// libc-ares
+#include <ares.h>
+
 #include "addr_util.h"
 #include "str_util.h"
 #include "json_util.h"
@@ -36,6 +39,9 @@ static void parse_http_headers(std::string & res, http_message_t *parsed, bool i
 
 struct http_context_t
 {
+    timerfd_manager_t *tfd = NULL;
+    ares_channel ares = NULL;
+
     std::string ssl_cert;
     std::string ssl_key;
     std::string ssl_ca;
@@ -46,6 +52,8 @@ struct http_context_t
 
     ~http_context_t()
     {
+        ares_destroy(ares);
+        ares = NULL;
 #ifdef WITH_OPENSSL
         if (ssl_ctx)
         {
@@ -72,12 +80,14 @@ struct http_co_t
     BIO *ssl_bio = NULL;
 #endif
 
-    timerfd_manager_t *tfd;
+    timerfd_manager_t *tfd = NULL;
+
     std::function<void(http_message_t*)> response_callback;
 
     int request_timeout = 0;
     bool ssl = false;
     std::string host;
+    std::string host_port;
     std::string request;
     std::string ws_outbox;
     std::string response;
@@ -106,7 +116,8 @@ struct http_co_t
     inline void stackout() { onstack--; if (!onstack && ended) end(); }
     inline void end() { ended = true; if (!onstack) { delete this; } }
     void run_cb_and_clear();
-    void start_connection();
+    void resolve_and_connect();
+    void start_connection(sockaddr *addr, size_t addr_size);
     void start_ws_connection();
     void close_connection();
     void next_request();
@@ -123,26 +134,44 @@ struct http_co_t
     void reply(const std::string & msg);
     void send_request(const std::string & host, const std::string & request,
         const http_options_t & options, std::function<void(http_message_t *response)> response_callback);
+
+    void ares_addrinfo_cb(int status, int timeouts, struct ares_addrinfo *result);
 };
 
 #define HTTP_CO_CLOSED 0
-#define HTTP_CO_CONNECTING 1
+#define HTTP_CO_RESOLVING 1
-#define HTTP_CO_SENDING_REQUEST 2
+#define HTTP_CO_CONNECTING 2
-#define HTTP_CO_REQUEST_SENT 3
+#define HTTP_CO_SENDING_REQUEST 3
-#define HTTP_CO_HEADERS_RECEIVED 4
+#define HTTP_CO_REQUEST_SENT 4
-#define HTTP_CO_WEBSOCKET 5
+#define HTTP_CO_HEADERS_RECEIVED 5
-#define HTTP_CO_CHUNKED 6
+#define HTTP_CO_WEBSOCKET 6
-#define HTTP_CO_KEEPALIVE 7
+#define HTTP_CO_CHUNKED 7
-#define HTTP_CO_SERVER 8
+#define HTTP_CO_KEEPALIVE 8
-#define HTTP_CO_REQ_HDR_RECEIVED 9
+#define HTTP_CO_SERVER 9
-#define HTTP_CO_REQUEST_RECEIVED 10
+#define HTTP_CO_REQ_HDR_RECEIVED 10
+#define HTTP_CO_REQUEST_RECEIVED 11
 
 #define DEFAULT_TIMEOUT 5000
 
-http_context_t* http_context_init(const std::string & ssl_cert, const std::string & ssl_key,
+void http_ares_cb(void *data, ares_socket_t socket_fd, int readable, int writable)
+{
+    http_context_t *ctx = (http_context_t *)data;
+    ctx->tfd->set_fd_handler(socket_fd, writable, [ctx](int fd, int epoll_events)
+    {
+        ares_process_fd(ctx->ares, (epoll_events & (EPOLLIN|EPOLLRDHUP)) ? fd : 0, (epoll_events & EPOLLOUT) ? fd : 0);
+    });
+}
+
+http_context_t* http_context_init(timerfd_manager_t *tfd, const std::string & ssl_cert, const std::string & ssl_key,
     const std::string & ssl_ca, bool verify_peer, std::string & error)
 {
     http_context_t *ctx = new http_context_t;
+    ctx->tfd = tfd;
+    ares_options opts = {
+        .sock_state_cb = http_ares_cb,
+        .sock_state_cb_data = ctx,
+    };
+    ares_init_options(&ctx->ares, &opts, ARES_OPT_SOCK_STATE_CB);
 #ifdef WITH_OPENSSL
     SSL_CTX *ssl_ctx = SSL_CTX_new(TLS_method());
     ctx->ssl_cert = ssl_cert;
@@ -175,10 +204,10 @@ void http_context_destroy(http_context_t *ctx)
     delete ctx;
 }
 
-http_co_t *http_init(timerfd_manager_t *tfd, http_context_t *ctx)
+http_co_t *http_init(http_context_t *ctx)
 {
     http_co_t *handler = new http_co_t();
-    handler->tfd = tfd;
+    handler->tfd = ctx->tfd;
     handler->state = HTTP_CO_CLOSED;
     handler->ctx = ctx;
     return handler;
@@ -199,6 +228,7 @@ void open_websocket(http_co_t *handler, const std::string & host, const std::str
         "Sec-WebSocket-Version: 13\r\n"
         "\r\n";
     handler->host = host;
+    handler->host_port = "";
     handler->request_timeout = options.timeout < 0 ? -1 : (options.timeout == 0 ? DEFAULT_TIMEOUT : options.timeout);
     handler->want_streaming = false;
     handler->keepalive = false;
@@ -223,6 +253,7 @@ void http_serve(http_co_t *handler, int peer_fd, const http_options_t & options,
     if (handler->state != HTTP_CO_SERVER || handler->peer_fd != peer_fd)
         handler->close_connection();
     handler->host = "";
+    handler->host_port = "";
     handler->request_timeout = options.timeout < 0 ? -1 : (options.timeout == 0 ? DEFAULT_TIMEOUT : options.timeout);
     handler->want_streaming = false;
     handler->keepalive = false;
@@ -277,6 +308,7 @@ void http_co_t::send_request(const std::string & host, const std::string & reque
     this->keepalive = options.keepalive;
     this->ssl = options.ssl;
     this->host = host;
+    this->host_port = "";
     this->request = request;
     this->response = "";
     this->sent = 0;
@@ -289,7 +321,7 @@ void http_co_t::send_request(const std::string & host, const std::string & reque
     }
     else
     {
-        start_connection();
+        resolve_and_connect();
     }
     // Do it _after_ state assignment because set_timer() can actually trigger
     // other timers and requests (reenterability is our friend)
@@ -435,7 +467,7 @@ void http_co_t::close_connection()
 void http_co_t::start_ws_connection()
 {
     stackin();
-    start_connection();
+    resolve_and_connect();
     if (request_timeout > 0)
     {
         timeout_id = tfd->set_timer(request_timeout, false, [this](int timer_id)
@@ -453,19 +485,72 @@ void http_co_t::start_ws_connection()
     stackout();
 }
 
-void http_co_t::start_connection()
+void http_ares_addrinfo_cb(void *data, int status, int timeouts, struct ares_addrinfo *result)
+{
+    ((http_co_t*)data)->ares_addrinfo_cb(status, timeouts, result);
+}
+
+void http_co_t::ares_addrinfo_cb(int status, int timeouts, struct ares_addrinfo *result)
 {
     stackin();
-    struct sockaddr_storage addr;
+    if (status != ARES_SUCCESS)
-    if (!string_to_addr(host.c_str(), 1, 80, &addr))
     {
         close_connection();
-        parsed = { .error = "Invalid address: "+host, .status_code = EINVAL };
+        parsed = { .error = ares_strerror(status), .status_code = EINVAL };
         run_cb_and_clear();
         stackout();
         return;
     }
-    peer_fd = socket(addr.ss_family, SOCK_STREAM, 0);
+    ares_addrinfo_node *node = NULL;
+    int count = 0;
+    for (node = result->nodes; node; node = node->ai_next)
+    {
+        count++;
+    }
+    assert(count > 0);
+    int pos = lrand48() % count;
+    for (node = result->nodes; node && pos > 0; node = node->ai_next)
+    {
+        pos--;
+    }
+    assert(node);
+    start_connection(node->ai_addr, node->ai_addrlen);
+    stackout();
+}
+
+void http_co_t::resolve_and_connect()
+{
+    stackin();
+    struct sockaddr_storage addr;
+    if (!string_to_addr(host.c_str(), 1, ssl ? 443 : 80, &addr))
+    {
+        // Try to resolve host via c-ares
+        state = HTTP_CO_RESOLVING;
+        ares_addrinfo_hints hints = { .ai_flags = ARES_AI_NOSORT|ARES_AI_NUMERICSERV };
+        if (host_port.empty())
+        {
+            auto pos = host.rfind(':');
+            if (pos != std::string::npos)
+            {
+                host_port = host.substr(pos+1);
+                host = host.substr(0, pos);
+            }
+            else
+                host_port = ssl ? "443" : "80";
+        }
+        ares_getaddrinfo(ctx->ares, host.c_str(), host_port.c_str(), &hints, http_ares_addrinfo_cb, this);
+    }
+    else
+    {
+        start_connection((sockaddr*)&addr, sizeof(addr));
+    }
+    stackout();
+}
+
+void http_co_t::start_connection(sockaddr *addr, size_t addr_size)
+{
+    stackin();
+    peer_fd = socket(addr->sa_family, SOCK_STREAM, 0);
     if (peer_fd < 0)
     {
         close_connection();
@@ -514,7 +599,7 @@ init_err:
     }
 #endif
     // Finally call connect
-    int r = ::connect(peer_fd, (sockaddr*)&addr, sizeof(addr));
+    int r = ::connect(peer_fd, addr, addr_size);
     if (r < 0 && errno != EINPROGRESS)
     {
         close_connection();

src/client/http_client.h

@@ -46,10 +46,10 @@ struct http_message_t
 // Opened websocket or keepalive HTTP connection
 struct http_co_t;
 
-http_context_t* http_context_init(const std::string & ssl_cert, const std::string & ssl_key,
+http_context_t* http_context_init(timerfd_manager_t *tfd, const std::string & ssl_cert, const std::string & ssl_key,
     const std::string & ssl_ca, bool verify_peer, std::string & error);
 void http_context_destroy(http_context_t *ctx);
-http_co_t* http_init(timerfd_manager_t *tfd, http_context_t *ctx = NULL);
+http_co_t* http_init(http_context_t *ctx = NULL);
 void open_websocket(http_co_t *handler, const std::string & host, const std::string & path,
     const http_options_t & options, std::function<void(http_message_t *msg)> on_message);
 void http_request(http_co_t *handler, const std::string & host, const std::string & request,

src/cmd/cli_serve.cpp

@@ -114,7 +114,7 @@ struct cli_serve_t
             std::string ssl_key = options["ssl_key"].string_value();
             std::string ssl_ca = options["ssl_ca"].string_value();
             std::string error;
-            http_ctx = http_context_init(ssl_cert, ssl_key, ssl_ca, ssl_ca != "", error);
+            http_ctx = http_context_init(parent->epmgr->tfd, ssl_cert, ssl_key, ssl_ca, ssl_ca != "", error);
             if (error != "")
             {
                 result = (cli_result_t){ .err = EINVAL, .text = error };
@@ -176,7 +176,7 @@ struct cli_serve_t
             cli_serve_conn_t *conn = new cli_serve_conn_t;
             conn->peer_fd = peer_fd;
             conn->peer_addr = peer_addr_str;
-            conn->co = http_init(parent->epmgr->tfd, http_ctx);
+            conn->co = http_init(http_ctx);
             http_serve(conn->co, peer_fd, (http_options_t){ .ssl = ssl }, [this, conn](http_message_t *msg)
             {
                 process_request(conn, msg);