Implement etcd SSL support via OpenSSL

Maybe I should remove all of this and use libwebsockets :)
2026-04-17 13:53:40 +03:00
parent a76d5ccc0d
commit 2846eba3af
12 changed files with 338 additions and 33 deletions
@@ -234,6 +234,60 @@ jobs:
          echo ""
        done

+  test_etcd_fail_https:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: ETCD_SCHEME=https /root/vitastor/tests/test_etcd_fail.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_etcd_fail_https_antietcd:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: ETCD_SCHEME=https ANTIETCD=1 /root/vitastor/tests/test_etcd_fail.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_snapshot_https:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: ETCD_SCHEME=https /root/vitastor/tests/test_snapshot.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
  test_interrupted_rebalance:
    runs-on: ubuntu-latest
    needs: build
@@ -38,6 +38,10 @@ for my $line (<>)
            {
                $test_name .= '_antietcd';
            }
+            elsif ($1 eq 'ETCD_SCHEME' && $2 eq 'https')
+            {
+                $test_name .= '_https';
+            }
            elsif ($1 eq 'OLD')
            {
                $test_name =~ s/^test_/test_old_/s;
@@ -1,7 +1,9 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)

+const fs = require('fs');
 const http = require('http');
+const https = require('https');
 const WebSocket = require('ws');
 const { b64, local_ips } = require('./utils.js');

@@ -15,11 +17,14 @@ class EtcdAdapter
        this.ws = null;
        this.ws_alive = false;
        this.ws_keepalive_timer = null;
+        this.opts = {};
    }

    parse_config(config)
    {
        this.parse_etcd_addresses(config.etcd_address||config.etcd_url);
+        if (config.etcd_ca)
+            this.opts.ca = fs.readFileSync(config.etcd_ca, { encoding: 'utf-8' });
    }

    parse_etcd_addresses(addrs)
@@ -39,7 +44,7 @@ class EtcdAdapter
        for (let url of addrs)
        {
            let scheme = 'http';
-            url = url.trim().replace(/^(https?):\/\//, (m, m1) => { scheme = m1; return ''; });
+            url = url.trim().replace(/^(https?):\/\//i, (m, m1) => { scheme = m1.toLowerCase(); return ''; });
            const slash = url.indexOf('/');
            const colon = url.indexOf(':');
            const is_local = is_local_ip[colon >= 0 ? url.substr(0, colon) : (slash >= 0 ? url.substr(0, slash) : url)];
@@ -130,7 +135,7 @@ class EtcdAdapter
                    }
                    ok(false);
                }, this.mon.config.etcd_mon_timeout);
-                this.ws = new WebSocket(base+'/watch');
+                this.ws = new WebSocket(base+'/watch', this.opts);
                this.ws_used_url = cur_addr;
                const fail = () =>
                {
@@ -272,7 +277,7 @@ class EtcdAdapter
            {
                throw new Error(MON_STOPPED);
            }
-            const res = await POST(base+path, body, timeout);
+            const res = await POST(base+path, body, timeout, this.opts);
            if (this.mon.stopped)
            {
                throw new Error(MON_STOPPED);
@@ -298,7 +303,7 @@ class EtcdAdapter
    }
 }

-function POST(url, body, timeout)
+function POST(url, body, timeout, opts)
 {
    return new Promise(ok =>
    {
@@ -310,10 +315,10 @@ function POST(url, body, timeout)
            req = null;
            ok({ error: 'timeout' });
        }, timeout) : null;
-        let req = http.request(url, { method: 'POST', headers: {
+        let req = (url.substr(0, 5) == 'https' ? https : http).request(url, { method: 'POST', headers: {
            'Content-Type': 'application/json',
            'Content-Length': body_text.length,
-        } }, (res) =>
+        }, ...(opts||{}) }, (res) =>
        {
            if (!req)
            {
@@ -74,6 +74,11 @@ if (RDMACM_LIBRARIES)
 	add_definitions(-DWITH_RDMACM)
 endif (RDMACM_LIBRARIES)

+find_package(OpenSSL)
+if (OPENSSL_FOUND)
+	add_definitions(-DWITH_OPENSSL)
+endif (OPENSSL_FOUND)
+
 if (${WITH_SYSTEM_LIBURING})
 	pkg_check_modules(LIBURING REQUIRED liburing>=2.10)
 	include_directories(${LIBURING_INCLUDE_DIRS})
@@ -33,6 +33,7 @@ target_link_libraries(vitastor_client
 	${LIBURING_LIBRARIES}
 	${IBVERBS_LIBRARIES}
 	${RDMACM_LIBRARIES}
+	${OPENSSL_LIBRARIES}
 )
 set_target_properties(vitastor_client PROPERTIES VERSION ${VITASTOR_VERSION} SOVERSION 0)
 configure_file(vitastor.pc.in vitastor.pc @ONLY)
@@ -76,6 +76,8 @@ void etcd_state_client_t::etcd_call_oneshot(std::string etcd_address, std::strin
    int timeout, std::function<void(std::string, json11::Json)> callback)
 {
    std::string etcd_api_path;
+    bool ssl = etcd_address.substr(0, 8) == "https://";
+    etcd_address = etcd_address.substr(ssl ? 8 : 7);
    int pos = etcd_address.find('/');
    if (pos >= 0)
    {
@@ -98,7 +100,7 @@ void etcd_state_client_t::etcd_call_oneshot(std::string etcd_address, std::strin
        callback(err, data);
        http_close(http_cli);
    };
-    http_request(http_cli, etcd_address, req, { .timeout = timeout }, cb);
+    http_request(http_cli, etcd_address, req, { .timeout = timeout, .ssl = ssl, .ssl_ca = etcd_ca }, cb);
 }

 void etcd_state_client_t::etcd_call(std::string api, json11::Json payload, int timeout,
@@ -112,6 +114,8 @@ void etcd_state_client_t::etcd_call(std::string api, json11::Json payload, int t
    pick_next_etcd();
    std::string etcd_address = selected_etcd_address;
    std::string etcd_api_path;
+    bool ssl = etcd_address.substr(0, 8) == "https://";
+    etcd_address = etcd_address.substr(ssl ? 8 : 7);
    int pos = etcd_address.find('/');
    if (pos >= 0)
    {
@@ -167,19 +171,20 @@ void etcd_state_client_t::etcd_call(std::string api, json11::Json payload, int t
    {
        keepalive_client = http_init(tfd);
    }
-    http_request(keepalive_client, etcd_address, req, { .timeout = timeout, .keepalive = true }, cb);
+    http_request(keepalive_client, etcd_address, req, { .timeout = timeout, .keepalive = true, .ssl = ssl, .ssl_ca = etcd_ca }, cb);
 }

 void etcd_state_client_t::add_etcd_url(std::string addr)
 {
    if (addr.length() > 0)
    {
+        bool ssl = false;
        if (strtolower(addr.substr(0, 7)) == "http://")
            addr = addr.substr(7);
        else if (strtolower(addr.substr(0, 8)) == "https://")
        {
-            fprintf(stderr, "HTTPS is unsupported for etcd. Either use plain HTTP or setup a local proxy for etcd interaction\n");
-            exit(1);
+            addr = addr.substr(8);
+            ssl = true;
        }
        if (!local_ips.size())
            local_ips = getifaddr_list(std::vector<addr_mask_t>(), true);
@@ -194,6 +199,7 @@ void etcd_state_client_t::add_etcd_url(std::string addr)
            check_addr = addr;
        if (pos == std::string::npos)
            addr += "/v3";
+        addr = (ssl ? "https://" : "http://") + addr;
        bool local = false;
        int i;
        for (i = 0; i < local_ips.size(); i++)
@@ -239,6 +245,7 @@ void etcd_state_client_t::parse_config(const json11::Json & config)
            add_etcd_url(ea.string_value());
        }
    }
+    this->etcd_ca = config["etcd_ca"].string_value();
    this->etcd_prefix = config["etcd_prefix"].string_value();
    if (this->etcd_prefix == "")
    {
@@ -331,6 +338,8 @@ void etcd_state_client_t::start_etcd_watcher()
    pick_next_etcd();
    std::string etcd_address = selected_etcd_address;
    std::string etcd_api_path;
+    bool ssl = etcd_address.substr(0, 8) == "https://";
+    etcd_address = etcd_address.substr(ssl ? 8 : 7);
    int pos = etcd_address.find('/');
    if (pos >= 0)
    {
@@ -349,7 +358,7 @@ void etcd_state_client_t::start_etcd_watcher()
        fprintf(stderr, "Trying to connect to etcd websocket at %s, watch from revision %ju/%ju/%ju\n", etcd_address.c_str(),
            etcd_watch_revision_config, etcd_watch_revision_osd, etcd_watch_revision_pg);
    }
-    etcd_watch_ws = open_websocket(tfd, etcd_address, etcd_api_path+"/watch", etcd_slow_timeout,
+    etcd_watch_ws = open_websocket(tfd, etcd_address, etcd_api_path+"/watch", { .timeout = etcd_slow_timeout, .ssl = ssl, .ssl_ca = etcd_ca },
        [this, cur_addr = selected_etcd_address](const http_response_t *msg)
    {
        if (msg->body.length())
@@ -125,6 +125,7 @@ public:
    uint32_t global_immediate_commit = IMMEDIATE_NONE;

    std::string etcd_prefix;
+    std::string etcd_ca;
    int log_level = 0;
    timerfd_manager_t *tfd = NULL;

@@ -10,9 +10,17 @@
 #include <unistd.h>
 #include <fcntl.h>
 #include <string.h>
+#include <assert.h>

 #include <stdexcept>

+#ifdef WITH_OPENSSL
+#include <openssl/bio.h>
+#include <openssl/err.h>
+#include <openssl/pem.h>
+#include <openssl/ssl.h>
+#endif
+
 #include "addr_util.h"
 #include "str_util.h"
 #include "json_util.h"
@@ -28,10 +36,18 @@ static void parse_http_headers(std::string & res, http_response_t *parsed);

 struct http_co_t
 {
+#ifdef WITH_OPENSSL
+    SSL_CTX *ssl_ctx = NULL;
+    SSL *ssl_cli = NULL;
+    BIO *ssl_bio = NULL;
+#endif
+
    timerfd_manager_t *tfd;
    std::function<void(const http_response_t*)> response_callback;

    int request_timeout = 0;
+    bool ssl = false;
+    std::string ssl_ca;
    std::string host;
    std::string request;
    std::string ws_outbox;
@@ -47,7 +63,7 @@ struct http_co_t
    int timeout_id = -1;
    int epoll_events = 0;
    int sent = 0;
-    std::vector<char> rbuf;
+    std::vector<uint8_t> rbuf;
    iovec read_iov, send_iov;
    msghdr read_msg = { 0 }, send_msg = { 0 };
    http_response_t parsed;
@@ -70,6 +86,10 @@ struct http_co_t
    void submit_read(bool check_timeout);
    void submit_send();
    bool handle_read();
+#ifdef WITH_OPENSSL
+    bool do_ssl_handshake(bool init_send);
+    void on_ssl_error(int res);
+#endif
    void post_message(uint8_t type, const std::string & msg);
    void send_request(const std::string & host, const std::string & request,
        const http_options_t & options, std::function<void(const http_response_t *response)> response_callback);
@@ -95,7 +115,7 @@ http_co_t *http_init(timerfd_manager_t *tfd)
 }

 http_co_t* open_websocket(timerfd_manager_t *tfd, const std::string & host, const std::string & path,
-    int timeout, std::function<void(const http_response_t *msg)> response_callback)
+    const http_options_t & options, std::function<void(const http_response_t *msg)> response_callback)
 {
    std::string request = "GET "+path+" HTTP/1.1\r\n"
        "Host: "+host+"\r\n"
@@ -108,9 +128,11 @@ http_co_t* open_websocket(timerfd_manager_t *tfd, const std::string & host, cons
    handler->tfd = tfd;
    handler->state = HTTP_CO_CLOSED;
    handler->host = host;
-    handler->request_timeout = timeout < 0 ? -1 : (timeout == 0 ? DEFAULT_TIMEOUT : timeout);
+    handler->request_timeout = options.timeout < 0 ? -1 : (options.timeout == 0 ? DEFAULT_TIMEOUT : options.timeout);
    handler->want_streaming = false;
    handler->keepalive = false;
+    handler->ssl = options.ssl;
+    handler->ssl_ca = options.ssl_ca;
    handler->request = request;
    handler->response_callback = response_callback;
    handler->start_ws_connection();
@@ -152,13 +174,15 @@ void http_co_t::send_request(const std::string & host, const std::string & reque
        stackout();
        return;
    }
-    if (state == HTTP_CO_KEEPALIVE && connected_host != host)
+    if (state == HTTP_CO_KEEPALIVE && (connected_host != host || ssl != options.ssl))
    {
        close_connection();
    }
    this->request_timeout = options.timeout < 0 ? 0 : (options.timeout == 0 ? DEFAULT_TIMEOUT : options.timeout);
    this->want_streaming = options.want_streaming;
    this->keepalive = options.keepalive;
+    this->ssl = options.ssl;
+    this->ssl_ca = options.ssl_ca;
    this->host = host;
    this->request = request;
    this->response = "";
@@ -261,6 +285,19 @@ void http_response_t::parse_json_response(std::string & error, json11::Json & r)

 http_co_t::~http_co_t()
 {
+#ifdef WITH_OPENSSL
+    ssl_bio = NULL;
+    if (ssl_cli)
+    {
+        SSL_free(ssl_cli);
+        ssl_cli = NULL;
+    }
+    if (ssl_ctx)
+    {
+        SSL_CTX_free(ssl_ctx);
+        ssl_ctx = NULL;
+    }
+#endif
    close_connection();
 }

@@ -277,6 +314,15 @@ void http_co_t::close_connection()
        close(peer_fd);
        peer_fd = -1;
    }
+#ifdef WITH_OPENSSL
+    if (ssl_cli)
+    {
+        // Frees client and bios at once
+        SSL_free(ssl_cli);
+        ssl_bio = NULL;
+        ssl_cli = NULL;
+    }
+#endif
    state = HTTP_CO_CLOSED;
    connected_host = "";
    response = "";
@@ -327,6 +373,56 @@ void http_co_t::start_connection()
    }
    fcntl(peer_fd, F_SETFL, fcntl(peer_fd, F_GETFL, 0) | O_NONBLOCK);
    epoll_events = 0;
+#ifdef WITH_OPENSSL
+    // https://wiki.openssl.org/index.php/Hostname_validation
+    if (ssl)
+    {
+        ssl_ctx = SSL_CTX_new(TLS_method());
+        if (!ssl_ctx)
+            goto init_err;
+        SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, NULL);
+        if (!SSL_CTX_set_min_proto_version(ssl_ctx, TLS1_2_VERSION))
+            goto init_err;
+        if ((ssl_ca != "")
+            ? !SSL_CTX_load_verify_locations(ssl_ctx, ssl_ca.c_str(), NULL)
+            : !SSL_CTX_set_default_verify_paths(ssl_ctx))
+            goto init_err;
+        ssl_bio = BIO_new(BIO_s_socket());
+        if (!ssl_bio)
+            goto init_err;
+        if (!BIO_set_fd(ssl_bio, peer_fd, BIO_NOCLOSE))
+            goto init_err;
+        ssl_cli = SSL_new(ssl_ctx);
+        if (!ssl_cli)
+            goto init_err;
+        SSL_set_bio(ssl_cli, ssl_bio, ssl_bio);
+        if (!SSL_set_tlsext_host_name(ssl_cli, host.c_str()))
+        {
+init_err:
+            if (ssl_cli)
+            {
+                SSL_free(ssl_cli);
+                ssl_cli = NULL;
+            }
+            else if (ssl_bio)
+            {
+                BIO_free(ssl_bio);
+                ssl_bio = NULL;
+            }
+            if (ssl_ctx)
+            {
+                SSL_CTX_free(ssl_ctx);
+                ssl_ctx = NULL;
+            }
+            parsed = { .error = std::string("openssl initialization failed: ")+ERR_error_string(ERR_get_error(), NULL) };
+            response_callback(&parsed);
+            response_callback = NULL;
+            stackout();
+            return;
+        }
+        SSL_set_connect_state(ssl_cli);
+    }
+#endif
    // Finally call connect
    int r = ::connect(peer_fd, (sockaddr*)&addr, sizeof(addr));
    if (r < 0 && errno != EINPROGRESS)
@@ -408,18 +504,44 @@ void http_co_t::handle_connect_result()
 void http_co_t::submit_send()
 {
    stackin();
-    int res;
+    ssize_t res = 0;
 again:
    if (sent < request.size())
    {
-        send_iov = (iovec){ .iov_base = (void*)(request.c_str()+sent), .iov_len = request.size()-sent };
-        send_msg.msg_iov = &send_iov;
-        send_msg.msg_iovlen = 1;
-        res = sendmsg(peer_fd, &send_msg, MSG_NOSIGNAL);
-        if (res < 0)
+        send_iov = (iovec){ .iov_base = (void*)(request.data()+sent), .iov_len = request.size()-sent };
+#ifdef WITH_OPENSSL
+        if (!ssl)
+#endif
        {
-            res = -errno;
+            send_msg.msg_iov = &send_iov;
+            send_msg.msg_iovlen = 1;
+            res = sendmsg(peer_fd, &send_msg, MSG_NOSIGNAL);
+            if (res < 0)
+                res = -errno;
        }
+#ifdef WITH_OPENSSL
+        else
+        {
+            if (!do_ssl_handshake(false))
+                goto out;
+            int ok = SSL_write_ex(ssl_cli, send_iov.iov_base, send_iov.iov_len, (size_t*)&res);
+            if (!ok)
+            {
+                res = SSL_get_error(ssl_cli, ok);
+                if (res == SSL_ERROR_WANT_WRITE || res == 0)
+                    res = 0;
+                else if (res == SSL_ERROR_WANT_READ)
+                    goto out;
+                else if (res == SSL_ERROR_SYSCALL)
+                    res = -errno;
+                else
+                {
+                    on_ssl_error(res);
+                    goto out;
+                }
+            }
+        }
+#endif
        if (res == -EAGAIN || res == -EINTR)
        {
            res = 0;
@@ -447,26 +569,53 @@ again:
            goto again;
        }
    }
+out:
    stackout();
 }

 void http_co_t::submit_read(bool check_timeout)
 {
    stackin();
-    int res;
+    ssize_t res = 0;
 again:
    if (rbuf.size() != READ_BUFFER_SIZE)
    {
        rbuf.resize(READ_BUFFER_SIZE);
    }
    read_iov = { .iov_base = rbuf.data(), .iov_len = READ_BUFFER_SIZE };
-    read_msg.msg_iov = &read_iov;
-    read_msg.msg_iovlen = 1;
-    res = recvmsg(peer_fd, &read_msg, 0);
-    if (res < 0)
+#ifdef WITH_OPENSSL
+    if (!ssl)
+#endif
    {
-        res = -errno;
+        read_msg.msg_iov = &read_iov;
+        read_msg.msg_iovlen = 1;
+        res = recvmsg(peer_fd, &read_msg, 0);
+        if (res < 0)
+            res = -errno;
    }
+#ifdef WITH_OPENSSL
+    else
+    {
+        if (!do_ssl_handshake(true))
+            goto out;
+        int ok = SSL_read_ex(ssl_cli, read_iov.iov_base, read_iov.iov_len, (size_t*)&res);
+        if (!ok)
+        {
+            res = SSL_get_error(ssl_cli, ok);
+            if (res == SSL_ERROR_WANT_READ)
+                res = -EAGAIN;
+            else if (res == SSL_ERROR_SYSCALL)
+                res = -errno;
+            else if (res == SSL_ERROR_ZERO_RETURN)
+                res = 0;
+            else
+            {
+                on_ssl_error(res);
+                goto out;
+            }
+        }
+    }
+#endif
    if (res == -EAGAIN || res == -EINTR)
    {
        if (check_timeout)
@@ -499,12 +648,60 @@ again:
    }
    else
    {
-        response += std::string(rbuf.data(), res);
+        response += std::string((char*)rbuf.data(), res);
        handle_read();
    }
+out:
    stackout();
 }

+#ifdef WITH_OPENSSL
+void http_co_t::on_ssl_error(int res)
+{
+    close_connection();
+    if (res == SSL_ERROR_ZERO_RETURN)
+    {
+        // Client closed the connection
+        parsed = { .error = "peer closed the SSL connection" };
+    }
+    else
+        parsed = { .error = std::string("SSL error: ")+ERR_error_string(ERR_get_error(), NULL) };
+    run_cb_and_clear();
+}
+
+bool http_co_t::do_ssl_handshake(bool init_send)
+{
+    if (SSL_is_init_finished(ssl_cli))
+        return true;
+    int r;
+    while (1)
+    {
+        r = SSL_do_handshake(ssl_cli);
+        if (r > 0)
+        {
+            // OK
+            if (init_send)
+                submit_send();
+            return true;
+        }
+        r = SSL_get_error(ssl_cli, r);
+        if (r == SSL_ERROR_WANT_READ)
+        {
+            break;
+        }
+        else
+        {
+            int errcode = ERR_get_error();
+            parsed = { .error = ERR_error_string(errcode, NULL) };
+            close_connection();
+            run_cb_and_clear();
+            return false;
+        }
+    }
+    return false;
+}
+#endif
+
 bool http_co_t::handle_read()
 {
    stackin();
@@ -22,6 +22,8 @@ struct http_options_t
    int timeout;
    bool want_streaming;
    bool keepalive;
+    bool ssl;
+    std::string ssl_ca;
 };

 struct http_response_t
@@ -43,7 +45,7 @@ struct http_co_t;

 http_co_t* http_init(timerfd_manager_t *tfd);
 http_co_t* open_websocket(timerfd_manager_t *tfd, const std::string & host, const std::string & path,
-    int timeout, std::function<void(const http_response_t *msg)> on_message);
+    const http_options_t & options, std::function<void(const http_response_t *msg)> on_message);
 void http_request(http_co_t *handler, const std::string & host, const std::string & request,
    const http_options_t & options, std::function<void(const http_response_t *response)> response_callback);
 void http_post_message(http_co_t *handler, uint8_t type, const std::string & msg);
@@ -15,6 +15,7 @@ target_link_libraries(vitastor-osd
 	${ISAL_LIBRARIES}
 	${IBVERBS_LIBRARIES}
 	${RDMACM_LIBRARIES}
+	${OPENSSL_LIBRARIES}
 )

 # osd_rmw_test
@@ -26,6 +26,7 @@ ETCD_PORT=${ETCD_PORT:-12379}
 ETCD_COUNT=${ETCD_COUNT:-1}
 ANTIETCD=${ANTIETCD}
 USE_RAMDISK=${USE_RAMDISK}
+ETCD_SCHEME=${ETCD_SCHEME:-http}

 RAMDISK=/run/user/$(id -u)
 findmnt $RAMDISK >/dev/null || (sudo mkdir -p $RAMDISK && sudo mount -t tmpfs tmpfs $RAMDISK)
@@ -48,17 +49,28 @@ if [[ -n "$OLD" ]]; then
    OFFSET_ARGS="$OFFSET_ARGS --meta_format 2"
 fi

-ETCD_URL="http://$ETCD_IP:$ETCD_PORT"
+ETCD_URL="$ETCD_SCHEME://$ETCD_IP:$ETCD_PORT"
 for i in $(seq 2 $ETCD_COUNT); do
-    ETCD_URL="$ETCD_URL,http://$ETCD_IP:$((ETCD_PORT+2*i-2))"
+    ETCD_URL="$ETCD_URL,$ETCD_SCHEME://$ETCD_IP:$((ETCD_PORT+2*i-2))"
 done

+if [[ "$ETCD_SCHEME" = "https" ]]; then
+    openssl req -days 3650 -x509 -new -newkey rsa:4096 -nodes -keyout ./testdata/etcd.key -out ./testdata/etcd.crt \
+        -subj '/C=RU/ST=Russia/L=Moscow/O=Vitastor/CN=etcd.local' \
+        -addext 'subjectAltName = IP:'$ETCD_IP
+fi
+
 start_etcd()
 {
    local i=$1
    if [[ -z "$ANTIETCD" ]]; then
+        ETCD_CERT=
+        if [[ "$ETCD_SCHEME" = "https" ]]; then
+            ETCD_CERT="--cert-file ./testdata/etcd.crt --key-file=./testdata/etcd.key"
+        fi
        ionice -c2 -n0 $ETCD -name etcd$i --data-dir $RAMDISK/testdata_etcd$i \
-            --advertise-client-urls http://$ETCD_IP:$((ETCD_PORT+2*i-2)) --listen-client-urls http://$ETCD_IP:$((ETCD_PORT+2*i-2)) \
+            --advertise-client-urls $ETCD_SCHEME://$ETCD_IP:$((ETCD_PORT+2*i-2)) --listen-client-urls $ETCD_SCHEME://$ETCD_IP:$((ETCD_PORT+2*i-2)) \
+            $ETCD_CERT \
            --initial-advertise-peer-urls http://$ETCD_IP:$((ETCD_PORT+2*i-1)) --listen-peer-urls http://$ETCD_IP:$((ETCD_PORT+2*i-1)) \
            --initial-cluster-token vitastor-tests-etcd --initial-cluster-state new \
            --initial-cluster "$ETCD_CLUSTER" --max-request-bytes=104857600 \
@@ -109,6 +121,9 @@ wait_condition()
 }

 VITASTOR_CFG='"etcd_address":"'$ETCD_URL'"'
+if [[ "$ETCD_SCHEME" = "https" ]]; then
+    VITASTOR_CFG="$VITASTOR_CFG"',"etcd_ca":"'$(pwd)'/testdata/etcd.crt"'
+fi
 echo "{$VITASTOR_CFG}" > ./testdata/vitastor.conf
 VITASTOR_CFG=./testdata/vitastor.conf
 VITASTOR_CLI="build/src/cmd/vitastor-cli --config_path $VITASTOR_CFG"
@@ -120,8 +135,15 @@ MON_PARAMS="$MON_PARAMS --config_path $VITASTOR_CFG"
 if [[ -n "$ANTIETCD" ]]; then
    ETCDCTL="node mon/node_modules/.bin/anticli -e $ETCD_URL"
    MON_PARAMS="--use_antietcd 1 --antietcd_data_dir ./testdata --antietcd_persist_interval 500 $MON_PARAMS"
+    if [[ "$ETCD_SCHEME" = "https" ]]; then
+        ETCDCTL="$ETCDCTL --ca ./testdata/etcd.crt"
+        MON_PARAMS="--antietcd_cert ./testdata/etcd.crt --antietcd_key ./testdata/etcd.key $MON_PARAMS"
+    fi
 else
    ETCDCTL="${ETCD}ctl --endpoints=$ETCD_URL --dial-timeout=5s --command-timeout=10s"
+    if [[ "$ETCD_SCHEME" = "https" ]]; then
+        ETCDCTL="$ETCDCTL --cacert=./testdata/etcd.crt"
+    fi
    start_etcd_cluster
 fi

@@ -19,6 +19,10 @@ SCHEME=ec ./test_change_pg_count.sh
 ./test_etcd_fail.sh
 ANTIETCD=1 ./test_etcd_fail.sh

+ETCD_SCHEME=https ./test_etcd_fail.sh
+ETCD_SCHEME=https ANTIETCD=1 ./test_etcd_fail.sh
+ETCD_SCHEME=https ./test_snapshot.sh
+
 ./test_interrupted_rebalance.sh
 IMMEDIATE_COMMIT=1 ./test_interrupted_rebalance.sh
 SCHEME=ec ./test_interrupted_rebalance.sh