Implement etcd SSL support via OpenSSL

Maybe I should remove all of this and use libwebsockets :)

.gitea/workflows/test.yml

@@ -234,6 +234,60 @@ jobs:
           echo ""
         done
 
+  test_etcd_fail_https:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: ETCD_SCHEME=https /root/vitastor/tests/test_etcd_fail.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_etcd_fail_https_antietcd:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: ETCD_SCHEME=https ANTIETCD=1 /root/vitastor/tests/test_etcd_fail.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_snapshot_https:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: ETCD_SCHEME=https /root/vitastor/tests/test_snapshot.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_interrupted_rebalance:
     runs-on: ubuntu-latest
     needs: build

.gitea/workflows/tests-to-yaml.pl

@@ -38,6 +38,10 @@ for my $line (<>)
             {
                 $test_name .= '_antietcd';
             }
+            elsif ($1 eq 'ETCD_SCHEME' && $2 eq 'https')
+            {
+                $test_name .= '_https';
+            }
             elsif ($1 eq 'OLD')
             {
                 $test_name =~ s/^test_/test_old_/s;

mon/etcd_adapter.js

@@ -1,7 +1,9 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)
 
+const fs = require('fs');
 const http = require('http');
+const https = require('https');
 const WebSocket = require('ws');
 const { b64, local_ips } = require('./utils.js');
 
@@ -15,11 +17,14 @@ class EtcdAdapter
         this.ws = null;
         this.ws_alive = false;
         this.ws_keepalive_timer = null;
+        this.opts = {};
     }
 
     parse_config(config)
     {
         this.parse_etcd_addresses(config.etcd_address||config.etcd_url);
+        if (config.etcd_ca)
+            this.opts.ca = fs.readFileSync(config.etcd_ca, { encoding: 'utf-8' });
     }
 
     parse_etcd_addresses(addrs)
@@ -39,7 +44,7 @@ class EtcdAdapter
         for (let url of addrs)
         {
             let scheme = 'http';
-            url = url.trim().replace(/^(https?):\/\//, (m, m1) => { scheme = m1; return ''; });
+            url = url.trim().replace(/^(https?):\/\//i, (m, m1) => { scheme = m1.toLowerCase(); return ''; });
             const slash = url.indexOf('/');
             const colon = url.indexOf(':');
             const is_local = is_local_ip[colon >= 0 ? url.substr(0, colon) : (slash >= 0 ? url.substr(0, slash) : url)];
@@ -130,7 +135,7 @@ class EtcdAdapter
                     }
                     ok(false);
                 }, this.mon.config.etcd_mon_timeout);
-                this.ws = new WebSocket(base+'/watch');
+                this.ws = new WebSocket(base+'/watch', this.opts);
                 this.ws_used_url = cur_addr;
                 const fail = () =>
                 {
@@ -272,7 +277,7 @@ class EtcdAdapter
             {
                 throw new Error(MON_STOPPED);
             }
-            const res = await POST(base+path, body, timeout);
+            const res = await POST(base+path, body, timeout, this.opts);
             if (this.mon.stopped)
             {
                 throw new Error(MON_STOPPED);
@@ -298,7 +303,7 @@ class EtcdAdapter
     }
 }
 
-function POST(url, body, timeout)
+function POST(url, body, timeout, opts)
 {
     return new Promise(ok =>
     {
@@ -310,10 +315,10 @@ function POST(url, body, timeout)
             req = null;
             ok({ error: 'timeout' });
         }, timeout) : null;
-        let req = http.request(url, { method: 'POST', headers: {
+        let req = (url.substr(0, 5) == 'https' ? https : http).request(url, { method: 'POST', headers: {
             'Content-Type': 'application/json',
             'Content-Length': body_text.length,
-        } }, (res) =>
+        }, ...(opts||{}) }, (res) =>
         {
             if (!req)
             {

src/CMakeLists.txt

@@ -75,6 +75,11 @@ if (RDMACM_LIBRARIES)
 	add_definitions(-DWITH_RDMACM)
 endif (RDMACM_LIBRARIES)
 
+find_package(OpenSSL)
+if (OPENSSL_FOUND)
+	add_definitions(-DWITH_OPENSSL)
+endif (OPENSSL_FOUND)
+
 if (${WITH_SYSTEM_LIBURING})
 	pkg_check_modules(LIBURING REQUIRED liburing>=2.10)
 	include_directories(${LIBURING_INCLUDE_DIRS})

src/client/CMakeLists.txt

@@ -33,6 +33,7 @@ target_link_libraries(vitastor_client
 	${LIBURING_LIBRARIES}
 	${IBVERBS_LIBRARIES}
 	${RDMACM_LIBRARIES}
+	${OPENSSL_LIBRARIES}
 )
 set_target_properties(vitastor_client PROPERTIES VERSION ${VITASTOR_VERSION} SOVERSION 0)
 configure_file(vitastor.pc.in vitastor.pc @ONLY)

src/client/etcd_state_client.cpp

@@ -76,6 +76,8 @@ void etcd_state_client_t::etcd_call_oneshot(std::string etcd_address, std::strin
     int timeout, std::function<void(std::string, json11::Json)> callback)
 {
     std::string etcd_api_path;
+    bool ssl = etcd_address.substr(0, 8) == "https://";
+    etcd_address = etcd_address.substr(ssl ? 8 : 7);
     int pos = etcd_address.find('/');
     if (pos >= 0)
     {
@@ -98,7 +100,7 @@ void etcd_state_client_t::etcd_call_oneshot(std::string etcd_address, std::strin
         callback(err, data);
         http_close(http_cli);
     };
-    http_request(http_cli, etcd_address, req, { .timeout = timeout }, cb);
+    http_request(http_cli, etcd_address, req, { .timeout = timeout, .ssl = ssl, .ssl_ca = etcd_ca }, cb);
 }
 
 void etcd_state_client_t::etcd_call(std::string api, json11::Json payload, int timeout,
@@ -112,6 +114,8 @@ void etcd_state_client_t::etcd_call(std::string api, json11::Json payload, int t
     pick_next_etcd();
     std::string etcd_address = selected_etcd_address;
     std::string etcd_api_path;
+    bool ssl = etcd_address.substr(0, 8) == "https://";
+    etcd_address = etcd_address.substr(ssl ? 8 : 7);
     int pos = etcd_address.find('/');
     if (pos >= 0)
     {
@@ -167,19 +171,20 @@ void etcd_state_client_t::etcd_call(std::string api, json11::Json payload, int t
     {
         keepalive_client = http_init(tfd);
     }
-    http_request(keepalive_client, etcd_address, req, { .timeout = timeout, .keepalive = true }, cb);
+    http_request(keepalive_client, etcd_address, req, { .timeout = timeout, .keepalive = true, .ssl = ssl, .ssl_ca = etcd_ca }, cb);
 }
 
 void etcd_state_client_t::add_etcd_url(std::string addr)
 {
     if (addr.length() > 0)
     {
+        bool ssl = false;
         if (strtolower(addr.substr(0, 7)) == "http://")
             addr = addr.substr(7);
         else if (strtolower(addr.substr(0, 8)) == "https://")
         {
-            fprintf(stderr, "HTTPS is unsupported for etcd. Either use plain HTTP or setup a local proxy for etcd interaction\n");
+            addr = addr.substr(8);
-            exit(1);
+            ssl = true;
         }
         if (!local_ips.size())
             local_ips = getifaddr_list(std::vector<addr_mask_t>(), true);
@@ -194,6 +199,7 @@ void etcd_state_client_t::add_etcd_url(std::string addr)
             check_addr = addr;
         if (pos == std::string::npos)
             addr += "/v3";
+        addr = (ssl ? "https://" : "http://") + addr;
         bool local = false;
         int i;
         for (i = 0; i < local_ips.size(); i++)
@@ -239,6 +245,7 @@ void etcd_state_client_t::parse_config(const json11::Json & config)
             add_etcd_url(ea.string_value());
         }
     }
+    this->etcd_ca = config["etcd_ca"].string_value();
     this->etcd_prefix = config["etcd_prefix"].string_value();
     if (this->etcd_prefix == "")
     {
@@ -331,6 +338,8 @@ void etcd_state_client_t::start_etcd_watcher()
     pick_next_etcd();
     std::string etcd_address = selected_etcd_address;
     std::string etcd_api_path;
+    bool ssl = etcd_address.substr(0, 8) == "https://";
+    etcd_address = etcd_address.substr(ssl ? 8 : 7);
     int pos = etcd_address.find('/');
     if (pos >= 0)
     {
@@ -349,7 +358,7 @@ void etcd_state_client_t::start_etcd_watcher()
         fprintf(stderr, "Trying to connect to etcd websocket at %s, watch from revision %ju/%ju/%ju\n", etcd_address.c_str(),
             etcd_watch_revision_config, etcd_watch_revision_osd, etcd_watch_revision_pg);
     }
-    etcd_watch_ws = open_websocket(tfd, etcd_address, etcd_api_path+"/watch", etcd_slow_timeout,
+    etcd_watch_ws = open_websocket(tfd, etcd_address, etcd_api_path+"/watch", { .timeout = etcd_slow_timeout, .ssl = ssl, .ssl_ca = etcd_ca },
         [this, cur_addr = selected_etcd_address](const http_response_t *msg)
     {
         if (msg->body.length())

src/client/etcd_state_client.h

@@ -125,6 +125,7 @@ public:
     uint32_t global_immediate_commit = IMMEDIATE_NONE;
 
     std::string etcd_prefix;
+    std::string etcd_ca;
     int log_level = 0;
     timerfd_manager_t *tfd = NULL;
 

src/client/http_client.cpp

@@ -10,9 +10,17 @@
 #include <unistd.h>
 #include <fcntl.h>
 #include <string.h>
+#include <assert.h>
 
 #include <stdexcept>
 
+#ifdef WITH_OPENSSL
+#include <openssl/bio.h>
+#include <openssl/err.h>
+#include <openssl/pem.h>
+#include <openssl/ssl.h>
+#endif
+
 #include "addr_util.h"
 #include "str_util.h"
 #include "json_util.h"
@@ -28,10 +36,18 @@ static void parse_http_headers(std::string & res, http_response_t *parsed);
 
 struct http_co_t
 {
+#ifdef WITH_OPENSSL
+    SSL_CTX *ssl_ctx = NULL;
+    SSL *ssl_cli = NULL;
+    BIO *ssl_bio = NULL;
+#endif
+
     timerfd_manager_t *tfd;
     std::function<void(const http_response_t*)> response_callback;
 
     int request_timeout = 0;
+    bool ssl = false;
+    std::string ssl_ca;
     std::string host;
     std::string request;
     std::string ws_outbox;
@@ -47,7 +63,7 @@ struct http_co_t
     int timeout_id = -1;
     int epoll_events = 0;
     int sent = 0;
-    std::vector<char> rbuf;
+    std::vector<uint8_t> rbuf;
     iovec read_iov, send_iov;
     msghdr read_msg = { 0 }, send_msg = { 0 };
     http_response_t parsed;
@@ -70,6 +86,10 @@ struct http_co_t
     void submit_read(bool check_timeout);
     void submit_send();
     bool handle_read();
+#ifdef WITH_OPENSSL
+    bool do_ssl_handshake(bool init_send);
+    void on_ssl_error(int res);
+#endif
     void post_message(uint8_t type, const std::string & msg);
     void send_request(const std::string & host, const std::string & request,
         const http_options_t & options, std::function<void(const http_response_t *response)> response_callback);
@@ -95,7 +115,7 @@ http_co_t *http_init(timerfd_manager_t *tfd)
 }
 
 http_co_t* open_websocket(timerfd_manager_t *tfd, const std::string & host, const std::string & path,
-    int timeout, std::function<void(const http_response_t *msg)> response_callback)
+    const http_options_t & options, std::function<void(const http_response_t *msg)> response_callback)
 {
     std::string request = "GET "+path+" HTTP/1.1\r\n"
         "Host: "+host+"\r\n"
@@ -108,9 +128,11 @@ http_co_t* open_websocket(timerfd_manager_t *tfd, const std::string & host, cons
     handler->tfd = tfd;
     handler->state = HTTP_CO_CLOSED;
     handler->host = host;
-    handler->request_timeout = timeout < 0 ? -1 : (timeout == 0 ? DEFAULT_TIMEOUT : timeout);
+    handler->request_timeout = options.timeout < 0 ? -1 : (options.timeout == 0 ? DEFAULT_TIMEOUT : options.timeout);
     handler->want_streaming = false;
     handler->keepalive = false;
+    handler->ssl = options.ssl;
+    handler->ssl_ca = options.ssl_ca;
     handler->request = request;
     handler->response_callback = response_callback;
     handler->start_ws_connection();
@@ -152,13 +174,15 @@ void http_co_t::send_request(const std::string & host, const std::string & reque
         stackout();
         return;
     }
-    if (state == HTTP_CO_KEEPALIVE && connected_host != host)
+    if (state == HTTP_CO_KEEPALIVE && (connected_host != host || ssl != options.ssl))
     {
         close_connection();
     }
     this->request_timeout = options.timeout < 0 ? 0 : (options.timeout == 0 ? DEFAULT_TIMEOUT : options.timeout);
     this->want_streaming = options.want_streaming;
     this->keepalive = options.keepalive;
+    this->ssl = options.ssl;
+    this->ssl_ca = options.ssl_ca;
     this->host = host;
     this->request = request;
     this->response = "";
@@ -261,6 +285,19 @@ void http_response_t::parse_json_response(std::string & error, json11::Json & r)
 
 http_co_t::~http_co_t()
 {
+#ifdef WITH_OPENSSL
+    ssl_bio = NULL;
+    if (ssl_cli)
+    {
+        SSL_free(ssl_cli);
+        ssl_cli = NULL;
+    }
+    if (ssl_ctx)
+    {
+        SSL_CTX_free(ssl_ctx);
+        ssl_ctx = NULL;
+    }
+#endif
     close_connection();
 }
 
@@ -277,6 +314,15 @@ void http_co_t::close_connection()
         close(peer_fd);
         peer_fd = -1;
     }
+#ifdef WITH_OPENSSL
+    if (ssl_cli)
+    {
+        // Frees client and bios at once
+        SSL_free(ssl_cli);
+        ssl_bio = NULL;
+        ssl_cli = NULL;
+    }
+#endif
     state = HTTP_CO_CLOSED;
     connected_host = "";
     response = "";
@@ -327,6 +373,56 @@ void http_co_t::start_connection()
     }
     fcntl(peer_fd, F_SETFL, fcntl(peer_fd, F_GETFL, 0) | O_NONBLOCK);
     epoll_events = 0;
+#ifdef WITH_OPENSSL
+    // https://wiki.openssl.org/index.php/Hostname_validation
+    if (ssl)
+    {
+        ssl_ctx = SSL_CTX_new(TLS_method());
+        if (!ssl_ctx)
+            goto init_err;
+        SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, NULL);
+        if (!SSL_CTX_set_min_proto_version(ssl_ctx, TLS1_2_VERSION))
+            goto init_err;
+        if ((ssl_ca != "")
+            ? !SSL_CTX_load_verify_locations(ssl_ctx, ssl_ca.c_str(), NULL)
+            : !SSL_CTX_set_default_verify_paths(ssl_ctx))
+            goto init_err;
+        ssl_bio = BIO_new(BIO_s_socket());
+        if (!ssl_bio)
+            goto init_err;
+        if (!BIO_set_fd(ssl_bio, peer_fd, BIO_NOCLOSE))
+            goto init_err;
+        ssl_cli = SSL_new(ssl_ctx);
+        if (!ssl_cli)
+            goto init_err;
+        SSL_set_bio(ssl_cli, ssl_bio, ssl_bio);
+        if (!SSL_set_tlsext_host_name(ssl_cli, host.c_str()))
+        {
+init_err:
+            if (ssl_cli)
+            {
+                SSL_free(ssl_cli);
+                ssl_cli = NULL;
+            }
+            else if (ssl_bio)
+            {
+                BIO_free(ssl_bio);
+                ssl_bio = NULL;
+            }
+            if (ssl_ctx)
+            {
+                SSL_CTX_free(ssl_ctx);
+                ssl_ctx = NULL;
+            }
+            parsed = { .error = std::string("openssl initialization failed: ")+ERR_error_string(ERR_get_error(), NULL) };
+            response_callback(&parsed);
+            response_callback = NULL;
+            stackout();
+            return;
+        }
+        SSL_set_connect_state(ssl_cli);
+    }
+#endif
     // Finally call connect
     int r = ::connect(peer_fd, (sockaddr*)&addr, sizeof(addr));
     if (r < 0 && errno != EINPROGRESS)
@@ -408,18 +504,44 @@ void http_co_t::handle_connect_result()
 void http_co_t::submit_send()
 {
     stackin();
-    int res;
+    ssize_t res = 0;
 again:
     if (sent < request.size())
     {
-        send_iov = (iovec){ .iov_base = (void*)(request.c_str()+sent), .iov_len = request.size()-sent };
+        send_iov = (iovec){ .iov_base = (void*)(request.data()+sent), .iov_len = request.size()-sent };
-        send_msg.msg_iov = &send_iov;
+#ifdef WITH_OPENSSL
-        send_msg.msg_iovlen = 1;
+        if (!ssl)
-        res = sendmsg(peer_fd, &send_msg, MSG_NOSIGNAL);
+#endif
-        if (res < 0)
         {
-            res = -errno;
+            send_msg.msg_iov = &send_iov;
+            send_msg.msg_iovlen = 1;
+            res = sendmsg(peer_fd, &send_msg, MSG_NOSIGNAL);
+            if (res < 0)
+                res = -errno;
         }
+#ifdef WITH_OPENSSL
+        else
+        {
+            if (!do_ssl_handshake(false))
+                goto out;
+            int ok = SSL_write_ex(ssl_cli, send_iov.iov_base, send_iov.iov_len, (size_t*)&res);
+            if (!ok)
+            {
+                res = SSL_get_error(ssl_cli, ok);
+                if (res == SSL_ERROR_WANT_WRITE || res == 0)
+                    res = 0;
+                else if (res == SSL_ERROR_WANT_READ)
+                    goto out;
+                else if (res == SSL_ERROR_SYSCALL)
+                    res = -errno;
+                else
+                {
+                    on_ssl_error(res);
+                    goto out;
+                }
+            }
+        }
+#endif
         if (res == -EAGAIN || res == -EINTR)
         {
             res = 0;
@@ -447,26 +569,53 @@ again:
             goto again;
         }
     }
+out:
     stackout();
 }
 
 void http_co_t::submit_read(bool check_timeout)
 {
     stackin();
-    int res;
+    ssize_t res = 0;
 again:
     if (rbuf.size() != READ_BUFFER_SIZE)
     {
         rbuf.resize(READ_BUFFER_SIZE);
     }
     read_iov = { .iov_base = rbuf.data(), .iov_len = READ_BUFFER_SIZE };
-    read_msg.msg_iov = &read_iov;
+#ifdef WITH_OPENSSL
-    read_msg.msg_iovlen = 1;
+    if (!ssl)
-    res = recvmsg(peer_fd, &read_msg, 0);
+#endif
-    if (res < 0)
     {
-        res = -errno;
+        read_msg.msg_iov = &read_iov;
+        read_msg.msg_iovlen = 1;
+        res = recvmsg(peer_fd, &read_msg, 0);
+        if (res < 0)
+            res = -errno;
     }
+#ifdef WITH_OPENSSL
+    else
+    {
+        if (!do_ssl_handshake(true))
+            goto out;
+        int ok = SSL_read_ex(ssl_cli, read_iov.iov_base, read_iov.iov_len, (size_t*)&res);
+        if (!ok)
+        {
+            res = SSL_get_error(ssl_cli, ok);
+            if (res == SSL_ERROR_WANT_READ)
+                res = -EAGAIN;
+            else if (res == SSL_ERROR_SYSCALL)
+                res = -errno;
+            else if (res == SSL_ERROR_ZERO_RETURN)
+                res = 0;
+            else
+            {
+                on_ssl_error(res);
+                goto out;
+            }
+        }
+    }
+#endif
     if (res == -EAGAIN || res == -EINTR)
     {
         if (check_timeout)
@@ -499,12 +648,60 @@ again:
     }
     else
     {
-        response += std::string(rbuf.data(), res);
+        response += std::string((char*)rbuf.data(), res);
         handle_read();
     }
+out:
     stackout();
 }
 
+#ifdef WITH_OPENSSL
+void http_co_t::on_ssl_error(int res)
+{
+    close_connection();
+    if (res == SSL_ERROR_ZERO_RETURN)
+    {
+        // Client closed the connection
+        parsed = { .error = "peer closed the SSL connection" };
+    }
+    else
+        parsed = { .error = std::string("SSL error: ")+ERR_error_string(ERR_get_error(), NULL) };
+    run_cb_and_clear();
+}
+
+bool http_co_t::do_ssl_handshake(bool init_send)
+{
+    if (SSL_is_init_finished(ssl_cli))
+        return true;
+    int r;
+    while (1)
+    {
+        r = SSL_do_handshake(ssl_cli);
+        if (r > 0)
+        {
+            // OK
+            if (init_send)
+                submit_send();
+            return true;
+        }
+        r = SSL_get_error(ssl_cli, r);
+        if (r == SSL_ERROR_WANT_READ)
+        {
+            break;
+        }
+        else
+        {
+            int errcode = ERR_get_error();
+            parsed = { .error = ERR_error_string(errcode, NULL) };
+            close_connection();
+            run_cb_and_clear();
+            return false;
+        }
+    }
+    return false;
+}
+#endif
+
 bool http_co_t::handle_read()
 {
     stackin();

src/client/http_client.h

@@ -22,6 +22,8 @@ struct http_options_t
     int timeout;
     bool want_streaming;
     bool keepalive;
+    bool ssl;
+    std::string ssl_ca;
 };
 
 struct http_response_t
@@ -43,7 +45,7 @@ struct http_co_t;
 
 http_co_t* http_init(timerfd_manager_t *tfd);
 http_co_t* open_websocket(timerfd_manager_t *tfd, const std::string & host, const std::string & path,
-    int timeout, std::function<void(const http_response_t *msg)> on_message);
+    const http_options_t & options, std::function<void(const http_response_t *msg)> on_message);
 void http_request(http_co_t *handler, const std::string & host, const std::string & request,
     const http_options_t & options, std::function<void(const http_response_t *response)> response_callback);
 void http_post_message(http_co_t *handler, uint8_t type, const std::string & msg);

src/osd/CMakeLists.txt

@@ -15,6 +15,7 @@ target_link_libraries(vitastor-osd
 	${ISAL_LIBRARIES}
 	${IBVERBS_LIBRARIES}
 	${RDMACM_LIBRARIES}
+	${OPENSSL_LIBRARIES}
 )
 
 # osd_rmw_test

tests/common.sh

@@ -26,6 +26,7 @@ ETCD_PORT=${ETCD_PORT:-12379}
 ETCD_COUNT=${ETCD_COUNT:-1}
 ANTIETCD=${ANTIETCD}
 USE_RAMDISK=${USE_RAMDISK}
+ETCD_SCHEME=${ETCD_SCHEME:-http}
 
 RAMDISK=/run/user/$(id -u)
 findmnt $RAMDISK >/dev/null || (sudo mkdir -p $RAMDISK && sudo mount -t tmpfs tmpfs $RAMDISK)
@@ -48,17 +49,28 @@ if [[ -n "$OLD" ]]; then
     OFFSET_ARGS="$OFFSET_ARGS --meta_format 2"
 fi
 
-ETCD_URL="http://$ETCD_IP:$ETCD_PORT"
+ETCD_URL="$ETCD_SCHEME://$ETCD_IP:$ETCD_PORT"
 for i in $(seq 2 $ETCD_COUNT); do
-    ETCD_URL="$ETCD_URL,http://$ETCD_IP:$((ETCD_PORT+2*i-2))"
+    ETCD_URL="$ETCD_URL,$ETCD_SCHEME://$ETCD_IP:$((ETCD_PORT+2*i-2))"
 done
 
+if [[ "$ETCD_SCHEME" = "https" ]]; then
+    openssl req -days 3650 -x509 -new -newkey rsa:4096 -nodes -keyout ./testdata/etcd.key -out ./testdata/etcd.crt \
+        -subj '/C=RU/ST=Russia/L=Moscow/O=Vitastor/CN=etcd.local' \
+        -addext 'subjectAltName = IP:'$ETCD_IP
+fi
+
 start_etcd()
 {
     local i=$1
     if [[ -z "$ANTIETCD" ]]; then
+        ETCD_CERT=
+        if [[ "$ETCD_SCHEME" = "https" ]]; then
+            ETCD_CERT="--cert-file ./testdata/etcd.crt --key-file=./testdata/etcd.key"
+        fi
         ionice -c2 -n0 $ETCD -name etcd$i --data-dir $RAMDISK/testdata_etcd$i \
-            --advertise-client-urls http://$ETCD_IP:$((ETCD_PORT+2*i-2)) --listen-client-urls http://$ETCD_IP:$((ETCD_PORT+2*i-2)) \
+            --advertise-client-urls $ETCD_SCHEME://$ETCD_IP:$((ETCD_PORT+2*i-2)) --listen-client-urls $ETCD_SCHEME://$ETCD_IP:$((ETCD_PORT+2*i-2)) \
+            $ETCD_CERT \
             --initial-advertise-peer-urls http://$ETCD_IP:$((ETCD_PORT+2*i-1)) --listen-peer-urls http://$ETCD_IP:$((ETCD_PORT+2*i-1)) \
             --initial-cluster-token vitastor-tests-etcd --initial-cluster-state new \
             --initial-cluster "$ETCD_CLUSTER" --max-request-bytes=104857600 \
@@ -109,6 +121,9 @@ wait_condition()
 }
 
 VITASTOR_CFG='"etcd_address":"'$ETCD_URL'"'
+if [[ "$ETCD_SCHEME" = "https" ]]; then
+    VITASTOR_CFG="$VITASTOR_CFG"',"etcd_ca":"'$(pwd)'/testdata/etcd.crt"'
+fi
 echo "{$VITASTOR_CFG}" > ./testdata/vitastor.conf
 VITASTOR_CFG=./testdata/vitastor.conf
 VITASTOR_CLI="build/src/cmd/vitastor-cli --config_path $VITASTOR_CFG"
@@ -120,8 +135,15 @@ MON_PARAMS="$MON_PARAMS --config_path $VITASTOR_CFG"
 if [[ -n "$ANTIETCD" ]]; then
     ETCDCTL="node mon/node_modules/.bin/anticli -e $ETCD_URL"
     MON_PARAMS="--use_antietcd 1 --antietcd_data_dir ./testdata --antietcd_persist_interval 500 $MON_PARAMS"
+    if [[ "$ETCD_SCHEME" = "https" ]]; then
+        ETCDCTL="$ETCDCTL --ca ./testdata/etcd.crt"
+        MON_PARAMS="--antietcd_cert ./testdata/etcd.crt --antietcd_key ./testdata/etcd.key $MON_PARAMS"
+    fi
 else
     ETCDCTL="${ETCD}ctl --endpoints=$ETCD_URL --dial-timeout=5s --command-timeout=10s"
+    if [[ "$ETCD_SCHEME" = "https" ]]; then
+        ETCDCTL="$ETCDCTL --cacert=./testdata/etcd.crt"
+    fi
     start_etcd_cluster
 fi
 

tests/run_tests.sh

@@ -19,6 +19,10 @@ SCHEME=ec ./test_change_pg_count.sh
 ./test_etcd_fail.sh
 ANTIETCD=1 ./test_etcd_fail.sh
 
+ETCD_SCHEME=https ./test_etcd_fail.sh
+ETCD_SCHEME=https ANTIETCD=1 ./test_etcd_fail.sh
+ETCD_SCHEME=https ./test_snapshot.sh
+
 ./test_interrupted_rebalance.sh
 IMMEDIATE_COMMIT=1 ./test_interrupted_rebalance.sh
 SCHEME=ec ./test_interrupted_rebalance.sh