Support storing image encryption keys in Vault

docs/usage/cli.en.md

@@ -135,6 +135,7 @@ Create an image. Options:
   If parent is not a snapshot, it must be a read-only image.
 * `--enc-key random` - Generate a new random AES-256-XTS encryption key for the new image.
 * `--enc-key HEX` - Set a specified AES-256-XTS key (64 bytes in hex) for the new image.
+* `--enc-key vault:ID` - Use an encryption key from an external Vault secret with specified ID.
 
 ```
 vitastor-cli create --snapshot <snapshot> [OPTIONS] <image>
@@ -147,7 +148,7 @@ Options:
 
 * `-p|--pool POOL` - Move image to pool POOL, leaving the snapshot in the old pool.
 * `--enc-key random` - Change image encryption key to a new random AES-256-XTS key.
-* `--enc-key HEX` - Change image encryption key to a specified key or to an empty key.
+* `--enc-key KEY` - Change image encryption key to a specified key, Vault key or to an empty key.
   By default, the image retains its old encryption key when taking a snapshot.
 
 See also about [how to export snapshots](qemu.en.md#exporting-snapshots).

docs/usage/cli.ru.md

@@ -137,6 +137,7 @@ bench-kaveri         kaveri    10 G  10 G    0 B/s  0     0      0 us  0 B/s  0
   Если `PARENT` - не снимок, он должен быть помечен как образ только для чтения.
 * `--enc-key random` - Сгенерировать случайный ключ шифрования AES-256-XTS для нового образа.
 * `--enc-key HEX` - Установить заданный ключ AES-256-XTS (64 байта в hex) для нового образа.
+* `--enc-key vault:ID` - Использовать ключ из внешнего секрета с заданным ID из Vault.
 
 ```
 vitastor-cli create --snapshot <snapshot> [ОПЦИИ] <image>
@@ -150,7 +151,7 @@ vitastor-cli snap-create [ОПЦИИ] <image>@<snapshot>
 
 * `-p|--pool POOL` - Переместить образ в пул POOL, оставив снимок в старом пуле.
 * `--enc-key random` - Изменить ключ шифрования образа на новый случайный ключ AES-256-XTS.
-* `--enc-key HEX` - Изменить ключ шифрования образа на заданный или пустой ключ.
+* `--enc-key KEY` - Изменить ключ шифрования образа на заданный ключ, ключ из Vault или пустой ключ.
   По умолчанию шифрованные образы сохраняют старый ключ при снятии снимка.
 
 Смотрите также информацию о том, [как экспортировать снимки](qemu.ru.md#экспорт-снимков).

src/client/CMakeLists.txt

@@ -24,6 +24,7 @@ add_library(vitastor_client SHARED
 	cluster_client.cpp
 	cluster_client_list.cpp
 	cluster_client_wb.cpp
+	cluster_client_icache.cpp
 	vitastor_c.cpp
 )
 set_target_properties(vitastor_client PROPERTIES PUBLIC_HEADER "client/vitastor_c.h")
@@ -99,7 +100,7 @@ endif (${WITH_QEMU})
 add_executable(test_cluster_client
 	EXCLUDE_FROM_ALL
 	../test/test_cluster_client.cpp
-	pg_states.cpp osd_ops.cpp cluster_client.cpp cluster_client_list.cpp cluster_client_wb.cpp msgr_op.cpp ../test/mock/messenger.cpp msgr_stop.cpp msgr_encrypt.cpp
+	pg_states.cpp osd_ops.cpp cluster_client.cpp cluster_client_list.cpp cluster_client_wb.cpp cluster_client_icache.cpp msgr_op.cpp ../test/mock/messenger.cpp msgr_stop.cpp msgr_encrypt.cpp
 	etcd_state_client.cpp ../util/timerfd_manager.cpp ../util/addr_util.cpp ../util/str_util.cpp ../util/json_util.cpp ../../json11/json11.cpp
 )
 target_link_libraries(test_cluster_client ${OPENSSL_LIBRARIES})

src/client/cluster_client.cpp

@@ -11,16 +11,6 @@
 #define TRY_SEND_CONNECTING 1
 #define TRY_SEND_OK 2
 
-inode_cache_t::~inode_cache_t()
-{
-    if (key_data)
-    {
-        free(key_data);
-        key_data = NULL;
-        op_enc = NULL;
-    }
-}
-
 cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json config)
 {
     wb = new writeback_cache_t();
@@ -88,6 +78,7 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
 
 cluster_client_t::~cluster_client_t()
 {
+    vault_destroy();
     if (retry_timeout_id >= 0)
     {
         tfd->clear_timer(retry_timeout_id);
@@ -492,6 +483,8 @@ void cluster_client_t::on_load_config_hook(json11::Json::object & etcd_global_co
         self_tree_metrics.clear();
         client_hostname = new_hostname;
     }
+    // vault
+    vault_parse_config();
     msgr.parse_config(config);
     st_cli.parse_config(config);
     st_cli.load_pgs();
@@ -620,6 +613,7 @@ void cluster_client_t::on_change_pool_config_hook()
     }
     inode_cache.clear();
     inode_cache_children.clear();
+    vault_keys.clear();
     continue_ops();
 }
 
@@ -665,136 +659,6 @@ void cluster_client_t::on_change_node_placement_hook()
     self_tree_metrics.clear();
 }
 
-// FIXME: Rework client API by adding open/close and cache inode information in the "FD" (maybe)
-void cluster_client_t::on_change_inode_hook(uint64_t inode, bool removed)
-{
-    std::vector<inode_t> children = { inode };
-    for (size_t i = 0; i < children.size(); i++)
-    {
-        auto it = inode_cache_children.lower_bound(std::make_pair(children[i], (inode_t)0));
-        while (it != inode_cache_children.end() && it->first == children[i])
-        {
-            children.push_back(it->second);
-            it++;
-        }
-    }
-    for (auto & inode: children)
-    {
-        auto it = inode_cache.find(inode);
-        if (it != inode_cache.end())
-        {
-            auto icache = it->second;
-            for (auto & parent: icache->chain)
-            {
-                inode_cache_children.erase(std::make_pair(parent, inode));
-            }
-            inode_cache.erase(it);
-        }
-    }
-}
-
-std::shared_ptr<inode_cache_t> cluster_client_t::inode_cache_get(inode_t ino)
-{
-    auto icache_it = inode_cache.find(ino);
-    if (icache_it != inode_cache.end())
-    {
-        return icache_it->second;
-    }
-    // Fill inode cache
-    auto ino_it = st_cli.inode_config.find(ino);
-    if (ino_it == st_cli.inode_config.end())
-    {
-        inode_cache[ino] = NULL;
-        return NULL;
-    }
-    auto pool_it = st_cli.pool_config.find(INODE_POOL(ino));
-    if (pool_it == st_cli.pool_config.end())
-    {
-        inode_cache[ino] = NULL;
-        return NULL;
-    }
-    auto & inode_cfg = ino_it->second;
-    auto & pool_cfg = pool_it->second;
-    std::shared_ptr<inode_cache_t> icache = std::make_shared<inode_cache_t>();
-    icache->readonly = inode_cfg.readonly;
-    icache->chain.push_back(ino);
-    std::vector<inode_config_t*> chain_cfg;
-    int enc_key_count = !inode_cfg.enc_key.empty() ? 1 : 0;
-    if (inode_cfg.parent_id)
-    {
-        // Check for loops and cache the chain
-        robin_hood::unordered_flat_set<inode_t> seen;
-        seen.insert(ino);
-        uint64_t parent_id = inode_cfg.parent_id;
-        while (parent_id)
-        {
-            if (seen.find(parent_id) != seen.end())
-            {
-                icache->has_parent_loop = true;
-                break;
-            }
-            seen.insert(parent_id);
-            ino_it = st_cli.inode_config.find(parent_id);
-            if (INODE_POOL(parent_id) == INODE_POOL(ino))
-            {
-                icache->chain.push_back(parent_id);
-                if (ino_it == st_cli.inode_config.end())
-                    chain_cfg.push_back(NULL);
-                else
-                {
-                    chain_cfg.push_back(&ino_it->second);
-                    if (!ino_it->second.enc_key.empty())
-                        enc_key_count++;
-                }
-            }
-            else if (!icache->other_pool_parent_id)
-                icache->other_pool_parent_id = parent_id;
-            if (ino_it == st_cli.inode_config.end())
-                break;
-            parent_id = ino_it->second.parent_id;
-        }
-    }
-    // Generate encryption key chain, if applicable
-    if (enc_key_count)
-    {
-        uint8_t *key_data = (uint8_t*)malloc_or_die(
-            AES_256_XTS_KEY_SIZE * enc_key_count +
-            sizeof(uint8_t*) * icache->chain.size() +
-            sizeof(osd_op_enc_t)
-        );
-        uint8_t **keys = (uint8_t**)(key_data + AES_256_XTS_KEY_SIZE * enc_key_count);
-        osd_op_enc_t *enc = (osd_op_enc_t*)((uint8_t*)keys + sizeof(uint8_t*)*icache->chain.size());
-        size_t key_pos = 0;
-        for (size_t i = 0; i <= chain_cfg.size(); i++)
-        {
-            inode_config_t *cfg = !i ? &inode_cfg : chain_cfg[i-1];
-            if (cfg && !cfg->enc_key.empty())
-            {
-                assert(key_pos < AES_256_XTS_KEY_SIZE * enc_key_count);
-                assert(cfg->enc_key.size() == AES_256_XTS_KEY_SIZE);
-                keys[i] = key_data + key_pos;
-                memcpy(key_data + key_pos, cfg->enc_key.data(), AES_256_XTS_KEY_SIZE);
-                key_pos += AES_256_XTS_KEY_SIZE;
-            }
-            else
-                keys[i] = NULL;
-        }
-        enc->key_chain = keys;
-        enc->chain_size = icache->chain.size();
-        enc->read_chain_bitmap_pos = pool_cfg.data_block_size/pool_cfg.bitmap_granularity/8;
-        enc->bitmap_granularity = pool_cfg.bitmap_granularity;
-        icache->key_data = key_data;
-        icache->op_enc = enc;
-    }
-    inode_cache[ino] = icache;
-    for (auto & parent: icache->chain)
-    {
-        if (parent != ino)
-            inode_cache_children.insert(std::make_pair(parent, ino));
-    }
-    return icache;
-}
-
 bool cluster_client_t::is_ready()
 {
     return pgs_loaded;
@@ -816,6 +680,10 @@ bool cluster_client_t::flush()
 {
     if (!ringloop)
     {
+        if (vault_loading)
+        {
+            return false;
+        }
         if (wb->writeback_queue.size())
         {
             wb->start_writebacks(this, 0);
@@ -838,7 +706,7 @@ bool cluster_client_t::flush()
         sync_done = true;
     };
     execute(sync);
-    while (!sync_done)
+    while (!sync_done || vault_loading)
     {
         ringloop->loop();
         if (!sync_done)
@@ -1163,6 +1031,21 @@ bool cluster_client_t::check_rw(cluster_op_t *op)
             }
         }
     }
+    if (icache && icache->err_code)
+    {
+        if (icache->err_code == EPERM)
+        {
+            op->retval = -EPERM;
+            auto cb = std::move(op->callback);
+            cb(op);
+            return false;
+        }
+        else if (icache->err_code == EAGAIN)
+        {
+            key_wait_ops.push_back(op);
+            return false;
+        }
+    }
     return true;
 }
 

src/client/cluster_client.h

@@ -90,10 +90,17 @@ struct inode_cache_t
     bool readonly = false;
     bool has_parent_loop = false;
     inode_t other_pool_parent_id = 0;
+    int err_code = 0;
 
     ~inode_cache_t();
 };
 
+struct vault_load_key_t
+{
+    int key_state = 0;
+    std::string key;
+};
+
 // FIXME: Split into public and private interfaces
 class __attribute__((visibility("default"))) cluster_client_t
 {
@@ -103,8 +110,8 @@ public:
     timerfd_manager_t *tfd = NULL;
     ring_loop_t *ringloop = NULL;
 
-    std::map<pool_id_t, uint64_t> pg_counts;
+    // config:
-    std::map<pool_pg_num_t, osd_num_t> pg_primary;
+
     // client_max_dirty_* is actually "max unsynced", for the case when immediate_commit is off
     uint64_t client_max_dirty_bytes = 0;
     uint64_t client_max_dirty_ops = 0;
@@ -116,12 +123,23 @@ public:
     uint64_t client_max_writeback_iodepth = 0;
     std::string conf_hostname;
 
+    std::string vault_url;
+    std::string vault_client_cert;
+    std::string vault_client_key;
+    std::string vault_ca;
+    std::string vault_secret_api_path;
+    uint64_t vault_timeout_ms = 0;
+    uint64_t vault_error_timeout_sec = 0;
+    uint64_t vault_refresh_leeway_sec = 0;
+
     int log_level = 0;
     int client_retry_interval = 50; // ms
     int client_eio_retry_interval = 1000; // ms
     bool client_retry_enospc = true;
     int client_wait_up_timeout = 16; // sec (for listings)
 
+    // state:
+
     std::string client_hostname;
     std::map<std::string, int> self_tree_metrics;
     std::map<osd_num_t, int> osd_tree_metrics;
@@ -129,6 +147,7 @@ public:
     int retry_timeout_id = -1;
     int retry_timeout_duration = 0;
     std::vector<cluster_op_t*> offline_ops;
+    std::vector<cluster_op_t*> key_wait_ops;
     cluster_op_t *op_queue_head = NULL, *op_queue_tail = NULL;
     writeback_cache_t *wb = NULL;
     std::set<osd_num_t> dirty_osds;
@@ -142,7 +161,17 @@ public:
     robin_hood::unordered_flat_map<inode_t, std::shared_ptr<inode_cache_t>> inode_cache;
     std::set<std::pair<inode_t, inode_t>> inode_cache_children;
 
+    http_context_t *vault_http_ctx = NULL;
+    http_co_t *vault_http_cli = NULL;
+    bool vault_loading = false;
+    std::string vault_token;
+    bool vault_auth_error = false;
+    timespec vault_token_expire = {};
+    std::vector<std::string> vault_key_load_queue;
+    std::map<std::string, vault_load_key_t> vault_keys;
+
     bool pgs_loaded = false;
+    std::map<pool_id_t, uint64_t> pg_counts;
     ring_consumer_t consumer;
     std::vector<std::function<void(void)>> on_ready_hooks;
     int list_retry_timeout_id = -1;
@@ -182,6 +211,13 @@ protected:
 #endif
     void continue_ops(int time_passed = 0);
 
+    std::shared_ptr<inode_cache_t> inode_cache_get(inode_t ino);
+    void vault_parse_config();
+    bool vault_check_token();
+    void vault_load_keys();
+    void vault_destroy();
+    void vault_parse_secret(const std::string & key_id, const std::string & err, json11::Json data);
+
 protected:
     bool affects_osd(uint64_t inode, uint64_t offset, uint64_t len, osd_num_t osd);
     bool affects_pg(uint64_t inode, uint64_t offset, uint64_t len, pool_id_t pool_id, pg_num_t pg_num);
@@ -194,8 +230,6 @@ protected:
     void on_change_node_placement_hook();
     void on_change_inode_hook(uint64_t inode, bool removed);
 
-    std::shared_ptr<inode_cache_t> inode_cache_get(inode_t ino);
-
     void execute_internal(cluster_op_t *op);
     void execute_cas(cluster_op_t *op);
     void unshift_op(cluster_op_t *op);

src/client/cluster_client_icache.cpp

@@ -0,0 +1,366 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+#include <stdexcept>
+#include <assert.h>
+#include "cluster_client_impl.h"
+#include "http_client.h"
+#include "str_util.h"
+
+#define VAULT_KEY_NOT_LOADED 0
+#define VAULT_KEY_LOADING 1
+#define VAULT_KEY_LOADED 2
+#define VAULT_KEY_ERROR 3
+
+inode_cache_t::~inode_cache_t()
+{
+    if (key_data)
+    {
+        free(key_data);
+        key_data = NULL;
+        op_enc = NULL;
+    }
+}
+
+void cluster_client_t::vault_destroy()
+{
+    if (vault_http_ctx)
+    {
+#ifndef __MOCK__
+        http_destroy(vault_http_cli);
+        http_context_destroy(vault_http_ctx);
+        vault_http_cli = NULL;
+        vault_http_ctx = NULL;
+#endif
+    }
+}
+
+void cluster_client_t::vault_parse_config()
+{
+    vault_url = config["vault_url"].string_value();
+    vault_client_cert = config["vault_client_cert"].string_value();
+    vault_client_key = config["vault_client_key"].string_value();
+    vault_ca = config["vault_ca"].string_value();
+    vault_secret_api_path = "/v1/secret/";
+    if (config["vault_secret_api_path"].is_string())
+        vault_secret_api_path = config["vault_secret_api_path"].string_value();
+    vault_timeout_ms = config["vault_timeout_ms"].uint64_value();
+    if (!vault_timeout_ms)
+        vault_timeout_ms = 5000;
+    vault_error_timeout_sec = config["vault_error_timeout_sec"].uint64_value();
+    if (!vault_error_timeout_sec)
+        vault_error_timeout_sec = 60;
+    vault_refresh_leeway_sec = config["vault_refresh_leeway_sec"].uint64_value();
+    if (!vault_refresh_leeway_sec)
+        vault_refresh_leeway_sec = 60;
+}
+
+// FIXME: Rework client API by adding open/close and cache inode information in the "FD" (maybe)
+void cluster_client_t::on_change_inode_hook(uint64_t inode, bool removed)
+{
+    std::vector<inode_t> children = { inode };
+    for (size_t i = 0; i < children.size(); i++)
+    {
+        auto it = inode_cache_children.lower_bound(std::make_pair(children[i], (inode_t)0));
+        while (it != inode_cache_children.end() && it->first == children[i])
+        {
+            children.push_back(it->second);
+            it++;
+        }
+    }
+    for (auto & inode: children)
+    {
+        auto it = inode_cache.find(inode);
+        if (it != inode_cache.end())
+        {
+            auto icache = it->second;
+            for (auto & parent: icache->chain)
+            {
+                inode_cache_children.erase(std::make_pair(parent, inode));
+            }
+            inode_cache.erase(it);
+        }
+    }
+}
+
+std::shared_ptr<inode_cache_t> cluster_client_t::inode_cache_get(inode_t ino)
+{
+    auto icache_it = inode_cache.find(ino);
+    if (icache_it != inode_cache.end())
+    {
+        return icache_it->second;
+    }
+    // Fill inode cache
+    auto ino_it = st_cli.inode_config.find(ino);
+    if (ino_it == st_cli.inode_config.end())
+    {
+        inode_cache[ino] = NULL;
+        return NULL;
+    }
+    auto pool_it = st_cli.pool_config.find(INODE_POOL(ino));
+    if (pool_it == st_cli.pool_config.end())
+    {
+        inode_cache[ino] = NULL;
+        return NULL;
+    }
+    auto & inode_cfg = ino_it->second;
+    auto & pool_cfg = pool_it->second;
+    std::shared_ptr<inode_cache_t> icache = std::make_shared<inode_cache_t>();
+    icache->readonly = inode_cfg.readonly;
+    icache->chain.push_back(ino);
+    std::vector<inode_config_t*> chain_cfg;
+    int enc_key_count = !inode_cfg.enc_key.empty() ? 1 : 0;
+    if (inode_cfg.parent_id)
+    {
+        // Check for loops and cache the chain
+        robin_hood::unordered_flat_set<inode_t> seen;
+        seen.insert(ino);
+        uint64_t parent_id = inode_cfg.parent_id;
+        while (parent_id)
+        {
+            if (seen.find(parent_id) != seen.end())
+            {
+                icache->has_parent_loop = true;
+                break;
+            }
+            seen.insert(parent_id);
+            ino_it = st_cli.inode_config.find(parent_id);
+            if (INODE_POOL(parent_id) == INODE_POOL(ino))
+            {
+                icache->chain.push_back(parent_id);
+                if (ino_it == st_cli.inode_config.end())
+                    chain_cfg.push_back(NULL);
+                else
+                {
+                    chain_cfg.push_back(&ino_it->second);
+                    if (!ino_it->second.enc_key.empty())
+                        enc_key_count++;
+                }
+            }
+            else if (!icache->other_pool_parent_id)
+                icache->other_pool_parent_id = parent_id;
+            if (ino_it == st_cli.inode_config.end())
+                break;
+            parent_id = ino_it->second.parent_id;
+        }
+    }
+    // Check external keys and wait for loading, if required
+    if (enc_key_count)
+    {
+        for (size_t i = 0; i <= chain_cfg.size(); i++)
+        {
+            inode_config_t *cfg = !i ? &inode_cfg : chain_cfg[i-1];
+            if (cfg && cfg->enc_key.substr(0, strlen(VAULT_KEY_PREFIX)) == VAULT_KEY_PREFIX)
+            {
+                auto & ik = vault_keys[inode_cfg.enc_key];
+                if (ik.key_state == VAULT_KEY_ERROR || vault_url.empty())
+                {
+                    icache->err_code = EPERM;
+                    enc_key_count = 0;
+                }
+                else if (ik.key_state == VAULT_KEY_NOT_LOADED)
+                {
+                    ik.key_state = VAULT_KEY_LOADING;
+                    vault_key_load_queue.push_back(inode_cfg.enc_key);
+                    vault_load_keys();
+                    icache->err_code = EAGAIN;
+                    enc_key_count = 0;
+                }
+                else if (ik.key_state == VAULT_KEY_LOADING)
+                {
+                    icache->err_code = EAGAIN;
+                    enc_key_count = 0;
+                }
+                else
+                {
+                    assert(ik.key_state == VAULT_KEY_LOADED);
+                }
+            }
+        }
+    }
+    // Generate encryption key chain, if applicable
+    if (enc_key_count)
+    {
+        uint8_t *key_data = (uint8_t*)malloc_or_die(
+            AES_256_XTS_KEY_SIZE * enc_key_count +
+            sizeof(uint8_t*) * icache->chain.size() +
+            sizeof(osd_op_enc_t)
+        );
+        uint8_t **keys = (uint8_t**)(key_data + AES_256_XTS_KEY_SIZE * enc_key_count);
+        osd_op_enc_t *enc = (osd_op_enc_t*)((uint8_t*)keys + sizeof(uint8_t*)*icache->chain.size());
+        size_t key_pos = 0;
+        for (size_t i = 0; i <= chain_cfg.size(); i++)
+        {
+            inode_config_t *cfg = !i ? &inode_cfg : chain_cfg[i-1];
+            if (cfg && !cfg->enc_key.empty())
+            {
+                const auto & key = cfg->enc_key.substr(0, strlen(VAULT_KEY_PREFIX)) == VAULT_KEY_PREFIX
+                    ? vault_keys.at(cfg->enc_key).key
+                    : cfg->enc_key;
+                assert(key_pos < AES_256_XTS_KEY_SIZE * enc_key_count);
+                assert(key.size() == 2*AES_256_XTS_KEY_SIZE);
+                keys[i] = key_data + key_pos;
+                fromhexstr(key, AES_256_XTS_KEY_SIZE, key_data + key_pos);
+                key_pos += AES_256_XTS_KEY_SIZE;
+            }
+            else
+                keys[i] = NULL;
+        }
+        enc->key_chain = keys;
+        enc->chain_size = icache->chain.size();
+        enc->read_chain_bitmap_pos = pool_cfg.data_block_size/pool_cfg.bitmap_granularity/8;
+        enc->bitmap_granularity = pool_cfg.bitmap_granularity;
+        icache->key_data = key_data;
+        icache->op_enc = enc;
+    }
+    inode_cache[ino] = icache;
+    for (auto & parent: icache->chain)
+    {
+        if (parent != ino)
+            inode_cache_children.insert(std::make_pair(parent, ino));
+    }
+    return icache;
+}
+
+#ifndef __MOCK__
+bool cluster_client_t::vault_check_token()
+{
+    timespec now;
+    clock_gettime(CLOCK_REALTIME, &now);
+    if (!vault_token_expire.tv_sec || vault_token_expire.tv_sec < now.tv_sec)
+    {
+        vault_loading = true;
+        http_json_post(
+            vault_http_cli, vault_url+"/v1/auth/cert/login", json11::Json::object{}, "",
+            (http_options_t){ .timeout = (int)vault_timeout_ms, .keepalive = true },
+            [this](http_message_t *response)
+            {
+                clock_gettime(CLOCK_REALTIME, &vault_token_expire);
+                vault_loading = false;
+                std::string err;
+                json11::Json data;
+                response->parse_json_response(err, data);
+                if (err != "")
+                {
+                    vault_token_expire.tv_sec += vault_error_timeout_sec;
+                    fprintf(stderr, "Vault request failed: %s\n", err.c_str());
+                }
+                else
+                {
+                    uint64_t ttl = data["auth"]["lease_duration"].uint64_value();
+                    vault_token = data["auth"]["client_token"].string_value();
+                    if (vault_token.empty() || !ttl)
+                    {
+                        vault_token_expire.tv_sec += vault_error_timeout_sec;
+                        fprintf(stderr, "No token or lease_duration in Vault response: %s\n", data.dump().c_str());
+                    }
+                    else
+                    {
+                        if (ttl < vault_refresh_leeway_sec)
+                            vault_token_expire.tv_sec += ttl/2;
+                        else
+                            vault_token_expire.tv_sec += ttl - vault_refresh_leeway_sec;
+                    }
+                }
+                vault_load_keys();
+            }
+        );
+        return false;
+    }
+    if (vault_token.empty())
+    {
+        // Auth error happened, mark all loads as failed
+        for (auto & key_id: vault_key_load_queue)
+        {
+            auto & k = vault_keys[key_id];
+            k.key_state = VAULT_KEY_ERROR;
+        }
+        vault_key_load_queue.clear();
+        auto ops = std::move(key_wait_ops);
+        for (cluster_op_t *op: ops)
+            inode_cache.erase(op->inode);
+        for (cluster_op_t *op: ops)
+            execute_internal(op);
+        return false;
+    }
+    return true;
+}
+#endif
+
+void cluster_client_t::vault_load_keys()
+{
+    if (vault_loading || !vault_key_load_queue.size())
+    {
+        return;
+    }
+#ifdef __MOCK__
+    vault_loading = true;
+#else
+    if (!vault_http_ctx)
+    {
+        std::string error;
+        vault_http_ctx = http_context_init(tfd, vault_client_cert, vault_client_key, vault_ca, true, error);
+        if (!vault_http_ctx)
+        {
+            fprintf(stderr, "Failed to initialize HTTP context for Vault: %s\n", error.c_str());
+            exit(1);
+        }
+        vault_http_cli = http_init(vault_http_ctx);
+    }
+    if (!vault_check_token())
+    {
+        return;
+    }
+    std::string key_id = vault_key_load_queue[0];
+    vault_key_load_queue.erase(vault_key_load_queue.begin());
+    vault_loading = true;
+    http_get(
+        vault_http_cli, vault_url+vault_secret_api_path+key_id.substr(strlen(VAULT_KEY_PREFIX)), "X-Vault-Token: "+vault_token+"\r\n",
+        (http_options_t){ .timeout = (int)vault_timeout_ms, .keepalive = true },
+        [this, key_id](http_message_t *response)
+        {
+            vault_loading = false;
+            std::string err;
+            json11::Json data;
+            response->parse_json_response(err, data);
+            vault_parse_secret(key_id, err, data);
+        }
+    );
+#endif
+}
+
+void cluster_client_t::vault_parse_secret(const std::string & key_id, const std::string & err, json11::Json data)
+{
+    vault_loading = false;
+    auto & k = vault_keys[key_id];
+    if (err != "")
+    {
+        k.key_state = VAULT_KEY_ERROR;
+        fprintf(stderr, "Vault %s%s%s request failed: %s\n", vault_url.c_str(),
+            vault_secret_api_path.c_str(), key_id.c_str()+strlen(VAULT_KEY_PREFIX), err.c_str());
+    }
+    else
+    {
+        auto hexkey = data["data"]["key"].string_value();
+        if (hexkey.empty() || !ishexstr(hexkey) || hexkey.size() != 2*AES_256_XTS_KEY_SIZE)
+        {
+            k.key_state = VAULT_KEY_ERROR;
+            fprintf(stderr, "Vault /v1/secret/%s request failed: 'key' is empty or has invalid format\n", key_id.c_str());
+        }
+        else
+        {
+            k.key_state = VAULT_KEY_LOADED;
+            k.key = hexkey;
+        }
+    }
+    if (vault_key_load_queue.empty())
+    {
+        auto ops = std::move(key_wait_ops);
+        for (cluster_op_t *op: ops)
+            inode_cache.erase(op->inode);
+        for (cluster_op_t *op: ops)
+            execute_internal(op);
+    }
+    else
+        vault_load_keys();
+}

src/client/etcd_state_client.cpp

@@ -98,32 +98,15 @@ http_context_t *etcd_state_client_t::get_http_ctx()
 void etcd_state_client_t::etcd_call_oneshot(const std::string & etcd_url, const std::string & api, json11::Json payload,
     int timeout, std::function<void(std::string, json11::Json)> callback)
 {
-    std::string etcd_api_path;
-    bool ssl = etcd_url.substr(0, 8) == "https://";
-    auto etcd_address = etcd_url.substr(ssl ? 8 : 7);
-    int pos = etcd_address.find('/');
-    if (pos >= 0)
-    {
-        etcd_api_path = etcd_address.substr(pos);
-        etcd_address = etcd_address.substr(0, pos);
-    }
-    std::string req = payload.dump();
-    req = "POST "+etcd_api_path+api+" HTTP/1.1\r\n"
-        "Host: "+etcd_address+"\r\n"
-        "Content-Type: application/json\r\n"
-        "Content-Length: "+std::to_string(req.size())+"\r\n"
-        "Connection: close\r\n"
-        "\r\n"+req;
     auto http_cli = http_init(get_http_ctx());
-    auto cb = [http_cli, callback](http_message_t *response)
+    http_json_post(http_cli, etcd_url+api, payload, "", { .timeout = timeout }, [http_cli, callback](http_message_t *response)
     {
         std::string err;
         json11::Json data;
         response->parse_json_response(err, data);
         callback(err, data);
         http_destroy(http_cli);
-    };
+    });
-    http_request(http_cli, etcd_address, req, { .timeout = timeout, .ssl = ssl }, cb);
 }
 
 void etcd_state_client_t::etcd_call(const std::string & api, json11::Json payload, int timeout,
@@ -1448,7 +1431,7 @@ json11::Json::object etcd_state_client_t::serialize_inode_cfg(inode_config_t *cf
     }
     if (!cfg->enc_key.empty())
     {
-        new_cfg["enc_key"] = tohexstr(cfg->enc_key.data(), cfg->enc_key.size());
+        new_cfg["enc_key"] = cfg->enc_key;
     }
     if (cfg->readonly)
     {
@@ -1484,18 +1467,15 @@ inode_config_t etcd_state_client_t::deserialize_inode_cfg(uint64_t inode_num, js
         else
             parent_inode_num |= parent_pool_id << (64-POOL_ID_BITS);
     }
-    std::vector<uint8_t> enc_key;
+    std::string enc_key;
     if (!value["enc_key"].is_null())
     {
-        if (value["enc_key"].string_value().size() == 2*AES_256_XTS_KEY_SIZE)
+        enc_key = value["enc_key"].string_value();
+        if (enc_key.substr(0, strlen(VAULT_KEY_PREFIX)) != VAULT_KEY_PREFIX &&
+            (enc_key.size() != 2*AES_256_XTS_KEY_SIZE || !ishexstr(enc_key)))
         {
-            enc_key.resize(AES_256_XTS_KEY_SIZE);
+            enc_key = "";
-            if (fromhexstr(value["enc_key"].string_value(), AES_256_XTS_KEY_SIZE, enc_key.data()) < AES_256_XTS_KEY_SIZE)
+            fprintf(stderr, "Inode %u/%ju has invalid enc_key, should be %u bit hex string or Vault key reference\n",
-                enc_key.clear();
-        }
-        if (enc_key.empty())
-        {
-            fprintf(stderr, "Inode %u/%ju has invalid enc_key, should be %u bit hex string\n",
                 INODE_POOL(inode_num), INODE_NO_POOL(inode_num), AES_256_XTS_KEY_SIZE);
         }
     }

src/client/etcd_state_client.h

@@ -20,6 +20,8 @@
 #define MAX_DATA_BLOCK_SIZE 128*1024*1024
 #define DEFAULT_BITMAP_GRANULARITY 4096
 
+#define VAULT_KEY_PREFIX "vault:"
+
 #ifndef IMMEDIATE_NONE
 #define IMMEDIATE_NONE 0
 #define IMMEDIATE_SMALL 1
@@ -84,7 +86,7 @@ struct inode_config_t
     inode_t parent_id = 0;
     bool readonly = false;
     bool deleted = false;
-    std::vector<uint8_t> enc_key;
+    std::string enc_key;
     // Arbitrary metadata
     json11::Json meta;
     // Change revision of the metadata in etcd

src/client/http_client.cpp

@@ -287,6 +287,48 @@ void http_request(http_co_t *handler, const std::string & host, const std::strin
     handler->send_request(host, request, options, response_callback);
 }
 
+void http_get(http_co_t *handler, const std::string & url, const std::string & headers,
+    const http_options_t & options, std::function<void(http_message_t *response)> response_callback)
+{
+    std::string path;
+    auto ssl = url.substr(0, 8) == "https://";
+    auto host = url.substr(ssl ? 8 : 7);
+    auto pos = host.find('/');
+    if (pos != std::string::npos)
+    {
+        path = host.substr(pos);
+        host = host.substr(0, pos);
+    }
+    std::string req = "GET "+path+" HTTP/1.1\r\n"
+        "Host: "+host+"\r\n"
+        "Connection: "+(options.keepalive ? "keep-alive" : "close")+"\r\n"+
+        headers+"\r\n";
+    handler->send_request(host, req, { .timeout = options.timeout, .keepalive = options.keepalive, .ssl = ssl }, response_callback);
+}
+
+void http_json_post(http_co_t *handler, const std::string & url, json11::Json body, const std::string & headers,
+    const http_options_t & options, std::function<void(http_message_t *response)> response_callback)
+{
+    std::string path;
+    auto ssl = url.substr(0, 8) == "https://";
+    auto host = url.substr(ssl ? 8 : 7);
+    auto pos = host.find('/');
+    if (pos != std::string::npos)
+    {
+        path = host.substr(pos);
+        host = host.substr(0, pos);
+    }
+    std::string req = body.dump();
+    req = "POST "+path+" HTTP/1.1\r\n"
+        "Host: "+host+"\r\n"
+        "Content-Type: application/json\r\n"
+        "Content-Length: "+std::to_string(req.size())+"\r\n"
+        "Connection: "+(options.keepalive ? "keep-alive" : "close")+"\r\n"+
+        headers+
+        "\r\n"+req;
+    handler->send_request(host, req, { .timeout = options.timeout, .keepalive = options.keepalive, .ssl = ssl }, response_callback);
+}
+
 void http_serve(http_co_t *handler, int peer_fd, const http_options_t & options, std::function<void(http_message_t *msg)> request_callback)
 {
     if (handler->state != HTTP_CO_SERVER || handler->peer_fd != peer_fd)

src/client/http_client.h

@@ -56,6 +56,10 @@ void open_websocket(http_co_t *handler, const std::string & addr, const std::str
     const http_options_t & options, std::function<void(http_message_t *msg)> on_message);
 void http_request(http_co_t *handler, const std::string & host, const std::string & request,
     const http_options_t & options, std::function<void(http_message_t *response)> response_callback);
+void http_get(http_co_t *handler, const std::string & url, const std::string & headers,
+    const http_options_t & options, std::function<void(http_message_t *response)> response_callback);
+void http_json_post(http_co_t *handler, const std::string & url, json11::Json body, const std::string & headers,
+    const http_options_t & options, std::function<void(http_message_t *response)> response_callback);
 void http_post_message(http_co_t *handler, uint8_t type, const std::string & msg);
 void http_serve(http_co_t *handler, int peer_fd, const http_options_t & options,
     std::function<void(http_message_t *msg)> request_callback);

src/cmd/cli.cpp

@@ -42,12 +42,13 @@ static const char* help_text =
     "\n"
     "vitastor-cli create -s|--size SIZE [OPTIONS] <name>\n"
     "  Create an image. Options:\n"
-    "  -s|--size SIZE    New image size in bytes or with a K/M/G/T unit suffix.\n"
+    "  -s|--size SIZE      New image size in bytes or with a K/M/G/T unit suffix.\n"
-    "  -p|--pool POOL    Specify pool for the new image (may be omitted if there is only 1 pool).\n"
+    "  -p|--pool POOL      Specify pool for the new image (may be omitted if there is only 1 pool).\n"
-    "  --parent PARENT   Create a copy-on-write image clone based on PARENT (or PARENT@SNAPSHOT).\n"
+    "  --parent PARENT     Create a copy-on-write image clone based on PARENT (or PARENT@SNAPSHOT).\n"
-    "                    If parent is not a snapshot, it must be a read-only image.\n"
+    "                      If parent is not a snapshot, it must be a read-only image.\n"
-    "  --enc-key random  Generate a new random AES-256-XTS encryption key for the new image.\n"
+    "  --enc-key random    Generate a new random AES-256-XTS encryption key for the new image.\n"
-    "  --enc-key HEX     Set a specified AES-256-XTS key (64 bytes in hex) for the new image.\n"
+    "  --enc-key HEX       Set a specified AES-256-XTS key (64 bytes in hex) for the new image.\n"
+    "  --enc-key vault:ID  Use an encryption key from an external Vault secret with specified ID.\n"
     "\n"
     "vitastor-cli create --snapshot <snapshot> [OPTIONS] <image>\n"
     "vitastor-cli snap-create [OPTIONS] <image>@<snapshot>\n"
@@ -55,7 +56,7 @@ static const char* help_text =
     "  Options:\n"
     "  -p|--pool POOL    Move image to pool POOL, leaving the snapshot in the old pool.\n"
     "  --enc-key random  Change image encryption key to a new random AES-256-XTS key.\n"
-    "  --enc-key HEX     Change image encryption key to a specified key or to an empty key.\n"
+    "  --enc-key KEY     Change image encryption key to a specified key, Vault key or to an empty key.\n"
     "                    By default, the image retains its old key when taking a snapshot.\n"
     "\n"
     "vitastor-cli modify <name> [--rename <new-name>] [--resize <size>] [--readonly | --readwrite] [-f|--force] [--down-ok]\n"

src/cmd/cli_create.cpp

@@ -456,11 +456,12 @@ resume_3:
         };
         if (set_key)
         {
-            new_cfg.enc_key.resize(enc_key.size()/2);
+            new_cfg.enc_key = enc_key;
-            fromhexstr(enc_key, new_cfg.enc_key.size(), new_cfg.enc_key.data());
         }
         else if (new_snap != "")
+        {
             new_cfg.enc_key = cur_cfg.enc_key;
+        }
         json11::Json::array checks = json11::Json::array {
             json11::Json::object {
                 { "target", "VERSION" },
@@ -600,19 +601,6 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_create(json11::Json cfg)
     if (!cfg["enc_key"].is_null())
     {
         image_creator->set_key = true;
-        image_creator->enc_key = cfg["enc_key"].string_value();
-        if (image_creator->enc_key != "" &&
-#ifdef WITH_OPENSSL
-            image_creator->enc_key != "random" &&
-#endif
-            (!ishexstr(image_creator->enc_key) || image_creator->enc_key.size() != 128))
-        {
-            return [](cli_result_t & result)
-            {
-                result = (cli_result_t){ .err = EINVAL, .text = "Encryption key is not a 512-bit hex string, not \"\" and not \"random\"" };
-                return true;
-            };
-        }
 #ifdef WITH_OPENSSL
         if (image_creator->enc_key == "random")
         {
@@ -621,6 +609,20 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_create(json11::Json cfg)
             image_creator->enc_key = tohexstr(newkey, 64);
         }
 #endif
+        else
+        {
+            image_creator->enc_key = cfg["enc_key"].string_value();
+            if (image_creator->enc_key != "" &&
+                image_creator->enc_key.substr(0, strlen(VAULT_KEY_PREFIX)) != VAULT_KEY_PREFIX &&
+                (!ishexstr(image_creator->enc_key) || image_creator->enc_key.size() != 128))
+            {
+                return [](cli_result_t & result)
+                {
+                    result = (cli_result_t){ .err = EINVAL, .text = "Encryption key is not a 512-bit hex string, not \"\" and not \"random\"" };
+                    return true;
+                };
+            }
+        }
     }
     image_creator->new_parent = cfg["parent"].string_value();
     if (!cfg["size"].is_null())

src/cmd/cli_modify.cpp

@@ -159,14 +159,15 @@ resume_1:
                 state = 100;
                 return;
             }
-            if (enc_key != "" && (!ishexstr(enc_key) || enc_key.size() != 128))
+            if (enc_key != "" &&
+                enc_key.substr(0, strlen(VAULT_KEY_PREFIX)) != VAULT_KEY_PREFIX &&
+                (!ishexstr(enc_key) || enc_key.size() != 128))
             {
                 result = (cli_result_t){ .err = EINVAL, .text = "Encryption key is not a 512-bit hex string and not \"\"" };
                 state = 100;
                 return;
             }
-            cfg.enc_key.resize(enc_key.size()/2);
+            cfg.enc_key = enc_key;
-            fromhexstr(enc_key, cfg.enc_key.size(), cfg.enc_key.data());
         }
         {
             std::string cur_cfg_key = base64_encode(parent->cli->st_cli.etcd_prefix+

src/cmd/openapi.json

@@ -106,7 +106,7 @@
                         "description": "Encryption key for the new/cloned image",
                         "oneOf": [
                             { "type": "string", "enum": [ "", "random" ] },
-                            { "type": "string", "pattern": "^[0-9a-fA-F]{128}$" }
+                            { "type": "string", "pattern": "^[0-9a-fA-F]{128}$|^vault:" }
                         ]
                     }
                 }
@@ -143,7 +143,7 @@
                         "description": "Change encryption key for the image",
                         "oneOf": [
                             { "type": "string", "enum": [ "" ] },
-                            { "type": "string", "pattern": "^[0-9a-fA-F]{128}$" }
+                            { "type": "string", "pattern": "^[0-9a-fA-F]{128}$|^vault:" }
                         ]
                     },
                     "force": { "type": "boolean", "description": "Proceed with shrinking or setting readwrite flag even if the image has children" },

src/test/test_cluster_client.cpp

@@ -55,7 +55,7 @@ int *test_write(cluster_client_t *cli, uint64_t offset, uint64_t len, uint8_t c,
 {
     printf("Post write %jx+%jx\n", offset, len);
     int *r = new int;
-    *r = instant ? -2 : -1;
+    *r = instant ? -1001 : -1000;
     cluster_op_t *op = new cluster_op_t();
     op->opcode = OSD_OP_WRITE;
     op->inode = 0x1000000000001;
@@ -65,10 +65,11 @@ int *test_write(cluster_client_t *cli, uint64_t offset, uint64_t len, uint8_t c,
     memset(op->iov.buf[0].iov_base, c, len);
     op->callback = [r, cb](cluster_op_t *op)
     {
-        if (*r == -1)
+        if (*r == -1000)
             printf("Error: Not allowed to complete yet\n");
-        assert(*r != -1);
+        assert(*r != -1000);
-        *r = op->retval == op->len ? 1 : 0;
+        assert(op->retval == op->len || op->retval < 0);
+        *r = op->retval == op->len ? 1 : op->retval;
         free(op->iov.buf[0].iov_base);
         printf("Done write %jx+%jx r=%d\n", op->offset, op->len, op->retval);
         delete op;
@@ -90,14 +91,14 @@ int *test_sync(cluster_client_t *cli)
 {
     printf("Post sync\n");
     int *r = new int;
-    *r = -1;
+    *r = -1000;
     cluster_op_t *op = new cluster_op_t();
     op->opcode = OSD_OP_SYNC;
     op->callback = [r](cluster_op_t *op)
     {
-        if (*r == -1)
+        if (*r == -1000)
             printf("Error: Not allowed to complete yet\n");
-        assert(*r != -1);
+        assert(*r != -1000);
         *r = op->retval == 0 ? 1 : 0;
         printf("Done sync r=%d\n", op->retval);
         delete op;
@@ -110,7 +111,7 @@ void can_complete(int *r)
 {
     // Allow the operation to proceed so the test verifies
     // that it doesn't complete earlier than expected
-    *r = -2;
+    *r = -1001;
 }
 
 void check_completed(int *r)
@@ -723,6 +724,67 @@ void test_msgr_decrypt_chain()
 }
 #endif
 
+void test_vault()
+{
+    json11::Json::object config;
+    config["vault_url"] = "http://vault";
+    timerfd_manager_t *tfd = new timerfd_manager_t([](int fd, bool wr, std::function<void(int, int)> callback){});
+    cluster_client_t *cli = new cluster_client_t(NULL, tfd, config);
+
+    configure_single_pg_pool(cli);
+    pretend_connected(cli, 1);
+    cli->st_cli.parse_state((etcd_kv_t){
+        .key = "/config/inode/1/1",
+        .value = json11::Json::object {
+            { "name", "testimg" },
+            { "size", (uint64_t)10*1024*1024*1024 },
+            { "enc_key", "vault:key1" },
+        },
+    });
+
+    // No key -> fetch successfully -> complete op
+
+    int *r1 = test_write(cli, 0, 4096, 0x55);
+    check_op_count(cli, 1, 0);
+
+    assert(cli->vault_key_load_queue == std::vector<std::string>{"vault:key1"});
+    cli->vault_key_load_queue.clear();
+
+    cli->vault_parse_secret("vault:key1", "", json11::Json::object{
+        {"data", json11::Json::object {
+            {"key", "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"}
+        }}
+    });
+
+    can_complete(r1);
+    check_op_count(cli, 1, 1);
+    pretend_op_completed(cli, find_op(cli, 1, OSD_OP_WRITE, 0, 4096), 0);
+    check_completed(r1);
+
+    // No key -> error -> EPERM
+
+    cli->vault_keys.clear();
+    cli->inode_cache.clear();
+    cli->inode_cache_children.clear();
+
+    r1 = test_write(cli, 0, 4096, 0x55);
+    check_op_count(cli, 1, 0);
+
+    assert(cli->vault_key_load_queue == std::vector<std::string>{"vault:key1"});
+    cli->vault_key_load_queue.clear();
+
+    can_complete(r1);
+    cli->vault_parse_secret("vault:key1", "HTTP 403 Forbidden", json11::Json());
+    check_op_count(cli, 1, 0);
+    assert(*r1 == -EPERM);
+    delete r1;
+
+    // Free client
+    delete cli;
+    delete tfd;
+    printf("[ok] basic vault key fetch test\n");
+}
+
 int main(int narg, char *args[])
 {
     test1();
@@ -733,5 +795,6 @@ int main(int narg, char *args[])
     test_msgr_encrypt();
     test_msgr_decrypt_chain();
 #endif
+    test_vault();
     return 0;
 }