diff --git a/docs/usage/cli.en.md b/docs/usage/cli.en.md index 843d7eed..85bf404b 100644 --- a/docs/usage/cli.en.md +++ b/docs/usage/cli.en.md @@ -135,6 +135,7 @@ Create an image. Options: If parent is not a snapshot, it must be a read-only image. * `--enc-key random` - Generate a new random AES-256-XTS encryption key for the new image. * `--enc-key HEX` - Set a specified AES-256-XTS key (64 bytes in hex) for the new image. +* `--enc-key vault:ID` - Use an encryption key from an external Vault secret with specified ID. ``` vitastor-cli create --snapshot [OPTIONS] @@ -147,7 +148,7 @@ Options: * `-p|--pool POOL` - Move image to pool POOL, leaving the snapshot in the old pool. * `--enc-key random` - Change image encryption key to a new random AES-256-XTS key. -* `--enc-key HEX` - Change image encryption key to a specified key or to an empty key. +* `--enc-key KEY` - Change image encryption key to a specified key, Vault key or to an empty key. By default, the image retains its old encryption key when taking a snapshot. See also about [how to export snapshots](qemu.en.md#exporting-snapshots). diff --git a/docs/usage/cli.ru.md b/docs/usage/cli.ru.md index f32b4857..11aa2474 100644 --- a/docs/usage/cli.ru.md +++ b/docs/usage/cli.ru.md @@ -137,6 +137,7 @@ bench-kaveri kaveri 10 G 10 G 0 B/s 0 0 0 us 0 B/s 0 Если `PARENT` - не снимок, он должен быть помечен как образ только для чтения. * `--enc-key random` - Сгенерировать случайный ключ шифрования AES-256-XTS для нового образа. * `--enc-key HEX` - Установить заданный ключ AES-256-XTS (64 байта в hex) для нового образа. +* `--enc-key vault:ID` - Использовать ключ из внешнего секрета с заданным ID из Vault. ``` vitastor-cli create --snapshot [ОПЦИИ] @@ -150,7 +151,7 @@ vitastor-cli snap-create [ОПЦИИ] @ * `-p|--pool POOL` - Переместить образ в пул POOL, оставив снимок в старом пуле. * `--enc-key random` - Изменить ключ шифрования образа на новый случайный ключ AES-256-XTS. -* `--enc-key HEX` - Изменить ключ шифрования образа на заданный или пустой ключ. +* `--enc-key KEY` - Изменить ключ шифрования образа на заданный ключ, ключ из Vault или пустой ключ. По умолчанию шифрованные образы сохраняют старый ключ при снятии снимка. Смотрите также информацию о том, [как экспортировать снимки](qemu.ru.md#экспорт-снимков). diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 013df631..c0378e2a 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -24,6 +24,7 @@ add_library(vitastor_client SHARED cluster_client.cpp cluster_client_list.cpp cluster_client_wb.cpp + cluster_client_icache.cpp vitastor_c.cpp ) set_target_properties(vitastor_client PROPERTIES PUBLIC_HEADER "client/vitastor_c.h") @@ -96,7 +97,7 @@ endif (${WITH_QEMU}) add_executable(test_cluster_client EXCLUDE_FROM_ALL ../test/test_cluster_client.cpp - pg_states.cpp osd_ops.cpp cluster_client.cpp cluster_client_list.cpp cluster_client_wb.cpp msgr_op.cpp ../test/mock/messenger.cpp msgr_stop.cpp msgr_encrypt.cpp + pg_states.cpp osd_ops.cpp cluster_client.cpp cluster_client_list.cpp cluster_client_wb.cpp cluster_client_icache.cpp msgr_op.cpp ../test/mock/messenger.cpp msgr_stop.cpp msgr_encrypt.cpp etcd_state_client.cpp ../util/timerfd_manager.cpp ../util/addr_util.cpp ../util/str_util.cpp ../util/json_util.cpp ../../json11/json11.cpp ) target_link_libraries(test_cluster_client ${LIBURING_LIBRARIES} ${OPENSSL_LIBRARIES}) diff --git a/src/client/cluster_client.cpp b/src/client/cluster_client.cpp index 628aa36d..52c9fbe3 100644 --- a/src/client/cluster_client.cpp +++ b/src/client/cluster_client.cpp @@ -11,16 +11,6 @@ #define TRY_SEND_CONNECTING 1 #define TRY_SEND_OK 2 -inode_cache_t::~inode_cache_t() -{ - if (key_data) - { - free(key_data); - key_data = NULL; - op_enc = NULL; - } -} - cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json config) { wb = new writeback_cache_t(); @@ -88,6 +78,7 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd cluster_client_t::~cluster_client_t() { + vault_destroy(); if (retry_timeout_id >= 0) { tfd->clear_timer(retry_timeout_id); @@ -492,6 +483,8 @@ void cluster_client_t::on_load_config_hook(json11::Json::object & etcd_global_co self_tree_metrics.clear(); client_hostname = new_hostname; } + // vault + vault_parse_config(); msgr.parse_config(config); st_cli.parse_config(config); st_cli.load_pgs(); @@ -620,6 +613,7 @@ void cluster_client_t::on_change_pool_config_hook() } inode_cache.clear(); inode_cache_children.clear(); + vault_keys.clear(); continue_ops(); } @@ -665,136 +659,6 @@ void cluster_client_t::on_change_node_placement_hook() self_tree_metrics.clear(); } -// FIXME: Rework client API by adding open/close and cache inode information in the "FD" (maybe) -void cluster_client_t::on_change_inode_hook(uint64_t inode, bool removed) -{ - std::vector children = { inode }; - for (size_t i = 0; i < children.size(); i++) - { - auto it = inode_cache_children.lower_bound(std::make_pair(children[i], (inode_t)0)); - while (it != inode_cache_children.end() && it->first == children[i]) - { - children.push_back(it->second); - it++; - } - } - for (auto & inode: children) - { - auto it = inode_cache.find(inode); - if (it != inode_cache.end()) - { - auto icache = it->second; - for (auto & parent: icache->chain) - { - inode_cache_children.erase(std::make_pair(parent, inode)); - } - inode_cache.erase(it); - } - } -} - -std::shared_ptr cluster_client_t::inode_cache_get(inode_t ino) -{ - auto icache_it = inode_cache.find(ino); - if (icache_it != inode_cache.end()) - { - return icache_it->second; - } - // Fill inode cache - auto ino_it = st_cli.inode_config.find(ino); - if (ino_it == st_cli.inode_config.end()) - { - inode_cache[ino] = NULL; - return NULL; - } - auto pool_it = st_cli.pool_config.find(INODE_POOL(ino)); - if (pool_it == st_cli.pool_config.end()) - { - inode_cache[ino] = NULL; - return NULL; - } - auto & inode_cfg = ino_it->second; - auto & pool_cfg = pool_it->second; - std::shared_ptr icache = std::make_shared(); - icache->readonly = inode_cfg.readonly; - icache->chain.push_back(ino); - std::vector chain_cfg; - int enc_key_count = !inode_cfg.enc_key.empty() ? 1 : 0; - if (inode_cfg.parent_id) - { - // Check for loops and cache the chain - robin_hood::unordered_flat_set seen; - seen.insert(ino); - uint64_t parent_id = inode_cfg.parent_id; - while (parent_id) - { - if (seen.find(parent_id) != seen.end()) - { - icache->has_parent_loop = true; - break; - } - seen.insert(parent_id); - ino_it = st_cli.inode_config.find(parent_id); - if (INODE_POOL(parent_id) == INODE_POOL(ino)) - { - icache->chain.push_back(parent_id); - if (ino_it == st_cli.inode_config.end()) - chain_cfg.push_back(NULL); - else - { - chain_cfg.push_back(&ino_it->second); - if (!ino_it->second.enc_key.empty()) - enc_key_count++; - } - } - else if (!icache->other_pool_parent_id) - icache->other_pool_parent_id = parent_id; - if (ino_it == st_cli.inode_config.end()) - break; - parent_id = ino_it->second.parent_id; - } - } - // Generate encryption key chain, if applicable - if (enc_key_count) - { - uint8_t *key_data = (uint8_t*)malloc_or_die( - AES_256_XTS_KEY_SIZE * enc_key_count + - sizeof(uint8_t*) * icache->chain.size() + - sizeof(osd_op_enc_t) - ); - uint8_t **keys = (uint8_t**)(key_data + AES_256_XTS_KEY_SIZE * enc_key_count); - osd_op_enc_t *enc = (osd_op_enc_t*)((uint8_t*)keys + sizeof(uint8_t*)*icache->chain.size()); - size_t key_pos = 0; - for (size_t i = 0; i <= chain_cfg.size(); i++) - { - inode_config_t *cfg = !i ? &inode_cfg : chain_cfg[i-1]; - if (cfg && !cfg->enc_key.empty()) - { - assert(key_pos < AES_256_XTS_KEY_SIZE * enc_key_count); - assert(cfg->enc_key.size() == AES_256_XTS_KEY_SIZE); - keys[i] = key_data + key_pos; - memcpy(key_data + key_pos, cfg->enc_key.data(), AES_256_XTS_KEY_SIZE); - key_pos += AES_256_XTS_KEY_SIZE; - } - else - keys[i] = NULL; - } - enc->key_chain = keys; - enc->chain_size = icache->chain.size(); - enc->read_chain_bitmap_pos = pool_cfg.data_block_size/pool_cfg.bitmap_granularity/8; - enc->bitmap_granularity = pool_cfg.bitmap_granularity; - icache->key_data = key_data; - icache->op_enc = enc; - } - inode_cache[ino] = icache; - for (auto & parent: icache->chain) - { - if (parent != ino) - inode_cache_children.insert(std::make_pair(parent, ino)); - } - return icache; -} - bool cluster_client_t::is_ready() { return pgs_loaded; @@ -816,6 +680,10 @@ bool cluster_client_t::flush() { if (!ringloop) { + if (vault_loading) + { + return false; + } if (wb->writeback_queue.size()) { wb->start_writebacks(this, 0); @@ -838,7 +706,7 @@ bool cluster_client_t::flush() sync_done = true; }; execute(sync); - while (!sync_done) + while (!sync_done || vault_loading) { ringloop->loop(); if (!sync_done) @@ -1163,6 +1031,21 @@ bool cluster_client_t::check_rw(cluster_op_t *op) } } } + if (icache && icache->err_code) + { + if (icache->err_code == EPERM) + { + op->retval = -EPERM; + auto cb = std::move(op->callback); + cb(op); + return false; + } + else if (icache->err_code == EAGAIN) + { + key_wait_ops.push_back(op); + return false; + } + } return true; } diff --git a/src/client/cluster_client.h b/src/client/cluster_client.h index 8709c1b6..d65f6dec 100644 --- a/src/client/cluster_client.h +++ b/src/client/cluster_client.h @@ -90,18 +90,25 @@ struct inode_cache_t bool readonly = false; bool has_parent_loop = false; inode_t other_pool_parent_id = 0; + int err_code = 0; ~inode_cache_t(); }; +struct vault_load_key_t +{ + int key_state = 0; + std::string key; +}; + // FIXME: Split into public and private interfaces class __attribute__((visibility("default"))) cluster_client_t { timerfd_manager_t *tfd = NULL; ring_loop_t *ringloop = NULL; - std::map pg_counts; - std::map pg_primary; + // config: + // client_max_dirty_* is actually "max unsynced", for the case when immediate_commit is off uint64_t client_max_dirty_bytes = 0; uint64_t client_max_dirty_ops = 0; @@ -113,12 +120,23 @@ class __attribute__((visibility("default"))) cluster_client_t uint64_t client_max_writeback_iodepth = 0; std::string conf_hostname; + std::string vault_url; + std::string vault_client_cert; + std::string vault_client_key; + std::string vault_ca; + std::string vault_secret_api_path; + uint64_t vault_timeout_ms = 0; + uint64_t vault_error_timeout_sec = 0; + uint64_t vault_refresh_leeway_sec = 0; + int log_level = 0; int client_retry_interval = 50; // ms int client_eio_retry_interval = 1000; // ms bool client_retry_enospc = true; int client_wait_up_timeout = 16; // sec (for listings) + // state: + std::string client_hostname; std::map self_tree_metrics; std::map osd_tree_metrics; @@ -126,6 +144,7 @@ class __attribute__((visibility("default"))) cluster_client_t int retry_timeout_id = -1; int retry_timeout_duration = 0; std::vector offline_ops; + std::vector key_wait_ops; cluster_op_t *op_queue_head = NULL, *op_queue_tail = NULL; writeback_cache_t *wb = NULL; std::set dirty_osds; @@ -139,7 +158,17 @@ class __attribute__((visibility("default"))) cluster_client_t robin_hood::unordered_flat_map> inode_cache; std::set> inode_cache_children; + http_context_t *vault_http_ctx = NULL; + http_co_t *vault_http_cli = NULL; + bool vault_loading = false; + std::string vault_token; + bool vault_auth_error = false; + timespec vault_token_expire = {}; + std::vector vault_key_load_queue; + std::map vault_keys; + bool pgs_loaded = false; + std::map pg_counts; ring_consumer_t consumer; std::vector> on_ready_hooks; int list_retry_timeout_id = -1; @@ -174,6 +203,13 @@ public: protected: void continue_ops(int time_passed = 0); + std::shared_ptr inode_cache_get(inode_t ino); + void vault_parse_config(); + bool vault_check_token(); + void vault_load_keys(); + void vault_destroy(); + void vault_parse_secret(const std::string & key_id, const std::string & err, json11::Json data); + bool affects_osd(uint64_t inode, uint64_t offset, uint64_t len, osd_num_t osd); bool affects_pg(uint64_t inode, uint64_t offset, uint64_t len, pool_id_t pool_id, pg_num_t pg_num); @@ -185,8 +221,6 @@ protected: void on_change_node_placement_hook(); void on_change_inode_hook(uint64_t inode, bool removed); - std::shared_ptr inode_cache_get(inode_t ino); - void execute_internal(cluster_op_t *op); void execute_cas(cluster_op_t *op); void unshift_op(cluster_op_t *op); diff --git a/src/client/cluster_client_icache.cpp b/src/client/cluster_client_icache.cpp new file mode 100644 index 00000000..626b2d73 --- /dev/null +++ b/src/client/cluster_client_icache.cpp @@ -0,0 +1,366 @@ +// Copyright (c) Vitaliy Filippov, 2019+ +// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details) + +#include +#include +#include "cluster_client_impl.h" +#include "http_client.h" +#include "str_util.h" + +#define VAULT_KEY_NOT_LOADED 0 +#define VAULT_KEY_LOADING 1 +#define VAULT_KEY_LOADED 2 +#define VAULT_KEY_ERROR 3 + +inode_cache_t::~inode_cache_t() +{ + if (key_data) + { + free(key_data); + key_data = NULL; + op_enc = NULL; + } +} + +void cluster_client_t::vault_destroy() +{ + if (vault_http_ctx) + { +#ifndef __MOCK__ + http_destroy(vault_http_cli); + http_context_destroy(vault_http_ctx); + vault_http_cli = NULL; + vault_http_ctx = NULL; +#endif + } +} + +void cluster_client_t::vault_parse_config() +{ + vault_url = config["vault_url"].string_value(); + vault_client_cert = config["vault_client_cert"].string_value(); + vault_client_key = config["vault_client_key"].string_value(); + vault_ca = config["vault_ca"].string_value(); + vault_secret_api_path = "/v1/secret/"; + if (config["vault_secret_api_path"].is_string()) + vault_secret_api_path = config["vault_secret_api_path"].string_value(); + vault_timeout_ms = config["vault_timeout_ms"].uint64_value(); + if (!vault_timeout_ms) + vault_timeout_ms = 5000; + vault_error_timeout_sec = config["vault_error_timeout_sec"].uint64_value(); + if (!vault_error_timeout_sec) + vault_error_timeout_sec = 60; + vault_refresh_leeway_sec = config["vault_refresh_leeway_sec"].uint64_value(); + if (!vault_refresh_leeway_sec) + vault_refresh_leeway_sec = 60; +} + +// FIXME: Rework client API by adding open/close and cache inode information in the "FD" (maybe) +void cluster_client_t::on_change_inode_hook(uint64_t inode, bool removed) +{ + std::vector children = { inode }; + for (size_t i = 0; i < children.size(); i++) + { + auto it = inode_cache_children.lower_bound(std::make_pair(children[i], (inode_t)0)); + while (it != inode_cache_children.end() && it->first == children[i]) + { + children.push_back(it->second); + it++; + } + } + for (auto & inode: children) + { + auto it = inode_cache.find(inode); + if (it != inode_cache.end()) + { + auto icache = it->second; + for (auto & parent: icache->chain) + { + inode_cache_children.erase(std::make_pair(parent, inode)); + } + inode_cache.erase(it); + } + } +} + +std::shared_ptr cluster_client_t::inode_cache_get(inode_t ino) +{ + auto icache_it = inode_cache.find(ino); + if (icache_it != inode_cache.end()) + { + return icache_it->second; + } + // Fill inode cache + auto ino_it = st_cli.inode_config.find(ino); + if (ino_it == st_cli.inode_config.end()) + { + inode_cache[ino] = NULL; + return NULL; + } + auto pool_it = st_cli.pool_config.find(INODE_POOL(ino)); + if (pool_it == st_cli.pool_config.end()) + { + inode_cache[ino] = NULL; + return NULL; + } + auto & inode_cfg = ino_it->second; + auto & pool_cfg = pool_it->second; + std::shared_ptr icache = std::make_shared(); + icache->readonly = inode_cfg.readonly; + icache->chain.push_back(ino); + std::vector chain_cfg; + int enc_key_count = !inode_cfg.enc_key.empty() ? 1 : 0; + if (inode_cfg.parent_id) + { + // Check for loops and cache the chain + robin_hood::unordered_flat_set seen; + seen.insert(ino); + uint64_t parent_id = inode_cfg.parent_id; + while (parent_id) + { + if (seen.find(parent_id) != seen.end()) + { + icache->has_parent_loop = true; + break; + } + seen.insert(parent_id); + ino_it = st_cli.inode_config.find(parent_id); + if (INODE_POOL(parent_id) == INODE_POOL(ino)) + { + icache->chain.push_back(parent_id); + if (ino_it == st_cli.inode_config.end()) + chain_cfg.push_back(NULL); + else + { + chain_cfg.push_back(&ino_it->second); + if (!ino_it->second.enc_key.empty()) + enc_key_count++; + } + } + else if (!icache->other_pool_parent_id) + icache->other_pool_parent_id = parent_id; + if (ino_it == st_cli.inode_config.end()) + break; + parent_id = ino_it->second.parent_id; + } + } + // Check external keys and wait for loading, if required + if (enc_key_count) + { + for (size_t i = 0; i <= chain_cfg.size(); i++) + { + inode_config_t *cfg = !i ? &inode_cfg : chain_cfg[i-1]; + if (cfg && cfg->enc_key.substr(0, strlen(VAULT_KEY_PREFIX)) == VAULT_KEY_PREFIX) + { + auto & ik = vault_keys[inode_cfg.enc_key]; + if (ik.key_state == VAULT_KEY_ERROR || vault_url.empty()) + { + icache->err_code = EPERM; + enc_key_count = 0; + } + else if (ik.key_state == VAULT_KEY_NOT_LOADED) + { + ik.key_state = VAULT_KEY_LOADING; + vault_key_load_queue.push_back(inode_cfg.enc_key); + vault_load_keys(); + icache->err_code = EAGAIN; + enc_key_count = 0; + } + else if (ik.key_state == VAULT_KEY_LOADING) + { + icache->err_code = EAGAIN; + enc_key_count = 0; + } + else + { + assert(ik.key_state == VAULT_KEY_LOADED); + } + } + } + } + // Generate encryption key chain, if applicable + if (enc_key_count) + { + uint8_t *key_data = (uint8_t*)malloc_or_die( + AES_256_XTS_KEY_SIZE * enc_key_count + + sizeof(uint8_t*) * icache->chain.size() + + sizeof(osd_op_enc_t) + ); + uint8_t **keys = (uint8_t**)(key_data + AES_256_XTS_KEY_SIZE * enc_key_count); + osd_op_enc_t *enc = (osd_op_enc_t*)((uint8_t*)keys + sizeof(uint8_t*)*icache->chain.size()); + size_t key_pos = 0; + for (size_t i = 0; i <= chain_cfg.size(); i++) + { + inode_config_t *cfg = !i ? &inode_cfg : chain_cfg[i-1]; + if (cfg && !cfg->enc_key.empty()) + { + const auto & key = cfg->enc_key.substr(0, strlen(VAULT_KEY_PREFIX)) == VAULT_KEY_PREFIX + ? vault_keys.at(cfg->enc_key).key + : cfg->enc_key; + assert(key_pos < AES_256_XTS_KEY_SIZE * enc_key_count); + assert(key.size() == 2*AES_256_XTS_KEY_SIZE); + keys[i] = key_data + key_pos; + fromhexstr(key, AES_256_XTS_KEY_SIZE, key_data + key_pos); + key_pos += AES_256_XTS_KEY_SIZE; + } + else + keys[i] = NULL; + } + enc->key_chain = keys; + enc->chain_size = icache->chain.size(); + enc->read_chain_bitmap_pos = pool_cfg.data_block_size/pool_cfg.bitmap_granularity/8; + enc->bitmap_granularity = pool_cfg.bitmap_granularity; + icache->key_data = key_data; + icache->op_enc = enc; + } + inode_cache[ino] = icache; + for (auto & parent: icache->chain) + { + if (parent != ino) + inode_cache_children.insert(std::make_pair(parent, ino)); + } + return icache; +} + +#ifndef __MOCK__ +bool cluster_client_t::vault_check_token() +{ + timespec now; + clock_gettime(CLOCK_REALTIME, &now); + if (!vault_token_expire.tv_sec || vault_token_expire.tv_sec < now.tv_sec) + { + vault_loading = true; + http_json_post( + vault_http_cli, vault_url+"/v1/auth/cert/login", json11::Json::object{}, "", + (http_options_t){ .timeout = (int)vault_timeout_ms, .keepalive = true }, + [this](http_message_t *response) + { + clock_gettime(CLOCK_REALTIME, &vault_token_expire); + vault_loading = false; + std::string err; + json11::Json data; + response->parse_json_response(err, data); + if (err != "") + { + vault_token_expire.tv_sec += vault_error_timeout_sec; + fprintf(stderr, "Vault request failed: %s\n", err.c_str()); + } + else + { + uint64_t ttl = data["auth"]["lease_duration"].uint64_value(); + vault_token = data["auth"]["client_token"].string_value(); + if (vault_token.empty() || !ttl) + { + vault_token_expire.tv_sec += vault_error_timeout_sec; + fprintf(stderr, "No token or lease_duration in Vault response: %s\n", data.dump().c_str()); + } + else + { + if (ttl < vault_refresh_leeway_sec) + vault_token_expire.tv_sec += ttl/2; + else + vault_token_expire.tv_sec += ttl - vault_refresh_leeway_sec; + } + } + vault_load_keys(); + } + ); + return false; + } + if (vault_token.empty()) + { + // Auth error happened, mark all loads as failed + for (auto & key_id: vault_key_load_queue) + { + auto & k = vault_keys[key_id]; + k.key_state = VAULT_KEY_ERROR; + } + vault_key_load_queue.clear(); + auto ops = std::move(key_wait_ops); + for (cluster_op_t *op: ops) + inode_cache.erase(op->inode); + for (cluster_op_t *op: ops) + execute_internal(op); + return false; + } + return true; +} +#endif + +void cluster_client_t::vault_load_keys() +{ + if (vault_loading || !vault_key_load_queue.size()) + { + return; + } +#ifdef __MOCK__ + vault_loading = true; +#else + if (!vault_http_ctx) + { + std::string error; + vault_http_ctx = http_context_init(tfd, vault_client_cert, vault_client_key, vault_ca, true, error); + if (!vault_http_ctx) + { + fprintf(stderr, "Failed to initialize HTTP context for Vault: %s\n", error.c_str()); + exit(1); + } + vault_http_cli = http_init(vault_http_ctx); + } + if (!vault_check_token()) + { + return; + } + std::string key_id = vault_key_load_queue[0]; + vault_key_load_queue.erase(vault_key_load_queue.begin()); + vault_loading = true; + http_get( + vault_http_cli, vault_url+vault_secret_api_path+key_id.substr(strlen(VAULT_KEY_PREFIX)), "X-Vault-Token: "+vault_token+"\r\n", + (http_options_t){ .timeout = (int)vault_timeout_ms, .keepalive = true }, + [this, key_id](http_message_t *response) + { + vault_loading = false; + std::string err; + json11::Json data; + response->parse_json_response(err, data); + vault_parse_secret(key_id, err, data); + } + ); +#endif +} + +void cluster_client_t::vault_parse_secret(const std::string & key_id, const std::string & err, json11::Json data) +{ + vault_loading = false; + auto & k = vault_keys[key_id]; + if (err != "") + { + k.key_state = VAULT_KEY_ERROR; + fprintf(stderr, "Vault %s%s%s request failed: %s\n", vault_url.c_str(), + vault_secret_api_path.c_str(), key_id.c_str()+strlen(VAULT_KEY_PREFIX), err.c_str()); + } + else + { + auto hexkey = data["data"]["key"].string_value(); + if (hexkey.empty() || !ishexstr(hexkey) || hexkey.size() != 2*AES_256_XTS_KEY_SIZE) + { + k.key_state = VAULT_KEY_ERROR; + fprintf(stderr, "Vault /v1/secret/%s request failed: 'key' is empty or has invalid format\n", key_id.c_str()); + } + else + { + k.key_state = VAULT_KEY_LOADED; + k.key = hexkey; + } + } + if (vault_key_load_queue.empty()) + { + auto ops = std::move(key_wait_ops); + for (cluster_op_t *op: ops) + inode_cache.erase(op->inode); + for (cluster_op_t *op: ops) + execute_internal(op); + } + else + vault_load_keys(); +} diff --git a/src/client/etcd_state_client.cpp b/src/client/etcd_state_client.cpp index 305d2c09..60944848 100644 --- a/src/client/etcd_state_client.cpp +++ b/src/client/etcd_state_client.cpp @@ -98,32 +98,15 @@ http_context_t *etcd_state_client_t::get_http_ctx() void etcd_state_client_t::etcd_call_oneshot(const std::string & etcd_url, const std::string & api, json11::Json payload, int timeout, std::function callback) { - std::string etcd_api_path; - bool ssl = etcd_url.substr(0, 8) == "https://"; - auto etcd_address = etcd_url.substr(ssl ? 8 : 7); - int pos = etcd_address.find('/'); - if (pos >= 0) - { - etcd_api_path = etcd_address.substr(pos); - etcd_address = etcd_address.substr(0, pos); - } - std::string req = payload.dump(); - req = "POST "+etcd_api_path+api+" HTTP/1.1\r\n" - "Host: "+etcd_address+"\r\n" - "Content-Type: application/json\r\n" - "Content-Length: "+std::to_string(req.size())+"\r\n" - "Connection: close\r\n" - "\r\n"+req; auto http_cli = http_init(get_http_ctx()); - auto cb = [http_cli, callback](http_message_t *response) + http_json_post(http_cli, etcd_url+api, payload, "", { .timeout = timeout }, [http_cli, callback](http_message_t *response) { std::string err; json11::Json data; response->parse_json_response(err, data); callback(err, data); http_destroy(http_cli); - }; - http_request(http_cli, etcd_address, req, { .timeout = timeout, .ssl = ssl }, cb); + }); } void etcd_state_client_t::etcd_call(const std::string & api, json11::Json payload, int timeout, @@ -1446,7 +1429,7 @@ json11::Json::object etcd_state_client_t::serialize_inode_cfg(inode_config_t *cf } if (!cfg->enc_key.empty()) { - new_cfg["enc_key"] = tohexstr(cfg->enc_key.data(), cfg->enc_key.size()); + new_cfg["enc_key"] = cfg->enc_key; } if (cfg->readonly) { @@ -1482,18 +1465,15 @@ inode_config_t etcd_state_client_t::deserialize_inode_cfg(uint64_t inode_num, js else parent_inode_num |= parent_pool_id << (64-POOL_ID_BITS); } - std::vector enc_key; + std::string enc_key; if (!value["enc_key"].is_null()) { - if (value["enc_key"].string_value().size() == 2*AES_256_XTS_KEY_SIZE) + enc_key = value["enc_key"].string_value(); + if (enc_key.substr(0, strlen(VAULT_KEY_PREFIX)) != VAULT_KEY_PREFIX && + (enc_key.size() != 2*AES_256_XTS_KEY_SIZE || !ishexstr(enc_key))) { - enc_key.resize(AES_256_XTS_KEY_SIZE); - if (fromhexstr(value["enc_key"].string_value(), AES_256_XTS_KEY_SIZE, enc_key.data()) < AES_256_XTS_KEY_SIZE) - enc_key.clear(); - } - if (enc_key.empty()) - { - fprintf(stderr, "Inode %u/%ju has invalid enc_key, should be %u bit hex string\n", + enc_key = ""; + fprintf(stderr, "Inode %u/%ju has invalid enc_key, should be %u bit hex string or Vault key reference\n", INODE_POOL(inode_num), INODE_NO_POOL(inode_num), AES_256_XTS_KEY_SIZE); } } diff --git a/src/client/etcd_state_client.h b/src/client/etcd_state_client.h index d9b37f4e..b0b25418 100644 --- a/src/client/etcd_state_client.h +++ b/src/client/etcd_state_client.h @@ -20,6 +20,8 @@ #define MAX_DATA_BLOCK_SIZE 128*1024*1024 #define DEFAULT_BITMAP_GRANULARITY 4096 +#define VAULT_KEY_PREFIX "vault:" + #ifndef IMMEDIATE_NONE #define IMMEDIATE_NONE 0 #define IMMEDIATE_SMALL 1 @@ -84,7 +86,7 @@ struct inode_config_t inode_t parent_id = 0; bool readonly = false; bool deleted = false; - std::vector enc_key; + std::string enc_key; // Arbitrary metadata json11::Json meta; // Change revision of the metadata in etcd diff --git a/src/client/http_client.cpp b/src/client/http_client.cpp index 7cf10c8c..7ea7e360 100644 --- a/src/client/http_client.cpp +++ b/src/client/http_client.cpp @@ -287,6 +287,48 @@ void http_request(http_co_t *handler, const std::string & host, const std::strin handler->send_request(host, request, options, response_callback); } +void http_get(http_co_t *handler, const std::string & url, const std::string & headers, + const http_options_t & options, std::function response_callback) +{ + std::string path; + auto ssl = url.substr(0, 8) == "https://"; + auto host = url.substr(ssl ? 8 : 7); + auto pos = host.find('/'); + if (pos != std::string::npos) + { + path = host.substr(pos); + host = host.substr(0, pos); + } + std::string req = "GET "+path+" HTTP/1.1\r\n" + "Host: "+host+"\r\n" + "Connection: "+(options.keepalive ? "keep-alive" : "close")+"\r\n"+ + headers+"\r\n"; + handler->send_request(host, req, { .timeout = options.timeout, .keepalive = options.keepalive, .ssl = ssl }, response_callback); +} + +void http_json_post(http_co_t *handler, const std::string & url, json11::Json body, const std::string & headers, + const http_options_t & options, std::function response_callback) +{ + std::string path; + auto ssl = url.substr(0, 8) == "https://"; + auto host = url.substr(ssl ? 8 : 7); + auto pos = host.find('/'); + if (pos != std::string::npos) + { + path = host.substr(pos); + host = host.substr(0, pos); + } + std::string req = body.dump(); + req = "POST "+path+" HTTP/1.1\r\n" + "Host: "+host+"\r\n" + "Content-Type: application/json\r\n" + "Content-Length: "+std::to_string(req.size())+"\r\n" + "Connection: "+(options.keepalive ? "keep-alive" : "close")+"\r\n"+ + headers+ + "\r\n"+req; + handler->send_request(host, req, { .timeout = options.timeout, .keepalive = options.keepalive, .ssl = ssl }, response_callback); +} + void http_serve(http_co_t *handler, int peer_fd, const http_options_t & options, std::function request_callback) { if (handler->state != HTTP_CO_SERVER || handler->peer_fd != peer_fd) diff --git a/src/client/http_client.h b/src/client/http_client.h index 457b27bf..72761d0c 100644 --- a/src/client/http_client.h +++ b/src/client/http_client.h @@ -56,6 +56,10 @@ void open_websocket(http_co_t *handler, const std::string & addr, const std::str const http_options_t & options, std::function on_message); void http_request(http_co_t *handler, const std::string & host, const std::string & request, const http_options_t & options, std::function response_callback); +void http_get(http_co_t *handler, const std::string & url, const std::string & headers, + const http_options_t & options, std::function response_callback); +void http_json_post(http_co_t *handler, const std::string & url, json11::Json body, const std::string & headers, + const http_options_t & options, std::function response_callback); void http_post_message(http_co_t *handler, uint8_t type, const std::string & msg); void http_serve(http_co_t *handler, int peer_fd, const http_options_t & options, std::function request_callback); diff --git a/src/cmd/cli.cpp b/src/cmd/cli.cpp index e710c2c2..c9aaaf2a 100644 --- a/src/cmd/cli.cpp +++ b/src/cmd/cli.cpp @@ -42,12 +42,13 @@ static const char* help_text = "\n" "vitastor-cli create -s|--size SIZE [OPTIONS] \n" " Create an image. Options:\n" - " -s|--size SIZE New image size in bytes or with a K/M/G/T unit suffix.\n" - " -p|--pool POOL Specify pool for the new image (may be omitted if there is only 1 pool).\n" - " --parent PARENT Create a copy-on-write image clone based on PARENT (or PARENT@SNAPSHOT).\n" - " If parent is not a snapshot, it must be a read-only image.\n" - " --enc-key random Generate a new random AES-256-XTS encryption key for the new image.\n" - " --enc-key HEX Set a specified AES-256-XTS key (64 bytes in hex) for the new image.\n" + " -s|--size SIZE New image size in bytes or with a K/M/G/T unit suffix.\n" + " -p|--pool POOL Specify pool for the new image (may be omitted if there is only 1 pool).\n" + " --parent PARENT Create a copy-on-write image clone based on PARENT (or PARENT@SNAPSHOT).\n" + " If parent is not a snapshot, it must be a read-only image.\n" + " --enc-key random Generate a new random AES-256-XTS encryption key for the new image.\n" + " --enc-key HEX Set a specified AES-256-XTS key (64 bytes in hex) for the new image.\n" + " --enc-key vault:ID Use an encryption key from an external Vault secret with specified ID.\n" "\n" "vitastor-cli create --snapshot [OPTIONS] \n" "vitastor-cli snap-create [OPTIONS] @\n" @@ -55,7 +56,7 @@ static const char* help_text = " Options:\n" " -p|--pool POOL Move image to pool POOL, leaving the snapshot in the old pool.\n" " --enc-key random Change image encryption key to a new random AES-256-XTS key.\n" - " --enc-key HEX Change image encryption key to a specified key or to an empty key.\n" + " --enc-key KEY Change image encryption key to a specified key, Vault key or to an empty key.\n" " By default, the image retains its old key when taking a snapshot.\n" "\n" "vitastor-cli modify [--rename ] [--resize ] [--readonly | --readwrite] [-f|--force] [--down-ok]\n" diff --git a/src/cmd/cli_create.cpp b/src/cmd/cli_create.cpp index 6672078b..b2e6acef 100644 --- a/src/cmd/cli_create.cpp +++ b/src/cmd/cli_create.cpp @@ -456,11 +456,12 @@ resume_3: }; if (set_key) { - new_cfg.enc_key.resize(enc_key.size()/2); - fromhexstr(enc_key, new_cfg.enc_key.size(), new_cfg.enc_key.data()); + new_cfg.enc_key = enc_key; } else if (new_snap != "") + { new_cfg.enc_key = cur_cfg.enc_key; + } json11::Json::array checks = json11::Json::array { json11::Json::object { { "target", "VERSION" }, @@ -600,19 +601,6 @@ std::function cli_tool_t::start_create(json11::Json cfg) if (!cfg["enc_key"].is_null()) { image_creator->set_key = true; - image_creator->enc_key = cfg["enc_key"].string_value(); - if (image_creator->enc_key != "" && -#ifdef WITH_OPENSSL - image_creator->enc_key != "random" && -#endif - (!ishexstr(image_creator->enc_key) || image_creator->enc_key.size() != 128)) - { - return [](cli_result_t & result) - { - result = (cli_result_t){ .err = EINVAL, .text = "Encryption key is not a 512-bit hex string, not \"\" and not \"random\"" }; - return true; - }; - } #ifdef WITH_OPENSSL if (image_creator->enc_key == "random") { @@ -621,6 +609,20 @@ std::function cli_tool_t::start_create(json11::Json cfg) image_creator->enc_key = tohexstr(newkey, 64); } #endif + else + { + image_creator->enc_key = cfg["enc_key"].string_value(); + if (image_creator->enc_key != "" && + image_creator->enc_key.substr(0, strlen(VAULT_KEY_PREFIX)) != VAULT_KEY_PREFIX && + (!ishexstr(image_creator->enc_key) || image_creator->enc_key.size() != 128)) + { + return [](cli_result_t & result) + { + result = (cli_result_t){ .err = EINVAL, .text = "Encryption key is not a 512-bit hex string, not \"\" and not \"random\"" }; + return true; + }; + } + } } image_creator->new_parent = cfg["parent"].string_value(); if (!cfg["size"].is_null()) diff --git a/src/cmd/cli_modify.cpp b/src/cmd/cli_modify.cpp index 5967078b..18c3d8f3 100644 --- a/src/cmd/cli_modify.cpp +++ b/src/cmd/cli_modify.cpp @@ -159,14 +159,15 @@ resume_1: state = 100; return; } - if (enc_key != "" && (!ishexstr(enc_key) || enc_key.size() != 128)) + if (enc_key != "" && + enc_key.substr(0, strlen(VAULT_KEY_PREFIX)) != VAULT_KEY_PREFIX && + (!ishexstr(enc_key) || enc_key.size() != 128)) { result = (cli_result_t){ .err = EINVAL, .text = "Encryption key is not a 512-bit hex string and not \"\"" }; state = 100; return; } - cfg.enc_key.resize(enc_key.size()/2); - fromhexstr(enc_key, cfg.enc_key.size(), cfg.enc_key.data()); + cfg.enc_key = enc_key; } { std::string cur_cfg_key = base64_encode(parent->cli->st_cli.etcd_prefix+ diff --git a/src/cmd/openapi.json b/src/cmd/openapi.json index f710c48b..c192b3f9 100644 --- a/src/cmd/openapi.json +++ b/src/cmd/openapi.json @@ -106,7 +106,7 @@ "description": "Encryption key for the new/cloned image", "oneOf": [ { "type": "string", "enum": [ "", "random" ] }, - { "type": "string", "pattern": "^[0-9a-fA-F]{128}$" } + { "type": "string", "pattern": "^[0-9a-fA-F]{128}$|^vault:" } ] } } @@ -143,7 +143,7 @@ "description": "Change encryption key for the image", "oneOf": [ { "type": "string", "enum": [ "" ] }, - { "type": "string", "pattern": "^[0-9a-fA-F]{128}$" } + { "type": "string", "pattern": "^[0-9a-fA-F]{128}$|^vault:" } ] }, "force": { "type": "boolean", "description": "Proceed with shrinking or setting readwrite flag even if the image has children" }, diff --git a/src/test/test_cluster_client.cpp b/src/test/test_cluster_client.cpp index 5fe57853..f83e91a5 100644 --- a/src/test/test_cluster_client.cpp +++ b/src/test/test_cluster_client.cpp @@ -18,6 +18,8 @@ public: { cli->continue_ops(cli->client_retry_interval); } + + static void test_vault(); }; void configure_single_pg_pool(cluster_client_t *cli) @@ -64,7 +66,7 @@ int *test_write(cluster_client_t *cli, uint64_t offset, uint64_t len, uint8_t c, { printf("Post write %jx+%jx\n", offset, len); int *r = new int; - *r = instant ? -2 : -1; + *r = instant ? -1001 : -1000; cluster_op_t *op = new cluster_op_t(); op->opcode = OSD_OP_WRITE; op->inode = 0x1000000000001; @@ -74,10 +76,11 @@ int *test_write(cluster_client_t *cli, uint64_t offset, uint64_t len, uint8_t c, memset(op->iov.buf[0].iov_base, c, len); op->callback = [r, cb](cluster_op_t *op) { - if (*r == -1) + if (*r == -1000) printf("Error: Not allowed to complete yet\n"); - assert(*r != -1); - *r = op->retval == op->len ? 1 : 0; + assert(*r != -1000); + assert(op->retval == op->len || op->retval < 0); + *r = op->retval == op->len ? 1 : op->retval; free(op->iov.buf[0].iov_base); printf("Done write %jx+%jx r=%d\n", op->offset, op->len, op->retval); delete op; @@ -99,14 +102,14 @@ int *test_sync(cluster_client_t *cli) { printf("Post sync\n"); int *r = new int; - *r = -1; + *r = -1000; cluster_op_t *op = new cluster_op_t(); op->opcode = OSD_OP_SYNC; op->callback = [r](cluster_op_t *op) { - if (*r == -1) + if (*r == -1000) printf("Error: Not allowed to complete yet\n"); - assert(*r != -1); + assert(*r != -1000); *r = op->retval == 0 ? 1 : 0; printf("Done sync r=%d\n", op->retval); delete op; @@ -119,7 +122,7 @@ void can_complete(int *r) { // Allow the operation to proceed so the test verifies // that it doesn't complete earlier than expected - *r = -2; + *r = -1001; } void check_completed(int *r) @@ -735,6 +738,67 @@ void test_msgr_decrypt_chain() } #endif +void cluster_client_test_t::test_vault() +{ + json11::Json::object config; + config["vault_url"] = "http://vault"; + timerfd_manager_t *tfd = new timerfd_manager_t([](int fd, bool wr, std::function callback){}); + cluster_client_t *cli = new cluster_client_t(NULL, tfd, config); + + configure_single_pg_pool(cli); + pretend_connected(cli, 1); + cli->st_cli.parse_state((etcd_kv_t){ + .key = "/config/inode/1/1", + .value = json11::Json::object { + { "name", "testimg" }, + { "size", (uint64_t)10*1024*1024*1024 }, + { "enc_key", "vault:key1" }, + }, + }); + + // No key -> fetch successfully -> complete op + + int *r1 = test_write(cli, 0, 4096, 0x55); + check_op_count(cli, 1, 0); + + assert(cli->vault_key_load_queue == std::vector{"vault:key1"}); + cli->vault_key_load_queue.clear(); + + cli->vault_parse_secret("vault:key1", "", json11::Json::object{ + {"data", json11::Json::object { + {"key", "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"} + }} + }); + + can_complete(r1); + check_op_count(cli, 1, 1); + pretend_op_completed(cli, find_op(cli, 1, OSD_OP_WRITE, 0, 4096), 0); + check_completed(r1); + + // No key -> error -> EPERM + + cli->vault_keys.clear(); + cli->inode_cache.clear(); + cli->inode_cache_children.clear(); + + r1 = test_write(cli, 0, 4096, 0x55); + check_op_count(cli, 1, 0); + + assert(cli->vault_key_load_queue == std::vector{"vault:key1"}); + cli->vault_key_load_queue.clear(); + + can_complete(r1); + cli->vault_parse_secret("vault:key1", "HTTP 403 Forbidden", json11::Json()); + check_op_count(cli, 1, 0); + assert(*r1 == -EPERM); + delete r1; + + // Free client + delete cli; + delete tfd; + printf("[ok] basic vault key fetch test\n"); +} + int main(int narg, char *args[]) { test1(); @@ -745,5 +809,6 @@ int main(int narg, char *args[]) test_msgr_encrypt(); test_msgr_decrypt_chain(); #endif + cluster_client_test_t::test_vault(); return 0; }