thebestvibedev/tromcho.net
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support storing image encryption keys in Vault

Browse Source
This commit is contained in:
Vitaliy Filippov
2026-04-17 13:53:40 +03:00
parent 4f9de7a6fb
commit 273b641820
15 changed files with 594 additions and 213 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/usage/cli.en.md
+2 -1
View File
@@ -135,6 +135,7 @@ Create an image. Options:
If parent is not a snapshot, it must be a read-only image.
* `--enc-key random` - Generate a new random AES-256-XTS encryption key for the new image.
* `--enc-key HEX` - Set a specified AES-256-XTS key (64 bytes in hex) for the new image.
* `--enc-key vault:ID` - Use an encryption key from an external Vault secret with specified ID.
```
vitastor-cli create --snapshot <snapshot> [OPTIONS] <image>
@@ -147,7 +148,7 @@ Options:
* `-p|--pool POOL` - Move image to pool POOL, leaving the snapshot in the old pool.
* `--enc-key random` - Change image encryption key to a new random AES-256-XTS key.
* `--enc-key HEX` - Change image encryption key to a specified key or to an empty key.
* `--enc-key KEY` - Change image encryption key to a specified key, Vault key or to an empty key.
By default, the image retains its old encryption key when taking a snapshot.
See also about [how to export snapshots](qemu.en.md#exporting-snapshots).
docs/usage/cli.ru.md
+2 -1
View File
@@ -137,6 +137,7 @@ bench-kaveri kaveri 10 G 10 G 0 B/s 0 0 0 us 0 B/s 0
Если `PARENT` - не снимок, он должен быть помечен как образ только для чтения.
* `--enc-key random` - Сгенерировать случайный ключ шифрования AES-256-XTS для нового образа.
* `--enc-key HEX` - Установить заданный ключ AES-256-XTS (64 байта в hex) для нового образа.
* `--enc-key vault:ID` - Использовать ключ из внешнего секрета с заданным ID из Vault.
```
vitastor-cli create --snapshot <snapshot> [ОПЦИИ] <image>
@@ -150,7 +151,7 @@ vitastor-cli snap-create [ОПЦИИ] <image>@<snapshot>
* `-p|--pool POOL` - Переместить образ в пул POOL, оставив снимок в старом пуле.
* `--enc-key random` - Изменить ключ шифрования образа на новый случайный ключ AES-256-XTS.
* `--enc-key HEX` - Изменить ключ шифрования образа на заданный или пустой ключ.
* `--enc-key KEY` - Изменить ключ шифрования образа на заданный ключ, ключ из Vault или пустой ключ.
По умолчанию шифрованные образы сохраняют старый ключ при снятии снимка.
Смотрите также информацию о том, [как экспортировать снимки](qemu.ru.md#экспорт-снимков).
src/client/CMakeLists.txt
+2 -1
View File
@@ -24,6 +24,7 @@ add_library(vitastor_client SHARED
cluster_client.cpp
cluster_client_list.cpp
cluster_client_wb.cpp
cluster_client_icache.cpp
vitastor_c.cpp
)
set_target_properties(vitastor_client PROPERTIES PUBLIC_HEADER "client/vitastor_c.h")
@@ -99,7 +100,7 @@ endif (${WITH_QEMU})
add_executable(test_cluster_client
EXCLUDE_FROM_ALL
../test/test_cluster_client.cpp
pg_states.cpp osd_ops.cpp cluster_client.cpp cluster_client_list.cpp cluster_client_wb.cpp msgr_op.cpp ../test/mock/messenger.cpp msgr_stop.cpp msgr_encrypt.cpp
pg_states.cpp osd_ops.cpp cluster_client.cpp cluster_client_list.cpp cluster_client_wb.cpp cluster_client_icache.cpp msgr_op.cpp ../test/mock/messenger.cpp msgr_stop.cpp msgr_encrypt.cpp
etcd_state_client.cpp ../util/timerfd_manager.cpp ../util/addr_util.cpp ../util/str_util.cpp ../util/json_util.cpp ../../json11/json11.cpp
)
target_link_libraries(test_cluster_client ${OPENSSL_LIBRARIES})
src/client/cluster_client.cpp
+24 -141
View File
@@ -11,16 +11,6 @@
#define TRY_SEND_CONNECTING 1
#define TRY_SEND_OK 2
inode_cache_t::~inode_cache_t()
{
if (key_data)
{
free(key_data);
key_data = NULL;
op_enc = NULL;
}
}
cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json config)
{
wb = new writeback_cache_t();
@@ -88,6 +78,7 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
cluster_client_t::~cluster_client_t()
{
vault_destroy();
if (retry_timeout_id >= 0)
{
tfd->clear_timer(retry_timeout_id);
@@ -492,6 +483,8 @@ void cluster_client_t::on_load_config_hook(json11::Json::object & etcd_global_co
self_tree_metrics.clear();
client_hostname = new_hostname;
}
// vault
vault_parse_config();
msgr.parse_config(config);
st_cli.parse_config(config);
st_cli.load_pgs();
@@ -620,6 +613,7 @@ void cluster_client_t::on_change_pool_config_hook()
}
inode_cache.clear();
inode_cache_children.clear();
vault_keys.clear();
continue_ops();
}
@@ -665,136 +659,6 @@ void cluster_client_t::on_change_node_placement_hook()
self_tree_metrics.clear();
}
// FIXME: Rework client API by adding open/close and cache inode information in the "FD" (maybe)
void cluster_client_t::on_change_inode_hook(uint64_t inode, bool removed)
{
std::vector<inode_t> children = { inode };
for (size_t i = 0; i < children.size(); i++)
{
auto it = inode_cache_children.lower_bound(std::make_pair(children[i], (inode_t)0));
while (it != inode_cache_children.end() && it->first == children[i])
{
children.push_back(it->second);
it++;
}
}
for (auto & inode: children)
{
auto it = inode_cache.find(inode);
if (it != inode_cache.end())
{
auto icache = it->second;
for (auto & parent: icache->chain)
{
inode_cache_children.erase(std::make_pair(parent, inode));
}
inode_cache.erase(it);
}
}
}
std::shared_ptr<inode_cache_t> cluster_client_t::inode_cache_get(inode_t ino)
{
auto icache_it = inode_cache.find(ino);
if (icache_it != inode_cache.end())
{
return icache_it->second;
}
// Fill inode cache
auto ino_it = st_cli.inode_config.find(ino);
if (ino_it == st_cli.inode_config.end())
{
inode_cache[ino] = NULL;
return NULL;
}
auto pool_it = st_cli.pool_config.find(INODE_POOL(ino));
if (pool_it == st_cli.pool_config.end())
{
inode_cache[ino] = NULL;
return NULL;
}
auto & inode_cfg = ino_it->second;
auto & pool_cfg = pool_it->second;
std::shared_ptr<inode_cache_t> icache = std::make_shared<inode_cache_t>();
icache->readonly = inode_cfg.readonly;
icache->chain.push_back(ino);
std::vector<inode_config_t*> chain_cfg;
int enc_key_count = !inode_cfg.enc_key.empty() ? 1 : 0;
if (inode_cfg.parent_id)
{
// Check for loops and cache the chain
robin_hood::unordered_flat_set<inode_t> seen;
seen.insert(ino);
uint64_t parent_id = inode_cfg.parent_id;
while (parent_id)
{
if (seen.find(parent_id) != seen.end())
{
icache->has_parent_loop = true;
break;
}
seen.insert(parent_id);
ino_it = st_cli.inode_config.find(parent_id);
if (INODE_POOL(parent_id) == INODE_POOL(ino))
{
icache->chain.push_back(parent_id);
if (ino_it == st_cli.inode_config.end())
chain_cfg.push_back(NULL);
else
{
chain_cfg.push_back(&ino_it->second);
if (!ino_it->second.enc_key.empty())
enc_key_count++;
}
}
else if (!icache->other_pool_parent_id)
icache->other_pool_parent_id = parent_id;
if (ino_it == st_cli.inode_config.end())
break;
parent_id = ino_it->second.parent_id;
}
}
// Generate encryption key chain, if applicable
if (enc_key_count)
{
uint8_t *key_data = (uint8_t*)malloc_or_die(
AES_256_XTS_KEY_SIZE * enc_key_count +
sizeof(uint8_t*) * icache->chain.size() +
sizeof(osd_op_enc_t)
);
uint8_t **keys = (uint8_t**)(key_data + AES_256_XTS_KEY_SIZE * enc_key_count);
osd_op_enc_t *enc = (osd_op_enc_t*)((uint8_t*)keys + sizeof(uint8_t*)*icache->chain.size());
size_t key_pos = 0;
for (size_t i = 0; i <= chain_cfg.size(); i++)
{
inode_config_t *cfg = !i ? &inode_cfg : chain_cfg[i-1];
if (cfg && !cfg->enc_key.empty())
{
assert(key_pos < AES_256_XTS_KEY_SIZE * enc_key_count);
assert(cfg->enc_key.size() == AES_256_XTS_KEY_SIZE);
keys[i] = key_data + key_pos;
memcpy(key_data + key_pos, cfg->enc_key.data(), AES_256_XTS_KEY_SIZE);
key_pos += AES_256_XTS_KEY_SIZE;
}
else
keys[i] = NULL;
}
enc->key_chain = keys;
enc->chain_size = icache->chain.size();
enc->read_chain_bitmap_pos = pool_cfg.data_block_size/pool_cfg.bitmap_granularity/8;
enc->bitmap_granularity = pool_cfg.bitmap_granularity;
icache->key_data = key_data;
icache->op_enc = enc;
}
inode_cache[ino] = icache;
for (auto & parent: icache->chain)
{
if (parent != ino)
inode_cache_children.insert(std::make_pair(parent, ino));
}
return icache;
}
bool cluster_client_t::is_ready()
{
return pgs_loaded;
@@ -816,6 +680,10 @@ bool cluster_client_t::flush()
{
if (!ringloop)
{
if (vault_loading)
{
return false;
}
if (wb->writeback_queue.size())
{
wb->start_writebacks(this, 0);
@@ -838,7 +706,7 @@ bool cluster_client_t::flush()
sync_done = true;
};
execute(sync);
while (!sync_done)
while (!sync_done || vault_loading)
{
ringloop->loop();
if (!sync_done)
@@ -1163,6 +1031,21 @@ bool cluster_client_t::check_rw(cluster_op_t *op)
}
}
}
if (icache && icache->err_code)
{
if (icache->err_code == EPERM)
{
op->retval = -EPERM;
auto cb = std::move(op->callback);
cb(op);
return false;
}
else if (icache->err_code == EAGAIN)
{
key_wait_ops.push_back(op);
return false;
}
}
return true;
}
src/client/cluster_client.h
+38 -4
View File
@@ -90,10 +90,17 @@ struct inode_cache_t
bool readonly = false;
bool has_parent_loop = false;
inode_t other_pool_parent_id = 0;
int err_code = 0;
~inode_cache_t();
};
struct vault_load_key_t
{
int key_state = 0;
std::string key;
};
// FIXME: Split into public and private interfaces
class __attribute__((visibility("default"))) cluster_client_t
{
@@ -103,8 +110,8 @@ public:
timerfd_manager_t *tfd = NULL;
ring_loop_t *ringloop = NULL;
std::map<pool_id_t, uint64_t> pg_counts;
std::map<pool_pg_num_t, osd_num_t> pg_primary;
// config:
// client_max_dirty_* is actually "max unsynced", for the case when immediate_commit is off
uint64_t client_max_dirty_bytes = 0;
uint64_t client_max_dirty_ops = 0;
@@ -116,12 +123,23 @@ public:
uint64_t client_max_writeback_iodepth = 0;
std::string conf_hostname;
std::string vault_url;
std::string vault_client_cert;
std::string vault_client_key;
std::string vault_ca;
std::string vault_secret_api_path;
uint64_t vault_timeout_ms = 0;
uint64_t vault_error_timeout_sec = 0;
uint64_t vault_refresh_leeway_sec = 0;
int log_level = 0;
int client_retry_interval = 50; // ms
int client_eio_retry_interval = 1000; // ms
bool client_retry_enospc = true;
int client_wait_up_timeout = 16; // sec (for listings)
// state:
std::string client_hostname;
std::map<std::string, int> self_tree_metrics;
std::map<osd_num_t, int> osd_tree_metrics;
@@ -129,6 +147,7 @@ public:
int retry_timeout_id = -1;
int retry_timeout_duration = 0;
std::vector<cluster_op_t*> offline_ops;
std::vector<cluster_op_t*> key_wait_ops;
cluster_op_t *op_queue_head = NULL, *op_queue_tail = NULL;
writeback_cache_t *wb = NULL;
std::set<osd_num_t> dirty_osds;
@@ -142,7 +161,17 @@ public:
robin_hood::unordered_flat_map<inode_t, std::shared_ptr<inode_cache_t>> inode_cache;
std::set<std::pair<inode_t, inode_t>> inode_cache_children;
http_context_t *vault_http_ctx = NULL;
http_co_t *vault_http_cli = NULL;
bool vault_loading = false;
std::string vault_token;
bool vault_auth_error = false;
timespec vault_token_expire = {};
std::vector<std::string> vault_key_load_queue;
std::map<std::string, vault_load_key_t> vault_keys;
bool pgs_loaded = false;
std::map<pool_id_t, uint64_t> pg_counts;
ring_consumer_t consumer;
std::vector<std::function<void(void)>> on_ready_hooks;
int list_retry_timeout_id = -1;
@@ -182,6 +211,13 @@ protected:
#endif
void continue_ops(int time_passed = 0);
std::shared_ptr<inode_cache_t> inode_cache_get(inode_t ino);
void vault_parse_config();
bool vault_check_token();
void vault_load_keys();
void vault_destroy();
void vault_parse_secret(const std::string & key_id, const std::string & err, json11::Json data);
protected:
bool affects_osd(uint64_t inode, uint64_t offset, uint64_t len, osd_num_t osd);
bool affects_pg(uint64_t inode, uint64_t offset, uint64_t len, pool_id_t pool_id, pg_num_t pg_num);
@@ -194,8 +230,6 @@ protected:
void on_change_node_placement_hook();
void on_change_inode_hook(uint64_t inode, bool removed);
std::shared_ptr<inode_cache_t> inode_cache_get(inode_t ino);
void execute_internal(cluster_op_t *op);
void execute_cas(cluster_op_t *op);
void unshift_op(cluster_op_t *op);
src/client/cluster_client_icache.cpp
+366
View File
@@ -0,0 +1,366 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
#include <stdexcept>
#include <assert.h>
#include "cluster_client_impl.h"
#include "http_client.h"
#include "str_util.h"
#define VAULT_KEY_NOT_LOADED 0
#define VAULT_KEY_LOADING 1
#define VAULT_KEY_LOADED 2
#define VAULT_KEY_ERROR 3
inode_cache_t::~inode_cache_t()
{
if (key_data)
{
free(key_data);
key_data = NULL;
op_enc = NULL;
}
}
void cluster_client_t::vault_destroy()
{
if (vault_http_ctx)
{
#ifndef __MOCK__
http_destroy(vault_http_cli);
http_context_destroy(vault_http_ctx);
vault_http_cli = NULL;
vault_http_ctx = NULL;
#endif
}
}
void cluster_client_t::vault_parse_config()
{
vault_url = config["vault_url"].string_value();
vault_client_cert = config["vault_client_cert"].string_value();
vault_client_key = config["vault_client_key"].string_value();
vault_ca = config["vault_ca"].string_value();
vault_secret_api_path = "/v1/secret/";
if (config["vault_secret_api_path"].is_string())
vault_secret_api_path = config["vault_secret_api_path"].string_value();
vault_timeout_ms = config["vault_timeout_ms"].uint64_value();
if (!vault_timeout_ms)
vault_timeout_ms = 5000;
vault_error_timeout_sec = config["vault_error_timeout_sec"].uint64_value();
if (!vault_error_timeout_sec)
vault_error_timeout_sec = 60;
vault_refresh_leeway_sec = config["vault_refresh_leeway_sec"].uint64_value();
if (!vault_refresh_leeway_sec)
vault_refresh_leeway_sec = 60;
}
// FIXME: Rework client API by adding open/close and cache inode information in the "FD" (maybe)
void cluster_client_t::on_change_inode_hook(uint64_t inode, bool removed)
{
std::vector<inode_t> children = { inode };
for (size_t i = 0; i < children.size(); i++)
{
auto it = inode_cache_children.lower_bound(std::make_pair(children[i], (inode_t)0));
while (it != inode_cache_children.end() && it->first == children[i])
{
children.push_back(it->second);
it++;
}
}
for (auto & inode: children)
{
auto it = inode_cache.find(inode);
if (it != inode_cache.end())
{
auto icache = it->second;
for (auto & parent: icache->chain)
{
inode_cache_children.erase(std::make_pair(parent, inode));
}
inode_cache.erase(it);
}
}
}
std::shared_ptr<inode_cache_t> cluster_client_t::inode_cache_get(inode_t ino)
{
auto icache_it = inode_cache.find(ino);
if (icache_it != inode_cache.end())
{
return icache_it->second;
}
// Fill inode cache
auto ino_it = st_cli.inode_config.find(ino);
if (ino_it == st_cli.inode_config.end())
{
inode_cache[ino] = NULL;
return NULL;
}
auto pool_it = st_cli.pool_config.find(INODE_POOL(ino));
if (pool_it == st_cli.pool_config.end())
{
inode_cache[ino] = NULL;
return NULL;
}
auto & inode_cfg = ino_it->second;
auto & pool_cfg = pool_it->second;
std::shared_ptr<inode_cache_t> icache = std::make_shared<inode_cache_t>();
icache->readonly = inode_cfg.readonly;
icache->chain.push_back(ino);
std::vector<inode_config_t*> chain_cfg;
int enc_key_count = !inode_cfg.enc_key.empty() ? 1 : 0;
if (inode_cfg.parent_id)
{
// Check for loops and cache the chain
robin_hood::unordered_flat_set<inode_t> seen;
seen.insert(ino);
uint64_t parent_id = inode_cfg.parent_id;
while (parent_id)
{
if (seen.find(parent_id) != seen.end())
{
icache->has_parent_loop = true;
break;
}
seen.insert(parent_id);
ino_it = st_cli.inode_config.find(parent_id);
if (INODE_POOL(parent_id) == INODE_POOL(ino))
{
icache->chain.push_back(parent_id);
if (ino_it == st_cli.inode_config.end())
chain_cfg.push_back(NULL);
else
{
chain_cfg.push_back(&ino_it->second);
if (!ino_it->second.enc_key.empty())
enc_key_count++;
}
}
else if (!icache->other_pool_parent_id)
icache->other_pool_parent_id = parent_id;
if (ino_it == st_cli.inode_config.end())
break;
parent_id = ino_it->second.parent_id;
}
}
// Check external keys and wait for loading, if required
if (enc_key_count)
{
for (size_t i = 0; i <= chain_cfg.size(); i++)
{
inode_config_t *cfg = !i ? &inode_cfg : chain_cfg[i-1];
if (cfg && cfg->enc_key.substr(0, strlen(VAULT_KEY_PREFIX)) == VAULT_KEY_PREFIX)
{
auto & ik = vault_keys[inode_cfg.enc_key];
if (ik.key_state == VAULT_KEY_ERROR || vault_url.empty())
{
icache->err_code = EPERM;
enc_key_count = 0;
}
else if (ik.key_state == VAULT_KEY_NOT_LOADED)
{
ik.key_state = VAULT_KEY_LOADING;
vault_key_load_queue.push_back(inode_cfg.enc_key);
vault_load_keys();
icache->err_code = EAGAIN;
enc_key_count = 0;
}
else if (ik.key_state == VAULT_KEY_LOADING)
{
icache->err_code = EAGAIN;
enc_key_count = 0;
}
else
{
assert(ik.key_state == VAULT_KEY_LOADED);
}
}
}
}
// Generate encryption key chain, if applicable
if (enc_key_count)
{
uint8_t *key_data = (uint8_t*)malloc_or_die(
AES_256_XTS_KEY_SIZE * enc_key_count +
sizeof(uint8_t*) * icache->chain.size() +
sizeof(osd_op_enc_t)
);
uint8_t **keys = (uint8_t**)(key_data + AES_256_XTS_KEY_SIZE * enc_key_count);
osd_op_enc_t *enc = (osd_op_enc_t*)((uint8_t*)keys + sizeof(uint8_t*)*icache->chain.size());
size_t key_pos = 0;
for (size_t i = 0; i <= chain_cfg.size(); i++)
{
inode_config_t *cfg = !i ? &inode_cfg : chain_cfg[i-1];
if (cfg && !cfg->enc_key.empty())
{
const auto & key = cfg->enc_key.substr(0, strlen(VAULT_KEY_PREFIX)) == VAULT_KEY_PREFIX
? vault_keys.at(cfg->enc_key).key
: cfg->enc_key;
assert(key_pos < AES_256_XTS_KEY_SIZE * enc_key_count);
assert(key.size() == 2*AES_256_XTS_KEY_SIZE);
keys[i] = key_data + key_pos;
fromhexstr(key, AES_256_XTS_KEY_SIZE, key_data + key_pos);
key_pos += AES_256_XTS_KEY_SIZE;
}
else
keys[i] = NULL;
}
enc->key_chain = keys;
enc->chain_size = icache->chain.size();
enc->read_chain_bitmap_pos = pool_cfg.data_block_size/pool_cfg.bitmap_granularity/8;
enc->bitmap_granularity = pool_cfg.bitmap_granularity;
icache->key_data = key_data;
icache->op_enc = enc;
}
inode_cache[ino] = icache;
for (auto & parent: icache->chain)
{
if (parent != ino)
inode_cache_children.insert(std::make_pair(parent, ino));
}
return icache;
}
#ifndef __MOCK__
bool cluster_client_t::vault_check_token()
{
timespec now;
clock_gettime(CLOCK_REALTIME, &now);
if (!vault_token_expire.tv_sec || vault_token_expire.tv_sec < now.tv_sec)
{
vault_loading = true;
http_json_post(
vault_http_cli, vault_url+"/v1/auth/cert/login", json11::Json::object{}, "",
(http_options_t){ .timeout = (int)vault_timeout_ms, .keepalive = true },
[this](http_message_t *response)
{
clock_gettime(CLOCK_REALTIME, &vault_token_expire);
vault_loading = false;
std::string err;
json11::Json data;
response->parse_json_response(err, data);
if (err != "")
{
vault_token_expire.tv_sec += vault_error_timeout_sec;
fprintf(stderr, "Vault request failed: %s\n", err.c_str());
}
else
{
uint64_t ttl = data["auth"]["lease_duration"].uint64_value();
vault_token = data["auth"]["client_token"].string_value();
if (vault_token.empty() || !ttl)
{
vault_token_expire.tv_sec += vault_error_timeout_sec;
fprintf(stderr, "No token or lease_duration in Vault response: %s\n", data.dump().c_str());
}
else
{
if (ttl < vault_refresh_leeway_sec)
vault_token_expire.tv_sec += ttl/2;
else
vault_token_expire.tv_sec += ttl - vault_refresh_leeway_sec;
}
}
vault_load_keys();
}
);
return false;
}
if (vault_token.empty())
{
// Auth error happened, mark all loads as failed
for (auto & key_id: vault_key_load_queue)
{
auto & k = vault_keys[key_id];
k.key_state = VAULT_KEY_ERROR;
}
vault_key_load_queue.clear();
auto ops = std::move(key_wait_ops);
for (cluster_op_t *op: ops)
inode_cache.erase(op->inode);
for (cluster_op_t *op: ops)
execute_internal(op);
return false;
}
return true;
}
#endif
void cluster_client_t::vault_load_keys()
{
if (vault_loading || !vault_key_load_queue.size())
{
return;
}
#ifdef __MOCK__
vault_loading = true;
#else
if (!vault_http_ctx)
{
std::string error;
vault_http_ctx = http_context_init(tfd, vault_client_cert, vault_client_key, vault_ca, true, error);
if (!vault_http_ctx)
{
fprintf(stderr, "Failed to initialize HTTP context for Vault: %s\n", error.c_str());
exit(1);
}
vault_http_cli = http_init(vault_http_ctx);
}
if (!vault_check_token())
{
return;
}
std::string key_id = vault_key_load_queue[0];
vault_key_load_queue.erase(vault_key_load_queue.begin());
vault_loading = true;
http_get(
vault_http_cli, vault_url+vault_secret_api_path+key_id.substr(strlen(VAULT_KEY_PREFIX)), "X-Vault-Token: "+vault_token+"\r\n",
(http_options_t){ .timeout = (int)vault_timeout_ms, .keepalive = true },
[this, key_id](http_message_t *response)
{
vault_loading = false;
std::string err;
json11::Json data;
response->parse_json_response(err, data);
vault_parse_secret(key_id, err, data);
}
);
#endif
}
void cluster_client_t::vault_parse_secret(const std::string & key_id, const std::string & err, json11::Json data)
{
vault_loading = false;
auto & k = vault_keys[key_id];
if (err != "")
{
k.key_state = VAULT_KEY_ERROR;
fprintf(stderr, "Vault %s%s%s request failed: %s\n", vault_url.c_str(),
vault_secret_api_path.c_str(), key_id.c_str()+strlen(VAULT_KEY_PREFIX), err.c_str());
}
else
{
auto hexkey = data["data"]["key"].string_value();
if (hexkey.empty() || !ishexstr(hexkey) || hexkey.size() != 2*AES_256_XTS_KEY_SIZE)
{
k.key_state = VAULT_KEY_ERROR;
fprintf(stderr, "Vault /v1/secret/%s request failed: 'key' is empty or has invalid format\n", key_id.c_str());
}
else
{
k.key_state = VAULT_KEY_LOADED;
k.key = hexkey;
}
}
if (vault_key_load_queue.empty())
{
auto ops = std::move(key_wait_ops);
for (cluster_op_t *op: ops)
inode_cache.erase(op->inode);
for (cluster_op_t *op: ops)
execute_internal(op);
}
else
vault_load_keys();
}
src/client/etcd_state_client.cpp
+9 -29
View File
@@ -98,32 +98,15 @@ http_context_t *etcd_state_client_t::get_http_ctx()
void etcd_state_client_t::etcd_call_oneshot(const std::string & etcd_url, const std::string & api, json11::Json payload,
int timeout, std::function<void(std::string, json11::Json)> callback)
{
std::string etcd_api_path;
bool ssl = etcd_url.substr(0, 8) == "https://";
auto etcd_address = etcd_url.substr(ssl ? 8 : 7);
int pos = etcd_address.find('/');
if (pos >= 0)
{
etcd_api_path = etcd_address.substr(pos);
etcd_address = etcd_address.substr(0, pos);
}
std::string req = payload.dump();
req = "POST "+etcd_api_path+api+" HTTP/1.1\r\n"
"Host: "+etcd_address+"\r\n"
"Content-Type: application/json\r\n"
"Content-Length: "+std::to_string(req.size())+"\r\n"
"Connection: close\r\n"
"\r\n"+req;
auto http_cli = http_init(get_http_ctx());
auto cb = [http_cli, callback](http_message_t *response)
http_json_post(http_cli, etcd_url+api, payload, "", { .timeout = timeout }, [http_cli, callback](http_message_t *response)
{
std::string err;
json11::Json data;
response->parse_json_response(err, data);
callback(err, data);
http_destroy(http_cli);
};
http_request(http_cli, etcd_address, req, { .timeout = timeout, .ssl = ssl }, cb);
});
}
void etcd_state_client_t::etcd_call(const std::string & api, json11::Json payload, int timeout,
@@ -1448,7 +1431,7 @@ json11::Json::object etcd_state_client_t::serialize_inode_cfg(inode_config_t *cf
}
if (!cfg->enc_key.empty())
{
new_cfg["enc_key"] = tohexstr(cfg->enc_key.data(), cfg->enc_key.size());
new_cfg["enc_key"] = cfg->enc_key;
}
if (cfg->readonly)
{
@@ -1484,18 +1467,15 @@ inode_config_t etcd_state_client_t::deserialize_inode_cfg(uint64_t inode_num, js
else
parent_inode_num |= parent_pool_id << (64-POOL_ID_BITS);
}
std::vector<uint8_t> enc_key;
std::string enc_key;
if (!value["enc_key"].is_null())
{
if (value["enc_key"].string_value().size() == 2*AES_256_XTS_KEY_SIZE)
enc_key = value["enc_key"].string_value();
if (enc_key.substr(0, strlen(VAULT_KEY_PREFIX)) != VAULT_KEY_PREFIX &&
(enc_key.size() != 2*AES_256_XTS_KEY_SIZE || !ishexstr(enc_key)))
{
enc_key.resize(AES_256_XTS_KEY_SIZE);
if (fromhexstr(value["enc_key"].string_value(), AES_256_XTS_KEY_SIZE, enc_key.data()) < AES_256_XTS_KEY_SIZE)
enc_key.clear();
}
if (enc_key.empty())
{
fprintf(stderr, "Inode %u/%ju has invalid enc_key, should be %u bit hex string\n",
enc_key = "";
fprintf(stderr, "Inode %u/%ju has invalid enc_key, should be %u bit hex string or Vault key reference\n",
INODE_POOL(inode_num), INODE_NO_POOL(inode_num), AES_256_XTS_KEY_SIZE);
}
}
src/client/etcd_state_client.h
+3 -1
View File
@@ -20,6 +20,8 @@
#define MAX_DATA_BLOCK_SIZE 128*1024*1024
#define DEFAULT_BITMAP_GRANULARITY 4096
#define VAULT_KEY_PREFIX "vault:"
#ifndef IMMEDIATE_NONE
#define IMMEDIATE_NONE 0
#define IMMEDIATE_SMALL 1
@@ -84,7 +86,7 @@ struct inode_config_t
inode_t parent_id = 0;
bool readonly = false;
bool deleted = false;
std::vector<uint8_t> enc_key;
std::string enc_key;
// Arbitrary metadata
json11::Json meta;
// Change revision of the metadata in etcd
src/client/http_client.cpp
+42
View File
@@ -287,6 +287,48 @@ void http_request(http_co_t *handler, const std::string & host, const std::strin
handler->send_request(host, request, options, response_callback);
}
void http_get(http_co_t *handler, const std::string & url, const std::string & headers,
const http_options_t & options, std::function<void(http_message_t *response)> response_callback)
{
std::string path;
auto ssl = url.substr(0, 8) == "https://";
auto host = url.substr(ssl ? 8 : 7);
auto pos = host.find('/');
if (pos != std::string::npos)
{
path = host.substr(pos);
host = host.substr(0, pos);
}
std::string req = "GET "+path+" HTTP/1.1\r\n"
"Host: "+host+"\r\n"
"Connection: "+(options.keepalive ? "keep-alive" : "close")+"\r\n"+
headers+"\r\n";
handler->send_request(host, req, { .timeout = options.timeout, .keepalive = options.keepalive, .ssl = ssl }, response_callback);
}
void http_json_post(http_co_t *handler, const std::string & url, json11::Json body, const std::string & headers,
const http_options_t & options, std::function<void(http_message_t *response)> response_callback)
{
std::string path;
auto ssl = url.substr(0, 8) == "https://";
auto host = url.substr(ssl ? 8 : 7);
auto pos = host.find('/');
if (pos != std::string::npos)
{
path = host.substr(pos);
host = host.substr(0, pos);
}
std::string req = body.dump();
req = "POST "+path+" HTTP/1.1\r\n"
"Host: "+host+"\r\n"
"Content-Type: application/json\r\n"
"Content-Length: "+std::to_string(req.size())+"\r\n"
"Connection: "+(options.keepalive ? "keep-alive" : "close")+"\r\n"+
headers+
"\r\n"+req;
handler->send_request(host, req, { .timeout = options.timeout, .keepalive = options.keepalive, .ssl = ssl }, response_callback);
}
void http_serve(http_co_t *handler, int peer_fd, const http_options_t & options, std::function<void(http_message_t *msg)> request_callback)
{
if (handler->state != HTTP_CO_SERVER || handler->peer_fd != peer_fd)
src/client/http_client.h
+4
View File
@@ -56,6 +56,10 @@ void open_websocket(http_co_t *handler, const std::string & addr, const std::str
const http_options_t & options, std::function<void(http_message_t *msg)> on_message);
void http_request(http_co_t *handler, const std::string & host, const std::string & request,
const http_options_t & options, std::function<void(http_message_t *response)> response_callback);
void http_get(http_co_t *handler, const std::string & url, const std::string & headers,
const http_options_t & options, std::function<void(http_message_t *response)> response_callback);
void http_json_post(http_co_t *handler, const std::string & url, json11::Json body, const std::string & headers,
const http_options_t & options, std::function<void(http_message_t *response)> response_callback);
void http_post_message(http_co_t *handler, uint8_t type, const std::string & msg);
void http_serve(http_co_t *handler, int peer_fd, const http_options_t & options,
std::function<void(http_message_t *msg)> request_callback);
src/cmd/cli.cpp
+8 -7
View File
@@ -42,12 +42,13 @@ static const char* help_text =
"\n"
"vitastor-cli create -s|--size SIZE [OPTIONS] <name>\n"
" Create an image. Options:\n"
" -s|--size SIZE New image size in bytes or with a K/M/G/T unit suffix.\n"
" -p|--pool POOL Specify pool for the new image (may be omitted if there is only 1 pool).\n"
" --parent PARENT Create a copy-on-write image clone based on PARENT (or PARENT@SNAPSHOT).\n"
" If parent is not a snapshot, it must be a read-only image.\n"
" --enc-key random Generate a new random AES-256-XTS encryption key for the new image.\n"
" --enc-key HEX Set a specified AES-256-XTS key (64 bytes in hex) for the new image.\n"
" -s|--size SIZE New image size in bytes or with a K/M/G/T unit suffix.\n"
" -p|--pool POOL Specify pool for the new image (may be omitted if there is only 1 pool).\n"
" --parent PARENT Create a copy-on-write image clone based on PARENT (or PARENT@SNAPSHOT).\n"
" If parent is not a snapshot, it must be a read-only image.\n"
" --enc-key random Generate a new random AES-256-XTS encryption key for the new image.\n"
" --enc-key HEX Set a specified AES-256-XTS key (64 bytes in hex) for the new image.\n"
" --enc-key vault:ID Use an encryption key from an external Vault secret with specified ID.\n"
"\n"
"vitastor-cli create --snapshot <snapshot> [OPTIONS] <image>\n"
"vitastor-cli snap-create [OPTIONS] <image>@<snapshot>\n"
@@ -55,7 +56,7 @@ static const char* help_text =
" Options:\n"
" -p|--pool POOL Move image to pool POOL, leaving the snapshot in the old pool.\n"
" --enc-key random Change image encryption key to a new random AES-256-XTS key.\n"
" --enc-key HEX Change image encryption key to a specified key or to an empty key.\n"
" --enc-key KEY Change image encryption key to a specified key, Vault key or to an empty key.\n"
" By default, the image retains its old key when taking a snapshot.\n"
"\n"
"vitastor-cli modify <name> [--rename <new-name>] [--resize <size>] [--readonly | --readwrite] [-f|--force] [--down-ok]\n"
src/cmd/cli_create.cpp
+17 -15
View File
@@ -456,11 +456,12 @@ resume_3:
};
if (set_key)
{
new_cfg.enc_key.resize(enc_key.size()/2);
fromhexstr(enc_key, new_cfg.enc_key.size(), new_cfg.enc_key.data());
new_cfg.enc_key = enc_key;
}
else if (new_snap != "")
{
new_cfg.enc_key = cur_cfg.enc_key;
}
json11::Json::array checks = json11::Json::array {
json11::Json::object {
{ "target", "VERSION" },
@@ -600,19 +601,6 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_create(json11::Json cfg)
if (!cfg["enc_key"].is_null())
{
image_creator->set_key = true;
image_creator->enc_key = cfg["enc_key"].string_value();
if (image_creator->enc_key != "" &&
#ifdef WITH_OPENSSL
image_creator->enc_key != "random" &&
#endif
(!ishexstr(image_creator->enc_key) || image_creator->enc_key.size() != 128))
{
return [](cli_result_t & result)
{
result = (cli_result_t){ .err = EINVAL, .text = "Encryption key is not a 512-bit hex string, not \"\" and not \"random\"" };
return true;
};
}
#ifdef WITH_OPENSSL
if (image_creator->enc_key == "random")
{
@@ -621,6 +609,20 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_create(json11::Json cfg)
image_creator->enc_key = tohexstr(newkey, 64);
}
#endif
else
{
image_creator->enc_key = cfg["enc_key"].string_value();
if (image_creator->enc_key != "" &&
image_creator->enc_key.substr(0, strlen(VAULT_KEY_PREFIX)) != VAULT_KEY_PREFIX &&
(!ishexstr(image_creator->enc_key) || image_creator->enc_key.size() != 128))
{
return [](cli_result_t & result)
{
result = (cli_result_t){ .err = EINVAL, .text = "Encryption key is not a 512-bit hex string, not \"\" and not \"random\"" };
return true;
};
}
}
}
image_creator->new_parent = cfg["parent"].string_value();
if (!cfg["size"].is_null())
src/cmd/cli_modify.cpp
+4 -3
View File
@@ -159,14 +159,15 @@ resume_1:
state = 100;
return;
}
if (enc_key != "" && (!ishexstr(enc_key) || enc_key.size() != 128))
if (enc_key != "" &&
enc_key.substr(0, strlen(VAULT_KEY_PREFIX)) != VAULT_KEY_PREFIX &&
(!ishexstr(enc_key) || enc_key.size() != 128))
{
result = (cli_result_t){ .err = EINVAL, .text = "Encryption key is not a 512-bit hex string and not \"\"" };
state = 100;
return;
}
cfg.enc_key.resize(enc_key.size()/2);
fromhexstr(enc_key, cfg.enc_key.size(), cfg.enc_key.data());
cfg.enc_key = enc_key;
}
{
std::string cur_cfg_key = base64_encode(parent->cli->st_cli.etcd_prefix+
src/cmd/openapi.json
+2 -2
View File
@@ -106,7 +106,7 @@
"description": "Encryption key for the new/cloned image",
"oneOf": [
{ "type": "string", "enum": [ "", "random" ] },
{ "type": "string", "pattern": "^[0-9a-fA-F]{128}$" }
{ "type": "string", "pattern": "^[0-9a-fA-F]{128}$|^vault:" }
]
}
}
@@ -143,7 +143,7 @@
"description": "Change encryption key for the image",
"oneOf": [
{ "type": "string", "enum": [ "" ] },
{ "type": "string", "pattern": "^[0-9a-fA-F]{128}$" }
{ "type": "string", "pattern": "^[0-9a-fA-F]{128}$|^vault:" }
]
},
"force": { "type": "boolean", "description": "Proceed with shrinking or setting readwrite flag even if the image has children" },
src/test/test_cluster_client.cpp
+71 -8
View File
@@ -55,7 +55,7 @@ int *test_write(cluster_client_t *cli, uint64_t offset, uint64_t len, uint8_t c,
{
printf("Post write %jx+%jx\n", offset, len);
int *r = new int;
*r = instant ? -2 : -1;
*r = instant ? -1001 : -1000;
cluster_op_t *op = new cluster_op_t();
op->opcode = OSD_OP_WRITE;
op->inode = 0x1000000000001;
@@ -65,10 +65,11 @@ int *test_write(cluster_client_t *cli, uint64_t offset, uint64_t len, uint8_t c,
memset(op->iov.buf[0].iov_base, c, len);
op->callback = [r, cb](cluster_op_t *op)
{
if (*r == -1)
if (*r == -1000)
printf("Error: Not allowed to complete yet\n");
assert(*r != -1);
*r = op->retval == op->len ? 1 : 0;
assert(*r != -1000);
assert(op->retval == op->len || op->retval < 0);
*r = op->retval == op->len ? 1 : op->retval;
free(op->iov.buf[0].iov_base);
printf("Done write %jx+%jx r=%d\n", op->offset, op->len, op->retval);
delete op;
@@ -90,14 +91,14 @@ int *test_sync(cluster_client_t *cli)
{
printf("Post sync\n");
int *r = new int;
*r = -1;
*r = -1000;
cluster_op_t *op = new cluster_op_t();
op->opcode = OSD_OP_SYNC;
op->callback = [r](cluster_op_t *op)
{
if (*r == -1)
if (*r == -1000)
printf("Error: Not allowed to complete yet\n");
assert(*r != -1);
assert(*r != -1000);
*r = op->retval == 0 ? 1 : 0;
printf("Done sync r=%d\n", op->retval);
delete op;
@@ -110,7 +111,7 @@ void can_complete(int *r)
{
// Allow the operation to proceed so the test verifies
// that it doesn't complete earlier than expected
*r = -2;
*r = -1001;
}
void check_completed(int *r)
@@ -726,6 +727,67 @@ void test_msgr_decrypt_chain()
}
#endif
void test_vault()
{
json11::Json::object config;
config["vault_url"] = "http://vault";
timerfd_manager_t *tfd = new timerfd_manager_t([](int fd, bool wr, std::function<void(int, int)> callback){});
cluster_client_t *cli = new cluster_client_t(NULL, tfd, config);
configure_single_pg_pool(cli);
pretend_connected(cli, 1);
cli->st_cli.parse_state((etcd_kv_t){
.key = "/config/inode/1/1",
.value = json11::Json::object {
{ "name", "testimg" },
{ "size", (uint64_t)10*1024*1024*1024 },
{ "enc_key", "vault:key1" },
},
});
// No key -> fetch successfully -> complete op
int *r1 = test_write(cli, 0, 4096, 0x55);
check_op_count(cli, 1, 0);
assert(cli->vault_key_load_queue == std::vector<std::string>{"vault:key1"});
cli->vault_key_load_queue.clear();
cli->vault_parse_secret("vault:key1", "", json11::Json::object{
{"data", json11::Json::object {
{"key", "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"}
}}
});
can_complete(r1);
check_op_count(cli, 1, 1);
pretend_op_completed(cli, find_op(cli, 1, OSD_OP_WRITE, 0, 4096), 0);
check_completed(r1);
// No key -> error -> EPERM
cli->vault_keys.clear();
cli->inode_cache.clear();
cli->inode_cache_children.clear();
r1 = test_write(cli, 0, 4096, 0x55);
check_op_count(cli, 1, 0);
assert(cli->vault_key_load_queue == std::vector<std::string>{"vault:key1"});
cli->vault_key_load_queue.clear();
can_complete(r1);
cli->vault_parse_secret("vault:key1", "HTTP 403 Forbidden", json11::Json());
check_op_count(cli, 1, 0);
assert(*r1 == -EPERM);
delete r1;
// Free client
delete cli;
delete tfd;
printf("[ok] basic vault key fetch test\n");
}
int main(int narg, char *args[])
{
test1();
@@ -736,5 +798,6 @@ int main(int narg, char *args[])
test_msgr_encrypt();
test_msgr_decrypt_chain();
#endif
test_vault();
return 0;
}