diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index d57a045a..49d01342 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -702,6 +702,24 @@ jobs: echo "" done + test_snapshot_chain_encrypted: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: ENCRYPTED=1 /root/vitastor/tests/test_snapshot_chain.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + test_old_snapshot_chain: runs-on: ubuntu-latest needs: build diff --git a/.gitea/workflows/tests-to-yaml.pl b/.gitea/workflows/tests-to-yaml.pl index 34a24f26..1320db2e 100755 --- a/.gitea/workflows/tests-to-yaml.pl +++ b/.gitea/workflows/tests-to-yaml.pl @@ -42,6 +42,10 @@ for my $line (<>) { $test_name .= '_https'; } + elsif ($1 eq 'ENCRYPTED') + { + $test_name .= '_encrypted'; + } elsif ($1 eq 'OLD') { $test_name =~ s/^test_/test_old_/s; diff --git a/src/client/cluster_client.cpp b/src/client/cluster_client.cpp index 1a907c07..628aa36d 100644 --- a/src/client/cluster_client.cpp +++ b/src/client/cluster_client.cpp @@ -11,6 +11,16 @@ #define TRY_SEND_CONNECTING 1 #define TRY_SEND_OK 2 +inode_cache_t::~inode_cache_t() +{ + if (key_data) + { + free(key_data); + key_data = NULL; + op_enc = NULL; + } +} + cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json config) { wb = new writeback_cache_t(); @@ -62,6 +72,7 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd st_cli.on_change_node_placement_hook = [this]() { on_change_node_placement_hook(); }; st_cli.on_load_pgs_hook = [this](bool success) { on_load_pgs_hook(success); }; st_cli.on_reload_hook = [this]() { st_cli.load_global_config(); }; + st_cli.on_inode_change_hook = [this](uint64_t inode, bool removed) { on_change_inode_hook(inode, removed); }; st_cli.parse_config(config); st_cli.infinite_start = false; @@ -607,6 +618,8 @@ void cluster_client_t::on_change_pool_config_hook() pg_counts[pool_item.first] = pool_item.second.real_pg_count; } } + inode_cache.clear(); + inode_cache_children.clear(); continue_ops(); } @@ -652,6 +665,136 @@ void cluster_client_t::on_change_node_placement_hook() self_tree_metrics.clear(); } +// FIXME: Rework client API by adding open/close and cache inode information in the "FD" (maybe) +void cluster_client_t::on_change_inode_hook(uint64_t inode, bool removed) +{ + std::vector children = { inode }; + for (size_t i = 0; i < children.size(); i++) + { + auto it = inode_cache_children.lower_bound(std::make_pair(children[i], (inode_t)0)); + while (it != inode_cache_children.end() && it->first == children[i]) + { + children.push_back(it->second); + it++; + } + } + for (auto & inode: children) + { + auto it = inode_cache.find(inode); + if (it != inode_cache.end()) + { + auto icache = it->second; + for (auto & parent: icache->chain) + { + inode_cache_children.erase(std::make_pair(parent, inode)); + } + inode_cache.erase(it); + } + } +} + +std::shared_ptr cluster_client_t::inode_cache_get(inode_t ino) +{ + auto icache_it = inode_cache.find(ino); + if (icache_it != inode_cache.end()) + { + return icache_it->second; + } + // Fill inode cache + auto ino_it = st_cli.inode_config.find(ino); + if (ino_it == st_cli.inode_config.end()) + { + inode_cache[ino] = NULL; + return NULL; + } + auto pool_it = st_cli.pool_config.find(INODE_POOL(ino)); + if (pool_it == st_cli.pool_config.end()) + { + inode_cache[ino] = NULL; + return NULL; + } + auto & inode_cfg = ino_it->second; + auto & pool_cfg = pool_it->second; + std::shared_ptr icache = std::make_shared(); + icache->readonly = inode_cfg.readonly; + icache->chain.push_back(ino); + std::vector chain_cfg; + int enc_key_count = !inode_cfg.enc_key.empty() ? 1 : 0; + if (inode_cfg.parent_id) + { + // Check for loops and cache the chain + robin_hood::unordered_flat_set seen; + seen.insert(ino); + uint64_t parent_id = inode_cfg.parent_id; + while (parent_id) + { + if (seen.find(parent_id) != seen.end()) + { + icache->has_parent_loop = true; + break; + } + seen.insert(parent_id); + ino_it = st_cli.inode_config.find(parent_id); + if (INODE_POOL(parent_id) == INODE_POOL(ino)) + { + icache->chain.push_back(parent_id); + if (ino_it == st_cli.inode_config.end()) + chain_cfg.push_back(NULL); + else + { + chain_cfg.push_back(&ino_it->second); + if (!ino_it->second.enc_key.empty()) + enc_key_count++; + } + } + else if (!icache->other_pool_parent_id) + icache->other_pool_parent_id = parent_id; + if (ino_it == st_cli.inode_config.end()) + break; + parent_id = ino_it->second.parent_id; + } + } + // Generate encryption key chain, if applicable + if (enc_key_count) + { + uint8_t *key_data = (uint8_t*)malloc_or_die( + AES_256_XTS_KEY_SIZE * enc_key_count + + sizeof(uint8_t*) * icache->chain.size() + + sizeof(osd_op_enc_t) + ); + uint8_t **keys = (uint8_t**)(key_data + AES_256_XTS_KEY_SIZE * enc_key_count); + osd_op_enc_t *enc = (osd_op_enc_t*)((uint8_t*)keys + sizeof(uint8_t*)*icache->chain.size()); + size_t key_pos = 0; + for (size_t i = 0; i <= chain_cfg.size(); i++) + { + inode_config_t *cfg = !i ? &inode_cfg : chain_cfg[i-1]; + if (cfg && !cfg->enc_key.empty()) + { + assert(key_pos < AES_256_XTS_KEY_SIZE * enc_key_count); + assert(cfg->enc_key.size() == AES_256_XTS_KEY_SIZE); + keys[i] = key_data + key_pos; + memcpy(key_data + key_pos, cfg->enc_key.data(), AES_256_XTS_KEY_SIZE); + key_pos += AES_256_XTS_KEY_SIZE; + } + else + keys[i] = NULL; + } + enc->key_chain = keys; + enc->chain_size = icache->chain.size(); + enc->read_chain_bitmap_pos = pool_cfg.data_block_size/pool_cfg.bitmap_granularity/8; + enc->bitmap_granularity = pool_cfg.bitmap_granularity; + icache->key_data = key_data; + icache->op_enc = enc; + } + inode_cache[ino] = icache; + for (auto & parent: icache->chain) + { + if (parent != ino) + inode_cache_children.insert(std::make_pair(parent, ino)); + } + return icache; +} + bool cluster_client_t::is_ready() { return pgs_loaded; @@ -958,37 +1101,40 @@ bool cluster_client_t::check_rw(cluster_op_t *op) { op->flags |= OP_IMMEDIATE_COMMIT; } - // FIXME: Rework client API by adding open/close and cache inode information in the "FD" bool searched = false; - std::map::iterator ino_it; + std::shared_ptr icache; if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_WRITE) { if (!searched) { - ino_it = st_cli.inode_config.find(op->inode); + icache = inode_cache_get(op->inode); searched = true; } - if (ino_it != st_cli.inode_config.end() && ino_it->second.enc_key) + if (icache && icache->has_parent_loop && op->opcode == OSD_OP_READ) { - op->enc = std::shared_ptr(ino_it->second.enc_key, ino_it->second.enc_key->op_enc); - if (!op->enc->bitmap_granularity) - { - op->enc->bitmap_granularity = pool_it->second.bitmap_granularity; - } + op->retval = -EINVAL; + auto cb = std::move(op->callback); + cb(op); + return false; } + if (icache && icache->op_enc) + { + // Use shared_ptr aliasing to attach op_enc to the inode cache entry + op->enc = std::shared_ptr(icache, icache->op_enc); + } + else + op->enc.reset(); } else - { op->enc.reset(); - } if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && !(op->flags & OSD_OP_IGNORE_READONLY)) { if (!searched) { - ino_it = st_cli.inode_config.find(op->inode); + icache = inode_cache_get(op->inode); searched = true; } - if (ino_it != st_cli.inode_config.end() && ino_it->second.readonly) + if (icache && icache->readonly) { op->retval = -EROFS; auto cb = std::move(op->callback); @@ -1001,32 +1147,19 @@ bool cluster_client_t::check_rw(cluster_op_t *op) { if (!searched) { - ino_it = st_cli.inode_config.find(op->inode); + icache = inode_cache_get(op->inode); searched = true; } - if (ino_it != st_cli.inode_config.end()) + if (icache) { - int chain_size = 0; - while (ino_it != st_cli.inode_config.end() && ino_it->second.parent_id) + for (auto & parent: icache->chain) { - // Check for loops - FIXME check it in etcd_state_client - if (ino_it->second.parent_id == op->inode || - chain_size > st_cli.inode_config.size()) - { - op->retval = -EINVAL; - auto cb = std::move(op->callback); - cb(op); - return false; - } - if (INODE_POOL(ino_it->second.parent_id) == INODE_POOL(ino_it->first) && - wb->has_inode(ino_it->second.parent_id)) + if (INODE_POOL(parent) == INODE_POOL(op->inode) && wb->has_inode(parent)) { // Deoptimise reads - we have dirty data for one of the parent layer(s). op->deoptimise_snapshot = true; break; } - chain_size++; - ino_it = st_cli.inode_config.find(ino_it->second.parent_id); } } } @@ -1159,31 +1292,33 @@ resume_2: } if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_CHAIN_BITMAP) { - // Check parent inode - auto ino_it = st_cli.inode_config.find(op->cur_inode); - // Skip parents from the same pool - int skipped = 0; - while (!op->deoptimise_snapshot && - ino_it != st_cli.inode_config.end() && ino_it->second.parent_id && - INODE_POOL(ino_it->second.parent_id) == INODE_POOL(op->cur_inode)) + uint64_t next_inode = 0; + auto icache = inode_cache_get(op->cur_inode); + if (icache) { - // Check for loops - FIXME check it in etcd_state_client - if (ino_it->second.parent_id == op->inode || - skipped > st_cli.inode_config.size()) + if (icache->has_parent_loop) { op->retval = -EINVAL; erase_op(op); return 1; } - skipped++; - ino_it = st_cli.inode_config.find(ino_it->second.parent_id); + if (op->deoptimise_snapshot) + { + if (icache->chain.size() > 1) + next_inode = icache->chain[1]; + } + else + { + if (icache->other_pool_parent_id) + next_inode = icache->other_pool_parent_id; + } } - if (ino_it != st_cli.inode_config.end() && - ino_it->second.parent_id && - ino_it->second.parent_id != op->inode) + if (next_inode) { // Continue reading from the parent inode - op->cur_inode = ino_it->second.parent_id; + icache = inode_cache_get(next_inode); + op->cur_inode = next_inode; + op->enc = (icache && icache->op_enc ? std::shared_ptr(icache, icache->op_enc) : nullptr); op->parts.clear(); op->done_count = 0; goto resume_0; @@ -1470,7 +1605,7 @@ int cluster_client_t::try_send(cluster_op_t *op, int i, std::functioncur_inode, .offset = part->offset, .len = part->len, - .flags = op->opcode == OSD_OP_READ && op->enc ? OSD_OP_RETURN_CHAIN : 0, + .flags = op->opcode == OSD_OP_READ && op->enc && !op->deoptimise_snapshot ? OSD_OP_RETURN_CHAIN : 0, .meta_revision = meta_rev, .version = op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE ? op->version : 0, } }, diff --git a/src/client/cluster_client.h b/src/client/cluster_client.h index a579549e..8709c1b6 100644 --- a/src/client/cluster_client.h +++ b/src/client/cluster_client.h @@ -5,6 +5,7 @@ #include "messenger.h" #include "etcd_state_client.h" +#include "../util/robin_hood.h" #define DEFAULT_CLIENT_MAX_DIRTY_BYTES 32*1024*1024 #define DEFAULT_CLIENT_MAX_DIRTY_OPS 1024 @@ -81,6 +82,18 @@ struct inode_list_osd_t; struct inode_list_pg_t; class writeback_cache_t; +struct inode_cache_t +{ + std::vector chain; + uint8_t *key_data = NULL; + osd_op_enc_t *op_enc = NULL; + bool readonly = false; + bool has_parent_loop = false; + inode_t other_pool_parent_id = 0; + + ~inode_cache_t(); +}; + // FIXME: Split into public and private interfaces class __attribute__((visibility("default"))) cluster_client_t { @@ -121,6 +134,11 @@ class __attribute__((visibility("default"))) cluster_client_t void *scrap_buffer = NULL; unsigned scrap_buffer_size = 0; + // inodes require some extra state for read/write, it's stored here. + // moreover, robin_hood access is slightly faster than std::map :) + robin_hood::unordered_flat_map> inode_cache; + std::set> inode_cache_children; + bool pgs_loaded = false; ring_consumer_t consumer; std::vector> on_ready_hooks; @@ -165,6 +183,9 @@ protected: void on_change_pg_state_hook(pool_id_t pool_id, pg_num_t pg_num, osd_num_t prev_primary); void on_change_osd_state_hook(uint64_t peer_osd); void on_change_node_placement_hook(); + void on_change_inode_hook(uint64_t inode, bool removed); + + std::shared_ptr inode_cache_get(inode_t ino); void execute_internal(cluster_op_t *op); void execute_cas(cluster_op_t *op); @@ -181,6 +202,7 @@ protected: void erase_op(cluster_op_t *op); void calc_wait(cluster_op_t *op); void inc_wait(uint64_t opcode, uint64_t flags, cluster_op_t *next, int inc); + void continue_lists(); bool continue_listing(inode_list_t *lst); bool restart_listing(inode_list_t* lst); diff --git a/src/client/etcd_state_client.cpp b/src/client/etcd_state_client.cpp index cbaf41ca..70ad82a2 100644 --- a/src/client/etcd_state_client.cpp +++ b/src/client/etcd_state_client.cpp @@ -12,14 +12,6 @@ #endif #include "str_util.h" -inode_key_t::~inode_key_t() -{ - if (op_enc) - { - free(op_enc); - } -} - etcd_state_client_t::~etcd_state_client_t() { for (auto watch: watches) @@ -1298,48 +1290,7 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv) } else { - inode_t parent_inode_num = value["parent_id"].uint64_value(); - if (parent_inode_num && !(parent_inode_num >> (64-POOL_ID_BITS))) - { - uint64_t parent_pool_id = value["parent_pool"].uint64_value(); - if (!parent_pool_id) - parent_inode_num |= pool_id << (64-POOL_ID_BITS); - else if (parent_pool_id >= POOL_ID_MAX) - { - fprintf( - stderr, "Inode %ju/%ju parent_pool value is invalid, ignoring parent setting\n", - inode_num >> (64-POOL_ID_BITS), inode_num & (((uint64_t)1 << (64-POOL_ID_BITS)) - 1) - ); - parent_inode_num = 0; - } - else - parent_inode_num |= parent_pool_id << (64-POOL_ID_BITS); - } - std::shared_ptr enc_key; - if (!value["enc_key"].string_value().empty()) - { - std::vector k = hexdecode(value["enc_key"].string_value()); - if (k.size() == 512/8) // AES-256-XTS - { - enc_key = std::make_shared(); - enc_key->key = std::move(k); - enc_key->op_enc = (osd_op_enc_t*)calloc_or_die(1, sizeof(osd_op_enc_t) + sizeof(uint8_t*)); - enc_key->op_enc->key_chain = (uint8_t**)(enc_key->op_enc + 1); - enc_key->op_enc->key_chain[0] = enc_key->key.data(); - enc_key->op_enc->chain_size = 1; - } - } - insert_inode_config((inode_config_t){ - .num = inode_num, - .name = value["name"].string_value(), - .size = value["size"].uint64_value(), - .parent_id = parent_inode_num, - .readonly = value["readonly"].bool_value(), - .deleted = value["deleted"].bool_value(), - .enc_key = enc_key, - .meta = value["meta"], - .mod_revision = kv.mod_revision, - }); + insert_inode_config(deserialize_inode_cfg(inode_num, kv.value, kv.mod_revision)); } } } @@ -1428,6 +1379,10 @@ json11::Json::object etcd_state_client_t::serialize_inode_cfg(inode_config_t *cf new_cfg["parent_pool"] = (uint64_t)INODE_POOL(cfg->parent_id); new_cfg["parent_id"] = (uint64_t)INODE_NO_POOL(cfg->parent_id); } + if (!cfg->enc_key.empty()) + { + new_cfg["enc_key"] = tohexstr(cfg->enc_key.data(), cfg->enc_key.size()); + } if (cfg->readonly) { new_cfg["readonly"] = true; @@ -1443,6 +1398,53 @@ json11::Json::object etcd_state_client_t::serialize_inode_cfg(inode_config_t *cf return new_cfg; } +inode_config_t etcd_state_client_t::deserialize_inode_cfg(uint64_t inode_num, json11::Json value, uint64_t mod_revision) +{ + inode_t parent_inode_num = value["parent_id"].uint64_value(); + if (parent_inode_num && !INODE_POOL(parent_inode_num)) + { + uint64_t parent_pool_id = value["parent_pool"].uint64_value(); + if (!parent_pool_id) + parent_inode_num = INODE_WITH_POOL(INODE_POOL(inode_num), parent_inode_num); + else if (parent_pool_id >= POOL_ID_MAX) + { + fprintf( + stderr, "Inode %u/%ju parent_pool value is invalid, ignoring parent setting\n", + INODE_POOL(inode_num), INODE_NO_POOL(inode_num) + ); + parent_inode_num = 0; + } + else + parent_inode_num |= parent_pool_id << (64-POOL_ID_BITS); + } + std::vector enc_key; + if (!value["enc_key"].is_null()) + { + if (value["enc_key"].string_value().size() == 2*AES_256_XTS_KEY_SIZE) + { + enc_key.resize(AES_256_XTS_KEY_SIZE); + if (fromhexstr(value["enc_key"].string_value(), AES_256_XTS_KEY_SIZE, enc_key.data()) < AES_256_XTS_KEY_SIZE) + enc_key.clear(); + } + if (enc_key.empty()) + { + fprintf(stderr, "Inode %u/%ju has invalid enc_key, should be %u bit hex string\n", + INODE_POOL(inode_num), INODE_NO_POOL(inode_num), AES_256_XTS_KEY_SIZE); + } + } + return (inode_config_t){ + .num = inode_num, + .name = value["name"].string_value(), + .size = value["size"].uint64_value(), + .parent_id = parent_inode_num, + .readonly = value["readonly"].bool_value(), + .deleted = value["deleted"].bool_value(), + .enc_key = std::move(enc_key), + .meta = value["meta"], + .mod_revision = mod_revision, + }; +} + int etcd_state_client_t::address_count() { return etcd_addresses.size() + etcd_local.size(); diff --git a/src/client/etcd_state_client.h b/src/client/etcd_state_client.h index 92a24ce9..5284dce9 100644 --- a/src/client/etcd_state_client.h +++ b/src/client/etcd_state_client.h @@ -76,16 +76,6 @@ struct pool_config_t void *reshard_state = NULL; }; -struct osd_op_enc_t; - -struct inode_key_t -{ - std::vector key; - osd_op_enc_t *op_enc; - - ~inode_key_t(); -}; - struct inode_config_t { uint64_t num = 0; @@ -94,7 +84,7 @@ struct inode_config_t inode_t parent_id = 0; bool readonly = false; bool deleted = false; - std::shared_ptr enc_key; + std::vector enc_key; // Arbitrary metadata json11::Json meta; // Change revision of the metadata in etcd @@ -176,6 +166,7 @@ public: std::function on_start_watcher_hook; json11::Json::object serialize_inode_cfg(inode_config_t *cfg); + inode_config_t deserialize_inode_cfg(uint64_t inode_num, json11::Json value, uint64_t mod_revision); etcd_kv_t parse_etcd_kv(const json11::Json & kv_json); std::vector get_addresses(); http_context_t *get_http_ctx(); diff --git a/src/client/msgr_op.h b/src/client/msgr_op.h index 8af442dd..6abe54b1 100644 --- a/src/client/msgr_op.h +++ b/src/client/msgr_op.h @@ -18,6 +18,8 @@ #define OSD_OP_INLINE_BUF_COUNT 16 +#define AES_256_XTS_KEY_SIZE 64 + // Kind of a vector with small-list-optimisation struct osd_op_buf_list_t { diff --git a/src/cmd/cli_create.cpp b/src/cmd/cli_create.cpp index fc7f5d20..302fd4c9 100644 --- a/src/cmd/cli_create.cpp +++ b/src/cmd/cli_create.cpp @@ -33,7 +33,8 @@ struct image_creator_t pool_id_t old_pool_id = 0; inode_t new_parent_id = 0; inode_t new_id = 0, old_id = 0; - uint64_t max_id_mod_rev = 0, cfg_mod_rev = 0, idx_mod_rev = 0; + uint64_t max_id_mod_rev = 0, idx_mod_rev = 0; + inode_config_t cur_cfg; inode_config_t new_cfg; int state = 0; @@ -250,7 +251,7 @@ resume_3: } do { - // In addition to next_id, get: size, old_id, old_pool_id, new_parent, cfg_mod_rev, idx_mod_rev + // In addition to next_id, get: cur_cfg, old_id, old_pool_id, size, idx_mod_rev resume_2: resume_3: get_image_details(); @@ -350,17 +351,6 @@ resume_4: goto resume_2; else if (state == 3) goto resume_3; - if (!new_pool_id) - { - for (auto & ic: parent->cli->st_cli.inode_config) - { - if (ic.second.name == image_name) - { - new_pool_id = INODE_POOL(ic.first); - break; - } - } - } parent->etcd_txn(json11::Json::object { { "success", json11::Json::array { get_next_id(), json11::Json::object { @@ -384,7 +374,7 @@ resume_2: extract_next_id(parent->etcd_result["responses"][0]); old_id = 0; old_pool_id = 0; - cfg_mod_rev = idx_mod_rev = 0; + idx_mod_rev = 0; if (parent->etcd_result["responses"][1]["response_range"]["kvs"].array_items().size() == 0) { for (auto & ic: parent->cli->st_cli.inode_config) @@ -393,9 +383,8 @@ resume_2: { old_id = INODE_NO_POOL(ic.first); old_pool_id = INODE_POOL(ic.first); + cur_cfg = ic.second; size = ic.second.size; - new_parent_id = ic.second.parent_id; - cfg_mod_rev = ic.second.mod_revision; break; } } @@ -439,16 +428,14 @@ resume_3: } { auto kv = parent->cli->st_cli.parse_etcd_kv(parent->etcd_result["responses"][0]["response_range"]["kvs"][0]); - size = kv.value["size"].uint64_value(); - new_parent_id = kv.value["parent_id"].uint64_value(); - uint64_t parent_pool_id = kv.value["parent_pool"].uint64_value(); - if (new_parent_id) - { - new_parent_id = INODE_WITH_POOL(parent_pool_id ? parent_pool_id : old_pool_id, new_parent_id); - } - cfg_mod_rev = kv.mod_revision; + cur_cfg = parent->cli->st_cli.deserialize_inode_cfg(INODE_WITH_POOL(old_pool_id, old_id), kv.value, kv.mod_revision); + size = cur_cfg.size; } } + if (!new_pool_id) + { + new_pool_id = old_pool_id; + } } void attempt_create() @@ -527,16 +514,12 @@ resume_3: }; if (new_snap != "") { - inode_config_t snap_cfg = { - .num = INODE_WITH_POOL(old_pool_id, old_id), - .name = image_name+"@"+new_snap, - .size = size, - .parent_id = new_parent_id, - .readonly = true, - }; + inode_config_t snap_cfg = cur_cfg; + snap_cfg.name = image_name+"@"+new_snap; + snap_cfg.readonly = true; checks.push_back(json11::Json::object { { "target", "MOD" }, - { "mod_revision", cfg_mod_rev }, + { "mod_revision", cur_cfg.mod_revision }, { "key", base64_encode( parent->cli->st_cli.etcd_prefix+"/config/inode/"+ std::to_string(old_pool_id)+"/"+std::to_string(old_id) diff --git a/src/cmd/cli_rm.cpp b/src/cmd/cli_rm.cpp index 43b53025..fcc87c31 100644 --- a/src/cmd/cli_rm.cpp +++ b/src/cmd/cli_rm.cpp @@ -451,6 +451,7 @@ resume_100: inode_config_t new_cfg = *child_cfg; new_cfg.deleted = false; new_cfg.num = target_cfg->num; + new_cfg.enc_key = target_cfg->enc_key; new_cfg.parent_id = new_parent; json11::Json::array cmp = json11::Json::array { json11::Json::object { diff --git a/src/util/str_util.cpp b/src/util/str_util.cpp index c0cf8543..b2ec85db 100644 --- a/src/util/str_util.cpp +++ b/src/util/str_util.cpp @@ -551,3 +551,17 @@ size_t fromhexstr(const std::string & from, size_t bytes, uint8_t *to) } return i; } + +std::string tohexstr(const uint8_t *from, size_t bytes) +{ + std::string res; + res.resize(bytes*2); + for (size_t i = 0; i < bytes; i++) + { + uint8_t x = from[i] / 16; + uint8_t y = from[i] % 16; + res[2*i] = (x < 10 ? '0' : 'a'-10) + x; + res[2*i+1] = (y < 10 ? '0' : 'a'-10) + y; + } + return res; +} diff --git a/src/util/str_util.h b/src/util/str_util.h index 4ae01f35..ac8e9f25 100644 --- a/src/util/str_util.h +++ b/src/util/str_util.h @@ -36,5 +36,6 @@ std::string format_datetime(uint64_t unixtime); bool is_zero(void *buf, size_t size); std::string urldecode(const std::string & orig); size_t fromhexstr(const std::string & from, size_t bytes, uint8_t *to); +std::string tohexstr(const uint8_t *from, size_t bytes); #pragma GCC visibility pop diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 9340eb42..c1184626 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -57,6 +57,7 @@ OLD=1 ./test_move_reappear.sh ./test_snapshot_chain.sh SCHEME=ec ./test_snapshot_chain.sh +ENCRYPTED=1 ./test_snapshot_chain.sh OLD=1 ./test_snapshot_chain.sh OLD=1 SCHEME=ec ./test_snapshot_chain.sh diff --git a/tests/test_snapshot_chain.sh b/tests/test_snapshot_chain.sh index 4ae95455..5b2e8d69 100755 --- a/tests/test_snapshot_chain.sh +++ b/tests/test_snapshot_chain.sh @@ -1,11 +1,15 @@ #!/bin/bash -ex +ENCRYPTED=${ENCRYPTED:-} . `dirname $0`/run_3osds.sh check_qemu # Test multiple snapshots $VITASTOR_CLI create -s 32M testchain +if [[ -n "$ENCRYPTED" ]]; then + $ETCDCTL put /vitastor/config/inode/1/1 '{"name":"testchain","size":33554432,"enc_key":"'$(openssl rand -hex 64)'"}' +fi $VITASTOR_FIO -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \ -image=testchain -mirror_file=./testdata/bin/mirror.bin @@ -13,6 +17,10 @@ $VITASTOR_FIO -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \ for i in {1..10}; do # Create a snapshot $VITASTOR_CLI snap-create testchain@$i + if [[ -n "$ENCRYPTED" ]]; then + # Generate different keys for each layer + $ETCDCTL put /vitastor/config/inode/1/$((i+1)) '{"name":"testchain","parent_id":'$((i))',"size":33554432,"enc_key":"'$(openssl rand -hex 64)'"}' + fi # Check that the new snapshot is see-through qemu-img convert -p \ -f raw "vitastor:config_path=$VITASTOR_CFG:image=testchain" \