Support reading from snapshots encrypted with different keys

.gitea/workflows/test.yml

@@ -702,6 +702,24 @@ jobs:
           echo ""
         done
 
+  test_snapshot_chain_encrypted:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: ENCRYPTED=1 /root/vitastor/tests/test_snapshot_chain.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_old_snapshot_chain:
     runs-on: ubuntu-latest
     needs: build

.gitea/workflows/tests-to-yaml.pl

@@ -42,6 +42,10 @@ for my $line (<>)
             {
                 $test_name .= '_https';
             }
+            elsif ($1 eq 'ENCRYPTED')
+            {
+                $test_name .= '_encrypted';
+            }
             elsif ($1 eq 'OLD')
             {
                 $test_name =~ s/^test_/test_old_/s;

src/client/cluster_client.cpp

@@ -11,6 +11,16 @@
 #define TRY_SEND_CONNECTING 1
 #define TRY_SEND_OK 2
 
+inode_cache_t::~inode_cache_t()
+{
+    if (key_data)
+    {
+        free(key_data);
+        key_data = NULL;
+        op_enc = NULL;
+    }
+}
+
 cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json config)
 {
     wb = new writeback_cache_t();
@@ -62,6 +72,7 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
     st_cli.on_change_node_placement_hook = [this]() { on_change_node_placement_hook(); };
     st_cli.on_load_pgs_hook = [this](bool success) { on_load_pgs_hook(success); };
     st_cli.on_reload_hook = [this]() { st_cli.load_global_config(); };
+    st_cli.on_inode_change_hook = [this](uint64_t inode, bool removed) { on_change_inode_hook(inode, removed); };
 
     st_cli.parse_config(config);
     st_cli.infinite_start = false;
@@ -607,6 +618,8 @@ void cluster_client_t::on_change_pool_config_hook()
             pg_counts[pool_item.first] = pool_item.second.real_pg_count;
         }
     }
+    inode_cache.clear();
+    inode_cache_children.clear();
     continue_ops();
 }
 
@@ -652,6 +665,136 @@ void cluster_client_t::on_change_node_placement_hook()
     self_tree_metrics.clear();
 }
 
+// FIXME: Rework client API by adding open/close and cache inode information in the "FD" (maybe)
+void cluster_client_t::on_change_inode_hook(uint64_t inode, bool removed)
+{
+    std::vector<inode_t> children = { inode };
+    for (size_t i = 0; i < children.size(); i++)
+    {
+        auto it = inode_cache_children.lower_bound(std::make_pair(children[i], (inode_t)0));
+        while (it != inode_cache_children.end() && it->first == children[i])
+        {
+            children.push_back(it->second);
+            it++;
+        }
+    }
+    for (auto & inode: children)
+    {
+        auto it = inode_cache.find(inode);
+        if (it != inode_cache.end())
+        {
+            auto icache = it->second;
+            for (auto & parent: icache->chain)
+            {
+                inode_cache_children.erase(std::make_pair(parent, inode));
+            }
+            inode_cache.erase(it);
+        }
+    }
+}
+
+std::shared_ptr<inode_cache_t> cluster_client_t::inode_cache_get(inode_t ino)
+{
+    auto icache_it = inode_cache.find(ino);
+    if (icache_it != inode_cache.end())
+    {
+        return icache_it->second;
+    }
+    // Fill inode cache
+    auto ino_it = st_cli.inode_config.find(ino);
+    if (ino_it == st_cli.inode_config.end())
+    {
+        inode_cache[ino] = NULL;
+        return NULL;
+    }
+    auto pool_it = st_cli.pool_config.find(INODE_POOL(ino));
+    if (pool_it == st_cli.pool_config.end())
+    {
+        inode_cache[ino] = NULL;
+        return NULL;
+    }
+    auto & inode_cfg = ino_it->second;
+    auto & pool_cfg = pool_it->second;
+    std::shared_ptr<inode_cache_t> icache = std::make_shared<inode_cache_t>();
+    icache->readonly = inode_cfg.readonly;
+    icache->chain.push_back(ino);
+    std::vector<inode_config_t*> chain_cfg;
+    int enc_key_count = !inode_cfg.enc_key.empty() ? 1 : 0;
+    if (inode_cfg.parent_id)
+    {
+        // Check for loops and cache the chain
+        robin_hood::unordered_flat_set<inode_t> seen;
+        seen.insert(ino);
+        uint64_t parent_id = inode_cfg.parent_id;
+        while (parent_id)
+        {
+            if (seen.find(parent_id) != seen.end())
+            {
+                icache->has_parent_loop = true;
+                break;
+            }
+            seen.insert(parent_id);
+            ino_it = st_cli.inode_config.find(parent_id);
+            if (INODE_POOL(parent_id) == INODE_POOL(ino))
+            {
+                icache->chain.push_back(parent_id);
+                if (ino_it == st_cli.inode_config.end())
+                    chain_cfg.push_back(NULL);
+                else
+                {
+                    chain_cfg.push_back(&ino_it->second);
+                    if (!ino_it->second.enc_key.empty())
+                        enc_key_count++;
+                }
+            }
+            else if (!icache->other_pool_parent_id)
+                icache->other_pool_parent_id = parent_id;
+            if (ino_it == st_cli.inode_config.end())
+                break;
+            parent_id = ino_it->second.parent_id;
+        }
+    }
+    // Generate encryption key chain, if applicable
+    if (enc_key_count)
+    {
+        uint8_t *key_data = (uint8_t*)malloc_or_die(
+            AES_256_XTS_KEY_SIZE * enc_key_count +
+            sizeof(uint8_t*) * icache->chain.size() +
+            sizeof(osd_op_enc_t)
+        );
+        uint8_t **keys = (uint8_t**)(key_data + AES_256_XTS_KEY_SIZE * enc_key_count);
+        osd_op_enc_t *enc = (osd_op_enc_t*)((uint8_t*)keys + sizeof(uint8_t*)*icache->chain.size());
+        size_t key_pos = 0;
+        for (size_t i = 0; i <= chain_cfg.size(); i++)
+        {
+            inode_config_t *cfg = !i ? &inode_cfg : chain_cfg[i-1];
+            if (cfg && !cfg->enc_key.empty())
+            {
+                assert(key_pos < AES_256_XTS_KEY_SIZE * enc_key_count);
+                assert(cfg->enc_key.size() == AES_256_XTS_KEY_SIZE);
+                keys[i] = key_data + key_pos;
+                memcpy(key_data + key_pos, cfg->enc_key.data(), AES_256_XTS_KEY_SIZE);
+                key_pos += AES_256_XTS_KEY_SIZE;
+            }
+            else
+                keys[i] = NULL;
+        }
+        enc->key_chain = keys;
+        enc->chain_size = icache->chain.size();
+        enc->read_chain_bitmap_pos = pool_cfg.data_block_size/pool_cfg.bitmap_granularity/8;
+        enc->bitmap_granularity = pool_cfg.bitmap_granularity;
+        icache->key_data = key_data;
+        icache->op_enc = enc;
+    }
+    inode_cache[ino] = icache;
+    for (auto & parent: icache->chain)
+    {
+        if (parent != ino)
+            inode_cache_children.insert(std::make_pair(parent, ino));
+    }
+    return icache;
+}
+
 bool cluster_client_t::is_ready()
 {
     return pgs_loaded;
@@ -958,37 +1101,40 @@ bool cluster_client_t::check_rw(cluster_op_t *op)
     {
         op->flags |= OP_IMMEDIATE_COMMIT;
     }
-    // FIXME: Rework client API by adding open/close and cache inode information in the "FD"
     bool searched = false;
-    std::map<inode_t, inode_config_t>::iterator ino_it;
+    std::shared_ptr<inode_cache_t> icache;
     if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_WRITE)
     {
         if (!searched)
         {
-            ino_it = st_cli.inode_config.find(op->inode);
+            icache = inode_cache_get(op->inode);
             searched = true;
         }
-        if (ino_it != st_cli.inode_config.end() && ino_it->second.enc_key)
+        if (icache && icache->has_parent_loop && op->opcode == OSD_OP_READ)
         {
-            op->enc = std::shared_ptr<osd_op_enc_t>(ino_it->second.enc_key, ino_it->second.enc_key->op_enc);
-            if (!op->enc->bitmap_granularity)
-            {
-                op->enc->bitmap_granularity = pool_it->second.bitmap_granularity;
-            }
+            op->retval = -EINVAL;
+            auto cb = std::move(op->callback);
+            cb(op);
+            return false;
         }
+        if (icache && icache->op_enc)
+        {
+            // Use shared_ptr aliasing to attach op_enc to the inode cache entry
+            op->enc = std::shared_ptr<osd_op_enc_t>(icache, icache->op_enc);
+        }
+        else
+            op->enc.reset();
     }
     else
-    {
         op->enc.reset();
-    }
     if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && !(op->flags & OSD_OP_IGNORE_READONLY))
     {
         if (!searched)
         {
-            ino_it = st_cli.inode_config.find(op->inode);
+            icache = inode_cache_get(op->inode);
             searched = true;
         }
-        if (ino_it != st_cli.inode_config.end() && ino_it->second.readonly)
+        if (icache && icache->readonly)
         {
             op->retval = -EROFS;
             auto cb = std::move(op->callback);
@@ -1001,32 +1147,19 @@ bool cluster_client_t::check_rw(cluster_op_t *op)
     {
         if (!searched)
         {
-            ino_it = st_cli.inode_config.find(op->inode);
+            icache = inode_cache_get(op->inode);
             searched = true;
         }
-        if (ino_it != st_cli.inode_config.end())
+        if (icache)
         {
-            int chain_size = 0;
-            while (ino_it != st_cli.inode_config.end() && ino_it->second.parent_id)
+            for (auto & parent: icache->chain)
             {
-                // Check for loops - FIXME check it in etcd_state_client
-                if (ino_it->second.parent_id == op->inode ||
-                    chain_size > st_cli.inode_config.size())
-                {
-                    op->retval = -EINVAL;
-                    auto cb = std::move(op->callback);
-                    cb(op);
-                    return false;
-                }
-                if (INODE_POOL(ino_it->second.parent_id) == INODE_POOL(ino_it->first) &&
-                    wb->has_inode(ino_it->second.parent_id))
+                if (INODE_POOL(parent) == INODE_POOL(op->inode) && wb->has_inode(parent))
                 {
                     // Deoptimise reads - we have dirty data for one of the parent layer(s).
                     op->deoptimise_snapshot = true;
                     break;
                 }
-                chain_size++;
-                ino_it = st_cli.inode_config.find(ino_it->second.parent_id);
             }
         }
     }
@@ -1152,31 +1285,33 @@ resume_2:
         // because if some operations were invalid for the new PG count we'd get errors
         if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
         {
-            // Check parent inode
-            auto ino_it = st_cli.inode_config.find(op->cur_inode);
-            // Skip parents from the same pool
-            int skipped = 0;
-            while (!op->deoptimise_snapshot &&
-                ino_it != st_cli.inode_config.end() && ino_it->second.parent_id &&
-                INODE_POOL(ino_it->second.parent_id) == INODE_POOL(op->cur_inode))
+            uint64_t next_inode = 0;
+            auto icache = inode_cache_get(op->cur_inode);
+            if (icache)
             {
-                // Check for loops - FIXME check it in etcd_state_client
-                if (ino_it->second.parent_id == op->inode ||
-                    skipped > st_cli.inode_config.size())
+                if (icache->has_parent_loop)
                 {
                     op->retval = -EINVAL;
                     erase_op(op);
                     return 1;
                 }
-                skipped++;
-                ino_it = st_cli.inode_config.find(ino_it->second.parent_id);
+                if (op->deoptimise_snapshot)
+                {
+                    if (icache->chain.size() > 1)
+                        next_inode = icache->chain[1];
+                }
+                else
+                {
+                    if (icache->other_pool_parent_id)
+                        next_inode = icache->other_pool_parent_id;
+                }
             }
-            if (ino_it != st_cli.inode_config.end() &&
-                ino_it->second.parent_id &&
-                ino_it->second.parent_id != op->inode)
+            if (next_inode)
             {
                 // Continue reading from the parent inode
-                op->cur_inode = ino_it->second.parent_id;
+                icache = inode_cache_get(next_inode);
+                op->cur_inode = next_inode;
+                op->enc = (icache && icache->op_enc ? std::shared_ptr<osd_op_enc_t>(icache, icache->op_enc) : nullptr);
                 op->parts.clear();
                 op->done_count = 0;
                 goto resume_0;
@@ -1463,7 +1598,7 @@ int cluster_client_t::try_send(cluster_op_t *op, int i, std::function<void(osd_o
                     .inode = op->cur_inode,
                     .offset = part->offset,
                     .len = part->len,
-                    .flags = op->opcode == OSD_OP_READ && op->enc ? OSD_OP_RETURN_CHAIN : 0,
+                    .flags = op->opcode == OSD_OP_READ && op->enc && !op->deoptimise_snapshot ? OSD_OP_RETURN_CHAIN : 0,
                     .meta_revision = meta_rev,
                     .version = op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE ? op->version : 0,
                 } },

src/client/cluster_client.h

@@ -5,6 +5,7 @@
 
 #include "messenger.h"
 #include "etcd_state_client.h"
+#include "../util/robin_hood.h"
 
 #define DEFAULT_CLIENT_MAX_DIRTY_BYTES 32*1024*1024
 #define DEFAULT_CLIENT_MAX_DIRTY_OPS 1024
@@ -81,6 +82,18 @@ struct inode_list_osd_t;
 struct inode_list_pg_t;
 class writeback_cache_t;
 
+struct inode_cache_t
+{
+    std::vector<inode_t> chain;
+    uint8_t *key_data = NULL;
+    osd_op_enc_t *op_enc = NULL;
+    bool readonly = false;
+    bool has_parent_loop = false;
+    inode_t other_pool_parent_id = 0;
+
+    ~inode_cache_t();
+};
+
 // FIXME: Split into public and private interfaces
 class __attribute__((visibility("default"))) cluster_client_t
 {
@@ -124,6 +137,11 @@ public:
     void *scrap_buffer = NULL;
     unsigned scrap_buffer_size = 0;
 
+    // inodes require some extra state for read/write, it's stored here.
+    // moreover, robin_hood access is slightly faster than std::map :)
+    robin_hood::unordered_flat_map<inode_t, std::shared_ptr<inode_cache_t>> inode_cache;
+    std::set<std::pair<inode_t, inode_t>> inode_cache_children;
+
     bool pgs_loaded = false;
     ring_consumer_t consumer;
     std::vector<std::function<void(void)>> on_ready_hooks;
@@ -174,6 +192,9 @@ protected:
     void on_change_pg_state_hook(pool_id_t pool_id, pg_num_t pg_num, osd_num_t prev_primary);
     void on_change_osd_state_hook(uint64_t peer_osd);
     void on_change_node_placement_hook();
+    void on_change_inode_hook(uint64_t inode, bool removed);
+
+    std::shared_ptr<inode_cache_t> inode_cache_get(inode_t ino);
 
     void execute_internal(cluster_op_t *op);
     void execute_cas(cluster_op_t *op);
@@ -190,6 +211,7 @@ protected:
     void erase_op(cluster_op_t *op);
     void calc_wait(cluster_op_t *op);
     void inc_wait(uint64_t opcode, uint64_t flags, cluster_op_t *next, int inc);
+
     void continue_lists();
     bool continue_listing(inode_list_t *lst);
     bool restart_listing(inode_list_t* lst);

src/client/etcd_state_client.cpp

@@ -12,14 +12,6 @@
 #endif
 #include "str_util.h"
 
-inode_key_t::~inode_key_t()
-{
-    if (op_enc)
-    {
-        free(op_enc);
-    }
-}
-
 etcd_state_client_t::~etcd_state_client_t()
 {
     for (auto watch: watches)
@@ -1300,48 +1292,7 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
             }
             else
             {
-                inode_t parent_inode_num = value["parent_id"].uint64_value();
-                if (parent_inode_num && !(parent_inode_num >> (64-POOL_ID_BITS)))
-                {
-                    uint64_t parent_pool_id = value["parent_pool"].uint64_value();
-                    if (!parent_pool_id)
-                        parent_inode_num |= pool_id << (64-POOL_ID_BITS);
-                    else if (parent_pool_id >= POOL_ID_MAX)
-                    {
-                        fprintf(
-                            stderr, "Inode %ju/%ju parent_pool value is invalid, ignoring parent setting\n",
-                            inode_num >> (64-POOL_ID_BITS), inode_num & (((uint64_t)1 << (64-POOL_ID_BITS)) - 1)
-                        );
-                        parent_inode_num = 0;
-                    }
-                    else
-                        parent_inode_num |= parent_pool_id << (64-POOL_ID_BITS);
-                }
-                std::shared_ptr<inode_key_t> enc_key;
-                if (!value["enc_key"].string_value().empty())
-                {
-                    std::vector<uint8_t> k = hexdecode(value["enc_key"].string_value());
-                    if (k.size() == 512/8) // AES-256-XTS
-                    {
-                        enc_key = std::make_shared<inode_key_t>();
-                        enc_key->key = std::move(k);
-                        enc_key->op_enc = (osd_op_enc_t*)calloc_or_die(1, sizeof(osd_op_enc_t) + sizeof(uint8_t*));
-                        enc_key->op_enc->key_chain = (uint8_t**)(enc_key->op_enc + 1);
-                        enc_key->op_enc->key_chain[0] = enc_key->key.data();
-                        enc_key->op_enc->chain_size = 1;
-                    }
-                }
-                insert_inode_config((inode_config_t){
-                    .num = inode_num,
-                    .name = value["name"].string_value(),
-                    .size = value["size"].uint64_value(),
-                    .parent_id = parent_inode_num,
-                    .readonly = value["readonly"].bool_value(),
-                    .deleted = value["deleted"].bool_value(),
-                    .enc_key = enc_key,
-                    .meta = value["meta"],
-                    .mod_revision = kv.mod_revision,
-                });
+                insert_inode_config(deserialize_inode_cfg(inode_num, kv.value, kv.mod_revision));
             }
         }
     }
@@ -1430,6 +1381,10 @@ json11::Json::object etcd_state_client_t::serialize_inode_cfg(inode_config_t *cf
             new_cfg["parent_pool"] = (uint64_t)INODE_POOL(cfg->parent_id);
         new_cfg["parent_id"] = (uint64_t)INODE_NO_POOL(cfg->parent_id);
     }
+    if (!cfg->enc_key.empty())
+    {
+        new_cfg["enc_key"] = tohexstr(cfg->enc_key.data(), cfg->enc_key.size());
+    }
     if (cfg->readonly)
     {
         new_cfg["readonly"] = true;
@@ -1445,6 +1400,53 @@ json11::Json::object etcd_state_client_t::serialize_inode_cfg(inode_config_t *cf
     return new_cfg;
 }
 
+inode_config_t etcd_state_client_t::deserialize_inode_cfg(uint64_t inode_num, json11::Json value, uint64_t mod_revision)
+{
+    inode_t parent_inode_num = value["parent_id"].uint64_value();
+    if (parent_inode_num && !INODE_POOL(parent_inode_num))
+    {
+        uint64_t parent_pool_id = value["parent_pool"].uint64_value();
+        if (!parent_pool_id)
+            parent_inode_num = INODE_WITH_POOL(INODE_POOL(inode_num), parent_inode_num);
+        else if (parent_pool_id >= POOL_ID_MAX)
+        {
+            fprintf(
+                stderr, "Inode %u/%ju parent_pool value is invalid, ignoring parent setting\n",
+                INODE_POOL(inode_num), INODE_NO_POOL(inode_num)
+            );
+            parent_inode_num = 0;
+        }
+        else
+            parent_inode_num |= parent_pool_id << (64-POOL_ID_BITS);
+    }
+    std::vector<uint8_t> enc_key;
+    if (!value["enc_key"].is_null())
+    {
+        if (value["enc_key"].string_value().size() == 2*AES_256_XTS_KEY_SIZE)
+        {
+            enc_key.resize(AES_256_XTS_KEY_SIZE);
+            if (fromhexstr(value["enc_key"].string_value(), AES_256_XTS_KEY_SIZE, enc_key.data()) < AES_256_XTS_KEY_SIZE)
+                enc_key.clear();
+        }
+        if (enc_key.empty())
+        {
+            fprintf(stderr, "Inode %u/%ju has invalid enc_key, should be %u bit hex string\n",
+                INODE_POOL(inode_num), INODE_NO_POOL(inode_num), AES_256_XTS_KEY_SIZE);
+        }
+    }
+    return (inode_config_t){
+        .num = inode_num,
+        .name = value["name"].string_value(),
+        .size = value["size"].uint64_value(),
+        .parent_id = parent_inode_num,
+        .readonly = value["readonly"].bool_value(),
+        .deleted = value["deleted"].bool_value(),
+        .enc_key = std::move(enc_key),
+        .meta = value["meta"],
+        .mod_revision = mod_revision,
+    };
+}
+
 int etcd_state_client_t::address_count()
 {
     return etcd_addresses.size() + etcd_local.size();

src/client/etcd_state_client.h

@@ -76,16 +76,6 @@ struct pool_config_t
     void *reshard_state = NULL;
 };
 
-struct osd_op_enc_t;
-
-struct inode_key_t
-{
-    std::vector<uint8_t> key;
-    osd_op_enc_t *op_enc;
-
-    ~inode_key_t();
-};
-
 struct inode_config_t
 {
     uint64_t num = 0;
@@ -94,7 +84,7 @@ struct inode_config_t
     inode_t parent_id = 0;
     bool readonly = false;
     bool deleted = false;
-    std::shared_ptr<inode_key_t> enc_key;
+    std::vector<uint8_t> enc_key;
     // Arbitrary metadata
     json11::Json meta;
     // Change revision of the metadata in etcd
@@ -176,6 +166,7 @@ public:
     std::function<void(http_co_t *)> on_start_watcher_hook;
 
     json11::Json::object serialize_inode_cfg(inode_config_t *cfg);
+    inode_config_t deserialize_inode_cfg(uint64_t inode_num, json11::Json value, uint64_t mod_revision);
     etcd_kv_t parse_etcd_kv(const json11::Json & kv_json);
     std::vector<std::string> get_addresses();
     http_context_t *get_http_ctx();

src/client/msgr_op.h

@@ -18,6 +18,8 @@
 
 #define OSD_OP_INLINE_BUF_COUNT 16
 
+#define AES_256_XTS_KEY_SIZE 64
+
 // Kind of a vector with small-list-optimisation
 struct osd_op_buf_list_t
 {

src/cmd/cli_create.cpp

@@ -33,7 +33,8 @@ struct image_creator_t
     pool_id_t old_pool_id = 0;
     inode_t new_parent_id = 0;
     inode_t new_id = 0, old_id = 0;
-    uint64_t max_id_mod_rev = 0, cfg_mod_rev = 0, idx_mod_rev = 0;
+    uint64_t max_id_mod_rev = 0, idx_mod_rev = 0;
+    inode_config_t cur_cfg;
     inode_config_t new_cfg;
 
     int state = 0;
@@ -250,7 +251,7 @@ resume_3:
         }
         do
         {
-            // In addition to next_id, get: size, old_id, old_pool_id, new_parent, cfg_mod_rev, idx_mod_rev
+            // In addition to next_id, get: cur_cfg, old_id, old_pool_id, size, idx_mod_rev
 resume_2:
 resume_3:
             get_image_details();
@@ -350,17 +351,6 @@ resume_4:
             goto resume_2;
         else if (state == 3)
             goto resume_3;
-        if (!new_pool_id)
-        {
-            for (auto & ic: parent->cli->st_cli.inode_config)
-            {
-                if (ic.second.name == image_name)
-                {
-                    new_pool_id = INODE_POOL(ic.first);
-                    break;
-                }
-            }
-        }
         parent->etcd_txn(json11::Json::object { { "success", json11::Json::array {
             get_next_id(),
             json11::Json::object {
@@ -384,7 +374,7 @@ resume_2:
         extract_next_id(parent->etcd_result["responses"][0]);
         old_id = 0;
         old_pool_id = 0;
-        cfg_mod_rev = idx_mod_rev = 0;
+        idx_mod_rev = 0;
         if (parent->etcd_result["responses"][1]["response_range"]["kvs"].array_items().size() == 0)
         {
             for (auto & ic: parent->cli->st_cli.inode_config)
@@ -393,9 +383,8 @@ resume_2:
                 {
                     old_id = INODE_NO_POOL(ic.first);
                     old_pool_id = INODE_POOL(ic.first);
+                    cur_cfg = ic.second;
                     size = ic.second.size;
-                    new_parent_id = ic.second.parent_id;
-                    cfg_mod_rev = ic.second.mod_revision;
                     break;
                 }
             }
@@ -439,16 +428,14 @@ resume_3:
             }
             {
                 auto kv = parent->cli->st_cli.parse_etcd_kv(parent->etcd_result["responses"][0]["response_range"]["kvs"][0]);
-                size = kv.value["size"].uint64_value();
-                new_parent_id = kv.value["parent_id"].uint64_value();
-                uint64_t parent_pool_id = kv.value["parent_pool"].uint64_value();
-                if (new_parent_id)
-                {
-                    new_parent_id = INODE_WITH_POOL(parent_pool_id ? parent_pool_id : old_pool_id, new_parent_id);
-                }
-                cfg_mod_rev = kv.mod_revision;
+                cur_cfg = parent->cli->st_cli.deserialize_inode_cfg(INODE_WITH_POOL(old_pool_id, old_id), kv.value, kv.mod_revision);
+                size = cur_cfg.size;
             }
         }
+        if (!new_pool_id)
+        {
+            new_pool_id = old_pool_id;
+        }
     }
 
     void attempt_create()
@@ -527,16 +514,12 @@ resume_3:
         };
         if (new_snap != "")
         {
-            inode_config_t snap_cfg = {
-                .num = INODE_WITH_POOL(old_pool_id, old_id),
-                .name = image_name+"@"+new_snap,
-                .size = size,
-                .parent_id = new_parent_id,
-                .readonly = true,
-            };
+            inode_config_t snap_cfg = cur_cfg;
+            snap_cfg.name = image_name+"@"+new_snap;
+            snap_cfg.readonly = true;
             checks.push_back(json11::Json::object {
                 { "target", "MOD" },
-                { "mod_revision", cfg_mod_rev },
+                { "mod_revision", cur_cfg.mod_revision },
                 { "key", base64_encode(
                     parent->cli->st_cli.etcd_prefix+"/config/inode/"+
                     std::to_string(old_pool_id)+"/"+std::to_string(old_id)

src/cmd/cli_rm.cpp

@@ -451,6 +451,7 @@ resume_100:
         inode_config_t new_cfg = *child_cfg;
         new_cfg.deleted = false;
         new_cfg.num = target_cfg->num;
+        new_cfg.enc_key = target_cfg->enc_key;
         new_cfg.parent_id = new_parent;
         json11::Json::array cmp = json11::Json::array {
             json11::Json::object {

src/util/str_util.cpp

@@ -551,3 +551,17 @@ size_t fromhexstr(const std::string & from, size_t bytes, uint8_t *to)
     }
     return i;
 }
+
+std::string tohexstr(const uint8_t *from, size_t bytes)
+{
+    std::string res;
+    res.resize(bytes*2);
+    for (size_t i = 0; i < bytes; i++)
+    {
+        uint8_t x = from[i] / 16;
+        uint8_t y = from[i] % 16;
+        res[2*i] = (x < 10 ? '0' : 'a'-10) + x;
+        res[2*i+1] = (y < 10 ? '0' : 'a'-10) + y;
+    }
+    return res;
+}

src/util/str_util.h

@@ -36,5 +36,6 @@ std::string format_datetime(uint64_t unixtime);
 bool is_zero(void *buf, size_t size);
 std::string urldecode(const std::string & orig);
 size_t fromhexstr(const std::string & from, size_t bytes, uint8_t *to);
+std::string tohexstr(const uint8_t *from, size_t bytes);
 
 #pragma GCC visibility pop

tests/run_tests.sh

@@ -57,6 +57,7 @@ OLD=1 ./test_move_reappear.sh
 
 ./test_snapshot_chain.sh
 SCHEME=ec ./test_snapshot_chain.sh
+ENCRYPTED=1 ./test_snapshot_chain.sh
 OLD=1 ./test_snapshot_chain.sh
 OLD=1 SCHEME=ec ./test_snapshot_chain.sh
 

tests/test_snapshot_chain.sh

@@ -1,11 +1,15 @@
 #!/bin/bash -ex
 
+ENCRYPTED=${ENCRYPTED:-}
 . `dirname $0`/run_3osds.sh
 check_qemu
 
 # Test multiple snapshots
 
 $VITASTOR_CLI create -s 32M testchain
+if [[ -n "$ENCRYPTED" ]]; then
+    $ETCDCTL put /vitastor/config/inode/1/1 '{"name":"testchain","size":33554432,"enc_key":"'$(openssl rand -hex 64)'"}'
+fi
 
 $VITASTOR_FIO -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \
     -image=testchain -mirror_file=./testdata/bin/mirror.bin
@@ -13,6 +17,10 @@ $VITASTOR_FIO -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \
 for i in {1..10}; do
     # Create a snapshot
     $VITASTOR_CLI snap-create testchain@$i
+    if [[ -n "$ENCRYPTED" ]]; then
+        # Generate different keys for each layer
+        $ETCDCTL put /vitastor/config/inode/1/$((i+1)) '{"name":"testchain","parent_id":'$((i))',"size":33554432,"enc_key":"'$(openssl rand -hex 64)'"}'
+    fi
     # Check that the new snapshot is see-through
     qemu-img convert -p \
         -f raw "vitastor:config_path=$VITASTOR_CFG:image=testchain" \