Support decryption with multiple keys

src/client/cluster_client.cpp

@@ -968,9 +968,9 @@ bool cluster_client_t::check_rw(cluster_op_t *op)
             ino_it = st_cli.inode_config.find(op->inode);
             searched = true;
         }
-        if (ino_it != st_cli.inode_config.end() && ino_it->second.enc)
+        if (ino_it != st_cli.inode_config.end() && ino_it->second.enc_key)
         {
-            op->enc = ino_it->second.enc;
+            op->enc = std::shared_ptr<osd_op_enc_t>(ino_it->second.enc_key, ino_it->second.enc_key->op_enc);
             if (!op->enc->bitmap_granularity)
             {
                 op->enc->bitmap_granularity = pool_it->second.bitmap_granularity;

src/client/cluster_client.h

@@ -71,7 +71,7 @@ protected:
     cluster_op_t *prev = NULL, *next = NULL;
     int prev_wait = 0;
     uint64_t flush_id = 0;
-    std::shared_ptr<inode_enc_t> enc;
+    std::shared_ptr<osd_op_enc_t> enc;
     friend class cluster_client_t;
     friend class writeback_cache_t;
 };

src/client/etcd_state_client.cpp

@@ -1,7 +1,9 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
+#include "malloc_or_die.h"
 #include "osd_ops.h"
+#include "msgr_op.h"
 #include "pg_states.h"
 #include "etcd_state_client.h"
 #ifndef __MOCK__
@@ -10,6 +12,14 @@
 #endif
 #include "str_util.h"
 
+inode_key_t::~inode_key_t()
+{
+    if (op_enc)
+    {
+        free(op_enc);
+    }
+}
+
 etcd_state_client_t::~etcd_state_client_t()
 {
     for (auto watch: watches)
@@ -1305,14 +1315,18 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
                     else
                         parent_inode_num |= parent_pool_id << (64-POOL_ID_BITS);
                 }
-                std::shared_ptr<inode_enc_t> enc;
+                std::shared_ptr<inode_key_t> enc_key;
                 if (!value["enc_key"].string_value().empty())
                 {
                     std::vector<uint8_t> k = hexdecode(value["enc_key"].string_value());
-                    if (k.size() == 512/8)
+                    if (k.size() == 512/8) // AES-256-XTS
                     {
-                        enc = std::make_shared<inode_enc_t>();
+                        enc_key = std::make_shared<inode_key_t>();
-                        enc->key = std::move(k);
+                        enc_key->key = std::move(k);
+                        enc_key->op_enc = (osd_op_enc_t*)calloc_or_die(1, sizeof(osd_op_enc_t) + sizeof(uint8_t*));
+                        enc_key->op_enc->key_chain = (uint8_t**)(enc_key->op_enc + 1);
+                        enc_key->op_enc->key_chain[0] = enc_key->key.data();
+                        enc_key->op_enc->chain_size = 1;
                     }
                 }
                 insert_inode_config((inode_config_t){
@@ -1322,7 +1336,7 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
                     .parent_id = parent_inode_num,
                     .readonly = value["readonly"].bool_value(),
                     .deleted = value["deleted"].bool_value(),
-                    .enc = enc,
+                    .enc_key = enc_key,
                     .meta = value["meta"],
                     .mod_revision = kv.mod_revision,
                 });

src/client/etcd_state_client.h

@@ -76,12 +76,14 @@ struct pool_config_t
     void *reshard_state = NULL;
 };
 
-struct inode_enc_t
+struct osd_op_enc_t;
+
+struct inode_key_t
 {
-    int refs = 0;
     std::vector<uint8_t> key;
-    // FIXME It may also contain snapshot chain and key information
+    osd_op_enc_t *op_enc;
-    uint32_t bitmap_granularity = 0;
+
+    ~inode_key_t();
 };
 
 struct inode_config_t
@@ -92,7 +94,7 @@ struct inode_config_t
     inode_t parent_id = 0;
     bool readonly = false;
     bool deleted = false;
-    std::shared_ptr<inode_enc_t> enc;
+    std::shared_ptr<inode_key_t> enc_key;
     // Arbitrary metadata
     json11::Json meta;
     // Change revision of the metadata in etcd

src/client/msgr_encrypt.cpp

@@ -37,7 +37,7 @@ op_aes_xts_encrypt_t::~op_aes_xts_encrypt_t()
         free(tmp);
 }
 
-void op_aes_xts_encrypt_t::start(const uint8_t *key, uint64_t start_offset, size_t block_size)
+void op_aes_xts_encrypt_t::start(uint8_t *key, uint64_t start_offset, size_t block_size)
 {
     assert(!encrypted);
     this->start_offset = start_offset;
@@ -187,11 +187,14 @@ op_aes_xts_decrypt_t::~op_aes_xts_decrypt_t()
         free(tmp);
 }
 
-void op_aes_xts_decrypt_t::start(const uint8_t *key, uint64_t start_offset, size_t block_size)
+void op_aes_xts_decrypt_t::start(uint8_t **key_chain, size_t chain_size, uint8_t *key_indexes, uint64_t start_offset, size_t block_size)
 {
     assert(!decrypted);
     this->start_offset = start_offset;
-    this->key = key;
+    this->key_chain = chain_size > 1 ? key_chain : 0;
+    this->chain_size = chain_size > 1 ? chain_size : 0;
+    this->key_indexes = chain_size > 1 ? key_indexes : NULL;
+    assert(chain_size <= 1 || key_indexes != NULL);
     this->block_size = block_size;
     this->offset = 0;
     this->tmp_pos = 0;
@@ -202,7 +205,7 @@ void op_aes_xts_decrypt_t::start(const uint8_t *key, uint64_t start_offset, size
         tmp_size = 0;
     }
 #ifdef WITH_OPENSSL
-    if (EVP_DecryptInit_ex(ctx, NULL, NULL, key, NULL) != 1)
+    if (chain_size == 1 && key_chain[0] && EVP_DecryptInit_ex(ctx, NULL, NULL, key_chain[0], NULL) != 1)
     {
         ERR_print_errors_fp(stderr);
         abort();
@@ -212,10 +215,22 @@ void op_aes_xts_decrypt_t::start(const uint8_t *key, uint64_t start_offset, size
 
 void op_aes_xts_decrypt_t::decrypt_block(uint8_t *in, uint8_t *out)
 {
+    uint8_t *key = NULL;
+    if (chain_size)
+    {
+        assert(key_indexes[offset/block_size] < chain_size);
+        key = key_chain[key_indexes[offset/block_size]];
+        if (!key)
+        {
+            if (in != out)
+                memcpy(out, in, block_size);
+            return;
+        }
+    }
 #ifdef WITH_OPENSSL
     uint8_t iv[16] = { 0 };
     *((uint64_t*)iv) = start_offset + offset - offset%block_size;
-    if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) != 1)
+    if (EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv) != 1)
     {
         ERR_print_errors_fp(stderr);
         abort();
@@ -322,8 +337,8 @@ bool osd_messenger_t::op_encrypted_copy_data_to(osd_client_t* cl, uint8_t *enc_b
             else
                 cl->encrypt_ctx = new op_aes_xts_encrypt_t();
         }
-        assert(op->enc->key.size() == 512/8);
+        assert(op->enc->key_chain[0]);
-        cl->encrypt_ctx->start(op->enc->key.data(), op->req.rw.offset, op->enc->bitmap_granularity);
+        cl->encrypt_ctx->start(op->enc->key_chain[0], op->req.rw.offset, op->enc->bitmap_granularity);
     }
     for (int i = 0; i < op->iov.count; i++)
     {
@@ -393,8 +408,10 @@ void osd_messenger_t::op_decrypt_start(osd_client_t* cl)
         }
         else
             cl->decrypt_ctx = new op_aes_xts_decrypt_t();
-        assert(cl->read_op->enc->key.size() == 512/8);
+        auto & enc = cl->read_op->enc;
-        cl->decrypt_ctx->start(cl->read_op->enc->key.data(), cl->read_op->req.rw.offset, cl->read_op->enc->bitmap_granularity);
+        cl->decrypt_ctx->start(enc->key_chain, enc->chain_size,
+            (cl->read_op->req.rw.flags & OSD_OP_RETURN_CHAIN) ? (uint8_t*)cl->read_op->bitmap + enc->read_chain_bitmap_pos : 0,
+            cl->read_op->req.rw.offset, enc->bitmap_granularity);
     }
 }
 

src/client/msgr_encrypt.h

@@ -16,7 +16,7 @@ class op_aes_xts_encrypt_t
     EVP_CIPHER_CTX *ctx = NULL;
 #endif
     uint64_t start_offset = 0;
-    const uint8_t *key = NULL;
+    uint8_t *key = NULL;
     size_t offset = 0;
     size_t block_size = 0;
     uint8_t *tmp = NULL;
@@ -31,7 +31,7 @@ public:
     ~op_aes_xts_encrypt_t();
 
     inline bool has_buffered() { return encrypted; };
-    void start(const uint8_t *key, uint64_t start_offset, size_t block_size);
+    void start(uint8_t *key, uint64_t start_offset, size_t block_size);
     void update(uint8_t *in, size_t max_in, uint8_t *out, size_t max_out, size_t & done_in, size_t & done_out);
 };
 
@@ -43,7 +43,9 @@ class op_aes_xts_decrypt_t
     EVP_CIPHER_CTX *ctx = NULL;
 #endif
     uint64_t start_offset = 0;
-    const uint8_t *key = NULL;
+    uint8_t **key_chain = NULL;
+    size_t chain_size = 0;
+    uint8_t *key_indexes = NULL;
     size_t offset = 0;
     size_t block_size = 0;
     uint8_t *tmp = NULL;
@@ -58,7 +60,7 @@ public:
     ~op_aes_xts_decrypt_t();
 
     inline bool has_buffered() { return decrypted; };
-    void start(const uint8_t *key, uint64_t start_offset, size_t block_size);
+    void start(uint8_t **key_chain, size_t chain_size, uint8_t *key_indexes, uint64_t start_offset, size_t block_size);
     void update(uint8_t *in, size_t max_in, uint8_t *out, size_t max_out, size_t & done_in, size_t & done_out);
 };
 

src/client/msgr_op.h

@@ -154,7 +154,18 @@ struct blockstore_op_t;
 
 struct osd_primary_op_data_t;
 
-struct inode_enc_t;
+struct osd_op_enc_t
+{
+    // Keys may contain more information in the future, like encryption algorithm and key ID
+    // In this case, key_chain will become inode_key_t* with inode_key_t also being a structure
+    // Currently all keys are required to be 512 bit (64 byte) long, for AES-256-XTS
+    // Raw pointers are convenient for messenger code; external users may use shared_ptr aliasing
+    // to implement complex freeing of osd_op_enc_t along with their external inode cache info
+    uint8_t** key_chain = NULL;
+    size_t chain_size = 0;
+    uint32_t read_chain_bitmap_pos = 0;
+    uint32_t bitmap_granularity = 0;
+};
 
 struct __attribute__((visibility("default"))) osd_op_t
 {
@@ -172,7 +183,7 @@ struct __attribute__((visibility("default"))) osd_op_t
     unsigned bmp_data = 0;
     void *bitmap_buf = NULL;
     void *rmw_buf = NULL;
-    std::shared_ptr<inode_enc_t> enc;
+    std::shared_ptr<osd_op_enc_t> enc;
     uint8_t *enc_buf = NULL;
     osd_primary_op_data_t* op_data = NULL;
     std::function<void(osd_op_t*)> callback;

src/test/test_cluster_client.cpp

@@ -589,7 +589,7 @@ void test_msgr_encrypt()
     }
 
     auto dec = new op_aes_xts_decrypt_t();
-    dec->start(key, 4096 * 113, 4096);
+    dec->start(&key, 1, NULL, 4096 * 113, 4096);
     in_pos = out_pos = 0;
     while (out_pos < sz)
     {
@@ -625,7 +625,7 @@ void test_msgr_encrypt()
     // Fragmented decrypt
     // Input: 1000 + 2000 + 3000 + 2192, output: 500 + 3000 + 1000 + 3000 + 692
     printf("...fragmented decrypt\n");
-    dec->start(key, 4096 * 114, 4096);
+    dec->start(&key, 1, NULL, 4096 * 114, 4096);
     in_pos = out_pos = 0;
     dec->update(crypt+4096, 1000, decrypt, 500, in_pos, out_pos);
     assert(in_pos == 1000);
@@ -659,7 +659,7 @@ void test_msgr_encrypt()
     // Extra size decrypt
     // Input: 8192, output: 4096
     printf("...extra size decrypt\n");
-    dec->start(key, 4096 * 114, 4096);
+    dec->start(&key, 1, NULL, 4096 * 114, 4096);
     in_pos = out_pos = 0;
     dec->update(crypt+4096, 8192, decrypt, 4096, in_pos, out_pos);
     assert(in_pos == 4096);
@@ -676,6 +676,63 @@ void test_msgr_encrypt()
     free(src);
     printf("[ok] msgr aes-xts encryption test\n");
 }
+
+void test_msgr_decrypt_chain()
+{
+    const size_t sz = 4096 * 4;
+    uint8_t *src = (uint8_t*)malloc_or_die(sz);
+    for (size_t i = 0; i < sz; i++)
+        src[i] = (i*0x1001) % 256;
+    uint8_t *crypt = (uint8_t*)malloc_or_die(sz);
+    uint8_t *decrypt = (uint8_t*)malloc_or_die(sz);
+    uint8_t *key = (uint8_t*)malloc_or_die(64);
+    RAND_bytes(key, 64);
+    uint8_t *key2 = (uint8_t*)malloc_or_die(64);
+    RAND_bytes(key2, 64);
+
+    // Chained decryption with multiple keys
+
+    // encrypt:
+    size_t in_pos = 0, out_pos = 0;
+    auto enc = new op_aes_xts_encrypt_t();
+    // block 1 with key1
+    enc->start(key, 4096 * 113, 4096);
+    enc->update(src, 4096, crypt, 4096, in_pos, out_pos);
+    assert(in_pos == 4096 && out_pos == 4096);
+    // block 2 as plain
+    memcpy(crypt + 4096, src + 4096, 4096);
+    // block 3 with key2
+    enc->start(key2, 4096 * 115, 4096);
+    enc->update(src + 2*4096, 4096, crypt + 2*4096, 4096, in_pos, out_pos);
+    assert(in_pos == 2*4096 && out_pos == 2*4096);
+    // block 4 again with key1
+    enc->start(key, 4096 * 116, 4096);
+    enc->update(src + 3*4096, 4096, crypt + 3*4096, 4096, in_pos, out_pos);
+    assert(in_pos == 3*4096 && out_pos == 3*4096);
+
+    // decrypt:
+    uint8_t* keys[3] = { key, key2, NULL };
+    uint8_t chain_info[4] = { 0, 2, 1, 0 };
+    auto dec = new op_aes_xts_decrypt_t();
+    dec->start(keys, 3, chain_info, 4096 * 113, 4096);
+    in_pos = out_pos = 0;
+    while (out_pos < sz)
+    {
+        dec->update(crypt+in_pos, sz-in_pos, decrypt+out_pos, sz-out_pos, in_pos, out_pos);
+    }
+
+    assert(memcmp(src, decrypt, sz) == 0);
+
+    delete dec;
+    delete enc;
+
+    free(key2);
+    free(key);
+    free(decrypt);
+    free(crypt);
+    free(src);
+    printf("[ok] msgr aes-xts chained decrypt\n");
+}
 #endif
 
 int main(int narg, char *args[])
@@ -686,6 +743,7 @@ int main(int narg, char *args[])
     test_writeback_merge();
 #ifdef WITH_OPENSSL
     test_msgr_encrypt();
+    test_msgr_decrypt_chain();
 #endif
     return 0;
 }