From 2ba4be470f136939e3e2f5df304fb4876609cc32 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Mon, 23 Feb 2026 15:38:20 +0300 Subject: [PATCH] Support decryption with multiple keys --- src/client/cluster_client.cpp | 4 +- src/client/cluster_client.h | 2 +- src/client/etcd_state_client.cpp | 24 +++++++++--- src/client/etcd_state_client.h | 12 +++--- src/client/msgr_encrypt.cpp | 35 ++++++++++++----- src/client/msgr_encrypt.h | 10 +++-- src/client/msgr_op.h | 15 +++++++- src/test/test_cluster_client.cpp | 64 ++++++++++++++++++++++++++++++-- 8 files changed, 135 insertions(+), 31 deletions(-) diff --git a/src/client/cluster_client.cpp b/src/client/cluster_client.cpp index fd4f0558..0094e470 100644 --- a/src/client/cluster_client.cpp +++ b/src/client/cluster_client.cpp @@ -968,9 +968,9 @@ bool cluster_client_t::check_rw(cluster_op_t *op) ino_it = st_cli.inode_config.find(op->inode); searched = true; } - if (ino_it != st_cli.inode_config.end() && ino_it->second.enc) + if (ino_it != st_cli.inode_config.end() && ino_it->second.enc_key) { - op->enc = ino_it->second.enc; + op->enc = std::shared_ptr(ino_it->second.enc_key, ino_it->second.enc_key->op_enc); if (!op->enc->bitmap_granularity) { op->enc->bitmap_granularity = pool_it->second.bitmap_granularity; diff --git a/src/client/cluster_client.h b/src/client/cluster_client.h index ed861d46..c05b1778 100644 --- a/src/client/cluster_client.h +++ b/src/client/cluster_client.h @@ -71,7 +71,7 @@ protected: cluster_op_t *prev = NULL, *next = NULL; int prev_wait = 0; uint64_t flush_id = 0; - std::shared_ptr enc; + std::shared_ptr enc; friend class cluster_client_t; friend class writeback_cache_t; }; diff --git a/src/client/etcd_state_client.cpp b/src/client/etcd_state_client.cpp index ae46c75e..b1058382 100644 --- a/src/client/etcd_state_client.cpp +++ b/src/client/etcd_state_client.cpp @@ -1,7 +1,9 @@ // Copyright (c) Vitaliy Filippov, 2019+ // License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details) +#include "malloc_or_die.h" #include "osd_ops.h" +#include "msgr_op.h" #include "pg_states.h" #include "etcd_state_client.h" #ifndef __MOCK__ @@ -10,6 +12,14 @@ #endif #include "str_util.h" +inode_key_t::~inode_key_t() +{ + if (op_enc) + { + free(op_enc); + } +} + etcd_state_client_t::~etcd_state_client_t() { for (auto watch: watches) @@ -1307,14 +1317,18 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv) else parent_inode_num |= parent_pool_id << (64-POOL_ID_BITS); } - std::shared_ptr enc; + std::shared_ptr enc_key; if (!value["enc_key"].string_value().empty()) { std::vector k = hexdecode(value["enc_key"].string_value()); - if (k.size() == 512/8) + if (k.size() == 512/8) // AES-256-XTS { - enc = std::make_shared(); - enc->key = std::move(k); + enc_key = std::make_shared(); + enc_key->key = std::move(k); + enc_key->op_enc = (osd_op_enc_t*)calloc_or_die(1, sizeof(osd_op_enc_t) + sizeof(uint8_t*)); + enc_key->op_enc->key_chain = (uint8_t**)(enc_key->op_enc + 1); + enc_key->op_enc->key_chain[0] = enc_key->key.data(); + enc_key->op_enc->chain_size = 1; } } insert_inode_config((inode_config_t){ @@ -1324,7 +1338,7 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv) .parent_id = parent_inode_num, .readonly = value["readonly"].bool_value(), .deleted = value["deleted"].bool_value(), - .enc = enc, + .enc_key = enc_key, .meta = value["meta"], .mod_revision = kv.mod_revision, }); diff --git a/src/client/etcd_state_client.h b/src/client/etcd_state_client.h index a08619f0..92a24ce9 100644 --- a/src/client/etcd_state_client.h +++ b/src/client/etcd_state_client.h @@ -76,12 +76,14 @@ struct pool_config_t void *reshard_state = NULL; }; -struct inode_enc_t +struct osd_op_enc_t; + +struct inode_key_t { - int refs = 0; std::vector key; - // FIXME It may also contain snapshot chain and key information - uint32_t bitmap_granularity = 0; + osd_op_enc_t *op_enc; + + ~inode_key_t(); }; struct inode_config_t @@ -92,7 +94,7 @@ struct inode_config_t inode_t parent_id = 0; bool readonly = false; bool deleted = false; - std::shared_ptr enc; + std::shared_ptr enc_key; // Arbitrary metadata json11::Json meta; // Change revision of the metadata in etcd diff --git a/src/client/msgr_encrypt.cpp b/src/client/msgr_encrypt.cpp index 3fc563ff..d25824ab 100644 --- a/src/client/msgr_encrypt.cpp +++ b/src/client/msgr_encrypt.cpp @@ -37,7 +37,7 @@ op_aes_xts_encrypt_t::~op_aes_xts_encrypt_t() free(tmp); } -void op_aes_xts_encrypt_t::start(const uint8_t *key, uint64_t start_offset, size_t block_size) +void op_aes_xts_encrypt_t::start(uint8_t *key, uint64_t start_offset, size_t block_size) { assert(!encrypted); this->start_offset = start_offset; @@ -187,11 +187,14 @@ op_aes_xts_decrypt_t::~op_aes_xts_decrypt_t() free(tmp); } -void op_aes_xts_decrypt_t::start(const uint8_t *key, uint64_t start_offset, size_t block_size) +void op_aes_xts_decrypt_t::start(uint8_t **key_chain, size_t chain_size, uint8_t *key_indexes, uint64_t start_offset, size_t block_size) { assert(!decrypted); this->start_offset = start_offset; - this->key = key; + this->key_chain = chain_size > 1 ? key_chain : 0; + this->chain_size = chain_size > 1 ? chain_size : 0; + this->key_indexes = chain_size > 1 ? key_indexes : NULL; + assert(chain_size <= 1 || key_indexes != NULL); this->block_size = block_size; this->offset = 0; this->tmp_pos = 0; @@ -202,7 +205,7 @@ void op_aes_xts_decrypt_t::start(const uint8_t *key, uint64_t start_offset, size tmp_size = 0; } #ifdef WITH_OPENSSL - if (EVP_DecryptInit_ex(ctx, NULL, NULL, key, NULL) != 1) + if (chain_size == 1 && key_chain[0] && EVP_DecryptInit_ex(ctx, NULL, NULL, key_chain[0], NULL) != 1) { ERR_print_errors_fp(stderr); abort(); @@ -212,10 +215,22 @@ void op_aes_xts_decrypt_t::start(const uint8_t *key, uint64_t start_offset, size void op_aes_xts_decrypt_t::decrypt_block(uint8_t *in, uint8_t *out) { + uint8_t *key = NULL; + if (chain_size) + { + assert(key_indexes[offset/block_size] < chain_size); + key = key_chain[key_indexes[offset/block_size]]; + if (!key) + { + if (in != out) + memcpy(out, in, block_size); + return; + } + } #ifdef WITH_OPENSSL uint8_t iv[16] = { 0 }; *((uint64_t*)iv) = start_offset + offset - offset%block_size; - if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) != 1) + if (EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv) != 1) { ERR_print_errors_fp(stderr); abort(); @@ -322,8 +337,8 @@ bool osd_messenger_t::op_encrypted_copy_data_to(osd_client_t* cl, uint8_t *enc_b else cl->encrypt_ctx = new op_aes_xts_encrypt_t(); } - assert(op->enc->key.size() == 512/8); - cl->encrypt_ctx->start(op->enc->key.data(), op->req.rw.offset, op->enc->bitmap_granularity); + assert(op->enc->key_chain[0]); + cl->encrypt_ctx->start(op->enc->key_chain[0], op->req.rw.offset, op->enc->bitmap_granularity); } for (int i = 0; i < op->iov.count; i++) { @@ -393,8 +408,10 @@ void osd_messenger_t::op_decrypt_start(osd_client_t* cl) } else cl->decrypt_ctx = new op_aes_xts_decrypt_t(); - assert(cl->read_op->enc->key.size() == 512/8); - cl->decrypt_ctx->start(cl->read_op->enc->key.data(), cl->read_op->req.rw.offset, cl->read_op->enc->bitmap_granularity); + auto & enc = cl->read_op->enc; + cl->decrypt_ctx->start(enc->key_chain, enc->chain_size, + (cl->read_op->req.rw.flags & OSD_OP_RETURN_CHAIN) ? (uint8_t*)cl->read_op->bitmap + enc->read_chain_bitmap_pos : 0, + cl->read_op->req.rw.offset, enc->bitmap_granularity); } } diff --git a/src/client/msgr_encrypt.h b/src/client/msgr_encrypt.h index 9283ffc8..41bf9fbd 100644 --- a/src/client/msgr_encrypt.h +++ b/src/client/msgr_encrypt.h @@ -16,7 +16,7 @@ class op_aes_xts_encrypt_t EVP_CIPHER_CTX *ctx = NULL; #endif uint64_t start_offset = 0; - const uint8_t *key = NULL; + uint8_t *key = NULL; size_t offset = 0; size_t block_size = 0; uint8_t *tmp = NULL; @@ -31,7 +31,7 @@ public: ~op_aes_xts_encrypt_t(); inline bool has_buffered() { return encrypted; }; - void start(const uint8_t *key, uint64_t start_offset, size_t block_size); + void start(uint8_t *key, uint64_t start_offset, size_t block_size); void update(uint8_t *in, size_t max_in, uint8_t *out, size_t max_out, size_t & done_in, size_t & done_out); }; @@ -43,7 +43,9 @@ class op_aes_xts_decrypt_t EVP_CIPHER_CTX *ctx = NULL; #endif uint64_t start_offset = 0; - const uint8_t *key = NULL; + uint8_t **key_chain = NULL; + size_t chain_size = 0; + uint8_t *key_indexes = NULL; size_t offset = 0; size_t block_size = 0; uint8_t *tmp = NULL; @@ -58,7 +60,7 @@ public: ~op_aes_xts_decrypt_t(); inline bool has_buffered() { return decrypted; }; - void start(const uint8_t *key, uint64_t start_offset, size_t block_size); + void start(uint8_t **key_chain, size_t chain_size, uint8_t *key_indexes, uint64_t start_offset, size_t block_size); void update(uint8_t *in, size_t max_in, uint8_t *out, size_t max_out, size_t & done_in, size_t & done_out); }; diff --git a/src/client/msgr_op.h b/src/client/msgr_op.h index 04105c80..e96f65f3 100644 --- a/src/client/msgr_op.h +++ b/src/client/msgr_op.h @@ -154,7 +154,18 @@ struct blockstore_op_t; struct osd_primary_op_data_t; -struct inode_enc_t; +struct osd_op_enc_t +{ + // Keys may contain more information in the future, like encryption algorithm and key ID + // In this case, key_chain will become inode_key_t* with inode_key_t also being a structure + // Currently all keys are required to be 512 bit (64 byte) long, for AES-256-XTS + // Raw pointers are convenient for messenger code; external users may use shared_ptr aliasing + // to implement complex freeing of osd_op_enc_t along with their external inode cache info + uint8_t** key_chain = NULL; + size_t chain_size = 0; + uint32_t read_chain_bitmap_pos = 0; + uint32_t bitmap_granularity = 0; +}; struct __attribute__((visibility("default"))) osd_op_t { @@ -171,7 +182,7 @@ struct __attribute__((visibility("default"))) osd_op_t unsigned bmp_data = 0; void *bitmap_buf = NULL; void *rmw_buf = NULL; - std::shared_ptr enc; + std::shared_ptr enc; uint8_t *enc_buf = NULL; osd_primary_op_data_t* op_data = NULL; std::function callback; diff --git a/src/test/test_cluster_client.cpp b/src/test/test_cluster_client.cpp index 02bba461..7696c83e 100644 --- a/src/test/test_cluster_client.cpp +++ b/src/test/test_cluster_client.cpp @@ -580,7 +580,7 @@ void test_msgr_encrypt() } auto dec = new op_aes_xts_decrypt_t(); - dec->start(key, 4096 * 113, 4096); + dec->start(&key, 1, NULL, 4096 * 113, 4096); in_pos = out_pos = 0; while (out_pos < sz) { @@ -616,7 +616,7 @@ void test_msgr_encrypt() // Fragmented decrypt // Input: 1000 + 2000 + 3000 + 2192, output: 500 + 3000 + 1000 + 3000 + 692 printf("...fragmented decrypt\n"); - dec->start(key, 4096 * 114, 4096); + dec->start(&key, 1, NULL, 4096 * 114, 4096); in_pos = out_pos = 0; dec->update(crypt+4096, 1000, decrypt, 500, in_pos, out_pos); assert(in_pos == 1000); @@ -650,7 +650,7 @@ void test_msgr_encrypt() // Extra size decrypt // Input: 8192, output: 4096 printf("...extra size decrypt\n"); - dec->start(key, 4096 * 114, 4096); + dec->start(&key, 1, NULL, 4096 * 114, 4096); in_pos = out_pos = 0; dec->update(crypt+4096, 8192, decrypt, 4096, in_pos, out_pos); assert(in_pos == 4096); @@ -667,6 +667,63 @@ void test_msgr_encrypt() free(src); printf("[ok] msgr aes-xts encryption test\n"); } + +void test_msgr_decrypt_chain() +{ + const size_t sz = 4096 * 4; + uint8_t *src = (uint8_t*)malloc_or_die(sz); + for (size_t i = 0; i < sz; i++) + src[i] = (i*0x1001) % 256; + uint8_t *crypt = (uint8_t*)malloc_or_die(sz); + uint8_t *decrypt = (uint8_t*)malloc_or_die(sz); + uint8_t *key = (uint8_t*)malloc_or_die(64); + RAND_bytes(key, 64); + uint8_t *key2 = (uint8_t*)malloc_or_die(64); + RAND_bytes(key2, 64); + + // Chained decryption with multiple keys + + // encrypt: + size_t in_pos = 0, out_pos = 0; + auto enc = new op_aes_xts_encrypt_t(); + // block 1 with key1 + enc->start(key, 4096 * 113, 4096); + enc->update(src, 4096, crypt, 4096, in_pos, out_pos); + assert(in_pos == 4096 && out_pos == 4096); + // block 2 as plain + memcpy(crypt + 4096, src + 4096, 4096); + // block 3 with key2 + enc->start(key2, 4096 * 115, 4096); + enc->update(src + 2*4096, 4096, crypt + 2*4096, 4096, in_pos, out_pos); + assert(in_pos == 2*4096 && out_pos == 2*4096); + // block 4 again with key1 + enc->start(key, 4096 * 116, 4096); + enc->update(src + 3*4096, 4096, crypt + 3*4096, 4096, in_pos, out_pos); + assert(in_pos == 3*4096 && out_pos == 3*4096); + + // decrypt: + uint8_t* keys[3] = { key, key2, NULL }; + uint8_t chain_info[4] = { 0, 2, 1, 0 }; + auto dec = new op_aes_xts_decrypt_t(); + dec->start(keys, 3, chain_info, 4096 * 113, 4096); + in_pos = out_pos = 0; + while (out_pos < sz) + { + dec->update(crypt+in_pos, sz-in_pos, decrypt+out_pos, sz-out_pos, in_pos, out_pos); + } + + assert(memcmp(src, decrypt, sz) == 0); + + delete dec; + delete enc; + + free(key2); + free(key); + free(decrypt); + free(crypt); + free(src); + printf("[ok] msgr aes-xts chained decrypt\n"); +} #endif int main(int narg, char *args[]) @@ -677,6 +734,7 @@ int main(int narg, char *args[]) test_writeback_merge(); #ifdef WITH_OPENSSL test_msgr_encrypt(); + test_msgr_decrypt_chain(); #endif return 0; }