From 039c524f0afbc236ccebca7b318797a5895f2cb6 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sun, 26 Apr 2026 11:26:34 +0300 Subject: [PATCH] Implement handshake for AES-256-GCM by capturing TLS secrets (pretty shitty approach actually...) --- src/client/messenger.cpp | 9 -- src/client/messenger.h | 18 ++-- src/client/msgr_encrypt.cpp | 176 +++++++++++++++++++++++++++++++++++- src/client/msgr_receive.cpp | 107 +++++++++++++++------- src/client/msgr_send.cpp | 81 +++++++++++++---- src/util/openssl_util.cpp | 65 +++++++------ src/util/openssl_util.h | 1 - src/util/str_util.cpp | 13 ++- src/util/str_util.h | 1 + tests/common.sh | 3 +- 10 files changed, 368 insertions(+), 106 deletions(-) diff --git a/src/client/messenger.cpp b/src/client/messenger.cpp index 4db3a40b..cb0fce51 100644 --- a/src/client/messenger.cpp +++ b/src/client/messenger.cpp @@ -364,15 +364,6 @@ void osd_messenger_t::parse_config(const json11::Json & config) osd_tls_ca = config["osd_tls_ca"].string_value(); client_tls_ca = config["client_tls_ca"].string_value(); } - test_osd_aes_key.resize(32); - if (fromhexstr(config["test_osd_aes_key"].string_value(), 32, (uint8_t*)test_osd_aes_key.data()) != 32) - test_osd_aes_key.clear(); - else - { -#ifdef WITH_ISAL_CRYPTO - isal_aes_gcm_pre_256(test_osd_aes_key.data(), &test_osd_aes_key_isal); -#endif - } if (!osd_num) this->iothread_count = (uint32_t)config["client_iothread_count"].uint64_value(); else diff --git a/src/client/messenger.h b/src/client/messenger.h index 3f8d3259..d465ac14 100644 --- a/src/client/messenger.h +++ b/src/client/messenger.h @@ -46,6 +46,9 @@ #define DEFAULT_MIN_ZEROCOPY_SEND_SIZE 32*1024 +#define AES_256_GCM_KEY_SIZE 32 +#define AES_256_GCM_IV_SIZE 12 + struct msgr_sendp_t { osd_op_t *op; @@ -98,13 +101,17 @@ struct osd_client_t BIO *read_from_ssl = NULL; uint8_t *ssl_out_buf = NULL; size_t ssl_out_buf_size = 0, ssl_out_buf_cap = 0; - bool ssl_handshake_done = false; + int ssl_handshake_pending = 0; msgr_tls_record_hdr_t ssl_read_record; size_t ssl_read_header_size = 0; bool ssl_more_to_buffer = false; bool gcm_enabled = false; + std::vector my_secret, peer_secret; + std::vector my_key, peer_key; + uint64_t my_iv_ctr, peer_iv_ctr; #ifdef WITH_ISAL_CRYPTO + isal_gcm_key_data my_key_isal, peer_key_isal; isal_gcm_context_data *enc_ctx = NULL; isal_gcm_context_data *dec_ctx = NULL; #else @@ -266,10 +273,6 @@ protected: std::string tls_key; std::string osd_tls_ca; std::string client_tls_ca; - std::string test_osd_aes_key; // FIXME Insecure, only for PoC tests -#ifdef WITH_ISAL_CRYPTO - isal_gcm_key_data test_osd_aes_key_isal; -#endif #ifdef WITH_RDMA bool use_rdma = true; @@ -288,6 +291,7 @@ protected: #endif SSL_CTX *ssl_ctx = NULL; + EVP_KDF_CTX *kdf_ctx = NULL; X509 *tls_cert_obj = NULL; X509 *osd_tls_ca_obj = NULL; X509 *client_tls_ca_obj = NULL; @@ -297,6 +301,8 @@ protected: void destroy_tls(); void init_tls_client(osd_client_t *cl); bool do_tls_handshake(osd_client_t *cl, bool from_recv = false); + bool finalize_tls_handshake(osd_client_t *cl); + bool derive_aes_keys(osd_client_t *cl, bool update_my, bool update_peer); std::vector iothreads; std::vector read_ready_clients; @@ -391,7 +397,7 @@ protected: void handle_read(int result, osd_client_t *cl); bool handle_read_buffer(osd_client_t *cl, uint8_t *curbuf, size_t bufsize); - template bool handle_buffer_with(osd_client_t *cl, uint8_t *curbuf, size_t bufsize); + template size_t handle_buffer_with(osd_client_t *cl, uint8_t *curbuf, size_t bufsize); bool handle_hdr(osd_client_t *cl); bool allocate_op_buffers(osd_client_t *cl); bool allocate_reply_buffers(osd_client_t *cl, osd_op_t *op); diff --git a/src/client/msgr_encrypt.cpp b/src/client/msgr_encrypt.cpp index cf77c2c4..5c17038c 100644 --- a/src/client/msgr_encrypt.cpp +++ b/src/client/msgr_encrypt.cpp @@ -7,6 +7,8 @@ #include #endif +#include + #include "str_util.h" #include "etcd_state_client.h" #include "messenger.h" @@ -14,9 +16,14 @@ #include "http_client.h" #include "openssl_util.h" +#include #include #include +#define MSGR_HSP_HS 1 +#define MSGR_HSP_SEND 2 +#define MSGR_HSP_RECV 4 + op_aes_xts_encrypt_t::op_aes_xts_encrypt_t() { #ifndef WITH_ISAL_CRYPTO @@ -486,6 +493,116 @@ void osd_messenger_t::op_encrypt_free(osd_client_t* cl) } } +struct tls_secrets_t +{ + std::vector client_secret; + std::vector server_secret; +}; + +// Sadly we have to use a global variable to capture TLS 1.3 secrets +static std::mutex logged_secrets_mu; +static std::map logged_secrets; + +static void openssl_key_log(const SSL *ssl, const char *line) +{ + // Format: _TRAFFIC_SECRET_0 + bool is_client_secret = !strncmp(line, "CLIENT_TRAFFIC_SECRET_0 ", strlen("CLIENT_TRAFFIC_SECRET_0 ")); + bool is_server_secret = !strncmp(line, "SERVER_TRAFFIC_SECRET_0 ", strlen("SERVER_TRAFFIC_SECRET_0 ")); + if (!is_client_secret && !is_server_secret) + return; + const char *hex = strchr(line+strlen("CLIENT_TRAFFIC_SECRET_0 "), ' '); + if (!hex) + return; + hex++; + size_t len = strlen(hex); + logged_secrets_mu.lock(); + auto & secrets = logged_secrets[ssl]; + logged_secrets_mu.unlock(); + auto & secret = is_client_secret ? secrets.client_secret : secrets.server_secret; + secret.resize(len/2); + fromhexstr(hex, len, secret.data(), secret.size()); +} + +static bool derive_kdf(EVP_KDF_CTX* kdf_ctx, const uint8_t* insecret, size_t insecret_len, + const uint8_t* salt, size_t salt_len, const char *label, uint8_t *key, size_t size) +{ + OSSL_PARAM params[5]; + int n = 0; + params[n++] = OSSL_PARAM_construct_utf8_string("digest", (char*)"sha384", (size_t)7); + params[n++] = OSSL_PARAM_construct_octet_string("key", (void*)insecret, insecret_len); + params[n++] = OSSL_PARAM_construct_octet_string("info", (void*)label, strlen(label)+1); + if (salt) + params[n++] = OSSL_PARAM_construct_octet_string("salt", (void*)salt, salt_len); + params[n++] = OSSL_PARAM_construct_end(); + assert(n <= sizeof(params)/sizeof(OSSL_PARAM)); + if (EVP_KDF_CTX_set_params(kdf_ctx, params) <= 0) + { + ERR_print_errors_fp(stderr); + return false; + } + if (EVP_KDF_derive(kdf_ctx, key, size, NULL) <= 0) + { + ERR_print_errors_fp(stderr); + return false; + } + return true; +} + +bool osd_messenger_t::derive_aes_keys(osd_client_t *cl, bool update_my, bool update_peer) +{ + std::vector old_my = cl->my_key, old_peer = cl->peer_key; + if (!cl->my_secret.size() || !cl->peer_secret.size()) + { + assert(cl->ssl_cli); + logged_secrets_mu.lock(); + auto & secrets = logged_secrets[cl->ssl_cli]; + cl->my_secret = std::move(cl->is_incoming ? secrets.client_secret : secrets.server_secret); + cl->peer_secret = std::move(!cl->is_incoming ? secrets.client_secret : secrets.server_secret); + logged_secrets.erase(cl->ssl_cli); + logged_secrets_mu.unlock(); + SSL_free(cl->ssl_cli); + cl->ssl_cli = NULL; + cl->gcm_enabled = true; + cl->write_to_ssl = NULL; + cl->read_from_ssl = NULL; + if (cl->my_secret.size() < 32 || cl->peer_secret.size() < 32) + { + fprintf(stderr, "Client %ju error: failed to capture TLS handshake results\n", cl->client_id); + return false; + } + } + // Both keys include AES key and iv + xxhash3 secret + const auto len = AES_256_GCM_KEY_SIZE + AES_256_GCM_IV_SIZE + XXH_SECRET_DEFAULT_SIZE; + cl->my_key.resize(len); + cl->peer_key.resize(len); + bool ok = true; + if (update_my || !old_my.size()) + { + ok = ok && derive_kdf(kdf_ctx, cl->my_secret.data(), cl->my_secret.size(), + old_my.size() ? old_my.data() : NULL, old_my.size(), + cl->is_incoming ? "server key" : "client key", + cl->my_key.data(), len); +#ifdef WITH_ISAL_CRYPTO + if (ok) + isal_aes_gcm_pre_256(cl->my_key.data(), &cl->my_key_isal); +#endif + cl->my_iv_ctr = 0; + } + if (update_peer || !old_peer.size()) + { + ok = ok && derive_kdf(kdf_ctx, cl->peer_secret.data(), cl->peer_secret.size(), + old_peer.size() ? old_peer.data() : NULL, old_peer.size(), + !cl->is_incoming ? "server key" : "client key", + cl->peer_key.data(), len); +#ifdef WITH_ISAL_CRYPTO + if (ok) + isal_aes_gcm_pre_256(cl->peer_key.data(), &cl->peer_key_isal); +#endif + cl->peer_iv_ctr = 0; + } + return ok; +} + void osd_messenger_t::init_tls() { if (!tls_cert.empty() || !tls_key.empty() || !osd_tls_ca.empty() || !client_tls_ca.empty()) @@ -512,6 +629,7 @@ init_err: SSL_CTX_set_min_proto_version(ssl_ctx, TLS1_3_VERSION); SSL_CTX_set_max_proto_version(ssl_ctx, TLS1_3_VERSION); SSL_CTX_set_ciphersuites(ssl_ctx, "TLS_AES_256_GCM_SHA384"); + SSL_CTX_set_keylog_callback(ssl_ctx, openssl_key_log); SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, NULL); bool ok = SSL_CTX_set_min_proto_version(ssl_ctx, TLS1_3_VERSION); ok = ok && (osd_tls_ca_obj = openssl_load_cert(osd_tls_ca)); @@ -524,6 +642,11 @@ init_err: } ok = ok && openssl_ctx_use_cert(ssl_ctx, tls_cert, tls_cn); ok = ok && openssl_ctx_use_key(ssl_ctx, tls_key); + EVP_KDF *kdf; + ok = ok && (kdf = EVP_KDF_fetch(NULL, "hkdf", NULL)); + ok = ok && (kdf_ctx = EVP_KDF_CTX_new(kdf)); + if (kdf) + EVP_KDF_free(kdf); if (!ok) { SSL_CTX_free(ssl_ctx); @@ -541,6 +664,7 @@ void osd_messenger_t::init_tls_client(osd_client_t *cl) cl->write_to_ssl = BIO_new(BIO_s_mem()); cl->read_from_ssl = BIO_new(BIO_s_mem()); cl->ssl_cli = SSL_new(ssl_ctx); + cl->ssl_handshake_pending = MSGR_HSP_HS; if (!cl->ssl_cli) { fprintf(stderr, "OpenSSL initialization failed: %s\n", ERR_error_string(ERR_get_error(), NULL)); @@ -562,12 +686,20 @@ void osd_messenger_t::init_tls_client(osd_client_t *cl) bool osd_messenger_t::do_tls_handshake(osd_client_t *cl, bool from_recv) { - if (cl->ssl_handshake_done) + if (!(cl->ssl_handshake_pending & MSGR_HSP_HS)) return true; int r = SSL_do_handshake(cl->ssl_cli); if (r > 0) { - cl->ssl_handshake_done = true; + // Server-side OpenSSL treats handshake as finalized only when receiving + // the first message, so we transmit 1 byte after connecting and only then + // finalize the handshake + cl->ssl_handshake_pending = MSGR_HSP_SEND|MSGR_HSP_RECV; + if (cl->write_state == 0 && from_recv) + { + cl->write_state = CL_WRITE_READY; + write_ready_clients.push_back(cl->client_id); + } } else { @@ -578,13 +710,44 @@ bool osd_messenger_t::do_tls_handshake(osd_client_t *cl, bool from_recv) cl->io_error = true; return false; } + if (from_recv && cl->write_state == 0 && openssl_bio_nonempty(cl->read_from_ssl)) + { + cl->write_state = CL_WRITE_READY; + write_ready_clients.push_back(cl->client_id); + } } - if (from_recv && cl->write_state == 0 && openssl_bio_nonempty(cl->read_from_ssl)) + return true; +} + +bool osd_messenger_t::finalize_tls_handshake(osd_client_t *cl) +{ + if (cl->ssl_handshake_pending) + return true; + // Capture secrets and switch to direct AES-256-GCM encryption + if (!derive_aes_keys(cl, true, true)) + { + cl->io_error = true; + return false; + } + if (cl->read_op) + { + assert(!cl->read_op_pos); + delete cl->read_op; + cl->read_op = NULL; + } + if (cl->write_op) + { + assert(!cl->write_op_pos); + cl->write_ops.insert(cl->write_ops.begin(), cl->write_op); + cl->write_op = NULL; + } + if (cl->write_state == 0) { cl->write_state = CL_WRITE_READY; write_ready_clients.push_back(cl->client_id); } - return true; + // Switched to direct AES-GCM, stop SSL callers + return false; } void osd_messenger_t::destroy_tls() @@ -623,4 +786,9 @@ void osd_messenger_t::destroy_tls() SSL_CTX_free(ssl_ctx); ssl_ctx = NULL; } + if (kdf_ctx) + { + EVP_KDF_CTX_free(kdf_ctx); + kdf_ctx = NULL; + } } diff --git a/src/client/msgr_receive.cpp b/src/client/msgr_receive.cpp index 67a92cb7..08091f5e 100644 --- a/src/client/msgr_receive.cpp +++ b/src/client/msgr_receive.cpp @@ -15,6 +15,10 @@ #define RDR_XTS 2 #define RDR_NO_CSUM 4 +#define MSGR_HSP_HS 1 +#define MSGR_HSP_SEND 2 +#define MSGR_HSP_RECV 4 + class msgr_op_reader_t { public: @@ -100,6 +104,30 @@ class ssl_op_reader_t: public msgr_op_reader_t size_t bufsize; size_t done; + bool read_ssl(void *buf, size_t & len) + { + int ok = SSL_read_ex(cl->ssl_cli, buf, len, &len); + if (ok > 0) + { + return true; + } + len = 0; + ok = SSL_get_error(cl->ssl_cli, ok); + if (ok == SSL_ERROR_ZERO_RETURN) + { + fprintf(stderr, "Client %ju TLS disconnected\n", cl->client_id); + cl->io_error = true; + return false; + } + else if (ok != 0 && ok != SSL_ERROR_WANT_WRITE && ok != SSL_ERROR_WANT_READ) + { + fprintf(stderr, "Client %ju TLS read error: %s. Disconnecting client\n", cl->client_id, ERR_error_string(ERR_get_error(), NULL)); + cl->io_error = true; + return false; + } + return true; + } + public: ssl_op_reader_t(osd_messenger_t* msgr, osd_client_t* cl, uint8_t *curbuf, size_t bufsize): msgr(msgr), cl(cl), from(cl->read_op_pos), curbuf(curbuf), bufsize(bufsize), done(0) @@ -201,30 +229,31 @@ public: assert(dst != NULL); buffer_again: buffer_encrypted(); - if (!cl->ssl_handshake_done) + if (cl->ssl_handshake_pending) { if (!msgr->do_tls_handshake(cl, true)) return false; + if (cl->ssl_handshake_pending & MSGR_HSP_RECV) + { + uint8_t first_byte = 0; + size_t b = 1; + if (!read_ssl(&first_byte, b)) + return false; + if (!b) + { + if (done < bufsize) + goto buffer_again; + return false; + } + cl->ssl_handshake_pending &= ~MSGR_HSP_RECV; + if (!msgr->finalize_tls_handshake(cl)) + return false; + } } - int ok = SSL_read_ex(cl->ssl_cli, dst+from, n, &n); - if (!ok) + if (!read_ssl(dst+from, n)) { - ok = SSL_get_error(cl->ssl_cli, ok); - if (ok == SSL_ERROR_WANT_READ) - { - if (done < bufsize) - goto buffer_again; - } - else if (ok == SSL_ERROR_ZERO_RETURN) - { - fprintf(stderr, "Client %ju TLS disconnected\n", cl->client_id); - cl->io_error = true; - } - else if (ok != 0 && ok != SSL_ERROR_WANT_WRITE) - { - fprintf(stderr, "Client %ju TLS read error: %s. Disconnecting client\n", cl->client_id, ERR_error_string(ERR_get_error(), NULL)); - cl->io_error = true; - } + if (done < bufsize) + goto buffer_again; return false; } if (cl->read_csum_state && !(flags & RDR_NO_CSUM)) @@ -298,16 +327,15 @@ public: #endif } } - uint8_t iv[12] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 }; #ifdef WITH_ISAL_CRYPTO - int r = isal_aes_gcm_init_256(&msgr->test_osd_aes_key_isal, cl->dec_ctx, iv, NULL, 0); + int r = isal_aes_gcm_init_256(&cl->peer_key_isal, cl->dec_ctx, cl->peer_key.data() + AES_256_GCM_KEY_SIZE, NULL, 0); if (r != 0) { fprintf(stderr, "isal_aes_gcm_init_256 error %d\n", r); abort(); } #else - int r = EVP_DecryptInit_ex(cl->dec_ctx, NULL, NULL, (uint8_t*)msgr->test_osd_aes_key.data(), iv); + int r = EVP_DecryptInit_ex(cl->dec_ctx, NULL, NULL, cl->peer_key.data(), cl->peer_key.data() + AES_256_GCM_KEY_SIZE); if (r != 1) { fprintf(stderr, "DecryptInit error: "); @@ -315,6 +343,9 @@ public: abort(); } #endif + // Increase IV + cl->peer_iv_ctr++; + (*(uint64_t*)(cl->peer_key.data() + AES_256_GCM_KEY_SIZE))++; } bool read(uint8_t *dst, size_t dst_len, int flags) override @@ -364,7 +395,7 @@ public: if (n > bufsize-done) n = bufsize-done; #ifdef WITH_ISAL_CRYPTO - int r = isal_aes_gcm_dec_256_update(&msgr->test_osd_aes_key_isal, cl->dec_ctx, dst+from, curbuf+done, n); + int r = isal_aes_gcm_dec_256_update(&cl->peer_key_isal, cl->dec_ctx, dst+from, curbuf+done, n); assert(!r); #else int actual_out; @@ -404,7 +435,7 @@ public: } #ifdef WITH_ISAL_CRYPTO uint8_t calc_tag[16]; - int r = isal_aes_gcm_dec_256_finalize(&msgr->test_osd_aes_key_isal, cl->dec_ctx, calc_tag, 16); + int r = isal_aes_gcm_dec_256_finalize(&cl->peer_key_isal, cl->dec_ctx, calc_tag, 16); assert(r == 0); if (cl->dec_tag_size > 0) { @@ -745,19 +776,29 @@ void osd_messenger_t::handle_immediate_ops() bool osd_messenger_t::handle_read_buffer(osd_client_t *cl, uint8_t *curbuf, size_t bufsize) { + size_t done; if (cl->ssl_cli) { - return handle_buffer_with(cl, curbuf, bufsize); + done = handle_buffer_with(cl, curbuf, bufsize); + if (done > 0 && done < bufsize && !cl->ssl_cli && cl->gcm_enabled) + { + done += handle_buffer_with(cl, curbuf+done, bufsize-done); + } } else if (cl->gcm_enabled) { - return handle_buffer_with(cl, curbuf, bufsize); + done = handle_buffer_with(cl, curbuf, bufsize); } - return handle_buffer_with(cl, curbuf, bufsize); + else + { + done = handle_buffer_with(cl, curbuf, bufsize); + } + assert(!done || done == bufsize); + return !!done; } template -bool osd_messenger_t::handle_buffer_with(osd_client_t *cl, uint8_t *curbuf, size_t bufsize) +size_t osd_messenger_t::handle_buffer_with(osd_client_t *cl, uint8_t *curbuf, size_t bufsize) { T rdr(this, cl, curbuf, bufsize); // Reset OSD ping state @@ -781,20 +822,22 @@ bool osd_messenger_t::handle_buffer_with(osd_client_t *cl, uint8_t *curbuf, size { if (!cl->read_csum_state) cl->read_csum_state = XXH3_createState(); - XXH3_64bits_reset(cl->read_csum_state); + if (cl->peer_key.size() == AES_256_GCM_KEY_SIZE + AES_256_GCM_IV_SIZE + XXH_SECRET_DEFAULT_SIZE) + XXH3_64bits_reset_withSecret(cl->read_csum_state, cl->peer_key.data() + AES_256_GCM_KEY_SIZE + AES_256_GCM_IV_SIZE, XXH_SECRET_DEFAULT_SIZE); + else + XXH3_64bits_reset(cl->read_csum_state); } if (!op_read_from(cl, rdr) || !handle_finished_op(cl)) { if (cl->io_error) { stop_client(cl->client_id); - return false; + return 0; } break; } } - assert(rdr.get_done() == bufsize); - return true; + return rdr.get_done(); } bool osd_messenger_t::handle_hdr(osd_client_t *cl) diff --git a/src/client/msgr_send.cpp b/src/client/msgr_send.cpp index 9ecad1da..fe98010d 100644 --- a/src/client/msgr_send.cpp +++ b/src/client/msgr_send.cpp @@ -16,6 +16,10 @@ #define WR_XTS 2 #define WR_NO_CSUM 4 +#define MSGR_HSP_HS 1 +#define MSGR_HSP_SEND 2 +#define MSGR_HSP_RECV 4 + class msgr_op_writer_t { public: @@ -111,11 +115,25 @@ public: bool flush_ssl() { - if (!cl->ssl_handshake_done) + if (cl->ssl_handshake_pending) { if (!msgr->do_tls_handshake(cl)) return false; - return _flush_ssl(); + if (cl->ssl_handshake_pending & MSGR_HSP_SEND) + { + uint8_t first_byte = 0; + size_t f = 0; + if (!write_to_ssl(cl, &first_byte, 1, 0, f)) + return false; + cl->write_op_pos--; + if (!_flush_ssl()) + return false; + cl->ssl_handshake_pending &= ~MSGR_HSP_SEND; + if (!msgr->finalize_tls_handshake(cl)) + return false; + } + else if (!_flush_ssl()) + return false; } return true; } @@ -202,12 +220,12 @@ public: } else { - if (!cl->ssl_handshake_done) + if (cl->ssl_handshake_pending) { - if (!flush_ssl()) + if (!_flush_ssl()) return false; } - if (cl->ssl_handshake_done) + if (!cl->ssl_handshake_pending) { if (!write_to_ssl(cl, src, src_len, flags, from)) return false; @@ -282,16 +300,15 @@ public: #endif } } - uint8_t iv[12] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 }; #ifdef WITH_ISAL_CRYPTO - int r = isal_aes_gcm_init_256(&msgr->test_osd_aes_key_isal, cl->enc_ctx, iv, NULL, 0); + int r = isal_aes_gcm_init_256(&cl->my_key_isal, cl->enc_ctx, cl->my_key.data() + AES_256_GCM_KEY_SIZE, NULL, 0); if (r != 0) { fprintf(stderr, "isal_aes_gcm_init_256 error %d\n", r); abort(); } #else - int r = EVP_EncryptInit_ex(cl->enc_ctx, NULL, NULL, (uint8_t*)msgr->test_osd_aes_key.data(), iv); + int r = EVP_EncryptInit_ex(cl->enc_ctx, NULL, NULL, (uint8_t*)cl->my_key.data(), cl->my_key.data() + AES_256_GCM_KEY_SIZE); if (r != 1) { fprintf(stderr, "EncryptInit error: "); @@ -299,6 +316,9 @@ public: abort(); } #endif + // Increase IV + cl->my_iv_ctr++; + (*(uint64_t*)(cl->my_key.data() + AES_256_GCM_KEY_SIZE))++; } static void free_ctx(osd_messenger_t* msgr, osd_client_t *cl) @@ -352,7 +372,7 @@ public: if (!n) return false; #ifdef WITH_ISAL_CRYPTO - int r = isal_aes_gcm_enc_256_update(&msgr->test_osd_aes_key_isal, cl->enc_ctx, curbuf+done, src+from, n); + int r = isal_aes_gcm_enc_256_update(&cl->my_key_isal, cl->enc_ctx, curbuf+done, src+from, n); assert(!r); #else int actual_out; @@ -379,7 +399,7 @@ public: static void write_tag_to(osd_messenger_t *msgr, osd_client_t *cl, uint8_t *dst) { #ifdef WITH_ISAL_CRYPTO - int r = isal_aes_gcm_enc_256_finalize(&msgr->test_osd_aes_key_isal, cl->enc_ctx, dst, 16); + int r = isal_aes_gcm_enc_256_finalize(&cl->my_key_isal, cl->enc_ctx, dst, 16); assert(!r); #else int actual_out = 0; @@ -517,11 +537,28 @@ public: bool flush_ssl() { - if (cl->ssl_cli && !cl->ssl_handshake_done) + if (cl->ssl_cli && cl->ssl_handshake_pending) { if (!msgr->do_tls_handshake(cl)) return false; - copy_ssl(); + if (cl->ssl_handshake_pending & MSGR_HSP_SEND) + { + uint8_t first_byte = 0; + size_t f = 0; + if (!ssl_op_writer_t::write_to_ssl(cl, &first_byte, 1, 0, f)) + return false; + cl->write_op_pos--; + copy_ssl(); + cl->ssl_handshake_pending &= ~MSGR_HSP_SEND; + if (!msgr->finalize_tls_handshake(cl)) + { + if (cl->gcm_enabled) + return true; + return false; + } + } + else + copy_ssl(); } return true; } @@ -542,12 +579,14 @@ public: { if (cl->ssl_cli) { - if (!cl->ssl_handshake_done) + if (cl->ssl_handshake_pending) { if (!flush_ssl()) return false; + if (cl->gcm_enabled) + goto try_gcm; } - if (cl->ssl_handshake_done) + if (!cl->ssl_handshake_pending) { if (!ssl_op_writer_t::write_to_ssl(cl, src, src_len, flags, from)) return false; @@ -559,13 +598,14 @@ public: from = 0; return true; } - else if (cl->enc_ctx) +try_gcm: + if (cl->gcm_enabled) { // Encrypt data to client's temporary output buffer (all at once) size_t n = src_len-from; ssl_extend_buf(n); #ifdef WITH_ISAL_CRYPTO - int r = isal_aes_gcm_enc_256_update(&msgr->test_osd_aes_key_isal, cl->enc_ctx, cl->ssl_out_buf+cl->ssl_out_buf_size, src+from, n); + int r = isal_aes_gcm_enc_256_update(&cl->my_key_isal, cl->enc_ctx, cl->ssl_out_buf+cl->ssl_out_buf_size, src+from, n); assert(!r); #else int actual_out; @@ -840,7 +880,9 @@ size_t osd_messenger_t::copy_ops_to(osd_client_t *cl, uint8_t *dst, size_t dst_l { if (cl->ssl_cli) { - return copy_ops_to_with(cl, dst, dst_len); + size_t done = copy_ops_to_with(cl, dst, dst_len); + if (done > 0 || cl->ssl_cli || !cl->gcm_enabled) + return done; } if (cl->gcm_enabled) { @@ -891,7 +933,10 @@ void osd_messenger_t::next_write_op(osd_client_t *cl) { if (!cl->write_csum_state) cl->write_csum_state = XXH3_createState(); - XXH3_64bits_reset(cl->write_csum_state); + if (cl->my_key.size() == AES_256_GCM_KEY_SIZE + AES_256_GCM_IV_SIZE + XXH_SECRET_DEFAULT_SIZE) + XXH3_64bits_reset_withSecret(cl->write_csum_state, cl->my_key.data() + AES_256_GCM_KEY_SIZE + AES_256_GCM_IV_SIZE, XXH_SECRET_DEFAULT_SIZE); + else + XXH3_64bits_reset(cl->write_csum_state); } } diff --git a/src/util/openssl_util.cpp b/src/util/openssl_util.cpp index e566d547..b9900b5e 100644 --- a/src/util/openssl_util.cpp +++ b/src/util/openssl_util.cpp @@ -24,24 +24,6 @@ X509 *openssl_load_cert(const std::string & file_or_pem) return x509; } -EVP_PKEY *openssl_load_key(const std::string & file_or_pem) -{ - std::string pem; - BIO *bio = NULL; - if (file_or_pem.substr(0, 5) != "-----") - { - pem = read_file(file_or_pem); - bio = BIO_new_mem_buf(pem.data(), pem.size()); - } - else - bio = BIO_new_mem_buf(file_or_pem.data(), file_or_pem.size()); - if (!bio) - return NULL; - EVP_PKEY *pkey = PEM_read_bio_PrivateKey(bio, NULL, NULL, NULL); - BIO_free(bio); - return pkey; -} - bool openssl_ctx_add_ca(SSL_CTX *ssl_ctx, const std::string & file_or_pem) { X509 *cert = openssl_load_cert(file_or_pem); @@ -81,27 +63,50 @@ std::string openssl_get_cn(X509 *x509) bool openssl_ctx_use_cert(SSL_CTX *ssl_ctx, const std::string & file_or_pem, std::string & common_name) { - X509 *cert = openssl_load_cert(file_or_pem); - bool ok = false; - if (cert) + BIO *bio = NULL; + std::string contents; + if (file_or_pem.substr(0, 5) == "-----") + bio = BIO_new_mem_buf(file_or_pem.data(), file_or_pem.size()); + else { - common_name = openssl_get_cn(cert); - ok = SSL_CTX_use_certificate(ssl_ctx, cert); - X509_free(cert); + contents = read_file(file_or_pem); + if (!contents.size()) + return false; + bio = BIO_new_mem_buf(contents.data(), contents.size()); } + if (!bio) + return false; + X509 *x509 = PEM_read_bio_X509(bio, NULL, 0, NULL); + bool ok = !!x509; + if (x509) + { + ok = SSL_CTX_use_certificate(ssl_ctx, x509); + if (ok) + common_name = openssl_get_cn(x509); + X509_free(x509); + } + BIO_free(bio); return ok; } bool openssl_ctx_use_key(SSL_CTX *ssl_ctx, const std::string & file_or_pem) { - EVP_PKEY *pkey = openssl_load_key(file_or_pem); - bool ok = false; - if (pkey) + if (file_or_pem.substr(0, 5) == "-----") { - ok = SSL_CTX_use_PrivateKey(ssl_ctx, pkey); - EVP_PKEY_free(pkey); + BIO *bio = BIO_new_mem_buf(file_or_pem.data(), file_or_pem.size()); + if (!bio) + return false; + EVP_PKEY *pkey = PEM_read_bio_PrivateKey(bio, NULL, NULL, NULL); + bool ok = !!pkey; + if (pkey) + { + ok = SSL_CTX_use_PrivateKey(ssl_ctx, pkey); + EVP_PKEY_free(pkey); + } + BIO_free(bio); + return ok; } - return ok; + return !!SSL_CTX_use_PrivateKey_file(ssl_ctx, file_or_pem.c_str(), SSL_FILETYPE_PEM); } bool openssl_bio_nonempty(BIO *bio) diff --git a/src/util/openssl_util.h b/src/util/openssl_util.h index 93f055c5..dc0e20de 100644 --- a/src/util/openssl_util.h +++ b/src/util/openssl_util.h @@ -10,7 +10,6 @@ #endif X509 *openssl_load_cert(const std::string & file_or_pem); -EVP_PKEY *openssl_load_key(const std::string & file_or_pem); bool openssl_ctx_add_ca(SSL_CTX *ssl_ctx, const std::string & file_or_pem); bool openssl_ctx_use_ca(SSL_CTX *ssl_ctx, const std::string & file_or_pem); std::string openssl_get_cn(X509 *x509); diff --git a/src/util/str_util.cpp b/src/util/str_util.cpp index 41a9587d..5002c39f 100644 --- a/src/util/str_util.cpp +++ b/src/util/str_util.cpp @@ -535,12 +535,12 @@ std::string urldecode(const std::string & orig) return res; } -size_t fromhexstr(const std::string & from, size_t bytes, uint8_t *to) +size_t fromhexstr(const char *from, size_t from_len, uint8_t *to, size_t to_len) { - if (bytes > from.size()/2) - bytes = from.size()/2; + if (to_len > from_len/2) + to_len = from_len/2; size_t i = 0; - while (i < bytes) + while (i < to_len) { uint8_t x = fromhexchar(from[2*i], 16); uint8_t y = fromhexchar(from[2*i+1], 16); @@ -552,6 +552,11 @@ size_t fromhexstr(const std::string & from, size_t bytes, uint8_t *to) return i; } +size_t fromhexstr(const std::string & from, size_t bytes, uint8_t *to) +{ + return fromhexstr(from.data(), from.size(), to, bytes); +} + std::string tohexstr(const uint8_t *from, size_t bytes) { std::string res; diff --git a/src/util/str_util.h b/src/util/str_util.h index 5b558c01..f93e987b 100644 --- a/src/util/str_util.h +++ b/src/util/str_util.h @@ -35,6 +35,7 @@ std::string realpath_str(std::string path, bool nofail = true); std::string format_datetime(uint64_t unixtime); bool is_zero(void *buf, size_t size); std::string urldecode(const std::string & orig); +size_t fromhexstr(const char *from, size_t from_len, uint8_t *to, size_t to_len); size_t fromhexstr(const std::string & from, size_t bytes, uint8_t *to); std::string tohexstr(const uint8_t *from, size_t bytes); bool ishexstr(const std::string & str); diff --git a/tests/common.sh b/tests/common.sh index cde05510..0df8b0a3 100644 --- a/tests/common.sh +++ b/tests/common.sh @@ -27,7 +27,7 @@ ETCD_COUNT=${ETCD_COUNT:-1} ANTIETCD=${ANTIETCD} USE_RAMDISK=${USE_RAMDISK} ETCD_SCHEME=${ETCD_SCHEME:-http} -OSD_TLS=${OSD_TLS} +OSD_TLS=${OSD_TLS:-1} RAMDISK=/run/user/$(id -u) findmnt $RAMDISK >/dev/null || (sudo mkdir -p $RAMDISK && sudo mount -t tmpfs tmpfs $RAMDISK) @@ -142,7 +142,6 @@ if [[ "$OSD_TLS" = "1" ]]; then VITASTOR_CFG="$VITASTOR_CFG"',"tls_cert":"'$(pwd)'/testdata/cli.crt"' VITASTOR_CFG="$VITASTOR_CFG"',"tls_key":"'$(pwd)'/testdata/cli.key"' fi -VITASTOR_CFG="$VITASTOR_CFG"',"test_osd_aes_key":"'$(openssl rand -hex 32)'"' echo "{$VITASTOR_CFG}" > ./testdata/vitastor.conf VITASTOR_CFG=./testdata/vitastor.conf VITASTOR_CLI="build/src/cmd/vitastor-cli --config_path $VITASTOR_CFG"