thebestvibedev/tromcho.net
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement TLS 1.3-like handshake manually

Browse Source
This commit is contained in:
Vitaliy Filippov
2026-07-05 14:58:24 +03:00
parent fdeacdcf30
commit 3afed2473e
12 changed files with 1086 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/client/CMakeLists.txt
+1
View File
@@ -11,6 +11,7 @@ add_library(vitastor_common STATIC
osd_ops.cpp
pg_states.cpp
msgr_encrypt.cpp
msgr_handshake.cpp
../util/allocator.cpp
../util/addr_util.cpp
../util/timerfd_manager.cpp
src/client/messenger.cpp
+1 -9
View File
@@ -240,6 +240,7 @@ void osd_messenger_t::parse_config(const json11::Json & config)
this->use_proto_checksums = config["proto_checksums"].string_value() == "full" ? MSGR_CSUM_FULL : MSGR_CSUM_PAYLOAD;
else
this->use_proto_checksums = 0;
gcm_enabled = true;
if (!osd_num)
{
tls_cert = config["tls_cert"].string_value();
@@ -253,15 +254,6 @@ void osd_messenger_t::parse_config(const json11::Json & config)
osd_tls_ca = config["osd_tls_ca"].string_value();
client_tls_ca = config["client_tls_ca"].string_value();
}
test_osd_aes_key.resize(32);
if (fromhexstr(config["test_osd_aes_key"].string_value(), 32, (uint8_t*)test_osd_aes_key.data()) != 32)
test_osd_aes_key.clear();
else
{
#ifdef WITH_ISAL_CRYPTO
isal_aes_gcm_pre_256(test_osd_aes_key.data(), &test_osd_aes_key_isal);
#endif
}
if (!osd_num)
this->iothread_count = (uint32_t)config["client_iothread_count"].uint64_value();
else
src/client/messenger.h
+14 -5
View File
@@ -25,6 +25,7 @@
#include "msgr_op.h"
#include "timerfd_manager.h"
#include "addr_util.h"
#include "msgr_handshake.h"
#include <ringloop.h>
#define CL_READ_HDR 1
@@ -46,6 +47,9 @@
#define DEFAULT_MIN_ZEROCOPY_SEND_SIZE 32*1024
#define AES_256_GCM_KEY_SIZE 32
#define AES_256_GCM_IV_SIZE 12
#define MAX_SIMPLE_PAYLOAD_SIZE 1048576
struct msgr_sendp_t
@@ -100,13 +104,18 @@ struct osd_client_t
BIO *read_from_ssl = NULL;
uint8_t *ssl_out_buf = NULL;
size_t ssl_out_buf_size = 0, ssl_out_buf_cap = 0;
bool ssl_handshake_done = false;
bool handshake_done = false;
msgr_tls_record_hdr_t ssl_read_record;
size_t ssl_read_header_size = 0;
bool ssl_more_to_buffer = false;
bool gcm_enabled = false;
msgr_handshake_i *hs = NULL;
msgr_handshake_result_t hs_result;
std::vector<uint8_t> my_key, peer_key;
uint64_t my_iv_ctr = 0, peer_iv_ctr = 0;
#ifdef WITH_ISAL_CRYPTO
isal_gcm_key_data my_key_isal, peer_key_isal;
isal_gcm_context_data *enc_ctx = NULL;
isal_gcm_context_data *dec_ctx = NULL;
#else
@@ -232,10 +241,6 @@ protected:
std::string tls_key;
std::string osd_tls_ca;
std::string client_tls_ca;
std::string test_osd_aes_key; // FIXME Insecure, only for PoC tests
#ifdef WITH_ISAL_CRYPTO
isal_gcm_key_data test_osd_aes_key_isal;
#endif
#ifdef WITH_RDMA
bool use_rdma = true;
@@ -259,10 +264,14 @@ protected:
X509 *client_tls_ca_obj = NULL;
std::string tls_cn;
bool gcm_enabled = false;
msgr_handshake_ctx_i *hs_ctx = NULL;
void init_tls();
void destroy_tls();
void init_tls_client(osd_client_t *cl);
bool do_tls_handshake(osd_client_t *cl, bool from_recv = false);
bool derive_aes_keys(osd_client_t *cl, bool update_my, bool update_peer);
std::vector<msgr_iothread_t*> iothreads;
std::vector<uint64_t> read_ready_clients;
src/client/msgr_encrypt.cpp
+71 -4
View File
@@ -486,6 +486,43 @@ void osd_messenger_t::op_encrypt_free(osd_client_t* cl)
}
}
bool osd_messenger_t::derive_aes_keys(osd_client_t *cl, bool update_my, bool update_peer)
{
if (!cl->hs_result.shared_secret.size())
{
cl->hs_result = cl->hs->get_result();
}
std::vector<uint8_t> old_my = cl->my_key, old_peer = cl->peer_key;
// Both keys include AES key and iv + xxhash3 secret
const auto len = AES_256_GCM_KEY_SIZE + AES_256_GCM_IV_SIZE + XXH_SECRET_DEFAULT_SIZE;
cl->my_key.resize(len);
cl->peer_key.resize(len);
bool ok = true;
if (update_my || !old_my.size())
{
ok = ok && hs_ctx->derive_kdf(cl->hs_result.shared_secret.data(), cl->hs_result.shared_secret.size(),
old_my.size() ? old_my.data() : NULL, old_my.size(),
cl->is_incoming ? "server key" : "client key", cl->my_key.data(), len);
#ifdef WITH_ISAL_CRYPTO
if (ok)
isal_aes_gcm_pre_256(cl->my_key.data(), &cl->my_key_isal);
#endif
cl->my_iv_ctr = 0;
}
if (update_peer || !old_peer.size())
{
ok = ok && hs_ctx->derive_kdf(cl->hs_result.shared_secret.data(), cl->hs_result.shared_secret.size(),
old_peer.size() ? old_peer.data() : NULL, old_peer.size(),
!cl->is_incoming ? "server key" : "client key", cl->peer_key.data(), len);
#ifdef WITH_ISAL_CRYPTO
if (ok)
isal_aes_gcm_pre_256(cl->peer_key.data(), &cl->peer_key_isal);
#endif
cl->peer_iv_ctr = 0;
}
return ok;
}
void osd_messenger_t::init_tls()
{
if (!tls_cert.empty() || !tls_key.empty() || !osd_tls_ca.empty() || !client_tls_ca.empty())
@@ -499,7 +536,7 @@ void osd_messenger_t::init_tls()
fprintf(stderr, "Vitastor client TLS requires tls_cert, tls_key and osd_tls_ca\n");
exit(1);
}
else
else if (!gcm_enabled)
{
ssl_ctx = SSL_CTX_new(TLS_method());
if (!ssl_ctx)
@@ -531,12 +568,37 @@ init_err:
goto init_err;
}
}
else
{
#ifndef __MOCK__
hs_ctx = msgr_handshake_ctx_i::create_ctx();
if (!hs_ctx->init(tls_cert, tls_key, osd_tls_ca, client_tls_ca))
{
fprintf(stderr, "Error: %s\n", hs_ctx->get_error().c_str());
exit(1);
}
#endif
}
}
}
void osd_messenger_t::init_tls_client(osd_client_t *cl)
{
if (!tls_cert.empty())
if (gcm_enabled)
{
cl->gcm_enabled = true;
cl->hs = hs_ctx->create();
cl->hs->init(cl->is_incoming);
if (cl->hs->get_out().size())
{
if (cl->write_state == 0)
{
cl->write_state = CL_WRITE_READY;
write_ready_clients.push_back(cl->client_id);
}
}
}
else if (!tls_cert.empty())
{
cl->write_to_ssl = BIO_new(BIO_s_mem());
cl->read_from_ssl = BIO_new(BIO_s_mem());
@@ -562,12 +624,12 @@ void osd_messenger_t::init_tls_client(osd_client_t *cl)
bool osd_messenger_t::do_tls_handshake(osd_client_t *cl, bool from_recv)
{
if (cl->ssl_handshake_done)
if (cl->handshake_done)
return true;
int r = SSL_do_handshake(cl->ssl_cli);
if (r > 0)
{
cl->ssl_handshake_done = true;
cl->handshake_done = true;
}
else
{
@@ -623,4 +685,9 @@ void osd_messenger_t::destroy_tls()
SSL_CTX_free(ssl_ctx);
ssl_ctx = NULL;
}
if (hs_ctx)
{
delete hs_ctx;
hs_ctx = NULL;
}
}
src/client/msgr_handshake.cpp
+765
View File
@@ -0,0 +1,765 @@
// Copyright (c) Vitaliy Filippov, 2026+
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
#include <stdint.h>
#include <assert.h>
#include <string>
#include <vector>
#include <memory>
#include <openssl/conf.h>
#include <openssl/evp.h>
#include <openssl/obj_mac.h>
#include <openssl/ec.h>
#include <openssl/kdf.h>
#include <openssl/bio.h>
#include <openssl/pem.h>
#include <openssl/err.h>
#include "msgr_handshake.h"
#include "openssl_util.h"
#include "str_util.h"
#define AES_256_GCM_KEY_SIZE 32
#define AES_256_GCM_IV_SIZE 12
// TLS 1.3-like handshake
// 1. Client->server: EC public key
// 2. Server->client: EC public key, encrypted certificate and digital signature of the handshake
// 3. Client->server: encrypted certificate and digital signature of the handshake
// "vitaECDH" interleaved
#define MSGR_HS_MAGIC 0x4861447443694576l
#define MSGR_HS_MAX_LEN 131072
#define MSGR_HS_SERVER_INIT 0
#define MSGR_HS_CLIENT_INIT 1
#define MSGR_HS_SERVER_REPLY 2
#define MSGR_HS_CLIENT_REPLY 3
#define MSGR_HS_DONE 100
#define MSGR_HS_ERROR 101
struct __attribute__((__packed__)) msgr_handshake_hdr_t
{
uint32_t msg_len;
uint64_t magic;
uint32_t type;
};
class msgr_handshake_ctx_t: public msgr_handshake_ctx_i
{
friend class msgr_handshake_t;
EVP_PKEY_CTX *pctx = NULL;
EVP_PKEY *params = NULL;
X509_STORE *ca = NULL;
X509 *osd_ca = NULL;
X509 *client_ca = NULL;
std::string my_cert_pem;
X509 *my_cert = NULL;
EVP_PKEY *my_pubkey = NULL;
const EVP_MD *md = NULL;
EVP_PKEY *my_privkey = NULL;
EVP_KDF_CTX* kdf_ctx = NULL;
std::string error;
bool on_error(const std::string & prefix);
public:
~msgr_handshake_ctx_t();
msgr_handshake_i* create() override;
bool init(const std::string & pem_cert, const std::string & pem_key,
const std::string & pem_osd_ca, const std::string & pem_client_ca) override;
std::string get_error() override;
bool derive_kdf(const uint8_t* insecret, size_t insecret_len,
const uint8_t* salt, size_t salt_len, const char *label, uint8_t *outsecret, size_t outsize) override;
};
class msgr_handshake_t: public msgr_handshake_i
{
msgr_handshake_ctx_t *ctx = NULL;
bool is_server = false;
std::vector<uint8_t> full_handshake;
std::vector<uint8_t> in_buf;
std::vector<uint8_t> out_buf;
std::string error;
int state = 0;
EVP_PKEY *ec_key = NULL;
X509 *peer_cert = NULL;
bool peer_is_osd = false;
std::vector<uint8_t> shared_secret;
std::vector<uint8_t> hs_key, peer_hs_key;
msgr_handshake_hdr_t *cur_hdr = NULL;
uint8_t *cur_buf = NULL;
size_t cur_left = 0;
bool on_error(const std::string & prefix);
bool derive_shared_secret(EVP_PKEY *peer_ec_key);
bool derive_hs_keys(const uint8_t *encoded_peer_key, size_t encoded_key_len);
bool sign(std::vector<uint8_t> & out);
bool verify(const uint8_t *signature, size_t signature_len);
bool encrypt(const uint8_t* src, size_t len, uint8_t* dest);
bool decrypt(const uint8_t* src, size_t & len, uint8_t* dest);
bool make_client_init();
bool make_server_reply();
bool make_client_reply();
bool verify_peer(const uint8_t *peer_cert_pem, size_t peer_cert_len);
ssize_t start_msg(uint8_t* src, size_t len, uint32_t expected_type);
bool read_with_len(const uint8_t* & dst, uint32_t & dst_len);
bool handle_client_init();
bool handle_server_reply();
bool handle_client_reply();
bool handle_peer_cert(const uint8_t *key, uint32_t key_len);
void complete();
public:
// Workflow: create -> init -> handle -> get_result/get_error -> destruct
msgr_handshake_t(msgr_handshake_ctx_t *ctx): ctx(ctx) {}
~msgr_handshake_t();
bool init(bool server_mode) override;
ssize_t handle(uint8_t* in_buf, size_t in_size) override;
bool done() override;
std::vector<uint8_t>& get_out() override;
msgr_handshake_result_t get_result() override;
std::string get_error() override;
};
msgr_handshake_ctx_i* msgr_handshake_ctx_i::create_ctx()
{
return new msgr_handshake_ctx_t();
}
msgr_handshake_i* msgr_handshake_ctx_t::create()
{
return new msgr_handshake_t(this);
}
bool msgr_handshake_ctx_t::init(const std::string & pem_cert, const std::string & pem_key,
const std::string & pem_osd_ca, const std::string & pem_client_ca)
{
if (pem_cert.substr(0, 5) == "-----")
my_cert_pem = pem_cert;
else
{
my_cert_pem = read_file(pem_cert);
if (my_cert_pem.empty())
{
error = "Failed to load certificate file";
return false;
}
}
{
BIO *bio = BIO_new_mem_buf(my_cert_pem.data(), my_cert_pem.size());
if (!bio)
return on_error("BIO_new_mem_buf: ");
my_cert = PEM_read_bio_X509(bio, NULL, 0, NULL);
BIO_free(bio);
if (!my_cert)
return on_error("Failed to load certificate: ");
}
if (!(my_pubkey = X509_get0_pubkey(my_cert)))
return on_error("X509_get0_pubkey: ");
if (!(md = EVP_get_digestbynid(NID_sha384)))
return on_error("EVP_get_digestbynid SHA384: ");
if (!(my_privkey = openssl_load_key(pem_key)))
return on_error("Failed to load private key: ");
if (!(ca = X509_STORE_new()))
return on_error("X509_STORE_CTX_new: ");
if (!(osd_ca = openssl_load_cert(pem_osd_ca)))
return on_error("Failed to load OSD CA certificate: ");
if (X509_STORE_add_cert(ca, osd_ca) <= 0)
return on_error("X509_STORE_add_cert OSD CA: ");
if (!pem_client_ca.empty() && !(client_ca = openssl_load_cert(pem_client_ca)))
return on_error("Failed to load client CA certificate: ");
if (client_ca && X509_STORE_add_cert(ca, client_ca) <= 0)
return on_error("X509_STORE_add_cert client CA: ");
if (!(pctx = EVP_PKEY_CTX_new_id(EVP_PKEY_EC, NULL)))
return on_error("EVP_PKEY_CTX_new_id: ");
if (EVP_PKEY_paramgen_init(pctx) <= 0)
return on_error("EVP_PKEY_paramgen_init: ");
if (EVP_PKEY_CTX_set_ec_paramgen_curve_nid(pctx, /*NID_X9_62_prime256v1*/NID_secp384r1) <= 0)
return on_error("EVP_PKEY_CTX_set_ec_paramgen_curve_nid: ");
if (EVP_PKEY_paramgen(pctx, &params) <= 0)
return on_error("EVP_PKEY_paramgen: ");
EVP_KDF *kdf = EVP_KDF_fetch(NULL, "hkdf", NULL);
if (!kdf)
return on_error("EVP_KDF_fetch: ");
kdf_ctx = EVP_KDF_CTX_new(kdf);
EVP_KDF_free(kdf);
return true;
}
std::string msgr_handshake_ctx_t::get_error()
{
return error;
}
msgr_handshake_ctx_t::~msgr_handshake_ctx_t()
{
if (pctx)
EVP_PKEY_CTX_free(pctx);
if (params)
EVP_PKEY_free(params);
if (ca)
X509_STORE_free(ca);
if (osd_ca)
X509_free(osd_ca);
if (client_ca)
X509_free(client_ca);
my_pubkey = NULL;
if (my_cert)
X509_free(my_cert);
if (my_privkey)
EVP_PKEY_free(my_privkey);
if (kdf_ctx)
EVP_KDF_CTX_free(kdf_ctx);
}
bool msgr_handshake_ctx_t::on_error(const std::string & prefix)
{
error = prefix+ERR_error_string(ERR_get_error(), NULL);
return false;
}
bool msgr_handshake_ctx_t::derive_kdf(const uint8_t* insecret, size_t insecret_len,
const uint8_t* salt, size_t salt_len, const char *label, uint8_t *outsecret, size_t outsize)
{
OSSL_PARAM params[5];
int n = 0;
params[n++] = OSSL_PARAM_construct_utf8_string("digest", (char*)"sha384", (size_t)7);
params[n++] = OSSL_PARAM_construct_octet_string("key", (void*)insecret, insecret_len);
params[n++] = OSSL_PARAM_construct_octet_string("info", (void*)label, strlen(label)+1);
params[n++] = OSSL_PARAM_construct_octet_string("salt", (salt ? (void*)salt : (void*)""), salt_len);
params[n++] = OSSL_PARAM_construct_end();
assert(n <= sizeof(params)/sizeof(OSSL_PARAM));
if (EVP_KDF_CTX_set_params(kdf_ctx, params) <= 0)
return false;
if (EVP_KDF_derive(kdf_ctx, outsecret, outsize, NULL) <= 0)
return false;
return true;
}
msgr_handshake_t::~msgr_handshake_t()
{
if (peer_cert)
X509_free(peer_cert);
if (ec_key)
EVP_PKEY_free(ec_key);
}
bool msgr_handshake_t::on_error(const std::string & prefix)
{
error = prefix+ERR_error_string(ERR_get_error(), NULL);
state = MSGR_HS_ERROR;
return false;
}
bool msgr_handshake_t::init(bool server_mode)
{
this->ctx = ctx;
std::unique_ptr<EVP_PKEY_CTX, decltype(&EVP_PKEY_CTX_free)> kctx(EVP_PKEY_CTX_new(ctx->params, NULL), EVP_PKEY_CTX_free);
if (!kctx)
return on_error("EVP_PKEY_CTX_new with EC params: ");
if (EVP_PKEY_keygen_init(kctx.get()) <= 0)
return on_error("EVP_PKEY_keygen_init: ");
if (EVP_PKEY_keygen(kctx.get(), &ec_key) <= 0)
return on_error("EVP_PKEY_keygen: ");
if (!server_mode)
{
// Send initial message - only the EC public key
if (!make_client_init())
return false;
}
this->is_server = server_mode;
this->state = server_mode ? MSGR_HS_SERVER_INIT : MSGR_HS_CLIENT_INIT;
return true;
}
static void copy_to(std::vector<uint8_t> & buf, const void* src, uint32_t len)
{
size_t old_size = buf.size();
buf.resize(buf.size() + len);
memcpy(buf.data() + old_size, src, len);
}
static void copy_to_with_len(std::vector<uint8_t> & buf, const void* src, uint32_t len)
{
copy_to(buf, &len, sizeof(len));
copy_to(buf, src, len);
}
bool msgr_handshake_t::derive_shared_secret(EVP_PKEY *peer_ec_key)
{
EVP_PKEY_CTX *dh_ctx = NULL;
if (!(dh_ctx = EVP_PKEY_CTX_new(ec_key, NULL)))
return on_error("EVP_PKEY_CTX_new for ECDH: ");
if (!EVP_PKEY_derive_init(dh_ctx))
{
EVP_PKEY_CTX_free(dh_ctx);
return on_error("EVP_PKEY_derive_init: ");
}
if (!EVP_PKEY_derive_set_peer(dh_ctx, peer_ec_key))
{
EVP_PKEY_CTX_free(dh_ctx);
return on_error("EVP_PKEY_derive_set_peer: ");
}
size_t len = 0;
if (!EVP_PKEY_derive(dh_ctx, NULL, &len))
{
EVP_PKEY_CTX_free(dh_ctx);
return on_error("EVP_PKEY_derive get length: ");
}
shared_secret.resize(len);
assert(len == 48);
if (!EVP_PKEY_derive(dh_ctx, shared_secret.data(), &len))
{
EVP_PKEY_CTX_free(dh_ctx);
return on_error("EVP_PKEY_derive: ");
}
assert(len == shared_secret.size());
shared_secret.resize(len);
EVP_PKEY_CTX_free(dh_ctx);
return true;
}
bool msgr_handshake_t::derive_hs_keys(const uint8_t *encoded_peer_key, size_t encoded_key_len)
{
std::unique_ptr<EVP_PKEY, decltype(&EVP_PKEY_free)> peer_ec_key(EVP_PKEY_new(), EVP_PKEY_free);
if (!peer_ec_key)
return on_error("EVP_PKEY_new: ");
if (EVP_PKEY_copy_parameters(peer_ec_key.get(), ec_key) <= 0)
return on_error("EVP_PKEY_copy_parameters: ");
if (EVP_PKEY_set1_encoded_public_key(peer_ec_key.get(), encoded_peer_key, encoded_key_len) <= 0)
return on_error("Invalid handshake peer key: ");
if (!derive_shared_secret(peer_ec_key.get()))
return false;
hs_key.resize(AES_256_GCM_KEY_SIZE + AES_256_GCM_IV_SIZE);
peer_hs_key.resize(AES_256_GCM_KEY_SIZE + AES_256_GCM_IV_SIZE);
if (!ctx->derive_kdf(shared_secret.data(), shared_secret.size(),
NULL, 0, (state == MSGR_HS_SERVER_INIT ? "server hs key" : "client hs key"),
hs_key.data(), hs_key.size()))
return on_error("derive_kdf: ");
if (!ctx->derive_kdf(shared_secret.data(), shared_secret.size(),
NULL, 0, (state != MSGR_HS_SERVER_INIT ? "server hs key" : "client hs key"),
peer_hs_key.data(), peer_hs_key.size()))
return on_error("derive_kdf: ");
return true;
}
bool msgr_handshake_t::sign(std::vector<uint8_t> & out)
{
std::unique_ptr<EVP_MD_CTX, decltype(&EVP_MD_CTX_free)> md_ctx(EVP_MD_CTX_new(), EVP_MD_CTX_free);
if (!md_ctx)
return on_error("EVP_MD_CTX_create: ");
if (EVP_DigestSignInit(md_ctx.get(), NULL, ctx->md, NULL, ctx->my_privkey) <= 0)
return on_error("EVP_DigestSignInit: ");
if (EVP_DigestSignUpdate(md_ctx.get(), full_handshake.data(), full_handshake.size()) <= 0)
return on_error("EVP_DigestSignUpdate: ");
size_t siglen = 0;
if (EVP_DigestSignFinal(md_ctx.get(), NULL, &siglen) <= 0)
return on_error("EVP_DigestSignFinal get length: ");
size_t oldsize = out.size();
out.resize(oldsize + siglen);
if (EVP_DigestSignFinal(md_ctx.get(), out.data() + oldsize, &siglen) <= 0)
return on_error("EVP_DigestSignFinal: ");
out.resize(oldsize + siglen);
return true;
}
bool msgr_handshake_t::verify(const uint8_t *signature, size_t signature_len)
{
std::unique_ptr<EVP_MD_CTX, decltype(&EVP_MD_CTX_free)> md_ctx(EVP_MD_CTX_new(), EVP_MD_CTX_free);
if (!md_ctx)
return on_error("EVP_MD_CTX_create: ");
if (EVP_DigestVerifyInit(md_ctx.get(), NULL, ctx->md, NULL, X509_get0_pubkey(peer_cert)) <= 0)
return on_error("EVP_DigestVerifyInit: ");
if (EVP_DigestVerifyUpdate(md_ctx.get(), full_handshake.data(), full_handshake.size()) <= 0)
return on_error("EVP_DigestVerifyUpdate: ");
if (EVP_DigestVerifyFinal(md_ctx.get(), signature, signature_len) <= 0)
return false;
return true;
}
bool msgr_handshake_t::encrypt(const uint8_t* src, size_t len, uint8_t* dest)
{
std::unique_ptr<EVP_CIPHER_CTX, decltype(&EVP_CIPHER_CTX_free)> enc_ctx(EVP_CIPHER_CTX_new(), EVP_CIPHER_CTX_free);
if (!ctx)
return on_error("EVP_CIPHER_CTX_new: ");
if (EVP_EncryptInit_ex(enc_ctx.get(), EVP_aes_256_gcm(), NULL, hs_key.data(), hs_key.data() + AES_256_GCM_KEY_SIZE) <= 0)
return on_error("EVP_EncryptInit AES-256-GCM: ");
int actual_out;
if (EVP_EncryptUpdate(enc_ctx.get(), dest, &actual_out, src, len) <= 0)
return on_error("EVP_EncryptUpdate: ");
assert(actual_out == len);
if (EVP_EncryptFinal_ex(enc_ctx.get(), NULL, &actual_out) <= 0)
return on_error("EVP_EncryptFinal: ");
if (EVP_CIPHER_CTX_ctrl(enc_ctx.get(), EVP_CTRL_GCM_GET_TAG, 16, dest+len) <= 0)
return on_error("EVP_CTRL_GCM_GET_TAG: ");
(*(uint64_t*)(hs_key.data() + AES_256_GCM_KEY_SIZE))++; // change IV
return true;
}
bool msgr_handshake_t::decrypt(const uint8_t *src, size_t & len, uint8_t* dest)
{
if (len <= 16) // only tag?!
{
len = 0;
error = "Handshake decryption failed";
state = MSGR_HS_ERROR;
return false;
}
std::unique_ptr<EVP_CIPHER_CTX, decltype(&EVP_CIPHER_CTX_free)> dec_ctx(EVP_CIPHER_CTX_new(), EVP_CIPHER_CTX_free);
if (!ctx)
return on_error("EVP_CIPHER_CTX_new: ");
if (EVP_DecryptInit_ex(dec_ctx.get(), EVP_aes_256_gcm(), NULL, peer_hs_key.data(), peer_hs_key.data() + AES_256_GCM_KEY_SIZE) <= 0)
return on_error("EVP_DecryptInit AES-256-GCM: ");
int actual_out;
len -= 16;
if (EVP_DecryptUpdate(dec_ctx.get(), dest, &actual_out, src, len) <= 0)
return on_error("EVP_DecryptUpdate: ");
assert(actual_out == len);
if (EVP_CIPHER_CTX_ctrl(dec_ctx.get(), EVP_CTRL_GCM_SET_TAG, 16, (void*)(src+len)) <= 0)
return on_error("EVP_CTRL_GCM_SET_TAG: ");
if (EVP_DecryptFinal_ex(dec_ctx.get(), NULL, &actual_out) <= 0)
{
error = "Handshake decryption failed";
state = MSGR_HS_ERROR;
return false;
}
(*(uint64_t*)(peer_hs_key.data() + AES_256_GCM_KEY_SIZE))++; // change IV
return true;
}
bool msgr_handshake_t::make_client_init()
{
uint8_t *key = NULL;
size_t key_len = EVP_PKEY_get1_encoded_public_key(ec_key, &key);
if (!key_len)
return on_error("EVP_PKEY_get1_encoded_public_key: ");
const size_t old_len = out_buf.size();
out_buf.resize(out_buf.size() + key_len + sizeof(msgr_handshake_hdr_t));
uint8_t *buf = out_buf.data() + old_len;
msgr_handshake_hdr_t *hdr = (msgr_handshake_hdr_t *)buf;
hdr->msg_len = key_len + sizeof(msgr_handshake_hdr_t);
hdr->magic = MSGR_HS_MAGIC;
hdr->type = MSGR_HS_CLIENT_INIT;
memcpy(buf + sizeof(msgr_handshake_hdr_t), key, key_len);
copy_to(full_handshake, &hdr->type, sizeof(hdr->type));
copy_to_with_len(full_handshake, key, key_len);
OPENSSL_free(key);
return true;
}
bool msgr_handshake_t::make_server_reply()
{
uint8_t *key = NULL;
size_t key_len = EVP_PKEY_get1_encoded_public_key(ec_key, &key);
if (!key_len)
return on_error("EVP_PKEY_get1_encoded_public_key: ");
// Append type, key and raw certificate to signed data and sign it
msgr_handshake_hdr_t hdr = { .magic = MSGR_HS_MAGIC, .type = MSGR_HS_SERVER_REPLY };
copy_to(full_handshake, &hdr.type, sizeof(hdr.type));
copy_to_with_len(full_handshake, key, key_len);
copy_to_with_len(full_handshake, ctx->my_cert_pem.data(), ctx->my_cert_pem.size());
std::vector<uint8_t> signature;
if (!sign(signature))
{
OPENSSL_free(key);
return false;
}
// Encrypt certificate and signature
std::vector<uint8_t> encrypt_data;
copy_to_with_len(encrypt_data, ctx->my_cert_pem.data(), ctx->my_cert_pem.size());
copy_to_with_len(encrypt_data, signature.data(), signature.size());
encrypt_data.resize(encrypt_data.size()+16);
if (!encrypt(encrypt_data.data(), encrypt_data.size()-16, encrypt_data.data()))
{
OPENSSL_free(key);
return false;
}
// Construct message
hdr.msg_len = sizeof(msgr_handshake_hdr_t) + 4 + key_len + encrypt_data.size();
copy_to(out_buf, &hdr, sizeof(hdr));
copy_to_with_len(out_buf, key, key_len);
copy_to(out_buf, encrypt_data.data(), encrypt_data.size());
OPENSSL_free(key);
return true;
}
bool msgr_handshake_t::make_client_reply()
{
// Append type and raw certificate to signed data and sign it
msgr_handshake_hdr_t hdr = { .magic = MSGR_HS_MAGIC, .type = MSGR_HS_CLIENT_REPLY };
copy_to(full_handshake, &hdr.type, sizeof(hdr.type));
copy_to_with_len(full_handshake, ctx->my_cert_pem.data(), ctx->my_cert_pem.size());
std::vector<uint8_t> signature;
if (!sign(signature))
return false;
// Encrypt certificate and signature
std::vector<uint8_t> encrypt_data;
copy_to_with_len(encrypt_data, ctx->my_cert_pem.data(), ctx->my_cert_pem.size());
copy_to_with_len(encrypt_data, signature.data(), signature.size());
encrypt_data.resize(encrypt_data.size()+16);
if (!encrypt(encrypt_data.data(), encrypt_data.size()-16, encrypt_data.data()))
return false;
// Construct message
hdr.msg_len = sizeof(msgr_handshake_hdr_t) + encrypt_data.size();
copy_to(out_buf, &hdr, sizeof(hdr));
copy_to(out_buf, encrypt_data.data(), encrypt_data.size());
return true;
}
bool msgr_handshake_t::verify_peer(const uint8_t *peer_cert_pem, size_t peer_cert_len)
{
BIO *bio = BIO_new_mem_buf(peer_cert_pem, peer_cert_len);
if (!bio)
return on_error("BIO_new_mem_buf: ");
peer_cert = PEM_read_bio_X509(bio, NULL, 0, NULL);
BIO_free(bio);
if (!peer_cert)
{
error = "Invalid peer certificate";
state = MSGR_HS_ERROR;
return false;
}
std::unique_ptr<X509_STORE_CTX, decltype(&X509_STORE_CTX_free)> ca_ctx(X509_STORE_CTX_new(), X509_STORE_CTX_free);
if (!ca_ctx)
return on_error("X509_STORE_CTX_new: ");
if (X509_STORE_CTX_init(ca_ctx.get(), ctx->ca, peer_cert, NULL) <= 0)
return on_error("X509_STORE_CTX_init: ");
// Maybe use X509_VERIFY_PARAM_set_auth_level(X509_STORE_CTX_get0_param(ca_ctx.get()), 2) ?
X509_STORE_CTX_set_default(ca_ctx.get(), is_server ? "ssl_client" : "ssl_server");
if (X509_verify_cert(ca_ctx.get()) <= 0)
{
error = "Peer certificate verification failed: ";
error += X509_verify_cert_error_string(X509_STORE_CTX_get_error(ca_ctx.get()));
state = MSGR_HS_ERROR;
return false;
}
peer_is_osd = (X509_verify(peer_cert, X509_get0_pubkey(ctx->osd_ca)) > 0);
if (!is_server && !peer_is_osd)
{
error = "Peer is not an OSD";
state = MSGR_HS_ERROR;
return false;
}
return true;
}
ssize_t msgr_handshake_t::start_msg(uint8_t* src, size_t len, uint32_t expected_type)
{
size_t orig_len = len;
size_t to_buffer = (len < sizeof(msgr_handshake_hdr_t)-in_buf.size()
? len : sizeof(msgr_handshake_hdr_t)-in_buf.size());
in_buf.insert(in_buf.end(), src, src+to_buffer);
len -= to_buffer;
src += to_buffer;
if (in_buf.size() < sizeof(msgr_handshake_hdr_t))
return 0;
cur_hdr = (msgr_handshake_hdr_t *)in_buf.data();
if (cur_hdr->magic != MSGR_HS_MAGIC ||
cur_hdr->type != expected_type ||
cur_hdr->msg_len <= sizeof(msgr_handshake_hdr_t) ||
cur_hdr->msg_len >= MSGR_HS_MAX_LEN)
{
error = "Invalid handshake packet magic, type or size";
state = MSGR_HS_ERROR;
return -1;
}
to_buffer = (len < cur_hdr->msg_len-in_buf.size()
? len : cur_hdr->msg_len-in_buf.size());
in_buf.insert(in_buf.end(), src, src+to_buffer);
cur_hdr = (msgr_handshake_hdr_t *)in_buf.data();
len -= to_buffer;
src += to_buffer;
if (in_buf.size() < cur_hdr->msg_len)
return 0;
cur_left = cur_hdr->msg_len - sizeof(msgr_handshake_hdr_t);
cur_buf = in_buf.data() + sizeof(msgr_handshake_hdr_t);
return orig_len - len;
}
bool msgr_handshake_t::read_with_len(const uint8_t* & dst, uint32_t & dst_len)
{
if (cur_left < 4)
{
error = "Handshake packet too short";
state = MSGR_HS_ERROR;
return false;
}
dst_len = *(uint32_t*)cur_buf;
cur_buf += 4;
cur_left -= 4;
if (cur_left < dst_len)
{
error = "Handshake packet too short";
state = MSGR_HS_ERROR;
return false;
}
dst = cur_buf;
cur_buf += dst_len;
cur_left -= dst_len;
return true;
}
bool msgr_handshake_t::handle_client_init()
{
// Derive shared secret and handshake keys
if (!derive_hs_keys(cur_buf, cur_left))
return false;
// Add type and key to full_handshake
copy_to(full_handshake, &cur_hdr->type, sizeof(cur_hdr->type));
copy_to_with_len(full_handshake, cur_buf, cur_left);
in_buf.clear();
return true;
}
// Decrypt and check peer certificate
bool msgr_handshake_t::handle_peer_cert(const uint8_t *key, uint32_t key_len)
{
if (!decrypt(cur_buf, cur_left, cur_buf))
return false;
const uint8_t *peer_cert_pem = NULL;
uint32_t peer_cert_len = 0;
if (!read_with_len(peer_cert_pem, peer_cert_len))
return false;
// Parse and verify certificate
if (!verify_peer(peer_cert_pem, peer_cert_len))
return false;
// Verify signature
copy_to(full_handshake, &cur_hdr->type, sizeof(cur_hdr->type));
if (key)
copy_to_with_len(full_handshake, key, key_len);
copy_to_with_len(full_handshake, peer_cert_pem, peer_cert_len);
const uint8_t *signature = NULL;
uint32_t signature_len = 0;
if (!read_with_len(signature, signature_len))
return false;
if (!verify(signature, signature_len))
return false;
return true;
}
bool msgr_handshake_t::handle_server_reply()
{
// Derive shared secret and handshake keys
const uint8_t *key = NULL;
uint32_t key_len = 0;
if (!read_with_len(key, key_len))
return false;
if (!derive_hs_keys(key, key_len))
return false;
// Decrypt and check peer certificate
if (!handle_peer_cert(key, key_len))
return false;
in_buf.clear();
return true;
}
bool msgr_handshake_t::handle_client_reply()
{
// Decrypt and check peer certificate
if (!handle_peer_cert(NULL, 0))
return false;
in_buf.clear();
return true;
}
void msgr_handshake_t::complete()
{
state = MSGR_HS_DONE;
hs_key.clear();
peer_hs_key.clear();
full_handshake.clear();
}
ssize_t msgr_handshake_t::handle(uint8_t* in_buf, size_t in_size)
{
if (state == MSGR_HS_SERVER_INIT)
{
ssize_t r = start_msg(in_buf, in_size, MSGR_HS_CLIENT_INIT);
if (r < 0)
return r;
if (r == 0)
return in_size;
if (!handle_client_init())
return -1;
// Send encrypted & signed response
if (!make_server_reply())
return -1;
state = MSGR_HS_SERVER_REPLY;
return r;
}
else if (state == MSGR_HS_CLIENT_INIT)
{
ssize_t r = start_msg(in_buf, in_size, MSGR_HS_SERVER_REPLY);
if (r < 0)
return r;
if (r == 0)
return in_size;
if (!handle_server_reply())
return -1;
// Verification passed, send certificate to the server
if (!make_client_reply())
return -1;
// Finished!
complete();
return r;
}
else if (state == MSGR_HS_SERVER_REPLY)
{
ssize_t r = start_msg(in_buf, in_size, MSGR_HS_CLIENT_REPLY);
if (r < 0)
return r;
if (r == 0)
return in_size;
if (!handle_client_reply())
return -1;
// Verification passed
// Finished!
complete();
return r;
}
else if (state == MSGR_HS_DONE)
{
return 0;
}
else if (state != MSGR_HS_ERROR)
{
error = "Unexpected handshake state: "+std::to_string(state);
}
return -1;
}
bool msgr_handshake_t::done()
{
return (state == MSGR_HS_DONE);
}
std::vector<uint8_t>& msgr_handshake_t::get_out()
{
return out_buf;
}
msgr_handshake_result_t msgr_handshake_t::get_result()
{
if (state != MSGR_HS_DONE)
return msgr_handshake_result_t{};
X509_up_ref(peer_cert);
return msgr_handshake_result_t{
.peer_cert = peer_cert,
.peer_is_osd = peer_is_osd,
.shared_secret = shared_secret,
};
}
std::string msgr_handshake_t::get_error()
{
return error;
}
src/client/msgr_handshake.h
+54
View File
@@ -0,0 +1,54 @@
// Copyright (c) Vitaliy Filippov, 2026+
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
#pragma once
#include <stdint.h>
#include <string>
#include <vector>
#include <openssl/types.h>
#define AES_256_GCM_KEY_SIZE 32
#define AES_256_GCM_IV_SIZE 12
#define AES_256_GCM_MAX_IV_CTR ((uint64_t)1 << 32)
// TLS 1.3-like handshake
// 1. Client->server: EC public key
// 2. Server->client: EC public key, encrypted certificate and digital signature of the handshake
// 3. Client->server: encrypted certificate and digital signature of the handshake
struct msgr_handshake_result_t
{
X509 *peer_cert = NULL;
bool peer_is_osd = false;
std::vector<uint8_t> shared_secret;
};
class msgr_handshake_i
{
public:
// Workflow: create -> init -> handle_msg -> get_result/get_error -> destruct
virtual ~msgr_handshake_i() = default;
virtual bool init(bool server_mode) = 0;
virtual ssize_t handle(uint8_t* in_buf, size_t in_size) = 0;
virtual bool done() = 0;
virtual std::vector<uint8_t> & get_out() = 0;
virtual msgr_handshake_result_t get_result() = 0;
virtual std::string get_error() = 0;
};
class msgr_handshake_ctx_i
{
public:
static msgr_handshake_ctx_i* create_ctx();
virtual ~msgr_handshake_ctx_i() = default;
virtual msgr_handshake_i* create() = 0;
virtual bool init(const std::string & pem_cert, const std::string & pem_key,
const std::string & pem_osd_ca, const std::string & pem_client_ca) = 0;
virtual std::string get_error() = 0;
virtual bool derive_kdf(const uint8_t* insecret, size_t insecret_len,
const uint8_t* salt, size_t salt_len, const char *label, uint8_t *outsecret, size_t outsize) = 0;
};
src/client/msgr_receive.cpp
+59 -7
View File
@@ -202,7 +202,7 @@ public:
assert(dst != NULL);
buffer_again:
buffer_encrypted();
if (!cl->ssl_handshake_done)
if (!cl->handshake_done)
{
if (!msgr->do_tls_handshake(cl, true))
return false;
@@ -299,16 +299,21 @@ public:
#endif
}
}
uint8_t iv[12] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 };
if (cl->peer_iv_ctr >= AES_256_GCM_MAX_IV_CTR)
{
// Rotate key every 2^32 messages
bool ok = msgr->derive_aes_keys(cl, false, true);
assert(ok);
}
#ifdef WITH_ISAL_CRYPTO
int r = isal_aes_gcm_init_256(&msgr->test_osd_aes_key_isal, cl->dec_ctx, iv, NULL, 0);
int r = isal_aes_gcm_init_256(&cl->peer_key_isal, cl->dec_ctx, cl->peer_key.data() + AES_256_GCM_KEY_SIZE, NULL, 0);
if (r != 0)
{
fprintf(stderr, "isal_aes_gcm_init_256 error %d\n", r);
abort();
}
#else
int r = EVP_DecryptInit_ex(cl->dec_ctx, NULL, NULL, (uint8_t*)msgr->test_osd_aes_key.data(), iv);
int r = EVP_DecryptInit_ex(cl->dec_ctx, NULL, NULL, cl->peer_key.data(), cl->peer_key.data() + AES_256_GCM_KEY_SIZE);
if (r != 1)
{
fprintf(stderr, "DecryptInit error: ");
@@ -316,6 +321,9 @@ public:
abort();
}
#endif
// Increase IV
cl->peer_iv_ctr++;
(*(uint64_t*)(cl->peer_key.data() + AES_256_GCM_KEY_SIZE))++;
}
bool read(uint8_t *dst, size_t dst_len, int flags) override
@@ -365,7 +373,7 @@ public:
if (n > bufsize-done)
n = bufsize-done;
#ifdef WITH_ISAL_CRYPTO
int r = isal_aes_gcm_dec_256_update(&msgr->test_osd_aes_key_isal, cl->dec_ctx, dst+from, curbuf+done, n);
int r = isal_aes_gcm_dec_256_update(&cl->peer_key_isal, cl->dec_ctx, dst+from, curbuf+done, n);
assert(!r);
#else
int actual_out;
@@ -405,7 +413,7 @@ public:
}
#ifdef WITH_ISAL_CRYPTO
uint8_t calc_tag[16];
int r = isal_aes_gcm_dec_256_finalize(&msgr->test_osd_aes_key_isal, cl->dec_ctx, calc_tag, 16);
int r = isal_aes_gcm_dec_256_finalize(&cl->peer_key_isal, cl->dec_ctx, calc_tag, 16);
assert(r == 0);
if (cl->dec_tag_size > 0)
{
@@ -752,6 +760,47 @@ bool osd_messenger_t::handle_read_buffer(osd_client_t *cl, uint8_t *curbuf, size
}
else if (cl->gcm_enabled)
{
if (cl->hs)
{
ssize_t done = cl->hs->handle(curbuf, bufsize);
if (done < 0)
{
fprintf(stderr, "Client %ju handshake failed: %s\n", cl->client_id, cl->hs->get_error().c_str());
stop_client(cl->client_id);
return false;
}
if (cl->hs->done() && !derive_aes_keys(cl, true, true))
{
stop_client(cl->client_id);
return false;
}
curbuf += done;
bufsize -= done;
if (cl->hs->get_out().size())
{
if (cl->write_state == 0)
{
cl->write_state = CL_WRITE_READY;
write_ready_clients.push_back(cl->client_id);
}
}
if (cl->hs->done() && !cl->hs->get_out().size())
{
// Delete hs when done and nothing to send
delete cl->hs;
cl->hs = NULL;
}
else
{
if (done < bufsize)
{
fprintf(stderr, "Client %ju extra data after handshake\n", cl->client_id);
stop_client(cl->client_id);
return false;
}
return true;
}
}
return handle_buffer_with<gcm_op_reader_t>(cl, curbuf, bufsize);
}
return handle_buffer_with<copy_op_reader_t>(cl, curbuf, bufsize);
@@ -782,7 +831,10 @@ bool osd_messenger_t::handle_buffer_with(osd_client_t *cl, uint8_t *curbuf, size
{
if (!cl->read_csum_state)
cl->read_csum_state = XXH3_createState();
XXH3_64bits_reset(cl->read_csum_state);
if (cl->peer_key.size() == AES_256_GCM_KEY_SIZE + AES_256_GCM_IV_SIZE + XXH_SECRET_DEFAULT_SIZE)
XXH3_64bits_reset_withSecret(cl->read_csum_state, cl->peer_key.data() + AES_256_GCM_KEY_SIZE + AES_256_GCM_IV_SIZE, XXH_SECRET_DEFAULT_SIZE);
else
XXH3_64bits_reset(cl->read_csum_state);
}
if (!op_read_from(cl, rdr) || !handle_finished_op(cl))
{
src/client/msgr_send.cpp
+100 -46
View File
@@ -112,7 +112,7 @@ public:
bool flush_ssl()
{
if (!cl->ssl_handshake_done)
if (!cl->handshake_done)
{
if (!msgr->do_tls_handshake(cl))
return false;
@@ -203,12 +203,12 @@ public:
}
else
{
if (!cl->ssl_handshake_done)
if (!cl->handshake_done)
{
if (!flush_ssl())
return false;
}
if (cl->ssl_handshake_done)
if (cl->handshake_done)
{
if (!write_to_ssl(cl, src, src_len, flags, from))
return false;
@@ -283,16 +283,21 @@ public:
#endif
}
}
uint8_t iv[12] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 };
if (cl->my_iv_ctr >= AES_256_GCM_MAX_IV_CTR)
{
// Rotate key every 2^32 messages
bool ok = msgr->derive_aes_keys(cl, true, false);
assert(ok);
}
#ifdef WITH_ISAL_CRYPTO
int r = isal_aes_gcm_init_256(&msgr->test_osd_aes_key_isal, cl->enc_ctx, iv, NULL, 0);
int r = isal_aes_gcm_init_256(&cl->my_key_isal, cl->enc_ctx, cl->my_key.data() + AES_256_GCM_KEY_SIZE, NULL, 0);
if (r != 0)
{
fprintf(stderr, "isal_aes_gcm_init_256 error %d\n", r);
abort();
}
#else
int r = EVP_EncryptInit_ex(cl->enc_ctx, NULL, NULL, (uint8_t*)msgr->test_osd_aes_key.data(), iv);
int r = EVP_EncryptInit_ex(cl->enc_ctx, NULL, NULL, (uint8_t*)cl->my_key.data(), cl->my_key.data() + AES_256_GCM_KEY_SIZE);
if (r != 1)
{
fprintf(stderr, "EncryptInit error: ");
@@ -300,6 +305,9 @@ public:
abort();
}
#endif
// Increase IV
cl->my_iv_ctr++;
(*(uint64_t*)(cl->my_key.data() + AES_256_GCM_KEY_SIZE))++;
}
static void free_ctx(osd_messenger_t* msgr, osd_client_t *cl)
@@ -353,7 +361,7 @@ public:
if (!n)
return false;
#ifdef WITH_ISAL_CRYPTO
int r = isal_aes_gcm_enc_256_update(&msgr->test_osd_aes_key_isal, cl->enc_ctx, curbuf+done, src+from, n);
int r = isal_aes_gcm_enc_256_update(&cl->my_key_isal, cl->enc_ctx, curbuf+done, src+from, n);
assert(!r);
#else
int actual_out;
@@ -380,7 +388,7 @@ public:
static void write_tag_to(osd_messenger_t *msgr, osd_client_t *cl, uint8_t *dst)
{
#ifdef WITH_ISAL_CRYPTO
int r = isal_aes_gcm_enc_256_finalize(&msgr->test_osd_aes_key_isal, cl->enc_ctx, dst, 16);
int r = isal_aes_gcm_enc_256_finalize(&cl->my_key_isal, cl->enc_ctx, dst, 16);
assert(!r);
#else
int actual_out = 0;
@@ -444,7 +452,33 @@ class get_op_writer_t: public msgr_op_writer_t
size_t enc_size;
size_t done_enc;
void ssl_extend_buf(size_t more = 0)
void copy_ssl()
{
size_t n = 0;
do
{
ssl_extend_buf(cl);
int r = BIO_read(cl->read_from_ssl, cl->ssl_out_buf+cl->ssl_out_buf_size, cl->ssl_out_buf_cap-cl->ssl_out_buf_size);
if (r > 0)
n += r;
} while (cl->ssl_out_buf_size+n >= cl->ssl_out_buf_cap);
cl->ssl_more_to_buffer = false;
if (n > 0)
{
send_out_buf(cl, n);
done += n;
}
}
public:
constexpr static bool is_ssl = true;
get_op_writer_t(osd_messenger_t* msgr, osd_client_t* cl, uint8_t*, size_t):
msgr(msgr), cl(cl), from(cl->write_op_pos), done(0), enc_size(0), done_enc(0)
{
}
static void ssl_extend_buf(osd_client_t *cl, size_t more = 0)
{
size_t min_cap = cl->ssl_out_buf_size*2;
if (min_cap < cl->ssl_out_buf_size+more)
@@ -465,7 +499,7 @@ class get_op_writer_t: public msgr_op_writer_t
}
}
void send_out_buf(size_t n)
static void send_out_buf(osd_client_t *cl, size_t n)
{
if (cl->send_list.size() > 0)
{
@@ -478,33 +512,9 @@ class get_op_writer_t: public msgr_op_writer_t
}
}
cl->send_list.push_back((iovec){ .iov_base = cl->ssl_out_buf+cl->ssl_out_buf_size, .iov_len = n });
done += n;
cl->ssl_out_buf_size += n;
}
void copy_ssl()
{
size_t n = 0;
do
{
ssl_extend_buf();
int r = BIO_read(cl->read_from_ssl, cl->ssl_out_buf+cl->ssl_out_buf_size, cl->ssl_out_buf_cap-cl->ssl_out_buf_size);
if (r > 0)
n += r;
} while (cl->ssl_out_buf_size+n >= cl->ssl_out_buf_cap);
cl->ssl_more_to_buffer = false;
if (n > 0)
send_out_buf(n);
}
public:
constexpr static bool is_ssl = true;
get_op_writer_t(osd_messenger_t* msgr, osd_client_t* cl, uint8_t*, size_t):
msgr(msgr), cl(cl), from(cl->write_op_pos), done(0), enc_size(0), done_enc(0)
{
}
void reset()
{
from = cl->write_op_pos;
@@ -518,7 +528,7 @@ public:
bool flush_ssl()
{
if (cl->ssl_cli && !cl->ssl_handshake_done)
if (cl->ssl_cli && !cl->handshake_done)
{
if (!msgr->do_tls_handshake(cl))
return false;
@@ -543,12 +553,12 @@ public:
{
if (cl->ssl_cli)
{
if (!cl->ssl_handshake_done)
if (!cl->handshake_done)
{
if (!flush_ssl())
return false;
}
if (cl->ssl_handshake_done)
if (cl->handshake_done)
{
if (!ssl_op_writer_t::write_to_ssl(cl, src, src_len, flags, from))
return false;
@@ -560,13 +570,13 @@ public:
from = 0;
return true;
}
else if (cl->enc_ctx)
if (cl->gcm_enabled)
{
// Encrypt data to client's temporary output buffer (all at once)
size_t n = src_len-from;
ssl_extend_buf(n);
ssl_extend_buf(cl, n);
#ifdef WITH_ISAL_CRYPTO
int r = isal_aes_gcm_enc_256_update(&msgr->test_osd_aes_key_isal, cl->enc_ctx, cl->ssl_out_buf+cl->ssl_out_buf_size, src+from, n);
int r = isal_aes_gcm_enc_256_update(&cl->my_key_isal, cl->enc_ctx, cl->ssl_out_buf+cl->ssl_out_buf_size, src+from, n);
assert(!r);
#else
int actual_out;
@@ -580,7 +590,8 @@ public:
#endif
if (cl->write_csum_state && !(flags & WR_NO_CSUM))
XXH3_64bits_update(cl->write_csum_state, src+from, n);
send_out_buf(n);
send_out_buf(cl, n);
done += n;
cl->write_op_pos += n;
from += n;
if (from < src_len)
@@ -635,9 +646,10 @@ public:
if (cl->send_list.size() >= IOV_MAX)
return false;
// Tag is 16 bytes
ssl_extend_buf(16);
ssl_extend_buf(cl, 16);
gcm_op_writer_t::write_tag_to(msgr, cl, cl->ssl_out_buf+cl->ssl_out_buf_size);
send_out_buf(16);
send_out_buf(cl, 16);
done += 16;
gcm_op_writer_t::free_ctx(msgr, cl);
}
return true;
@@ -720,7 +732,28 @@ bool osd_messenger_t::try_send(osd_client_t *cl)
return false;
}
assert(cl->peer_state != PEER_RDMA);
copy_ops_to_with<get_op_writer_t>(cl, NULL, 0);
if (cl->hs)
{
// Send handshake message
if (cl->hs->get_out().size())
{
get_op_writer_t::ssl_extend_buf(cl, cl->hs->get_out().size());
memcpy(cl->ssl_out_buf+cl->ssl_out_buf_size, cl->hs->get_out().data(), cl->hs->get_out().size());
get_op_writer_t::send_out_buf(cl, cl->hs->get_out().size());
cl->hs->get_out().clear();
}
if (!cl->hs->get_out().size() && cl->hs->done())
{
delete cl->hs;
cl->hs = NULL;
goto copy_ops;
}
}
else
{
copy_ops:
copy_ops_to_with<get_op_writer_t>(cl, NULL, 0);
}
if (cl->io_error)
{
stop_client(cl->client_id);
@@ -796,6 +829,24 @@ size_t osd_messenger_t::copy_ops_to(osd_client_t *cl, uint8_t *dst, size_t dst_l
}
if (cl->gcm_enabled)
{
if (cl->hs)
{
// Send handshake message
size_t n = 0;
if (cl->hs->get_out().size())
{
n = cl->hs->get_out().size() < dst_len ? cl->hs->get_out().size() : dst_len;
memcpy(dst, cl->hs->get_out().data(), n);
cl->hs->get_out().erase(cl->hs->get_out().begin(), cl->hs->get_out().begin() + n);
}
if (!cl->hs->get_out().size() && cl->hs->done())
{
delete cl->hs;
cl->hs = NULL;
n += copy_ops_to_with<gcm_op_writer_t>(cl, dst+n, dst_len-n);
}
return n;
}
return copy_ops_to_with<gcm_op_writer_t>(cl, dst, dst_len);
}
return copy_ops_to_with<copy_op_writer_t>(cl, dst, dst_len);
@@ -809,8 +860,8 @@ size_t osd_messenger_t::copy_ops_to_with(osd_client_t *cl, uint8_t *dst, size_t
{
if (!cl->write_op)
{
next_write_op(cl);
wr.reset();
next_write_op(cl);
}
osd_op_t *op = cl->write_op;
if (!op_write_to(cl, wr))
@@ -843,7 +894,10 @@ void osd_messenger_t::next_write_op(osd_client_t *cl)
{
if (!cl->write_csum_state)
cl->write_csum_state = XXH3_createState();
XXH3_64bits_reset(cl->write_csum_state);
if (cl->my_key.size() == AES_256_GCM_KEY_SIZE + AES_256_GCM_IV_SIZE + XXH_SECRET_DEFAULT_SIZE)
XXH3_64bits_reset_withSecret(cl->write_csum_state, cl->my_key.data() + AES_256_GCM_KEY_SIZE + AES_256_GCM_IV_SIZE, XXH_SECRET_DEFAULT_SIZE);
else
XXH3_64bits_reset(cl->write_csum_state);
}
}
src/client/msgr_stop.cpp
+10
View File
@@ -257,4 +257,14 @@ osd_client_t::~osd_client_t()
free(ssl_out_buf);
ssl_out_buf = NULL;
}
if (hs)
{
delete hs;
hs = NULL;
}
if (hs_result.peer_cert)
{
X509_free(hs_result.peer_cert);
hs_result.peer_cert = NULL;
}
}
src/util/str_util.cpp
+9 -4
View File
@@ -558,12 +558,12 @@ std::string urldecode(const std::string & orig)
return res;
}
size_t fromhexstr(const std::string & from, size_t bytes, uint8_t *to)
size_t fromhexstr(const char *from, size_t from_len, uint8_t *to, size_t to_len)
{
if (bytes > from.size()/2)
bytes = from.size()/2;
if (to_len > from_len/2)
to_len = from_len/2;
size_t i = 0;
while (i < bytes)
while (i < to_len)
{
uint8_t x = fromhexchar(from[2*i], 16);
uint8_t y = fromhexchar(from[2*i+1], 16);
@@ -575,6 +575,11 @@ size_t fromhexstr(const std::string & from, size_t bytes, uint8_t *to)
return i;
}
size_t fromhexstr(const std::string & from, size_t bytes, uint8_t *to)
{
return fromhexstr(from.data(), from.size(), to, bytes);
}
std::string tohexstr(const uint8_t *from, size_t bytes)
{
std::string res;
src/util/str_util.h
+1
View File
@@ -37,6 +37,7 @@ std::string format_datetime(uint64_t unixtime);
bool is_zero(void *buf, size_t size);
bool memcheck(uint8_t *buf, uint8_t byte, size_t len);
std::string urldecode(const std::string & orig);
size_t fromhexstr(const char *from, size_t from_len, uint8_t *to, size_t to_len);
size_t fromhexstr(const std::string & from, size_t bytes, uint8_t *to);
std::string tohexstr(const uint8_t *from, size_t bytes);
bool ishexstr(const std::string & str);
tests/common.sh
+1 -2
View File
@@ -27,7 +27,7 @@ ETCD_COUNT=${ETCD_COUNT:-1}
ANTIETCD=${ANTIETCD}
USE_RAMDISK=${USE_RAMDISK}
ETCD_SCHEME=${ETCD_SCHEME:-http}
OSD_TLS=${OSD_TLS}
OSD_TLS=${OSD_TLS:-1}
RAMDISK=/run/user/$(id -u)
findmnt $RAMDISK >/dev/null || (sudo mkdir -p $RAMDISK && sudo mount -t tmpfs tmpfs $RAMDISK)
@@ -142,7 +142,6 @@ if [[ "$OSD_TLS" = "1" ]]; then
VITASTOR_CFG="$VITASTOR_CFG"',"tls_cert":"'$(pwd)'/testdata/cli.crt"'
VITASTOR_CFG="$VITASTOR_CFG"',"tls_key":"'$(pwd)'/testdata/cli.key"'
fi
VITASTOR_CFG="$VITASTOR_CFG"',"test_osd_aes_key":"'$(openssl rand -hex 32)'"'
echo "{$VITASTOR_CFG}" > ./testdata/vitastor.conf
VITASTOR_CFG=./testdata/vitastor.conf
VITASTOR_CLI="build/src/cmd/vitastor-cli --config_path $VITASTOR_CFG"