diff --git a/src/client/messenger.cpp b/src/client/messenger.cpp index f368d548..cb0fce51 100644 --- a/src/client/messenger.cpp +++ b/src/client/messenger.cpp @@ -351,7 +351,6 @@ void osd_messenger_t::parse_config(const json11::Json & config) this->use_proto_checksums = config["proto_checksums"].string_value() == "full" ? MSGR_CSUM_FULL : MSGR_CSUM_PAYLOAD; else this->use_proto_checksums = 0; - gcm_enabled = true; if (!osd_num) { tls_cert = config["tls_cert"].string_value(); diff --git a/src/client/messenger.h b/src/client/messenger.h index e0e11229..4687ae65 100644 --- a/src/client/messenger.h +++ b/src/client/messenger.h @@ -66,14 +66,6 @@ struct op_aes_xts_decrypt_t; void destroy_aes_xts_encrypt(op_aes_xts_encrypt_t *encrypt_ctx); void destroy_aes_xts_decrypt(op_aes_xts_decrypt_t *decrypt_ctx); -// Standard TLS record header. We are only interested in the record size -struct __attribute__((__packed__)) msgr_tls_record_hdr_t -{ - uint8_t content_type; - uint16_t version; - uint16_t size; -}; - struct osd_client_t { uint64_t client_id = 0; @@ -96,16 +88,8 @@ struct osd_client_t msgr_rdma_connection_t *rdma_conn = NULL; #endif - SSL *ssl_cli = NULL; - BIO *write_to_ssl = NULL; - // FIXME: use custom bio to avoid 1 more memory copy? - BIO *read_from_ssl = NULL; uint8_t *ssl_out_buf = NULL; size_t ssl_out_buf_size = 0, ssl_out_buf_cap = 0; - bool handshake_done = false; - msgr_tls_record_hdr_t ssl_read_record; - size_t ssl_read_header_size = 0; - bool ssl_more_to_buffer = false; bool gcm_enabled = false; msgr_handshake_i *hs = NULL; @@ -250,11 +234,9 @@ struct __attribute__((visibility("default"))) osd_messenger_t { protected: friend class copy_op_reader_t; - friend class ssl_op_reader_t; friend class gcm_op_reader_t; friend class get_op_reader_t; friend class copy_op_writer_t; - friend class ssl_op_writer_t; friend class gcm_op_writer_t; friend class get_op_writer_t; @@ -292,12 +274,6 @@ protected: robin_hood::unordered_flat_map rdmacm_connecting; #endif - SSL_CTX *ssl_ctx = NULL; - X509 *tls_cert_obj = NULL; - X509 *osd_tls_ca_obj = NULL; - X509 *client_tls_ca_obj = NULL; - std::string tls_cn; - bool gcm_enabled = false; msgr_handshake_ctx_i *hs_ctx = NULL; diff --git a/src/client/msgr_encrypt.cpp b/src/client/msgr_encrypt.cpp index e7ae35ec..44d26ed3 100644 --- a/src/client/msgr_encrypt.cpp +++ b/src/client/msgr_encrypt.cpp @@ -527,7 +527,6 @@ void osd_messenger_t::init_tls() { if (!tls_cert.empty() || !tls_key.empty() || !osd_tls_ca.empty() || !client_tls_ca.empty()) { - // Initialize TLS context if (tls_cert.empty() || tls_key.empty() || osd_tls_ca.empty() || osd_num && client_tls_ca.empty()) { if (osd_num) @@ -536,41 +535,10 @@ void osd_messenger_t::init_tls() fprintf(stderr, "Vitastor client TLS requires tls_cert, tls_key and osd_tls_ca\n"); exit(1); } - else if (!gcm_enabled) - { - ssl_ctx = SSL_CTX_new(TLS_method()); - if (!ssl_ctx) - { -init_err: - fprintf(stderr, "OpenSSL initialization failed: %s\n", ERR_error_string(ERR_get_error(), NULL)); - exit(1); - } - // Always use TLS 1.3 with AES-256-GCM - SSL_CTX_set_min_proto_version(ssl_ctx, TLS1_3_VERSION); - SSL_CTX_set_max_proto_version(ssl_ctx, TLS1_3_VERSION); - SSL_CTX_set_ciphersuites(ssl_ctx, "TLS_AES_256_GCM_SHA384"); - SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, NULL); - bool ok = SSL_CTX_set_min_proto_version(ssl_ctx, TLS1_3_VERSION); - ok = ok && (osd_tls_ca_obj = openssl_load_cert(osd_tls_ca)); - ok = ok && X509_STORE_add_cert(SSL_CTX_get_cert_store(ssl_ctx), osd_tls_ca_obj); - if (osd_num) - { - // OSD uses 2 separate root certificates to distinguish between clients and peer OSDs - ok = ok && (client_tls_ca_obj = openssl_load_cert(client_tls_ca)); - ok = ok && X509_STORE_add_cert(SSL_CTX_get_cert_store(ssl_ctx), client_tls_ca_obj); - } - ok = ok && openssl_ctx_use_cert(ssl_ctx, tls_cert, tls_cn); - ok = ok && openssl_ctx_use_key(ssl_ctx, tls_key); - if (!ok) - { - SSL_CTX_free(ssl_ctx); - ssl_ctx = NULL; - goto init_err; - } - } else { #ifndef __MOCK__ + gcm_enabled = true; hs_ctx = msgr_handshake_ctx_i::create_ctx(); if (!hs_ctx->init(tls_cert, tls_key, osd_tls_ca, client_tls_ca)) { @@ -598,55 +566,6 @@ void osd_messenger_t::init_tls_client(osd_client_t *cl) } } } - else if (!tls_cert.empty()) - { - cl->write_to_ssl = BIO_new(BIO_s_mem()); - cl->read_from_ssl = BIO_new(BIO_s_mem()); - cl->ssl_cli = SSL_new(ssl_ctx); - if (!cl->ssl_cli) - { - fprintf(stderr, "OpenSSL initialization failed: %s\n", ERR_error_string(ERR_get_error(), NULL)); - exit(1); - } - if (cl->is_incoming) - { - SSL_set_accept_state(cl->ssl_cli); - } - else - { - SSL_set_connect_state(cl->ssl_cli); - } - SSL_set_bio(cl->ssl_cli, cl->write_to_ssl, cl->read_from_ssl); - bool ok = do_tls_handshake(cl); - assert(ok); - } -} - -bool osd_messenger_t::do_tls_handshake(osd_client_t *cl, bool from_recv) -{ - if (cl->handshake_done) - return true; - int r = SSL_do_handshake(cl->ssl_cli); - if (r > 0) - { - cl->handshake_done = true; - } - else - { - r = SSL_get_error(cl->ssl_cli, r); - if (r != 0 && r != SSL_ERROR_WANT_READ && r != SSL_ERROR_WANT_WRITE) - { - fprintf(stderr, "Client %ju TLS handshake error: %s, stopping client\n", cl->client_id, ERR_error_string(ERR_get_error(), NULL)); - cl->io_error = true; - return false; - } - } - if (from_recv && cl->write_state == 0 && openssl_bio_nonempty(cl->read_from_ssl)) - { - cl->write_state = CL_WRITE_READY; - write_ready_clients.push_back(cl->client_id); - } - return true; } void osd_messenger_t::destroy_tls() @@ -670,21 +589,6 @@ void osd_messenger_t::destroy_tls() EVP_CIPHER_CTX_free(ctx); } #endif - if (osd_tls_ca_obj) - { - X509_free(osd_tls_ca_obj); - osd_tls_ca_obj = NULL; - } - if (client_tls_ca_obj) - { - X509_free(client_tls_ca_obj); - client_tls_ca_obj = NULL; - } - if (ssl_ctx) - { - SSL_CTX_free(ssl_ctx); - ssl_ctx = NULL; - } if (hs_ctx) { delete hs_ctx; diff --git a/src/client/msgr_receive.cpp b/src/client/msgr_receive.cpp index ae9820cf..33caa932 100644 --- a/src/client/msgr_receive.cpp +++ b/src/client/msgr_receive.cpp @@ -6,10 +6,8 @@ #include "messenger.h" #include "openssl_util.h" -#include +#include #include -#include -#include #define RDR_GCM 1 #define RDR_XTS 2 @@ -90,171 +88,6 @@ public: } }; -class ssl_op_reader_t: public msgr_op_reader_t -{ - osd_messenger_t* msgr; - osd_client_t* cl; - size_t from; - - uint8_t *curbuf; - size_t bufsize; - size_t done; - -public: - ssl_op_reader_t(osd_messenger_t* msgr, osd_client_t* cl, uint8_t *curbuf, size_t bufsize): - msgr(msgr), cl(cl), from(cl->read_op_pos), curbuf(curbuf), bufsize(bufsize), done(0) - { - } - - void reset() - { - from = cl->read_op_pos; - } - - void buffer_encrypted() - { - if (cl->ssl_read_header_size < sizeof(msgr_tls_record_hdr_t)) - { - size_t h = bufsize-done; - if (bufsize-done <= sizeof(msgr_tls_record_hdr_t)-cl->ssl_read_header_size) - { - // Less than record header or just record header - memcpy(((uint8_t*)&cl->ssl_read_record) + cl->ssl_read_header_size, curbuf+done, h); - cl->ssl_read_header_size += h; - if (cl->ssl_read_header_size == sizeof(msgr_tls_record_hdr_t)) - cl->ssl_read_record.size = ntohs(cl->ssl_read_record.size); - int r = BIO_write(cl->write_to_ssl, curbuf+done, h); - assert(r == h); - done += h; - return; - } - // Record header and at least some data - copy both to BIO in a one BIO_write() call - h = sizeof(msgr_tls_record_hdr_t)-cl->ssl_read_header_size; - memcpy(((uint8_t*)&cl->ssl_read_record) + cl->ssl_read_header_size, curbuf+done, h); - cl->ssl_read_header_size = sizeof(msgr_tls_record_hdr_t); - cl->ssl_read_record.size = ntohs(cl->ssl_read_record.size); - size_t n = h + cl->ssl_read_record.size; - if (n > bufsize-done) - n = bufsize-done; - int r = BIO_write(cl->write_to_ssl, curbuf+done, n); - assert(r == n); - done += n; - cl->ssl_read_record.size -= (n - h); - if (!cl->ssl_read_record.size) - cl->ssl_read_header_size = 0; - return; - } - // Continued TLS data - buffer it to BIO - size_t n = cl->ssl_read_record.size; - if (n > bufsize-done) - n = bufsize-done; - int r = BIO_write(cl->write_to_ssl, curbuf+done, n); - assert(r == n); - done += n; - cl->ssl_read_record.size -= n; - if (!cl->ssl_read_record.size) - cl->ssl_read_header_size = 0; - } - - bool read(uint8_t *dst, size_t dst_len, int flags) override - { - if (from >= dst_len) - { - // Skip - from -= dst_len; - return true; - } - if (done >= bufsize) - return false; - size_t n = dst_len-from; - if (!(flags & RDR_GCM) || !cl->ssl_cli) - { - if (n > bufsize-done) - n = bufsize-done; - if (flags & RDR_XTS) - { - msgr->op_decrypted_copy_buf(cl, curbuf, bufsize, dst, dst_len, from, done); - n = 0; - } - else - { - if (cl->read_csum_state && !(flags & RDR_NO_CSUM)) - { - // data may be skipped if dst == NULL but checksum is still calculated - XXH3_64bits_update(cl->read_csum_state, curbuf+done, n); - } - // Here, dst == NULL is allowed - if (dst != NULL) - memcpy(dst+from, curbuf+done, n); - done += n; - } - cl->read_op_pos += n; - from += n; - if (from < dst_len) - { - return false; - } - } - else - { - // Here, dst == NULL is not allowed - assert(dst != NULL); -buffer_again: - buffer_encrypted(); - if (!cl->handshake_done) - { - if (!msgr->do_tls_handshake(cl, true)) - return false; - } - int ok = SSL_read_ex(cl->ssl_cli, dst+from, n, &n); - if (!ok) - { - ok = SSL_get_error(cl->ssl_cli, ok); - if (ok == SSL_ERROR_WANT_READ) - { - if (done < bufsize) - goto buffer_again; - } - else if (ok == SSL_ERROR_ZERO_RETURN) - { - fprintf(stderr, "Client %ju TLS disconnected\n", cl->client_id); - cl->io_error = true; - } - else if (ok != 0 && ok != SSL_ERROR_WANT_WRITE) - { - fprintf(stderr, "Client %ju TLS read error: %s. Disconnecting client\n", cl->client_id, ERR_error_string(ERR_get_error(), NULL)); - cl->io_error = true; - } - return false; - } - if (cl->read_csum_state && !(flags & RDR_NO_CSUM)) - { - XXH3_64bits_update(cl->read_csum_state, dst+from, n); - } - cl->read_op_pos += n; - from += n; - if (from < dst_len) - { - if (done < bufsize) - goto buffer_again; - return false; - } - } - from = 0; - return true; - } - - bool finish() override - { - return true; - } - - size_t get_done() - { - return done; - } -}; - class gcm_op_reader_t: public msgr_op_reader_t { osd_messenger_t* msgr; @@ -531,9 +364,9 @@ public: from -= dst_len; return true; } - if ((flags & RDR_GCM) && (cl->ssl_cli || cl->gcm_enabled)) + if ((flags & RDR_GCM) && cl->gcm_enabled) { - // Can't inplace read TLS/GCM data + // Can't inplace read encrypted data return false; } if (cl->recv_list.size() >= IOV_MAX) @@ -753,11 +586,7 @@ void osd_messenger_t::handle_immediate_ops() bool osd_messenger_t::handle_read_buffer(osd_client_t *cl, uint8_t *curbuf, size_t bufsize) { - if (cl->ssl_cli) - { - return handle_buffer_with(cl, curbuf, bufsize); - } - else if (cl->gcm_enabled) + if (cl->gcm_enabled) { if (cl->hs) { diff --git a/src/client/msgr_send.cpp b/src/client/msgr_send.cpp index a464ba05..1e22e8e7 100644 --- a/src/client/msgr_send.cpp +++ b/src/client/msgr_send.cpp @@ -7,10 +7,8 @@ #include "messenger.h" -#include +#include #include -#include -#include #define WR_GCM 1 #define WR_XTS 2 @@ -35,8 +33,6 @@ protected: size_t done; public: - constexpr static bool is_ssl = false; - copy_op_writer_t(osd_messenger_t* msgr, osd_client_t* cl, uint8_t *curbuf, size_t bufsize): msgr(msgr), cl(cl), from(cl->write_op_pos), curbuf(curbuf), bufsize(bufsize), done(0) {} @@ -86,152 +82,6 @@ public: } }; -class ssl_op_writer_t: public msgr_op_writer_t -{ - osd_messenger_t* msgr; - osd_client_t* cl; - size_t from; - - uint8_t *curbuf; - size_t bufsize; - size_t done; - -public: - constexpr static bool is_ssl = true; - - ssl_op_writer_t(osd_messenger_t* msgr, osd_client_t* cl, uint8_t *curbuf, size_t bufsize): - msgr(msgr), cl(cl), from(cl->write_op_pos), curbuf(curbuf), bufsize(bufsize), done(0) - { - } - - void reset() - { - from = cl->write_op_pos; - } - - bool flush_ssl() - { - if (!cl->handshake_done) - { - if (!msgr->do_tls_handshake(cl)) - return false; - return _flush_ssl(); - } - return true; - } - - bool _flush_ssl() - { - int r = BIO_read(cl->read_from_ssl, curbuf+done, bufsize-done); - if (r > 0) - done += r; - if (done >= bufsize) - { - // Check if we've sent all buffered TLS data - // ...Because we can't return true from this->write() if we haven't - char *bio_buf = NULL; - size_t bio_sz = BIO_get_mem_data(cl->read_from_ssl, &bio_buf); - if (bio_sz > 0) - { - cl->ssl_more_to_buffer = true; - return false; - } - else - cl->ssl_more_to_buffer = false; - } - return true; - } - - static inline bool write_to_ssl(osd_client_t *cl, uint8_t *src, size_t src_len, int flags, size_t & from) - { - size_t n = src_len-from; - int ok = SSL_write_ex(cl->ssl_cli, src+from, n, &n); - if (ok) - { - if (cl->write_csum_state && !(flags & WR_NO_CSUM)) - XXH3_64bits_update(cl->write_csum_state, src+from, n); - cl->write_op_pos += n; - from += n; - } - else - { - ok = SSL_get_error(cl->ssl_cli, ok); - if (ok == SSL_ERROR_ZERO_RETURN) - { - fprintf(stderr, "Client %ju TLS disconnected\n", cl->client_id); - cl->io_error = true; - return false; - } - else if (ok != SSL_ERROR_WANT_READ && ok != SSL_ERROR_WANT_WRITE) - { - fprintf(stderr, "Client %ju TLS write error: %s. Disconnecting client\n", cl->client_id, ERR_error_string(ERR_get_error(), NULL)); - cl->io_error = true; - return false; - } - } - return true; - } - - bool write(uint8_t *src, size_t src_len, int flags) override - { - if (from >= src_len) - { - if (cl->ssl_more_to_buffer && !_flush_ssl()) - return false; - from -= src_len; - return true; - } - if (!(flags & WR_GCM) || !cl->ssl_cli) - { - if (flags & WR_XTS) - { - msgr->op_encrypted_copy_buf(cl, curbuf, bufsize, src, src_len, from, done); - } - else - { - size_t n = src_len-from; - if (n > bufsize-done) - n = bufsize-done; - if (cl->write_csum_state && !(flags & WR_NO_CSUM)) - XXH3_64bits_update(cl->write_csum_state, src+from, n); - memcpy(curbuf+done, src+from, n); - done += n; - cl->write_op_pos += n; - from += n; - } - } - else - { - if (!cl->handshake_done) - { - if (!flush_ssl()) - return false; - } - if (cl->handshake_done) - { - if (!write_to_ssl(cl, src, src_len, flags, from)) - return false; - } - if (!_flush_ssl()) - return false; - } - if (from < src_len) - return false; - from = 0; - return true; - } - - bool finish() override - { - return _flush_ssl(); - } - - size_t get_done() - { - return done; - } -}; - class gcm_op_writer_t: public msgr_op_writer_t { osd_messenger_t* msgr; @@ -243,8 +93,6 @@ class gcm_op_writer_t: public msgr_op_writer_t size_t done; public: - constexpr static bool is_ssl = false; - gcm_op_writer_t(osd_messenger_t* msgr, osd_client_t* cl, uint8_t *curbuf, size_t bufsize): msgr(msgr), cl(cl), from(cl->write_op_pos), curbuf(curbuf), bufsize(bufsize), done(0) { @@ -451,27 +299,7 @@ class get_op_writer_t: public msgr_op_writer_t size_t enc_size; size_t done_enc; - void copy_ssl() - { - size_t n = 0; - do - { - ssl_extend_buf(cl); - int r = BIO_read(cl->read_from_ssl, cl->ssl_out_buf+cl->ssl_out_buf_size, cl->ssl_out_buf_cap-cl->ssl_out_buf_size); - if (r > 0) - n += r; - } while (cl->ssl_out_buf_size+n >= cl->ssl_out_buf_cap); - cl->ssl_more_to_buffer = false; - if (n > 0) - { - send_out_buf(cl, n); - done += n; - } - } - public: - constexpr static bool is_ssl = true; - get_op_writer_t(osd_messenger_t* msgr, osd_client_t* cl, uint8_t*, size_t): msgr(msgr), cl(cl), from(cl->write_op_pos), done(0), enc_size(0), done_enc(0) { @@ -525,17 +353,6 @@ public: } } - bool flush_ssl() - { - if (cl->ssl_cli && !cl->handshake_done) - { - if (!msgr->do_tls_handshake(cl)) - return false; - copy_ssl(); - } - return true; - } - bool write(uint8_t *src, size_t src_len, int flags) override { if (from >= src_len) @@ -550,25 +367,6 @@ public: } if (flags & WR_GCM) { - if (cl->ssl_cli) - { - if (!cl->handshake_done) - { - if (!flush_ssl()) - return false; - } - if (cl->handshake_done) - { - if (!ssl_op_writer_t::write_to_ssl(cl, src, src_len, flags, from)) - return false; - } - // Copy data to client's temporary SSL output buffer - copy_ssl(); - if (from < src_len) - return false; - from = 0; - return true; - } if (cl->gcm_enabled) { // Encrypt data to client's temporary output buffer (all at once) @@ -634,13 +432,7 @@ public: bool finish() override { - if (cl->ssl_cli) - { - if (cl->send_list.size() >= IOV_MAX) - return false; - copy_ssl(); - } - else if (cl->enc_ctx) + if (cl->enc_ctx) { if (cl->send_list.size() >= IOV_MAX) return false; @@ -871,10 +663,6 @@ copy_ops: size_t osd_messenger_t::copy_ops_to(osd_client_t *cl, uint8_t *dst, size_t dst_len) { - if (cl->ssl_cli) - { - return copy_ops_to_with(cl, dst, dst_len); - } if (cl->gcm_enabled) { if (cl->hs) @@ -924,13 +712,6 @@ size_t osd_messenger_t::copy_ops_to_with(osd_client_t *cl, uint8_t *dst, size_t cl->send_free_ops.push_back(op); } } - if constexpr (T::is_ssl) - { - if (!wr.get_done()) - { - wr.flush_ssl(); - } - } return wr.get_done(); } diff --git a/src/client/msgr_stop.cpp b/src/client/msgr_stop.cpp index 92bc60aa..7e81dcc1 100644 --- a/src/client/msgr_stop.cpp +++ b/src/client/msgr_stop.cpp @@ -9,10 +9,8 @@ #ifdef WITH_RDMA #include "msgr_rdma.h" #endif -#include -#include +#include #include -#include void osd_client_t::cancel_ops() { @@ -264,13 +262,6 @@ osd_client_t::~osd_client_t() #endif dec_ctx = NULL; } - if (ssl_cli) - { - SSL_free(ssl_cli); - ssl_cli = NULL; - write_to_ssl = NULL; - read_from_ssl = NULL; - } if (ssl_out_buf) { free(ssl_out_buf);