Support isa-l_crypto for AES-GCM

debian/vitastor-buildenv.Dockerfile

@@ -12,20 +12,18 @@ ARG REL=
 WORKDIR /root
 
 RUN set -e -x; \
-    if [ "$REL" = "buster" ]; then \
+    perl -i -pe 's/deb.debian.org/archive.debian.org/' /etc/apt/sources.list; \
-        perl -i -pe 's/deb.debian.org/archive.debian.org/' /etc/apt/sources.list; \
+    apt-get update; \
-        apt-get update; \
+    apt-get -y install wget; \
-        apt-get -y install wget; \
+    wget https://vitastor.io/debian/pubkey.gpg -O /etc/apt/trusted.gpg.d/vitastor.gpg; \
-        wget https://vitastor.io/debian/pubkey.gpg -O /etc/apt/trusted.gpg.d/vitastor.gpg; \
+    echo "deb https://vitastor.io/debian $REL main" >> /etc/apt/sources.list; \
-        echo "deb https://vitastor.io/debian $REL main" >> /etc/apt/sources.list; \
-    fi; \
     grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb/deb-src/' >> /etc/apt/sources.list; \
     perl -i -pe 's/Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/*.sources || true; \
     echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf; \
     echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
 
 RUN apt-get update && \
-    apt-get -y install fio libgoogle-perftools-dev devscripts libjerasure-dev cmake libc-ares-dev \
+    apt-get -y install fio libgoogle-perftools-dev devscripts libjerasure-dev cmake libc-ares-dev libisal-crypto-dev \
         libibverbs-dev librdmacm-dev libisal-dev libnl-3-dev libnl-genl-3-dev curl nodejs npm node-nan node-bindings && \
     apt-get -y build-dep fio && \
     apt-get --download-only source fio

src/CMakeLists.txt

@@ -69,6 +69,10 @@ pkg_check_modules(ISAL libisal)
 if (ISAL_LIBRARIES)
 	add_definitions(-DWITH_ISAL)
 endif (ISAL_LIBRARIES)
+pkg_check_modules(ISAL_CRYPTO libisal_crypto)
+if (ISAL_CRYPTO_LIBRARIES)
+	add_definitions(-DWITH_ISAL_CRYPTO)
+endif (ISAL_CRYPTO_LIBRARIES)
 pkg_check_modules(RDMACM librdmacm)
 if (RDMACM_LIBRARIES)
 	add_definitions(-DWITH_RDMACM)

src/client/CMakeLists.txt

@@ -16,7 +16,7 @@ add_library(vitastor_common STATIC
 	msgr_encrypt.cpp msgr_stop.cpp msgr_op.cpp msgr_send.cpp msgr_receive.cpp ../util/ringloop.cpp ../../json11/json11.cpp
 	http_client.cpp osd_ops.cpp pg_states.cpp ../util/timerfd_manager.cpp ../util/str_util.cpp ../util/json_util.cpp ${MSGR_RDMA} ${MSGR_RDMACM}
 )
-target_link_libraries(vitastor_common pthread ${OPENSSL_LIBRARIES} ${CARES_LIBRARIES})
+target_link_libraries(vitastor_common pthread ${OPENSSL_LIBRARIES} ${CARES_LIBRARIES} ${ISAL_CRYPTO_LIBRARIES})
 target_compile_options(vitastor_common PUBLIC -fPIC)
 
 # libvitastor_client.so
@@ -35,6 +35,7 @@ target_link_libraries(vitastor_client
 	${IBVERBS_LIBRARIES}
 	${RDMACM_LIBRARIES}
 	${OPENSSL_LIBRARIES}
+	${ISAL_CRYPTO_LIBRARIES}
 )
 set_target_properties(vitastor_client PROPERTIES VERSION ${VITASTOR_VERSION} SOVERSION 0)
 configure_file(vitastor.pc.in vitastor.pc @ONLY)

src/client/messenger.cpp

@@ -347,6 +347,16 @@ osd_messenger_t::~osd_messenger_t()
     {
         destroy_aes_xts_decrypt(decrypt_ctx);
     }
+#ifdef WITH_ISAL_CRYPTO
+    for (isal_gcm_context_data *ctx: encrypt_gcm_pool)
+    {
+        free(ctx);
+    }
+    for (isal_gcm_context_data *ctx: decrypt_gcm_pool)
+    {
+        free(ctx);
+    }
+#else
     for (EVP_CIPHER_CTX *ctx: encrypt_gcm_pool)
     {
         EVP_CIPHER_CTX_free(ctx);
@@ -355,6 +365,7 @@ osd_messenger_t::~osd_messenger_t()
     {
         EVP_CIPHER_CTX_free(ctx);
     }
+#endif
     if (ssl_ctx)
     {
         SSL_CTX_free(ssl_ctx);
@@ -421,6 +432,12 @@ void osd_messenger_t::parse_config(const json11::Json & config)
     test_osd_aes_key.resize(32);
     if (fromhexstr(config["test_osd_aes_key"].string_value(), 32, (uint8_t*)test_osd_aes_key.data()) != 32)
         test_osd_aes_key.clear();
+    else
+    {
+#ifdef WITH_ISAL_CRYPTO
+        isal_aes_gcm_pre_256(test_osd_aes_key.data(), &test_osd_aes_key_isal);
+#endif
+    }
     if (!osd_num)
         this->iothread_count = (uint32_t)config["client_iothread_count"].uint64_value();
     else

src/client/messenger.h

@@ -14,6 +14,10 @@
 
 #include <openssl/types.h>
 
+#ifdef WITH_ISAL_CRYPTO
+#include <isa-l_crypto/aes_gcm.h>
+#endif
+
 #include "../util/xxh_x86dispatch.h"
 #include "../util/robin_hood.h"
 #include "malloc_or_die.h"
@@ -100,10 +104,15 @@ struct osd_client_t
     bool ssl_more_to_buffer = false;
 
     bool gcm_enabled = false;
+#ifdef WITH_ISAL_CRYPTO
+    isal_gcm_context_data *enc_ctx = NULL;
+    isal_gcm_context_data *dec_ctx = NULL;
+#else
     EVP_CIPHER_CTX *enc_ctx = NULL;
+    EVP_CIPHER_CTX *dec_ctx = NULL;
+#endif
     uint8_t enc_tag[16];
     size_t enc_tag_size = 0;
-    EVP_CIPHER_CTX *dec_ctx = NULL;
     uint8_t dec_tag[16];
     size_t dec_tag_size = 0;
 
@@ -258,6 +267,9 @@ protected:
     std::string osd_tls_ca;
     std::string client_tls_ca;
     std::string test_osd_aes_key; // FIXME Insecure, only for PoC tests
+#ifdef WITH_ISAL_CRYPTO
+    isal_gcm_key_data test_osd_aes_key_isal;
+#endif
 
 #ifdef WITH_RDMA
     bool use_rdma = true;
@@ -290,8 +302,13 @@ protected:
     std::vector<op_aes_xts_encrypt_t*> encrypt_xts_pool;
     std::vector<op_aes_xts_decrypt_t*> decrypt_xts_pool;
 
+#ifdef WITH_ISAL_CRYPTO
+    std::vector<isal_gcm_context_data*> encrypt_gcm_pool;
+    std::vector<isal_gcm_context_data*> decrypt_gcm_pool;
+#else
     std::vector<EVP_CIPHER_CTX*> encrypt_gcm_pool;
     std::vector<EVP_CIPHER_CTX*> decrypt_gcm_pool;
+#endif
 
 public:
     timerfd_manager_t *tfd = NULL;

src/client/msgr_receive.cpp

@@ -294,6 +294,9 @@ public:
             }
             else
             {
+#ifdef WITH_ISAL_CRYPTO
+                cl->dec_ctx = (isal_gcm_context_data*)malloc_or_die(sizeof(isal_gcm_context_data));
+#else
                 cl->dec_ctx = EVP_CIPHER_CTX_new();
                 assert(cl->dec_ctx);
                 int r = EVP_DecryptInit_ex(cl->dec_ctx, EVP_aes_256_gcm(), NULL, NULL, NULL);
@@ -303,9 +306,18 @@ public:
                     ERR_print_errors_fp(stderr);
                     abort();
                 }
+#endif
             }
         }
         uint8_t iv[12] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 };
+#ifdef WITH_ISAL_CRYPTO
+        int r = isal_aes_gcm_init_256(&msgr->test_osd_aes_key_isal, cl->dec_ctx, iv, NULL, 0);
+        if (r != 0)
+        {
+            fprintf(stderr, "isal_aes_gcm_init_256 error %d\n", r);
+            abort();
+        }
+#else
         int r = EVP_DecryptInit_ex(cl->dec_ctx, NULL, NULL, (uint8_t*)msgr->test_osd_aes_key.data(), iv);
         if (r != 1)
         {
@@ -313,6 +325,7 @@ public:
             ERR_print_errors_fp(stderr);
             abort();
         }
+#endif
     }
 
     bool read(uint8_t *dst, size_t dst_len, int flags) override
@@ -361,6 +374,10 @@ public:
             size_t n = dst_len-from;
             if (n > bufsize-done)
                 n = bufsize-done;
+#ifdef WITH_ISAL_CRYPTO
+            int r = isal_aes_gcm_dec_256_update(&msgr->test_osd_aes_key_isal, cl->dec_ctx, dst+from, curbuf+done, n);
+            assert(!r);
+#else
             int actual_out;
             if (EVP_DecryptUpdate(cl->dec_ctx, dst+from, &actual_out, curbuf+done, n) != 1)
             {
@@ -369,6 +386,7 @@ public:
                 abort();
             }
             assert(actual_out == n);
+#endif
             if (cl->read_csum_state && !(flags & RDR_NO_CSUM))
             {
                 XXH3_64bits_update(cl->read_csum_state, dst+from, n);
@@ -395,23 +413,44 @@ public:
             done = bufsize;
             return false;
         }
+#ifdef WITH_ISAL_CRYPTO
+        uint8_t calc_tag[16];
+        int r = isal_aes_gcm_dec_256_finalize(&msgr->test_osd_aes_key_isal, cl->dec_ctx, calc_tag, 16);
+        assert(r == 0);
+        if (cl->dec_tag_size > 0)
+        {
+            // Tag is partially buffered, append to it and compare
+            memcpy(cl->dec_tag+cl->dec_tag_size, curbuf+done, 16-cl->dec_tag_size);
+            done += 16-cl->dec_tag_size;
+            r = !memcmp(calc_tag, cl->dec_tag, 16);
+        }
+        else
+        {
+            // Compare the full tag directly from the source buffer
+            r = !memcmp(calc_tag, curbuf+done, 16);
+            done += 16;
+        }
+#else
         int r;
         if (cl->dec_tag_size > 0)
         {
             // Tag is partially buffered, append to it and use it from there
             memcpy(cl->dec_tag+cl->dec_tag_size, curbuf+done, 16-cl->dec_tag_size);
-            done += 16-cl->dec_tag_size;
             r = EVP_CIPHER_CTX_ctrl(cl->dec_ctx, EVP_CTRL_GCM_SET_TAG, 16, cl->dec_tag);
+            assert(r == 1);
+            done += 16-cl->dec_tag_size;
         }
         else
         {
             // Take full tag directly from the source buffer
             r = EVP_CIPHER_CTX_ctrl(cl->dec_ctx, EVP_CTRL_GCM_SET_TAG, 16, curbuf+done);
+            assert(r == 1);
             done += 16;
         }
-        assert(r == 1);
         int len = 0;
         r = EVP_DecryptFinal_ex(cl->dec_ctx, NULL, &len);
+        assert(len == 0);
+#endif
         if (r != 1)
         {
             fprintf(stderr, "Client %ju AES-GCM decryption failed\n", cl->client_id);
@@ -421,10 +460,15 @@ public:
         if (msgr->decrypt_gcm_pool.size() < msgr->max_cipher_pool_size)
             msgr->decrypt_gcm_pool.push_back(cl->dec_ctx);
         else
+        {
+#ifdef WITH_ISAL_CRYPTO
+            free(cl->dec_ctx);
+#else
             EVP_CIPHER_CTX_free(cl->dec_ctx);
+#endif
+        }
         cl->dec_ctx = NULL;
         cl->dec_tag_size = 0;
-        assert(len == 0);
         return true;
     }
 

src/client/msgr_send.cpp

@@ -258,6 +258,9 @@ public:
             }
             else
             {
+#ifdef WITH_ISAL_CRYPTO
+                cl->enc_ctx = (isal_gcm_context_data*)malloc_or_die(sizeof(isal_gcm_context_data));
+#else
                 cl->enc_ctx = EVP_CIPHER_CTX_new();
                 assert(cl->enc_ctx);
                 int r = EVP_EncryptInit_ex(cl->enc_ctx, EVP_aes_256_gcm(), NULL, NULL, NULL);
@@ -267,9 +270,18 @@ public:
                     ERR_print_errors_fp(stderr);
                     abort();
                 }
+#endif
             }
         }
         uint8_t iv[12] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 };
+#ifdef WITH_ISAL_CRYPTO
+        int r = isal_aes_gcm_init_256(&msgr->test_osd_aes_key_isal, cl->enc_ctx, iv, NULL, 0);
+        if (r != 0)
+        {
+            fprintf(stderr, "isal_aes_gcm_init_256 error %d\n", r);
+            abort();
+        }
+#else
         int r = EVP_EncryptInit_ex(cl->enc_ctx, NULL, NULL, (uint8_t*)msgr->test_osd_aes_key.data(), iv);
         if (r != 1)
         {
@@ -277,6 +289,7 @@ public:
             ERR_print_errors_fp(stderr);
             abort();
         }
+#endif
     }
 
     static void free_ctx(osd_messenger_t* msgr, osd_client_t *cl)
@@ -284,7 +297,13 @@ public:
         if (msgr->encrypt_gcm_pool.size() < msgr->max_cipher_pool_size)
             msgr->encrypt_gcm_pool.push_back(cl->enc_ctx);
         else
+        {
+#ifdef WITH_ISAL_CRYPTO
+            free(cl->enc_ctx);
+#else
             EVP_CIPHER_CTX_free(cl->enc_ctx);
+#endif
+        }
         cl->enc_ctx = NULL;
     }
 
@@ -323,6 +342,10 @@ public:
                 n = bufsize-done;
             if (!n)
                 return false;
+#ifdef WITH_ISAL_CRYPTO
+            int r = isal_aes_gcm_enc_256_update(&msgr->test_osd_aes_key_isal, cl->enc_ctx, curbuf+done, src+from, n);
+            assert(!r);
+#else
             int actual_out;
             if (EVP_EncryptUpdate(cl->enc_ctx, curbuf+done, &actual_out, src+from, n) != 1)
             {
@@ -331,6 +354,7 @@ public:
                 abort();
             }
             assert(actual_out == n);
+#endif
             if (cl->write_csum_state && !(flags & WR_NO_CSUM))
                 XXH3_64bits_update(cl->write_csum_state, src+from, n);
             done += n;
@@ -343,8 +367,12 @@ public:
         return true;
     }
 
-    static void write_tag_to(osd_client_t *cl, uint8_t *dst)
+    static void write_tag_to(osd_messenger_t *msgr, osd_client_t *cl, uint8_t *dst)
     {
+#ifdef WITH_ISAL_CRYPTO
+        int r = isal_aes_gcm_enc_256_finalize(&msgr->test_osd_aes_key_isal, cl->enc_ctx, dst, 16);
+        assert(!r);
+#else
         int actual_out = 0;
         int r = EVP_EncryptFinal_ex(cl->enc_ctx, NULL, &actual_out);
         if (r != 1)
@@ -356,6 +384,7 @@ public:
         assert(actual_out == 0);
         r = EVP_CIPHER_CTX_ctrl(cl->enc_ctx, EVP_CTRL_GCM_GET_TAG, 16, dst);
         assert(r == 1);
+#endif
     }
 
     bool finish() override
@@ -368,7 +397,7 @@ public:
             // No space for the full tag, but msgr_rdma expects us to always fill the whole buffer
             if (!cl->enc_tag_size)
             {
-                write_tag_to(cl, cl->enc_tag);
+                write_tag_to(msgr, cl, cl->enc_tag);
                 cl->enc_tag_size = 16;
             }
             size_t n = bufsize-done;
@@ -383,7 +412,7 @@ public:
         else
         {
             // The whole tag fits at once
-            write_tag_to(cl, curbuf+done);
+            write_tag_to(msgr, cl, curbuf+done);
             done += 16;
         }
         free_ctx(msgr, cl);
@@ -511,6 +540,10 @@ public:
                 // Encrypt data to client's temporary output buffer (all at once)
                 size_t n = src_len-from;
                 ssl_extend_buf(n);
+#ifdef WITH_ISAL_CRYPTO
+                int r = isal_aes_gcm_enc_256_update(&msgr->test_osd_aes_key_isal, cl->enc_ctx, cl->ssl_out_buf+cl->ssl_out_buf_size, src+from, n);
+                assert(!r);
+#else
                 int actual_out;
                 if (EVP_EncryptUpdate(cl->enc_ctx, cl->ssl_out_buf+cl->ssl_out_buf_size, &actual_out, src+from, n) != 1)
                 {
@@ -519,6 +552,7 @@ public:
                     abort();
                 }
                 assert(actual_out == n);
+#endif
                 if (cl->write_csum_state && !(flags & WR_NO_CSUM))
                     XXH3_64bits_update(cl->write_csum_state, src+from, n);
                 cl->send_list.push_back((iovec){ .iov_base = cl->ssl_out_buf+cl->ssl_out_buf_size, .iov_len = n });
@@ -576,7 +610,7 @@ public:
                 return false;
             // Tag is 16 bytes
             ssl_extend_buf(16);
-            gcm_op_writer_t::write_tag_to(cl, cl->ssl_out_buf+cl->ssl_out_buf_size);
+            gcm_op_writer_t::write_tag_to(msgr, cl, cl->ssl_out_buf+cl->ssl_out_buf_size);
             // FIXME coalesce entries in ssl_out_buf
             cl->send_list.push_back((iovec){ .iov_base = cl->ssl_out_buf+cl->ssl_out_buf_size, .iov_len = 16 });
             cl->ssl_out_buf_size += 16;

src/client/msgr_stop.cpp

@@ -248,12 +248,20 @@ osd_client_t::~osd_client_t()
     }
     if (enc_ctx)
     {
+#ifdef WITH_ISAL_CRYPTO
+        free(enc_ctx);
+#else
         EVP_CIPHER_CTX_free(enc_ctx);
+#endif
         enc_ctx = NULL;
     }
     if (dec_ctx)
     {
+#ifdef WITH_ISAL_CRYPTO
+        free(dec_ctx);
+#else
         EVP_CIPHER_CTX_free(dec_ctx);
+#endif
         dec_ctx = NULL;
     }
     if (ssl_cli)