Allow to create OSD with mocked blockstore and network

src/blockstore/blockstore_mock.cpp

@@ -0,0 +1,113 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 (see README.md for details)
+
+#include "blockstore_mock.h"
+
+blockstore_mock_t::blockstore_mock_t(const blockstore_config_t & config)
+{
+}
+
+void blockstore_mock_t::parse_config(blockstore_config_t & config)
+{
+}
+
+void* blockstore_mock_t::reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit)
+{
+    return NULL;
+}
+
+bool blockstore_mock_t::reshard_continue(void *reshard_state, uint64_t chunk_limit)
+{
+    return true;
+}
+
+void blockstore_mock_t::loop()
+{
+}
+
+bool blockstore_mock_t::is_started()
+{
+    return true;
+}
+
+bool blockstore_mock_t::is_stalled()
+{
+    return false;
+}
+
+bool blockstore_mock_t::is_safe_to_stop()
+{
+    return true;
+}
+
+void blockstore_mock_t::enqueue_op(blockstore_op_t *op)
+{
+}
+
+int blockstore_mock_t::read_bitmap(object_id oid, uint64_t target_version, void *bitmap, uint64_t *result_version)
+{
+    return -EIO;
+}
+
+const std::map<uint64_t, uint64_t> & blockstore_mock_t::get_inode_space_stats()
+{
+    return inode_space;
+}
+
+void blockstore_mock_t::set_no_inode_stats(const std::vector<uint64_t> & pool_ids)
+{
+}
+
+void blockstore_mock_t::dump_diagnostics()
+{
+}
+
+std::string blockstore_mock_t::get_op_diag(blockstore_op_t *op)
+{
+    return "";
+}
+
+uint32_t blockstore_mock_t::get_block_size()
+{
+    return 0;
+}
+
+uint64_t blockstore_mock_t::get_block_count()
+{
+    return 0;
+}
+
+uint64_t blockstore_mock_t::get_free_block_count()
+{
+    return 0;
+}
+
+uint64_t blockstore_mock_t::get_journal_size()
+{
+    return 0;
+}
+
+uint32_t blockstore_mock_t::get_bitmap_granularity()
+{
+    return 0;
+}
+
+uint64_t blockstore_mock_t::get_live_entries()
+{
+    return 0;
+}
+
+uint64_t blockstore_mock_t::get_live_memory()
+{
+    return 0;
+}
+
+uint64_t blockstore_mock_t::get_garbage_entries()
+{
+    return 0;
+}
+
+uint64_t blockstore_mock_t::get_garbage_memory()
+{
+    return 0;
+}

src/blockstore/blockstore_mock.h

@@ -0,0 +1,36 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 (see README.md for details)
+
+#pragma once
+
+#include "blockstore.h"
+
+class blockstore_mock_t: public blockstore_i
+{
+public:
+    std::map<uint64_t, uint64_t> inode_space;
+
+    blockstore_mock_t(const blockstore_config_t & config);
+    void parse_config(blockstore_config_t & config) override;
+    void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit) override;
+    bool reshard_continue(void *reshard_state, uint64_t chunk_limit) override;
+    void loop() override;
+    bool is_started() override;
+    bool is_stalled() override;
+    bool is_safe_to_stop() override;
+    void enqueue_op(blockstore_op_t *op) override;
+    int read_bitmap(object_id oid, uint64_t target_version, void *bitmap, uint64_t *result_version = NULL) override;
+    const std::map<uint64_t, uint64_t> & get_inode_space_stats() override;
+    void set_no_inode_stats(const std::vector<uint64_t> & pool_ids) override;
+    void dump_diagnostics() override;
+    std::string get_op_diag(blockstore_op_t *op) override;
+    uint32_t get_block_size() override;
+    uint64_t get_block_count() override;
+    uint64_t get_free_block_count() override;
+    uint64_t get_journal_size() override;
+    uint32_t get_bitmap_granularity() override;
+    uint64_t get_live_entries() override;
+    uint64_t get_live_memory() override;
+    uint64_t get_garbage_entries() override;
+    uint64_t get_garbage_memory() override;
+};

src/client/CMakeLists.txt

@@ -3,6 +3,22 @@ cmake_minimum_required(VERSION 2.8...3.30)
 project(vitastor)
 
 # libvitastor_common.a
+add_library(vitastor_common STATIC
+	etcd_state_client.cpp
+	msgr_stop.cpp
+	msgr_op.cpp
+	../../json11/json11.cpp
+	osd_ops.cpp
+	pg_states.cpp
+	../util/allocator.cpp
+	../util/addr_util.cpp
+	../util/timerfd_manager.cpp
+	../util/str_util.cpp
+	../util/json_util.cpp
+)
+target_compile_options(vitastor_common PUBLIC -fPIC)
+
+# libvitastor_net.a
 set(MSGR_RDMA "")
 if (IBVERBS_LIBRARIES)
 	set(MSGR_RDMA "msgr_rdma.cpp")
@@ -11,13 +27,20 @@ set(MSGR_RDMACM "")
 if (RDMACM_LIBRARIES)
 	set(MSGR_RDMACM "msgr_rdmacm.cpp")
 endif (RDMACM_LIBRARIES)
-add_library(vitastor_common STATIC
+add_library(vitastor_net STATIC
-	../util/epoll_manager.cpp etcd_state_client.cpp etcd_state_client_http.cpp messenger.cpp msgr_iothread.cpp ../util/addr_util.cpp
+	../util/epoll_manager.cpp
-	msgr_stop.cpp msgr_op.cpp msgr_send.cpp msgr_receive.cpp ../util/ringloop.cpp ../../json11/json11.cpp
+	etcd_state_client_http.cpp
-	http_client.cpp osd_ops.cpp pg_states.cpp ../util/timerfd_manager.cpp ../util/str_util.cpp ../util/json_util.cpp ${MSGR_RDMA} ${MSGR_RDMACM}
+	messenger.cpp
+	msgr_iothread.cpp
+	msgr_send.cpp
+	msgr_receive.cpp
+	../util/ringloop.cpp
+	http_client.cpp
+	${MSGR_RDMA}
+	${MSGR_RDMACM}
 )
-target_link_libraries(vitastor_common pthread)
+target_link_libraries(vitastor_net pthread vitastor_common)
-target_compile_options(vitastor_common PUBLIC -fPIC)
+target_compile_options(vitastor_net PUBLIC -fPIC)
 
 # libvitastor_client.so
 add_library(vitastor_client SHARED
@@ -29,7 +52,7 @@ add_library(vitastor_client SHARED
 )
 set_target_properties(vitastor_client PROPERTIES PUBLIC_HEADER "client/vitastor_c.h")
 target_link_libraries(vitastor_client
-	vitastor_common
+	vitastor_net
 	vitastor_cli
 	${LIBURING_LIBRARIES}
 	${IBVERBS_LIBRARIES}
@@ -96,10 +119,13 @@ endif (${WITH_QEMU})
 add_executable(test_cluster_client
 	EXCLUDE_FROM_ALL
 	../test/test_cluster_client.cpp
-	pg_states.cpp osd_ops.cpp cluster_client.cpp cluster_client_list.cpp cluster_client_wb.cpp msgr_op.cpp ../test/mock/messenger.cpp msgr_stop.cpp
+	cluster_client.cpp
-	etcd_state_client.cpp etcd_state_client_mock.cpp ../util/timerfd_manager.cpp ../util/addr_util.cpp ../util/str_util.cpp ../util/json_util.cpp ../../json11/json11.cpp
+	cluster_client_list.cpp
+	cluster_client_wb.cpp
+	../test/mock/messenger.cpp
+	etcd_state_client_mock.cpp
 )
-target_link_libraries(test_cluster_client ${LIBURING_LIBRARIES})
+target_link_libraries(test_cluster_client vitastor_common ${LIBURING_LIBRARIES})
 target_include_directories(test_cluster_client BEFORE PUBLIC ${CMAKE_SOURCE_DIR}/src/test/mock)
 add_dependencies(build_tests test_cluster_client)
 add_test(NAME test_cluster_client COMMAND test_cluster_client)

src/client/messenger.h

@@ -222,7 +222,7 @@ public:
 
 #ifdef WITH_RDMA
     bool is_rdma_enabled();
-    bool connect_rdma(uint64_t client_id, std::string rdma_address, uint64_t client_max_msg);
+    json11::Json connect_rdma(uint64_t client_id, std::string rdma_address, uint64_t client_max_msg);
 #endif
 #ifdef WITH_RDMACM
     bool is_use_rdmacm();

src/client/msgr_op.cpp

@@ -3,6 +3,7 @@
 
 #include <assert.h>
 
+#include "messenger.h"
 #include "msgr_op.h"
 
 osd_op_t::~osd_op_t()
@@ -38,3 +39,52 @@ bool osd_op_t::is_recovery_related()
         req.hdr.opcode == OSD_OP_SEC_SYNC &&
         (req.sec_sync.flags & OSD_OP_RECOVERY_RELATED);
 }
+
+void osd_messenger_t::measure_exec(osd_op_t *cur_op)
+{
+    // Measure execution latency
+    if (cur_op->req.hdr.opcode > OSD_OP_MAX)
+    {
+        return;
+    }
+    if (!cur_op->tv_end.tv_sec)
+    {
+        clock_gettime(CLOCK_REALTIME, &cur_op->tv_end);
+    }
+    uint64_t len = 0;
+    if (cur_op->req.hdr.opcode == OSD_OP_READ ||
+        cur_op->req.hdr.opcode == OSD_OP_WRITE ||
+        cur_op->req.hdr.opcode == OSD_OP_SCRUB)
+    {
+        // req.rw.len is internally set to the full object size for scrubs
+        len = cur_op->req.rw.len;
+    }
+    else if (cur_op->req.hdr.opcode == OSD_OP_SEC_READ ||
+        cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE ||
+        cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE_STABLE)
+    {
+        len = cur_op->req.sec_rw.len;
+    }
+    inc_op_stats(stats, cur_op->req.hdr.opcode, cur_op->tv_begin, cur_op->tv_end, len);
+    if (cur_op->is_recovery_related())
+    {
+        inc_op_stats(recovery_stats, cur_op->req.hdr.opcode, cur_op->tv_begin, cur_op->tv_end, len);
+    }
+}
+
+void osd_messenger_t::inc_op_stats(osd_op_stats_t & stats, uint64_t opcode, timespec & tv_begin, timespec & tv_end, uint64_t len)
+{
+    uint64_t usecs = (
+        (tv_end.tv_sec - tv_begin.tv_sec)*1000000 +
+        (tv_end.tv_nsec - tv_begin.tv_nsec)/1000
+    );
+    stats.op_stat_count[opcode]++;
+    if (!stats.op_stat_count[opcode])
+    {
+        stats.op_stat_count[opcode] = 1;
+        stats.op_stat_sum[opcode] = 0;
+        stats.op_stat_bytes[opcode] = 0;
+    }
+    stats.op_stat_sum[opcode] += usecs;
+    stats.op_stat_bytes[opcode] += len;
+}

src/client/msgr_rdma.cpp

@@ -507,7 +507,7 @@ int msgr_rdma_connection_t::connect(msgr_rdma_address_t *dest)
     return 0;
 }
 
-bool osd_messenger_t::connect_rdma(uint64_t client_id, std::string rdma_address, uint64_t client_max_msg)
+json11::Json osd_messenger_t::connect_rdma(uint64_t client_id, std::string rdma_address, uint64_t client_max_msg)
 {
     // Try to connect to the peer using RDMA
     msgr_rdma_address_t addr;
@@ -523,7 +523,7 @@ bool osd_messenger_t::connect_rdma(uint64_t client_id, std::string rdma_address,
         {
             if (log_level > 0)
                 fprintf(stderr, "No RDMA context for peer %ju, using only TCP\n", client_id);
-            return false;
+            return json11::Json();
         }
         msgr_rdma_connection_t *rdma_conn = msgr_rdma_connection_t::create(selected_ctx, rdma_max_send, rdma_max_recv, rdma_max_sge, client_max_msg);
         if (rdma_conn)
@@ -542,11 +542,14 @@ bool osd_messenger_t::connect_rdma(uint64_t client_id, std::string rdma_address,
                 // Remember connection, but switch to RDMA only after sending the configuration response
                 cl->rdma_conn = rdma_conn;
                 cl->peer_state = PEER_RDMA_CONNECTING;
-                return true;
+                return json11::Json::object{
+                    {"rdma_address", rdma_conn->addr.to_string()},
+                    {"rdma_max_msg", rdma_conn->max_msg},
+                };
             }
         }
     }
-    return false;
+    return json11::Json();
 }
 
 static void try_send_rdma_wr(osd_client_t *cl, ibv_sge *sge, int op_sge)

src/client/msgr_send.cpp

@@ -140,55 +140,6 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
     }
 }
 
-void osd_messenger_t::inc_op_stats(osd_op_stats_t & stats, uint64_t opcode, timespec & tv_begin, timespec & tv_end, uint64_t len)
-{
-    uint64_t usecs = (
-        (tv_end.tv_sec - tv_begin.tv_sec)*1000000 +
-        (tv_end.tv_nsec - tv_begin.tv_nsec)/1000
-    );
-    stats.op_stat_count[opcode]++;
-    if (!stats.op_stat_count[opcode])
-    {
-        stats.op_stat_count[opcode] = 1;
-        stats.op_stat_sum[opcode] = 0;
-        stats.op_stat_bytes[opcode] = 0;
-    }
-    stats.op_stat_sum[opcode] += usecs;
-    stats.op_stat_bytes[opcode] += len;
-}
-
-void osd_messenger_t::measure_exec(osd_op_t *cur_op)
-{
-    // Measure execution latency
-    if (cur_op->req.hdr.opcode > OSD_OP_MAX)
-    {
-        return;
-    }
-    if (!cur_op->tv_end.tv_sec)
-    {
-        clock_gettime(CLOCK_REALTIME, &cur_op->tv_end);
-    }
-    uint64_t len = 0;
-    if (cur_op->req.hdr.opcode == OSD_OP_READ ||
-        cur_op->req.hdr.opcode == OSD_OP_WRITE ||
-        cur_op->req.hdr.opcode == OSD_OP_SCRUB)
-    {
-        // req.rw.len is internally set to the full object size for scrubs
-        len = cur_op->req.rw.len;
-    }
-    else if (cur_op->req.hdr.opcode == OSD_OP_SEC_READ ||
-        cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE ||
-        cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE_STABLE)
-    {
-        len = cur_op->req.sec_rw.len;
-    }
-    inc_op_stats(stats, cur_op->req.hdr.opcode, cur_op->tv_begin, cur_op->tv_end, len);
-    if (cur_op->is_recovery_related())
-    {
-        inc_op_stats(recovery_stats, cur_op->req.hdr.opcode, cur_op->tv_begin, cur_op->tv_end, len);
-    }
-}
-
 bool osd_messenger_t::try_send(osd_client_t *cl)
 {
     if (!cl->send_list.size() || cl->write_msg.msg_iovlen > 0 || cl->peer_state == PEER_STOPPED || cl->peer_fd < 0)

src/osd/CMakeLists.txt

@@ -2,14 +2,22 @@ cmake_minimum_required(VERSION 2.8...3.30)
 
 project(vitastor)
 
-# vitastor-osd
+# libosd
-add_executable(vitastor-osd
+add_library(osd STATIC
-	osd_main.cpp osd.cpp osd_secondary.cpp osd_peering.cpp osd_flush.cpp osd_peering_pg.cpp
+	osd.cpp osd_secondary.cpp osd_peering.cpp osd_flush.cpp osd_peering_pg.cpp
 	osd_primary.cpp osd_primary_chain.cpp osd_primary_sync.cpp osd_primary_write.cpp osd_primary_subops.cpp
 	osd_cluster.cpp osd_rmw.cpp osd_scrub.cpp osd_primary_describe.cpp
 )
+target_link_libraries(osd pthread)
+target_compile_options(osd PUBLIC -fPIC)
+
+# vitastor-osd
+add_executable(vitastor-osd
+	osd_main.cpp
+)
 target_link_libraries(vitastor-osd
-	vitastor_common
+	osd
+	vitastor_net
 	vitastor_blk
 	Jerasure
 	${ISAL_LIBRARIES}
@@ -17,6 +25,22 @@ target_link_libraries(vitastor-osd
 	${RDMACM_LIBRARIES}
 )
 
+# osd_test
+add_executable(osd_test
+	EXCLUDE_FROM_ALL
+	../test/osd_test.cpp
+	../client/etcd_state_client_mock.cpp
+	../blockstore/blockstore_mock.cpp
+	../test/mock/messenger.cpp
+	../test/ringloop_mock.cpp
+)
+target_link_libraries(osd_test
+	osd
+	vitastor_common
+	Jerasure
+	${ISAL_LIBRARIES}
+)
+
 # osd_rmw_test
 add_executable(osd_rmw_test EXCLUDE_FROM_ALL osd_rmw_test.cpp ../util/allocator.cpp)
 target_link_libraries(osd_rmw_test Jerasure ${ISAL_LIBRARIES})

src/osd/osd.cpp

@@ -15,7 +15,8 @@
 #include "str_util.h"
 #include "json_util.h"
 
-osd_t::osd_t(const json11::Json & config, ring_loop_i *ringloop, timerfd_manager_t *tfd, std::unique_ptr<etcd_state_client_t> st_cli_ptr)
+osd_t::osd_t(const json11::Json & config, ring_loop_i *ringloop, timerfd_manager_t *tfd,
+    std::unique_ptr<etcd_state_client_t> st_cli_ptr, std::function<blockstore_i*(blockstore_config_t & config)> bs_factory)
 {
     zero_buffer_size = 1<<20;
     zero_buffer = malloc_or_die(zero_buffer_size);
@@ -24,6 +25,7 @@ osd_t::osd_t(const json11::Json & config, ring_loop_i *ringloop, timerfd_manager
     this->ringloop = ringloop;
     this->tfd = tfd;
     this->st_cli = std::move(st_cli_ptr);
+    this->bs_factory = bs_factory;
 
     this->cli_config = config.object_items();
     this->file_config = msgr.read_config(this->cli_config);
@@ -115,7 +117,7 @@ void osd_t::init_blockstore(std::function<void()> on_init)
     if (!json_is_true(this->config["disable_blockstore"]))
     {
         auto bs_cfg = json_to_string_map(this->config);
-        this->bs = blockstore_i::create(bs_cfg, ringloop, tfd);
+        this->bs = bs_factory(bs_cfg);
         // Pre-configure pool PG shards
         for (auto & pool_item: st_cli->pool_config)
         {

src/osd/osd.h

@@ -153,6 +153,7 @@ class osd_t
     // cluster state
 
     std::unique_ptr<etcd_state_client_t> st_cli;
+    std::function<blockstore_i*(blockstore_config_t & config)> bs_factory;
     osd_messenger_t msgr;
     int etcd_failed_attempts = 0;
     std::string etcd_lease_id;
@@ -390,7 +391,8 @@ class osd_t
     }
 
 public:
-    osd_t(const json11::Json & config, ring_loop_i *ringloop, timerfd_manager_t *tfd, std::unique_ptr<etcd_state_client_t> st_cli_ptr);
+    osd_t(const json11::Json & config, ring_loop_i *ringloop, timerfd_manager_t *tfd, std::unique_ptr<etcd_state_client_t> st_cli_ptr,
+        std::function<blockstore_i*(blockstore_config_t & config)> bs_factory);
     ~osd_t();
     void force_stop(int exitcode);
     bool shutdown();

src/osd/osd_main.cpp

@@ -67,7 +67,13 @@ int main(int narg, char *args[])
     ring_loop_t *ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
     epoll_manager_t *epmgr = new epoll_manager_t(ringloop);
     auto st_cli = new etcd_state_client_http_t(epmgr->tfd);
-    osd = new osd_t(config, ringloop, epmgr->tfd, std::unique_ptr<etcd_state_client_t>(st_cli));
+    osd = new osd_t(
+        config, ringloop, epmgr->tfd, std::unique_ptr<etcd_state_client_t>(st_cli),
+        [ringloop, tfd = epmgr->tfd](blockstore_config_t & cfg)
+        {
+            return blockstore_i::create(cfg, ringloop, tfd);
+        }
+    );
     while (1)
     {
         ringloop->loop();

src/osd/osd_secondary.cpp

@@ -2,9 +2,6 @@
 // License: VNPL-1.1 (see README.md for details)
 
 #include "osd.h"
-#ifdef WITH_RDMA
-#include "msgr_rdma.h"
-#endif
 
 #include "json11/json11.hpp"
 
@@ -365,12 +362,11 @@ void osd_t::exec_show_config(osd_op_t *cur_op)
         if (req_json["connect_rdma"].is_string())
         {
             // Peer is trying to connect using RDMA, try to satisfy him
-            bool ok = msgr.connect_rdma(cur_op->client_id, req_json["connect_rdma"].string_value(), req_json["rdma_max_msg"].uint64_value());
+            json11::Json data = msgr.connect_rdma(cur_op->client_id, req_json["connect_rdma"].string_value(), req_json["rdma_max_msg"].uint64_value());
-            if (ok)
+            if (!data.is_null())
             {
-                auto rc = cl->rdma_conn;
+                for (auto & kv: data.object_items())
-                wire_config["rdma_address"] = rc->addr.to_string();
+                    wire_config[kv.first] = kv.second;
-                wire_config["rdma_max_msg"] = rc->max_msg;
             }
         }
     }

src/test/CMakeLists.txt

@@ -19,7 +19,7 @@ add_executable(stub_uring_osd
 	stub_uring_osd.cpp
 )
 target_link_libraries(stub_uring_osd
-	vitastor_common
+	vitastor_net
 	${LIBURING_LIBRARIES}
 	${IBVERBS_LIBRARIES}
 	${RDMACM_LIBRARIES}

src/test/mock/messenger.cpp

@@ -61,3 +61,30 @@ json11::Json::object osd_messenger_t::merge_configs(const json11::Json::object &
 void osd_messenger_t::destroy_rdma_conn(msgr_rdma_connection_t *rdma_conn)
 {
 }
+
+void osd_messenger_t::accept_connections(int listen_fd)
+{
+}
+
+#ifdef WITH_RDMA
+json11::Json osd_messenger_t::connect_rdma(uint64_t client_id, std::string rdma_address, uint64_t client_max_msg)
+{
+    return json11::Json();
+}
+
+bool osd_messenger_t::is_rdma_enabled()
+{
+    return false;
+}
+#endif
+
+#ifdef WITH_RDMACM
+rdma_cm_id *osd_messenger_t::rdmacm_listen(const std::string & bind_address, int rdmacm_port, int *bound_port, int log_level)
+{
+    return NULL;
+}
+
+void osd_messenger_t::rdmacm_destroy_listener(rdma_cm_id *listener)
+{
+}
+#endif

src/test/osd_test.cpp

@@ -0,0 +1,29 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 (see README.md for details)
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include "osd.h"
+#include "etcd_state_client_mock.h"
+#include "ringloop_mock.h"
+#include "blockstore_mock.h"
+
+void test1()
+{
+    json11::Json config;
+    timerfd_manager_t *tfd = new timerfd_manager_t([](int fd, bool wr, std::function<void(int, int)> callback){});
+    etcd_state_client_mock_t *st_cli = new etcd_state_client_mock_t();
+    ring_loop_mock_t *ringloop = new ring_loop_mock_t(RINGLOOP_DEFAULT_SIZE, [&](io_uring_sqe *sqe) {});
+    st_cli->pause();
+    osd_t *osd = new osd_t(config, ringloop, tfd, std::unique_ptr<etcd_state_client_t>(st_cli), [](blockstore_config_t & cfg)
+    {
+        return new blockstore_mock_t({});
+    });
+}
+
+int main(int narg, char *args[])
+{
+    test1();
+    return 0;
+}