thebestvibedev/tromcho.net
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allow to create OSD with mocked blockstore and network

Browse Source
This commit is contained in:
Vitaliy Filippov
2026-06-13 19:56:06 +03:00
parent ef0e61be1b
commit 27bd38d95e
15 changed files with 346 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/client/CMakeLists.txt
+36 -10
View File
@@ -3,6 +3,22 @@ cmake_minimum_required(VERSION 2.8...3.30)
project(vitastor)
# libvitastor_common.a
add_library(vitastor_common STATIC
etcd_state_client.cpp
msgr_stop.cpp
msgr_op.cpp
../../json11/json11.cpp
osd_ops.cpp
pg_states.cpp
../util/allocator.cpp
../util/addr_util.cpp
../util/timerfd_manager.cpp
../util/str_util.cpp
../util/json_util.cpp
)
target_compile_options(vitastor_common PUBLIC -fPIC)
# libvitastor_net.a
set(MSGR_RDMA "")
if (IBVERBS_LIBRARIES)
set(MSGR_RDMA "msgr_rdma.cpp")
@@ -11,13 +27,20 @@ set(MSGR_RDMACM "")
if (RDMACM_LIBRARIES)
set(MSGR_RDMACM "msgr_rdmacm.cpp")
endif (RDMACM_LIBRARIES)
add_library(vitastor_common STATIC
../util/epoll_manager.cpp etcd_state_client.cpp etcd_state_client_http.cpp messenger.cpp msgr_iothread.cpp ../util/addr_util.cpp
msgr_stop.cpp msgr_op.cpp msgr_send.cpp msgr_receive.cpp ../util/ringloop.cpp ../../json11/json11.cpp
http_client.cpp osd_ops.cpp pg_states.cpp ../util/timerfd_manager.cpp ../util/str_util.cpp ../util/json_util.cpp ${MSGR_RDMA} ${MSGR_RDMACM}
add_library(vitastor_net STATIC
../util/epoll_manager.cpp
etcd_state_client_http.cpp
messenger.cpp
msgr_iothread.cpp
msgr_send.cpp
msgr_receive.cpp
../util/ringloop.cpp
http_client.cpp
${MSGR_RDMA}
${MSGR_RDMACM}
)
target_link_libraries(vitastor_common pthread)
target_compile_options(vitastor_common PUBLIC -fPIC)
target_link_libraries(vitastor_net pthread vitastor_common)
target_compile_options(vitastor_net PUBLIC -fPIC)
# libvitastor_client.so
add_library(vitastor_client SHARED
@@ -29,7 +52,7 @@ add_library(vitastor_client SHARED
)
set_target_properties(vitastor_client PROPERTIES PUBLIC_HEADER "client/vitastor_c.h")
target_link_libraries(vitastor_client
vitastor_common
vitastor_net
vitastor_cli
${LIBURING_LIBRARIES}
${IBVERBS_LIBRARIES}
@@ -96,10 +119,13 @@ endif (${WITH_QEMU})
add_executable(test_cluster_client
EXCLUDE_FROM_ALL
../test/test_cluster_client.cpp
pg_states.cpp osd_ops.cpp cluster_client.cpp cluster_client_list.cpp cluster_client_wb.cpp msgr_op.cpp ../test/mock/messenger.cpp msgr_stop.cpp
etcd_state_client.cpp etcd_state_client_mock.cpp ../util/timerfd_manager.cpp ../util/addr_util.cpp ../util/str_util.cpp ../util/json_util.cpp ../../json11/json11.cpp
cluster_client.cpp
cluster_client_list.cpp
cluster_client_wb.cpp
../test/mock/messenger.cpp
etcd_state_client_mock.cpp
)
target_link_libraries(test_cluster_client ${LIBURING_LIBRARIES})
target_link_libraries(test_cluster_client vitastor_common ${LIBURING_LIBRARIES})
target_include_directories(test_cluster_client BEFORE PUBLIC ${CMAKE_SOURCE_DIR}/src/test/mock)
add_dependencies(build_tests test_cluster_client)
add_test(NAME test_cluster_client COMMAND test_cluster_client)
src/client/messenger.h
+1 -1
View File
@@ -222,7 +222,7 @@ public:
#ifdef WITH_RDMA
bool is_rdma_enabled();
bool connect_rdma(uint64_t client_id, std::string rdma_address, uint64_t client_max_msg);
json11::Json connect_rdma(uint64_t client_id, std::string rdma_address, uint64_t client_max_msg);
#endif
#ifdef WITH_RDMACM
bool is_use_rdmacm();
src/client/msgr_op.cpp
+50
View File
@@ -3,6 +3,7 @@
#include <assert.h>
#include "messenger.h"
#include "msgr_op.h"
osd_op_t::~osd_op_t()
@@ -38,3 +39,52 @@ bool osd_op_t::is_recovery_related()
req.hdr.opcode == OSD_OP_SEC_SYNC &&
(req.sec_sync.flags & OSD_OP_RECOVERY_RELATED);
}
void osd_messenger_t::measure_exec(osd_op_t *cur_op)
{
// Measure execution latency
if (cur_op->req.hdr.opcode > OSD_OP_MAX)
{
return;
}
if (!cur_op->tv_end.tv_sec)
{
clock_gettime(CLOCK_REALTIME, &cur_op->tv_end);
}
uint64_t len = 0;
if (cur_op->req.hdr.opcode == OSD_OP_READ ||
cur_op->req.hdr.opcode == OSD_OP_WRITE ||
cur_op->req.hdr.opcode == OSD_OP_SCRUB)
{
// req.rw.len is internally set to the full object size for scrubs
len = cur_op->req.rw.len;
}
else if (cur_op->req.hdr.opcode == OSD_OP_SEC_READ ||
cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE ||
cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE_STABLE)
{
len = cur_op->req.sec_rw.len;
}
inc_op_stats(stats, cur_op->req.hdr.opcode, cur_op->tv_begin, cur_op->tv_end, len);
if (cur_op->is_recovery_related())
{
inc_op_stats(recovery_stats, cur_op->req.hdr.opcode, cur_op->tv_begin, cur_op->tv_end, len);
}
}
void osd_messenger_t::inc_op_stats(osd_op_stats_t & stats, uint64_t opcode, timespec & tv_begin, timespec & tv_end, uint64_t len)
{
uint64_t usecs = (
(tv_end.tv_sec - tv_begin.tv_sec)*1000000 +
(tv_end.tv_nsec - tv_begin.tv_nsec)/1000
);
stats.op_stat_count[opcode]++;
if (!stats.op_stat_count[opcode])
{
stats.op_stat_count[opcode] = 1;
stats.op_stat_sum[opcode] = 0;
stats.op_stat_bytes[opcode] = 0;
}
stats.op_stat_sum[opcode] += usecs;
stats.op_stat_bytes[opcode] += len;
}
src/client/msgr_rdma.cpp
+7 -4
View File
@@ -507,7 +507,7 @@ int msgr_rdma_connection_t::connect(msgr_rdma_address_t *dest)
return 0;
}
bool osd_messenger_t::connect_rdma(uint64_t client_id, std::string rdma_address, uint64_t client_max_msg)
json11::Json osd_messenger_t::connect_rdma(uint64_t client_id, std::string rdma_address, uint64_t client_max_msg)
{
// Try to connect to the peer using RDMA
msgr_rdma_address_t addr;
@@ -523,7 +523,7 @@ bool osd_messenger_t::connect_rdma(uint64_t client_id, std::string rdma_address,
{
if (log_level > 0)
fprintf(stderr, "No RDMA context for peer %ju, using only TCP\n", client_id);
return false;
return json11::Json();
}
msgr_rdma_connection_t *rdma_conn = msgr_rdma_connection_t::create(selected_ctx, rdma_max_send, rdma_max_recv, rdma_max_sge, client_max_msg);
if (rdma_conn)
@@ -542,11 +542,14 @@ bool osd_messenger_t::connect_rdma(uint64_t client_id, std::string rdma_address,
// Remember connection, but switch to RDMA only after sending the configuration response
cl->rdma_conn = rdma_conn;
cl->peer_state = PEER_RDMA_CONNECTING;
return true;
return json11::Json::object{
{"rdma_address", rdma_conn->addr.to_string()},
{"rdma_max_msg", rdma_conn->max_msg},
};
}
}
}
return false;
return json11::Json();
}
static void try_send_rdma_wr(osd_client_t *cl, ibv_sge *sge, int op_sge)
src/client/msgr_send.cpp
-49
View File
@@ -140,55 +140,6 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
}
}
void osd_messenger_t::inc_op_stats(osd_op_stats_t & stats, uint64_t opcode, timespec & tv_begin, timespec & tv_end, uint64_t len)
{
uint64_t usecs = (
(tv_end.tv_sec - tv_begin.tv_sec)*1000000 +
(tv_end.tv_nsec - tv_begin.tv_nsec)/1000
);
stats.op_stat_count[opcode]++;
if (!stats.op_stat_count[opcode])
{
stats.op_stat_count[opcode] = 1;
stats.op_stat_sum[opcode] = 0;
stats.op_stat_bytes[opcode] = 0;
}
stats.op_stat_sum[opcode] += usecs;
stats.op_stat_bytes[opcode] += len;
}
void osd_messenger_t::measure_exec(osd_op_t *cur_op)
{
// Measure execution latency
if (cur_op->req.hdr.opcode > OSD_OP_MAX)
{
return;
}
if (!cur_op->tv_end.tv_sec)
{
clock_gettime(CLOCK_REALTIME, &cur_op->tv_end);
}
uint64_t len = 0;
if (cur_op->req.hdr.opcode == OSD_OP_READ ||
cur_op->req.hdr.opcode == OSD_OP_WRITE ||
cur_op->req.hdr.opcode == OSD_OP_SCRUB)
{
// req.rw.len is internally set to the full object size for scrubs
len = cur_op->req.rw.len;
}
else if (cur_op->req.hdr.opcode == OSD_OP_SEC_READ ||
cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE ||
cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE_STABLE)
{
len = cur_op->req.sec_rw.len;
}
inc_op_stats(stats, cur_op->req.hdr.opcode, cur_op->tv_begin, cur_op->tv_end, len);
if (cur_op->is_recovery_related())
{
inc_op_stats(recovery_stats, cur_op->req.hdr.opcode, cur_op->tv_begin, cur_op->tv_end, len);
}
}
bool osd_messenger_t::try_send(osd_client_t *cl)
{
if (!cl->send_list.size() || cl->write_msg.msg_iovlen > 0 || cl->peer_state == PEER_STOPPED || cl->peer_fd < 0)