Move all sources to subdirs

src/client/CMakeLists.txt

@@ -0,0 +1,95 @@
+cmake_minimum_required(VERSION 2.8.12)
+
+project(vitastor)
+
+# libvitastor_common.a
+set(MSGR_RDMA "")
+if (IBVERBS_LIBRARIES)
+	set(MSGR_RDMA "msgr_rdma.cpp")
+endif (IBVERBS_LIBRARIES)
+add_library(vitastor_common STATIC
+	../util/epoll_manager.cpp etcd_state_client.cpp messenger.cpp ../util/addr_util.cpp
+	msgr_stop.cpp msgr_op.cpp msgr_send.cpp msgr_receive.cpp ../util/ringloop.cpp ../../json11/json11.cpp
+	http_client.cpp osd_ops.cpp pg_states.cpp ../util/timerfd_manager.cpp ../util/str_util.cpp ${MSGR_RDMA}
+)
+target_compile_options(vitastor_common PUBLIC -fPIC)
+
+# libvitastor_client.so
+add_library(vitastor_client SHARED
+	cluster_client.cpp
+	cluster_client_list.cpp
+	cluster_client_wb.cpp
+	vitastor_c.cpp
+)
+set_target_properties(vitastor_client PROPERTIES PUBLIC_HEADER "vitastor_c.h")
+target_link_libraries(vitastor_client
+	vitastor_common
+	vitastor_cli
+	${LIBURING_LIBRARIES}
+	${IBVERBS_LIBRARIES}
+)
+set_target_properties(vitastor_client PROPERTIES VERSION ${VERSION} SOVERSION 0)
+configure_file(vitastor.pc.in vitastor.pc @ONLY)
+
+if (${WITH_FIO})
+	# libfio_vitastor.so
+	add_library(fio_vitastor SHARED
+		fio_cluster.cpp
+	)
+	target_link_libraries(fio_vitastor
+		vitastor_client
+	)
+
+	# libfio_vitastor_sec.so
+	add_library(fio_vitastor_sec SHARED
+		fio_sec_osd.cpp
+		../util/rw_blocking.cpp
+		../util/addr_util.cpp
+	)
+	target_link_libraries(fio_vitastor_sec
+		tcmalloc_minimal
+	)
+endif (${WITH_FIO})
+
+# vitastor-nbd
+pkg_check_modules(NL3 libnl-3.0 libnl-genl-3.0)
+add_executable(vitastor-nbd
+	nbd_proxy.cpp
+)
+target_include_directories(vitastor-nbd PUBLIC ${NL3_INCLUDE_DIRS})
+target_link_libraries(vitastor-nbd vitastor_client ${NL3_LIBRARIES})
+if (HAVE_NBD_NETLINK_H AND NL3_LIBRARIES)
+	target_compile_definitions(vitastor-nbd PUBLIC HAVE_NBD_NETLINK_H)
+endif (HAVE_NBD_NETLINK_H AND NL3_LIBRARIES)
+
+if (${WITH_QEMU})
+	# qemu_driver.so
+	add_library(qemu_vitastor SHARED
+		qemu_driver.c
+	)
+	target_compile_options(qemu_vitastor PUBLIC -DVITASTOR_SOURCE_TREE)
+	target_include_directories(qemu_vitastor PUBLIC
+		../../qemu/b/qemu
+		../../qemu/include
+		${GLIB_INCLUDE_DIRS}
+	)
+	target_link_libraries(qemu_vitastor
+		vitastor_client
+	)
+	set_target_properties(qemu_vitastor PROPERTIES
+		PREFIX ""
+		OUTPUT_NAME "block-vitastor"
+	)
+endif (${WITH_QEMU})
+
+# test_cluster_client
+add_executable(test_cluster_client
+	EXCLUDE_FROM_ALL
+	../test/test_cluster_client.cpp
+	pg_states.cpp osd_ops.cpp cluster_client.cpp cluster_client_list.cpp cluster_client_wb.cpp msgr_op.cpp ../test/mock/messenger.cpp msgr_stop.cpp
+	etcd_state_client.cpp ../util/timerfd_manager.cpp ../util/str_util.cpp ../../json11/json11.cpp
+)
+target_compile_definitions(test_cluster_client PUBLIC -D__MOCK__)
+target_include_directories(test_cluster_client BEFORE PUBLIC ${CMAKE_SOURCE_DIR}/src/test/mock)
+add_dependencies(build_tests test_cluster_client)
+add_test(NAME test_cluster_client COMMAND test_cluster_client)

src/client/cluster_client.h

@@ -0,0 +1,173 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+#pragma once
+
+#include "messenger.h"
+#include "etcd_state_client.h"
+
+#define DEFAULT_CLIENT_MAX_DIRTY_BYTES 32*1024*1024
+#define DEFAULT_CLIENT_MAX_DIRTY_OPS 1024
+#define DEFAULT_CLIENT_MAX_BUFFERED_BYTES 32*1024*1024
+#define DEFAULT_CLIENT_MAX_BUFFERED_OPS 1024
+#define DEFAULT_CLIENT_MAX_WRITEBACK_IODEPTH 256
+#define INODE_LIST_DONE 1
+#define INODE_LIST_HAS_UNSTABLE 2
+#define OSD_OP_READ_BITMAP OSD_OP_SEC_READ_BMP
+#define OSD_OP_READ_CHAIN_BITMAP 0x102
+
+#define OSD_OP_IGNORE_READONLY 0x08
+
+struct cluster_op_t;
+
+struct cluster_op_part_t
+{
+    cluster_op_t *parent;
+    uint64_t offset;
+    uint32_t len;
+    pg_num_t pg_num;
+    osd_num_t osd_num;
+    osd_op_buf_list_t iov;
+    unsigned flags;
+    osd_op_t op;
+};
+
+struct cluster_op_t
+{
+    uint64_t opcode; // OSD_OP_READ, OSD_OP_WRITE, OSD_OP_SYNC, OSD_OP_DELETE, OSD_OP_READ_BITMAP, OSD_OP_READ_CHAIN_BITMAP
+    uint64_t inode;
+    uint64_t offset;
+    uint64_t len;
+    // for reads and writes within a single object (stripe),
+    // reads can return current version and writes can use "CAS" semantics
+    uint64_t version = 0;
+    // now only OSD_OP_IGNORE_READONLY is supported
+    uint64_t flags = 0;
+    // negative retval is an error number
+    // write and read return len on success
+    // sync and delete return 0 on success
+    // read_bitmap and read_chain_bitmap return the length of bitmap in bits(!)
+    int retval;
+    osd_op_buf_list_t iov;
+    // READ, READ_BITMAP, READ_CHAIN_BITMAP return the bitmap here
+    void *bitmap_buf = NULL;
+    std::function<void(cluster_op_t*)> callback;
+    ~cluster_op_t();
+protected:
+    int state = 0;
+    uint64_t cur_inode; // for snapshot reads
+    void *buf = NULL;
+    cluster_op_t *orig_op = NULL;
+    bool needs_reslice = false;
+    int retry_after = 0;
+    int inflight_count = 0, done_count = 0;
+    std::vector<cluster_op_part_t> parts;
+    void *part_bitmaps = NULL;
+    unsigned bitmap_buf_size = 0;
+    cluster_op_t *prev = NULL, *next = NULL;
+    int prev_wait = 0;
+    friend class cluster_client_t;
+    friend class writeback_cache_t;
+};
+
+struct inode_list_t;
+struct inode_list_osd_t;
+class writeback_cache_t;
+
+// FIXME: Split into public and private interfaces
+class cluster_client_t
+{
+    timerfd_manager_t *tfd;
+    ring_loop_t *ringloop;
+
+    std::map<pool_id_t, uint64_t> pg_counts;
+    // client_max_dirty_* is actually "max unsynced", for the case when immediate_commit is off
+    uint64_t client_max_dirty_bytes = 0;
+    uint64_t client_max_dirty_ops = 0;
+    // writeback improves (1) small consecutive writes and (2) Q1 writes without fsync
+    bool enable_writeback = false;
+    // client_max_buffered_* is the real "dirty limit" - maximum amount of writes buffered in memory
+    uint64_t client_max_buffered_bytes = 0;
+    uint64_t client_max_buffered_ops = 0;
+    uint64_t client_max_writeback_iodepth = 0;
+
+    int log_level = 0;
+    int client_retry_interval = 50; // ms
+    int client_eio_retry_interval = 1000; // ms
+    bool client_retry_enospc = true;
+
+    int retry_timeout_id = 0;
+    int retry_timeout_duration = 0;
+    std::vector<cluster_op_t*> offline_ops;
+    cluster_op_t *op_queue_head = NULL, *op_queue_tail = NULL;
+    writeback_cache_t *wb = NULL;
+    std::set<osd_num_t> dirty_osds;
+    uint64_t dirty_bytes = 0, dirty_ops = 0;
+
+    void *scrap_buffer = NULL;
+    unsigned scrap_buffer_size = 0;
+
+    bool pgs_loaded = false;
+    ring_consumer_t consumer;
+    std::vector<std::function<void(void)>> on_ready_hooks;
+    std::vector<inode_list_t*> lists;
+    std::multimap<osd_num_t, osd_op_t*> raw_ops;
+    int continuing_ops = 0;
+    bool msgr_initialized = false;
+
+public:
+    etcd_state_client_t st_cli;
+
+    osd_messenger_t msgr;
+    void init_msgr();
+
+    json11::Json::object cli_config, file_config, etcd_global_config;
+    json11::Json::object config;
+
+    cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json config);
+    ~cluster_client_t();
+    void execute(cluster_op_t *op);
+    void execute_raw(osd_num_t osd_num, osd_op_t *op);
+    bool is_ready();
+    void on_ready(std::function<void(void)> fn);
+    bool flush();
+
+    bool get_immediate_commit(uint64_t inode);
+
+    void continue_ops(int time_passed = 0);
+    inode_list_t *list_inode_start(inode_t inode,
+        std::function<void(inode_list_t* lst, std::set<object_id>&& objects, pg_num_t pg_num, osd_num_t primary_osd, int status)> callback);
+    int list_pg_count(inode_list_t *lst);
+    const std::vector<osd_num_t> & list_inode_get_inactive_osds(inode_list_t *lst);
+    void list_inode_next(inode_list_t *lst, int next_pgs);
+    //inline uint32_t get_bs_bitmap_granularity() { return st_cli.global_bitmap_granularity; }
+    //inline uint64_t get_bs_block_size() { return st_cli.global_block_size; }
+    uint64_t next_op_id();
+
+protected:
+    bool affects_osd(uint64_t inode, uint64_t offset, uint64_t len, osd_num_t osd);
+    void on_load_config_hook(json11::Json::object & config);
+    void on_load_pgs_hook(bool success);
+    void on_change_hook(std::map<std::string, etcd_kv_t> & changes);
+    void on_change_osd_state_hook(uint64_t peer_osd);
+    void execute_internal(cluster_op_t *op);
+    void unshift_op(cluster_op_t *op);
+    int continue_rw(cluster_op_t *op);
+    bool check_rw(cluster_op_t *op);
+    void slice_rw(cluster_op_t *op);
+    void reset_retry_timer(int new_duration);
+    bool try_send(cluster_op_t *op, int i);
+    int continue_sync(cluster_op_t *op);
+    void send_sync(cluster_op_t *op, cluster_op_part_t *part);
+    void handle_op_part(cluster_op_part_t *part);
+    void copy_part_bitmap(cluster_op_t *op, cluster_op_part_t *part);
+    void erase_op(cluster_op_t *op);
+    void calc_wait(cluster_op_t *op);
+    void inc_wait(uint64_t opcode, uint64_t flags, cluster_op_t *next, int inc);
+    void continue_lists();
+    void continue_listing(inode_list_t *lst);
+    void send_list(inode_list_osd_t *cur_list);
+    void continue_raw_ops(osd_num_t peer_osd);
+
+    friend class writeback_cache_t;
+};

src/client/cluster_client_impl.h

@@ -0,0 +1,57 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+#pragma once
+
+#include "cluster_client.h"
+
+#define SCRAP_BUFFER_SIZE 4*1024*1024
+#define PART_SENT 1
+#define PART_DONE 2
+#define PART_ERROR 4
+#define PART_RETRY 8
+#define CACHE_DIRTY 1
+#define CACHE_WRITTEN 2
+#define CACHE_FLUSHING 3
+#define CACHE_REPEATING 4
+#define OP_FLUSH_BUFFER 0x02
+#define OP_IMMEDIATE_COMMIT 0x04
+
+struct cluster_buffer_t
+{
+    uint8_t *buf;
+    uint64_t len;
+    int state;
+    uint64_t flush_id;
+    uint64_t *refcnt;
+};
+
+typedef std::map<object_id, cluster_buffer_t>::iterator dirty_buf_it_t;
+
+class writeback_cache_t
+{
+public:
+    uint64_t writeback_bytes = 0;
+    int writeback_queue_size = 0;
+    int writebacks_active = 0;
+    uint64_t last_flush_id = 0;
+
+    std::map<object_id, cluster_buffer_t> dirty_buffers;
+    std::vector<cluster_op_t*> writeback_overflow;
+    std::vector<object_id> writeback_queue;
+    std::multimap<uint64_t, uint64_t*> flushed_buffers; // flush_id => refcnt
+
+    ~writeback_cache_t();
+    dirty_buf_it_t find_dirty(uint64_t inode, uint64_t offset);
+    bool is_left_merged(dirty_buf_it_t dirty_it);
+    bool is_right_merged(dirty_buf_it_t dirty_it);
+    bool is_merged(const dirty_buf_it_t & dirty_it);
+    void copy_write(cluster_op_t *op, int state);
+    int repeat_ops_for(cluster_client_t *cli, osd_num_t peer_osd);
+    void start_writebacks(cluster_client_t *cli, int count);
+    bool read_from_cache(cluster_op_t *op, uint32_t bitmap_granularity);
+    void flush_buffers(cluster_client_t *cli, dirty_buf_it_t from_it, dirty_buf_it_t to_it);
+    void fsync_start();
+    void fsync_error();
+    void fsync_ok();
+};

src/client/cluster_client_list.cpp

@@ -0,0 +1,310 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+#include <algorithm>
+#include "pg_states.h"
+#include "cluster_client.h"
+
+struct inode_list_t;
+
+struct inode_list_pg_t;
+
+struct inode_list_osd_t
+{
+    inode_list_pg_t *pg = NULL;
+    osd_num_t osd_num = 0;
+    bool sent = false;
+};
+
+struct inode_list_pg_t
+{
+    inode_list_t *lst = NULL;
+    int pos = 0;
+    pg_num_t pg_num;
+    osd_num_t cur_primary;
+    bool has_unstable = false;
+    int sent = 0;
+    int done = 0;
+    std::vector<inode_list_osd_t> list_osds;
+    std::set<object_id> objects;
+};
+
+struct inode_list_t
+{
+    cluster_client_t *cli = NULL;
+    pool_id_t pool_id = 0;
+    inode_t inode = 0;
+    int done_pgs = 0;
+    int want = 0;
+    std::vector<osd_num_t> inactive_osds;
+    std::vector<inode_list_pg_t*> pgs;
+    std::function<void(inode_list_t* lst, std::set<object_id>&& objects, pg_num_t pg_num, osd_num_t primary_osd, int status)> callback;
+};
+
+inode_list_t* cluster_client_t::list_inode_start(inode_t inode,
+    std::function<void(inode_list_t* lst, std::set<object_id>&& objects, pg_num_t pg_num, osd_num_t primary_osd, int status)> callback)
+{
+    init_msgr();
+    int skipped_pgs = 0;
+    pool_id_t pool_id = INODE_POOL(inode);
+    if (!pool_id || st_cli.pool_config.find(pool_id) == st_cli.pool_config.end())
+    {
+        if (log_level > 0)
+        {
+            fprintf(stderr, "Pool %u does not exist\n", pool_id);
+        }
+        return NULL;
+    }
+    inode_list_t *lst = new inode_list_t();
+    lst->cli = this;
+    lst->pool_id = pool_id;
+    lst->inode = inode;
+    lst->callback = callback;
+    auto pool_cfg = st_cli.pool_config[pool_id];
+    std::set<osd_num_t> inactive_osd_set;
+    for (auto & pg_item: pool_cfg.pg_config)
+    {
+        auto & pg = pg_item.second;
+        if (pg.pause || !pg.cur_primary || !(pg.cur_state & PG_ACTIVE))
+        {
+            skipped_pgs++;
+            if (log_level > 0)
+            {
+                fprintf(stderr, "PG %u is inactive, skipping\n", pg_item.first);
+            }
+            continue;
+        }
+        inode_list_pg_t *r = new inode_list_pg_t();
+        r->lst = lst;
+        r->pg_num = pg_item.first;
+        r->cur_primary = pg.cur_primary;
+        if (pg.cur_state != PG_ACTIVE)
+        {
+            // Not clean
+            std::set<osd_num_t> all_peers;
+            for (osd_num_t pg_osd: pg.target_set)
+            {
+                if (pg_osd != 0)
+                {
+                    all_peers.insert(pg_osd);
+                }
+            }
+            for (osd_num_t pg_osd: pg.all_peers)
+            {
+                if (pg_osd != 0)
+                {
+                    all_peers.insert(pg_osd);
+                }
+            }
+            for (auto & hist_item: pg.target_history)
+            {
+                for (auto pg_osd: hist_item)
+                {
+                    if (pg_osd != 0)
+                    {
+                        all_peers.insert(pg_osd);
+                    }
+                }
+            }
+            for (osd_num_t peer_osd: all_peers)
+            {
+                if (st_cli.peer_states.find(peer_osd) != st_cli.peer_states.end())
+                {
+                    r->list_osds.push_back((inode_list_osd_t){
+                        .pg = r,
+                        .osd_num = peer_osd,
+                        .sent = false,
+                    });
+                }
+                else
+                {
+                    inactive_osd_set.insert(peer_osd);
+                }
+            }
+        }
+        else
+        {
+            // Clean
+            r->list_osds.push_back((inode_list_osd_t){
+                .pg = r,
+                .osd_num = pg.cur_primary,
+                .sent = false,
+            });
+        }
+        lst->pgs.push_back(r);
+    }
+    std::sort(lst->pgs.begin(), lst->pgs.end(), [](inode_list_pg_t *a, inode_list_pg_t *b)
+    {
+        return a->cur_primary < b->cur_primary ? true : false;
+    });
+    for (int i = 0; i < lst->pgs.size(); i++)
+    {
+        lst->pgs[i]->pos = i;
+    }
+    lst->inactive_osds.insert(lst->inactive_osds.end(), inactive_osd_set.begin(), inactive_osd_set.end());
+    lists.push_back(lst);
+    return lst;
+}
+
+int cluster_client_t::list_pg_count(inode_list_t *lst)
+{
+    return lst->pgs.size();
+}
+
+const std::vector<osd_num_t> & cluster_client_t::list_inode_get_inactive_osds(inode_list_t *lst)
+{
+    return lst->inactive_osds;
+}
+
+void cluster_client_t::list_inode_next(inode_list_t *lst, int next_pgs)
+{
+    if (next_pgs >= 0)
+    {
+        lst->want += next_pgs;
+    }
+    continue_listing(lst);
+}
+
+void cluster_client_t::continue_listing(inode_list_t *lst)
+{
+    if (lst->done_pgs >= lst->pgs.size())
+    {
+        return;
+    }
+    if (lst->want <= 0)
+    {
+        return;
+    }
+    for (int i = 0; i < lst->pgs.size(); i++)
+    {
+        if (lst->pgs[i] && lst->pgs[i]->sent < lst->pgs[i]->list_osds.size())
+        {
+            for (int j = 0; j < lst->pgs[i]->list_osds.size(); j++)
+            {
+                send_list(&lst->pgs[i]->list_osds[j]);
+                if (lst->want <= 0)
+                {
+                    return;
+                }
+            }
+        }
+    }
+}
+
+void cluster_client_t::send_list(inode_list_osd_t *cur_list)
+{
+    if (cur_list->sent)
+    {
+        return;
+    }
+    if (msgr.osd_peer_fds.find(cur_list->osd_num) == msgr.osd_peer_fds.end())
+    {
+        // Initiate connection
+        msgr.connect_peer(cur_list->osd_num, st_cli.peer_states[cur_list->osd_num]);
+        return;
+    }
+    auto & pool_cfg = st_cli.pool_config[cur_list->pg->lst->pool_id];
+    osd_op_t *op = new osd_op_t();
+    op->op_type = OSD_OP_OUT;
+    // Already checked that it exists above, but anyway
+    op->peer_fd = msgr.osd_peer_fds.at(cur_list->osd_num);
+    op->req = (osd_any_op_t){
+        .sec_list = {
+            .header = {
+                .magic = SECONDARY_OSD_OP_MAGIC,
+                .id = next_op_id(),
+                .opcode = OSD_OP_SEC_LIST,
+            },
+            .list_pg = cur_list->pg->pg_num,
+            .pg_count = (pg_num_t)pool_cfg.real_pg_count,
+            .pg_stripe_size = pool_cfg.pg_stripe_size,
+            .min_inode = cur_list->pg->lst->inode,
+            .max_inode = cur_list->pg->lst->inode,
+        },
+    };
+    op->callback = [this, cur_list](osd_op_t *op)
+    {
+        if (op->reply.hdr.retval < 0)
+        {
+            fprintf(stderr, "Failed to get PG %u/%u object list from OSD %ju (retval=%jd), skipping\n",
+                cur_list->pg->lst->pool_id, cur_list->pg->pg_num, cur_list->osd_num, op->reply.hdr.retval);
+        }
+        else
+        {
+            if (op->reply.sec_list.stable_count < op->reply.hdr.retval)
+            {
+                // Unstable objects, if present, mean that someone still writes into the inode. Warn the user about it.
+                cur_list->pg->has_unstable = true;
+                fprintf(
+                    stderr, "[PG %u/%u] Inode still has %ju unstable object versions out of total %ju - is it still open?\n",
+                    cur_list->pg->lst->pool_id, cur_list->pg->pg_num, op->reply.hdr.retval - op->reply.sec_list.stable_count,
+                    op->reply.hdr.retval
+                );
+            }
+            if (log_level > 0)
+            {
+                fprintf(
+                    stderr, "[PG %u/%u] Got inode object list from OSD %ju: %jd object versions\n",
+                    cur_list->pg->lst->pool_id, cur_list->pg->pg_num, cur_list->osd_num, op->reply.hdr.retval
+                );
+            }
+            for (uint64_t i = 0; i < op->reply.hdr.retval; i++)
+            {
+                object_id oid = ((obj_ver_id*)op->buf)[i].oid;
+                oid.stripe = oid.stripe & ~STRIPE_MASK;
+                cur_list->pg->objects.insert(oid);
+            }
+        }
+        delete op;
+        auto lst = cur_list->pg->lst;
+        auto pg = cur_list->pg;
+        pg->done++;
+        if (pg->done >= pg->list_osds.size())
+        {
+            int status = 0;
+            lst->done_pgs++;
+            if (lst->done_pgs >= lst->pgs.size())
+            {
+                status |= INODE_LIST_DONE;
+            }
+            if (pg->has_unstable)
+            {
+                status |= INODE_LIST_HAS_UNSTABLE;
+            }
+            lst->callback(lst, std::move(pg->objects), pg->pg_num, pg->cur_primary, status);
+            lst->pgs[pg->pos] = NULL;
+            delete pg;
+            if (lst->done_pgs >= lst->pgs.size())
+            {
+                // All done
+                for (int i = 0; i < lists.size(); i++)
+                {
+                    if (lists[i] == lst)
+                    {
+                        lists.erase(lists.begin()+i, lists.begin()+i+1);
+                        break;
+                    }
+                }
+                delete lst;
+                return;
+            }
+        }
+        else
+        {
+            lst->want++;
+        }
+        continue_listing(lst);
+    };
+    msgr.outbox_push(op);
+    cur_list->sent = true;
+    cur_list->pg->sent++;
+    cur_list->pg->lst->want--;
+}
+
+void cluster_client_t::continue_lists()
+{
+    for (auto lst: lists)
+    {
+        continue_listing(lst);
+    }
+}

src/client/cluster_client_wb.cpp

@@ -0,0 +1,498 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+#include <cassert>
+
+#include "cluster_client_impl.h"
+
+writeback_cache_t::~writeback_cache_t()
+{
+    for (auto & bp: dirty_buffers)
+    {
+        if (!--(*bp.second.refcnt))
+        {
+            free(bp.second.refcnt); // refcnt is allocated with the buffer
+        }
+    }
+    dirty_buffers.clear();
+}
+
+dirty_buf_it_t writeback_cache_t::find_dirty(uint64_t inode, uint64_t offset)
+{
+    auto dirty_it = dirty_buffers.lower_bound((object_id){
+        .inode = inode,
+        .stripe = offset,
+    });
+    while (dirty_it != dirty_buffers.begin())
+    {
+        dirty_it--;
+        if (dirty_it->first.inode != inode ||
+            (dirty_it->first.stripe + dirty_it->second.len) <= offset)
+        {
+            dirty_it++;
+            break;
+        }
+    }
+    return dirty_it;
+}
+
+bool writeback_cache_t::is_left_merged(dirty_buf_it_t dirty_it)
+{
+    if (dirty_it != dirty_buffers.begin())
+    {
+        auto prev_it = dirty_it;
+        prev_it--;
+        if (prev_it->first.inode == dirty_it->first.inode &&
+            prev_it->first.stripe+prev_it->second.len == dirty_it->first.stripe &&
+            prev_it->second.state == CACHE_DIRTY)
+        {
+            return true;
+        }
+    }
+    return false;
+}
+
+bool writeback_cache_t::is_right_merged(dirty_buf_it_t dirty_it)
+{
+    auto next_it = dirty_it;
+    next_it++;
+    if (next_it != dirty_buffers.end() &&
+        next_it->first.inode == dirty_it->first.inode &&
+        next_it->first.stripe == dirty_it->first.stripe+dirty_it->second.len &&
+        next_it->second.state == CACHE_DIRTY)
+    {
+        return true;
+    }
+    return false;
+}
+
+bool writeback_cache_t::is_merged(const dirty_buf_it_t & dirty_it)
+{
+    return is_left_merged(dirty_it) || is_right_merged(dirty_it);
+}
+
+void writeback_cache_t::copy_write(cluster_op_t *op, int state)
+{
+    // Save operation for replay when one of PGs goes out of sync
+    // (primary OSD drops our connection in this case)
+    // ...or just save it for writeback if write buffering is enabled
+    if (op->len == 0)
+    {
+        return;
+    }
+    auto dirty_it = find_dirty(op->inode, op->offset);
+    auto new_end = op->offset + op->len;
+    while (dirty_it != dirty_buffers.end() &&
+        dirty_it->first.inode == op->inode &&
+        dirty_it->first.stripe < op->offset+op->len)
+    {
+        assert(dirty_it->first.stripe + dirty_it->second.len > op->offset);
+        // Remove overlapping part(s) of buffers
+        auto old_end = dirty_it->first.stripe + dirty_it->second.len;
+        if (dirty_it->first.stripe < op->offset)
+        {
+            if (old_end > new_end)
+            {
+                // Split into end and start
+                dirty_it->second.len = op->offset - dirty_it->first.stripe;
+                dirty_it = dirty_buffers.emplace_hint(dirty_it, (object_id){
+                    .inode = op->inode,
+                    .stripe = new_end,
+                }, (cluster_buffer_t){
+                    .buf = dirty_it->second.buf + new_end - dirty_it->first.stripe,
+                    .len = old_end - new_end,
+                    .state = dirty_it->second.state,
+                    .flush_id = dirty_it->second.flush_id,
+                    .refcnt = dirty_it->second.refcnt,
+                });
+                (*dirty_it->second.refcnt)++;
+                if (dirty_it->second.state == CACHE_DIRTY)
+                {
+                    writeback_bytes -= op->len;
+                    writeback_queue_size++;
+                }
+                break;
+            }
+            else
+            {
+                // Only leave the beginning
+                if (dirty_it->second.state == CACHE_DIRTY)
+                {
+                    writeback_bytes -= old_end - op->offset;
+                    if (is_left_merged(dirty_it) && !is_right_merged(dirty_it))
+                    {
+                        writeback_queue_size++;
+                    }
+                }
+                dirty_it->second.len = op->offset - dirty_it->first.stripe;
+                dirty_it++;
+            }
+        }
+        else if (old_end > new_end)
+        {
+            // Only leave the end
+            if (dirty_it->second.state == CACHE_DIRTY)
+            {
+                writeback_bytes -= new_end - dirty_it->first.stripe;
+                if (!is_left_merged(dirty_it) && is_right_merged(dirty_it))
+                {
+                    writeback_queue_size++;
+                }
+            }
+            auto new_dirty_it = dirty_buffers.emplace_hint(dirty_it, (object_id){
+                .inode = op->inode,
+                .stripe = new_end,
+            }, (cluster_buffer_t){
+                .buf = dirty_it->second.buf + new_end - dirty_it->first.stripe,
+                .len = old_end - new_end,
+                .state = dirty_it->second.state,
+                .flush_id = dirty_it->second.flush_id,
+                .refcnt = dirty_it->second.refcnt,
+            });
+            dirty_buffers.erase(dirty_it);
+            dirty_it = new_dirty_it;
+            break;
+        }
+        else
+        {
+            // Remove the whole buffer
+            if (dirty_it->second.state == CACHE_DIRTY && !is_merged(dirty_it))
+            {
+                writeback_bytes -= dirty_it->second.len;
+                assert(writeback_queue_size > 0);
+                writeback_queue_size--;
+            }
+            if (!--(*dirty_it->second.refcnt))
+            {
+                free(dirty_it->second.refcnt);
+            }
+            dirty_buffers.erase(dirty_it++);
+        }
+    }
+    // Overlapping buffers are removed, just insert the new one
+    uint64_t *refcnt = (uint64_t*)malloc_or_die(sizeof(uint64_t) + op->len);
+    uint8_t *buf = (uint8_t*)refcnt + sizeof(uint64_t);
+    *refcnt = 1;
+    dirty_it = dirty_buffers.emplace_hint(dirty_it, (object_id){
+        .inode = op->inode,
+        .stripe = op->offset,
+    }, (cluster_buffer_t){
+        .buf = buf,
+        .len = op->len,
+        .state = state,
+        .refcnt = refcnt,
+    });
+    if (state == CACHE_DIRTY)
+    {
+        writeback_bytes += op->len;
+        // Track consecutive write-back operations
+        if (!is_merged(dirty_it))
+        {
+            // <writeback_queue> is OK to contain more than actual number of consecutive
+            // requests as long as it doesn't miss anything. But <writeback_queue_size>
+            // is always calculated correctly.
+            writeback_queue_size++;
+            writeback_queue.push_back((object_id){
+                .inode = op->inode,
+                .stripe = op->offset,
+            });
+        }
+    }
+    uint64_t pos = 0, len = op->len, iov_idx = 0;
+    while (len > 0 && iov_idx < op->iov.count)
+    {
+        auto & iov = op->iov.buf[iov_idx];
+        memcpy(buf + pos, iov.iov_base, iov.iov_len);
+        pos += iov.iov_len;
+        iov_idx++;
+    }
+}
+
+int writeback_cache_t::repeat_ops_for(cluster_client_t *cli, osd_num_t peer_osd)
+{
+    int repeated = 0;
+    if (dirty_buffers.size())
+    {
+        // peer_osd just dropped connection
+        // determine WHICH dirty_buffers are now obsolete and repeat them
+        for (auto wr_it = dirty_buffers.begin(), flush_it = wr_it, last_it = wr_it; ; )
+        {
+            bool end = wr_it == dirty_buffers.end();
+            bool flush_this = !end && wr_it->second.state != CACHE_REPEATING &&
+                cli->affects_osd(wr_it->first.inode, wr_it->first.stripe, wr_it->second.len, peer_osd);
+            if (flush_it != wr_it && (end || !flush_this ||
+                wr_it->first.inode != flush_it->first.inode ||
+                wr_it->first.stripe != last_it->first.stripe+last_it->second.len))
+            {
+                repeated++;
+                flush_buffers(cli, flush_it, wr_it);
+                flush_it = wr_it;
+            }
+            if (end)
+                break;
+            last_it = wr_it;
+            wr_it++;
+            if (!flush_this)
+                flush_it = wr_it;
+        }
+    }
+    return repeated;
+}
+
+void writeback_cache_t::flush_buffers(cluster_client_t *cli, dirty_buf_it_t from_it, dirty_buf_it_t to_it)
+{
+    auto prev_it = to_it;
+    prev_it--;
+    bool is_writeback = from_it->second.state == CACHE_DIRTY;
+    cluster_op_t *op = new cluster_op_t;
+    op->flags = OSD_OP_IGNORE_READONLY|OP_FLUSH_BUFFER;
+    op->opcode = OSD_OP_WRITE;
+    op->cur_inode = op->inode = from_it->first.inode;
+    op->offset = from_it->first.stripe;
+    op->len = prev_it->first.stripe + prev_it->second.len - from_it->first.stripe;
+    uint32_t calc_len = 0;
+    uint64_t flush_id = ++last_flush_id;
+    for (auto it = from_it; it != to_it; it++)
+    {
+        it->second.state = CACHE_REPEATING;
+        it->second.flush_id = flush_id;
+        (*it->second.refcnt)++;
+        flushed_buffers.emplace(flush_id, it->second.refcnt);
+        op->iov.push_back(it->second.buf, it->second.len);
+        calc_len += it->second.len;
+    }
+    assert(calc_len == op->len);
+    writebacks_active++;
+    op->callback = [this, flush_id](cluster_op_t* op)
+    {
+        // Buffer flushes should be always retried, regardless of the error,
+        // so they should never result in an error here
+        assert(op->retval == op->len);
+        for (auto fl_it = flushed_buffers.find(flush_id);
+            fl_it != flushed_buffers.end() && fl_it->first == flush_id; )
+        {
+            if (!--(*fl_it->second)) // refcnt
+            {
+                free(fl_it->second);
+            }
+            flushed_buffers.erase(fl_it++);
+        }
+        for (auto dirty_it = find_dirty(op->inode, op->offset);
+            dirty_it != dirty_buffers.end() && dirty_it->first.inode == op->inode &&
+            dirty_it->first.stripe < op->offset+op->len; dirty_it++)
+        {
+            if (dirty_it->second.flush_id == flush_id && dirty_it->second.state == CACHE_REPEATING)
+            {
+                dirty_it->second.flush_id = 0;
+                dirty_it->second.state = CACHE_WRITTEN;
+            }
+        }
+        delete op;
+        writebacks_active--;
+        // We can't call execute_internal because it affects an invalid copy of the list here
+        // (erase_op remembers `next` after writeback callback)
+    };
+    if (is_writeback)
+    {
+        cli->execute_internal(op);
+    }
+    else
+    {
+        // Insert repeated flushes into the beginning
+        cli->unshift_op(op);
+        cli->continue_rw(op);
+    }
+}
+
+void writeback_cache_t::start_writebacks(cluster_client_t *cli, int count)
+{
+    if (!writeback_queue.size())
+    {
+        return;
+    }
+    std::vector<object_id> queue_copy;
+    queue_copy.swap(writeback_queue);
+    int started = 0, i = 0;
+    for (i = 0; i < queue_copy.size() && (!count || started < count); i++)
+    {
+        object_id & req = queue_copy[i];
+        auto dirty_it = find_dirty(req.inode, req.stripe);
+        if (dirty_it == dirty_buffers.end() ||
+            dirty_it->first.inode != req.inode ||
+            dirty_it->second.state != CACHE_DIRTY)
+        {
+            continue;
+        }
+        auto from_it = dirty_it;
+        uint64_t off = dirty_it->first.stripe;
+        while (from_it != dirty_buffers.begin())
+        {
+            from_it--;
+            if (from_it->second.state != CACHE_DIRTY ||
+                from_it->first.inode != req.inode ||
+                from_it->first.stripe+from_it->second.len != off)
+            {
+                from_it++;
+                break;
+            }
+            off = from_it->first.stripe;
+        }
+        off = dirty_it->first.stripe + dirty_it->second.len;
+        auto to_it = dirty_it;
+        to_it++;
+        while (to_it != dirty_buffers.end())
+        {
+            if (to_it->second.state != CACHE_DIRTY ||
+                to_it->first.inode != req.inode ||
+                to_it->first.stripe != off)
+            {
+                break;
+            }
+            off = to_it->first.stripe + to_it->second.len;
+            to_it++;
+        }
+        started++;
+        assert(writeback_queue_size > 0);
+        writeback_queue_size--;
+        writeback_bytes -= off - from_it->first.stripe;
+        flush_buffers(cli, from_it, to_it);
+    }
+    queue_copy.erase(queue_copy.begin(), queue_copy.begin()+i);
+    if (writeback_queue.size())
+    {
+        queue_copy.insert(queue_copy.end(), writeback_queue.begin(), writeback_queue.end());
+    }
+    queue_copy.swap(writeback_queue);
+}
+
+static void copy_to_op(cluster_op_t *op, uint64_t offset, uint8_t *buf, uint64_t len, uint32_t bitmap_granularity)
+{
+    if (op->opcode == OSD_OP_READ)
+    {
+        // Not OSD_OP_READ_BITMAP or OSD_OP_READ_CHAIN_BITMAP
+        int iov_idx = 0;
+        uint64_t cur_offset = op->offset;
+        while (iov_idx < op->iov.count && cur_offset+op->iov.buf[iov_idx].iov_len <= offset)
+        {
+            cur_offset += op->iov.buf[iov_idx].iov_len;
+            iov_idx++;
+        }
+        while (iov_idx < op->iov.count && cur_offset < offset+len)
+        {
+            auto & v = op->iov.buf[iov_idx];
+            auto begin = (cur_offset < offset ? offset : cur_offset);
+            auto end = (cur_offset+v.iov_len > offset+len ? offset+len : cur_offset+v.iov_len);
+            memcpy(
+                (uint8_t*)v.iov_base + begin - cur_offset,
+                buf + (cur_offset <= offset ? 0 : cur_offset-offset),
+                end - begin
+            );
+            cur_offset += v.iov_len;
+            iov_idx++;
+        }
+    }
+    // Set bitmap bits
+    int start_bit = (offset-op->offset)/bitmap_granularity;
+    int end_bit = (offset-op->offset+len)/bitmap_granularity;
+    for (int bit = start_bit; bit < end_bit;)
+    {
+        if (!(bit%8) && bit <= end_bit-8)
+        {
+            ((uint8_t*)op->bitmap_buf)[bit/8] = 0xFF;
+            bit += 8;
+        }
+        else
+        {
+            ((uint8_t*)op->bitmap_buf)[bit/8] |= (1 << (bit%8));
+            bit++;
+        }
+    }
+}
+
+bool writeback_cache_t::read_from_cache(cluster_op_t *op, uint32_t bitmap_granularity)
+{
+    bool dirty_copied = false;
+    if (dirty_buffers.size() && (op->opcode == OSD_OP_READ ||
+        op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP))
+    {
+        // We also have to return reads from CACHE_REPEATING buffers - they are not
+        // guaranteed to be present on target OSDs at the moment of repeating
+        // And we're also free to return data from other cached buffers just
+        // because it's faster
+        auto dirty_it = find_dirty(op->cur_inode, op->offset);
+        while (dirty_it != dirty_buffers.end() && dirty_it->first.inode == op->cur_inode &&
+            dirty_it->first.stripe < op->offset+op->len)
+        {
+            uint64_t begin = dirty_it->first.stripe, end = dirty_it->first.stripe + dirty_it->second.len;
+            if (begin < op->offset)
+                begin = op->offset;
+            if (end > op->offset+op->len)
+                end = op->offset+op->len;
+            bool skip_prev = true;
+            uint64_t cur = begin, prev = begin;
+            while (cur < end)
+            {
+                unsigned bmp_loc = (cur - op->offset)/bitmap_granularity;
+                bool skip = (((*((uint8_t*)op->bitmap_buf + bmp_loc/8)) >> (bmp_loc%8)) & 0x1);
+                if (skip_prev != skip)
+                {
+                    if (cur > prev && !skip)
+                    {
+                        // Copy data
+                        dirty_copied = true;
+                        copy_to_op(op, prev, dirty_it->second.buf + prev - dirty_it->first.stripe, cur-prev, bitmap_granularity);
+                    }
+                    skip_prev = skip;
+                    prev = cur;
+                }
+                cur += bitmap_granularity;
+            }
+            assert(cur > prev);
+            if (!skip_prev)
+            {
+                // Copy data
+                dirty_copied = true;
+                copy_to_op(op, prev, dirty_it->second.buf + prev - dirty_it->first.stripe, cur-prev, bitmap_granularity);
+            }
+            dirty_it++;
+        }
+    }
+    return dirty_copied;
+}
+
+void writeback_cache_t::fsync_start()
+{
+    for (auto & prev_op: dirty_buffers)
+    {
+        if (prev_op.second.state == CACHE_WRITTEN)
+        {
+            prev_op.second.state = CACHE_FLUSHING;
+        }
+    }
+}
+
+void writeback_cache_t::fsync_error()
+{
+    for (auto & prev_op: dirty_buffers)
+    {
+        if (prev_op.second.state == CACHE_FLUSHING)
+        {
+            prev_op.second.state = CACHE_WRITTEN;
+        }
+    }
+}
+
+void writeback_cache_t::fsync_ok()
+{
+    for (auto uw_it = dirty_buffers.begin(); uw_it != dirty_buffers.end(); )
+    {
+        if (uw_it->second.state == CACHE_FLUSHING)
+        {
+            if (!--(*uw_it->second.refcnt))
+                free(uw_it->second.refcnt);
+            dirty_buffers.erase(uw_it++);
+        }
+        else
+            uw_it++;
+    }
+}

src/client/etcd_state_client.h

@@ -0,0 +1,158 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+#pragma once
+
+#include <set>
+
+#include "json11/json11.hpp"
+#include "osd_id.h"
+#include "timerfd_manager.h"
+
+#define ETCD_CONFIG_WATCH_ID 1
+#define ETCD_PG_STATE_WATCH_ID 2
+#define ETCD_PG_HISTORY_WATCH_ID 3
+#define ETCD_OSD_STATE_WATCH_ID 4
+#define ETCD_TOTAL_WATCHES 4
+
+#define DEFAULT_BLOCK_SIZE 128*1024
+#define MIN_DATA_BLOCK_SIZE 4*1024
+#define MAX_DATA_BLOCK_SIZE 128*1024*1024
+#define DEFAULT_BITMAP_GRANULARITY 4096
+
+#define IMMEDIATE_NONE 0
+#define IMMEDIATE_SMALL 1
+#define IMMEDIATE_ALL 2
+
+struct etcd_kv_t
+{
+    std::string key;
+    json11::Json value;
+    uint64_t mod_revision = 0;
+};
+
+struct pg_config_t
+{
+    bool config_exists, history_exists, state_exists;
+    osd_num_t primary;
+    std::vector<osd_num_t> target_set;
+    std::vector<std::vector<osd_num_t>> target_history;
+    std::vector<osd_num_t> all_peers;
+    bool pause;
+    osd_num_t cur_primary;
+    int cur_state;
+    uint64_t epoch;
+    uint64_t next_scrub;
+};
+
+struct pool_config_t
+{
+    bool exists;
+    pool_id_t id;
+    std::string name;
+    uint64_t scheme;
+    uint64_t pg_size, pg_minsize, parity_chunks;
+    uint32_t data_block_size, bitmap_granularity, immediate_commit;
+    uint64_t pg_count;
+    uint64_t real_pg_count;
+    std::string failure_domain;
+    uint64_t max_osd_combinations;
+    uint64_t pg_stripe_size;
+    std::map<pg_num_t, pg_config_t> pg_config;
+    uint64_t scrub_interval;
+    std::string used_for_fs;
+};
+
+struct inode_config_t
+{
+    uint64_t num = 0;
+    std::string name;
+    uint64_t size = 0;
+    inode_t parent_id = 0;
+    bool readonly = false;
+    // Arbitrary metadata
+    json11::Json meta;
+    // Change revision of the metadata in etcd
+    uint64_t mod_revision = 0;
+};
+
+struct inode_watch_t
+{
+    std::string name;
+    inode_config_t cfg = {};
+};
+
+struct http_co_t;
+
+struct etcd_state_client_t
+{
+protected:
+    std::vector<std::string> local_ips;
+    std::vector<std::string> etcd_addresses;
+    std::vector<std::string> etcd_local;
+    std::string selected_etcd_address;
+    std::vector<std::string> addresses_to_try;
+    std::vector<inode_watch_t*> watches;
+    http_co_t *etcd_watch_ws = NULL, *keepalive_client = NULL;
+    int ws_keepalive_timer = -1;
+    int ws_alive = 0;
+    bool rand_initialized = false;
+    void add_etcd_url(std::string);
+    void pick_next_etcd();
+public:
+    int etcd_keepalive_timeout = 30;
+    int etcd_ws_keepalive_interval = 30;
+    int max_etcd_attempts = 5;
+    int etcd_quick_timeout = 1000;
+    int etcd_slow_timeout = 5000;
+    uint64_t global_block_size = DEFAULT_BLOCK_SIZE;
+    uint32_t global_bitmap_granularity = DEFAULT_BITMAP_GRANULARITY;
+    uint32_t global_immediate_commit = IMMEDIATE_NONE;
+
+    std::string etcd_prefix;
+    int log_level = 0;
+    timerfd_manager_t *tfd = NULL;
+
+    int etcd_watches_initialised = 0;
+    uint64_t etcd_watch_revision = 0;
+    std::map<pool_id_t, pool_config_t> pool_config;
+    std::map<osd_num_t, json11::Json> peer_states;
+    std::set<osd_num_t> seen_peers;
+    std::map<inode_t, inode_config_t> inode_config;
+    std::map<std::string, inode_t> inode_by_name;
+
+    std::function<void(std::map<std::string, etcd_kv_t> &)> on_change_hook;
+    std::function<void(json11::Json::object &)> on_load_config_hook;
+    std::function<json11::Json()> load_pgs_checks_hook;
+    std::function<void(bool)> on_load_pgs_hook;
+    std::function<void(pool_id_t, pg_num_t)> on_change_pg_history_hook;
+    std::function<void(osd_num_t)> on_change_osd_state_hook;
+    std::function<void()> on_reload_hook;
+    std::function<void(inode_t, bool)> on_inode_change_hook;
+    std::function<void(http_co_t *)> on_start_watcher_hook;
+
+    json11::Json::object serialize_inode_cfg(inode_config_t *cfg);
+    etcd_kv_t parse_etcd_kv(const json11::Json & kv_json);
+    std::vector<std::string> get_addresses();
+    void etcd_call_oneshot(std::string etcd_address, std::string api, json11::Json payload, int timeout, std::function<void(std::string, json11::Json)> callback);
+    void etcd_call(std::string api, json11::Json payload, int timeout, int retries, int interval, std::function<void(std::string, json11::Json)> callback);
+    void etcd_txn(json11::Json txn, int timeout, int retries, int interval, std::function<void(std::string, json11::Json)> callback);
+    void etcd_txn_slow(json11::Json txn, std::function<void(std::string, json11::Json)> callback);
+    void start_etcd_watcher();
+    void stop_ws_keepalive();
+    void start_ws_keepalive();
+    void load_global_config();
+    void load_pgs();
+    void reset_pg_exists();
+    void clean_nonexistent_pgs();
+    void parse_state(const etcd_kv_t & kv);
+    void parse_config(const json11::Json & config);
+    void insert_inode_config(const inode_config_t & cfg);
+    inode_watch_t* watch_inode(std::string name);
+    void close_watch(inode_watch_t* watch);
+    int address_count();
+    ~etcd_state_client_t();
+
+    static uint32_t parse_immediate_commit(const std::string & immediate_commit_str);
+    static uint32_t parse_scheme(const std::string & scheme_str);
+};

src/client/fio_cluster.cpp

@@ -0,0 +1,554 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+// FIO engine to test cluster I/O
+//
+// Random write:
+//
+// fio -thread -ioengine=./libfio_cluster.so -name=test -bs=4k -direct=1 -fsync=16 -iodepth=16 -rw=randwrite \
+//     -etcd=127.0.0.1:2379 [-etcd_prefix=/vitastor] (-image=testimg | -pool=1 -inode=1 -size=1000M)
+//
+// Linear write:
+//
+// fio -thread -ioengine=./libfio_cluster.so -name=test -bs=128k -direct=1 -fsync=32 -iodepth=32 -rw=write \
+//     -etcd=127.0.0.1:2379 [-etcd_prefix=/vitastor] -image=testimg
+//
+// Random read (run with -iodepth=32 or -iodepth=1):
+//
+// fio -thread -ioengine=./libfio_cluster.so -name=test -bs=4k -direct=1 -iodepth=32 -rw=randread \
+//     -etcd=127.0.0.1:2379 [-etcd_prefix=/vitastor] -image=testimg
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+
+#include <vector>
+#include <string>
+
+#include "vitastor_c.h"
+#include "fio_headers.h"
+
+struct sec_data
+{
+    vitastor_c *cli = NULL;
+    bool epoll_based = false;
+    void *watch = NULL;
+    bool last_sync = false;
+    /* The list of completed io_u structs. */
+    std::vector<io_u*> completed;
+    uint64_t inflight = 0;
+    int mirror_fd = -1;
+    bool trace = false;
+};
+
+struct sec_options
+{
+    int __pad;
+    char *config_path = NULL;
+    char *etcd_host = NULL;
+    char *etcd_prefix = NULL;
+    char *image = NULL;
+    char *mirror_file = NULL;
+    uint64_t pool = 0;
+    uint64_t inode = 0;
+    int cluster_log = 0;
+    int trace = 0;
+    int use_rdma = 0;
+    char *rdma_device = NULL;
+    int rdma_port_num = 0;
+    int rdma_gid_index = 0;
+    int rdma_mtu = 0;
+    int no_io_uring = 0;
+};
+
+static struct fio_option options[] = {
+    {
+        .name   = "conf",
+        .lname  = "Vitastor config path",
+        .type   = FIO_OPT_STR_STORE,
+        .off1   = offsetof(struct sec_options, config_path),
+        .help   = "Vitastor config path",
+        .category = FIO_OPT_C_ENGINE,
+        .group  = FIO_OPT_G_FILENAME,
+    },
+    {
+        .name   = "etcd",
+        .lname  = "etcd address",
+        .type   = FIO_OPT_STR_STORE,
+        .off1   = offsetof(struct sec_options, etcd_host),
+        .help   = "etcd address in the form HOST:PORT[/PATH]",
+        .category = FIO_OPT_C_ENGINE,
+        .group  = FIO_OPT_G_FILENAME,
+    },
+    {
+        .name   = "etcd_prefix",
+        .lname  = "etcd key prefix",
+        .type   = FIO_OPT_STR_STORE,
+        .off1   = offsetof(struct sec_options, etcd_prefix),
+        .help   = "etcd key prefix, by default /vitastor",
+        .category = FIO_OPT_C_ENGINE,
+        .group  = FIO_OPT_G_FILENAME,
+    },
+    {
+        .name   = "image",
+        .lname  = "Vitastor image name",
+        .type   = FIO_OPT_STR_STORE,
+        .off1   = offsetof(struct sec_options, image),
+        .help   = "Vitastor image name to run tests on",
+        .category = FIO_OPT_C_ENGINE,
+        .group  = FIO_OPT_G_FILENAME,
+    },
+    {
+        .name   = "pool",
+        .lname  = "pool number for the inode",
+        .type   = FIO_OPT_INT,
+        .off1   = offsetof(struct sec_options, pool),
+        .help   = "pool number for the inode to run tests on",
+        .category = FIO_OPT_C_ENGINE,
+        .group  = FIO_OPT_G_FILENAME,
+    },
+    {
+        .name   = "inode",
+        .lname  = "inode to run tests on",
+        .type   = FIO_OPT_INT,
+        .off1   = offsetof(struct sec_options, inode),
+        .help   = "inode number to run tests on",
+        .category = FIO_OPT_C_ENGINE,
+        .group  = FIO_OPT_G_FILENAME,
+    },
+    {
+        .name   = "cluster_log_level",
+        .lname  = "cluster log level",
+        .type   = FIO_OPT_INT,
+        .off1   = offsetof(struct sec_options, cluster_log),
+        .help   = "Set log level for the Vitastor client",
+        .def    = "0",
+        .category = FIO_OPT_C_ENGINE,
+        .group  = FIO_OPT_G_FILENAME,
+    },
+    {
+        .name   = "osd_trace",
+        .lname  = "OSD trace",
+        .type   = FIO_OPT_BOOL,
+        .off1   = offsetof(struct sec_options, trace),
+        .help   = "Trace OSD operations",
+        .def    = "0",
+        .category = FIO_OPT_C_ENGINE,
+        .group  = FIO_OPT_G_FILENAME,
+    },
+    {
+        .name   = "mirror_file",
+        .lname  = "File name to mirror writes to",
+        .type   = FIO_OPT_STR_STORE,
+        .off1   = offsetof(struct sec_options, mirror_file),
+        .help   = "File name to mirror writes to (for debug purpose)",
+        .category = FIO_OPT_C_ENGINE,
+        .group  = FIO_OPT_G_FILENAME,
+    },
+    {
+        .name   = "use_rdma",
+        .lname  = "Use RDMA",
+        .type   = FIO_OPT_BOOL,
+        .off1   = offsetof(struct sec_options, use_rdma),
+        .help   = "Use RDMA",
+        .def    = "-1",
+        .category = FIO_OPT_C_ENGINE,
+        .group  = FIO_OPT_G_FILENAME,
+    },
+    {
+        .name   = "rdma_device",
+        .lname  = "RDMA device name",
+        .type   = FIO_OPT_STR_STORE,
+        .off1   = offsetof(struct sec_options, rdma_device),
+        .help   = "RDMA device name",
+        .category = FIO_OPT_C_ENGINE,
+        .group  = FIO_OPT_G_FILENAME,
+    },
+    {
+        .name   = "rdma_port_num",
+        .lname  = "RDMA port number",
+        .type   = FIO_OPT_INT,
+        .off1   = offsetof(struct sec_options, rdma_port_num),
+        .help   = "RDMA port number",
+        .def    = "0",
+        .category = FIO_OPT_C_ENGINE,
+        .group  = FIO_OPT_G_FILENAME,
+    },
+    {
+        .name   = "rdma_gid_index",
+        .lname  = "RDMA gid index",
+        .type   = FIO_OPT_INT,
+        .off1   = offsetof(struct sec_options, rdma_gid_index),
+        .help   = "RDMA gid index",
+        .def    = "0",
+        .category = FIO_OPT_C_ENGINE,
+        .group  = FIO_OPT_G_FILENAME,
+    },
+    {
+        .name   = "rdma_mtu",
+        .lname  = "RDMA path MTU",
+        .type   = FIO_OPT_INT,
+        .off1   = offsetof(struct sec_options, rdma_mtu),
+        .help   = "RDMA path MTU",
+        .def    = "0",
+        .category = FIO_OPT_C_ENGINE,
+        .group  = FIO_OPT_G_FILENAME,
+    },
+    {
+        .name   = "no_io_uring",
+        .lname  = "Disable io_uring",
+        .type   = FIO_OPT_BOOL,
+        .off1   = offsetof(struct sec_options, no_io_uring),
+        .help   = "Use epoll and plain sendmsg/recvmsg instead of io_uring (slower)",
+        .def    = "0",
+        .category = FIO_OPT_C_ENGINE,
+        .group  = FIO_OPT_G_FILENAME,
+    },
+    {
+        .name = NULL,
+    },
+};
+
+static void watch_callback(void *opaque, long watch)
+{
+    struct sec_data *bsd = (struct sec_data*)opaque;
+    bsd->watch = (void*)watch;
+}
+
+static void opt_push(std::vector<char *> & options, const char *opt, const char *value)
+{
+    if (value)
+    {
+        options.push_back(strdup(opt));
+        options.push_back(strdup(value));
+    }
+}
+
+static int sec_setup(struct thread_data *td)
+{
+    sec_options *o = (sec_options*)td->eo;
+    sec_data *bsd;
+
+    bsd = new sec_data;
+    if (!bsd)
+    {
+        td_verror(td, errno, "calloc");
+        return 1;
+    }
+    td->io_ops_data = bsd;
+
+    if (!td->files_index)
+    {
+        add_file(td, "osd_cluster", 0, 0);
+        td->o.nr_files = td->o.nr_files ? : 1;
+        td->o.open_files++;
+    }
+
+    if (o->mirror_file)
+    {
+        bsd->mirror_fd = open(o->mirror_file, O_CREAT|O_RDWR, 0666);
+        if (bsd->mirror_fd < 0)
+        {
+            td_verror(td, errno, "open mirror file");
+            return 1;
+        }
+    }
+
+    if (!o->image)
+    {
+        if (!(o->inode & (((uint64_t)1 << (64-POOL_ID_BITS)) - 1)))
+        {
+            td_verror(td, EINVAL, "inode number is missing");
+            return 1;
+        }
+        if (o->pool)
+        {
+            o->inode = (o->inode & (((uint64_t)1 << (64-POOL_ID_BITS)) - 1)) | (o->pool << (64-POOL_ID_BITS));
+        }
+        if (!(o->inode >> (64-POOL_ID_BITS)))
+        {
+            td_verror(td, EINVAL, "pool is missing");
+            return 1;
+        }
+    }
+    else
+    {
+        o->inode = 0;
+    }
+    std::vector<char *> options;
+    opt_push(options, "config_path", o->config_path);
+    opt_push(options, "etcd_address", o->etcd_host);
+    opt_push(options, "etcd_prefix", o->etcd_prefix);
+    if (o->use_rdma != -1)
+        opt_push(options, "use_rdma", std::to_string(o->use_rdma).c_str());
+    opt_push(options, "rdma_device", o->rdma_device);
+    if (o->rdma_port_num)
+        opt_push(options, "rdma_port_num", std::to_string(o->rdma_port_num).c_str());
+    if (o->rdma_gid_index)
+        opt_push(options, "rdma_gid_index", std::to_string(o->rdma_gid_index).c_str());
+    if (o->rdma_mtu)
+        opt_push(options, "rdma_mtu", std::to_string(o->rdma_mtu).c_str());
+    if (o->cluster_log)
+        opt_push(options, "log_level", std::to_string(o->cluster_log).c_str());
+    // allow writeback caching if -direct is not set
+    opt_push(options, "client_writeback_allowed", td->o.odirect ? "0" : "1");
+    bsd->cli = o->no_io_uring ? NULL : vitastor_c_create_uring_json((const char**)options.data(), options.size());
+    bsd->epoll_based = false;
+    if (!bsd->cli)
+    {
+        if (o->no_io_uring)
+            fprintf(stderr, "vitastor: io_uring disabled - I/O will be slower\n");
+        else
+            fprintf(stderr, "vitastor: failed to create io_uring: %s - I/O will be slower\n", strerror(errno));
+        bsd->cli = vitastor_c_create_epoll_json((const char**)options.data(), options.size());
+        bsd->epoll_based = true;
+    }
+    for (auto opt: options)
+        free(opt);
+    options.clear();
+    if (o->image)
+    {
+        bsd->watch = NULL;
+        vitastor_c_watch_inode(bsd->cli, o->image, watch_callback, bsd);
+        if (!bsd->epoll_based)
+        {
+            while (true)
+            {
+                vitastor_c_uring_handle_events(bsd->cli);
+                if (bsd->watch)
+                    break;
+                vitastor_c_uring_wait_events(bsd->cli);
+            }
+        }
+        else
+        {
+            while (true)
+            {
+                if (bsd->watch)
+                    break;
+                vitastor_c_epoll_handle_events(bsd->cli, 1000);
+            }
+        }
+        td->files[0]->real_file_size = vitastor_c_inode_get_size(bsd->watch);
+        if (!vitastor_c_inode_get_num(bsd->watch) ||
+            !td->files[0]->real_file_size)
+        {
+            td_verror(td, EINVAL, "image does not exist");
+            return 1;
+        }
+    }
+
+    bsd->trace = o->trace ? true : false;
+
+    return 0;
+}
+
+static void sec_cleanup(struct thread_data *td)
+{
+    sec_data *bsd = (sec_data*)td->io_ops_data;
+    if (bsd)
+    {
+        if (bsd->mirror_fd >= 0)
+        {
+            close(bsd->mirror_fd);
+        }
+        if (bsd->watch)
+        {
+            vitastor_c_close_watch(bsd->cli, bsd->watch);
+        }
+        vitastor_c_destroy(bsd->cli);
+        delete bsd;
+    }
+}
+
+/* Connect to the server from each thread. */
+static int sec_init(struct thread_data *td)
+{
+    return 0;
+}
+
+static void io_callback(void *opaque, long retval)
+{
+    struct io_u *io = (struct io_u*)opaque;
+    io->error = retval < 0 ? -retval : 0;
+    sec_data *bsd = (sec_data*)io->engine_data;
+    bsd->inflight--;
+    bsd->completed.push_back(io);
+    if (bsd->trace)
+    {
+        printf("--- %s 0x%jx retval=%ld\n", io->ddir == DDIR_READ ? "READ" :
+            (io->ddir == DDIR_WRITE ? "WRITE" : "SYNC"), (uint64_t)io, retval);
+    }
+}
+
+static void read_callback(void *opaque, long retval, uint64_t version)
+{
+    io_callback(opaque, retval);
+}
+
+/* Begin read or write request. */
+static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
+{
+    sec_options *opt = (sec_options*)td->eo;
+    sec_data *bsd = (sec_data*)td->io_ops_data;
+    struct iovec iov;
+
+    fio_ro_check(td, io);
+    if (io->ddir == DDIR_SYNC && bsd->last_sync)
+    {
+        return FIO_Q_COMPLETED;
+    }
+
+    io->engine_data = bsd;
+    io->error = 0;
+    bsd->inflight++;
+
+    uint64_t inode = opt->image ? vitastor_c_inode_get_num(bsd->watch) : opt->inode;
+    assert(io->xfer_buflen < (size_t)-1);
+    switch (io->ddir)
+    {
+    case DDIR_READ:
+        iov = { .iov_base = io->xfer_buf, .iov_len = (size_t)io->xfer_buflen };
+        vitastor_c_read(bsd->cli, inode, io->offset, io->xfer_buflen, &iov, 1, read_callback, io);
+        bsd->last_sync = false;
+        break;
+    case DDIR_WRITE:
+        if (opt->mirror_file)
+        {
+            size_t done = 0;
+            while (done < io->xfer_buflen)
+            {
+                ssize_t r = pwrite(bsd->mirror_fd, (uint8_t*)io->xfer_buf+done, io->xfer_buflen-done, io->offset+done);
+                if (r < 0 && errno != EAGAIN)
+                {
+                    fprintf(stderr, "Error writing mirror file: %s\n", strerror(errno));
+                    io->error = errno;
+                    return FIO_Q_COMPLETED;
+                }
+                if (r > 0)
+                {
+                    done += r;
+                }
+            }
+        }
+        if (opt->image && vitastor_c_inode_get_readonly(bsd->watch))
+        {
+            io->error = EROFS;
+            return FIO_Q_COMPLETED;
+        }
+        iov = { .iov_base = io->xfer_buf, .iov_len = (size_t)io->xfer_buflen };
+        vitastor_c_write(bsd->cli, inode, io->offset, io->xfer_buflen, 0, &iov, 1, io_callback, io);
+        bsd->last_sync = false;
+        break;
+    case DDIR_SYNC:
+        vitastor_c_sync(bsd->cli, io_callback, io);
+        bsd->last_sync = true;
+        break;
+    default:
+        io->error = EINVAL;
+        return FIO_Q_COMPLETED;
+    }
+
+    if (opt->trace)
+    {
+        if (io->ddir == DDIR_SYNC)
+        {
+            printf("+++ SYNC 0x%jx\n", (uint64_t)io);
+        }
+        else
+        {
+            printf("+++ %s 0x%jx 0x%llx+%jx\n",
+                io->ddir == DDIR_READ ? "READ" : "WRITE",
+                (uint64_t)io, io->offset, (uint64_t)io->xfer_buflen);
+        }
+    }
+
+    if (io->error != 0)
+        return FIO_Q_COMPLETED;
+    return FIO_Q_QUEUED;
+}
+
+static int sec_getevents(struct thread_data *td, unsigned int min, unsigned int max, const struct timespec *t)
+{
+    sec_data *bsd = (sec_data*)td->io_ops_data;
+    if (!bsd->epoll_based)
+    {
+        while (true)
+        {
+            vitastor_c_uring_handle_events(bsd->cli);
+            if (bsd->completed.size() >= min)
+                break;
+            vitastor_c_uring_wait_events(bsd->cli);
+        }
+    }
+    else
+    {
+        while (true)
+        {
+            if (bsd->completed.size() >= min)
+                break;
+            vitastor_c_epoll_handle_events(bsd->cli, 1000);
+        }
+    }
+    return bsd->completed.size();
+}
+
+static struct io_u *sec_event(struct thread_data *td, int event)
+{
+    sec_data *bsd = (sec_data*)td->io_ops_data;
+    if (bsd->completed.size() == 0)
+        return NULL;
+    /* FIXME We ignore the event number and assume fio calls us exactly once for [0..nr_events-1] */
+    struct io_u *ev = bsd->completed.back();
+    bsd->completed.pop_back();
+    return ev;
+}
+
+static int sec_io_u_init(struct thread_data *td, struct io_u *io)
+{
+    io->engine_data = NULL;
+    return 0;
+}
+
+static void sec_io_u_free(struct thread_data *td, struct io_u *io)
+{
+}
+
+static int sec_open_file(struct thread_data *td, struct fio_file *f)
+{
+    return 0;
+}
+
+static int sec_invalidate(struct thread_data *td, struct fio_file *f)
+{
+    return 0;
+}
+
+struct ioengine_ops ioengine = {
+    .name               = "vitastor_cluster",
+    .version            = FIO_IOOPS_VERSION,
+    .flags              = FIO_MEMALIGN | FIO_DISKLESSIO | FIO_NOEXTEND,
+    .setup              = sec_setup,
+    .init               = sec_init,
+    .queue              = sec_queue,
+    .getevents          = sec_getevents,
+    .event              = sec_event,
+    .cleanup            = sec_cleanup,
+    .open_file          = sec_open_file,
+    .invalidate         = sec_invalidate,
+    .io_u_init          = sec_io_u_init,
+    .io_u_free          = sec_io_u_free,
+    .option_struct_size = sizeof(struct sec_options),
+    .options            = options,
+};
+
+static void fio_init fio_sec_register(void)
+{
+    register_ioengine(&ioengine);
+}
+
+static void fio_exit fio_sec_unregister(void)
+{
+    unregister_ioengine(&ioengine);
+}

src/client/fio_sec_osd.cpp

@@ -0,0 +1,466 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+// FIO engine to test Blockstore through Secondary OSD interface
+//
+// Prepare storage like in fio_engine.cpp, then start OSD with ./osd, then test it
+//
+// Random write:
+//
+// fio -thread -ioengine=./libfio_sec_osd.so -name=test -bs=4k -direct=1 -fsync=16 -iodepth=16 -rw=randwrite \
+//     -host=127.0.0.1 -port=11203 [-block_size_order=17] [-single_primary=1] -size=1000M
+//
+// Linear write:
+//
+// fio -thread -ioengine=./libfio_sec_osd.so -name=test -bs=128k -direct=1 -fsync=32 -iodepth=32 -rw=write \
+//     -host=127.0.0.1 -port=11203 -size=1000M
+//
+// Random read (run with -iodepth=32 or -iodepth=1):
+//
+// fio -thread -ioengine=./libfio_sec_osd.so -name=test -bs=4k -direct=1 -iodepth=32 -rw=randread \
+//     -host=127.0.0.1 -port=11203 -size=1000M
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+
+#include <vector>
+#include <unordered_map>
+
+#include "addr_util.h"
+#include "rw_blocking.h"
+#include "osd_ops.h"
+#include "fio_headers.h"
+
+struct op_buf_t
+{
+    osd_any_op_t buf;
+    io_u* fio_op;
+};
+
+struct sec_data
+{
+    int connect_fd;
+    /* block_size = 1 << block_order (128KB by default) */
+    uint64_t block_order = 17, block_size = 1 << 17;
+    std::unordered_map<uint64_t, op_buf_t*> queue;
+    bool last_sync = false;
+    /* The list of completed io_u structs. */
+    std::vector<io_u*> completed;
+    uint64_t op_n = 0, inflight = 0;
+};
+
+struct sec_options
+{
+    int __pad;
+    char *host = NULL;
+    int port = 0;
+    int single_primary = 0;
+    int trace = 0;
+    int block_order = 17;
+    int zerocopy_send = 0;
+};
+
+static struct fio_option options[] = {
+    {
+        .name   = "host",
+        .lname  = "Test Secondary OSD host",
+        .type   = FIO_OPT_STR_STORE,
+        .off1   = offsetof(struct sec_options, host),
+        .help   = "Test Secondary OSD host",
+        .category = FIO_OPT_C_ENGINE,
+        .group  = FIO_OPT_G_FILENAME,
+    },
+    {
+        .name   = "port",
+        .lname  = "Test Secondary OSD port",
+        .type   = FIO_OPT_INT,
+        .off1   = offsetof(struct sec_options, port),
+        .help   = "Test Secondary OSD port",
+        .category = FIO_OPT_C_ENGINE,
+        .group  = FIO_OPT_G_FILENAME,
+    },
+    {
+        .name   = "block_size_order",
+        .lname  = "Blockstore block size order",
+        .type   = FIO_OPT_INT,
+        .off1   = offsetof(struct sec_options, block_order),
+        .help   = "Blockstore block size order (size = 2^order)",
+        .category = FIO_OPT_C_ENGINE,
+        .group  = FIO_OPT_G_FILENAME,
+    },
+    {
+        .name   = "single_primary",
+        .lname  = "Single Primary",
+        .type   = FIO_OPT_BOOL,
+        .off1   = offsetof(struct sec_options, single_primary),
+        .help   = "Test single Primary OSD (one PG) instead of Secondary",
+        .def    = "0",
+        .category = FIO_OPT_C_ENGINE,
+        .group  = FIO_OPT_G_FILENAME,
+    },
+    {
+        .name   = "osd_trace",
+        .lname  = "OSD trace",
+        .type   = FIO_OPT_BOOL,
+        .off1   = offsetof(struct sec_options, trace),
+        .help   = "Trace OSD operations",
+        .def    = "0",
+        .category = FIO_OPT_C_ENGINE,
+        .group  = FIO_OPT_G_FILENAME,
+    },
+    {
+        .name   = "zerocopy_send",
+        .lname  = "Use zero-copy send",
+        .type   = FIO_OPT_BOOL,
+        .off1   = offsetof(struct sec_options, zerocopy_send),
+        .help   = "Use zero-copy send (MSG_ZEROCOPY)",
+        .def    = "0",
+        .category = FIO_OPT_C_ENGINE,
+        .group  = FIO_OPT_G_FILENAME,
+    },
+    {
+        .name = NULL,
+    },
+};
+
+static int sec_setup(struct thread_data *td)
+{
+    sec_data *bsd;
+    //fio_file *f;
+    //int r;
+    //int64_t size;
+
+    bsd = new sec_data;
+    if (!bsd)
+    {
+        td_verror(td, errno, "calloc");
+        return 1;
+    }
+    td->io_ops_data = bsd;
+
+    if (!td->files_index)
+    {
+        add_file(td, "bs_sec_osd", 0, 0);
+        td->o.nr_files = td->o.nr_files ? : 1;
+        td->o.open_files++;
+    }
+
+    //f = td->files[0];
+    //f->real_file_size = size;
+    return 0;
+}
+
+static void sec_cleanup(struct thread_data *td)
+{
+    sec_data *bsd = (sec_data*)td->io_ops_data;
+    if (bsd)
+    {
+        close(bsd->connect_fd);
+        delete bsd;
+    }
+}
+
+/* Connect to the server from each thread. */
+static int sec_init(struct thread_data *td)
+{
+    sec_options *o = (sec_options*)td->eo;
+    sec_data *bsd = (sec_data*)td->io_ops_data;
+    bsd->block_order = o->block_order == 0 ? 17 : o->block_order;
+    bsd->block_size = 1 << o->block_order;
+
+    sockaddr_storage addr;
+    if (!string_to_addr(std::string(o->host ? o->host : "127.0.0.1"), false, o->port > 0 ? o->port : 11203, &addr))
+    {
+        fprintf(stderr, "server address: %s is not valid\n", o->host ? o->host : "127.0.0.1");
+        return 1;
+    }
+
+    bsd->connect_fd = socket(addr.ss_family, SOCK_STREAM, 0);
+    if (bsd->connect_fd < 0)
+    {
+        perror("socket");
+        return 1;
+    }
+    if (connect(bsd->connect_fd, (sockaddr*)&addr, sizeof(addr)) < 0)
+    {
+        perror("connect");
+        return 1;
+    }
+    int one = 1;
+    setsockopt(bsd->connect_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
+    if (o->zerocopy_send)
+    {
+#ifndef SO_ZEROCOPY
+        perror("zerocopy send not supported on your system (socket.h misses SO_ZEROCOPY)");
+        return 1;
+#else
+        if (setsockopt(bsd->connect_fd, SOL_SOCKET, SO_ZEROCOPY, &one, sizeof(one)) < 0)
+        {
+            perror("setsockopt zerocopy");
+            return 1;
+        }
+#endif
+    }
+
+    // FIXME: read config (block size) from OSD
+
+    return 0;
+}
+
+/* Begin read or write request. */
+static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
+{
+    sec_options *opt = (sec_options*)td->eo;
+    sec_data *bsd = (sec_data*)td->io_ops_data;
+    int n = bsd->op_n;
+
+    fio_ro_check(td, io);
+    if (io->ddir == DDIR_SYNC && bsd->last_sync)
+    {
+        return FIO_Q_COMPLETED;
+    }
+
+    io->engine_data = bsd;
+    op_buf_t *op_buf = new op_buf_t;
+    op_buf->fio_op = io;
+    osd_any_op_t &op = op_buf->buf;
+
+    op.hdr.magic = SECONDARY_OSD_OP_MAGIC;
+    op.hdr.id = n;
+    switch (io->ddir)
+    {
+    case DDIR_READ:
+        if (!opt->single_primary)
+        {
+            op.hdr.opcode = OSD_OP_SEC_READ;
+            op.sec_rw.oid = {
+                .inode = 1,
+                .stripe = io->offset >> bsd->block_order,
+            };
+            op.sec_rw.version = UINT64_MAX; // last unstable
+            op.sec_rw.offset = io->offset % bsd->block_size;
+            op.sec_rw.len = io->xfer_buflen;
+            op.sec_rw.attr_len = 0;
+        }
+        else
+        {
+            op.hdr.opcode = OSD_OP_READ;
+            op.rw.inode = 1;
+            op.rw.offset = io->offset;
+            op.rw.len = io->xfer_buflen;
+        }
+        bsd->last_sync = false;
+        break;
+    case DDIR_WRITE:
+        if (!opt->single_primary)
+        {
+            op.hdr.opcode = OSD_OP_SEC_WRITE;
+            op.sec_rw.oid = {
+                .inode = 1,
+                .stripe = io->offset >> bsd->block_order,
+            };
+            op.sec_rw.version = 0; // assign automatically
+            op.sec_rw.offset = io->offset % bsd->block_size;
+            op.sec_rw.len = io->xfer_buflen;
+            op.sec_rw.attr_len = 0;
+        }
+        else
+        {
+            op.hdr.opcode = OSD_OP_WRITE;
+            op.rw.inode = 1;
+            op.rw.offset = io->offset;
+            op.rw.len = io->xfer_buflen;
+        }
+        bsd->last_sync = false;
+        break;
+    case DDIR_SYNC:
+        if (!opt->single_primary)
+        {
+            // Allowed only for testing: sync & stabilize all unstable object versions
+            op.hdr.opcode = OSD_OP_TEST_SYNC_STAB_ALL;
+        }
+        else
+        {
+            op.hdr.opcode = OSD_OP_SYNC;
+        }
+        // fio sends 32 syncs with -fsync=32. we omit 31 of them even though
+        // generally it may not be 100% correct (FIXME: fix fio itself)
+        bsd->last_sync = true;
+        break;
+    default:
+        io->error = EINVAL;
+        delete op_buf;
+        return FIO_Q_COMPLETED;
+    }
+
+    if (opt->trace)
+    {
+        printf("+++ %s # %d\n", io->ddir == DDIR_READ ? "READ" :
+            (io->ddir == DDIR_WRITE ? "WRITE" : "SYNC"), n);
+    }
+
+    io->error = 0;
+    bsd->inflight++;
+    bsd->op_n++;
+    bsd->queue[n] = op_buf;
+
+    iovec iov[2] = { { .iov_base = op.buf, .iov_len = OSD_PACKET_SIZE } };
+    int iovcnt = 1, wtotal = OSD_PACKET_SIZE;
+    if (io->ddir == DDIR_WRITE)
+    {
+        assert(io->xfer_buflen <= 0x7fffffff);
+        iov[iovcnt++] = { .iov_base = io->xfer_buf, .iov_len = (size_t)io->xfer_buflen };
+        wtotal += io->xfer_buflen;
+    }
+    if (sendv_blocking(bsd->connect_fd, iov, iovcnt,
+#ifdef SO_ZEROCOPY
+        opt->zerocopy_send ? MSG_ZEROCOPY : 0
+#else
+        0
+#endif
+    ) != wtotal)
+    {
+        perror("sendmsg");
+        exit(1);
+    }
+
+    if (io->error != 0)
+        return FIO_Q_COMPLETED;
+    return FIO_Q_QUEUED;
+}
+
+static int sec_getevents(struct thread_data *td, unsigned int min, unsigned int max, const struct timespec *t)
+{
+    sec_options *opt = (sec_options*)td->eo;
+    sec_data *bsd = (sec_data*)td->io_ops_data;
+    // FIXME timeout, at least poll. Now it's the stupidest implementation possible
+    osd_any_reply_t reply;
+    while (bsd->completed.size() < min)
+    {
+        read_blocking(bsd->connect_fd, reply.buf, OSD_PACKET_SIZE);
+        if (reply.hdr.magic != SECONDARY_OSD_REPLY_MAGIC)
+        {
+            fprintf(stderr, "bad reply: magic = %jx instead of %jx\n", reply.hdr.magic, SECONDARY_OSD_REPLY_MAGIC);
+            exit(1);
+        }
+        auto it = bsd->queue.find(reply.hdr.id);
+        if (it == bsd->queue.end())
+        {
+            fprintf(stderr, "bad reply: op id %jx missing in local queue\n", reply.hdr.id);
+            exit(1);
+        }
+        io_u* io = it->second->fio_op;
+        delete it->second;
+        bsd->queue.erase(it);
+        if (io->ddir == DDIR_READ)
+        {
+            if (reply.hdr.retval != io->xfer_buflen)
+            {
+                fprintf(stderr, "Short read: retval = %jd instead of %ju\n", reply.hdr.retval, (uint64_t)io->xfer_buflen);
+                exit(1);
+            }
+            // Support bitmap
+            uint64_t bitmap = 0;
+            int iovcnt = 0;
+            iovec iov[2];
+            if (reply.sec_rw.attr_len > 0)
+            {
+                if (reply.sec_rw.attr_len <= 8)
+                    iov[iovcnt++] = { .iov_base = &bitmap, .iov_len = reply.sec_rw.attr_len };
+                else
+                    iov[iovcnt++] = { .iov_base = (void*)(bitmap = (uint64_t)malloc(reply.sec_rw.attr_len)), .iov_len = reply.sec_rw.attr_len };
+            }
+            assert(io->xfer_buflen <= 0x7FFFFFFF);
+            iov[iovcnt++] = { .iov_base = io->xfer_buf, .iov_len = (size_t)io->xfer_buflen };
+            readv_blocking(bsd->connect_fd, iov, iovcnt);
+            if (reply.sec_rw.attr_len > 8)
+            {
+                free((void*)bitmap);
+            }
+        }
+        else if (io->ddir == DDIR_WRITE)
+        {
+            if (reply.hdr.retval != io->xfer_buflen)
+            {
+                fprintf(stderr, "Short write: retval = %jd instead of %ju\n", reply.hdr.retval, (uint64_t)io->xfer_buflen);
+                exit(1);
+            }
+        }
+        else if (io->ddir == DDIR_SYNC)
+        {
+            if (reply.hdr.retval != 0)
+            {
+                fprintf(stderr, "Sync failed: retval = %jd\n", reply.hdr.retval);
+                exit(1);
+            }
+        }
+        if (opt->trace)
+        {
+            printf("--- %s # %ju\n", io->ddir == DDIR_READ ? "READ" :
+                (io->ddir == DDIR_WRITE ? "WRITE" : "SYNC"), reply.hdr.id);
+        }
+        bsd->completed.push_back(io);
+    }
+    return bsd->completed.size();
+}
+
+static struct io_u *sec_event(struct thread_data *td, int event)
+{
+    sec_data *bsd = (sec_data*)td->io_ops_data;
+    if (bsd->completed.size() == 0)
+        return NULL;
+    /* FIXME We ignore the event number and assume fio calls us exactly once for [0..nr_events-1] */
+    struct io_u *ev = bsd->completed.back();
+    bsd->completed.pop_back();
+    return ev;
+}
+
+static int sec_io_u_init(struct thread_data *td, struct io_u *io)
+{
+    io->engine_data = NULL;
+    return 0;
+}
+
+static void sec_io_u_free(struct thread_data *td, struct io_u *io)
+{
+}
+
+static int sec_open_file(struct thread_data *td, struct fio_file *f)
+{
+    return 0;
+}
+
+static int sec_invalidate(struct thread_data *td, struct fio_file *f)
+{
+    return 0;
+}
+
+struct ioengine_ops ioengine = {
+    .name               = "vitastor_secondary_osd",
+    .version            = FIO_IOOPS_VERSION,
+    .flags              = FIO_MEMALIGN | FIO_DISKLESSIO | FIO_NOEXTEND,
+    .setup              = sec_setup,
+    .init               = sec_init,
+    .queue              = sec_queue,
+    .getevents          = sec_getevents,
+    .event              = sec_event,
+    .cleanup            = sec_cleanup,
+    .open_file          = sec_open_file,
+    .invalidate         = sec_invalidate,
+    .io_u_init          = sec_io_u_init,
+    .io_u_free          = sec_io_u_free,
+    .option_struct_size = sizeof(struct sec_options),
+    .options            = options,
+};
+
+static void fio_init fio_sec_register(void)
+{
+    register_ioengine(&ioengine);
+}
+
+static void fio_exit fio_sec_unregister(void)
+{
+    unregister_ioengine(&ioengine);
+}

src/client/http_client.cpp

@@ -0,0 +1,745 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+#include <netinet/tcp.h>
+#include <sys/epoll.h>
+
+#include <arpa/inet.h>
+
+#include <ctype.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <string.h>
+
+#include <stdexcept>
+
+#include "addr_util.h"
+#include "str_util.h"
+#include "json11/json11.hpp"
+#include "http_client.h"
+#include "timerfd_manager.h"
+
+#define READ_BUFFER_SIZE 9000
+
+static std::string ws_format_frame(int type, uint64_t size);
+static bool ws_parse_frame(std::string & buf, int & type, std::string & res);
+static void parse_http_headers(std::string & res, http_response_t *parsed);
+
+struct http_co_t
+{
+    timerfd_manager_t *tfd;
+    std::function<void(const http_response_t*)> response_callback;
+
+    int request_timeout = 0;
+    std::string host;
+    std::string request;
+    std::string ws_outbox;
+    std::string response;
+    bool want_streaming;
+    bool keepalive;
+
+    std::vector<std::function<void()>> keepalive_queue;
+
+    int state = 0;
+    std::string connected_host;
+    int peer_fd = -1;
+    int timeout_id = -1;
+    int epoll_events = 0;
+    int sent = 0;
+    std::vector<char> rbuf;
+    iovec read_iov, send_iov;
+    msghdr read_msg = { 0 }, send_msg = { 0 };
+    http_response_t parsed;
+    uint64_t target_response_size = 0;
+
+    int onstack = 0;
+    bool ended = false;
+
+    ~http_co_t();
+    inline void stackin() { onstack++; }
+    inline void stackout() { onstack--; if (!onstack && ended) end(); }
+    inline void end() { ended = true; if (!onstack) { delete this; } }
+    void run_cb_and_clear();
+    void start_connection();
+    void close_connection();
+    void next_request();
+    void handle_events();
+    void handle_connect_result();
+    void submit_read(bool check_timeout);
+    void submit_send();
+    bool handle_read();
+    void post_message(int type, const std::string & msg);
+    void send_request(const std::string & host, const std::string & request,
+        const http_options_t & options, std::function<void(const http_response_t *response)> response_callback);
+};
+
+#define HTTP_CO_CLOSED 0
+#define HTTP_CO_CONNECTING 1
+#define HTTP_CO_SENDING_REQUEST 2
+#define HTTP_CO_REQUEST_SENT 3
+#define HTTP_CO_HEADERS_RECEIVED 4
+#define HTTP_CO_WEBSOCKET 5
+#define HTTP_CO_CHUNKED 6
+#define HTTP_CO_KEEPALIVE 7
+
+#define DEFAULT_TIMEOUT 5000
+
+http_co_t *http_init(timerfd_manager_t *tfd)
+{
+    http_co_t *handler = new http_co_t();
+    handler->tfd = tfd;
+    handler->state = HTTP_CO_CLOSED;
+    return handler;
+}
+
+http_co_t* open_websocket(timerfd_manager_t *tfd, const std::string & host, const std::string & path,
+    int timeout, std::function<void(const http_response_t *msg)> response_callback)
+{
+    std::string request = "GET "+path+" HTTP/1.1\r\n"
+        "Host: "+host+"\r\n"
+        "Upgrade: websocket\r\n"
+        "Connection: upgrade\r\n"
+        "Sec-WebSocket-Key: x3JJHMbDL1EzLkh9GBhXDw==\r\n"
+        "Sec-WebSocket-Version: 13\r\n"
+        "\r\n";
+    http_co_t *handler = new http_co_t();
+    handler->tfd = tfd;
+    handler->state = HTTP_CO_CLOSED;
+    handler->host = host;
+    handler->request_timeout = timeout < 0 ? -1 : (timeout == 0 ? DEFAULT_TIMEOUT : timeout);
+    handler->want_streaming = false;
+    handler->keepalive = false;
+    handler->request = request;
+    handler->response_callback = response_callback;
+    handler->start_connection();
+    return handler;
+}
+
+void http_request(http_co_t *handler, const std::string & host, const std::string & request,
+    const http_options_t & options, std::function<void(const http_response_t *response)> response_callback)
+{
+    handler->send_request(host, request, options, response_callback);
+}
+
+void http_co_t::run_cb_and_clear()
+{
+    parsed.eof = true;
+    std::function<void(const http_response_t*)> cb;
+    cb.swap(response_callback);
+    // Call callback after clearing it because otherwise we may hit reenterability problems
+    if (cb != NULL)
+        cb(&parsed);
+    next_request();
+}
+
+void http_co_t::send_request(const std::string & host, const std::string & request,
+    const http_options_t & options, std::function<void(const http_response_t *response)> response_callback)
+{
+    stackin();
+    if (state == HTTP_CO_WEBSOCKET)
+    {
+        stackout();
+        throw std::runtime_error("Attempt to send HTTP request into a websocket or chunked stream");
+    }
+    else if (state != HTTP_CO_KEEPALIVE && state != HTTP_CO_CLOSED)
+    {
+        keepalive_queue.push_back([this, host, request, options, response_callback]()
+        {
+            this->send_request(host, request, options, response_callback);
+        });
+        stackout();
+        return;
+    }
+    if (state == HTTP_CO_KEEPALIVE && connected_host != host)
+    {
+        close_connection();
+    }
+    this->request_timeout = options.timeout < 0 ? 0 : (options.timeout == 0 ? DEFAULT_TIMEOUT : options.timeout);
+    this->want_streaming = options.want_streaming;
+    this->keepalive = options.keepalive;
+    this->host = host;
+    this->request = request;
+    this->response = "";
+    this->sent = 0;
+    this->response_callback = response_callback;
+    this->parsed = {};
+    if (state == HTTP_CO_KEEPALIVE)
+    {
+        state = HTTP_CO_SENDING_REQUEST;
+        submit_send();
+    }
+    else
+    {
+        start_connection();
+    }
+    // Do it _after_ state assignment because set_timer() can actually trigger
+    // other timers and requests (reenterability is our friend)
+    if (request_timeout > 0)
+    {
+        timeout_id = tfd->set_timer(request_timeout, false, [this](int timer_id)
+        {
+            stackin();
+            if (state == HTTP_CO_REQUEST_SENT)
+            {
+                // In case of high CPU load, we may not handle etcd responses in time
+                // For this case, first check the socket and only then terminate request with the timeout
+                submit_read(true);
+            }
+            else
+            {
+                close_connection();
+                parsed = { .error = "HTTP request timed out" };
+                run_cb_and_clear();
+            }
+            stackout();
+        });
+    }
+    stackout();
+}
+
+void http_post_message(http_co_t *handler, int type, const std::string & msg)
+{
+    handler->post_message(type, msg);
+}
+
+void http_co_t::post_message(int type, const std::string & msg)
+{
+    stackin();
+    if (state == HTTP_CO_WEBSOCKET)
+    {
+        request += ws_format_frame(type, msg.size());
+        request += msg;
+        submit_send();
+    }
+    else if (state == HTTP_CO_KEEPALIVE || state == HTTP_CO_CHUNKED)
+    {
+        throw std::runtime_error("Attempt to send websocket message on a regular HTTP connection");
+    }
+    else
+    {
+        ws_outbox += ws_format_frame(type, msg.size());
+        ws_outbox += msg;
+    }
+    stackout();
+}
+
+void http_close(http_co_t *handler)
+{
+    handler->end();
+}
+
+void http_response_t::parse_json_response(std::string & error, json11::Json & r) const
+{
+    if (this->error != "")
+    {
+        error = this->error;
+        r = json11::Json();
+    }
+    else if (status_code != 200)
+    {
+        error = "HTTP "+std::to_string(status_code)+" "+status_line+" body: "+trim(body);
+        r = json11::Json();
+    }
+    else
+    {
+        std::string json_err;
+        json11::Json data = json11::Json::parse(body, json_err);
+        if (json_err != "")
+        {
+            error = "Bad JSON: "+json_err+" (response: "+trim(body)+")";
+            r = json11::Json();
+        }
+        else
+        {
+            error = "";
+            r = data;
+        }
+    }
+}
+
+http_co_t::~http_co_t()
+{
+    close_connection();
+}
+
+void http_co_t::close_connection()
+{
+    if (timeout_id >= 0)
+    {
+        tfd->clear_timer(timeout_id);
+        timeout_id = -1;
+    }
+    if (peer_fd >= 0)
+    {
+        tfd->set_fd_handler(peer_fd, false, NULL);
+        close(peer_fd);
+        peer_fd = -1;
+    }
+    state = HTTP_CO_CLOSED;
+    connected_host = "";
+    response = "";
+    epoll_events = 0;
+}
+
+void http_co_t::start_connection()
+{
+    stackin();
+    struct sockaddr_storage addr;
+    if (!string_to_addr(host.c_str(), 1, 80, &addr))
+    {
+        close_connection();
+        parsed = { .error = "Invalid address: "+host };
+        run_cb_and_clear();
+        stackout();
+        return;
+    }
+    peer_fd = socket(addr.ss_family, SOCK_STREAM, 0);
+    if (peer_fd < 0)
+    {
+        close_connection();
+        parsed = { .error = std::string("socket: ")+strerror(errno) };
+        run_cb_and_clear();
+        stackout();
+        return;
+    }
+    fcntl(peer_fd, F_SETFL, fcntl(peer_fd, F_GETFL, 0) | O_NONBLOCK);
+    epoll_events = 0;
+    // Finally call connect
+    int r = ::connect(peer_fd, (sockaddr*)&addr, sizeof(addr));
+    if (r < 0 && errno != EINPROGRESS)
+    {
+        close_connection();
+        parsed = { .error = std::string("connect: ")+strerror(errno) };
+        run_cb_and_clear();
+        stackout();
+        return;
+    }
+    tfd->set_fd_handler(peer_fd, true, [this](int peer_fd, int epoll_events)
+    {
+        this->epoll_events |= epoll_events;
+        handle_events();
+    });
+    connected_host = host;
+    state = HTTP_CO_CONNECTING;
+    stackout();
+}
+
+void http_co_t::handle_events()
+{
+    stackin();
+    while (epoll_events)
+    {
+        if (state == HTTP_CO_CONNECTING)
+        {
+            handle_connect_result();
+        }
+        else
+        {
+            epoll_events &= ~EPOLLOUT;
+            if (epoll_events & EPOLLIN)
+            {
+                submit_read(false);
+            }
+            else if (epoll_events & (EPOLLRDHUP|EPOLLERR))
+            {
+                if (state == HTTP_CO_HEADERS_RECEIVED)
+                    std::swap(parsed.body, response);
+                close_connection();
+                run_cb_and_clear();
+                break;
+            }
+        }
+    }
+    stackout();
+}
+
+void http_co_t::handle_connect_result()
+{
+    stackin();
+    int result = 0;
+    socklen_t result_len = sizeof(result);
+    if (getsockopt(peer_fd, SOL_SOCKET, SO_ERROR, &result, &result_len) < 0)
+    {
+        result = errno;
+    }
+    if (result != 0)
+    {
+        close_connection();
+        parsed = { .error = std::string("connect: ")+strerror(result) };
+        run_cb_and_clear();
+        stackout();
+        return;
+    }
+    int one = 1;
+    setsockopt(peer_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
+    tfd->set_fd_handler(peer_fd, false, [this](int peer_fd, int epoll_events)
+    {
+        this->epoll_events |= epoll_events;
+        handle_events();
+    });
+    state = HTTP_CO_SENDING_REQUEST;
+    submit_send();
+    stackout();
+}
+
+void http_co_t::submit_send()
+{
+    stackin();
+    int res;
+again:
+    if (sent < request.size())
+    {
+        send_iov = (iovec){ .iov_base = (void*)(request.c_str()+sent), .iov_len = request.size()-sent };
+        send_msg.msg_iov = &send_iov;
+        send_msg.msg_iovlen = 1;
+        res = sendmsg(peer_fd, &send_msg, MSG_NOSIGNAL);
+        if (res < 0)
+        {
+            res = -errno;
+        }
+        if (res == -EAGAIN || res == -EINTR)
+        {
+            res = 0;
+        }
+        else if (res < 0)
+        {
+            close_connection();
+            parsed = { .error = std::string("sendmsg: ")+strerror(errno) };
+            run_cb_and_clear();
+            stackout();
+            return;
+        }
+        sent += res;
+        if (state == HTTP_CO_SENDING_REQUEST)
+        {
+            if (sent >= request.size())
+                state = HTTP_CO_REQUEST_SENT;
+            else
+                goto again;
+        }
+        else if (state == HTTP_CO_WEBSOCKET)
+        {
+            request = request.substr(sent);
+            sent = 0;
+            goto again;
+        }
+    }
+    stackout();
+}
+
+void http_co_t::submit_read(bool check_timeout)
+{
+    stackin();
+    int res;
+again:
+    if (rbuf.size() != READ_BUFFER_SIZE)
+    {
+        rbuf.resize(READ_BUFFER_SIZE);
+    }
+    read_iov = { .iov_base = rbuf.data(), .iov_len = READ_BUFFER_SIZE };
+    read_msg.msg_iov = &read_iov;
+    read_msg.msg_iovlen = 1;
+    res = recvmsg(peer_fd, &read_msg, 0);
+    if (res < 0)
+    {
+        res = -errno;
+    }
+    if (res == -EAGAIN || res == -EINTR)
+    {
+        if (check_timeout)
+        {
+            if (res == -EINTR)
+                goto again;
+            else
+            {
+                // Timeout happened and there is no data to read
+                close_connection();
+                parsed = { .error = "HTTP request timed out" };
+                run_cb_and_clear();
+            }
+        }
+        else
+        {
+            epoll_events = epoll_events & ~EPOLLIN;
+        }
+    }
+    else if (res <= 0)
+    {
+        // < 0 means error, 0 means EOF
+        epoll_events = epoll_events & ~EPOLLIN;
+        if (state == HTTP_CO_HEADERS_RECEIVED)
+            std::swap(parsed.body, response);
+        close_connection();
+        if (res < 0)
+            parsed = { .error = std::string("recvmsg: ")+strerror(-res) };
+        run_cb_and_clear();
+    }
+    else
+    {
+        response += std::string(rbuf.data(), res);
+        handle_read();
+    }
+    stackout();
+}
+
+bool http_co_t::handle_read()
+{
+    stackin();
+    if (state == HTTP_CO_REQUEST_SENT)
+    {
+        int pos = response.find("\r\n\r\n");
+        if (pos >= 0)
+        {
+            if (timeout_id >= 0)
+            {
+                // Timeout is cleared when headers are received
+                tfd->clear_timer(timeout_id);
+                timeout_id = -1;
+            }
+            state = HTTP_CO_HEADERS_RECEIVED;
+            parse_http_headers(response, &parsed);
+            if (parsed.status_code == 101 &&
+                parsed.headers.find("sec-websocket-accept") != parsed.headers.end() &&
+                parsed.headers["upgrade"] == "websocket" &&
+                parsed.headers["connection"] == "upgrade")
+            {
+                // Don't care about validating the key
+                state = HTTP_CO_WEBSOCKET;
+                request = ws_outbox;
+                ws_outbox = "";
+                sent = 0;
+                submit_send();
+            }
+            else if (parsed.headers["transfer-encoding"] == "chunked")
+            {
+                state = HTTP_CO_CHUNKED;
+            }
+            else if (parsed.headers["connection"] != "close")
+            {
+                target_response_size = stoull_full(parsed.headers["content-length"]);
+                if (!target_response_size)
+                {
+                    // Sorry, unsupported response
+                    close_connection();
+                    parsed = { .error = "Response has neither Connection: close, nor Transfer-Encoding: chunked nor Content-Length headers" };
+                    run_cb_and_clear();
+                    stackout();
+                    return false;
+                }
+            }
+            else
+            {
+                keepalive = false;
+            }
+        }
+    }
+    if (state == HTTP_CO_HEADERS_RECEIVED && target_response_size > 0 && response.size() >= target_response_size)
+    {
+        std::swap(parsed.body, response);
+        if (!keepalive)
+            close_connection();
+        else
+            state = HTTP_CO_KEEPALIVE;
+        run_cb_and_clear();
+    }
+    else if (state == HTTP_CO_CHUNKED && response.size() > 0)
+    {
+        int prev = 0, pos = 0;
+        while ((pos = response.find("\r\n", prev)) >= prev)
+        {
+            uint64_t len = strtoull(response.c_str()+prev, NULL, 16);
+            if (!len)
+            {
+                // Zero length chunk indicates EOF
+                parsed.eof = true;
+                break;
+            }
+            if (response.size() < pos+2+len+2)
+            {
+                break;
+            }
+            parsed.body += response.substr(pos+2, len);
+            prev = pos+2+len+2;
+        }
+        if (prev > 0)
+        {
+            response = response.substr(prev);
+        }
+        if (want_streaming)
+        {
+            // Streaming response
+            response_callback(&parsed);
+            parsed.body = "";
+        }
+        else if (parsed.eof)
+        {
+            // Normal response
+            if (!keepalive)
+                close_connection();
+            else
+                state = HTTP_CO_KEEPALIVE;
+            run_cb_and_clear();
+        }
+    }
+    else if (state == HTTP_CO_WEBSOCKET && response.size() > 0)
+    {
+        while (ws_parse_frame(response, parsed.ws_msg_type, parsed.body))
+        {
+            response_callback(&parsed);
+            parsed.body = "";
+        }
+    }
+    stackout();
+    return true;
+}
+
+void http_co_t::next_request()
+{
+    if (keepalive_queue.size() > 0)
+    {
+        auto next = keepalive_queue[0];
+        keepalive_queue.erase(keepalive_queue.begin(), keepalive_queue.begin()+1);
+        next();
+    }
+}
+
+static void parse_http_headers(std::string & res, http_response_t *parsed)
+{
+    int pos = res.find("\r\n");
+    pos = pos < 0 ? res.length() : pos+2;
+    std::string status_line = res.substr(0, pos);
+    int http_version;
+    char *status_text = NULL;
+    sscanf(status_line.c_str(), "HTTP/1.%d %d %ms", &http_version, &parsed->status_code, &status_text);
+    if (status_text)
+    {
+        parsed->status_line = status_text;
+        // %ms = allocate a buffer
+        free(status_text);
+        status_text = NULL;
+    }
+    int prev = pos;
+    while ((pos = res.find("\r\n", prev)) >= prev)
+    {
+        if (pos == prev)
+        {
+            res = res.substr(pos+2);
+            break;
+        }
+        std::string header = res.substr(prev, pos-prev);
+        int p2 = header.find(":");
+        if (p2 >= 0)
+        {
+            std::string key = strtolower(header.substr(0, p2));
+            int p3 = p2+1;
+            while (p3 < header.length() && isblank(header[p3]))
+                p3++;
+            parsed->headers[key] = key == "connection" || key == "upgrade" || key == "transfer-encoding"
+                ? strtolower(header.substr(p3)) : header.substr(p3);
+        }
+        prev = pos+2;
+    }
+}
+
+static std::string ws_format_frame(int type, uint64_t size)
+{
+    // Always zero mask
+    std::string res;
+    int p = 0;
+    res.resize(2 + (size >= 126 ? 2 : 0) + (size >= 65536 ? 6 : 0) + /*mask*/4);
+    res[p++] = 0x80 | type;
+    if (size < 126)
+        res[p++] = size | /*mask*/0x80;
+    else if (size < 65536)
+    {
+        res[p++] = (char)(126 | /*mask*/0x80);
+        res[p++] = (size >> 8) & 0xFF;
+        res[p++] = (size >> 0) & 0xFF;
+    }
+    else
+    {
+        res[p++] = (char)(127 | /*mask*/0x80);
+        res[p++] = (size >> 56) & 0xFF;
+        res[p++] = (size >> 48) & 0xFF;
+        res[p++] = (size >> 40) & 0xFF;
+        res[p++] = (size >> 32) & 0xFF;
+        res[p++] = (size >> 24) & 0xFF;
+        res[p++] = (size >> 16) & 0xFF;
+        res[p++] = (size >>  8) & 0xFF;
+        res[p++] = (size >>  0) & 0xFF;
+    }
+    res[p++] = 0;
+    res[p++] = 0;
+    res[p++] = 0;
+    res[p++] = 0;
+    return res;
+}
+
+static bool ws_parse_frame(std::string & buf, int & type, std::string & res)
+{
+    uint64_t hdr = 2;
+    if (buf.size() < hdr)
+    {
+        return false;
+    }
+    type = buf[0] & ~0x80;
+    bool mask = !!(buf[1] & 0x80);
+    hdr += mask ? 4 : 0;
+    uint64_t len = ((uint8_t)buf[1] & ~0x80);
+    if (len == 126)
+    {
+        hdr += 2;
+        if (buf.size() < hdr)
+        {
+            return false;
+        }
+        len = ((uint64_t)(uint8_t)buf[2] << 8) | ((uint64_t)(uint8_t)buf[3] << 0);
+    }
+    else if (len == 127)
+    {
+        hdr += 8;
+        if (buf.size() < hdr)
+        {
+            return false;
+        }
+        len = ((uint64_t)(uint8_t)buf[2] << 56) |
+            ((uint64_t)(uint8_t)buf[3] << 48) |
+            ((uint64_t)(uint8_t)buf[4] << 40) |
+            ((uint64_t)(uint8_t)buf[5] << 32) |
+            ((uint64_t)(uint8_t)buf[6] << 24) |
+            ((uint64_t)(uint8_t)buf[7] << 16) |
+            ((uint64_t)(uint8_t)buf[8] << 8) |
+            ((uint64_t)(uint8_t)buf[9] << 0);
+    }
+    if (buf.size() < hdr+len)
+    {
+        return false;
+    }
+    if (mask)
+    {
+        for (int i = 0; i < len; i++)
+            buf[hdr+i] ^= buf[hdr-4+(i & 3)];
+    }
+    res += buf.substr(hdr, len);
+    buf = buf.substr(hdr+len);
+    return true;
+}
+
+// FIXME: move to utils
+bool json_is_true(const json11::Json & val)
+{
+    if (val.is_string())
+        return val == "true" || val == "yes" || val == "1";
+    return val.bool_value();
+}
+
+bool json_is_false(const json11::Json & val)
+{
+    if (val.is_string())
+        return val.string_value() == "false" || val.string_value() == "no" || val.string_value() == "0";
+    if (val.is_number())
+        return val.number_value() == 0;
+    if (val.is_bool())
+        return !val.bool_value();
+    return false;
+}

src/client/http_client.h

@@ -0,0 +1,56 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+#pragma once
+#include <string>
+#include <vector>
+#include <map>
+#include <functional>
+#include "json11/json11.hpp"
+
+#define WS_CONTINUATION 0
+#define WS_TEXT 1
+#define WS_BINARY 2
+#define WS_CLOSE 8
+#define WS_PING 9
+#define WS_PONG 10
+
+class timerfd_manager_t;
+
+struct http_options_t
+{
+    int timeout;
+    bool want_streaming;
+    bool keepalive;
+};
+
+struct http_response_t
+{
+    std::string error;
+
+    bool eof = false;
+    int status_code = 0;
+    std::string status_line;
+    std::map<std::string, std::string> headers;
+    int ws_msg_type = -1;
+    std::string body;
+
+    void parse_json_response(std::string & error, json11::Json & r) const;
+};
+
+// Opened websocket or keepalive HTTP connection
+struct http_co_t;
+
+http_co_t* http_init(timerfd_manager_t *tfd);
+http_co_t* open_websocket(timerfd_manager_t *tfd, const std::string & host, const std::string & path,
+    int timeout, std::function<void(const http_response_t *msg)> on_message);
+void http_request(http_co_t *handler, const std::string & host, const std::string & request,
+    const http_options_t & options, std::function<void(const http_response_t *response)> response_callback);
+void http_post_message(http_co_t *handler, int type, const std::string & msg);
+void http_close(http_co_t *co);
+
+// Utils
+std::string strtolower(const std::string & in);
+// FIXME: move to json11
+bool json_is_true(const json11::Json & val);
+bool json_is_false(const json11::Json & val);

src/client/messenger.cpp

@@ -0,0 +1,701 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/epoll.h>
+#include <netinet/tcp.h>
+#include <stdexcept>
+
+#include "addr_util.h"
+#include "messenger.h"
+#ifdef WITH_RDMA
+#include "msgr_rdma.h"
+#endif
+
+void osd_messenger_t::init()
+{
+#ifdef WITH_RDMA
+    if (use_rdma)
+    {
+        rdma_context = msgr_rdma_context_t::create(
+            rdma_device != "" ? rdma_device.c_str() : NULL,
+            rdma_port_num, rdma_gid_index, rdma_mtu, rdma_odp, log_level
+        );
+        if (!rdma_context)
+        {
+            if (log_level > 0)
+                fprintf(stderr, "[OSD %ju] Couldn't initialize RDMA, proceeding with TCP only\n", osd_num);
+        }
+        else
+        {
+            rdma_max_sge = rdma_max_sge < rdma_context->attrx.orig_attr.max_sge
+                ? rdma_max_sge : rdma_context->attrx.orig_attr.max_sge;
+            fprintf(stderr, "[OSD %ju] RDMA initialized successfully\n", osd_num);
+            fcntl(rdma_context->channel->fd, F_SETFL, fcntl(rdma_context->channel->fd, F_GETFL, 0) | O_NONBLOCK);
+            tfd->set_fd_handler(rdma_context->channel->fd, false, [this](int notify_fd, int epoll_events)
+            {
+                handle_rdma_events();
+            });
+            handle_rdma_events();
+        }
+    }
+#endif
+    keepalive_timer_id = tfd->set_timer(1000, true, [this](int)
+    {
+        auto cl_it = clients.begin();
+        while (cl_it != clients.end())
+        {
+            auto cl = cl_it->second;
+            cl_it++;
+            auto peer_fd = cl->peer_fd;
+            if (!cl->osd_num || cl->peer_state != PEER_CONNECTED && cl->peer_state != PEER_RDMA)
+            {
+                // Do not run keepalive on regular clients
+                continue;
+            }
+            if (cl->ping_time_remaining > 0)
+            {
+                cl->ping_time_remaining--;
+                if (!cl->ping_time_remaining)
+                {
+                    // Ping timed out, stop the client
+                    fprintf(stderr, "Ping timed out for OSD %ju (client %d), disconnecting peer\n", cl->osd_num, cl->peer_fd);
+                    stop_client(peer_fd, true);
+                    // Restart iterator because it may be invalidated
+                    cl_it = clients.upper_bound(peer_fd);
+                }
+            }
+            else if (cl->idle_time_remaining > 0)
+            {
+                cl->idle_time_remaining--;
+                if (!cl->idle_time_remaining)
+                {
+                    // Connection is idle for <osd_idle_time>, send ping
+                    osd_op_t *op = new osd_op_t();
+                    op->op_type = OSD_OP_OUT;
+                    op->peer_fd = cl->peer_fd;
+                    op->req = (osd_any_op_t){
+                        .hdr = {
+                            .magic = SECONDARY_OSD_OP_MAGIC,
+                            .id = this->next_subop_id++,
+                            .opcode = OSD_OP_PING,
+                        },
+                    };
+                    op->callback = [this, cl](osd_op_t *op)
+                    {
+                        auto cl_it = clients.find(op->peer_fd);
+                        if (cl_it == clients.end() || cl_it->second != cl)
+                        {
+                            // client is already dropped
+                            delete op;
+                            return;
+                        }
+                        int fail_fd = (op->reply.hdr.retval != 0 ? op->peer_fd : -1);
+                        auto fail_osd_num = cl->osd_num;
+                        cl->ping_time_remaining = 0;
+                        delete op;
+                        if (fail_fd >= 0)
+                        {
+                            fprintf(stderr, "Ping failed for OSD %ju (client %d), disconnecting peer\n", fail_osd_num, fail_fd);
+                            stop_client(fail_fd, true);
+                        }
+                    };
+                    cl->ping_time_remaining = osd_ping_timeout;
+                    cl->idle_time_remaining = osd_idle_timeout;
+                    outbox_push(op);
+                    // Restart iterator because it may be invalidated
+                    cl_it = clients.upper_bound(peer_fd);
+                }
+            }
+            else
+            {
+                cl->idle_time_remaining = osd_idle_timeout;
+            }
+        }
+    });
+}
+
+osd_messenger_t::~osd_messenger_t()
+{
+    if (keepalive_timer_id >= 0)
+    {
+        tfd->clear_timer(keepalive_timer_id);
+        keepalive_timer_id = -1;
+    }
+    while (clients.size() > 0)
+    {
+        stop_client(clients.begin()->first, true, true);
+    }
+#ifdef WITH_RDMA
+    if (rdma_context)
+    {
+        delete rdma_context;
+    }
+#endif
+}
+
+void osd_messenger_t::parse_config(const json11::Json & config)
+{
+#ifdef WITH_RDMA
+    if (!config["use_rdma"].is_null())
+    {
+        // RDMA is on by default in RDMA-enabled builds
+        this->use_rdma = config["use_rdma"].bool_value() || config["use_rdma"].uint64_value() != 0;
+    }
+    this->rdma_device = config["rdma_device"].string_value();
+    this->rdma_port_num = (uint8_t)config["rdma_port_num"].uint64_value();
+    if (!this->rdma_port_num)
+        this->rdma_port_num = 1;
+    this->rdma_gid_index = (uint8_t)config["rdma_gid_index"].uint64_value();
+    this->rdma_mtu = (uint32_t)config["rdma_mtu"].uint64_value();
+    this->rdma_max_sge = config["rdma_max_sge"].uint64_value();
+    if (!this->rdma_max_sge)
+        this->rdma_max_sge = 128;
+    this->rdma_max_send = config["rdma_max_send"].uint64_value();
+    if (!this->rdma_max_send)
+        this->rdma_max_send = 8;
+    this->rdma_max_recv = config["rdma_max_recv"].uint64_value();
+    if (!this->rdma_max_recv)
+        this->rdma_max_recv = 16;
+    this->rdma_max_msg = config["rdma_max_msg"].uint64_value();
+    if (!this->rdma_max_msg || this->rdma_max_msg > 128*1024*1024)
+        this->rdma_max_msg = 129*1024;
+    this->rdma_odp = config["rdma_odp"].bool_value();
+#endif
+    this->receive_buffer_size = (uint32_t)config["tcp_header_buffer_size"].uint64_value();
+    if (!this->receive_buffer_size || this->receive_buffer_size > 1024*1024*1024)
+        this->receive_buffer_size = 65536;
+    this->use_sync_send_recv = config["use_sync_send_recv"].bool_value() ||
+        config["use_sync_send_recv"].uint64_value();
+    this->peer_connect_interval = config["peer_connect_interval"].uint64_value();
+    if (!this->peer_connect_interval)
+        this->peer_connect_interval = 5;
+    this->peer_connect_timeout = config["peer_connect_timeout"].uint64_value();
+    if (!this->peer_connect_timeout)
+        this->peer_connect_timeout = 5;
+    this->osd_idle_timeout = config["osd_idle_timeout"].uint64_value();
+    if (!this->osd_idle_timeout)
+        this->osd_idle_timeout = 5;
+    this->osd_ping_timeout = config["osd_ping_timeout"].uint64_value();
+    if (!this->osd_ping_timeout)
+        this->osd_ping_timeout = 5;
+    this->log_level = config["log_level"].uint64_value();
+}
+
+void osd_messenger_t::connect_peer(uint64_t peer_osd, json11::Json peer_state)
+{
+    if (wanted_peers.find(peer_osd) == wanted_peers.end())
+    {
+        wanted_peers[peer_osd] = (osd_wanted_peer_t){
+            .address_list = peer_state["addresses"],
+            .port = (int)peer_state["port"].int64_value(),
+        };
+    }
+    else
+    {
+        wanted_peers[peer_osd].address_list = peer_state["addresses"];
+        wanted_peers[peer_osd].port = (int)peer_state["port"].int64_value();
+    }
+    wanted_peers[peer_osd].address_changed = true;
+    try_connect_peer(peer_osd);
+}
+
+void osd_messenger_t::try_connect_peer(uint64_t peer_osd)
+{
+    auto wp_it = wanted_peers.find(peer_osd);
+    if (wp_it == wanted_peers.end() || wp_it->second.connecting ||
+        (time(NULL) - wp_it->second.last_connect_attempt) < peer_connect_interval)
+    {
+        return;
+    }
+    if (osd_peer_fds.find(peer_osd) != osd_peer_fds.end())
+    {
+        wanted_peers.erase(peer_osd);
+        return;
+    }
+    auto & wp = wp_it->second;
+    if (wp.address_index >= wp.address_list.array_items().size())
+    {
+        return;
+    }
+    wp.cur_addr = wp.address_list[wp.address_index].string_value();
+    wp.cur_port = wp.port;
+    wp.connecting = true;
+    try_connect_peer_addr(peer_osd, wp.cur_addr.c_str(), wp.cur_port);
+}
+
+void osd_messenger_t::try_connect_peer_addr(osd_num_t peer_osd, const char *peer_host, int peer_port)
+{
+    assert(peer_osd != this->osd_num);
+    struct sockaddr_storage addr;
+    if (!string_to_addr(peer_host, 0, peer_port, &addr))
+    {
+        on_connect_peer(peer_osd, -EINVAL);
+        return;
+    }
+    int peer_fd = socket(addr.ss_family, SOCK_STREAM, 0);
+    if (peer_fd < 0)
+    {
+        on_connect_peer(peer_osd, -errno);
+        return;
+    }
+    fcntl(peer_fd, F_SETFL, fcntl(peer_fd, F_GETFL, 0) | O_NONBLOCK);
+    int r = connect(peer_fd, (sockaddr*)&addr, sizeof(addr));
+    if (r < 0 && errno != EINPROGRESS)
+    {
+        close(peer_fd);
+        on_connect_peer(peer_osd, -errno);
+        return;
+    }
+    clients[peer_fd] = new osd_client_t();
+    if (log_level > 0)
+    {
+        fprintf(stderr, "Connecting to OSD %ju at %s:%d (client %d)\n", peer_osd, peer_host, peer_port, peer_fd);
+    }
+    clients[peer_fd]->peer_addr = addr;
+    clients[peer_fd]->peer_port = peer_port;
+    clients[peer_fd]->peer_fd = peer_fd;
+    clients[peer_fd]->peer_state = PEER_CONNECTING;
+    clients[peer_fd]->connect_timeout_id = -1;
+    clients[peer_fd]->osd_num = peer_osd;
+    clients[peer_fd]->in_buf = malloc_or_die(receive_buffer_size);
+    tfd->set_fd_handler(peer_fd, true, [this](int peer_fd, int epoll_events)
+    {
+        // Either OUT (connected) or HUP
+        handle_connect_epoll(peer_fd);
+    });
+    if (peer_connect_timeout > 0)
+    {
+        clients[peer_fd]->connect_timeout_id = tfd->set_timer(1000*peer_connect_timeout, false, [this, peer_fd](int timer_id)
+        {
+            osd_num_t peer_osd = clients.at(peer_fd)->osd_num;
+            stop_client(peer_fd, true);
+            on_connect_peer(peer_osd, -EPIPE);
+            return;
+        });
+    }
+}
+
+void osd_messenger_t::handle_connect_epoll(int peer_fd)
+{
+    auto cl = clients[peer_fd];
+    if (cl->connect_timeout_id >= 0)
+    {
+        tfd->clear_timer(cl->connect_timeout_id);
+        cl->connect_timeout_id = -1;
+    }
+    osd_num_t peer_osd = cl->osd_num;
+    int result = 0;
+    socklen_t result_len = sizeof(result);
+    if (getsockopt(peer_fd, SOL_SOCKET, SO_ERROR, &result, &result_len) < 0)
+    {
+        result = errno;
+    }
+    if (result != 0)
+    {
+        stop_client(peer_fd, true);
+        on_connect_peer(peer_osd, -result);
+        return;
+    }
+    int one = 1;
+    setsockopt(peer_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
+    cl->peer_state = PEER_CONNECTED;
+    tfd->set_fd_handler(peer_fd, false, [this](int peer_fd, int epoll_events)
+    {
+        handle_peer_epoll(peer_fd, epoll_events);
+    });
+    // Check OSD number
+    check_peer_config(cl);
+}
+
+void osd_messenger_t::handle_peer_epoll(int peer_fd, int epoll_events)
+{
+    // Mark client as ready (i.e. some data is available)
+    if (epoll_events & EPOLLRDHUP)
+    {
+        // Stop client
+        if (log_level > 0)
+        {
+            fprintf(stderr, "[OSD %ju] client %d disconnected\n", this->osd_num, peer_fd);
+        }
+        stop_client(peer_fd, true);
+    }
+    else if (epoll_events & EPOLLIN)
+    {
+        // Mark client as ready (i.e. some data is available)
+        auto cl = clients[peer_fd];
+        cl->read_ready++;
+        if (cl->read_ready == 1)
+        {
+            read_ready_clients.push_back(cl->peer_fd);
+            if (ringloop)
+                ringloop->wakeup();
+            else
+                read_requests();
+        }
+    }
+}
+
+void osd_messenger_t::on_connect_peer(osd_num_t peer_osd, int peer_fd)
+{
+    auto & wp = wanted_peers.at(peer_osd);
+    wp.connecting = false;
+    if (peer_fd < 0)
+    {
+        fprintf(stderr, "Failed to connect to peer OSD %ju address %s port %d: %s\n", peer_osd, wp.cur_addr.c_str(), wp.cur_port, strerror(-peer_fd));
+        if (wp.address_changed)
+        {
+            wp.address_changed = false;
+            wp.address_index = 0;
+            try_connect_peer(peer_osd);
+        }
+        else if (wp.address_index < wp.address_list.array_items().size()-1)
+        {
+            // Try other addresses
+            wp.address_index++;
+            try_connect_peer(peer_osd);
+        }
+        else
+        {
+            // Retry again in <peer_connect_interval> seconds
+            wp.last_connect_attempt = time(NULL);
+            wp.address_index = 0;
+            tfd->set_timer(1000*peer_connect_interval, false, [this, peer_osd](int)
+            {
+                try_connect_peer(peer_osd);
+            });
+        }
+        return;
+    }
+    if (log_level > 0)
+    {
+        fprintf(stderr, "[OSD %ju] Connected with peer OSD %ju (client %d)\n", osd_num, peer_osd, peer_fd);
+    }
+    wanted_peers.erase(peer_osd);
+    repeer_pgs(peer_osd);
+}
+
+void osd_messenger_t::check_peer_config(osd_client_t *cl)
+{
+    osd_op_t *op = new osd_op_t();
+    op->op_type = OSD_OP_OUT;
+    op->peer_fd = cl->peer_fd;
+    op->req = (osd_any_op_t){
+        .show_conf = {
+            .header = {
+                .magic = SECONDARY_OSD_OP_MAGIC,
+                .id = this->next_subop_id++,
+                .opcode = OSD_OP_SHOW_CONFIG,
+            },
+        },
+    };
+#ifdef WITH_RDMA
+    if (rdma_context)
+    {
+        cl->rdma_conn = msgr_rdma_connection_t::create(rdma_context, rdma_max_send, rdma_max_recv, rdma_max_sge, rdma_max_msg);
+        if (cl->rdma_conn)
+        {
+            json11::Json payload = json11::Json::object {
+                { "connect_rdma", cl->rdma_conn->addr.to_string() },
+                { "rdma_max_msg", cl->rdma_conn->max_msg },
+            };
+            std::string payload_str = payload.dump();
+            op->req.show_conf.json_len = payload_str.size();
+            op->buf = malloc_or_die(payload_str.size());
+            op->iov.push_back(op->buf, payload_str.size());
+            memcpy(op->buf, payload_str.c_str(), payload_str.size());
+        }
+    }
+#endif
+    op->callback = [this, cl](osd_op_t *op)
+    {
+        std::string json_err;
+        json11::Json config;
+        bool err = false;
+        if (op->reply.hdr.retval < 0)
+        {
+            err = true;
+            fprintf(stderr, "Failed to get config from OSD %ju (retval=%jd), disconnecting peer\n", cl->osd_num, op->reply.hdr.retval);
+        }
+        else
+        {
+            config = json11::Json::parse(std::string((char*)op->buf), json_err);
+            if (json_err != "")
+            {
+                err = true;
+                fprintf(stderr, "Failed to get config from OSD %ju: bad JSON: %s, disconnecting peer\n", cl->osd_num, json_err.c_str());
+            }
+            else if (config["osd_num"].uint64_value() != cl->osd_num)
+            {
+                err = true;
+                fprintf(stderr, "Connected to OSD %ju instead of OSD %ju, peer state is outdated, disconnecting peer\n", config["osd_num"].uint64_value(), cl->osd_num);
+            }
+            else if (config["protocol_version"].uint64_value() != OSD_PROTOCOL_VERSION)
+            {
+                err = true;
+                fprintf(
+                    stderr, "OSD %ju protocol version is %ju, but only version %u is supported.\n"
+                    " If you need to upgrade from 0.5.x please request it via the issue tracker.\n",
+                    cl->osd_num, config["protocol_version"].uint64_value(), OSD_PROTOCOL_VERSION
+                );
+            }
+            if (check_config_hook)
+            {
+                err = !check_config_hook(cl, config);
+            }
+        }
+        if (err)
+        {
+            osd_num_t peer_osd = cl->osd_num;
+            stop_client(op->peer_fd);
+            on_connect_peer(peer_osd, -1);
+            delete op;
+            return;
+        }
+#ifdef WITH_RDMA
+        if (config["rdma_address"].is_string())
+        {
+            msgr_rdma_address_t addr;
+            if (!msgr_rdma_address_t::from_string(config["rdma_address"].string_value().c_str(), &addr) ||
+                cl->rdma_conn->connect(&addr) != 0)
+            {
+                fprintf(
+                    stderr, "Failed to connect to OSD %ju (address %s) using RDMA\n",
+                    cl->osd_num, config["rdma_address"].string_value().c_str()
+                );
+                delete cl->rdma_conn;
+                cl->rdma_conn = NULL;
+                // FIXME: Keep TCP connection in this case
+                osd_num_t peer_osd = cl->osd_num;
+                stop_client(cl->peer_fd);
+                on_connect_peer(peer_osd, -1);
+                delete op;
+                return;
+            }
+            else
+            {
+                uint64_t server_max_msg = config["rdma_max_msg"].uint64_value();
+                if (cl->rdma_conn->max_msg > server_max_msg)
+                {
+                    cl->rdma_conn->max_msg = server_max_msg;
+                }
+                if (log_level > 0)
+                {
+                    fprintf(stderr, "Connected to OSD %ju using RDMA\n", cl->osd_num);
+                }
+                cl->peer_state = PEER_RDMA;
+                tfd->set_fd_handler(cl->peer_fd, false, [this](int peer_fd, int epoll_events)
+                {
+                    // Do not miss the disconnection!
+                    if (epoll_events & EPOLLRDHUP)
+                    {
+                        handle_peer_epoll(peer_fd, epoll_events);
+                    }
+                });
+                // Add the initial receive request
+                try_recv_rdma(cl);
+            }
+        }
+#endif
+        osd_peer_fds[cl->osd_num] = cl->peer_fd;
+        on_connect_peer(cl->osd_num, cl->peer_fd);
+        delete op;
+    };
+    outbox_push(op);
+}
+
+void osd_messenger_t::accept_connections(int listen_fd)
+{
+    // Accept new connections
+    sockaddr_storage addr;
+    socklen_t peer_addr_size = sizeof(addr);
+    int peer_fd;
+    while ((peer_fd = accept(listen_fd, (sockaddr*)&addr, &peer_addr_size)) >= 0)
+    {
+        assert(peer_fd != 0);
+        fprintf(stderr, "[OSD %ju] new client %d: connection from %s\n", this->osd_num, peer_fd,
+            addr_to_string(addr).c_str());
+        fcntl(peer_fd, F_SETFL, fcntl(peer_fd, F_GETFL, 0) | O_NONBLOCK);
+        int one = 1;
+        setsockopt(peer_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
+        clients[peer_fd] = new osd_client_t();
+        clients[peer_fd]->peer_addr = addr;
+        clients[peer_fd]->peer_port = ntohs(((sockaddr_in*)&addr)->sin_port);
+        clients[peer_fd]->peer_fd = peer_fd;
+        clients[peer_fd]->peer_state = PEER_CONNECTED;
+        clients[peer_fd]->in_buf = malloc_or_die(receive_buffer_size);
+        // Add FD to epoll
+        tfd->set_fd_handler(peer_fd, false, [this](int peer_fd, int epoll_events)
+        {
+            handle_peer_epoll(peer_fd, epoll_events);
+        });
+        // Try to accept next connection
+        peer_addr_size = sizeof(addr);
+    }
+    if (peer_fd == -1 && errno != EAGAIN)
+    {
+        throw std::runtime_error(std::string("accept: ") + strerror(errno));
+    }
+}
+
+#ifdef WITH_RDMA
+bool osd_messenger_t::is_rdma_enabled()
+{
+    return rdma_context != NULL;
+}
+#endif
+
+json11::Json::object osd_messenger_t::read_config(const json11::Json & config)
+{
+    json11::Json::object file_config;
+    const char *config_path = config["config_path"].string_value() != ""
+        ? config["config_path"].string_value().c_str() : VITASTOR_CONFIG_PATH;
+    int fd = open(config_path, O_RDONLY);
+    if (fd < 0)
+    {
+        if (errno != ENOENT)
+            fprintf(stderr, "Error reading %s: %s\n", config_path, strerror(errno));
+        return file_config;
+    }
+    struct stat st;
+    if (fstat(fd, &st) != 0)
+    {
+        fprintf(stderr, "Error reading %s: %s\n", config_path, strerror(errno));
+        close(fd);
+        return file_config;
+    }
+    std::string buf;
+    buf.resize(st.st_size);
+    int done = 0;
+    while (done < st.st_size)
+    {
+        int r = read(fd, (uint8_t*)buf.data()+done, st.st_size-done);
+        if (r < 0)
+        {
+            fprintf(stderr, "Error reading %s: %s\n", config_path, strerror(errno));
+            close(fd);
+            return file_config;
+        }
+        done += r;
+    }
+    close(fd);
+    std::string json_err;
+    file_config = json11::Json::parse(buf, json_err).object_items();
+    if (json_err != "")
+    {
+        fprintf(stderr, "Invalid JSON in %s: %s\n", config_path, json_err.c_str());
+    }
+    return file_config;
+}
+
+static const char* cli_only_params[] = {
+    // The list has to be sorted
+    "bitmap_granularity",
+    "block_size",
+    "data_device",
+    "data_offset",
+    "data_size",
+    "disable_data_fsync",
+    "disable_device_lock",
+    "disable_journal_fsync",
+    "disable_meta_fsync",
+    "disk_alignment",
+    "flush_journal",
+    "immediate_commit",
+    "inmemory_journal",
+    "inmemory_metadata",
+    "journal_block_size",
+    "journal_device",
+    "journal_no_same_sector_overwrites",
+    "journal_offset",
+    "journal_sector_buffer_count",
+    "journal_size",
+    "meta_block_size",
+    "meta_buf_size",
+    "meta_device",
+    "meta_offset",
+    "osd_num",
+    "readonly",
+};
+
+static const char **cli_only_end = cli_only_params + (sizeof(cli_only_params)/sizeof(cli_only_params[0]));
+
+static const char* local_only_params[] = {
+    // The list has to be sorted
+    "config_path",
+    "rdma_device",
+    "rdma_gid_index",
+    "rdma_max_msg",
+    "rdma_max_recv",
+    "rdma_max_send",
+    "rdma_max_sge",
+    "rdma_mtu",
+    "rdma_port_num",
+    "tcp_header_buffer_size",
+    "use_rdma",
+    "use_sync_send_recv",
+};
+
+static const char **local_only_end = local_only_params + (sizeof(local_only_params)/sizeof(local_only_params[0]));
+
+// Basically could be replaced by std::lower_bound()...
+static int find_str_array(const char **start, const char **end, const std::string & s)
+{
+    int min = 0, max = end-start;
+    while (max-min >= 2)
+    {
+        int mid = (min+max)/2;
+        int r = strcmp(s.c_str(), start[mid]);
+        if (r < 0)
+            max = mid;
+        else if (r > 0)
+            min = mid;
+        else
+            return mid;
+    }
+    if (min < end-start && !strcmp(s.c_str(), start[min]))
+        return min;
+    return -1;
+}
+
+json11::Json::object osd_messenger_t::merge_configs(const json11::Json::object & cli_config,
+    const json11::Json::object & file_config,
+    const json11::Json::object & etcd_global_config,
+    const json11::Json::object & etcd_osd_config)
+{
+    // Priority: most important -> less important:
+    // etcd_osd_config -> cli_config -> etcd_global_config -> file_config
+    json11::Json::object res = file_config;
+    for (auto & kv: file_config)
+    {
+        int cli_only = find_str_array(cli_only_params, cli_only_end, kv.first);
+        if (cli_only < 0)
+        {
+            res[kv.first] = kv.second;
+        }
+    }
+    for (auto & kv: etcd_global_config)
+    {
+        int local_only = find_str_array(local_only_params, local_only_end, kv.first);
+        if (local_only < 0)
+        {
+            res[kv.first] = kv.second;
+        }
+    }
+    for (auto & kv: cli_config)
+    {
+        res[kv.first] = kv.second;
+    }
+    for (auto & kv: etcd_osd_config)
+    {
+        int local_only = find_str_array(local_only_params, local_only_end, kv.first);
+        if (local_only < 0)
+        {
+            res[kv.first] = kv.second;
+        }
+    }
+    return res;
+}

src/client/messenger.h

@@ -0,0 +1,208 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+#pragma once
+
+#include <sys/types.h>
+#include <stdint.h>
+#include <arpa/inet.h>
+
+#include <set>
+#include <map>
+#include <deque>
+#include <vector>
+
+#include "malloc_or_die.h"
+#include "json11/json11.hpp"
+#include "msgr_op.h"
+#include "timerfd_manager.h"
+#include <ringloop.h>
+
+#define CL_READ_HDR 1
+#define CL_READ_DATA 2
+#define CL_READ_REPLY_DATA 3
+#define CL_WRITE_READY 1
+
+#define PEER_CONNECTING 1
+#define PEER_CONNECTED 2
+#define PEER_RDMA_CONNECTING 3
+#define PEER_RDMA 4
+#define PEER_STOPPED 5
+
+#define VITASTOR_CONFIG_PATH "/etc/vitastor/vitastor.conf"
+
+#define MSGR_SENDP_HDR 1
+#define MSGR_SENDP_FREE 2
+
+struct msgr_sendp_t
+{
+    osd_op_t *op;
+    int flags;
+};
+
+#ifdef WITH_RDMA
+struct msgr_rdma_connection_t;
+struct msgr_rdma_context_t;
+#endif
+
+struct osd_client_t
+{
+    int refs = 0;
+
+    sockaddr_storage peer_addr;
+    int peer_port;
+    int peer_fd = -1;
+    int peer_state;
+    int connect_timeout_id = -1;
+    int ping_time_remaining = 0;
+    int idle_time_remaining = 0;
+    osd_num_t osd_num = 0;
+
+    void *in_buf = NULL;
+
+#ifdef WITH_RDMA
+    msgr_rdma_connection_t *rdma_conn = NULL;
+#endif
+
+    // Read state
+    int read_ready = 0;
+    osd_op_t *read_op = NULL;
+    iovec read_iov = { 0 };
+    msghdr read_msg = { 0 };
+    int read_remaining = 0;
+    int read_state = 0;
+    osd_op_buf_list_t recv_list;
+
+    // Incoming operations
+    std::vector<osd_op_t*> received_ops;
+
+    // Outbound operations
+    std::map<uint64_t, osd_op_t*> sent_ops;
+
+    // PGs dirtied by this client's primary-writes
+    std::set<pool_pg_num_t> dirty_pgs;
+
+    // Write state
+    msghdr write_msg = { 0 };
+    int write_state = 0;
+    std::vector<iovec> send_list, next_send_list;
+    std::vector<msgr_sendp_t> outbox, next_outbox;
+
+    ~osd_client_t();
+};
+
+struct osd_wanted_peer_t
+{
+    json11::Json address_list;
+    int port;
+    time_t last_connect_attempt;
+    bool connecting, address_changed;
+    int address_index;
+    std::string cur_addr;
+    int cur_port;
+};
+
+struct osd_op_stats_t
+{
+    uint64_t op_stat_sum[OSD_OP_MAX+1] = { 0 };
+    uint64_t op_stat_count[OSD_OP_MAX+1] = { 0 };
+    uint64_t op_stat_bytes[OSD_OP_MAX+1] = { 0 };
+    uint64_t subop_stat_sum[OSD_OP_MAX+1] = { 0 };
+    uint64_t subop_stat_count[OSD_OP_MAX+1] = { 0 };
+};
+
+struct osd_messenger_t
+{
+protected:
+    int keepalive_timer_id = -1;
+
+    uint32_t receive_buffer_size = 0;
+    int peer_connect_interval = 0;
+    int peer_connect_timeout = 0;
+    int osd_idle_timeout = 0;
+    int osd_ping_timeout = 0;
+    int log_level = 0;
+    bool use_sync_send_recv = false;
+
+#ifdef WITH_RDMA
+    bool use_rdma = true;
+    std::string rdma_device;
+    uint64_t rdma_port_num = 1, rdma_gid_index = 0, rdma_mtu = 0;
+    msgr_rdma_context_t *rdma_context = NULL;
+    uint64_t rdma_max_sge = 0, rdma_max_send = 0, rdma_max_recv = 0;
+    uint64_t rdma_max_msg = 0;
+    bool rdma_odp = false;
+#endif
+
+    std::vector<int> read_ready_clients;
+    std::vector<int> write_ready_clients;
+    // We don't use ringloop->set_immediate here because we may have no ringloop in client :)
+    std::vector<std::function<void()>> set_immediate;
+
+public:
+    timerfd_manager_t *tfd;
+    ring_loop_t *ringloop;
+    // osd_num_t is only for logging and asserts
+    osd_num_t osd_num;
+    uint64_t next_subop_id = 1;
+    std::map<int, osd_client_t*> clients;
+    std::map<osd_num_t, osd_wanted_peer_t> wanted_peers;
+    std::map<uint64_t, int> osd_peer_fds;
+    // op statistics
+    osd_op_stats_t stats, recovery_stats;
+
+    void init();
+    void parse_config(const json11::Json & config);
+    void connect_peer(uint64_t osd_num, json11::Json peer_state);
+    void stop_client(int peer_fd, bool force = false, bool force_delete = false);
+    void outbox_push(osd_op_t *cur_op);
+    std::function<void(osd_op_t*)> exec_op;
+    std::function<void(osd_num_t)> repeer_pgs;
+    std::function<bool(osd_client_t*, json11::Json)> check_config_hook;
+    void read_requests();
+    void send_replies();
+    void accept_connections(int listen_fd);
+    ~osd_messenger_t();
+
+    static json11::Json::object read_config(const json11::Json & config);
+    static json11::Json::object merge_configs(const json11::Json::object & cli_config,
+        const json11::Json::object & file_config,
+        const json11::Json::object & etcd_global_config,
+        const json11::Json::object & etcd_osd_config);
+
+#ifdef WITH_RDMA
+    bool is_rdma_enabled();
+    bool connect_rdma(int peer_fd, std::string rdma_address, uint64_t client_max_msg);
+#endif
+
+    void inc_op_stats(osd_op_stats_t & stats, uint64_t opcode, timespec & tv_begin, timespec & tv_end, uint64_t len);
+    void measure_exec(osd_op_t *cur_op);
+
+protected:
+    void try_connect_peer(uint64_t osd_num);
+    void try_connect_peer_addr(osd_num_t peer_osd, const char *peer_host, int peer_port);
+    void handle_peer_epoll(int peer_fd, int epoll_events);
+    void handle_connect_epoll(int peer_fd);
+    void on_connect_peer(osd_num_t peer_osd, int peer_fd);
+    void check_peer_config(osd_client_t *cl);
+    void cancel_osd_ops(osd_client_t *cl);
+    void cancel_op(osd_op_t *op);
+
+    bool try_send(osd_client_t *cl);
+    void handle_send(int result, osd_client_t *cl);
+
+    bool handle_read(int result, osd_client_t *cl);
+    bool handle_read_buffer(osd_client_t *cl, void *curbuf, int remain);
+    bool handle_finished_read(osd_client_t *cl);
+    void handle_op_hdr(osd_client_t *cl);
+    bool handle_reply_hdr(osd_client_t *cl);
+    void handle_reply_ready(osd_op_t *op);
+
+#ifdef WITH_RDMA
+    void try_send_rdma(osd_client_t *cl);
+    void try_send_rdma_odp(osd_client_t *cl);
+    void try_send_rdma_nodp(osd_client_t *cl);
+    bool try_recv_rdma(osd_client_t *cl);
+    void handle_rdma_events();
+#endif
+};

src/client/msgr_op.cpp

@@ -0,0 +1,40 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+#include <assert.h>
+
+#include "msgr_op.h"
+
+osd_op_t::~osd_op_t()
+{
+    assert(!bs_op);
+    assert(!op_data);
+    if (bitmap_buf)
+    {
+        free(bitmap_buf);
+    }
+    if (rmw_buf)
+    {
+        free(rmw_buf);
+    }
+    if (buf)
+    {
+        // Note: reusing osd_op_t WILL currently lead to memory leaks
+        // So we don't reuse it, but free it every time
+        free(buf);
+    }
+}
+
+bool osd_op_t::is_recovery_related()
+{
+    return (req.hdr.opcode == OSD_OP_SEC_READ ||
+        req.hdr.opcode == OSD_OP_SEC_WRITE ||
+        req.hdr.opcode == OSD_OP_SEC_WRITE_STABLE) &&
+        (req.sec_rw.flags & OSD_OP_RECOVERY_RELATED) ||
+        req.hdr.opcode == OSD_OP_SEC_DELETE &&
+        (req.sec_del.flags & OSD_OP_RECOVERY_RELATED) ||
+        req.hdr.opcode == OSD_OP_SEC_STABILIZE &&
+        (req.sec_stab.flags & OSD_OP_RECOVERY_RELATED) ||
+        req.hdr.opcode == OSD_OP_SEC_SYNC &&
+        (req.sec_sync.flags & OSD_OP_RECOVERY_RELATED);
+}

src/client/msgr_op.h

@@ -0,0 +1,178 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+#pragma once
+
+#include <sys/uio.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+#include "osd_ops.h"
+
+#define OSD_OP_IN 0
+#define OSD_OP_OUT 1
+
+#define OSD_OP_INLINE_BUF_COUNT 16
+
+// Kind of a vector with small-list-optimisation
+struct osd_op_buf_list_t
+{
+    int count = 0, alloc = OSD_OP_INLINE_BUF_COUNT, done = 0;
+    iovec *buf = NULL;
+    iovec inline_buf[OSD_OP_INLINE_BUF_COUNT];
+
+    inline osd_op_buf_list_t()
+    {
+        buf = inline_buf;
+    }
+
+    inline osd_op_buf_list_t(const osd_op_buf_list_t & other)
+    {
+        buf = inline_buf;
+        append(other);
+    }
+
+    inline osd_op_buf_list_t & operator = (const osd_op_buf_list_t & other)
+    {
+        reset();
+        append(other);
+        return *this;
+    }
+
+    inline ~osd_op_buf_list_t()
+    {
+        if (buf && buf != inline_buf)
+        {
+            free(buf);
+        }
+    }
+
+    inline void reset()
+    {
+        count = 0;
+        done = 0;
+    }
+
+    inline iovec* get_iovec()
+    {
+        return buf + done;
+    }
+
+    inline int get_size()
+    {
+        return count - done;
+    }
+
+    inline void append(const osd_op_buf_list_t & other)
+    {
+        if (count+other.count > alloc)
+        {
+            if (buf == inline_buf)
+            {
+                int old = alloc;
+                alloc = (((count+other.count+15)/16)*16);
+                buf = (iovec*)malloc(sizeof(iovec) * alloc);
+                if (!buf)
+                {
+                    fprintf(stderr, "Failed to allocate %u bytes\n", (int)sizeof(iovec) * alloc);
+                    exit(1);
+                }
+                memcpy(buf, inline_buf, sizeof(iovec) * old);
+            }
+            else
+            {
+                alloc = (((count+other.count+15)/16)*16);
+                buf = (iovec*)realloc(buf, sizeof(iovec) * alloc);
+                if (!buf)
+                {
+                    fprintf(stderr, "Failed to allocate %u bytes\n", (int)sizeof(iovec) * alloc);
+                    exit(1);
+                }
+            }
+        }
+        for (int i = 0; i < other.count; i++)
+        {
+            buf[count++] = other.buf[i];
+        }
+    }
+
+    inline void push_back(void *nbuf, size_t len)
+    {
+        if (count >= alloc)
+        {
+            if (buf == inline_buf)
+            {
+                int old = alloc;
+                alloc = ((alloc/16)*16 + 1);
+                buf = (iovec*)malloc(sizeof(iovec) * alloc);
+                if (!buf)
+                {
+                    fprintf(stderr, "Failed to allocate %u bytes\n", (int)sizeof(iovec) * alloc);
+                    exit(1);
+                }
+                memcpy(buf, inline_buf, sizeof(iovec)*old);
+            }
+            else
+            {
+                alloc = alloc < 16 ? 16 : (alloc+16);
+                buf = (iovec*)realloc(buf, sizeof(iovec) * alloc);
+                if (!buf)
+                {
+                    fprintf(stderr, "Failed to allocate %u bytes\n", (int)sizeof(iovec) * alloc);
+                    exit(1);
+                }
+            }
+        }
+        buf[count++] = { .iov_base = nbuf, .iov_len = len };
+    }
+
+    inline void eat(int result)
+    {
+        while (result > 0 && done < count)
+        {
+            iovec & iov = buf[done];
+            if (iov.iov_len <= result)
+            {
+                result -= iov.iov_len;
+                done++;
+            }
+            else
+            {
+                iov.iov_len -= result;
+                iov.iov_base = (uint8_t*)iov.iov_base + result;
+                break;
+            }
+        }
+    }
+};
+
+struct blockstore_op_t;
+
+struct osd_primary_op_data_t;
+
+struct osd_op_t
+{
+    timespec tv_begin = { 0 }, tv_end = { 0 };
+    uint64_t op_type = OSD_OP_IN;
+    int peer_fd;
+    osd_any_op_t req;
+    osd_any_reply_t reply;
+    blockstore_op_t *bs_op = NULL;
+    void *buf = NULL;
+    // bitmap, bitmap_len, bmp_data are only meaningful for reads
+    void *bitmap = NULL;
+    unsigned bitmap_len = 0;
+    unsigned bmp_data = 0;
+    void *bitmap_buf = NULL;
+    void *rmw_buf = NULL;
+    osd_primary_op_data_t* op_data = NULL;
+    std::function<void(osd_op_t*)> callback;
+
+    osd_op_buf_list_t iov;
+
+    ~osd_op_t();
+
+    bool is_recovery_related();
+};

src/client/msgr_rdma.cpp

@@ -0,0 +1,674 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+#include <stdio.h>
+#include <stdlib.h>
+#include "msgr_rdma.h"
+#include "messenger.h"
+
+std::string msgr_rdma_address_t::to_string()
+{
+    char msg[sizeof "0000:00000000:00000000:00000000000000000000000000000000"];
+    sprintf(
+        msg, "%04x:%06x:%06x:%016jx%016jx", lid, qpn, psn,
+        htobe64(((uint64_t*)&gid)[0]), htobe64(((uint64_t*)&gid)[1])
+    );
+    return std::string(msg);
+}
+
+bool msgr_rdma_address_t::from_string(const char *str, msgr_rdma_address_t *dest)
+{
+    uint64_t* gid = (uint64_t*)&dest->gid;
+    int scanned = sscanf(
+        str, "%hx:%x:%x:%16jx%16jx", &dest->lid, &dest->qpn, &dest->psn, gid, gid+1
+    );
+    gid[0] = be64toh(gid[0]);
+    gid[1] = be64toh(gid[1]);
+    return scanned == 5;
+}
+
+msgr_rdma_context_t::~msgr_rdma_context_t()
+{
+    if (cq)
+        ibv_destroy_cq(cq);
+    if (channel)
+        ibv_destroy_comp_channel(channel);
+    if (mr)
+        ibv_dereg_mr(mr);
+    if (pd)
+        ibv_dealloc_pd(pd);
+    if (context)
+        ibv_close_device(context);
+}
+
+msgr_rdma_connection_t::~msgr_rdma_connection_t()
+{
+    ctx->used_max_cqe -= max_send+max_recv;
+    if (qp)
+        ibv_destroy_qp(qp);
+    if (recv_buffers.size())
+    {
+        for (auto b: recv_buffers)
+        {
+            if (b.mr)
+                ibv_dereg_mr(b.mr);
+            free(b.buf);
+        }
+        recv_buffers.clear();
+    }
+    if (send_out.mr)
+    {
+        ibv_dereg_mr(send_out.mr);
+        send_out.mr = NULL;
+    }
+    if (send_out.buf)
+    {
+        free(send_out.buf);
+        send_out.buf = NULL;
+    }
+    send_out_size = 0;
+}
+
+msgr_rdma_context_t *msgr_rdma_context_t::create(const char *ib_devname, uint8_t ib_port, uint8_t gid_index, uint32_t mtu, bool odp, int log_level)
+{
+    int res;
+    ibv_device **dev_list = NULL;
+    msgr_rdma_context_t *ctx = new msgr_rdma_context_t();
+    ctx->mtu = mtu;
+
+    timespec tv;
+    clock_gettime(CLOCK_REALTIME, &tv);
+    srand48(tv.tv_sec*1000000000 + tv.tv_nsec);
+    dev_list = ibv_get_device_list(NULL);
+    if (!dev_list)
+    {
+        if (errno == -ENOSYS || errno == ENOSYS)
+        {
+            if (log_level > 0)
+                fprintf(stderr, "No RDMA devices found (RDMA device list returned ENOSYS)\n");
+        }
+        else
+            fprintf(stderr, "Failed to get RDMA device list: %s\n", strerror(errno));
+        goto cleanup;
+    }
+    if (!ib_devname)
+    {
+        ctx->dev = *dev_list;
+        if (!ctx->dev)
+        {
+            if (log_level > 0)
+                fprintf(stderr, "No RDMA devices found\n");
+            goto cleanup;
+        }
+    }
+    else
+    {
+        int i;
+        for (i = 0; dev_list[i]; ++i)
+            if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname))
+                break;
+        ctx->dev = dev_list[i];
+        if (!ctx->dev)
+        {
+            fprintf(stderr, "RDMA device %s not found\n", ib_devname);
+            goto cleanup;
+        }
+    }
+
+    ctx->context = ibv_open_device(ctx->dev);
+    if (!ctx->context)
+    {
+        fprintf(stderr, "Couldn't get RDMA context for %s\n", ibv_get_device_name(ctx->dev));
+        goto cleanup;
+    }
+
+    ctx->ib_port = ib_port;
+    ctx->gid_index = gid_index;
+    if ((res = ibv_query_port(ctx->context, ib_port, &ctx->portinfo)) != 0)
+    {
+        fprintf(stderr, "Couldn't get RDMA device %s port %d info: %s\n", ibv_get_device_name(ctx->dev), ib_port, strerror(res));
+        goto cleanup;
+    }
+    ctx->my_lid = ctx->portinfo.lid;
+    if (ctx->portinfo.link_layer != IBV_LINK_LAYER_ETHERNET && !ctx->my_lid)
+    {
+        fprintf(stderr, "RDMA device %s must have local LID because it's not Ethernet, but LID is zero\n", ibv_get_device_name(ctx->dev));
+        goto cleanup;
+    }
+    if (ibv_query_gid(ctx->context, ib_port, gid_index, &ctx->my_gid))
+    {
+        fprintf(stderr, "Couldn't read RDMA device %s GID index %d\n", ibv_get_device_name(ctx->dev), gid_index);
+        goto cleanup;
+    }
+
+    ctx->pd = ibv_alloc_pd(ctx->context);
+    if (!ctx->pd)
+    {
+        fprintf(stderr, "Couldn't allocate RDMA protection domain\n");
+        goto cleanup;
+    }
+
+    {
+        if (ibv_query_device_ex(ctx->context, NULL, &ctx->attrx))
+        {
+            fprintf(stderr, "Couldn't query RDMA device for its features\n");
+            goto cleanup;
+        }
+        ctx->odp = odp;
+        if (ctx->odp &&
+            (!(ctx->attrx.odp_caps.general_caps & IBV_ODP_SUPPORT) ||
+            !(ctx->attrx.odp_caps.general_caps & IBV_ODP_SUPPORT_IMPLICIT) ||
+            !(ctx->attrx.odp_caps.per_transport_caps.rc_odp_caps & IBV_ODP_SUPPORT_SEND) ||
+            !(ctx->attrx.odp_caps.per_transport_caps.rc_odp_caps & IBV_ODP_SUPPORT_RECV)))
+        {
+            ctx->odp = false;
+            if (log_level > 0)
+                fprintf(stderr, "The RDMA device isn't implicit ODP (On-Demand Paging) capable, disabling it\n");
+        }
+    }
+
+    if (ctx->odp)
+    {
+        ctx->mr = ibv_reg_mr(ctx->pd, NULL, SIZE_MAX, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_ON_DEMAND);
+        if (!ctx->mr)
+        {
+            fprintf(stderr, "Couldn't register RDMA memory region\n");
+            goto cleanup;
+        }
+    }
+
+    ctx->channel = ibv_create_comp_channel(ctx->context);
+    if (!ctx->channel)
+    {
+        fprintf(stderr, "Couldn't create RDMA completion channel\n");
+        goto cleanup;
+    }
+
+    ctx->max_cqe = 4096;
+    ctx->cq = ibv_create_cq(ctx->context, ctx->max_cqe, NULL, ctx->channel, 0);
+    if (!ctx->cq)
+    {
+        fprintf(stderr, "Couldn't create RDMA completion queue\n");
+        goto cleanup;
+    }
+
+    if (dev_list)
+        ibv_free_device_list(dev_list);
+    return ctx;
+
+cleanup:
+    delete ctx;
+    if (dev_list)
+        ibv_free_device_list(dev_list);
+    return NULL;
+}
+
+msgr_rdma_connection_t *msgr_rdma_connection_t::create(msgr_rdma_context_t *ctx, uint32_t max_send,
+    uint32_t max_recv, uint32_t max_sge, uint32_t max_msg)
+{
+    msgr_rdma_connection_t *conn = new msgr_rdma_connection_t;
+
+    max_sge = max_sge > ctx->attrx.orig_attr.max_sge ? ctx->attrx.orig_attr.max_sge : max_sge;
+
+    conn->ctx = ctx;
+    conn->max_send = max_send;
+    conn->max_recv = max_recv;
+    conn->max_sge = max_sge;
+    conn->max_msg = max_msg;
+
+    ctx->used_max_cqe += max_send+max_recv;
+    if (ctx->used_max_cqe > ctx->max_cqe)
+    {
+        // Resize CQ
+        // Mellanox ConnectX-4 supports up to 4194303 CQEs, so it's fine to put everything into a single CQ
+        int new_max_cqe = ctx->max_cqe;
+        while (ctx->used_max_cqe > new_max_cqe)
+        {
+            new_max_cqe *= 2;
+        }
+        if (ibv_resize_cq(ctx->cq, new_max_cqe) != 0)
+        {
+            fprintf(stderr, "Couldn't resize RDMA completion queue to %d entries\n", new_max_cqe);
+            delete conn;
+            return NULL;
+        }
+        ctx->max_cqe = new_max_cqe;
+    }
+
+    ibv_qp_init_attr init_attr = {
+        .send_cq = ctx->cq,
+        .recv_cq = ctx->cq,
+        .cap     = {
+            .max_send_wr  = max_send,
+            .max_recv_wr  = max_recv,
+            .max_send_sge = max_sge,
+            .max_recv_sge = max_sge,
+        },
+        .qp_type = IBV_QPT_RC,
+    };
+    conn->qp = ibv_create_qp(ctx->pd, &init_attr);
+    if (!conn->qp)
+    {
+        fprintf(stderr, "Couldn't create RDMA queue pair\n");
+        delete conn;
+        return NULL;
+    }
+
+    conn->addr.lid = ctx->my_lid;
+    conn->addr.gid = ctx->my_gid;
+    conn->addr.qpn = conn->qp->qp_num;
+    conn->addr.psn = lrand48() & 0xffffff;
+
+    ibv_qp_attr attr = {
+        .qp_state        = IBV_QPS_INIT,
+        .qp_access_flags = 0,
+        .pkey_index      = 0,
+        .port_num        = ctx->ib_port,
+    };
+
+    if (ibv_modify_qp(conn->qp, &attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS))
+    {
+        fprintf(stderr, "Failed to switch RDMA queue pair to INIT state\n");
+        delete conn;
+        return NULL;
+    }
+
+    return conn;
+}
+
+static ibv_mtu mtu_to_ibv_mtu(uint32_t mtu)
+{
+    switch (mtu)
+    {
+        case 256:  return IBV_MTU_256;
+        case 512:  return IBV_MTU_512;
+        case 1024: return IBV_MTU_1024;
+        case 2048: return IBV_MTU_2048;
+        case 4096: return IBV_MTU_4096;
+    }
+    return IBV_MTU_4096;
+}
+
+int msgr_rdma_connection_t::connect(msgr_rdma_address_t *dest)
+{
+    auto conn = this;
+    ibv_qp_attr attr = {
+        .qp_state       = IBV_QPS_RTR,
+        .path_mtu       = mtu_to_ibv_mtu(conn->ctx->mtu),
+        .rq_psn         = dest->psn,
+        .sq_psn         = conn->addr.psn,
+        .dest_qp_num    = dest->qpn,
+        .ah_attr        = {
+            .grh        = {
+                .dgid = dest->gid,
+                .sgid_index = conn->ctx->gid_index,
+                .hop_limit = 1, // FIXME can it vary?
+            },
+            .dlid       = dest->lid,
+            .sl         = 0, // service level
+            .src_path_bits = 0,
+            .is_global  = (uint8_t)(dest->gid.global.interface_id ? 1 : 0),
+            .port_num   = conn->ctx->ib_port,
+        },
+        .max_rd_atomic  = 1,
+        .max_dest_rd_atomic = 1,
+        // Timeout and min_rnr_timer actual values seem to be 4.096us*2^(timeout+1)
+        .min_rnr_timer  = 1,
+        .timeout        = 14,
+        .retry_cnt      = 7,
+        .rnr_retry      = 7,
+    };
+    // FIXME No idea if ibv_modify_qp is a blocking operation or not. No idea if it has a timeout and what it is.
+    if (ibv_modify_qp(conn->qp, &attr, IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU |
+        IBV_QP_DEST_QPN | IBV_QP_RQ_PSN | IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER))
+    {
+        fprintf(stderr, "Failed to switch RDMA queue pair to RTR (ready-to-receive) state\n");
+        return 1;
+    }
+    attr.qp_state = IBV_QPS_RTS;
+    if (ibv_modify_qp(conn->qp, &attr, IBV_QP_STATE | IBV_QP_TIMEOUT |
+        IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY | IBV_QP_SQ_PSN | IBV_QP_MAX_QP_RD_ATOMIC))
+    {
+        fprintf(stderr, "Failed to switch RDMA queue pair to RTS (ready-to-send) state\n");
+        return 1;
+    }
+    return 0;
+}
+
+bool osd_messenger_t::connect_rdma(int peer_fd, std::string rdma_address, uint64_t client_max_msg)
+{
+    // Try to connect to the peer using RDMA
+    msgr_rdma_address_t addr;
+    if (msgr_rdma_address_t::from_string(rdma_address.c_str(), &addr))
+    {
+        if (client_max_msg > rdma_max_msg)
+        {
+            client_max_msg = rdma_max_msg;
+        }
+        auto rdma_conn = msgr_rdma_connection_t::create(rdma_context, rdma_max_send, rdma_max_recv, rdma_max_sge, client_max_msg);
+        if (rdma_conn)
+        {
+            int r = rdma_conn->connect(&addr);
+            if (r != 0)
+            {
+                delete rdma_conn;
+                fprintf(
+                    stderr, "Failed to connect RDMA queue pair to %s (client %d)\n",
+                    addr.to_string().c_str(), peer_fd
+                );
+            }
+            else
+            {
+                // Remember connection, but switch to RDMA only after sending the configuration response
+                auto cl = clients.at(peer_fd);
+                cl->rdma_conn = rdma_conn;
+                cl->peer_state = PEER_RDMA_CONNECTING;
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
+static void try_send_rdma_wr(osd_client_t *cl, ibv_sge *sge, int op_sge)
+{
+    ibv_send_wr *bad_wr = NULL;
+    ibv_send_wr wr = {
+        .wr_id = (uint64_t)(cl->peer_fd*2+1),
+        .sg_list = sge,
+        .num_sge = op_sge,
+        .opcode = IBV_WR_SEND,
+        .send_flags = IBV_SEND_SIGNALED,
+    };
+    int err = ibv_post_send(cl->rdma_conn->qp, &wr, &bad_wr);
+    if (err || bad_wr)
+    {
+        fprintf(stderr, "RDMA send failed: %s\n", strerror(err));
+        exit(1);
+    }
+    cl->rdma_conn->cur_send++;
+}
+
+static int try_send_rdma_copy(osd_client_t *cl, uint8_t *dst, int dst_len)
+{
+    auto rc = cl->rdma_conn;
+    int total_dst_len = dst_len;
+    while (dst_len > 0 && rc->send_pos < cl->send_list.size())
+    {
+        iovec & iov = cl->send_list[rc->send_pos];
+        uint32_t len = (uint32_t)(iov.iov_len-rc->send_buf_pos < dst_len
+            ? iov.iov_len-rc->send_buf_pos : dst_len);
+        memcpy(dst, iov.iov_base+rc->send_buf_pos, len);
+        dst += len;
+        dst_len -= len;
+        rc->send_buf_pos += len;
+        if (rc->send_buf_pos >= iov.iov_len)
+        {
+            rc->send_pos++;
+            rc->send_buf_pos = 0;
+        }
+    }
+    return total_dst_len-dst_len;
+}
+
+void osd_messenger_t::try_send_rdma_odp(osd_client_t *cl)
+{
+    auto rc = cl->rdma_conn;
+    if (!cl->send_list.size() || rc->cur_send >= rc->max_send)
+    {
+        return;
+    }
+    uint64_t op_size = 0, op_sge = 0;
+    ibv_sge sge[rc->max_sge];
+    while (rc->send_pos < cl->send_list.size())
+    {
+        iovec & iov = cl->send_list[rc->send_pos];
+        if (op_size >= rc->max_msg || op_sge >= rc->max_sge)
+        {
+            rc->send_sizes.push_back(op_size);
+            try_send_rdma_wr(cl, sge, op_sge);
+            op_sge = 0;
+            op_size = 0;
+            if (rc->cur_send >= rc->max_send)
+            {
+                break;
+            }
+        }
+        uint32_t len = (uint32_t)(op_size+iov.iov_len-rc->send_buf_pos < rc->max_msg
+            ? iov.iov_len-rc->send_buf_pos : rc->max_msg-op_size);
+        sge[op_sge++] = {
+            .addr = (uintptr_t)((uint8_t*)iov.iov_base+rc->send_buf_pos),
+            .length = len,
+            .lkey = rc->ctx->mr->lkey,
+        };
+        op_size += len;
+        rc->send_buf_pos += len;
+        if (rc->send_buf_pos >= iov.iov_len)
+        {
+            rc->send_pos++;
+            rc->send_buf_pos = 0;
+        }
+    }
+    if (op_sge > 0)
+    {
+        rc->send_sizes.push_back(op_size);
+        try_send_rdma_wr(cl, sge, op_sge);
+    }
+}
+
+void osd_messenger_t::try_send_rdma_nodp(osd_client_t *cl)
+{
+    auto rc = cl->rdma_conn;
+    if (!rc->send_out_size)
+    {
+        // Allocate send ring buffer, if not yet
+        rc->send_out_size = rc->max_msg*rdma_max_send;
+        rc->send_out.buf = malloc_or_die(rc->send_out_size);
+        if (!rdma_context->odp)
+        {
+            rc->send_out.mr = ibv_reg_mr(rdma_context->pd, rc->send_out.buf, rc->send_out_size, 0);
+            if (!rc->send_out.mr)
+            {
+                fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
+                exit(1);
+            }
+        }
+    }
+    // Copy data into the buffer and send it
+    uint8_t *dst = NULL;
+    int dst_len = 0;
+    int copied = 1;
+    while (!rc->send_out_full && copied > 0 && rc->cur_send < rc->max_send)
+    {
+        dst = (uint8_t*)rc->send_out.buf + rc->send_out_pos;
+        dst_len = (rc->send_out_pos < rc->send_out_size ? rc->send_out_size-rc->send_out_pos : rc->send_done_pos-rc->send_out_pos);
+        if (dst_len > rc->max_msg)
+            dst_len = rc->max_msg;
+        copied = try_send_rdma_copy(cl, dst, dst_len);
+        if (copied > 0)
+        {
+            rc->send_out_pos += copied;
+            if (rc->send_out_pos == rc->send_out_size)
+                rc->send_out_pos = 0;
+            assert(rc->send_out_pos < rc->send_out_size);
+            if (rc->send_out_pos >= rc->send_done_pos)
+                rc->send_out_full = true;
+            ibv_sge sge = {
+                .addr = (uintptr_t)dst,
+                .length = (uint32_t)copied,
+                .lkey = rdma_context->odp ? rdma_context->mr->lkey : rc->send_out.mr->lkey,
+            };
+            try_send_rdma_wr(cl, &sge, 1);
+            rc->send_sizes.push_back(copied);
+        }
+    }
+}
+
+void osd_messenger_t::try_send_rdma(osd_client_t *cl)
+{
+    if (rdma_context->odp)
+        try_send_rdma_odp(cl);
+    else
+        try_send_rdma_nodp(cl);
+}
+
+static void try_recv_rdma_wr(osd_client_t *cl, msgr_rdma_buf_t b)
+{
+    ibv_sge sge = {
+        .addr = (uintptr_t)b.buf,
+        .length = (uint32_t)cl->rdma_conn->max_msg,
+        .lkey = cl->rdma_conn->ctx->odp ? cl->rdma_conn->ctx->mr->lkey : b.mr->lkey,
+    };
+    ibv_recv_wr *bad_wr = NULL;
+    ibv_recv_wr wr = {
+        .wr_id = (uint64_t)(cl->peer_fd*2),
+        .sg_list = &sge,
+        .num_sge = 1,
+    };
+    int err = ibv_post_recv(cl->rdma_conn->qp, &wr, &bad_wr);
+    if (err || bad_wr)
+    {
+        fprintf(stderr, "RDMA receive failed: %s\n", strerror(err));
+        exit(1);
+    }
+    cl->rdma_conn->cur_recv++;
+}
+
+bool osd_messenger_t::try_recv_rdma(osd_client_t *cl)
+{
+    auto rc = cl->rdma_conn;
+    while (rc->cur_recv < rc->max_recv)
+    {
+        msgr_rdma_buf_t b;
+        b.buf = malloc_or_die(rc->max_msg);
+        if (!rdma_context->odp)
+        {
+            b.mr = ibv_reg_mr(rdma_context->pd, b.buf, rc->max_msg, IBV_ACCESS_LOCAL_WRITE);
+            if (!b.mr)
+            {
+                fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
+                exit(1);
+            }
+        }
+        rc->recv_buffers.push_back(b);
+        try_recv_rdma_wr(cl, b);
+    }
+    return true;
+}
+
+#define RDMA_EVENTS_AT_ONCE 32
+
+void osd_messenger_t::handle_rdma_events()
+{
+    // Request next notification
+    ibv_cq *ev_cq;
+    void *ev_ctx;
+    // FIXME: This is inefficient as it calls read()...
+    if (ibv_get_cq_event(rdma_context->channel, &ev_cq, &ev_ctx) == 0)
+    {
+        ibv_ack_cq_events(rdma_context->cq, 1);
+    }
+    if (ibv_req_notify_cq(rdma_context->cq, 0) != 0)
+    {
+        fprintf(stderr, "Failed to request RDMA completion notification, exiting\n");
+        exit(1);
+    }
+    ibv_wc wc[RDMA_EVENTS_AT_ONCE];
+    int event_count;
+    do
+    {
+        event_count = ibv_poll_cq(rdma_context->cq, RDMA_EVENTS_AT_ONCE, wc);
+        for (int i = 0; i < event_count; i++)
+        {
+            int client_id = wc[i].wr_id >> 1;
+            bool is_send = wc[i].wr_id & 1;
+            auto cl_it = clients.find(client_id);
+            if (cl_it == clients.end())
+            {
+                continue;
+            }
+            osd_client_t *cl = cl_it->second;
+            auto rc = cl->rdma_conn;
+            if (wc[i].status != IBV_WC_SUCCESS)
+            {
+                fprintf(stderr, "RDMA work request failed for client %d", client_id);
+                if (cl->osd_num)
+                {
+                    fprintf(stderr, " (OSD %ju)", cl->osd_num);
+                }
+                fprintf(stderr, " with status: %s, stopping client\n", ibv_wc_status_str(wc[i].status));
+                stop_client(client_id);
+                continue;
+            }
+            if (!is_send)
+            {
+                rc->cur_recv--;
+                if (!handle_read_buffer(cl, rc->recv_buffers[rc->next_recv_buf].buf, wc[i].byte_len))
+                {
+                    // handle_read_buffer may stop the client
+                    continue;
+                }
+                try_recv_rdma_wr(cl, rc->recv_buffers[rc->next_recv_buf]);
+                rc->next_recv_buf = (rc->next_recv_buf+1) % rc->recv_buffers.size();
+            }
+            else
+            {
+                rc->cur_send--;
+                uint64_t sent_size = rc->send_sizes.at(0);
+                rc->send_sizes.erase(rc->send_sizes.begin(), rc->send_sizes.begin()+1);
+                if (!rdma_context->odp)
+                {
+                    rc->send_done_pos += sent_size;
+                    rc->send_out_full = false;
+                    if (rc->send_done_pos == rc->send_out_size)
+                        rc->send_done_pos = 0;
+                    assert(rc->send_done_pos < rc->send_out_size);
+                }
+                int send_pos = 0, send_buf_pos = 0;
+                while (sent_size > 0)
+                {
+                    if (sent_size >= cl->send_list.at(send_pos).iov_len)
+                    {
+                        sent_size -= cl->send_list[send_pos].iov_len;
+                        send_pos++;
+                    }
+                    else
+                    {
+                        send_buf_pos = sent_size;
+                        sent_size = 0;
+                    }
+                }
+                assert(rc->send_pos >= send_pos);
+                if (rc->send_pos == send_pos)
+                {
+                    rc->send_buf_pos -= send_buf_pos;
+                }
+                rc->send_pos -= send_pos;
+                for (int i = 0; i < send_pos; i++)
+                {
+                    if (cl->outbox[i].flags & MSGR_SENDP_FREE)
+                    {
+                        // Reply fully sent
+                        delete cl->outbox[i].op;
+                    }
+                }
+                if (send_pos > 0)
+                {
+                    cl->send_list.erase(cl->send_list.begin(), cl->send_list.begin()+send_pos);
+                    cl->outbox.erase(cl->outbox.begin(), cl->outbox.begin()+send_pos);
+                }
+                if (send_buf_pos > 0)
+                {
+                    cl->send_list[0].iov_base = (uint8_t*)cl->send_list[0].iov_base + send_buf_pos;
+                    cl->send_list[0].iov_len -= send_buf_pos;
+                }
+                try_send_rdma(cl);
+            }
+        }
+    } while (event_count > 0);
+    for (auto cb: set_immediate)
+    {
+        cb();
+    }
+    set_immediate.clear();
+}

src/client/msgr_rdma.h

@@ -0,0 +1,69 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+#pragma once
+#include <infiniband/verbs.h>
+#include <string>
+#include <vector>
+
+struct msgr_rdma_address_t
+{
+    ibv_gid gid;
+    uint16_t lid;
+    uint32_t qpn;
+    uint32_t psn;
+
+    std::string to_string();
+    static bool from_string(const char *str, msgr_rdma_address_t *dest);
+};
+
+struct msgr_rdma_context_t
+{
+    ibv_context *context = NULL;
+    ibv_device *dev = NULL;
+    ibv_device_attr_ex attrx;
+    ibv_pd *pd = NULL;
+    bool odp = false;
+    ibv_mr *mr = NULL;
+    ibv_comp_channel *channel = NULL;
+    ibv_cq *cq = NULL;
+    ibv_port_attr portinfo;
+    uint8_t ib_port;
+    uint8_t gid_index;
+    uint16_t my_lid;
+    ibv_gid my_gid;
+    uint32_t mtu;
+    int max_cqe = 0;
+    int used_max_cqe = 0;
+
+    static msgr_rdma_context_t *create(const char *ib_devname, uint8_t ib_port, uint8_t gid_index, uint32_t mtu, bool odp, int log_level);
+    ~msgr_rdma_context_t();
+};
+
+struct msgr_rdma_buf_t
+{
+    void *buf = NULL;
+    ibv_mr *mr = NULL;
+};
+
+struct msgr_rdma_connection_t
+{
+    msgr_rdma_context_t *ctx = NULL;
+    ibv_qp *qp = NULL;
+    msgr_rdma_address_t addr;
+    int max_send = 0, max_recv = 0, max_sge = 0;
+    int cur_send = 0, cur_recv = 0;
+    uint64_t max_msg = 0;
+
+    int send_pos = 0, send_buf_pos = 0;
+    int next_recv_buf = 0;
+    std::vector<msgr_rdma_buf_t> recv_buffers;
+    std::vector<uint64_t> send_sizes;
+    msgr_rdma_buf_t send_out;
+    int send_out_pos = 0, send_done_pos = 0, send_out_size = 0;
+    bool send_out_full = false;
+
+    ~msgr_rdma_connection_t();
+    static msgr_rdma_connection_t *create(msgr_rdma_context_t *ctx, uint32_t max_send, uint32_t max_recv, uint32_t max_sge, uint32_t max_msg);
+    int connect(msgr_rdma_address_t *dest);
+};

src/client/msgr_receive.cpp

@@ -0,0 +1,414 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+#include "messenger.h"
+
+void osd_messenger_t::read_requests()
+{
+    for (int i = 0; i < read_ready_clients.size(); i++)
+    {
+        int peer_fd = read_ready_clients[i];
+        osd_client_t *cl = clients[peer_fd];
+        if (cl->read_msg.msg_iovlen)
+        {
+            continue;
+        }
+        if (cl->read_remaining < receive_buffer_size)
+        {
+            cl->read_iov.iov_base = cl->in_buf;
+            cl->read_iov.iov_len = receive_buffer_size;
+            cl->read_msg.msg_iov = &cl->read_iov;
+            cl->read_msg.msg_iovlen = 1;
+        }
+        else
+        {
+            cl->read_iov.iov_base = 0;
+            cl->read_iov.iov_len = cl->read_remaining;
+            cl->read_msg.msg_iov = cl->recv_list.get_iovec();
+            cl->read_msg.msg_iovlen = cl->recv_list.get_size();
+        }
+        cl->refs++;
+        if (ringloop && !use_sync_send_recv)
+        {
+            io_uring_sqe* sqe = ringloop->get_sqe();
+            if (!sqe)
+            {
+                cl->read_msg.msg_iovlen = 0;
+                read_ready_clients.erase(read_ready_clients.begin(), read_ready_clients.begin() + i);
+                return;
+            }
+            ring_data_t* data = ((ring_data_t*)sqe->user_data);
+            data->callback = [this, cl](ring_data_t *data) { handle_read(data->res, cl); };
+            my_uring_prep_recvmsg(sqe, peer_fd, &cl->read_msg, 0);
+        }
+        else
+        {
+            int result = recvmsg(peer_fd, &cl->read_msg, 0);
+            if (result < 0)
+            {
+                result = -errno;
+            }
+            handle_read(result, cl);
+        }
+    }
+    read_ready_clients.clear();
+}
+
+bool osd_messenger_t::handle_read(int result, osd_client_t *cl)
+{
+    bool ret = false;
+    cl->read_msg.msg_iovlen = 0;
+    cl->refs--;
+    if (cl->peer_state == PEER_STOPPED)
+    {
+        if (cl->refs <= 0)
+        {
+            delete cl;
+        }
+        return false;
+    }
+    if (result <= 0 && result != -EAGAIN && result != -EINTR)
+    {
+        // this is a client socket, so don't panic on error. just disconnect it
+        if (result != 0)
+        {
+            fprintf(stderr, "Client %d socket read error: %d (%s). Disconnecting client\n", cl->peer_fd, -result, strerror(-result));
+        }
+        stop_client(cl->peer_fd);
+        return false;
+    }
+    if (result == -EAGAIN || result == -EINTR || result < cl->read_iov.iov_len)
+    {
+        cl->read_ready--;
+        if (cl->read_ready > 0)
+            read_ready_clients.push_back(cl->peer_fd);
+    }
+    else
+    {
+        read_ready_clients.push_back(cl->peer_fd);
+    }
+    if (result > 0)
+    {
+        if (cl->read_iov.iov_base == cl->in_buf)
+        {
+            if (!handle_read_buffer(cl, cl->in_buf, result))
+            {
+                goto fin;
+            }
+        }
+        else
+        {
+            // Long data
+            cl->read_remaining -= result;
+            cl->recv_list.eat(result);
+            if (cl->recv_list.done >= cl->recv_list.count)
+            {
+                if (!handle_finished_read(cl))
+                {
+                    goto fin;
+                }
+            }
+        }
+        if (result >= cl->read_iov.iov_len)
+        {
+            ret = true;
+        }
+    }
+fin:
+    for (auto cb: set_immediate)
+    {
+        cb();
+    }
+    set_immediate.clear();
+    return ret;
+}
+
+bool osd_messenger_t::handle_read_buffer(osd_client_t *cl, void *curbuf, int remain)
+{
+    // Compose operation(s) from the buffer
+    while (remain > 0)
+    {
+        if (!cl->read_op)
+        {
+            cl->read_op = new osd_op_t;
+            cl->read_op->peer_fd = cl->peer_fd;
+            cl->read_op->op_type = OSD_OP_IN;
+            cl->recv_list.push_back(cl->read_op->req.buf, OSD_PACKET_SIZE);
+            cl->read_remaining = OSD_PACKET_SIZE;
+            cl->read_state = CL_READ_HDR;
+        }
+        while (cl->recv_list.done < cl->recv_list.count && remain > 0)
+        {
+            iovec* cur = cl->recv_list.get_iovec();
+            if (cur->iov_len > remain)
+            {
+                memcpy(cur->iov_base, curbuf, remain);
+                cl->read_remaining -= remain;
+                cur->iov_len -= remain;
+                cur->iov_base = (uint8_t*)cur->iov_base + remain;
+                remain = 0;
+            }
+            else
+            {
+                memcpy(cur->iov_base, curbuf, cur->iov_len);
+                curbuf = (uint8_t*)curbuf + cur->iov_len;
+                cl->read_remaining -= cur->iov_len;
+                remain -= cur->iov_len;
+                cur->iov_len = 0;
+                cl->recv_list.done++;
+            }
+        }
+        if (cl->recv_list.done >= cl->recv_list.count)
+        {
+            if (!handle_finished_read(cl))
+            {
+                return false;
+            }
+        }
+    }
+    return true;
+}
+
+bool osd_messenger_t::handle_finished_read(osd_client_t *cl)
+{
+    cl->recv_list.reset();
+    if (cl->read_state == CL_READ_HDR)
+    {
+        if (cl->read_op->req.hdr.magic == SECONDARY_OSD_REPLY_MAGIC)
+            return handle_reply_hdr(cl);
+        else if (cl->read_op->req.hdr.magic == SECONDARY_OSD_OP_MAGIC)
+            handle_op_hdr(cl);
+        else
+        {
+            fprintf(stderr, "Received garbage: magic=%jx id=%ju opcode=%jx from %d\n", cl->read_op->req.hdr.magic, cl->read_op->req.hdr.id, cl->read_op->req.hdr.opcode, cl->peer_fd);
+            stop_client(cl->peer_fd);
+            return false;
+        }
+    }
+    else if (cl->read_state == CL_READ_DATA)
+    {
+        // Operation is ready
+        cl->received_ops.push_back(cl->read_op);
+        set_immediate.push_back([this, op = cl->read_op]() { exec_op(op); });
+        cl->read_op = NULL;
+        cl->read_state = 0;
+    }
+    else if (cl->read_state == CL_READ_REPLY_DATA)
+    {
+        // Reply is ready
+        handle_reply_ready(cl->read_op);
+        cl->read_op = NULL;
+        cl->read_state = 0;
+    }
+    else
+    {
+        assert(0);
+    }
+    return true;
+}
+
+void osd_messenger_t::handle_op_hdr(osd_client_t *cl)
+{
+    osd_op_t *cur_op = cl->read_op;
+    if (cur_op->req.hdr.opcode == OSD_OP_SEC_READ)
+    {
+        cl->read_remaining = 0;
+    }
+    else if (cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE ||
+        cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE_STABLE)
+    {
+        if (cur_op->req.sec_rw.attr_len > 0)
+        {
+            if (cur_op->req.sec_rw.attr_len > sizeof(unsigned))
+                cur_op->bitmap = cur_op->rmw_buf = malloc_or_die(cur_op->req.sec_rw.attr_len);
+            else
+                cur_op->bitmap = &cur_op->bmp_data;
+            cl->recv_list.push_back(cur_op->bitmap, cur_op->req.sec_rw.attr_len);
+        }
+        if (cur_op->req.sec_rw.len > 0)
+        {
+            cur_op->buf = memalign_or_die(MEM_ALIGNMENT, cur_op->req.sec_rw.len);
+            cl->recv_list.push_back(cur_op->buf, cur_op->req.sec_rw.len);
+        }
+        cl->read_remaining = cur_op->req.sec_rw.len + cur_op->req.sec_rw.attr_len;
+    }
+    else if (cur_op->req.hdr.opcode == OSD_OP_SEC_STABILIZE ||
+        cur_op->req.hdr.opcode == OSD_OP_SEC_ROLLBACK)
+    {
+        if (cur_op->req.sec_stab.len > 0)
+        {
+            cur_op->buf = memalign_or_die(MEM_ALIGNMENT, cur_op->req.sec_stab.len);
+            cl->recv_list.push_back(cur_op->buf, cur_op->req.sec_stab.len);
+        }
+        cl->read_remaining = cur_op->req.sec_stab.len;
+    }
+    else if (cur_op->req.hdr.opcode == OSD_OP_SEC_READ_BMP)
+    {
+        if (cur_op->req.sec_read_bmp.len > 0)
+        {
+            cur_op->buf = memalign_or_die(MEM_ALIGNMENT, cur_op->req.sec_read_bmp.len);
+            cl->recv_list.push_back(cur_op->buf, cur_op->req.sec_read_bmp.len);
+        }
+        cl->read_remaining = cur_op->req.sec_read_bmp.len;
+    }
+    else if (cur_op->req.hdr.opcode == OSD_OP_WRITE)
+    {
+        if (cur_op->req.rw.len > 0)
+        {
+            cur_op->buf = memalign_or_die(MEM_ALIGNMENT, cur_op->req.rw.len);
+            cl->recv_list.push_back(cur_op->buf, cur_op->req.rw.len);
+        }
+        cl->read_remaining = cur_op->req.rw.len;
+    }
+    else if (cur_op->req.hdr.opcode == OSD_OP_SHOW_CONFIG)
+    {
+        if (cur_op->req.show_conf.json_len > 0)
+        {
+            cur_op->buf = malloc_or_die(cur_op->req.show_conf.json_len+1);
+            ((uint8_t*)cur_op->buf)[cur_op->req.show_conf.json_len] = 0;
+            cl->recv_list.push_back(cur_op->buf, cur_op->req.show_conf.json_len);
+        }
+        cl->read_remaining = cur_op->req.show_conf.json_len;
+    }
+    /*else if (cur_op->req.hdr.opcode == OSD_OP_READ ||
+        cur_op->req.hdr.opcode == OSD_OP_SCRUB ||
+        cur_op->req.hdr.opcode == OSD_OP_DESCRIBE)
+    {
+        cl->read_remaining = 0;
+    }*/
+    if (cl->read_remaining > 0)
+    {
+        // Read data
+        cl->read_state = CL_READ_DATA;
+    }
+    else
+    {
+        // Operation is ready
+        cl->received_ops.push_back(cur_op);
+        set_immediate.push_back([this, cur_op]() { exec_op(cur_op); });
+        cl->read_op = NULL;
+        cl->read_state = 0;
+    }
+}
+
+bool osd_messenger_t::handle_reply_hdr(osd_client_t *cl)
+{
+    auto req_it = cl->sent_ops.find(cl->read_op->req.hdr.id);
+    if (req_it == cl->sent_ops.end())
+    {
+        // Command out of sync. Drop connection
+        fprintf(stderr, "Client %d command out of sync: id %ju\n", cl->peer_fd, cl->read_op->req.hdr.id);
+        stop_client(cl->peer_fd);
+        return false;
+    }
+    osd_op_t *op = req_it->second;
+    memcpy(op->reply.buf, cl->read_op->req.buf, OSD_PACKET_SIZE);
+    cl->sent_ops.erase(req_it);
+    if (op->reply.hdr.opcode == OSD_OP_SEC_READ || op->reply.hdr.opcode == OSD_OP_READ)
+    {
+        // Read data. In this case we assume that the buffer is preallocated by the caller (!)
+        unsigned bmp_len = (op->reply.hdr.opcode == OSD_OP_SEC_READ ? op->reply.sec_rw.attr_len : op->reply.rw.bitmap_len);
+        unsigned expected_size = (op->reply.hdr.opcode == OSD_OP_SEC_READ ? op->req.sec_rw.len : op->req.rw.len);
+        if (op->reply.hdr.retval >= 0 && (op->reply.hdr.retval != expected_size || bmp_len > op->bitmap_len))
+        {
+            // Check reply length to not overflow the buffer
+            fprintf(stderr, "Client %d read reply of different length: expected %u+%u, got %jd+%u\n",
+                cl->peer_fd, expected_size, op->bitmap_len, op->reply.hdr.retval, bmp_len);
+            cl->sent_ops[op->req.hdr.id] = op;
+            stop_client(cl->peer_fd);
+            return false;
+        }
+        if (bmp_len > 0)
+        {
+            assert(op->bitmap);
+            cl->recv_list.push_back(op->bitmap, bmp_len);
+            cl->read_remaining += bmp_len;
+        }
+        if (op->reply.hdr.retval > 0)
+        {
+            assert(op->iov.count > 0);
+            cl->recv_list.append(op->iov);
+            cl->read_remaining += op->reply.hdr.retval;
+        }
+        if (cl->read_remaining == 0)
+        {
+            goto reuse;
+        }
+        delete cl->read_op;
+        cl->read_op = op;
+        cl->read_state = CL_READ_REPLY_DATA;
+    }
+    else if (op->reply.hdr.opcode == OSD_OP_SEC_LIST && op->reply.hdr.retval > 0)
+    {
+        assert(!op->iov.count);
+        delete cl->read_op;
+        cl->read_op = op;
+        cl->read_state = CL_READ_REPLY_DATA;
+        cl->read_remaining = sizeof(obj_ver_id) * op->reply.hdr.retval;
+        op->buf = memalign_or_die(MEM_ALIGNMENT, cl->read_remaining);
+        cl->recv_list.push_back(op->buf, cl->read_remaining);
+    }
+    else if (op->reply.hdr.opcode == OSD_OP_SEC_READ_BMP && op->reply.hdr.retval > 0)
+    {
+        assert(!op->iov.count);
+        delete cl->read_op;
+        cl->read_op = op;
+        cl->read_state = CL_READ_REPLY_DATA;
+        cl->read_remaining = op->reply.hdr.retval;
+        free(op->buf);
+        op->buf = memalign_or_die(MEM_ALIGNMENT, cl->read_remaining);
+        cl->recv_list.push_back(op->buf, cl->read_remaining);
+    }
+    else if (op->reply.hdr.opcode == OSD_OP_SHOW_CONFIG && op->reply.hdr.retval > 0)
+    {
+        delete cl->read_op;
+        cl->read_op = op;
+        cl->read_state = CL_READ_REPLY_DATA;
+        cl->read_remaining = op->reply.hdr.retval;
+        free(op->buf);
+        op->buf = malloc_or_die(op->reply.hdr.retval);
+        cl->recv_list.push_back(op->buf, op->reply.hdr.retval);
+    }
+    else if (op->reply.hdr.opcode == OSD_OP_DESCRIBE && op->reply.describe.result_bytes > 0)
+    {
+        delete cl->read_op;
+        cl->read_op = op;
+        cl->read_state = CL_READ_REPLY_DATA;
+        cl->read_remaining = op->reply.describe.result_bytes;
+        free(op->buf);
+        op->buf = malloc_or_die(op->reply.describe.result_bytes);
+        cl->recv_list.push_back(op->buf, op->reply.describe.result_bytes);
+    }
+    else
+    {
+reuse:
+        // It's fine to reuse cl->read_op for the next reply
+        handle_reply_ready(op);
+        cl->recv_list.push_back(cl->read_op->req.buf, OSD_PACKET_SIZE);
+        cl->read_remaining = OSD_PACKET_SIZE;
+        cl->read_state = CL_READ_HDR;
+    }
+    return true;
+}
+
+void osd_messenger_t::handle_reply_ready(osd_op_t *op)
+{
+    // Measure subop latency
+    timespec tv_end;
+    clock_gettime(CLOCK_REALTIME, &tv_end);
+    stats.subop_stat_count[op->req.hdr.opcode]++;
+    if (!stats.subop_stat_count[op->req.hdr.opcode])
+    {
+        stats.subop_stat_count[op->req.hdr.opcode]++;
+        stats.subop_stat_sum[op->req.hdr.opcode] = 0;
+    }
+    stats.subop_stat_sum[op->req.hdr.opcode] += (
+        (tv_end.tv_sec - op->tv_begin.tv_sec)*1000000 +
+        (tv_end.tv_nsec - op->tv_begin.tv_nsec)/1000
+    );
+    set_immediate.push_back([op]()
+    {
+        // Copy lambda to be unaffected by `delete op`
+        std::function<void(osd_op_t*)>(op->callback)(op);
+    });
+}

src/client/msgr_send.cpp

@@ -0,0 +1,316 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+#define _XOPEN_SOURCE
+#include <limits.h>
+#include <sys/epoll.h>
+
+#include "messenger.h"
+
+void osd_messenger_t::outbox_push(osd_op_t *cur_op)
+{
+    assert(cur_op->peer_fd);
+    osd_client_t *cl = clients.at(cur_op->peer_fd);
+    if (cur_op->op_type == OSD_OP_OUT)
+    {
+        clock_gettime(CLOCK_REALTIME, &cur_op->tv_begin);
+    }
+    else
+    {
+        // Check that operation actually belongs to this client
+        // FIXME: Review if this is still needed
+        bool found = false;
+        for (auto it = cl->received_ops.begin(); it != cl->received_ops.end(); it++)
+        {
+            if (*it == cur_op)
+            {
+                found = true;
+                cl->received_ops.erase(it, it+1);
+                break;
+            }
+        }
+        if (!found)
+        {
+            delete cur_op;
+            return;
+        }
+    }
+    auto & to_send_list = cl->write_msg.msg_iovlen ? cl->next_send_list : cl->send_list;
+    auto & to_outbox = cl->write_msg.msg_iovlen ? cl->next_outbox : cl->outbox;
+    if (cur_op->op_type == OSD_OP_IN)
+    {
+        measure_exec(cur_op);
+        to_send_list.push_back((iovec){ .iov_base = cur_op->reply.buf, .iov_len = OSD_PACKET_SIZE });
+    }
+    else
+    {
+        to_send_list.push_back((iovec){ .iov_base = cur_op->req.buf, .iov_len = OSD_PACKET_SIZE });
+        cl->sent_ops[cur_op->req.hdr.id] = cur_op;
+    }
+    to_outbox.push_back((msgr_sendp_t){ .op = cur_op, .flags = MSGR_SENDP_HDR });
+    // Bitmap
+    if (cur_op->op_type == OSD_OP_IN &&
+        cur_op->req.hdr.opcode == OSD_OP_SEC_READ &&
+        cur_op->reply.sec_rw.attr_len > 0)
+    {
+        to_send_list.push_back((iovec){
+            .iov_base = cur_op->bitmap,
+            .iov_len = cur_op->reply.sec_rw.attr_len,
+        });
+        to_outbox.push_back((msgr_sendp_t){ .op = cur_op, .flags = 0 });
+    }
+    else if (cur_op->op_type == OSD_OP_OUT &&
+        (cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE || cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE_STABLE) &&
+        cur_op->req.sec_rw.attr_len > 0)
+    {
+        to_send_list.push_back((iovec){
+            .iov_base = cur_op->bitmap,
+            .iov_len = cur_op->req.sec_rw.attr_len,
+        });
+        to_outbox.push_back((msgr_sendp_t){ .op = cur_op, .flags = 0 });
+    }
+    // Operation data
+    if ((cur_op->op_type == OSD_OP_IN
+        ? (cur_op->req.hdr.opcode == OSD_OP_READ ||
+        cur_op->req.hdr.opcode == OSD_OP_SEC_READ ||
+        cur_op->req.hdr.opcode == OSD_OP_SEC_LIST ||
+        cur_op->req.hdr.opcode == OSD_OP_SHOW_CONFIG ||
+        cur_op->req.hdr.opcode == OSD_OP_DESCRIBE)
+        : (cur_op->req.hdr.opcode == OSD_OP_WRITE ||
+        cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE ||
+        cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE_STABLE ||
+        cur_op->req.hdr.opcode == OSD_OP_SEC_STABILIZE ||
+        cur_op->req.hdr.opcode == OSD_OP_SEC_ROLLBACK ||
+        cur_op->req.hdr.opcode == OSD_OP_SHOW_CONFIG)) && cur_op->iov.count > 0)
+    {
+        for (int i = 0; i < cur_op->iov.count; i++)
+        {
+            if (cur_op->iov.buf[i].iov_len > 0)
+            {
+                assert(cur_op->iov.buf[i].iov_base);
+                to_send_list.push_back(cur_op->iov.buf[i]);
+                to_outbox.push_back((msgr_sendp_t){ .op = cur_op, .flags = 0 });
+            }
+        }
+    }
+    if (cur_op->req.hdr.opcode == OSD_OP_SEC_READ_BMP)
+    {
+        if (cur_op->op_type == OSD_OP_IN && cur_op->reply.hdr.retval > 0)
+            to_send_list.push_back((iovec){ .iov_base = cur_op->buf, .iov_len = (size_t)cur_op->reply.hdr.retval });
+        else if (cur_op->op_type == OSD_OP_OUT && cur_op->req.sec_read_bmp.len > 0)
+            to_send_list.push_back((iovec){ .iov_base = cur_op->buf, .iov_len = (size_t)cur_op->req.sec_read_bmp.len });
+        to_outbox.push_back((msgr_sendp_t){ .op = cur_op, .flags = 0 });
+    }
+    if (cur_op->op_type == OSD_OP_IN)
+    {
+        to_outbox[to_outbox.size()-1].flags |= MSGR_SENDP_FREE;
+    }
+#ifdef WITH_RDMA
+    if (cl->peer_state == PEER_RDMA)
+    {
+        try_send_rdma(cl);
+        return;
+    }
+#endif
+    if (!ringloop)
+    {
+        // FIXME: It's worse because it doesn't allow batching
+        while (cl->outbox.size())
+        {
+            try_send(cl);
+        }
+    }
+    else
+    {
+        if ((cl->write_msg.msg_iovlen > 0 || !try_send(cl)) && (cl->write_state == 0))
+        {
+            cl->write_state = CL_WRITE_READY;
+            write_ready_clients.push_back(cur_op->peer_fd);
+        }
+        ringloop->wakeup();
+    }
+}
+
+void osd_messenger_t::inc_op_stats(osd_op_stats_t & stats, uint64_t opcode, timespec & tv_begin, timespec & tv_end, uint64_t len)
+{
+    uint64_t usecs = (
+        (tv_end.tv_sec - tv_begin.tv_sec)*1000000 +
+        (tv_end.tv_nsec - tv_begin.tv_nsec)/1000
+    );
+    stats.op_stat_count[opcode]++;
+    if (!stats.op_stat_count[opcode])
+    {
+        stats.op_stat_count[opcode] = 1;
+        stats.op_stat_sum[opcode] = 0;
+        stats.op_stat_bytes[opcode] = 0;
+    }
+    stats.op_stat_sum[opcode] += usecs;
+    stats.op_stat_bytes[opcode] += len;
+}
+
+void osd_messenger_t::measure_exec(osd_op_t *cur_op)
+{
+    // Measure execution latency
+    if (cur_op->req.hdr.opcode > OSD_OP_MAX)
+    {
+        return;
+    }
+    if (!cur_op->tv_end.tv_sec)
+    {
+        clock_gettime(CLOCK_REALTIME, &cur_op->tv_end);
+    }
+    uint64_t len = 0;
+    if (cur_op->req.hdr.opcode == OSD_OP_READ ||
+        cur_op->req.hdr.opcode == OSD_OP_WRITE ||
+        cur_op->req.hdr.opcode == OSD_OP_SCRUB)
+    {
+        // req.rw.len is internally set to the full object size for scrubs
+        len = cur_op->req.rw.len;
+    }
+    else if (cur_op->req.hdr.opcode == OSD_OP_SEC_READ ||
+        cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE ||
+        cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE_STABLE)
+    {
+        len = cur_op->req.sec_rw.len;
+    }
+    inc_op_stats(stats, cur_op->req.hdr.opcode, cur_op->tv_begin, cur_op->tv_end, len);
+    if (cur_op->is_recovery_related())
+    {
+        inc_op_stats(recovery_stats, cur_op->req.hdr.opcode, cur_op->tv_begin, cur_op->tv_end, len);
+    }
+}
+
+bool osd_messenger_t::try_send(osd_client_t *cl)
+{
+    int peer_fd = cl->peer_fd;
+    if (!cl->send_list.size() || cl->write_msg.msg_iovlen > 0)
+    {
+        return true;
+    }
+    if (ringloop && !use_sync_send_recv)
+    {
+        io_uring_sqe* sqe = ringloop->get_sqe();
+        if (!sqe)
+        {
+            return false;
+        }
+        cl->write_msg.msg_iov = cl->send_list.data();
+        cl->write_msg.msg_iovlen = cl->send_list.size() < IOV_MAX ? cl->send_list.size() : IOV_MAX;
+        cl->refs++;
+        ring_data_t* data = ((ring_data_t*)sqe->user_data);
+        data->callback = [this, cl](ring_data_t *data) { handle_send(data->res, cl); };
+        my_uring_prep_sendmsg(sqe, peer_fd, &cl->write_msg, 0);
+    }
+    else
+    {
+        cl->write_msg.msg_iov = cl->send_list.data();
+        cl->write_msg.msg_iovlen = cl->send_list.size() < IOV_MAX ? cl->send_list.size() : IOV_MAX;
+        cl->refs++;
+        int result = sendmsg(peer_fd, &cl->write_msg, MSG_NOSIGNAL);
+        if (result < 0)
+        {
+            result = -errno;
+        }
+        handle_send(result, cl);
+    }
+    return true;
+}
+
+void osd_messenger_t::send_replies()
+{
+    for (int i = 0; i < write_ready_clients.size(); i++)
+    {
+        int peer_fd = write_ready_clients[i];
+        auto cl_it = clients.find(peer_fd);
+        if (cl_it != clients.end() && !try_send(cl_it->second))
+        {
+            write_ready_clients.erase(write_ready_clients.begin(), write_ready_clients.begin() + i);
+            return;
+        }
+    }
+    write_ready_clients.clear();
+}
+
+void osd_messenger_t::handle_send(int result, osd_client_t *cl)
+{
+    cl->write_msg.msg_iovlen = 0;
+    cl->refs--;
+    if (cl->peer_state == PEER_STOPPED)
+    {
+        if (cl->refs <= 0)
+        {
+            delete cl;
+        }
+        return;
+    }
+    if (result < 0 && result != -EAGAIN && result != -EINTR)
+    {
+        // this is a client socket, so don't panic. just disconnect it
+        fprintf(stderr, "Client %d socket write error: %d (%s). Disconnecting client\n", cl->peer_fd, -result, strerror(-result));
+        stop_client(cl->peer_fd);
+        return;
+    }
+    if (result >= 0)
+    {
+        int done = 0;
+        while (result > 0 && done < cl->send_list.size())
+        {
+            iovec & iov = cl->send_list[done];
+            if (iov.iov_len <= result)
+            {
+                if (cl->outbox[done].flags & MSGR_SENDP_FREE)
+                {
+                    // Reply fully sent
+                    delete cl->outbox[done].op;
+                }
+                result -= iov.iov_len;
+                done++;
+            }
+            else
+            {
+                iov.iov_len -= result;
+                iov.iov_base = (uint8_t*)iov.iov_base + result;
+                break;
+            }
+        }
+        if (done > 0)
+        {
+            cl->send_list.erase(cl->send_list.begin(), cl->send_list.begin()+done);
+            cl->outbox.erase(cl->outbox.begin(), cl->outbox.begin()+done);
+        }
+        if (cl->next_send_list.size())
+        {
+            cl->send_list.insert(cl->send_list.end(), cl->next_send_list.begin(), cl->next_send_list.end());
+            cl->outbox.insert(cl->outbox.end(), cl->next_outbox.begin(), cl->next_outbox.end());
+            cl->next_send_list.clear();
+            cl->next_outbox.clear();
+        }
+        cl->write_state = cl->outbox.size() > 0 ? CL_WRITE_READY : 0;
+#ifdef WITH_RDMA
+        if (cl->rdma_conn && !cl->outbox.size() && cl->peer_state == PEER_RDMA_CONNECTING)
+        {
+            // FIXME: Do something better than just forgetting the FD
+            // FIXME: Ignore pings during RDMA state transition
+            if (log_level > 0)
+            {
+                fprintf(stderr, "Successfully connected with client %d using RDMA\n", cl->peer_fd);
+            }
+            cl->peer_state = PEER_RDMA;
+            tfd->set_fd_handler(cl->peer_fd, false, [this](int peer_fd, int epoll_events)
+            {
+                // Do not miss the disconnection!
+                if (epoll_events & EPOLLRDHUP)
+                {
+                    handle_peer_epoll(peer_fd, epoll_events);
+                }
+            });
+            // Add the initial receive request
+            try_recv_rdma(cl);
+        }
+#endif
+    }
+    if (cl->write_state != 0)
+    {
+        write_ready_clients.push_back(cl->peer_fd);
+    }
+}

src/client/msgr_stop.cpp

@@ -0,0 +1,161 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+#include <unistd.h>
+#include <assert.h>
+
+#include "messenger.h"
+#ifdef WITH_RDMA
+#include "msgr_rdma.h"
+#endif
+
+void osd_messenger_t::cancel_osd_ops(osd_client_t *cl)
+{
+    std::vector<osd_op_t*> cancel_ops;
+    cancel_ops.resize(cl->sent_ops.size());
+    int i = 0;
+    for (auto p: cl->sent_ops)
+    {
+        cancel_ops[i++] = p.second;
+    }
+    cl->sent_ops.clear();
+    cl->outbox.clear();
+    for (auto op: cancel_ops)
+    {
+        cancel_op(op);
+    }
+}
+
+void osd_messenger_t::cancel_op(osd_op_t *op)
+{
+    if (op->op_type == OSD_OP_OUT)
+    {
+        op->reply.hdr.magic = SECONDARY_OSD_REPLY_MAGIC;
+        op->reply.hdr.id = op->req.hdr.id;
+        op->reply.hdr.opcode = op->req.hdr.opcode;
+        op->reply.hdr.retval = -EPIPE;
+        // Copy lambda to be unaffected by `delete op`
+        std::function<void(osd_op_t*)>(op->callback)(op);
+    }
+    else
+    {
+        // This function is only called in stop_client(), so it's fine to destroy the operation
+        delete op;
+    }
+}
+
+void osd_messenger_t::stop_client(int peer_fd, bool force, bool force_delete)
+{
+    assert(peer_fd != 0);
+    auto it = clients.find(peer_fd);
+    if (it == clients.end())
+    {
+        return;
+    }
+    osd_client_t *cl = it->second;
+    if (cl->peer_state == PEER_CONNECTING && !force || cl->peer_state == PEER_STOPPED)
+    {
+        return;
+    }
+    if (log_level > 0)
+    {
+        if (cl->osd_num)
+        {
+            fprintf(stderr, "[OSD %ju] Stopping client %d (OSD peer %ju)\n", osd_num, peer_fd, cl->osd_num);
+        }
+        else
+        {
+            fprintf(stderr, "[OSD %ju] Stopping client %d (regular client)\n", osd_num, peer_fd);
+        }
+    }
+    // First set state to STOPPED so another stop_client() call doesn't try to free it again
+    cl->refs++;
+    cl->peer_state = PEER_STOPPED;
+    if (cl->osd_num)
+    {
+        // ...and forget OSD peer
+        osd_peer_fds.erase(cl->osd_num);
+    }
+#ifndef __MOCK__
+    // Then remove FD from the eventloop so we don't accidentally read something
+    tfd->set_fd_handler(peer_fd, false, NULL);
+    if (cl->connect_timeout_id >= 0)
+    {
+        tfd->clear_timer(cl->connect_timeout_id);
+        cl->connect_timeout_id = -1;
+    }
+    for (auto rit = read_ready_clients.begin(); rit != read_ready_clients.end(); rit++)
+    {
+        if (*rit == peer_fd)
+        {
+            read_ready_clients.erase(rit);
+            break;
+        }
+    }
+    for (auto wit = write_ready_clients.begin(); wit != write_ready_clients.end(); wit++)
+    {
+        if (*wit == peer_fd)
+        {
+            write_ready_clients.erase(wit);
+            break;
+        }
+    }
+#endif
+    if (cl->osd_num)
+    {
+        // Then repeer PGs because cancel_op() callbacks can try to perform
+        // some actions and we need correct PG states to not do something silly
+        repeer_pgs(cl->osd_num);
+    }
+    // Then cancel all operations
+    if (cl->read_op)
+    {
+        if (!cl->read_op->callback)
+        {
+            delete cl->read_op;
+        }
+        else
+        {
+            cancel_op(cl->read_op);
+        }
+        cl->read_op = NULL;
+    }
+    if (cl->osd_num)
+    {
+        // Cancel outbound operations
+        cancel_osd_ops(cl);
+    }
+    // Find the item again because it can be invalidated at this point
+    it = clients.find(peer_fd);
+    if (it != clients.end())
+    {
+        clients.erase(it);
+    }
+    cl->refs--;
+    if (cl->refs <= 0 || force_delete)
+    {
+        delete cl;
+    }
+}
+
+osd_client_t::~osd_client_t()
+{
+    free(in_buf);
+    in_buf = NULL;
+    if (peer_fd >= 0)
+    {
+        // Close the FD only when the client is actually destroyed
+        // Which only happens when all references are cleared
+        close(peer_fd);
+        peer_fd = -1;
+    }
+#ifndef __MOCK__
+#ifdef WITH_RDMA
+    if (rdma_conn)
+    {
+        delete rdma_conn;
+        rdma_conn = NULL;
+    }
+#endif
+#endif
+}

src/client/object_id.h

@@ -0,0 +1,78 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+#pragma once
+
+#include <stdint.h>
+#include <functional>
+
+typedef uint64_t inode_t;
+
+// 16 bytes per object/stripe id
+// stripe = (start of the parity stripe + peer role)
+// i.e. for example (256KB + one of 0,1,2)
+struct __attribute__((__packed__)) object_id
+{
+    inode_t inode;
+    uint64_t stripe;
+};
+
+inline bool operator == (const object_id & a, const object_id & b)
+{
+    return a.inode == b.inode && a.stripe == b.stripe;
+}
+
+inline bool operator != (const object_id & a, const object_id & b)
+{
+    return a.inode != b.inode || a.stripe != b.stripe;
+}
+
+inline bool operator < (const object_id & a, const object_id & b)
+{
+    return a.inode < b.inode || a.inode == b.inode && a.stripe < b.stripe;
+}
+
+// 56 = 24 + 32 bytes per dirty entry in memory (obj_ver_id => dirty_entry)
+struct __attribute__((__packed__)) obj_ver_id
+{
+    object_id oid;
+    uint64_t version;
+};
+
+inline bool operator == (const obj_ver_id & a, const obj_ver_id & b)
+{
+    return a.oid == b.oid && a.version == b.version;
+}
+
+inline bool operator < (const obj_ver_id & a, const obj_ver_id & b)
+{
+    return a.oid < b.oid || a.oid == b.oid && a.version < b.version;
+}
+
+namespace std
+{
+    template<> struct hash<object_id>
+    {
+        inline size_t operator()(const object_id &s) const
+        {
+            size_t seed = 0;
+            // Copy-pasted from spp::hash_combine()
+            seed ^= (s.inode + 0xc6a4a7935bd1e995 + (seed << 6) + (seed >> 2));
+            seed ^= (s.stripe + 0xc6a4a7935bd1e995 + (seed << 6) + (seed >> 2));
+            return seed;
+        }
+    };
+
+    template<> struct hash<obj_ver_id>
+    {
+        inline size_t operator()(const obj_ver_id &s) const
+        {
+            size_t seed = 0;
+            // Copy-pasted from spp::hash_combine()
+            seed ^= (s.oid.inode + 0xc6a4a7935bd1e995 + (seed << 6) + (seed >> 2));
+            seed ^= (s.oid.stripe + 0xc6a4a7935bd1e995 + (seed << 6) + (seed >> 2));
+            seed ^= (s.version + 0xc6a4a7935bd1e995 + (seed << 6) + (seed >> 2));
+            return seed;
+        }
+    };
+}

src/client/osd_ops.cpp

@@ -0,0 +1,26 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+#include "osd_ops.h"
+
+const char* osd_op_names[] = {
+    "",
+    "read",
+    "write",
+    "write_stable",
+    "sync",
+    "stabilize",
+    "rollback",
+    "delete",
+    "sync_stab_all",
+    "list",
+    "show_config",
+    "primary_read",
+    "primary_write",
+    "primary_sync",
+    "primary_delete",
+    "ping",
+    "sec_read_bmp",
+    "scrub",
+    "describe",
+};

src/client/osd_ops.h

@@ -0,0 +1,315 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+#pragma once
+
+#include "object_id.h"
+#include "osd_id.h"
+
+// Magic numbers
+#define SECONDARY_OSD_OP_MAGIC      0x2bd7b10325434553l
+#define SECONDARY_OSD_REPLY_MAGIC   0xbaa699b87b434553l
+// Operation request / reply headers have fixed size after which comes data
+#define OSD_PACKET_SIZE             0x80
+// Opcodes
+#define OSD_OP_MIN                  1
+#define OSD_OP_SEC_READ             1
+#define OSD_OP_SEC_WRITE            2
+#define OSD_OP_SEC_WRITE_STABLE     3
+#define OSD_OP_SEC_SYNC             4
+#define OSD_OP_SEC_STABILIZE        5
+#define OSD_OP_SEC_ROLLBACK         6
+#define OSD_OP_SEC_DELETE           7
+#define OSD_OP_TEST_SYNC_STAB_ALL   8
+#define OSD_OP_SEC_LIST             9
+#define OSD_OP_SHOW_CONFIG          10
+#define OSD_OP_READ                 11
+#define OSD_OP_WRITE                12
+#define OSD_OP_SYNC                 13
+#define OSD_OP_DELETE               14
+#define OSD_OP_PING                 15
+#define OSD_OP_SEC_READ_BMP         16
+#define OSD_OP_SCRUB                17
+#define OSD_OP_DESCRIBE             18
+#define OSD_OP_MAX                  18
+#define OSD_RW_MAX                  64*1024*1024
+#define OSD_PROTOCOL_VERSION        1
+#define OSD_OP_RECOVERY_RELATED     (uint32_t)1
+
+// Memory alignment for direct I/O (usually 512 bytes)
+#ifndef DIRECT_IO_ALIGNMENT
+#define DIRECT_IO_ALIGNMENT 512
+#endif
+
+// Memory allocation alignment (page size is usually optimal)
+#ifndef MEM_ALIGNMENT
+#define MEM_ALIGNMENT 4096
+#endif
+
+// Constants for osd_reply_describe_item_t.loc_bad
+#define LOC_OUTDATED 1
+#define LOC_CORRUPTED 2
+#define LOC_INCONSISTENT 4
+
+// common request and reply headers
+struct __attribute__((__packed__)) osd_op_header_t
+{
+    // magic & protocol version
+    uint64_t magic;
+    // operation id
+    uint64_t id;
+    // operation type
+    uint64_t opcode;
+};
+
+struct __attribute__((__packed__)) osd_reply_header_t
+{
+    // magic & protocol version
+    uint64_t magic;
+    // operation id
+    uint64_t id;
+    // operation type
+    uint64_t opcode;
+    // return value
+    int64_t retval;
+};
+
+// read or write to the secondary OSD
+struct __attribute__((__packed__)) osd_op_sec_rw_t
+{
+    osd_op_header_t header;
+    // object
+    object_id oid;
+    // read/write version (automatic or specific)
+    // FIXME deny values close to UINT64_MAX
+    uint64_t version;
+    // offset
+    uint32_t offset;
+    // length
+    uint32_t len;
+    // bitmap/attribute length - bitmap comes after header, but before data
+    uint32_t attr_len;
+    // the only possible flag is OSD_OP_RECOVERY_RELATED
+    uint32_t flags;
+};
+
+struct __attribute__((__packed__)) osd_reply_sec_rw_t
+{
+    osd_reply_header_t header;
+    // for reads and writes: assigned or read version number
+    uint64_t version;
+    // for reads: bitmap/attribute length (just to double-check)
+    uint32_t attr_len;
+    uint32_t pad0;
+};
+
+// delete object on the secondary OSD
+struct __attribute__((__packed__)) osd_op_sec_del_t
+{
+    osd_op_header_t header;
+    // object
+    object_id oid;
+    // delete version (automatic or specific)
+    uint64_t version;
+    // the only possible flag is OSD_OP_RECOVERY_RELATED
+    uint32_t flags;
+    uint32_t pad0;
+};
+
+struct __attribute__((__packed__)) osd_reply_sec_del_t
+{
+    osd_reply_header_t header;
+    uint64_t version;
+};
+
+// sync to the secondary OSD
+struct __attribute__((__packed__)) osd_op_sec_sync_t
+{
+    osd_op_header_t header;
+    // the only possible flag is OSD_OP_RECOVERY_RELATED
+    uint32_t flags;
+    uint32_t pad0;
+};
+
+struct __attribute__((__packed__)) osd_reply_sec_sync_t
+{
+    osd_reply_header_t header;
+};
+
+// stabilize or rollback objects on the secondary OSD
+struct __attribute__((__packed__)) osd_op_sec_stab_t
+{
+    osd_op_header_t header;
+    // obj_ver_id array length in bytes
+    uint64_t len;
+    // the only possible flag is OSD_OP_RECOVERY_RELATED
+    uint32_t flags;
+    uint32_t pad0;
+};
+typedef osd_op_sec_stab_t osd_op_sec_rollback_t;
+
+struct __attribute__((__packed__)) osd_reply_sec_stab_t
+{
+    osd_reply_header_t header;
+};
+typedef osd_reply_sec_stab_t osd_reply_sec_rollback_t;
+
+// bulk read bitmaps from a secondary OSD
+struct __attribute__((__packed__)) osd_op_sec_read_bmp_t
+{
+    osd_op_header_t header;
+    // obj_ver_id array length in bytes
+    uint64_t len;
+};
+
+struct __attribute__((__packed__)) osd_reply_sec_read_bmp_t
+{
+    // retval is payload length in bytes. payload is {version,bitmap}[]
+    osd_reply_header_t header;
+};
+
+// show configuration
+struct __attribute__((__packed__)) osd_op_show_config_t
+{
+    osd_op_header_t header;
+    // JSON request length
+    uint64_t json_len;
+};
+
+struct __attribute__((__packed__)) osd_reply_show_config_t
+{
+    osd_reply_header_t header;
+};
+
+// list objects on replica
+struct __attribute__((__packed__)) osd_op_sec_list_t
+{
+    osd_op_header_t header;
+    // placement group total number and total count
+    pg_num_t list_pg, pg_count;
+    // size of an area that maps to one PG continuously
+    uint64_t pg_stripe_size;
+    // inode range (used to select pools)
+    uint64_t min_inode, max_inode;
+    // min/max oid stripe, added after inodes for backwards compatibility
+    // also for backwards compatibility, max_stripe=UINT64_MAX means 0 and 0 means UINT64_MAX O_o
+    uint64_t min_stripe, max_stripe;
+    // max stable object count
+    uint32_t stable_limit;
+};
+
+struct __attribute__((__packed__)) osd_reply_sec_list_t
+{
+    osd_reply_header_t header;
+    // stable object version count. header.retval = total object version count
+    // FIXME: maybe change to the number of bytes in the reply...
+    uint64_t stable_count;
+};
+
+// read or write to the primary OSD (must be within individual stripe)
+struct __attribute__((__packed__)) osd_op_rw_t
+{
+    osd_op_header_t header;
+    // inode
+    uint64_t inode;
+    // offset
+    uint64_t offset;
+    // length. 0 means to read all bitmaps of the specified range, but no data.
+    uint32_t len;
+    // flags (for future)
+    uint32_t flags;
+    // inode metadata revision
+    uint64_t meta_revision;
+    // object version for atomic "CAS" (compare-and-set) writes
+    // writes and deletes fail with -EINTR if object version differs from (version-1)
+    uint64_t version;
+};
+
+struct __attribute__((__packed__)) osd_reply_rw_t
+{
+    osd_reply_header_t header;
+    // for reads: bitmap length
+    uint32_t bitmap_len;
+    uint32_t pad0;
+    // for reads and writes: object version
+    uint64_t version;
+};
+
+// sync to the primary OSD
+struct __attribute__((__packed__)) osd_op_sync_t
+{
+    osd_op_header_t header;
+};
+
+struct __attribute__((__packed__)) osd_reply_sync_t
+{
+    osd_reply_header_t header;
+};
+
+// describe unclean object states in detail
+struct __attribute__((__packed__)) osd_op_describe_t
+{
+    osd_op_header_t header;
+    // state mask to filter objects by state (0 or 0xfff..ff = all objects)
+    uint64_t object_state;
+    // minimum inode and offset
+    uint64_t min_inode, min_offset;
+    // maximum inode and offset
+    uint64_t max_inode, max_offset;
+    // limit
+    uint64_t limit;
+    // pool and PG
+    uint32_t pool_id;
+    uint32_t pg_num;
+};
+
+struct __attribute__((__packed__)) osd_reply_describe_t
+{
+    osd_reply_header_t header;
+    // size of the resulting <osd_reply_describe_item_t> array in bytes
+    uint64_t result_bytes;
+};
+
+struct __attribute__((__packed__)) osd_reply_describe_item_t
+{
+    uint64_t inode;
+    uint64_t stripe;
+    uint32_t role;      // part number: 0 for replicas, 0..pg_size-1 for EC
+    uint32_t loc_bad;   // LOC_OUTDATED / LOC_CORRUPTED / LOC_INCONSISTENT
+    osd_num_t osd_num;  // OSD number
+};
+
+// FIXME it would be interesting to try to unify blockstore_op and osd_op formats
+union osd_any_op_t
+{
+    osd_op_header_t hdr;
+    osd_op_sec_rw_t sec_rw;
+    osd_op_sec_del_t sec_del;
+    osd_op_sec_sync_t sec_sync;
+    osd_op_sec_stab_t sec_stab;
+    osd_op_sec_read_bmp_t sec_read_bmp;
+    osd_op_sec_list_t sec_list;
+    osd_op_show_config_t show_conf;
+    osd_op_rw_t rw;
+    osd_op_sync_t sync;
+    osd_op_describe_t describe;
+    uint8_t buf[OSD_PACKET_SIZE];
+};
+
+union osd_any_reply_t
+{
+    osd_reply_header_t hdr;
+    osd_reply_sec_rw_t sec_rw;
+    osd_reply_sec_del_t sec_del;
+    osd_reply_sec_sync_t sec_sync;
+    osd_reply_sec_stab_t sec_stab;
+    osd_reply_sec_read_bmp_t sec_read_bmp;
+    osd_reply_sec_list_t sec_list;
+    osd_reply_show_config_t show_conf;
+    osd_reply_rw_t rw;
+    osd_reply_sync_t sync;
+    osd_reply_describe_t describe;
+    uint8_t buf[OSD_PACKET_SIZE];
+};
+
+extern const char* osd_op_names[];

src/client/pg_states.cpp

@@ -0,0 +1,70 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+#include "pg_states.h"
+
+const int pg_state_bit_count = 17;
+
+const int pg_state_bits[17] = {
+    PG_STARTING,
+    PG_PEERING,
+    PG_INCOMPLETE,
+    PG_ACTIVE,
+    PG_REPEERING,
+    PG_STOPPING,
+    PG_OFFLINE,
+    PG_DEGRADED,
+    PG_HAS_INCONSISTENT,
+    PG_HAS_CORRUPTED,
+    PG_HAS_INCOMPLETE,
+    PG_HAS_DEGRADED,
+    PG_HAS_MISPLACED,
+    PG_HAS_UNCLEAN,
+    PG_HAS_INVALID,
+    PG_LEFT_ON_DEAD,
+    PG_SCRUBBING,
+};
+
+const char *pg_state_names[17] = {
+    "starting",
+    "peering",
+    "incomplete",
+    "active",
+    "repeering",
+    "stopping",
+    "offline",
+    "degraded",
+    "has_inconsistent",
+    "has_corrupted",
+    "has_incomplete",
+    "has_degraded",
+    "has_misplaced",
+    "has_unclean",
+    "has_invalid",
+    "left_on_dead",
+    "scrubbing",
+};
+
+const int object_state_bit_count = 8;
+
+const int object_state_bits[8] = {
+    OBJ_DEGRADED,
+    OBJ_INCOMPLETE,
+    OBJ_MISPLACED,
+    OBJ_CORRUPTED,
+    OBJ_INCONSISTENT,
+    OBJ_NEEDS_STABLE,
+    OBJ_NEEDS_ROLLBACK,
+    0,
+};
+
+const char *object_state_names[8] = {
+    "degraded",
+    "incomplete",
+    "misplaced",
+    "corrupted",
+    "inconsistent",
+    "needs_stable",
+    "needs_rollback",
+    "clean",
+};

src/client/pg_states.h

@@ -0,0 +1,52 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+#pragma once
+
+// Placement group states
+// STARTING -> [acquire lock] -> PEERING -> INCOMPLETE|ACTIVE
+// ACTIVE -> REPEERING -> PEERING
+// ACTIVE -> STOPPING -> OFFLINE -> [release lock]
+// Exactly one of these:
+#define PG_STARTING (1<<0)
+#define PG_PEERING (1<<1)
+#define PG_INCOMPLETE (1<<2)
+#define PG_ACTIVE (1<<3)
+#define PG_REPEERING (1<<4)
+#define PG_STOPPING (1<<5)
+#define PG_OFFLINE (1<<6)
+// Plus any of these:
+#define PG_DEGRADED (1<<7)
+#define PG_HAS_INCOMPLETE (1<<8)
+#define PG_HAS_DEGRADED (1<<9)
+#define PG_HAS_MISPLACED (1<<10)
+#define PG_HAS_UNCLEAN (1<<11)
+#define PG_HAS_INVALID (1<<12)
+#define PG_HAS_CORRUPTED (1<<13)
+#define PG_HAS_INCONSISTENT (1<<14)
+#define PG_LEFT_ON_DEAD (1<<15)
+#define PG_SCRUBBING (1<<16)
+
+// Lower bits that represent object role (EC 0/1/2... or always 0 with replication)
+// 12 bits is a safe default that doesn't depend on pg_stripe_size or pg_block_size
+#define STRIPE_MASK ((uint64_t)4096 - 1)
+
+// OSD object states
+#define OBJ_DEGRADED 0x02
+#define OBJ_INCOMPLETE 0x04
+#define OBJ_MISPLACED 0x08
+// OBJ_CORRUPTED is always set with one of OBJ_INCOMPLETE/OBJ_DEGRADED/OBJ_MISPLACED
+#define OBJ_CORRUPTED 0x10
+// OBJ_INCONSISTENT is when its replicas don't match, but it's unclear which one is correct
+// OBJ_INCONSISTENT may be set with CORRUPTED, but never with other states
+#define OBJ_INCONSISTENT 0x20
+#define OBJ_NEEDS_STABLE 0x10000
+#define OBJ_NEEDS_ROLLBACK 0x20000
+
+extern const int pg_state_bits[];
+extern const char *pg_state_names[];
+extern const int pg_state_bit_count;
+
+extern const int object_state_bits[];
+extern const char *object_state_names[];
+extern const int object_state_bit_count;

src/client/vitastor.pc.in

@@ -0,0 +1,12 @@
+prefix=@CMAKE_INSTALL_PREFIX@
+exec_prefix=${prefix}
+libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@
+includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
+
+
+Name: Vitastor
+Description: Vitastor client library
+Version: 1.6.1
+Libs: -L${libdir} -lvitastor_client
+Cflags: -I${includedir}
+

src/client/vitastor_c.cpp

@@ -0,0 +1,387 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+// Simplified C client library for QEMU, fio and other external drivers
+// Also acts as a C-C++ proxy for the QEMU driver (QEMU headers don't compile with g++)
+
+#include <sys/epoll.h>
+#include <sys/eventfd.h>
+
+#include "ringloop.h"
+#include "epoll_manager.h"
+#include "cluster_client.h"
+
+#include "vitastor_c.h"
+
+struct vitastor_qemu_fd_t
+{
+    int fd;
+    std::function<void(int, int)> callback;
+};
+
+struct vitastor_c
+{
+    std::map<int, vitastor_qemu_fd_t> handlers;
+    ring_loop_t *ringloop = NULL;
+    epoll_manager_t *epmgr = NULL;
+    timerfd_manager_t *tfd = NULL;
+    cluster_client_t *cli = NULL;
+    int uring_eventfd = -1;
+
+    QEMUSetFDHandler *aio_set_fd_handler = NULL;
+    void *aio_ctx = NULL;
+};
+
+extern "C" {
+
+static json11::Json vitastor_c_common_config(const char *config_path, const char *etcd_host, const char *etcd_prefix,
+    int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level)
+{
+    json11::Json::object cfg;
+    if (config_path)
+        cfg["config_path"] = std::string(config_path);
+    if (etcd_host)
+        cfg["etcd_address"] = std::string(etcd_host);
+    if (etcd_prefix)
+        cfg["etcd_prefix"] = std::string(etcd_prefix);
+    // -1 means unspecified
+    if (use_rdma >= 0)
+        cfg["use_rdma"] = use_rdma > 0;
+    if (rdma_device)
+        cfg["rdma_device"] = std::string(rdma_device);
+    if (rdma_port_num)
+        cfg["rdma_port_num"] = rdma_port_num;
+    if (rdma_gid_index)
+        cfg["rdma_gid_index"] = rdma_gid_index;
+    if (rdma_mtu)
+        cfg["rdma_mtu"] = rdma_mtu;
+    if (log_level)
+        cfg["log_level"] = log_level;
+    return json11::Json(cfg);
+}
+
+static void vitastor_c_read_handler(void *opaque)
+{
+    vitastor_qemu_fd_t *data = (vitastor_qemu_fd_t *)opaque;
+    data->callback(data->fd, EPOLLIN);
+}
+
+static void vitastor_c_write_handler(void *opaque)
+{
+    vitastor_qemu_fd_t *data = (vitastor_qemu_fd_t *)opaque;
+    data->callback(data->fd, EPOLLOUT);
+}
+
+static vitastor_c *vitastor_c_create_qemu_common(QEMUSetFDHandler *aio_set_fd_handler, void *aio_context)
+{
+    vitastor_c *self = new vitastor_c;
+    self->aio_set_fd_handler = aio_set_fd_handler;
+    self->aio_ctx = aio_context;
+    self->tfd = new timerfd_manager_t([self](int fd, bool wr, std::function<void(int, int)> callback)
+    {
+        if (callback != NULL)
+        {
+            self->handlers[fd] = { .fd = fd, .callback = callback };
+            self->aio_set_fd_handler(self->aio_ctx, fd, false,
+                vitastor_c_read_handler, wr ? vitastor_c_write_handler : NULL, NULL, &self->handlers[fd]);
+        }
+        else
+        {
+            self->handlers.erase(fd);
+            self->aio_set_fd_handler(self->aio_ctx, fd, false, NULL, NULL, NULL, NULL);
+        }
+    });
+    return self;
+}
+
+vitastor_c *vitastor_c_create_qemu(QEMUSetFDHandler *aio_set_fd_handler, void *aio_context,
+    const char *config_path, const char *etcd_host, const char *etcd_prefix,
+    int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level)
+{
+    json11::Json cfg_json = vitastor_c_common_config(
+        config_path, etcd_host, etcd_prefix, use_rdma,
+        rdma_device, rdma_port_num, rdma_gid_index, rdma_mtu, log_level
+    );
+    auto self = vitastor_c_create_qemu_common(aio_set_fd_handler, aio_context);
+    self->cli = new cluster_client_t(NULL, self->tfd, cfg_json);
+    return self;
+}
+
+vitastor_c *vitastor_c_create_qemu_uring(QEMUSetFDHandler *aio_set_fd_handler, void *aio_context,
+    const char *config_path, const char *etcd_host, const char *etcd_prefix,
+    int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level)
+{
+    ring_loop_t *ringloop = NULL;
+    try
+    {
+        ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
+    }
+    catch (std::exception & e)
+    {
+        return NULL;
+    }
+    json11::Json cfg_json = vitastor_c_common_config(
+        config_path, etcd_host, etcd_prefix, use_rdma,
+        rdma_device, rdma_port_num, rdma_gid_index, rdma_mtu, log_level
+    );
+    auto self = vitastor_c_create_qemu_common(aio_set_fd_handler, aio_context);
+    self->ringloop = ringloop;
+    self->cli = new cluster_client_t(self->ringloop, self->tfd, cfg_json);
+    return self;
+}
+
+vitastor_c *vitastor_c_create_uring(const char *config_path, const char *etcd_host, const char *etcd_prefix,
+    int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level)
+{
+    ring_loop_t *ringloop = NULL;
+    try
+    {
+        ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
+    }
+    catch (std::exception & e)
+    {
+        return NULL;
+    }
+    json11::Json cfg_json = vitastor_c_common_config(
+        config_path, etcd_host, etcd_prefix, use_rdma,
+        rdma_device, rdma_port_num, rdma_gid_index, rdma_mtu, log_level
+    );
+    vitastor_c *self = new vitastor_c;
+    self->ringloop = ringloop;
+    self->epmgr = new epoll_manager_t(self->ringloop);
+    self->cli = new cluster_client_t(self->ringloop, self->epmgr->tfd, cfg_json);
+    return self;
+}
+
+int vitastor_c_uring_register_eventfd(vitastor_c *client)
+{
+    if (!client->ringloop)
+    {
+        return -EINVAL;
+    }
+    return client->ringloop->register_eventfd();
+}
+
+vitastor_c *vitastor_c_create_uring_json(const char **options, int options_len)
+{
+    ring_loop_t *ringloop = NULL;
+    try
+    {
+        ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
+    }
+    catch (std::exception & e)
+    {
+        return NULL;
+    }
+    json11::Json::object cfg;
+    for (int i = 0; i < options_len-1; i += 2)
+    {
+        cfg[options[i]] = std::string(options[i+1]);
+    }
+    json11::Json cfg_json(cfg);
+    vitastor_c *self = new vitastor_c;
+    self->ringloop = ringloop;
+    self->epmgr = new epoll_manager_t(self->ringloop);
+    self->cli = new cluster_client_t(self->ringloop, self->epmgr->tfd, cfg_json);
+    return self;
+}
+
+vitastor_c *vitastor_c_create_epoll_json(const char **options, int options_len)
+{
+    json11::Json::object cfg;
+    for (int i = 0; i < options_len-1; i += 2)
+    {
+        cfg[options[i]] = std::string(options[i+1]);
+    }
+    json11::Json cfg_json(cfg);
+    vitastor_c *self = new vitastor_c;
+    self->epmgr = new epoll_manager_t(NULL);
+    self->cli = new cluster_client_t(NULL, self->epmgr->tfd, cfg_json);
+    return self;
+}
+
+void* vitastor_c_get_internal_client(vitastor_c *client)
+{
+    return client->cli;
+}
+
+void vitastor_c_destroy(vitastor_c *client)
+{
+    delete client->cli;
+    if (client->epmgr)
+        delete client->epmgr;
+    else if (client->tfd)
+        delete client->tfd;
+    if (client->ringloop)
+        delete client->ringloop;
+    delete client;
+}
+
+int vitastor_c_is_ready(vitastor_c *client)
+{
+    return client->cli->is_ready();
+}
+
+void vitastor_c_uring_wait_ready(vitastor_c *client)
+{
+    while (!client->cli->is_ready())
+    {
+        client->ringloop->loop();
+        if (client->cli->is_ready())
+            break;
+        client->ringloop->wait();
+    }
+}
+
+void vitastor_c_uring_handle_events(vitastor_c *client)
+{
+    client->ringloop->loop();
+}
+
+void vitastor_c_uring_wait_events(vitastor_c *client)
+{
+    client->ringloop->wait();
+}
+
+int vitastor_c_uring_has_work(vitastor_c *client)
+{
+    return client->ringloop->has_work();
+}
+
+int vitastor_c_epoll_get_fd(vitastor_c *client)
+{
+    return !client->ringloop && client->epmgr ? client->epmgr->get_fd() : -1;
+}
+
+void vitastor_c_epoll_handle_events(vitastor_c *client, int timeout)
+{
+    return client->epmgr->handle_events(timeout);
+}
+
+void vitastor_c_read(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len,
+    struct iovec *iov, int iovcnt, VitastorReadHandler cb, void *opaque)
+{
+    cluster_op_t *op = new cluster_op_t;
+    op->opcode = OSD_OP_READ;
+    op->inode = inode;
+    op->offset = offset;
+    op->len = len;
+    for (int i = 0; i < iovcnt; i++)
+    {
+        op->iov.push_back(iov[i].iov_base, iov[i].iov_len);
+    }
+    op->callback = [cb, opaque](cluster_op_t *op)
+    {
+        cb(opaque, op->retval, op->version);
+        delete op;
+    };
+    client->cli->execute(op);
+}
+
+void vitastor_c_write(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, uint64_t check_version,
+    struct iovec *iov, int iovcnt, VitastorIOHandler cb, void *opaque)
+{
+    cluster_op_t *op = new cluster_op_t;
+    op->opcode = OSD_OP_WRITE;
+    op->inode = inode;
+    op->offset = offset;
+    op->len = len;
+    op->version = check_version;
+    for (int i = 0; i < iovcnt; i++)
+    {
+        op->iov.push_back(iov[i].iov_base, iov[i].iov_len);
+    }
+    op->callback = [cb, opaque](cluster_op_t *op)
+    {
+        cb(opaque, op->retval);
+        delete op;
+    };
+    client->cli->execute(op);
+}
+
+void vitastor_c_read_bitmap(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len,
+    int with_parents, VitastorReadBitmapHandler cb, void *opaque)
+{
+    cluster_op_t *op = new cluster_op_t;
+    op->opcode = with_parents ? OSD_OP_READ_CHAIN_BITMAP : OSD_OP_READ_BITMAP;
+    op->inode = inode;
+    op->offset = offset;
+    op->len = len;
+    op->callback = [cb, opaque](cluster_op_t *op)
+    {
+        uint8_t *bitmap = NULL;
+        if (op->retval >= 0)
+        {
+            bitmap = (uint8_t*)op->bitmap_buf;
+            op->bitmap_buf = NULL;
+        }
+        cb(opaque, op->retval, bitmap);
+        delete op;
+    };
+    client->cli->execute(op);
+}
+
+void vitastor_c_sync(vitastor_c *client, VitastorIOHandler cb, void *opaque)
+{
+    cluster_op_t *op = new cluster_op_t;
+    op->opcode = OSD_OP_SYNC;
+    op->callback = [cb, opaque](cluster_op_t *op)
+    {
+        cb(opaque, op->retval);
+        delete op;
+    };
+    client->cli->execute(op);
+}
+
+void vitastor_c_watch_inode(vitastor_c *client, char *image, VitastorIOHandler cb, void *opaque)
+{
+    client->cli->on_ready([=]()
+    {
+        auto watch = client->cli->st_cli.watch_inode(std::string(image));
+        cb(opaque, (long)watch);
+    });
+}
+
+void vitastor_c_close_watch(vitastor_c *client, void *handle)
+{
+    client->cli->st_cli.close_watch((inode_watch_t*)handle);
+}
+
+uint64_t vitastor_c_inode_get_size(void *handle)
+{
+    inode_watch_t *watch = (inode_watch_t*)handle;
+    return watch->cfg.size;
+}
+
+uint64_t vitastor_c_inode_get_num(void *handle)
+{
+    inode_watch_t *watch = (inode_watch_t*)handle;
+    return watch->cfg.num;
+}
+
+uint32_t vitastor_c_inode_get_block_size(vitastor_c *client, uint64_t inode_num)
+{
+    auto pool_it = client->cli->st_cli.pool_config.find(INODE_POOL(inode_num));
+    if (pool_it == client->cli->st_cli.pool_config.end())
+        return 0;
+    auto & pool_cfg = pool_it->second;
+    uint32_t pg_data_size = (pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks);
+    return pool_cfg.data_block_size * pg_data_size;
+}
+
+uint32_t vitastor_c_inode_get_bitmap_granularity(vitastor_c *client, uint64_t inode_num)
+{
+    auto pool_it = client->cli->st_cli.pool_config.find(INODE_POOL(inode_num));
+    if (pool_it == client->cli->st_cli.pool_config.end())
+        return 0;
+    // FIXME: READ_BITMAP may fails if parent bitmap granularity differs from inode bitmap granularity
+    return pool_it->second.bitmap_granularity;
+}
+
+int vitastor_c_inode_get_readonly(void *handle)
+{
+    inode_watch_t *watch = (inode_watch_t*)handle;
+    return watch->cfg.readonly;
+}
+
+}

src/client/vitastor_c.h

@@ -0,0 +1,73 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+// Simplified C client library for QEMU, fio and other external drivers
+
+#ifndef VITASTOR_QEMU_PROXY_H
+#define VITASTOR_QEMU_PROXY_H
+
+// C API wrapper version
+#define VITASTOR_C_API_VERSION 4
+
+#ifndef POOL_ID_BITS
+#define POOL_ID_BITS 16
+#endif
+#include <stdint.h>
+#include <sys/uio.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct vitastor_c;
+typedef struct vitastor_c vitastor_c;
+
+typedef void VitastorReadHandler(void *opaque, long retval, uint64_t version);
+typedef void VitastorIOHandler(void *opaque, long retval);
+typedef void VitastorReadBitmapHandler(void *opaque, long retval, uint8_t *bitmap);
+
+// QEMU
+typedef void IOHandler(void *opaque);
+// is_external and poll_fn are not required, but are here for compatibility
+typedef void QEMUSetFDHandler(void *ctx, int fd, int is_external, IOHandler *fd_read, IOHandler *fd_write, void *poll_fn, void *opaque);
+
+vitastor_c *vitastor_c_create_qemu(QEMUSetFDHandler *aio_set_fd_handler, void *aio_context,
+    const char *config_path, const char *etcd_host, const char *etcd_prefix,
+    int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level);
+vitastor_c *vitastor_c_create_qemu_uring(QEMUSetFDHandler *aio_set_fd_handler, void *aio_context,
+    const char *config_path, const char *etcd_host, const char *etcd_prefix,
+    int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level);
+vitastor_c *vitastor_c_create_uring(const char *config_path, const char *etcd_host, const char *etcd_prefix,
+    int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level);
+vitastor_c *vitastor_c_create_uring_json(const char **options, int options_len);
+vitastor_c *vitastor_c_create_epoll_json(const char **options, int options_len);
+void* vitastor_c_get_internal_client(vitastor_c *client);
+void vitastor_c_destroy(vitastor_c *client);
+int vitastor_c_is_ready(vitastor_c *client);
+int vitastor_c_uring_register_eventfd(vitastor_c *client);
+void vitastor_c_uring_wait_ready(vitastor_c *client);
+void vitastor_c_uring_handle_events(vitastor_c *client);
+void vitastor_c_uring_wait_events(vitastor_c *client);
+int vitastor_c_uring_has_work(vitastor_c *client);
+int vitastor_c_epoll_get_fd(vitastor_c *client);
+void vitastor_c_epoll_handle_events(vitastor_c *client, int timeout);
+void vitastor_c_read(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len,
+    struct iovec *iov, int iovcnt, VitastorReadHandler cb, void *opaque);
+void vitastor_c_write(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, uint64_t check_version,
+    struct iovec *iov, int iovcnt, VitastorIOHandler cb, void *opaque);
+void vitastor_c_read_bitmap(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len,
+    int with_parents, VitastorReadBitmapHandler cb, void *opaque);
+void vitastor_c_sync(vitastor_c *client, VitastorIOHandler cb, void *opaque);
+void vitastor_c_watch_inode(vitastor_c *client, char *image, VitastorIOHandler cb, void *opaque);
+void vitastor_c_close_watch(vitastor_c *client, void *handle);
+uint64_t vitastor_c_inode_get_size(void *handle);
+uint64_t vitastor_c_inode_get_num(void *handle);
+uint32_t vitastor_c_inode_get_block_size(vitastor_c *client, uint64_t inode_num);
+uint32_t vitastor_c_inode_get_bitmap_granularity(vitastor_c *client, uint64_t inode_num);
+int vitastor_c_inode_get_readonly(void *handle);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif