diff --git a/src/client/messenger.cpp b/src/client/messenger.cpp index d9ccc76f..2182a519 100644 --- a/src/client/messenger.cpp +++ b/src/client/messenger.cpp @@ -182,12 +182,13 @@ void osd_messenger_t::init() } keepalive_timer_id = tfd->set_timer(1000, true, [this](int) { + std::vector clients_to_stop; + std::vector ops_to_send; auto cl_it = clients.begin(); while (cl_it != clients.end()) { auto cl = cl_it->second; cl_it++; - auto client_id = cl->client_id; if (!cl->osd_num && !cl->in_osd_num || cl->peer_state != PEER_CONNECTED && cl->peer_state != PEER_RDMA) { // Do not run keepalive on regular clients @@ -201,9 +202,7 @@ void osd_messenger_t::init() // Ping timed out, stop the client fprintf(stderr, "Ping timed out for OSD %ju (client %ju), disconnecting peer\n", cl->in_osd_num ? cl->in_osd_num : cl->osd_num, cl->client_id); - stop_client(cl->client_id); - // Restart iterator because it may be invalidated - cl_it = clients.upper_bound(client_id); + clients_to_stop.push_back(cl->client_id); } } else if (cl->idle_time_remaining > 0) @@ -242,9 +241,7 @@ void osd_messenger_t::init() }; cl->ping_time_remaining = osd_ping_timeout; cl->idle_time_remaining = osd_idle_timeout; - outbox_push(op); - // Restart iterator because it may be invalidated - cl_it = clients.upper_bound(client_id); + ops_to_send.push_back(op); } } else @@ -252,6 +249,14 @@ void osd_messenger_t::init() cl->idle_time_remaining = osd_idle_timeout; } } + for (uint64_t client_id: clients_to_stop) + { + stop_client(client_id); + } + for (osd_op_t *op: ops_to_send) + { + outbox_push(op); + } }); } diff --git a/src/client/messenger.h b/src/client/messenger.h index 83e07ce6..88b34894 100644 --- a/src/client/messenger.h +++ b/src/client/messenger.h @@ -12,6 +12,7 @@ #include #include +#include "../util/robin_hood.h" #include "malloc_or_die.h" #include "json11/json11.hpp" #include "msgr_op.h" @@ -86,7 +87,7 @@ struct osd_client_t std::vector received_ops; // Outbound operations - std::map sent_ops; + robin_hood::unordered_flat_map sent_ops; uint64_t send_op_id = 0; // PGs dirtied by this client's primary-writes @@ -202,8 +203,8 @@ protected: uint64_t rdma_max_sge = 0, rdma_max_send = 0, rdma_max_recv = 0; uint64_t rdma_max_msg = 0; rdma_event_channel *rdmacm_evch = NULL; - std::map rdmacm_connections; - std::map rdmacm_connecting; + robin_hood::unordered_flat_map rdmacm_connections; + robin_hood::unordered_flat_map rdmacm_connecting; #endif std::vector iothreads; @@ -219,10 +220,10 @@ public: // osd_num_t is only for logging and asserts uint64_t next_client_id = 1; osd_num_t osd_num; - std::map clients; - std::map osd_peers; - std::map clients_by_fd; - std::map wanted_peers; + robin_hood::unordered_flat_map clients; + robin_hood::unordered_flat_map osd_peers; + robin_hood::unordered_flat_map clients_by_fd; + robin_hood::unordered_flat_map wanted_peers; std::vector osd_networks; std::vector osd_network_masks; std::vector osd_cluster_networks; diff --git a/src/osd/osd_primary_subops.cpp b/src/osd/osd_primary_subops.cpp index 4f2ff8ce..e3314b44 100644 --- a/src/osd/osd_primary_subops.cpp +++ b/src/osd/osd_primary_subops.cpp @@ -641,7 +641,7 @@ int osd_t::submit_primary_sync_subops(osd_op_t *cur_op) op_data->done = op_data->errors = op_data->errcode = 0; op_data->n_subops = n_osds; op_data->subops = subops; - std::map::iterator peer_it; + robin_hood::unordered_flat_map::iterator peer_it; for (int i = 0; i < n_osds; i++) { osd_num_t sync_osd = op_data->dirty_osds[i];