Lock PGs on secondary OSDs to allow local reads and guarantee splitbrain prevention

src/client/messenger.cpp

@@ -773,12 +773,15 @@ void osd_messenger_t::accept_connections(int listen_fd)
         fcntl(peer_fd, F_SETFL, fcntl(peer_fd, F_GETFL, 0) | O_NONBLOCK);
         int one = 1;
         setsockopt(peer_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
-        clients[peer_fd] = new osd_client_t();
-        clients[peer_fd]->peer_addr = addr;
-        clients[peer_fd]->peer_port = ntohs(((sockaddr_in*)&addr)->sin_port);
-        clients[peer_fd]->peer_fd = peer_fd;
-        clients[peer_fd]->peer_state = PEER_CONNECTED;
-        clients[peer_fd]->in_buf = malloc_or_die(receive_buffer_size);
+        auto cl = new osd_client_t();
+        clients[peer_fd] = cl;
+        cl->is_incoming = true;
+        cl->peer_addr = addr;
+        cl->peer_addr = addr;
+        cl->peer_port = ntohs(((sockaddr_in*)&addr)->sin_port);
+        cl->peer_fd = peer_fd;
+        cl->peer_state = PEER_CONNECTED;
+        cl->in_buf = malloc_or_die(receive_buffer_size);
         // Add FD to epoll
         tfd->set_fd_handler(peer_fd, false, [this](int peer_fd, int epoll_events)
         {

src/client/messenger.h

@@ -60,6 +60,7 @@ struct osd_client_t
     int ping_time_remaining = 0;
     int idle_time_remaining = 0;
     osd_num_t osd_num = 0;
+    bool is_incoming = false;
 
     void *in_buf = NULL;
 
@@ -77,6 +78,7 @@ struct osd_client_t
     osd_op_buf_list_t recv_list;
     uint64_t read_op_id = 1;
     bool check_sequencing = false;
+    bool enable_pg_locks = false;
 
     // Incoming operations
     std::vector<osd_op_t*> received_ops;

src/client/msgr_rdmacm.cpp

@@ -510,6 +510,7 @@ void osd_messenger_t::rdmacm_established(rdma_cm_event *ev)
     rc->qp = conn->cmid->qp;
     // And an osd_client_t
     auto cl = new osd_client_t();
+    cl->is_incoming = true;
     cl->peer_addr = conn->parsed_addr;
     cl->peer_port = conn->rdmacm_port;
     cl->peer_fd = conn->peer_fd;

src/client/osd_ops.cpp

@@ -23,4 +23,5 @@ const char* osd_op_names[] = {
     "sec_read_bmp",
     "scrub",
     "describe",
+    "sec_lock",
 };

src/client/osd_ops.h

@@ -31,10 +31,13 @@
 #define OSD_OP_SEC_READ_BMP         16
 #define OSD_OP_SCRUB                17
 #define OSD_OP_DESCRIBE             18
-#define OSD_OP_MAX                  18
+#define OSD_OP_SEC_LOCK             19
+#define OSD_OP_MAX                  19
 #define OSD_RW_MAX                  64*1024*1024
 #define OSD_PROTOCOL_VERSION        1
+
 #define OSD_OP_RECOVERY_RELATED     (uint32_t)1
+#define OSD_OP_IGNORE_PG_LOCK       (uint32_t)2
 
 // Memory alignment for direct I/O (usually 512 bytes)
 #ifndef DIRECT_IO_ALIGNMENT
@@ -56,6 +59,9 @@
 #define OSD_DEL_SUPPORT_LEFT_ON_DEAD 1
 #define OSD_DEL_LEFT_ON_DEAD         2
 
+#define OSD_SEC_LOCK_PG 1
+#define OSD_SEC_UNLOCK_PG 2
+
 // common request and reply headers
 struct __attribute__((__packed__)) osd_op_header_t
 {
@@ -94,7 +100,7 @@ struct __attribute__((__packed__)) osd_op_sec_rw_t
     uint32_t len;
     // bitmap/attribute length - bitmap comes after header, but before data
     uint32_t attr_len;
-    // the only possible flag is OSD_OP_RECOVERY_RELATED
+    // OSD_OP_RECOVERY_RELATED, OSD_OP_IGNORE_PG_LOCK
     uint32_t flags;
 };
 
@@ -116,7 +122,7 @@ struct __attribute__((__packed__)) osd_op_sec_del_t
     object_id oid;
     // delete version (automatic or specific)
     uint64_t version;
-    // the only possible flag is OSD_OP_RECOVERY_RELATED
+    // OSD_OP_RECOVERY_RELATED, OSD_OP_IGNORE_PG_LOCK
     uint32_t flags;
     uint32_t pad0;
 };
@@ -131,7 +137,7 @@ struct __attribute__((__packed__)) osd_reply_sec_del_t
 struct __attribute__((__packed__)) osd_op_sec_sync_t
 {
     osd_op_header_t header;
-    // the only possible flag is OSD_OP_RECOVERY_RELATED
+    // OSD_OP_RECOVERY_RELATED, OSD_OP_IGNORE_PG_LOCK
     uint32_t flags;
     uint32_t pad0;
 };
@@ -147,7 +153,7 @@ struct __attribute__((__packed__)) osd_op_sec_stab_t
     osd_op_header_t header;
     // obj_ver_id array length in bytes
     uint64_t len;
-    // the only possible flag is OSD_OP_RECOVERY_RELATED
+    // OSD_OP_RECOVERY_RELATED, OSD_OP_IGNORE_PG_LOCK
     uint32_t flags;
     uint32_t pad0;
 };
@@ -165,6 +171,8 @@ struct __attribute__((__packed__)) osd_op_sec_read_bmp_t
     osd_op_header_t header;
     // obj_ver_id array length in bytes
     uint64_t len;
+    // OSD_OP_RECOVERY_RELATED, OSD_OP_IGNORE_PG_LOCK
+    uint32_t flags;
 };
 
 struct __attribute__((__packed__)) osd_reply_sec_read_bmp_t
@@ -173,7 +181,7 @@ struct __attribute__((__packed__)) osd_reply_sec_read_bmp_t
     osd_reply_header_t header;
 };
 
-// show configuration
+// show configuration and remember peer information
 struct __attribute__((__packed__)) osd_op_show_config_t
 {
     osd_op_header_t header;
@@ -303,6 +311,25 @@ struct __attribute__((__packed__)) osd_reply_describe_item_t
     osd_num_t osd_num;  // OSD number
 };
 
+// lock/unlock PG for use by a primary OSD
+struct __attribute__((__packed__)) osd_op_sec_lock_t
+{
+    osd_op_header_t header;
+    // OSD_SEC_LOCK_PG or OSD_SEC_UNLOCK_PG
+    uint64_t flags;
+    // Pool ID and PG number
+    uint64_t pool_id;
+    uint64_t pg_num;
+    // PG state as calculated by the primary OSD
+    uint64_t pg_state;
+};
+
+struct __attribute__((__packed__)) osd_reply_sec_lock_t
+{
+    osd_reply_header_t header;
+    uint64_t cur_primary;
+};
+
 // FIXME it would be interesting to try to unify blockstore_op and osd_op formats
 union osd_any_op_t
 {
@@ -313,6 +340,7 @@ union osd_any_op_t
     osd_op_sec_stab_t sec_stab;
     osd_op_sec_read_bmp_t sec_read_bmp;
     osd_op_sec_list_t sec_list;
+    osd_op_sec_lock_t sec_lock;
     osd_op_show_config_t show_conf;
     osd_op_rw_t rw;
     osd_op_sync_t sync;
@@ -329,6 +357,7 @@ union osd_any_reply_t
     osd_reply_sec_stab_t sec_stab;
     osd_reply_sec_read_bmp_t sec_read_bmp;
     osd_reply_sec_list_t sec_list;
+    osd_reply_sec_lock_t sec_lock;
     osd_reply_show_config_t show_conf;
     osd_reply_rw_t rw;
     osd_reply_del_t del;