Decouple break_pg_locks from outbound OSD disconnections

src/osd/osd.cpp

@@ -85,6 +85,7 @@ osd_t::osd_t(const json11::Json & config, ring_loop_t *ringloop)
     msgr.ringloop = this->ringloop;
     msgr.exec_op = [this](osd_op_t *op) { exec_op(op); };
     msgr.repeer_pgs = [this](osd_num_t peer_osd) { repeer_pgs(peer_osd); };
+    msgr.break_pg_locks = [this](osd_num_t peer_osd) { break_pg_locks(peer_osd); };
     msgr.check_config_hook = [this](osd_client_t *cl, json11::Json conf) { return check_peer_config(cl, conf); };
     msgr.init();
 

src/osd/osd.h

@@ -279,6 +279,7 @@ class osd_t
     bool check_peer_config(osd_client_t *cl, json11::Json conf);
     void repeer_pgs(osd_num_t osd_num);
     void repeer_pg(pg_t & pg);
+    void break_pg_locks(osd_num_t osd_num);
     void start_pg_peering(pg_t & pg);
     void drop_dirty_pg_connections(pool_pg_num_t pg);
     void record_pg_lock(pg_t & pg, osd_num_t peer_osd, uint64_t pg_state);

src/osd/osd_peering.cpp

@@ -73,18 +73,25 @@ void osd_t::handle_peers()
     }
 }
 
+void osd_t::break_pg_locks(osd_num_t peer_osd)
+{
+    for (auto lock_it = pg_locks.begin(); lock_it != pg_locks.end(); )
+    {
+        if (lock_it->second.primary_osd == peer_osd)
+        {
+            if (log_level > 3)
+            {
+                printf("Break PG %u/%u lock on disconnection of OSD %ju\n", lock_it->first.pool_id, lock_it->first.pg_num, peer_osd);
+            }
+            pg_locks.erase(lock_it++);
+        }
+        else
+            lock_it++;
+    }
+}
+
 void osd_t::repeer_pgs(osd_num_t peer_osd)
 {
-    if (msgr.osd_peer_fds.find(peer_osd) == msgr.osd_peer_fds.end())
-    {
-        for (auto lock_it = pg_locks.begin(); lock_it != pg_locks.end(); )
-        {
-            if (lock_it->second.primary_osd == peer_osd)
-                pg_locks.erase(lock_it++);
-            else
-                lock_it++;
-        }
-    }
     // Re-peer affected PGs
     for (auto & p: pgs)
     {
@@ -471,6 +478,7 @@ void osd_t::relock_pg(pg_t & pg)
             auto pg_it = pgs.find(pg_id);
             if (pg_it == pgs.end())
             {
+                printf("Warning: PG %u/%u is gone during lock attempt\n", pg_id.pool_id, pg_id.pg_num);
                 return;
             }
             auto & pg = pg_it->second;

src/osd/osd_secondary.cpp

@@ -304,6 +304,10 @@ void osd_t::exec_sec_lock(osd_op_t *cur_op)
             finish_op(cur_op, -EBUSY);
             return;
         }
+        if (log_level > 3)
+        {
+            printf("Lock PG %u/%u for OSD %ju\n", ppg.pool_id, ppg.pg_num, cl->in_osd_num);
+        }
         pg_locks[ppg] = (osd_pg_lock_t){
             .primary_osd = cl->in_osd_num,
             .state = cur_op->req.sec_lock.pg_state,
@@ -311,6 +315,10 @@ void osd_t::exec_sec_lock(osd_op_t *cur_op)
     }
     else if (lock_it != pg_locks.end() && lock_it->second.primary_osd == cl->in_osd_num)
     {
+        if (log_level > 3)
+        {
+            printf("Unlock PG %u/%u by OSD %ju\n", ppg.pool_id, ppg.pg_num, cl->in_osd_num);
+        }
         pg_locks.erase(lock_it);
     }
     finish_op(cur_op, 0);