Return left_on_dead OSD list in DELETE replies and use it in rm-data

src/osd/osd_peering.cpp

@@ -133,6 +133,7 @@ void osd_t::repeer_pgs(osd_num_t peer_osd)
 void osd_t::reset_pg(pg_t & pg)
 {
     pg.cur_peers.clear();
+    pg.dead_peers.clear();
     pg.state_dict.clear();
     copies_to_delete_after_sync_count -= pg.copies_to_delete_after_sync.size();
     pg.copies_to_delete_after_sync.clear();
@@ -235,13 +236,16 @@ void osd_t::start_pg_peering(pg_t & pg)
         return;
     }
     std::set<osd_num_t> cur_peers;
+    std::set<osd_num_t> dead_peers;
     for (auto pg_osd: pg.all_peers)
     {
         if (pg_osd == this->osd_num || msgr.osd_peer_fds.find(pg_osd) != msgr.osd_peer_fds.end())
-        {
             cur_peers.insert(pg_osd);
-        }
+        else
+            dead_peers.insert(pg_osd);
     }
+    pg.cur_peers.insert(pg.cur_peers.begin(), cur_peers.begin(), cur_peers.end());
+    pg.dead_peers.insert(pg.dead_peers.begin(), dead_peers.begin(), dead_peers.end());
     if (pg.target_history.size())
     {
         // Refuse to start PG if no peers are available from any of the historical OSD sets
@@ -269,7 +273,6 @@ void osd_t::start_pg_peering(pg_t & pg)
             }
         }
     }
-    pg.cur_peers.insert(pg.cur_peers.begin(), cur_peers.begin(), cur_peers.end());
     if (pg.peering_state)
     {
         // Adjust the peering operation that's still in progress - discard unneeded results

src/osd/osd_peering_pg.h

@@ -97,6 +97,8 @@ struct pg_t
     bool history_changed = false;
     // peer list from the last peering event
     std::vector<osd_num_t> cur_peers;
+    // dead_peers = all_peers - cur_peers
+    std::vector<osd_num_t> dead_peers;
     // target_set is the "correct" peer OSD set for this PG
     std::vector<osd_num_t> target_set;
     // cur_set is the current set of connected peer OSDs for this PG

src/osd/osd_primary.cpp

@@ -783,6 +783,20 @@ resume_5:
     }
     pg.total_count--;
     cur_op->reply.hdr.retval = 0;
+    // indicate possibly unfinished (left_on_dead) deletions
+    cur_op->reply.del.flags = OSD_DEL_SUPPORT_LEFT_ON_DEAD;
+    if (pg.dead_peers.size() > 0)
+    {
+        int max_del = (OSD_PACKET_SIZE-sizeof(cur_op->reply.del)) / sizeof(uint32_t);
+        cur_op->reply.del.flags |= OSD_DEL_LEFT_ON_DEAD;
+        cur_op->reply.del.left_on_dead_count = pg.dead_peers.size() < max_del
+            ? pg.dead_peers.size() : max_del;
+        uint32_t *left_on_dead = (uint32_t*)((&cur_op->reply.del) + 1);
+        for (int i = 0; i < cur_op->reply.del.left_on_dead_count; i++)
+        {
+            left_on_dead[i] = pg.dead_peers[i];
+        }
+    }
 continue_others:
     osd_op_t *next_op = NULL;
     auto next_it = pg.write_queue.find(op_data->oid);