Trigger double autosync when switching PG state to prevent leaving garbage in non-immediate_commit clusters

src/osd/osd.h

@@ -150,7 +150,7 @@ class osd_t
     bool pg_config_applied = false;
     bool etcd_reporting_pg_state = false;
     bool etcd_reporting_stats = false;
-    int autosync_timer_id = -1, print_stats_timer_id = -1, slow_log_timer_id = -1;
+    int print_stats_timer_id = -1, slow_log_timer_id = -1;
 
     // peers and PGs
 
@@ -168,6 +168,8 @@ class osd_t
     object_id recovery_last_oid;
     int recovery_pg_done = 0, recovery_done = 0;
     osd_op_t *autosync_op = NULL;
+    int autosync_copies_to_delete = 0;
+    int autosync_timer_id = -1;
 
     // Scrubbing
     uint64_t scrub_nearest_ts = 0;

src/osd/osd_primary.cpp

@@ -645,6 +645,18 @@ void osd_t::remove_object_from_state(object_id & oid, pg_osd_set_state_t **objec
     {
         throw std::runtime_error("BUG: Invalid object state: "+std::to_string((*object_state)->state));
     }
+    if (changed && immediate_commit != IMMEDIATE_ALL)
+    {
+        // Trigger double automatic sync after changing PG state when we're running with fsyncs.
+        // First autosync commits all written objects and applies copies_to_delete_after_sync;
+        // Second autosync commits all deletions run by the first sync.
+        // Without it, rebalancing in a cluster without load may result in some small amount of
+        // garbage left on "extra" OSDs of the PG, because last deletions are not synced at all.
+        // FIXME: 1000% correct way is to switch PG state only after copies_to_delete_after_sync.
+        // But it's much more complicated.
+        unstable_write_count += autosync_writes;
+        autosync_copies_to_delete = 2;
+    }
     if (changed && report)
     {
         report_pg_state(pg);

src/osd/osd_primary_subops.cpp

@@ -9,6 +9,10 @@ void osd_t::autosync()
 {
     if (immediate_commit != IMMEDIATE_ALL && !autosync_op)
     {
+        if (autosync_copies_to_delete > 0)
+        {
+            autosync_copies_to_delete--;
+        }
         autosync_op = new osd_op_t();
         autosync_op->op_type = OSD_OP_IN;
         autosync_op->peer_fd = SELF_FD;
@@ -29,6 +33,11 @@ void osd_t::autosync()
             }
             delete autosync_op;
             autosync_op = NULL;
+            if (autosync_copies_to_delete > 0)
+            {
+                // Trigger the second "copies_to_delete" autosync
+                autosync();
+            }
         };
         exec_op(autosync_op);
     }

src/osd/osd_primary_sync.cpp

@@ -213,6 +213,15 @@ resume_8:
         {
             goto resume_6;
         }
+        if (immediate_commit == IMMEDIATE_NONE)
+        {
+            // Mark OSDs as dirty because deletions have to be synced too!
+            for (int i = 0; i < op_data->copies_to_delete_count; i++)
+            {
+                auto & chunk = op_data->copies_to_delete[i];
+                this->dirty_osds.insert(chunk.osd_num);
+            }
+        }
     }
     for (int i = 0; i < op_data->dirty_pg_count; i++)
     {
@@ -227,7 +236,7 @@ resume_8:
             start_pg_peering(pg);
         }
     }
-    // FIXME: Free those in the destructor?
+    // FIXME: Free those in the destructor (not here)?
     free(op_data->dirty_pgs);
     op_data->dirty_pgs = NULL;
     op_data->dirty_osds = NULL;