diff --git a/src/osd/osd.h b/src/osd/osd.h index 66015e2d..19351b08 100644 --- a/src/osd/osd.h +++ b/src/osd/osd.h @@ -150,7 +150,7 @@ class osd_t bool pg_config_applied = false; bool etcd_reporting_pg_state = false; bool etcd_reporting_stats = false; - int autosync_timer_id = -1, print_stats_timer_id = -1, slow_log_timer_id = -1; + int print_stats_timer_id = -1, slow_log_timer_id = -1; // peers and PGs @@ -168,6 +168,8 @@ class osd_t object_id recovery_last_oid; int recovery_pg_done = 0, recovery_done = 0; osd_op_t *autosync_op = NULL; + int autosync_copies_to_delete = 0; + int autosync_timer_id = -1; // Scrubbing uint64_t scrub_nearest_ts = 0; diff --git a/src/osd/osd_primary.cpp b/src/osd/osd_primary.cpp index c5a11325..69f57a60 100644 --- a/src/osd/osd_primary.cpp +++ b/src/osd/osd_primary.cpp @@ -645,6 +645,18 @@ void osd_t::remove_object_from_state(object_id & oid, pg_osd_set_state_t **objec { throw std::runtime_error("BUG: Invalid object state: "+std::to_string((*object_state)->state)); } + if (changed && immediate_commit != IMMEDIATE_ALL) + { + // Trigger double automatic sync after changing PG state when we're running with fsyncs. + // First autosync commits all written objects and applies copies_to_delete_after_sync; + // Second autosync commits all deletions run by the first sync. + // Without it, rebalancing in a cluster without load may result in some small amount of + // garbage left on "extra" OSDs of the PG, because last deletions are not synced at all. + // FIXME: 1000% correct way is to switch PG state only after copies_to_delete_after_sync. + // But it's much more complicated. + unstable_write_count += autosync_writes; + autosync_copies_to_delete = 2; + } if (changed && report) { report_pg_state(pg); diff --git a/src/osd/osd_primary_subops.cpp b/src/osd/osd_primary_subops.cpp index d4d562fb..b09103a3 100644 --- a/src/osd/osd_primary_subops.cpp +++ b/src/osd/osd_primary_subops.cpp @@ -9,6 +9,10 @@ void osd_t::autosync() { if (immediate_commit != IMMEDIATE_ALL && !autosync_op) { + if (autosync_copies_to_delete > 0) + { + autosync_copies_to_delete--; + } autosync_op = new osd_op_t(); autosync_op->op_type = OSD_OP_IN; autosync_op->peer_fd = SELF_FD; @@ -29,6 +33,11 @@ void osd_t::autosync() } delete autosync_op; autosync_op = NULL; + if (autosync_copies_to_delete > 0) + { + // Trigger the second "copies_to_delete" autosync + autosync(); + } }; exec_op(autosync_op); } diff --git a/src/osd/osd_primary_sync.cpp b/src/osd/osd_primary_sync.cpp index 61d55971..0808773b 100644 --- a/src/osd/osd_primary_sync.cpp +++ b/src/osd/osd_primary_sync.cpp @@ -213,6 +213,15 @@ resume_8: { goto resume_6; } + if (immediate_commit == IMMEDIATE_NONE) + { + // Mark OSDs as dirty because deletions have to be synced too! + for (int i = 0; i < op_data->copies_to_delete_count; i++) + { + auto & chunk = op_data->copies_to_delete[i]; + this->dirty_osds.insert(chunk.osd_num); + } + } } for (int i = 0; i < op_data->dirty_pg_count; i++) { @@ -227,7 +236,7 @@ resume_8: start_pg_peering(pg); } } - // FIXME: Free those in the destructor? + // FIXME: Free those in the destructor (not here)? free(op_data->dirty_pgs); op_data->dirty_pgs = NULL; op_data->dirty_osds = NULL;