Add a regression test for LOC_CORRUPTED cleared on second scrub

2026-06-13 19:56:06 +03:00
parent 91698404a7
commit 213f76c66c
2 changed files with 157 additions and 2 deletions
@@ -19,6 +19,8 @@
 // for without having to drive the rest of the startup sequence.
 void test_load_global_config()
 {
+    printf("test_load_global_config\n");
+
    osd_test_fixture_t f;
    f.st_cli->pause();
    f.start(json11::Json::object {
@@ -64,6 +66,8 @@ static osd_op_t *make_write_op(inode_t inode, uint64_t offset, uint64_t len, uin
 // Layout: pool 1, replicated x2, primary = OSD 1 (us), secondary = OSD 2.
 void test_replicated_write()
 {
+    printf("test_replicated_write\n");
+
    osd_test_fixture_t f;
    f.configure_replicated_pool(/*pool_id*/ 1, /*pg_size*/ 2, /*pg_minsize*/ 1, /*pg_count*/ 1,
        { { 1, 2 } });
@@ -141,9 +145,160 @@ void test_replicated_write()
    printf("test_replicated_write passed\n");
 }

+// Regression test for a bug where a second scrub on a still-failing replica
+// (already marked LOC_CORRUPTED) would clear the corruption flag.
+void test_scrub_corruption_persists()
+{
+    printf("test_scrub_corruption_persists\n");
+
+    osd_test_fixture_t f;
+    f.configure_replicated_pool(/*pool_id*/ 1, /*pg_size*/ 2, /*pg_minsize*/ 1, /*pg_count*/ 1,
+        { { 1, 2 } });
+    f.start(json11::Json::object {
+        { "osd_num", 1 },
+        { "etcd_address", "127.0.0.1:2379" },
+        { "immediate_commit", "all" },
+        { "block_size", 131072 },
+        { "bitmap_granularity", 4096 },
+    });
+    f.connect_peer(2);
+    f.complete_peering_empty();
+    assert(f.pg(1, 1).state & PG_ACTIVE);
+
+    // ---- Write one object so there is something to scrub ----
+    inode_t inode = INODE_WITH_POOL(1, 1);
+    auto *write_op = make_write_op(inode, 0, 4096, 0xab);
+    int final_retval = -1;
+    write_op->callback = [&final_retval](osd_op_t *op) {
+        final_retval = op->reply.hdr.retval;
+    };
+    f.exec(write_op);
+
+    // Stage 1: zero-length read for version resolution
+    assert(f.bs->queued.size() == 1);
+    auto *zero_read = f.bs->take();
+    assert(zero_read->opcode == BS_OP_READ);
+    assert(zero_read->len == 0);
+    zero_read->version = 0;
+    zero_read->retval = 0;
+    zero_read->callback(zero_read);
+
+    // Stage 2: local write + peer write
+    assert(f.bs->queued.size() == 1);
+    assert(f.peer(2)->sent_ops.size() == 1);
+    auto *local_write = f.bs->take();
+    auto *peer_write = f.peer_take(2, OSD_OP_SEC_WRITE_STABLE);
+
+    // Complete both
+    local_write->retval = local_write->len;
+    local_write->callback(local_write);
+    assert(final_retval == -1); // still waiting for peer
+    peer_write->reply.hdr.retval = peer_write->req.sec_rw.len;
+    peer_write->reply.sec_rw.version = 1;
+    peer_write->callback(peer_write);
+
+    assert(final_retval == 4096);
+    assert(f.pg(1, 1).inflight == 0);
+    assert(f.pg(1, 1).write_queue.empty());
+    delete write_op;
+
+    object_id oid = { .inode = inode, .stripe = 0 };
+
+    // ============ SCRUB 1 — peer returns -EIO ============
+
+    auto *scrub1 = new osd_op_t();
+    scrub1->op_type = OSD_OP_IN;
+    scrub1->client_id = 0;
+    scrub1->req.rw.header.magic = SECONDARY_OSD_OP_MAGIC;
+    scrub1->req.rw.header.id = 2;
+    scrub1->req.rw.header.opcode = OSD_OP_SCRUB;
+    scrub1->req.rw.inode = inode;
+    scrub1->req.rw.offset = 0;
+    scrub1->req.rw.len = 0;
+
+    int scrub1_retval = -1;
+    scrub1->callback = [&scrub1_retval](osd_op_t *op) {
+        scrub1_retval = op->reply.hdr.retval;
+    };
+    f.exec(scrub1);
+
+    // submit_scrub_subops submitted local BS_OP_READ + peer OSD_OP_SEC_READ
+    assert(f.bs->queued.size() == 1);
+    auto *local_r1 = f.bs->take(BS_OP_READ);
+    memset(local_r1->buf, 0xab, local_r1->len);
+    local_r1->retval = local_r1->len;
+    local_r1->version = 1;
+    local_r1->callback(local_r1);
+
+    auto *peer_r1 = f.peer_take(2, OSD_OP_SEC_READ);
+    peer_r1->reply.hdr.retval = -EIO;
+    peer_r1->callback(peer_r1);
+
+    assert(scrub1_retval == 0);
+
+    // Verify: OSD 2 chunk is now LOC_CORRUPTED
+    auto check_corrupted = [&]()
+    {
+        if (!(f.pg(1, 1).state & PG_HAS_CORRUPTED))
+            return false;
+        auto st_it = f.pg(1, 1).degraded_objects.find(oid);
+        if (st_it == f.pg(1, 1).degraded_objects.end())
+            return false;
+        for (auto & chunk : st_it->second->osd_set)
+            if (chunk.osd_num == 2 && (chunk.loc_bad & LOC_CORRUPTED))
+                return true;
+        return false;
+    };
+    bool osd2_corrupted = check_corrupted();
+    assert(osd2_corrupted);
+    printf("test_scrub_corruption_persists: scrub 1 -> LOC_CORRUPTED set\n");
+    delete scrub1;
+
+    // ============ SCRUB 2 — peer still returns -EIO ============
+
+    auto *scrub2 = new osd_op_t();
+    scrub2->op_type = OSD_OP_IN;
+    scrub2->client_id = 0;
+    scrub2->req.rw.header.magic = SECONDARY_OSD_OP_MAGIC;
+    scrub2->req.rw.header.id = 3;
+    scrub2->req.rw.header.opcode = OSD_OP_SCRUB;
+    scrub2->req.rw.inode = inode;
+    scrub2->req.rw.offset = 0;
+    scrub2->req.rw.len = 0;
+
+    int scrub2_retval = -1;
+    scrub2->callback = [&scrub2_retval](osd_op_t *op) {
+        scrub2_retval = op->reply.hdr.retval;
+    };
+    f.exec(scrub2);
+
+    assert(f.bs->queued.size() == 1);
+    auto *local_r2 = f.bs->take(BS_OP_READ);
+    memset(local_r2->buf, 0xab, local_r2->len);
+    local_r2->retval = local_r2->len;
+    local_r2->version = 1;
+    local_r2->callback(local_r2);
+
+    auto *peer_r2 = f.peer_take(2, OSD_OP_SEC_READ);
+    peer_r2->reply.hdr.retval = -EIO;
+    peer_r2->callback(peer_r2);
+
+    assert(scrub2_retval == 0);
+
+    // THE KEY ASSERTION — LOC_CORRUPTED must persist
+    osd2_corrupted = check_corrupted();
+    if (!osd2_corrupted)
+        printf("BUG: LOC_CORRUPTED cleared on second scrub\n");
+    assert(osd2_corrupted);
+
+    delete scrub2;
+    printf("test_scrub_corruption_persists passed\n");
+}
+
 int main(int narg, char *args[])
 {
    test_load_global_config();
    test_replicated_write();
+    test_scrub_corruption_persists();
    return 0;
 }
@@ -185,11 +185,11 @@ struct osd_test_fixture_t
            auto *op = *it;
            if (op->opcode == BS_OP_LIST)
            {
-                it = bs->queued.erase(it);
+                it = bs->queued.erase(it++);
                op->retval = 0;
                op->version = 0;
                op->buf = NULL;
-                op->callback(op);
+                std::function<void(blockstore_op_t*)>(op->callback)(op);
            }
            else
                ++it;