Support primary-reads from clean replicated PGs on secondary OSDs

src/osd/osd.h

@@ -330,6 +330,7 @@ class osd_t
     bool prepare_primary_rw(osd_op_t *cur_op);
     void continue_primary_read(osd_op_t *cur_op);
     void continue_primary_scrub(osd_op_t *cur_op);
+    void continue_local_secondary_read(osd_op_t *cur_op);
     void continue_primary_describe(osd_op_t *cur_op);
     void continue_primary_list(osd_op_t *cur_op);
     void continue_primary_write(osd_op_t *cur_op);
@@ -367,13 +368,13 @@ class osd_t
     uint64_t* get_object_osd_set(pg_t &pg, object_id &oid, pg_osd_set_state_t **object_state);
 
     void continue_chained_read(osd_op_t *cur_op);
-    int submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op);
+    int submit_chained_read_requests(pg_t *pg, osd_op_t *cur_op);
     void check_corrupted_chained(pg_t & pg, osd_op_t *cur_op);
-    void send_chained_read_results(pg_t & pg, osd_op_t *cur_op);
+    void send_chained_read_results(pg_t *pg, osd_op_t *cur_op);
     std::vector<osd_chain_read_t> collect_chained_read_requests(osd_op_t *cur_op);
     int collect_bitmap_requests(osd_op_t *cur_op, pg_t & pg, std::vector<bitmap_request_t> & bitmap_requests);
     int submit_bitmap_subops(osd_op_t *cur_op, pg_t & pg);
-    int read_bitmaps(osd_op_t *cur_op, pg_t & pg, int base_state);
+    int read_bitmaps(osd_op_t *cur_op, pg_t *pg, int base_state);
 
     inline pg_num_t map_to_pg(object_id oid, uint64_t pg_stripe_size)
     {

src/osd/osd_primary.cpp

@@ -37,7 +37,20 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
     };
     pg_num_t pg_num = (oid.stripe/pool_cfg.pg_stripe_size) % pg_counts[pool_id] + 1; // like map_to_pg()
     auto pg_it = pgs.find({ .pool_id = pool_id, .pg_num = pg_num });
-    if (pg_it == pgs.end() || !(pg_it->second.state & PG_ACTIVE))
+    if (pg_it == pgs.end() || pg_it->second.state == PG_OFFLINE)
+    {
+        // Check for a local replicated read from secondary OSD
+        auto lock_it = cur_op->req.hdr.opcode == OSD_OP_READ && pool_cfg.scheme == POOL_SCHEME_REPLICATED
+            ? pg_locks.find({ .pool_id = pool_id, .pg_num = pg_num })
+            : pg_locks.end();
+        if (lock_it == pg_locks.end() || lock_it->second.state != PG_ACTIVE && lock_it->second.state != (PG_ACTIVE|PG_LEFT_ON_DEAD))
+        {
+            // FIXME: Change EPIPE to something else
+            finish_op(cur_op, -EPIPE);
+            return false;
+        }
+    }
+    else if (!(pg_it->second.state & PG_ACTIVE))
     {
         // This OSD is not primary for this PG or the PG is inactive
         // FIXME: Allow reads from PGs degraded under pg_minsize, but don't allow writes
@@ -69,7 +82,7 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
         // Find parents from the same pool. Optimized reads only work within pools
         while (inode_it != st_cli.inode_config.end() &&
             inode_it->second.parent_id &&
-            INODE_POOL(inode_it->second.parent_id) == pg_it->second.pool_id)
+            INODE_POOL(inode_it->second.parent_id) == pool_cfg.id)
         {
             // Check for loops - FIXME check it in etcd_state_client
             if (inode_it->second.parent_id == cur_op->req.rw.inode ||
@@ -109,7 +122,7 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
     );
     void *data_buf = (uint8_t*)op_data + sizeof(osd_primary_op_data_t);
     op_data->pg_num = pg_num;
-    op_data->pg = &pg_it->second;
+    op_data->pg = pg_it == pgs.end() ? NULL : &pg_it->second;
     op_data->oid = oid;
     op_data->stripes = (osd_rmw_stripe_t*)data_buf;
     op_data->stripe_count = stripe_count;
@@ -144,7 +157,7 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
         chain_num++;
         auto inode_it = st_cli.inode_config.find(cur_op->req.rw.inode);
         while (inode_it != st_cli.inode_config.end() && inode_it->second.parent_id &&
-            INODE_POOL(inode_it->second.parent_id) == pg_it->second.pool_id &&
+            INODE_POOL(inode_it->second.parent_id) == pool_cfg.id &&
             // Check for loops
             inode_it->second.parent_id != cur_op->req.rw.inode)
         {
@@ -154,7 +167,10 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
             chain_num++;
         }
     }
-    pg_it->second.inflight++;
+    if (op_data->pg)
+    {
+        op_data->pg->inflight++;
+    }
     return true;
 }
 
@@ -194,6 +210,7 @@ void osd_t::continue_primary_read(osd_op_t *cur_op)
         return;
     }
     osd_primary_op_data_t *op_data = cur_op->op_data;
+    pg_t *pg = op_data->pg;
     if (op_data->chain_size)
     {
         continue_chained_read(cur_op);
@@ -206,11 +223,10 @@ void osd_t::continue_primary_read(osd_op_t *cur_op)
 resume_0:
     cur_op->reply.rw.bitmap_len = 0;
     {
-        auto & pg = *op_data->pg;
         if (cur_op->req.rw.len == 0)
         {
             // len=0 => bitmap read
-            for (int role = 0; role < pg.pg_data_size; role++)
+            for (int role = 0; role < (pg ? pg->pg_data_size : 1); role++)
             {
                 op_data->stripes[role].read_start = 0;
                 op_data->stripes[role].read_end = UINT32_MAX;
@@ -218,40 +234,48 @@ resume_0:
         }
         else
         {
-            for (int role = 0; role < pg.pg_data_size; role++)
+            for (int role = 0; role < (pg ? pg->pg_data_size : 1); role++)
             {
                 op_data->stripes[role].read_start = op_data->stripes[role].req_start;
                 op_data->stripes[role].read_end = op_data->stripes[role].req_end;
             }
         }
         // Determine version
-        auto vo_it = pg.ver_override.find(op_data->oid);
+        if (pg)
-        op_data->target_ver = vo_it != pg.ver_override.end() ? vo_it->second : UINT64_MAX;
+        {
-        // PG may have degraded or misplaced objects
+            auto vo_it = pg->ver_override.find(op_data->oid);
-        op_data->prev_set = get_object_osd_set(pg, op_data->oid, &op_data->object_state);
+            op_data->target_ver = vo_it != pg->ver_override.end() ? vo_it->second : UINT64_MAX;
-        if (pg.state == PG_ACTIVE || pg.scheme == POOL_SCHEME_REPLICATED)
+            // PG may have degraded or misplaced objects
+            op_data->prev_set = get_object_osd_set(*pg, op_data->oid, &op_data->object_state);
+        }
+        else
+        {
+            op_data->target_ver = UINT64_MAX;
+            op_data->prev_set = &this->osd_num;
+        }
+        if (!pg || pg->state == PG_ACTIVE || pg->scheme == POOL_SCHEME_REPLICATED)
         {
             // Fast happy-path
-            if (pg.scheme == POOL_SCHEME_REPLICATED &&
+            if (pg && pg->scheme == POOL_SCHEME_REPLICATED &&
                 op_data->object_state && (op_data->object_state->state & OBJ_INCOMPLETE))
             {
                 finish_op(cur_op, -EIO);
                 return;
             }
-            cur_op->buf = alloc_read_buffer(op_data->stripes, pg.pg_data_size, 0);
+            cur_op->buf = alloc_read_buffer(op_data->stripes, pg ? pg->pg_data_size : 1, 0);
             submit_primary_subops(SUBMIT_RMW_READ, op_data->target_ver, op_data->prev_set, cur_op);
             op_data->st = 1;
         }
         else
         {
-            if (extend_missing_stripes(op_data->stripes, op_data->prev_set, pg.pg_data_size, pg.pg_size) < 0)
+            if (extend_missing_stripes(op_data->stripes, op_data->prev_set, pg->pg_data_size, pg->pg_size) < 0)
             {
                 finish_op(cur_op, -EIO);
                 return;
             }
             // Submit reads
             op_data->degraded = 1;
-            cur_op->buf = alloc_read_buffer(op_data->stripes, pg.pg_size, 0);
+            cur_op->buf = alloc_read_buffer(op_data->stripes, pg->pg_size, 0);
             submit_primary_subops(SUBMIT_RMW_READ, op_data->target_ver, op_data->prev_set, cur_op);
             op_data->st = 1;
         }
@@ -261,32 +285,32 @@ resume_1:
 resume_2:
     if (op_data->errors > 0)
     {
-        if (op_data->errcode == -EIO || op_data->errcode == -EDOM)
+        if (pg && (op_data->errcode == -EIO || op_data->errcode == -EDOM))
         {
             // I/O or checksum error
             // FIXME: ref = true ideally... because new_state != state is not necessarily true if it's freed and recreated
-            op_data->object_state = mark_object_corrupted(*op_data->pg, op_data->oid, op_data->object_state, op_data->stripes, false);
+            op_data->object_state = mark_object_corrupted(*pg, op_data->oid, op_data->object_state, op_data->stripes, false);
             goto resume_0;
         }
         finish_op(cur_op, op_data->errcode);
         return;
     }
     cur_op->reply.rw.version = op_data->fact_ver;
-    cur_op->reply.rw.bitmap_len = op_data->pg->pg_data_size * clean_entry_bitmap_size;
+    cur_op->reply.rw.bitmap_len = (pg ? pg->pg_data_size : 1) * clean_entry_bitmap_size;
     if (op_data->degraded)
     {
         // Reconstruct missing stripes
         osd_rmw_stripe_t *stripes = op_data->stripes;
-        if (op_data->pg->scheme == POOL_SCHEME_XOR)
+        if (pg->scheme == POOL_SCHEME_XOR)
         {
-            reconstruct_stripes_xor(stripes, op_data->pg->pg_size, clean_entry_bitmap_size);
+            reconstruct_stripes_xor(stripes, pg->pg_size, clean_entry_bitmap_size);
         }
-        else if (op_data->pg->scheme == POOL_SCHEME_EC)
+        else if (pg->scheme == POOL_SCHEME_EC)
         {
-            reconstruct_stripes_ec(stripes, op_data->pg->pg_size, op_data->pg->pg_data_size, clean_entry_bitmap_size);
+            reconstruct_stripes_ec(stripes, pg->pg_size, pg->pg_data_size, clean_entry_bitmap_size);
         }
         cur_op->iov.push_back(op_data->stripes[0].bmp_buf, cur_op->reply.rw.bitmap_len);
-        for (int role = 0; role < op_data->pg->pg_size; role++)
+        for (int role = 0; role < pg->pg_size; role++)
         {
             if (stripes[role].req_end != 0)
             {

src/osd/osd_primary_chain.cpp

@@ -7,7 +7,7 @@
 void osd_t::continue_chained_read(osd_op_t *cur_op)
 {
     osd_primary_op_data_t *op_data = cur_op->op_data;
-    auto & pg = *op_data->pg;
+    auto pg = op_data->pg;
     if (op_data->st == 1)
         goto resume_1;
     else if (op_data->st == 2)
@@ -17,7 +17,7 @@ void osd_t::continue_chained_read(osd_op_t *cur_op)
     else if (op_data->st == 4)
         goto resume_4;
     cur_op->reply.rw.bitmap_len = 0;
-    for (int role = 0; role < pg.pg_data_size; role++)
+    for (int role = 0; role < (pg ? pg->pg_data_size : 1); role++)
     {
         op_data->stripes[role].read_start = op_data->stripes[role].req_start;
         op_data->stripes[role].read_end = op_data->stripes[role].req_end;
@@ -40,10 +40,10 @@ resume_3:
 resume_4:
     if (op_data->errors > 0)
     {
-        if (op_data->errcode == -EIO || op_data->errcode == -EDOM)
+        if (pg && (op_data->errcode == -EIO || op_data->errcode == -EDOM))
         {
             // Handle corrupted reads and retry...
-            check_corrupted_chained(pg, cur_op);
+            check_corrupted_chained(*pg, cur_op);
             free(cur_op->buf);
             cur_op->buf = NULL;
             free(op_data->chain_reads);
@@ -63,31 +63,30 @@ resume_4:
     finish_op(cur_op, cur_op->req.rw.len);
 }
 
-int osd_t::read_bitmaps(osd_op_t *cur_op, pg_t & pg, int base_state)
+int osd_t::read_bitmaps(osd_op_t *cur_op, pg_t *pg, int base_state)
 {
     osd_primary_op_data_t *op_data = cur_op->op_data;
     if (op_data->st == base_state)
         goto resume_0;
     else if (op_data->st == base_state+1)
         goto resume_1;
-    if (pg.state == PG_ACTIVE && pg.scheme == POOL_SCHEME_REPLICATED)
+    if (!pg || pg->state == PG_ACTIVE && pg->scheme == POOL_SCHEME_REPLICATED)
     {
         // Happy path for clean replicated PGs (all bitmaps are available locally)
+        osd_primary_op_data_t *op_data = cur_op->op_data;
         for (int chain_num = 0; chain_num < op_data->chain_size; chain_num++)
         {
             object_id cur_oid = { .inode = op_data->read_chain[chain_num], .stripe = op_data->oid.stripe };
-            auto vo_it = pg.ver_override.find(cur_oid);
-            auto read_version = (vo_it != pg.ver_override.end() ? vo_it->second : UINT64_MAX);
             // Read bitmap synchronously from the local database
             bs->read_bitmap(
-                cur_oid, read_version, (uint8_t*)op_data->snapshot_bitmaps + chain_num*clean_entry_bitmap_size,
+                cur_oid, UINT64_MAX, (uint8_t*)op_data->snapshot_bitmaps + chain_num*clean_entry_bitmap_size,
                 !chain_num ? &cur_op->reply.rw.version : NULL
             );
         }
     }
     else
     {
-        if (submit_bitmap_subops(cur_op, pg) < 0)
+        if (submit_bitmap_subops(cur_op, *pg) < 0)
         {
             // Failure
             finish_op(cur_op, -EIO);
@@ -101,32 +100,32 @@ resume_0:
             return 1;
         }
 resume_1:
-        if (pg.scheme != POOL_SCHEME_REPLICATED)
+        if (pg->scheme != POOL_SCHEME_REPLICATED)
         {
             for (int chain_num = 0; chain_num < op_data->chain_size; chain_num++)
             {
                 // Check if we need to reconstruct any bitmaps
-                for (int i = 0; i < pg.pg_size; i++)
+                for (int i = 0; i < pg->pg_size; i++)
                 {
-                    if (op_data->missing_flags[chain_num*pg.pg_size + i])
+                    if (op_data->missing_flags[chain_num*pg->pg_size + i])
                     {
-                        osd_rmw_stripe_t local_stripes[pg.pg_size];
+                        osd_rmw_stripe_t local_stripes[pg->pg_size];
-                        for (i = 0; i < pg.pg_size; i++)
+                        for (i = 0; i < pg->pg_size; i++)
                         {
                             local_stripes[i] = (osd_rmw_stripe_t){
-                                .bmp_buf = (uint8_t*)op_data->snapshot_bitmaps + (chain_num*pg.pg_size + i)*clean_entry_bitmap_size,
+                                .bmp_buf = (uint8_t*)op_data->snapshot_bitmaps + (chain_num*pg->pg_size + i)*clean_entry_bitmap_size,
                                 .read_start = 1,
                                 .read_end = 1,
-                                .missing = op_data->missing_flags[chain_num*pg.pg_size + i] && true,
+                                .missing = op_data->missing_flags[chain_num*pg->pg_size + i] && true,
                             };
                         }
-                        if (pg.scheme == POOL_SCHEME_XOR)
+                        if (pg->scheme == POOL_SCHEME_XOR)
                         {
-                            reconstruct_stripes_xor(local_stripes, pg.pg_size, clean_entry_bitmap_size);
+                            reconstruct_stripes_xor(local_stripes, pg->pg_size, clean_entry_bitmap_size);
                         }
-                        else if (pg.scheme == POOL_SCHEME_EC)
+                        else if (pg->scheme == POOL_SCHEME_EC)
                         {
-                            reconstruct_stripes_ec(local_stripes, pg.pg_size, pg.pg_data_size, clean_entry_bitmap_size);
+                            reconstruct_stripes_ec(local_stripes, pg->pg_size, pg->pg_data_size, clean_entry_bitmap_size);
                         }
                         break;
                     }
@@ -139,6 +138,7 @@ resume_1:
 
 int osd_t::collect_bitmap_requests(osd_op_t *cur_op, pg_t & pg, std::vector<bitmap_request_t> & bitmap_requests)
 {
+    assert(&pg);
     osd_primary_op_data_t *op_data = cur_op->op_data;
     for (int chain_num = 0; chain_num < op_data->chain_size; chain_num++)
     {
@@ -216,6 +216,7 @@ int osd_t::collect_bitmap_requests(osd_op_t *cur_op, pg_t & pg, std::vector<bitm
 
 int osd_t::submit_bitmap_subops(osd_op_t *cur_op, pg_t & pg)
 {
+    assert(&pg);
     osd_primary_op_data_t *op_data = cur_op->op_data;
     std::vector<bitmap_request_t> *bitmap_requests = new std::vector<bitmap_request_t>();
     if (collect_bitmap_requests(cur_op, pg, *bitmap_requests) < 0)
@@ -382,12 +383,12 @@ std::vector<osd_chain_read_t> osd_t::collect_chained_read_requests(osd_op_t *cur
     return chain_reads;
 }
 
-int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
+int osd_t::submit_chained_read_requests(pg_t *pg, osd_op_t *cur_op)
 {
     // Decide which parts of which objects we need to read based on bitmaps
     osd_primary_op_data_t *op_data = cur_op->op_data;
     auto chain_reads = collect_chained_read_requests(cur_op);
-    int stripe_count = (pg.scheme == POOL_SCHEME_REPLICATED ? 1 : pg.pg_size);
+    int stripe_count = (!pg || pg->scheme == POOL_SCHEME_REPLICATED ? 1 : pg->pg_size);
     op_data->chain_read_count = chain_reads.size();
     op_data->chain_reads = (osd_chain_read_t*)calloc_or_die(
         1, sizeof(osd_chain_read_t) * chain_reads.size()
@@ -408,23 +409,23 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
         object_id cur_oid = { .inode = chain_reads[cri].inode, .stripe = op_data->oid.stripe };
         // FIXME: maybe introduce split_read_stripes to shorten these lines and to remove read_start=req_start
         osd_rmw_stripe_t *stripes = chain_stripes + chain_reads[cri].chain_pos*stripe_count;
-        split_stripes(pg.pg_data_size, bs_block_size, chain_reads[cri].offset, chain_reads[cri].len, stripes);
+        split_stripes(pg ? pg->pg_data_size : 1, bs_block_size, chain_reads[cri].offset, chain_reads[cri].len, stripes);
-        if (pg.scheme == POOL_SCHEME_REPLICATED && !stripes[0].req_end)
+        if ((!pg || pg->scheme == POOL_SCHEME_REPLICATED) && !stripes[0].req_end)
         {
             continue;
         }
-        for (int role = 0; role < pg.pg_data_size; role++)
+        for (int role = 0; role < (pg ? pg->pg_data_size : 1); role++)
         {
             stripes[role].read_start = stripes[role].req_start;
             stripes[role].read_end = stripes[role].req_end;
         }
-        uint64_t *cur_set = pg.cur_set.data();
+        uint64_t *cur_set = pg ? pg->cur_set.data() : &this->osd_num;
-        if (pg.state != PG_ACTIVE)
+        if (pg && pg->state != PG_ACTIVE)
         {
-            cur_set = get_object_osd_set(pg, cur_oid, &op_data->chain_states[chain_reads[cri].chain_pos]);
+            cur_set = get_object_osd_set(*pg, cur_oid, &op_data->chain_states[chain_reads[cri].chain_pos]);
-            if (pg.scheme != POOL_SCHEME_REPLICATED)
+            if (pg->scheme != POOL_SCHEME_REPLICATED)
             {
-                if (extend_missing_stripes(stripes, cur_set, pg.pg_data_size, pg.pg_size) < 0)
+                if (extend_missing_stripes(stripes, cur_set, pg->pg_data_size, pg->pg_size) < 0)
                 {
                     free(op_data->chain_reads);
                     op_data->chain_reads = NULL;
@@ -445,14 +446,14 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
                 }
             }
         }
-        if (pg.scheme == POOL_SCHEME_REPLICATED)
+        if (!pg || pg->scheme == POOL_SCHEME_REPLICATED)
         {
             n_subops++;
             read_buffer_size += stripes[0].read_end - stripes[0].read_start;
         }
         else
         {
-            for (int role = 0; role < pg.pg_size; role++)
+            for (int role = 0; role < pg->pg_size; role++)
             {
                 if (stripes[role].read_end > 0 && cur_set[role] != 0)
                     n_subops++;
@@ -490,19 +491,23 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
     for (int cri = 0; cri < chain_reads.size(); cri++)
     {
         osd_rmw_stripe_t *stripes = chain_stripes + chain_reads[cri].chain_pos*stripe_count;
-        if (pg.scheme == POOL_SCHEME_REPLICATED && !stripes[0].req_end)
+        if ((!pg || pg->scheme == POOL_SCHEME_REPLICATED) && !stripes[0].req_end)
         {
             continue;
         }
         object_id cur_oid = { .inode = chain_reads[cri].inode, .stripe = op_data->oid.stripe };
-        auto vo_it = pg.ver_override.find(cur_oid);
+        uint64_t target_ver = UINT64_MAX;
-        uint64_t target_ver = vo_it != pg.ver_override.end() ? vo_it->second : UINT64_MAX;
+        if (pg)
-        auto cur_state = op_data->chain_states[chain_reads[cri].chain_pos];
-        uint64_t *cur_set = (pg.state != PG_ACTIVE && cur_state ? cur_state->read_target.data() : pg.cur_set.data());
-        int zero_read = -1;
-        if (pg.scheme == POOL_SCHEME_REPLICATED)
         {
-            for (int role = 0; role < pg.pg_size; role++)
+            auto vo_it = pg->ver_override.find(cur_oid);
+            target_ver = vo_it != pg->ver_override.end() ? vo_it->second : UINT64_MAX;
+        }
+        auto cur_state = op_data->chain_states[chain_reads[cri].chain_pos];
+        uint64_t *cur_set = (!pg ? &this->osd_num : (pg->state != PG_ACTIVE && cur_state ? cur_state->read_target.data() : pg->cur_set.data()));
+        int zero_read = -1;
+        if (!pg || pg->scheme == POOL_SCHEME_REPLICATED)
+        {
+            for (int role = 0; role < (pg ? pg->pg_size : 1); role++)
                 if (cur_set[role] == this->osd_num || zero_read == -1)
                     zero_read = role;
         }
@@ -514,6 +519,7 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
 
 void osd_t::check_corrupted_chained(pg_t & pg, osd_op_t *cur_op)
 {
+    assert(&pg);
     osd_primary_op_data_t *op_data = cur_op->op_data;
     int stripe_count = (pg.scheme == POOL_SCHEME_REPLICATED ? 1 : pg.pg_size);
     osd_rmw_stripe_t *chain_stripes = (osd_rmw_stripe_t*)(
@@ -539,33 +545,32 @@ void osd_t::check_corrupted_chained(pg_t & pg, osd_op_t *cur_op)
     }
 }
 
-void osd_t::send_chained_read_results(pg_t & pg, osd_op_t *cur_op)
+void osd_t::send_chained_read_results(pg_t *pg, osd_op_t *cur_op)
 {
     osd_primary_op_data_t *op_data = cur_op->op_data;
-    int stripe_count = (pg.scheme == POOL_SCHEME_REPLICATED ? 1 : pg.pg_size);
+    int stripe_count = (!pg || pg->scheme == POOL_SCHEME_REPLICATED ? 1 : pg->pg_size);
     osd_rmw_stripe_t *chain_stripes = (osd_rmw_stripe_t*)(
         (uint8_t*)op_data->chain_reads + sizeof(osd_chain_read_t) * op_data->chain_read_count
     );
     // Reconstruct parts if needed
     if (op_data->degraded)
     {
-        int stripe_count = (pg.scheme == POOL_SCHEME_REPLICATED ? 1 : pg.pg_size);
         for (int cri = 0; cri < op_data->chain_read_count; cri++)
         {
             // Reconstruct missing stripes
             osd_rmw_stripe_t *stripes = chain_stripes + op_data->chain_reads[cri].chain_pos*stripe_count;
-            if (pg.scheme == POOL_SCHEME_XOR)
+            if (pg->scheme == POOL_SCHEME_XOR)
             {
-                reconstruct_stripes_xor(stripes, pg.pg_size, clean_entry_bitmap_size);
+                reconstruct_stripes_xor(stripes, pg->pg_size, clean_entry_bitmap_size);
             }
-            else if (pg.scheme == POOL_SCHEME_EC)
+            else if (pg->scheme == POOL_SCHEME_EC)
             {
-                reconstruct_stripes_ec(stripes, pg.pg_size, pg.pg_data_size, clean_entry_bitmap_size);
+                reconstruct_stripes_ec(stripes, pg->pg_size, pg->pg_data_size, clean_entry_bitmap_size);
             }
         }
     }
     // Send bitmap
-    cur_op->reply.rw.bitmap_len = pg.pg_data_size * clean_entry_bitmap_size;
+    cur_op->reply.rw.bitmap_len = (pg ? pg->pg_data_size : 1) * clean_entry_bitmap_size;
     cur_op->iov.push_back(op_data->stripes[0].bmp_buf, cur_op->reply.rw.bitmap_len);
     // And finally compose the result
     uint64_t sent = 0;

src/osd/osd_primary_subops.cpp

@@ -67,14 +67,17 @@ void osd_t::finish_op(osd_op_t *cur_op, int retval)
         if (cur_op->req.hdr.opcode == OSD_OP_DELETE)
         {
             if (cur_op->op_data)
-                inode_stats[cur_op->req.rw.inode].op_bytes[inode_st_op] += cur_op->op_data->pg->pg_data_size * bs_block_size;
+            {
+                inode_stats[cur_op->req.rw.inode].op_bytes[inode_st_op] += (cur_op->op_data->pg
+                    ? cur_op->op_data->pg->pg_data_size : 1) * bs_block_size;
+            }
         }
         else
             inode_stats[cur_op->req.rw.inode].op_bytes[inode_st_op] += cur_op->req.rw.len;
     }
     if (cur_op->op_data)
     {
-        if (cur_op->op_data->pg_num > 0)
+        if (cur_op->op_data->pg)
         {
             auto & pg = *cur_op->op_data->pg;
             rm_inflight(pg);
@@ -117,10 +120,10 @@ void osd_t::submit_primary_subops(int submit_type, uint64_t op_version, const ui
     bool wr = submit_type == SUBMIT_WRITE;
     osd_primary_op_data_t *op_data = cur_op->op_data;
     osd_rmw_stripe_t *stripes = op_data->stripes;
-    bool rep = op_data->pg->scheme == POOL_SCHEME_REPLICATED;
+    bool rep = !op_data->pg || op_data->pg->scheme == POOL_SCHEME_REPLICATED;
     // Allocate subops
     int n_subops = 0, zero_read = -1;
-    for (int role = 0; role < op_data->pg->pg_size; role++)
+    for (int role = 0; role < (op_data->pg ? op_data->pg->pg_size : 1); role++)
     {
         if (osd_set[role] == this->osd_num || osd_set[role] != 0 && zero_read == -1)
             zero_read = role;
@@ -143,11 +146,11 @@ void osd_t::submit_primary_subops(int submit_type, uint64_t op_version, const ui
 int osd_t::submit_primary_subop_batch(int submit_type, inode_t inode, uint64_t op_version,
     osd_rmw_stripe_t *stripes, const uint64_t* osd_set, osd_op_t *cur_op, int subop_idx, int zero_read)
 {
-    bool rep = cur_op->op_data->pg->scheme == POOL_SCHEME_REPLICATED;
+    bool rep = !cur_op->op_data->pg || cur_op->op_data->pg->scheme == POOL_SCHEME_REPLICATED;
     bool wr = submit_type == SUBMIT_WRITE;
     osd_primary_op_data_t *op_data = cur_op->op_data;
     int i = subop_idx;
-    for (int role = 0; role < op_data->pg->pg_size; role++)
+    for (int role = 0; role < (op_data->pg ? op_data->pg->pg_size : 1); role++)
     {
         // We always submit zero-length writes to all replicas, even if the stripe is not modified
         if (!(wr || !rep && stripes[role].read_end != 0 || zero_read == role || submit_type == SUBMIT_SCRUB_READ))