thebestvibedev/tromcho.net
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP Integrate LSMeta :-)

Browse Source
This commit is contained in:
Vitaliy Filippov
2025-12-02 01:52:12 +03:00
parent e512e1eeb1
commit 5afef7ca6d
16 changed files with 742 additions and 802 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/blockstore/blockstore_flush.cpp
+162 -207
View File
@@ -24,6 +24,11 @@ journal_flusher_t::journal_flusher_t(blockstore_impl_t *bs)
{
co[i].co_id = i;
co[i].bs = bs;
if (bs->dsk.csum_block_size > bs->dsk.bitmap_granularity)
{
co[i].new_csums = (uint8_t*)malloc_or_die(bs->dsk.data_block_size / bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF));
co[i].new_bmp = (uint8_t*)malloc_or_die(bs->dsk.clean_entry_bitmap_size);
}
co[i].flusher = this;
}
}
@@ -54,6 +59,16 @@ journal_flusher_t::~journal_flusher_t()
journal_flusher_co::~journal_flusher_co()
{
if (new_csums)
{
free(new_csums);
new_csums = NULL;
}
if (new_bmp)
{
free(new_bmp);
new_bmp = NULL;
}
free_buffers();
}
@@ -151,14 +166,6 @@ bool journal_flusher_co::loop()
else if (wait_state == 14) goto resume_14;
else if (wait_state == 15) goto resume_15;
else if (wait_state == 16) goto resume_16;
else if (wait_state == 17) goto resume_17;
else if (wait_state == 18) goto resume_18;
else if (wait_state == 19) goto resume_19;
else if (wait_state == 20) goto resume_20;
else if (wait_state == 21) goto resume_21;
else if (wait_state == 22) goto resume_22;
else if (wait_state == 23) goto resume_23;
else if (wait_state == 24) goto resume_24;
resume_0:
wait_state = 0;
wait_count = 0;
@@ -168,10 +175,10 @@ resume_0:
(!bs->dsk.disable_journal_fsync || !bs->dsk.disable_meta_fsync))
{
flusher->active_flushers++;
resume_21:
resume_22:
res = fsync_buffer(21);
if (!res)
resume_14:
resume_15:
resume_16:
if (!fsync_buffer(14))
{
return false;
}
@@ -180,18 +187,6 @@ resume_22:
}
if (res == ENOENT)
{
if (co_id == 0 && flusher->force_start > 0)
{
flusher->active_flushers++;
resume_16:
resume_17:
resume_18:
resume_19:
resume_20:
if (!trim_lsn(16))
return false;
flusher->active_flushers--;
}
cur_oid = {};
wait_state = 0;
return true;
@@ -206,35 +201,37 @@ resume_20:
}
}
resume_1:
wait_state = 1;
should_repeat = false;
cur_obj = bs->heap->lock_and_read_entry(cur_oid, copy_id);
cur_obj = bs->heap->lock_and_read_entry(cur_oid);
if (!cur_obj)
{
// Object does not exist
goto resume_0;
}
cur_version = cur_obj->get_writes()->version;
// Find the range to compact
compact_lsn = bs->heap->get_fsynced_lsn();
bs->heap->get_compact_range(cur_obj, compact_lsn, &begin_wr, &end_wr);
if (!begin_wr)
{
// Nothing to flush
bs->heap->unlock_entry(cur_oid, copy_id);
goto resume_0;
}
assert(!end_wr->next() && end_wr->entry_type == (BS_HEAP_BIG_WRITE|BS_HEAP_STABLE));
clean_loc = end_wr->big_location(bs->heap);
if (bs->log_level > 10)
printf("Compacting %jx:%jx l%ju .. l%ju (last l%ju)\n", cur_oid.inode, cur_oid.stripe, end_wr->lsn, begin_wr->lsn, compact_lsn);
flusher->active_flushers++;
// Scan versions to flush
free_buffers();
copy_count = 0;
for (auto wr = begin_wr; wr != end_wr; wr = wr->next())
fsynced_lsn = bs->heap->get_fsynced_lsn();
compact_info = bs->heap->iterate_compaction(cur_obj, fsynced_lsn, flusher->force_start, [&](heap_entry_t *wr)
{
bs->prepare_read(read_vec, cur_obj, wr, 0, bs->dsk.data_block_size);
copy_count++;
if (wr->type() == BS_HEAP_SMALL_WRITE ||
wr->type() == BS_HEAP_INTENT_WRITE && bs->dsk.csum_block_size > bs->dsk.bitmap_granularity)
{
bs->prepare_read(read_vec, cur_obj, wr, 0, bs->dsk.data_block_size);
copy_count++;
}
});
if (!compact_info.compact_lsn)
{
// Flushing is aborted
bs->heap->unlock_entry(cur_oid);
goto resume_0;
}
flusher->active_flushers++;
if (bs->log_level > 10)
{
printf("Compacting %jx:%jx l%ju .. l%ju\n", cur_oid.inode, cur_oid.stripe, compact_info.clean_lsn, compact_info.compact_lsn);
}
overwrite_start = overwrite_end = 0;
if (read_vec.size() > 0)
@@ -254,7 +251,6 @@ resume_1:
}
// Read buffered data
cur_obj = NULL;
begin_wr = end_wr = NULL;
resume_2:
resume_3:
if (!read_buffered(2))
@@ -272,29 +268,45 @@ resume_3:
flusher->wanting_meta_fsync--;
}
res = check_and_punch_checksums();
if (res == EBUSY)
{
resume_4:
resume_5:
if (!write_meta_block(4))
{
return false;
}
resume_6:
resume_7:
resume_8:
if (!fsync_meta(6))
{
return false;
}
res = 0;
}
else if (res == ENOENT || res == EDOM)
if (res == ENOENT || res == EDOM)
{
// Abort compaction
flusher->active_flushers--;
goto resume_0;
}
if (res == EBUSY)
{
resume_4:
modified_block = UINT32_MAX;
res = bs->heap->add_punch_holes(cur_obj, compact_info.clean_lsn, compact_info.clean_version, new_bmp, new_csums, &modified_block);
if (res == ENOENT)
{
// Abort compaction
flusher->active_flushers--;
goto resume_0;
}
if (res == EAGAIN)
{
// Retry, block is busy
wait_state = 4;
return false;
}
assert(res == 0);
resume_5:
resume_6:
if (!write_meta_block(5))
{
return false;
}
resume_7:
resume_8:
resume_9:
if (!fsync_meta(7))
{
return false;
}
res = 0;
}
assert(res == 0);
// Submit data writes
for (i = 0; i < read_vec.size(); i++)
@@ -304,26 +316,26 @@ resume_8:
(read_vec[i].copy_flags & COPY_BUF_PADDED)) // FIXME Shit, simplify these flags
{
assert(read_vec[i].buf);
await_sqe(9);
await_sqe(10);
data->iov = (struct iovec){ read_vec[i].buf + (read_vec[i].copy_flags & COPY_BUF_PADDED
? read_vec[i].offset - read_vec[i].disk_offset : 0), (size_t)read_vec[i].len };
data->callback = simple_callback_w;
io_uring_prep_writev(sqe, bs->dsk.data_fd, &data->iov, 1, bs->dsk.data_offset + clean_loc + read_vec[i].offset);
io_uring_prep_writev(sqe, bs->dsk.data_fd, &data->iov, 1, bs->dsk.data_offset + compact_info.clean_loc + read_vec[i].offset);
wait_count++;
}
}
resume_10:
resume_11:
if (wait_count > 0)
{
wait_state = 10;
wait_state = 11;
return false;
}
// Lock is only needed to prevent freeing the big_write because we overwrite it...
bs->heap->unlock_entry(cur_oid, copy_id);
bs->heap->unlock_entry(cur_oid);
// Mark the object compacted, but don't free and remove small_writes
// We'll free and remove them only when trimming
// The only thing we modify here are big_write block checksums if >4k block is used
cur_obj = bs->heap->read_entry(cur_oid, &modified_block);
cur_obj = bs->heap->read_entry(cur_oid);
if (!cur_obj)
{
// Abort compaction
@@ -334,40 +346,20 @@ resume_10:
// Abort compaction
goto resume_0;
}
if (read_to_fill_incomplete)
{
resume_23:
resume_24:
if (!write_meta_block(23))
{
return false;
}
// Recheck the object because it could be invalidated again
cur_obj = bs->heap->read_entry(cur_oid, &modified_block);
if (!cur_obj)
{
// Abort compaction
goto resume_0;
}
}
bs->heap->mark_object_compacted(cur_obj, compact_lsn);
// Done
if (bs->log_level > 10)
printf("Compacted %jx:%jx l%ju (%d writes)\n", cur_oid.inode, cur_oid.stripe, compact_lsn, copy_count);
flusher->compact_counter++;
flusher->active_flushers--;
// Advance compacted_lsn every <journal_trim_interval> objects
if (co_id == 0 && !((++flusher->advance_lsn_counter) % bs->journal_trim_interval))
{
flusher->advance_lsn_counter = 0;
resume_11:
bs->heap->add_compact(cur_obj, compact_info.compact_lsn, &modified_block, new_csums);
resume_12:
resume_13:
resume_14:
resume_15:
if (!trim_lsn(11))
return false;
if (!write_meta_block(12))
{
return false;
}
// Done
if (bs->log_level > 10)
{
printf("Compacted %jx:%jx l%ju (%d writes)\n", cur_oid.inode, cur_oid.stripe, compact_info.compact_lsn, copy_count);
}
flusher->compact_counter++;
flusher->active_flushers--;
if (should_repeat)
{
// Flush the same object again
@@ -409,11 +401,11 @@ void journal_flusher_co::fill_partial_checksum_blocks()
.copy_flags = COPY_BUF_DATA | copy_flags,
.offset = blk_begin,
.len = blk_end - blk_begin,
.disk_loc = end_wr->big_location(bs->heap),
.disk_loc = compact_info.clean_loc,
.disk_offset = blk_begin,
.disk_len = blk_end - blk_begin,
.buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, blk_end - blk_begin),
.wr_lsn = end_wr->lsn,
.wr_lsn = compact_info.clean_lsn,
});
}
auto & vec = read_vec[read_vec.size()-1];
@@ -451,16 +443,18 @@ int journal_flusher_co::check_and_punch_checksums()
return 0;
}
// Verify data checksums
cur_obj = bs->heap->read_locked_entry(cur_oid, copy_id);
cur_obj = bs->heap->read_entry(cur_oid);
bool csum_ok = true;
for (int i = 0; i < read_vec.size(); i++)
{
auto & vec = read_vec[i];
if (!(vec.copy_flags & (COPY_BUF_COALESCED|COPY_BUF_ZERO|COPY_BUF_SKIP_CSUM)))
{
heap_write_t *wr = cur_obj->get_writes();
heap_entry_t *wr = cur_obj;
while (wr && wr->lsn != vec.wr_lsn)
wr = wr->next();
{
wr = bs->heap->prev(wr);
}
assert(wr);
uint32_t *csums = (uint32_t*)(wr->get_checksums(bs->heap)
+ (vec.disk_offset/bs->dsk.csum_block_size)*(bs->dsk.data_csum_type & 0xFF)
@@ -489,20 +483,30 @@ int journal_flusher_co::check_and_punch_checksums()
// Nothing to do
return 0;
}
cur_obj = bs->heap->read_entry(cur_oid, &modified_block);
cur_obj = bs->heap->read_entry(cur_oid);
if (!cur_obj)
{
// Object is deleted, abort compaction
return ENOENT;
}
bs->heap->get_compact_range(cur_obj, compact_lsn, &begin_wr, &end_wr);
if (!begin_wr)
heap_entry_t *clean_wr = NULL;
for (auto wr = cur_obj; wr; wr = bs->heap->prev(wr))
{
// Object is overwritten, abort compaction
return ENOENT;
if (wr->is_overwrite() && wr->lsn > compact_info.clean_lsn &&
wr->lsn <= fsynced_lsn)
{
// Object is overwritten, abort compaction
return ENOENT;
}
if (wr->lsn == compact_info.clean_lsn)
{
clean_wr = wr;
break;
}
}
uint8_t *bmp = end_wr->get_int_bitmap(bs->heap);
uint8_t *csums = end_wr->get_checksums(bs->heap);
assert(clean_wr);
memcpy(new_bmp, clean_wr->get_int_bitmap(bs->heap), bs->dsk.clean_entry_bitmap_size);
memcpy(new_csums, clean_wr->get_checksums(bs->heap), bs->dsk.data_block_size/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF));
// Clear bits
for (auto & vec: read_vec)
{
@@ -513,7 +517,7 @@ int journal_flusher_co::check_and_punch_checksums()
if (!(vec.copy_flags & COPY_BUF_COALESCED) &&
((vec.offset % bs->dsk.csum_block_size) || (vec.len % bs->dsk.csum_block_size)))
{
bitmap_clear(bmp, vec.offset, vec.len, bs->dsk.bitmap_granularity);
bitmap_clear(new_bmp, vec.offset, vec.len, bs->dsk.bitmap_granularity);
}
}
// Update partial block checksums
@@ -522,17 +526,10 @@ int journal_flusher_co::check_and_punch_checksums()
if (vec.copy_flags & COPY_BUF_CSUM_FILL)
{
uint32_t csum_off = vec.offset/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF);
bs->heap->calc_block_checksums((uint32_t*)(csums+csum_off), vec.buf, bmp, vec.offset, vec.offset+vec.len, true, NULL);
bs->heap->calc_block_checksums((uint32_t*)(new_csums+csum_off), vec.buf, new_bmp, vec.offset, vec.offset+vec.len, true, NULL);
}
}
cur_obj->crc32c = cur_obj->calc_crc32c();
if (res == ENOENT)
{
// Object is deleted, abort compaction
return ENOENT;
}
// Modified, we should write the block to disk
assert(!res);
// Modified, we should add_punch_holes and then write the block to disk
return EBUSY;
}
@@ -542,20 +539,30 @@ bool journal_flusher_co::calc_block_checksums()
{
return true;
}
bs->heap->get_compact_range(cur_obj, compact_lsn, &begin_wr, &end_wr);
if (!begin_wr)
heap_entry_t *clean_wr = NULL;
for (auto wr = cur_obj; wr; wr = bs->heap->prev(wr))
{
// Object is overwritten, abort compaction
return false;
if (wr->is_overwrite() && wr->lsn > compact_info.clean_lsn &&
wr->lsn <= fsynced_lsn)
{
// Object is overwritten, abort compaction
return false;
}
if (wr->lsn == compact_info.clean_lsn)
{
clean_wr = wr;
break;
}
}
uint8_t *bmp = end_wr->get_int_bitmap(bs->heap);
uint8_t *csums = end_wr->get_checksums(bs->heap);
assert(clean_wr);
memcpy(new_bmp, clean_wr->get_int_bitmap(bs->heap), bs->dsk.clean_entry_bitmap_size);
memcpy(new_csums, clean_wr->get_checksums(bs->heap), bs->dsk.data_block_size/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF));
// Set bits
for (auto & vec: read_vec)
{
if (!(vec.copy_flags & (COPY_BUF_COALESCED|COPY_BUF_CSUM_FILL)))
{
bitmap_set(bmp, vec.offset, vec.len, bs->dsk.bitmap_granularity);
bitmap_set(new_bmp, vec.offset, vec.len, bs->dsk.bitmap_granularity);
}
}
// Update block checksums
@@ -576,7 +583,7 @@ bool journal_flusher_co::calc_block_checksums()
assert(!(end % bs->dsk.csum_block_size));
uint32_t csum_off = start/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF);
bs->heap->calc_block_checksums(
(uint32_t*)(csums+csum_off), bmp, start, end,
(uint32_t*)(new_csums+csum_off), new_bmp, start, end,
[&](uint32_t start, uint32_t & len)
{
// O(n^2) search, may be fixed later :-p
@@ -593,7 +600,6 @@ bool journal_flusher_co::calc_block_checksums()
}, true, NULL
);
}
cur_obj->crc32c = cur_obj->calc_crc32c();
return true;
}
@@ -603,13 +609,15 @@ bool journal_flusher_co::write_meta_block(int wait_base)
goto resume_0;
else if (wait_state == wait_base+1)
goto resume_1;
await_sqe(0);
data->iov = (struct iovec){ bs->heap->get_meta_block(modified_block), (size_t)bs->dsk.meta_block_size };
data->callback = simple_callback_w;
io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + (modified_block+1)*bs->dsk.meta_block_size);
wait_count++;
resume_0:
if (bs->ringloop->space_left() < 1)
{
wait_state = wait_base+0;
return 0;
}
bs->prepare_meta_block_write(modified_block);
resume_1:
if (wait_count > 0)
if (bs->meta_block_is_pending(modified_block))
{
wait_state = wait_base+1;
return false;
@@ -664,11 +672,11 @@ bool journal_flusher_co::fsync_meta(int wait_base)
if (wait_state == wait_base) goto resume_0;
else if (wait_state == wait_base+1) goto resume_1;
else if (wait_state == wait_base+2) goto resume_2;
resume_0:
if (bs->dsk.disable_meta_fsync)
{
return true;
}
resume_0:
if (flusher->wanting_meta_fsync || flusher->fsyncing_meta > 0)
{
wait_state = wait_base;
@@ -693,69 +701,31 @@ resume_2:
return true;
}
int journal_flusher_co::fsync_buffer(int wait_base)
{
if (wait_state == wait_base) goto resume_0;
else if (wait_state == wait_base+1) goto resume_1;
if (bs->dsk.disable_journal_fsync && bs->dsk.disable_meta_fsync && bs->dsk.disable_data_fsync || !bs->unsynced_big_write_count && !bs->unsynced_small_write_count)
{
return 1;
}
if (flusher->syncing_buffer)
{
return 0;
}
flusher->active_flushers++;
flusher->syncing_buffer++;
resume_0:
assert(!wait_count);
compact_lsn = bs->heap->get_completed_lsn();
if (!bs->submit_fsyncs(wait_count))
{
wait_state = wait_base+0;
return 0;
}
resume_1:
if (wait_count > 0)
{
wait_state = wait_base+1;
return 0;
}
bs->heap->mark_lsn_fsynced(compact_lsn);
flusher->active_flushers--;
flusher->syncing_buffer--;
return 2;
}
bool journal_flusher_co::trim_lsn(int wait_base)
bool journal_flusher_co::fsync_buffer(int wait_base)
{
if (wait_state == wait_base) goto resume_0;
else if (wait_state == wait_base+1) goto resume_1;
else if (wait_state == wait_base+2) goto resume_2;
else if (wait_state == wait_base+3) goto resume_3;
else if (wait_state == wait_base+4) goto resume_4;
compact_lsn = bs->heap->get_compacted_lsn();
if (((blockstore_meta_header_v3_t*)bs->meta_superblock)->compacted_lsn == compact_lsn)
if (bs->dsk.disable_journal_fsync && bs->dsk.disable_meta_fsync && bs->dsk.disable_data_fsync ||
!bs->unsynced_big_write_count && !bs->unsynced_small_write_count)
{
return true;
}
flusher->active_flushers++;
assert(!wait_count);
if (!bs->dsk.disable_meta_fsync)
resume_0:
if (flusher->syncing_buffer)
{
await_sqe(0);
data->iov = { 0 };
data->callback = simple_callback_w;
io_uring_prep_fsync(sqe, bs->dsk.meta_fd, IORING_FSYNC_DATASYNC);
wait_count++;
wait_state = wait_base+0;
return false;
}
if (!bs->dsk.disable_data_fsync && bs->dsk.data_fd != bs->dsk.meta_fd)
flusher->active_flushers++;
flusher->syncing_buffer++;
resume_1:
assert(!wait_count);
fsynced_lsn = bs->heap->get_completed_lsn();
if (!bs->submit_fsyncs(wait_count))
{
await_sqe(1);
data->iov = { 0 };
data->callback = simple_callback_w;
io_uring_prep_fsync(sqe, bs->dsk.data_fd, IORING_FSYNC_DATASYNC);
wait_count++;
wait_state = wait_base+1;
return false;
}
resume_2:
if (wait_count > 0)
@@ -763,23 +733,8 @@ resume_2:
wait_state = wait_base+2;
return false;
}
((blockstore_meta_header_v3_t*)bs->meta_superblock)->compacted_lsn = compact_lsn;
((blockstore_meta_header_v3_t*)bs->meta_superblock)->set_crc32c();
await_sqe(3);
data->iov = (struct iovec){ bs->meta_superblock, (size_t)bs->dsk.meta_block_size };
data->callback = simple_callback_w;
io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset);
// Update superblock with datasync
sqe->rw_flags = RWF_DSYNC;
wait_count++;
resume_4:
if (wait_count > 0)
{
wait_state = wait_base+4;
return false;
}
bs->heap->mark_lsn_trimmed(compact_lsn);
flusher->compact_counter++;
bs->heap->mark_lsn_fsynced(fsynced_lsn);
flusher->active_flushers--;
flusher->syncing_buffer--;
return true;
}
src/blockstore/blockstore_flush.h
+6 -10
View File
@@ -36,26 +36,23 @@ class journal_flusher_co
int wait_state, wait_count;
struct io_uring_sqe *sqe;
struct ring_data_t *data;
uint8_t *new_csums = NULL;
uint8_t *new_bmp = NULL;
std::function<void(ring_data_t*)> simple_callback_r, simple_callback_w;
object_id cur_oid;
uint64_t copy_id;
uint64_t compact_lsn;
uint64_t cur_version;
heap_object_t *cur_obj;
heap_write_t *begin_wr, *end_wr;
heap_entry_t *cur_obj;
uint64_t fsynced_lsn;
heap_compact_t compact_info;
uint32_t modified_block;
bool should_repeat;
std::vector<copy_buffer_t> read_vec;
uint32_t overwrite_start, overwrite_end;
uint32_t big_start, big_end;
int i, res;
bool read_to_fill_incomplete;
int copy_count;
uint64_t clean_loc;
flusher_meta_write_t meta_old, meta_new;
bool do_repeat = false;
friend class journal_flusher_t;
@@ -68,8 +65,7 @@ class journal_flusher_co
bool write_meta_block(int wait_base);
bool read_buffered(int wait_base);
bool fsync_meta(int wait_base);
int fsync_buffer(int wait_base);
bool trim_lsn(int wait_base);
bool fsync_buffer(int wait_base);
public:
journal_flusher_co();
~journal_flusher_co();
src/blockstore/blockstore_impl.cpp
+6
View File
@@ -197,6 +197,12 @@ void blockstore_impl_t::loop()
{
throw std::runtime_error(std::string("io_uring_submit: ") + strerror(-ret));
}
for (auto & block_num: pending_modified_blocks)
{
heap->start_block_write(block_num);
modified_blocks.insert(block_num);
}
pending_modified_blocks.clear();
if ((initial_ring_space - ringloop->space_left()) > 0)
{
live = true;
src/blockstore/blockstore_impl.h
+12 -11
View File
@@ -53,8 +53,8 @@ struct blockstore_op_private_t
int pending_ops;
int op_state;
// Read, write, sync, stabilize
uint64_t lsn;
// Write, sync, stabilize
uint32_t modified_block, modified_block2;
// Read
std::vector<copy_buffer_t> read_vec;
@@ -66,9 +66,6 @@ struct blockstore_op_private_t
// Stabilize, rollback
int stab_pos;
// Stabilize
uint64_t to_lsn;
// Write
struct iovec iov_zerofill[3];
timespec tv_begin;
@@ -119,6 +116,9 @@ public:
int unsynced_queued_ops = 0;
uint8_t *zero_object = NULL;
std::vector<uint32_t> pending_modified_blocks;
robin_hood::unordered_flat_set<uint32_t> modified_blocks;
journal_flusher_t *flusher;
int write_iodepth = 0;
int inflight_big = 0;
@@ -153,11 +153,11 @@ public:
// Read
int dequeue_read(blockstore_op_t *op);
int fulfill_read(blockstore_op_t *op);
uint32_t prepare_read(std::vector<copy_buffer_t> & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end);
uint32_t prepare_read_with_bitmaps(std::vector<copy_buffer_t> & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end);
uint32_t prepare_read(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end);
uint32_t prepare_read_with_bitmaps(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end);
uint32_t prepare_read_zero(std::vector<copy_buffer_t> & read_vec, uint32_t start, uint32_t end);
uint32_t prepare_read_simple(std::vector<copy_buffer_t> & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end);
void prepare_disk_read(std::vector<copy_buffer_t> & read_vec, int pos, heap_object_t *obj, heap_write_t *wr,
uint32_t prepare_read_simple(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end);
void prepare_disk_read(std::vector<copy_buffer_t> & read_vec, int pos, heap_entry_t *obj, heap_entry_t *wr,
uint32_t blk_start, uint32_t blk_end, uint32_t start, uint32_t end, uint32_t copy_flags);
void find_holes(std::vector<copy_buffer_t> & read_vec, uint32_t item_start, uint32_t item_end,
std::function<void(int&, uint32_t, uint32_t)> callback);
@@ -167,9 +167,10 @@ public:
// Write
bool enqueue_write(blockstore_op_t *op);
void prepare_meta_block_write(blockstore_op_t *op, uint64_t modified_block, io_uring_sqe *sqe = NULL);
void prepare_meta_block_write(uint32_t modified_block);
bool meta_block_is_pending(uint32_t modified_block);
bool intent_write_allowed(blockstore_op_t *op, heap_entry_t *obj);
int dequeue_write(blockstore_op_t *op);
int make_big_write(blockstore_op_t *op, uint32_t offset, uint32_t len, uint32_t *modified_block, uint32_t *moved_from_block);
int continue_write(blockstore_op_t *op);
void handle_write_event(ring_data_t *data, blockstore_op_t *op);
src/blockstore/blockstore_init.cpp
+5 -3
View File
@@ -79,7 +79,6 @@ resume_1:
hdr->meta_block_size = bs->dsk.meta_block_size;
hdr->data_block_size = bs->dsk.data_block_size;
hdr->bitmap_granularity = bs->dsk.bitmap_granularity;
hdr->compacted_lsn = 0;
if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_V2)
{
hdr->data_csum_type = bs->dsk.data_csum_type;
@@ -156,7 +155,6 @@ resume_1:
bs->dsk.check_lengths();
}
bs->init();
bs->heap->set_compacted_lsn(((blockstore_meta_header_v3_t *)bs->meta_superblock)->compacted_lsn);
if (bs->dsk.inmemory_journal)
{
// Read buffer area
@@ -224,7 +222,11 @@ resume_4:
if (bufs[i].state == INIT_META_READ_DONE)
{
// Handle result
entries_loaded += bs->heap->load_blocks(bufs[i].offset-bs->dsk.meta_block_size, bufs[i].size, bufs[i].buf);
uint64_t loaded = 0;
int r = bs->heap->load_blocks(bufs[i].offset-bs->dsk.meta_block_size, bufs[i].size, bufs[i].buf, loaded);
if (r != 0)
exit(1);
entries_loaded += loaded;
bufs[i].state = 0;
bs->ringloop->wakeup();
}
src/blockstore/blockstore_read.cpp
+62 -36
View File
@@ -7,7 +7,7 @@
int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
{
heap_object_t *obj = heap->lock_and_read_entry(op->oid, PRIV(op)->lsn);
heap_entry_t *obj = heap->lock_and_read_entry(op->oid);
if (!obj)
{
op->version = 0;
@@ -20,11 +20,11 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
auto & rv = PRIV(op)->read_vec;
uint64_t result_version = 0;
bool found = false;
for (auto wr = obj->get_writes(); wr; wr = wr->next())
heap->iterate_with_stable(obj, obj->lsn, [&](heap_entry_t *wr, bool stable)
{
if (op->version < wr->version)
{
continue;
return true;
}
if (!found)
{
@@ -37,16 +37,16 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
}
fulfilled += prepare_read(PRIV(op)->read_vec, obj, wr, op->offset, op->offset+op->len);
if (fulfilled == op->len ||
wr->type() == BS_HEAP_BIG_WRITE ||
wr->type() == BS_HEAP_TOMBSTONE)
wr->type() == BS_HEAP_BIG_WRITE || wr->type() == BS_HEAP_DELETE)
{
break;
return false;
}
}
return true;
});
if (!found)
{
// May happen if there are entries but all of them are > requested version
heap->unlock_entry(op->oid, PRIV(op)->lsn);
heap->unlock_entry(op->oid);
op->version = 0;
op->retval = -ENOENT;
FINISH_OP(op);
@@ -56,7 +56,7 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
if (!fulfill_read(op))
{
// Need to wait. undo added requests, unlock lsn
heap->unlock_entry(op->oid, PRIV(op)->lsn);
heap->unlock_entry(op->oid);
free_read_buffers(rv);
rv.clear();
return 0;
@@ -65,7 +65,7 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *op)
if (!PRIV(op)->pending_ops)
{
// everything is fulfilled from memory
heap->unlock_entry(op->oid, PRIV(op)->lsn);
heap->unlock_entry(op->oid);
op->retval = op->len;
free_read_buffers(rv);
FINISH_OP(op);
@@ -108,20 +108,20 @@ int blockstore_impl_t::fulfill_read(blockstore_op_t *op)
return 1;
}
uint32_t blockstore_impl_t::prepare_read(std::vector<copy_buffer_t> & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end)
uint32_t blockstore_impl_t::prepare_read(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end)
{
if (wr->type() == BS_HEAP_BIG_WRITE)
{
return prepare_read_with_bitmaps(read_vec, obj, wr, start, end);
}
if (wr->type() == BS_HEAP_TOMBSTONE)
if (wr->type() == BS_HEAP_DELETE)
{
return prepare_read_zero(read_vec, start, end);
}
return prepare_read_simple(read_vec, obj, wr, start, end);
}
uint32_t blockstore_impl_t::prepare_read_with_bitmaps(std::vector<copy_buffer_t> & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end)
uint32_t blockstore_impl_t::prepare_read_with_bitmaps(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end)
{
// BIG_WRITEs contain a bitmap and we have to handle its holes
uint32_t res = 0;
@@ -166,7 +166,7 @@ uint32_t blockstore_impl_t::prepare_read_zero(std::vector<copy_buffer_t> & read_
return res;
}
uint32_t blockstore_impl_t::prepare_read_simple(std::vector<copy_buffer_t> & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end)
uint32_t blockstore_impl_t::prepare_read_simple(std::vector<copy_buffer_t> & read_vec, heap_entry_t *obj, heap_entry_t *wr, uint32_t start, uint32_t end)
{
uint32_t res = 0;
if (wr->type() == BS_HEAP_SMALL_WRITE || wr->type() == BS_HEAP_INTENT_WRITE)
@@ -212,10 +212,14 @@ uint32_t blockstore_impl_t::prepare_read_simple(std::vector<copy_buffer_t> & rea
uint32_t skip_csum = 0;
if (!perfect_csum_update && wr->type() == BS_HEAP_BIG_WRITE)
{
for (auto owr = obj->get_writes(); owr && owr != wr; owr = owr->next())
for (auto owr = obj; owr && owr != wr; owr = heap->prev(owr))
{
if ((owr->type() == BS_HEAP_INTENT_WRITE || owr->type() == BS_HEAP_SMALL_WRITE) &&
owr->small().offset < blk_end && owr->small().offset+owr->small().len > blk_start)
{
skip_csum = COPY_BUF_SKIP_CSUM;
}
}
}
if ((blk_end-1)/dsk.csum_block_size == blk_start/dsk.csum_block_size ||
blk_end/dsk.csum_block_size == blk_start/dsk.csum_block_size+1 && blk_end != end && blk_start != start ||
@@ -243,18 +247,34 @@ uint32_t blockstore_impl_t::prepare_read_simple(std::vector<copy_buffer_t> & rea
return res;
}
void blockstore_impl_t::prepare_disk_read(std::vector<copy_buffer_t> & read_vec, int pos, heap_object_t *obj, heap_write_t *wr,
void blockstore_impl_t::prepare_disk_read(std::vector<copy_buffer_t> & read_vec, int pos, heap_entry_t *obj, heap_entry_t *wr,
uint32_t blk_start, uint32_t blk_end, uint32_t start, uint32_t end, uint32_t copy_flags)
{
// Only one INTENT_WRITE is allowed at a time
assert(wr->type() != BS_HEAP_INTENT_WRITE || wr->next()->type() == BS_HEAP_BIG_WRITE);
uint64_t loc = 0;
if (wr->type() == BS_HEAP_INTENT_WRITE)
{
heap_entry_t *big_wr = wr;
while (big_wr && big_wr->type() == BS_HEAP_INTENT_WRITE)
{
big_wr = heap->prev(big_wr);
}
assert(big_wr);
loc = big_wr->big_location(heap);
}
else if (wr->type() == BS_HEAP_SMALL_WRITE)
{
loc = wr->small().location-wr->small().offset;
}
else /*if (wr->type() == BS_HEAP_BIG_WRITE)*/
{
loc = wr->big_location(heap);
}
copy_buffer_t vec = {
.copy_flags = (wr->type() == BS_HEAP_SMALL_WRITE ? COPY_BUF_JOURNAL : COPY_BUF_DATA) | copy_flags,
.offset = start,
.len = end-start,
.disk_loc = (wr->type() == BS_HEAP_INTENT_WRITE ? wr->next()->big_location(heap)
: (wr->type() == BS_HEAP_SMALL_WRITE ? wr->small().location-wr->small().offset
: wr->big_location(heap))),
.disk_loc = loc,
.disk_offset = blk_start,
.disk_len = blk_end - blk_start,
.wr_lsn = wr->lsn,
@@ -349,7 +369,7 @@ void blockstore_impl_t::handle_read_event(ring_data_t *data, blockstore_op_t *op
op->retval = -EDOM;
else if (op->retval == 0)
op->retval = op->len;
heap->unlock_entry(op->oid, PRIV(op)->lsn);
heap->unlock_entry(op->oid);
free_read_buffers(PRIV(op)->read_vec);
FINISH_OP(op);
}
@@ -357,7 +377,7 @@ void blockstore_impl_t::handle_read_event(ring_data_t *data, blockstore_op_t *op
bool blockstore_impl_t::verify_read_checksums(blockstore_op_t *op)
{
heap_object_t *obj = heap->read_locked_entry(op->oid, PRIV(op)->lsn);
heap_entry_t *obj = heap->read_entry(op->oid);
auto & rv = PRIV(op)->read_vec;
for (auto & vec: rv)
{
@@ -367,9 +387,9 @@ bool blockstore_impl_t::verify_read_checksums(blockstore_op_t *op)
memcpy(op->buf + vec.offset - op->offset, vec.buf + vec.offset - vec.disk_offset, vec.len);
if (vec.copy_flags & (COPY_BUF_COALESCED|COPY_BUF_SKIP_CSUM))
continue;
heap_write_t *wr = obj->get_writes();
heap_entry_t *wr = obj;
while (wr && wr->lsn != vec.wr_lsn)
wr = wr->next();
wr = heap->prev(wr);
assert(wr);
uint8_t *buf = vec.buf ? vec.buf : (op->buf + vec.offset - op->offset);
uint32_t *csums = (uint32_t*)(wr->get_checksums(heap)
@@ -394,23 +414,29 @@ bool blockstore_impl_t::verify_read_checksums(blockstore_op_t *op)
int blockstore_impl_t::read_bitmap(object_id oid, uint64_t target_version, void *bitmap, uint64_t *result_version)
{
heap_object_t *obj = heap->read_entry(oid, NULL);
heap_entry_t *obj = heap->read_entry(oid);
if (obj)
{
for (auto wr = obj->get_writes(); wr; wr = wr->next())
bool found = false;
heap->iterate_with_stable(obj, obj->lsn, [&](heap_entry_t *wr, bool stable)
{
if (target_version < wr->version)
if (target_version >= wr->version)
{
continue;
}
if (result_version)
{
*result_version = wr->version;
}
if (bitmap)
{
memcpy(bitmap, wr->get_ext_bitmap(heap), dsk.clean_entry_bitmap_size);
found = true;
if (result_version)
{
*result_version = wr->version;
}
if (bitmap)
{
memcpy(bitmap, wr->get_ext_bitmap(heap), dsk.clean_entry_bitmap_size);
}
return false;
}
return true;
});
if (found)
{
return 0;
}
}
src/blockstore/blockstore_stable.cpp
+57 -54
View File
@@ -13,73 +13,76 @@ int blockstore_impl_t::dequeue_stable(blockstore_op_t *op)
else if (priv->op_state == 2) goto resume_2;
else if (priv->op_state == 3) goto resume_3;
else if (priv->op_state == 4) goto resume_4;
else if (priv->op_state == 5) goto resume_5;
assert(!priv->op_state);
// Modify in-memory state and assign contiguous LSNs
priv->stab_pos = 0;
priv->lsn = priv->to_lsn = 0;
op->retval = 0;
while (priv->stab_pos < op->len)
priv->modified_block = priv->modified_block2 = UINT32_MAX;
for (priv->stab_pos = 0; priv->stab_pos < op->len; priv->stab_pos++)
{
uint32_t modified_block = 0;
uint64_t new_lsn = 0;
uint64_t new_to_lsn = 0;
int res = op->opcode == BS_OP_STABLE
? heap->post_stabilize(v[priv->stab_pos].oid, v[priv->stab_pos].version, &modified_block, &new_lsn, &new_to_lsn)
: heap->post_rollback(v[priv->stab_pos].oid, v[priv->stab_pos].version, &new_lsn, &modified_block);
if (res != 0)
{
assert(res == ENOENT || res == EBUSY);
op->retval = -res;
}
if (new_lsn)
{
assert(priv->lsn == 0 || priv->to_lsn == new_lsn-1);
if (!priv->lsn)
priv->lsn = new_lsn;
priv->to_lsn = op->opcode == BS_OP_STABLE ? new_to_lsn : new_lsn;
}
priv->stab_pos++;
}
// Submit metadata writes
priv->stab_pos = 0;
resume_1:
priv->op_state = 1;
while (priv->stab_pos < op->len)
{
uint32_t block_num = 0;
heap_object_t *obj = heap->read_entry(v[priv->stab_pos].oid, &block_num);
if (obj)
{
io_uring_sqe *sqe = get_sqe();
if (!sqe)
auto obj = heap->read_entry(v[priv->stab_pos].oid);
if (!obj)
{
if (priv->pending_ops > 0)
return 1;
priv->wait_detail = 1;
priv->wait_for = WAIT_SQE;
op->retval = -ENOENT;
FINISH_OP(op);
return 2;
}
int res = op->opcode == BS_OP_STABLE
? heap->add_commit(obj, v[priv->stab_pos].version, &priv->modified_block2)
: heap->add_rollback(obj, v[priv->stab_pos].version, &priv->modified_block2);
if (res == EBUSY)
{
op->retval = -EBUSY;
FINISH_OP(op);
return 2;
}
if (res == ENOSPC)
{
if (!heap->get_inflight_queue_size())
{
// no space
op->retval = -ENOSPC;
FINISH_OP(op);
return 2;
}
if (priv->modified_block2 != UINT32_MAX)
{
priv->stab_pos--;
goto resume_1;
}
priv->wait_for = WAIT_COMPACTION;
priv->wait_detail = flusher->get_compact_counter();
flusher->request_trim();
return 0;
}
prepare_meta_block_write(op, block_num, sqe);
assert(res == 0);
}
if (priv->modified_block != UINT32_MAX && priv->modified_block2 != priv->modified_block)
{
resume_1:
BS_SUBMIT_CHECK_SQES(1);
prepare_meta_block_write(priv->modified_block);
resume_2:
if (meta_block_is_pending(priv->modified_block))
{
priv->op_state = 2;
return 1;
}
}
priv->modified_block = priv->modified_block2;
if (priv->stab_pos == op->len-1 && priv->modified_block2 != UINT32_MAX)
{
priv->modified_block2 = UINT32_MAX;
goto resume_1;
}
priv->stab_pos++;
}
if (priv->pending_ops > 0)
{
priv->op_state = 1;
return 1;
}
// Mark writes as completed to allow compaction
for (uint64_t lsn = priv->lsn; lsn <= priv->to_lsn; lsn++)
{
heap->mark_lsn_completed(lsn);
}
unsynced_meta_write_count++;
// Fsync, just because our semantics imply that commit (stabilize) is immediately fsynced
priv->op_state = 2;
resume_2:
priv->op_state = 3;
resume_3:
resume_4:
int res = do_sync(op, 2);
resume_5:
int res = do_sync(op, 3);
if (res != 2)
{
return res;
src/blockstore/blockstore_sync.cpp
+2 -2
View File
@@ -96,7 +96,7 @@ int blockstore_impl_t::do_sync(blockstore_op_t *op, int base_state)
unsynced_big_write_count = unsynced_small_write_count = unsynced_meta_write_count = 0;
return 2;
}
PRIV(op)->lsn = heap->get_completed_lsn();
PRIV(op)->modified_block = heap->get_completed_lsn();
if (!submit_fsyncs(PRIV(op)->pending_ops))
{
PRIV(op)->wait_detail = 1;
@@ -110,6 +110,6 @@ resume_1:
return 1;
}
resume_2:
heap->mark_lsn_fsynced(PRIV(op)->lsn);
heap->mark_lsn_fsynced(PRIV(op)->modified_block);
return 2;
}
src/blockstore/blockstore_write.cpp
+127 -110
View File
@@ -11,23 +11,108 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
return true;
}
void blockstore_impl_t::prepare_meta_block_write(blockstore_op_t *op, uint64_t modified_block, io_uring_sqe *sqe)
void blockstore_impl_t::prepare_meta_block_write(uint32_t modified_block)
{
if (!sqe)
//assert(modified_blocks.find(modified_block) == modified_blocks.end());
for (auto & block_num: pending_modified_blocks)
{
sqe = get_sqe();
assert(sqe != NULL);
if (block_num == modified_block)
return;
}
io_uring_sqe *sqe = get_sqe();
assert(sqe != NULL); // FIXME
pending_modified_blocks.push_back(modified_block);
ring_data_t *data = ((ring_data_t*)sqe->user_data);
data->iov = (struct iovec){ heap->get_meta_block(modified_block), (size_t)dsk.meta_block_size };
data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
PRIV(op)->pending_ops++;
data->callback = [this, modified_block](ring_data_t *data)
{
live = true;
if (data->res != data->iov.iov_len)
{
// FIXME: our state becomes corrupted after a write error. maybe do something better than just die
disk_error_abort("data write", data->res, data->iov.iov_len);
}
modified_blocks.erase(modified_block);
heap->complete_block_write(modified_block);
ringloop->wakeup();
};
io_uring_prep_writev(
sqe, dsk.meta_fd, &data->iov, 1, dsk.meta_offset + (modified_block+1)*dsk.meta_block_size
);
}
// First step of the write algorithm: dequeue operation and submit initial write(s)
bool blockstore_impl_t::meta_block_is_pending(uint32_t modified_block)
{
auto mb_it = modified_blocks.find(modified_block);
if (mb_it != modified_blocks.end())
return true;
for (auto & block_num: pending_modified_blocks)
{
if (block_num == modified_block)
return true;
}
return false;
}
bool blockstore_impl_t::intent_write_allowed(blockstore_op_t *op, heap_entry_t *obj)
{
// Parallel writes to the same object are forbidden so "one intent at a time" is fulfilled automatically
// Intent writes are disabled when metadata fsync is enabled
if (!dsk.disable_meta_fsync)
{
return false;
}
// Intent writes are only for replication
if (op->opcode != BS_OP_WRITE_STABLE)
{
return false;
}
// Operation size should be less than or equal to atomic write size
if (!op->len || op->len > dsk.atomic_write_size)
{
return false;
}
// Intent-writes are disabled if "absolutely correct during compaction" checksum validation algorithm is enabled
// We could also do RMW here when perfect_csum_update is enabled, but it's unclear if we need it
if (perfect_csum_update && dsk.csum_block_size > dsk.bitmap_granularity &&
((op->offset % dsk.csum_block_size) || (op->len % dsk.csum_block_size)))
{
return false;
}
bool ok = true, has_intent = false;
heap->iterate_with_stable(obj, obj->lsn, [&](heap_entry_t *wr, bool stable)
{
// Intent writes are not allowed over buffered writes
if (wr->type() == BS_HEAP_SMALL_WRITE)
{
ok = false;
return false;
}
// Intent writes are not allowed over unstable writes
if (!stable)
{
ok = false;
return false;
}
// One intent-write is allowed even with fsyncs because BIG_WRITE is always counted as fsynced
if (dsk.disable_data_fsync && wr->type() == BS_HEAP_INTENT_WRITE)
{
if (has_intent)
{
ok = false;
return false;
}
has_intent = true;
}
if (wr->type() == BS_HEAP_BIG_WRITE)
{
return false;
}
return true;
});
return ok;
}
int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
{
if (PRIV(op)->op_state)
@@ -38,13 +123,13 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
{
return 0;
}
PRIV(op)->modified_block = UINT32_MAX;
PRIV(op)->is_big = false;
uint32_t modified_block = UINT32_MAX, moved_from_block = UINT32_MAX;
heap_object_t *obj = heap->read_entry(op->oid, &modified_block);
heap_entry_t *obj = heap->read_entry(op->oid);
if (op->opcode == BS_OP_DELETE)
{
// Delete
if (!obj)
if (!obj || obj->type() == BS_HEAP_DELETE)
{
// Already deleted
op->retval = 0;
@@ -52,24 +137,23 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
return 2;
}
BS_SUBMIT_CHECK_SQES(1);
int res = heap->post_delete(modified_block, obj, &PRIV(op)->lsn);
int res = heap->add_delete(obj, &PRIV(op)->modified_block);
assert(res == 0);
prepare_meta_block_write(op, modified_block);
prepare_meta_block_write(PRIV(op)->modified_block);
PRIV(op)->pending_ops++;
PRIV(op)->op_state = 5;
write_iodepth++;
}
// FIXME: Add 'big_intent' write mode
// FIXME: Allow to do initial writes as buffered, not redirected
// FIXME: Allow to do direct writes over holes
else if (!obj || obj->get_writes()->type() == BS_HEAP_TOMBSTONE ||
op->offset == 0 && op->len == dsk.data_block_size)
else if (!obj || obj->type() == BS_HEAP_DELETE || op->offset == 0 && op->len == dsk.data_block_size)
{
// Big (redirect) write
BS_SUBMIT_CHECK_SQES(1);
PRIV(op)->is_big = true;
uint32_t tmp_block;
uint64_t loc = heap->find_free_data();
if (loc == UINT64_MAX ||
!obj && heap->get_block_for_new_object(tmp_block) != 0)
if (loc == UINT64_MAX)
{
if (!heap->get_inflight_queue_size())
{
@@ -91,17 +175,6 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
);
#endif
heap->use_data(op->oid.inode, PRIV(op)->location);
if (!dsk.disable_data_fsync && dsk.disable_meta_fsync)
{
// Do big_write as an INTENT to avoid data fsync
int res = make_big_write(op, 0, 0, &modified_block, &moved_from_block);
assert(res == 0);
if (moved_from_block != UINT32_MAX)
prepare_meta_block_write(op, moved_from_block);
obj = heap->read_entry(op->oid, &modified_block);
heap->mark_lsn_completed(PRIV(op)->lsn);
goto process_intent;
}
io_uring_sqe *sqe = get_sqe();
ring_data_t *data = ((ring_data_t*)sqe->user_data);
uint64_t stripe_offset = (op->offset % dsk.bitmap_granularity);
@@ -128,46 +201,19 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
write_iodepth++;
inflight_big++;
}
// Only one INTENT_WRITE is allowed at a time, but in fact,
// parallel writes to the same object are forbidden anyway
else if (op->len > 0 && op->len <= dsk.atomic_write_size &&
// Intent-writes are disabled if "absolutely correct during compaction" checksum validation algorithm is enabled
// We could also do RMW here when perfect_csum_update is enabled, but it's unclear if we need it
(!perfect_csum_update || dsk.csum_block_size <= dsk.bitmap_granularity ||
!(op->offset % dsk.csum_block_size) &&
!(op->len % dsk.csum_block_size) &&
(obj->get_writes()->entry_type != (BS_HEAP_INTENT_WRITE|BS_HEAP_STABLE) ||
obj->get_writes()->can_be_collapsed(heap))) &&
// One intent-write is allowed even with fsyncs because BIG_WRITE is always counted as fsynced
dsk.disable_meta_fsync &&
(op->opcode == BS_OP_WRITE_STABLE &&
(obj->get_writes()->entry_type == (BS_HEAP_BIG_WRITE|BS_HEAP_STABLE) ||
obj->get_writes()->entry_type == (BS_HEAP_INTENT_WRITE|BS_HEAP_STABLE) && dsk.disable_data_fsync) ||
op->opcode == BS_OP_WRITE && obj->get_writes()->entry_type == BS_HEAP_BIG_WRITE))
else if (intent_write_allowed(op, obj))
{
// Direct intent-write
BS_SUBMIT_CHECK_SQES(1);
if (obj->get_writes()->type() == BS_HEAP_BIG_WRITE)
auto wr = obj;
while (wr && (wr->type() == BS_HEAP_INTENT_WRITE || wr->type() == BS_HEAP_COMMIT || wr->type() == BS_HEAP_ROLLBACK))
{
PRIV(op)->location = obj->get_writes()->big_location(heap);
wr = heap->prev(wr);
}
else
{
assert(obj->get_writes()->next()->type() == BS_HEAP_BIG_WRITE);
PRIV(op)->location = obj->get_writes()->next()->big_location(heap);
}
process_intent:
uint8_t wr_buf[heap->get_max_write_entry_size()];
heap_write_t *wr = (heap_write_t*)wr_buf;
wr->version = op->version;
wr->entry_type = BS_HEAP_INTENT_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0);
wr->small().offset = op->offset;
wr->small().len = op->len;
wr->small().location = 0;
if (op->bitmap)
memcpy(wr->get_ext_bitmap(heap), op->bitmap, dsk.clean_entry_bitmap_size);
heap->calc_checksums(wr, (uint8_t*)op->buf, true);
int res = heap->post_write(modified_block, op->oid, obj, wr, &moved_from_block);
assert(wr->type() == BS_HEAP_BIG_WRITE);
PRIV(op)->location = wr->big_location(heap);
int res = heap->add_small_write(op->oid, obj, (BS_HEAP_INTENT_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0)),
op->version, op->offset, op->len, 0, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
if (res == EAGAIN)
{
assert(heap->get_inflight_queue_size());
@@ -184,10 +230,8 @@ process_intent:
return 2;
}
assert(res == 0);
PRIV(op)->lsn = wr->lsn;
if (moved_from_block != UINT32_MAX)
prepare_meta_block_write(op, moved_from_block);
prepare_meta_block_write(op, modified_block);
prepare_meta_block_write(PRIV(op)->modified_block);
PRIV(op)->pending_ops++;
PRIV(op)->op_state = 9;
write_iodepth++;
}
@@ -205,18 +249,8 @@ process_intent:
}
// There is sufficient space. Check SQE(s)
BS_SUBMIT_CHECK_SQES(1 + (op->len > 0 ? 1 : 0));
uint8_t wr_buf[heap->get_max_write_entry_size()];
heap_write_t *wr = (heap_write_t*)wr_buf;
wr->version = op->version;
wr->entry_type = BS_HEAP_SMALL_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0);
wr->small().offset = op->offset;
wr->small().len = op->len;
wr->small().location = loc;
PRIV(op)->location = loc;
if (op->bitmap)
memcpy(wr->get_ext_bitmap(heap), op->bitmap, dsk.clean_entry_bitmap_size);
heap->calc_checksums(wr, (uint8_t*)op->buf, true);
int res = heap->post_write(modified_block, op->oid, obj, wr, &moved_from_block);
int res = heap->add_small_write(op->oid, obj, (BS_HEAP_SMALL_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0)),
op->version, op->offset, op->len, loc, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
if (res == EAGAIN)
{
assert(heap->get_inflight_queue_size());
@@ -233,12 +267,10 @@ process_intent:
return 2;
}
assert(res == 0);
PRIV(op)->lsn = wr->lsn;
if (op->len)
heap->use_buffer_area(op->oid.inode, loc, op->len);
if (moved_from_block != UINT32_MAX)
prepare_meta_block_write(op, moved_from_block);
prepare_meta_block_write(op, modified_block);
prepare_meta_block_write(PRIV(op)->modified_block);
PRIV(op)->pending_ops++;
if (op->len > 0)
{
// Prepare buffered data write
@@ -263,26 +295,6 @@ process_intent:
return 1;
}
int blockstore_impl_t::make_big_write(blockstore_op_t *op, uint32_t offset, uint32_t len, uint32_t *modified_block, uint32_t *moved_from_block)
{
uint8_t wr_buf[heap->get_max_write_entry_size()];
heap_write_t *wr = (heap_write_t*)wr_buf;
wr->entry_type = BS_HEAP_BIG_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0);
wr->version = op->version;
wr->set_big_location(heap, PRIV(op)->location);
if (op->bitmap)
memcpy(wr->get_ext_bitmap(heap), op->bitmap, dsk.clean_entry_bitmap_size);
memset(wr->get_int_bitmap(heap), 0, dsk.clean_entry_bitmap_size);
bitmap_set(wr->get_int_bitmap(heap), offset, len, dsk.bitmap_granularity);
heap->calc_checksums(wr, (uint8_t*)op->buf, true, offset, len);
int res = heap->post_write(op->oid, wr, modified_block, moved_from_block);
if (res != 0)
return res;
assert(res == 0);
PRIV(op)->lsn = wr->lsn;
return 0;
}
int blockstore_impl_t::continue_write(blockstore_op_t *op)
{
int op_state = PRIV(op)->op_state;
@@ -305,6 +317,12 @@ again:
{
// In progress
assert(op_state < 10);
if (PRIV(op)->modified_block != UINT32_MAX &&
!meta_block_is_pending(PRIV(op)->modified_block))
{
PRIV(op)->pending_ops--;
PRIV(op)->modified_block = UINT32_MAX;
}
if (PRIV(op)->pending_ops > 0)
return 1;
op_state++;
@@ -313,7 +331,6 @@ again:
resume_2:
// We must fsync all big writes to avoid complex write workflows
// It's OK for all HDDs and for server SSDs, but slightly worse for desktop SSDs
// The other way is to add another type of MVCC to blockstore_heap: "forward" MVCC :)
inflight_big--;
if (!dsk.disable_data_fsync)
{
@@ -349,8 +366,9 @@ resume_12:
}
resume_4:
{
uint32_t modified_block = UINT32_MAX, moved_from_block = UINT32_MAX;
int res = make_big_write(op, op->offset, op->len, &modified_block, &moved_from_block);
auto obj = heap->read_entry(op->oid);
int res = heap->add_big_write(op->oid, obj, (op->opcode == BS_OP_WRITE_STABLE), op->version,
op->offset, op->len, PRIV(op)->location, op->bitmap, (uint8_t*)op->buf, &PRIV(op)->modified_block);
if (res == EAGAIN)
{
assert(heap->get_inflight_queue_size());
@@ -367,9 +385,9 @@ resume_4:
FINISH_OP(op);
return 2;
}
if (moved_from_block != UINT32_MAX)
prepare_meta_block_write(op, moved_from_block);
prepare_meta_block_write(op, modified_block);
assert(res == 0);
prepare_meta_block_write(PRIV(op)->modified_block);
PRIV(op)->pending_ops++;
PRIV(op)->op_state = 5;
return 1;
}
@@ -411,7 +429,6 @@ resume_8:
printf("Ack write %jx:%jx v%ju\n", op->oid.inode, op->oid.stripe, op->version);
#endif
op->retval = op->len;
heap->mark_lsn_completed(PRIV(op)->lsn);
if (PRIV(op)->is_big)
unsynced_big_write_count++;
else