700 lines
23 KiB
C++
700 lines
23 KiB
C++
// Copyright (c) Vitaliy Filippov, 2019+
|
|
// License: VNPL-1.1 (see README.md for details)
|
|
|
|
#include "blockstore_impl.h"
|
|
#include "blockstore_internal.h"
|
|
#include "crc32c.h"
|
|
#include "allocator.h"
|
|
|
|
#define META_BLOCK_UNREAD 0
|
|
#define META_BLOCK_READ 1
|
|
|
|
// FIXME rename to compactor_t
|
|
journal_flusher_t::journal_flusher_t(blockstore_impl_t *bs)
|
|
{
|
|
this->bs = bs;
|
|
this->max_flusher_count = bs->max_flusher_count;
|
|
this->min_flusher_count = bs->min_flusher_count;
|
|
this->cur_flusher_count = bs->min_flusher_count;
|
|
this->target_flusher_count = bs->min_flusher_count;
|
|
active_flushers = 0;
|
|
syncing_flushers = 0;
|
|
advance_lsn_counter = 0;
|
|
co = new journal_flusher_co[max_flusher_count];
|
|
for (int i = 0; i < max_flusher_count; i++)
|
|
{
|
|
co[i].co_id = i;
|
|
co[i].bs = bs;
|
|
co[i].flusher = this;
|
|
if (bs->dsk.csum_block_size)
|
|
co[i].csum_buf = (uint8_t*)malloc_or_die(bs->dsk.data_block_size/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF));
|
|
}
|
|
}
|
|
|
|
journal_flusher_co::journal_flusher_co()
|
|
{
|
|
wait_state = 0;
|
|
simple_callback_r = [this](ring_data_t* data)
|
|
{
|
|
bs->live = true;
|
|
if (data->res != data->iov.iov_len)
|
|
bs->disk_error_abort("read operation during flush", data->res, data->iov.iov_len);
|
|
wait_count--;
|
|
};
|
|
simple_callback_w = [this](ring_data_t* data)
|
|
{
|
|
bs->live = true;
|
|
if (data->res != data->iov.iov_len)
|
|
bs->disk_error_abort("write operation during flush", data->res, data->iov.iov_len);
|
|
wait_count--;
|
|
};
|
|
}
|
|
|
|
journal_flusher_t::~journal_flusher_t()
|
|
{
|
|
delete[] co;
|
|
}
|
|
|
|
journal_flusher_co::~journal_flusher_co()
|
|
{
|
|
if (csum_buf)
|
|
{
|
|
free(csum_buf);
|
|
}
|
|
}
|
|
|
|
int journal_flusher_t::get_active()
|
|
{
|
|
return active_flushers;
|
|
}
|
|
|
|
uint64_t journal_flusher_t::get_counter()
|
|
{
|
|
return compact_counter;
|
|
}
|
|
|
|
bool journal_flusher_t::is_active()
|
|
{
|
|
return active_flushers > 0 || bs->heap->get_compact_queue_size() > (force_start > 0 ? 0 : bs->flusher_start_threshold);
|
|
}
|
|
|
|
void journal_flusher_t::request_trim()
|
|
{
|
|
force_start++;
|
|
bs->ringloop->wakeup();
|
|
}
|
|
|
|
void journal_flusher_t::release_trim()
|
|
{
|
|
force_start--;
|
|
}
|
|
|
|
void journal_flusher_t::dump_diagnostics()
|
|
{
|
|
printf(
|
|
"Compaction queue: %u items, data: %ju/%ju blocks used, meta: %ju/%ju bytes used, %u/%ju blocks nearfull\n",
|
|
bs->heap->get_compact_queue_size(),
|
|
bs->heap->get_data_used_space()/bs->dsk.data_block_size, bs->dsk.block_count,
|
|
bs->heap->get_meta_used_space(), bs->heap->get_meta_total_space(),
|
|
bs->heap->get_meta_nearfull_blocks(), bs->dsk.meta_area_size/bs->dsk.meta_block_size-1
|
|
);
|
|
}
|
|
|
|
void journal_flusher_t::loop()
|
|
{
|
|
target_flusher_count = bs->write_iodepth*2;
|
|
if (target_flusher_count < min_flusher_count)
|
|
target_flusher_count = min_flusher_count;
|
|
else if (target_flusher_count > max_flusher_count)
|
|
target_flusher_count = max_flusher_count;
|
|
if (target_flusher_count > cur_flusher_count)
|
|
cur_flusher_count = target_flusher_count;
|
|
else if (target_flusher_count < cur_flusher_count)
|
|
{
|
|
while (target_flusher_count < cur_flusher_count)
|
|
{
|
|
if (co[cur_flusher_count-1].wait_state)
|
|
break;
|
|
cur_flusher_count--;
|
|
}
|
|
}
|
|
int prev_active = active_flushers;
|
|
for (int i = 0; is_active() && i < cur_flusher_count; i++)
|
|
co[i].loop();
|
|
if (prev_active && !active_flushers && force_start > 0)
|
|
bs->ringloop->wakeup();
|
|
}
|
|
|
|
#define await_sqe(label) \
|
|
resume_##label:\
|
|
sqe = bs->get_sqe();\
|
|
if (!sqe)\
|
|
{\
|
|
wait_state = wait_base+label;\
|
|
return false;\
|
|
}\
|
|
data = ((ring_data_t*)sqe->user_data);
|
|
|
|
bool journal_flusher_co::loop()
|
|
{
|
|
int wait_base = 0;
|
|
// This is much better than implementing the whole function as an FSM
|
|
// Maybe I should consider a coroutine library like https://github.com/hnes/libaco ...
|
|
// Or just C++ coroutines, but they require some wrappers
|
|
if (wait_state == 1) goto resume_1;
|
|
else if (wait_state == 2) goto resume_2;
|
|
else if (wait_state == 3) goto resume_3;
|
|
else if (wait_state == 4) goto resume_4;
|
|
else if (wait_state == 5) goto resume_5;
|
|
else if (wait_state == 6) goto resume_6;
|
|
else if (wait_state == 7) goto resume_7;
|
|
else if (wait_state == 8) goto resume_8;
|
|
else if (wait_state == 9) goto resume_9;
|
|
else if (wait_state == 10) goto resume_10;
|
|
else if (wait_state == 11) goto resume_11;
|
|
else if (wait_state == 12) goto resume_12;
|
|
else if (wait_state == 13) goto resume_13;
|
|
else if (wait_state == 14) goto resume_14;
|
|
else if (wait_state == 15) goto resume_15;
|
|
else if (wait_state == 16) goto resume_16;
|
|
else if (wait_state == 17) goto resume_17;
|
|
resume_0:
|
|
res = bs->heap->get_next_compact(cur_oid);
|
|
if (res == ENOENT)
|
|
{
|
|
cur_oid = {};
|
|
wait_state = 0;
|
|
return true;
|
|
}
|
|
for (int i = 0; i < flusher->cur_flusher_count; i++)
|
|
{
|
|
if (i != co_id && flusher->co[i].cur_oid == cur_oid)
|
|
{
|
|
// Already flushing it
|
|
flusher->co[i].should_repeat = true;
|
|
goto resume_0;
|
|
}
|
|
}
|
|
resume_1:
|
|
should_repeat = false;
|
|
cur_obj = bs->heap->lock_and_read_entry(cur_oid, copy_id);
|
|
if (!cur_obj)
|
|
{
|
|
// Object does not exist
|
|
goto resume_0;
|
|
}
|
|
cur_version = cur_obj->get_writes()->version;
|
|
// Find the range to compact
|
|
compact_lsn = bs->heap->get_completed_lsn();
|
|
bs->heap->get_compact_range(cur_obj, compact_lsn, &begin_wr, &end_wr);
|
|
if (!begin_wr)
|
|
{
|
|
// Nothing to flush
|
|
bs->heap->unlock_entry(cur_oid, copy_id);
|
|
goto resume_0;
|
|
}
|
|
assert(!end_wr->next() && end_wr->flags == (BS_HEAP_BIG_WRITE|BS_HEAP_STABLE));
|
|
clean_loc = end_wr->location;
|
|
#ifdef BLOCKSTORE_DEBUG
|
|
printf("Flushing %jx:%jx v%ju .. v%ju\n", cur_oid.inode, cur_oid.stripe, end_wr->version, begin_wr->version);
|
|
#endif
|
|
flusher->active_flushers++;
|
|
// Scan versions to flush
|
|
read_vec.clear();
|
|
for (auto wr = begin_wr; wr != end_wr; wr = wr->next())
|
|
{
|
|
min_compact_lsn = wr->lsn;
|
|
bs->prepare_read(read_vec, cur_obj, wr, 0, bs->dsk.data_block_size);
|
|
}
|
|
overwrite_start = overwrite_end = 0;
|
|
if (read_vec.size() > 0)
|
|
{
|
|
overwrite_start = read_vec[0].offset;
|
|
overwrite_end = read_vec[read_vec.size()-1].offset + read_vec[read_vec.size()-1].len;
|
|
}
|
|
read_to_fill_incomplete = false;
|
|
if (bs->dsk.csum_block_size > bs->dsk.bitmap_granularity && end_wr->next())
|
|
{
|
|
// Read original checksum blocks to calculate padded checksums if required
|
|
fill_partial_checksum_blocks();
|
|
}
|
|
// Read buffered data
|
|
cur_obj = NULL;
|
|
begin_wr = end_wr = NULL;
|
|
resume_2:
|
|
resume_3:
|
|
if (!read_buffered(2))
|
|
return false;
|
|
// Now, if csum_block_size is > bitmap_granularity and if we are doing partial checksum block updates,
|
|
// perform a trick: clear bitmap bits in the metadata entry and recalculate block checksum with zeros
|
|
// in place of overwritten parts. Then, even if the actual partial update fully or partially fails,
|
|
// we'll have a correct checksum because it won't include overwritten parts!
|
|
// The same thing actually happens even when csum_block_size == bitmap_granularity, but in that case
|
|
// we never need to read (and thus verify) overwritten parts from the data device.
|
|
res = check_and_punch_checksums();
|
|
if (res == EBUSY)
|
|
{
|
|
resume_4:
|
|
resume_5:
|
|
if (!write_meta_block(4))
|
|
return false;
|
|
resume_6:
|
|
resume_7:
|
|
resume_8:
|
|
if (!fsync_batch(true, 6)) // FIXME: is it correct to batch here
|
|
return false;
|
|
}
|
|
else if (res == ENOENT || res == EDOM)
|
|
{
|
|
// Abort compaction
|
|
goto release_oid;
|
|
}
|
|
assert(res == 0);
|
|
// Submit data writes
|
|
copy_count = 0;
|
|
for (i = 0; i < read_vec.size(); i++)
|
|
{
|
|
if (read_vec[i].copy_flags & COPY_BUF_JOURNAL)
|
|
{
|
|
assert(read_vec[i].buf);
|
|
await_sqe(9);
|
|
data->iov = (struct iovec){ read_vec[i].buf, (size_t)read_vec[i].len };
|
|
data->callback = simple_callback_w;
|
|
io_uring_prep_writev(sqe, bs->dsk.data_fd, &data->iov, 1, bs->dsk.data_offset + clean_loc + read_vec[i].offset);
|
|
wait_count++;
|
|
copy_count++;
|
|
}
|
|
}
|
|
resume_10:
|
|
if (wait_count > 0)
|
|
{
|
|
wait_state = 10;
|
|
return false;
|
|
}
|
|
// Sync data before modifying metadata
|
|
resume_11:
|
|
resume_12:
|
|
resume_13:
|
|
if (copy_count && !fsync_batch(false, 11))
|
|
return false;
|
|
bs->heap->unlock_entry(cur_oid, copy_id);
|
|
// Modify the metadata entry; don't write anything. Metadata block will be written on the next write
|
|
calc_block_checksums();
|
|
bs->heap->compact_object(cur_oid, compact_lsn, new_data_csums);
|
|
// Done, free all buffers
|
|
free_buffers();
|
|
#ifdef BLOCKSTORE_DEBUG
|
|
printf("Compacted %jx:%jx v%ju (%d writes)\n", cur_oid.inode, cur_oid.stripe, cur_version, copy_count);
|
|
#endif
|
|
// Advance compacted_lsn every <journal_trim_interval> objects
|
|
bs->heap->set_compacted_lsn(min_compact_lsn);
|
|
if (bs->journal_trim_interval && !((++flusher->advance_lsn_counter) % bs->journal_trim_interval))
|
|
{
|
|
resume_14:
|
|
resume_15:
|
|
resume_16:
|
|
resume_17:
|
|
if (!trim_lsn(14))
|
|
return false;
|
|
}
|
|
release_oid:
|
|
flusher->active_flushers--;
|
|
if (should_repeat)
|
|
{
|
|
// Flush the same object again
|
|
goto resume_1;
|
|
}
|
|
// All done
|
|
flusher->compact_counter++;
|
|
wait_state = 0;
|
|
goto resume_0;
|
|
}
|
|
|
|
void journal_flusher_co::iterate_partial_overwrites(std::function<int(int, uint32_t, uint32_t)> cb)
|
|
{
|
|
int prev = 0;
|
|
uint32_t prev_begin = 0, prev_end = 0;
|
|
for (int i = 0; i < read_vec.size() && !(read_vec[i].copy_flags & COPY_BUF_CSUM_FILL); i++)
|
|
{
|
|
if (read_vec[i].copy_flags != (COPY_BUF_JOURNAL|COPY_BUF_COALESCED))
|
|
{
|
|
if (read_vec[i].offset > prev_end)
|
|
{
|
|
i += cb(prev, prev_begin, prev_end);
|
|
prev = i;
|
|
prev_begin = read_vec[i].offset;
|
|
}
|
|
prev_end = read_vec[i].offset + read_vec[i].len;
|
|
}
|
|
}
|
|
if (prev_end > prev_begin)
|
|
{
|
|
cb(prev, prev_begin, prev_end);
|
|
}
|
|
}
|
|
|
|
void journal_flusher_co::iterate_checksum_holes(std::function<void(int, uint32_t, uint32_t)> cb)
|
|
{
|
|
iterate_partial_overwrites([&](int pos, uint32_t prev_begin, uint32_t prev_end)
|
|
{
|
|
int r = 0;
|
|
if ((prev_begin % bs->dsk.csum_block_size) &&
|
|
(prev_begin / bs->dsk.csum_block_size) != (prev_end / bs->dsk.csum_block_size))
|
|
{
|
|
cb(pos, prev_begin, prev_begin + bs->dsk.csum_block_size - prev_begin%bs->dsk.csum_block_size);
|
|
r++;
|
|
}
|
|
if ((prev_end % bs->dsk.csum_block_size) ||
|
|
(prev_begin % bs->dsk.csum_block_size) &&
|
|
(prev_end / bs->dsk.csum_block_size) == (prev_begin / bs->dsk.csum_block_size))
|
|
{
|
|
cb(i, prev_end - (prev_end % bs->dsk.csum_block_size ? (prev_end % bs->dsk.csum_block_size) : bs->dsk.csum_block_size), prev_end);
|
|
r++;
|
|
}
|
|
return r;
|
|
});
|
|
}
|
|
|
|
void journal_flusher_co::fill_partial_checksum_blocks()
|
|
{
|
|
iterate_checksum_holes([&](int vec_pos, uint32_t hole_start, uint32_t hole_end)
|
|
{
|
|
read_to_fill_incomplete = true;
|
|
int pos = read_vec.size();
|
|
bs->prepare_disk_read(read_vec, pos, cur_obj, end_wr,
|
|
hole_start - hole_start % bs->dsk.csum_block_size, hole_start - hole_start % bs->dsk.csum_block_size + bs->dsk.csum_block_size,
|
|
hole_start - hole_start % bs->dsk.csum_block_size, hole_start - hole_start % bs->dsk.csum_block_size + bs->dsk.csum_block_size);
|
|
pos--;
|
|
read_vec[pos].copy_flags |= COPY_BUF_CSUM_FILL;
|
|
read_vec.insert(read_vec.begin()+vec_pos, (copy_buffer_t){
|
|
.copy_flags = COPY_BUF_JOURNAL|COPY_BUF_COALESCED,
|
|
.offset = hole_start,
|
|
.len = hole_end-hole_start,
|
|
.buf = read_vec[pos].buf + hole_start - read_vec[pos].offset,
|
|
});
|
|
});
|
|
}
|
|
|
|
void journal_flusher_co::free_buffers()
|
|
{
|
|
for (auto it = read_vec.begin(); it != read_vec.end(); it++)
|
|
{
|
|
// Free it if it's not taken from the journal
|
|
if (it->buf && !(it->copy_flags & COPY_BUF_COALESCED) &&
|
|
(!bs->dsk.inmemory_journal || it->buf < bs->buffer_area || it->buf >= (uint8_t*)bs->buffer_area + bs->dsk.journal_len))
|
|
{
|
|
free(it->buf);
|
|
}
|
|
}
|
|
read_vec.clear();
|
|
}
|
|
|
|
// FIXME: Write tests for it
|
|
int journal_flusher_co::check_and_punch_checksums()
|
|
{
|
|
if (!bs->dsk.csum_block_size)
|
|
{
|
|
// Nothing to do
|
|
return 0;
|
|
}
|
|
// Verify data checksums
|
|
cur_obj = bs->heap->read_locked_entry(cur_oid, copy_id);
|
|
bool csum_ok = true;
|
|
for (int i = 0; i < read_vec.size(); i++)
|
|
{
|
|
auto & vec = read_vec[i];
|
|
if (!(vec.copy_flags & (COPY_BUF_COALESCED|COPY_BUF_ZERO)))
|
|
{
|
|
heap_write_t *wr = cur_obj->get_writes();
|
|
while (wr && wr->lsn != vec.wr_lsn)
|
|
wr = wr->next();
|
|
assert(wr);
|
|
bs->heap->calc_block_checksums(
|
|
(uint32_t*)wr->get_checksums(bs->heap), vec.buf, wr->get_int_bitmap(bs->heap), vec.offset, vec.offset+vec.len, false,
|
|
[&](uint32_t mismatch_pos, uint32_t expected_csum, uint32_t real_csum)
|
|
{
|
|
printf("Checksum mismatch in object %jx:%jx v%ju in %s area at offset 0x%jx: got %08x, expected %08x\n",
|
|
cur_oid.inode, cur_oid.stripe, wr->version,
|
|
(vec.copy_flags & COPY_BUF_JOURNAL ? "buffer" : "data"),
|
|
vec.disk_offset, real_csum, expected_csum);
|
|
csum_ok = false;
|
|
}
|
|
);
|
|
}
|
|
}
|
|
if (!csum_ok)
|
|
{
|
|
// Checksum error, abort compaction
|
|
// FIXME: Report the corrupted object to the upper layer
|
|
return EDOM;
|
|
}
|
|
if (!read_to_fill_incomplete)
|
|
{
|
|
// Nothing to do
|
|
return 0;
|
|
}
|
|
cur_obj = bs->heap->read_entry(cur_oid, &modified_block, true);
|
|
if (!cur_obj)
|
|
{
|
|
// Object is deleted, abort compaction
|
|
return ENOENT;
|
|
}
|
|
bs->heap->get_compact_range(cur_obj, compact_lsn, &begin_wr, &end_wr);
|
|
if (!begin_wr || begin_wr->lsn != compact_lsn)
|
|
{
|
|
// Object is overwritten, abort compaction
|
|
return ENOENT;
|
|
}
|
|
uint8_t *bmp = end_wr->get_int_bitmap(bs->heap);
|
|
uint8_t *csums = end_wr->get_checksums(bs->heap);
|
|
// Clear bits
|
|
iterate_partial_overwrites([&](int pos, uint32_t start, uint32_t end)
|
|
{
|
|
bitmap_clear(bmp, start, end-start, bs->dsk.bitmap_granularity);
|
|
return 0;
|
|
});
|
|
// Update partial block checksums
|
|
for (auto & vec: read_vec)
|
|
{
|
|
if (vec.copy_flags & COPY_BUF_CSUM_FILL)
|
|
{
|
|
uint32_t csum_off = (vec.offset/bs->dsk.csum_block_size - end_wr->offset/bs->dsk.csum_block_size) * (bs->dsk.data_csum_type & 0xFF);
|
|
bs->heap->calc_block_checksums((uint32_t*)(csums+csum_off), vec.buf, bmp, vec.offset, vec.offset+vec.len, true, NULL);
|
|
}
|
|
}
|
|
cur_obj->crc32c = cur_obj->calc_crc32c();
|
|
if (res == ENOENT)
|
|
{
|
|
// Object is deleted, abort compaction
|
|
return ENOENT;
|
|
}
|
|
// Modified, we should write the block to disk
|
|
assert(!res);
|
|
return EBUSY;
|
|
}
|
|
|
|
void journal_flusher_co::calc_block_checksums()
|
|
{
|
|
new_data_csums = NULL;
|
|
if (bs->dsk.csum_block_size <= bs->dsk.bitmap_granularity)
|
|
return;
|
|
new_data_csums = csum_buf + overwrite_start/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF);
|
|
cur_obj = bs->heap->read_locked_entry(cur_oid, copy_id);
|
|
uint64_t block_offset = 0;
|
|
uint32_t block_done = 0;
|
|
uint32_t block_csum = 0;
|
|
for (auto it = read_vec.begin(); it != read_vec.end(); it++)
|
|
{
|
|
if (it->copy_flags & COPY_BUF_CSUM_FILL)
|
|
break;
|
|
if (block_done == 0)
|
|
{
|
|
// `read_vec` should contain aligned items, possibly split into pieces
|
|
assert(!(it->offset % bs->dsk.csum_block_size));
|
|
block_offset = it->offset;
|
|
}
|
|
bool zero = (it->copy_flags & COPY_BUF_ZERO);
|
|
auto len = it->len;
|
|
while ((block_done+len) >= bs->dsk.csum_block_size)
|
|
{
|
|
if (!block_done && it->wr_lsn)
|
|
{
|
|
// We may take existing checksums if an overwrite contains a full block
|
|
heap_write_t *wr = cur_obj->get_writes();
|
|
while (wr && wr->lsn != it->wr_lsn)
|
|
wr = wr->next();
|
|
assert(wr);
|
|
assert(!(it->offset % bs->dsk.csum_block_size));
|
|
assert(!(wr->offset % bs->dsk.csum_block_size));
|
|
auto full_csum_offset = (it->offset - wr->offset) / bs->dsk.csum_block_size;
|
|
auto full_csum_count = len/bs->dsk.csum_block_size;
|
|
memcpy(new_data_csums + block_offset/bs->dsk.csum_block_size,
|
|
wr->get_checksums(bs->heap) + full_csum_offset*4, full_csum_count*4);
|
|
len -= full_csum_count*bs->dsk.csum_block_size;
|
|
block_offset += full_csum_count*bs->dsk.csum_block_size;
|
|
}
|
|
else
|
|
{
|
|
auto cur_len = bs->dsk.csum_block_size-block_done;
|
|
block_csum = zero
|
|
? crc32c_pad(block_csum, NULL, 0, cur_len, 0)
|
|
: crc32c(block_csum, (uint8_t*)it->buf+(it->len-len), cur_len);
|
|
new_data_csums[block_offset / bs->dsk.csum_block_size] = block_csum;
|
|
block_csum = 0;
|
|
block_done = 0;
|
|
block_offset += bs->dsk.csum_block_size;
|
|
len -= cur_len;
|
|
}
|
|
}
|
|
if (len > 0)
|
|
{
|
|
block_csum = zero
|
|
? crc32c_pad(block_csum, NULL, 0, len, 0)
|
|
: crc32c(block_csum, (uint8_t*)it->buf+(it->len-len), len);
|
|
block_done += len;
|
|
}
|
|
}
|
|
// `read_vec` should contain aligned items, possibly split into pieces
|
|
assert(!block_done);
|
|
}
|
|
|
|
bool journal_flusher_co::write_meta_block(int wait_base)
|
|
{
|
|
if (wait_state == wait_base)
|
|
goto resume_0;
|
|
else if (wait_state == wait_base+1)
|
|
goto resume_1;
|
|
await_sqe(0);
|
|
data->iov = (struct iovec){ bs->heap->get_meta_block(modified_block), (size_t)bs->dsk.meta_block_size };
|
|
data->callback = simple_callback_w;
|
|
io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + (modified_block+1)*bs->dsk.meta_block_size);
|
|
wait_count++;
|
|
resume_1:
|
|
if (wait_count > 0)
|
|
{
|
|
wait_state = wait_base+1;
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool journal_flusher_co::read_buffered(int wait_base)
|
|
{
|
|
if (wait_state == wait_base)
|
|
goto resume_0;
|
|
else if (wait_state == wait_base+1)
|
|
goto resume_1;
|
|
wait_count = 0;
|
|
if (bs->dsk.inmemory_journal && !read_to_fill_incomplete)
|
|
{
|
|
// Happy path: nothing to read :)
|
|
return true;
|
|
}
|
|
for (i = 0; i < read_vec.size(); i++)
|
|
{
|
|
if (read_vec[i].copy_flags == COPY_BUF_JOURNAL && !bs->dsk.inmemory_journal ||
|
|
(read_vec[i].copy_flags & COPY_BUF_DATA) && !(read_vec[i].copy_flags & COPY_BUF_COALESCED))
|
|
{
|
|
await_sqe(0);
|
|
auto & vec = read_vec[i];
|
|
data->iov = (struct iovec){ vec.buf, (size_t)vec.disk_len };
|
|
wait_count++;
|
|
io_uring_prep_readv(
|
|
sqe,
|
|
(vec.copy_flags & COPY_BUF_JOURNAL) ? bs->dsk.journal_fd : bs->dsk.data_fd,
|
|
&data->iov, 1,
|
|
((vec.copy_flags & COPY_BUF_JOURNAL) ? bs->dsk.journal_offset : bs->dsk.data_offset) + vec.disk_offset
|
|
);
|
|
data->callback = simple_callback_r;
|
|
}
|
|
}
|
|
// Wait for reads/writes if the journal is not inmemory
|
|
resume_1:
|
|
if (wait_count > 0)
|
|
{
|
|
wait_state = wait_base+1;
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool journal_flusher_co::fsync_batch(bool fsync_meta, int wait_base)
|
|
{
|
|
if (wait_state == wait_base) goto resume_0;
|
|
else if (wait_state == wait_base+1) goto resume_1;
|
|
else if (wait_state == wait_base+2) goto resume_2;
|
|
if (!(fsync_meta ? bs->disable_meta_fsync : bs->disable_data_fsync))
|
|
{
|
|
cur_sync = flusher->syncs.end();
|
|
while (cur_sync != flusher->syncs.begin())
|
|
{
|
|
cur_sync--;
|
|
if (cur_sync->fsync_meta == fsync_meta && cur_sync->state == 0)
|
|
{
|
|
goto sync_found;
|
|
}
|
|
}
|
|
cur_sync = flusher->syncs.emplace(flusher->syncs.end(), (flusher_sync_t){
|
|
.fsync_meta = fsync_meta,
|
|
.ready_count = 0,
|
|
.state = 0,
|
|
});
|
|
sync_found:
|
|
cur_sync->ready_count++;
|
|
flusher->syncing_flushers++;
|
|
resume_1:
|
|
if (!cur_sync->state)
|
|
{
|
|
if (flusher->syncing_flushers >= flusher->active_flushers || true /*FIXME*/)
|
|
{
|
|
// Sync batch is ready. Do it.
|
|
await_sqe(0);
|
|
data->iov = { 0 };
|
|
data->callback = simple_callback_w;
|
|
io_uring_prep_fsync(sqe, fsync_meta ? bs->dsk.meta_fd : bs->dsk.data_fd, IORING_FSYNC_DATASYNC);
|
|
cur_sync->state = 1;
|
|
wait_count++;
|
|
resume_2:
|
|
if (wait_count > 0)
|
|
{
|
|
wait_state = wait_base+2;
|
|
return false;
|
|
}
|
|
// Sync completed. All previous coroutines waiting for it must be resumed
|
|
cur_sync->state = 2;
|
|
bs->ringloop->wakeup();
|
|
}
|
|
else
|
|
{
|
|
// Wait until someone else sends and completes a sync.
|
|
wait_state = wait_base+1;
|
|
return false;
|
|
}
|
|
}
|
|
flusher->syncing_flushers--;
|
|
cur_sync->ready_count--;
|
|
if (cur_sync->ready_count == 0)
|
|
{
|
|
flusher->syncs.erase(cur_sync);
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool journal_flusher_co::trim_lsn(int wait_base)
|
|
{
|
|
if (wait_state == wait_base) goto resume_0;
|
|
else if (wait_state == wait_base+1) goto resume_1;
|
|
else if (wait_state == wait_base+2) goto resume_2;
|
|
else if (wait_state == wait_base+3) goto resume_3;
|
|
if (!bs->disable_meta_fsync)
|
|
{
|
|
await_sqe(0);
|
|
data->iov = { 0 };
|
|
data->callback = simple_callback_w;
|
|
io_uring_prep_fsync(sqe, bs->dsk.meta_fd, IORING_FSYNC_DATASYNC);
|
|
wait_count++;
|
|
resume_1:
|
|
if (wait_count > 0)
|
|
{
|
|
wait_state = wait_base+1;
|
|
return false;
|
|
}
|
|
}
|
|
((blockstore_meta_header_v3_t*)bs->meta_superblock)->compacted_lsn = bs->heap->get_compacted_lsn();
|
|
((blockstore_meta_header_v3_t*)bs->meta_superblock)->set_crc32c();
|
|
await_sqe(2);
|
|
data->iov = (struct iovec){ bs->meta_superblock, (size_t)bs->dsk.meta_block_size };
|
|
data->callback = simple_callback_w;
|
|
io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset);
|
|
wait_count++;
|
|
resume_3:
|
|
if (wait_count > 0)
|
|
{
|
|
wait_state = wait_base+3;
|
|
return false;
|
|
}
|
|
flusher->advance_lsn_counter = 0;
|
|
return true;
|
|
}
|