From 19a386e4b3f4a26b3b31e951706ebec313d42422 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sat, 14 Jun 2025 15:24:18 +0300 Subject: [PATCH] Integrate "heap" metadata storage into blockstore --- src/blockstore/CMakeLists.txt | 7 +- src/blockstore/blockstore.cpp | 9 +- src/blockstore/blockstore.h | 4 +- src/blockstore/blockstore_disk.cpp | 58 +- src/blockstore/blockstore_disk.h | 14 +- src/blockstore/blockstore_flush.cpp | 1586 +++++------------ src/blockstore/blockstore_flush.h | 98 +- src/blockstore/blockstore_impl.cpp | 524 +----- src/blockstore/blockstore_impl.h | 179 +- src/blockstore/blockstore_init.cpp | 1094 +----------- src/blockstore/blockstore_init.h | 37 - src/blockstore/blockstore_internal.h | 40 +- src/blockstore/blockstore_open.cpp | 78 +- src/blockstore/blockstore_read.cpp | 1268 ++++--------- src/blockstore/blockstore_stable.cpp | 575 +----- src/blockstore/blockstore_sync.cpp | 238 +-- src/blockstore/blockstore_write.cpp | 783 ++------ src/blockstore/fio_engine.cpp | 6 +- src/blockstore/ondisk_formats.h | 17 + src/blockstore/v1/flush.cpp | 1469 +++++++++++++++ src/blockstore/v1/flush.h | 134 ++ src/blockstore/v1/impl.cpp | 806 +++++++++ src/blockstore/v1/impl.h | 329 ++++ src/blockstore/v1/init.cpp | 1219 +++++++++++++ src/blockstore/v1/init.h | 71 + src/blockstore/v1/internal.h | 85 + .../journal.cpp} | 0 .../{blockstore_journal.h => v1/journal.h} | 0 src/blockstore/v1/open.cpp | 183 ++ src/blockstore/v1/read.cpp | 1033 +++++++++++ .../rollback.cpp} | 0 src/blockstore/v1/stable.cpp | 562 ++++++ src/blockstore/v1/sync.cpp | 234 +++ src/blockstore/v1/write.cpp | 824 +++++++++ 34 files changed, 8166 insertions(+), 5398 deletions(-) create mode 100644 src/blockstore/v1/flush.cpp create mode 100644 src/blockstore/v1/flush.h create mode 100644 src/blockstore/v1/impl.cpp create mode 100644 src/blockstore/v1/impl.h create mode 100644 src/blockstore/v1/init.cpp create mode 100644 src/blockstore/v1/init.h create mode 100644 src/blockstore/v1/internal.h rename src/blockstore/{blockstore_journal.cpp => v1/journal.cpp} (100%) rename src/blockstore/{blockstore_journal.h => v1/journal.h} (100%) create mode 100644 src/blockstore/v1/open.cpp create mode 100644 src/blockstore/v1/read.cpp rename src/blockstore/{blockstore_rollback.cpp => v1/rollback.cpp} (100%) create mode 100644 src/blockstore/v1/stable.cpp create mode 100644 src/blockstore/v1/sync.cpp create mode 100644 src/blockstore/v1/write.cpp diff --git a/src/blockstore/CMakeLists.txt b/src/blockstore/CMakeLists.txt index 34cc071c..daf8ce25 100644 --- a/src/blockstore/CMakeLists.txt +++ b/src/blockstore/CMakeLists.txt @@ -4,8 +4,11 @@ project(vitastor) # libvitastor_blk.so add_library(vitastor_blk SHARED - ../util/allocator.cpp blockstore.cpp blockstore_impl.cpp blockstore_disk.cpp blockstore_init.cpp blockstore_open.cpp blockstore_journal.cpp blockstore_read.cpp - blockstore_write.cpp blockstore_sync.cpp blockstore_stable.cpp blockstore_rollback.cpp blockstore_flush.cpp ../util/crc32c.c ../util/ringloop.cpp + ../util/allocator.cpp ../util/crc32c.c ../util/ringloop.cpp + blockstore_heap.cpp blockstore_disk.cpp + blockstore.cpp blockstore_impl.cpp blockstore_init.cpp blockstore_open.cpp + blockstore_flush.cpp blockstore_read.cpp blockstore_stable.cpp blockstore_sync.cpp blockstore_write.cpp + v1/flush.cpp v1/impl.cpp v1/init.cpp v1/journal.cpp v1/open.cpp v1/read.cpp v1/rollback.cpp v1/stable.cpp v1/sync.cpp v1/write.cpp ) target_link_libraries(vitastor_blk ${LIBURING_LIBRARIES} diff --git a/src/blockstore/blockstore.cpp b/src/blockstore/blockstore.cpp index 65d8ad33..74e1c306 100644 --- a/src/blockstore/blockstore.cpp +++ b/src/blockstore/blockstore.cpp @@ -1,9 +1,16 @@ // Copyright (c) Vitaliy Filippov, 2019+ // License: VNPL-1.1 (see README.md for details) +#include "str_util.h" + #include "blockstore_impl.h" +#include "v1/impl.h" blockstore_i* blockstore_i::create(blockstore_config_t & config, ring_loop_t *ringloop, timerfd_manager_t *tfd) { - return new blockstore_impl_t(config, ringloop, tfd); + auto meta_format = stoull_full(config["meta_format"]); + if (meta_format == BLOCKSTORE_META_FORMAT_HEAP) + return new blockstore_impl_t(config, ringloop, tfd); + else + return new v1::blockstore_impl_t(config, ringloop, tfd); } diff --git a/src/blockstore/blockstore.h b/src/blockstore/blockstore.h index d6b5e51d..222d2fee 100644 --- a/src/blockstore/blockstore.h +++ b/src/blockstore/blockstore.h @@ -164,8 +164,8 @@ struct __attribute__ ((visibility("default"))) blockstore_op_t uint32_t list_stable_limit; }; }; - uint8_t *buf = NULL; - uint8_t *bitmap = NULL; + void *buf = NULL; + void *bitmap = NULL; int retval = 0; uint8_t private_data[BS_OP_PRIVATE_DATA_SIZE]; diff --git a/src/blockstore/blockstore_disk.cpp b/src/blockstore/blockstore_disk.cpp index 111bacce..57dea67e 100644 --- a/src/blockstore/blockstore_disk.cpp +++ b/src/blockstore/blockstore_disk.cpp @@ -2,10 +2,12 @@ // License: VNPL-1.1 (see README.md for details) #include +#include #include -#include "blockstore_impl.h" +#include "blockstore.h" +#include "ondisk_formats.h" #include "blockstore_disk.h" #include "str_util.h" #include "allocator.h" @@ -44,6 +46,7 @@ void blockstore_disk_t::parse_config(std::map & config disk_alignment = parse_size(config["disk_alignment"]); journal_block_size = parse_size(config["journal_block_size"]); meta_block_size = parse_size(config["meta_block_size"]); + meta_block_target_free_space = parse_size(config["meta_block_target_free_space"]); bitmap_granularity = parse_size(config["bitmap_granularity"]); meta_format = stoull_full(config["meta_format"]); if (config.find("data_io") == config.end() && @@ -90,12 +93,16 @@ void blockstore_disk_t::parse_config(std::map & config if (!min_discard_size) min_discard_size = 1024*1024; discard_granularity = parse_size(config["discard_granularity"]); + inmemory_meta = config["inmemory_metadata"] != "false" && config["inmemory_metadata"] != "0" && + config["inmemory_metadata"] != "no"; + inmemory_journal = config["inmemory_journal"] != "false" && config["inmemory_journal"] != "0" && + config["inmemory_journal"] != "no"; // Validate if (!data_block_size) { data_block_size = (1 << DEFAULT_DATA_BLOCK_ORDER); } - if (is_power_of_two(data_block_size) >= 64 || data_block_size < MIN_DATA_BLOCK_SIZE || data_block_size >= MAX_DATA_BLOCK_SIZE) + if ((block_order = is_power_of_two(data_block_size)) >= 64 || data_block_size < MIN_DATA_BLOCK_SIZE || data_block_size >= MAX_DATA_BLOCK_SIZE) { throw std::runtime_error("Bad block size"); } @@ -131,6 +138,14 @@ void blockstore_disk_t::parse_config(std::map & config { throw std::runtime_error("meta_block_size must not exceed "+std::to_string(MAX_DATA_BLOCK_SIZE)); } + if (!meta_block_target_free_space) + { + meta_block_target_free_space = 800; + } + if (meta_block_target_free_space >= meta_block_size) + { + throw std::runtime_error("meta_block_target_free_space must not exceed "+std::to_string(meta_block_size)); + } if (data_offset % disk_alignment) { throw std::runtime_error("data_offset must be a multiple of disk_alignment = "+std::to_string(disk_alignment)); @@ -204,7 +219,7 @@ void blockstore_disk_t::calc_lengths(bool skip_meta_check) data_len = cfg_data_size; } // meta - uint64_t meta_area_size = (meta_fd == data_fd ? data_device_size : meta_device_size) - meta_offset; + meta_area_size = (meta_fd == data_fd ? data_device_size : meta_device_size) - meta_offset; if (meta_fd == data_fd && meta_offset <= data_offset) { meta_area_size = data_offset - meta_offset; @@ -230,34 +245,13 @@ void blockstore_disk_t::calc_lengths(bool skip_meta_check) clean_entry_bitmap_size = data_block_size / bitmap_granularity / 8; clean_dyn_size = clean_entry_bitmap_size*2 + (csum_block_size ? data_block_size/csum_block_size*(data_csum_type & 0xFF) : 0); - clean_entry_size = sizeof(clean_disk_entry) + clean_dyn_size + 4 /*entry_csum*/; - meta_len = (1 + (block_count - 1 + meta_block_size / clean_entry_size) / (meta_block_size / clean_entry_size)) * meta_block_size; - bool new_doesnt_fit = (!meta_format && !skip_meta_check && meta_area_size < meta_len && !data_csum_type); - if (meta_format == BLOCKSTORE_META_FORMAT_V1 || new_doesnt_fit) + uint32_t entries_per_block = ((meta_block_size-meta_block_target_free_space) / + (24 /*sizeof(heap_object_t)*/ + 33 /*sizeof(heap_write_t)*/ + clean_dyn_size)); + min_meta_len = (block_count+entries_per_block-1) / entries_per_block * meta_block_size; + meta_format = BLOCKSTORE_META_FORMAT_HEAP; + if (!skip_meta_check && meta_area_size < min_meta_len) { - uint64_t clean_entry_v0_size = sizeof(clean_disk_entry) + 2*clean_entry_bitmap_size; - uint64_t meta_v0_len = (1 + (block_count - 1 + meta_block_size / clean_entry_v0_size) - / (meta_block_size / clean_entry_v0_size)) * meta_block_size; - if (meta_format == BLOCKSTORE_META_FORMAT_V1 || meta_area_size >= meta_v0_len) - { - // Old metadata fits. - if (new_doesnt_fit) - { - printf("Warning: Using old metadata format without checksums because the new format" - " doesn't fit into provided area (%ju bytes required, %ju bytes available)\n", meta_len, meta_area_size); - } - clean_entry_size = clean_entry_v0_size; - meta_len = meta_v0_len; - meta_format = BLOCKSTORE_META_FORMAT_V1; - } - else - meta_format = BLOCKSTORE_META_FORMAT_V2; - } - else - meta_format = BLOCKSTORE_META_FORMAT_V2; - if (!skip_meta_check && meta_area_size < meta_len) - { - throw std::runtime_error("Metadata area is too small, need at least "+std::to_string(meta_len)+" bytes, have only "+std::to_string(meta_area_size)+" bytes"); + throw std::runtime_error("Metadata area is too small, need at least "+std::to_string(min_meta_len)+" bytes, have only "+std::to_string(meta_area_size)+" bytes"); } // requested journal size if (!skip_meta_check && cfg_journal_size > journal_len) @@ -429,14 +423,14 @@ void blockstore_disk_t::close_all() // Sadly DISCARD only works through ioctl(), but it seems to always block the device queue, // so it's not a big deal that we can only run it synchronously. -int blockstore_disk_t::trim_data(allocator_t *alloc) +int blockstore_disk_t::trim_data(std::function is_free) { int r = 0; uint64_t j = 0, i = 0; uint64_t discarded = 0; for (; i <= block_count; i++) { - if (i >= block_count || alloc->get(i)) + if (i >= block_count || is_free(i)) { if (i > j && (i-j)*data_block_size >= min_discard_size) { diff --git a/src/blockstore/blockstore_disk.h b/src/blockstore/blockstore_disk.h index a3d45b1d..dbfc74de 100644 --- a/src/blockstore/blockstore_disk.h +++ b/src/blockstore/blockstore_disk.h @@ -30,6 +30,8 @@ struct blockstore_disk_t uint64_t journal_block_size = 4096; // Metadata block size - minimum_io_size of the metadata device is the best choice uint64_t meta_block_size = 4096; + // Target free space in metadata blocks + uint32_t meta_block_target_free_space = 800; // Sparse write tracking granularity. 4 KB is a good choice. Must be a multiple of disk_alignment uint64_t bitmap_granularity = 4096; // Data checksum type, BLOCKSTORE_CSUM_NONE or BLOCKSTORE_CSUM_CRC32C @@ -41,18 +43,24 @@ struct blockstore_disk_t // I/O modes for data, metadata and journal: direct or "" = O_DIRECT, cached = O_SYNC, directsync = O_DIRECT|O_SYNC // O_SYNC without O_DIRECT = use Linux page cache for reads and writes std::string data_io, meta_io, journal_io; + // Keep journal (buffered data) in memory? + bool inmemory_meta = true; + // Keep metadata in memory? + bool inmemory_journal = true; // Data discard granularity and minimum size (for the sake of performance) bool discard_on_start = false; uint64_t min_discard_size = 1024*1024; uint64_t discard_granularity = 0; int meta_fd = -1, data_fd = -1, journal_fd = -1; - uint64_t meta_offset, meta_device_sect, meta_device_size, meta_len, meta_format = 0; + uint64_t meta_offset, meta_device_sect, meta_device_size, meta_area_size, min_meta_len, meta_format = 0; uint64_t data_offset, data_device_sect, data_device_size, data_len; uint64_t journal_offset, journal_device_sect, journal_device_size, journal_len; + uint32_t block_order = 0; uint64_t block_count = 0; - uint32_t clean_entry_bitmap_size = 0, clean_entry_size = 0, clean_dyn_size = 0; + uint32_t clean_entry_bitmap_size = 0; + uint32_t clean_entry_size = 0, clean_dyn_size = 0; // for meta_v1/2 void parse_config(std::map & config); void open_data(); @@ -60,7 +68,7 @@ struct blockstore_disk_t void open_journal(); void calc_lengths(bool skip_meta_check = false); void close_all(); - int trim_data(allocator_t *alloc); + int trim_data(std::function is_free); inline uint64_t dirty_dyn_size(uint64_t offset, uint64_t len) { diff --git a/src/blockstore/blockstore_flush.cpp b/src/blockstore/blockstore_flush.cpp index 4ce2ff91..e8023d10 100644 --- a/src/blockstore/blockstore_flush.cpp +++ b/src/blockstore/blockstore_flush.cpp @@ -3,10 +3,13 @@ #include "blockstore_impl.h" #include "blockstore_internal.h" +#include "crc32c.h" +#include "allocator.h" #define META_BLOCK_UNREAD 0 #define META_BLOCK_READ 1 +// FIXME rename to compactor_t journal_flusher_t::journal_flusher_t(blockstore_impl_t *bs) { this->bs = bs; @@ -14,20 +17,16 @@ journal_flusher_t::journal_flusher_t(blockstore_impl_t *bs) this->min_flusher_count = bs->min_flusher_count; this->cur_flusher_count = bs->min_flusher_count; this->target_flusher_count = bs->min_flusher_count; - dequeuing = false; - trimming = false; active_flushers = 0; syncing_flushers = 0; - // FIXME: allow to configure flusher_start_threshold and journal_trim_interval - flusher_start_threshold = bs->dsk.journal_block_size / sizeof(journal_entry_stable); - journal_trim_counter = bs->journal.flush_journal ? 1 : 0; - trim_wanted = bs->journal.flush_journal ? 1 : 0; - journal_superblock = bs->journal.inmemory ? bs->journal.buffer : memalign_or_die(MEM_ALIGNMENT, bs->dsk.journal_block_size); + advance_lsn_counter = 0; co = new journal_flusher_co[max_flusher_count]; for (int i = 0; i < max_flusher_count; i++) { co[i].bs = bs; co[i].flusher = this; + if (bs->dsk.csum_block_size) + co[i].csum_buf = (uint8_t*)malloc_or_die(bs->dsk.data_block_size/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF)); } } @@ -41,13 +40,6 @@ journal_flusher_co::journal_flusher_co() bs->disk_error_abort("read operation during flush", data->res, data->iov.iov_len); wait_count--; }; - simple_callback_rj = [this](ring_data_t* data) - { - bs->live = true; - if (data->res != data->iov.iov_len) - bs->disk_error_abort("read operation during flush", data->res, data->iov.iov_len); - wait_journal_count--; - }; simple_callback_w = [this](ring_data_t* data) { bs->live = true; @@ -59,14 +51,42 @@ journal_flusher_co::journal_flusher_co() journal_flusher_t::~journal_flusher_t() { - if (!bs->journal.inmemory) - free(journal_superblock); delete[] co; } +journal_flusher_co::~journal_flusher_co() +{ + if (csum_buf) + { + free(csum_buf); + } +} + bool journal_flusher_t::is_active() { - return active_flushers > 0 || dequeuing; + return active_flushers > 0 || bs->heap->get_compact_queue_size() > (force_start > 0 ? 0 : bs->flusher_start_threshold); +} + +void journal_flusher_t::request_trim() +{ + force_start++; + bs->ringloop->wakeup(); +} + +void journal_flusher_t::release_trim() +{ + force_start--; +} + +void journal_flusher_t::dump_diagnostics() +{ + printf( + "Compaction queue: %u items, data: %ju/%ju blocks used, meta: %ju/%ju bytes used, %u/%ju blocks nearfull\n", + bs->heap->get_compact_queue_size(), + bs->heap->get_data_used_space()/bs->dsk.data_block_size, bs->dsk.block_count, + bs->heap->get_meta_used_space(), bs->heap->get_meta_total_space(), + bs->heap->get_meta_nearfull_blocks(), bs->dsk.meta_area_size/bs->dsk.meta_block_size-1 + ); } void journal_flusher_t::loop() @@ -87,235 +107,10 @@ void journal_flusher_t::loop() cur_flusher_count--; } } - if (trim_wanted) - co[0].try_trim = true; - for (int i = 0; (active_flushers > 0 || dequeuing || trim_wanted > 0) && i < cur_flusher_count; i++) + for (int i = 0; is_active() && i < cur_flusher_count; i++) co[i].loop(); } -void journal_flusher_t::enqueue_flush(obj_ver_id ov) -{ -#ifdef BLOCKSTORE_DEBUG - printf("enqueue_flush %jx:%jx v%ju\n", ov.oid.inode, ov.oid.stripe, ov.version); -#endif - auto it = flush_versions.find(ov.oid); - if (it != flush_versions.end()) - { - if (it->second < ov.version) - it->second = ov.version; - } - else - { - flush_versions[ov.oid] = ov.version; - flush_queue.push_back(ov.oid); - } - if (!dequeuing && (flush_queue.size() >= flusher_start_threshold || trim_wanted > 0)) - { - dequeuing = true; - bs->ringloop->wakeup(); - } -} - -void journal_flusher_t::unshift_flush(obj_ver_id ov, bool force) -{ -#ifdef BLOCKSTORE_DEBUG - printf("unshift_flush %jx:%jx v%ju\n", ov.oid.inode, ov.oid.stripe, ov.version); -#endif - auto it = flush_versions.find(ov.oid); - if (it != flush_versions.end()) - { - if (it->second < ov.version) - it->second = ov.version; - } - else - { - flush_versions[ov.oid] = ov.version; - if (!force) - flush_queue.push_front(ov.oid); - } - if (force) - flush_queue.push_front(ov.oid); - if (force || !dequeuing && (flush_queue.size() >= flusher_start_threshold || trim_wanted > 0)) - { - dequeuing = true; - bs->ringloop->wakeup(); - } -} - -void journal_flusher_t::remove_flush(object_id oid) -{ -#ifdef BLOCKSTORE_DEBUG - printf("undo_flush %jx:%jx\n", oid.inode, oid.stripe); -#endif - auto v_it = flush_versions.find(oid); - if (v_it != flush_versions.end()) - { - flush_versions.erase(v_it); - for (auto q_it = flush_queue.begin(); q_it != flush_queue.end(); q_it++) - { - if (*q_it == oid) - { - flush_queue.erase(q_it); - break; - } - } - } -} - -bool journal_flusher_t::is_mutated(uint64_t clean_loc) -{ - for (int i = 0; i < cur_flusher_count; i++) - { - if (co[i].clean_loc == clean_loc && co[i].copy_count > 0) - { - return true; - } - } - return false; -} - -void journal_flusher_t::request_trim() -{ - dequeuing = true; - trim_wanted++; - bs->ringloop->wakeup(); -} - -void journal_flusher_t::mark_trim_possible() -{ - if (trim_wanted > 0) - { - dequeuing = true; - journal_trim_counter = 0; - bs->ringloop->wakeup(); - } -} - -void journal_flusher_t::release_trim() -{ - trim_wanted--; -} - -void journal_flusher_t::dump_diagnostics() -{ - const char *unflushable_type = ""; - obj_ver_id unflushable = {}; - // Try to find out if there is a flushable object for information - for (object_id cur_oid: flush_queue) - { - obj_ver_id cur = { .oid = cur_oid, .version = flush_versions[cur_oid] }; - auto dirty_end = bs->dirty_db.find(cur); - if (dirty_end == bs->dirty_db.end()) - { - // Already flushed - continue; - } - auto repeat_it = sync_to_repeat.find(cur.oid); - if (repeat_it != sync_to_repeat.end()) - { - // Someone is already flushing it - unflushable_type = "locked,"; - unflushable = cur; - break; - } - if (dirty_end->second.journal_sector >= bs->journal.dirty_start && - (bs->journal.dirty_start >= bs->journal.used_start || - dirty_end->second.journal_sector < bs->journal.used_start)) - { - // Object is more recent than possible to flush - bool found = try_find_older(dirty_end, cur); - if (!found) - { - unflushable_type = "dirty,"; - unflushable = cur; - break; - } - } - unflushable_type = "ok,"; - unflushable = cur; - break; - } - printf( - "Flusher: queued=%zd first=%s%jx:%jx trim_wanted=%d dequeuing=%d trimming=%d cur=%d target=%d active=%d syncing=%d\n", - flush_queue.size(), unflushable_type, unflushable.oid.inode, unflushable.oid.stripe, - trim_wanted, dequeuing, trimming, cur_flusher_count, target_flusher_count, - active_flushers, syncing_flushers - ); -} - -bool journal_flusher_t::try_find_older(std::map::iterator & dirty_end, obj_ver_id & cur) -{ - bool found = false; - while (dirty_end != bs->dirty_db.begin()) - { - dirty_end--; - if (dirty_end->first.oid != cur.oid) - { - break; - } - if (!(dirty_end->second.journal_sector >= bs->journal.dirty_start && - (bs->journal.dirty_start >= bs->journal.used_start || - dirty_end->second.journal_sector < bs->journal.used_start))) - { - found = true; - cur.version = dirty_end->first.version; - break; - } - } - return found; -} - -bool journal_flusher_t::try_find_other(std::map::iterator & dirty_end, obj_ver_id & cur) -{ - int search_left = flush_queue.size() - 1; -#ifdef BLOCKSTORE_DEBUG - printf("Flusher overran writers (%jx:%jx v%ju, dirty_start=%08jx) - searching for older flushes (%d left)\n", - cur.oid.inode, cur.oid.stripe, cur.version, bs->journal.dirty_start, search_left); -#endif - while (search_left > 0) - { - cur.oid = flush_queue.front(); - cur.version = flush_versions[cur.oid]; - flush_queue.pop_front(); - flush_versions.erase(cur.oid); - dirty_end = bs->dirty_db.find(cur); - if (dirty_end != bs->dirty_db.end()) - { - if (dirty_end->second.journal_sector >= bs->journal.dirty_start && - (bs->journal.dirty_start >= bs->journal.used_start || - dirty_end->second.journal_sector < bs->journal.used_start)) - { -#ifdef BLOCKSTORE_DEBUG - printf("Write %jx:%jx v%ju is too new: offset=%08jx\n", cur.oid.inode, cur.oid.stripe, cur.version, dirty_end->second.journal_sector); -#endif - enqueue_flush(cur); - } - else - { - auto repeat_it = sync_to_repeat.find(cur.oid); - if (repeat_it != sync_to_repeat.end()) - { - if (repeat_it->second < cur.version) - repeat_it->second = cur.version; - } - else - { - sync_to_repeat[cur.oid] = 0; - break; - } - } - } - search_left--; - } - if (search_left <= 0) - { -#ifdef BLOCKSTORE_DEBUG - printf("No older flushes, stopping\n"); -#endif - } - return search_left > 0; -} - #define await_sqe(label) \ resume_##label:\ sqe = bs->get_sqe();\ @@ -349,579 +144,364 @@ bool journal_flusher_co::loop() else if (wait_state == 15) goto resume_15; else if (wait_state == 16) goto resume_16; else if (wait_state == 17) goto resume_17; - else if (wait_state == 18) goto resume_18; - else if (wait_state == 19) goto resume_19; - else if (wait_state == 20) goto resume_20; - else if (wait_state == 21) goto resume_21; - else if (wait_state == 22) goto resume_22; - else if (wait_state == 23) goto resume_23; - else if (wait_state == 24) goto resume_24; - else if (wait_state == 25) goto resume_25; - else if (wait_state == 26) goto resume_26; - else if (wait_state == 27) goto resume_27; - else if (wait_state == 28) goto resume_28; - else if (wait_state == 29) goto resume_29; - else if (wait_state == 30) goto resume_30; - else if (wait_state == 31) goto resume_31; - else if (wait_state == 32) goto resume_32; - else if (wait_state == 33) goto resume_33; - else if (wait_state == 34) goto resume_34; resume_0: - if (flusher->flush_queue.size() < flusher->min_flusher_count && !flusher->trim_wanted || - !flusher->flush_queue.size() || !flusher->dequeuing) + res = bs->heap->get_next_compact(cur_oid); + if (res == ENOENT) { -stop_flusher: - flusher->dequeuing = false; - if (flusher->trim_wanted > 0 && try_trim) - { - // Attempt forced trim - try_trim = false; - flusher->active_flushers++; - goto trim_journal; - } wait_state = 0; return true; } - try_trim = true; - cur.oid = flusher->flush_queue.front(); - cur.version = flusher->flush_versions[cur.oid]; - flusher->flush_queue.pop_front(); - flusher->flush_versions.erase(cur.oid); - dirty_end = bs->dirty_db.find(cur); - if (dirty_end != bs->dirty_db.end()) - { - repeat_it = flusher->sync_to_repeat.find(cur.oid); - if (repeat_it != flusher->sync_to_repeat.end()) - { -#ifdef BLOCKSTORE_DEBUG - printf("Postpone %jx:%jx v%ju\n", cur.oid.inode, cur.oid.stripe, cur.version); -#endif - // We don't flush different parts of history of the same object in parallel - // So we check if someone is already flushing this object - // In that case we set sync_to_repeat and pick another object - // Another coroutine will see it and re-queue the object after it finishes - if (repeat_it->second < cur.version) - repeat_it->second = cur.version; - wait_state = 0; - goto resume_0; - } - else - flusher->sync_to_repeat[cur.oid] = 0; - if (dirty_end->second.journal_sector >= bs->journal.dirty_start && - (bs->journal.dirty_start >= bs->journal.used_start || - dirty_end->second.journal_sector < bs->journal.used_start)) - { - flusher->enqueue_flush(cur); - // We can't flush journal sectors that are still written to - // However, as we group flushes by oid, current oid may have older writes to flush! - // And it may even block writes if we don't flush the older version - // (if it's in the beginning of the journal)... - // So first try to find an older version of the same object to flush. - if (!flusher->try_find_older(dirty_end, cur)) - { - // Try other objects - flusher->sync_to_repeat.erase(cur.oid); - if (!flusher->try_find_other(dirty_end, cur)) - { - cur.oid = {}; - goto stop_flusher; - } - } - } -#ifdef BLOCKSTORE_DEBUG - printf("Flushing %jx:%jx v%ju\n", cur.oid.inode, cur.oid.stripe, cur.version); -#endif - flusher->active_flushers++; - // Find it in clean_db - { - auto & clean_db = bs->clean_db_shard(cur.oid); - auto clean_it = clean_db.find(cur.oid); - old_clean_ver = (clean_it != clean_db.end() ? clean_it->second.version : 0); - old_clean_loc = (clean_it != clean_db.end() ? clean_it->second.location : UINT64_MAX); - } - // Scan dirty versions of the object to determine what we need to read - scan_dirty(); - // Writes and deletes shouldn't happen at the same time - assert(!has_writes || !has_delete); - if (!has_writes && !has_delete || has_delete && old_clean_loc == UINT64_MAX) - { - // Nothing to flush - bs->erase_dirty(dirty_start, std::next(dirty_end), clean_loc); - goto release_oid; - } - if (clean_loc == UINT64_MAX) - { - if (old_clean_loc == UINT64_MAX) - { - // Object not allocated. This is a bug. - char err[1024]; - snprintf( - err, 1024, "BUG: Object %jx:%jx v%ju that we are trying to flush is not allocated on the data device", - cur.oid.inode, cur.oid.stripe, cur.version - ); - throw std::runtime_error(err); - } - else - { - clean_loc = old_clean_loc; - clean_ver = old_clean_ver; - } - } - // Submit dirty data and old checksum data reads resume_1: -resume_2: - if (!read_dirty(1)) - return false; - // Also we may need to read metadata. We do read-modify-write cycle(s) for every operation. - resume_3: - resume_4: - if (!modify_meta_do_reads(3)) - return false; - // Now, if csum_block_size is > bitmap_granularity and if we are doing partial checksum block updates, - // perform a trick: clear bitmap bits in the metadata entry and recalculate block checksum with zeros - // in place of overwritten parts. Then, even if the actual partial update fully or partially fails, - // we'll have a correct checksum because it won't include overwritten parts! - // The same thing actually happens even when csum_block_size == bitmap_granularity, but in that case - // we never need to read (and thus verify) overwritten parts from the data device. - resume_5: - resume_6: - resume_7: - resume_8: - resume_9: - resume_10: - resume_11: - resume_12: - resume_13: - if (fill_incomplete && !clear_incomplete_csum_block_bits(5)) - return false; - // Wait for journal data reads if the journal is not inmemory - resume_14: - if (wait_journal_count > 0) - { - wait_state = wait_base+14; - return false; - } - if (bs->dsk.csum_block_size) - { - // Mark objects used by reads as modified - auto uo_it = bs->used_clean_objects.find(clean_loc); - if (uo_it != bs->used_clean_objects.end()) - { - uo_it->second.was_changed = true; - } - } - // Submit data writes - for (it = v.begin(); it != v.end(); it++) - { - if (it->copy_flags == COPY_BUF_JOURNAL || it->copy_flags == (COPY_BUF_JOURNAL|COPY_BUF_COALESCED)) - { - await_sqe(15); - data->iov = (struct iovec){ it->buf, (size_t)it->len }; - data->callback = simple_callback_w; - io_uring_prep_writev( - sqe, bs->dsk.data_fd, &data->iov, 1, bs->dsk.data_offset + clean_loc + it->offset - ); - wait_count++; - } - } - // Wait for data writes and metadata reads - resume_16: - resume_17: - if (!wait_meta_reads(16)) - return false; - // Sync data before writing metadata - resume_18: - resume_19: - resume_20: - if (copy_count && !fsync_batch(false, 18)) - return false; - if (old_clean_loc != UINT64_MAX && old_clean_loc != clean_loc) - { - // zero out old metadata entry - { - clean_disk_entry *old_entry = (clean_disk_entry*)((uint8_t*)meta_old.buf + meta_old.pos*bs->dsk.clean_entry_size); - if (old_entry->oid.inode != 0 && old_entry->oid != cur.oid) - { - printf("Fatal error (metadata corruption or bug): tried to wipe metadata entry %ju (%jx:%jx v%ju) as old location of %jx:%jx\n", - old_clean_loc / bs->dsk.data_block_size, old_entry->oid.inode, old_entry->oid.stripe, - old_entry->version, cur.oid.inode, cur.oid.stripe); - exit(1); - } - } - memset((uint8_t*)meta_old.buf + meta_old.pos*bs->dsk.clean_entry_size, 0, bs->dsk.clean_entry_size); - if (meta_old.sector != meta_new.sector) - { - resume_21: - if (flusher->inflight_meta_sectors.find(meta_old.sector) != flusher->inflight_meta_sectors.end()) - { - wait_state = wait_base+21; - return false; - } - flusher->inflight_meta_sectors.insert(meta_old.sector); - resume_22: - if (!write_meta_block(meta_old, 22)) - return false; - resume_23: - if (wait_count > 0) - { - wait_state = wait_base+23; - return false; - } - flusher->inflight_meta_sectors.erase(meta_old.sector); - } - } - resume_24: - if (flusher->inflight_meta_sectors.find(meta_new.sector) != flusher->inflight_meta_sectors.end()) - { - wait_state = wait_base+24; - return false; - } - flusher->inflight_meta_sectors.insert(meta_new.sector); - // Modify the new metadata entry - update_metadata_entry(); - // Update clean_db - it must be equal to the metadata entry - update_clean_db(); - // And write metadata entries - resume_25: - if (!write_meta_block(meta_new, 25)) - return false; - resume_26: - if (wait_count > 0) - { - wait_state = wait_base+26; - return false; - } - flusher->inflight_meta_sectors.erase(meta_new.sector); - // Done, free all buffers - free_buffers(); - // And sync metadata (in batches - not per each operation!) - resume_27: - resume_28: - resume_29: - if (!fsync_batch(true, 27)) - return false; - // Free the data block only when metadata is synced - free_data_blocks(); - // Erase dirty_db entries - bs->erase_dirty(dirty_start, std::next(dirty_end), clean_loc); -#ifdef BLOCKSTORE_DEBUG - printf("Flushed %jx:%jx v%ju (%d copies, wr:%d, del:%d), %jd left\n", cur.oid.inode, cur.oid.stripe, cur.version, - copy_count, has_writes, has_delete, flusher->flush_queue.size()); -#endif - release_oid: - repeat_it = flusher->sync_to_repeat.find(cur.oid); - if (repeat_it != flusher->sync_to_repeat.end() && repeat_it->second > cur.version) - { - // Requeue version - flusher->unshift_flush({ .oid = cur.oid, .version = repeat_it->second }, false); - } - flusher->sync_to_repeat.erase(repeat_it); - trim_journal: - // Clear unused part of the journal every flushes - if (bs->journal_trim_interval && !((++flusher->journal_trim_counter) % bs->journal_trim_interval) || - flusher->trim_wanted > 0) - { - resume_30: - resume_31: - resume_32: - resume_33: - resume_34: - if (!trim_journal(30)) - return false; - } - // All done - flusher->active_flushers--; - wait_state = 0; + cur_obj = bs->heap->lock_and_read_entry(cur_oid, cur_lsn); + if (!cur_obj) + { + // Object does not exist goto resume_0; } + cur_version = cur_obj->get_writes()->version; + // Find the range to compact + bs->heap->get_compact_range(cur_obj, cur_lsn, &begin_wr, &end_wr); + if (!begin_wr) + { + // Nothing to flush + bs->heap->unlock_entry(cur_oid, cur_lsn); + goto resume_0; + } + compact_lsn = begin_wr->lsn; + assert(end_wr < (heap_write_t*)cur_obj->next() && end_wr->flags == (BS_HEAP_BIG_WRITE|BS_HEAP_STABLE)); + clean_loc = end_wr->location; + // "Lock" object for flushing + repeat_it = flusher->sync_to_repeat.find(cur_oid); + if (repeat_it != flusher->sync_to_repeat.end()) + { +#ifdef BLOCKSTORE_DEBUG + printf("Postpone %jx:%jx v%ju\n", cur_oid.inode, cur_oid.stripe, cur_version); +#endif + // We don't flush different parts of history of the same object in parallel + // So we check if someone is already flushing this object + // In that case we set sync_to_repeat and pick another object + // Another coroutine will see it and re-queue the object after it finishes + if (repeat_it->second < cur_version) + repeat_it->second = cur_version; + bs->heap->unlock_entry(cur_oid, cur_lsn); + goto resume_0; + } + else + flusher->sync_to_repeat[cur_oid] = 0; +#ifdef BLOCKSTORE_DEBUG + printf("Flushing %jx:%jx v%ju .. v%ju\n", cur_oid.inode, cur_oid.stripe, end_wr->version, begin_wr->version); +#endif + flusher->active_flushers++; + // Scan versions to flush + read_vec.clear(); + for (auto wr = begin_wr; wr != end_wr; wr = wr->next(bs->heap)) + { + min_compact_lsn = wr->lsn; + bs->prepare_read(read_vec, cur_obj, wr, 0, bs->dsk.data_block_size); + } + overwrite_start = overwrite_end = 0; + if (read_vec.size() > 0) + { + overwrite_start = read_vec[0].offset; + overwrite_end = read_vec[read_vec.size()-1].offset + read_vec[read_vec.size()-1].len; + } + read_to_fill_incomplete = false; + if (bs->dsk.csum_block_size > bs->dsk.bitmap_granularity && + end_wr < (heap_write_t*)cur_obj->next()) + { + // Read original checksum blocks to calculate padded checksums if required + fill_partial_checksum_blocks(); + } + // Read buffered data + cur_obj = NULL; + begin_wr = end_wr = NULL; +resume_2: +resume_3: + if (!read_buffered(2)) + return false; + // Now, if csum_block_size is > bitmap_granularity and if we are doing partial checksum block updates, + // perform a trick: clear bitmap bits in the metadata entry and recalculate block checksum with zeros + // in place of overwritten parts. Then, even if the actual partial update fully or partially fails, + // we'll have a correct checksum because it won't include overwritten parts! + // The same thing actually happens even when csum_block_size == bitmap_granularity, but in that case + // we never need to read (and thus verify) overwritten parts from the data device. + res = check_and_punch_checksums(); + if (res == EBUSY) + { +resume_4: +resume_5: + if (!write_meta_block(4)) + return false; +resume_6: +resume_7: +resume_8: + if (!fsync_batch(true, 6)) // FIXME: is it correct to batch here + return false; + } + else if (res == ENOENT || res == EDOM) + { + // Abort compaction + goto release_oid; + } + assert(res == 0); + // Submit data writes + copy_count = 0; + for (i = 0; i < read_vec.size(); i++) + { + if (read_vec[i].copy_flags & COPY_BUF_JOURNAL) + { + assert(read_vec[i].buf); + await_sqe(9); + data->iov = (struct iovec){ read_vec[i].buf, (size_t)read_vec[i].len }; + data->callback = simple_callback_w; + io_uring_prep_writev(sqe, bs->dsk.data_fd, &data->iov, 1, bs->dsk.data_offset + clean_loc + read_vec[i].offset); + wait_count++; + copy_count++; + } + } +resume_10: + if (wait_count > 0) + { + wait_state = 10; + return false; + } + // Sync data before modifying metadata +resume_11: +resume_12: +resume_13: + if (copy_count && !fsync_batch(false, 11)) + return false; + // Modify the metadata entry; don't write anything. Metadata block will be written on the next write + calc_block_checksums(); + bs->heap->compact_object(cur_oid, cur_lsn, new_data_csums); + // Done, free all buffers + free_buffers(); + // Unlock entry and free referenced block only after fsync + bs->heap->unlock_entry(cur_oid, cur_lsn); +#ifdef BLOCKSTORE_DEBUG + printf("Compacted %jx:%jx v%ju (%d writes)\n", cur_oid.inode, cur_oid.stripe, cur_version, copy_count); +#endif + // Advance compacted_lsn every objects + bs->heap->set_compacted_lsn(min_compact_lsn); + if (bs->journal_trim_interval && !((++flusher->advance_lsn_counter) % bs->journal_trim_interval)) + { +resume_14: +resume_15: +resume_16: +resume_17: + if (!trim_lsn(14)) + return false; + } +release_oid: + repeat_it = flusher->sync_to_repeat.find(cur_oid); + do_repeat = (repeat_it != flusher->sync_to_repeat.end() && repeat_it->second > cur_version); + flusher->sync_to_repeat.erase(repeat_it); + if (do_repeat) + { + // Flush the same object again + goto resume_1; + } + // All done + flusher->active_flushers--; + wait_state = 0; + goto resume_0; return true; } -void journal_flusher_co::update_metadata_entry() +void journal_flusher_co::iterate_partial_overwrites(std::function cb) { - clean_disk_entry *new_entry = (clean_disk_entry*)((uint8_t*)meta_new.buf + meta_new.pos*bs->dsk.clean_entry_size); - if (new_entry->oid.inode != 0 && new_entry->oid != cur.oid) + int prev = 0; + uint32_t prev_begin = 0, prev_end = 0; + for (int i = 0; i < read_vec.size() && !(read_vec[i].copy_flags & COPY_BUF_CSUM_FILL); i++) { - printf( - has_delete - ? "Fatal error (metadata corruption or bug): tried to delete metadata entry %ju (%jx:%jx v%ju) while deleting %jx:%jx v%ju\n" - : "Fatal error (metadata corruption or bug): tried to overwrite non-zero metadata entry %ju (%jx:%jx v%ju) with %jx:%jx v%ju\n", - clean_loc / bs->dsk.data_block_size, new_entry->oid.inode, new_entry->oid.stripe, - new_entry->version, cur.oid.inode, cur.oid.stripe, cur.version - ); - exit(1); - } - if (has_delete) - { - // Zero out the new metadata entry - memset((uint8_t*)meta_new.buf + meta_new.pos*bs->dsk.clean_entry_size, 0, bs->dsk.clean_entry_size); - } - else - { - // Set initial internal bitmap bits from the big write - if (clean_init_bitmap) + if (read_vec[i].copy_flags != (COPY_BUF_JOURNAL|COPY_BUF_COALESCED)) { - memset(new_clean_bitmap, 0, bs->dsk.clean_entry_bitmap_size); - bitmap_set(new_clean_bitmap, clean_bitmap_offset, clean_bitmap_len, bs->dsk.bitmap_granularity); - } - for (auto it = v.begin(); it != v.end(); it++) - { - // Set internal bitmap bits from small writes - if (it->copy_flags == COPY_BUF_JOURNAL || it->copy_flags == (COPY_BUF_JOURNAL|COPY_BUF_COALESCED)) - bitmap_set(new_clean_bitmap, it->offset, it->len, bs->dsk.bitmap_granularity); - } - // Copy latest external bitmap/attributes - { - void *dyn_ptr = bs->alloc_dyn_data - ? (uint8_t*)dirty_end->second.dyn_data+sizeof(int) : (uint8_t*)&dirty_end->second.dyn_data; - memcpy(new_clean_bitmap + bs->dsk.clean_entry_bitmap_size, dyn_ptr, bs->dsk.clean_entry_bitmap_size); - } - // Copy initial (big_write) data checksums - if (bs->dsk.csum_block_size && clean_init_bitmap) - { - uint8_t *new_clean_data_csum = new_clean_bitmap + 2*bs->dsk.clean_entry_bitmap_size; - // big_write partial checksums are calculated from a padded csum_block_size, we can just copy them - memset(new_clean_data_csum, 0, bs->dsk.data_block_size / bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF)); - uint64_t dyn_size = bs->dsk.dirty_dyn_size(clean_bitmap_offset, clean_bitmap_len); - uint32_t *csums = (uint32_t*)(clean_init_dyn_ptr + bs->dsk.clean_entry_bitmap_size); - memcpy(new_clean_data_csum + clean_bitmap_offset / bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF), - csums, dyn_size - bs->dsk.clean_entry_bitmap_size); - } - // Calculate or copy small_write checksums - uint32_t *new_data_csums = (uint32_t*)(new_clean_bitmap + 2*bs->dsk.clean_entry_bitmap_size); - if (bs->dsk.csum_block_size) - calc_block_checksums(new_data_csums, false); - // Update entry - new_entry->oid = cur.oid; - new_entry->version = cur.version; - if (!bs->inmemory_meta) - { - auto inmem_bmp = (uint8_t*)bs->clean_bitmaps + (clean_loc / bs->dsk.data_block_size)*2*bs->dsk.clean_entry_bitmap_size; - memcpy(inmem_bmp, new_clean_bitmap, 2*bs->dsk.clean_entry_bitmap_size); - } - if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_V2) - { - // Calculate metadata entry checksum - uint32_t *new_entry_csum = (uint32_t*)((uint8_t*)new_entry + bs->dsk.clean_entry_size - 4); - *new_entry_csum = crc32c(0, new_entry, bs->dsk.clean_entry_size - 4); + if (read_vec[i].offset > prev_end) + { + i += cb(prev, prev_begin, prev_end); + prev = i; + prev_begin = read_vec[i].offset; + } + prev_end = read_vec[i].offset + read_vec[i].len; } } + if (prev_end > prev_begin) + { + cb(prev, prev_begin, prev_end); + } +} + +void journal_flusher_co::iterate_checksum_holes(std::function cb) +{ + iterate_partial_overwrites([&](int pos, uint32_t prev_begin, uint32_t prev_end) + { + int r = 0; + if ((prev_begin % bs->dsk.csum_block_size) && + (prev_begin / bs->dsk.csum_block_size) != (prev_end / bs->dsk.csum_block_size)) + { + cb(pos, prev_begin, prev_begin + bs->dsk.csum_block_size - prev_begin%bs->dsk.csum_block_size); + r++; + } + if ((prev_end % bs->dsk.csum_block_size) || + (prev_begin % bs->dsk.csum_block_size) && + (prev_end / bs->dsk.csum_block_size) == (prev_begin / bs->dsk.csum_block_size)) + { + cb(i, prev_end - (prev_end % bs->dsk.csum_block_size ? (prev_end % bs->dsk.csum_block_size) : bs->dsk.csum_block_size), prev_end); + r++; + } + return r; + }); +} + +void journal_flusher_co::fill_partial_checksum_blocks() +{ + iterate_checksum_holes([&](int vec_pos, uint32_t hole_start, uint32_t hole_end) + { + read_to_fill_incomplete = true; + int pos = read_vec.size(); + bs->prepare_disk_read(read_vec, pos, cur_obj, end_wr, + hole_start - hole_start % bs->dsk.csum_block_size, hole_start - hole_start % bs->dsk.csum_block_size + bs->dsk.csum_block_size, + hole_start - hole_start % bs->dsk.csum_block_size, hole_start - hole_start % bs->dsk.csum_block_size + bs->dsk.csum_block_size); + pos--; + read_vec[pos].copy_flags |= COPY_BUF_CSUM_FILL; + read_vec.insert(read_vec.begin()+vec_pos, (copy_buffer_t){ + .copy_flags = COPY_BUF_JOURNAL|COPY_BUF_COALESCED, + .offset = hole_start, + .len = hole_end-hole_start, + .buf = read_vec[pos].buf + hole_start - read_vec[pos].offset, + }); + }); } void journal_flusher_co::free_buffers() { - if (!bs->inmemory_meta) - { - meta_new.it->second.usage_count--; - if (meta_new.it->second.usage_count == 0) - { - free(meta_new.it->second.buf); - flusher->meta_sectors.erase(meta_new.it); - } - if (old_clean_loc != UINT64_MAX && old_clean_loc != clean_loc) - { - meta_old.it->second.usage_count--; - if (meta_old.it->second.usage_count == 0) - { - free(meta_old.it->second.buf); - flusher->meta_sectors.erase(meta_old.it); - } - } - } - for (auto it = v.begin(); it != v.end(); it++) + for (auto it = read_vec.begin(); it != read_vec.end(); it++) { // Free it if it's not taken from the journal - if (it->buf && (it->copy_flags == COPY_BUF_JOURNAL || (it->copy_flags & COPY_BUF_CSUM_FILL)) && - (!bs->journal.inmemory || it->buf < bs->journal.buffer || it->buf >= (uint8_t*)bs->journal.buffer + bs->journal.len)) + if (it->buf && !(it->copy_flags & COPY_BUF_COALESCED) && + (!bs->dsk.inmemory_journal || it->buf < bs->buffer_area || it->buf >= (uint8_t*)bs->buffer_area + bs->dsk.journal_len)) { free(it->buf); } } - v.clear(); + read_vec.clear(); } -bool journal_flusher_co::write_meta_block(flusher_meta_write_t & meta_block, int wait_base) +// FIXME: Write tests for it +int journal_flusher_co::check_and_punch_checksums() { - if (wait_state == wait_base) - goto resume_0; - await_sqe(0); - data->iov = (struct iovec){ meta_block.buf, (size_t)bs->dsk.meta_block_size }; - data->callback = simple_callback_w; - io_uring_prep_writev( - sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bs->dsk.meta_block_size + meta_block.sector - ); - wait_count++; - return true; -} - -// Punch holes in incomplete checksum blocks -bool journal_flusher_co::clear_incomplete_csum_block_bits(int wait_base) -{ - if (wait_state == wait_base) goto resume_0; - else if (wait_state == wait_base+1) goto resume_1; - else if (wait_state == wait_base+2) goto resume_2; - else if (wait_state == wait_base+3) goto resume_3; - else if (wait_state == wait_base+4) goto resume_4; - else if (wait_state == wait_base+5) goto resume_5; - else if (wait_state == wait_base+6) goto resume_6; - else if (wait_state == wait_base+7) goto resume_7; - else if (wait_state == wait_base+8) goto resume_8; - cleared_incomplete = false; - for (auto it = v.begin(); it != v.end(); it++) + if (!bs->dsk.csum_block_size) { - if ((it->copy_flags == COPY_BUF_JOURNAL || it->copy_flags == (COPY_BUF_JOURNAL|COPY_BUF_COALESCED)) && - bitmap_check(new_clean_bitmap, it->offset, it->len, bs->dsk.bitmap_granularity)) + // Nothing to do + return 0; + } + // Verify data checksums + cur_obj = bs->heap->read_locked_entry(cur_oid, cur_lsn); + bool csum_ok = true; + for (int i = 0; i < read_vec.size(); i++) + { + auto & vec = read_vec[i]; + if (!(vec.copy_flags & (COPY_BUF_COALESCED|COPY_BUF_ZERO))) { - cleared_incomplete = true; - break; + heap_write_t *wr = (heap_write_t*)((uint8_t*)cur_obj + vec.wr_offset); + bs->heap->calc_block_checksums( + (uint32_t*)wr->get_checksums(bs->heap), vec.buf, wr->get_int_bitmap(bs->heap), vec.offset, vec.offset+vec.len, false, + [&](uint32_t mismatch_pos, uint32_t expected_csum, uint32_t real_csum) + { + printf("Checksum mismatch in object %jx:%jx v%ju in %s area at offset 0x%jx: got %08x, expected %08x\n", + cur_oid.inode, cur_oid.stripe, wr->version, + (vec.copy_flags & COPY_BUF_JOURNAL ? "buffer" : "data"), + vec.disk_offset, real_csum, expected_csum); + csum_ok = false; + } + ); } } - if (cleared_incomplete) + if (!csum_ok) { - // This modification may only happen in place - assert(old_clean_loc == clean_loc); - // Wait for data writes and metadata reads - resume_0: - resume_1: - if (!wait_meta_reads(wait_base+0)) - return false; - resume_2: - if (flusher->inflight_meta_sectors.find(meta_new.sector) != flusher->inflight_meta_sectors.end()) - { - wait_state = wait_base+2; - return false; - } - flusher->inflight_meta_sectors.insert(meta_new.sector); - resume_3: - if (wait_journal_count > 0) - { - wait_state = wait_base+3; - return false; - } - // Verify data checksums - for (i = v.size()-1; i >= 0 && (v[i].copy_flags & COPY_BUF_CSUM_FILL); i--) - { - // If we encounter bad checksums during flush, we still update the bad block, - // but intentionally mangle checksums to avoid hiding the corruption. - iovec iov = { .iov_base = v[i].buf, .iov_len = (size_t)v[i].len }; - if (!(v[i].copy_flags & COPY_BUF_JOURNAL)) - { - assert(!(v[i].offset % bs->dsk.csum_block_size)); - assert(!(v[i].len % bs->dsk.csum_block_size)); - bs->verify_padded_checksums(new_clean_bitmap, new_clean_bitmap + 2*bs->dsk.clean_entry_bitmap_size, - v[i].offset, &iov, 1, [&](uint32_t bad_block, uint32_t calc_csum, uint32_t stored_csum) - { - printf("Checksum mismatch in object %jx:%jx v%ju in data area at offset 0x%jx+0x%x: got %08x, expected %08x\n", - cur.oid.inode, cur.oid.stripe, old_clean_ver, old_clean_loc, bad_block, calc_csum, stored_csum); - for (uint32_t j = 0; j < bs->dsk.csum_block_size; j += bs->dsk.bitmap_granularity) - { - // Simplest method of mangling: flip one byte in every sector - ((uint8_t*)v[i].buf)[j+bad_block-v[i].offset] ^= 0xff; - } - }); - } - else - { - bs->verify_journal_checksums(v[i].csum_buf, v[i].offset, &iov, 1, [&](uint32_t bad_block, uint32_t calc_csum, uint32_t stored_csum) - { - printf("Checksum mismatch in object %jx:%jx v%ju in journal at offset 0x%jx+0x%x (block offset 0x%jx): got %08x, expected %08x\n", - cur.oid.inode, cur.oid.stripe, old_clean_ver, - v[i].disk_offset, bad_block, v[i].offset, calc_csum, stored_csum); - bad_block += (v[i].offset/bs->dsk.csum_block_size) * bs->dsk.csum_block_size; - uint32_t bad_block_end = bad_block + bs->dsk.csum_block_size + (v[i].offset/bs->dsk.csum_block_size) * bs->dsk.csum_block_size; - if (bad_block < v[i].offset) - bad_block = v[i].offset; - if (bad_block_end > v[i].offset+v[i].len) - bad_block_end = v[i].offset+v[i].len; - bad_block -= v[i].offset; - bad_block_end -= v[i].offset; - for (uint32_t j = bad_block; j < bad_block_end; j += bs->dsk.bitmap_granularity) - { - // Simplest method of mangling: flip one byte in every sector - ((uint8_t*)v[i].buf)[j] ^= 0xff; - } - }); - } - } - { - clean_disk_entry *new_entry = (clean_disk_entry*)((uint8_t*)meta_new.buf + meta_new.pos*bs->dsk.clean_entry_size); - if (new_entry->oid != cur.oid) - { - printf( - "Fatal error (metadata corruption or bug): tried to make holes in %ju (%jx:%jx v%ju) with %jx:%jx v%ju\n", - clean_loc / bs->dsk.data_block_size, new_entry->oid.inode, new_entry->oid.stripe, - new_entry->version, cur.oid.inode, cur.oid.stripe, cur.version - ); - } - assert(new_entry->oid == cur.oid); - // Actually clear bits - for (auto it = v.begin(); it != v.end(); it++) - { - if (it->copy_flags == COPY_BUF_JOURNAL || it->copy_flags == (COPY_BUF_JOURNAL|COPY_BUF_COALESCED)) - bitmap_clear(new_clean_bitmap, it->offset, it->len, bs->dsk.bitmap_granularity); - } - // Calculate block checksums with new holes - uint32_t *new_data_csums = (uint32_t*)(new_clean_bitmap + 2*bs->dsk.clean_entry_bitmap_size); - calc_block_checksums(new_data_csums, true); - if (!bs->inmemory_meta) - { - auto inmem_bmp = (uint8_t*)bs->clean_bitmaps + (clean_loc / bs->dsk.data_block_size)*2*bs->dsk.clean_entry_bitmap_size; - memcpy(inmem_bmp, new_clean_bitmap, 2*bs->dsk.clean_entry_bitmap_size); - } - if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_V2) - { - // calculate metadata entry checksum - uint32_t *new_entry_csum = (uint32_t*)((uint8_t*)new_entry + bs->dsk.clean_entry_size - 4); - *new_entry_csum = crc32c(0, new_entry, bs->dsk.clean_entry_size - 4); - } - } - // Write and fsync the modified metadata entry - resume_4: - if (!write_meta_block(meta_new, wait_base+4)) - return false; - resume_5: - if (wait_count > 0) - { - wait_state = wait_base+5; - return false; - } - flusher->inflight_meta_sectors.erase(meta_new.sector); - resume_6: - resume_7: - resume_8: - if (!fsync_batch(true, wait_base+6)) - return false; + // Checksum error, abort compaction + // FIXME: Report the corrupted object to the upper layer + return EDOM; } - return true; + if (!read_to_fill_incomplete) + { + // Nothing to do + return 0; + } + cur_obj = bs->heap->read_entry(cur_oid, &modified_block); + if (!cur_obj) + { + // Object is deleted, abort compaction + return ENOENT; + } + bs->heap->get_compact_range(cur_obj, cur_lsn, &begin_wr, &end_wr); + if (!begin_wr || begin_wr->lsn != compact_lsn) + { + // Object is overwritten, abort compaction + return ENOENT; + } + uint8_t *bmp = end_wr->get_int_bitmap(bs->heap); + uint8_t *csums = end_wr->get_checksums(bs->heap); + // Clear bits + iterate_partial_overwrites([&](int pos, uint32_t start, uint32_t end) + { + bitmap_clear(bmp, start, end-start, bs->dsk.bitmap_granularity); + return 0; + }); + // Update partial block checksums + for (auto & vec: read_vec) + { + if (vec.copy_flags & COPY_BUF_CSUM_FILL) + { + uint32_t csum_off = (vec.offset/bs->dsk.csum_block_size - end_wr->offset/bs->dsk.csum_block_size) * (bs->dsk.data_csum_type & 0xFF); + bs->heap->calc_block_checksums((uint32_t*)(csums+csum_off), vec.buf, bmp, vec.offset, vec.offset+vec.len, true, NULL); + } + } + cur_obj->crc32c = cur_obj->calc_crc32c(); + if (res == ENOENT) + { + // Object is deleted, abort compaction + return ENOENT; + } + // Modified, we should write the block to disk + assert(!res); + return EBUSY; } -void journal_flusher_co::calc_block_checksums(uint32_t *new_data_csums, bool skip_overwrites) +void journal_flusher_co::calc_block_checksums() { + new_data_csums = NULL; + if (bs->dsk.csum_block_size <= bs->dsk.bitmap_granularity) + return; + new_data_csums = csum_buf + overwrite_start/bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF); + cur_obj = bs->heap->read_locked_entry(cur_oid, cur_lsn); uint64_t block_offset = 0; uint32_t block_done = 0; uint32_t block_csum = 0; - for (auto it = v.begin(); it != v.end(); it++) + for (auto it = read_vec.begin(); it != read_vec.end(); it++) { if (it->copy_flags & COPY_BUF_CSUM_FILL) break; if (block_done == 0) { - // `v` should contain aligned items, possibly split into pieces + // `read_vec` should contain aligned items, possibly split into pieces assert(!(it->offset % bs->dsk.csum_block_size)); block_offset = it->offset; } - bool zero = (it->copy_flags & COPY_BUF_ZERO) || (skip_overwrites && (it->copy_flags & COPY_BUF_JOURNAL)); + bool zero = (it->copy_flags & COPY_BUF_ZERO); auto len = it->len; while ((block_done+len) >= bs->dsk.csum_block_size) { - if (!skip_overwrites && !block_done && it->csum_buf) + if (!block_done && it->wr_offset) { // We may take existing checksums if an overwrite contains a full block - auto full_csum_offset = (it->offset+it->len-len+bs->dsk.csum_block_size-1) / bs->dsk.csum_block_size - - it->offset / bs->dsk.csum_block_size; + heap_write_t *wr = (heap_write_t*)((uint8_t*)cur_obj + it->wr_offset); + assert(!(it->offset % bs->dsk.csum_block_size)); + assert(!(wr->offset % bs->dsk.csum_block_size)); + auto full_csum_offset = (it->offset - wr->offset) / bs->dsk.csum_block_size; auto full_csum_count = len/bs->dsk.csum_block_size; memcpy(new_data_csums + block_offset/bs->dsk.csum_block_size, - it->csum_buf + full_csum_offset*4, full_csum_count*4); + wr->get_checksums(bs->heap) + full_csum_offset*4, full_csum_count*4); len -= full_csum_count*bs->dsk.csum_block_size; block_offset += full_csum_count*bs->dsk.csum_block_size; } @@ -946,380 +526,70 @@ void journal_flusher_co::calc_block_checksums(uint32_t *new_data_csums, bool ski block_done += len; } } - // `v` should contain aligned items, possibly split into pieces + // `read_vec` should contain aligned items, possibly split into pieces assert(!block_done); } -void journal_flusher_co::scan_dirty() +bool journal_flusher_co::write_meta_block(int wait_base) { - dirty_it = dirty_start = dirty_end; - v.clear(); - copy_count = 0; - clean_loc = UINT64_MAX; - clean_ver = 0; - has_delete = false; - has_writes = false; - skip_copy = false; - clean_init_bitmap = false; - fill_incomplete = false; - read_to_fill_incomplete = 0; - while (1) + if (wait_state == wait_base) + goto resume_0; + else if (wait_state == wait_base+1) + goto resume_1; + await_sqe(0); + data->iov = (struct iovec){ bs->heap->get_meta_block(modified_block), (size_t)bs->dsk.meta_block_size }; + data->callback = simple_callback_w; + io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + (modified_block+1)*bs->dsk.meta_block_size); + wait_count++; +resume_1: + if (wait_count > 0) { - if (!IS_STABLE(dirty_it->second.state)) - { - char err[1024]; - snprintf( - err, 1024, "BUG: Unexpected dirty_entry %jx:%jx v%ju unstable state during flush: 0x%x", - dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version, dirty_it->second.state - ); - throw std::runtime_error(err); - } - else if (IS_JOURNAL(dirty_it->second.state) && !skip_copy) - { - // Partial dirty overwrite - has_writes = true; - if (dirty_it->second.len != 0) - { - uint64_t blk_begin = 0, blk_end = 0; - uint8_t *blk_buf = NULL; - bs->find_holes( - v, dirty_it->second.offset, dirty_it->second.offset + dirty_it->second.len, - [&](int pos, bool alloc, uint32_t cur_start, uint32_t cur_end) - { - if (alloc) - return 0; - copy_count++; - uint64_t submit_offset = dirty_it->second.location + cur_start - dirty_it->second.offset; - auto it = v.insert(v.begin()+pos, (copy_buffer_t){ - .copy_flags = COPY_BUF_JOURNAL, - .offset = cur_start, - .len = cur_end-cur_start, - .disk_offset = submit_offset, - }); - if (bs->journal.inmemory) - { - // Take it from memory, don't copy it - it->buf = (uint8_t*)bs->journal.buffer + submit_offset; - } - if (bs->dsk.csum_block_size) - { - // FIXME Remove this > sizeof(void*) inline perversion from everywhere. - // I think it doesn't matter but I couldn't stop myself from implementing it :) - uint8_t* dyn_from = (uint8_t*)(bs->alloc_dyn_data - ? (uint8_t*)dirty_it->second.dyn_data+sizeof(int) : (uint8_t*)&dirty_it->second.dyn_data) + - bs->dsk.clean_entry_bitmap_size; - it->csum_buf = dyn_from + (it->offset/bs->dsk.csum_block_size - - dirty_it->second.offset/bs->dsk.csum_block_size) * (bs->dsk.data_csum_type & 0xFF); - if (cur_start % bs->dsk.csum_block_size || cur_end % bs->dsk.csum_block_size) - { - // Small write not aligned for checksums. We may have to pad it - fill_incomplete = true; - if (!bs->journal.inmemory) - { - bs->pad_journal_read(v, *it, dirty_it->second.offset, - dirty_it->second.offset + dirty_it->second.len, dirty_it->second.location, - dyn_from, NULL, cur_start, cur_end-cur_start, blk_begin, blk_end, blk_buf); - } - } - } - return 0; - } - ); - } - } - else if (IS_BIG_WRITE(dirty_it->second.state) && !skip_copy) - { - // There is an unflushed big write. Copy small writes in its position - has_writes = true; - clean_loc = dirty_it->second.location; - clean_ver = dirty_it->first.version; - clean_init_bitmap = true; - clean_bitmap_offset = dirty_it->second.offset; - clean_bitmap_len = dirty_it->second.len; - clean_init_dyn_ptr = bs->alloc_dyn_data - ? (uint8_t*)dirty_it->second.dyn_data+sizeof(int) : (uint8_t*)&dirty_it->second.dyn_data; - skip_copy = true; - } - else if (IS_DELETE(dirty_it->second.state) && !skip_copy) - { - // There is an unflushed delete - has_delete = true; - skip_copy = true; - } - dirty_start = dirty_it; - if (dirty_it == bs->dirty_db.begin()) - { - break; - } - dirty_it--; - if (dirty_it->first.oid != cur.oid) - { - break; - } - } - if (fill_incomplete && !clean_init_bitmap) - { - // Rescan and fill incomplete writes with old data to calculate checksums - if (old_clean_loc == UINT64_MAX) - { - // May happen if the metadata entry is corrupt, but journal isn't - // FIXME: Report corrupted object to the upper layer (OSD) - printf( - "Warning: object %jx:%jx has overwrites, but doesn't have a clean version." - " Metadata is likely corrupted. Dropping object from the DB.\n", - cur.oid.inode, cur.oid.stripe - ); - v.clear(); - has_writes = false; - has_delete = skip_copy = true; - copy_count = 0; - fill_incomplete = false; - read_to_fill_incomplete = 0; - return; - } - uint8_t *bmp_ptr = bs->get_clean_entry_bitmap(old_clean_loc, 0); - uint64_t fulfilled = 0; - int last = v.size()-1; - while (last >= 0 && (v[last].copy_flags & COPY_BUF_CSUM_FILL)) - last--; - read_to_fill_incomplete = bs->fill_partial_checksum_blocks( - v, fulfilled, bmp_ptr, NULL, false, NULL, v[0].offset/bs->dsk.csum_block_size * bs->dsk.csum_block_size, - ((v[last].offset+v[last].len-1) / bs->dsk.csum_block_size + 1) * bs->dsk.csum_block_size - ); - } - else if (fill_incomplete && clean_init_bitmap) - { - // If we actually have partial checksum block overwrites AND a new clean_loc - // at the same time then we can't use our fancy checksum block mutation algorithm. - // So in this case we'll have to first flush the clean write separately. - while (!IS_BIG_WRITE(dirty_end->second.state)) - { - assert(dirty_end != bs->dirty_db.begin()); - dirty_end--; - } - flusher->enqueue_flush(cur); - cur.version = dirty_end->first.version; -#ifdef BLOCKSTORE_DEBUG - printf("Partial checksum block overwrites found - rewinding flush back to %jx:%jx v%ju\n", cur.oid.inode, cur.oid.stripe, cur.version); -#endif - v.clear(); - copy_count = 0; - fill_incomplete = false; - read_to_fill_incomplete = 0; + wait_state = wait_base+1; + return false; } + return true; } -bool journal_flusher_co::read_dirty(int wait_base) +bool journal_flusher_co::read_buffered(int wait_base) { - if (wait_state == wait_base) goto resume_0; - else if (wait_state == wait_base+1) goto resume_1; - wait_count = wait_journal_count = 0; - if (bs->journal.inmemory && !read_to_fill_incomplete) + if (wait_state == wait_base) + goto resume_0; + else if (wait_state == wait_base+1) + goto resume_1; + wait_count = 0; + if (bs->dsk.inmemory_journal && !read_to_fill_incomplete) { // Happy path: nothing to read :) return true; } - for (i = 1; i <= v.size() && (v[v.size()-i].copy_flags & COPY_BUF_CSUM_FILL); i++) + for (i = 0; i < read_vec.size(); i++) { - if (v[v.size()-i].copy_flags & COPY_BUF_JOURNAL) - continue; - // Read old data from disk to calculate checksums - await_sqe(0); - auto & vi = v[v.size()-i]; - assert(vi.len != 0); - vi.buf = memalign_or_die(MEM_ALIGNMENT, vi.len); - data->iov = (struct iovec){ vi.buf, (size_t)vi.len }; - data->callback = simple_callback_r; - io_uring_prep_readv( - sqe, bs->dsk.data_fd, &data->iov, 1, bs->dsk.data_offset + old_clean_loc + vi.offset - ); - wait_count++; - bs->find_holes(v, vi.offset, vi.offset+vi.len, [this, buf = (uint8_t*)vi.buf-vi.offset](int pos, bool alloc, uint32_t cur_start, uint32_t cur_end) + if (read_vec[i].copy_flags == COPY_BUF_JOURNAL && !bs->dsk.inmemory_journal || + (read_vec[i].copy_flags & COPY_BUF_DATA) && !(read_vec[i].copy_flags & COPY_BUF_COALESCED)) { - if (!alloc) - { - v.insert(v.begin()+pos, (copy_buffer_t){ - .copy_flags = COPY_BUF_DATA, - .offset = cur_start, - .len = cur_end-cur_start, - .buf = buf+cur_start, - }); - return 1; - } - return 0; - }); - } - if (!bs->journal.inmemory) - { - for (i = 0; i < v.size(); i++) - { - if (v[i].copy_flags == COPY_BUF_JOURNAL || - v[i].copy_flags == (COPY_BUF_JOURNAL | COPY_BUF_CSUM_FILL)) - { - // Read journal data from disk - if (!v[i].buf) - v[i].buf = memalign_or_die(MEM_ALIGNMENT, v[i].len); - await_sqe(1); - data->iov = (struct iovec){ v[i].buf, (size_t)v[i].len }; - data->callback = simple_callback_rj; - io_uring_prep_readv( - sqe, bs->dsk.journal_fd, &data->iov, 1, bs->journal.offset + v[i].disk_offset - ); - wait_journal_count++; - } + await_sqe(0); + auto & vec = read_vec[i]; + data->iov = (struct iovec){ vec.buf, (size_t)vec.disk_len }; + wait_count++; + io_uring_prep_readv( + sqe, + (vec.copy_flags & COPY_BUF_JOURNAL) ? bs->dsk.journal_fd : bs->dsk.data_fd, + &data->iov, 1, + ((vec.copy_flags & COPY_BUF_JOURNAL) ? bs->dsk.journal_offset : bs->dsk.data_offset) + vec.disk_offset + ); + data->callback = simple_callback_r; } } - return true; -} - -bool journal_flusher_co::modify_meta_do_reads(int wait_base) -{ - if (wait_state == wait_base) goto resume_0; - else if (wait_state == wait_base+1) goto resume_1; -resume_0: - if (!modify_meta_read(clean_loc, meta_new, wait_base+0)) - return false; - new_clean_bitmap = (uint8_t*)meta_new.buf + meta_new.pos*bs->dsk.clean_entry_size + sizeof(clean_disk_entry); - if (old_clean_loc != UINT64_MAX && old_clean_loc != clean_loc) - { - resume_1: - if (!modify_meta_read(old_clean_loc, meta_old, wait_base+1)) - return false; - } - else - meta_old.submitted = false; - return true; -} - -bool journal_flusher_co::wait_meta_reads(int wait_base) -{ - if (wait_state == wait_base) goto resume_0; - else if (wait_state == wait_base+1) goto resume_1; -resume_0: + // Wait for reads/writes if the journal is not inmemory +resume_1: if (wait_count > 0) { - wait_state = wait_base+0; - return false; - } - // Our own reads completed - if (meta_new.submitted) - { - meta_new.it->second.state = META_BLOCK_READ; - bs->ringloop->wakeup(); - } - if (meta_old.submitted) - { - meta_old.it->second.state = META_BLOCK_READ; - bs->ringloop->wakeup(); - } -resume_1: - if (!bs->inmemory_meta && (meta_new.it->second.state == META_BLOCK_UNREAD || - (old_clean_loc != UINT64_MAX && old_clean_loc != clean_loc) && meta_old.it->second.state == META_BLOCK_UNREAD)) - { - // Metadata block is being read by another coroutine wait_state = wait_base+1; return false; } - // All reads completed return true; } -bool journal_flusher_co::modify_meta_read(uint64_t meta_loc, flusher_meta_write_t &wr, int wait_base) -{ - if (wait_state == wait_base) - goto resume_0; - // We must check if the same sector is already in memory if we don't keep all metadata in memory all the time. - // And yet another option is to use LSM trees for metadata, but it sophisticates everything a lot, - // so I'll avoid it as long as I can. - wr.submitted = false; - wr.sector = ((meta_loc / bs->dsk.data_block_size) / (bs->dsk.meta_block_size / bs->dsk.clean_entry_size)) * bs->dsk.meta_block_size; - wr.pos = ((meta_loc / bs->dsk.data_block_size) % (bs->dsk.meta_block_size / bs->dsk.clean_entry_size)); - if (bs->inmemory_meta) - { - wr.buf = (uint8_t*)bs->metadata_buffer + wr.sector; - return true; - } - wr.it = flusher->meta_sectors.find(wr.sector); - if (wr.it == flusher->meta_sectors.end()) - { - // Not in memory yet, read it - wr.buf = memalign_or_die(MEM_ALIGNMENT, bs->dsk.meta_block_size); - wr.it = flusher->meta_sectors.emplace(wr.sector, (meta_sector_t){ - .offset = wr.sector, - .len = bs->dsk.meta_block_size, - .state = META_BLOCK_UNREAD, // 0 = not read yet - .buf = wr.buf, - .usage_count = 1, - }).first; - await_sqe(0); - data->iov = (struct iovec){ wr.it->second.buf, (size_t)bs->dsk.meta_block_size }; - data->callback = simple_callback_r; - wr.submitted = true; - io_uring_prep_readv( - sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bs->dsk.meta_block_size + wr.sector - ); - wait_count++; - } - else - { - wr.buf = wr.it->second.buf; - wr.it->second.usage_count++; - } - return true; -} - -void journal_flusher_co::update_clean_db() -{ - auto & clean_db = bs->clean_db_shard(cur.oid); - if (has_delete) - { - clean_db.erase(cur.oid); - } - else - { - clean_db[cur.oid] = { - .version = cur.version, - .location = clean_loc, - }; - } -} - -void journal_flusher_co::free_data_blocks() -{ - if (old_clean_loc != UINT64_MAX && old_clean_loc != clean_loc) - { - auto uo_it = bs->used_clean_objects.find(old_clean_loc); - bool used = uo_it != bs->used_clean_objects.end(); -#ifdef BLOCKSTORE_DEBUG - printf("%s block %ju from %jx:%jx v%ju (new location is %ju)\n", - used ? "Postpone free" : "Free", - old_clean_loc / bs->dsk.data_block_size, - cur.oid.inode, cur.oid.stripe, cur.version, - clean_loc / bs->dsk.data_block_size); -#endif - if (used) - uo_it->second.was_freed = true; - else - bs->data_alloc->set(old_clean_loc / bs->dsk.data_block_size, false); - } - if (has_delete) - { - assert(clean_loc == old_clean_loc); - auto uo_it = bs->used_clean_objects.find(old_clean_loc); - bool used = uo_it != bs->used_clean_objects.end(); -#ifdef BLOCKSTORE_DEBUG - printf("%s block %ju from %jx:%jx v%ju (delete)\n", - used ? "Postpone free" : "Free", - old_clean_loc / bs->dsk.data_block_size, - cur.oid.inode, cur.oid.stripe, cur.version); -#endif - if (used) - uo_it->second.was_freed = true; - else - bs->data_alloc->set(old_clean_loc / bs->dsk.data_block_size, false); - } -} - bool journal_flusher_co::fsync_batch(bool fsync_meta, int wait_base) { if (wait_state == wait_base) goto resume_0; @@ -1347,7 +617,7 @@ bool journal_flusher_co::fsync_batch(bool fsync_meta, int wait_base) resume_1: if (!cur_sync->state) { - if (flusher->syncing_flushers >= flusher->active_flushers || !flusher->flush_queue.size()) + if (flusher->syncing_flushers >= flusher->active_flushers || true /*FIXME*/) { // Sync batch is ready. Do it. await_sqe(0); @@ -1383,87 +653,39 @@ bool journal_flusher_co::fsync_batch(bool fsync_meta, int wait_base) return true; } -bool journal_flusher_co::trim_journal(int wait_base) +bool journal_flusher_co::trim_lsn(int wait_base) { if (wait_state == wait_base) goto resume_0; else if (wait_state == wait_base+1) goto resume_1; else if (wait_state == wait_base+2) goto resume_2; else if (wait_state == wait_base+3) goto resume_3; - else if (wait_state == wait_base+4) goto resume_4; - new_trim_pos = bs->journal.get_trim_pos(); - if (new_trim_pos != bs->journal.used_start) + if (!bs->disable_meta_fsync) { - resume_0: - // Wait for other coroutines trimming the journal, if any - if (flusher->trimming) + await_sqe(0); + data->iov = { 0 }; + data->callback = simple_callback_w; + io_uring_prep_fsync(sqe, bs->dsk.meta_fd, IORING_FSYNC_DATASYNC); + wait_count++; +resume_1: + if (wait_count > 0) { - wait_state = wait_base+0; + wait_state = wait_base+1; return false; } - flusher->trimming = true; - // Recheck the position with the "lock" taken - new_trim_pos = bs->journal.get_trim_pos(); - if (new_trim_pos != bs->journal.used_start) - { - // First update journal "superblock" and only then update in memory - await_sqe(1); - *((journal_entry_start*)flusher->journal_superblock) = { - .crc32 = 0, - .magic = JOURNAL_MAGIC, - .type = JE_START, - .size = ((!bs->dsk.data_csum_type && ((journal_entry_start*)flusher->journal_superblock)->version == JOURNAL_VERSION_V1) - ? (uint32_t)JE_START_V1_SIZE : (uint32_t)JE_START_V2_SIZE), - .reserved = 0, - .journal_start = new_trim_pos, - .version = (uint64_t)(!bs->dsk.data_csum_type && ((journal_entry_start*)flusher->journal_superblock)->version == JOURNAL_VERSION_V1 - ? JOURNAL_VERSION_V1 : JOURNAL_VERSION_V2), - .data_csum_type = bs->dsk.data_csum_type, - .csum_block_size = bs->dsk.csum_block_size, - }; - ((journal_entry_start*)flusher->journal_superblock)->crc32 = je_crc32((journal_entry*)flusher->journal_superblock); - data->iov = (struct iovec){ flusher->journal_superblock, (size_t)bs->dsk.journal_block_size }; - data->callback = simple_callback_w; - io_uring_prep_writev(sqe, bs->dsk.journal_fd, &data->iov, 1, bs->journal.offset); - wait_count++; - resume_2: - if (wait_count > 0) - { - wait_state = wait_base+2; - return false; - } - if (!bs->disable_journal_fsync) - { - await_sqe(3); - io_uring_prep_fsync(sqe, bs->dsk.journal_fd, IORING_FSYNC_DATASYNC); - data->iov = { 0 }; - data->callback = simple_callback_w; - wait_count++; - resume_4: - if (wait_count > 0) - { - wait_state = wait_base+4; - return false; - } - } - if (new_trim_pos < bs->journal.used_start - ? (bs->journal.dirty_start >= bs->journal.used_start || bs->journal.dirty_start < new_trim_pos) - : (bs->journal.dirty_start >= bs->journal.used_start && bs->journal.dirty_start < new_trim_pos)) - { - bs->journal.dirty_start = new_trim_pos; - } - bs->journal.used_start = new_trim_pos; -#ifdef BLOCKSTORE_DEBUG - printf("Journal trimmed to %08jx (next_free=%08jx dirty_start=%08jx)\n", bs->journal.used_start, bs->journal.next_free, bs->journal.dirty_start); -#endif - if (bs->journal.flush_journal && !flusher->flush_queue.size()) - { - assert(bs->journal.used_start == bs->journal.next_free); - printf("Journal flushed\n"); - exit(0); - } - } - flusher->journal_trim_counter = 0; - flusher->trimming = false; } + ((blockstore_meta_header_v3_t*)bs->meta_superblock)->compacted_lsn = bs->heap->get_compacted_lsn(); + ((blockstore_meta_header_v3_t*)bs->meta_superblock)->set_crc32c(); + await_sqe(2); + data->iov = (struct iovec){ bs->meta_superblock, (size_t)bs->dsk.meta_block_size }; + data->callback = simple_callback_w; + io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset); + wait_count++; +resume_3: + if (wait_count > 0) + { + wait_state = wait_base+3; + return false; + } + flusher->advance_lsn_counter = 0; return true; } diff --git a/src/blockstore/blockstore_flush.h b/src/blockstore/blockstore_flush.h index 3bf18f48..e96589b5 100644 --- a/src/blockstore/blockstore_flush.h +++ b/src/blockstore/blockstore_flush.h @@ -4,11 +4,9 @@ struct copy_buffer_t { int copy_flags; - uint64_t offset, len, disk_offset; - uint64_t journal_sector; // only for reads: sector+1 if used and !journal.inmemory, otherwise 0 - void *buf; - uint8_t *csum_buf; - int *dyn_data; + uint64_t offset, len, disk_offset, disk_len; + uint8_t *buf; + uint32_t wr_offset; }; struct meta_sector_t @@ -41,94 +39,74 @@ class journal_flusher_co { blockstore_impl_t *bs; journal_flusher_t *flusher; - int wait_state, wait_count, wait_journal_count; + int wait_state, wait_count; struct io_uring_sqe *sqe; struct ring_data_t *data; std::list::iterator cur_sync; - - obj_ver_id cur; - std::map::iterator dirty_it, dirty_start, dirty_end; std::map::iterator repeat_it; - std::function simple_callback_r, simple_callback_rj, simple_callback_w; + std::function simple_callback_r, simple_callback_w; - bool try_trim = false; - bool skip_copy, has_delete, has_writes; - std::vector v; - std::vector::iterator it; - int i; - bool fill_incomplete, cleared_incomplete; - int read_to_fill_incomplete; + object_id cur_oid; + uint64_t cur_lsn; + uint64_t compact_lsn; + uint64_t min_compact_lsn; + uint64_t cur_version; + heap_object_t *cur_obj; + heap_write_t *begin_wr, *end_wr; + uint32_t modified_block; + + std::vector read_vec; + uint32_t overwrite_start, overwrite_end; + int i, res; + bool read_to_fill_incomplete; int copy_count; - uint64_t clean_loc, clean_ver, old_clean_loc, old_clean_ver; + uint64_t clean_loc; flusher_meta_write_t meta_old, meta_new; - bool clean_init_bitmap; - uint64_t clean_bitmap_offset, clean_bitmap_len; - uint8_t *clean_init_dyn_ptr; - uint8_t *new_clean_bitmap; - - uint64_t new_trim_pos; + uint8_t *csum_buf = NULL; + uint8_t *new_data_csums = NULL; + bool do_repeat = false; friend class journal_flusher_t; - void scan_dirty(); - bool read_dirty(int wait_base); - bool modify_meta_do_reads(int wait_base); - bool wait_meta_reads(int wait_base); - bool modify_meta_read(uint64_t meta_loc, flusher_meta_write_t &wr, int wait_base); - bool clear_incomplete_csum_block_bits(int wait_base); - void calc_block_checksums(uint32_t *new_data_csums, bool skip_overwrites); - void update_metadata_entry(); - bool write_meta_block(flusher_meta_write_t & meta_block, int wait_base); - void update_clean_db(); - void free_data_blocks(); - bool fsync_batch(bool fsync_meta, int wait_base); - bool trim_journal(int wait_base); + + void iterate_partial_overwrites(std::function cb); + void iterate_checksum_holes(std::function cb); + void fill_partial_checksum_blocks(); void free_buffers(); + int check_and_punch_checksums(); + void calc_block_checksums(); + bool write_meta_block(int wait_base); + bool read_buffered(int wait_base); + bool fsync_batch(bool fsync_meta, int wait_base); + bool trim_lsn(int wait_base); public: journal_flusher_co(); + ~journal_flusher_co(); bool loop(); }; // Journal flusher itself class journal_flusher_t { - int trim_wanted = 0; - bool dequeuing; - int min_flusher_count, max_flusher_count, cur_flusher_count, target_flusher_count; - int flusher_start_threshold; + int force_start = 0; + int min_flusher_count = 0, max_flusher_count = 0, cur_flusher_count = 0, target_flusher_count = 0; journal_flusher_co *co; blockstore_impl_t *bs; friend class journal_flusher_co; - int journal_trim_counter; - bool trimming; - void* journal_superblock; + int advance_lsn_counter = 0; - int active_flushers; - int syncing_flushers; + int active_flushers = 0; + int syncing_flushers = 0; std::list syncs; std::map sync_to_repeat; - std::map meta_sectors; - std::deque flush_queue; - std::unordered_map flush_versions; - std::unordered_set inflight_meta_sectors; - - bool try_find_older(std::map::iterator & dirty_end, obj_ver_id & cur); - bool try_find_other(std::map::iterator & dirty_end, obj_ver_id & cur); - public: journal_flusher_t(blockstore_impl_t *bs); ~journal_flusher_t(); void loop(); - bool is_trim_wanted() { return trim_wanted; } bool is_active(); - void mark_trim_possible(); void request_trim(); void release_trim(); - void enqueue_flush(obj_ver_id oid); - void unshift_flush(obj_ver_id oid, bool force); - void remove_flush(object_id oid); void dump_diagnostics(); - bool is_mutated(uint64_t clean_loc); }; diff --git a/src/blockstore/blockstore_impl.cpp b/src/blockstore/blockstore_impl.cpp index 1715918e..0ecb78b9 100644 --- a/src/blockstore/blockstore_impl.cpp +++ b/src/blockstore/blockstore_impl.cpp @@ -3,6 +3,7 @@ #include "blockstore_impl.h" #include "blockstore_internal.h" +#include "crc32c.h" blockstore_impl_t::blockstore_impl_t(blockstore_config_t & config, ring_loop_t *ringloop, timerfd_manager_t *tfd) { @@ -18,31 +19,37 @@ blockstore_impl_t::blockstore_impl_t(blockstore_config_t & config, ring_loop_t * dsk.open_data(); dsk.open_meta(); dsk.open_journal(); - calc_lengths(); - alloc_dyn_data = dsk.clean_dyn_size > sizeof(void*) || dsk.csum_block_size > 0; + dsk.calc_lengths(); zero_object = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.data_block_size); - data_alloc = new allocator_t(dsk.block_count); } catch (std::exception & e) { dsk.close_all(); throw; } + meta_superblock = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size); + memset(meta_superblock, 0, dsk.meta_block_size); flusher = new journal_flusher_t(this); + if (dsk.inmemory_journal) + { + buffer_area = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.journal_len); + } + heap = new blockstore_heap_t(&dsk, buffer_area, log_level); } blockstore_impl_t::~blockstore_impl_t() { - delete data_alloc; + if (heap) + delete heap; + if (buffer_area) + free(buffer_area); delete flusher; + if (meta_superblock) + free(meta_superblock); if (zero_object) free(zero_object); ringloop->unregister_consumer(&ring_consumer); dsk.close_all(); - if (metadata_buffer) - free(metadata_buffer); - if (clean_bitmaps) - free(clean_bitmaps); } bool blockstore_impl_t::is_started() @@ -58,10 +65,9 @@ bool blockstore_impl_t::is_stalled() // main event loop - produce requests void blockstore_impl_t::loop() { - // FIXME: initialized == 10 is ugly if (initialized != 10) { - // read metadata, then journal + // read metadata if (initialized == 0) { metadata_init_reader = new blockstore_init_meta(this); @@ -74,39 +80,16 @@ void blockstore_impl_t::loop() { delete metadata_init_reader; metadata_init_reader = NULL; - journal_init_reader = new blockstore_init_journal(this); - initialized = 2; - } - } - if (initialized == 2) - { - int res = journal_init_reader->loop(); - if (!res) - { - delete journal_init_reader; - journal_init_reader = NULL; initialized = 3; - ringloop->wakeup(); } } if (initialized == 3) { if (!readonly && dsk.discard_on_start) - dsk.trim_data(data_alloc); - if (journal.flush_journal) - initialized = 4; - else - initialized = 10; - } - if (initialized == 4) - { - if (readonly) { - printf("Can't flush the journal in readonly mode\n"); - exit(1); + dsk.trim_data([this](uint64_t block_num){ return heap->is_data_used(block_num * dsk.data_block_size); }); } - flusher->loop(); - ringloop->submit(); + initialized = 10; } } else @@ -149,7 +132,7 @@ void blockstore_impl_t::loop() { wr_st = dequeue_read(op); } - else if (op->opcode == BS_OP_WRITE || op->opcode == BS_OP_WRITE_STABLE) + else if (op->opcode == BS_OP_WRITE || op->opcode == BS_OP_WRITE_STABLE || op->opcode == BS_OP_DELETE) { if (has_writes == 2) { @@ -159,16 +142,6 @@ void blockstore_impl_t::loop() wr_st = dequeue_write(op); has_writes = wr_st > 0 ? 1 : 2; } - else if (op->opcode == BS_OP_DELETE) - { - if (has_writes == 2) - { - // Some writes already could not be submitted - continue; - } - wr_st = dequeue_del(op); - has_writes = wr_st > 0 ? 1 : 2; - } else if (op->opcode == BS_OP_SYNC) { // sync only completed writes? @@ -176,14 +149,10 @@ void blockstore_impl_t::loop() // then submit an fsync operation wr_st = continue_sync(op); } - else if (op->opcode == BS_OP_STABLE) + else if (op->opcode == BS_OP_STABLE || op->opcode == BS_OP_ROLLBACK) { wr_st = dequeue_stable(op); } - else if (op->opcode == BS_OP_ROLLBACK) - { - wr_st = dequeue_rollback(op); - } else if (op->opcode == BS_OP_LIST) { // LIST doesn't have to be blocked by previous modifications @@ -203,10 +172,6 @@ void blockstore_impl_t::loop() // ring is full, stop submission break; } - else if (PRIV(op)->wait_for == WAIT_JOURNAL) - { - PRIV(op)->wait_detail2 = (unstable_writes.size()+unstable_unsynced); - } } } if (op_idx != new_idx) @@ -226,14 +191,6 @@ void blockstore_impl_t::loop() { throw std::runtime_error(std::string("io_uring_submit: ") + strerror(-ret)); } - for (auto s: journal.submitting_sectors) - { - // Mark journal sector writes as submitted - if (journal.sector_info[s].submit_id) - journal.sector_info[s].written = true; - journal.sector_info[s].submit_id = 0; - } - journal.submitting_sectors.clear(); if ((initial_ring_space - ringloop->space_left()) > 0) { live = true; @@ -251,7 +208,7 @@ bool blockstore_impl_t::is_safe_to_stop() { return false; } - if (unsynced_big_writes.size() > 0 || unsynced_small_writes.size() > 0) + if (unsynced_big_write_count > 0 || unsynced_small_write_count > 0) { if (!readonly && !stop_sync_submitted) { @@ -285,40 +242,13 @@ void blockstore_impl_t::check_wait(blockstore_op_t *op) } PRIV(op)->wait_for = 0; } - else if (PRIV(op)->wait_for == WAIT_JOURNAL) + else if (PRIV(op)->wait_for == WAIT_COMPACTION) { - if (journal.used_start == PRIV(op)->wait_detail && - (unstable_writes.size()+unstable_unsynced) == PRIV(op)->wait_detail2) + if (heap->get_compact_queue_size() >= PRIV(op)->wait_detail) { // do not submit #ifdef BLOCKSTORE_DEBUG - printf("Still waiting to flush journal offset %08jx\n", PRIV(op)->wait_detail); -#endif - return; - } - flusher->release_trim(); - PRIV(op)->wait_for = 0; - } - else if (PRIV(op)->wait_for == WAIT_JOURNAL_BUFFER) - { - int next = ((journal.cur_sector + 1) % journal.sector_count); - if (journal.sector_info[next].flush_count > 0 || - journal.sector_info[next].dirty) - { - // do not submit -#ifdef BLOCKSTORE_DEBUG - printf("Still waiting for a journal buffer\n"); -#endif - return; - } - PRIV(op)->wait_for = 0; - } - else if (PRIV(op)->wait_for == WAIT_FREE) - { - if (!data_alloc->get_free_count() && big_to_flush > 0) - { -#ifdef BLOCKSTORE_DEBUG - printf("Still waiting for free space on the data device\n"); + printf("Still waiting to reduce compaction queue size below %ju\n", PRIV(op)->wait_detail); #endif return; } @@ -364,75 +294,11 @@ void blockstore_impl_t::init_op(blockstore_op_t *op) { // Call constructor without allocating memory. We'll call destructor before returning op back new ((void*)op->private_data) blockstore_op_private_t; - PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 0; PRIV(op)->wait_for = 0; PRIV(op)->op_state = 0; PRIV(op)->pending_ops = 0; } -static bool replace_stable(object_id oid, uint64_t version, int search_start, int search_end, obj_ver_id* list) -{ - while (search_start < search_end) - { - int pos = search_start+(search_end-search_start)/2; - if (oid < list[pos].oid) - { - search_end = pos; - } - else if (list[pos].oid < oid) - { - search_start = pos+1; - } - else - { - list[pos].version = version; - return true; - } - } - return false; -} - -blockstore_clean_db_t& blockstore_impl_t::clean_db_shard(object_id oid) -{ - uint64_t pg_num = 0; - uint64_t pool_id = (oid.inode >> (64-POOL_ID_BITS)); - auto sh_it = clean_db_settings.find(pool_id); - if (sh_it != clean_db_settings.end()) - { - // like map_to_pg() - pg_num = (oid.stripe / sh_it->second.pg_stripe_size) % sh_it->second.pg_count + 1; - } - return clean_db_shards[(pool_id << (64-POOL_ID_BITS)) | pg_num]; -} - -void blockstore_impl_t::reshard_clean_db(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size) -{ - uint64_t pool_id = (uint64_t)pool; - std::map new_shards; - auto sh_it = clean_db_shards.lower_bound((pool_id << (64-POOL_ID_BITS))); - while (sh_it != clean_db_shards.end() && - (sh_it->first >> (64-POOL_ID_BITS)) == pool_id) - { - for (auto & pair: sh_it->second) - { - // like map_to_pg() - uint64_t pg_num = (pair.first.stripe / pg_stripe_size) % pg_count + 1; - uint64_t shard_id = (pool_id << (64-POOL_ID_BITS)) | pg_num; - new_shards[shard_id][pair.first] = pair.second; - } - clean_db_shards.erase(sh_it++); - } - for (sh_it = new_shards.begin(); sh_it != new_shards.end(); sh_it++) - { - auto & to = clean_db_shards[sh_it->first]; - to.swap(sh_it->second); - } - clean_db_settings[pool_id] = (pool_shard_settings_t){ - .pg_count = pg_count, - .pg_stripe_size = pg_stripe_size, - }; -} - void blockstore_impl_t::process_list(blockstore_op_t *op) { uint32_t list_pg = op->pg_number+1; @@ -441,7 +307,8 @@ void blockstore_impl_t::process_list(blockstore_op_t *op) uint64_t min_inode = op->min_oid.inode; uint64_t max_inode = op->max_oid.inode; // Check PG - if (pg_count != 0 && (pg_stripe_size < MIN_DATA_BLOCK_SIZE || list_pg > pg_count)) + if (!pg_count || (pg_stripe_size < MIN_DATA_BLOCK_SIZE || list_pg > pg_count) || + !INODE_POOL(min_inode) || INODE_POOL(min_inode) != INODE_POOL(max_inode)) { op->retval = -EINVAL; FINISH_OP(op); @@ -449,250 +316,32 @@ void blockstore_impl_t::process_list(blockstore_op_t *op) } // Check if the DB needs resharding // (we don't know about PGs from the beginning, we only create "shards" here) - uint64_t first_shard = 0, last_shard = UINT64_MAX; - if (min_inode != 0 && - // Check if min_inode == max_inode == pool_id<> (64-POOL_ID_BITS)) == (max_inode >> (64-POOL_ID_BITS))) - { - pool_id_t pool_id = (min_inode >> (64-POOL_ID_BITS)); - if (pg_count > 1) - { - // Per-pg listing - auto sh_it = clean_db_settings.find(pool_id); - if (sh_it == clean_db_settings.end() || - sh_it->second.pg_count != pg_count || - sh_it->second.pg_stripe_size != pg_stripe_size) - { - reshard_clean_db(pool_id, pg_count, pg_stripe_size); - } - first_shard = last_shard = ((uint64_t)pool_id << (64-POOL_ID_BITS)) | list_pg; - } - else - { - // Per-pool listing - first_shard = ((uint64_t)pool_id << (64-POOL_ID_BITS)); - last_shard = ((uint64_t)(pool_id+1) << (64-POOL_ID_BITS)) - 1; - } - } - // Copy clean_db entries - int stable_count = 0, stable_alloc = 0; - if (min_inode != max_inode) - { - for (auto shard_it = clean_db_shards.lower_bound(first_shard); - shard_it != clean_db_shards.end() && shard_it->first <= last_shard; - shard_it++) - { - auto & clean_db = shard_it->second; - stable_alloc += clean_db.size(); - } - } - if (op->list_stable_limit > 0) - { - stable_alloc = op->list_stable_limit; - if (stable_alloc > 1024*1024) - stable_alloc = 1024*1024; - } - if (stable_alloc < 32768) - { - stable_alloc = 32768; - } - obj_ver_id *stable = (obj_ver_id*)malloc(sizeof(obj_ver_id) * stable_alloc); - if (!stable) - { - op->retval = -ENOMEM; - FINISH_OP(op); - return; - } - auto max_oid = op->max_oid; - bool limited = false; - pool_pg_id_t last_shard_id = 0; - for (auto shard_it = clean_db_shards.lower_bound(first_shard); - shard_it != clean_db_shards.end() && shard_it->first <= last_shard; - shard_it++) - { - auto & clean_db = shard_it->second; - auto clean_it = clean_db.begin(), clean_end = clean_db.end(); - if (op->min_oid.inode != 0 || op->min_oid.stripe != 0) - { - clean_it = clean_db.lower_bound(op->min_oid); - } - if ((max_oid.inode != 0 || max_oid.stripe != 0) && !(max_oid < op->min_oid)) - { - clean_end = clean_db.upper_bound(max_oid); - } - for (; clean_it != clean_end; clean_it++) - { - if (stable_count >= stable_alloc) - { - stable_alloc *= 2; - obj_ver_id* nst = (obj_ver_id*)realloc(stable, sizeof(obj_ver_id) * stable_alloc); - if (!nst) - { - op->retval = -ENOMEM; - FINISH_OP(op); - return; - } - stable = nst; - } - stable[stable_count++] = { - .oid = clean_it->first, - .version = clean_it->second.version, - }; - if (op->list_stable_limit > 0 && stable_count >= op->list_stable_limit) - { - if (!limited) - { - limited = true; - max_oid = stable[stable_count-1].oid; - } - break; - } - } - if (op->list_stable_limit > 0) - { - // To maintain the order, we have to include objects in the same range from other shards - if (last_shard_id != 0 && last_shard_id != shard_it->first) - std::sort(stable, stable+stable_count); - if (stable_count > op->list_stable_limit) - stable_count = op->list_stable_limit; - } - last_shard_id = shard_it->first; - } - if (op->list_stable_limit == 0 && first_shard != last_shard) - { - // If that's not a per-PG listing, sort clean entries (already sorted if list_stable_limit != 0) - std::sort(stable, stable+stable_count); - } - int clean_stable_count = stable_count; - // Copy dirty_db entries (sorted, too) - int unstable_count = 0, unstable_alloc = 0; - obj_ver_id *unstable = NULL; - { - auto dirty_it = dirty_db.begin(), dirty_end = dirty_db.end(); - if (op->min_oid.inode != 0 || op->min_oid.stripe != 0) - { - dirty_it = dirty_db.lower_bound({ - .oid = op->min_oid, - .version = 0, - }); - } - if ((max_oid.inode != 0 || max_oid.stripe != 0) && !(max_oid < op->min_oid)) - { - dirty_end = dirty_db.upper_bound({ - .oid = max_oid, - .version = UINT64_MAX, - }); - } - for (; dirty_it != dirty_end; dirty_it++) - { - if (!pg_count || ((dirty_it->first.oid.stripe / pg_stripe_size) % pg_count + 1) == list_pg) // like map_to_pg() - { - if (IS_DELETE(dirty_it->second.state)) - { - // Deletions are always stable, so try to zero out two possible entries - if (!replace_stable(dirty_it->first.oid, 0, 0, clean_stable_count, stable)) - { - replace_stable(dirty_it->first.oid, 0, clean_stable_count, stable_count, stable); - } - } - else if (IS_STABLE(dirty_it->second.state) || (dirty_it->second.state & BS_ST_INSTANT)) - { - // First try to replace a clean stable version in the first part of the list - if (!replace_stable(dirty_it->first.oid, dirty_it->first.version, 0, clean_stable_count, stable)) - { - // Then try to replace the last dirty stable version in the second part of the list - if (stable_count > 0 && stable[stable_count-1].oid == dirty_it->first.oid) - { - stable[stable_count-1].version = dirty_it->first.version; - } - else - { - if (stable_count >= stable_alloc) - { - stable_alloc += 32768; - obj_ver_id *nst = (obj_ver_id*)realloc(stable, sizeof(obj_ver_id) * stable_alloc); - if (!nst) - { - if (unstable) - free(unstable); - op->retval = -ENOMEM; - FINISH_OP(op); - return; - } - stable = nst; - } - stable[stable_count++] = dirty_it->first; - } - } - if (op->list_stable_limit > 0 && stable_count >= op->list_stable_limit) - { - // Stop here - break; - } - } - else - { - if (unstable_count >= unstable_alloc) - { - unstable_alloc += 32768; - obj_ver_id *nst = (obj_ver_id*)realloc(unstable, sizeof(obj_ver_id) * unstable_alloc); - if (!nst) - { - if (stable) - free(stable); - op->retval = -ENOMEM; - FINISH_OP(op); - return; - } - unstable = nst; - } - unstable[unstable_count++] = dirty_it->first; - } - } - } - } - // Remove zeroed out stable entries - int j = 0; - for (int i = 0; i < stable_count; i++) - { - if (stable[i].version != 0) - { - stable[j++] = stable[i]; - } - } - stable_count = j; - if (stable_count+unstable_count > stable_alloc) - { - stable_alloc = stable_count+unstable_count; - obj_ver_id *nst = (obj_ver_id*)realloc(stable, sizeof(obj_ver_id) * stable_alloc); - if (!nst) - { - if (unstable) - free(unstable); - op->retval = -ENOMEM; - FINISH_OP(op); - return; - } - stable = nst; - } - // Copy unstable entries - for (int i = 0; i < unstable_count; i++) - { - stable[j++] = unstable[i]; - } - free(unstable); + heap->reshard(INODE_POOL(min_inode), pg_count, pg_stripe_size); + obj_ver_id *result = NULL; + size_t stable_count = 0, unstable_count = 0; + int res = heap->list_objects(list_pg, min_inode, max_inode, &result, &stable_count, &unstable_count); op->version = stable_count; - op->retval = stable_count+unstable_count; - op->buf = (uint8_t*)stable; + op->retval = res == 0 ? stable_count+unstable_count : -res; + op->buf = result; FINISH_OP(op); } +void blockstore_impl_t::set_no_inode_stats(const std::vector & pool_ids) +{ +} + void blockstore_impl_t::dump_diagnostics() { - journal.dump_diagnostics(); flusher->dump_diagnostics(); } +void blockstore_meta_header_v3_t::set_crc32c() +{ + header_csum = 0; + uint32_t calc = crc32c(0, this, sizeof(*this)); + header_csum = calc; +} + void blockstore_impl_t::disk_error_abort(const char *op, int retval, int expected) { if (retval == -EAGAIN) @@ -706,92 +355,9 @@ void blockstore_impl_t::disk_error_abort(const char *op, int retval, int expecte exit(1); } -const std::map & blockstore_impl_t::get_inode_space_stats() +uint64_t blockstore_impl_t::get_free_block_count() { - return inode_space_stats; -} - -void blockstore_impl_t::set_no_inode_stats(const std::vector & pool_ids) -{ - for (auto & np: no_inode_stats) - { - np.second = 2; - } - for (auto pool_id: pool_ids) - { - if (!no_inode_stats[pool_id]) - recalc_inode_space_stats(pool_id, false); - no_inode_stats[pool_id] = 1; - } - for (auto np_it = no_inode_stats.begin(); np_it != no_inode_stats.end(); ) - { - if (np_it->second == 2) - { - recalc_inode_space_stats(np_it->first, true); - no_inode_stats.erase(np_it++); - } - else - np_it++; - } -} - -void blockstore_impl_t::recalc_inode_space_stats(uint64_t pool_id, bool per_inode) -{ - auto sp_begin = inode_space_stats.lower_bound((pool_id << (64-POOL_ID_BITS))); - auto sp_end = inode_space_stats.lower_bound(((pool_id+1) << (64-POOL_ID_BITS))); - inode_space_stats.erase(sp_begin, sp_end); - auto sh_it = clean_db_shards.lower_bound((pool_id << (64-POOL_ID_BITS))); - while (sh_it != clean_db_shards.end() && - (sh_it->first >> (64-POOL_ID_BITS)) == pool_id) - { - for (auto & pair: sh_it->second) - { - uint64_t space_id = per_inode ? pair.first.inode : (pool_id << (64-POOL_ID_BITS)); - inode_space_stats[space_id] += dsk.data_block_size; - } - sh_it++; - } - object_id last_oid = {}; - bool last_exists = false; - auto dirty_it = dirty_db.lower_bound((obj_ver_id){ .oid = { .inode = (pool_id << (64-POOL_ID_BITS)) } }); - while (dirty_it != dirty_db.end() && (dirty_it->first.oid.inode >> (64-POOL_ID_BITS)) == pool_id) - { - if (IS_STABLE(dirty_it->second.state) && (IS_BIG_WRITE(dirty_it->second.state) || IS_DELETE(dirty_it->second.state))) - { - bool exists = false; - if (last_oid == dirty_it->first.oid) - { - exists = last_exists; - } - else - { - auto & clean_db = clean_db_shard(dirty_it->first.oid); - auto clean_it = clean_db.find(dirty_it->first.oid); - exists = clean_it != clean_db.end(); - } - uint64_t space_id = per_inode ? dirty_it->first.oid.inode : (pool_id << (64-POOL_ID_BITS)); - if (IS_BIG_WRITE(dirty_it->second.state)) - { - if (!exists) - inode_space_stats[space_id] += dsk.data_block_size; - last_exists = true; - } - else - { - if (exists) - { - auto & sp = inode_space_stats[space_id]; - if (sp > dsk.data_block_size) - sp -= dsk.data_block_size; - else - inode_space_stats.erase(space_id); - } - last_exists = false; - } - last_oid = dirty_it->first.oid; - } - dirty_it++; - } + return dsk.block_count - heap->get_data_used_space()/dsk.data_block_size; } std::string blockstore_impl_t::get_op_diag(blockstore_op_t *op) diff --git a/src/blockstore/blockstore_impl.h b/src/blockstore/blockstore_impl.h index 7365ff75..4ccd5ef2 100644 --- a/src/blockstore/blockstore_impl.h +++ b/src/blockstore/blockstore_impl.h @@ -5,6 +5,7 @@ #include "blockstore.h" #include "blockstore_disk.h" +#include "blockstore_heap.h" #include "ondisk_formats.h" #include @@ -22,63 +23,23 @@ #include #include -#include "cpp-btree/btree_map.h" - #include "malloc_or_die.h" -#include "allocator.h" + +class blockstore_impl_t; //#define BLOCKSTORE_DEBUG -#include "blockstore_journal.h" - -// 32 = 16 + 16 bytes per "clean" entry in memory (object_id => clean_entry) -struct __attribute__((__packed__)) clean_entry -{ - uint64_t version; - uint64_t location; -}; - -// 64 = 24 + 40 bytes per dirty entry in memory (obj_ver_id => dirty_entry). Plus checksums -struct __attribute__((__packed__)) dirty_entry -{ - uint32_t state; - uint32_t flags; // unneeded, but present for alignment - uint64_t location; // location in either journal or data -> in BYTES - uint32_t offset; // data offset within object (stripe) - uint32_t len; // data length - uint64_t journal_sector; // journal sector used for this entry - void* dyn_data; // dynamic data: external bitmap and data block checksums. may be a pointer to the in-memory journal -}; - // - Sync must be submitted after previous writes/deletes (not before!) -// - Reads to the same object must be submitted after previous writes/deletes -// are written (not necessarily synced) in their location. This is because we -// rely on read-modify-write for erasure coding and we must return new data -// to calculate parity for subsequent writes +// - Reads may be submitted in parallel with writes/deletes because we use MVCC // - Writes may be submitted in any order, because they don't overlap. Each write // goes into a new location - either on the journal device or on the data device // - Stable (stabilize) must be submitted after sync of that object is completed // It's even OK to return an error to the caller if that object is not synced yet -// - Journal trim may be processed only after all versions are moved to -// the main storage AND after all read operations for older versions complete +// - compacted_lsn should be moved forward only after all versions are moved to the main storage // - If an operation can not be submitted because the ring is full // we should stop submission of other operations. Otherwise some "scatter" reads // may end up blocked for a long time. -// Otherwise, the submit order is free, that is all operations may be submitted immediately -// In fact, adding a write operation must immediately result in dirty_db being populated - -struct used_clean_obj_t -{ - int refs; - bool was_freed; // was freed by a parallel flush? - bool was_changed; // was changed by a parallel flush? -}; - -// https://github.com/algorithm-ninja/cpp-btree -// https://github.com/greg7mdp/sparsepp/ was used previously, but it was TERRIBLY slow after resizing -// with sparsepp, random reads dropped to ~700 iops very fast with just as much as ~32k objects in the DB -typedef btree::btree_map blockstore_clean_db_t; -typedef std::map blockstore_dirty_db_t; +// Otherwise, the submission order is free. #include "blockstore_init.h" @@ -88,35 +49,26 @@ struct blockstore_op_private_t { // Wait status int wait_for; - uint64_t wait_detail, wait_detail2; + uint64_t wait_detail; int pending_ops; int op_state; // Read - uint64_t clean_block_used; + uint64_t lsn; std::vector read_vec; - // Sync, write - uint64_t min_flushed_journal_sector, max_flushed_journal_sector; + // Write + uint64_t location; + bool is_big; + + // Stabilize/rollback + int stab_pos; // Write struct iovec iov_zerofill[3]; - // Warning: must not have a default value here because it's written to before calling constructor in blockstore_write.cpp O_o - uint64_t real_version; timespec tv_begin; - - // Sync - std::vector sync_big_writes, sync_small_writes; }; -struct pool_shard_settings_t -{ - uint32_t pg_count; - uint32_t pg_stripe_size; -}; - -typedef uint64_t pool_pg_id_t; - class blockstore_impl_t: public blockstore_i { blockstore_disk_t dsk; @@ -129,9 +81,11 @@ class blockstore_impl_t: public blockstore_i // Suitable only for server SSDs with capacitors, requires disabled data and journal fsyncs int immediate_commit = IMMEDIATE_NONE; bool inmemory_meta = false; + uint32_t meta_write_recheck_parallelism = 0; // Maximum and minimum flusher count - unsigned max_flusher_count, min_flusher_count; - unsigned journal_trim_interval; + unsigned max_flusher_count = 0, min_flusher_count = 0; + unsigned journal_trim_interval = 0; + unsigned flusher_start_threshold = 0; // Maximum queue depth unsigned max_write_iodepth = 128; // Enable small (journaled) write throttling, useful for the SSD+HDD case @@ -150,30 +104,16 @@ class blockstore_impl_t: public blockstore_i struct ring_consumer_t ring_consumer; - std::map clean_db_settings; - std::map clean_db_shards; - std::map no_inode_stats; - std::map inode_space_stats; - uint8_t *clean_bitmaps = NULL; - blockstore_dirty_db_t dirty_db; + blockstore_heap_t *heap = NULL; + uint8_t* meta_superblock = NULL; + uint8_t *buffer_area = NULL; std::vector submit_queue; - std::vector unsynced_big_writes, unsynced_small_writes; - int unsynced_big_write_count = 0, unstable_unsynced = 0; + int unsynced_big_write_count = 0, unsynced_small_write_count = 0; int unsynced_queued_ops = 0; - allocator_t *data_alloc = NULL; - uint64_t used_blocks = 0; uint8_t *zero_object = NULL; - void *metadata_buffer = NULL; - - struct journal_t journal; journal_flusher_t *flusher; - int big_to_flush = 0; int write_iodepth = 0; - bool alloc_dyn_data = false; - - // clean data blocks referenced by read operations - std::map used_clean_objects; bool live = false, queue_stall = false; ring_loop_t *ringloop; @@ -187,92 +127,50 @@ class blockstore_impl_t: public blockstore_i } friend class blockstore_init_meta; - friend class blockstore_init_journal; - friend struct blockstore_journal_check_t; friend class journal_flusher_t; friend class journal_flusher_co; - void calc_lengths(); void open_data(); void open_meta(); void open_journal(); - uint8_t* get_clean_entry_bitmap(uint64_t block_loc, int offset); - blockstore_clean_db_t& clean_db_shard(object_id oid); - void reshard_clean_db(pool_id_t pool_id, uint32_t pg_count, uint32_t pg_stripe_size); - void recalc_inode_space_stats(uint64_t pool_id, bool per_inode); - - // Journaling - void prepare_journal_sector_write(int sector, blockstore_op_t *op); - void handle_journal_write(ring_data_t *data, uint64_t flush_id); void disk_error_abort(const char *op, int retval, int expected); // Asynchronous init int initialized; int metadata_buf_size; blockstore_init_meta* metadata_init_reader; - blockstore_init_journal* journal_init_reader; void check_wait(blockstore_op_t *op); void init_op(blockstore_op_t *op); // Read - int dequeue_read(blockstore_op_t *read_op); + int dequeue_read(blockstore_op_t *op); + int fulfill_read(blockstore_op_t *op); + uint32_t prepare_read(std::vector & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end); + uint32_t prepare_read_with_bitmaps(std::vector & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end); + uint32_t prepare_read_zero(std::vector & read_vec, uint32_t start, uint32_t end); + uint32_t prepare_read_simple(std::vector & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end); + void prepare_disk_read(std::vector & read_vec, int & pos, heap_object_t *obj, heap_write_t *wr, + uint32_t blk_start, uint32_t blk_end, uint32_t start, uint32_t end); void find_holes(std::vector & read_vec, uint32_t item_start, uint32_t item_end, - std::function callback); - int fulfill_read(blockstore_op_t *read_op, - uint64_t &fulfilled, uint32_t item_start, uint32_t item_end, - uint32_t item_state, uint64_t item_version, uint64_t item_location, - uint64_t journal_sector, uint8_t *csum, int *dyn_data); - bool fulfill_clean_read(blockstore_op_t *read_op, uint64_t & fulfilled, - uint8_t *clean_entry_bitmap, int *dyn_data, - uint32_t item_start, uint32_t item_end, uint64_t clean_loc, uint64_t clean_ver); - int fill_partial_checksum_blocks(std::vector & rv, uint64_t & fulfilled, - uint8_t *clean_entry_bitmap, int *dyn_data, bool from_journal, uint8_t *read_buf, uint64_t read_offset, uint64_t read_end); - int pad_journal_read(std::vector & rv, copy_buffer_t & cp, - uint64_t dirty_offset, uint64_t dirty_end, uint64_t dirty_loc, uint8_t *csum_ptr, int *dyn_data, - uint64_t offset, uint64_t submit_len, uint64_t & blk_begin, uint64_t & blk_end, uint8_t* & blk_buf); - bool read_range_fulfilled(std::vector & rv, uint64_t & fulfilled, uint8_t *read_buf, - uint8_t *clean_entry_bitmap, uint32_t item_start, uint32_t item_end); - bool read_checksum_block(blockstore_op_t *op, int rv_pos, uint64_t &fulfilled, uint64_t clean_loc); - uint8_t* read_clean_meta_block(blockstore_op_t *read_op, uint64_t clean_loc, int rv_pos); - bool verify_padded_checksums(uint8_t *clean_entry_bitmap, uint8_t *csum_buf, uint32_t offset, - iovec *iov, int n_iov, std::function bad_block_cb); - bool verify_journal_checksums(uint8_t *csums, uint32_t offset, - iovec *iov, int n_iov, std::function bad_block_cb); - bool verify_clean_padded_checksums(blockstore_op_t *op, uint64_t clean_loc, uint8_t *dyn_data, bool from_journal, - iovec *iov, int n_iov, std::function bad_block_cb); - int fulfill_read_push(blockstore_op_t *op, void *buf, uint64_t offset, uint64_t len, - uint32_t item_state, uint64_t item_version); + std::function callback); void handle_read_event(ring_data_t *data, blockstore_op_t *op); + bool verify_read_checksums(blockstore_op_t *op); // Write bool enqueue_write(blockstore_op_t *op); - void cancel_all_writes(blockstore_op_t *op, blockstore_dirty_db_t::iterator dirty_it, int retval); + void cancel_all_writes(blockstore_op_t *op, int retval); + void prepare_meta_block_write(blockstore_op_t *op, uint64_t modified_block); int dequeue_write(blockstore_op_t *op); - int dequeue_del(blockstore_op_t *op); int continue_write(blockstore_op_t *op); - void release_journal_sectors(blockstore_op_t *op); void handle_write_event(ring_data_t *data, blockstore_op_t *op); // Sync int continue_sync(blockstore_op_t *op); - void ack_sync(blockstore_op_t *op); // Stabilize int dequeue_stable(blockstore_op_t *op); - int continue_stable(blockstore_op_t *op); - void mark_stable(obj_ver_id ov, bool forget_dirty = false); - void stabilize_object(object_id oid, uint64_t max_ver); - blockstore_op_t* selective_sync(blockstore_op_t *op); - int split_stab_op(blockstore_op_t *op, std::function decider); - - // Rollback - int dequeue_rollback(blockstore_op_t *op); - int continue_rollback(blockstore_op_t *op); - void mark_rolled_back(const obj_ver_id & ov); - void erase_dirty(blockstore_dirty_db_t::iterator dirty_start, blockstore_dirty_db_t::iterator dirty_end, uint64_t clean_loc); - void free_dirty_dyn_data(dirty_entry & e); // List void process_list(blockstore_op_t *op); @@ -306,12 +204,6 @@ public: // Simplified synchronous operation: get object bitmap & current version int read_bitmap(object_id oid, uint64_t target_version, void *bitmap, uint64_t *result_version = NULL); - // Unstable writes are added here (map of object_id -> version) - std::unordered_map unstable_writes; - - // Get space usage statistics - const std::map & get_inode_space_stats(); - // Set per-pool no_inode_stats void set_no_inode_stats(const std::vector & pool_ids); @@ -321,9 +213,10 @@ public: // Get diagnostic string for an operation std::string get_op_diag(blockstore_op_t *op); + const std::map & get_inode_space_stats() { return heap->get_inode_space_stats(); } inline uint32_t get_block_size() { return dsk.data_block_size; } inline uint64_t get_block_count() { return dsk.block_count; } - inline uint64_t get_free_block_count() { return dsk.block_count - used_blocks; } + uint64_t get_free_block_count(); inline uint32_t get_bitmap_granularity() { return dsk.disk_alignment; } inline uint64_t get_journal_size() { return dsk.journal_len; } }; diff --git a/src/blockstore/blockstore_init.cpp b/src/blockstore/blockstore_init.cpp index 34969858..52158859 100644 --- a/src/blockstore/blockstore_init.cpp +++ b/src/blockstore/blockstore_init.cpp @@ -3,6 +3,7 @@ #include "blockstore_impl.h" #include "blockstore_internal.h" +#include "crc32c.h" #define INIT_META_EMPTY 0 #define INIT_META_READING 1 @@ -55,17 +56,14 @@ int blockstore_init_meta::loop() else if (wait_state == 4) goto resume_4; else if (wait_state == 5) goto resume_5; else if (wait_state == 6) goto resume_6; - printf("Reading blockstore metadata\n"); - if (bs->inmemory_meta) - metadata_buffer = bs->metadata_buffer; - else - metadata_buffer = memalign(MEM_ALIGNMENT, 2*bs->metadata_buf_size); + else if (wait_state == 7) goto resume_7; + metadata_buffer = memalign(MEM_ALIGNMENT, 2*bs->metadata_buf_size); if (!metadata_buffer) throw std::runtime_error("Failed to allocate metadata read buffer"); - // Read superblock + // Read metadata superblock GET_SQE(); last_read_offset = 0; - data->iov = { metadata_buffer, (size_t)bs->dsk.meta_block_size }; + data->iov = { bs->meta_superblock, (size_t)bs->dsk.meta_block_size }; data->callback = [this](ring_data_t *data) { handle_event(data, -1); }; io_uring_prep_readv(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset); bs->ringloop->submit(); @@ -76,23 +74,23 @@ resume_1: wait_state = 1; return 1; } - if (iszero((uint64_t*)metadata_buffer, bs->dsk.meta_block_size / sizeof(uint64_t))) + if (iszero((uint64_t*)bs->meta_superblock, bs->dsk.meta_block_size / sizeof(uint64_t))) { { - blockstore_meta_header_v2_t *hdr = (blockstore_meta_header_v2_t *)metadata_buffer; + blockstore_meta_header_v3_t *hdr = (blockstore_meta_header_v3_t *)bs->meta_superblock; hdr->zero = 0; hdr->magic = BLOCKSTORE_META_MAGIC_V1; hdr->version = bs->dsk.meta_format; hdr->meta_block_size = bs->dsk.meta_block_size; hdr->data_block_size = bs->dsk.data_block_size; hdr->bitmap_granularity = bs->dsk.bitmap_granularity; + hdr->compacted_lsn = 0; if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_V2) { hdr->data_csum_type = bs->dsk.data_csum_type; hdr->csum_block_size = bs->dsk.csum_block_size; - hdr->header_csum = 0; - hdr->header_csum = crc32c(0, hdr, sizeof(*hdr)); } + hdr->set_crc32c(); } if (bs->readonly) { @@ -103,15 +101,15 @@ resume_1: printf("Initializing metadata area\n"); GET_SQE(); last_read_offset = 0; - data->iov = (struct iovec){ metadata_buffer, (size_t)bs->dsk.meta_block_size }; + data->iov = (struct iovec){ bs->meta_superblock, (size_t)bs->dsk.meta_block_size }; data->callback = [this](ring_data_t *data) { handle_event(data, -1); }; io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset); bs->ringloop->submit(); submitted++; - resume_3: + resume_2: if (submitted > 0) { - wait_state = 3; + wait_state = 2; return 1; } zero_on_init = true; @@ -119,7 +117,7 @@ resume_1: } else { - blockstore_meta_header_v2_t *hdr = (blockstore_meta_header_v2_t *)metadata_buffer; + blockstore_meta_header_v2_t *hdr = (blockstore_meta_header_v2_t *)bs->meta_superblock; if (hdr->zero != 0 || hdr->magic != BLOCKSTORE_META_MAGIC_V1 || hdr->version < BLOCKSTORE_META_FORMAT_V1) { printf( @@ -129,43 +127,12 @@ resume_1: ); exit(1); } - if (hdr->version == BLOCKSTORE_META_FORMAT_V2) - { - uint32_t csum = hdr->header_csum; - hdr->header_csum = 0; - if (crc32c(0, hdr, sizeof(*hdr)) != csum) - { - printf("Metadata header is corrupt (checksum mismatch).\n"); - exit(1); - } - hdr->header_csum = csum; - if (bs->dsk.meta_format != BLOCKSTORE_META_FORMAT_V2) - { - bs->dsk.meta_format = BLOCKSTORE_META_FORMAT_V2; - bs->dsk.calc_lengths(); - } - } - else if (hdr->version == BLOCKSTORE_META_FORMAT_V1) - { - hdr->data_csum_type = 0; - hdr->csum_block_size = 0; - hdr->header_csum = 0; - // Enable compatibility mode - entries without checksums - if (bs->dsk.meta_format != BLOCKSTORE_META_FORMAT_V1 || - bs->dsk.data_csum_type != 0 || bs->dsk.csum_block_size != 0) - { - bs->dsk.data_csum_type = 0; - bs->dsk.csum_block_size = 0; - bs->dsk.meta_format = BLOCKSTORE_META_FORMAT_V1; - bs->dsk.calc_lengths(); - printf("Warning: Starting with metadata in the old format without checksums, as stored on disk\n"); - } - } - else if (hdr->version > BLOCKSTORE_META_FORMAT_V2) + if (hdr->version != BLOCKSTORE_META_FORMAT_HEAP) { printf( - "Metadata format is too new for me (stored version is %ju, max supported %u).\n", - hdr->version, BLOCKSTORE_META_FORMAT_V2 + "OSD is started with meta_format 3, but actually stored format is %ju on disk." + " Please update the OSD superblock or startup options.\n", + hdr->version ); exit(1); } @@ -187,25 +154,48 @@ resume_1: exit(1); } } + if (bs->dsk.inmemory_journal) + { + // Read buffer area + printf("Reading buffered data\n"); + md_offset = 0; + while (md_offset < bs->dsk.journal_len) + { + GET_SQE(); + data->iov = (iovec){ + bs->buffer_area + md_offset, + (size_t)(bs->dsk.journal_len - md_offset < bs->metadata_buf_size ? bs->dsk.journal_len - md_offset : bs->metadata_buf_size), + }; + data->callback = [this](ring_data_t *data) { handle_event(data, -1); }; + io_uring_prep_readv(sqe, bs->dsk.journal_fd, &data->iov, 1, bs->dsk.journal_offset + md_offset); + md_offset += data->iov.iov_len; + submitted++; + bs->ringloop->submit(); +resume_3: + if (submitted > 0) + { + wait_state = 3; + return 1; + } + } + } + printf("Reading blockstore heap metadata\n"); // Skip superblock md_offset = bs->dsk.meta_block_size; next_offset = md_offset; - entries_per_block = bs->dsk.meta_block_size / bs->dsk.clean_entry_size; // Read the rest of the metadata -resume_2: - if (next_offset < bs->dsk.meta_len && submitted == 0) +resume_4: + if (next_offset < bs->dsk.meta_area_size && submitted == 0) { // Submit one read for (int i = 0; i < 2; i++) { if (!bufs[i].state) { - bufs[i].buf = (uint8_t*)metadata_buffer + (bs->inmemory_meta - ? next_offset-md_offset - : i*bs->metadata_buf_size); + bufs[i].buf = (uint8_t*)metadata_buffer + i*bs->metadata_buf_size; bufs[i].offset = next_offset; - bufs[i].size = bs->dsk.meta_len-next_offset > bs->metadata_buf_size - ? bs->metadata_buf_size : bs->dsk.meta_len-next_offset; + bufs[i].size = bs->dsk.meta_area_size-next_offset > bs->metadata_buf_size + ? bs->metadata_buf_size : bs->dsk.meta_area_size-next_offset; bufs[i].state = INIT_META_READING; submitted++; next_offset += bufs[i].size; @@ -231,86 +221,19 @@ resume_2: if (bufs[i].state == INIT_META_READ_DONE) { // Handle result - bool changed = false; - for (uint64_t sector = 0; sector < bufs[i].size; sector += bs->dsk.meta_block_size) - { - // handle entries - if (handle_meta_block(bufs[i].buf + sector, entries_per_block, - ((bufs[i].offset + sector - md_offset) / bs->dsk.meta_block_size) * entries_per_block)) - changed = true; - } - if (changed && !bs->inmemory_meta && !bs->readonly) - { - // write the modified buffer back - GET_SQE(); - assert(bufs[i].size <= 0x7fffffff); - data->iov = { bufs[i].buf, (size_t)bufs[i].size }; - data->callback = [this, i](ring_data_t *data) { handle_event(data, i); }; - io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bufs[i].offset); - bs->ringloop->submit(); - bufs[i].state = INIT_META_WRITING; - submitted++; - } - else - { - bufs[i].state = 0; - } + entries_loaded += bs->heap->load_blocks(bufs[i].offset-bs->dsk.meta_block_size, bufs[i].size, bufs[i].buf); + bufs[i].state = 0; bs->ringloop->wakeup(); } } if (submitted > 0) { - wait_state = 2; + wait_state = 4; return 1; } - if (entries_to_zero.size() && !bs->inmemory_meta && !bs->readonly) - { - std::sort(entries_to_zero.begin(), entries_to_zero.end()); - // we have to zero out additional entries - for (i = 0; i < entries_to_zero.size(); ) - { - next_offset = entries_to_zero[i]/entries_per_block; - for (j = i; j < entries_to_zero.size() && entries_to_zero[j]/entries_per_block == next_offset; j++) {} - GET_SQE(); - last_read_offset = (1+next_offset)*bs->dsk.meta_block_size; - data->iov = { metadata_buffer, (size_t)bs->dsk.meta_block_size }; - data->callback = [this](ring_data_t *data) { handle_event(data, -1); }; - io_uring_prep_readv(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + (1+next_offset)*bs->dsk.meta_block_size); - bs->ringloop->submit(); - submitted++; -resume_5: - if (submitted > 0) - { - wait_state = 5; - return 1; - } - for (; i < j; i++) - { - uint64_t pos = (entries_to_zero[i] % entries_per_block); - memset((uint8_t*)metadata_buffer + pos*bs->dsk.clean_entry_size, 0, bs->dsk.clean_entry_size); - } - GET_SQE(); - data->iov = { metadata_buffer, (size_t)bs->dsk.meta_block_size }; - data->callback = [this](ring_data_t *data) { handle_event(data, -1); }; - io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + (1+next_offset)*bs->dsk.meta_block_size); - bs->ringloop->submit(); - submitted++; -resume_6: - if (submitted > 0) - { - wait_state = 6; - return 1; - } - } - entries_to_zero.clear(); - } // metadata read finished - printf("Metadata entries loaded: %ju, free blocks: %ju / %ju\n", entries_loaded, bs->data_alloc->get_free_count(), bs->dsk.block_count); - if (!bs->inmemory_meta) - { - free(metadata_buffer); - metadata_buffer = NULL; - } + bs->heap->finish_load(); + printf("Metadata entries loaded: %ju, used blocks: %ju / %ju\n", entries_loaded, bs->heap->get_data_used_space() / bs->dsk.data_block_size, bs->dsk.block_count); if (zero_on_init && !bs->disable_meta_fsync) { GET_SQE(); @@ -320,900 +243,45 @@ resume_6: data->callback = [this](ring_data_t *data) { handle_event(data, -1); }; submitted++; bs->ringloop->submit(); - resume_4: + resume_5: if (submitted > 0) { - wait_state = 4; + wait_state = 5; return 1; } } - return 0; -} - -bool blockstore_init_meta::handle_meta_block(uint8_t *buf, uint64_t entries_per_block, uint64_t done_cnt) -{ - bool updated = false; - uint64_t max_i = entries_per_block; - if (max_i > bs->dsk.block_count-done_cnt) - max_i = bs->dsk.block_count-done_cnt; - for (uint64_t i = 0; i < max_i; i++) + if (!bs->dsk.inmemory_journal) { - clean_disk_entry *entry = (clean_disk_entry*)(buf + i*bs->dsk.clean_entry_size); - if (entry->oid.inode > 0) + // asynchronous recheck + bs->heap->recheck_small_writes([this](uint64_t offset, uint64_t len, uint8_t *buf, std::function cb) { - if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_V2) + if (!buf) { - // Check entry crc32 - uint32_t *entry_csum = (uint32_t*)((uint8_t*)entry + bs->dsk.clean_entry_size - 4); - if (*entry_csum != crc32c(0, entry, bs->dsk.clean_entry_size - 4)) - { - printf("Metadata entry %ju is corrupt (checksum mismatch: %08x vs %08x), skipping\n", done_cnt+i, *entry_csum, crc32c(0, entry, bs->dsk.clean_entry_size - 4)); - // zero out the invalid entry, otherwise we'll hit "tried to overwrite non-zero metadata entry" later - if (bs->inmemory_meta) - { - memset(entry, 0, bs->dsk.clean_entry_size); - } - else - { - entries_to_zero.push_back(done_cnt+i); - } - continue; - } + wait_state = 7; + bs->ringloop->wakeup(); + return; } - if (!bs->inmemory_meta && bs->dsk.clean_entry_bitmap_size) - { - memcpy(bs->clean_bitmaps + (done_cnt+i) * 2 * bs->dsk.clean_entry_bitmap_size, &entry->bitmap, 2 * bs->dsk.clean_entry_bitmap_size); - } - auto & clean_db = bs->clean_db_shard(entry->oid); - auto clean_it = clean_db.find(entry->oid); - if (clean_it == clean_db.end() || clean_it->second.version < entry->version) - { - if (clean_it != clean_db.end()) - { - // free the previous block - // here we have to zero out the previous entry because otherwise we'll hit - // "tried to overwrite non-zero metadata entry" later - uint64_t old_clean_loc = clean_it->second.location / bs->dsk.data_block_size; - if (bs->inmemory_meta) - { - uint64_t sector = (old_clean_loc / entries_per_block) * bs->dsk.meta_block_size; - uint64_t pos = (old_clean_loc % entries_per_block); - clean_disk_entry *old_entry = (clean_disk_entry*)((uint8_t*)bs->metadata_buffer + sector + pos*bs->dsk.clean_entry_size); - memset(old_entry, 0, bs->dsk.clean_entry_size); - } - else if (old_clean_loc >= done_cnt) - { - updated = true; - uint64_t sector = ((old_clean_loc - done_cnt) / entries_per_block) * bs->dsk.meta_block_size; - uint64_t pos = (old_clean_loc % entries_per_block); - clean_disk_entry *old_entry = (clean_disk_entry*)(buf + sector + pos*bs->dsk.clean_entry_size); - memset(old_entry, 0, bs->dsk.clean_entry_size); - } - else - { - entries_to_zero.push_back(clean_it->second.location / bs->dsk.data_block_size); - } -#ifdef BLOCKSTORE_DEBUG - printf("Free block %ju from %jx:%jx v%ju (new location is %ju)\n", - old_clean_loc, - clean_it->first.inode, clean_it->first.stripe, clean_it->second.version, - done_cnt+i); -#endif - bs->data_alloc->set(old_clean_loc, false); - } - else - { - bs->inode_space_stats[entry->oid.inode] += bs->dsk.data_block_size; - bs->used_blocks++; - } - entries_loaded++; -#ifdef BLOCKSTORE_DEBUG - printf("Allocate block (clean entry) %ju: %jx:%jx v%ju\n", done_cnt+i, entry->oid.inode, entry->oid.stripe, entry->version); -#endif - bs->data_alloc->set(done_cnt+i, true); - clean_db[entry->oid] = (struct clean_entry){ - .version = entry->version, - .location = (done_cnt+i) * bs->dsk.data_block_size, - }; - } - else - { - // here we also have to zero out the entry - updated = true; - memset(entry, 0, bs->dsk.clean_entry_size); -#ifdef BLOCKSTORE_DEBUG - printf("Old clean entry %ju: %jx:%jx v%ju\n", done_cnt+i, entry->oid.inode, entry->oid.stripe, entry->version); -#endif - } - } - } - return updated; -} - -blockstore_init_journal::blockstore_init_journal(blockstore_impl_t *bs) -{ - this->bs = bs; - next_free = bs->journal.block_size; - simple_callback = [this](ring_data_t *data1) - { - if (data1->res != data1->iov.iov_len) - { - throw std::runtime_error(std::string("I/O operation failed while reading journal: ") + strerror(-data1->res)); - } - wait_count--; - }; -} - -void blockstore_init_journal::handle_event(ring_data_t *data1) -{ - if (data1->res <= 0) - { - throw std::runtime_error( - std::string("read journal failed at offset ") + std::to_string(journal_pos) + - std::string(": ") + strerror(-data1->res) - ); - } - done.push_back({ - .buf = submitted_buf, - .pos = journal_pos, - .len = (uint64_t)data1->res, - }); - journal_pos += data1->res; - if (journal_pos >= bs->journal.len) - { - // Continue from the beginning - journal_pos = bs->journal.block_size; - wrapped = true; - } - submitted_buf = NULL; -} - -int blockstore_init_journal::loop() -{ - if (wait_state == 1) - goto resume_1; - else if (wait_state == 2) - goto resume_2; - else if (wait_state == 3) - goto resume_3; - else if (wait_state == 4) - goto resume_4; - else if (wait_state == 5) - goto resume_5; - else if (wait_state == 6) - goto resume_6; - else if (wait_state == 7) - goto resume_7; - printf("Reading blockstore journal\n"); - if (!bs->journal.inmemory) - submitted_buf = memalign_or_die(MEM_ALIGNMENT, 2*bs->journal.block_size); - else - submitted_buf = bs->journal.buffer; - // Read first block of the journal - sqe = bs->get_sqe(); - if (!sqe) - throw std::runtime_error("io_uring is full while trying to read journal"); - data = ((ring_data_t*)sqe->user_data); - data->iov = { submitted_buf, (size_t)bs->journal.block_size }; - data->callback = simple_callback; - io_uring_prep_readv(sqe, bs->dsk.journal_fd, &data->iov, 1, bs->journal.offset); - bs->ringloop->submit(); - wait_count = 1; -resume_1: - if (wait_count > 0) - { - wait_state = 1; - return 1; - } - if (iszero((uint64_t*)submitted_buf, bs->journal.block_size / sizeof(uint64_t))) - { - // Journal is empty - // FIXME handle this wrapping to journal_block_size better (maybe) - bs->journal.used_start = bs->journal.block_size; - bs->journal.next_free = bs->journal.block_size; - // Initialize journal "superblock" and the first block - memset(submitted_buf, 0, 2*bs->journal.block_size); - *((journal_entry_start*)submitted_buf) = { - .crc32 = 0, - .magic = JOURNAL_MAGIC, - .type = JE_START, - .size = sizeof(journal_entry_start), - .reserved = 0, - .journal_start = bs->journal.block_size, - .version = JOURNAL_VERSION_V2, - .data_csum_type = bs->dsk.data_csum_type, - .csum_block_size = bs->dsk.csum_block_size, - }; - ((journal_entry_start*)submitted_buf)->crc32 = je_crc32((journal_entry*)submitted_buf); - if (bs->readonly) - { - printf("Skipping journal initialization because blockstore is readonly\n"); - } - else - { - // Cool effect. Same operations result in journal replay. - // FIXME: Randomize initial crc32. Track crc32 when trimming. - printf("Resetting journal\n"); GET_SQE(); - data->iov = (struct iovec){ submitted_buf, (size_t)(2*bs->journal.block_size) }; - data->callback = simple_callback; - io_uring_prep_writev(sqe, bs->dsk.journal_fd, &data->iov, 1, bs->journal.offset); - wait_count++; + data->iov = (iovec){ buf, len }; + data->callback = [this, offset, cb](ring_data_t *data) + { + if (data->res < 0) + { + fprintf(stderr, "Buffer area read failed at offset %ju: %d\n", offset, data->res); + exit(1); + } + cb(); + }; + io_uring_prep_readv(sqe, bs->dsk.journal_fd, &data->iov, 1, bs->dsk.journal_offset + offset); bs->ringloop->submit(); - resume_6: - if (wait_count > 0) - { - wait_state = 6; - return 1; - } - if (!bs->disable_journal_fsync) - { - GET_SQE(); - io_uring_prep_fsync(sqe, bs->dsk.journal_fd, IORING_FSYNC_DATASYNC); - data->iov = { 0 }; - data->callback = simple_callback; - wait_count++; - bs->ringloop->submit(); - } - resume_4: - if (wait_count > 0) - { - wait_state = 4; - return 1; - } - } - if (!bs->journal.inmemory) - { - free(submitted_buf); - } + }, bs->meta_write_recheck_parallelism); +resume_6: + wait_state = 6; + return 1; +resume_7: + ; } - else - { - // First block always contains a single JE_START entry - je_start = (journal_entry_start*)submitted_buf; - if (je_start->magic != JOURNAL_MAGIC || - je_start->type != JE_START || - je_crc32((journal_entry*)je_start) != je_start->crc32 || - je_start->size != JE_START_V0_SIZE && je_start->size != JE_START_V1_SIZE && je_start->size != JE_START_V2_SIZE) - { - // Entry is corrupt - fprintf(stderr, "First entry of the journal is corrupt or unsupported\n"); - exit(1); - } - if (je_start->size == JE_START_V0_SIZE || - (je_start->version != JOURNAL_VERSION_V1 || je_start->size != JE_START_V1_SIZE) && - (je_start->version != JOURNAL_VERSION_V2 || je_start->size != JE_START_V2_SIZE && je_start->size != JE_START_V1_SIZE)) - { - fprintf( - stderr, "The code only supports journal versions 2 and 1, but it is %ju on disk." - " Please use vitastor-disk to rewrite the journal\n", - je_start->size == JE_START_V0_SIZE ? 0 : je_start->version - ); - exit(1); - } - if (je_start->version == JOURNAL_VERSION_V1 || - je_start->version == JOURNAL_VERSION_V2 && je_start->size == JE_START_V1_SIZE) - { - je_start->data_csum_type = 0; - je_start->csum_block_size = 0; - } - if (je_start->data_csum_type != bs->dsk.data_csum_type || - je_start->csum_block_size != bs->dsk.csum_block_size) - { - printf( - "Configuration stored in journal superblock (data_csum_type=%u, csum_block_size=%u)" - " differs from OSD configuration (%u/%u).\n", - je_start->data_csum_type, je_start->csum_block_size, - bs->dsk.data_csum_type, bs->dsk.csum_block_size - ); - exit(1); - } - next_free = journal_pos = bs->journal.used_start = je_start->journal_start; - if (!bs->journal.inmemory) - free(submitted_buf); - submitted_buf = NULL; - crc32_last = 0; - // Read journal - while (1) - { - resume_2: - if (submitted_buf) - { - wait_state = 2; - return 1; - } - if (!wrapped || journal_pos < bs->journal.used_start) - { - GET_SQE(); - uint64_t end = bs->journal.len; - if (journal_pos < bs->journal.used_start) - end = bs->journal.used_start; - if (!bs->journal.inmemory) - submitted_buf = memalign_or_die(MEM_ALIGNMENT, JOURNAL_BUFFER_SIZE); - else - submitted_buf = (uint8_t*)bs->journal.buffer + journal_pos; - data->iov = { - submitted_buf, - (size_t)(end - journal_pos < JOURNAL_BUFFER_SIZE ? end - journal_pos : JOURNAL_BUFFER_SIZE), - }; - data->callback = [this](ring_data_t *data1) { handle_event(data1); }; - io_uring_prep_readv(sqe, bs->dsk.journal_fd, &data->iov, 1, bs->journal.offset + journal_pos); - bs->ringloop->submit(); - } - while (done.size() > 0) - { - handle_res = handle_journal_part(done[0].buf, done[0].pos, done[0].len); - if (handle_res == 0) - { - // journal ended - // zero out corrupted entry, if required - if (init_write_buf && !bs->readonly) - { - GET_SQE(); - data->iov = { init_write_buf, (size_t)bs->journal.block_size }; - data->callback = simple_callback; - io_uring_prep_writev(sqe, bs->dsk.journal_fd, &data->iov, 1, bs->journal.offset + init_write_sector); - wait_count++; - bs->ringloop->submit(); - resume_7: - if (wait_count > 0) - { - wait_state = 7; - return 1; - } - if (!bs->disable_journal_fsync) - { - GET_SQE(); - data->iov = { 0 }; - data->callback = simple_callback; - io_uring_prep_fsync(sqe, bs->dsk.journal_fd, IORING_FSYNC_DATASYNC); - wait_count++; - bs->ringloop->submit(); - } - resume_5: - if (wait_count > 0) - { - wait_state = 5; - return 1; - } - } - // wait for the next read to complete, then stop - resume_3: - if (submitted_buf) - { - wait_state = 3; - return 1; - } - // free buffers - if (!bs->journal.inmemory) - for (auto & e: done) - free(e.buf); - done.clear(); - break; - } - else if (handle_res == 1) - { - // OK, remove it - if (!bs->journal.inmemory) - { - free(done[0].buf); - } - done.erase(done.begin()); - } - else if (handle_res == 2) - { - // Need to wait for more reads - break; - } - } - if (!submitted_buf) - { - break; - } - } - } - for (auto ov: double_allocs) - { - auto dirty_it = bs->dirty_db.find(ov); - if (dirty_it != bs->dirty_db.end() && - IS_BIG_WRITE(dirty_it->second.state) && - dirty_it->second.location == UINT64_MAX) - { - printf("Fatal error (bug): %jx:%jx v%ju big_write journal_entry was allocated over another object\n", - dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version); - exit(1); - } - } - bs->flusher->mark_trim_possible(); - bs->journal.dirty_start = bs->journal.next_free; - printf( - "Journal entries loaded: %ju, free journal space: %ju bytes (%08jx..%08jx is used), free blocks: %ju / %ju\n", - entries_loaded, - (bs->journal.next_free >= bs->journal.used_start - ? bs->journal.len-bs->journal.block_size - (bs->journal.next_free-bs->journal.used_start) - : bs->journal.used_start - bs->journal.next_free), - bs->journal.used_start, bs->journal.next_free, - bs->data_alloc->get_free_count(), bs->dsk.block_count - ); - bs->journal.crc32_last = crc32_last; + free(metadata_buffer); + metadata_buffer = NULL; return 0; } - -int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, uint64_t len) -{ - uint64_t proc_pos, pos; - if (continue_pos != 0) - { - proc_pos = (continue_pos / bs->journal.block_size) * bs->journal.block_size; - pos = continue_pos % bs->journal.block_size; - continue_pos = 0; - goto resume; - } - while (next_free >= done_pos && next_free < done_pos+len) - { - proc_pos = next_free; - pos = 0; - next_free += bs->journal.block_size; - if (next_free >= bs->journal.len) - { - next_free = bs->journal.block_size; - } - resume: - while (pos < bs->journal.block_size) - { - auto buf_pos = proc_pos - done_pos + pos; - journal_entry *je = (journal_entry*)((uint8_t*)buf + buf_pos); - if (je->magic != JOURNAL_MAGIC || buf_pos+je->size > len || je_crc32(je) != je->crc32 || - je->type < JE_MIN || je->type > JE_MAX || started && je->crc32_prev != crc32_last) - { - if (pos == 0) - { - // invalid entry in the beginning, this is definitely the end of the journal - bs->journal.next_free = proc_pos; - return 0; - } - else - { - // allow partially filled sectors - break; - } - } - if (je->type == JE_SMALL_WRITE || je->type == JE_SMALL_WRITE_INSTANT) - { -#ifdef BLOCKSTORE_DEBUG - printf( - "je_small_write%s oid=%jx:%jx ver=%ju offset=%u len=%u\n", - je->type == JE_SMALL_WRITE_INSTANT ? "_instant" : "", - je->small_write.oid.inode, je->small_write.oid.stripe, je->small_write.version, - je->small_write.offset, je->small_write.len - ); -#endif - // oid, version, offset, len - uint64_t prev_free = next_free; - if (next_free + je->small_write.len > bs->journal.len) - { - // data continues from the beginning of the journal - next_free = bs->journal.block_size; - } - uint64_t location = next_free; - next_free += je->small_write.len; - if (next_free >= bs->journal.len) - { - next_free = bs->journal.block_size; - } - if (location != je->small_write.data_offset) - { - char err[1024]; - snprintf(err, 1024, "BUG: calculated journal data offset (%08jx) != stored journal data offset (%08jx)", location, je->small_write.data_offset); - throw std::runtime_error(err); - } - small_write_data.clear(); - if (location >= done_pos && location+je->small_write.len <= done_pos+len) - { - // data is within this buffer - small_write_data.push_back((iovec){ - .iov_base = (uint8_t*)buf + location - done_pos, - .iov_len = je->small_write.len, - }); - } - else - { - // this case is even more interesting because we must carry data crc32 check to next buffer(s) - uint64_t covered = 0; - for (int i = 0; i < done.size(); i++) - { - if (location+je->small_write.len > done[i].pos && - location < done[i].pos+done[i].len) - { - uint64_t part_end = (location+je->small_write.len < done[i].pos+done[i].len - ? location+je->small_write.len : done[i].pos+done[i].len); - uint64_t part_begin = (location < done[i].pos ? done[i].pos : location); - covered += part_end - part_begin; - small_write_data.push_back((iovec){ - .iov_base = (uint8_t*)done[i].buf + part_begin - done[i].pos, - .iov_len = (size_t)(part_end - part_begin), - }); - } - } - if (covered < je->small_write.len) - { - continue_pos = proc_pos+pos; - next_free = prev_free; - return 2; - } - } - bool data_csum_valid = true; - if (!bs->dsk.csum_block_size) - { - uint32_t data_crc32 = 0; - for (auto & sd: small_write_data) - { - data_crc32 = crc32c(data_crc32, sd.iov_base, sd.iov_len); - } - data_csum_valid = data_crc32 == je->small_write.crc32_data; - if (!data_csum_valid) - { - printf( - "Journal entry data is corrupt for small_write%s oid=%jx:%jx ver=%ju offset=%u len=%u - data crc32 %x != %x\n", - je->type == JE_SMALL_WRITE_INSTANT ? "_instant" : "", - je->small_write.oid.inode, je->small_write.oid.stripe, je->small_write.version, - je->small_write.offset, je->small_write.len, - data_crc32, je->small_write.crc32_data - ); - } - } - else if (je->small_write.len > 0) - { - // FIXME: deduplicate with disk_tool_journal.cpp - // like in enqueue_write() - uint32_t start = je->small_write.offset / bs->dsk.csum_block_size; - uint32_t end = (je->small_write.offset+je->small_write.len-1) / bs->dsk.csum_block_size; - uint32_t data_csum_size = (end-start+1) * (bs->dsk.data_csum_type & 0xFF); - uint32_t required_size = sizeof(journal_entry_small_write) + bs->dsk.clean_entry_bitmap_size + data_csum_size; - if (je->size != required_size) - { - printf( - "Journal entry data has invalid size for small_write%s oid=%jx:%jx ver=%ju offset=%u len=%u - should be %u bytes but is %u bytes\n", - je->type == JE_SMALL_WRITE_INSTANT ? "_instant" : "", - je->small_write.oid.inode, je->small_write.oid.stripe, je->small_write.version, - je->small_write.offset, je->small_write.len, - required_size, je->size - ); - data_csum_valid = false; - } - else - { - int sd_num = 0; - size_t sd_pos = 0; - uint32_t *block_csums = (uint32_t*)((uint8_t*)je + sizeof(journal_entry_small_write) + bs->dsk.clean_entry_bitmap_size); - for (uint32_t pos = start; pos <= end; pos++, block_csums++) - { - size_t block_left = (pos == start - ? (start == end - ? je->small_write.len - : bs->dsk.csum_block_size - je->small_write.offset%bs->dsk.csum_block_size) - : (pos < end - ? bs->dsk.csum_block_size - : (je->small_write.offset + je->small_write.len)%bs->dsk.csum_block_size)); - if (pos > start && pos == end && block_left == 0) - { - // full last block - block_left = bs->dsk.csum_block_size; - } - uint32_t block_crc32 = 0; - while (block_left > 0) - { - assert(sd_num < small_write_data.size()); - if (small_write_data[sd_num].iov_len >= sd_pos+block_left) - { - block_crc32 = crc32c(block_crc32, (uint8_t*)small_write_data[sd_num].iov_base+sd_pos, block_left); - sd_pos += block_left; - break; - } - else - { - block_crc32 = crc32c(block_crc32, (uint8_t*)small_write_data[sd_num].iov_base+sd_pos, small_write_data[sd_num].iov_len-sd_pos); - block_left -= (small_write_data[sd_num].iov_len-sd_pos); - sd_pos = 0; - sd_num++; - } - } - if (block_crc32 != *block_csums) - { - printf( - "Journal entry data is corrupt for small_write%s oid=%jx:%jx ver=%ju offset=%u len=%u - block %u crc32 %x != %x\n", - je->type == JE_SMALL_WRITE_INSTANT ? "_instant" : "", - je->small_write.oid.inode, je->small_write.oid.stripe, je->small_write.version, - je->small_write.offset, je->small_write.len, - pos, block_crc32, *block_csums - ); - data_csum_valid = false; - break; - } - } - } - } - if (!data_csum_valid) - { - // journal entry is corrupt, stop here - // interesting thing is that we must clear the corrupt entry if we're not readonly, - // because we don't write next entries in the same journal block - memset((uint8_t*)buf + proc_pos - done_pos + pos, 0, bs->journal.block_size - pos); - bs->journal.next_free = prev_free; - init_write_buf = (uint8_t*)buf + proc_pos - done_pos; - init_write_sector = proc_pos; - return 0; - } - auto & clean_db = bs->clean_db_shard(je->small_write.oid); - auto clean_it = clean_db.find(je->small_write.oid); - if (clean_it == clean_db.end() || - clean_it->second.version < je->small_write.version) - { - obj_ver_id ov = { - .oid = je->small_write.oid, - .version = je->small_write.version, - }; - uint64_t dyn_size = bs->dsk.dirty_dyn_size(je->small_write.offset, je->small_write.len); - void *dyn = NULL; - void *dyn_from = (uint8_t*)je + sizeof(journal_entry_small_write); - if (!bs->alloc_dyn_data) - { - // Bitmap without checksum is only 4 bytes for 128k objects, save it inline - // It can even contain 4 byte bitmap + 4 byte CRC32 for 4 kb writes :) - memcpy(&dyn, dyn_from, dyn_size); - } - else - { - // FIXME Using large blockstore objects will result in a lot of small - // allocations for entry bitmaps. This can only be fixed by using - // a patched map with dynamic entry size, but not the btree_map, - // because it doesn't keep iterators valid all the time. - dyn = malloc_or_die(dyn_size+sizeof(int)); - *((int*)dyn) = 1; - memcpy((uint8_t*)dyn+sizeof(int), dyn_from, dyn_size); - } - bs->dirty_db.emplace(ov, (dirty_entry){ - .state = (BS_ST_SMALL_WRITE | BS_ST_SYNCED), - .flags = 0, - .location = location, - .offset = je->small_write.offset, - .len = je->small_write.len, - .journal_sector = proc_pos, - .dyn_data = dyn, - }); - bs->journal.used_sectors[proc_pos]++; -#ifdef BLOCKSTORE_DEBUG - printf( - "journal offset %08jx is used by %jx:%jx v%ju (%ju refs)\n", - proc_pos, ov.oid.inode, ov.oid.stripe, ov.version, bs->journal.used_sectors[proc_pos] - ); -#endif - auto & unstab = bs->unstable_writes[ov.oid]; - unstab = unstab < ov.version ? ov.version : unstab; - if (je->type == JE_SMALL_WRITE_INSTANT) - { - bs->mark_stable(ov, true); - } - } - } - else if (je->type == JE_BIG_WRITE || je->type == JE_BIG_WRITE_INSTANT) - { -#ifdef BLOCKSTORE_DEBUG - printf( - "je_big_write%s oid=%jx:%jx ver=%ju loc=%ju\n", - je->type == JE_BIG_WRITE_INSTANT ? "_instant" : "", - je->big_write.oid.inode, je->big_write.oid.stripe, je->big_write.version, je->big_write.location / bs->dsk.data_block_size - ); -#endif - auto dirty_it = bs->dirty_db.upper_bound((obj_ver_id){ - .oid = je->big_write.oid, - .version = UINT64_MAX, - }); - if (dirty_it != bs->dirty_db.begin() && bs->dirty_db.size() > 0) - { - dirty_it--; - if (dirty_it->first.oid == je->big_write.oid && - dirty_it->first.version >= je->big_write.version && - (dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_DELETE) - { - // It is allowed to overwrite a deleted object with a - // version number smaller than deletion version number, - // because the presence of a BIG_WRITE entry means that - // its data and metadata are already flushed. - // We don't know if newer versions are flushed, but - // the previous delete definitely is. - // So we forget previous dirty entries, but retain the clean one. - // This feature is required for writes happening shortly - // after deletes. - erase_dirty_object(dirty_it); - } - } - auto & clean_db = bs->clean_db_shard(je->big_write.oid); - auto clean_it = clean_db.find(je->big_write.oid); - if (clean_it == clean_db.end() || - clean_it->second.version < je->big_write.version) - { - // oid, version, block - obj_ver_id ov = { - .oid = je->big_write.oid, - .version = je->big_write.version, - }; - uint64_t dyn_size = bs->dsk.dirty_dyn_size(je->big_write.offset, je->big_write.len); - void *dyn = NULL; - void *dyn_from = (uint8_t*)je + sizeof(journal_entry_big_write); - if (!bs->alloc_dyn_data) - { - // Bitmap without checksum is only 4 bytes for 128k objects, save it inline - memcpy(&dyn, dyn_from, dyn_size); - } - else - { - // FIXME Using large blockstore objects will result in a lot of small - // allocations for entry bitmaps. This can only be fixed by using - // a patched map with dynamic entry size, but not the btree_map, - // because it doesn't keep iterators valid all the time. - dyn = malloc_or_die(dyn_size+sizeof(int)); - *((int*)dyn) = 1; - memcpy((uint8_t*)dyn+sizeof(int), dyn_from, dyn_size); - } - auto dirty_it = bs->dirty_db.emplace(ov, (dirty_entry){ - .state = (BS_ST_BIG_WRITE | BS_ST_SYNCED), - .flags = 0, - .location = je->big_write.location, - .offset = je->big_write.offset, - .len = je->big_write.len, - .journal_sector = proc_pos, - .dyn_data = dyn, - }).first; - if (bs->data_alloc->get(je->big_write.location / bs->dsk.data_block_size)) - { - // This is probably a big_write that's already flushed and freed, but it may - // also indicate a bug. So we remember such entries and recheck them afterwards. - // If it's not a bug they won't be present after reading the whole journal. - dirty_it->second.location = UINT64_MAX; - double_allocs.push_back(ov); - } - else - { -#ifdef BLOCKSTORE_DEBUG - printf( - "Allocate block (journal) %ju: %jx:%jx v%ju\n", - je->big_write.location / bs->dsk.data_block_size, - ov.oid.inode, ov.oid.stripe, ov.version - ); -#endif - bs->data_alloc->set(je->big_write.location / bs->dsk.data_block_size, true); - } - bs->journal.used_sectors[proc_pos]++; -#ifdef BLOCKSTORE_DEBUG - printf( - "journal offset %08jx is used by %jx:%jx v%ju (%ju refs)\n", - proc_pos, ov.oid.inode, ov.oid.stripe, ov.version, bs->journal.used_sectors[proc_pos] - ); -#endif - auto & unstab = bs->unstable_writes[ov.oid]; - unstab = unstab < ov.version ? ov.version : unstab; - if (je->type == JE_BIG_WRITE_INSTANT) - { - bs->mark_stable(ov, true); - } - } - } - else if (je->type == JE_STABLE) - { -#ifdef BLOCKSTORE_DEBUG - printf("je_stable oid=%jx:%jx ver=%ju\n", je->stable.oid.inode, je->stable.oid.stripe, je->stable.version); -#endif - // oid, version - obj_ver_id ov = { - .oid = je->stable.oid, - .version = je->stable.version, - }; - bs->mark_stable(ov, true); - } - else if (je->type == JE_ROLLBACK) - { -#ifdef BLOCKSTORE_DEBUG - printf("je_rollback oid=%jx:%jx ver=%ju\n", je->rollback.oid.inode, je->rollback.oid.stripe, je->rollback.version); -#endif - // rollback dirty writes of up to - obj_ver_id ov = { - .oid = je->rollback.oid, - .version = je->rollback.version, - }; - bs->mark_rolled_back(ov); - } - else if (je->type == JE_DELETE) - { -#ifdef BLOCKSTORE_DEBUG - printf("je_delete oid=%jx:%jx ver=%ju\n", je->del.oid.inode, je->del.oid.stripe, je->del.version); -#endif - bool dirty_exists = false; - auto dirty_it = bs->dirty_db.upper_bound((obj_ver_id){ - .oid = je->del.oid, - .version = UINT64_MAX, - }); - if (dirty_it != bs->dirty_db.begin()) - { - dirty_it--; - dirty_exists = dirty_it->first.oid == je->del.oid; - } - auto & clean_db = bs->clean_db_shard(je->del.oid); - auto clean_it = clean_db.find(je->del.oid); - bool clean_exists = (clean_it != clean_db.end() && - clean_it->second.version < je->del.version); - if (!clean_exists && dirty_exists) - { - // Clean entry doesn't exist. This means that the delete is already flushed. - // So we must not flush this object anymore. - erase_dirty_object(dirty_it); - } - else if (clean_exists || dirty_exists) - { - // oid, version - obj_ver_id ov = { - .oid = je->del.oid, - .version = je->del.version, - }; - bs->dirty_db.emplace(ov, (dirty_entry){ - .state = (BS_ST_DELETE | BS_ST_SYNCED), - .flags = 0, - .location = 0, - .offset = 0, - .len = 0, - .journal_sector = proc_pos, - }); - bs->journal.used_sectors[proc_pos]++; - // Deletions are treated as immediately stable, because - // "2-phase commit" (write->stabilize) isn't sufficient for them anyway - bs->mark_stable(ov, true); - } - // Ignore delete if neither preceding dirty entries nor the clean one are present - } - started = true; - pos += je->size; - crc32_last = je->crc32; - entries_loaded++; - } - } - bs->journal.next_free = next_free; - return 1; -} - -void blockstore_init_journal::erase_dirty_object(blockstore_dirty_db_t::iterator dirty_it) -{ - auto oid = dirty_it->first.oid; - bool exists = !IS_DELETE(dirty_it->second.state); - auto dirty_end = dirty_it; - dirty_end++; - while (1) - { - if (dirty_it == bs->dirty_db.begin()) - { - break; - } - dirty_it--; - if (dirty_it->first.oid != oid) - { - dirty_it++; - break; - } - } - auto & clean_db = bs->clean_db_shard(oid); - auto clean_it = clean_db.find(oid); - uint64_t clean_loc = clean_it != clean_db.end() - ? clean_it->second.location : UINT64_MAX; - if (exists && clean_loc == UINT64_MAX) - { - auto & sp = bs->inode_space_stats[oid.inode]; - if (sp > bs->dsk.data_block_size) - sp -= bs->dsk.data_block_size; - else - bs->inode_space_stats.erase(oid.inode); - bs->used_blocks--; - } - bs->erase_dirty(dirty_it, dirty_end, clean_loc); - // Remove it from the flusher's queue, too - // Otherwise it may end up referring to a small unstable write after reading the rest of the journal - bs->flusher->remove_flush(oid); -} diff --git a/src/blockstore/blockstore_init.h b/src/blockstore/blockstore_init.h index 1df9304b..077bf23a 100644 --- a/src/blockstore/blockstore_init.h +++ b/src/blockstore/blockstore_init.h @@ -25,47 +25,10 @@ class blockstore_init_meta uint64_t next_offset = 0; uint64_t last_read_offset = 0; uint64_t entries_loaded = 0; - unsigned entries_per_block = 0; int i = 0, j = 0; - std::vector entries_to_zero; bool handle_meta_block(uint8_t *buf, uint64_t count, uint64_t done_cnt); void handle_event(ring_data_t *data, int buf_num); public: blockstore_init_meta(blockstore_impl_t *bs); int loop(); }; - -struct bs_init_journal_done -{ - void *buf; - uint64_t pos, len; -}; - -class blockstore_init_journal -{ - blockstore_impl_t *bs; - int wait_state = 0, wait_count = 0, handle_res = 0; - uint64_t entries_loaded = 0; - uint32_t crc32_last = 0; - bool started = false; - uint64_t next_free; - std::vector done; - std::vector double_allocs; - std::vector small_write_data; - uint64_t journal_pos = 0; - uint64_t continue_pos = 0; - void *init_write_buf = NULL; - uint64_t init_write_sector = 0; - bool wrapped = false; - void *submitted_buf; - struct io_uring_sqe *sqe; - struct ring_data_t *data; - journal_entry_start *je_start; - std::function simple_callback; - int handle_journal_part(void *buf, uint64_t done_pos, uint64_t len); - void handle_event(ring_data_t *data); - void erase_dirty_object(blockstore_dirty_db_t::iterator dirty_it); -public: - blockstore_init_journal(blockstore_impl_t* bs); - int loop(); -}; diff --git a/src/blockstore/blockstore_internal.h b/src/blockstore/blockstore_internal.h index f28046e4..cd07b688 100644 --- a/src/blockstore/blockstore_internal.h +++ b/src/blockstore/blockstore_internal.h @@ -1,31 +1,5 @@ #pragma once -// States are not stored on disk. Instead, they're deduced from the journal - -#define BS_ST_SMALL_WRITE 0x01 -#define BS_ST_BIG_WRITE 0x02 -#define BS_ST_DELETE 0x03 - -#define BS_ST_WAIT_DEL 0x10 -#define BS_ST_WAIT_BIG 0x20 -#define BS_ST_IN_FLIGHT 0x30 -#define BS_ST_SUBMITTED 0x40 -#define BS_ST_WRITTEN 0x50 -#define BS_ST_SYNCED 0x60 -#define BS_ST_STABLE 0x70 - -#define BS_ST_INSTANT 0x100 - -#define BS_ST_TYPE_MASK 0x0F -#define BS_ST_WORKFLOW_MASK 0xF0 -#define IS_IN_FLIGHT(st) (((st) & 0xF0) <= BS_ST_SUBMITTED) -#define IS_STABLE(st) (((st) & 0xF0) == BS_ST_STABLE) -#define IS_SYNCED(st) (((st) & 0xF0) >= BS_ST_SYNCED) -#define IS_JOURNAL(st) (((st) & 0x0F) == BS_ST_SMALL_WRITE) -#define IS_BIG_WRITE(st) (((st) & 0x0F) == BS_ST_BIG_WRITE) -#define IS_DELETE(st) (((st) & 0x0F) == BS_ST_DELETE) -#define IS_INSTANT(st) (((st) & BS_ST_TYPE_MASK) == BS_ST_DELETE || ((st) & BS_ST_INSTANT)) - #define BS_SUBMIT_CHECK_SQES(n) \ if (ringloop->space_left() < (n))\ {\ @@ -65,21 +39,11 @@ // Suspend operation until there are more free SQEs #define WAIT_SQE 1 // Suspend operation until there are bytes of free space in the journal on disk -#define WAIT_JOURNAL 3 -// Suspend operation until the next journal sector buffer is free -#define WAIT_JOURNAL_BUFFER 4 -// Suspend operation until there is some free space on the data device -#define WAIT_FREE 5 +#define WAIT_COMPACTION 2 #define COPY_BUF_JOURNAL 1 #define COPY_BUF_DATA 2 #define COPY_BUF_ZERO 4 #define COPY_BUF_CSUM_FILL 8 #define COPY_BUF_COALESCED 16 -#define COPY_BUF_META_BLOCK 32 -#define COPY_BUF_JOURNALED_BIG 64 - -#define STAB_SPLIT_DONE 1 -#define STAB_SPLIT_WAIT 2 -#define STAB_SPLIT_SYNC 3 -#define STAB_SPLIT_TODO 4 +#define COPY_BUF_PADDED 32 diff --git a/src/blockstore/blockstore_open.cpp b/src/blockstore/blockstore_open.cpp index 4341150e..1fc8dae5 100644 --- a/src/blockstore/blockstore_open.cpp +++ b/src/blockstore/blockstore_open.cpp @@ -19,6 +19,7 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init) } min_flusher_count = strtoull(config["min_flusher_count"].c_str(), NULL, 10); journal_trim_interval = strtoull(config["journal_trim_interval"].c_str(), NULL, 10); + flusher_start_threshold = strtoull(config["flusher_start_threshold"].c_str(), NULL, 10); max_write_iodepth = strtoull(config["max_write_iodepth"].c_str(), NULL, 10); throttle_small_writes = config["throttle_small_writes"] == "true" || config["throttle_small_writes"] == "1" || config["throttle_small_writes"] == "yes"; throttle_target_iops = strtoull(config["throttle_target_iops"].c_str(), NULL, 10); @@ -33,13 +34,17 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init) { max_flusher_count = 256; } - if (!min_flusher_count || journal.flush_journal) + if (!min_flusher_count) { min_flusher_count = 1; } if (!journal_trim_interval) { - journal_trim_interval = 512; + journal_trim_interval = 1024; + } + if (!flusher_start_threshold) + { + flusher_start_threshold = 32; } if (!max_write_iodepth) { @@ -85,11 +90,6 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init) { disable_journal_fsync = true; } - if (config["flush_journal"] == "true" || config["flush_journal"] == "1" || config["flush_journal"] == "yes") - { - // Only flush journal and exit - journal.flush_journal = true; - } if (config["immediate_commit"] == "all") { immediate_commit = IMMEDIATE_ALL; @@ -99,23 +99,17 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init) immediate_commit = IMMEDIATE_SMALL; } metadata_buf_size = strtoull(config["meta_buf_size"].c_str(), NULL, 10); - inmemory_meta = config["inmemory_metadata"] != "false" && config["inmemory_metadata"] != "0" && - config["inmemory_metadata"] != "no"; - journal.sector_count = strtoull(config["journal_sector_buffer_count"].c_str(), NULL, 10); - journal.no_same_sector_overwrites = config["journal_no_same_sector_overwrites"] == "true" || - config["journal_no_same_sector_overwrites"] == "1" || config["journal_no_same_sector_overwrites"] == "yes"; - journal.inmemory = config["inmemory_journal"] != "false" && config["inmemory_journal"] != "0" && - config["inmemory_journal"] != "no"; + meta_write_recheck_parallelism = strtoull(config["meta_write_recheck_parallelism"].c_str(), NULL, 10); log_level = strtoull(config["log_level"].c_str(), NULL, 10); // Validate - if (journal.sector_count < 2) - { - journal.sector_count = 32; - } if (metadata_buf_size < 65536) { metadata_buf_size = 4*1024*1024; } + if (!meta_write_recheck_parallelism) + { + meta_write_recheck_parallelism = 16; + } if (dsk.meta_device == dsk.data_device) { disable_meta_fsync = disable_data_fsync; @@ -132,52 +126,4 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init) { throw std::runtime_error("immediate_commit=all requires disable_journal_fsync and disable_data_fsync"); } - // init some fields - journal.block_size = dsk.journal_block_size; - journal.next_free = dsk.journal_block_size; - journal.used_start = dsk.journal_block_size; - // no free space because sector is initially unmapped - journal.in_sector_pos = dsk.journal_block_size; -} - -void blockstore_impl_t::calc_lengths() -{ - dsk.calc_lengths(); - journal.len = dsk.journal_len; - journal.block_size = dsk.journal_block_size; - journal.offset = dsk.journal_offset; - if (inmemory_meta) - { - metadata_buffer = memalign(MEM_ALIGNMENT, dsk.meta_len); - if (!metadata_buffer) - throw std::runtime_error("Failed to allocate memory for the metadata ("+std::to_string(dsk.meta_len/1024/1024)+" MB)"); - } - else if (dsk.clean_entry_bitmap_size || dsk.data_csum_type) - { - clean_bitmaps = (uint8_t*)malloc(dsk.block_count * 2 * dsk.clean_entry_bitmap_size); - if (!clean_bitmaps) - { - throw std::runtime_error( - "Failed to allocate memory for the metadata sparse write bitmap ("+ - std::to_string(dsk.block_count * 2 * dsk.clean_entry_bitmap_size / 1024 / 1024)+" MB)" - ); - } - } - if (journal.inmemory) - { - journal.buffer = memalign(MEM_ALIGNMENT, journal.len); - if (!journal.buffer) - throw std::runtime_error("Failed to allocate memory for journal ("+std::to_string(journal.len/1024/1024)+" MB)"); - } - else - { - journal.sector_buf = (uint8_t*)memalign(MEM_ALIGNMENT, journal.sector_count * dsk.journal_block_size); - if (!journal.sector_buf) - throw std::bad_alloc(); - } - journal.sector_info = (journal_sector_info_t*)calloc(journal.sector_count, sizeof(journal_sector_info_t)); - if (!journal.sector_info) - { - throw std::bad_alloc(); - } } diff --git a/src/blockstore/blockstore_read.cpp b/src/blockstore/blockstore_read.cpp index 4c30ac91..f6870867 100644 --- a/src/blockstore/blockstore_read.cpp +++ b/src/blockstore/blockstore_read.cpp @@ -5,59 +5,284 @@ #include "blockstore_impl.h" #include "blockstore_internal.h" -int blockstore_impl_t::fulfill_read_push(blockstore_op_t *op, void *buf, uint64_t offset, uint64_t len, - uint32_t item_state, uint64_t item_version) +int blockstore_impl_t::dequeue_read(blockstore_op_t *op) { - if (!len) + heap_object_t *obj = heap->lock_and_read_entry(op->oid, PRIV(op)->lsn); + if (!obj) { - // Zero-length read - return 1; + op->version = 0; + op->retval = -ENOENT; + FINISH_OP(op); + return 2; } - else if (IS_DELETE(item_state)) + uint32_t fulfilled = 0; + PRIV(op)->pending_ops = 0; + auto & rv = PRIV(op)->read_vec; + uint64_t result_version = 0; + for (heap_write_t *wr = obj->get_writes(); wr < (heap_write_t*)obj->next(); wr = wr->next(heap)) { - // item is unallocated - return zeroes - memset(buf, 0, len); - return 1; + if (op->version < wr->version) + { + continue; + } + if (!result_version) + { + result_version = wr->version; + if (op->bitmap) + { + memcpy(op->bitmap, wr->get_ext_bitmap(heap), dsk.clean_entry_bitmap_size); + } + } + fulfilled += prepare_read(PRIV(op)->read_vec, obj, wr, op->offset, op->offset+op->len); + if (fulfilled == op->len) + { + break; + } } - assert(!IS_IN_FLIGHT(item_state)); - if (journal.inmemory && IS_JOURNAL(item_state)) + if (!result_version) { - memcpy(buf, (uint8_t*)journal.buffer + offset, len); - return 1; + // May happen if there are entries but all of them are > requested version + op->version = 0; + op->retval = -ENOENT; + FINISH_OP(op); + return 2; + } + assert(fulfilled == op->len); + if (!fulfill_read(op)) + { + // Need to wait. undo added requests, unlock lsn + heap->unlock_entry(op->oid, PRIV(op)->lsn); + if (dsk.csum_block_size > dsk.bitmap_granularity) + { + for (auto & vec: rv) + { + if (!(vec.copy_flags & COPY_BUF_COALESCED) && vec.buf) + { + free(vec.buf); + vec.buf = NULL; + } + } + } + rv.clear(); + return 0; + } + op->version = result_version; + if (!PRIV(op)->pending_ops) + { + // everything is fulfilled from memory + op->retval = op->len; + FINISH_OP(op); + return 2; + } + op->retval = 0; + return 2; +} + +int blockstore_impl_t::fulfill_read(blockstore_op_t *op) +{ + for (auto & vec: PRIV(op)->read_vec) + { + if (vec.copy_flags == COPY_BUF_ZERO) + { + memset(op->buf + vec.offset - op->offset, 0, vec.len); + } + else if (vec.copy_flags == COPY_BUF_JOURNAL && dsk.inmemory_journal) + { + memcpy(op->buf + vec.offset - op->offset, buffer_area + vec.disk_offset, vec.len); + } + else if (!(vec.copy_flags & COPY_BUF_COALESCED)) + { + BS_SUBMIT_GET_SQE(sqe, data); + data->iov = (struct iovec){ vec.buf ? vec.buf : (op->buf + vec.offset - op->offset), (size_t)vec.disk_len }; + PRIV(op)->pending_ops++; + io_uring_prep_readv( + sqe, + (vec.copy_flags & COPY_BUF_JOURNAL) ? dsk.journal_fd : dsk.data_fd, + &data->iov, 1, + ((vec.copy_flags & COPY_BUF_JOURNAL) ? dsk.journal_offset : dsk.data_offset) + vec.disk_offset + ); + data->callback = [this, op](ring_data_t *data) { handle_read_event(data, op); }; + } } - BS_SUBMIT_GET_SQE(sqe, data); - data->iov = (struct iovec){ buf, (size_t)len }; - PRIV(op)->pending_ops++; - io_uring_prep_readv( - sqe, - IS_JOURNAL(item_state) ? dsk.journal_fd : dsk.data_fd, - &data->iov, 1, - (IS_JOURNAL(item_state) ? dsk.journal_offset : dsk.data_offset) + offset - ); - data->callback = [this, op](ring_data_t *data) { handle_read_event(data, op); }; return 1; } +uint32_t blockstore_impl_t::prepare_read(std::vector & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end) +{ + if (wr->offset >= end || wr->offset+wr->len <= start) + { + return 0; + } + start = start < wr->offset ? wr->offset : start; + end = end > wr->offset+wr->len ? wr->offset+wr->len : end; + if ((wr->flags & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE) + { + return prepare_read_with_bitmaps(read_vec, obj, wr, start, end); + } + if ((wr->flags & BS_HEAP_TYPE) == BS_HEAP_TOMBSTONE) + { + return prepare_read_zero(read_vec, start, end); + } + return prepare_read_simple(read_vec, obj, wr, start, end); +} + +uint32_t blockstore_impl_t::prepare_read_with_bitmaps(std::vector & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end) +{ + // BIG_WRITEs contain a bitmap and we have to handle its holes at the upper level, especially with padded checksums + uint32_t res = 0; + uint8_t *bmp = wr->get_int_bitmap(heap); + uint64_t bmp_start = 0, bmp_end = 0, bmp_size = dsk.data_block_size/dsk.bitmap_granularity; + while (bmp_start < bmp_size) + { + while (!(bmp[bmp_end >> 3] & (1 << (bmp_end & 0x7))) && bmp_end < bmp_size) + { + bmp_end++; + } + if (bmp_end > bmp_start) + { + res += prepare_read_zero(read_vec, bmp_start * dsk.bitmap_granularity, bmp_end * dsk.bitmap_granularity); + } + bmp_start = bmp_end; + while (bmp[bmp_end >> 3] & (1 << (bmp_end & 0x7)) && bmp_end < bmp_size) + { + bmp_end++; + } + if (bmp_end > bmp_start) + { + res += prepare_read_simple(read_vec, obj, wr, bmp_start * dsk.bitmap_granularity, bmp_end * dsk.bitmap_granularity); + bmp_start = bmp_end; + } + } + return res; +} + +uint32_t blockstore_impl_t::prepare_read_zero(std::vector & read_vec, uint32_t start, uint32_t end) +{ + uint32_t res = 0; + find_holes(read_vec, start, end, [&](int & pos, bool alloc, uint32_t start, uint32_t end) + { + if (!alloc) + { + res += end-start; + read_vec.insert(read_vec.begin() + (pos++), (copy_buffer_t){ + .copy_flags = COPY_BUF_ZERO, + .offset = start, + .len = end-start, + }); + } + }); + return res; +} + +uint32_t blockstore_impl_t::prepare_read_simple(std::vector & read_vec, heap_object_t *obj, heap_write_t *wr, uint32_t start, uint32_t end) +{ + uint32_t res = 0; + find_holes(read_vec, start, end, [&](int & pos, bool alloc, uint32_t start, uint32_t end) + { + if (alloc) + { + return; + } + res += end-start; + if ((wr->flags & BS_HEAP_TYPE) == BS_HEAP_SMALL_WRITE && dsk.inmemory_journal) + { + // read buffered data from memory + read_vec.insert(read_vec.begin() + (pos++), (copy_buffer_t){ + .copy_flags = COPY_BUF_JOURNAL, + .offset = start, + .len = end-start, + .disk_offset = wr->location + start - wr->offset, + .disk_len = end-start, + .buf = buffer_area + wr->location + start - wr->offset, + .wr_offset = (uint32_t)((uint8_t*)wr - (uint8_t*)obj), + }); + } + else if (dsk.csum_block_size <= dsk.bitmap_granularity) + { + // simple disk read + prepare_disk_read(read_vec, pos, obj, wr, start, end, start, end); + } + else + { + // the most complex case: read data from disk with padding + uint32_t blk_start = start, blk_end = end; + blk_start = (start/dsk.csum_block_size) * dsk.csum_block_size; + blk_start = blk_start < wr->offset ? wr->offset : blk_start; + blk_end = ((end-1) / dsk.csum_block_size + 1) * dsk.csum_block_size; + blk_end = blk_end > wr->offset+wr->len ? wr->offset+wr->len : blk_end; + if (blk_end == blk_start+dsk.csum_block_size || + blk_end == blk_start+2*dsk.csum_block_size && blk_end != end && blk_start != start || + blk_end == end && blk_start == start) + { + // single block, two partial blocks, or any number of full blocks + prepare_disk_read(read_vec, pos, obj, wr, blk_start, blk_end, start, end); + } + else + { + // one or two partial blocks + uint32_t full_start = (blk_start != start ? blk_start+dsk.csum_block_size : blk_start); + uint32_t full_end = (blk_end != end ? blk_end-dsk.csum_block_size : blk_end); + if (blk_start != start) + prepare_disk_read(read_vec, pos, obj, wr, blk_start, full_start, start, full_start); + if (full_start > full_end) + prepare_disk_read(read_vec, pos, obj, wr, full_start, full_end, full_start, full_end); + if (blk_end != end) + prepare_disk_read(read_vec, pos, obj, wr, full_end, blk_end, full_end, end); + } + } + }); + return res; +} + +void blockstore_impl_t::prepare_disk_read(std::vector & read_vec, int & pos, heap_object_t *obj, heap_write_t *wr, + uint32_t blk_start, uint32_t blk_end, uint32_t start, uint32_t end) +{ + copy_buffer_t vec = { + .copy_flags = ((wr->flags & BS_HEAP_TYPE) == BS_HEAP_SMALL_WRITE ? COPY_BUF_JOURNAL : COPY_BUF_DATA), + .offset = start, + .len = end-start, + .disk_offset = wr->location + blk_start - wr->offset, + .disk_len = blk_end - blk_start, + .wr_offset = (uint32_t)((uint8_t*)wr - (uint8_t*)obj), + }; + if (blk_start != start || blk_end != end) + { + vec.copy_flags |= COPY_BUF_PADDED; + if (read_vec.size() > pos && (read_vec[pos].copy_flags & ~COPY_BUF_CSUM_FILL) == vec.copy_flags && + read_vec[pos].offset >= blk_start && read_vec[pos].offset+read_vec[pos].len <= blk_end) + { + // This is the same block as the previous one, we can read it only once + vec.copy_flags |= COPY_BUF_COALESCED; + vec.buf = read_vec[pos].buf; + } + else + { + vec.buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, blk_end-blk_start); + } + } + read_vec.insert(read_vec.begin() + (pos++), vec); +} + void blockstore_impl_t::find_holes(std::vector & read_vec, uint32_t item_start, uint32_t item_end, - std::function callback) + std::function callback) { auto cur_start = item_start; int i = 0; while (cur_start < item_end) { // COPY_BUF_CSUM_FILL items are fake items inserted in the end, their offsets aren't in order - if (i >= read_vec.size() || read_vec[i].copy_flags & COPY_BUF_CSUM_FILL || read_vec[i].offset >= item_end) + if (i >= read_vec.size() || (read_vec[i].copy_flags & COPY_BUF_CSUM_FILL) || read_vec[i].offset >= item_end) { // Hole (at end): cur_start .. item_end - i += callback(i, false, cur_start, item_end); + callback(i, false, cur_start, item_end); break; } else if (read_vec[i].offset > cur_start) { // Hole: cur_start .. min(read_vec[i].offset, item_end) auto cur_end = read_vec[i].offset > item_end ? item_end : read_vec[i].offset; - i += callback(i, false, cur_start, cur_end); + callback(i, false, cur_start, cur_end); cur_start = cur_end; } else if (read_vec[i].offset + read_vec[i].len > cur_start) @@ -65,7 +290,7 @@ void blockstore_impl_t::find_holes(std::vector & read_vec, // Allocated: cur_start .. min(read_vec[i].offset + read_vec[i].len, item_end) auto cur_end = read_vec[i].offset + read_vec[i].len; cur_end = cur_end > item_end ? item_end : cur_end; - i += callback(i, true, cur_start, cur_end); + callback(i, true, cur_start, cur_end); cur_start = cur_end; i++; } @@ -74,748 +299,6 @@ void blockstore_impl_t::find_holes(std::vector & read_vec, } } -int blockstore_impl_t::fulfill_read(blockstore_op_t *read_op, - uint64_t &fulfilled, uint32_t item_start, uint32_t item_end, // FIXME: Rename item_* to dirty_* - uint32_t item_state, uint64_t item_version, uint64_t item_location, - uint64_t journal_sector, uint8_t *csum, int *dyn_data) -{ - int r = 1; - if (item_start < read_op->offset + read_op->len && item_end > read_op->offset) - { - auto & rv = PRIV(read_op)->read_vec; - auto rd_start = item_start < read_op->offset ? read_op->offset : item_start; - auto rd_end = item_end > read_op->offset + read_op->len ? read_op->offset + read_op->len : item_end; - find_holes(rv, rd_start, rd_end, [&](int pos, bool alloc, uint32_t start, uint32_t end) - { - if (!r || alloc) - return 0; - if (!journal.inmemory && dsk.csum_block_size > dsk.bitmap_granularity && IS_JOURNAL(item_state) && !IS_DELETE(item_state)) - { - uint32_t blk_begin = (start/dsk.csum_block_size) * dsk.csum_block_size; - blk_begin = blk_begin < item_start ? item_start : blk_begin; - uint32_t blk_end = ((end-1) / dsk.csum_block_size + 1) * dsk.csum_block_size; - blk_end = blk_end > item_end ? item_end : blk_end; - rv.push_back((copy_buffer_t){ - .copy_flags = COPY_BUF_JOURNAL|COPY_BUF_CSUM_FILL, - .offset = blk_begin, - .len = blk_end-blk_begin, - .csum_buf = (csum + (blk_begin/dsk.csum_block_size - - item_start/dsk.csum_block_size) * (dsk.data_csum_type & 0xFF)), - .dyn_data = dyn_data, - }); - if (dyn_data) - { - (*dyn_data)++; - } - // Submit the journal checksum block read - if (!read_checksum_block(read_op, 1, fulfilled, item_location - item_start)) - { - r = 0; - } - return 0; - } - copy_buffer_t el = { - .copy_flags = (IS_JOURNAL(item_state) ? COPY_BUF_JOURNAL : COPY_BUF_DATA), - .offset = start, - .len = end-start, - .disk_offset = item_location + start - item_start, - .journal_sector = (IS_JOURNAL(item_state) ? journal_sector : 0), - .csum_buf = !csum ? NULL : (csum + (start - item_start) / dsk.csum_block_size * (dsk.data_csum_type & 0xFF)), - .dyn_data = dyn_data, - }; - if (dyn_data) - { - (*dyn_data)++; - } - if (IS_BIG_WRITE(item_state)) - { - // If we don't track it then we may IN THEORY read another object's data: - // submit read -> remove the object -> flush remove -> overwrite with another object -> finish read - // Very improbable, but possible - PRIV(read_op)->clean_block_used = 1; - } - rv.insert(rv.begin() + pos, el); - fulfilled += el.len; - if (!fulfill_read_push(read_op, - (uint8_t*)read_op->buf + el.offset - read_op->offset, - item_location + el.offset - item_start, - el.len, item_state, item_version)) - { - r = 0; - } - return 1; - }); - } - return r; -} - -uint8_t* blockstore_impl_t::get_clean_entry_bitmap(uint64_t block_loc, int offset) -{ - uint8_t *clean_entry_bitmap; - uint64_t meta_loc = block_loc / dsk.data_block_size; - if (inmemory_meta) - { - uint64_t sector = (meta_loc / (dsk.meta_block_size / dsk.clean_entry_size)) * dsk.meta_block_size; - uint64_t pos = (meta_loc % (dsk.meta_block_size / dsk.clean_entry_size)); - clean_entry_bitmap = ((uint8_t*)metadata_buffer + sector + pos*dsk.clean_entry_size + sizeof(clean_disk_entry) + offset); - } - else - clean_entry_bitmap = (uint8_t*)(clean_bitmaps + meta_loc*2*dsk.clean_entry_bitmap_size + offset); - return clean_entry_bitmap; -} - -int blockstore_impl_t::fill_partial_checksum_blocks(std::vector & rv, uint64_t & fulfilled, - uint8_t *clean_entry_bitmap, int *dyn_data, bool from_journal, uint8_t *read_buf, uint64_t read_offset, uint64_t read_end) -{ - if (read_end == read_offset) - return 0; - int required = 0; - read_buf -= read_offset; - uint32_t last_block = (read_end-1)/dsk.csum_block_size; - uint32_t start_block = read_offset/dsk.csum_block_size; - uint32_t end_block = 0; - while (start_block <= last_block) - { - if (read_range_fulfilled(rv, fulfilled, read_buf, clean_entry_bitmap, - start_block*dsk.csum_block_size < read_offset ? read_offset : start_block*dsk.csum_block_size, - (start_block+1)*dsk.csum_block_size > read_end ? read_end : (start_block+1)*dsk.csum_block_size)) - { - // read_range_fulfilled() also adds zero-filled areas - start_block++; - } - else - { - // Find a sequence of checksum blocks required to be read - end_block = start_block; - while ((end_block+1)*dsk.csum_block_size < read_end && - !read_range_fulfilled(rv, fulfilled, read_buf, clean_entry_bitmap, - (end_block+1)*dsk.csum_block_size < read_offset ? read_offset : (end_block+1)*dsk.csum_block_size, - (end_block+2)*dsk.csum_block_size > read_end ? read_end : (end_block+2)*dsk.csum_block_size)) - { - end_block++; - } - end_block++; - // OK, mark this range as required - rv.push_back((copy_buffer_t){ - .copy_flags = COPY_BUF_CSUM_FILL | (from_journal ? COPY_BUF_JOURNALED_BIG : 0), - .offset = start_block*dsk.csum_block_size, - .len = (end_block-start_block)*dsk.csum_block_size, - // save clean_entry_bitmap if we're reading clean data from the journal - .csum_buf = from_journal ? clean_entry_bitmap : NULL, - .dyn_data = dyn_data, - }); - if (dyn_data) - { - (*dyn_data)++; - } - start_block = end_block; - required++; - } - } - return required; -} - -// read_buf should be == op->buf - op->offset -bool blockstore_impl_t::read_range_fulfilled(std::vector & rv, uint64_t & fulfilled, uint8_t *read_buf, - uint8_t *clean_entry_bitmap, uint32_t item_start, uint32_t item_end) -{ - bool all_done = true; - find_holes(rv, item_start, item_end, [&](int pos, bool alloc, uint32_t cur_start, uint32_t cur_end) - { - if (alloc) - return 0; - int diff = 0; - uint32_t bmp_start = cur_start/dsk.bitmap_granularity; - uint32_t bmp_end = cur_end/dsk.bitmap_granularity; - uint32_t bmp_pos = bmp_start; - while (bmp_pos < bmp_end) - { - while (bmp_pos < bmp_end && !(clean_entry_bitmap[bmp_pos >> 3] & (1 << (bmp_pos & 0x7)))) - bmp_pos++; - if (bmp_pos > bmp_start) - { - // zero fill - copy_buffer_t el = { - .copy_flags = COPY_BUF_ZERO, - .offset = bmp_start*dsk.bitmap_granularity, - .len = (bmp_pos-bmp_start)*dsk.bitmap_granularity, - }; - rv.insert(rv.begin() + pos, el); - if (read_buf) - memset(read_buf + el.offset, 0, el.len); - fulfilled += el.len; - diff++; - } - bmp_start = bmp_pos; - while (bmp_pos < bmp_end && (clean_entry_bitmap[bmp_pos >> 3] & (1 << (bmp_pos & 0x7)))) - bmp_pos++; - if (bmp_pos > bmp_start) - { - // something is to be read - all_done = false; - } - bmp_start = bmp_pos; - } - return diff; - }); - return all_done; -} - -bool blockstore_impl_t::read_checksum_block(blockstore_op_t *op, int rv_pos, uint64_t &fulfilled, uint64_t clean_loc) -{ - auto & rv = PRIV(op)->read_vec; - auto *vi = &rv[rv.size()-rv_pos]; - uint32_t item_start = vi->offset, item_end = vi->offset+vi->len; - uint32_t fill_size = 0; - int n_iov = 0; - find_holes(rv, item_start, item_end, [&](int pos, bool alloc, uint32_t cur_start, uint32_t cur_end) - { - if (alloc) - { - fill_size += cur_end-cur_start; - n_iov++; - } - else - { - if (cur_start < op->offset) - { - fill_size += op->offset-cur_start; - n_iov++; - cur_start = op->offset; - } - if (cur_end > op->offset+op->len) - { - fill_size += cur_end-(op->offset+op->len); - n_iov++; - cur_end = op->offset+op->len; - } - if (cur_end > cur_start) - { - n_iov++; - } - } - return 0; - }); - void *buf = memalign_or_die(MEM_ALIGNMENT, fill_size + n_iov*sizeof(struct iovec)); - iovec *iov = (struct iovec*)((uint8_t*)buf+fill_size); - n_iov = 0; - fill_size = 0; - find_holes(rv, item_start, item_end, [&](int pos, bool alloc, uint32_t cur_start, uint32_t cur_end) - { - int res = 0; - if (alloc) - { - iov[n_iov++] = (struct iovec){ (uint8_t*)buf+fill_size, cur_end-cur_start }; - fill_size += cur_end-cur_start; - } - else - { - if (cur_start < op->offset) - { - iov[n_iov++] = (struct iovec){ (uint8_t*)buf+fill_size, op->offset-cur_start }; - fill_size += op->offset-cur_start; - cur_start = op->offset; - } - auto lim_end = cur_end > op->offset+op->len ? op->offset+op->len : cur_end; - if (lim_end > cur_start) - { - iov[n_iov++] = (struct iovec){ (uint8_t*)op->buf+cur_start-op->offset, lim_end-cur_start }; - rv.insert(rv.begin() + pos, (copy_buffer_t){ - .copy_flags = COPY_BUF_DATA, - .offset = cur_start, - .len = lim_end-cur_start, - }); - fulfilled += lim_end-cur_start; - res++; - } - if (cur_end > op->offset+op->len) - { - iov[n_iov++] = (struct iovec){ (uint8_t*)buf+fill_size, cur_end - (op->offset+op->len) }; - fill_size += cur_end - (op->offset+op->len); - cur_end = op->offset+op->len; - } - } - return res; - }); - vi = &rv[rv.size()-rv_pos]; - // Save buf into read_vec too but in a creepy way - // FIXME: Shit, something else should be invented %) - *vi = (copy_buffer_t){ - .copy_flags = vi->copy_flags, - .offset = vi->offset, - .len = ((uint64_t)n_iov << 32) | fill_size, - .disk_offset = clean_loc + item_start, - .buf = (uint8_t*)buf, - .csum_buf = vi->csum_buf, - .dyn_data = vi->dyn_data, - }; - int submit_fd = (vi->copy_flags & COPY_BUF_JOURNAL ? dsk.journal_fd : dsk.data_fd); - uint64_t submit_offset = (vi->copy_flags & COPY_BUF_JOURNAL ? journal.offset : dsk.data_offset); - uint32_t d_pos = 0; - for (int n_pos = 0; n_pos < n_iov; n_pos += IOV_MAX) - { - int n_cur = n_iov-n_pos < IOV_MAX ? n_iov-n_pos : IOV_MAX; - BS_SUBMIT_GET_SQE(sqe, data); - PRIV(op)->pending_ops++; - io_uring_prep_readv(sqe, submit_fd, iov + n_pos, n_cur, submit_offset + clean_loc + item_start + d_pos); - data->callback = [this, op](ring_data_t *data) { handle_read_event(data, op); }; - if (n_pos > 0 || n_pos + IOV_MAX < n_iov) - { - uint32_t d_len = 0; - for (int i = 0; i < IOV_MAX; i++) - d_len += iov[n_pos+i].iov_len; - data->iov.iov_len = d_len; - d_pos += d_len; - } - else - data->iov.iov_len = item_end-item_start; - } - if (!(vi->copy_flags & COPY_BUF_JOURNAL)) - { - // Reads running parallel to flushes of the same clean block may read - // a mixture of old and new data. So we don't verify checksums for such blocks. - PRIV(op)->clean_block_used = 1; - } - return true; -} - -int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op) -{ - auto & clean_db = clean_db_shard(read_op->oid); - auto clean_it = clean_db.find(read_op->oid); - auto dirty_it = dirty_db.upper_bound((obj_ver_id){ - .oid = read_op->oid, - .version = UINT64_MAX, - }); - if (dirty_it != dirty_db.begin()) - dirty_it--; - bool clean_found = clean_it != clean_db.end(); - bool dirty_found = (dirty_it != dirty_db.end() && dirty_it->first.oid == read_op->oid); - if (!clean_found && !dirty_found) - { - read_op->version = 0; - read_op->retval = -ENOENT; - FINISH_OP(read_op); - return 2; - } - uint64_t fulfilled = 0; - PRIV(read_op)->pending_ops = 0; - PRIV(read_op)->clean_block_used = 0; - auto & rv = PRIV(read_op)->read_vec; - uint64_t result_version = 0; - if (dirty_found) - { - while (dirty_it->first.oid == read_op->oid) - { - dirty_entry& dirty = dirty_it->second; - bool version_ok = !IS_IN_FLIGHT(dirty.state) && read_op->version >= dirty_it->first.version; - if (version_ok) - { - if (IS_DELETE(dirty.state)) - { - assert(!result_version); - read_op->version = 0; - read_op->retval = -ENOENT; - FINISH_OP(read_op); - return 2; - } - int *dyn_data = (int*)(dsk.csum_block_size > 0 && alloc_dyn_data ? dirty.dyn_data : NULL); - uint8_t *bmp_ptr = (alloc_dyn_data - ? (uint8_t*)dirty.dyn_data + sizeof(int) : (uint8_t*)&dirty.dyn_data); - if (!result_version) - { - result_version = dirty_it->first.version; - if (read_op->bitmap) - { - memcpy(read_op->bitmap, bmp_ptr, dsk.clean_entry_bitmap_size); - } - } - // If inmemory_journal is false, journal trim will have to wait until the read is completed - if (!IS_JOURNAL(dirty.state)) - { - // Read from data disk, possibly checking checksums - if (!fulfill_clean_read(read_op, fulfilled, bmp_ptr, dyn_data, - dirty.offset, dirty.offset+dirty.len, dirty.location, dirty_it->first.version)) - { - goto undo_read; - } - } - else - { - // Copy from memory or read from journal, possibly checking checksums - if (!fulfill_read(read_op, fulfilled, dirty.offset, dirty.offset + dirty.len, - dirty.state, dirty_it->first.version, dirty.location, dirty.journal_sector+1, - journal.inmemory ? NULL : bmp_ptr+dsk.clean_entry_bitmap_size, dyn_data)) - { - goto undo_read; - } - } - } - if (fulfilled == read_op->len || dirty_it == dirty_db.begin()) - { - break; - } - dirty_it--; - } - } - if (clean_found) - { - if (!result_version) - { - result_version = clean_it->second.version; - if (read_op->bitmap) - { - void *bmp_ptr = get_clean_entry_bitmap(clean_it->second.location, dsk.clean_entry_bitmap_size); - memcpy(read_op->bitmap, bmp_ptr, dsk.clean_entry_bitmap_size); - } - } - if (fulfilled < read_op->len) - { - if (!fulfill_clean_read(read_op, fulfilled, NULL, NULL, 0, dsk.data_block_size, - clean_it->second.location, clean_it->second.version)) - { - goto undo_read; - } - } - } - if (!result_version) - { - // May happen if there are entries in dirty_db but all of them are !version_ok - read_op->version = 0; - read_op->retval = -ENOENT; - FINISH_OP(read_op); - return 2; - } - assert(fulfilled == read_op->len); - read_op->version = result_version; - if (!PRIV(read_op)->pending_ops) - { - // everything is fulfilled from memory - if (!PRIV(read_op)->read_vec.size()) - { - // region is not allocated - return zeroes - memset(read_op->buf, 0, read_op->len); - } - read_op->retval = read_op->len; - FINISH_OP(read_op); - return 2; - } - if (!journal.inmemory) - { - // Journal trim has to wait until the read is completed - record journal sector usage - for (auto & rv: PRIV(read_op)->read_vec) - { - if (rv.journal_sector) - journal.used_sectors.at(rv.journal_sector-1)++; - } - } - read_op->retval = 0; - return 2; -undo_read: - // need to wait. undo added requests, don't dequeue op - if (dsk.csum_block_size > dsk.bitmap_granularity) - { - for (auto & vec: rv) - { - if ((vec.copy_flags & COPY_BUF_CSUM_FILL) && vec.buf) - { - free(vec.buf); - vec.buf = NULL; - } - if (vec.dyn_data && --(*vec.dyn_data) == 0) // refcount - { - free(vec.dyn_data); - vec.dyn_data = NULL; - } - } - } - rv.clear(); - return 0; -} - -int blockstore_impl_t::pad_journal_read(std::vector & rv, copy_buffer_t & cp, - // FIXME Passing dirty_entry& would be nicer - uint64_t dirty_offset, uint64_t dirty_end, uint64_t dirty_loc, uint8_t *csum_ptr, int *dyn_data, - uint64_t offset, uint64_t submit_len, uint64_t & blk_begin, uint64_t & blk_end, uint8_t* & blk_buf) -{ - if (offset % dsk.csum_block_size || submit_len % dsk.csum_block_size) - { - if (offset < blk_end) - { - // Already being read as a part of the previous checksum block series - cp.buf = blk_buf + offset - blk_begin; - cp.copy_flags |= COPY_BUF_COALESCED; - if (offset+submit_len > blk_end) - cp.len = blk_end-offset; - return 2; - } - else - { - // We don't use fill_partial_checksum_blocks for journal because journal writes never have holes (internal bitmap) - blk_begin = (offset/dsk.csum_block_size) * dsk.csum_block_size; - blk_begin = blk_begin < dirty_offset ? dirty_offset : blk_begin; - blk_end = ((offset+submit_len-1)/dsk.csum_block_size + 1) * dsk.csum_block_size; - blk_end = blk_end > dirty_end ? dirty_end : blk_end; - if (blk_begin < offset || blk_end > offset+submit_len) - { - blk_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, blk_end-blk_begin); - cp.buf = blk_buf + offset - blk_begin; - cp.copy_flags |= COPY_BUF_COALESCED; - rv.push_back((copy_buffer_t){ - .copy_flags = COPY_BUF_JOURNAL|COPY_BUF_CSUM_FILL, - .offset = blk_begin, - .len = blk_end-blk_begin, - .disk_offset = dirty_loc + blk_begin - dirty_offset, - .buf = blk_buf, - .csum_buf = (csum_ptr + (blk_begin/dsk.csum_block_size - - dirty_offset/dsk.csum_block_size) * (dsk.data_csum_type & 0xFF)), - .dyn_data = dyn_data, - }); - if (dyn_data) - { - (*dyn_data)++; - } - return 1; - } - } - } - return 0; -} - -bool blockstore_impl_t::fulfill_clean_read(blockstore_op_t *read_op, uint64_t & fulfilled, - uint8_t *clean_entry_bitmap, int *dyn_data, uint32_t item_start, uint32_t item_end, uint64_t clean_loc, uint64_t clean_ver) -{ - bool from_journal = clean_entry_bitmap != NULL; - if (!clean_entry_bitmap) - { - // NULL clean_entry_bitmap means we're reading from data, not from the journal, - // and the bitmap location is obvious - clean_entry_bitmap = get_clean_entry_bitmap(clean_loc, 0); - } - if (dsk.csum_block_size > dsk.bitmap_granularity) - { - auto & rv = PRIV(read_op)->read_vec; - int req = fill_partial_checksum_blocks(rv, fulfilled, clean_entry_bitmap, dyn_data, from_journal, - (uint8_t*)read_op->buf, read_op->offset, read_op->offset+read_op->len); - if (!inmemory_meta && !from_journal && req > 0) - { - // Read checksums from disk - uint8_t *csum_buf = read_clean_meta_block(read_op, clean_loc, rv.size()-req); - for (int i = req; i > 0; i--) - { - rv[rv.size()-i].csum_buf = csum_buf; - } - } - for (int i = req; i > 0; i--) - { - if (!read_checksum_block(read_op, i, fulfilled, clean_loc)) - { - return false; - } - } - PRIV(read_op)->clean_block_used = req > 0; - } - else if (from_journal) - { - // Don't scan bitmap - journal writes don't have holes (internal bitmap)! - uint8_t *csum = !dsk.csum_block_size ? 0 : (clean_entry_bitmap + dsk.clean_entry_bitmap_size + - item_start/dsk.csum_block_size*(dsk.data_csum_type & 0xFF)); - if (!fulfill_read(read_op, fulfilled, item_start, item_end, - (BS_ST_BIG_WRITE | BS_ST_STABLE), 0, clean_loc + item_start, 0, csum, dyn_data)) - { - return false; - } - if (item_start > 0 && fulfilled < read_op->len) - { - // fill with zeroes - assert(fulfill_read(read_op, fulfilled, 0, item_start, (BS_ST_DELETE | BS_ST_STABLE), 0, 0, 0, NULL, NULL)); - } - if (item_end < dsk.data_block_size && fulfilled < read_op->len) - { - // fill with zeroes - assert(fulfill_read(read_op, fulfilled, item_end, dsk.data_block_size, (BS_ST_DELETE | BS_ST_STABLE), 0, 0, 0, NULL, NULL)); - } - } - else - { - bool csum_done = !dsk.csum_block_size || inmemory_meta; - uint8_t *csum_buf = clean_entry_bitmap; - uint64_t bmp_start = 0, bmp_end = 0, bmp_size = dsk.data_block_size/dsk.bitmap_granularity; - while (bmp_start < bmp_size) - { - while (!(clean_entry_bitmap[bmp_end >> 3] & (1 << (bmp_end & 0x7))) && bmp_end < bmp_size) - { - bmp_end++; - } - if (bmp_end > bmp_start) - { - // fill with zeroes - assert(fulfill_read(read_op, fulfilled, bmp_start * dsk.bitmap_granularity, - bmp_end * dsk.bitmap_granularity, (BS_ST_DELETE | BS_ST_STABLE), 0, 0, 0, NULL, NULL)); - } - bmp_start = bmp_end; - while (clean_entry_bitmap[bmp_end >> 3] & (1 << (bmp_end & 0x7)) && bmp_end < bmp_size) - { - bmp_end++; - } - if (bmp_end > bmp_start) - { - if (!csum_done) - { - // Read checksums from disk - csum_buf = read_clean_meta_block(read_op, clean_loc, PRIV(read_op)->read_vec.size()); - csum_done = true; - } - uint8_t *csum = !dsk.csum_block_size ? 0 : (csum_buf + 2*dsk.clean_entry_bitmap_size + bmp_start*(dsk.data_csum_type & 0xFF)); - if (!fulfill_read(read_op, fulfilled, bmp_start * dsk.bitmap_granularity, - bmp_end * dsk.bitmap_granularity, (BS_ST_BIG_WRITE | BS_ST_STABLE), 0, - clean_loc + bmp_start * dsk.bitmap_granularity, 0, csum, dyn_data)) - { - return false; - } - bmp_start = bmp_end; - } - } - } - // Increment reference counter if clean data is being read from the disk - if (PRIV(read_op)->clean_block_used) - { - auto & uo = used_clean_objects[clean_loc]; - uo.refs++; - if (dsk.csum_block_size && flusher->is_mutated(clean_loc)) - uo.was_changed = true; - PRIV(read_op)->clean_block_used = clean_loc; - } - return true; -} - -uint8_t* blockstore_impl_t::read_clean_meta_block(blockstore_op_t *op, uint64_t clean_loc, int rv_pos) -{ - auto & rv = PRIV(op)->read_vec; - auto sector = ((clean_loc / dsk.data_block_size) / (dsk.meta_block_size / dsk.clean_entry_size)) * dsk.meta_block_size; - auto pos = ((clean_loc / dsk.data_block_size) % (dsk.meta_block_size / dsk.clean_entry_size)) * dsk.clean_entry_size; - uint8_t *buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size); - rv.insert(rv.begin()+rv_pos, (copy_buffer_t){ - .copy_flags = COPY_BUF_META_BLOCK|COPY_BUF_CSUM_FILL, - .offset = pos, - .buf = buf, - }); - BS_SUBMIT_GET_SQE(sqe, data); - data->iov = (struct iovec){ buf, (size_t)dsk.meta_block_size }; - PRIV(op)->pending_ops++; - io_uring_prep_readv(sqe, dsk.meta_fd, &data->iov, 1, dsk.meta_offset + dsk.meta_block_size + sector); - data->callback = [this, op](ring_data_t *data) { handle_read_event(data, op); }; - // return pointer to checksums + bitmap - return buf + pos + sizeof(clean_disk_entry); -} - -bool blockstore_impl_t::verify_padded_checksums(uint8_t *clean_entry_bitmap, uint8_t *csum_buf, uint32_t offset, - iovec *iov, int n_iov, std::function bad_block_cb) -{ - assert(!(offset % dsk.csum_block_size)); - uint32_t *csums = (uint32_t*)csum_buf; - uint32_t block_csum = 0; - uint32_t block_done = 0; - uint32_t block_num = clean_entry_bitmap ? offset/dsk.csum_block_size : 0; - uint32_t bmp_pos = offset/dsk.bitmap_granularity; - for (int i = 0; i < n_iov; i++) - { - uint32_t pos = 0; - while (pos < iov[i].iov_len) - { - uint32_t start = pos; - uint8_t bit = (clean_entry_bitmap[bmp_pos >> 3] >> (bmp_pos & 0x7)) & 1; - while (pos < iov[i].iov_len && ((clean_entry_bitmap[bmp_pos >> 3] >> (bmp_pos & 0x7)) & 1) == bit) - { - pos += dsk.bitmap_granularity; - bmp_pos++; - } - uint32_t len = pos-start; - auto buf = (uint8_t*)iov[i].iov_base+start; - while (block_done+len >= dsk.csum_block_size) - { - auto cur_len = dsk.csum_block_size-block_done; - block_csum = crc32c_pad(block_csum, buf, bit ? cur_len : 0, bit ? 0 : cur_len, 0); - if (block_csum != csums[block_num]) - { - if (bad_block_cb) - bad_block_cb(block_num*dsk.csum_block_size, block_csum, csums[block_num]); - else - return false; - } - block_num++; - buf += cur_len; - len -= cur_len; - block_done = block_csum = 0; - } - if (len > 0) - { - block_csum = crc32c_pad(block_csum, buf, bit ? len : 0, bit ? 0 : len, 0); - block_done += len; - } - } - } - assert(!block_done); - return true; -} - -bool blockstore_impl_t::verify_journal_checksums(uint8_t *csums, uint32_t offset, - iovec *iov, int n_iov, std::function bad_block_cb) -{ - uint32_t block_csum = 0; - uint32_t block_num = 0; - uint32_t block_done = offset%dsk.csum_block_size; - for (int i = 0; i < n_iov; i++) - { - uint32_t len = iov[i].iov_len; - auto buf = (uint8_t*)iov[i].iov_base; - while (block_done+len >= dsk.csum_block_size) - { - auto cur_len = dsk.csum_block_size-block_done; - block_csum = crc32c(block_csum, buf, cur_len); - if (block_csum != ((uint32_t*)csums)[block_num]) - { - if (bad_block_cb) - bad_block_cb(block_num*dsk.csum_block_size, block_csum, ((uint32_t*)csums)[block_num]); - else - return false; - } - block_num++; - buf += cur_len; - len -= cur_len; - block_done = block_csum = 0; - } - if (len > 0) - { - block_csum = crc32c(block_csum, buf, len); - block_done += len; - } - } - if (block_done > 0 && block_csum != ((uint32_t*)csums)[block_num]) - { - if (bad_block_cb) - bad_block_cb(block_num*dsk.csum_block_size, block_csum, ((uint32_t*)csums)[block_num]); - else - return false; - } - return true; -} - -bool blockstore_impl_t::verify_clean_padded_checksums(blockstore_op_t *op, uint64_t clean_loc, uint8_t *dyn_data, bool from_journal, - iovec *iov, int n_iov, std::function bad_block_cb) -{ - uint32_t offset = clean_loc % dsk.data_block_size; - if (from_journal) - return verify_padded_checksums(dyn_data, dyn_data + dsk.clean_entry_bitmap_size, offset, iov, n_iov, bad_block_cb); - clean_loc = (clean_loc / dsk.data_block_size) * dsk.data_block_size; - if (!dyn_data) - { - assert(inmemory_meta); - dyn_data = get_clean_entry_bitmap(clean_loc, 0); - } - return verify_padded_checksums(dyn_data, dyn_data + 2*dsk.clean_entry_bitmap_size, offset, iov, n_iov, bad_block_cb); -} - void blockstore_impl_t::handle_read_event(ring_data_t *data, blockstore_op_t *op) { live = true; @@ -827,207 +310,82 @@ void blockstore_impl_t::handle_read_event(ring_data_t *data, blockstore_op_t *op } if (PRIV(op)->pending_ops == 0) { - if (dsk.csum_block_size) - { - // verify checksums if required - auto & rv = PRIV(op)->read_vec; - void *meta_block = NULL; - if (dsk.csum_block_size > dsk.bitmap_granularity) - { - for (int i = rv.size()-1; i >= 0 && (rv[i].copy_flags & COPY_BUF_CSUM_FILL); i--) - { - if (rv[i].copy_flags & COPY_BUF_META_BLOCK) - { - // Metadata read. Skip - assert(!meta_block); - meta_block = rv[i].buf; - rv[i].buf = NULL; - continue; - } - struct iovec *iov = (struct iovec*)((uint8_t*)rv[i].buf + (rv[i].len & 0xFFFFFFFF)); - int n_iov = rv[i].len >> 32; - bool ok = true; - if (rv[i].copy_flags & COPY_BUF_JOURNAL) - { - // SMALL_WRITE from journal - verify_journal_checksums( - rv[i].csum_buf, rv[i].offset, iov, n_iov, - [&](uint32_t bad_block, uint32_t calc_csum, uint32_t stored_csum) - { - ok = false; - printf( - "Checksum mismatch in object %jx:%jx v%ju in journal at 0x%jx, checksum block #%u: got %08x, expected %08x\n", - op->oid.inode, op->oid.stripe, op->version, - rv[i].disk_offset, bad_block / dsk.csum_block_size, calc_csum, stored_csum - ); - } - ); - } - else - { - // BIG_WRITE from journal or clean data - // Do not verify checksums if the data location is/was mutated by flushers - auto & uo = used_clean_objects.at((rv[i].disk_offset / dsk.data_block_size) * dsk.data_block_size); - if (!uo.was_changed) - { - verify_clean_padded_checksums( - op, rv[i].disk_offset, rv[i].csum_buf, (rv[i].copy_flags & COPY_BUF_JOURNALED_BIG), iov, n_iov, - [&](uint32_t bad_block, uint32_t calc_csum, uint32_t stored_csum) - { - ok = false; - printf( - "Checksum mismatch in object %jx:%jx v%ju in %s data at 0x%jx, checksum block #%u: got %08x, expected %08x\n", - op->oid.inode, op->oid.stripe, op->version, - (rv[i].copy_flags & COPY_BUF_JOURNALED_BIG ? "redirect-write" : "clean"), - rv[i].disk_offset, bad_block / dsk.csum_block_size, calc_csum, stored_csum - ); - } - ); - } - } - if (!ok) - { - op->retval = -EDOM; - } - free(rv[i].buf); - rv[i].buf = NULL; - if (rv[i].dyn_data && --(*rv[i].dyn_data) == 0) // refcount - { - free(rv[i].dyn_data); - rv[i].dyn_data = NULL; - } - } - } - else - { - for (auto & vec: rv) - { - if (vec.copy_flags & COPY_BUF_META_BLOCK) - { - // Metadata read. Skip - assert(!meta_block); - meta_block = vec.buf; - vec.buf = NULL; - continue; - } - if (vec.csum_buf) - { - uint32_t *csum = (uint32_t*)vec.csum_buf; - for (size_t p = 0; p < vec.len; p += dsk.csum_block_size, csum++) - { - if (crc32c(0, (uint8_t*)op->buf + vec.offset - op->offset + p, dsk.csum_block_size) != *csum) - { - // checksum error - printf( - "Checksum mismatch in object %jx:%jx v%ju in %s area at offset 0x%jx+0x%zx: %08x vs %08x\n", - op->oid.inode, op->oid.stripe, op->version, - (vec.copy_flags & COPY_BUF_JOURNAL) ? "journal" : "data", vec.disk_offset, p, - crc32c(0, (uint8_t*)op->buf + vec.offset - op->offset + p, dsk.csum_block_size), *csum - ); - op->retval = -EDOM; - break; - } - } - } - if (vec.dyn_data && --(*vec.dyn_data) == 0) // refcount - { - free(vec.dyn_data); - vec.dyn_data = NULL; - } - } - } - if (meta_block) - { - // Free after checking - free(meta_block); - meta_block = NULL; - } - } - if (PRIV(op)->clean_block_used) - { - // Release clean data block - auto uo_it = used_clean_objects.find(PRIV(op)->clean_block_used); - if (uo_it != used_clean_objects.end()) - { - uo_it->second.refs--; - if (uo_it->second.refs <= 0) - { - if (uo_it->second.was_freed) - { - data_alloc->set(PRIV(op)->clean_block_used, false); - } - used_clean_objects.erase(uo_it); - } - } - } - if (!journal.inmemory) - { - // Release journal sector usage - for (auto & rv: PRIV(op)->read_vec) - { - if (rv.journal_sector) - { - auto used = --journal.used_sectors.at(rv.journal_sector-1); - if (used == 0) - { - journal.used_sectors.erase(rv.journal_sector-1); - flusher->mark_trim_possible(); - } - } - } - } - if (op->retval == 0) + // verify checksums if required + if (dsk.csum_block_size && !verify_read_checksums(op)) + op->retval = -EDOM; + else if (op->retval == 0) op->retval = op->len; + heap->unlock_entry(op->oid, PRIV(op)->lsn); FINISH_OP(op); } } +bool blockstore_impl_t::verify_read_checksums(blockstore_op_t *op) +{ + heap_object_t *obj = heap->read_locked_entry(op->oid, PRIV(op)->lsn); + auto & rv = PRIV(op)->read_vec; + for (auto & vec: rv) + { + if (vec.copy_flags & COPY_BUF_COALESCED) + { + continue; + } + heap_write_t *wr = (heap_write_t*)((uint8_t*)obj + vec.wr_offset); + uint32_t blk_start = vec.offset, blk_end = vec.offset + vec.len; + if (vec.copy_flags & COPY_BUF_PADDED) + { + blk_start = (blk_start/dsk.csum_block_size) * dsk.csum_block_size; + blk_start = blk_start < wr->offset ? wr->offset : blk_start; + blk_end = ((blk_end-1) / dsk.csum_block_size + 1) * dsk.csum_block_size; + blk_end = blk_end > wr->offset+wr->len ? wr->offset+wr->len : blk_end; + memcpy(op->buf + vec.offset - op->offset, vec.buf + vec.offset - blk_start, vec.len); + } + if (!heap->calc_block_checksums((uint32_t*)wr->get_checksums(heap), vec.buf, wr->get_int_bitmap(heap), + blk_start, blk_end, false, [&](uint32_t mismatch_pos, uint32_t expected_csum, uint32_t real_csum) + { + printf( + "Checksum mismatch in object %jx:%jx v%ju in %s area at offset 0x%jx+%x: %08x vs %08x\n", + op->oid.inode, op->oid.stripe, op->version, + (vec.copy_flags & COPY_BUF_JOURNAL) ? "buffer" : "data", vec.disk_offset, + mismatch_pos, expected_csum, real_csum + ); + })) + { + return false; + } + } + return true; +} + int blockstore_impl_t::read_bitmap(object_id oid, uint64_t target_version, void *bitmap, uint64_t *result_version) { - auto dirty_it = dirty_db.upper_bound((obj_ver_id){ - .oid = oid, - .version = UINT64_MAX, - }); - if (dirty_it != dirty_db.begin()) - dirty_it--; - if (dirty_it != dirty_db.end()) + heap_object_t *obj = heap->read_entry(oid, NULL); + if (obj) { - while (dirty_it->first.oid == oid) + for (heap_write_t *wr = obj->get_writes(); wr < (heap_write_t*)obj->next(); wr = wr->next(heap)) { - // Condition has to be the same as in dequeue_read() - if (!IS_IN_FLIGHT(dirty_it->second.state) && target_version >= dirty_it->first.version) + if (target_version < wr->version) { - if (result_version) - *result_version = dirty_it->first.version; - if (bitmap) - { - void *dyn_ptr = (alloc_dyn_data - ? (uint8_t*)dirty_it->second.dyn_data + sizeof(int) : (uint8_t*)&dirty_it->second.dyn_data); - memcpy(bitmap, dyn_ptr, dsk.clean_entry_bitmap_size); - } - return 0; + continue; } - if (dirty_it == dirty_db.begin()) - break; - dirty_it--; + if (result_version) + { + *result_version = wr->version; + } + if (bitmap) + { + memcpy(bitmap, wr->get_ext_bitmap(heap), dsk.clean_entry_bitmap_size); + } + return 0; } } - auto & clean_db = clean_db_shard(oid); - auto clean_it = clean_db.find(oid); - if (clean_it != clean_db.end()) - { - if (result_version) - *result_version = clean_it->second.version; - if (bitmap) - { - void *bmp_ptr = get_clean_entry_bitmap(clean_it->second.location, dsk.clean_entry_bitmap_size); - memcpy(bitmap, bmp_ptr, dsk.clean_entry_bitmap_size); - } - return 0; - } if (result_version) + { *result_version = 0; + } if (bitmap) + { memset(bitmap, 0, dsk.clean_entry_bitmap_size); + } return -ENOENT; } diff --git a/src/blockstore/blockstore_stable.cpp b/src/blockstore/blockstore_stable.cpp index 2559bec8..7d5b82f7 100644 --- a/src/blockstore/blockstore_stable.cpp +++ b/src/blockstore/blockstore_stable.cpp @@ -4,559 +4,66 @@ #include "blockstore_impl.h" #include "blockstore_internal.h" -// Stabilize small write: -// 1) Copy data from the journal to the data device -// 2) Increase version on the metadata device and sync it -// 3) Advance clean_db entry's version, clear previous journal entries -// -// This makes 1 4K small write+sync look like: -// 512b+4K (journal) + sync + 512b (journal) + sync + 4K (data) [+ sync?] + 512b (metadata) + sync. -// WA = 2.375. It's not the best, SSD FTL-like redirect-write could probably be lower -// even with defragmentation. But it's fixed and it's still better than in Ceph. :) -// except for HDD-only clusters, because each write results in 3 seeks. - -// Stabilize big write: -// 1) Copy metadata from the journal to the metadata device -// 2) Move dirty_db entry to clean_db and clear previous journal entries -// -// This makes 1 128K big write+sync look like: -// 128K (data) + sync + 512b (journal) + sync + 512b (journal) + sync + 512b (metadata) + sync. -// WA = 1.012. Very good :) - -// Stabilize delete: -// 1) Remove metadata entry and sync it -// 2) Remove dirty_db entry and clear previous journal entries -// We have 2 problems here: -// - In the cluster environment, we must store the "tombstones" of deleted objects until -// all replicas (not just quorum) agrees about their deletion. That is, "stabilize" is -// not possible for deletes in degraded placement groups -// - With simple "fixed" metadata tables we can't just clear the metadata entry of the latest -// object version. We must clear all previous entries, too. -// FIXME Fix both problems - probably, by switching from "fixed" metadata tables to "dynamic" - -// AND We must do it in batches, for the sake of reduced fsync call count -// AND We must know what we stabilize. Basic workflow is like: -// 1) primary OSD receives sync request -// 2) it submits syncs to blockstore and peers -// 3) after everyone acks sync it acks sync to the client -// 4) after a while it takes his synced object list and sends stabilize requests -// to peers and to its own blockstore, thus freeing the old version - -struct ver_vector_t +// Handles both stabilize (commit) and rollback +int blockstore_impl_t::dequeue_stable(blockstore_op_t *op) { - obj_ver_id *items = NULL; - uint64_t alloc = 0, size = 0; -}; - -static void init_versions(ver_vector_t & vec, obj_ver_id *start, obj_ver_id *end, uint64_t len) -{ - if (!vec.items) + obj_ver_id *v = (obj_ver_id*)op->buf; + auto priv = PRIV(op); + if (priv->op_state == 1) goto resume_1; + else if (priv->op_state == 2) goto resume_2; + else if (priv->op_state == 3) goto resume_3; + else if (priv->op_state == 4) goto resume_4; + assert(!priv->op_state); + priv->stab_pos = 0; + op->retval = 0; + while (priv->stab_pos < op->len) { - vec.alloc = len; - vec.items = (obj_ver_id*)malloc_or_die(sizeof(obj_ver_id) * vec.alloc); - for (auto sv = start; sv < end; sv++) + io_uring_sqe *sqe = get_sqe(); + if (!sqe) { - vec.items[vec.size++] = *sv; + if (priv->pending_ops > 0) + return 1; + priv->wait_detail = 1; + priv->wait_for = WAIT_SQE; + return 0; } - } -} - -static void append_version(ver_vector_t & vec, obj_ver_id ov) -{ - if (vec.size >= vec.alloc) - { - vec.alloc = !vec.alloc ? 4 : vec.alloc*2; - vec.items = (obj_ver_id*)realloc_or_die(vec.items, sizeof(obj_ver_id) * vec.alloc); - } - vec.items[vec.size++] = ov; -} - -static bool check_unsynced(std::vector & check, obj_ver_id ov, std::vector & to, int *count) -{ - bool found = false; - int j = 0, k = 0; - while (j < check.size()) - { - if (check[j] == ov) - found = true; - if (check[j].oid == ov.oid && check[j].version <= ov.version) + uint32_t modified_block; + int res = op->opcode == BS_OP_STABLE + ? heap->post_stabilize(v[priv->stab_pos].oid, v[priv->stab_pos].version, &modified_block) + : heap->post_rollback(v[priv->stab_pos].oid, v[priv->stab_pos].version, &modified_block); + if (res != 0) { - to.push_back(check[j++]); - if (count) - (*count)--; - } - else - check[k++] = check[j++]; - } - check.resize(k); - return found; -} - -blockstore_op_t* blockstore_impl_t::selective_sync(blockstore_op_t *op) -{ - unsynced_big_write_count -= unsynced_big_writes.size(); - unsynced_big_writes.swap(PRIV(op)->sync_big_writes); - unsynced_big_write_count += unsynced_big_writes.size(); - unsynced_small_writes.swap(PRIV(op)->sync_small_writes); - // Create a sync operation, insert into the end of the queue - // And move ourselves into the end too! - // Rather hacky but that's what we need... - blockstore_op_t *sync_op = new blockstore_op_t; - sync_op->opcode = BS_OP_SYNC; - sync_op->buf = NULL; - sync_op->callback = [](blockstore_op_t *sync_op) - { - delete sync_op; - }; - init_op(sync_op); - int sync_res = continue_sync(sync_op); - if (sync_res != 2) - { - // Put SYNC into the queue if it's not finished yet - submit_queue.push_back(sync_op); - } - // Restore unsynced_writes - unsynced_small_writes.swap(PRIV(op)->sync_small_writes); - unsynced_big_write_count -= unsynced_big_writes.size(); - unsynced_big_writes.swap(PRIV(op)->sync_big_writes); - unsynced_big_write_count += unsynced_big_writes.size(); - if (sync_res == 2) - { - // Sync is immediately completed - return NULL; - } - return sync_op; -} - -// Returns: 2 = stop processing and dequeue, 0 = stop processing and do not dequeue, 1 = proceed with op itself -int blockstore_impl_t::split_stab_op(blockstore_op_t *op, std::function decider) -{ - bool add_sync = false; - ver_vector_t good_vers, bad_vers; - obj_ver_id* v; - int i, todo = 0; - for (i = 0, v = (obj_ver_id*)op->buf; i < op->len; i++, v++) - { - int action = decider(*v); - if (action < 0) - { - // Rollback changes - for (auto & ov: PRIV(op)->sync_big_writes) - { - unsynced_big_writes.push_back(ov); - unsynced_big_write_count++; - } - for (auto & ov: PRIV(op)->sync_small_writes) - { - unsynced_small_writes.push_back(ov); - } - free(good_vers.items); - good_vers.items = NULL; - free(bad_vers.items); - bad_vers.items = NULL; - // Error - op->retval = action; + op->retval = -res; FINISH_OP(op); return 2; } - else if (action == STAB_SPLIT_DONE) - { - // Already done - init_versions(good_vers, (obj_ver_id*)op->buf, v, op->len); - } - else if (action == STAB_SPLIT_WAIT) - { - // Already in progress, we just have to wait until it finishes - init_versions(good_vers, (obj_ver_id*)op->buf, v, op->len); - append_version(bad_vers, *v); - } - else if (action == STAB_SPLIT_SYNC) - { - // Needs a SYNC, we have to send a SYNC if not already in progress - // - // If the object is not present in unsynced_(big|small)_writes then - // it's currently being synced. If it's present then we can initiate - // its sync ourselves. - init_versions(good_vers, (obj_ver_id*)op->buf, v, op->len); - append_version(bad_vers, *v); - if (!add_sync) - { - PRIV(op)->sync_big_writes.clear(); - PRIV(op)->sync_small_writes.clear(); - add_sync = true; - } - check_unsynced(unsynced_small_writes, *v, PRIV(op)->sync_small_writes, NULL); - check_unsynced(unsynced_big_writes, *v, PRIV(op)->sync_big_writes, &unsynced_big_write_count); - } - else /* if (action == STAB_SPLIT_TODO) */ - { - if (good_vers.items) - { - // If we're selecting versions then append it - // Main idea is that 99% of the time all versions passed to BS_OP_STABLE are synced - // And we don't want to select/allocate anything in that optimistic case - append_version(good_vers, *v); - } - todo++; - } + prepare_meta_block_write(op, modified_block); + priv->pending_ops++; + priv->stab_pos++; } - // In a pessimistic scenario, an operation may be split into 3: - // - Stabilize synced entries - // - Sync unsynced entries - // - Continue for unsynced entries after sync - add_sync = add_sync && (PRIV(op)->sync_big_writes.size() || PRIV(op)->sync_small_writes.size()); - if (!todo && !bad_vers.size) +resume_1: + if (priv->pending_ops > 0) { - // Already stable - op->retval = 0; - FINISH_OP(op); - return 2; - } - op->retval = 0; - if (!todo && !add_sync) - { - // Only wait for inflight writes or current in-progress syncs + priv->op_state = 1; return 0; } - blockstore_op_t *sync_op = NULL, *split_stab_op = NULL; - if (add_sync) - { - // Initiate a selective sync for PRIV(op)->sync_(big|small)_writes - sync_op = selective_sync(op); - } - if (bad_vers.size) - { - // Split part of the request into a separate operation - split_stab_op = new blockstore_op_t; - split_stab_op->opcode = op->opcode; - split_stab_op->buf = (uint8_t*)bad_vers.items; - split_stab_op->len = bad_vers.size; - init_op(split_stab_op); - submit_queue.push_back(split_stab_op); - } - if (sync_op || split_stab_op || good_vers.items) - { - uint8_t *orig_buf = op->buf; - if (good_vers.items) - { - op->buf = (uint8_t*)good_vers.items; - op->len = good_vers.size; - } - // Make a wrapped callback - int *split_op_counter = (int*)malloc_or_die(sizeof(int)); - *split_op_counter = (sync_op ? 1 : 0) + (split_stab_op ? 1 : 0) + (todo ? 1 : 0); - auto cb = [op, good_items = good_vers.items, - bad_items = bad_vers.items, split_op_counter, - orig_buf, real_cb = op->callback](blockstore_op_t *split_op) - { - if (split_op->retval != 0) - op->retval = split_op->retval; - (*split_op_counter)--; - assert((*split_op_counter) >= 0); - if (op != split_op) - delete split_op; - if (!*split_op_counter) - { - free(good_items); - free(bad_items); - free(split_op_counter); - op->buf = orig_buf; - real_cb(op); - } - }; - if (sync_op) - { - sync_op->callback = cb; - } - if (split_stab_op) - { - split_stab_op->callback = cb; - } - op->callback = cb; - } - if (!todo) - { - // All work is postponed - op->callback = NULL; - return 2; - } - return 1; -} - -int blockstore_impl_t::dequeue_stable(blockstore_op_t *op) -{ - if (PRIV(op)->op_state) - { - return continue_stable(op); - } - int r = split_stab_op(op, [this](obj_ver_id ov) - { - auto dirty_it = dirty_db.find(ov); - if (dirty_it == dirty_db.end()) - { - auto & clean_db = clean_db_shard(ov.oid); - auto clean_it = clean_db.find(ov.oid); - if (clean_it == clean_db.end() || clean_it->second.version < ov.version) - { - // No such object version - printf("Error: %jx:%jx v%ju not found while stabilizing\n", ov.oid.inode, ov.oid.stripe, ov.version); - return -ENOENT; - } - else - { - // Already stable - return STAB_SPLIT_DONE; - } - } - else if (IS_STABLE(dirty_it->second.state)) - { - // Already stable - return STAB_SPLIT_DONE; - } - while (true) - { - if (IS_IN_FLIGHT(dirty_it->second.state)) - { - // Object write is still in progress. Wait until the write request completes - return STAB_SPLIT_WAIT; - } - else if (!IS_SYNCED(dirty_it->second.state)) - { - // Object not synced yet - sync it - // In previous versions we returned EBUSY here and required - // the caller (OSD) to issue a global sync first. But a global sync - // waits for all writes in the queue including inflight writes. And - // inflight writes may themselves be blocked by unstable writes being - // still present in the journal and not flushed away from it. - // So we must sync specific objects here. - // - // Even more, we have to process "stabilize" request in parts. That is, - // we must stabilize all objects which are already synced. Otherwise - // they may block objects which are NOT synced yet. - return STAB_SPLIT_SYNC; - } - else if (IS_STABLE(dirty_it->second.state)) - { - break; - } - // Check previous versions too - if (dirty_it == dirty_db.begin()) - { - break; - } - dirty_it--; - if (dirty_it->first.oid != ov.oid) - { - break; - } - } - return STAB_SPLIT_TODO; - }); - if (r != 1) - { - return r; - } - // Check journal space - blockstore_journal_check_t space_check(this); - if (!space_check.check_available(op, op->len, sizeof(journal_entry_stable), 0)) - { - return 0; - } - // There is sufficient space. Check SQEs - BS_SUBMIT_CHECK_SQES(space_check.sectors_to_write); - // Prepare and submit journal entries - int s = 0; - auto v = (obj_ver_id*)op->buf; - for (int i = 0; i < op->len; i++, v++) - { - if (!journal.entry_fits(sizeof(journal_entry_stable)) && - journal.sector_info[journal.cur_sector].dirty) - { - prepare_journal_sector_write(journal.cur_sector, op); - s++; - } - journal_entry_stable *je = (journal_entry_stable*) - prefill_single_journal_entry(journal, JE_STABLE, sizeof(journal_entry_stable)); - je->oid = v->oid; - je->version = v->version; - je->crc32 = je_crc32((journal_entry*)je); - journal.crc32_last = je->crc32; - } - prepare_journal_sector_write(journal.cur_sector, op); - s++; - assert(s == space_check.sectors_to_write); - PRIV(op)->op_state = 1; - return 1; -} - -int blockstore_impl_t::continue_stable(blockstore_op_t *op) -{ - if (PRIV(op)->op_state == 2) - goto resume_2; - else if (PRIV(op)->op_state == 4) - goto resume_4; - else - return 1; resume_2: - if (!disable_journal_fsync) + if (!disable_meta_fsync) { BS_SUBMIT_GET_SQE(sqe, data); - io_uring_prep_fsync(sqe, dsk.journal_fd, IORING_FSYNC_DATASYNC); + io_uring_prep_fsync(sqe, dsk.meta_fd, IORING_FSYNC_DATASYNC); data->iov = { 0 }; data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); }; - PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 0; - PRIV(op)->pending_ops = 1; - PRIV(op)->op_state = 3; - return 1; + priv->pending_ops++; + } +resume_3: + if (priv->pending_ops > 0) + { + priv->op_state = 3; + return 0; } resume_4: - // Mark dirty_db entries as stable, acknowledge op completion - obj_ver_id* v; - int i; - for (i = 0, v = (obj_ver_id*)op->buf; i < op->len; i++, v++) - { - // Mark all dirty_db entries up to op->version as stable -#ifdef BLOCKSTORE_DEBUG - printf("Stabilize %jx:%jx v%ju\n", v->oid.inode, v->oid.stripe, v->version); -#endif - mark_stable(*v); - } - // Acknowledge op - op->retval = 0; + // Done. Don't touch op->retval - if anything resulted in ENOENT, return it as is FINISH_OP(op); return 2; } - -void blockstore_impl_t::mark_stable(obj_ver_id v, bool forget_dirty) -{ - auto dirty_it = dirty_db.find(v); - if (dirty_it != dirty_db.end()) - { - if (IS_INSTANT(dirty_it->second.state)) - { - // 'Instant' (non-EC) operations may complete and try to become stable out of order. Prevent it. - auto back_it = dirty_it; - while (back_it != dirty_db.begin()) - { - back_it--; - if (back_it->first.oid != v.oid) - { - break; - } - if (!IS_STABLE(back_it->second.state)) - { - // There are preceding unstable versions, can't flush - return; - } - } - while (true) - { - dirty_it++; - if (dirty_it == dirty_db.end() || dirty_it->first.oid != v.oid || - !IS_SYNCED(dirty_it->second.state)) - { - dirty_it--; - break; - } - v.version = dirty_it->first.version; - } - } - while (1) - { - bool was_stable = IS_STABLE(dirty_it->second.state); - if ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_SYNCED) - { - dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | BS_ST_STABLE; - // Allocations and deletions are counted when they're stabilized - if (IS_BIG_WRITE(dirty_it->second.state)) - { - int exists = -1; - if (dirty_it != dirty_db.begin()) - { - auto prev_it = dirty_it; - prev_it--; - if (prev_it->first.oid == v.oid) - { - exists = IS_DELETE(prev_it->second.state) ? 0 : 1; - } - } - if (exists == -1) - { - auto & clean_db = clean_db_shard(v.oid); - auto clean_it = clean_db.find(v.oid); - exists = clean_it != clean_db.end() ? 1 : 0; - } - if (!exists) - { - uint64_t space_id = dirty_it->first.oid.inode; - if (no_inode_stats[dirty_it->first.oid.inode >> (64-POOL_ID_BITS)]) - space_id = space_id & ~(((uint64_t)1 << (64-POOL_ID_BITS)) - 1); - inode_space_stats[space_id] += dsk.data_block_size; - used_blocks++; - } - big_to_flush++; - } - else if (IS_DELETE(dirty_it->second.state)) - { - uint64_t space_id = dirty_it->first.oid.inode; - if (no_inode_stats[dirty_it->first.oid.inode >> (64-POOL_ID_BITS)]) - space_id = space_id & ~(((uint64_t)1 << (64-POOL_ID_BITS)) - 1); - auto & sp = inode_space_stats[space_id]; - if (sp > dsk.data_block_size) - sp -= dsk.data_block_size; - else - inode_space_stats.erase(space_id); - used_blocks--; - big_to_flush++; - } - } - else if (IS_IN_FLIGHT(dirty_it->second.state)) - { - // mark_stable should never be called for in-flight or submitted writes - printf( - "BUG: Attempt to mark_stable object %jx:%jx v%ju state of which is %x\n", - dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version, - dirty_it->second.state - ); - exit(1); - } - if (forget_dirty && (IS_BIG_WRITE(dirty_it->second.state) || - IS_DELETE(dirty_it->second.state))) - { - // Big write overrides all previous dirty entries - auto erase_end = dirty_it; - while (dirty_it != dirty_db.begin()) - { - dirty_it--; - if (dirty_it->first.oid != v.oid) - { - dirty_it++; - break; - } - } - auto & clean_db = clean_db_shard(v.oid); - auto clean_it = clean_db.find(v.oid); - uint64_t clean_loc = clean_it != clean_db.end() - ? clean_it->second.location : UINT64_MAX; - erase_dirty(dirty_it, erase_end, clean_loc); - break; - } - if (was_stable || dirty_it == dirty_db.begin()) - { - break; - } - dirty_it--; - if (dirty_it->first.oid != v.oid) - { - break; - } - } - flusher->enqueue_flush(v); - } - auto unstab_it = unstable_writes.find(v.oid); - if (unstab_it != unstable_writes.end() && - unstab_it->second <= v.version) - { - unstable_writes.erase(unstab_it); - } -} diff --git a/src/blockstore/blockstore_sync.cpp b/src/blockstore/blockstore_sync.cpp index 868c31b5..4be66253 100644 --- a/src/blockstore/blockstore_sync.cpp +++ b/src/blockstore/blockstore_sync.cpp @@ -4,231 +4,49 @@ #include "blockstore_impl.h" #include "blockstore_internal.h" -#define SYNC_HAS_SMALL 1 -#define SYNC_HAS_BIG 2 -#define SYNC_DATA_SYNC_SENT 3 -#define SYNC_DATA_SYNC_DONE 4 -#define SYNC_JOURNAL_WRITE_SENT 5 -#define SYNC_JOURNAL_WRITE_DONE 6 -#define SYNC_JOURNAL_SYNC_SENT 7 -#define SYNC_DONE 8 - int blockstore_impl_t::continue_sync(blockstore_op_t *op) { - if (immediate_commit == IMMEDIATE_ALL) + if (immediate_commit == IMMEDIATE_ALL || !unsynced_big_write_count && !unsynced_small_write_count) { // We can return immediately because sync is only dequeued after all previous writes + unsynced_big_write_count = unsynced_small_write_count = 0; op->retval = 0; FINISH_OP(op); return 2; } - if (PRIV(op)->op_state == 0) + int op_state = PRIV(op)->op_state; + if (op_state == 1) goto resume_1; + if (op_state == 2) goto resume_2; + assert(!op_state); + stop_sync_submitted = false; + if (unsynced_small_write_count > 0 && !disable_journal_fsync) { - stop_sync_submitted = false; - unsynced_big_write_count -= unsynced_big_writes.size(); - PRIV(op)->sync_big_writes.swap(unsynced_big_writes); - PRIV(op)->sync_small_writes.swap(unsynced_small_writes); - unsynced_big_writes.clear(); - unsynced_small_writes.clear(); - if (PRIV(op)->sync_big_writes.size() > 0) - PRIV(op)->op_state = SYNC_HAS_BIG; - else if (PRIV(op)->sync_small_writes.size() > 0) - PRIV(op)->op_state = SYNC_HAS_SMALL; - else - PRIV(op)->op_state = SYNC_DONE; + // fsync buffer + BS_SUBMIT_GET_SQE(sqe, data); + io_uring_prep_fsync(sqe, dsk.journal_fd, IORING_FSYNC_DATASYNC); + data->iov = { 0 }; + data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); }; + PRIV(op)->pending_ops++; } - if (PRIV(op)->op_state == SYNC_HAS_SMALL) + if (!disable_meta_fsync) { - // No big writes, just fsync the journal - if (journal.sector_info[journal.cur_sector].dirty) - { - // Write out the last journal sector if it happens to be dirty - BS_SUBMIT_CHECK_SQES(1); - prepare_journal_sector_write(journal.cur_sector, op); - PRIV(op)->op_state = SYNC_JOURNAL_WRITE_SENT; - return 1; - } - else - { - PRIV(op)->op_state = SYNC_JOURNAL_WRITE_DONE; - } + // fsync meta + BS_SUBMIT_GET_SQE(sqe, data); + io_uring_prep_fsync(sqe, dsk.meta_fd, IORING_FSYNC_DATASYNC); + data->iov = { 0 }; + data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); }; + PRIV(op)->pending_ops++; } - if (PRIV(op)->op_state == SYNC_HAS_BIG) + unsynced_big_write_count = 0; + unsynced_small_write_count = 0; +resume_1: + if (PRIV(op)->pending_ops > 0) { - // 1st step: fsync data - if (!disable_data_fsync) - { - BS_SUBMIT_GET_SQE(sqe, data); - io_uring_prep_fsync(sqe, dsk.data_fd, IORING_FSYNC_DATASYNC); - data->iov = { 0 }; - data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); }; - PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 0; - PRIV(op)->pending_ops = 1; - PRIV(op)->op_state = SYNC_DATA_SYNC_SENT; - return 1; - } - else - { - PRIV(op)->op_state = SYNC_DATA_SYNC_DONE; - } - } - if (PRIV(op)->op_state == SYNC_DATA_SYNC_DONE) - { - // 2nd step: Data device is synced, prepare & write journal entries - // Check space in the journal and journal memory buffers - blockstore_journal_check_t space_check(this); - if (dsk.csum_block_size) - { - // More complex check because all journal entries have different lengths - int left = PRIV(op)->sync_big_writes.size(); - for (auto & sbw: PRIV(op)->sync_big_writes) - { - left--; - auto & dirty_entry = dirty_db.at(sbw); - uint64_t dyn_size = dsk.dirty_dyn_size(dirty_entry.offset, dirty_entry.len); - if (!space_check.check_available(op, 1, sizeof(journal_entry_big_write) + dyn_size, 0)) - { - return 0; - } - } - } - else if (!space_check.check_available(op, PRIV(op)->sync_big_writes.size(), - sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size, 0)) - { - return 0; - } - // Check SQEs. Don't bother about merging, submit each journal sector as a separate request - BS_SUBMIT_CHECK_SQES(space_check.sectors_to_write); - // Prepare and submit journal entries - auto it = PRIV(op)->sync_big_writes.begin(); - int s = 0; - while (it != PRIV(op)->sync_big_writes.end()) - { - auto & dirty_entry = dirty_db.at(*it); - uint64_t dyn_size = dsk.dirty_dyn_size(dirty_entry.offset, dirty_entry.len); - if (!journal.entry_fits(sizeof(journal_entry_big_write) + dyn_size) && - journal.sector_info[journal.cur_sector].dirty) - { - prepare_journal_sector_write(journal.cur_sector, op); - s++; - } - journal_entry_big_write *je = (journal_entry_big_write*)prefill_single_journal_entry( - journal, (dirty_entry.state & BS_ST_INSTANT) ? JE_BIG_WRITE_INSTANT : JE_BIG_WRITE, - sizeof(journal_entry_big_write) + dyn_size - ); - auto jsec = dirty_entry.journal_sector = journal.sector_info[journal.cur_sector].offset; - assert(journal.next_free >= journal.used_start - ? (jsec >= journal.used_start && jsec < journal.next_free) - : (jsec >= journal.used_start || jsec < journal.next_free)); - journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++; -#ifdef BLOCKSTORE_DEBUG - printf( - "journal offset %08jx is used by %jx:%jx v%ju (%ju refs)\n", - dirty_entry.journal_sector, it->oid.inode, it->oid.stripe, it->version, - journal.used_sectors[journal.sector_info[journal.cur_sector].offset] - ); -#endif - je->oid = it->oid; - je->version = it->version; - je->offset = dirty_entry.offset; - je->len = dirty_entry.len; - je->location = dirty_entry.location; - memcpy((void*)(je+1), (alloc_dyn_data - ? (uint8_t*)dirty_entry.dyn_data+sizeof(int) : (uint8_t*)&dirty_entry.dyn_data), dyn_size); - je->crc32 = je_crc32((journal_entry*)je); - journal.crc32_last = je->crc32; - it++; - } - prepare_journal_sector_write(journal.cur_sector, op); - s++; - assert(s == space_check.sectors_to_write); - PRIV(op)->op_state = SYNC_JOURNAL_WRITE_SENT; + PRIV(op)->op_state = 1; return 1; } - if (PRIV(op)->op_state == SYNC_JOURNAL_WRITE_DONE) - { - if (!disable_journal_fsync) - { - BS_SUBMIT_GET_SQE(sqe, data); - io_uring_prep_fsync(sqe, dsk.journal_fd, IORING_FSYNC_DATASYNC); - data->iov = { 0 }; - data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); }; - PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 0; - PRIV(op)->pending_ops = 1; - PRIV(op)->op_state = SYNC_JOURNAL_SYNC_SENT; - return 1; - } - else - { - PRIV(op)->op_state = SYNC_DONE; - } - } - if (PRIV(op)->op_state == SYNC_DONE) - { - ack_sync(op); - return 2; - } - return 1; -} - -void blockstore_impl_t::ack_sync(blockstore_op_t *op) -{ - // Handle states - for (auto it = PRIV(op)->sync_big_writes.begin(); it != PRIV(op)->sync_big_writes.end(); it++) - { -#ifdef BLOCKSTORE_DEBUG - printf("Ack sync big %jx:%jx v%ju\n", it->oid.inode, it->oid.stripe, it->version); -#endif - auto & unstab = unstable_writes[it->oid]; - unstab = unstab < it->version ? it->version : unstab; - auto dirty_it = dirty_db.find(*it); - dirty_it->second.state = ((dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | BS_ST_SYNCED); - if (dirty_it->second.state & BS_ST_INSTANT) - { - mark_stable(dirty_it->first); - } - else - { - unstable_unsynced--; - assert(unstable_unsynced >= 0); - } - dirty_it++; - while (dirty_it != dirty_db.end() && dirty_it->first.oid == it->oid) - { - if ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_WAIT_BIG) - { - dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | BS_ST_IN_FLIGHT; - } - dirty_it++; - } - } - for (auto it = PRIV(op)->sync_small_writes.begin(); it != PRIV(op)->sync_small_writes.end(); it++) - { -#ifdef BLOCKSTORE_DEBUG - printf("Ack sync small %jx:%jx v%ju\n", it->oid.inode, it->oid.stripe, it->version); -#endif - auto & unstab = unstable_writes[it->oid]; - unstab = unstab < it->version ? it->version : unstab; - if (dirty_db[*it].state == (BS_ST_DELETE | BS_ST_WRITTEN)) - { - dirty_db[*it].state = (BS_ST_DELETE | BS_ST_SYNCED); - // Deletions are treated as immediately stable - mark_stable(*it); - } - else /* (BS_ST_INSTANT?) | BS_ST_SMALL_WRITE | BS_ST_WRITTEN */ - { - dirty_db[*it].state = (dirty_db[*it].state & ~BS_ST_WORKFLOW_MASK) | BS_ST_SYNCED; - if (dirty_db[*it].state & BS_ST_INSTANT) - { - mark_stable(*it); - } - else - { - unstable_unsynced--; - assert(unstable_unsynced >= 0); - } - } - } +resume_2: op->retval = 0; FINISH_OP(op); + return 2; } diff --git a/src/blockstore/blockstore_write.cpp b/src/blockstore/blockstore_write.cpp index dcefb4a8..cba251cc 100644 --- a/src/blockstore/blockstore_write.cpp +++ b/src/blockstore/blockstore_write.cpp @@ -6,219 +6,12 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op) { - // Check or assign version number - bool found = false, deleted = false, unsynced = false, is_del = (op->opcode == BS_OP_DELETE); - bool wait_big = false, wait_del = false; - void *dyn = NULL; - if (is_del) - { - op->len = 0; - } - size_t dyn_size = dsk.dirty_dyn_size(op->offset, op->len); - if (!is_del && alloc_dyn_data) - { - // FIXME: Working with `dyn_data` has to be refactored somehow but I first have to decide how :) - // +sizeof(int) = refcount - dyn = calloc_or_die(1, dyn_size+sizeof(int)); - *((int*)dyn) = 1; - } - uint8_t *dyn_ptr = (alloc_dyn_data ? (uint8_t*)dyn+sizeof(int) : (uint8_t*)&dyn); - uint64_t version = 1; - if (dirty_db.size() > 0) - { - auto dirty_it = dirty_db.upper_bound((obj_ver_id){ - .oid = op->oid, - .version = UINT64_MAX, - }); - dirty_it--; // segfaults when dirty_db is empty - if (dirty_it != dirty_db.end() && dirty_it->first.oid == op->oid) - { - found = true; - version = dirty_it->first.version + 1; - deleted = IS_DELETE(dirty_it->second.state); - unsynced = !IS_SYNCED(dirty_it->second.state); - wait_del = ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_WAIT_DEL); - wait_big = (dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_BIG_WRITE - ? !IS_SYNCED(dirty_it->second.state) - : ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_WAIT_BIG); - if (!is_del && !deleted) - { - void *dyn_from = alloc_dyn_data - ? (uint8_t*)dirty_it->second.dyn_data + sizeof(int) : (uint8_t*)&dirty_it->second.dyn_data; - memcpy(dyn_ptr, dyn_from, dsk.clean_entry_bitmap_size); - } - } - } - if (!found) - { - auto & clean_db = clean_db_shard(op->oid); - auto clean_it = clean_db.find(op->oid); - if (clean_it != clean_db.end()) - { - version = clean_it->second.version + 1; - if (!is_del) - { - void *bmp_ptr = get_clean_entry_bitmap(clean_it->second.location, dsk.clean_entry_bitmap_size); - memcpy(dyn_ptr, bmp_ptr, dsk.clean_entry_bitmap_size); - } - } - else - { - deleted = true; - } - } - if (deleted && is_del) - { - // Already deleted - op->retval = 0; - return false; - } - PRIV(op)->real_version = 0; - if (op->version == 0) - { - op->version = version; - } - else if (op->version < version) - { - // Implicit operations must be added like that: DEL [FLUSH] BIG [SYNC] SMALL SMALL - if (deleted || wait_del) - { - // It's allowed to write versions with low numbers over deletes - // However, we have to flush those deletes first as we use version number for ordering -#ifdef BLOCKSTORE_DEBUG - printf("Write %jx:%jx v%ju over delete (real v%ju) offset=%u len=%u\n", op->oid.inode, op->oid.stripe, version, op->version, op->offset, op->len); -#endif - wait_del = true; - PRIV(op)->real_version = op->version; - op->version = version; - if (unsynced) - { - // Issue an additional sync so the delete reaches the journal - blockstore_op_t *sync_op = new blockstore_op_t; - sync_op->opcode = BS_OP_SYNC; - sync_op->oid = op->oid; - sync_op->version = op->version; - sync_op->callback = [this](blockstore_op_t *sync_op) - { - flusher->unshift_flush((obj_ver_id){ - .oid = sync_op->oid, - .version = sync_op->version-1, - }, true); - delete sync_op; - }; - enqueue_op(sync_op); - } - else - { - flusher->unshift_flush((obj_ver_id){ - .oid = op->oid, - .version = version-1, - }, true); - } - } - else - { - // Invalid version requested -#ifdef BLOCKSTORE_DEBUG - printf("Write %jx:%jx v%ju requested, but we already have v%ju\n", op->oid.inode, op->oid.stripe, op->version, version); -#endif - op->retval = -EEXIST; - if (!is_del && alloc_dyn_data) - { - free(dyn); - } - return false; - } - } - bool imm = (op->len < dsk.data_block_size ? (immediate_commit != IMMEDIATE_NONE) : (immediate_commit == IMMEDIATE_ALL)); - if (wait_big && !is_del && !deleted && op->len < dsk.data_block_size && !imm || - !imm && autosync_writes && unsynced_queued_ops >= autosync_writes) - { - // Issue an additional sync so that the previous big write can reach the journal - blockstore_op_t *sync_op = new blockstore_op_t; - sync_op->opcode = BS_OP_SYNC; - sync_op->callback = [](blockstore_op_t *sync_op) - { - delete sync_op; - }; - enqueue_op(sync_op); - } - else if (!imm) - unsynced_queued_ops++; -#ifdef BLOCKSTORE_DEBUG - if (is_del) - printf("Delete %jx:%jx v%ju\n", op->oid.inode, op->oid.stripe, op->version); - else if (!wait_del) - printf("Write %jx:%jx v%ju offset=%u len=%u\n", op->oid.inode, op->oid.stripe, op->version, op->offset, op->len); -#endif - // No strict need to add it into dirty_db here except maybe for listings to return - // correct data when there are inflight operations in the queue - uint32_t state; - if (is_del) - state = BS_ST_DELETE | BS_ST_IN_FLIGHT; - else - { - state = (op->len == dsk.data_block_size || deleted ? BS_ST_BIG_WRITE : BS_ST_SMALL_WRITE); - if (state == BS_ST_SMALL_WRITE && throttle_small_writes) - clock_gettime(CLOCK_REALTIME, &PRIV(op)->tv_begin); - if (wait_del) - state |= BS_ST_WAIT_DEL; - else if (state == BS_ST_SMALL_WRITE && wait_big) - state |= BS_ST_WAIT_BIG; - else - state |= BS_ST_IN_FLIGHT; - if (op->opcode == BS_OP_WRITE_STABLE) - state |= BS_ST_INSTANT; - if (op->bitmap) - memcpy(dyn_ptr, op->bitmap, dsk.clean_entry_bitmap_size); - } - // Calculate checksums - // FIXME: Allow to receive checksums from outside? - if (!is_del && dsk.data_csum_type && op->len > 0) - { - uint32_t *data_csums = (uint32_t*)(dyn_ptr + dsk.clean_entry_bitmap_size); - uint32_t start = op->offset / dsk.csum_block_size; - uint32_t end = (op->offset+op->len-1) / dsk.csum_block_size; - auto fn = state & BS_ST_BIG_WRITE ? crc32c_pad : crc32c_nopad; - if (start == end) - data_csums[0] = fn(0, op->buf, op->len, op->offset - start*dsk.csum_block_size, end*dsk.csum_block_size - (op->offset+op->len)); - else - { - // First block - data_csums[0] = fn(0, op->buf, dsk.csum_block_size*(start+1)-op->offset, op->offset - start*dsk.csum_block_size, 0); - // Intermediate blocks - for (uint32_t i = start+1; i < end; i++) - data_csums[i-start] = crc32c(0, (uint8_t*)op->buf + dsk.csum_block_size*i-op->offset, dsk.csum_block_size); - // Last block - data_csums[end-start] = fn( - 0, (uint8_t*)op->buf + end*dsk.csum_block_size - op->offset, - op->offset+op->len - end*dsk.csum_block_size, - 0, (end+1)*dsk.csum_block_size - (op->offset+op->len) - ); - } - } - dirty_db.emplace((obj_ver_id){ - .oid = op->oid, - .version = op->version, - }, (dirty_entry){ - .state = state, - .flags = 0, - .location = 0, - .offset = is_del ? 0 : op->offset, - .len = is_del ? 0 : op->len, - .journal_sector = 0, - .dyn_data = dyn, - }); + clock_gettime(CLOCK_REALTIME, &PRIV(op)->tv_begin); return true; } -void blockstore_impl_t::cancel_all_writes(blockstore_op_t *op, blockstore_dirty_db_t::iterator dirty_it, int retval) +void blockstore_impl_t::cancel_all_writes(blockstore_op_t *op, int retval) { - while (dirty_it != dirty_db.end() && dirty_it->first.oid == op->oid) - { - free_dirty_dyn_data(dirty_it->second); - dirty_db.erase(dirty_it++); - } bool found = false; for (auto other_op: submit_queue) { @@ -235,7 +28,7 @@ void blockstore_impl_t::cancel_all_writes(blockstore_op_t *op, blockstore_dirty_ (other_op->opcode == BS_OP_WRITE || other_op->opcode == BS_OP_WRITE_STABLE)) { // Mark operations to cancel them - PRIV(other_op)->real_version = UINT64_MAX; + PRIV(other_op)->op_state = 100; other_op->retval = retval; } } @@ -243,113 +36,89 @@ void blockstore_impl_t::cancel_all_writes(blockstore_op_t *op, blockstore_dirty_ FINISH_OP(op); } +void blockstore_impl_t::prepare_meta_block_write(blockstore_op_t *op, uint64_t modified_block) +{ + io_uring_sqe *sqe = get_sqe(); + assert(sqe != NULL); + ring_data_t *data = ((ring_data_t*)sqe->user_data); + data->iov = (struct iovec){ heap->get_meta_block(modified_block), (size_t)dsk.meta_block_size }; + data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); }; + PRIV(op)->pending_ops++; + io_uring_prep_writev( + sqe, dsk.meta_fd, &data->iov, 1, dsk.meta_offset + (modified_block+1)*dsk.meta_block_size + ); +} + // First step of the write algorithm: dequeue operation and submit initial write(s) int blockstore_impl_t::dequeue_write(blockstore_op_t *op) { + if (PRIV(op)->op_state == 100) + { + // This is the flag used to cancel ops + FINISH_OP(op); + return 2; + } if (PRIV(op)->op_state) { return continue_write(op); } - auto dirty_it = dirty_db.find((obj_ver_id){ - .oid = op->oid, - .version = op->version, - }); - assert(dirty_it != dirty_db.end()); - if ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) < BS_ST_IN_FLIGHT) - { - // Don't dequeue - return 0; - } - if (PRIV(op)->real_version != 0) - { - if (PRIV(op)->real_version == UINT64_MAX) - { - // This is the flag value used to cancel operations - FINISH_OP(op); - return 2; - } - // Restore original low version number for unblocked operations -#ifdef BLOCKSTORE_DEBUG - printf("Restoring %jx:%jx version: v%ju -> v%ju\n", op->oid.inode, op->oid.stripe, op->version, PRIV(op)->real_version); -#endif - auto prev_it = dirty_it; - if (prev_it != dirty_db.begin()) - { - prev_it--; - if (prev_it->first.oid == op->oid && prev_it->first.version >= PRIV(op)->real_version) - { - // Original version is still invalid - // All subsequent writes to the same object must be canceled too - printf("Tried to write %jx:%jx v%ju after delete (old version v%ju), but already have v%ju\n", - op->oid.inode, op->oid.stripe, PRIV(op)->real_version, op->version, prev_it->first.version); - cancel_all_writes(op, dirty_it, -EEXIST); - return 2; - } - } - op->version = PRIV(op)->real_version; - PRIV(op)->real_version = 0; - dirty_entry e = dirty_it->second; - dirty_db.erase(dirty_it); - dirty_it = dirty_db.emplace((obj_ver_id){ - .oid = op->oid, - .version = op->version, - }, e).first; - } if (write_iodepth >= max_write_iodepth) { return 0; } - if ((dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_BIG_WRITE) + PRIV(op)->is_big = false; + heap_object_t *obj = heap->read_entry(op->oid, NULL); + if (op->opcode == BS_OP_DELETE) { - blockstore_journal_check_t space_check(this); - if (!space_check.check_available(op, unsynced_big_write_count + 1, - sizeof(journal_entry_big_write) + dsk.clean_dyn_size, - (unstable_writes.size()+unstable_unsynced+((dirty_it->second.state & BS_ST_INSTANT) ? 0 : 1))*journal.block_size)) + // Delete + if (!obj) { - return 0; - } - // Big (redirect) write - uint64_t loc = data_alloc->find_free(); - if (loc == UINT64_MAX) - { - // no space - if (big_to_flush > 0) - { - // hope that some space will be available after flush - flusher->request_trim(); - PRIV(op)->wait_for = WAIT_FREE; - return 0; - } - cancel_all_writes(op, dirty_it, -ENOSPC); + // Already deleted + op->retval = 0; + FINISH_OP(op); return 2; } - if (inmemory_meta) + BS_SUBMIT_CHECK_SQES(1); + uint32_t modified_block; + int res = heap->post_delete(op->oid, &modified_block); + assert(res == 0); + prepare_meta_block_write(op, modified_block); + PRIV(op)->pending_ops++; + PRIV(op)->op_state = 5; + } + // FIXME: Allow to do initial writes as buffered, not redirected + // FIXME: Allow to do direct writes over holes + else if (!obj || op->offset == 0 && op->len == dsk.data_block_size) + { + // Big (redirect) write + PRIV(op)->is_big = true; + uint32_t tmp_block; + uint64_t loc = heap->find_free_data(); + if (loc == UINT64_MAX || + !obj && heap->get_block_for_new_object(tmp_block) != 0) { - // Check once more that metadata entry is zeroed (the reverse means a bug or corruption) - uint64_t sector = (loc / (dsk.meta_block_size / dsk.clean_entry_size)) * dsk.meta_block_size; - uint64_t pos = (loc % (dsk.meta_block_size / dsk.clean_entry_size)); - clean_disk_entry *entry = (clean_disk_entry*)((uint8_t*)metadata_buffer + sector + pos*dsk.clean_entry_size); - if (entry->oid.inode || entry->oid.stripe || entry->version) + auto queue_size = heap->get_compact_queue_size(); + if (!queue_size) { - printf( - "Fatal error (metadata corruption or bug): tried to write object %jx:%jx v%ju" - " over a non-zero metadata entry %ju with %jx:%jx v%ju\n", op->oid.inode, - op->oid.stripe, op->version, loc, entry->oid.inode, entry->oid.stripe, entry->version - ); - exit(1); + // no space + cancel_all_writes(op, -ENOSPC); + return 2; } + PRIV(op)->wait_for = WAIT_COMPACTION; + PRIV(op)->wait_detail = queue_size; + flusher->request_trim(); + return 0; } BS_SUBMIT_GET_SQE(sqe, data); write_iodepth++; - dirty_it->second.location = loc * dsk.data_block_size; - dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | BS_ST_SUBMITTED; + PRIV(op)->location = loc; #ifdef BLOCKSTORE_DEBUG printf( - "Allocate block %ju for %jx:%jx v%ju\n", + "Allocate offset %ju for %jx:%jx v%ju\n", loc, op->oid.inode, op->oid.stripe, op->version ); #endif - data_alloc->set(loc, true); + heap->use_data(op->oid.inode, PRIV(op)->location); uint64_t stripe_offset = (op->offset % dsk.bitmap_granularity); uint64_t stripe_end = (op->offset + op->len) % dsk.bitmap_granularity; // Zero fill up to dsk.bitmap_granularity @@ -367,161 +136,77 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op) data->iov.iov_len = op->len + stripe_offset + stripe_end; // to check it in the callback data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); }; io_uring_prep_writev( - sqe, dsk.data_fd, PRIV(op)->iov_zerofill, vcnt, dsk.data_offset + (loc * dsk.data_block_size) + op->offset - stripe_offset + sqe, dsk.data_fd, PRIV(op)->iov_zerofill, vcnt, dsk.data_offset + loc + op->offset - stripe_offset ); PRIV(op)->pending_ops = 1; - if (!(dirty_it->second.state & BS_ST_INSTANT)) - { - unstable_unsynced++; - } - if (immediate_commit != IMMEDIATE_ALL) - { - // Increase the counter, but don't save into unsynced_writes yet (can't sync until the write is finished) - unsynced_big_write_count++; - PRIV(op)->op_state = 3; - } - else - { - PRIV(op)->op_state = 1; - } + unsynced_big_write_count++; + PRIV(op)->op_state = 1; } - else /* if ((dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_SMALL_WRITE) */ + else { - // Small (journaled) write - // First check if the journal has sufficient space - uint64_t dyn_size = dsk.dirty_dyn_size(op->offset, op->len); - blockstore_journal_check_t space_check(this); - if (unsynced_big_write_count && - !space_check.check_available(op, unsynced_big_write_count, - sizeof(journal_entry_big_write) + dsk.clean_dyn_size, 0) - || !space_check.check_available(op, 1, - sizeof(journal_entry_small_write) + dyn_size, - op->len + (unstable_writes.size()+unstable_unsynced+((dirty_it->second.state & BS_ST_INSTANT) ? 0 : 1))*journal.block_size)) + // Small (buffered) overwrite + // First check if there is free buffer space + uint64_t loc = !op->len ? 0 : heap->find_free_buffer_area(op->len); + if (loc == UINT64_MAX) { + PRIV(op)->wait_for = WAIT_COMPACTION; + PRIV(op)->wait_detail = heap->get_compact_queue_size(); + flusher->request_trim(); return 0; } // There is sufficient space. Check SQE(s) - BS_SUBMIT_CHECK_SQES( - // Write current journal sector only if it's dirty and full, or in the immediate_commit mode - (immediate_commit != IMMEDIATE_NONE || - !journal.entry_fits(sizeof(journal_entry_small_write) + dyn_size) ? 1 : 0) + - (op->len > 0 ? 1 : 0) - ); + BS_SUBMIT_CHECK_SQES(1 + (op->len > 0 ? 1 : 0)); write_iodepth++; - // Got SQEs. Prepare previous journal sector write if required - if (immediate_commit == IMMEDIATE_NONE && - !journal.entry_fits(sizeof(journal_entry_small_write) + dyn_size)) + uint8_t wr_buf[heap->get_max_write_entry_size()]; + heap_write_t *wr = (heap_write_t*)wr_buf; + wr->version = op->version; + wr->offset = op->offset; + wr->len = op->len; + wr->location = loc; + PRIV(op)->location = loc; + wr->flags = BS_HEAP_SMALL_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0); + if (op->bitmap) + memcpy(wr->get_ext_bitmap(heap), op->bitmap, dsk.clean_entry_bitmap_size); + heap->calc_checksums(wr, (uint8_t*)op->buf, true); + uint32_t modified_block; + heap->use_buffer_area(op->oid.inode, loc, op->len); + int res = heap->post_write(op->oid, wr, &modified_block); + if (res == ENOSPC) { - prepare_journal_sector_write(journal.cur_sector, op); + cancel_all_writes(op, -ENOSPC); + return 2; } - // Then pre-fill journal entry - journal_entry_small_write *je = (journal_entry_small_write*)prefill_single_journal_entry( - journal, op->opcode == BS_OP_WRITE_STABLE ? JE_SMALL_WRITE_INSTANT : JE_SMALL_WRITE, - sizeof(journal_entry_small_write) + dyn_size - ); - auto jsec = dirty_it->second.journal_sector = journal.sector_info[journal.cur_sector].offset; - if (!(journal.next_free >= journal.used_start - ? (jsec >= journal.used_start && jsec < journal.next_free) - : (jsec >= journal.used_start || jsec < journal.next_free))) + else if (res == EAGAIN) { - printf( - "BUG: journal offset %08jx is used by %jx:%jx v%ju (%ju refs) BUT used_start=%jx next_free=%jx\n", - dirty_it->second.journal_sector, dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version, - journal.used_sectors[journal.sector_info[journal.cur_sector].offset], - journal.used_start, journal.next_free - ); - abort(); + // Pause submission, wait for compaction + return 0; } - journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++; -#ifdef BLOCKSTORE_DEBUG - printf( - "journal offset %08jx is used by %jx:%jx v%ju (%ju refs)\n", - dirty_it->second.journal_sector, dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version, - journal.used_sectors[journal.sector_info[journal.cur_sector].offset] - ); -#endif - // Figure out where data will be - auto next_next_free = (journal.next_free + op->len) <= journal.len ? journal.next_free : dsk.journal_block_size; + assert(res == 0); + prepare_meta_block_write(op, modified_block); if (op->len > 0) { - auto journal_used_it = journal.used_sectors.lower_bound(next_next_free); - if (journal_used_it != journal.used_sectors.end() && - journal_used_it->first < next_next_free + op->len) - { - printf( - "BUG: Attempt to overwrite used offset (%jx, %ju refs) of the journal with the object %jx:%jx v%ju: data at %jx, len %x!" - " Journal used_start=%08jx (%ju refs), next_free=%08jx, dirty_start=%08jx\n", - journal_used_it->first, journal_used_it->second, op->oid.inode, op->oid.stripe, op->version, next_next_free, op->len, - journal.used_start, journal.used_sectors[journal.used_start], journal.next_free, journal.dirty_start - ); - exit(1); - } - } - // double check that next_free doesn't cross used_start from the left - assert(journal.next_free >= journal.used_start && next_next_free >= journal.next_free || next_next_free < journal.used_start); - journal.next_free = next_next_free; - je->oid = op->oid; - je->version = op->version; - je->offset = op->offset; - je->len = op->len; - je->data_offset = journal.next_free; - je->crc32_data = dsk.csum_block_size ? 0 : crc32c(0, op->buf, op->len); - memcpy((void*)(je+1), (alloc_dyn_data - ? (uint8_t*)dirty_it->second.dyn_data+sizeof(int) : (uint8_t*)&dirty_it->second.dyn_data), dyn_size); - je->crc32 = je_crc32((journal_entry*)je); - journal.crc32_last = je->crc32; - if (immediate_commit != IMMEDIATE_NONE) - { - prepare_journal_sector_write(journal.cur_sector, op); - } - if (op->len > 0) - { - // Prepare journal data write - if (journal.inmemory) - { - // Copy data - memcpy((uint8_t*)journal.buffer + journal.next_free, op->buf, op->len); - } + // Prepare buffered data write + assert(dsk.inmemory_journal); + memcpy((uint8_t*)buffer_area + loc, op->buf, op->len); BS_SUBMIT_GET_SQE(sqe2, data2); data2->iov = (struct iovec){ op->buf, op->len }; - ++journal.submit_id; - assert(journal.submit_id != 0); // check overflow - // Make subsequent journal writes wait for our data write - journal.flushing_ops.emplace(journal.submit_id, (pending_journaling_t){ - .pending = 1, - .sector = -1, - .op = op, - }); - data2->callback = [this, flush_id = journal.submit_id](ring_data_t *data) { handle_journal_write(data, flush_id); }; - io_uring_prep_writev( - sqe2, dsk.journal_fd, &data2->iov, 1, journal.offset + journal.next_free - ); + data2->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); }; + io_uring_prep_writev(sqe2, dsk.journal_fd, &data2->iov, 1, dsk.journal_offset + loc); PRIV(op)->pending_ops++; } else { // Zero-length overwrite. Allowed to bump object version in EC placement groups without actually writing data } - dirty_it->second.location = journal.next_free; - dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | BS_ST_SUBMITTED; - next_next_free = journal.next_free + op->len; - if (next_next_free >= journal.len) - next_next_free = dsk.journal_block_size; - // double check that next_free doesn't cross used_start from the left - assert(journal.next_free >= journal.used_start && next_next_free >= journal.next_free || next_next_free < journal.used_start); - journal.next_free = next_next_free; - if (!(dirty_it->second.state & BS_ST_INSTANT)) - { - unstable_unsynced++; - } + unsynced_small_write_count++; if (!PRIV(op)->pending_ops) { - PRIV(op)->op_state = 4; + PRIV(op)->op_state = 6; return continue_write(op); } else { - PRIV(op)->op_state = 3; + PRIV(op)->op_state = 5; } } return 1; @@ -536,130 +221,60 @@ int blockstore_impl_t::continue_write(blockstore_op_t *op) goto resume_4; else if (op_state == 6) goto resume_6; + else if (op_state == 8) + goto resume_8; else { // In progress return 1; } resume_2: - // Only for the immediate_commit mode: prepare and submit big_write journal entry + // We must fsync all big writes to avoid complex write workflows + // It's anyway OK for all HDDs and for server SSDs + // The other way is to add another type of MVCC to blockstore_heap: "forward" MVCC :) + if (!disable_data_fsync) { - auto dirty_it = dirty_db.find((obj_ver_id){ - .oid = op->oid, - .version = op->version, - }); - assert(dirty_it != dirty_db.end()); - uint64_t dyn_size = dsk.dirty_dyn_size(op->offset, op->len); - blockstore_journal_check_t space_check(this); - if (!space_check.check_available(op, 1, sizeof(journal_entry_big_write) + dyn_size, - (unstable_writes.size()+unstable_unsynced+((dirty_it->second.state & BS_ST_INSTANT) ? 0 : 1))*journal.block_size)) - { - return 0; - } - BS_SUBMIT_CHECK_SQES(1); - journal_entry_big_write *je = (journal_entry_big_write*)prefill_single_journal_entry( - journal, op->opcode == BS_OP_WRITE_STABLE ? JE_BIG_WRITE_INSTANT : JE_BIG_WRITE, - sizeof(journal_entry_big_write) + dyn_size - ); - auto jsec = dirty_it->second.journal_sector = journal.sector_info[journal.cur_sector].offset; - if (!(journal.next_free >= journal.used_start - ? (jsec >= journal.used_start && jsec < journal.next_free) - : (jsec >= journal.used_start || jsec < journal.next_free))) - { - printf( - "BUG: journal offset %08jx is used by %jx:%jx v%ju (%ju refs) BUT used_start=%jx next_free=%jx\n", - dirty_it->second.journal_sector, dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version, - journal.used_sectors[journal.sector_info[journal.cur_sector].offset], - journal.used_start, journal.next_free - ); - abort(); - } - journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++; -#ifdef BLOCKSTORE_DEBUG - printf( - "journal offset %08jx is used by %jx:%jx v%ju (%ju refs)\n", - journal.sector_info[journal.cur_sector].offset, op->oid.inode, op->oid.stripe, op->version, - journal.used_sectors[journal.sector_info[journal.cur_sector].offset] - ); -#endif - je->oid = op->oid; - je->version = op->version; - je->offset = op->offset; - je->len = op->len; - je->location = dirty_it->second.location; - memcpy((void*)(je+1), (alloc_dyn_data - ? (uint8_t*)dirty_it->second.dyn_data+sizeof(int) : (uint8_t*)&dirty_it->second.dyn_data), dyn_size); - je->crc32 = je_crc32((journal_entry*)je); - journal.crc32_last = je->crc32; - prepare_journal_sector_write(journal.cur_sector, op); + // fsync data + // FIXME: Share fsyncs with fsync batches from flusher + BS_SUBMIT_GET_SQE(sqe, data); + io_uring_prep_fsync(sqe, dsk.data_fd, IORING_FSYNC_DATASYNC); + data->iov = { 0 }; + data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); }; + PRIV(op)->pending_ops++; PRIV(op)->op_state = 3; return 1; } resume_4: - // Switch object state { - auto dirty_it = dirty_db.find((obj_ver_id){ - .oid = op->oid, - .version = op->version, - }); - assert(dirty_it != dirty_db.end()); + uint8_t wr_buf[heap->get_max_write_entry_size()]; + heap_write_t *wr = (heap_write_t*)wr_buf; + wr->version = op->version; + wr->offset = op->offset; + wr->len = op->len; + wr->location = PRIV(op)->location; + wr->flags = BS_HEAP_BIG_WRITE | (op->opcode == BS_OP_WRITE_STABLE ? BS_HEAP_STABLE : 0); + if (op->bitmap) + memcpy(wr->get_ext_bitmap(heap), op->bitmap, dsk.clean_entry_bitmap_size); + heap->calc_checksums(wr, (uint8_t*)op->buf, true); + uint32_t modified_block; + int res = heap->post_write(op->oid, wr, &modified_block); + if (res == ENOSPC) + { + // wait for compaction + return 1; + } + assert(res == 0); + prepare_meta_block_write(op, modified_block); + PRIV(op)->op_state = 5; + return 1; + } +resume_6: + { #ifdef BLOCKSTORE_DEBUG - printf("Ack write %jx:%jx v%ju = state 0x%x\n", op->oid.inode, op->oid.stripe, op->version, dirty_it->second.state); + printf("Ack write %jx:%jx v%ju\n", op->oid.inode, op->oid.stripe, op->version); #endif - bool is_big = (dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_BIG_WRITE; - bool imm = is_big ? (immediate_commit == IMMEDIATE_ALL) : (immediate_commit != IMMEDIATE_NONE); - bool is_instant = IS_INSTANT(dirty_it->second.state); - if (imm) - { - auto & unstab = unstable_writes[op->oid]; - unstab = unstab < op->version ? op->version : unstab; - if (!is_instant) - { - unstable_unsynced--; - assert(unstable_unsynced >= 0); - } - } - dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) - | (imm ? BS_ST_SYNCED : BS_ST_WRITTEN); - if (imm && is_instant) - { - // Deletions and 'instant' operations are treated as immediately stable - mark_stable(dirty_it->first); - } - if (!imm) - { - if (is_big) - { - // Remember big write as unsynced - unsynced_big_writes.push_back((obj_ver_id){ - .oid = op->oid, - .version = op->version, - }); - } - else - { - // Remember small write as unsynced - unsynced_small_writes.push_back((obj_ver_id){ - .oid = op->oid, - .version = op->version, - }); - } - } - if (imm && (dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_BIG_WRITE) - { - // Unblock small writes - dirty_it++; - while (dirty_it != dirty_db.end() && dirty_it->first.oid == op->oid) - { - if ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_WAIT_BIG) - { - dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | BS_ST_IN_FLIGHT; - } - dirty_it++; - } - } // Apply throttling to not fill the journal too fast for the SSD+HDD case - if (!is_big && throttle_small_writes) + if (!PRIV(op)->is_big && throttle_small_writes) { // Apply throttling timespec tv_end; @@ -670,19 +285,16 @@ resume_4: // Compare with target execution time // 100% free -> target time = 0 // 0% free -> target time = iodepth/parallelism * (iops + size/bw) / write per second - uint64_t used_start = journal.get_trim_pos(); - uint64_t journal_free_space = journal.next_free < used_start - ? (used_start - journal.next_free) - : (journal.len - journal.next_free + used_start - journal.block_size); + uint64_t buffer_free_space = dsk.journal_len - heap->get_buffer_area_used_space(); uint64_t ref_us = (write_iodepth <= throttle_target_parallelism ? 100 : 100*write_iodepth/throttle_target_parallelism) * (1000000/throttle_target_iops + op->len*1000000/throttle_target_mbs/1024/1024) / 100; - ref_us -= ref_us * journal_free_space / journal.len; + ref_us -= ref_us * buffer_free_space / dsk.journal_len; if (ref_us > exec_us + throttle_threshold_us) { // Pause reply - PRIV(op)->op_state = 5; + PRIV(op)->op_state = 7; // Remember that the timer can in theory be called right here tfd->set_timer_us(ref_us-exec_us, false, [this, op](int timer_id) { @@ -693,7 +305,7 @@ resume_4: } } } -resume_6: +resume_8: // Acknowledge write op->retval = op->len; write_iodepth--; @@ -713,112 +325,7 @@ void blockstore_impl_t::handle_write_event(ring_data_t *data, blockstore_op_t *o assert(PRIV(op)->pending_ops >= 0); if (PRIV(op)->pending_ops == 0) { - release_journal_sectors(op); PRIV(op)->op_state++; ringloop->wakeup(); } } - -void blockstore_impl_t::release_journal_sectors(blockstore_op_t *op) -{ - // Release flushed journal sectors - if (PRIV(op)->min_flushed_journal_sector > 0 && - PRIV(op)->max_flushed_journal_sector > 0) - { - uint64_t s = PRIV(op)->min_flushed_journal_sector; - while (1) - { - if (!journal.sector_info[s-1].dirty && journal.sector_info[s-1].flush_count == 0) - { - if (s == (1+journal.cur_sector)) - { - // Forcibly move to the next sector and move dirty position - journal.in_sector_pos = journal.block_size; - } - // We know for sure that we won't write into this sector anymore - uint64_t new_ds = journal.sector_info[s-1].offset + journal.block_size; - if (new_ds >= journal.len) - { - new_ds = journal.block_size; - } - if ((journal.dirty_start + (journal.dirty_start >= journal.used_start ? 0 : journal.len)) < - (new_ds + (new_ds >= journal.used_start ? 0 : journal.len))) - { - journal.dirty_start = new_ds; - } - } - if (s == PRIV(op)->max_flushed_journal_sector) - break; - s = 1 + s % journal.sector_count; - } - PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 0; - } -} - -int blockstore_impl_t::dequeue_del(blockstore_op_t *op) -{ - if (PRIV(op)->op_state) - { - return continue_write(op); - } - auto dirty_it = dirty_db.find((obj_ver_id){ - .oid = op->oid, - .version = op->version, - }); - assert(dirty_it != dirty_db.end()); - blockstore_journal_check_t space_check(this); - if (!space_check.check_available(op, 1, sizeof(journal_entry_del), (unstable_writes.size()+unstable_unsynced)*journal.block_size)) - { - return 0; - } - // Write current journal sector only if it's dirty and full, or in the immediate_commit mode - BS_SUBMIT_CHECK_SQES( - (immediate_commit != IMMEDIATE_NONE || - (dsk.journal_block_size - journal.in_sector_pos) < sizeof(journal_entry_del) && - journal.sector_info[journal.cur_sector].dirty) ? 1 : 0 - ); - if (write_iodepth >= max_write_iodepth) - { - return 0; - } - write_iodepth++; - // Prepare journal sector write - if (immediate_commit == IMMEDIATE_NONE && - (dsk.journal_block_size - journal.in_sector_pos) < sizeof(journal_entry_del) && - journal.sector_info[journal.cur_sector].dirty) - { - prepare_journal_sector_write(journal.cur_sector, op); - } - // Pre-fill journal entry - journal_entry_del *je = (journal_entry_del*)prefill_single_journal_entry( - journal, JE_DELETE, sizeof(struct journal_entry_del) - ); - dirty_it->second.journal_sector = journal.sector_info[journal.cur_sector].offset; - journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++; -#ifdef BLOCKSTORE_DEBUG - printf( - "journal offset %08jx is used by %jx:%jx v%ju (%ju refs)\n", - dirty_it->second.journal_sector, dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version, - journal.used_sectors[journal.sector_info[journal.cur_sector].offset] - ); -#endif - je->oid = op->oid; - je->version = op->version; - je->crc32 = je_crc32((journal_entry*)je); - journal.crc32_last = je->crc32; - dirty_it->second.state = BS_ST_DELETE | BS_ST_SUBMITTED; - if (immediate_commit != IMMEDIATE_NONE) - { - prepare_journal_sector_write(journal.cur_sector, op); - } - if (!PRIV(op)->pending_ops) - { - PRIV(op)->op_state = 4; - return continue_write(op); - } - else - { - PRIV(op)->op_state = 3; - } - return 1; -} diff --git a/src/blockstore/fio_engine.cpp b/src/blockstore/fio_engine.cpp index ae938e82..7febf402 100644 --- a/src/blockstore/fio_engine.cpp +++ b/src/blockstore/fio_engine.cpp @@ -200,7 +200,7 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io) { case DDIR_READ: op->opcode = BS_OP_READ; - op->buf = (uint8_t*)io->xfer_buf; + op->buf = io->xfer_buf; op->oid = { .inode = 1, .stripe = io->offset / bsd->bs->get_block_size(), @@ -221,7 +221,7 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io) break; case DDIR_WRITE: op->opcode = bsd->ec ? BS_OP_WRITE : BS_OP_WRITE_STABLE; - op->buf = (uint8_t*)io->xfer_buf; + op->buf = io->xfer_buf; op->oid = { .inode = 1, .stripe = io->offset / bsd->bs->get_block_size(), @@ -247,7 +247,7 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io) { auto stab_op = new blockstore_op_t; stab_op->opcode = BS_OP_STABLE; - stab_op->buf = (uint8_t*)malloc_or_die(sizeof(obj_ver_id)); + stab_op->buf = malloc_or_die(sizeof(obj_ver_id)); obj_ver_id *ver = (obj_ver_id *)stab_op->buf; ver[0].oid = op->oid; ver[0].version = op->version; diff --git a/src/blockstore/ondisk_formats.h b/src/blockstore/ondisk_formats.h index 9b85127c..22bdc05e 100644 --- a/src/blockstore/ondisk_formats.h +++ b/src/blockstore/ondisk_formats.h @@ -148,6 +148,7 @@ inline uint32_t je_crc32(journal_entry *je) #define BLOCKSTORE_META_MAGIC_V1 0x726F747341544956l #define BLOCKSTORE_META_FORMAT_V1 1 #define BLOCKSTORE_META_FORMAT_V2 2 +#define BLOCKSTORE_META_FORMAT_HEAP 3 // metadata header (superblock) struct __attribute__((__packed__)) blockstore_meta_header_v1_t @@ -173,6 +174,22 @@ struct __attribute__((__packed__)) blockstore_meta_header_v2_t uint32_t header_csum; }; +struct __attribute__((__packed__)) blockstore_meta_header_v3_t +{ + uint64_t zero; + uint64_t magic; + uint64_t version; + uint32_t meta_block_size; + uint32_t data_block_size; + uint32_t bitmap_granularity; + uint32_t data_csum_type; + uint32_t csum_block_size; + uint32_t header_csum; + uint64_t compacted_lsn; + + void set_crc32c(); +}; + // 32 bytes = 24 bytes + block bitmap (4 bytes by default) + external attributes (also bitmap, 4 bytes by default) // per "clean" entry on disk with fixed metadata tables struct __attribute__((__packed__)) clean_disk_entry diff --git a/src/blockstore/v1/flush.cpp b/src/blockstore/v1/flush.cpp new file mode 100644 index 00000000..4ce2ff91 --- /dev/null +++ b/src/blockstore/v1/flush.cpp @@ -0,0 +1,1469 @@ +// Copyright (c) Vitaliy Filippov, 2019+ +// License: VNPL-1.1 (see README.md for details) + +#include "blockstore_impl.h" +#include "blockstore_internal.h" + +#define META_BLOCK_UNREAD 0 +#define META_BLOCK_READ 1 + +journal_flusher_t::journal_flusher_t(blockstore_impl_t *bs) +{ + this->bs = bs; + this->max_flusher_count = bs->max_flusher_count; + this->min_flusher_count = bs->min_flusher_count; + this->cur_flusher_count = bs->min_flusher_count; + this->target_flusher_count = bs->min_flusher_count; + dequeuing = false; + trimming = false; + active_flushers = 0; + syncing_flushers = 0; + // FIXME: allow to configure flusher_start_threshold and journal_trim_interval + flusher_start_threshold = bs->dsk.journal_block_size / sizeof(journal_entry_stable); + journal_trim_counter = bs->journal.flush_journal ? 1 : 0; + trim_wanted = bs->journal.flush_journal ? 1 : 0; + journal_superblock = bs->journal.inmemory ? bs->journal.buffer : memalign_or_die(MEM_ALIGNMENT, bs->dsk.journal_block_size); + co = new journal_flusher_co[max_flusher_count]; + for (int i = 0; i < max_flusher_count; i++) + { + co[i].bs = bs; + co[i].flusher = this; + } +} + +journal_flusher_co::journal_flusher_co() +{ + wait_state = 0; + simple_callback_r = [this](ring_data_t* data) + { + bs->live = true; + if (data->res != data->iov.iov_len) + bs->disk_error_abort("read operation during flush", data->res, data->iov.iov_len); + wait_count--; + }; + simple_callback_rj = [this](ring_data_t* data) + { + bs->live = true; + if (data->res != data->iov.iov_len) + bs->disk_error_abort("read operation during flush", data->res, data->iov.iov_len); + wait_journal_count--; + }; + simple_callback_w = [this](ring_data_t* data) + { + bs->live = true; + if (data->res != data->iov.iov_len) + bs->disk_error_abort("write operation during flush", data->res, data->iov.iov_len); + wait_count--; + }; +} + +journal_flusher_t::~journal_flusher_t() +{ + if (!bs->journal.inmemory) + free(journal_superblock); + delete[] co; +} + +bool journal_flusher_t::is_active() +{ + return active_flushers > 0 || dequeuing; +} + +void journal_flusher_t::loop() +{ + target_flusher_count = bs->write_iodepth*2; + if (target_flusher_count < min_flusher_count) + target_flusher_count = min_flusher_count; + else if (target_flusher_count > max_flusher_count) + target_flusher_count = max_flusher_count; + if (target_flusher_count > cur_flusher_count) + cur_flusher_count = target_flusher_count; + else if (target_flusher_count < cur_flusher_count) + { + while (target_flusher_count < cur_flusher_count) + { + if (co[cur_flusher_count-1].wait_state) + break; + cur_flusher_count--; + } + } + if (trim_wanted) + co[0].try_trim = true; + for (int i = 0; (active_flushers > 0 || dequeuing || trim_wanted > 0) && i < cur_flusher_count; i++) + co[i].loop(); +} + +void journal_flusher_t::enqueue_flush(obj_ver_id ov) +{ +#ifdef BLOCKSTORE_DEBUG + printf("enqueue_flush %jx:%jx v%ju\n", ov.oid.inode, ov.oid.stripe, ov.version); +#endif + auto it = flush_versions.find(ov.oid); + if (it != flush_versions.end()) + { + if (it->second < ov.version) + it->second = ov.version; + } + else + { + flush_versions[ov.oid] = ov.version; + flush_queue.push_back(ov.oid); + } + if (!dequeuing && (flush_queue.size() >= flusher_start_threshold || trim_wanted > 0)) + { + dequeuing = true; + bs->ringloop->wakeup(); + } +} + +void journal_flusher_t::unshift_flush(obj_ver_id ov, bool force) +{ +#ifdef BLOCKSTORE_DEBUG + printf("unshift_flush %jx:%jx v%ju\n", ov.oid.inode, ov.oid.stripe, ov.version); +#endif + auto it = flush_versions.find(ov.oid); + if (it != flush_versions.end()) + { + if (it->second < ov.version) + it->second = ov.version; + } + else + { + flush_versions[ov.oid] = ov.version; + if (!force) + flush_queue.push_front(ov.oid); + } + if (force) + flush_queue.push_front(ov.oid); + if (force || !dequeuing && (flush_queue.size() >= flusher_start_threshold || trim_wanted > 0)) + { + dequeuing = true; + bs->ringloop->wakeup(); + } +} + +void journal_flusher_t::remove_flush(object_id oid) +{ +#ifdef BLOCKSTORE_DEBUG + printf("undo_flush %jx:%jx\n", oid.inode, oid.stripe); +#endif + auto v_it = flush_versions.find(oid); + if (v_it != flush_versions.end()) + { + flush_versions.erase(v_it); + for (auto q_it = flush_queue.begin(); q_it != flush_queue.end(); q_it++) + { + if (*q_it == oid) + { + flush_queue.erase(q_it); + break; + } + } + } +} + +bool journal_flusher_t::is_mutated(uint64_t clean_loc) +{ + for (int i = 0; i < cur_flusher_count; i++) + { + if (co[i].clean_loc == clean_loc && co[i].copy_count > 0) + { + return true; + } + } + return false; +} + +void journal_flusher_t::request_trim() +{ + dequeuing = true; + trim_wanted++; + bs->ringloop->wakeup(); +} + +void journal_flusher_t::mark_trim_possible() +{ + if (trim_wanted > 0) + { + dequeuing = true; + journal_trim_counter = 0; + bs->ringloop->wakeup(); + } +} + +void journal_flusher_t::release_trim() +{ + trim_wanted--; +} + +void journal_flusher_t::dump_diagnostics() +{ + const char *unflushable_type = ""; + obj_ver_id unflushable = {}; + // Try to find out if there is a flushable object for information + for (object_id cur_oid: flush_queue) + { + obj_ver_id cur = { .oid = cur_oid, .version = flush_versions[cur_oid] }; + auto dirty_end = bs->dirty_db.find(cur); + if (dirty_end == bs->dirty_db.end()) + { + // Already flushed + continue; + } + auto repeat_it = sync_to_repeat.find(cur.oid); + if (repeat_it != sync_to_repeat.end()) + { + // Someone is already flushing it + unflushable_type = "locked,"; + unflushable = cur; + break; + } + if (dirty_end->second.journal_sector >= bs->journal.dirty_start && + (bs->journal.dirty_start >= bs->journal.used_start || + dirty_end->second.journal_sector < bs->journal.used_start)) + { + // Object is more recent than possible to flush + bool found = try_find_older(dirty_end, cur); + if (!found) + { + unflushable_type = "dirty,"; + unflushable = cur; + break; + } + } + unflushable_type = "ok,"; + unflushable = cur; + break; + } + printf( + "Flusher: queued=%zd first=%s%jx:%jx trim_wanted=%d dequeuing=%d trimming=%d cur=%d target=%d active=%d syncing=%d\n", + flush_queue.size(), unflushable_type, unflushable.oid.inode, unflushable.oid.stripe, + trim_wanted, dequeuing, trimming, cur_flusher_count, target_flusher_count, + active_flushers, syncing_flushers + ); +} + +bool journal_flusher_t::try_find_older(std::map::iterator & dirty_end, obj_ver_id & cur) +{ + bool found = false; + while (dirty_end != bs->dirty_db.begin()) + { + dirty_end--; + if (dirty_end->first.oid != cur.oid) + { + break; + } + if (!(dirty_end->second.journal_sector >= bs->journal.dirty_start && + (bs->journal.dirty_start >= bs->journal.used_start || + dirty_end->second.journal_sector < bs->journal.used_start))) + { + found = true; + cur.version = dirty_end->first.version; + break; + } + } + return found; +} + +bool journal_flusher_t::try_find_other(std::map::iterator & dirty_end, obj_ver_id & cur) +{ + int search_left = flush_queue.size() - 1; +#ifdef BLOCKSTORE_DEBUG + printf("Flusher overran writers (%jx:%jx v%ju, dirty_start=%08jx) - searching for older flushes (%d left)\n", + cur.oid.inode, cur.oid.stripe, cur.version, bs->journal.dirty_start, search_left); +#endif + while (search_left > 0) + { + cur.oid = flush_queue.front(); + cur.version = flush_versions[cur.oid]; + flush_queue.pop_front(); + flush_versions.erase(cur.oid); + dirty_end = bs->dirty_db.find(cur); + if (dirty_end != bs->dirty_db.end()) + { + if (dirty_end->second.journal_sector >= bs->journal.dirty_start && + (bs->journal.dirty_start >= bs->journal.used_start || + dirty_end->second.journal_sector < bs->journal.used_start)) + { +#ifdef BLOCKSTORE_DEBUG + printf("Write %jx:%jx v%ju is too new: offset=%08jx\n", cur.oid.inode, cur.oid.stripe, cur.version, dirty_end->second.journal_sector); +#endif + enqueue_flush(cur); + } + else + { + auto repeat_it = sync_to_repeat.find(cur.oid); + if (repeat_it != sync_to_repeat.end()) + { + if (repeat_it->second < cur.version) + repeat_it->second = cur.version; + } + else + { + sync_to_repeat[cur.oid] = 0; + break; + } + } + } + search_left--; + } + if (search_left <= 0) + { +#ifdef BLOCKSTORE_DEBUG + printf("No older flushes, stopping\n"); +#endif + } + return search_left > 0; +} + +#define await_sqe(label) \ + resume_##label:\ + sqe = bs->get_sqe();\ + if (!sqe)\ + {\ + wait_state = wait_base+label;\ + return false;\ + }\ + data = ((ring_data_t*)sqe->user_data); + +bool journal_flusher_co::loop() +{ + int wait_base = 0; + // This is much better than implementing the whole function as an FSM + // Maybe I should consider a coroutine library like https://github.com/hnes/libaco ... + // Or just C++ coroutines, but they require some wrappers + if (wait_state == 1) goto resume_1; + else if (wait_state == 2) goto resume_2; + else if (wait_state == 3) goto resume_3; + else if (wait_state == 4) goto resume_4; + else if (wait_state == 5) goto resume_5; + else if (wait_state == 6) goto resume_6; + else if (wait_state == 7) goto resume_7; + else if (wait_state == 8) goto resume_8; + else if (wait_state == 9) goto resume_9; + else if (wait_state == 10) goto resume_10; + else if (wait_state == 11) goto resume_11; + else if (wait_state == 12) goto resume_12; + else if (wait_state == 13) goto resume_13; + else if (wait_state == 14) goto resume_14; + else if (wait_state == 15) goto resume_15; + else if (wait_state == 16) goto resume_16; + else if (wait_state == 17) goto resume_17; + else if (wait_state == 18) goto resume_18; + else if (wait_state == 19) goto resume_19; + else if (wait_state == 20) goto resume_20; + else if (wait_state == 21) goto resume_21; + else if (wait_state == 22) goto resume_22; + else if (wait_state == 23) goto resume_23; + else if (wait_state == 24) goto resume_24; + else if (wait_state == 25) goto resume_25; + else if (wait_state == 26) goto resume_26; + else if (wait_state == 27) goto resume_27; + else if (wait_state == 28) goto resume_28; + else if (wait_state == 29) goto resume_29; + else if (wait_state == 30) goto resume_30; + else if (wait_state == 31) goto resume_31; + else if (wait_state == 32) goto resume_32; + else if (wait_state == 33) goto resume_33; + else if (wait_state == 34) goto resume_34; +resume_0: + if (flusher->flush_queue.size() < flusher->min_flusher_count && !flusher->trim_wanted || + !flusher->flush_queue.size() || !flusher->dequeuing) + { +stop_flusher: + flusher->dequeuing = false; + if (flusher->trim_wanted > 0 && try_trim) + { + // Attempt forced trim + try_trim = false; + flusher->active_flushers++; + goto trim_journal; + } + wait_state = 0; + return true; + } + try_trim = true; + cur.oid = flusher->flush_queue.front(); + cur.version = flusher->flush_versions[cur.oid]; + flusher->flush_queue.pop_front(); + flusher->flush_versions.erase(cur.oid); + dirty_end = bs->dirty_db.find(cur); + if (dirty_end != bs->dirty_db.end()) + { + repeat_it = flusher->sync_to_repeat.find(cur.oid); + if (repeat_it != flusher->sync_to_repeat.end()) + { +#ifdef BLOCKSTORE_DEBUG + printf("Postpone %jx:%jx v%ju\n", cur.oid.inode, cur.oid.stripe, cur.version); +#endif + // We don't flush different parts of history of the same object in parallel + // So we check if someone is already flushing this object + // In that case we set sync_to_repeat and pick another object + // Another coroutine will see it and re-queue the object after it finishes + if (repeat_it->second < cur.version) + repeat_it->second = cur.version; + wait_state = 0; + goto resume_0; + } + else + flusher->sync_to_repeat[cur.oid] = 0; + if (dirty_end->second.journal_sector >= bs->journal.dirty_start && + (bs->journal.dirty_start >= bs->journal.used_start || + dirty_end->second.journal_sector < bs->journal.used_start)) + { + flusher->enqueue_flush(cur); + // We can't flush journal sectors that are still written to + // However, as we group flushes by oid, current oid may have older writes to flush! + // And it may even block writes if we don't flush the older version + // (if it's in the beginning of the journal)... + // So first try to find an older version of the same object to flush. + if (!flusher->try_find_older(dirty_end, cur)) + { + // Try other objects + flusher->sync_to_repeat.erase(cur.oid); + if (!flusher->try_find_other(dirty_end, cur)) + { + cur.oid = {}; + goto stop_flusher; + } + } + } +#ifdef BLOCKSTORE_DEBUG + printf("Flushing %jx:%jx v%ju\n", cur.oid.inode, cur.oid.stripe, cur.version); +#endif + flusher->active_flushers++; + // Find it in clean_db + { + auto & clean_db = bs->clean_db_shard(cur.oid); + auto clean_it = clean_db.find(cur.oid); + old_clean_ver = (clean_it != clean_db.end() ? clean_it->second.version : 0); + old_clean_loc = (clean_it != clean_db.end() ? clean_it->second.location : UINT64_MAX); + } + // Scan dirty versions of the object to determine what we need to read + scan_dirty(); + // Writes and deletes shouldn't happen at the same time + assert(!has_writes || !has_delete); + if (!has_writes && !has_delete || has_delete && old_clean_loc == UINT64_MAX) + { + // Nothing to flush + bs->erase_dirty(dirty_start, std::next(dirty_end), clean_loc); + goto release_oid; + } + if (clean_loc == UINT64_MAX) + { + if (old_clean_loc == UINT64_MAX) + { + // Object not allocated. This is a bug. + char err[1024]; + snprintf( + err, 1024, "BUG: Object %jx:%jx v%ju that we are trying to flush is not allocated on the data device", + cur.oid.inode, cur.oid.stripe, cur.version + ); + throw std::runtime_error(err); + } + else + { + clean_loc = old_clean_loc; + clean_ver = old_clean_ver; + } + } + // Submit dirty data and old checksum data reads +resume_1: +resume_2: + if (!read_dirty(1)) + return false; + // Also we may need to read metadata. We do read-modify-write cycle(s) for every operation. + resume_3: + resume_4: + if (!modify_meta_do_reads(3)) + return false; + // Now, if csum_block_size is > bitmap_granularity and if we are doing partial checksum block updates, + // perform a trick: clear bitmap bits in the metadata entry and recalculate block checksum with zeros + // in place of overwritten parts. Then, even if the actual partial update fully or partially fails, + // we'll have a correct checksum because it won't include overwritten parts! + // The same thing actually happens even when csum_block_size == bitmap_granularity, but in that case + // we never need to read (and thus verify) overwritten parts from the data device. + resume_5: + resume_6: + resume_7: + resume_8: + resume_9: + resume_10: + resume_11: + resume_12: + resume_13: + if (fill_incomplete && !clear_incomplete_csum_block_bits(5)) + return false; + // Wait for journal data reads if the journal is not inmemory + resume_14: + if (wait_journal_count > 0) + { + wait_state = wait_base+14; + return false; + } + if (bs->dsk.csum_block_size) + { + // Mark objects used by reads as modified + auto uo_it = bs->used_clean_objects.find(clean_loc); + if (uo_it != bs->used_clean_objects.end()) + { + uo_it->second.was_changed = true; + } + } + // Submit data writes + for (it = v.begin(); it != v.end(); it++) + { + if (it->copy_flags == COPY_BUF_JOURNAL || it->copy_flags == (COPY_BUF_JOURNAL|COPY_BUF_COALESCED)) + { + await_sqe(15); + data->iov = (struct iovec){ it->buf, (size_t)it->len }; + data->callback = simple_callback_w; + io_uring_prep_writev( + sqe, bs->dsk.data_fd, &data->iov, 1, bs->dsk.data_offset + clean_loc + it->offset + ); + wait_count++; + } + } + // Wait for data writes and metadata reads + resume_16: + resume_17: + if (!wait_meta_reads(16)) + return false; + // Sync data before writing metadata + resume_18: + resume_19: + resume_20: + if (copy_count && !fsync_batch(false, 18)) + return false; + if (old_clean_loc != UINT64_MAX && old_clean_loc != clean_loc) + { + // zero out old metadata entry + { + clean_disk_entry *old_entry = (clean_disk_entry*)((uint8_t*)meta_old.buf + meta_old.pos*bs->dsk.clean_entry_size); + if (old_entry->oid.inode != 0 && old_entry->oid != cur.oid) + { + printf("Fatal error (metadata corruption or bug): tried to wipe metadata entry %ju (%jx:%jx v%ju) as old location of %jx:%jx\n", + old_clean_loc / bs->dsk.data_block_size, old_entry->oid.inode, old_entry->oid.stripe, + old_entry->version, cur.oid.inode, cur.oid.stripe); + exit(1); + } + } + memset((uint8_t*)meta_old.buf + meta_old.pos*bs->dsk.clean_entry_size, 0, bs->dsk.clean_entry_size); + if (meta_old.sector != meta_new.sector) + { + resume_21: + if (flusher->inflight_meta_sectors.find(meta_old.sector) != flusher->inflight_meta_sectors.end()) + { + wait_state = wait_base+21; + return false; + } + flusher->inflight_meta_sectors.insert(meta_old.sector); + resume_22: + if (!write_meta_block(meta_old, 22)) + return false; + resume_23: + if (wait_count > 0) + { + wait_state = wait_base+23; + return false; + } + flusher->inflight_meta_sectors.erase(meta_old.sector); + } + } + resume_24: + if (flusher->inflight_meta_sectors.find(meta_new.sector) != flusher->inflight_meta_sectors.end()) + { + wait_state = wait_base+24; + return false; + } + flusher->inflight_meta_sectors.insert(meta_new.sector); + // Modify the new metadata entry + update_metadata_entry(); + // Update clean_db - it must be equal to the metadata entry + update_clean_db(); + // And write metadata entries + resume_25: + if (!write_meta_block(meta_new, 25)) + return false; + resume_26: + if (wait_count > 0) + { + wait_state = wait_base+26; + return false; + } + flusher->inflight_meta_sectors.erase(meta_new.sector); + // Done, free all buffers + free_buffers(); + // And sync metadata (in batches - not per each operation!) + resume_27: + resume_28: + resume_29: + if (!fsync_batch(true, 27)) + return false; + // Free the data block only when metadata is synced + free_data_blocks(); + // Erase dirty_db entries + bs->erase_dirty(dirty_start, std::next(dirty_end), clean_loc); +#ifdef BLOCKSTORE_DEBUG + printf("Flushed %jx:%jx v%ju (%d copies, wr:%d, del:%d), %jd left\n", cur.oid.inode, cur.oid.stripe, cur.version, + copy_count, has_writes, has_delete, flusher->flush_queue.size()); +#endif + release_oid: + repeat_it = flusher->sync_to_repeat.find(cur.oid); + if (repeat_it != flusher->sync_to_repeat.end() && repeat_it->second > cur.version) + { + // Requeue version + flusher->unshift_flush({ .oid = cur.oid, .version = repeat_it->second }, false); + } + flusher->sync_to_repeat.erase(repeat_it); + trim_journal: + // Clear unused part of the journal every flushes + if (bs->journal_trim_interval && !((++flusher->journal_trim_counter) % bs->journal_trim_interval) || + flusher->trim_wanted > 0) + { + resume_30: + resume_31: + resume_32: + resume_33: + resume_34: + if (!trim_journal(30)) + return false; + } + // All done + flusher->active_flushers--; + wait_state = 0; + goto resume_0; + } + return true; +} + +void journal_flusher_co::update_metadata_entry() +{ + clean_disk_entry *new_entry = (clean_disk_entry*)((uint8_t*)meta_new.buf + meta_new.pos*bs->dsk.clean_entry_size); + if (new_entry->oid.inode != 0 && new_entry->oid != cur.oid) + { + printf( + has_delete + ? "Fatal error (metadata corruption or bug): tried to delete metadata entry %ju (%jx:%jx v%ju) while deleting %jx:%jx v%ju\n" + : "Fatal error (metadata corruption or bug): tried to overwrite non-zero metadata entry %ju (%jx:%jx v%ju) with %jx:%jx v%ju\n", + clean_loc / bs->dsk.data_block_size, new_entry->oid.inode, new_entry->oid.stripe, + new_entry->version, cur.oid.inode, cur.oid.stripe, cur.version + ); + exit(1); + } + if (has_delete) + { + // Zero out the new metadata entry + memset((uint8_t*)meta_new.buf + meta_new.pos*bs->dsk.clean_entry_size, 0, bs->dsk.clean_entry_size); + } + else + { + // Set initial internal bitmap bits from the big write + if (clean_init_bitmap) + { + memset(new_clean_bitmap, 0, bs->dsk.clean_entry_bitmap_size); + bitmap_set(new_clean_bitmap, clean_bitmap_offset, clean_bitmap_len, bs->dsk.bitmap_granularity); + } + for (auto it = v.begin(); it != v.end(); it++) + { + // Set internal bitmap bits from small writes + if (it->copy_flags == COPY_BUF_JOURNAL || it->copy_flags == (COPY_BUF_JOURNAL|COPY_BUF_COALESCED)) + bitmap_set(new_clean_bitmap, it->offset, it->len, bs->dsk.bitmap_granularity); + } + // Copy latest external bitmap/attributes + { + void *dyn_ptr = bs->alloc_dyn_data + ? (uint8_t*)dirty_end->second.dyn_data+sizeof(int) : (uint8_t*)&dirty_end->second.dyn_data; + memcpy(new_clean_bitmap + bs->dsk.clean_entry_bitmap_size, dyn_ptr, bs->dsk.clean_entry_bitmap_size); + } + // Copy initial (big_write) data checksums + if (bs->dsk.csum_block_size && clean_init_bitmap) + { + uint8_t *new_clean_data_csum = new_clean_bitmap + 2*bs->dsk.clean_entry_bitmap_size; + // big_write partial checksums are calculated from a padded csum_block_size, we can just copy them + memset(new_clean_data_csum, 0, bs->dsk.data_block_size / bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF)); + uint64_t dyn_size = bs->dsk.dirty_dyn_size(clean_bitmap_offset, clean_bitmap_len); + uint32_t *csums = (uint32_t*)(clean_init_dyn_ptr + bs->dsk.clean_entry_bitmap_size); + memcpy(new_clean_data_csum + clean_bitmap_offset / bs->dsk.csum_block_size * (bs->dsk.data_csum_type & 0xFF), + csums, dyn_size - bs->dsk.clean_entry_bitmap_size); + } + // Calculate or copy small_write checksums + uint32_t *new_data_csums = (uint32_t*)(new_clean_bitmap + 2*bs->dsk.clean_entry_bitmap_size); + if (bs->dsk.csum_block_size) + calc_block_checksums(new_data_csums, false); + // Update entry + new_entry->oid = cur.oid; + new_entry->version = cur.version; + if (!bs->inmemory_meta) + { + auto inmem_bmp = (uint8_t*)bs->clean_bitmaps + (clean_loc / bs->dsk.data_block_size)*2*bs->dsk.clean_entry_bitmap_size; + memcpy(inmem_bmp, new_clean_bitmap, 2*bs->dsk.clean_entry_bitmap_size); + } + if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_V2) + { + // Calculate metadata entry checksum + uint32_t *new_entry_csum = (uint32_t*)((uint8_t*)new_entry + bs->dsk.clean_entry_size - 4); + *new_entry_csum = crc32c(0, new_entry, bs->dsk.clean_entry_size - 4); + } + } +} + +void journal_flusher_co::free_buffers() +{ + if (!bs->inmemory_meta) + { + meta_new.it->second.usage_count--; + if (meta_new.it->second.usage_count == 0) + { + free(meta_new.it->second.buf); + flusher->meta_sectors.erase(meta_new.it); + } + if (old_clean_loc != UINT64_MAX && old_clean_loc != clean_loc) + { + meta_old.it->second.usage_count--; + if (meta_old.it->second.usage_count == 0) + { + free(meta_old.it->second.buf); + flusher->meta_sectors.erase(meta_old.it); + } + } + } + for (auto it = v.begin(); it != v.end(); it++) + { + // Free it if it's not taken from the journal + if (it->buf && (it->copy_flags == COPY_BUF_JOURNAL || (it->copy_flags & COPY_BUF_CSUM_FILL)) && + (!bs->journal.inmemory || it->buf < bs->journal.buffer || it->buf >= (uint8_t*)bs->journal.buffer + bs->journal.len)) + { + free(it->buf); + } + } + v.clear(); +} + +bool journal_flusher_co::write_meta_block(flusher_meta_write_t & meta_block, int wait_base) +{ + if (wait_state == wait_base) + goto resume_0; + await_sqe(0); + data->iov = (struct iovec){ meta_block.buf, (size_t)bs->dsk.meta_block_size }; + data->callback = simple_callback_w; + io_uring_prep_writev( + sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bs->dsk.meta_block_size + meta_block.sector + ); + wait_count++; + return true; +} + +// Punch holes in incomplete checksum blocks +bool journal_flusher_co::clear_incomplete_csum_block_bits(int wait_base) +{ + if (wait_state == wait_base) goto resume_0; + else if (wait_state == wait_base+1) goto resume_1; + else if (wait_state == wait_base+2) goto resume_2; + else if (wait_state == wait_base+3) goto resume_3; + else if (wait_state == wait_base+4) goto resume_4; + else if (wait_state == wait_base+5) goto resume_5; + else if (wait_state == wait_base+6) goto resume_6; + else if (wait_state == wait_base+7) goto resume_7; + else if (wait_state == wait_base+8) goto resume_8; + cleared_incomplete = false; + for (auto it = v.begin(); it != v.end(); it++) + { + if ((it->copy_flags == COPY_BUF_JOURNAL || it->copy_flags == (COPY_BUF_JOURNAL|COPY_BUF_COALESCED)) && + bitmap_check(new_clean_bitmap, it->offset, it->len, bs->dsk.bitmap_granularity)) + { + cleared_incomplete = true; + break; + } + } + if (cleared_incomplete) + { + // This modification may only happen in place + assert(old_clean_loc == clean_loc); + // Wait for data writes and metadata reads + resume_0: + resume_1: + if (!wait_meta_reads(wait_base+0)) + return false; + resume_2: + if (flusher->inflight_meta_sectors.find(meta_new.sector) != flusher->inflight_meta_sectors.end()) + { + wait_state = wait_base+2; + return false; + } + flusher->inflight_meta_sectors.insert(meta_new.sector); + resume_3: + if (wait_journal_count > 0) + { + wait_state = wait_base+3; + return false; + } + // Verify data checksums + for (i = v.size()-1; i >= 0 && (v[i].copy_flags & COPY_BUF_CSUM_FILL); i--) + { + // If we encounter bad checksums during flush, we still update the bad block, + // but intentionally mangle checksums to avoid hiding the corruption. + iovec iov = { .iov_base = v[i].buf, .iov_len = (size_t)v[i].len }; + if (!(v[i].copy_flags & COPY_BUF_JOURNAL)) + { + assert(!(v[i].offset % bs->dsk.csum_block_size)); + assert(!(v[i].len % bs->dsk.csum_block_size)); + bs->verify_padded_checksums(new_clean_bitmap, new_clean_bitmap + 2*bs->dsk.clean_entry_bitmap_size, + v[i].offset, &iov, 1, [&](uint32_t bad_block, uint32_t calc_csum, uint32_t stored_csum) + { + printf("Checksum mismatch in object %jx:%jx v%ju in data area at offset 0x%jx+0x%x: got %08x, expected %08x\n", + cur.oid.inode, cur.oid.stripe, old_clean_ver, old_clean_loc, bad_block, calc_csum, stored_csum); + for (uint32_t j = 0; j < bs->dsk.csum_block_size; j += bs->dsk.bitmap_granularity) + { + // Simplest method of mangling: flip one byte in every sector + ((uint8_t*)v[i].buf)[j+bad_block-v[i].offset] ^= 0xff; + } + }); + } + else + { + bs->verify_journal_checksums(v[i].csum_buf, v[i].offset, &iov, 1, [&](uint32_t bad_block, uint32_t calc_csum, uint32_t stored_csum) + { + printf("Checksum mismatch in object %jx:%jx v%ju in journal at offset 0x%jx+0x%x (block offset 0x%jx): got %08x, expected %08x\n", + cur.oid.inode, cur.oid.stripe, old_clean_ver, + v[i].disk_offset, bad_block, v[i].offset, calc_csum, stored_csum); + bad_block += (v[i].offset/bs->dsk.csum_block_size) * bs->dsk.csum_block_size; + uint32_t bad_block_end = bad_block + bs->dsk.csum_block_size + (v[i].offset/bs->dsk.csum_block_size) * bs->dsk.csum_block_size; + if (bad_block < v[i].offset) + bad_block = v[i].offset; + if (bad_block_end > v[i].offset+v[i].len) + bad_block_end = v[i].offset+v[i].len; + bad_block -= v[i].offset; + bad_block_end -= v[i].offset; + for (uint32_t j = bad_block; j < bad_block_end; j += bs->dsk.bitmap_granularity) + { + // Simplest method of mangling: flip one byte in every sector + ((uint8_t*)v[i].buf)[j] ^= 0xff; + } + }); + } + } + { + clean_disk_entry *new_entry = (clean_disk_entry*)((uint8_t*)meta_new.buf + meta_new.pos*bs->dsk.clean_entry_size); + if (new_entry->oid != cur.oid) + { + printf( + "Fatal error (metadata corruption or bug): tried to make holes in %ju (%jx:%jx v%ju) with %jx:%jx v%ju\n", + clean_loc / bs->dsk.data_block_size, new_entry->oid.inode, new_entry->oid.stripe, + new_entry->version, cur.oid.inode, cur.oid.stripe, cur.version + ); + } + assert(new_entry->oid == cur.oid); + // Actually clear bits + for (auto it = v.begin(); it != v.end(); it++) + { + if (it->copy_flags == COPY_BUF_JOURNAL || it->copy_flags == (COPY_BUF_JOURNAL|COPY_BUF_COALESCED)) + bitmap_clear(new_clean_bitmap, it->offset, it->len, bs->dsk.bitmap_granularity); + } + // Calculate block checksums with new holes + uint32_t *new_data_csums = (uint32_t*)(new_clean_bitmap + 2*bs->dsk.clean_entry_bitmap_size); + calc_block_checksums(new_data_csums, true); + if (!bs->inmemory_meta) + { + auto inmem_bmp = (uint8_t*)bs->clean_bitmaps + (clean_loc / bs->dsk.data_block_size)*2*bs->dsk.clean_entry_bitmap_size; + memcpy(inmem_bmp, new_clean_bitmap, 2*bs->dsk.clean_entry_bitmap_size); + } + if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_V2) + { + // calculate metadata entry checksum + uint32_t *new_entry_csum = (uint32_t*)((uint8_t*)new_entry + bs->dsk.clean_entry_size - 4); + *new_entry_csum = crc32c(0, new_entry, bs->dsk.clean_entry_size - 4); + } + } + // Write and fsync the modified metadata entry + resume_4: + if (!write_meta_block(meta_new, wait_base+4)) + return false; + resume_5: + if (wait_count > 0) + { + wait_state = wait_base+5; + return false; + } + flusher->inflight_meta_sectors.erase(meta_new.sector); + resume_6: + resume_7: + resume_8: + if (!fsync_batch(true, wait_base+6)) + return false; + } + return true; +} + +void journal_flusher_co::calc_block_checksums(uint32_t *new_data_csums, bool skip_overwrites) +{ + uint64_t block_offset = 0; + uint32_t block_done = 0; + uint32_t block_csum = 0; + for (auto it = v.begin(); it != v.end(); it++) + { + if (it->copy_flags & COPY_BUF_CSUM_FILL) + break; + if (block_done == 0) + { + // `v` should contain aligned items, possibly split into pieces + assert(!(it->offset % bs->dsk.csum_block_size)); + block_offset = it->offset; + } + bool zero = (it->copy_flags & COPY_BUF_ZERO) || (skip_overwrites && (it->copy_flags & COPY_BUF_JOURNAL)); + auto len = it->len; + while ((block_done+len) >= bs->dsk.csum_block_size) + { + if (!skip_overwrites && !block_done && it->csum_buf) + { + // We may take existing checksums if an overwrite contains a full block + auto full_csum_offset = (it->offset+it->len-len+bs->dsk.csum_block_size-1) / bs->dsk.csum_block_size + - it->offset / bs->dsk.csum_block_size; + auto full_csum_count = len/bs->dsk.csum_block_size; + memcpy(new_data_csums + block_offset/bs->dsk.csum_block_size, + it->csum_buf + full_csum_offset*4, full_csum_count*4); + len -= full_csum_count*bs->dsk.csum_block_size; + block_offset += full_csum_count*bs->dsk.csum_block_size; + } + else + { + auto cur_len = bs->dsk.csum_block_size-block_done; + block_csum = zero + ? crc32c_pad(block_csum, NULL, 0, cur_len, 0) + : crc32c(block_csum, (uint8_t*)it->buf+(it->len-len), cur_len); + new_data_csums[block_offset / bs->dsk.csum_block_size] = block_csum; + block_csum = 0; + block_done = 0; + block_offset += bs->dsk.csum_block_size; + len -= cur_len; + } + } + if (len > 0) + { + block_csum = zero + ? crc32c_pad(block_csum, NULL, 0, len, 0) + : crc32c(block_csum, (uint8_t*)it->buf+(it->len-len), len); + block_done += len; + } + } + // `v` should contain aligned items, possibly split into pieces + assert(!block_done); +} + +void journal_flusher_co::scan_dirty() +{ + dirty_it = dirty_start = dirty_end; + v.clear(); + copy_count = 0; + clean_loc = UINT64_MAX; + clean_ver = 0; + has_delete = false; + has_writes = false; + skip_copy = false; + clean_init_bitmap = false; + fill_incomplete = false; + read_to_fill_incomplete = 0; + while (1) + { + if (!IS_STABLE(dirty_it->second.state)) + { + char err[1024]; + snprintf( + err, 1024, "BUG: Unexpected dirty_entry %jx:%jx v%ju unstable state during flush: 0x%x", + dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version, dirty_it->second.state + ); + throw std::runtime_error(err); + } + else if (IS_JOURNAL(dirty_it->second.state) && !skip_copy) + { + // Partial dirty overwrite + has_writes = true; + if (dirty_it->second.len != 0) + { + uint64_t blk_begin = 0, blk_end = 0; + uint8_t *blk_buf = NULL; + bs->find_holes( + v, dirty_it->second.offset, dirty_it->second.offset + dirty_it->second.len, + [&](int pos, bool alloc, uint32_t cur_start, uint32_t cur_end) + { + if (alloc) + return 0; + copy_count++; + uint64_t submit_offset = dirty_it->second.location + cur_start - dirty_it->second.offset; + auto it = v.insert(v.begin()+pos, (copy_buffer_t){ + .copy_flags = COPY_BUF_JOURNAL, + .offset = cur_start, + .len = cur_end-cur_start, + .disk_offset = submit_offset, + }); + if (bs->journal.inmemory) + { + // Take it from memory, don't copy it + it->buf = (uint8_t*)bs->journal.buffer + submit_offset; + } + if (bs->dsk.csum_block_size) + { + // FIXME Remove this > sizeof(void*) inline perversion from everywhere. + // I think it doesn't matter but I couldn't stop myself from implementing it :) + uint8_t* dyn_from = (uint8_t*)(bs->alloc_dyn_data + ? (uint8_t*)dirty_it->second.dyn_data+sizeof(int) : (uint8_t*)&dirty_it->second.dyn_data) + + bs->dsk.clean_entry_bitmap_size; + it->csum_buf = dyn_from + (it->offset/bs->dsk.csum_block_size - + dirty_it->second.offset/bs->dsk.csum_block_size) * (bs->dsk.data_csum_type & 0xFF); + if (cur_start % bs->dsk.csum_block_size || cur_end % bs->dsk.csum_block_size) + { + // Small write not aligned for checksums. We may have to pad it + fill_incomplete = true; + if (!bs->journal.inmemory) + { + bs->pad_journal_read(v, *it, dirty_it->second.offset, + dirty_it->second.offset + dirty_it->second.len, dirty_it->second.location, + dyn_from, NULL, cur_start, cur_end-cur_start, blk_begin, blk_end, blk_buf); + } + } + } + return 0; + } + ); + } + } + else if (IS_BIG_WRITE(dirty_it->second.state) && !skip_copy) + { + // There is an unflushed big write. Copy small writes in its position + has_writes = true; + clean_loc = dirty_it->second.location; + clean_ver = dirty_it->first.version; + clean_init_bitmap = true; + clean_bitmap_offset = dirty_it->second.offset; + clean_bitmap_len = dirty_it->second.len; + clean_init_dyn_ptr = bs->alloc_dyn_data + ? (uint8_t*)dirty_it->second.dyn_data+sizeof(int) : (uint8_t*)&dirty_it->second.dyn_data; + skip_copy = true; + } + else if (IS_DELETE(dirty_it->second.state) && !skip_copy) + { + // There is an unflushed delete + has_delete = true; + skip_copy = true; + } + dirty_start = dirty_it; + if (dirty_it == bs->dirty_db.begin()) + { + break; + } + dirty_it--; + if (dirty_it->first.oid != cur.oid) + { + break; + } + } + if (fill_incomplete && !clean_init_bitmap) + { + // Rescan and fill incomplete writes with old data to calculate checksums + if (old_clean_loc == UINT64_MAX) + { + // May happen if the metadata entry is corrupt, but journal isn't + // FIXME: Report corrupted object to the upper layer (OSD) + printf( + "Warning: object %jx:%jx has overwrites, but doesn't have a clean version." + " Metadata is likely corrupted. Dropping object from the DB.\n", + cur.oid.inode, cur.oid.stripe + ); + v.clear(); + has_writes = false; + has_delete = skip_copy = true; + copy_count = 0; + fill_incomplete = false; + read_to_fill_incomplete = 0; + return; + } + uint8_t *bmp_ptr = bs->get_clean_entry_bitmap(old_clean_loc, 0); + uint64_t fulfilled = 0; + int last = v.size()-1; + while (last >= 0 && (v[last].copy_flags & COPY_BUF_CSUM_FILL)) + last--; + read_to_fill_incomplete = bs->fill_partial_checksum_blocks( + v, fulfilled, bmp_ptr, NULL, false, NULL, v[0].offset/bs->dsk.csum_block_size * bs->dsk.csum_block_size, + ((v[last].offset+v[last].len-1) / bs->dsk.csum_block_size + 1) * bs->dsk.csum_block_size + ); + } + else if (fill_incomplete && clean_init_bitmap) + { + // If we actually have partial checksum block overwrites AND a new clean_loc + // at the same time then we can't use our fancy checksum block mutation algorithm. + // So in this case we'll have to first flush the clean write separately. + while (!IS_BIG_WRITE(dirty_end->second.state)) + { + assert(dirty_end != bs->dirty_db.begin()); + dirty_end--; + } + flusher->enqueue_flush(cur); + cur.version = dirty_end->first.version; +#ifdef BLOCKSTORE_DEBUG + printf("Partial checksum block overwrites found - rewinding flush back to %jx:%jx v%ju\n", cur.oid.inode, cur.oid.stripe, cur.version); +#endif + v.clear(); + copy_count = 0; + fill_incomplete = false; + read_to_fill_incomplete = 0; + } +} + +bool journal_flusher_co::read_dirty(int wait_base) +{ + if (wait_state == wait_base) goto resume_0; + else if (wait_state == wait_base+1) goto resume_1; + wait_count = wait_journal_count = 0; + if (bs->journal.inmemory && !read_to_fill_incomplete) + { + // Happy path: nothing to read :) + return true; + } + for (i = 1; i <= v.size() && (v[v.size()-i].copy_flags & COPY_BUF_CSUM_FILL); i++) + { + if (v[v.size()-i].copy_flags & COPY_BUF_JOURNAL) + continue; + // Read old data from disk to calculate checksums + await_sqe(0); + auto & vi = v[v.size()-i]; + assert(vi.len != 0); + vi.buf = memalign_or_die(MEM_ALIGNMENT, vi.len); + data->iov = (struct iovec){ vi.buf, (size_t)vi.len }; + data->callback = simple_callback_r; + io_uring_prep_readv( + sqe, bs->dsk.data_fd, &data->iov, 1, bs->dsk.data_offset + old_clean_loc + vi.offset + ); + wait_count++; + bs->find_holes(v, vi.offset, vi.offset+vi.len, [this, buf = (uint8_t*)vi.buf-vi.offset](int pos, bool alloc, uint32_t cur_start, uint32_t cur_end) + { + if (!alloc) + { + v.insert(v.begin()+pos, (copy_buffer_t){ + .copy_flags = COPY_BUF_DATA, + .offset = cur_start, + .len = cur_end-cur_start, + .buf = buf+cur_start, + }); + return 1; + } + return 0; + }); + } + if (!bs->journal.inmemory) + { + for (i = 0; i < v.size(); i++) + { + if (v[i].copy_flags == COPY_BUF_JOURNAL || + v[i].copy_flags == (COPY_BUF_JOURNAL | COPY_BUF_CSUM_FILL)) + { + // Read journal data from disk + if (!v[i].buf) + v[i].buf = memalign_or_die(MEM_ALIGNMENT, v[i].len); + await_sqe(1); + data->iov = (struct iovec){ v[i].buf, (size_t)v[i].len }; + data->callback = simple_callback_rj; + io_uring_prep_readv( + sqe, bs->dsk.journal_fd, &data->iov, 1, bs->journal.offset + v[i].disk_offset + ); + wait_journal_count++; + } + } + } + return true; +} + +bool journal_flusher_co::modify_meta_do_reads(int wait_base) +{ + if (wait_state == wait_base) goto resume_0; + else if (wait_state == wait_base+1) goto resume_1; +resume_0: + if (!modify_meta_read(clean_loc, meta_new, wait_base+0)) + return false; + new_clean_bitmap = (uint8_t*)meta_new.buf + meta_new.pos*bs->dsk.clean_entry_size + sizeof(clean_disk_entry); + if (old_clean_loc != UINT64_MAX && old_clean_loc != clean_loc) + { + resume_1: + if (!modify_meta_read(old_clean_loc, meta_old, wait_base+1)) + return false; + } + else + meta_old.submitted = false; + return true; +} + +bool journal_flusher_co::wait_meta_reads(int wait_base) +{ + if (wait_state == wait_base) goto resume_0; + else if (wait_state == wait_base+1) goto resume_1; +resume_0: + if (wait_count > 0) + { + wait_state = wait_base+0; + return false; + } + // Our own reads completed + if (meta_new.submitted) + { + meta_new.it->second.state = META_BLOCK_READ; + bs->ringloop->wakeup(); + } + if (meta_old.submitted) + { + meta_old.it->second.state = META_BLOCK_READ; + bs->ringloop->wakeup(); + } +resume_1: + if (!bs->inmemory_meta && (meta_new.it->second.state == META_BLOCK_UNREAD || + (old_clean_loc != UINT64_MAX && old_clean_loc != clean_loc) && meta_old.it->second.state == META_BLOCK_UNREAD)) + { + // Metadata block is being read by another coroutine + wait_state = wait_base+1; + return false; + } + // All reads completed + return true; +} + +bool journal_flusher_co::modify_meta_read(uint64_t meta_loc, flusher_meta_write_t &wr, int wait_base) +{ + if (wait_state == wait_base) + goto resume_0; + // We must check if the same sector is already in memory if we don't keep all metadata in memory all the time. + // And yet another option is to use LSM trees for metadata, but it sophisticates everything a lot, + // so I'll avoid it as long as I can. + wr.submitted = false; + wr.sector = ((meta_loc / bs->dsk.data_block_size) / (bs->dsk.meta_block_size / bs->dsk.clean_entry_size)) * bs->dsk.meta_block_size; + wr.pos = ((meta_loc / bs->dsk.data_block_size) % (bs->dsk.meta_block_size / bs->dsk.clean_entry_size)); + if (bs->inmemory_meta) + { + wr.buf = (uint8_t*)bs->metadata_buffer + wr.sector; + return true; + } + wr.it = flusher->meta_sectors.find(wr.sector); + if (wr.it == flusher->meta_sectors.end()) + { + // Not in memory yet, read it + wr.buf = memalign_or_die(MEM_ALIGNMENT, bs->dsk.meta_block_size); + wr.it = flusher->meta_sectors.emplace(wr.sector, (meta_sector_t){ + .offset = wr.sector, + .len = bs->dsk.meta_block_size, + .state = META_BLOCK_UNREAD, // 0 = not read yet + .buf = wr.buf, + .usage_count = 1, + }).first; + await_sqe(0); + data->iov = (struct iovec){ wr.it->second.buf, (size_t)bs->dsk.meta_block_size }; + data->callback = simple_callback_r; + wr.submitted = true; + io_uring_prep_readv( + sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bs->dsk.meta_block_size + wr.sector + ); + wait_count++; + } + else + { + wr.buf = wr.it->second.buf; + wr.it->second.usage_count++; + } + return true; +} + +void journal_flusher_co::update_clean_db() +{ + auto & clean_db = bs->clean_db_shard(cur.oid); + if (has_delete) + { + clean_db.erase(cur.oid); + } + else + { + clean_db[cur.oid] = { + .version = cur.version, + .location = clean_loc, + }; + } +} + +void journal_flusher_co::free_data_blocks() +{ + if (old_clean_loc != UINT64_MAX && old_clean_loc != clean_loc) + { + auto uo_it = bs->used_clean_objects.find(old_clean_loc); + bool used = uo_it != bs->used_clean_objects.end(); +#ifdef BLOCKSTORE_DEBUG + printf("%s block %ju from %jx:%jx v%ju (new location is %ju)\n", + used ? "Postpone free" : "Free", + old_clean_loc / bs->dsk.data_block_size, + cur.oid.inode, cur.oid.stripe, cur.version, + clean_loc / bs->dsk.data_block_size); +#endif + if (used) + uo_it->second.was_freed = true; + else + bs->data_alloc->set(old_clean_loc / bs->dsk.data_block_size, false); + } + if (has_delete) + { + assert(clean_loc == old_clean_loc); + auto uo_it = bs->used_clean_objects.find(old_clean_loc); + bool used = uo_it != bs->used_clean_objects.end(); +#ifdef BLOCKSTORE_DEBUG + printf("%s block %ju from %jx:%jx v%ju (delete)\n", + used ? "Postpone free" : "Free", + old_clean_loc / bs->dsk.data_block_size, + cur.oid.inode, cur.oid.stripe, cur.version); +#endif + if (used) + uo_it->second.was_freed = true; + else + bs->data_alloc->set(old_clean_loc / bs->dsk.data_block_size, false); + } +} + +bool journal_flusher_co::fsync_batch(bool fsync_meta, int wait_base) +{ + if (wait_state == wait_base) goto resume_0; + else if (wait_state == wait_base+1) goto resume_1; + else if (wait_state == wait_base+2) goto resume_2; + if (!(fsync_meta ? bs->disable_meta_fsync : bs->disable_data_fsync)) + { + cur_sync = flusher->syncs.end(); + while (cur_sync != flusher->syncs.begin()) + { + cur_sync--; + if (cur_sync->fsync_meta == fsync_meta && cur_sync->state == 0) + { + goto sync_found; + } + } + cur_sync = flusher->syncs.emplace(flusher->syncs.end(), (flusher_sync_t){ + .fsync_meta = fsync_meta, + .ready_count = 0, + .state = 0, + }); + sync_found: + cur_sync->ready_count++; + flusher->syncing_flushers++; + resume_1: + if (!cur_sync->state) + { + if (flusher->syncing_flushers >= flusher->active_flushers || !flusher->flush_queue.size()) + { + // Sync batch is ready. Do it. + await_sqe(0); + data->iov = { 0 }; + data->callback = simple_callback_w; + io_uring_prep_fsync(sqe, fsync_meta ? bs->dsk.meta_fd : bs->dsk.data_fd, IORING_FSYNC_DATASYNC); + cur_sync->state = 1; + wait_count++; + resume_2: + if (wait_count > 0) + { + wait_state = wait_base+2; + return false; + } + // Sync completed. All previous coroutines waiting for it must be resumed + cur_sync->state = 2; + bs->ringloop->wakeup(); + } + else + { + // Wait until someone else sends and completes a sync. + wait_state = wait_base+1; + return false; + } + } + flusher->syncing_flushers--; + cur_sync->ready_count--; + if (cur_sync->ready_count == 0) + { + flusher->syncs.erase(cur_sync); + } + } + return true; +} + +bool journal_flusher_co::trim_journal(int wait_base) +{ + if (wait_state == wait_base) goto resume_0; + else if (wait_state == wait_base+1) goto resume_1; + else if (wait_state == wait_base+2) goto resume_2; + else if (wait_state == wait_base+3) goto resume_3; + else if (wait_state == wait_base+4) goto resume_4; + new_trim_pos = bs->journal.get_trim_pos(); + if (new_trim_pos != bs->journal.used_start) + { + resume_0: + // Wait for other coroutines trimming the journal, if any + if (flusher->trimming) + { + wait_state = wait_base+0; + return false; + } + flusher->trimming = true; + // Recheck the position with the "lock" taken + new_trim_pos = bs->journal.get_trim_pos(); + if (new_trim_pos != bs->journal.used_start) + { + // First update journal "superblock" and only then update in memory + await_sqe(1); + *((journal_entry_start*)flusher->journal_superblock) = { + .crc32 = 0, + .magic = JOURNAL_MAGIC, + .type = JE_START, + .size = ((!bs->dsk.data_csum_type && ((journal_entry_start*)flusher->journal_superblock)->version == JOURNAL_VERSION_V1) + ? (uint32_t)JE_START_V1_SIZE : (uint32_t)JE_START_V2_SIZE), + .reserved = 0, + .journal_start = new_trim_pos, + .version = (uint64_t)(!bs->dsk.data_csum_type && ((journal_entry_start*)flusher->journal_superblock)->version == JOURNAL_VERSION_V1 + ? JOURNAL_VERSION_V1 : JOURNAL_VERSION_V2), + .data_csum_type = bs->dsk.data_csum_type, + .csum_block_size = bs->dsk.csum_block_size, + }; + ((journal_entry_start*)flusher->journal_superblock)->crc32 = je_crc32((journal_entry*)flusher->journal_superblock); + data->iov = (struct iovec){ flusher->journal_superblock, (size_t)bs->dsk.journal_block_size }; + data->callback = simple_callback_w; + io_uring_prep_writev(sqe, bs->dsk.journal_fd, &data->iov, 1, bs->journal.offset); + wait_count++; + resume_2: + if (wait_count > 0) + { + wait_state = wait_base+2; + return false; + } + if (!bs->disable_journal_fsync) + { + await_sqe(3); + io_uring_prep_fsync(sqe, bs->dsk.journal_fd, IORING_FSYNC_DATASYNC); + data->iov = { 0 }; + data->callback = simple_callback_w; + wait_count++; + resume_4: + if (wait_count > 0) + { + wait_state = wait_base+4; + return false; + } + } + if (new_trim_pos < bs->journal.used_start + ? (bs->journal.dirty_start >= bs->journal.used_start || bs->journal.dirty_start < new_trim_pos) + : (bs->journal.dirty_start >= bs->journal.used_start && bs->journal.dirty_start < new_trim_pos)) + { + bs->journal.dirty_start = new_trim_pos; + } + bs->journal.used_start = new_trim_pos; +#ifdef BLOCKSTORE_DEBUG + printf("Journal trimmed to %08jx (next_free=%08jx dirty_start=%08jx)\n", bs->journal.used_start, bs->journal.next_free, bs->journal.dirty_start); +#endif + if (bs->journal.flush_journal && !flusher->flush_queue.size()) + { + assert(bs->journal.used_start == bs->journal.next_free); + printf("Journal flushed\n"); + exit(0); + } + } + flusher->journal_trim_counter = 0; + flusher->trimming = false; + } + return true; +} diff --git a/src/blockstore/v1/flush.h b/src/blockstore/v1/flush.h new file mode 100644 index 00000000..3bf18f48 --- /dev/null +++ b/src/blockstore/v1/flush.h @@ -0,0 +1,134 @@ +// Copyright (c) Vitaliy Filippov, 2019+ +// License: VNPL-1.1 (see README.md for details) + +struct copy_buffer_t +{ + int copy_flags; + uint64_t offset, len, disk_offset; + uint64_t journal_sector; // only for reads: sector+1 if used and !journal.inmemory, otherwise 0 + void *buf; + uint8_t *csum_buf; + int *dyn_data; +}; + +struct meta_sector_t +{ + uint64_t offset, len; + int state; + void *buf; + int usage_count; +}; + +struct flusher_sync_t +{ + bool fsync_meta; + int ready_count; + int state; +}; + +struct flusher_meta_write_t +{ + uint64_t sector, pos; + bool submitted; + void *buf; + std::map::iterator it; +}; + +class journal_flusher_t; + +// Journal flusher coroutine +class journal_flusher_co +{ + blockstore_impl_t *bs; + journal_flusher_t *flusher; + int wait_state, wait_count, wait_journal_count; + struct io_uring_sqe *sqe; + struct ring_data_t *data; + + std::list::iterator cur_sync; + + obj_ver_id cur; + std::map::iterator dirty_it, dirty_start, dirty_end; + std::map::iterator repeat_it; + std::function simple_callback_r, simple_callback_rj, simple_callback_w; + + bool try_trim = false; + bool skip_copy, has_delete, has_writes; + std::vector v; + std::vector::iterator it; + int i; + bool fill_incomplete, cleared_incomplete; + int read_to_fill_incomplete; + int copy_count; + uint64_t clean_loc, clean_ver, old_clean_loc, old_clean_ver; + flusher_meta_write_t meta_old, meta_new; + bool clean_init_bitmap; + uint64_t clean_bitmap_offset, clean_bitmap_len; + uint8_t *clean_init_dyn_ptr; + uint8_t *new_clean_bitmap; + + uint64_t new_trim_pos; + + friend class journal_flusher_t; + void scan_dirty(); + bool read_dirty(int wait_base); + bool modify_meta_do_reads(int wait_base); + bool wait_meta_reads(int wait_base); + bool modify_meta_read(uint64_t meta_loc, flusher_meta_write_t &wr, int wait_base); + bool clear_incomplete_csum_block_bits(int wait_base); + void calc_block_checksums(uint32_t *new_data_csums, bool skip_overwrites); + void update_metadata_entry(); + bool write_meta_block(flusher_meta_write_t & meta_block, int wait_base); + void update_clean_db(); + void free_data_blocks(); + bool fsync_batch(bool fsync_meta, int wait_base); + bool trim_journal(int wait_base); + void free_buffers(); +public: + journal_flusher_co(); + bool loop(); +}; + +// Journal flusher itself +class journal_flusher_t +{ + int trim_wanted = 0; + bool dequeuing; + int min_flusher_count, max_flusher_count, cur_flusher_count, target_flusher_count; + int flusher_start_threshold; + journal_flusher_co *co; + blockstore_impl_t *bs; + friend class journal_flusher_co; + + int journal_trim_counter; + bool trimming; + void* journal_superblock; + + int active_flushers; + int syncing_flushers; + std::list syncs; + std::map sync_to_repeat; + + std::map meta_sectors; + std::deque flush_queue; + std::unordered_map flush_versions; + std::unordered_set inflight_meta_sectors; + + bool try_find_older(std::map::iterator & dirty_end, obj_ver_id & cur); + bool try_find_other(std::map::iterator & dirty_end, obj_ver_id & cur); + +public: + journal_flusher_t(blockstore_impl_t *bs); + ~journal_flusher_t(); + void loop(); + bool is_trim_wanted() { return trim_wanted; } + bool is_active(); + void mark_trim_possible(); + void request_trim(); + void release_trim(); + void enqueue_flush(obj_ver_id oid); + void unshift_flush(obj_ver_id oid, bool force); + void remove_flush(object_id oid); + void dump_diagnostics(); + bool is_mutated(uint64_t clean_loc); +}; diff --git a/src/blockstore/v1/impl.cpp b/src/blockstore/v1/impl.cpp new file mode 100644 index 00000000..2b76e595 --- /dev/null +++ b/src/blockstore/v1/impl.cpp @@ -0,0 +1,806 @@ +// Copyright (c) Vitaliy Filippov, 2019+ +// License: VNPL-1.1 (see README.md for details) + +#include "blockstore_impl.h" +#include "blockstore_internal.h" + +blockstore_impl_t::blockstore_impl_t(blockstore_config_t & config, ring_loop_t *ringloop, timerfd_manager_t *tfd) +{ + assert(sizeof(blockstore_op_private_t) <= BS_OP_PRIVATE_DATA_SIZE); + this->tfd = tfd; + this->ringloop = ringloop; + ring_consumer.loop = [this]() { loop(); }; + ringloop->register_consumer(&ring_consumer); + initialized = 0; + parse_config(config, true); + try + { + dsk.open_data(); + dsk.open_meta(); + dsk.open_journal(); + calc_lengths(); + alloc_dyn_data = dsk.clean_dyn_size > sizeof(void*) || dsk.csum_block_size > 0; + zero_object = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.data_block_size); + data_alloc = new allocator_t(dsk.block_count); + } + catch (std::exception & e) + { + dsk.close_all(); + throw; + } + flusher = new journal_flusher_t(this); +} + +blockstore_impl_t::~blockstore_impl_t() +{ + delete data_alloc; + delete flusher; + if (zero_object) + free(zero_object); + ringloop->unregister_consumer(&ring_consumer); + dsk.close_all(); + if (metadata_buffer) + free(metadata_buffer); + if (clean_bitmaps) + free(clean_bitmaps); +} + +bool blockstore_impl_t::is_started() +{ + return initialized == 10; +} + +bool blockstore_impl_t::is_stalled() +{ + return queue_stall; +} + +// main event loop - produce requests +void blockstore_impl_t::loop() +{ + // FIXME: initialized == 10 is ugly + if (initialized != 10) + { + // read metadata, then journal + if (initialized == 0) + { + metadata_init_reader = new blockstore_init_meta(this); + initialized = 1; + } + if (initialized == 1) + { + int res = metadata_init_reader->loop(); + if (!res) + { + delete metadata_init_reader; + metadata_init_reader = NULL; + journal_init_reader = new blockstore_init_journal(this); + initialized = 2; + } + } + if (initialized == 2) + { + int res = journal_init_reader->loop(); + if (!res) + { + delete journal_init_reader; + journal_init_reader = NULL; + initialized = 3; + ringloop->wakeup(); + } + } + if (initialized == 3) + { + if (!readonly && dsk.discard_on_start) + dsk.trim_data(data_alloc); + if (journal.flush_journal) + initialized = 4; + else + initialized = 10; + } + if (initialized == 4) + { + if (readonly) + { + printf("Can't flush the journal in readonly mode\n"); + exit(1); + } + flusher->loop(); + ringloop->submit(); + } + } + else + { + // try to submit ops + unsigned initial_ring_space = ringloop->space_left(); + // has_writes == 0 - no writes before the current queue item + // has_writes == 1 - some writes in progress + // has_writes == 2 - tried to submit some writes, but failed + int has_writes = 0, op_idx = 0, new_idx = 0; + for (; op_idx < submit_queue.size(); op_idx++, new_idx++) + { + auto op = submit_queue[op_idx]; + submit_queue[new_idx] = op; + // FIXME: This needs some simplification + // Writes should not block reads if the ring is not full and reads don't depend on them + // In all other cases we should stop submission + if (PRIV(op)->wait_for) + { + check_wait(op); + if (PRIV(op)->wait_for == WAIT_SQE) + { + break; + } + else if (PRIV(op)->wait_for) + { + if (op->opcode == BS_OP_WRITE || op->opcode == BS_OP_WRITE_STABLE || op->opcode == BS_OP_DELETE) + { + has_writes = 2; + } + continue; + } + } + unsigned prev_sqe_pos = ringloop->save(); + // 0 = can't submit + // 1 = in progress + // 2 = can be removed from queue + int wr_st = 0; + if (op->opcode == BS_OP_READ) + { + wr_st = dequeue_read(op); + } + else if (op->opcode == BS_OP_WRITE || op->opcode == BS_OP_WRITE_STABLE) + { + if (has_writes == 2) + { + // Some writes already could not be submitted + continue; + } + wr_st = dequeue_write(op); + has_writes = wr_st > 0 ? 1 : 2; + } + else if (op->opcode == BS_OP_DELETE) + { + if (has_writes == 2) + { + // Some writes already could not be submitted + continue; + } + wr_st = dequeue_del(op); + has_writes = wr_st > 0 ? 1 : 2; + } + else if (op->opcode == BS_OP_SYNC) + { + // sync only completed writes? + // wait for the data device fsync to complete, then submit journal writes for big writes + // then submit an fsync operation + wr_st = continue_sync(op); + } + else if (op->opcode == BS_OP_STABLE) + { + wr_st = dequeue_stable(op); + } + else if (op->opcode == BS_OP_ROLLBACK) + { + wr_st = dequeue_rollback(op); + } + else if (op->opcode == BS_OP_LIST) + { + // LIST doesn't have to be blocked by previous modifications + process_list(op); + wr_st = 2; + } + if (wr_st == 2) + { + submit_queue[op_idx] = NULL; + new_idx--; + } + if (wr_st == 0) + { + ringloop->restore(prev_sqe_pos); + if (PRIV(op)->wait_for == WAIT_SQE) + { + // ring is full, stop submission + break; + } + else if (PRIV(op)->wait_for == WAIT_JOURNAL) + { + PRIV(op)->wait_detail2 = (unstable_writes.size()+unstable_unsynced); + } + } + } + if (op_idx != new_idx) + { + while (op_idx < submit_queue.size()) + { + submit_queue[new_idx++] = submit_queue[op_idx++]; + } + submit_queue.resize(new_idx); + } + if (!readonly) + { + flusher->loop(); + } + int ret = ringloop->submit(); + if (ret < 0) + { + throw std::runtime_error(std::string("io_uring_submit: ") + strerror(-ret)); + } + for (auto s: journal.submitting_sectors) + { + // Mark journal sector writes as submitted + if (journal.sector_info[s].submit_id) + journal.sector_info[s].written = true; + journal.sector_info[s].submit_id = 0; + } + journal.submitting_sectors.clear(); + if ((initial_ring_space - ringloop->space_left()) > 0) + { + live = true; + } + queue_stall = !live && !ringloop->has_work(); + live = false; + } +} + +bool blockstore_impl_t::is_safe_to_stop() +{ + // It's safe to stop blockstore when there are no in-flight operations, + // no in-progress syncs and flusher isn't doing anything + if (submit_queue.size() > 0 || !readonly && flusher->is_active()) + { + return false; + } + if (unsynced_big_writes.size() > 0 || unsynced_small_writes.size() > 0) + { + if (!readonly && !stop_sync_submitted) + { + // We should sync the blockstore before unmounting + blockstore_op_t *op = new blockstore_op_t; + op->opcode = BS_OP_SYNC; + op->buf = NULL; + op->callback = [](blockstore_op_t *op) + { + delete op; + }; + enqueue_op(op); + stop_sync_submitted = true; + } + return false; + } + return true; +} + +void blockstore_impl_t::check_wait(blockstore_op_t *op) +{ + if (PRIV(op)->wait_for == WAIT_SQE) + { + if (ringloop->sqes_left() < PRIV(op)->wait_detail) + { + // stop submission if there's still no free space +#ifdef BLOCKSTORE_DEBUG + printf("Still waiting for %ju SQE(s)\n", PRIV(op)->wait_detail); +#endif + return; + } + PRIV(op)->wait_for = 0; + } + else if (PRIV(op)->wait_for == WAIT_JOURNAL) + { + if (journal.used_start == PRIV(op)->wait_detail && + (unstable_writes.size()+unstable_unsynced) == PRIV(op)->wait_detail2) + { + // do not submit +#ifdef BLOCKSTORE_DEBUG + printf("Still waiting to flush journal offset %08jx\n", PRIV(op)->wait_detail); +#endif + return; + } + flusher->release_trim(); + PRIV(op)->wait_for = 0; + } + else if (PRIV(op)->wait_for == WAIT_JOURNAL_BUFFER) + { + int next = ((journal.cur_sector + 1) % journal.sector_count); + if (journal.sector_info[next].flush_count > 0 || + journal.sector_info[next].dirty) + { + // do not submit +#ifdef BLOCKSTORE_DEBUG + printf("Still waiting for a journal buffer\n"); +#endif + return; + } + PRIV(op)->wait_for = 0; + } + else if (PRIV(op)->wait_for == WAIT_FREE) + { + if (!data_alloc->get_free_count() && big_to_flush > 0) + { +#ifdef BLOCKSTORE_DEBUG + printf("Still waiting for free space on the data device\n"); +#endif + return; + } + flusher->release_trim(); + PRIV(op)->wait_for = 0; + } + else + { + throw std::runtime_error("BUG: op->wait_for value is unexpected"); + } +} + +void blockstore_impl_t::enqueue_op(blockstore_op_t *op) +{ + if (op->opcode < BS_OP_MIN || op->opcode > BS_OP_MAX || + ((op->opcode == BS_OP_READ || op->opcode == BS_OP_WRITE || op->opcode == BS_OP_WRITE_STABLE) && ( + op->offset >= dsk.data_block_size || + op->len > dsk.data_block_size-op->offset || + (op->len % dsk.disk_alignment) + )) || + readonly && op->opcode != BS_OP_READ && op->opcode != BS_OP_LIST) + { + // Basic verification not passed + op->retval = -EINVAL; + ringloop->set_immediate([op]() { std::function(op->callback)(op); }); + return; + } + if ((op->opcode == BS_OP_WRITE || op->opcode == BS_OP_WRITE_STABLE || op->opcode == BS_OP_DELETE) && !enqueue_write(op)) + { + ringloop->set_immediate([op]() { std::function(op->callback)(op); }); + return; + } + if (op->opcode == BS_OP_SYNC) + { + unsynced_queued_ops = 0; + } + init_op(op); + submit_queue.push_back(op); + ringloop->wakeup(); +} + +void blockstore_impl_t::init_op(blockstore_op_t *op) +{ + // Call constructor without allocating memory. We'll call destructor before returning op back + new ((void*)op->private_data) blockstore_op_private_t; + PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 0; + PRIV(op)->wait_for = 0; + PRIV(op)->op_state = 0; + PRIV(op)->pending_ops = 0; +} + +static bool replace_stable(object_id oid, uint64_t version, int search_start, int search_end, obj_ver_id* list) +{ + while (search_start < search_end) + { + int pos = search_start+(search_end-search_start)/2; + if (oid < list[pos].oid) + { + search_end = pos; + } + else if (list[pos].oid < oid) + { + search_start = pos+1; + } + else + { + list[pos].version = version; + return true; + } + } + return false; +} + +blockstore_clean_db_t& blockstore_impl_t::clean_db_shard(object_id oid) +{ + uint64_t pg_num = 0; + uint64_t pool_id = (oid.inode >> (64-POOL_ID_BITS)); + auto sh_it = clean_db_settings.find(pool_id); + if (sh_it != clean_db_settings.end()) + { + // like map_to_pg() + pg_num = (oid.stripe / sh_it->second.pg_stripe_size) % sh_it->second.pg_count + 1; + } + return clean_db_shards[(pool_id << (64-POOL_ID_BITS)) | pg_num]; +} + +void blockstore_impl_t::reshard_clean_db(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size) +{ + uint64_t pool_id = (uint64_t)pool; + std::map new_shards; + auto sh_it = clean_db_shards.lower_bound((pool_id << (64-POOL_ID_BITS))); + while (sh_it != clean_db_shards.end() && + (sh_it->first >> (64-POOL_ID_BITS)) == pool_id) + { + for (auto & pair: sh_it->second) + { + // like map_to_pg() + uint64_t pg_num = (pair.first.stripe / pg_stripe_size) % pg_count + 1; + uint64_t shard_id = (pool_id << (64-POOL_ID_BITS)) | pg_num; + new_shards[shard_id][pair.first] = pair.second; + } + clean_db_shards.erase(sh_it++); + } + for (sh_it = new_shards.begin(); sh_it != new_shards.end(); sh_it++) + { + auto & to = clean_db_shards[sh_it->first]; + to.swap(sh_it->second); + } + clean_db_settings[pool_id] = (pool_shard_settings_t){ + .pg_count = pg_count, + .pg_stripe_size = pg_stripe_size, + }; +} + +void blockstore_impl_t::process_list(blockstore_op_t *op) +{ + uint32_t list_pg = op->pg_number+1; + uint32_t pg_count = op->pg_count; + uint64_t pg_stripe_size = op->pg_alignment; + uint64_t min_inode = op->min_oid.inode; + uint64_t max_inode = op->max_oid.inode; + // Check PG + if (pg_count != 0 && (pg_stripe_size < MIN_DATA_BLOCK_SIZE || list_pg > pg_count)) + { + op->retval = -EINVAL; + FINISH_OP(op); + return; + } + // Check if the DB needs resharding + // (we don't know about PGs from the beginning, we only create "shards" here) + uint64_t first_shard = 0, last_shard = UINT64_MAX; + if (min_inode != 0 && + // Check if min_inode == max_inode == pool_id<> (64-POOL_ID_BITS)) == (max_inode >> (64-POOL_ID_BITS))) + { + pool_id_t pool_id = (min_inode >> (64-POOL_ID_BITS)); + if (pg_count > 1) + { + // Per-pg listing + auto sh_it = clean_db_settings.find(pool_id); + if (sh_it == clean_db_settings.end() || + sh_it->second.pg_count != pg_count || + sh_it->second.pg_stripe_size != pg_stripe_size) + { + reshard_clean_db(pool_id, pg_count, pg_stripe_size); + } + first_shard = last_shard = ((uint64_t)pool_id << (64-POOL_ID_BITS)) | list_pg; + } + else + { + // Per-pool listing + first_shard = ((uint64_t)pool_id << (64-POOL_ID_BITS)); + last_shard = ((uint64_t)(pool_id+1) << (64-POOL_ID_BITS)) - 1; + } + } + // Copy clean_db entries + int stable_count = 0, stable_alloc = 0; + if (min_inode != max_inode) + { + for (auto shard_it = clean_db_shards.lower_bound(first_shard); + shard_it != clean_db_shards.end() && shard_it->first <= last_shard; + shard_it++) + { + auto & clean_db = shard_it->second; + stable_alloc += clean_db.size(); + } + } + if (op->list_stable_limit > 0) + { + stable_alloc = op->list_stable_limit; + if (stable_alloc > 1024*1024) + stable_alloc = 1024*1024; + } + if (stable_alloc < 32768) + { + stable_alloc = 32768; + } + obj_ver_id *stable = (obj_ver_id*)malloc(sizeof(obj_ver_id) * stable_alloc); + if (!stable) + { + op->retval = -ENOMEM; + FINISH_OP(op); + return; + } + auto max_oid = op->max_oid; + bool limited = false; + pool_pg_id_t last_shard_id = 0; + for (auto shard_it = clean_db_shards.lower_bound(first_shard); + shard_it != clean_db_shards.end() && shard_it->first <= last_shard; + shard_it++) + { + auto & clean_db = shard_it->second; + auto clean_it = clean_db.begin(), clean_end = clean_db.end(); + if (op->min_oid.inode != 0 || op->min_oid.stripe != 0) + { + clean_it = clean_db.lower_bound(op->min_oid); + } + if ((max_oid.inode != 0 || max_oid.stripe != 0) && !(max_oid < op->min_oid)) + { + clean_end = clean_db.upper_bound(max_oid); + } + for (; clean_it != clean_end; clean_it++) + { + if (stable_count >= stable_alloc) + { + stable_alloc *= 2; + obj_ver_id* nst = (obj_ver_id*)realloc(stable, sizeof(obj_ver_id) * stable_alloc); + if (!nst) + { + op->retval = -ENOMEM; + FINISH_OP(op); + return; + } + stable = nst; + } + stable[stable_count++] = { + .oid = clean_it->first, + .version = clean_it->second.version, + }; + if (op->list_stable_limit > 0 && stable_count >= op->list_stable_limit) + { + if (!limited) + { + limited = true; + max_oid = stable[stable_count-1].oid; + } + break; + } + } + if (op->list_stable_limit > 0) + { + // To maintain the order, we have to include objects in the same range from other shards + if (last_shard_id != 0 && last_shard_id != shard_it->first) + std::sort(stable, stable+stable_count); + if (stable_count > op->list_stable_limit) + stable_count = op->list_stable_limit; + } + last_shard_id = shard_it->first; + } + if (op->list_stable_limit == 0 && first_shard != last_shard) + { + // If that's not a per-PG listing, sort clean entries (already sorted if list_stable_limit != 0) + std::sort(stable, stable+stable_count); + } + int clean_stable_count = stable_count; + // Copy dirty_db entries (sorted, too) + int unstable_count = 0, unstable_alloc = 0; + obj_ver_id *unstable = NULL; + { + auto dirty_it = dirty_db.begin(), dirty_end = dirty_db.end(); + if (op->min_oid.inode != 0 || op->min_oid.stripe != 0) + { + dirty_it = dirty_db.lower_bound({ + .oid = op->min_oid, + .version = 0, + }); + } + if ((max_oid.inode != 0 || max_oid.stripe != 0) && !(max_oid < op->min_oid)) + { + dirty_end = dirty_db.upper_bound({ + .oid = max_oid, + .version = UINT64_MAX, + }); + } + for (; dirty_it != dirty_end; dirty_it++) + { + if (!pg_count || ((dirty_it->first.oid.stripe / pg_stripe_size) % pg_count + 1) == list_pg) // like map_to_pg() + { + if (IS_DELETE(dirty_it->second.state)) + { + // Deletions are always stable, so try to zero out two possible entries + if (!replace_stable(dirty_it->first.oid, 0, 0, clean_stable_count, stable)) + { + replace_stable(dirty_it->first.oid, 0, clean_stable_count, stable_count, stable); + } + } + else if (IS_STABLE(dirty_it->second.state) || (dirty_it->second.state & BS_ST_INSTANT)) + { + // First try to replace a clean stable version in the first part of the list + if (!replace_stable(dirty_it->first.oid, dirty_it->first.version, 0, clean_stable_count, stable)) + { + // Then try to replace the last dirty stable version in the second part of the list + if (stable_count > 0 && stable[stable_count-1].oid == dirty_it->first.oid) + { + stable[stable_count-1].version = dirty_it->first.version; + } + else + { + if (stable_count >= stable_alloc) + { + stable_alloc += 32768; + obj_ver_id *nst = (obj_ver_id*)realloc(stable, sizeof(obj_ver_id) * stable_alloc); + if (!nst) + { + if (unstable) + free(unstable); + op->retval = -ENOMEM; + FINISH_OP(op); + return; + } + stable = nst; + } + stable[stable_count++] = dirty_it->first; + } + } + if (op->list_stable_limit > 0 && stable_count >= op->list_stable_limit) + { + // Stop here + break; + } + } + else + { + if (unstable_count >= unstable_alloc) + { + unstable_alloc += 32768; + obj_ver_id *nst = (obj_ver_id*)realloc(unstable, sizeof(obj_ver_id) * unstable_alloc); + if (!nst) + { + if (stable) + free(stable); + op->retval = -ENOMEM; + FINISH_OP(op); + return; + } + unstable = nst; + } + unstable[unstable_count++] = dirty_it->first; + } + } + } + } + // Remove zeroed out stable entries + int j = 0; + for (int i = 0; i < stable_count; i++) + { + if (stable[i].version != 0) + { + stable[j++] = stable[i]; + } + } + stable_count = j; + if (stable_count+unstable_count > stable_alloc) + { + stable_alloc = stable_count+unstable_count; + obj_ver_id *nst = (obj_ver_id*)realloc(stable, sizeof(obj_ver_id) * stable_alloc); + if (!nst) + { + if (unstable) + free(unstable); + op->retval = -ENOMEM; + FINISH_OP(op); + return; + } + stable = nst; + } + // Copy unstable entries + for (int i = 0; i < unstable_count; i++) + { + stable[j++] = unstable[i]; + } + free(unstable); + op->version = stable_count; + op->retval = stable_count+unstable_count; + op->buf = (uint8_t*)stable; + FINISH_OP(op); +} + +void blockstore_impl_t::dump_diagnostics() +{ + journal.dump_diagnostics(); + flusher->dump_diagnostics(); +} + +void blockstore_impl_t::disk_error_abort(const char *op, int retval, int expected) +{ + if (retval == -EAGAIN) + { + fprintf(stderr, "EAGAIN error received from a disk %s during flush." + " It must never happen with io_uring and indicates a kernel bug." + " Please upgrade your kernel. Aborting.\n", op); + exit(1); + } + fprintf(stderr, "Disk %s failed: result is %d, expected %d. Can't continue, sorry :-(\n", op, retval, expected); + exit(1); +} + +const std::map & blockstore_impl_t::get_inode_space_stats() +{ + return inode_space_stats; +} + +void blockstore_impl_t::set_no_inode_stats(const std::vector & pool_ids) +{ + for (auto & np: no_inode_stats) + { + np.second = 2; + } + for (auto pool_id: pool_ids) + { + if (!no_inode_stats[pool_id]) + recalc_inode_space_stats(pool_id, false); + no_inode_stats[pool_id] = 1; + } + for (auto np_it = no_inode_stats.begin(); np_it != no_inode_stats.end(); ) + { + if (np_it->second == 2) + { + recalc_inode_space_stats(np_it->first, true); + no_inode_stats.erase(np_it++); + } + else + np_it++; + } +} + +void blockstore_impl_t::recalc_inode_space_stats(uint64_t pool_id, bool per_inode) +{ + auto sp_begin = inode_space_stats.lower_bound((pool_id << (64-POOL_ID_BITS))); + auto sp_end = inode_space_stats.lower_bound(((pool_id+1) << (64-POOL_ID_BITS))); + inode_space_stats.erase(sp_begin, sp_end); + auto sh_it = clean_db_shards.lower_bound((pool_id << (64-POOL_ID_BITS))); + while (sh_it != clean_db_shards.end() && + (sh_it->first >> (64-POOL_ID_BITS)) == pool_id) + { + for (auto & pair: sh_it->second) + { + uint64_t space_id = per_inode ? pair.first.inode : (pool_id << (64-POOL_ID_BITS)); + inode_space_stats[space_id] += dsk.data_block_size; + } + sh_it++; + } + object_id last_oid = {}; + bool last_exists = false; + auto dirty_it = dirty_db.lower_bound((obj_ver_id){ .oid = { .inode = (pool_id << (64-POOL_ID_BITS)) } }); + while (dirty_it != dirty_db.end() && (dirty_it->first.oid.inode >> (64-POOL_ID_BITS)) == pool_id) + { + if (IS_STABLE(dirty_it->second.state) && (IS_BIG_WRITE(dirty_it->second.state) || IS_DELETE(dirty_it->second.state))) + { + bool exists = false; + if (last_oid == dirty_it->first.oid) + { + exists = last_exists; + } + else + { + auto & clean_db = clean_db_shard(dirty_it->first.oid); + auto clean_it = clean_db.find(dirty_it->first.oid); + exists = clean_it != clean_db.end(); + } + uint64_t space_id = per_inode ? dirty_it->first.oid.inode : (pool_id << (64-POOL_ID_BITS)); + if (IS_BIG_WRITE(dirty_it->second.state)) + { + if (!exists) + inode_space_stats[space_id] += dsk.data_block_size; + last_exists = true; + } + else + { + if (exists) + { + auto & sp = inode_space_stats[space_id]; + if (sp > dsk.data_block_size) + sp -= dsk.data_block_size; + else + inode_space_stats.erase(space_id); + } + last_exists = false; + } + last_oid = dirty_it->first.oid; + } + dirty_it++; + } +} + +std::string blockstore_impl_t::get_op_diag(blockstore_op_t *op) +{ + char buf[256]; + auto priv = PRIV(op); + if (priv->wait_for) + snprintf(buf, sizeof(buf), "state=%d wait=%d (detail=%ju)", priv->op_state, priv->wait_for, priv->wait_detail); + else + snprintf(buf, sizeof(buf), "state=%d", priv->op_state); + return std::string(buf); +} diff --git a/src/blockstore/v1/impl.h b/src/blockstore/v1/impl.h new file mode 100644 index 00000000..7365ff75 --- /dev/null +++ b/src/blockstore/v1/impl.h @@ -0,0 +1,329 @@ +// Copyright (c) Vitaliy Filippov, 2019+ +// License: VNPL-1.1 (see README.md for details) + +#pragma once + +#include "blockstore.h" +#include "blockstore_disk.h" +#include "ondisk_formats.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "cpp-btree/btree_map.h" + +#include "malloc_or_die.h" +#include "allocator.h" + +//#define BLOCKSTORE_DEBUG + +#include "blockstore_journal.h" + +// 32 = 16 + 16 bytes per "clean" entry in memory (object_id => clean_entry) +struct __attribute__((__packed__)) clean_entry +{ + uint64_t version; + uint64_t location; +}; + +// 64 = 24 + 40 bytes per dirty entry in memory (obj_ver_id => dirty_entry). Plus checksums +struct __attribute__((__packed__)) dirty_entry +{ + uint32_t state; + uint32_t flags; // unneeded, but present for alignment + uint64_t location; // location in either journal or data -> in BYTES + uint32_t offset; // data offset within object (stripe) + uint32_t len; // data length + uint64_t journal_sector; // journal sector used for this entry + void* dyn_data; // dynamic data: external bitmap and data block checksums. may be a pointer to the in-memory journal +}; + +// - Sync must be submitted after previous writes/deletes (not before!) +// - Reads to the same object must be submitted after previous writes/deletes +// are written (not necessarily synced) in their location. This is because we +// rely on read-modify-write for erasure coding and we must return new data +// to calculate parity for subsequent writes +// - Writes may be submitted in any order, because they don't overlap. Each write +// goes into a new location - either on the journal device or on the data device +// - Stable (stabilize) must be submitted after sync of that object is completed +// It's even OK to return an error to the caller if that object is not synced yet +// - Journal trim may be processed only after all versions are moved to +// the main storage AND after all read operations for older versions complete +// - If an operation can not be submitted because the ring is full +// we should stop submission of other operations. Otherwise some "scatter" reads +// may end up blocked for a long time. +// Otherwise, the submit order is free, that is all operations may be submitted immediately +// In fact, adding a write operation must immediately result in dirty_db being populated + +struct used_clean_obj_t +{ + int refs; + bool was_freed; // was freed by a parallel flush? + bool was_changed; // was changed by a parallel flush? +}; + +// https://github.com/algorithm-ninja/cpp-btree +// https://github.com/greg7mdp/sparsepp/ was used previously, but it was TERRIBLY slow after resizing +// with sparsepp, random reads dropped to ~700 iops very fast with just as much as ~32k objects in the DB +typedef btree::btree_map blockstore_clean_db_t; +typedef std::map blockstore_dirty_db_t; + +#include "blockstore_init.h" + +#include "blockstore_flush.h" + +struct blockstore_op_private_t +{ + // Wait status + int wait_for; + uint64_t wait_detail, wait_detail2; + int pending_ops; + int op_state; + + // Read + uint64_t clean_block_used; + std::vector read_vec; + + // Sync, write + uint64_t min_flushed_journal_sector, max_flushed_journal_sector; + + // Write + struct iovec iov_zerofill[3]; + // Warning: must not have a default value here because it's written to before calling constructor in blockstore_write.cpp O_o + uint64_t real_version; + timespec tv_begin; + + // Sync + std::vector sync_big_writes, sync_small_writes; +}; + +struct pool_shard_settings_t +{ + uint32_t pg_count; + uint32_t pg_stripe_size; +}; + +typedef uint64_t pool_pg_id_t; + +class blockstore_impl_t: public blockstore_i +{ + blockstore_disk_t dsk; + + /******* OPTIONS *******/ + bool readonly = false; + // It is safe to disable fsync() if drive write cache is writethrough + bool disable_data_fsync = false, disable_meta_fsync = false, disable_journal_fsync = false; + // Enable if you want every operation to be executed with an "implicit fsync" + // Suitable only for server SSDs with capacitors, requires disabled data and journal fsyncs + int immediate_commit = IMMEDIATE_NONE; + bool inmemory_meta = false; + // Maximum and minimum flusher count + unsigned max_flusher_count, min_flusher_count; + unsigned journal_trim_interval; + // Maximum queue depth + unsigned max_write_iodepth = 128; + // Enable small (journaled) write throttling, useful for the SSD+HDD case + bool throttle_small_writes = false; + // Target data device iops, bandwidth and parallelism for throttling (100/100/1 is the default for HDD) + int throttle_target_iops = 100; + int throttle_target_mbs = 100; + int throttle_target_parallelism = 1; + // Minimum difference in microseconds between target and real execution times to throttle the response + int throttle_threshold_us = 50; + // Maximum writes between automatically added fsync operations + uint64_t autosync_writes = 128; + // Log level (0-10) + int log_level = 0; + /******* END OF OPTIONS *******/ + + struct ring_consumer_t ring_consumer; + + std::map clean_db_settings; + std::map clean_db_shards; + std::map no_inode_stats; + std::map inode_space_stats; + uint8_t *clean_bitmaps = NULL; + blockstore_dirty_db_t dirty_db; + std::vector submit_queue; + std::vector unsynced_big_writes, unsynced_small_writes; + int unsynced_big_write_count = 0, unstable_unsynced = 0; + int unsynced_queued_ops = 0; + allocator_t *data_alloc = NULL; + uint64_t used_blocks = 0; + uint8_t *zero_object = NULL; + + void *metadata_buffer = NULL; + + struct journal_t journal; + journal_flusher_t *flusher; + int big_to_flush = 0; + int write_iodepth = 0; + bool alloc_dyn_data = false; + + // clean data blocks referenced by read operations + std::map used_clean_objects; + + bool live = false, queue_stall = false; + ring_loop_t *ringloop; + timerfd_manager_t *tfd; + + bool stop_sync_submitted; + + inline struct io_uring_sqe* get_sqe() + { + return ringloop->get_sqe(); + } + + friend class blockstore_init_meta; + friend class blockstore_init_journal; + friend struct blockstore_journal_check_t; + friend class journal_flusher_t; + friend class journal_flusher_co; + + void calc_lengths(); + void open_data(); + void open_meta(); + void open_journal(); + uint8_t* get_clean_entry_bitmap(uint64_t block_loc, int offset); + + blockstore_clean_db_t& clean_db_shard(object_id oid); + void reshard_clean_db(pool_id_t pool_id, uint32_t pg_count, uint32_t pg_stripe_size); + void recalc_inode_space_stats(uint64_t pool_id, bool per_inode); + + // Journaling + void prepare_journal_sector_write(int sector, blockstore_op_t *op); + void handle_journal_write(ring_data_t *data, uint64_t flush_id); + void disk_error_abort(const char *op, int retval, int expected); + + // Asynchronous init + int initialized; + int metadata_buf_size; + blockstore_init_meta* metadata_init_reader; + blockstore_init_journal* journal_init_reader; + + void check_wait(blockstore_op_t *op); + void init_op(blockstore_op_t *op); + + // Read + int dequeue_read(blockstore_op_t *read_op); + void find_holes(std::vector & read_vec, uint32_t item_start, uint32_t item_end, + std::function callback); + int fulfill_read(blockstore_op_t *read_op, + uint64_t &fulfilled, uint32_t item_start, uint32_t item_end, + uint32_t item_state, uint64_t item_version, uint64_t item_location, + uint64_t journal_sector, uint8_t *csum, int *dyn_data); + bool fulfill_clean_read(blockstore_op_t *read_op, uint64_t & fulfilled, + uint8_t *clean_entry_bitmap, int *dyn_data, + uint32_t item_start, uint32_t item_end, uint64_t clean_loc, uint64_t clean_ver); + int fill_partial_checksum_blocks(std::vector & rv, uint64_t & fulfilled, + uint8_t *clean_entry_bitmap, int *dyn_data, bool from_journal, uint8_t *read_buf, uint64_t read_offset, uint64_t read_end); + int pad_journal_read(std::vector & rv, copy_buffer_t & cp, + uint64_t dirty_offset, uint64_t dirty_end, uint64_t dirty_loc, uint8_t *csum_ptr, int *dyn_data, + uint64_t offset, uint64_t submit_len, uint64_t & blk_begin, uint64_t & blk_end, uint8_t* & blk_buf); + bool read_range_fulfilled(std::vector & rv, uint64_t & fulfilled, uint8_t *read_buf, + uint8_t *clean_entry_bitmap, uint32_t item_start, uint32_t item_end); + bool read_checksum_block(blockstore_op_t *op, int rv_pos, uint64_t &fulfilled, uint64_t clean_loc); + uint8_t* read_clean_meta_block(blockstore_op_t *read_op, uint64_t clean_loc, int rv_pos); + bool verify_padded_checksums(uint8_t *clean_entry_bitmap, uint8_t *csum_buf, uint32_t offset, + iovec *iov, int n_iov, std::function bad_block_cb); + bool verify_journal_checksums(uint8_t *csums, uint32_t offset, + iovec *iov, int n_iov, std::function bad_block_cb); + bool verify_clean_padded_checksums(blockstore_op_t *op, uint64_t clean_loc, uint8_t *dyn_data, bool from_journal, + iovec *iov, int n_iov, std::function bad_block_cb); + int fulfill_read_push(blockstore_op_t *op, void *buf, uint64_t offset, uint64_t len, + uint32_t item_state, uint64_t item_version); + void handle_read_event(ring_data_t *data, blockstore_op_t *op); + + // Write + bool enqueue_write(blockstore_op_t *op); + void cancel_all_writes(blockstore_op_t *op, blockstore_dirty_db_t::iterator dirty_it, int retval); + int dequeue_write(blockstore_op_t *op); + int dequeue_del(blockstore_op_t *op); + int continue_write(blockstore_op_t *op); + void release_journal_sectors(blockstore_op_t *op); + void handle_write_event(ring_data_t *data, blockstore_op_t *op); + + // Sync + int continue_sync(blockstore_op_t *op); + void ack_sync(blockstore_op_t *op); + + // Stabilize + int dequeue_stable(blockstore_op_t *op); + int continue_stable(blockstore_op_t *op); + void mark_stable(obj_ver_id ov, bool forget_dirty = false); + void stabilize_object(object_id oid, uint64_t max_ver); + blockstore_op_t* selective_sync(blockstore_op_t *op); + int split_stab_op(blockstore_op_t *op, std::function decider); + + // Rollback + int dequeue_rollback(blockstore_op_t *op); + int continue_rollback(blockstore_op_t *op); + void mark_rolled_back(const obj_ver_id & ov); + void erase_dirty(blockstore_dirty_db_t::iterator dirty_start, blockstore_dirty_db_t::iterator dirty_end, uint64_t clean_loc); + void free_dirty_dyn_data(dirty_entry & e); + + // List + void process_list(blockstore_op_t *op); + +public: + + blockstore_impl_t(blockstore_config_t & config, ring_loop_t *ringloop, timerfd_manager_t *tfd); + ~blockstore_impl_t(); + + void parse_config(blockstore_config_t & config); + void parse_config(blockstore_config_t & config, bool init); + + // Event loop + void loop(); + + // Returns true when blockstore is ready to process operations + // (Although you're free to enqueue them before that) + bool is_started(); + + // Returns true when it's safe to destroy the instance. If destroying the instance + // requires to purge some queues, starts that process. Should be called in the event + // loop until it returns true. + bool is_safe_to_stop(); + + // Returns true if stalled + bool is_stalled(); + + // Submission + void enqueue_op(blockstore_op_t *op); + + // Simplified synchronous operation: get object bitmap & current version + int read_bitmap(object_id oid, uint64_t target_version, void *bitmap, uint64_t *result_version = NULL); + + // Unstable writes are added here (map of object_id -> version) + std::unordered_map unstable_writes; + + // Get space usage statistics + const std::map & get_inode_space_stats(); + + // Set per-pool no_inode_stats + void set_no_inode_stats(const std::vector & pool_ids); + + // Print diagnostics to stdout + void dump_diagnostics(); + + // Get diagnostic string for an operation + std::string get_op_diag(blockstore_op_t *op); + + inline uint32_t get_block_size() { return dsk.data_block_size; } + inline uint64_t get_block_count() { return dsk.block_count; } + inline uint64_t get_free_block_count() { return dsk.block_count - used_blocks; } + inline uint32_t get_bitmap_granularity() { return dsk.disk_alignment; } + inline uint64_t get_journal_size() { return dsk.journal_len; } +}; diff --git a/src/blockstore/v1/init.cpp b/src/blockstore/v1/init.cpp new file mode 100644 index 00000000..34969858 --- /dev/null +++ b/src/blockstore/v1/init.cpp @@ -0,0 +1,1219 @@ +// Copyright (c) Vitaliy Filippov, 2019+ +// License: VNPL-1.1 (see README.md for details) + +#include "blockstore_impl.h" +#include "blockstore_internal.h" + +#define INIT_META_EMPTY 0 +#define INIT_META_READING 1 +#define INIT_META_READ_DONE 2 +#define INIT_META_WRITING 3 + +#define GET_SQE() \ + sqe = bs->get_sqe();\ + if (!sqe)\ + throw std::runtime_error("io_uring is full during initialization");\ + data = ((ring_data_t*)sqe->user_data) + +static bool iszero(uint64_t *buf, int len) +{ + for (int i = 0; i < len; i++) + if (buf[i] != 0) + return false; + return true; +} + +blockstore_init_meta::blockstore_init_meta(blockstore_impl_t *bs) +{ + this->bs = bs; +} + +void blockstore_init_meta::handle_event(ring_data_t *data, int buf_num) +{ + if (data->res < 0) + { + throw std::runtime_error( + std::string("read metadata failed at offset ") + std::to_string(buf_num >= 0 ? bufs[buf_num].offset : last_read_offset) + + std::string(": ") + strerror(-data->res) + ); + } + if (buf_num >= 0) + { + bufs[buf_num].state = (bufs[buf_num].state == INIT_META_READING + ? INIT_META_READ_DONE + : INIT_META_EMPTY); + } + submitted--; + bs->ringloop->wakeup(); +} + +int blockstore_init_meta::loop() +{ + if (wait_state == 1) goto resume_1; + else if (wait_state == 2) goto resume_2; + else if (wait_state == 3) goto resume_3; + else if (wait_state == 4) goto resume_4; + else if (wait_state == 5) goto resume_5; + else if (wait_state == 6) goto resume_6; + printf("Reading blockstore metadata\n"); + if (bs->inmemory_meta) + metadata_buffer = bs->metadata_buffer; + else + metadata_buffer = memalign(MEM_ALIGNMENT, 2*bs->metadata_buf_size); + if (!metadata_buffer) + throw std::runtime_error("Failed to allocate metadata read buffer"); + // Read superblock + GET_SQE(); + last_read_offset = 0; + data->iov = { metadata_buffer, (size_t)bs->dsk.meta_block_size }; + data->callback = [this](ring_data_t *data) { handle_event(data, -1); }; + io_uring_prep_readv(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset); + bs->ringloop->submit(); + submitted++; +resume_1: + if (submitted > 0) + { + wait_state = 1; + return 1; + } + if (iszero((uint64_t*)metadata_buffer, bs->dsk.meta_block_size / sizeof(uint64_t))) + { + { + blockstore_meta_header_v2_t *hdr = (blockstore_meta_header_v2_t *)metadata_buffer; + hdr->zero = 0; + hdr->magic = BLOCKSTORE_META_MAGIC_V1; + hdr->version = bs->dsk.meta_format; + hdr->meta_block_size = bs->dsk.meta_block_size; + hdr->data_block_size = bs->dsk.data_block_size; + hdr->bitmap_granularity = bs->dsk.bitmap_granularity; + if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_V2) + { + hdr->data_csum_type = bs->dsk.data_csum_type; + hdr->csum_block_size = bs->dsk.csum_block_size; + hdr->header_csum = 0; + hdr->header_csum = crc32c(0, hdr, sizeof(*hdr)); + } + } + if (bs->readonly) + { + printf("Skipping metadata initialization because blockstore is readonly\n"); + } + else + { + printf("Initializing metadata area\n"); + GET_SQE(); + last_read_offset = 0; + data->iov = (struct iovec){ metadata_buffer, (size_t)bs->dsk.meta_block_size }; + data->callback = [this](ring_data_t *data) { handle_event(data, -1); }; + io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset); + bs->ringloop->submit(); + submitted++; + resume_3: + if (submitted > 0) + { + wait_state = 3; + return 1; + } + zero_on_init = true; + } + } + else + { + blockstore_meta_header_v2_t *hdr = (blockstore_meta_header_v2_t *)metadata_buffer; + if (hdr->zero != 0 || hdr->magic != BLOCKSTORE_META_MAGIC_V1 || hdr->version < BLOCKSTORE_META_FORMAT_V1) + { + printf( + "Metadata is corrupt or too old (pre-0.6.x).\n" + " If this is a new OSD, please zero out the metadata area before starting it.\n" + " If you need to upgrade from 0.5.x, convert metadata with vitastor-disk.\n" + ); + exit(1); + } + if (hdr->version == BLOCKSTORE_META_FORMAT_V2) + { + uint32_t csum = hdr->header_csum; + hdr->header_csum = 0; + if (crc32c(0, hdr, sizeof(*hdr)) != csum) + { + printf("Metadata header is corrupt (checksum mismatch).\n"); + exit(1); + } + hdr->header_csum = csum; + if (bs->dsk.meta_format != BLOCKSTORE_META_FORMAT_V2) + { + bs->dsk.meta_format = BLOCKSTORE_META_FORMAT_V2; + bs->dsk.calc_lengths(); + } + } + else if (hdr->version == BLOCKSTORE_META_FORMAT_V1) + { + hdr->data_csum_type = 0; + hdr->csum_block_size = 0; + hdr->header_csum = 0; + // Enable compatibility mode - entries without checksums + if (bs->dsk.meta_format != BLOCKSTORE_META_FORMAT_V1 || + bs->dsk.data_csum_type != 0 || bs->dsk.csum_block_size != 0) + { + bs->dsk.data_csum_type = 0; + bs->dsk.csum_block_size = 0; + bs->dsk.meta_format = BLOCKSTORE_META_FORMAT_V1; + bs->dsk.calc_lengths(); + printf("Warning: Starting with metadata in the old format without checksums, as stored on disk\n"); + } + } + else if (hdr->version > BLOCKSTORE_META_FORMAT_V2) + { + printf( + "Metadata format is too new for me (stored version is %ju, max supported %u).\n", + hdr->version, BLOCKSTORE_META_FORMAT_V2 + ); + exit(1); + } + if (hdr->meta_block_size != bs->dsk.meta_block_size || + hdr->data_block_size != bs->dsk.data_block_size || + hdr->bitmap_granularity != bs->dsk.bitmap_granularity || + hdr->data_csum_type != bs->dsk.data_csum_type || + hdr->csum_block_size != bs->dsk.csum_block_size) + { + printf( + "Configuration stored in metadata superblock" + " (meta_block_size=%u, data_block_size=%u, bitmap_granularity=%u, data_csum_type=%u, csum_block_size=%u)" + " differs from OSD configuration (%ju/%u/%ju, %u/%u).\n", + hdr->meta_block_size, hdr->data_block_size, hdr->bitmap_granularity, + hdr->data_csum_type, hdr->csum_block_size, + bs->dsk.meta_block_size, bs->dsk.data_block_size, bs->dsk.bitmap_granularity, + bs->dsk.data_csum_type, bs->dsk.csum_block_size + ); + exit(1); + } + } + // Skip superblock + md_offset = bs->dsk.meta_block_size; + next_offset = md_offset; + entries_per_block = bs->dsk.meta_block_size / bs->dsk.clean_entry_size; + // Read the rest of the metadata +resume_2: + if (next_offset < bs->dsk.meta_len && submitted == 0) + { + // Submit one read + for (int i = 0; i < 2; i++) + { + if (!bufs[i].state) + { + bufs[i].buf = (uint8_t*)metadata_buffer + (bs->inmemory_meta + ? next_offset-md_offset + : i*bs->metadata_buf_size); + bufs[i].offset = next_offset; + bufs[i].size = bs->dsk.meta_len-next_offset > bs->metadata_buf_size + ? bs->metadata_buf_size : bs->dsk.meta_len-next_offset; + bufs[i].state = INIT_META_READING; + submitted++; + next_offset += bufs[i].size; + GET_SQE(); + assert(bufs[i].size <= 0x7fffffff); + data->iov = { bufs[i].buf, (size_t)bufs[i].size }; + data->callback = [this, i](ring_data_t *data) { handle_event(data, i); }; + if (!zero_on_init) + io_uring_prep_readv(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bufs[i].offset); + else + { + // Fill metadata with zeroes + memset(data->iov.iov_base, 0, data->iov.iov_len); + io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bufs[i].offset); + } + bs->ringloop->submit(); + break; + } + } + } + for (int i = 0; i < 2; i++) + { + if (bufs[i].state == INIT_META_READ_DONE) + { + // Handle result + bool changed = false; + for (uint64_t sector = 0; sector < bufs[i].size; sector += bs->dsk.meta_block_size) + { + // handle entries + if (handle_meta_block(bufs[i].buf + sector, entries_per_block, + ((bufs[i].offset + sector - md_offset) / bs->dsk.meta_block_size) * entries_per_block)) + changed = true; + } + if (changed && !bs->inmemory_meta && !bs->readonly) + { + // write the modified buffer back + GET_SQE(); + assert(bufs[i].size <= 0x7fffffff); + data->iov = { bufs[i].buf, (size_t)bufs[i].size }; + data->callback = [this, i](ring_data_t *data) { handle_event(data, i); }; + io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bufs[i].offset); + bs->ringloop->submit(); + bufs[i].state = INIT_META_WRITING; + submitted++; + } + else + { + bufs[i].state = 0; + } + bs->ringloop->wakeup(); + } + } + if (submitted > 0) + { + wait_state = 2; + return 1; + } + if (entries_to_zero.size() && !bs->inmemory_meta && !bs->readonly) + { + std::sort(entries_to_zero.begin(), entries_to_zero.end()); + // we have to zero out additional entries + for (i = 0; i < entries_to_zero.size(); ) + { + next_offset = entries_to_zero[i]/entries_per_block; + for (j = i; j < entries_to_zero.size() && entries_to_zero[j]/entries_per_block == next_offset; j++) {} + GET_SQE(); + last_read_offset = (1+next_offset)*bs->dsk.meta_block_size; + data->iov = { metadata_buffer, (size_t)bs->dsk.meta_block_size }; + data->callback = [this](ring_data_t *data) { handle_event(data, -1); }; + io_uring_prep_readv(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + (1+next_offset)*bs->dsk.meta_block_size); + bs->ringloop->submit(); + submitted++; +resume_5: + if (submitted > 0) + { + wait_state = 5; + return 1; + } + for (; i < j; i++) + { + uint64_t pos = (entries_to_zero[i] % entries_per_block); + memset((uint8_t*)metadata_buffer + pos*bs->dsk.clean_entry_size, 0, bs->dsk.clean_entry_size); + } + GET_SQE(); + data->iov = { metadata_buffer, (size_t)bs->dsk.meta_block_size }; + data->callback = [this](ring_data_t *data) { handle_event(data, -1); }; + io_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + (1+next_offset)*bs->dsk.meta_block_size); + bs->ringloop->submit(); + submitted++; +resume_6: + if (submitted > 0) + { + wait_state = 6; + return 1; + } + } + entries_to_zero.clear(); + } + // metadata read finished + printf("Metadata entries loaded: %ju, free blocks: %ju / %ju\n", entries_loaded, bs->data_alloc->get_free_count(), bs->dsk.block_count); + if (!bs->inmemory_meta) + { + free(metadata_buffer); + metadata_buffer = NULL; + } + if (zero_on_init && !bs->disable_meta_fsync) + { + GET_SQE(); + io_uring_prep_fsync(sqe, bs->dsk.meta_fd, IORING_FSYNC_DATASYNC); + last_read_offset = 0; + data->iov = { 0 }; + data->callback = [this](ring_data_t *data) { handle_event(data, -1); }; + submitted++; + bs->ringloop->submit(); + resume_4: + if (submitted > 0) + { + wait_state = 4; + return 1; + } + } + return 0; +} + +bool blockstore_init_meta::handle_meta_block(uint8_t *buf, uint64_t entries_per_block, uint64_t done_cnt) +{ + bool updated = false; + uint64_t max_i = entries_per_block; + if (max_i > bs->dsk.block_count-done_cnt) + max_i = bs->dsk.block_count-done_cnt; + for (uint64_t i = 0; i < max_i; i++) + { + clean_disk_entry *entry = (clean_disk_entry*)(buf + i*bs->dsk.clean_entry_size); + if (entry->oid.inode > 0) + { + if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_V2) + { + // Check entry crc32 + uint32_t *entry_csum = (uint32_t*)((uint8_t*)entry + bs->dsk.clean_entry_size - 4); + if (*entry_csum != crc32c(0, entry, bs->dsk.clean_entry_size - 4)) + { + printf("Metadata entry %ju is corrupt (checksum mismatch: %08x vs %08x), skipping\n", done_cnt+i, *entry_csum, crc32c(0, entry, bs->dsk.clean_entry_size - 4)); + // zero out the invalid entry, otherwise we'll hit "tried to overwrite non-zero metadata entry" later + if (bs->inmemory_meta) + { + memset(entry, 0, bs->dsk.clean_entry_size); + } + else + { + entries_to_zero.push_back(done_cnt+i); + } + continue; + } + } + if (!bs->inmemory_meta && bs->dsk.clean_entry_bitmap_size) + { + memcpy(bs->clean_bitmaps + (done_cnt+i) * 2 * bs->dsk.clean_entry_bitmap_size, &entry->bitmap, 2 * bs->dsk.clean_entry_bitmap_size); + } + auto & clean_db = bs->clean_db_shard(entry->oid); + auto clean_it = clean_db.find(entry->oid); + if (clean_it == clean_db.end() || clean_it->second.version < entry->version) + { + if (clean_it != clean_db.end()) + { + // free the previous block + // here we have to zero out the previous entry because otherwise we'll hit + // "tried to overwrite non-zero metadata entry" later + uint64_t old_clean_loc = clean_it->second.location / bs->dsk.data_block_size; + if (bs->inmemory_meta) + { + uint64_t sector = (old_clean_loc / entries_per_block) * bs->dsk.meta_block_size; + uint64_t pos = (old_clean_loc % entries_per_block); + clean_disk_entry *old_entry = (clean_disk_entry*)((uint8_t*)bs->metadata_buffer + sector + pos*bs->dsk.clean_entry_size); + memset(old_entry, 0, bs->dsk.clean_entry_size); + } + else if (old_clean_loc >= done_cnt) + { + updated = true; + uint64_t sector = ((old_clean_loc - done_cnt) / entries_per_block) * bs->dsk.meta_block_size; + uint64_t pos = (old_clean_loc % entries_per_block); + clean_disk_entry *old_entry = (clean_disk_entry*)(buf + sector + pos*bs->dsk.clean_entry_size); + memset(old_entry, 0, bs->dsk.clean_entry_size); + } + else + { + entries_to_zero.push_back(clean_it->second.location / bs->dsk.data_block_size); + } +#ifdef BLOCKSTORE_DEBUG + printf("Free block %ju from %jx:%jx v%ju (new location is %ju)\n", + old_clean_loc, + clean_it->first.inode, clean_it->first.stripe, clean_it->second.version, + done_cnt+i); +#endif + bs->data_alloc->set(old_clean_loc, false); + } + else + { + bs->inode_space_stats[entry->oid.inode] += bs->dsk.data_block_size; + bs->used_blocks++; + } + entries_loaded++; +#ifdef BLOCKSTORE_DEBUG + printf("Allocate block (clean entry) %ju: %jx:%jx v%ju\n", done_cnt+i, entry->oid.inode, entry->oid.stripe, entry->version); +#endif + bs->data_alloc->set(done_cnt+i, true); + clean_db[entry->oid] = (struct clean_entry){ + .version = entry->version, + .location = (done_cnt+i) * bs->dsk.data_block_size, + }; + } + else + { + // here we also have to zero out the entry + updated = true; + memset(entry, 0, bs->dsk.clean_entry_size); +#ifdef BLOCKSTORE_DEBUG + printf("Old clean entry %ju: %jx:%jx v%ju\n", done_cnt+i, entry->oid.inode, entry->oid.stripe, entry->version); +#endif + } + } + } + return updated; +} + +blockstore_init_journal::blockstore_init_journal(blockstore_impl_t *bs) +{ + this->bs = bs; + next_free = bs->journal.block_size; + simple_callback = [this](ring_data_t *data1) + { + if (data1->res != data1->iov.iov_len) + { + throw std::runtime_error(std::string("I/O operation failed while reading journal: ") + strerror(-data1->res)); + } + wait_count--; + }; +} + +void blockstore_init_journal::handle_event(ring_data_t *data1) +{ + if (data1->res <= 0) + { + throw std::runtime_error( + std::string("read journal failed at offset ") + std::to_string(journal_pos) + + std::string(": ") + strerror(-data1->res) + ); + } + done.push_back({ + .buf = submitted_buf, + .pos = journal_pos, + .len = (uint64_t)data1->res, + }); + journal_pos += data1->res; + if (journal_pos >= bs->journal.len) + { + // Continue from the beginning + journal_pos = bs->journal.block_size; + wrapped = true; + } + submitted_buf = NULL; +} + +int blockstore_init_journal::loop() +{ + if (wait_state == 1) + goto resume_1; + else if (wait_state == 2) + goto resume_2; + else if (wait_state == 3) + goto resume_3; + else if (wait_state == 4) + goto resume_4; + else if (wait_state == 5) + goto resume_5; + else if (wait_state == 6) + goto resume_6; + else if (wait_state == 7) + goto resume_7; + printf("Reading blockstore journal\n"); + if (!bs->journal.inmemory) + submitted_buf = memalign_or_die(MEM_ALIGNMENT, 2*bs->journal.block_size); + else + submitted_buf = bs->journal.buffer; + // Read first block of the journal + sqe = bs->get_sqe(); + if (!sqe) + throw std::runtime_error("io_uring is full while trying to read journal"); + data = ((ring_data_t*)sqe->user_data); + data->iov = { submitted_buf, (size_t)bs->journal.block_size }; + data->callback = simple_callback; + io_uring_prep_readv(sqe, bs->dsk.journal_fd, &data->iov, 1, bs->journal.offset); + bs->ringloop->submit(); + wait_count = 1; +resume_1: + if (wait_count > 0) + { + wait_state = 1; + return 1; + } + if (iszero((uint64_t*)submitted_buf, bs->journal.block_size / sizeof(uint64_t))) + { + // Journal is empty + // FIXME handle this wrapping to journal_block_size better (maybe) + bs->journal.used_start = bs->journal.block_size; + bs->journal.next_free = bs->journal.block_size; + // Initialize journal "superblock" and the first block + memset(submitted_buf, 0, 2*bs->journal.block_size); + *((journal_entry_start*)submitted_buf) = { + .crc32 = 0, + .magic = JOURNAL_MAGIC, + .type = JE_START, + .size = sizeof(journal_entry_start), + .reserved = 0, + .journal_start = bs->journal.block_size, + .version = JOURNAL_VERSION_V2, + .data_csum_type = bs->dsk.data_csum_type, + .csum_block_size = bs->dsk.csum_block_size, + }; + ((journal_entry_start*)submitted_buf)->crc32 = je_crc32((journal_entry*)submitted_buf); + if (bs->readonly) + { + printf("Skipping journal initialization because blockstore is readonly\n"); + } + else + { + // Cool effect. Same operations result in journal replay. + // FIXME: Randomize initial crc32. Track crc32 when trimming. + printf("Resetting journal\n"); + GET_SQE(); + data->iov = (struct iovec){ submitted_buf, (size_t)(2*bs->journal.block_size) }; + data->callback = simple_callback; + io_uring_prep_writev(sqe, bs->dsk.journal_fd, &data->iov, 1, bs->journal.offset); + wait_count++; + bs->ringloop->submit(); + resume_6: + if (wait_count > 0) + { + wait_state = 6; + return 1; + } + if (!bs->disable_journal_fsync) + { + GET_SQE(); + io_uring_prep_fsync(sqe, bs->dsk.journal_fd, IORING_FSYNC_DATASYNC); + data->iov = { 0 }; + data->callback = simple_callback; + wait_count++; + bs->ringloop->submit(); + } + resume_4: + if (wait_count > 0) + { + wait_state = 4; + return 1; + } + } + if (!bs->journal.inmemory) + { + free(submitted_buf); + } + } + else + { + // First block always contains a single JE_START entry + je_start = (journal_entry_start*)submitted_buf; + if (je_start->magic != JOURNAL_MAGIC || + je_start->type != JE_START || + je_crc32((journal_entry*)je_start) != je_start->crc32 || + je_start->size != JE_START_V0_SIZE && je_start->size != JE_START_V1_SIZE && je_start->size != JE_START_V2_SIZE) + { + // Entry is corrupt + fprintf(stderr, "First entry of the journal is corrupt or unsupported\n"); + exit(1); + } + if (je_start->size == JE_START_V0_SIZE || + (je_start->version != JOURNAL_VERSION_V1 || je_start->size != JE_START_V1_SIZE) && + (je_start->version != JOURNAL_VERSION_V2 || je_start->size != JE_START_V2_SIZE && je_start->size != JE_START_V1_SIZE)) + { + fprintf( + stderr, "The code only supports journal versions 2 and 1, but it is %ju on disk." + " Please use vitastor-disk to rewrite the journal\n", + je_start->size == JE_START_V0_SIZE ? 0 : je_start->version + ); + exit(1); + } + if (je_start->version == JOURNAL_VERSION_V1 || + je_start->version == JOURNAL_VERSION_V2 && je_start->size == JE_START_V1_SIZE) + { + je_start->data_csum_type = 0; + je_start->csum_block_size = 0; + } + if (je_start->data_csum_type != bs->dsk.data_csum_type || + je_start->csum_block_size != bs->dsk.csum_block_size) + { + printf( + "Configuration stored in journal superblock (data_csum_type=%u, csum_block_size=%u)" + " differs from OSD configuration (%u/%u).\n", + je_start->data_csum_type, je_start->csum_block_size, + bs->dsk.data_csum_type, bs->dsk.csum_block_size + ); + exit(1); + } + next_free = journal_pos = bs->journal.used_start = je_start->journal_start; + if (!bs->journal.inmemory) + free(submitted_buf); + submitted_buf = NULL; + crc32_last = 0; + // Read journal + while (1) + { + resume_2: + if (submitted_buf) + { + wait_state = 2; + return 1; + } + if (!wrapped || journal_pos < bs->journal.used_start) + { + GET_SQE(); + uint64_t end = bs->journal.len; + if (journal_pos < bs->journal.used_start) + end = bs->journal.used_start; + if (!bs->journal.inmemory) + submitted_buf = memalign_or_die(MEM_ALIGNMENT, JOURNAL_BUFFER_SIZE); + else + submitted_buf = (uint8_t*)bs->journal.buffer + journal_pos; + data->iov = { + submitted_buf, + (size_t)(end - journal_pos < JOURNAL_BUFFER_SIZE ? end - journal_pos : JOURNAL_BUFFER_SIZE), + }; + data->callback = [this](ring_data_t *data1) { handle_event(data1); }; + io_uring_prep_readv(sqe, bs->dsk.journal_fd, &data->iov, 1, bs->journal.offset + journal_pos); + bs->ringloop->submit(); + } + while (done.size() > 0) + { + handle_res = handle_journal_part(done[0].buf, done[0].pos, done[0].len); + if (handle_res == 0) + { + // journal ended + // zero out corrupted entry, if required + if (init_write_buf && !bs->readonly) + { + GET_SQE(); + data->iov = { init_write_buf, (size_t)bs->journal.block_size }; + data->callback = simple_callback; + io_uring_prep_writev(sqe, bs->dsk.journal_fd, &data->iov, 1, bs->journal.offset + init_write_sector); + wait_count++; + bs->ringloop->submit(); + resume_7: + if (wait_count > 0) + { + wait_state = 7; + return 1; + } + if (!bs->disable_journal_fsync) + { + GET_SQE(); + data->iov = { 0 }; + data->callback = simple_callback; + io_uring_prep_fsync(sqe, bs->dsk.journal_fd, IORING_FSYNC_DATASYNC); + wait_count++; + bs->ringloop->submit(); + } + resume_5: + if (wait_count > 0) + { + wait_state = 5; + return 1; + } + } + // wait for the next read to complete, then stop + resume_3: + if (submitted_buf) + { + wait_state = 3; + return 1; + } + // free buffers + if (!bs->journal.inmemory) + for (auto & e: done) + free(e.buf); + done.clear(); + break; + } + else if (handle_res == 1) + { + // OK, remove it + if (!bs->journal.inmemory) + { + free(done[0].buf); + } + done.erase(done.begin()); + } + else if (handle_res == 2) + { + // Need to wait for more reads + break; + } + } + if (!submitted_buf) + { + break; + } + } + } + for (auto ov: double_allocs) + { + auto dirty_it = bs->dirty_db.find(ov); + if (dirty_it != bs->dirty_db.end() && + IS_BIG_WRITE(dirty_it->second.state) && + dirty_it->second.location == UINT64_MAX) + { + printf("Fatal error (bug): %jx:%jx v%ju big_write journal_entry was allocated over another object\n", + dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version); + exit(1); + } + } + bs->flusher->mark_trim_possible(); + bs->journal.dirty_start = bs->journal.next_free; + printf( + "Journal entries loaded: %ju, free journal space: %ju bytes (%08jx..%08jx is used), free blocks: %ju / %ju\n", + entries_loaded, + (bs->journal.next_free >= bs->journal.used_start + ? bs->journal.len-bs->journal.block_size - (bs->journal.next_free-bs->journal.used_start) + : bs->journal.used_start - bs->journal.next_free), + bs->journal.used_start, bs->journal.next_free, + bs->data_alloc->get_free_count(), bs->dsk.block_count + ); + bs->journal.crc32_last = crc32_last; + return 0; +} + +int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, uint64_t len) +{ + uint64_t proc_pos, pos; + if (continue_pos != 0) + { + proc_pos = (continue_pos / bs->journal.block_size) * bs->journal.block_size; + pos = continue_pos % bs->journal.block_size; + continue_pos = 0; + goto resume; + } + while (next_free >= done_pos && next_free < done_pos+len) + { + proc_pos = next_free; + pos = 0; + next_free += bs->journal.block_size; + if (next_free >= bs->journal.len) + { + next_free = bs->journal.block_size; + } + resume: + while (pos < bs->journal.block_size) + { + auto buf_pos = proc_pos - done_pos + pos; + journal_entry *je = (journal_entry*)((uint8_t*)buf + buf_pos); + if (je->magic != JOURNAL_MAGIC || buf_pos+je->size > len || je_crc32(je) != je->crc32 || + je->type < JE_MIN || je->type > JE_MAX || started && je->crc32_prev != crc32_last) + { + if (pos == 0) + { + // invalid entry in the beginning, this is definitely the end of the journal + bs->journal.next_free = proc_pos; + return 0; + } + else + { + // allow partially filled sectors + break; + } + } + if (je->type == JE_SMALL_WRITE || je->type == JE_SMALL_WRITE_INSTANT) + { +#ifdef BLOCKSTORE_DEBUG + printf( + "je_small_write%s oid=%jx:%jx ver=%ju offset=%u len=%u\n", + je->type == JE_SMALL_WRITE_INSTANT ? "_instant" : "", + je->small_write.oid.inode, je->small_write.oid.stripe, je->small_write.version, + je->small_write.offset, je->small_write.len + ); +#endif + // oid, version, offset, len + uint64_t prev_free = next_free; + if (next_free + je->small_write.len > bs->journal.len) + { + // data continues from the beginning of the journal + next_free = bs->journal.block_size; + } + uint64_t location = next_free; + next_free += je->small_write.len; + if (next_free >= bs->journal.len) + { + next_free = bs->journal.block_size; + } + if (location != je->small_write.data_offset) + { + char err[1024]; + snprintf(err, 1024, "BUG: calculated journal data offset (%08jx) != stored journal data offset (%08jx)", location, je->small_write.data_offset); + throw std::runtime_error(err); + } + small_write_data.clear(); + if (location >= done_pos && location+je->small_write.len <= done_pos+len) + { + // data is within this buffer + small_write_data.push_back((iovec){ + .iov_base = (uint8_t*)buf + location - done_pos, + .iov_len = je->small_write.len, + }); + } + else + { + // this case is even more interesting because we must carry data crc32 check to next buffer(s) + uint64_t covered = 0; + for (int i = 0; i < done.size(); i++) + { + if (location+je->small_write.len > done[i].pos && + location < done[i].pos+done[i].len) + { + uint64_t part_end = (location+je->small_write.len < done[i].pos+done[i].len + ? location+je->small_write.len : done[i].pos+done[i].len); + uint64_t part_begin = (location < done[i].pos ? done[i].pos : location); + covered += part_end - part_begin; + small_write_data.push_back((iovec){ + .iov_base = (uint8_t*)done[i].buf + part_begin - done[i].pos, + .iov_len = (size_t)(part_end - part_begin), + }); + } + } + if (covered < je->small_write.len) + { + continue_pos = proc_pos+pos; + next_free = prev_free; + return 2; + } + } + bool data_csum_valid = true; + if (!bs->dsk.csum_block_size) + { + uint32_t data_crc32 = 0; + for (auto & sd: small_write_data) + { + data_crc32 = crc32c(data_crc32, sd.iov_base, sd.iov_len); + } + data_csum_valid = data_crc32 == je->small_write.crc32_data; + if (!data_csum_valid) + { + printf( + "Journal entry data is corrupt for small_write%s oid=%jx:%jx ver=%ju offset=%u len=%u - data crc32 %x != %x\n", + je->type == JE_SMALL_WRITE_INSTANT ? "_instant" : "", + je->small_write.oid.inode, je->small_write.oid.stripe, je->small_write.version, + je->small_write.offset, je->small_write.len, + data_crc32, je->small_write.crc32_data + ); + } + } + else if (je->small_write.len > 0) + { + // FIXME: deduplicate with disk_tool_journal.cpp + // like in enqueue_write() + uint32_t start = je->small_write.offset / bs->dsk.csum_block_size; + uint32_t end = (je->small_write.offset+je->small_write.len-1) / bs->dsk.csum_block_size; + uint32_t data_csum_size = (end-start+1) * (bs->dsk.data_csum_type & 0xFF); + uint32_t required_size = sizeof(journal_entry_small_write) + bs->dsk.clean_entry_bitmap_size + data_csum_size; + if (je->size != required_size) + { + printf( + "Journal entry data has invalid size for small_write%s oid=%jx:%jx ver=%ju offset=%u len=%u - should be %u bytes but is %u bytes\n", + je->type == JE_SMALL_WRITE_INSTANT ? "_instant" : "", + je->small_write.oid.inode, je->small_write.oid.stripe, je->small_write.version, + je->small_write.offset, je->small_write.len, + required_size, je->size + ); + data_csum_valid = false; + } + else + { + int sd_num = 0; + size_t sd_pos = 0; + uint32_t *block_csums = (uint32_t*)((uint8_t*)je + sizeof(journal_entry_small_write) + bs->dsk.clean_entry_bitmap_size); + for (uint32_t pos = start; pos <= end; pos++, block_csums++) + { + size_t block_left = (pos == start + ? (start == end + ? je->small_write.len + : bs->dsk.csum_block_size - je->small_write.offset%bs->dsk.csum_block_size) + : (pos < end + ? bs->dsk.csum_block_size + : (je->small_write.offset + je->small_write.len)%bs->dsk.csum_block_size)); + if (pos > start && pos == end && block_left == 0) + { + // full last block + block_left = bs->dsk.csum_block_size; + } + uint32_t block_crc32 = 0; + while (block_left > 0) + { + assert(sd_num < small_write_data.size()); + if (small_write_data[sd_num].iov_len >= sd_pos+block_left) + { + block_crc32 = crc32c(block_crc32, (uint8_t*)small_write_data[sd_num].iov_base+sd_pos, block_left); + sd_pos += block_left; + break; + } + else + { + block_crc32 = crc32c(block_crc32, (uint8_t*)small_write_data[sd_num].iov_base+sd_pos, small_write_data[sd_num].iov_len-sd_pos); + block_left -= (small_write_data[sd_num].iov_len-sd_pos); + sd_pos = 0; + sd_num++; + } + } + if (block_crc32 != *block_csums) + { + printf( + "Journal entry data is corrupt for small_write%s oid=%jx:%jx ver=%ju offset=%u len=%u - block %u crc32 %x != %x\n", + je->type == JE_SMALL_WRITE_INSTANT ? "_instant" : "", + je->small_write.oid.inode, je->small_write.oid.stripe, je->small_write.version, + je->small_write.offset, je->small_write.len, + pos, block_crc32, *block_csums + ); + data_csum_valid = false; + break; + } + } + } + } + if (!data_csum_valid) + { + // journal entry is corrupt, stop here + // interesting thing is that we must clear the corrupt entry if we're not readonly, + // because we don't write next entries in the same journal block + memset((uint8_t*)buf + proc_pos - done_pos + pos, 0, bs->journal.block_size - pos); + bs->journal.next_free = prev_free; + init_write_buf = (uint8_t*)buf + proc_pos - done_pos; + init_write_sector = proc_pos; + return 0; + } + auto & clean_db = bs->clean_db_shard(je->small_write.oid); + auto clean_it = clean_db.find(je->small_write.oid); + if (clean_it == clean_db.end() || + clean_it->second.version < je->small_write.version) + { + obj_ver_id ov = { + .oid = je->small_write.oid, + .version = je->small_write.version, + }; + uint64_t dyn_size = bs->dsk.dirty_dyn_size(je->small_write.offset, je->small_write.len); + void *dyn = NULL; + void *dyn_from = (uint8_t*)je + sizeof(journal_entry_small_write); + if (!bs->alloc_dyn_data) + { + // Bitmap without checksum is only 4 bytes for 128k objects, save it inline + // It can even contain 4 byte bitmap + 4 byte CRC32 for 4 kb writes :) + memcpy(&dyn, dyn_from, dyn_size); + } + else + { + // FIXME Using large blockstore objects will result in a lot of small + // allocations for entry bitmaps. This can only be fixed by using + // a patched map with dynamic entry size, but not the btree_map, + // because it doesn't keep iterators valid all the time. + dyn = malloc_or_die(dyn_size+sizeof(int)); + *((int*)dyn) = 1; + memcpy((uint8_t*)dyn+sizeof(int), dyn_from, dyn_size); + } + bs->dirty_db.emplace(ov, (dirty_entry){ + .state = (BS_ST_SMALL_WRITE | BS_ST_SYNCED), + .flags = 0, + .location = location, + .offset = je->small_write.offset, + .len = je->small_write.len, + .journal_sector = proc_pos, + .dyn_data = dyn, + }); + bs->journal.used_sectors[proc_pos]++; +#ifdef BLOCKSTORE_DEBUG + printf( + "journal offset %08jx is used by %jx:%jx v%ju (%ju refs)\n", + proc_pos, ov.oid.inode, ov.oid.stripe, ov.version, bs->journal.used_sectors[proc_pos] + ); +#endif + auto & unstab = bs->unstable_writes[ov.oid]; + unstab = unstab < ov.version ? ov.version : unstab; + if (je->type == JE_SMALL_WRITE_INSTANT) + { + bs->mark_stable(ov, true); + } + } + } + else if (je->type == JE_BIG_WRITE || je->type == JE_BIG_WRITE_INSTANT) + { +#ifdef BLOCKSTORE_DEBUG + printf( + "je_big_write%s oid=%jx:%jx ver=%ju loc=%ju\n", + je->type == JE_BIG_WRITE_INSTANT ? "_instant" : "", + je->big_write.oid.inode, je->big_write.oid.stripe, je->big_write.version, je->big_write.location / bs->dsk.data_block_size + ); +#endif + auto dirty_it = bs->dirty_db.upper_bound((obj_ver_id){ + .oid = je->big_write.oid, + .version = UINT64_MAX, + }); + if (dirty_it != bs->dirty_db.begin() && bs->dirty_db.size() > 0) + { + dirty_it--; + if (dirty_it->first.oid == je->big_write.oid && + dirty_it->first.version >= je->big_write.version && + (dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_DELETE) + { + // It is allowed to overwrite a deleted object with a + // version number smaller than deletion version number, + // because the presence of a BIG_WRITE entry means that + // its data and metadata are already flushed. + // We don't know if newer versions are flushed, but + // the previous delete definitely is. + // So we forget previous dirty entries, but retain the clean one. + // This feature is required for writes happening shortly + // after deletes. + erase_dirty_object(dirty_it); + } + } + auto & clean_db = bs->clean_db_shard(je->big_write.oid); + auto clean_it = clean_db.find(je->big_write.oid); + if (clean_it == clean_db.end() || + clean_it->second.version < je->big_write.version) + { + // oid, version, block + obj_ver_id ov = { + .oid = je->big_write.oid, + .version = je->big_write.version, + }; + uint64_t dyn_size = bs->dsk.dirty_dyn_size(je->big_write.offset, je->big_write.len); + void *dyn = NULL; + void *dyn_from = (uint8_t*)je + sizeof(journal_entry_big_write); + if (!bs->alloc_dyn_data) + { + // Bitmap without checksum is only 4 bytes for 128k objects, save it inline + memcpy(&dyn, dyn_from, dyn_size); + } + else + { + // FIXME Using large blockstore objects will result in a lot of small + // allocations for entry bitmaps. This can only be fixed by using + // a patched map with dynamic entry size, but not the btree_map, + // because it doesn't keep iterators valid all the time. + dyn = malloc_or_die(dyn_size+sizeof(int)); + *((int*)dyn) = 1; + memcpy((uint8_t*)dyn+sizeof(int), dyn_from, dyn_size); + } + auto dirty_it = bs->dirty_db.emplace(ov, (dirty_entry){ + .state = (BS_ST_BIG_WRITE | BS_ST_SYNCED), + .flags = 0, + .location = je->big_write.location, + .offset = je->big_write.offset, + .len = je->big_write.len, + .journal_sector = proc_pos, + .dyn_data = dyn, + }).first; + if (bs->data_alloc->get(je->big_write.location / bs->dsk.data_block_size)) + { + // This is probably a big_write that's already flushed and freed, but it may + // also indicate a bug. So we remember such entries and recheck them afterwards. + // If it's not a bug they won't be present after reading the whole journal. + dirty_it->second.location = UINT64_MAX; + double_allocs.push_back(ov); + } + else + { +#ifdef BLOCKSTORE_DEBUG + printf( + "Allocate block (journal) %ju: %jx:%jx v%ju\n", + je->big_write.location / bs->dsk.data_block_size, + ov.oid.inode, ov.oid.stripe, ov.version + ); +#endif + bs->data_alloc->set(je->big_write.location / bs->dsk.data_block_size, true); + } + bs->journal.used_sectors[proc_pos]++; +#ifdef BLOCKSTORE_DEBUG + printf( + "journal offset %08jx is used by %jx:%jx v%ju (%ju refs)\n", + proc_pos, ov.oid.inode, ov.oid.stripe, ov.version, bs->journal.used_sectors[proc_pos] + ); +#endif + auto & unstab = bs->unstable_writes[ov.oid]; + unstab = unstab < ov.version ? ov.version : unstab; + if (je->type == JE_BIG_WRITE_INSTANT) + { + bs->mark_stable(ov, true); + } + } + } + else if (je->type == JE_STABLE) + { +#ifdef BLOCKSTORE_DEBUG + printf("je_stable oid=%jx:%jx ver=%ju\n", je->stable.oid.inode, je->stable.oid.stripe, je->stable.version); +#endif + // oid, version + obj_ver_id ov = { + .oid = je->stable.oid, + .version = je->stable.version, + }; + bs->mark_stable(ov, true); + } + else if (je->type == JE_ROLLBACK) + { +#ifdef BLOCKSTORE_DEBUG + printf("je_rollback oid=%jx:%jx ver=%ju\n", je->rollback.oid.inode, je->rollback.oid.stripe, je->rollback.version); +#endif + // rollback dirty writes of up to + obj_ver_id ov = { + .oid = je->rollback.oid, + .version = je->rollback.version, + }; + bs->mark_rolled_back(ov); + } + else if (je->type == JE_DELETE) + { +#ifdef BLOCKSTORE_DEBUG + printf("je_delete oid=%jx:%jx ver=%ju\n", je->del.oid.inode, je->del.oid.stripe, je->del.version); +#endif + bool dirty_exists = false; + auto dirty_it = bs->dirty_db.upper_bound((obj_ver_id){ + .oid = je->del.oid, + .version = UINT64_MAX, + }); + if (dirty_it != bs->dirty_db.begin()) + { + dirty_it--; + dirty_exists = dirty_it->first.oid == je->del.oid; + } + auto & clean_db = bs->clean_db_shard(je->del.oid); + auto clean_it = clean_db.find(je->del.oid); + bool clean_exists = (clean_it != clean_db.end() && + clean_it->second.version < je->del.version); + if (!clean_exists && dirty_exists) + { + // Clean entry doesn't exist. This means that the delete is already flushed. + // So we must not flush this object anymore. + erase_dirty_object(dirty_it); + } + else if (clean_exists || dirty_exists) + { + // oid, version + obj_ver_id ov = { + .oid = je->del.oid, + .version = je->del.version, + }; + bs->dirty_db.emplace(ov, (dirty_entry){ + .state = (BS_ST_DELETE | BS_ST_SYNCED), + .flags = 0, + .location = 0, + .offset = 0, + .len = 0, + .journal_sector = proc_pos, + }); + bs->journal.used_sectors[proc_pos]++; + // Deletions are treated as immediately stable, because + // "2-phase commit" (write->stabilize) isn't sufficient for them anyway + bs->mark_stable(ov, true); + } + // Ignore delete if neither preceding dirty entries nor the clean one are present + } + started = true; + pos += je->size; + crc32_last = je->crc32; + entries_loaded++; + } + } + bs->journal.next_free = next_free; + return 1; +} + +void blockstore_init_journal::erase_dirty_object(blockstore_dirty_db_t::iterator dirty_it) +{ + auto oid = dirty_it->first.oid; + bool exists = !IS_DELETE(dirty_it->second.state); + auto dirty_end = dirty_it; + dirty_end++; + while (1) + { + if (dirty_it == bs->dirty_db.begin()) + { + break; + } + dirty_it--; + if (dirty_it->first.oid != oid) + { + dirty_it++; + break; + } + } + auto & clean_db = bs->clean_db_shard(oid); + auto clean_it = clean_db.find(oid); + uint64_t clean_loc = clean_it != clean_db.end() + ? clean_it->second.location : UINT64_MAX; + if (exists && clean_loc == UINT64_MAX) + { + auto & sp = bs->inode_space_stats[oid.inode]; + if (sp > bs->dsk.data_block_size) + sp -= bs->dsk.data_block_size; + else + bs->inode_space_stats.erase(oid.inode); + bs->used_blocks--; + } + bs->erase_dirty(dirty_it, dirty_end, clean_loc); + // Remove it from the flusher's queue, too + // Otherwise it may end up referring to a small unstable write after reading the rest of the journal + bs->flusher->remove_flush(oid); +} diff --git a/src/blockstore/v1/init.h b/src/blockstore/v1/init.h new file mode 100644 index 00000000..1df9304b --- /dev/null +++ b/src/blockstore/v1/init.h @@ -0,0 +1,71 @@ +// Copyright (c) Vitaliy Filippov, 2019+ +// License: VNPL-1.1 (see README.md for details) + +#pragma once + +struct blockstore_init_meta_buf +{ + uint8_t *buf = NULL; + uint64_t size = 0; + uint64_t offset = 0; + int state = 0; +}; + +class blockstore_init_meta +{ + blockstore_impl_t *bs; + int wait_state = 0; + bool zero_on_init = false; + void *metadata_buffer = NULL; + blockstore_init_meta_buf bufs[2] = {}; + int submitted = 0; + struct io_uring_sqe *sqe; + struct ring_data_t *data; + uint64_t md_offset = 0; + uint64_t next_offset = 0; + uint64_t last_read_offset = 0; + uint64_t entries_loaded = 0; + unsigned entries_per_block = 0; + int i = 0, j = 0; + std::vector entries_to_zero; + bool handle_meta_block(uint8_t *buf, uint64_t count, uint64_t done_cnt); + void handle_event(ring_data_t *data, int buf_num); +public: + blockstore_init_meta(blockstore_impl_t *bs); + int loop(); +}; + +struct bs_init_journal_done +{ + void *buf; + uint64_t pos, len; +}; + +class blockstore_init_journal +{ + blockstore_impl_t *bs; + int wait_state = 0, wait_count = 0, handle_res = 0; + uint64_t entries_loaded = 0; + uint32_t crc32_last = 0; + bool started = false; + uint64_t next_free; + std::vector done; + std::vector double_allocs; + std::vector small_write_data; + uint64_t journal_pos = 0; + uint64_t continue_pos = 0; + void *init_write_buf = NULL; + uint64_t init_write_sector = 0; + bool wrapped = false; + void *submitted_buf; + struct io_uring_sqe *sqe; + struct ring_data_t *data; + journal_entry_start *je_start; + std::function simple_callback; + int handle_journal_part(void *buf, uint64_t done_pos, uint64_t len); + void handle_event(ring_data_t *data); + void erase_dirty_object(blockstore_dirty_db_t::iterator dirty_it); +public: + blockstore_init_journal(blockstore_impl_t* bs); + int loop(); +}; diff --git a/src/blockstore/v1/internal.h b/src/blockstore/v1/internal.h new file mode 100644 index 00000000..f28046e4 --- /dev/null +++ b/src/blockstore/v1/internal.h @@ -0,0 +1,85 @@ +#pragma once + +// States are not stored on disk. Instead, they're deduced from the journal + +#define BS_ST_SMALL_WRITE 0x01 +#define BS_ST_BIG_WRITE 0x02 +#define BS_ST_DELETE 0x03 + +#define BS_ST_WAIT_DEL 0x10 +#define BS_ST_WAIT_BIG 0x20 +#define BS_ST_IN_FLIGHT 0x30 +#define BS_ST_SUBMITTED 0x40 +#define BS_ST_WRITTEN 0x50 +#define BS_ST_SYNCED 0x60 +#define BS_ST_STABLE 0x70 + +#define BS_ST_INSTANT 0x100 + +#define BS_ST_TYPE_MASK 0x0F +#define BS_ST_WORKFLOW_MASK 0xF0 +#define IS_IN_FLIGHT(st) (((st) & 0xF0) <= BS_ST_SUBMITTED) +#define IS_STABLE(st) (((st) & 0xF0) == BS_ST_STABLE) +#define IS_SYNCED(st) (((st) & 0xF0) >= BS_ST_SYNCED) +#define IS_JOURNAL(st) (((st) & 0x0F) == BS_ST_SMALL_WRITE) +#define IS_BIG_WRITE(st) (((st) & 0x0F) == BS_ST_BIG_WRITE) +#define IS_DELETE(st) (((st) & 0x0F) == BS_ST_DELETE) +#define IS_INSTANT(st) (((st) & BS_ST_TYPE_MASK) == BS_ST_DELETE || ((st) & BS_ST_INSTANT)) + +#define BS_SUBMIT_CHECK_SQES(n) \ + if (ringloop->space_left() < (n))\ + {\ + /* Pause until there are more requests available */\ + PRIV(op)->wait_detail = (n);\ + PRIV(op)->wait_for = WAIT_SQE;\ + return 0;\ + } + +#define BS_SUBMIT_GET_SQE(sqe, data) \ + BS_SUBMIT_GET_ONLY_SQE(sqe); \ + struct ring_data_t *data = ((ring_data_t*)sqe->user_data) + +#define BS_SUBMIT_GET_ONLY_SQE(sqe) \ + struct io_uring_sqe *sqe = get_sqe();\ + if (!sqe)\ + {\ + /* Pause until there are more requests available */\ + PRIV(op)->wait_detail = 1;\ + PRIV(op)->wait_for = WAIT_SQE;\ + return 0;\ + } + +#define BS_SUBMIT_GET_SQE_DECL(sqe) \ + sqe = get_sqe();\ + if (!sqe)\ + {\ + /* Pause until there are more requests available */\ + PRIV(op)->wait_detail = 1;\ + PRIV(op)->wait_for = WAIT_SQE;\ + return 0;\ + } + +#define PRIV(op) ((blockstore_op_private_t*)(op)->private_data) +#define FINISH_OP(op) PRIV(op)->~blockstore_op_private_t(); std::function(op->callback)(op) + +// Suspend operation until there are more free SQEs +#define WAIT_SQE 1 +// Suspend operation until there are bytes of free space in the journal on disk +#define WAIT_JOURNAL 3 +// Suspend operation until the next journal sector buffer is free +#define WAIT_JOURNAL_BUFFER 4 +// Suspend operation until there is some free space on the data device +#define WAIT_FREE 5 + +#define COPY_BUF_JOURNAL 1 +#define COPY_BUF_DATA 2 +#define COPY_BUF_ZERO 4 +#define COPY_BUF_CSUM_FILL 8 +#define COPY_BUF_COALESCED 16 +#define COPY_BUF_META_BLOCK 32 +#define COPY_BUF_JOURNALED_BIG 64 + +#define STAB_SPLIT_DONE 1 +#define STAB_SPLIT_WAIT 2 +#define STAB_SPLIT_SYNC 3 +#define STAB_SPLIT_TODO 4 diff --git a/src/blockstore/blockstore_journal.cpp b/src/blockstore/v1/journal.cpp similarity index 100% rename from src/blockstore/blockstore_journal.cpp rename to src/blockstore/v1/journal.cpp diff --git a/src/blockstore/blockstore_journal.h b/src/blockstore/v1/journal.h similarity index 100% rename from src/blockstore/blockstore_journal.h rename to src/blockstore/v1/journal.h diff --git a/src/blockstore/v1/open.cpp b/src/blockstore/v1/open.cpp new file mode 100644 index 00000000..4341150e --- /dev/null +++ b/src/blockstore/v1/open.cpp @@ -0,0 +1,183 @@ +// Copyright (c) Vitaliy Filippov, 2019+ +// License: VNPL-1.1 (see README.md for details) + +#include +#include "blockstore_impl.h" + +void blockstore_impl_t::parse_config(blockstore_config_t & config) +{ + return parse_config(config, false); +} + +void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init) +{ + // Online-configurable options: + max_flusher_count = strtoull(config["max_flusher_count"].c_str(), NULL, 10); + if (!max_flusher_count) + { + max_flusher_count = strtoull(config["flusher_count"].c_str(), NULL, 10); + } + min_flusher_count = strtoull(config["min_flusher_count"].c_str(), NULL, 10); + journal_trim_interval = strtoull(config["journal_trim_interval"].c_str(), NULL, 10); + max_write_iodepth = strtoull(config["max_write_iodepth"].c_str(), NULL, 10); + throttle_small_writes = config["throttle_small_writes"] == "true" || config["throttle_small_writes"] == "1" || config["throttle_small_writes"] == "yes"; + throttle_target_iops = strtoull(config["throttle_target_iops"].c_str(), NULL, 10); + throttle_target_mbs = strtoull(config["throttle_target_mbs"].c_str(), NULL, 10); + throttle_target_parallelism = strtoull(config["throttle_target_parallelism"].c_str(), NULL, 10); + throttle_threshold_us = strtoull(config["throttle_threshold_us"].c_str(), NULL, 10); + if (config["autosync_writes"] != "") + { + autosync_writes = strtoull(config["autosync_writes"].c_str(), NULL, 10); + } + if (!max_flusher_count) + { + max_flusher_count = 256; + } + if (!min_flusher_count || journal.flush_journal) + { + min_flusher_count = 1; + } + if (!journal_trim_interval) + { + journal_trim_interval = 512; + } + if (!max_write_iodepth) + { + max_write_iodepth = 128; + } + if (!throttle_target_iops) + { + throttle_target_iops = 100; + } + if (!throttle_target_mbs) + { + throttle_target_mbs = 100; + } + if (!throttle_target_parallelism) + { + throttle_target_parallelism = 1; + } + if (!throttle_threshold_us) + { + throttle_threshold_us = 50; + } + if (!init) + { + return; + } + // Offline-configurable options: + // Common disk options + dsk.parse_config(config); + // Parse + if (config["readonly"] == "true" || config["readonly"] == "1" || config["readonly"] == "yes") + { + readonly = true; + } + if (config["disable_data_fsync"] == "true" || config["disable_data_fsync"] == "1" || config["disable_data_fsync"] == "yes") + { + disable_data_fsync = true; + } + if (config["disable_meta_fsync"] == "true" || config["disable_meta_fsync"] == "1" || config["disable_meta_fsync"] == "yes") + { + disable_meta_fsync = true; + } + if (config["disable_journal_fsync"] == "true" || config["disable_journal_fsync"] == "1" || config["disable_journal_fsync"] == "yes") + { + disable_journal_fsync = true; + } + if (config["flush_journal"] == "true" || config["flush_journal"] == "1" || config["flush_journal"] == "yes") + { + // Only flush journal and exit + journal.flush_journal = true; + } + if (config["immediate_commit"] == "all") + { + immediate_commit = IMMEDIATE_ALL; + } + else if (config["immediate_commit"] == "small") + { + immediate_commit = IMMEDIATE_SMALL; + } + metadata_buf_size = strtoull(config["meta_buf_size"].c_str(), NULL, 10); + inmemory_meta = config["inmemory_metadata"] != "false" && config["inmemory_metadata"] != "0" && + config["inmemory_metadata"] != "no"; + journal.sector_count = strtoull(config["journal_sector_buffer_count"].c_str(), NULL, 10); + journal.no_same_sector_overwrites = config["journal_no_same_sector_overwrites"] == "true" || + config["journal_no_same_sector_overwrites"] == "1" || config["journal_no_same_sector_overwrites"] == "yes"; + journal.inmemory = config["inmemory_journal"] != "false" && config["inmemory_journal"] != "0" && + config["inmemory_journal"] != "no"; + log_level = strtoull(config["log_level"].c_str(), NULL, 10); + // Validate + if (journal.sector_count < 2) + { + journal.sector_count = 32; + } + if (metadata_buf_size < 65536) + { + metadata_buf_size = 4*1024*1024; + } + if (dsk.meta_device == dsk.data_device) + { + disable_meta_fsync = disable_data_fsync; + } + if (dsk.journal_device == dsk.meta_device) + { + disable_journal_fsync = disable_meta_fsync; + } + if (immediate_commit != IMMEDIATE_NONE && !disable_journal_fsync) + { + throw std::runtime_error("immediate_commit requires disable_journal_fsync"); + } + if (immediate_commit == IMMEDIATE_ALL && !disable_data_fsync) + { + throw std::runtime_error("immediate_commit=all requires disable_journal_fsync and disable_data_fsync"); + } + // init some fields + journal.block_size = dsk.journal_block_size; + journal.next_free = dsk.journal_block_size; + journal.used_start = dsk.journal_block_size; + // no free space because sector is initially unmapped + journal.in_sector_pos = dsk.journal_block_size; +} + +void blockstore_impl_t::calc_lengths() +{ + dsk.calc_lengths(); + journal.len = dsk.journal_len; + journal.block_size = dsk.journal_block_size; + journal.offset = dsk.journal_offset; + if (inmemory_meta) + { + metadata_buffer = memalign(MEM_ALIGNMENT, dsk.meta_len); + if (!metadata_buffer) + throw std::runtime_error("Failed to allocate memory for the metadata ("+std::to_string(dsk.meta_len/1024/1024)+" MB)"); + } + else if (dsk.clean_entry_bitmap_size || dsk.data_csum_type) + { + clean_bitmaps = (uint8_t*)malloc(dsk.block_count * 2 * dsk.clean_entry_bitmap_size); + if (!clean_bitmaps) + { + throw std::runtime_error( + "Failed to allocate memory for the metadata sparse write bitmap ("+ + std::to_string(dsk.block_count * 2 * dsk.clean_entry_bitmap_size / 1024 / 1024)+" MB)" + ); + } + } + if (journal.inmemory) + { + journal.buffer = memalign(MEM_ALIGNMENT, journal.len); + if (!journal.buffer) + throw std::runtime_error("Failed to allocate memory for journal ("+std::to_string(journal.len/1024/1024)+" MB)"); + } + else + { + journal.sector_buf = (uint8_t*)memalign(MEM_ALIGNMENT, journal.sector_count * dsk.journal_block_size); + if (!journal.sector_buf) + throw std::bad_alloc(); + } + journal.sector_info = (journal_sector_info_t*)calloc(journal.sector_count, sizeof(journal_sector_info_t)); + if (!journal.sector_info) + { + throw std::bad_alloc(); + } +} diff --git a/src/blockstore/v1/read.cpp b/src/blockstore/v1/read.cpp new file mode 100644 index 00000000..4c30ac91 --- /dev/null +++ b/src/blockstore/v1/read.cpp @@ -0,0 +1,1033 @@ +// Copyright (c) Vitaliy Filippov, 2019+ +// License: VNPL-1.1 (see README.md for details) + +#include +#include "blockstore_impl.h" +#include "blockstore_internal.h" + +int blockstore_impl_t::fulfill_read_push(blockstore_op_t *op, void *buf, uint64_t offset, uint64_t len, + uint32_t item_state, uint64_t item_version) +{ + if (!len) + { + // Zero-length read + return 1; + } + else if (IS_DELETE(item_state)) + { + // item is unallocated - return zeroes + memset(buf, 0, len); + return 1; + } + assert(!IS_IN_FLIGHT(item_state)); + if (journal.inmemory && IS_JOURNAL(item_state)) + { + memcpy(buf, (uint8_t*)journal.buffer + offset, len); + return 1; + } + BS_SUBMIT_GET_SQE(sqe, data); + data->iov = (struct iovec){ buf, (size_t)len }; + PRIV(op)->pending_ops++; + io_uring_prep_readv( + sqe, + IS_JOURNAL(item_state) ? dsk.journal_fd : dsk.data_fd, + &data->iov, 1, + (IS_JOURNAL(item_state) ? dsk.journal_offset : dsk.data_offset) + offset + ); + data->callback = [this, op](ring_data_t *data) { handle_read_event(data, op); }; + return 1; +} + +void blockstore_impl_t::find_holes(std::vector & read_vec, + uint32_t item_start, uint32_t item_end, + std::function callback) +{ + auto cur_start = item_start; + int i = 0; + while (cur_start < item_end) + { + // COPY_BUF_CSUM_FILL items are fake items inserted in the end, their offsets aren't in order + if (i >= read_vec.size() || read_vec[i].copy_flags & COPY_BUF_CSUM_FILL || read_vec[i].offset >= item_end) + { + // Hole (at end): cur_start .. item_end + i += callback(i, false, cur_start, item_end); + break; + } + else if (read_vec[i].offset > cur_start) + { + // Hole: cur_start .. min(read_vec[i].offset, item_end) + auto cur_end = read_vec[i].offset > item_end ? item_end : read_vec[i].offset; + i += callback(i, false, cur_start, cur_end); + cur_start = cur_end; + } + else if (read_vec[i].offset + read_vec[i].len > cur_start) + { + // Allocated: cur_start .. min(read_vec[i].offset + read_vec[i].len, item_end) + auto cur_end = read_vec[i].offset + read_vec[i].len; + cur_end = cur_end > item_end ? item_end : cur_end; + i += callback(i, true, cur_start, cur_end); + cur_start = cur_end; + i++; + } + else + i++; + } +} + +int blockstore_impl_t::fulfill_read(blockstore_op_t *read_op, + uint64_t &fulfilled, uint32_t item_start, uint32_t item_end, // FIXME: Rename item_* to dirty_* + uint32_t item_state, uint64_t item_version, uint64_t item_location, + uint64_t journal_sector, uint8_t *csum, int *dyn_data) +{ + int r = 1; + if (item_start < read_op->offset + read_op->len && item_end > read_op->offset) + { + auto & rv = PRIV(read_op)->read_vec; + auto rd_start = item_start < read_op->offset ? read_op->offset : item_start; + auto rd_end = item_end > read_op->offset + read_op->len ? read_op->offset + read_op->len : item_end; + find_holes(rv, rd_start, rd_end, [&](int pos, bool alloc, uint32_t start, uint32_t end) + { + if (!r || alloc) + return 0; + if (!journal.inmemory && dsk.csum_block_size > dsk.bitmap_granularity && IS_JOURNAL(item_state) && !IS_DELETE(item_state)) + { + uint32_t blk_begin = (start/dsk.csum_block_size) * dsk.csum_block_size; + blk_begin = blk_begin < item_start ? item_start : blk_begin; + uint32_t blk_end = ((end-1) / dsk.csum_block_size + 1) * dsk.csum_block_size; + blk_end = blk_end > item_end ? item_end : blk_end; + rv.push_back((copy_buffer_t){ + .copy_flags = COPY_BUF_JOURNAL|COPY_BUF_CSUM_FILL, + .offset = blk_begin, + .len = blk_end-blk_begin, + .csum_buf = (csum + (blk_begin/dsk.csum_block_size - + item_start/dsk.csum_block_size) * (dsk.data_csum_type & 0xFF)), + .dyn_data = dyn_data, + }); + if (dyn_data) + { + (*dyn_data)++; + } + // Submit the journal checksum block read + if (!read_checksum_block(read_op, 1, fulfilled, item_location - item_start)) + { + r = 0; + } + return 0; + } + copy_buffer_t el = { + .copy_flags = (IS_JOURNAL(item_state) ? COPY_BUF_JOURNAL : COPY_BUF_DATA), + .offset = start, + .len = end-start, + .disk_offset = item_location + start - item_start, + .journal_sector = (IS_JOURNAL(item_state) ? journal_sector : 0), + .csum_buf = !csum ? NULL : (csum + (start - item_start) / dsk.csum_block_size * (dsk.data_csum_type & 0xFF)), + .dyn_data = dyn_data, + }; + if (dyn_data) + { + (*dyn_data)++; + } + if (IS_BIG_WRITE(item_state)) + { + // If we don't track it then we may IN THEORY read another object's data: + // submit read -> remove the object -> flush remove -> overwrite with another object -> finish read + // Very improbable, but possible + PRIV(read_op)->clean_block_used = 1; + } + rv.insert(rv.begin() + pos, el); + fulfilled += el.len; + if (!fulfill_read_push(read_op, + (uint8_t*)read_op->buf + el.offset - read_op->offset, + item_location + el.offset - item_start, + el.len, item_state, item_version)) + { + r = 0; + } + return 1; + }); + } + return r; +} + +uint8_t* blockstore_impl_t::get_clean_entry_bitmap(uint64_t block_loc, int offset) +{ + uint8_t *clean_entry_bitmap; + uint64_t meta_loc = block_loc / dsk.data_block_size; + if (inmemory_meta) + { + uint64_t sector = (meta_loc / (dsk.meta_block_size / dsk.clean_entry_size)) * dsk.meta_block_size; + uint64_t pos = (meta_loc % (dsk.meta_block_size / dsk.clean_entry_size)); + clean_entry_bitmap = ((uint8_t*)metadata_buffer + sector + pos*dsk.clean_entry_size + sizeof(clean_disk_entry) + offset); + } + else + clean_entry_bitmap = (uint8_t*)(clean_bitmaps + meta_loc*2*dsk.clean_entry_bitmap_size + offset); + return clean_entry_bitmap; +} + +int blockstore_impl_t::fill_partial_checksum_blocks(std::vector & rv, uint64_t & fulfilled, + uint8_t *clean_entry_bitmap, int *dyn_data, bool from_journal, uint8_t *read_buf, uint64_t read_offset, uint64_t read_end) +{ + if (read_end == read_offset) + return 0; + int required = 0; + read_buf -= read_offset; + uint32_t last_block = (read_end-1)/dsk.csum_block_size; + uint32_t start_block = read_offset/dsk.csum_block_size; + uint32_t end_block = 0; + while (start_block <= last_block) + { + if (read_range_fulfilled(rv, fulfilled, read_buf, clean_entry_bitmap, + start_block*dsk.csum_block_size < read_offset ? read_offset : start_block*dsk.csum_block_size, + (start_block+1)*dsk.csum_block_size > read_end ? read_end : (start_block+1)*dsk.csum_block_size)) + { + // read_range_fulfilled() also adds zero-filled areas + start_block++; + } + else + { + // Find a sequence of checksum blocks required to be read + end_block = start_block; + while ((end_block+1)*dsk.csum_block_size < read_end && + !read_range_fulfilled(rv, fulfilled, read_buf, clean_entry_bitmap, + (end_block+1)*dsk.csum_block_size < read_offset ? read_offset : (end_block+1)*dsk.csum_block_size, + (end_block+2)*dsk.csum_block_size > read_end ? read_end : (end_block+2)*dsk.csum_block_size)) + { + end_block++; + } + end_block++; + // OK, mark this range as required + rv.push_back((copy_buffer_t){ + .copy_flags = COPY_BUF_CSUM_FILL | (from_journal ? COPY_BUF_JOURNALED_BIG : 0), + .offset = start_block*dsk.csum_block_size, + .len = (end_block-start_block)*dsk.csum_block_size, + // save clean_entry_bitmap if we're reading clean data from the journal + .csum_buf = from_journal ? clean_entry_bitmap : NULL, + .dyn_data = dyn_data, + }); + if (dyn_data) + { + (*dyn_data)++; + } + start_block = end_block; + required++; + } + } + return required; +} + +// read_buf should be == op->buf - op->offset +bool blockstore_impl_t::read_range_fulfilled(std::vector & rv, uint64_t & fulfilled, uint8_t *read_buf, + uint8_t *clean_entry_bitmap, uint32_t item_start, uint32_t item_end) +{ + bool all_done = true; + find_holes(rv, item_start, item_end, [&](int pos, bool alloc, uint32_t cur_start, uint32_t cur_end) + { + if (alloc) + return 0; + int diff = 0; + uint32_t bmp_start = cur_start/dsk.bitmap_granularity; + uint32_t bmp_end = cur_end/dsk.bitmap_granularity; + uint32_t bmp_pos = bmp_start; + while (bmp_pos < bmp_end) + { + while (bmp_pos < bmp_end && !(clean_entry_bitmap[bmp_pos >> 3] & (1 << (bmp_pos & 0x7)))) + bmp_pos++; + if (bmp_pos > bmp_start) + { + // zero fill + copy_buffer_t el = { + .copy_flags = COPY_BUF_ZERO, + .offset = bmp_start*dsk.bitmap_granularity, + .len = (bmp_pos-bmp_start)*dsk.bitmap_granularity, + }; + rv.insert(rv.begin() + pos, el); + if (read_buf) + memset(read_buf + el.offset, 0, el.len); + fulfilled += el.len; + diff++; + } + bmp_start = bmp_pos; + while (bmp_pos < bmp_end && (clean_entry_bitmap[bmp_pos >> 3] & (1 << (bmp_pos & 0x7)))) + bmp_pos++; + if (bmp_pos > bmp_start) + { + // something is to be read + all_done = false; + } + bmp_start = bmp_pos; + } + return diff; + }); + return all_done; +} + +bool blockstore_impl_t::read_checksum_block(blockstore_op_t *op, int rv_pos, uint64_t &fulfilled, uint64_t clean_loc) +{ + auto & rv = PRIV(op)->read_vec; + auto *vi = &rv[rv.size()-rv_pos]; + uint32_t item_start = vi->offset, item_end = vi->offset+vi->len; + uint32_t fill_size = 0; + int n_iov = 0; + find_holes(rv, item_start, item_end, [&](int pos, bool alloc, uint32_t cur_start, uint32_t cur_end) + { + if (alloc) + { + fill_size += cur_end-cur_start; + n_iov++; + } + else + { + if (cur_start < op->offset) + { + fill_size += op->offset-cur_start; + n_iov++; + cur_start = op->offset; + } + if (cur_end > op->offset+op->len) + { + fill_size += cur_end-(op->offset+op->len); + n_iov++; + cur_end = op->offset+op->len; + } + if (cur_end > cur_start) + { + n_iov++; + } + } + return 0; + }); + void *buf = memalign_or_die(MEM_ALIGNMENT, fill_size + n_iov*sizeof(struct iovec)); + iovec *iov = (struct iovec*)((uint8_t*)buf+fill_size); + n_iov = 0; + fill_size = 0; + find_holes(rv, item_start, item_end, [&](int pos, bool alloc, uint32_t cur_start, uint32_t cur_end) + { + int res = 0; + if (alloc) + { + iov[n_iov++] = (struct iovec){ (uint8_t*)buf+fill_size, cur_end-cur_start }; + fill_size += cur_end-cur_start; + } + else + { + if (cur_start < op->offset) + { + iov[n_iov++] = (struct iovec){ (uint8_t*)buf+fill_size, op->offset-cur_start }; + fill_size += op->offset-cur_start; + cur_start = op->offset; + } + auto lim_end = cur_end > op->offset+op->len ? op->offset+op->len : cur_end; + if (lim_end > cur_start) + { + iov[n_iov++] = (struct iovec){ (uint8_t*)op->buf+cur_start-op->offset, lim_end-cur_start }; + rv.insert(rv.begin() + pos, (copy_buffer_t){ + .copy_flags = COPY_BUF_DATA, + .offset = cur_start, + .len = lim_end-cur_start, + }); + fulfilled += lim_end-cur_start; + res++; + } + if (cur_end > op->offset+op->len) + { + iov[n_iov++] = (struct iovec){ (uint8_t*)buf+fill_size, cur_end - (op->offset+op->len) }; + fill_size += cur_end - (op->offset+op->len); + cur_end = op->offset+op->len; + } + } + return res; + }); + vi = &rv[rv.size()-rv_pos]; + // Save buf into read_vec too but in a creepy way + // FIXME: Shit, something else should be invented %) + *vi = (copy_buffer_t){ + .copy_flags = vi->copy_flags, + .offset = vi->offset, + .len = ((uint64_t)n_iov << 32) | fill_size, + .disk_offset = clean_loc + item_start, + .buf = (uint8_t*)buf, + .csum_buf = vi->csum_buf, + .dyn_data = vi->dyn_data, + }; + int submit_fd = (vi->copy_flags & COPY_BUF_JOURNAL ? dsk.journal_fd : dsk.data_fd); + uint64_t submit_offset = (vi->copy_flags & COPY_BUF_JOURNAL ? journal.offset : dsk.data_offset); + uint32_t d_pos = 0; + for (int n_pos = 0; n_pos < n_iov; n_pos += IOV_MAX) + { + int n_cur = n_iov-n_pos < IOV_MAX ? n_iov-n_pos : IOV_MAX; + BS_SUBMIT_GET_SQE(sqe, data); + PRIV(op)->pending_ops++; + io_uring_prep_readv(sqe, submit_fd, iov + n_pos, n_cur, submit_offset + clean_loc + item_start + d_pos); + data->callback = [this, op](ring_data_t *data) { handle_read_event(data, op); }; + if (n_pos > 0 || n_pos + IOV_MAX < n_iov) + { + uint32_t d_len = 0; + for (int i = 0; i < IOV_MAX; i++) + d_len += iov[n_pos+i].iov_len; + data->iov.iov_len = d_len; + d_pos += d_len; + } + else + data->iov.iov_len = item_end-item_start; + } + if (!(vi->copy_flags & COPY_BUF_JOURNAL)) + { + // Reads running parallel to flushes of the same clean block may read + // a mixture of old and new data. So we don't verify checksums for such blocks. + PRIV(op)->clean_block_used = 1; + } + return true; +} + +int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op) +{ + auto & clean_db = clean_db_shard(read_op->oid); + auto clean_it = clean_db.find(read_op->oid); + auto dirty_it = dirty_db.upper_bound((obj_ver_id){ + .oid = read_op->oid, + .version = UINT64_MAX, + }); + if (dirty_it != dirty_db.begin()) + dirty_it--; + bool clean_found = clean_it != clean_db.end(); + bool dirty_found = (dirty_it != dirty_db.end() && dirty_it->first.oid == read_op->oid); + if (!clean_found && !dirty_found) + { + read_op->version = 0; + read_op->retval = -ENOENT; + FINISH_OP(read_op); + return 2; + } + uint64_t fulfilled = 0; + PRIV(read_op)->pending_ops = 0; + PRIV(read_op)->clean_block_used = 0; + auto & rv = PRIV(read_op)->read_vec; + uint64_t result_version = 0; + if (dirty_found) + { + while (dirty_it->first.oid == read_op->oid) + { + dirty_entry& dirty = dirty_it->second; + bool version_ok = !IS_IN_FLIGHT(dirty.state) && read_op->version >= dirty_it->first.version; + if (version_ok) + { + if (IS_DELETE(dirty.state)) + { + assert(!result_version); + read_op->version = 0; + read_op->retval = -ENOENT; + FINISH_OP(read_op); + return 2; + } + int *dyn_data = (int*)(dsk.csum_block_size > 0 && alloc_dyn_data ? dirty.dyn_data : NULL); + uint8_t *bmp_ptr = (alloc_dyn_data + ? (uint8_t*)dirty.dyn_data + sizeof(int) : (uint8_t*)&dirty.dyn_data); + if (!result_version) + { + result_version = dirty_it->first.version; + if (read_op->bitmap) + { + memcpy(read_op->bitmap, bmp_ptr, dsk.clean_entry_bitmap_size); + } + } + // If inmemory_journal is false, journal trim will have to wait until the read is completed + if (!IS_JOURNAL(dirty.state)) + { + // Read from data disk, possibly checking checksums + if (!fulfill_clean_read(read_op, fulfilled, bmp_ptr, dyn_data, + dirty.offset, dirty.offset+dirty.len, dirty.location, dirty_it->first.version)) + { + goto undo_read; + } + } + else + { + // Copy from memory or read from journal, possibly checking checksums + if (!fulfill_read(read_op, fulfilled, dirty.offset, dirty.offset + dirty.len, + dirty.state, dirty_it->first.version, dirty.location, dirty.journal_sector+1, + journal.inmemory ? NULL : bmp_ptr+dsk.clean_entry_bitmap_size, dyn_data)) + { + goto undo_read; + } + } + } + if (fulfilled == read_op->len || dirty_it == dirty_db.begin()) + { + break; + } + dirty_it--; + } + } + if (clean_found) + { + if (!result_version) + { + result_version = clean_it->second.version; + if (read_op->bitmap) + { + void *bmp_ptr = get_clean_entry_bitmap(clean_it->second.location, dsk.clean_entry_bitmap_size); + memcpy(read_op->bitmap, bmp_ptr, dsk.clean_entry_bitmap_size); + } + } + if (fulfilled < read_op->len) + { + if (!fulfill_clean_read(read_op, fulfilled, NULL, NULL, 0, dsk.data_block_size, + clean_it->second.location, clean_it->second.version)) + { + goto undo_read; + } + } + } + if (!result_version) + { + // May happen if there are entries in dirty_db but all of them are !version_ok + read_op->version = 0; + read_op->retval = -ENOENT; + FINISH_OP(read_op); + return 2; + } + assert(fulfilled == read_op->len); + read_op->version = result_version; + if (!PRIV(read_op)->pending_ops) + { + // everything is fulfilled from memory + if (!PRIV(read_op)->read_vec.size()) + { + // region is not allocated - return zeroes + memset(read_op->buf, 0, read_op->len); + } + read_op->retval = read_op->len; + FINISH_OP(read_op); + return 2; + } + if (!journal.inmemory) + { + // Journal trim has to wait until the read is completed - record journal sector usage + for (auto & rv: PRIV(read_op)->read_vec) + { + if (rv.journal_sector) + journal.used_sectors.at(rv.journal_sector-1)++; + } + } + read_op->retval = 0; + return 2; +undo_read: + // need to wait. undo added requests, don't dequeue op + if (dsk.csum_block_size > dsk.bitmap_granularity) + { + for (auto & vec: rv) + { + if ((vec.copy_flags & COPY_BUF_CSUM_FILL) && vec.buf) + { + free(vec.buf); + vec.buf = NULL; + } + if (vec.dyn_data && --(*vec.dyn_data) == 0) // refcount + { + free(vec.dyn_data); + vec.dyn_data = NULL; + } + } + } + rv.clear(); + return 0; +} + +int blockstore_impl_t::pad_journal_read(std::vector & rv, copy_buffer_t & cp, + // FIXME Passing dirty_entry& would be nicer + uint64_t dirty_offset, uint64_t dirty_end, uint64_t dirty_loc, uint8_t *csum_ptr, int *dyn_data, + uint64_t offset, uint64_t submit_len, uint64_t & blk_begin, uint64_t & blk_end, uint8_t* & blk_buf) +{ + if (offset % dsk.csum_block_size || submit_len % dsk.csum_block_size) + { + if (offset < blk_end) + { + // Already being read as a part of the previous checksum block series + cp.buf = blk_buf + offset - blk_begin; + cp.copy_flags |= COPY_BUF_COALESCED; + if (offset+submit_len > blk_end) + cp.len = blk_end-offset; + return 2; + } + else + { + // We don't use fill_partial_checksum_blocks for journal because journal writes never have holes (internal bitmap) + blk_begin = (offset/dsk.csum_block_size) * dsk.csum_block_size; + blk_begin = blk_begin < dirty_offset ? dirty_offset : blk_begin; + blk_end = ((offset+submit_len-1)/dsk.csum_block_size + 1) * dsk.csum_block_size; + blk_end = blk_end > dirty_end ? dirty_end : blk_end; + if (blk_begin < offset || blk_end > offset+submit_len) + { + blk_buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, blk_end-blk_begin); + cp.buf = blk_buf + offset - blk_begin; + cp.copy_flags |= COPY_BUF_COALESCED; + rv.push_back((copy_buffer_t){ + .copy_flags = COPY_BUF_JOURNAL|COPY_BUF_CSUM_FILL, + .offset = blk_begin, + .len = blk_end-blk_begin, + .disk_offset = dirty_loc + blk_begin - dirty_offset, + .buf = blk_buf, + .csum_buf = (csum_ptr + (blk_begin/dsk.csum_block_size - + dirty_offset/dsk.csum_block_size) * (dsk.data_csum_type & 0xFF)), + .dyn_data = dyn_data, + }); + if (dyn_data) + { + (*dyn_data)++; + } + return 1; + } + } + } + return 0; +} + +bool blockstore_impl_t::fulfill_clean_read(blockstore_op_t *read_op, uint64_t & fulfilled, + uint8_t *clean_entry_bitmap, int *dyn_data, uint32_t item_start, uint32_t item_end, uint64_t clean_loc, uint64_t clean_ver) +{ + bool from_journal = clean_entry_bitmap != NULL; + if (!clean_entry_bitmap) + { + // NULL clean_entry_bitmap means we're reading from data, not from the journal, + // and the bitmap location is obvious + clean_entry_bitmap = get_clean_entry_bitmap(clean_loc, 0); + } + if (dsk.csum_block_size > dsk.bitmap_granularity) + { + auto & rv = PRIV(read_op)->read_vec; + int req = fill_partial_checksum_blocks(rv, fulfilled, clean_entry_bitmap, dyn_data, from_journal, + (uint8_t*)read_op->buf, read_op->offset, read_op->offset+read_op->len); + if (!inmemory_meta && !from_journal && req > 0) + { + // Read checksums from disk + uint8_t *csum_buf = read_clean_meta_block(read_op, clean_loc, rv.size()-req); + for (int i = req; i > 0; i--) + { + rv[rv.size()-i].csum_buf = csum_buf; + } + } + for (int i = req; i > 0; i--) + { + if (!read_checksum_block(read_op, i, fulfilled, clean_loc)) + { + return false; + } + } + PRIV(read_op)->clean_block_used = req > 0; + } + else if (from_journal) + { + // Don't scan bitmap - journal writes don't have holes (internal bitmap)! + uint8_t *csum = !dsk.csum_block_size ? 0 : (clean_entry_bitmap + dsk.clean_entry_bitmap_size + + item_start/dsk.csum_block_size*(dsk.data_csum_type & 0xFF)); + if (!fulfill_read(read_op, fulfilled, item_start, item_end, + (BS_ST_BIG_WRITE | BS_ST_STABLE), 0, clean_loc + item_start, 0, csum, dyn_data)) + { + return false; + } + if (item_start > 0 && fulfilled < read_op->len) + { + // fill with zeroes + assert(fulfill_read(read_op, fulfilled, 0, item_start, (BS_ST_DELETE | BS_ST_STABLE), 0, 0, 0, NULL, NULL)); + } + if (item_end < dsk.data_block_size && fulfilled < read_op->len) + { + // fill with zeroes + assert(fulfill_read(read_op, fulfilled, item_end, dsk.data_block_size, (BS_ST_DELETE | BS_ST_STABLE), 0, 0, 0, NULL, NULL)); + } + } + else + { + bool csum_done = !dsk.csum_block_size || inmemory_meta; + uint8_t *csum_buf = clean_entry_bitmap; + uint64_t bmp_start = 0, bmp_end = 0, bmp_size = dsk.data_block_size/dsk.bitmap_granularity; + while (bmp_start < bmp_size) + { + while (!(clean_entry_bitmap[bmp_end >> 3] & (1 << (bmp_end & 0x7))) && bmp_end < bmp_size) + { + bmp_end++; + } + if (bmp_end > bmp_start) + { + // fill with zeroes + assert(fulfill_read(read_op, fulfilled, bmp_start * dsk.bitmap_granularity, + bmp_end * dsk.bitmap_granularity, (BS_ST_DELETE | BS_ST_STABLE), 0, 0, 0, NULL, NULL)); + } + bmp_start = bmp_end; + while (clean_entry_bitmap[bmp_end >> 3] & (1 << (bmp_end & 0x7)) && bmp_end < bmp_size) + { + bmp_end++; + } + if (bmp_end > bmp_start) + { + if (!csum_done) + { + // Read checksums from disk + csum_buf = read_clean_meta_block(read_op, clean_loc, PRIV(read_op)->read_vec.size()); + csum_done = true; + } + uint8_t *csum = !dsk.csum_block_size ? 0 : (csum_buf + 2*dsk.clean_entry_bitmap_size + bmp_start*(dsk.data_csum_type & 0xFF)); + if (!fulfill_read(read_op, fulfilled, bmp_start * dsk.bitmap_granularity, + bmp_end * dsk.bitmap_granularity, (BS_ST_BIG_WRITE | BS_ST_STABLE), 0, + clean_loc + bmp_start * dsk.bitmap_granularity, 0, csum, dyn_data)) + { + return false; + } + bmp_start = bmp_end; + } + } + } + // Increment reference counter if clean data is being read from the disk + if (PRIV(read_op)->clean_block_used) + { + auto & uo = used_clean_objects[clean_loc]; + uo.refs++; + if (dsk.csum_block_size && flusher->is_mutated(clean_loc)) + uo.was_changed = true; + PRIV(read_op)->clean_block_used = clean_loc; + } + return true; +} + +uint8_t* blockstore_impl_t::read_clean_meta_block(blockstore_op_t *op, uint64_t clean_loc, int rv_pos) +{ + auto & rv = PRIV(op)->read_vec; + auto sector = ((clean_loc / dsk.data_block_size) / (dsk.meta_block_size / dsk.clean_entry_size)) * dsk.meta_block_size; + auto pos = ((clean_loc / dsk.data_block_size) % (dsk.meta_block_size / dsk.clean_entry_size)) * dsk.clean_entry_size; + uint8_t *buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size); + rv.insert(rv.begin()+rv_pos, (copy_buffer_t){ + .copy_flags = COPY_BUF_META_BLOCK|COPY_BUF_CSUM_FILL, + .offset = pos, + .buf = buf, + }); + BS_SUBMIT_GET_SQE(sqe, data); + data->iov = (struct iovec){ buf, (size_t)dsk.meta_block_size }; + PRIV(op)->pending_ops++; + io_uring_prep_readv(sqe, dsk.meta_fd, &data->iov, 1, dsk.meta_offset + dsk.meta_block_size + sector); + data->callback = [this, op](ring_data_t *data) { handle_read_event(data, op); }; + // return pointer to checksums + bitmap + return buf + pos + sizeof(clean_disk_entry); +} + +bool blockstore_impl_t::verify_padded_checksums(uint8_t *clean_entry_bitmap, uint8_t *csum_buf, uint32_t offset, + iovec *iov, int n_iov, std::function bad_block_cb) +{ + assert(!(offset % dsk.csum_block_size)); + uint32_t *csums = (uint32_t*)csum_buf; + uint32_t block_csum = 0; + uint32_t block_done = 0; + uint32_t block_num = clean_entry_bitmap ? offset/dsk.csum_block_size : 0; + uint32_t bmp_pos = offset/dsk.bitmap_granularity; + for (int i = 0; i < n_iov; i++) + { + uint32_t pos = 0; + while (pos < iov[i].iov_len) + { + uint32_t start = pos; + uint8_t bit = (clean_entry_bitmap[bmp_pos >> 3] >> (bmp_pos & 0x7)) & 1; + while (pos < iov[i].iov_len && ((clean_entry_bitmap[bmp_pos >> 3] >> (bmp_pos & 0x7)) & 1) == bit) + { + pos += dsk.bitmap_granularity; + bmp_pos++; + } + uint32_t len = pos-start; + auto buf = (uint8_t*)iov[i].iov_base+start; + while (block_done+len >= dsk.csum_block_size) + { + auto cur_len = dsk.csum_block_size-block_done; + block_csum = crc32c_pad(block_csum, buf, bit ? cur_len : 0, bit ? 0 : cur_len, 0); + if (block_csum != csums[block_num]) + { + if (bad_block_cb) + bad_block_cb(block_num*dsk.csum_block_size, block_csum, csums[block_num]); + else + return false; + } + block_num++; + buf += cur_len; + len -= cur_len; + block_done = block_csum = 0; + } + if (len > 0) + { + block_csum = crc32c_pad(block_csum, buf, bit ? len : 0, bit ? 0 : len, 0); + block_done += len; + } + } + } + assert(!block_done); + return true; +} + +bool blockstore_impl_t::verify_journal_checksums(uint8_t *csums, uint32_t offset, + iovec *iov, int n_iov, std::function bad_block_cb) +{ + uint32_t block_csum = 0; + uint32_t block_num = 0; + uint32_t block_done = offset%dsk.csum_block_size; + for (int i = 0; i < n_iov; i++) + { + uint32_t len = iov[i].iov_len; + auto buf = (uint8_t*)iov[i].iov_base; + while (block_done+len >= dsk.csum_block_size) + { + auto cur_len = dsk.csum_block_size-block_done; + block_csum = crc32c(block_csum, buf, cur_len); + if (block_csum != ((uint32_t*)csums)[block_num]) + { + if (bad_block_cb) + bad_block_cb(block_num*dsk.csum_block_size, block_csum, ((uint32_t*)csums)[block_num]); + else + return false; + } + block_num++; + buf += cur_len; + len -= cur_len; + block_done = block_csum = 0; + } + if (len > 0) + { + block_csum = crc32c(block_csum, buf, len); + block_done += len; + } + } + if (block_done > 0 && block_csum != ((uint32_t*)csums)[block_num]) + { + if (bad_block_cb) + bad_block_cb(block_num*dsk.csum_block_size, block_csum, ((uint32_t*)csums)[block_num]); + else + return false; + } + return true; +} + +bool blockstore_impl_t::verify_clean_padded_checksums(blockstore_op_t *op, uint64_t clean_loc, uint8_t *dyn_data, bool from_journal, + iovec *iov, int n_iov, std::function bad_block_cb) +{ + uint32_t offset = clean_loc % dsk.data_block_size; + if (from_journal) + return verify_padded_checksums(dyn_data, dyn_data + dsk.clean_entry_bitmap_size, offset, iov, n_iov, bad_block_cb); + clean_loc = (clean_loc / dsk.data_block_size) * dsk.data_block_size; + if (!dyn_data) + { + assert(inmemory_meta); + dyn_data = get_clean_entry_bitmap(clean_loc, 0); + } + return verify_padded_checksums(dyn_data, dyn_data + 2*dsk.clean_entry_bitmap_size, offset, iov, n_iov, bad_block_cb); +} + +void blockstore_impl_t::handle_read_event(ring_data_t *data, blockstore_op_t *op) +{ + live = true; + PRIV(op)->pending_ops--; + if (data->res != data->iov.iov_len) + { + // read error + op->retval = data->res; + } + if (PRIV(op)->pending_ops == 0) + { + if (dsk.csum_block_size) + { + // verify checksums if required + auto & rv = PRIV(op)->read_vec; + void *meta_block = NULL; + if (dsk.csum_block_size > dsk.bitmap_granularity) + { + for (int i = rv.size()-1; i >= 0 && (rv[i].copy_flags & COPY_BUF_CSUM_FILL); i--) + { + if (rv[i].copy_flags & COPY_BUF_META_BLOCK) + { + // Metadata read. Skip + assert(!meta_block); + meta_block = rv[i].buf; + rv[i].buf = NULL; + continue; + } + struct iovec *iov = (struct iovec*)((uint8_t*)rv[i].buf + (rv[i].len & 0xFFFFFFFF)); + int n_iov = rv[i].len >> 32; + bool ok = true; + if (rv[i].copy_flags & COPY_BUF_JOURNAL) + { + // SMALL_WRITE from journal + verify_journal_checksums( + rv[i].csum_buf, rv[i].offset, iov, n_iov, + [&](uint32_t bad_block, uint32_t calc_csum, uint32_t stored_csum) + { + ok = false; + printf( + "Checksum mismatch in object %jx:%jx v%ju in journal at 0x%jx, checksum block #%u: got %08x, expected %08x\n", + op->oid.inode, op->oid.stripe, op->version, + rv[i].disk_offset, bad_block / dsk.csum_block_size, calc_csum, stored_csum + ); + } + ); + } + else + { + // BIG_WRITE from journal or clean data + // Do not verify checksums if the data location is/was mutated by flushers + auto & uo = used_clean_objects.at((rv[i].disk_offset / dsk.data_block_size) * dsk.data_block_size); + if (!uo.was_changed) + { + verify_clean_padded_checksums( + op, rv[i].disk_offset, rv[i].csum_buf, (rv[i].copy_flags & COPY_BUF_JOURNALED_BIG), iov, n_iov, + [&](uint32_t bad_block, uint32_t calc_csum, uint32_t stored_csum) + { + ok = false; + printf( + "Checksum mismatch in object %jx:%jx v%ju in %s data at 0x%jx, checksum block #%u: got %08x, expected %08x\n", + op->oid.inode, op->oid.stripe, op->version, + (rv[i].copy_flags & COPY_BUF_JOURNALED_BIG ? "redirect-write" : "clean"), + rv[i].disk_offset, bad_block / dsk.csum_block_size, calc_csum, stored_csum + ); + } + ); + } + } + if (!ok) + { + op->retval = -EDOM; + } + free(rv[i].buf); + rv[i].buf = NULL; + if (rv[i].dyn_data && --(*rv[i].dyn_data) == 0) // refcount + { + free(rv[i].dyn_data); + rv[i].dyn_data = NULL; + } + } + } + else + { + for (auto & vec: rv) + { + if (vec.copy_flags & COPY_BUF_META_BLOCK) + { + // Metadata read. Skip + assert(!meta_block); + meta_block = vec.buf; + vec.buf = NULL; + continue; + } + if (vec.csum_buf) + { + uint32_t *csum = (uint32_t*)vec.csum_buf; + for (size_t p = 0; p < vec.len; p += dsk.csum_block_size, csum++) + { + if (crc32c(0, (uint8_t*)op->buf + vec.offset - op->offset + p, dsk.csum_block_size) != *csum) + { + // checksum error + printf( + "Checksum mismatch in object %jx:%jx v%ju in %s area at offset 0x%jx+0x%zx: %08x vs %08x\n", + op->oid.inode, op->oid.stripe, op->version, + (vec.copy_flags & COPY_BUF_JOURNAL) ? "journal" : "data", vec.disk_offset, p, + crc32c(0, (uint8_t*)op->buf + vec.offset - op->offset + p, dsk.csum_block_size), *csum + ); + op->retval = -EDOM; + break; + } + } + } + if (vec.dyn_data && --(*vec.dyn_data) == 0) // refcount + { + free(vec.dyn_data); + vec.dyn_data = NULL; + } + } + } + if (meta_block) + { + // Free after checking + free(meta_block); + meta_block = NULL; + } + } + if (PRIV(op)->clean_block_used) + { + // Release clean data block + auto uo_it = used_clean_objects.find(PRIV(op)->clean_block_used); + if (uo_it != used_clean_objects.end()) + { + uo_it->second.refs--; + if (uo_it->second.refs <= 0) + { + if (uo_it->second.was_freed) + { + data_alloc->set(PRIV(op)->clean_block_used, false); + } + used_clean_objects.erase(uo_it); + } + } + } + if (!journal.inmemory) + { + // Release journal sector usage + for (auto & rv: PRIV(op)->read_vec) + { + if (rv.journal_sector) + { + auto used = --journal.used_sectors.at(rv.journal_sector-1); + if (used == 0) + { + journal.used_sectors.erase(rv.journal_sector-1); + flusher->mark_trim_possible(); + } + } + } + } + if (op->retval == 0) + op->retval = op->len; + FINISH_OP(op); + } +} + +int blockstore_impl_t::read_bitmap(object_id oid, uint64_t target_version, void *bitmap, uint64_t *result_version) +{ + auto dirty_it = dirty_db.upper_bound((obj_ver_id){ + .oid = oid, + .version = UINT64_MAX, + }); + if (dirty_it != dirty_db.begin()) + dirty_it--; + if (dirty_it != dirty_db.end()) + { + while (dirty_it->first.oid == oid) + { + // Condition has to be the same as in dequeue_read() + if (!IS_IN_FLIGHT(dirty_it->second.state) && target_version >= dirty_it->first.version) + { + if (result_version) + *result_version = dirty_it->first.version; + if (bitmap) + { + void *dyn_ptr = (alloc_dyn_data + ? (uint8_t*)dirty_it->second.dyn_data + sizeof(int) : (uint8_t*)&dirty_it->second.dyn_data); + memcpy(bitmap, dyn_ptr, dsk.clean_entry_bitmap_size); + } + return 0; + } + if (dirty_it == dirty_db.begin()) + break; + dirty_it--; + } + } + auto & clean_db = clean_db_shard(oid); + auto clean_it = clean_db.find(oid); + if (clean_it != clean_db.end()) + { + if (result_version) + *result_version = clean_it->second.version; + if (bitmap) + { + void *bmp_ptr = get_clean_entry_bitmap(clean_it->second.location, dsk.clean_entry_bitmap_size); + memcpy(bitmap, bmp_ptr, dsk.clean_entry_bitmap_size); + } + return 0; + } + if (result_version) + *result_version = 0; + if (bitmap) + memset(bitmap, 0, dsk.clean_entry_bitmap_size); + return -ENOENT; +} diff --git a/src/blockstore/blockstore_rollback.cpp b/src/blockstore/v1/rollback.cpp similarity index 100% rename from src/blockstore/blockstore_rollback.cpp rename to src/blockstore/v1/rollback.cpp diff --git a/src/blockstore/v1/stable.cpp b/src/blockstore/v1/stable.cpp new file mode 100644 index 00000000..2559bec8 --- /dev/null +++ b/src/blockstore/v1/stable.cpp @@ -0,0 +1,562 @@ +// Copyright (c) Vitaliy Filippov, 2019+ +// License: VNPL-1.1 (see README.md for details) + +#include "blockstore_impl.h" +#include "blockstore_internal.h" + +// Stabilize small write: +// 1) Copy data from the journal to the data device +// 2) Increase version on the metadata device and sync it +// 3) Advance clean_db entry's version, clear previous journal entries +// +// This makes 1 4K small write+sync look like: +// 512b+4K (journal) + sync + 512b (journal) + sync + 4K (data) [+ sync?] + 512b (metadata) + sync. +// WA = 2.375. It's not the best, SSD FTL-like redirect-write could probably be lower +// even with defragmentation. But it's fixed and it's still better than in Ceph. :) +// except for HDD-only clusters, because each write results in 3 seeks. + +// Stabilize big write: +// 1) Copy metadata from the journal to the metadata device +// 2) Move dirty_db entry to clean_db and clear previous journal entries +// +// This makes 1 128K big write+sync look like: +// 128K (data) + sync + 512b (journal) + sync + 512b (journal) + sync + 512b (metadata) + sync. +// WA = 1.012. Very good :) + +// Stabilize delete: +// 1) Remove metadata entry and sync it +// 2) Remove dirty_db entry and clear previous journal entries +// We have 2 problems here: +// - In the cluster environment, we must store the "tombstones" of deleted objects until +// all replicas (not just quorum) agrees about their deletion. That is, "stabilize" is +// not possible for deletes in degraded placement groups +// - With simple "fixed" metadata tables we can't just clear the metadata entry of the latest +// object version. We must clear all previous entries, too. +// FIXME Fix both problems - probably, by switching from "fixed" metadata tables to "dynamic" + +// AND We must do it in batches, for the sake of reduced fsync call count +// AND We must know what we stabilize. Basic workflow is like: +// 1) primary OSD receives sync request +// 2) it submits syncs to blockstore and peers +// 3) after everyone acks sync it acks sync to the client +// 4) after a while it takes his synced object list and sends stabilize requests +// to peers and to its own blockstore, thus freeing the old version + +struct ver_vector_t +{ + obj_ver_id *items = NULL; + uint64_t alloc = 0, size = 0; +}; + +static void init_versions(ver_vector_t & vec, obj_ver_id *start, obj_ver_id *end, uint64_t len) +{ + if (!vec.items) + { + vec.alloc = len; + vec.items = (obj_ver_id*)malloc_or_die(sizeof(obj_ver_id) * vec.alloc); + for (auto sv = start; sv < end; sv++) + { + vec.items[vec.size++] = *sv; + } + } +} + +static void append_version(ver_vector_t & vec, obj_ver_id ov) +{ + if (vec.size >= vec.alloc) + { + vec.alloc = !vec.alloc ? 4 : vec.alloc*2; + vec.items = (obj_ver_id*)realloc_or_die(vec.items, sizeof(obj_ver_id) * vec.alloc); + } + vec.items[vec.size++] = ov; +} + +static bool check_unsynced(std::vector & check, obj_ver_id ov, std::vector & to, int *count) +{ + bool found = false; + int j = 0, k = 0; + while (j < check.size()) + { + if (check[j] == ov) + found = true; + if (check[j].oid == ov.oid && check[j].version <= ov.version) + { + to.push_back(check[j++]); + if (count) + (*count)--; + } + else + check[k++] = check[j++]; + } + check.resize(k); + return found; +} + +blockstore_op_t* blockstore_impl_t::selective_sync(blockstore_op_t *op) +{ + unsynced_big_write_count -= unsynced_big_writes.size(); + unsynced_big_writes.swap(PRIV(op)->sync_big_writes); + unsynced_big_write_count += unsynced_big_writes.size(); + unsynced_small_writes.swap(PRIV(op)->sync_small_writes); + // Create a sync operation, insert into the end of the queue + // And move ourselves into the end too! + // Rather hacky but that's what we need... + blockstore_op_t *sync_op = new blockstore_op_t; + sync_op->opcode = BS_OP_SYNC; + sync_op->buf = NULL; + sync_op->callback = [](blockstore_op_t *sync_op) + { + delete sync_op; + }; + init_op(sync_op); + int sync_res = continue_sync(sync_op); + if (sync_res != 2) + { + // Put SYNC into the queue if it's not finished yet + submit_queue.push_back(sync_op); + } + // Restore unsynced_writes + unsynced_small_writes.swap(PRIV(op)->sync_small_writes); + unsynced_big_write_count -= unsynced_big_writes.size(); + unsynced_big_writes.swap(PRIV(op)->sync_big_writes); + unsynced_big_write_count += unsynced_big_writes.size(); + if (sync_res == 2) + { + // Sync is immediately completed + return NULL; + } + return sync_op; +} + +// Returns: 2 = stop processing and dequeue, 0 = stop processing and do not dequeue, 1 = proceed with op itself +int blockstore_impl_t::split_stab_op(blockstore_op_t *op, std::function decider) +{ + bool add_sync = false; + ver_vector_t good_vers, bad_vers; + obj_ver_id* v; + int i, todo = 0; + for (i = 0, v = (obj_ver_id*)op->buf; i < op->len; i++, v++) + { + int action = decider(*v); + if (action < 0) + { + // Rollback changes + for (auto & ov: PRIV(op)->sync_big_writes) + { + unsynced_big_writes.push_back(ov); + unsynced_big_write_count++; + } + for (auto & ov: PRIV(op)->sync_small_writes) + { + unsynced_small_writes.push_back(ov); + } + free(good_vers.items); + good_vers.items = NULL; + free(bad_vers.items); + bad_vers.items = NULL; + // Error + op->retval = action; + FINISH_OP(op); + return 2; + } + else if (action == STAB_SPLIT_DONE) + { + // Already done + init_versions(good_vers, (obj_ver_id*)op->buf, v, op->len); + } + else if (action == STAB_SPLIT_WAIT) + { + // Already in progress, we just have to wait until it finishes + init_versions(good_vers, (obj_ver_id*)op->buf, v, op->len); + append_version(bad_vers, *v); + } + else if (action == STAB_SPLIT_SYNC) + { + // Needs a SYNC, we have to send a SYNC if not already in progress + // + // If the object is not present in unsynced_(big|small)_writes then + // it's currently being synced. If it's present then we can initiate + // its sync ourselves. + init_versions(good_vers, (obj_ver_id*)op->buf, v, op->len); + append_version(bad_vers, *v); + if (!add_sync) + { + PRIV(op)->sync_big_writes.clear(); + PRIV(op)->sync_small_writes.clear(); + add_sync = true; + } + check_unsynced(unsynced_small_writes, *v, PRIV(op)->sync_small_writes, NULL); + check_unsynced(unsynced_big_writes, *v, PRIV(op)->sync_big_writes, &unsynced_big_write_count); + } + else /* if (action == STAB_SPLIT_TODO) */ + { + if (good_vers.items) + { + // If we're selecting versions then append it + // Main idea is that 99% of the time all versions passed to BS_OP_STABLE are synced + // And we don't want to select/allocate anything in that optimistic case + append_version(good_vers, *v); + } + todo++; + } + } + // In a pessimistic scenario, an operation may be split into 3: + // - Stabilize synced entries + // - Sync unsynced entries + // - Continue for unsynced entries after sync + add_sync = add_sync && (PRIV(op)->sync_big_writes.size() || PRIV(op)->sync_small_writes.size()); + if (!todo && !bad_vers.size) + { + // Already stable + op->retval = 0; + FINISH_OP(op); + return 2; + } + op->retval = 0; + if (!todo && !add_sync) + { + // Only wait for inflight writes or current in-progress syncs + return 0; + } + blockstore_op_t *sync_op = NULL, *split_stab_op = NULL; + if (add_sync) + { + // Initiate a selective sync for PRIV(op)->sync_(big|small)_writes + sync_op = selective_sync(op); + } + if (bad_vers.size) + { + // Split part of the request into a separate operation + split_stab_op = new blockstore_op_t; + split_stab_op->opcode = op->opcode; + split_stab_op->buf = (uint8_t*)bad_vers.items; + split_stab_op->len = bad_vers.size; + init_op(split_stab_op); + submit_queue.push_back(split_stab_op); + } + if (sync_op || split_stab_op || good_vers.items) + { + uint8_t *orig_buf = op->buf; + if (good_vers.items) + { + op->buf = (uint8_t*)good_vers.items; + op->len = good_vers.size; + } + // Make a wrapped callback + int *split_op_counter = (int*)malloc_or_die(sizeof(int)); + *split_op_counter = (sync_op ? 1 : 0) + (split_stab_op ? 1 : 0) + (todo ? 1 : 0); + auto cb = [op, good_items = good_vers.items, + bad_items = bad_vers.items, split_op_counter, + orig_buf, real_cb = op->callback](blockstore_op_t *split_op) + { + if (split_op->retval != 0) + op->retval = split_op->retval; + (*split_op_counter)--; + assert((*split_op_counter) >= 0); + if (op != split_op) + delete split_op; + if (!*split_op_counter) + { + free(good_items); + free(bad_items); + free(split_op_counter); + op->buf = orig_buf; + real_cb(op); + } + }; + if (sync_op) + { + sync_op->callback = cb; + } + if (split_stab_op) + { + split_stab_op->callback = cb; + } + op->callback = cb; + } + if (!todo) + { + // All work is postponed + op->callback = NULL; + return 2; + } + return 1; +} + +int blockstore_impl_t::dequeue_stable(blockstore_op_t *op) +{ + if (PRIV(op)->op_state) + { + return continue_stable(op); + } + int r = split_stab_op(op, [this](obj_ver_id ov) + { + auto dirty_it = dirty_db.find(ov); + if (dirty_it == dirty_db.end()) + { + auto & clean_db = clean_db_shard(ov.oid); + auto clean_it = clean_db.find(ov.oid); + if (clean_it == clean_db.end() || clean_it->second.version < ov.version) + { + // No such object version + printf("Error: %jx:%jx v%ju not found while stabilizing\n", ov.oid.inode, ov.oid.stripe, ov.version); + return -ENOENT; + } + else + { + // Already stable + return STAB_SPLIT_DONE; + } + } + else if (IS_STABLE(dirty_it->second.state)) + { + // Already stable + return STAB_SPLIT_DONE; + } + while (true) + { + if (IS_IN_FLIGHT(dirty_it->second.state)) + { + // Object write is still in progress. Wait until the write request completes + return STAB_SPLIT_WAIT; + } + else if (!IS_SYNCED(dirty_it->second.state)) + { + // Object not synced yet - sync it + // In previous versions we returned EBUSY here and required + // the caller (OSD) to issue a global sync first. But a global sync + // waits for all writes in the queue including inflight writes. And + // inflight writes may themselves be blocked by unstable writes being + // still present in the journal and not flushed away from it. + // So we must sync specific objects here. + // + // Even more, we have to process "stabilize" request in parts. That is, + // we must stabilize all objects which are already synced. Otherwise + // they may block objects which are NOT synced yet. + return STAB_SPLIT_SYNC; + } + else if (IS_STABLE(dirty_it->second.state)) + { + break; + } + // Check previous versions too + if (dirty_it == dirty_db.begin()) + { + break; + } + dirty_it--; + if (dirty_it->first.oid != ov.oid) + { + break; + } + } + return STAB_SPLIT_TODO; + }); + if (r != 1) + { + return r; + } + // Check journal space + blockstore_journal_check_t space_check(this); + if (!space_check.check_available(op, op->len, sizeof(journal_entry_stable), 0)) + { + return 0; + } + // There is sufficient space. Check SQEs + BS_SUBMIT_CHECK_SQES(space_check.sectors_to_write); + // Prepare and submit journal entries + int s = 0; + auto v = (obj_ver_id*)op->buf; + for (int i = 0; i < op->len; i++, v++) + { + if (!journal.entry_fits(sizeof(journal_entry_stable)) && + journal.sector_info[journal.cur_sector].dirty) + { + prepare_journal_sector_write(journal.cur_sector, op); + s++; + } + journal_entry_stable *je = (journal_entry_stable*) + prefill_single_journal_entry(journal, JE_STABLE, sizeof(journal_entry_stable)); + je->oid = v->oid; + je->version = v->version; + je->crc32 = je_crc32((journal_entry*)je); + journal.crc32_last = je->crc32; + } + prepare_journal_sector_write(journal.cur_sector, op); + s++; + assert(s == space_check.sectors_to_write); + PRIV(op)->op_state = 1; + return 1; +} + +int blockstore_impl_t::continue_stable(blockstore_op_t *op) +{ + if (PRIV(op)->op_state == 2) + goto resume_2; + else if (PRIV(op)->op_state == 4) + goto resume_4; + else + return 1; +resume_2: + if (!disable_journal_fsync) + { + BS_SUBMIT_GET_SQE(sqe, data); + io_uring_prep_fsync(sqe, dsk.journal_fd, IORING_FSYNC_DATASYNC); + data->iov = { 0 }; + data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); }; + PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 0; + PRIV(op)->pending_ops = 1; + PRIV(op)->op_state = 3; + return 1; + } +resume_4: + // Mark dirty_db entries as stable, acknowledge op completion + obj_ver_id* v; + int i; + for (i = 0, v = (obj_ver_id*)op->buf; i < op->len; i++, v++) + { + // Mark all dirty_db entries up to op->version as stable +#ifdef BLOCKSTORE_DEBUG + printf("Stabilize %jx:%jx v%ju\n", v->oid.inode, v->oid.stripe, v->version); +#endif + mark_stable(*v); + } + // Acknowledge op + op->retval = 0; + FINISH_OP(op); + return 2; +} + +void blockstore_impl_t::mark_stable(obj_ver_id v, bool forget_dirty) +{ + auto dirty_it = dirty_db.find(v); + if (dirty_it != dirty_db.end()) + { + if (IS_INSTANT(dirty_it->second.state)) + { + // 'Instant' (non-EC) operations may complete and try to become stable out of order. Prevent it. + auto back_it = dirty_it; + while (back_it != dirty_db.begin()) + { + back_it--; + if (back_it->first.oid != v.oid) + { + break; + } + if (!IS_STABLE(back_it->second.state)) + { + // There are preceding unstable versions, can't flush + return; + } + } + while (true) + { + dirty_it++; + if (dirty_it == dirty_db.end() || dirty_it->first.oid != v.oid || + !IS_SYNCED(dirty_it->second.state)) + { + dirty_it--; + break; + } + v.version = dirty_it->first.version; + } + } + while (1) + { + bool was_stable = IS_STABLE(dirty_it->second.state); + if ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_SYNCED) + { + dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | BS_ST_STABLE; + // Allocations and deletions are counted when they're stabilized + if (IS_BIG_WRITE(dirty_it->second.state)) + { + int exists = -1; + if (dirty_it != dirty_db.begin()) + { + auto prev_it = dirty_it; + prev_it--; + if (prev_it->first.oid == v.oid) + { + exists = IS_DELETE(prev_it->second.state) ? 0 : 1; + } + } + if (exists == -1) + { + auto & clean_db = clean_db_shard(v.oid); + auto clean_it = clean_db.find(v.oid); + exists = clean_it != clean_db.end() ? 1 : 0; + } + if (!exists) + { + uint64_t space_id = dirty_it->first.oid.inode; + if (no_inode_stats[dirty_it->first.oid.inode >> (64-POOL_ID_BITS)]) + space_id = space_id & ~(((uint64_t)1 << (64-POOL_ID_BITS)) - 1); + inode_space_stats[space_id] += dsk.data_block_size; + used_blocks++; + } + big_to_flush++; + } + else if (IS_DELETE(dirty_it->second.state)) + { + uint64_t space_id = dirty_it->first.oid.inode; + if (no_inode_stats[dirty_it->first.oid.inode >> (64-POOL_ID_BITS)]) + space_id = space_id & ~(((uint64_t)1 << (64-POOL_ID_BITS)) - 1); + auto & sp = inode_space_stats[space_id]; + if (sp > dsk.data_block_size) + sp -= dsk.data_block_size; + else + inode_space_stats.erase(space_id); + used_blocks--; + big_to_flush++; + } + } + else if (IS_IN_FLIGHT(dirty_it->second.state)) + { + // mark_stable should never be called for in-flight or submitted writes + printf( + "BUG: Attempt to mark_stable object %jx:%jx v%ju state of which is %x\n", + dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version, + dirty_it->second.state + ); + exit(1); + } + if (forget_dirty && (IS_BIG_WRITE(dirty_it->second.state) || + IS_DELETE(dirty_it->second.state))) + { + // Big write overrides all previous dirty entries + auto erase_end = dirty_it; + while (dirty_it != dirty_db.begin()) + { + dirty_it--; + if (dirty_it->first.oid != v.oid) + { + dirty_it++; + break; + } + } + auto & clean_db = clean_db_shard(v.oid); + auto clean_it = clean_db.find(v.oid); + uint64_t clean_loc = clean_it != clean_db.end() + ? clean_it->second.location : UINT64_MAX; + erase_dirty(dirty_it, erase_end, clean_loc); + break; + } + if (was_stable || dirty_it == dirty_db.begin()) + { + break; + } + dirty_it--; + if (dirty_it->first.oid != v.oid) + { + break; + } + } + flusher->enqueue_flush(v); + } + auto unstab_it = unstable_writes.find(v.oid); + if (unstab_it != unstable_writes.end() && + unstab_it->second <= v.version) + { + unstable_writes.erase(unstab_it); + } +} diff --git a/src/blockstore/v1/sync.cpp b/src/blockstore/v1/sync.cpp new file mode 100644 index 00000000..868c31b5 --- /dev/null +++ b/src/blockstore/v1/sync.cpp @@ -0,0 +1,234 @@ +// Copyright (c) Vitaliy Filippov, 2019+ +// License: VNPL-1.1 (see README.md for details) + +#include "blockstore_impl.h" +#include "blockstore_internal.h" + +#define SYNC_HAS_SMALL 1 +#define SYNC_HAS_BIG 2 +#define SYNC_DATA_SYNC_SENT 3 +#define SYNC_DATA_SYNC_DONE 4 +#define SYNC_JOURNAL_WRITE_SENT 5 +#define SYNC_JOURNAL_WRITE_DONE 6 +#define SYNC_JOURNAL_SYNC_SENT 7 +#define SYNC_DONE 8 + +int blockstore_impl_t::continue_sync(blockstore_op_t *op) +{ + if (immediate_commit == IMMEDIATE_ALL) + { + // We can return immediately because sync is only dequeued after all previous writes + op->retval = 0; + FINISH_OP(op); + return 2; + } + if (PRIV(op)->op_state == 0) + { + stop_sync_submitted = false; + unsynced_big_write_count -= unsynced_big_writes.size(); + PRIV(op)->sync_big_writes.swap(unsynced_big_writes); + PRIV(op)->sync_small_writes.swap(unsynced_small_writes); + unsynced_big_writes.clear(); + unsynced_small_writes.clear(); + if (PRIV(op)->sync_big_writes.size() > 0) + PRIV(op)->op_state = SYNC_HAS_BIG; + else if (PRIV(op)->sync_small_writes.size() > 0) + PRIV(op)->op_state = SYNC_HAS_SMALL; + else + PRIV(op)->op_state = SYNC_DONE; + } + if (PRIV(op)->op_state == SYNC_HAS_SMALL) + { + // No big writes, just fsync the journal + if (journal.sector_info[journal.cur_sector].dirty) + { + // Write out the last journal sector if it happens to be dirty + BS_SUBMIT_CHECK_SQES(1); + prepare_journal_sector_write(journal.cur_sector, op); + PRIV(op)->op_state = SYNC_JOURNAL_WRITE_SENT; + return 1; + } + else + { + PRIV(op)->op_state = SYNC_JOURNAL_WRITE_DONE; + } + } + if (PRIV(op)->op_state == SYNC_HAS_BIG) + { + // 1st step: fsync data + if (!disable_data_fsync) + { + BS_SUBMIT_GET_SQE(sqe, data); + io_uring_prep_fsync(sqe, dsk.data_fd, IORING_FSYNC_DATASYNC); + data->iov = { 0 }; + data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); }; + PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 0; + PRIV(op)->pending_ops = 1; + PRIV(op)->op_state = SYNC_DATA_SYNC_SENT; + return 1; + } + else + { + PRIV(op)->op_state = SYNC_DATA_SYNC_DONE; + } + } + if (PRIV(op)->op_state == SYNC_DATA_SYNC_DONE) + { + // 2nd step: Data device is synced, prepare & write journal entries + // Check space in the journal and journal memory buffers + blockstore_journal_check_t space_check(this); + if (dsk.csum_block_size) + { + // More complex check because all journal entries have different lengths + int left = PRIV(op)->sync_big_writes.size(); + for (auto & sbw: PRIV(op)->sync_big_writes) + { + left--; + auto & dirty_entry = dirty_db.at(sbw); + uint64_t dyn_size = dsk.dirty_dyn_size(dirty_entry.offset, dirty_entry.len); + if (!space_check.check_available(op, 1, sizeof(journal_entry_big_write) + dyn_size, 0)) + { + return 0; + } + } + } + else if (!space_check.check_available(op, PRIV(op)->sync_big_writes.size(), + sizeof(journal_entry_big_write) + dsk.clean_entry_bitmap_size, 0)) + { + return 0; + } + // Check SQEs. Don't bother about merging, submit each journal sector as a separate request + BS_SUBMIT_CHECK_SQES(space_check.sectors_to_write); + // Prepare and submit journal entries + auto it = PRIV(op)->sync_big_writes.begin(); + int s = 0; + while (it != PRIV(op)->sync_big_writes.end()) + { + auto & dirty_entry = dirty_db.at(*it); + uint64_t dyn_size = dsk.dirty_dyn_size(dirty_entry.offset, dirty_entry.len); + if (!journal.entry_fits(sizeof(journal_entry_big_write) + dyn_size) && + journal.sector_info[journal.cur_sector].dirty) + { + prepare_journal_sector_write(journal.cur_sector, op); + s++; + } + journal_entry_big_write *je = (journal_entry_big_write*)prefill_single_journal_entry( + journal, (dirty_entry.state & BS_ST_INSTANT) ? JE_BIG_WRITE_INSTANT : JE_BIG_WRITE, + sizeof(journal_entry_big_write) + dyn_size + ); + auto jsec = dirty_entry.journal_sector = journal.sector_info[journal.cur_sector].offset; + assert(journal.next_free >= journal.used_start + ? (jsec >= journal.used_start && jsec < journal.next_free) + : (jsec >= journal.used_start || jsec < journal.next_free)); + journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++; +#ifdef BLOCKSTORE_DEBUG + printf( + "journal offset %08jx is used by %jx:%jx v%ju (%ju refs)\n", + dirty_entry.journal_sector, it->oid.inode, it->oid.stripe, it->version, + journal.used_sectors[journal.sector_info[journal.cur_sector].offset] + ); +#endif + je->oid = it->oid; + je->version = it->version; + je->offset = dirty_entry.offset; + je->len = dirty_entry.len; + je->location = dirty_entry.location; + memcpy((void*)(je+1), (alloc_dyn_data + ? (uint8_t*)dirty_entry.dyn_data+sizeof(int) : (uint8_t*)&dirty_entry.dyn_data), dyn_size); + je->crc32 = je_crc32((journal_entry*)je); + journal.crc32_last = je->crc32; + it++; + } + prepare_journal_sector_write(journal.cur_sector, op); + s++; + assert(s == space_check.sectors_to_write); + PRIV(op)->op_state = SYNC_JOURNAL_WRITE_SENT; + return 1; + } + if (PRIV(op)->op_state == SYNC_JOURNAL_WRITE_DONE) + { + if (!disable_journal_fsync) + { + BS_SUBMIT_GET_SQE(sqe, data); + io_uring_prep_fsync(sqe, dsk.journal_fd, IORING_FSYNC_DATASYNC); + data->iov = { 0 }; + data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); }; + PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 0; + PRIV(op)->pending_ops = 1; + PRIV(op)->op_state = SYNC_JOURNAL_SYNC_SENT; + return 1; + } + else + { + PRIV(op)->op_state = SYNC_DONE; + } + } + if (PRIV(op)->op_state == SYNC_DONE) + { + ack_sync(op); + return 2; + } + return 1; +} + +void blockstore_impl_t::ack_sync(blockstore_op_t *op) +{ + // Handle states + for (auto it = PRIV(op)->sync_big_writes.begin(); it != PRIV(op)->sync_big_writes.end(); it++) + { +#ifdef BLOCKSTORE_DEBUG + printf("Ack sync big %jx:%jx v%ju\n", it->oid.inode, it->oid.stripe, it->version); +#endif + auto & unstab = unstable_writes[it->oid]; + unstab = unstab < it->version ? it->version : unstab; + auto dirty_it = dirty_db.find(*it); + dirty_it->second.state = ((dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | BS_ST_SYNCED); + if (dirty_it->second.state & BS_ST_INSTANT) + { + mark_stable(dirty_it->first); + } + else + { + unstable_unsynced--; + assert(unstable_unsynced >= 0); + } + dirty_it++; + while (dirty_it != dirty_db.end() && dirty_it->first.oid == it->oid) + { + if ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_WAIT_BIG) + { + dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | BS_ST_IN_FLIGHT; + } + dirty_it++; + } + } + for (auto it = PRIV(op)->sync_small_writes.begin(); it != PRIV(op)->sync_small_writes.end(); it++) + { +#ifdef BLOCKSTORE_DEBUG + printf("Ack sync small %jx:%jx v%ju\n", it->oid.inode, it->oid.stripe, it->version); +#endif + auto & unstab = unstable_writes[it->oid]; + unstab = unstab < it->version ? it->version : unstab; + if (dirty_db[*it].state == (BS_ST_DELETE | BS_ST_WRITTEN)) + { + dirty_db[*it].state = (BS_ST_DELETE | BS_ST_SYNCED); + // Deletions are treated as immediately stable + mark_stable(*it); + } + else /* (BS_ST_INSTANT?) | BS_ST_SMALL_WRITE | BS_ST_WRITTEN */ + { + dirty_db[*it].state = (dirty_db[*it].state & ~BS_ST_WORKFLOW_MASK) | BS_ST_SYNCED; + if (dirty_db[*it].state & BS_ST_INSTANT) + { + mark_stable(*it); + } + else + { + unstable_unsynced--; + assert(unstable_unsynced >= 0); + } + } + } + op->retval = 0; + FINISH_OP(op); +} diff --git a/src/blockstore/v1/write.cpp b/src/blockstore/v1/write.cpp new file mode 100644 index 00000000..dcefb4a8 --- /dev/null +++ b/src/blockstore/v1/write.cpp @@ -0,0 +1,824 @@ +// Copyright (c) Vitaliy Filippov, 2019+ +// License: VNPL-1.1 (see README.md for details) + +#include "blockstore_impl.h" +#include "blockstore_internal.h" + +bool blockstore_impl_t::enqueue_write(blockstore_op_t *op) +{ + // Check or assign version number + bool found = false, deleted = false, unsynced = false, is_del = (op->opcode == BS_OP_DELETE); + bool wait_big = false, wait_del = false; + void *dyn = NULL; + if (is_del) + { + op->len = 0; + } + size_t dyn_size = dsk.dirty_dyn_size(op->offset, op->len); + if (!is_del && alloc_dyn_data) + { + // FIXME: Working with `dyn_data` has to be refactored somehow but I first have to decide how :) + // +sizeof(int) = refcount + dyn = calloc_or_die(1, dyn_size+sizeof(int)); + *((int*)dyn) = 1; + } + uint8_t *dyn_ptr = (alloc_dyn_data ? (uint8_t*)dyn+sizeof(int) : (uint8_t*)&dyn); + uint64_t version = 1; + if (dirty_db.size() > 0) + { + auto dirty_it = dirty_db.upper_bound((obj_ver_id){ + .oid = op->oid, + .version = UINT64_MAX, + }); + dirty_it--; // segfaults when dirty_db is empty + if (dirty_it != dirty_db.end() && dirty_it->first.oid == op->oid) + { + found = true; + version = dirty_it->first.version + 1; + deleted = IS_DELETE(dirty_it->second.state); + unsynced = !IS_SYNCED(dirty_it->second.state); + wait_del = ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_WAIT_DEL); + wait_big = (dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_BIG_WRITE + ? !IS_SYNCED(dirty_it->second.state) + : ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_WAIT_BIG); + if (!is_del && !deleted) + { + void *dyn_from = alloc_dyn_data + ? (uint8_t*)dirty_it->second.dyn_data + sizeof(int) : (uint8_t*)&dirty_it->second.dyn_data; + memcpy(dyn_ptr, dyn_from, dsk.clean_entry_bitmap_size); + } + } + } + if (!found) + { + auto & clean_db = clean_db_shard(op->oid); + auto clean_it = clean_db.find(op->oid); + if (clean_it != clean_db.end()) + { + version = clean_it->second.version + 1; + if (!is_del) + { + void *bmp_ptr = get_clean_entry_bitmap(clean_it->second.location, dsk.clean_entry_bitmap_size); + memcpy(dyn_ptr, bmp_ptr, dsk.clean_entry_bitmap_size); + } + } + else + { + deleted = true; + } + } + if (deleted && is_del) + { + // Already deleted + op->retval = 0; + return false; + } + PRIV(op)->real_version = 0; + if (op->version == 0) + { + op->version = version; + } + else if (op->version < version) + { + // Implicit operations must be added like that: DEL [FLUSH] BIG [SYNC] SMALL SMALL + if (deleted || wait_del) + { + // It's allowed to write versions with low numbers over deletes + // However, we have to flush those deletes first as we use version number for ordering +#ifdef BLOCKSTORE_DEBUG + printf("Write %jx:%jx v%ju over delete (real v%ju) offset=%u len=%u\n", op->oid.inode, op->oid.stripe, version, op->version, op->offset, op->len); +#endif + wait_del = true; + PRIV(op)->real_version = op->version; + op->version = version; + if (unsynced) + { + // Issue an additional sync so the delete reaches the journal + blockstore_op_t *sync_op = new blockstore_op_t; + sync_op->opcode = BS_OP_SYNC; + sync_op->oid = op->oid; + sync_op->version = op->version; + sync_op->callback = [this](blockstore_op_t *sync_op) + { + flusher->unshift_flush((obj_ver_id){ + .oid = sync_op->oid, + .version = sync_op->version-1, + }, true); + delete sync_op; + }; + enqueue_op(sync_op); + } + else + { + flusher->unshift_flush((obj_ver_id){ + .oid = op->oid, + .version = version-1, + }, true); + } + } + else + { + // Invalid version requested +#ifdef BLOCKSTORE_DEBUG + printf("Write %jx:%jx v%ju requested, but we already have v%ju\n", op->oid.inode, op->oid.stripe, op->version, version); +#endif + op->retval = -EEXIST; + if (!is_del && alloc_dyn_data) + { + free(dyn); + } + return false; + } + } + bool imm = (op->len < dsk.data_block_size ? (immediate_commit != IMMEDIATE_NONE) : (immediate_commit == IMMEDIATE_ALL)); + if (wait_big && !is_del && !deleted && op->len < dsk.data_block_size && !imm || + !imm && autosync_writes && unsynced_queued_ops >= autosync_writes) + { + // Issue an additional sync so that the previous big write can reach the journal + blockstore_op_t *sync_op = new blockstore_op_t; + sync_op->opcode = BS_OP_SYNC; + sync_op->callback = [](blockstore_op_t *sync_op) + { + delete sync_op; + }; + enqueue_op(sync_op); + } + else if (!imm) + unsynced_queued_ops++; +#ifdef BLOCKSTORE_DEBUG + if (is_del) + printf("Delete %jx:%jx v%ju\n", op->oid.inode, op->oid.stripe, op->version); + else if (!wait_del) + printf("Write %jx:%jx v%ju offset=%u len=%u\n", op->oid.inode, op->oid.stripe, op->version, op->offset, op->len); +#endif + // No strict need to add it into dirty_db here except maybe for listings to return + // correct data when there are inflight operations in the queue + uint32_t state; + if (is_del) + state = BS_ST_DELETE | BS_ST_IN_FLIGHT; + else + { + state = (op->len == dsk.data_block_size || deleted ? BS_ST_BIG_WRITE : BS_ST_SMALL_WRITE); + if (state == BS_ST_SMALL_WRITE && throttle_small_writes) + clock_gettime(CLOCK_REALTIME, &PRIV(op)->tv_begin); + if (wait_del) + state |= BS_ST_WAIT_DEL; + else if (state == BS_ST_SMALL_WRITE && wait_big) + state |= BS_ST_WAIT_BIG; + else + state |= BS_ST_IN_FLIGHT; + if (op->opcode == BS_OP_WRITE_STABLE) + state |= BS_ST_INSTANT; + if (op->bitmap) + memcpy(dyn_ptr, op->bitmap, dsk.clean_entry_bitmap_size); + } + // Calculate checksums + // FIXME: Allow to receive checksums from outside? + if (!is_del && dsk.data_csum_type && op->len > 0) + { + uint32_t *data_csums = (uint32_t*)(dyn_ptr + dsk.clean_entry_bitmap_size); + uint32_t start = op->offset / dsk.csum_block_size; + uint32_t end = (op->offset+op->len-1) / dsk.csum_block_size; + auto fn = state & BS_ST_BIG_WRITE ? crc32c_pad : crc32c_nopad; + if (start == end) + data_csums[0] = fn(0, op->buf, op->len, op->offset - start*dsk.csum_block_size, end*dsk.csum_block_size - (op->offset+op->len)); + else + { + // First block + data_csums[0] = fn(0, op->buf, dsk.csum_block_size*(start+1)-op->offset, op->offset - start*dsk.csum_block_size, 0); + // Intermediate blocks + for (uint32_t i = start+1; i < end; i++) + data_csums[i-start] = crc32c(0, (uint8_t*)op->buf + dsk.csum_block_size*i-op->offset, dsk.csum_block_size); + // Last block + data_csums[end-start] = fn( + 0, (uint8_t*)op->buf + end*dsk.csum_block_size - op->offset, + op->offset+op->len - end*dsk.csum_block_size, + 0, (end+1)*dsk.csum_block_size - (op->offset+op->len) + ); + } + } + dirty_db.emplace((obj_ver_id){ + .oid = op->oid, + .version = op->version, + }, (dirty_entry){ + .state = state, + .flags = 0, + .location = 0, + .offset = is_del ? 0 : op->offset, + .len = is_del ? 0 : op->len, + .journal_sector = 0, + .dyn_data = dyn, + }); + return true; +} + +void blockstore_impl_t::cancel_all_writes(blockstore_op_t *op, blockstore_dirty_db_t::iterator dirty_it, int retval) +{ + while (dirty_it != dirty_db.end() && dirty_it->first.oid == op->oid) + { + free_dirty_dyn_data(dirty_it->second); + dirty_db.erase(dirty_it++); + } + bool found = false; + for (auto other_op: submit_queue) + { + if (!other_op) + { + // freed operations during submitting are zeroed + } + else if (other_op == op) + { + // may be present in queue multiple times due to moving operations in submit_queue + found = true; + } + else if (found && other_op->oid == op->oid && + (other_op->opcode == BS_OP_WRITE || other_op->opcode == BS_OP_WRITE_STABLE)) + { + // Mark operations to cancel them + PRIV(other_op)->real_version = UINT64_MAX; + other_op->retval = retval; + } + } + op->retval = retval; + FINISH_OP(op); +} + +// First step of the write algorithm: dequeue operation and submit initial write(s) +int blockstore_impl_t::dequeue_write(blockstore_op_t *op) +{ + if (PRIV(op)->op_state) + { + return continue_write(op); + } + auto dirty_it = dirty_db.find((obj_ver_id){ + .oid = op->oid, + .version = op->version, + }); + assert(dirty_it != dirty_db.end()); + if ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) < BS_ST_IN_FLIGHT) + { + // Don't dequeue + return 0; + } + if (PRIV(op)->real_version != 0) + { + if (PRIV(op)->real_version == UINT64_MAX) + { + // This is the flag value used to cancel operations + FINISH_OP(op); + return 2; + } + // Restore original low version number for unblocked operations +#ifdef BLOCKSTORE_DEBUG + printf("Restoring %jx:%jx version: v%ju -> v%ju\n", op->oid.inode, op->oid.stripe, op->version, PRIV(op)->real_version); +#endif + auto prev_it = dirty_it; + if (prev_it != dirty_db.begin()) + { + prev_it--; + if (prev_it->first.oid == op->oid && prev_it->first.version >= PRIV(op)->real_version) + { + // Original version is still invalid + // All subsequent writes to the same object must be canceled too + printf("Tried to write %jx:%jx v%ju after delete (old version v%ju), but already have v%ju\n", + op->oid.inode, op->oid.stripe, PRIV(op)->real_version, op->version, prev_it->first.version); + cancel_all_writes(op, dirty_it, -EEXIST); + return 2; + } + } + op->version = PRIV(op)->real_version; + PRIV(op)->real_version = 0; + dirty_entry e = dirty_it->second; + dirty_db.erase(dirty_it); + dirty_it = dirty_db.emplace((obj_ver_id){ + .oid = op->oid, + .version = op->version, + }, e).first; + } + if (write_iodepth >= max_write_iodepth) + { + return 0; + } + if ((dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_BIG_WRITE) + { + blockstore_journal_check_t space_check(this); + if (!space_check.check_available(op, unsynced_big_write_count + 1, + sizeof(journal_entry_big_write) + dsk.clean_dyn_size, + (unstable_writes.size()+unstable_unsynced+((dirty_it->second.state & BS_ST_INSTANT) ? 0 : 1))*journal.block_size)) + { + return 0; + } + // Big (redirect) write + uint64_t loc = data_alloc->find_free(); + if (loc == UINT64_MAX) + { + // no space + if (big_to_flush > 0) + { + // hope that some space will be available after flush + flusher->request_trim(); + PRIV(op)->wait_for = WAIT_FREE; + return 0; + } + cancel_all_writes(op, dirty_it, -ENOSPC); + return 2; + } + if (inmemory_meta) + { + // Check once more that metadata entry is zeroed (the reverse means a bug or corruption) + uint64_t sector = (loc / (dsk.meta_block_size / dsk.clean_entry_size)) * dsk.meta_block_size; + uint64_t pos = (loc % (dsk.meta_block_size / dsk.clean_entry_size)); + clean_disk_entry *entry = (clean_disk_entry*)((uint8_t*)metadata_buffer + sector + pos*dsk.clean_entry_size); + if (entry->oid.inode || entry->oid.stripe || entry->version) + { + printf( + "Fatal error (metadata corruption or bug): tried to write object %jx:%jx v%ju" + " over a non-zero metadata entry %ju with %jx:%jx v%ju\n", op->oid.inode, + op->oid.stripe, op->version, loc, entry->oid.inode, entry->oid.stripe, entry->version + ); + exit(1); + } + } + BS_SUBMIT_GET_SQE(sqe, data); + write_iodepth++; + dirty_it->second.location = loc * dsk.data_block_size; + dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | BS_ST_SUBMITTED; +#ifdef BLOCKSTORE_DEBUG + printf( + "Allocate block %ju for %jx:%jx v%ju\n", + loc, op->oid.inode, op->oid.stripe, op->version + ); +#endif + data_alloc->set(loc, true); + uint64_t stripe_offset = (op->offset % dsk.bitmap_granularity); + uint64_t stripe_end = (op->offset + op->len) % dsk.bitmap_granularity; + // Zero fill up to dsk.bitmap_granularity + int vcnt = 0; + if (stripe_offset) + { + PRIV(op)->iov_zerofill[vcnt++] = (struct iovec){ zero_object, (size_t)stripe_offset }; + } + PRIV(op)->iov_zerofill[vcnt++] = (struct iovec){ op->buf, op->len }; + if (stripe_end) + { + stripe_end = dsk.bitmap_granularity - stripe_end; + PRIV(op)->iov_zerofill[vcnt++] = (struct iovec){ zero_object, (size_t)stripe_end }; + } + data->iov.iov_len = op->len + stripe_offset + stripe_end; // to check it in the callback + data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); }; + io_uring_prep_writev( + sqe, dsk.data_fd, PRIV(op)->iov_zerofill, vcnt, dsk.data_offset + (loc * dsk.data_block_size) + op->offset - stripe_offset + ); + PRIV(op)->pending_ops = 1; + if (!(dirty_it->second.state & BS_ST_INSTANT)) + { + unstable_unsynced++; + } + if (immediate_commit != IMMEDIATE_ALL) + { + // Increase the counter, but don't save into unsynced_writes yet (can't sync until the write is finished) + unsynced_big_write_count++; + PRIV(op)->op_state = 3; + } + else + { + PRIV(op)->op_state = 1; + } + } + else /* if ((dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_SMALL_WRITE) */ + { + // Small (journaled) write + // First check if the journal has sufficient space + uint64_t dyn_size = dsk.dirty_dyn_size(op->offset, op->len); + blockstore_journal_check_t space_check(this); + if (unsynced_big_write_count && + !space_check.check_available(op, unsynced_big_write_count, + sizeof(journal_entry_big_write) + dsk.clean_dyn_size, 0) + || !space_check.check_available(op, 1, + sizeof(journal_entry_small_write) + dyn_size, + op->len + (unstable_writes.size()+unstable_unsynced+((dirty_it->second.state & BS_ST_INSTANT) ? 0 : 1))*journal.block_size)) + { + return 0; + } + // There is sufficient space. Check SQE(s) + BS_SUBMIT_CHECK_SQES( + // Write current journal sector only if it's dirty and full, or in the immediate_commit mode + (immediate_commit != IMMEDIATE_NONE || + !journal.entry_fits(sizeof(journal_entry_small_write) + dyn_size) ? 1 : 0) + + (op->len > 0 ? 1 : 0) + ); + write_iodepth++; + // Got SQEs. Prepare previous journal sector write if required + if (immediate_commit == IMMEDIATE_NONE && + !journal.entry_fits(sizeof(journal_entry_small_write) + dyn_size)) + { + prepare_journal_sector_write(journal.cur_sector, op); + } + // Then pre-fill journal entry + journal_entry_small_write *je = (journal_entry_small_write*)prefill_single_journal_entry( + journal, op->opcode == BS_OP_WRITE_STABLE ? JE_SMALL_WRITE_INSTANT : JE_SMALL_WRITE, + sizeof(journal_entry_small_write) + dyn_size + ); + auto jsec = dirty_it->second.journal_sector = journal.sector_info[journal.cur_sector].offset; + if (!(journal.next_free >= journal.used_start + ? (jsec >= journal.used_start && jsec < journal.next_free) + : (jsec >= journal.used_start || jsec < journal.next_free))) + { + printf( + "BUG: journal offset %08jx is used by %jx:%jx v%ju (%ju refs) BUT used_start=%jx next_free=%jx\n", + dirty_it->second.journal_sector, dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version, + journal.used_sectors[journal.sector_info[journal.cur_sector].offset], + journal.used_start, journal.next_free + ); + abort(); + } + journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++; +#ifdef BLOCKSTORE_DEBUG + printf( + "journal offset %08jx is used by %jx:%jx v%ju (%ju refs)\n", + dirty_it->second.journal_sector, dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version, + journal.used_sectors[journal.sector_info[journal.cur_sector].offset] + ); +#endif + // Figure out where data will be + auto next_next_free = (journal.next_free + op->len) <= journal.len ? journal.next_free : dsk.journal_block_size; + if (op->len > 0) + { + auto journal_used_it = journal.used_sectors.lower_bound(next_next_free); + if (journal_used_it != journal.used_sectors.end() && + journal_used_it->first < next_next_free + op->len) + { + printf( + "BUG: Attempt to overwrite used offset (%jx, %ju refs) of the journal with the object %jx:%jx v%ju: data at %jx, len %x!" + " Journal used_start=%08jx (%ju refs), next_free=%08jx, dirty_start=%08jx\n", + journal_used_it->first, journal_used_it->second, op->oid.inode, op->oid.stripe, op->version, next_next_free, op->len, + journal.used_start, journal.used_sectors[journal.used_start], journal.next_free, journal.dirty_start + ); + exit(1); + } + } + // double check that next_free doesn't cross used_start from the left + assert(journal.next_free >= journal.used_start && next_next_free >= journal.next_free || next_next_free < journal.used_start); + journal.next_free = next_next_free; + je->oid = op->oid; + je->version = op->version; + je->offset = op->offset; + je->len = op->len; + je->data_offset = journal.next_free; + je->crc32_data = dsk.csum_block_size ? 0 : crc32c(0, op->buf, op->len); + memcpy((void*)(je+1), (alloc_dyn_data + ? (uint8_t*)dirty_it->second.dyn_data+sizeof(int) : (uint8_t*)&dirty_it->second.dyn_data), dyn_size); + je->crc32 = je_crc32((journal_entry*)je); + journal.crc32_last = je->crc32; + if (immediate_commit != IMMEDIATE_NONE) + { + prepare_journal_sector_write(journal.cur_sector, op); + } + if (op->len > 0) + { + // Prepare journal data write + if (journal.inmemory) + { + // Copy data + memcpy((uint8_t*)journal.buffer + journal.next_free, op->buf, op->len); + } + BS_SUBMIT_GET_SQE(sqe2, data2); + data2->iov = (struct iovec){ op->buf, op->len }; + ++journal.submit_id; + assert(journal.submit_id != 0); // check overflow + // Make subsequent journal writes wait for our data write + journal.flushing_ops.emplace(journal.submit_id, (pending_journaling_t){ + .pending = 1, + .sector = -1, + .op = op, + }); + data2->callback = [this, flush_id = journal.submit_id](ring_data_t *data) { handle_journal_write(data, flush_id); }; + io_uring_prep_writev( + sqe2, dsk.journal_fd, &data2->iov, 1, journal.offset + journal.next_free + ); + PRIV(op)->pending_ops++; + } + else + { + // Zero-length overwrite. Allowed to bump object version in EC placement groups without actually writing data + } + dirty_it->second.location = journal.next_free; + dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | BS_ST_SUBMITTED; + next_next_free = journal.next_free + op->len; + if (next_next_free >= journal.len) + next_next_free = dsk.journal_block_size; + // double check that next_free doesn't cross used_start from the left + assert(journal.next_free >= journal.used_start && next_next_free >= journal.next_free || next_next_free < journal.used_start); + journal.next_free = next_next_free; + if (!(dirty_it->second.state & BS_ST_INSTANT)) + { + unstable_unsynced++; + } + if (!PRIV(op)->pending_ops) + { + PRIV(op)->op_state = 4; + return continue_write(op); + } + else + { + PRIV(op)->op_state = 3; + } + } + return 1; +} + +int blockstore_impl_t::continue_write(blockstore_op_t *op) +{ + int op_state = PRIV(op)->op_state; + if (op_state == 2) + goto resume_2; + else if (op_state == 4) + goto resume_4; + else if (op_state == 6) + goto resume_6; + else + { + // In progress + return 1; + } +resume_2: + // Only for the immediate_commit mode: prepare and submit big_write journal entry + { + auto dirty_it = dirty_db.find((obj_ver_id){ + .oid = op->oid, + .version = op->version, + }); + assert(dirty_it != dirty_db.end()); + uint64_t dyn_size = dsk.dirty_dyn_size(op->offset, op->len); + blockstore_journal_check_t space_check(this); + if (!space_check.check_available(op, 1, sizeof(journal_entry_big_write) + dyn_size, + (unstable_writes.size()+unstable_unsynced+((dirty_it->second.state & BS_ST_INSTANT) ? 0 : 1))*journal.block_size)) + { + return 0; + } + BS_SUBMIT_CHECK_SQES(1); + journal_entry_big_write *je = (journal_entry_big_write*)prefill_single_journal_entry( + journal, op->opcode == BS_OP_WRITE_STABLE ? JE_BIG_WRITE_INSTANT : JE_BIG_WRITE, + sizeof(journal_entry_big_write) + dyn_size + ); + auto jsec = dirty_it->second.journal_sector = journal.sector_info[journal.cur_sector].offset; + if (!(journal.next_free >= journal.used_start + ? (jsec >= journal.used_start && jsec < journal.next_free) + : (jsec >= journal.used_start || jsec < journal.next_free))) + { + printf( + "BUG: journal offset %08jx is used by %jx:%jx v%ju (%ju refs) BUT used_start=%jx next_free=%jx\n", + dirty_it->second.journal_sector, dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version, + journal.used_sectors[journal.sector_info[journal.cur_sector].offset], + journal.used_start, journal.next_free + ); + abort(); + } + journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++; +#ifdef BLOCKSTORE_DEBUG + printf( + "journal offset %08jx is used by %jx:%jx v%ju (%ju refs)\n", + journal.sector_info[journal.cur_sector].offset, op->oid.inode, op->oid.stripe, op->version, + journal.used_sectors[journal.sector_info[journal.cur_sector].offset] + ); +#endif + je->oid = op->oid; + je->version = op->version; + je->offset = op->offset; + je->len = op->len; + je->location = dirty_it->second.location; + memcpy((void*)(je+1), (alloc_dyn_data + ? (uint8_t*)dirty_it->second.dyn_data+sizeof(int) : (uint8_t*)&dirty_it->second.dyn_data), dyn_size); + je->crc32 = je_crc32((journal_entry*)je); + journal.crc32_last = je->crc32; + prepare_journal_sector_write(journal.cur_sector, op); + PRIV(op)->op_state = 3; + return 1; + } +resume_4: + // Switch object state + { + auto dirty_it = dirty_db.find((obj_ver_id){ + .oid = op->oid, + .version = op->version, + }); + assert(dirty_it != dirty_db.end()); +#ifdef BLOCKSTORE_DEBUG + printf("Ack write %jx:%jx v%ju = state 0x%x\n", op->oid.inode, op->oid.stripe, op->version, dirty_it->second.state); +#endif + bool is_big = (dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_BIG_WRITE; + bool imm = is_big ? (immediate_commit == IMMEDIATE_ALL) : (immediate_commit != IMMEDIATE_NONE); + bool is_instant = IS_INSTANT(dirty_it->second.state); + if (imm) + { + auto & unstab = unstable_writes[op->oid]; + unstab = unstab < op->version ? op->version : unstab; + if (!is_instant) + { + unstable_unsynced--; + assert(unstable_unsynced >= 0); + } + } + dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) + | (imm ? BS_ST_SYNCED : BS_ST_WRITTEN); + if (imm && is_instant) + { + // Deletions and 'instant' operations are treated as immediately stable + mark_stable(dirty_it->first); + } + if (!imm) + { + if (is_big) + { + // Remember big write as unsynced + unsynced_big_writes.push_back((obj_ver_id){ + .oid = op->oid, + .version = op->version, + }); + } + else + { + // Remember small write as unsynced + unsynced_small_writes.push_back((obj_ver_id){ + .oid = op->oid, + .version = op->version, + }); + } + } + if (imm && (dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_BIG_WRITE) + { + // Unblock small writes + dirty_it++; + while (dirty_it != dirty_db.end() && dirty_it->first.oid == op->oid) + { + if ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_WAIT_BIG) + { + dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | BS_ST_IN_FLIGHT; + } + dirty_it++; + } + } + // Apply throttling to not fill the journal too fast for the SSD+HDD case + if (!is_big && throttle_small_writes) + { + // Apply throttling + timespec tv_end; + clock_gettime(CLOCK_REALTIME, &tv_end); + uint64_t exec_us = + (tv_end.tv_sec - PRIV(op)->tv_begin.tv_sec)*1000000 + + (tv_end.tv_nsec - PRIV(op)->tv_begin.tv_nsec)/1000; + // Compare with target execution time + // 100% free -> target time = 0 + // 0% free -> target time = iodepth/parallelism * (iops + size/bw) / write per second + uint64_t used_start = journal.get_trim_pos(); + uint64_t journal_free_space = journal.next_free < used_start + ? (used_start - journal.next_free) + : (journal.len - journal.next_free + used_start - journal.block_size); + uint64_t ref_us = + (write_iodepth <= throttle_target_parallelism ? 100 : 100*write_iodepth/throttle_target_parallelism) + * (1000000/throttle_target_iops + op->len*1000000/throttle_target_mbs/1024/1024) + / 100; + ref_us -= ref_us * journal_free_space / journal.len; + if (ref_us > exec_us + throttle_threshold_us) + { + // Pause reply + PRIV(op)->op_state = 5; + // Remember that the timer can in theory be called right here + tfd->set_timer_us(ref_us-exec_us, false, [this, op](int timer_id) + { + PRIV(op)->op_state++; + ringloop->wakeup(); + }); + return 1; + } + } + } +resume_6: + // Acknowledge write + op->retval = op->len; + write_iodepth--; + FINISH_OP(op); + return 2; +} + +void blockstore_impl_t::handle_write_event(ring_data_t *data, blockstore_op_t *op) +{ + live = true; + if (data->res != data->iov.iov_len) + { + // FIXME: our state becomes corrupted after a write error. maybe do something better than just die + disk_error_abort("data write", data->res, data->iov.iov_len); + } + PRIV(op)->pending_ops--; + assert(PRIV(op)->pending_ops >= 0); + if (PRIV(op)->pending_ops == 0) + { + release_journal_sectors(op); + PRIV(op)->op_state++; + ringloop->wakeup(); + } +} + +void blockstore_impl_t::release_journal_sectors(blockstore_op_t *op) +{ + // Release flushed journal sectors + if (PRIV(op)->min_flushed_journal_sector > 0 && + PRIV(op)->max_flushed_journal_sector > 0) + { + uint64_t s = PRIV(op)->min_flushed_journal_sector; + while (1) + { + if (!journal.sector_info[s-1].dirty && journal.sector_info[s-1].flush_count == 0) + { + if (s == (1+journal.cur_sector)) + { + // Forcibly move to the next sector and move dirty position + journal.in_sector_pos = journal.block_size; + } + // We know for sure that we won't write into this sector anymore + uint64_t new_ds = journal.sector_info[s-1].offset + journal.block_size; + if (new_ds >= journal.len) + { + new_ds = journal.block_size; + } + if ((journal.dirty_start + (journal.dirty_start >= journal.used_start ? 0 : journal.len)) < + (new_ds + (new_ds >= journal.used_start ? 0 : journal.len))) + { + journal.dirty_start = new_ds; + } + } + if (s == PRIV(op)->max_flushed_journal_sector) + break; + s = 1 + s % journal.sector_count; + } + PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 0; + } +} + +int blockstore_impl_t::dequeue_del(blockstore_op_t *op) +{ + if (PRIV(op)->op_state) + { + return continue_write(op); + } + auto dirty_it = dirty_db.find((obj_ver_id){ + .oid = op->oid, + .version = op->version, + }); + assert(dirty_it != dirty_db.end()); + blockstore_journal_check_t space_check(this); + if (!space_check.check_available(op, 1, sizeof(journal_entry_del), (unstable_writes.size()+unstable_unsynced)*journal.block_size)) + { + return 0; + } + // Write current journal sector only if it's dirty and full, or in the immediate_commit mode + BS_SUBMIT_CHECK_SQES( + (immediate_commit != IMMEDIATE_NONE || + (dsk.journal_block_size - journal.in_sector_pos) < sizeof(journal_entry_del) && + journal.sector_info[journal.cur_sector].dirty) ? 1 : 0 + ); + if (write_iodepth >= max_write_iodepth) + { + return 0; + } + write_iodepth++; + // Prepare journal sector write + if (immediate_commit == IMMEDIATE_NONE && + (dsk.journal_block_size - journal.in_sector_pos) < sizeof(journal_entry_del) && + journal.sector_info[journal.cur_sector].dirty) + { + prepare_journal_sector_write(journal.cur_sector, op); + } + // Pre-fill journal entry + journal_entry_del *je = (journal_entry_del*)prefill_single_journal_entry( + journal, JE_DELETE, sizeof(struct journal_entry_del) + ); + dirty_it->second.journal_sector = journal.sector_info[journal.cur_sector].offset; + journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++; +#ifdef BLOCKSTORE_DEBUG + printf( + "journal offset %08jx is used by %jx:%jx v%ju (%ju refs)\n", + dirty_it->second.journal_sector, dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version, + journal.used_sectors[journal.sector_info[journal.cur_sector].offset] + ); +#endif + je->oid = op->oid; + je->version = op->version; + je->crc32 = je_crc32((journal_entry*)je); + journal.crc32_last = je->crc32; + dirty_it->second.state = BS_ST_DELETE | BS_ST_SUBMITTED; + if (immediate_commit != IMMEDIATE_NONE) + { + prepare_journal_sector_write(journal.cur_sector, op); + } + if (!PRIV(op)->pending_ops) + { + PRIV(op)->op_state = 4; + return continue_write(op); + } + else + { + PRIV(op)->op_state = 3; + } + return 1; +}