// Metadata storage version 3 ("lsm heap") // Copyright (c) Vitaliy Filippov, 2025+ // License: VNPL-1.1 (see README.md for details) #pragma once #include #include #include #include #include #include "../client/object_id.h" #include "../util/robin_hood.h" #include "blockstore_disk.h" #include "multilist.h" struct pool_shard_settings_t { uint32_t pg_count; uint32_t pg_stripe_size; uint32_t no_inode_stats; }; #define BS_HEAP_TYPE 0x07 #define BS_HEAP_BIG_WRITE 1 #define BS_HEAP_SMALL_WRITE 2 #define BS_HEAP_INTENT_WRITE 3 #define BS_HEAP_BIG_INTENT 4 #define BS_HEAP_DELETE 5 #define BS_HEAP_COMMIT 6 #define BS_HEAP_ROLLBACK 7 #define BS_HEAP_STABLE 0x40 #define BS_HEAP_GARBAGE 0x80 class blockstore_heap_t; struct heap_small_write_t; struct heap_big_write_t; struct heap_big_intent_t; struct __attribute__((__packed__)) heap_entry_t { uint16_t size; uint16_t entry_type; uint32_t checksum; uint64_t lsn; uint64_t inode; uint64_t stripe; uint64_t version; // uint8_t[] external_bitmap // uint8_t[] internal_bitmap // uint32_t[] checksums inline uint8_t type() const { return (entry_type & BS_HEAP_TYPE); } inline heap_small_write_t& small() { return *(heap_small_write_t*)this; } inline heap_big_write_t& big() { return *(heap_big_write_t*)this; } inline heap_big_intent_t& big_intent() { return *(heap_big_intent_t*)this; } bool is_garbage(); void set_garbage(); bool is_overwrite(); bool is_compactable(); bool is_before(heap_entry_t *other); uint32_t get_size(blockstore_heap_t *heap); uint8_t *get_ext_bitmap(blockstore_heap_t *heap); uint8_t *get_int_bitmap(blockstore_heap_t *heap); uint8_t *get_checksums(blockstore_heap_t *heap); uint32_t *get_checksum(blockstore_heap_t *heap); uint64_t big_location(blockstore_heap_t *heap); void set_big_location(blockstore_heap_t *heap, uint64_t location); uint32_t calc_checksum(blockstore_heap_t *heap); uint32_t calc_checksum(blockstore_disk_t *dsk); }; struct __attribute__((__packed__)) heap_small_write_t { heap_entry_t hdr; uint64_t location; uint32_t offset; uint32_t len; // Also includes 1 bitmap and 1 checksum after the bitmap if block checksums are disabled }; struct __attribute__((__packed__)) heap_big_write_t { heap_entry_t hdr; uint32_t block_num; }; struct __attribute__((__packed__)) heap_big_intent_t { heap_entry_t hdr; uint32_t block_num; uint32_t offset; uint32_t len; // Also includes 2 bitmaps and 1 checksums if block checksums are disabled }; struct __attribute__((__packed__)) heap_list_item_t { heap_list_item_t *prev; heap_list_item_t *next; uint32_t block_num; heap_entry_t entry; }; struct heap_object_mvcc_t { uint32_t readers = 0; heap_entry_t *garbage_entry = NULL; }; struct heap_block_info_t { uint32_t used_space = 0; uint64_t mod_lsn = 0, mod_lsn_to = 0; // only 1 block write of LSN sequence is allowed at a moment bool is_writing: 1; bool has_garbage: 1; std::vector entries; }; struct heap_inflight_lsn_t { uint64_t flags; heap_entry_t *wr; }; struct heap_compact_t { uint64_t compact_lsn, compact_version; heap_entry_t *clean_wr; bool do_delete; }; struct heap_reshard_state_t; struct heap_li_hash { size_t operator()(const heap_list_item_t* li) const noexcept { return robin_hood::hash_int(li->entry.stripe); } }; struct heap_li_equal { constexpr bool operator()(const heap_list_item_t* a, const heap_list_item_t* b) const noexcept { return a->entry.stripe == b->entry.stripe; } }; using i64hash_t = robin_hood::hash; using heap_inode_map_t = robin_hood::unordered_flat_set; using heap_block_index_t = robin_hood::unordered_flat_map, i64hash_t>; using heap_mvcc_map_t = robin_hood::unordered_flat_map; class blockstore_heap_t { friend struct heap_entry_t; blockstore_disk_t *dsk = NULL; uint8_t* buffer_area = NULL; int log_level = 0; const uint32_t meta_block_count = 0; const uint32_t max_entry_size = 0; robin_hood::unordered_flat_map pool_shard_settings; // PG => inode => stripe => block number heap_block_index_t block_index; std::vector block_info; allocator_t *data_alloc = NULL; multilist_index_t *meta_alloc = NULL; uint32_t meta_nearfull_blocks = 0; uint64_t meta_used_space = 0; multilist_alloc_t *buffer_alloc = NULL; std::map inode_space_stats; uint64_t buffer_area_used_space = 0; uint64_t data_used_space = 0; uint64_t next_lsn = 0; uint32_t last_allocated_block = UINT32_MAX; heap_mvcc_map_t object_mvcc; // LSN queue: inflight (writing) -> completed [-> fsynced] std::deque inflight_lsn; uint32_t to_compact_count = 0; uint64_t compacted_count = 0; uint32_t inflight_overwrite_count = 0; uint64_t first_inflight_lsn = 0; uint64_t completed_lsn = 0; uint64_t fsynced_lsn = 0; std::deque compact_queue; bool marked_used_blocks = false; bool recheck_queue_filled = false; std::vector loaded_list_items; std::set recheck_modified_blocks; std::deque recheck_queue; int recheck_in_progress = 0; bool in_recheck = false; std::function)> recheck_cb; int recheck_queue_depth = 0; uint64_t get_pg_id(inode_t inode, uint64_t stripe); bool validate_object(heap_entry_t *obj); void fill_recheck_queue(); int mark_used_blocks(); void recheck_buffer(heap_entry_t *cwr, uint8_t *buf); void defragment_block(uint32_t block_num); void reshard_add(heap_reshard_state_t *st, heap_list_item_t *li); void gc_block(heap_block_info_t & inf); int allocate_entry(uint32_t entry_size, uint32_t *block_num, bool allow_last_free); void insert_list_item(heap_list_item_t *li); int add_entry(uint32_t wr_size, uint32_t *modified_block, bool allow_last_free, bool explicit_complete, std::function fill_entry); int add_simple(heap_entry_t *obj, uint64_t version, uint32_t *modified_block, uint32_t entry_type); uint32_t meta_alloc_pos(const heap_block_info_t & inf); void modify_alloc(uint32_t block_num, std::function change_cb); void mark_garbage_up_to(heap_entry_t *wr); void mark_garbage(uint32_t block_num, heap_entry_t *prev_wr, uint32_t used_big); void push_inflight_lsn(uint64_t lsn, heap_entry_t *wr, uint64_t flags); void mark_completed_lsns(uint64_t mod_lsn); void apply_inflight(heap_inflight_lsn_t & inflight); public: blockstore_heap_t(blockstore_disk_t *dsk, uint8_t *buffer_area, int log_level = 0); ~blockstore_heap_t(); void start_load(uint64_t completed_lsn); // load data from the disk, returns EDOM on corruption int read_blocks(uint64_t disk_offset, uint64_t size, uint8_t *buf, bool allow_corrupted, std::function handle_write, std::function handle_block); int load_blocks(uint64_t disk_offset, uint64_t size, uint8_t *buf, bool allow_corrupted, uint64_t &entries_loaded); // finish loading - should be called after load_blocks void finish_load(); // get blocks which are modified during loading and should be written to the disk // before finishing initialization if not R/O std::vector get_recheck_modified_blocks(); // recheck small write data after reading the database from disk bool recheck_small_writes(std::function)> read_buffer, int queue_depth); int finish_recheck(); // reshard database according to the pool's PG count void* reshard_start(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size, uint64_t chunk_limit); bool reshard_continue(void* reshard_state, uint64_t chunk_limit); bool reshard_check(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size); void set_no_inode_stats(const std::vector & pool_ids); void recalc_inode_space_stats(uint64_t pool_id, bool per_inode); // read an object entry and lock it against removal // in the future, may become asynchronous heap_entry_t *lock_and_read_entry(object_id oid); // read an object entry without locking it heap_entry_t *read_entry(object_id oid); // unlock an entry bool unlock_entry(object_id oid); // set or verify checksums in a write request bool calc_checksums(heap_entry_t *wr, uint8_t *data, bool set, uint32_t offset = UINT32_MAX, uint32_t len = UINT32_MAX); // set or verify raw block checksums bool calc_block_checksums(uint32_t *block_csums, uint8_t *data, uint8_t *bitmap, uint32_t start, uint32_t end, bool set, std::function bad_block_cb); bool calc_block_checksums(uint32_t *block_csums, uint8_t *bitmap, uint32_t start, uint32_t end, std::function next, bool set, std::function bad_block_cb); // adds a small_write or intent_write entry to an object // return 0 if OK, or maybe ENOSPC int add_small_write(object_id oid, heap_entry_t **obj_ptr, uint16_t type, uint64_t version, uint32_t offset, uint32_t len, uint64_t location, uint8_t *bitmap, uint8_t *data, uint32_t *modified_block); // adds a big_write (overwrite) entry to an object int add_big_write(object_id oid, heap_entry_t *old_head, bool stable, uint64_t version, uint32_t offset, uint32_t len, uint64_t location, uint8_t *bitmap, uint8_t *data, uint32_t *modified_block); // adds a "redirecting" big_intent entry to an object (same as big_write, used to avoid fsync on desktop SSDs) int add_redirect_intent(object_id oid, heap_entry_t **obj_ptr, uint64_t version, uint32_t offset, uint32_t len, uint64_t location, uint8_t *bitmap, uint8_t *data, uint32_t *modified_block); // adds a big_intent (atomic partial modification) entry to an object int add_big_intent(object_id oid, heap_entry_t **obj_ptr, uint64_t version, uint32_t offset, uint32_t len, uint8_t *bitmap, uint8_t *data, uint8_t *checksums, uint32_t *modified_block); // adds a compacted up to entry to an object int add_compact(heap_entry_t *obj, uint64_t compact_version, uint64_t compact_lsn, uint64_t compact_location, bool do_delete, uint32_t *modified_block, uint8_t *new_int_bitmap, uint8_t *new_ext_bitmap, uint8_t *new_csums); // "punch holes" in a big_entry int punch_holes(heap_entry_t *wr, uint8_t *new_bitmap, uint8_t *new_csums, uint32_t *modified_block); // stabilize an unstable object version // return 0 if OK, ENOENT if not exists int add_commit(heap_entry_t *obj, uint64_t version, uint32_t *modified_block); // rollback an unstable object version // return 0 if OK, ENOENT if not exists, EBUSY if already stable int add_rollback(heap_entry_t *obj, uint64_t version, uint32_t *modified_block); // forget an object // return error code int add_delete(heap_entry_t *obj, uint32_t *modified_block); // get the next object to compact // guaranteed to return objects in min lsn order // returns 0 if OK, ENOENT if nothing to compact int get_next_compact(object_id & oid); void iterate_with_stable(heap_entry_t *obj, uint64_t max_lsn, std::function cb); // iterate compactable entries heap_compact_t iterate_compaction(heap_entry_t *obj, uint64_t fsynced_lsn, bool under_pressure, std::function small_wr_cb); // iterate all objects void iterate_objects(std::function cb); // retrieve object listing from a PG int list_objects(uint32_t pg_num, object_id min_oid, object_id max_oid, obj_ver_id **result_list, size_t *stable_count, size_t *unstable_count); // inflight write tracking void start_block_write(uint32_t block_num); void complete_block_write(uint32_t block_num); void complete_lsn_write(uint64_t lsn); bool is_lsn_completed(uint64_t lsn); uint64_t get_completed_lsn(); uint64_t get_fsynced_lsn(); void mark_lsn_fsynced(uint64_t lsn); // data device block allocator functions uint64_t find_free_data(); bool is_data_used(uint64_t location); void use_data(inode_t inode, uint64_t location); void free_data(inode_t inode, uint64_t location); // buffer device allocator functions uint64_t find_free_buffer_area(uint64_t size); bool is_buffer_area_free(uint64_t location, uint64_t size); void use_buffer_area(inode_t inode, uint64_t location, uint64_t size); void free_buffer_area(inode_t inode, uint64_t location, uint64_t size); uint64_t get_buffer_area_used_space(); // get metadata block data buffer and used space void get_meta_block(uint32_t block_num, uint8_t *buffer); void fill_block_empty_space(uint8_t *buffer, uint32_t pos); uint32_t get_meta_block_used_space(uint32_t block_num); // get space usage statistics uint64_t get_data_used_space(); const std::map & get_inode_space_stats(); uint64_t get_meta_total_space(); uint64_t get_meta_used_space(); uint32_t get_meta_nearfull_blocks(); uint32_t get_compact_queue_size(); uint32_t get_to_compact_count(); uint64_t get_compacted_count(); uint64_t entry_pos(uint32_t block_num, uint32_t offset); heap_entry_t *entry_from_pos(uint64_t entry_pos, bool allow_unallocated = false); heap_entry_t *prev(heap_entry_t *wr); uint32_t get_simple_entry_size(); uint32_t get_big_entry_size(); uint32_t get_big_intent_entry_size(); uint32_t get_small_entry_size(uint32_t offset, uint32_t len); uint32_t get_csum_size(heap_entry_t *wr); uint32_t get_csum_size(uint32_t entry_type, uint32_t offset = 0, uint32_t len = 0); };