WIP dump/load heap

2025-12-02 01:52:12 +03:00
parent 97dfbfad75
commit 3ea9230ed0
16 changed files with 1161 additions and 218 deletions
@@ -228,11 +228,11 @@ void blockstore_disk_t::calc_lengths()
 {
    // data
    data_len = data_device_size - data_offset;
-    if (data_fd == meta_fd && data_offset < meta_offset)
+    if (data_device == meta_device && data_offset < meta_offset)
    {
        data_len = meta_offset - data_offset;
    }
-    if (data_fd == journal_fd && data_offset < journal_offset)
+    if (data_device == journal_device && data_offset < journal_offset)
    {
        data_len = data_len < journal_offset-data_offset
            ? data_len : journal_offset-data_offset;
@@ -247,23 +247,23 @@ void blockstore_disk_t::calc_lengths()
        data_len = cfg_data_size;
    }
    // meta
-    meta_area_size = (meta_fd == data_fd ? data_device_size : meta_device_size) - meta_offset;
-    if (meta_fd == data_fd && meta_offset <= data_offset)
+    meta_area_size = (meta_device == data_device ? data_device_size : meta_device_size) - meta_offset;
+    if (meta_device == data_device && meta_offset <= data_offset)
    {
        meta_area_size = data_offset - meta_offset;
    }
-    if (meta_fd == journal_fd && meta_offset <= journal_offset)
+    if (meta_device == journal_device && meta_offset <= journal_offset)
    {
        meta_area_size = meta_area_size < journal_offset-meta_offset
            ? meta_area_size : journal_offset-meta_offset;
    }
    // journal
-    journal_len = (journal_fd == data_fd ? data_device_size : (journal_fd == meta_fd ? meta_device_size : journal_device_size)) - journal_offset;
-    if (journal_fd == data_fd && journal_offset <= data_offset)
+    journal_len = (journal_device == data_device ? data_device_size : (journal_device == meta_device ? meta_device_size : journal_device_size)) - journal_offset;
+    if (journal_device == data_device && journal_offset <= data_offset)
    {
        journal_len = data_offset - journal_offset;
    }
-    if (journal_fd == meta_fd && journal_offset <= meta_offset)
+    if (journal_device == meta_device && journal_offset <= meta_offset)
    {
        journal_len = journal_len < meta_offset-journal_offset
            ? journal_len : meta_offset-journal_offset;
@@ -1285,18 +1285,11 @@ void blockstore_heap_t::allocate_block(heap_block_info_t & inf)
    }
 }

-int blockstore_heap_t::add_object(object_id oid, heap_write_t *wr, uint32_t *modified_block)
+int blockstore_heap_t::allocate_new_object(object_id oid, uint32_t full_object_size, uint32_t *modified_block, heap_object_t **new_obj)
 {
-    // By now, initial small_writes are not allowed
-    if (wr->type() != BS_HEAP_BIG_WRITE &&
-        wr->type() != BS_HEAP_TOMBSTONE)
-    {
-        return EINVAL;
-    }
-    const uint32_t wr_size = wr->get_size(this);
-    // Allocate block (always leave at least <max_write_entry_size> free_space in the block)
    uint32_t block_num = 0;
-    int res = get_block_for_new_object(block_num, sizeof(heap_object_t)+wr_size+max_write_entry_size);
+    // Allocate block (always leave at least <max_write_entry_size> free_space in the block)
+    int res = get_block_for_new_object(block_num, full_object_size+max_write_entry_size);
    if (res != 0)
    {
        return res;
@@ -1307,27 +1300,66 @@ int blockstore_heap_t::add_object(object_id oid, heap_write_t *wr, uint32_t *mod
    {
        *modified_block = block_num;
    }
-    const uint32_t offset = find_block_space(block_num, sizeof(heap_object_t)+wr_size);
+    const uint32_t offset = find_block_space(block_num, full_object_size);
    if (offset == UINT32_MAX)
    {
        return ENOSPC;
    }
+    add_used_space(block_num, full_object_size);
    block_index[get_pg_id(oid.inode, oid.stripe)][oid.inode][oid.stripe] = (uint64_t)block_num*dsk->meta_block_size + offset;
-    // and just append the object entry
-    heap_object_t *new_entry = (heap_object_t *)(inf.data + offset);
-    new_entry->write_pos = sizeof(heap_object_t);
-    new_entry->inode = oid.inode;
-    new_entry->stripe = oid.stripe;
-    heap_write_t *new_wr = new_entry->get_writes();
+    *new_obj = (heap_object_t *)(inf.data + offset);
+    return 0;
+}
+
+int blockstore_heap_t::add_object(object_id oid, heap_write_t *wr, uint32_t *modified_block)
+{
+    // By now, initial small_writes are not allowed
+    if (wr->type() != BS_HEAP_BIG_WRITE &&
+        wr->type() != BS_HEAP_TOMBSTONE)
+    {
+        return EINVAL;
+    }
+    const uint32_t wr_size = wr->get_size(this);
+    heap_object_t *new_obj = NULL;
+    int res = allocate_new_object(oid, sizeof(heap_object_t)+wr_size, modified_block, &new_obj);
+    if (res != 0)
+    {
+        return res;
+    }
+    // Fill the object entry
+    new_obj->size = sizeof(heap_object_t);
+    new_obj->write_pos = sizeof(heap_object_t);
+    new_obj->inode = oid.inode;
+    new_obj->stripe = oid.stripe;
+    heap_write_t *new_wr = new_obj->get_writes();
    memcpy(new_wr, wr, wr_size);
    new_wr->next_pos = 0;
    new_wr->size = wr_size;
    new_wr->lsn = ++next_lsn;
    wr->lsn = new_wr->lsn;
    push_inflight_lsn(oid, new_wr->lsn, new_wr->needs_compact(this) ? HEAP_INFLIGHT_COMPACTABLE : 0);
-    new_entry->size = sizeof(heap_object_t);
-    new_entry->crc32c = new_entry->calc_crc32c();
-    add_used_space(block_num, sizeof(heap_object_t) + wr_size);
+    new_obj->crc32c = new_obj->calc_crc32c();
+    return 0;
+}
+
+int blockstore_heap_t::copy_object(heap_object_t *obj, uint32_t *modified_block)
+{
+    // Allocate block (always leave at least <max_write_entry_size> free_space in the block)
+    auto oid = (object_id){ .inode = obj->inode, .stripe = obj->stripe };
+    assert(!read_entry(oid, NULL));
+    uint32_t full_object_size = obj->size;
+    for (auto wr = obj->get_writes(); wr; wr = wr->next())
+    {
+        full_object_size += wr->size;
+    }
+    heap_object_t *new_obj = NULL;
+    int res = allocate_new_object(oid, full_object_size, modified_block, &new_obj);
+    if (res != 0)
+    {
+        return res;
+    }
+    copy_full_object((uint8_t*)new_obj, obj);
+    new_obj->crc32c = new_obj->calc_crc32c();
    return 0;
 }

@@ -215,6 +215,7 @@ class blockstore_heap_t
    bool mvcc_check_tracking(object_id oid);
    void free_mvcc(heap_mvcc_map_t::iterator mvcc_it);
    void allocate_block(heap_block_info_t & inf);
+    int allocate_new_object(object_id oid, uint32_t full_object_size, uint32_t *modified_block, heap_object_t **new_obj);
    int add_object(object_id oid, heap_write_t *wr, uint32_t *modified_block);
    void mark_overwritten(uint64_t over_lsn, uint64_t inode, heap_write_t *wr, heap_write_t *end_wr, bool tracking_active);
    int update_object(uint32_t block_num, heap_object_t *obj, heap_write_t *wr, uint32_t *modified_block, uint32_t *moved_from_block);
@@ -268,6 +269,8 @@ public:
    bool calc_block_checksums(uint32_t *block_csums, uint8_t *bitmap,
        uint32_t start, uint32_t end, std::function<uint8_t*(uint32_t start, uint32_t & len)> next,
        bool set, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);
+    // copy an object as is
+    int copy_object(heap_object_t *obj, uint32_t *modified_block);
    // auto-compacts the object, then adds a write entry to it and to the compaction queue
    // return 0 if OK, or maybe ENOSPC
    int post_write(object_id oid, heap_write_t *wr, uint32_t *modified_block, uint32_t *moved_from_block);
@@ -355,7 +355,8 @@ void blockstore_impl_t::dump_diagnostics()
 void blockstore_meta_header_v3_t::set_crc32c()
 {
    header_csum = 0;
-    uint32_t calc = crc32c(0, this, sizeof(*this));
+    uint32_t calc = crc32c(0, this, version == BLOCKSTORE_META_FORMAT_HEAP
+        ? sizeof(blockstore_meta_header_v3_t) : sizeof(blockstore_meta_header_v2_t));
    header_csum = calc;
 }