thebestvibedev/tromcho.net
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Do not store offset & len in big_writes

Browse Source
This commit is contained in:
Vitaliy Filippov
2025-12-02 01:52:12 +03:00
parent 94f3634602
commit ac8e0ef231
10 changed files with 330 additions and 270 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/disk_tool/disk_tool_discard.cpp
+1 -1
View File
@@ -61,7 +61,7 @@ int disk_tool_t::trim_data(std::string device)
{
if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE)
{
data_alloc->set(wr->location / dsk.data_block_size, true);
data_alloc->set(wr->big().location / dsk.data_block_size, true);
}
}
},
src/disk_tool/disk_tool_meta.cpp
+55 -38
View File
@@ -349,7 +349,7 @@ void disk_tool_t::dump_heap_entry_as_old(blockstore_heap_t *heap, heap_object_t
#define ENTRY_FMT "{\"block\":%ju,\"pool\":%u,\"inode\":\"0x%jx\",\"stripe\":\"0x%jx\",\"version\":%ju"
(first_entry ? ENTRY_FMT : (",\n" ENTRY_FMT)),
#undef ENTRY_FMT
wr->location/dsk.data_block_size, INODE_POOL(obj->inode), INODE_NO_POOL(obj->inode),
wr->big().location/dsk.data_block_size, INODE_POOL(obj->inode), INODE_NO_POOL(obj->inode),
obj->stripe, wr->version
);
printf(",\"bitmap\":\"");
@@ -395,27 +395,36 @@ void disk_tool_t::dump_heap_entry(blockstore_heap_t *heap, heap_object_t *obj)
for (wr = obj->get_writes(); wr; wr = wr->next())
{
printf(
#define ENTRY_FMT "{\"lsn\":%ju,\"version\":%ju,\"type\":\"%s\",\"stable\":%s,\"offset\":%u,\"len\":%u"
#define ENTRY_FMT "{\"lsn\":%ju,\"version\":%ju,\"type\":\"%s\",\"stable\":%s"
(first_wr ? ENTRY_FMT : ("," ENTRY_FMT)),
#undef ENTRY_FMT
wr->lsn, wr->version, (wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_SMALL_WRITE ? "small" : (
(wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE ? "big" : (
(wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_INTENT_WRITE ? "intent" : (
(wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_TOMBSTONE ? "tombstone" : "unknown"))),
(wr->entry_type & BS_HEAP_STABLE) ? "true" : "false",
wr->offset, wr->len
(wr->entry_type & BS_HEAP_STABLE) ? "true" : "false"
);
if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE ||
(wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_SMALL_WRITE && !dump_with_data)
if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE)
{
printf(",\"location\":%ju", wr->location);
printf(",\"location\":%ju", wr->big().location);
}
else if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_SMALL_WRITE && dump_with_data)
else if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_INTENT_WRITE)
{
printf(",\"data\":\"");
for (uint32_t i = 0; i < wr->len; i++)
printf("%02x", buffer_area[wr->location + i]);
printf("\"");
printf(",\"offset\":%u,\"len\":%u", wr->small().offset, wr->small().len);
}
else if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_SMALL_WRITE)
{
if (!dump_with_data)
{
printf(",\"offset\":%u,\"len\":%u,\"location\":%ju", wr->small().offset, wr->small().len, wr->small().location);
}
else
{
printf(",\"data\":\"");
for (uint32_t i = 0; i < wr->small().len; i++)
printf("%02x", buffer_area[wr->small().location + i]);
printf("\"");
}
}
uint8_t* bitmap = wr->get_int_bitmap(heap);
if (bitmap)
@@ -640,21 +649,28 @@ int disk_tool_t::write_json_heap(json11::Json meta, json11::Json journal)
wr->entry_type = wr_type | (write_entry["stable"].bool_value() ? BS_HEAP_STABLE : 0);
wr->lsn = write_entry["lsn"].uint64_value();
wr->version = write_entry["version"].uint64_value();
wr->offset = write_entry["offset"].uint64_value();
wr->len = write_entry["len"].uint64_value();
wr->location = write_entry["location"].uint64_value();
wr->size = wr->get_size(new_heap);
wr->next_pos = wr->size;
if (wr_type == BS_HEAP_SMALL_WRITE && write_entry["data"].is_string() && wr->len > 0)
if (wr_type == BS_HEAP_SMALL_WRITE || wr_type == BS_HEAP_INTENT_WRITE)
{
if (!new_journal_buf)
wr->small().offset = write_entry["offset"].uint64_value();
wr->small().len = write_entry["len"].uint64_value();
wr->small().location = write_entry["location"].uint64_value();
if (wr_type == BS_HEAP_SMALL_WRITE && write_entry["data"].is_string() && wr->small().len > 0)
{
fprintf(stderr, "Loading small write data requires overwriting buffer area\n");
free_new_meta();
return 1;
if (!new_journal_buf)
{
fprintf(stderr, "Loading small write data requires overwriting buffer area\n");
free_new_meta();
return 1;
}
wr->small().location = new_heap->find_free_buffer_area(wr->small().len);
fromhexstr(write_entry["data"].string_value(), wr->small().len, new_journal_buf + wr->small().location);
}
wr->location = new_heap->find_free_buffer_area(wr->len);
fromhexstr(write_entry["data"].string_value(), wr->len, new_journal_buf + wr->location);
}
else if (wr_type == BS_HEAP_BIG_WRITE)
{
wr->big().location = write_entry["location"].uint64_value();
}
if (write_entry["bitmap"].is_string() && wr->get_int_bitmap(new_heap))
{
@@ -778,9 +794,7 @@ close_err:
wr->entry_type = BS_HEAP_BIG_WRITE|BS_HEAP_STABLE;
wr->lsn = ++next_lsn;
wr->version = sscanf_json(NULL, meta_entry["version"]);
wr->offset = 0;
wr->len = new_meta_hdr->data_block_size;
wr->location = meta_entry["block"].uint64_value() * new_meta_hdr->data_block_size;
wr->big().location = meta_entry["block"].uint64_value() * new_meta_hdr->data_block_size;
wr->size = wr->get_size(&heap);
fromhexstr(meta_entry["bitmap"].string_value(), new_clean_entry_bitmap_size, wr->get_int_bitmap(&heap));
fromhexstr(meta_entry["ext_bitmap"].string_value(), new_clean_entry_bitmap_size, wr->get_ext_bitmap(&heap));
@@ -795,36 +809,39 @@ close_err:
wr->next_pos = 0;
wr->lsn = ++next_lsn;
wr->version = rec["ver"].uint64_value();
wr->offset = rec["offset"].uint64_value();
wr->len = rec["len"].uint64_value();
uint64_t wr_offset = rec["offset"].uint64_value();
uint64_t wr_len = rec["len"].uint64_value();
if (rec["type"] == "small_write" || rec["type"] == "small_write_instant")
{
if (wr->len > 0 && !rec["data"].is_string())
if (wr_len > 0 && !rec["data"].is_string())
{
fprintf(stderr, "Error: entry data is missing, please generate the dump with --json --format data\n");
goto close_err;
}
wr->entry_type = BS_HEAP_SMALL_WRITE | (rec["type"] == "small_write_instant" ? BS_HEAP_STABLE : 0);
wr->small().offset = wr_offset;
wr->small().len = wr_len;
wr->small().location = buffer_pos;
fromhexstr(rec["bitmap"].string_value(), new_clean_entry_bitmap_size, wr->get_ext_bitmap(&heap));
fromhexstr(rec["data"].string_value(), wr->len, new_journal_buf+buffer_pos);
if (wr->len > 0)
fromhexstr(rec["data"].string_value(), wr_len, new_journal_buf+buffer_pos);
if (wr_len > 0)
{
if (!new_meta_hdr->data_csum_type)
*wr->get_checksum(&heap) = crc32c(0, new_journal_buf+buffer_pos, wr->len);
*wr->get_checksum(&heap) = crc32c(0, new_journal_buf+buffer_pos, wr_len);
else
heap.calc_block_checksums((uint32_t*)wr->get_checksums(&heap), new_journal_buf+buffer_pos, NULL, wr->offset, wr->offset+wr->len, true, NULL);
heap.calc_block_checksums((uint32_t*)wr->get_checksums(&heap), new_journal_buf+buffer_pos, NULL, wr_offset, wr_offset+wr_len, true, NULL);
}
buffer_pos += wr->len;
buffer_pos += wr_len;
}
else if (rec["type"] == "big_write" || rec["type"] == "big_write_instant")
{
wr->entry_type = BS_HEAP_BIG_WRITE | (rec["type"] == "big_write_instant" ? BS_HEAP_STABLE : 0);
wr->location = sscanf_json(NULL, rec["loc"]);
bitmap_set(wr->get_int_bitmap(&heap), wr->offset, wr->len, new_meta_hdr->bitmap_granularity);
wr->big().location = sscanf_json(NULL, rec["loc"]);
bitmap_set(wr->get_int_bitmap(&heap), wr_offset, wr_len, new_meta_hdr->bitmap_granularity);
fromhexstr(rec["bitmap"].string_value(), new_clean_entry_bitmap_size, wr->get_ext_bitmap(&heap));
if (new_meta_hdr->data_csum_type != 0)
{
if ((wr->offset % new_meta_hdr->csum_block_size) || (wr->len % new_meta_hdr->csum_block_size))
if ((wr_offset % new_meta_hdr->csum_block_size) || (wr_len % new_meta_hdr->csum_block_size))
{
fprintf(stderr,
"Error: big_write journal entries not aligned to csum_block_size can't be converted between v0.9 and v3.0 metadata\n"
@@ -832,8 +849,8 @@ close_err:
goto close_err;
}
fromhexstr(rec["block_csums"].string_value(),
((wr->offset+wr->len+new_meta_hdr->csum_block_size-1)/new_meta_hdr->csum_block_size
- wr->offset/new_meta_hdr->csum_block_size) * (new_meta_hdr->data_csum_type & 0xFF),
((wr_offset+wr_len+new_meta_hdr->csum_block_size-1)/new_meta_hdr->csum_block_size
- wr_offset/new_meta_hdr->csum_block_size) * (new_meta_hdr->data_csum_type & 0xFF),
wr->get_checksums(&heap));
}
}
src/disk_tool/disk_tool_resize.cpp
+21 -26
View File
@@ -43,7 +43,7 @@ int disk_tool_t::raw_resize()
{
if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE)
{
data_alloc->set(wr->location / dsk.data_block_size, true);
data_alloc->set(wr->big().location / dsk.data_block_size, true);
}
}
},
@@ -569,13 +569,7 @@ int disk_tool_t::resize_rebuild_meta()
{
if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_BIG_WRITE)
{
if (wr->next() && new_meta_hdr->data_csum_type != 0 &&
((wr->offset % new_meta_hdr->csum_block_size) || (wr->len % new_meta_hdr->csum_block_size)))
{
fprintf(stderr, "Error: big_write journal entries not aligned to csum_block_size can't be converted between v0.9 and v3.0 metadata\n");
exit(1);
}
auto block_num = wr->location / dsk.data_block_size;
auto block_num = wr->big().location / dsk.data_block_size;
auto remap_it = data_remap.find(block_num);
if (remap_it != data_remap.end())
block_num = remap_it->second;
@@ -585,20 +579,20 @@ int disk_tool_t::resize_rebuild_meta()
exit(1);
}
block_num += data_idx_diff;
wr->location = block_num * dsk.data_block_size;
wr->big().location = block_num * dsk.data_block_size;
}
else if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_SMALL_WRITE)
{
if (new_heap && wr->len > 0)
if (new_heap && wr->small().len > 0)
{
if (new_journal_ptr-new_journal_buf+wr->len > new_journal_len)
if (new_journal_ptr-new_journal_buf+wr->small().len > new_journal_len)
{
fprintf(stderr, "Small write data doesn't fit into the new buffer area\n");
exit(1);
}
memcpy(new_journal_ptr, buffer_area+wr->location, wr->len);
wr->location = new_journal_ptr-new_journal_buf;
new_journal_ptr += wr->len;
memcpy(new_journal_ptr, buffer_area+wr->small().location, wr->small().len);
wr->small().location = new_journal_ptr-new_journal_buf;
new_journal_ptr += wr->small().len;
}
}
else if (!new_heap)
@@ -626,8 +620,9 @@ int disk_tool_t::resize_rebuild_meta()
{
auto wr = writes[i];
assert((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_SMALL_WRITE || wr->entry_type == BS_HEAP_BIG_WRITE);
uint32_t je_size = dsk.dirty_dyn_size(wr->offset, wr->len) +
((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_SMALL_WRITE ? sizeof(journal_entry_small_write) : sizeof(journal_entry_big_write));
uint32_t je_size = ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_SMALL_WRITE
? sizeof(journal_entry_small_write) + dsk.dirty_dyn_size(wr->small().offset, wr->small().len)
: sizeof(journal_entry_big_write) + dsk.dirty_dyn_size(0, dsk.data_block_size));
choose_journal_block(je_size);
journal_entry *je = (journal_entry*)(new_journal_ptr + new_journal_in_pos);
je->magic = JOURNAL_MAGIC;
@@ -636,28 +631,30 @@ int disk_tool_t::resize_rebuild_meta()
je->crc32_prev = new_crc32_prev;
je->small_write.oid = (object_id){ .inode = obj->inode, .stripe = obj->stripe };
je->small_write.version = wr->version;
je->small_write.offset = wr->offset;
je->small_write.len = wr->len;
if ((wr->entry_type & BS_HEAP_TYPE) == BS_HEAP_SMALL_WRITE)
if (wr->type() == BS_HEAP_SMALL_WRITE)
{
je->small_write.offset = wr->small().offset;
je->small_write.len = wr->small().len;
je->small_write.data_offset = new_journal_data-new_journal_buf;
if (je->small_write.data_offset + je->small_write.len > new_journal_len)
{
fprintf(stderr, "Error: live entries don't fit to the new journal\n");
exit(1);
}
memcpy(new_journal_data, buffer_area+wr->location, je->small_write.len);
memcpy(new_journal_data, buffer_area+wr->small().location, je->small_write.len);
new_journal_data += je->small_write.len;
if (dsk.data_csum_type == 0 && wr->get_checksum(heap))
je->small_write.crc32_data = *wr->get_checksum(heap);
}
else
{
je->big_write.location = wr->location;
je->big_write.location = wr->big().location;
}
memcpy((uint8_t*)je + je->size, wr->get_ext_bitmap(heap), new_clean_entry_bitmap_size);
if (dsk.data_csum_type != 0 && wr->get_checksums(heap))
memcpy((uint8_t*)je + je->size + new_clean_entry_bitmap_size, wr->get_checksums(heap), new_data_csum_size);
{
memcpy((uint8_t*)je + je->size + new_clean_entry_bitmap_size, wr->get_checksums(heap), wr->get_csum_size(heap));
}
je->crc32 = je_crc32(je);
new_journal_in_pos += je->size;
new_crc32_prev = je->crc32;
@@ -666,7 +663,7 @@ int disk_tool_t::resize_rebuild_meta()
if (writes[writes.size()-1]->entry_type == BS_HEAP_BIG_WRITE|BS_HEAP_STABLE)
{
auto big_wr = writes[writes.size()-1];
uint64_t block_num = big_wr->location / dsk.data_block_size;
uint64_t block_num = big_wr->big().location / dsk.data_block_size;
clean_disk_entry *new_entry = (clean_disk_entry*)(new_meta_buf + dsk.meta_block_size +
dsk.meta_block_size*(block_num / new_entries_per_block) +
new_clean_entry_size*(block_num % new_entries_per_block));
@@ -697,10 +694,8 @@ int disk_tool_t::resize_rebuild_meta()
uint8_t wr_buf[new_heap->get_max_write_entry_size()];
heap_write_t *wr = (heap_write_t*)wr_buf;
wr->entry_type = BS_HEAP_BIG_WRITE|BS_HEAP_STABLE;
wr->location = block_num * dsk.data_block_size;
wr->big().location = block_num * dsk.data_block_size;
wr->next_pos = 0;
wr->offset = 0;
wr->len = 0;
wr->size = wr->get_size(new_heap);
if (bitmap)
{