thebestvibedev/tromcho.net
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Finish journal reader

Browse Source
This commit is contained in:
Vitaliy Filippov
2019-11-04 20:18:52 +03:00
parent e1c92d2227
commit c959948c82
5 changed files with 157 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
blockstore_init.cpp
+93 -41
View File
@@ -1,5 +1,4 @@
#include "blockstore.h"
#include "crc32c.h"
blockstore_init_meta::blockstore_init_meta(blockstore *bs)
{
@@ -101,13 +100,6 @@ bool iszero(uint64_t *buf, int len)
return true;
}
inline uint32_t je_crc32(journal_entry *je)
{
return crc32c_zero4(((uint8_t*)je)+4, je->size-4);
}
#define JOURNAL_BUFFER_SIZE 4*1024*1024
int blockstore_init_journal::read_loop()
{
if (step == 100)
@@ -148,8 +140,8 @@ int blockstore_init_journal::read_loop()
if (iszero((uint64_t*)journal_buffer, 3))
{
// Journal is empty
bs->journal_start = 0;
bs->journal_end = 0;
bs->journal_start = 512;
bs->journal_end = 512;
step = 99;
}
else
@@ -194,36 +186,51 @@ int blockstore_init_journal::read_loop()
{
// Continue from the beginning
journal_pos = 512;
wrapped = true;
}
submitted = 0;
}
}
if (!submitted && step != 3)
if (!submitted)
{
struct io_uring_sqe *sqe = io_uring_get_sqe(bs->ring);
if (!sqe)
if (step != 3)
{
throw new std::runtime_error("io_uring is full while trying to read journal");
if (journal_pos == bs->journal_start && wrapped)
{
step = 3;
}
else
{
struct io_uring_sqe *sqe = io_uring_get_sqe(bs->ring);
if (!sqe)
{
throw new std::runtime_error("io_uring is full while trying to read journal");
}
uint64_t end = bs->journal_len;
if (journal_pos < bs->journal_start)
{
end = bs->journal_start;
}
submit_iov = {
journal_buffer + (done_buf == 1 ? JOURNAL_BUFFER_SIZE : 0),
end - journal_pos < JOURNAL_BUFFER_SIZE ? end - journal_pos : JOURNAL_BUFFER_SIZE,
};
io_uring_prep_readv(sqe, bs->journal_fd, &submit_iov, 1, bs->journal_offset + journal_pos);
io_uring_submit(bs->ring);
submitted = done_buf == 1 ? 2 : 1;
}
}
uint64_t end = bs->journal_len;
if (journal_pos < bs->journal_start)
else
{
end = bs->journal_start;
step = 99;
}
submit_iov = {
journal_buffer + (done_buf == 1 ? JOURNAL_BUFFER_SIZE : 0),
end - journal_pos < JOURNAL_BUFFER_SIZE ? end - journal_pos : JOURNAL_BUFFER_SIZE,
};
io_uring_prep_readv(sqe, bs->journal_fd, &submit_iov, 1, bs->journal_offset + journal_pos);
io_uring_submit(bs->ring);
submitted = done_buf == 1 ? 2 : 1;
}
if (done_buf && step != 3)
{
// handle journal entries
if (handle_journal(journal_buffer + (done_buf == 1 ? 0 : JOURNAL_BUFFER_SIZE), done_len) == 0)
if (handle_journal_part(journal_buffer + (done_buf == 1 ? 0 : JOURNAL_BUFFER_SIZE), done_len) == 0)
{
// finish
// journal ended. wait for the next read to complete, then stop
step = 3;
}
done_buf = 0;
@@ -232,43 +239,69 @@ int blockstore_init_journal::read_loop()
if (step == 99)
{
free(journal_buffer);
bs->journal_crc32_last = crc32_last;
journal_buffer = NULL;
step = 100;
}
return 1;
}
int blockstore_init_journal::handle_journal(void *buf, int len)
int blockstore_init_journal::handle_journal_part(void *buf, uint64_t len)
{
int total_pos = 0;
uint64_t total_pos = 0;
if (cur_skip >= 0)
{
total_pos = cur_skip;
cur_skip = 0;
}
while (total_pos < len)
{
int pos = 0, skip = 0;
total_pos += 512;
uint64_t pos = 0;
while (pos < 512)
{
journal_entry *je = (journal_entry*)((uint8_t*)buf + total_pos + pos);
if (je->magic != JOURNAL_MAGIC || je_crc32(je) != je->crc32 ||
je->type < JE_SMALL_WRITE || je->type > JE_DELETE || je->crc32_prev != crc32_last)
{
// Invalid entry - end of the journal
bs->journal_end = done_pos + total_pos + pos;
// FIXME: save <skip>
return 0;
if (pos == 0)
{
// invalid entry in the beginning, this is definitely the end of the journal
bs->journal_end = done_pos + total_pos + pos;
return 0;
}
else
{
// allow partially filled sectors
break;
}
}
pos += je->size;
crc32_last = je->crc32;
if (je->type == JE_SMALL_WRITE)
{
// oid, version, offset, len
uint64_t location;
if (cur_skip > 0 || done_pos + total_pos + je->small_write.len > bs->journal_len)
{
// data continues from the beginning of the journal
location = 512 + cur_skip;
cur_skip += je->small_write.len;
}
else
{
// data is right next
location = done_pos + total_pos;
total_pos += je->small_write.len;
}
bs->dirty_queue[je->small_write.oid].push_back((dirty_entry){
.version = je->small_write.version,
.state = ST_J_SYNCED,
.flags = 0,
// FIXME: data in journal may never be non-contiguous
.location = done_pos + total_pos + 512 + skip,
.location = location,
.offset = je->small_write.offset,
.size = je->small_write.len,
});
skip += je->small_write.len;
}
else if (je->type == JE_BIG_WRITE)
{
@@ -288,28 +321,47 @@ int blockstore_init_journal::handle_journal(void *buf, int len)
auto it = bs->dirty_queue.find(je->stable.oid);
if (it == bs->dirty_queue.end())
{
// FIXME ignore entry, but warn
// journal contains a legitimate STABLE entry for a non-existing dirty write
// this probably means that journal was trimmed between WRITTEN and STABLE entries
// skip for now. but FIXME: maybe warn about it in the future
}
else
{
auto & lst = it->second;
for (int i = 0; i < lst.size(); i++)
int i;
for (i = 0; i < lst.size(); i++)
{
if (lst[i].version == je->stable.version)
{
lst[i].state = (lst[i].state == ST_D_META_SYNCED ? ST_D_STABLE : ST_J_STABLE);
lst[i].state = (lst[i].state == ST_D_META_SYNCED
? ST_D_STABLE
: (lst[i].state == ST_DEL_SYNCED ? ST_DEL_STABLE : ST_J_STABLE));
break;
}
}
if (i >= lst.size())
{
// same. STABLE entry for a missing object version
}
}
}
else if (je->type == JE_DELETE)
{
// oid, version
// FIXME
bs->dirty_queue[je->small_write.oid].push_back((dirty_entry){
.version = je->small_write.version,
.state = ST_DEL_SYNCED,
.flags = 0,
.location = 0,
.offset = 0,
.size = 0,
});
}
}
total_pos += 512 + skip;
}
if (cur_skip == 0 && total_pos > len)
{
cur_skip = total_pos - len;
}
return 1;
}