thebestvibedev/tromcho.net
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move all sources to subdirs

Browse Source
This commit is contained in:
Vitaliy Filippov
2024-05-15 11:06:01 +03:00
parent 44692d148a
commit c79b38bd26
219 changed files with 465 additions and 433 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/test/CMakeLists.txt
+51
View File
@@ -0,0 +1,51 @@
cmake_minimum_required(VERSION 2.8.12)
project(vitastor)
### Test stubs
# stub_osd, stub_bench, osd_test
add_executable(stub_osd stub_osd.cpp ../util/rw_blocking.cpp ../util/addr_util.cpp)
target_link_libraries(stub_osd tcmalloc_minimal)
add_executable(stub_bench stub_bench.cpp ../util/rw_blocking.cpp ../util/addr_util.cpp)
target_link_libraries(stub_bench tcmalloc_minimal)
add_executable(osd_test osd_test.cpp ../util/rw_blocking.cpp ../util/addr_util.cpp)
target_link_libraries(osd_test tcmalloc_minimal)
# stub_uring_osd
add_executable(stub_uring_osd
stub_uring_osd.cpp
)
target_link_libraries(stub_uring_osd
vitastor_common
${LIBURING_LIBRARIES}
${IBVERBS_LIBRARIES}
tcmalloc_minimal
)
# test_allocator
add_executable(test_allocator EXCLUDE_FROM_ALL test_allocator.cpp ../util/allocator.cpp)
add_dependencies(build_tests test_allocator)
add_test(NAME test_allocator COMMAND test_allocator)
# test_cas
add_executable(test_cas
test_cas.cpp
)
target_link_libraries(test_cas
vitastor_client
)
# test_crc32
add_executable(test_crc32
test_crc32.cpp
)
target_link_libraries(test_crc32
vitastor_blk
)
## test_blockstore, test_shit
#add_executable(test_blockstore test_blockstore.cpp)
#target_link_libraries(test_blockstore blockstore)
#add_executable(test_shit test_shit.cpp osd_peering_pg.cpp)
#target_link_libraries(test_shit ${LIBURING_LIBRARIES} m)
src/test/mock/messenger.cpp
+64
View File
@@ -0,0 +1,64 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
#include <unistd.h>
#include <stdexcept>
#include <assert.h>
#include "messenger.h"
void osd_messenger_t::init()
{
}
osd_messenger_t::~osd_messenger_t()
{
while (clients.size() > 0)
{
stop_client(clients.begin()->first, true, true);
}
}
void osd_messenger_t::outbox_push(osd_op_t *cur_op)
{
clients[cur_op->peer_fd]->sent_ops[cur_op->req.hdr.id] = cur_op;
}
void osd_messenger_t::parse_config(const json11::Json & config)
{
}
void osd_messenger_t::connect_peer(uint64_t peer_osd, json11::Json peer_state)
{
wanted_peers[peer_osd] = (osd_wanted_peer_t){
.port = 1,
};
}
void osd_messenger_t::read_requests()
{
}
void osd_messenger_t::send_replies()
{
}
json11::Json::object osd_messenger_t::read_config(const json11::Json & config)
{
return json11::Json::object();
}
json11::Json::object osd_messenger_t::merge_configs(const json11::Json::object & cli_config,
const json11::Json::object & file_config,
const json11::Json::object & etcd_global_config,
const json11::Json::object & etcd_osd_config)
{
return cli_config;
}
bool json_is_true(const json11::Json & val)
{
if (val.is_string())
return val == "true" || val == "yes" || val == "1";
return val.bool_value();
}
src/test/mock/ringloop.h
+31
View File
@@ -0,0 +1,31 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
#pragma once
#include <functional>
struct ring_consumer_t
{
std::function<void(void)> loop;
};
class ring_loop_t
{
public:
void register_consumer(ring_consumer_t *consumer)
{
}
void unregister_consumer(ring_consumer_t *consumer)
{
}
void submit()
{
}
void wait()
{
}
void loop()
{
}
};
src/test/osd_test.cpp
+388
View File
@@ -0,0 +1,388 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <arpa/inet.h>
#include <string.h>
#include <stdio.h>
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>
#include <stdlib.h>
#include <malloc.h>
#include <stdexcept>
#include "addr_util.h"
#include "osd_ops.h"
#include "rw_blocking.h"
#include "test_pattern.h"
int connect_osd(const char *osd_address, int osd_port);
uint64_t test_read(int connect_fd, uint64_t inode, uint64_t stripe, uint64_t version, uint64_t offset, uint64_t len);
uint64_t test_write(int connect_fd, uint64_t inode, uint64_t stripe, uint64_t version, uint64_t pattern);
void* test_primary_read(int connect_fd, uint64_t inode, uint64_t offset, uint64_t len);
void test_primary_write(int connect_fd, uint64_t inode, uint64_t offset, uint64_t len, uint64_t pattern);
void test_primary_sync(int connect_fd);
void test_sync_stab_all(int connect_fd);
void test_list_stab(int connect_fd);
int main0(int narg, char *args[])
{
int connect_fd;
// Prepare data for cluster read
connect_fd = connect_osd("127.0.0.1", 11203);
test_write(connect_fd, 2, 0, 1, PATTERN0);
close(connect_fd);
connect_fd = connect_osd("127.0.0.1", 11204);
test_write(connect_fd, 2, 1, 1, PATTERN1);
close(connect_fd);
connect_fd = connect_osd("127.0.0.1", 11205);
test_write(connect_fd, 2, 2, 1, PATTERN0^PATTERN1);
close(connect_fd);
return 0;
}
int main1(int narg, char *args[])
{
int connect_fd;
void *data;
// Cluster read
connect_fd = connect_osd("127.0.0.1", 11203);
data = test_primary_read(connect_fd, 2, 0, 128*1024);
if (data)
{
check_pattern(data, 128*1024, PATTERN0);
printf("inode=2 0-128K OK\n");
free(data);
}
data = test_primary_read(connect_fd, 2, 0, 256*1024);
if (data)
{
check_pattern(data, 128*1024, PATTERN0);
check_pattern(data+128*1024, 128*1024, PATTERN1);
printf("inode=2 0-256K OK\n");
free(data);
}
close(connect_fd);
return 0;
}
int main2(int narg, char *args[])
{
int connect_fd;
// Cluster write (sync not implemented yet)
connect_fd = connect_osd("127.0.0.1", 11203);
test_primary_write(connect_fd, 2, 0, 128*1024, PATTERN0);
test_primary_write(connect_fd, 2, 128*1024, 128*1024, PATTERN1);
test_sync_stab_all(connect_fd);
close(connect_fd);
connect_fd = connect_osd("127.0.0.1", 11204);
if (connect_fd >= 0)
{
test_sync_stab_all(connect_fd);
close(connect_fd);
}
connect_fd = connect_osd("127.0.0.1", 11205);
if (connect_fd >= 0)
{
test_sync_stab_all(connect_fd);
close(connect_fd);
}
return 0;
}
int main3(int narg, char *args[])
{
int connect_fd;
connect_fd = connect_osd("127.0.0.1", 11203);
test_list_stab(connect_fd);
close(connect_fd);
return 0;
}
int main4(int narg, char *args[])
{
int connect_fd;
// Cluster write (sync not implemented yet)
connect_fd = connect_osd("127.0.0.1", 11203);
test_primary_write(connect_fd, 2, 0, 128*1024, PATTERN0);
test_primary_write(connect_fd, 2, 128*1024, 128*1024, PATTERN1);
test_primary_sync(connect_fd);
close(connect_fd);
return 0;
}
int main(int narg, char *args[])
{
int connect_fd;
connect_fd = connect_osd("192.168.7.2", 43051);
test_read(connect_fd, 1, 1039663104, UINT64_MAX, 0, 128*1024);
close(connect_fd);
return 0;
}
int connect_osd(const char *osd_address, int osd_port)
{
struct sockaddr_storage addr;
if (!string_to_addr(osd_address, 0, osd_port, &addr))
{
fprintf(stderr, "server address: %s is not valid\n", osd_address);
return -1;
}
int connect_fd = socket(addr.ss_family, SOCK_STREAM, 0);
if (connect_fd < 0)
{
perror("socket");
return -1;
}
if (connect(connect_fd, (sockaddr*)&addr, sizeof(addr)) < 0)
{
perror("connect");
close(connect_fd);
return -1;
}
int one = 1;
setsockopt(connect_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
return connect_fd;
}
bool check_reply(int r, osd_any_op_t & op, osd_any_reply_t & reply, int expected)
{
if (r != OSD_PACKET_SIZE)
{
printf("read failed\n");
return false;
}
if (reply.hdr.magic != SECONDARY_OSD_REPLY_MAGIC ||
reply.hdr.id != op.hdr.id || reply.hdr.opcode != op.hdr.opcode)
{
printf("bad reply: magic, id or opcode does not match request\n");
return false;
}
if (expected >= 0 && reply.hdr.retval != expected)
{
printf("operation failed, retval=%jd\n", reply.hdr.retval);
return false;
}
return true;
}
uint64_t test_read(int connect_fd, uint64_t inode, uint64_t stripe, uint64_t version, uint64_t offset, uint64_t len)
{
osd_any_op_t op;
osd_any_reply_t reply;
op.hdr.magic = SECONDARY_OSD_OP_MAGIC;
op.hdr.id = 1;
op.hdr.opcode = OSD_OP_SEC_READ;
op.sec_rw.oid = {
.inode = inode,
.stripe = stripe,
};
op.sec_rw.version = version;
op.sec_rw.offset = offset;
op.sec_rw.len = len;
void *data = memalign(MEM_ALIGNMENT, op.sec_rw.len);
write_blocking(connect_fd, op.buf, OSD_PACKET_SIZE);
int r = read_blocking(connect_fd, reply.buf, OSD_PACKET_SIZE);
if (!check_reply(r, op, reply, op.sec_rw.len))
{
free(data);
return 0;
}
r = read_blocking(connect_fd, data, len);
if (r != len)
{
free(data);
perror("read data");
return 0;
}
free(data);
printf("Read %jx:%jx v%ju = v%ju\n", inode, stripe, version, reply.sec_rw.version);
op.hdr.opcode = OSD_OP_SEC_LIST;
op.sec_list.list_pg = 1;
op.sec_list.pg_count = 1;
op.sec_list.pg_stripe_size = 4*1024*1024;
write_blocking(connect_fd, op.buf, OSD_PACKET_SIZE);
r = read_blocking(connect_fd, reply.buf, OSD_PACKET_SIZE);
if (reply.hdr.retval < 0 || !check_reply(r, op, reply, reply.hdr.retval))
{
return 0;
}
data = memalign(MEM_ALIGNMENT, sizeof(obj_ver_id)*reply.hdr.retval);
r = read_blocking(connect_fd, data, sizeof(obj_ver_id)*reply.hdr.retval);
if (r != sizeof(obj_ver_id)*reply.hdr.retval)
{
free(data);
perror("read data");
return 0;
}
obj_ver_id *ov = (obj_ver_id*)data;
for (int i = 0; i < reply.hdr.retval; i++)
{
if (ov[i].oid.inode == inode && (ov[i].oid.stripe & ~(4096-1)) == (stripe & ~(4096-1)))
{
printf("list: %jx:%jx v%ju stable=%d\n", ov[i].oid.inode, ov[i].oid.stripe, ov[i].version, i < reply.sec_list.stable_count ? 1 : 0);
}
}
return 0;
}
uint64_t test_write(int connect_fd, uint64_t inode, uint64_t stripe, uint64_t version, uint64_t pattern)
{
osd_any_op_t op;
osd_any_reply_t reply;
op.hdr.magic = SECONDARY_OSD_OP_MAGIC;
op.hdr.id = 1;
op.hdr.opcode = OSD_OP_SEC_WRITE;
op.sec_rw.oid = {
.inode = inode,
.stripe = stripe,
};
op.sec_rw.version = version;
op.sec_rw.offset = 0;
op.sec_rw.len = 128*1024;
void *data = memalign(MEM_ALIGNMENT, op.sec_rw.len);
for (int i = 0; i < (op.sec_rw.len)/sizeof(uint64_t); i++)
((uint64_t*)data)[i] = pattern;
write_blocking(connect_fd, op.buf, OSD_PACKET_SIZE);
write_blocking(connect_fd, data, op.sec_rw.len);
int r = read_blocking(connect_fd, reply.buf, OSD_PACKET_SIZE);
if (!check_reply(r, op, reply, op.sec_rw.len))
{
free(data);
return 0;
}
version = reply.sec_rw.version;
op.hdr.opcode = OSD_OP_TEST_SYNC_STAB_ALL;
op.hdr.id = 2;
write_blocking(connect_fd, op.buf, OSD_PACKET_SIZE);
r = read_blocking(connect_fd, reply.buf, OSD_PACKET_SIZE);
if (!check_reply(r, op, reply, 0))
{
free(data);
return 0;
}
free(data);
return version;
}
void* test_primary_read(int connect_fd, uint64_t inode, uint64_t offset, uint64_t len)
{
osd_any_op_t op;
osd_any_reply_t reply;
op.hdr.magic = SECONDARY_OSD_OP_MAGIC;
op.hdr.id = 1;
op.hdr.opcode = OSD_OP_READ;
op.rw.inode = inode;
op.rw.offset = offset;
op.rw.len = len;
void *data = memalign(MEM_ALIGNMENT, len);
write_blocking(connect_fd, op.buf, OSD_PACKET_SIZE);
int r = read_blocking(connect_fd, reply.buf, OSD_PACKET_SIZE);
if (!check_reply(r, op, reply, len))
{
free(data);
return NULL;
}
r = read_blocking(connect_fd, data, len);
if (r != len)
{
free(data);
perror("read data");
return NULL;
}
return data;
}
void test_primary_write(int connect_fd, uint64_t inode, uint64_t offset, uint64_t len, uint64_t pattern)
{
osd_any_op_t op;
osd_any_reply_t reply;
op.hdr.magic = SECONDARY_OSD_OP_MAGIC;
op.hdr.id = 1;
op.hdr.opcode = OSD_OP_WRITE;
op.rw.inode = inode;
op.rw.offset = offset;
op.rw.len = len;
void *data = memalign(MEM_ALIGNMENT, len);
set_pattern(data, len, pattern);
write_blocking(connect_fd, op.buf, OSD_PACKET_SIZE);
write_blocking(connect_fd, data, len);
free(data);
int r = read_blocking(connect_fd, reply.buf, OSD_PACKET_SIZE);
assert(check_reply(r, op, reply, len));
}
void test_primary_sync(int connect_fd)
{
osd_any_op_t op;
osd_any_reply_t reply;
op.hdr.magic = SECONDARY_OSD_OP_MAGIC;
op.hdr.id = 1;
op.hdr.opcode = OSD_OP_SYNC;
write_blocking(connect_fd, op.buf, OSD_PACKET_SIZE);
int r = read_blocking(connect_fd, reply.buf, OSD_PACKET_SIZE);
assert(check_reply(r, op, reply, 0));
}
void test_sync_stab_all(int connect_fd)
{
osd_any_op_t op;
osd_any_reply_t reply;
op.hdr.magic = SECONDARY_OSD_OP_MAGIC;
op.hdr.id = 1;
op.hdr.opcode = OSD_OP_TEST_SYNC_STAB_ALL;
write_blocking(connect_fd, op.buf, OSD_PACKET_SIZE);
int r = read_blocking(connect_fd, reply.buf, OSD_PACKET_SIZE);
assert(check_reply(r, op, reply, 0));
}
void test_list_stab(int connect_fd)
{
osd_any_op_t op;
osd_any_reply_t reply;
op.hdr.magic = SECONDARY_OSD_OP_MAGIC;
op.hdr.id = 1;
op.hdr.opcode = OSD_OP_SEC_LIST;
op.sec_list.pg_count = 0;
assert(write_blocking(connect_fd, op.buf, OSD_PACKET_SIZE) == OSD_PACKET_SIZE);
int r = read_blocking(connect_fd, reply.buf, OSD_PACKET_SIZE);
assert(check_reply(r, op, reply, -1));
int total_count = reply.hdr.retval;
int stable_count = reply.sec_list.stable_count;
obj_ver_id *data = (obj_ver_id*)malloc(total_count * sizeof(obj_ver_id));
assert(data);
assert(read_blocking(connect_fd, data, total_count * sizeof(obj_ver_id)) == (total_count * sizeof(obj_ver_id)));
int last_start = stable_count;
for (int i = stable_count; i <= total_count; i++)
{
// Stabilize in portions of 32 entries
if (i - last_start >= 32 || i == total_count)
{
op.hdr.opcode = OSD_OP_SEC_STABILIZE;
op.sec_stab.len = sizeof(obj_ver_id) * (i - last_start);
assert(write_blocking(connect_fd, op.buf, OSD_PACKET_SIZE) == OSD_PACKET_SIZE);
assert(write_blocking(connect_fd, data + last_start, op.sec_stab.len) == op.sec_stab.len);
r = read_blocking(connect_fd, reply.buf, OSD_PACKET_SIZE);
assert(check_reply(r, op, reply, 0));
last_start = i;
}
}
obj_ver_id *data2 = (obj_ver_id*)malloc(sizeof(obj_ver_id) * 32);
assert(data2);
free(data2);
free(data);
}
src/test/stub_bench.cpp
+197
View File
@@ -0,0 +1,197 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
/**
* Stub benchmarker
*/
#include <sys/types.h>
#include <time.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <arpa/inet.h>
#include <string.h>
#include <stdio.h>
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>
#include <stdlib.h>
#include <signal.h>
#include <stdexcept>
#include "addr_util.h"
#include "rw_blocking.h"
#include "osd_ops.h"
int connect_stub(const char *server_address, int server_port);
void run_bench(int peer_fd);
static uint64_t read_sum = 0, read_count = 0;
static uint64_t write_sum = 0, write_count = 0;
static uint64_t sync_sum = 0, sync_count = 0;
void handle_sigint(int sig)
{
printf("4k randread: %ju us avg\n", read_count ? read_sum/read_count : 0);
printf("4k randwrite: %ju us avg\n", write_count ? write_sum/write_count : 0);
printf("sync: %ju us avg\n", sync_count ? sync_sum/sync_count : 0);
exit(0);
}
int main(int narg, char *args[])
{
if (narg < 2)
{
printf("USAGE: %s SERVER_IP [PORT]\n", args[0]);
return 1;
}
int port = 11203;
if (narg >= 3)
{
port = atoi(args[2]);
if (port <= 0 || port >= 65536)
{
printf("Bad port number\n");
return 1;
}
}
signal(SIGINT, handle_sigint);
int peer_fd = connect_stub(args[1], port);
run_bench(peer_fd);
close(peer_fd);
return 0;
}
int connect_stub(const char *server_address, int server_port)
{
struct sockaddr_storage addr;
if (!string_to_addr(server_address, 0, server_port, &addr))
{
fprintf(stderr, "server address: %s is not valid\n", server_address);
return -1;
}
int connect_fd = socket(addr.ss_family, SOCK_STREAM, 0);
if (connect_fd < 0)
{
perror("socket");
return -1;
}
if (connect(connect_fd, (sockaddr*)&addr, sizeof(addr)) < 0)
{
perror("connect");
close(connect_fd);
return -1;
}
int one = 1;
setsockopt(connect_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
return connect_fd;
}
bool check_reply(int r, osd_any_op_t & op, osd_any_reply_t & reply, int expected)
{
if (r != OSD_PACKET_SIZE)
{
printf("read failed\n");
return false;
}
if (reply.hdr.magic != SECONDARY_OSD_REPLY_MAGIC ||
reply.hdr.id != op.hdr.id || reply.hdr.opcode != op.hdr.opcode)
{
printf("bad reply: magic, id or opcode does not match request\n");
return false;
}
if (reply.hdr.retval != expected)
{
printf("operation failed, retval=%jd (%s)\n", reply.hdr.retval, strerror(-reply.hdr.retval));
return false;
}
return true;
}
void run_bench(int peer_fd)
{
osd_any_op_t op;
osd_any_reply_t reply;
void *buf = NULL;
int r;
iovec iov[2];
timespec tv_begin, tv_end;
clock_gettime(CLOCK_REALTIME, &tv_begin);
while (1)
{
// read
op.hdr.magic = SECONDARY_OSD_OP_MAGIC;
op.hdr.id = 1;
op.hdr.opcode = OSD_OP_SEC_READ;
op.sec_rw.oid.inode = 3;
op.sec_rw.oid.stripe = (rand() << 17) % (1 << 29); // 512 MB
op.sec_rw.version = 0;
op.sec_rw.len = 4096;
op.sec_rw.offset = (rand() * op.sec_rw.len) % (1 << 17);
r = write_blocking(peer_fd, op.buf, OSD_PACKET_SIZE) == OSD_PACKET_SIZE;
if (!r)
break;
buf = malloc(op.sec_rw.len);
iov[0] = { reply.buf, OSD_PACKET_SIZE };
iov[1] = { buf, op.sec_rw.len };
r = readv_blocking(peer_fd, iov, 2) == (OSD_PACKET_SIZE + op.sec_rw.len);
free(buf);
if (!r || !check_reply(OSD_PACKET_SIZE, op, reply, op.sec_rw.len))
break;
clock_gettime(CLOCK_REALTIME, &tv_end);
read_count++;
read_sum += (
(tv_end.tv_sec - tv_begin.tv_sec)*1000000 +
tv_end.tv_nsec/1000 - tv_begin.tv_nsec/1000
);
tv_begin = tv_end;
// write
op.hdr.magic = SECONDARY_OSD_OP_MAGIC;
op.hdr.id = 1;
op.hdr.opcode = OSD_OP_SEC_WRITE;
op.sec_rw.oid.inode = 3;
op.sec_rw.oid.stripe = (rand() << 17) % (1 << 29); // 512 MB
op.sec_rw.version = 0;
op.sec_rw.len = 4096;
op.sec_rw.offset = (rand() * op.sec_rw.len) % (1 << 17);
buf = malloc(op.sec_rw.len);
memset(buf, rand() % 255, op.sec_rw.len);
iov[0] = { op.buf, OSD_PACKET_SIZE };
iov[1] = { buf, op.sec_rw.len };
r = writev_blocking(peer_fd, iov, 2) == (OSD_PACKET_SIZE + op.sec_rw.len);
free(buf);
if (!r)
break;
r = read_blocking(peer_fd, reply.buf, OSD_PACKET_SIZE);
if (!check_reply(r, op, reply, op.sec_rw.len))
break;
clock_gettime(CLOCK_REALTIME, &tv_end);
write_count++;
write_sum += (
(tv_end.tv_sec - tv_begin.tv_sec)*1000000 +
tv_end.tv_nsec/1000 - tv_begin.tv_nsec/1000
);
tv_begin = tv_end;
// sync/stab
op.hdr.magic = SECONDARY_OSD_OP_MAGIC;
op.hdr.id = 1;
op.hdr.opcode = OSD_OP_TEST_SYNC_STAB_ALL;
r = write_blocking(peer_fd, op.buf, OSD_PACKET_SIZE) == OSD_PACKET_SIZE;
if (!r)
break;
r = read_blocking(peer_fd, reply.buf, OSD_PACKET_SIZE);
if (!check_reply(r, op, reply, 0))
break;
clock_gettime(CLOCK_REALTIME, &tv_end);
sync_count++;
sync_sum += (
(tv_end.tv_sec - tv_begin.tv_sec)*1000000 +
tv_end.tv_nsec/1000 - tv_begin.tv_nsec/1000
);
tv_begin = tv_end;
}
}
src/test/stub_osd.cpp
+134
View File
@@ -0,0 +1,134 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
/**
* Stub "OSD" to test & compare network performance with sync read/write and io_uring
*
* Core i7-6700HQ laptop
*
* stub_osd:
* randwrite Q1 S1: 36900 iops
* randwrite Q32 S32: 71000 iops
* randwrite Q32 S32 (multi-fsync fix): 113000 iops
* randread Q1: 67300 iops
* randread Q32: 144000 iops
*
* io_uring osd with #define OSD_STUB:
* randwrite Q1 S1: 30000 iops
* randwrite Q32 S32: 78600 iops
* randwrite Q32 S32 (multi-fsync fix): 125000 iops
* randread Q1: 50700 iops
* randread Q32: 86100 iops
*
* It seems io_uring is fine :)
*/
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <arpa/inet.h>
#include <string.h>
#include <stdio.h>
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>
#include <stdlib.h>
#include <stdexcept>
#include "addr_util.h"
#include "rw_blocking.h"
#include "osd_ops.h"
void run_stub(int peer_fd);
int main(int narg, char *args[])
{
int listen_fd = create_and_bind_socket("0.0.0.0", 11203, 128, NULL);
// Accept new connections
sockaddr_storage addr;
socklen_t peer_addr_size = sizeof(addr);
int peer_fd;
while (1)
{
printf("stub_osd: waiting for 1 client\n");
peer_fd = accept(listen_fd, (sockaddr*)&addr, &peer_addr_size);
if (peer_fd == -1)
{
if (errno == EAGAIN)
continue;
else
throw std::runtime_error(std::string("accept: ") + strerror(errno));
}
printf("stub_osd: new client %d: connection from %s\n", peer_fd,
addr_to_string(addr).c_str());
int one = 1;
setsockopt(peer_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
run_stub(peer_fd);
close(peer_fd);
printf("stub_osd: client %d disconnected\n", peer_fd);
// Try to accept next connection
peer_addr_size = sizeof(addr);
}
return 0;
}
void run_stub(int peer_fd)
{
osd_any_op_t op;
osd_any_reply_t reply = {};
void *buf = NULL;
while (1)
{
int r = read_blocking(peer_fd, op.buf, OSD_PACKET_SIZE);
if (r < OSD_PACKET_SIZE)
{
break;
}
if (op.hdr.magic != SECONDARY_OSD_OP_MAGIC)
{
printf("client %d: bad magic number in operation header\n", peer_fd);
break;
}
reply.hdr.magic = SECONDARY_OSD_REPLY_MAGIC;
reply.hdr.id = op.hdr.id;
reply.hdr.opcode = op.hdr.opcode;
if (op.hdr.opcode == OSD_OP_SEC_READ)
{
reply.hdr.retval = op.sec_rw.len;
buf = malloc(op.sec_rw.len);
r = write_blocking(peer_fd, reply.buf, OSD_PACKET_SIZE);
if (r == OSD_PACKET_SIZE)
r = write_blocking(peer_fd, buf, op.sec_rw.len);
free(buf);
if (r < op.sec_rw.len)
break;
}
else if (op.hdr.opcode == OSD_OP_SEC_WRITE || op.hdr.opcode == OSD_OP_SEC_WRITE_STABLE)
{
buf = malloc(op.sec_rw.len);
r = read_blocking(peer_fd, buf, op.sec_rw.len);
free(buf);
reply.hdr.retval = op.sec_rw.len;
if (r == op.sec_rw.len)
r = write_blocking(peer_fd, reply.buf, OSD_PACKET_SIZE);
else
r = 0;
if (r < OSD_PACKET_SIZE)
break;
}
else if (op.hdr.opcode == OSD_OP_TEST_SYNC_STAB_ALL)
{
reply.hdr.retval = 0;
r = write_blocking(peer_fd, reply.buf, OSD_PACKET_SIZE);
if (r < OSD_PACKET_SIZE)
break;
}
else
{
printf("client %d: unsupported stub opcode: %ju\n", peer_fd, op.hdr.opcode);
break;
}
}
}
src/test/stub_uring_osd.cpp
+96
View File
@@ -0,0 +1,96 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
/**
* Stub "OSD" implemented on top of osd_messenger to test & compare
* network performance with sync read/write and io_uring
*/
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <arpa/inet.h>
#include <string.h>
#include <stdio.h>
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>
#include <stdlib.h>
#include <stdexcept>
#include "addr_util.h"
#include "ringloop.h"
#include "epoll_manager.h"
#include "messenger.h"
void stub_exec_op(osd_messenger_t *msgr, osd_op_t *op);
int main(int narg, char *args[])
{
ring_consumer_t looper;
ring_loop_t *ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
epoll_manager_t *epmgr = new epoll_manager_t(ringloop);
osd_messenger_t *msgr = new osd_messenger_t();
msgr->osd_num = 1351;
msgr->tfd = epmgr->tfd;
msgr->ringloop = ringloop;
msgr->repeer_pgs = [](osd_num_t) {};
msgr->exec_op = [msgr](osd_op_t *op) { stub_exec_op(msgr, op); };
json11::Json config = json11::Json::object { { "log_level", 1 } };
msgr->parse_config(config);
// Accept new connections
int listen_fd = create_and_bind_socket("0.0.0.0", 11203, 128, NULL);
fcntl(listen_fd, F_SETFL, fcntl(listen_fd, F_GETFL, 0) | O_NONBLOCK);
epmgr->set_fd_handler(listen_fd, false, [listen_fd, msgr](int fd, int events)
{
msgr->accept_connections(listen_fd);
});
looper.loop = [msgr, ringloop]()
{
msgr->read_requests();
msgr->send_replies();
ringloop->submit();
};
ringloop->register_consumer(&looper);
printf("stub_uring_osd: waiting for clients\n");
while (true)
{
ringloop->loop();
ringloop->wait();
}
delete msgr;
delete epmgr;
delete ringloop;
return 0;
}
void stub_exec_op(osd_messenger_t *msgr, osd_op_t *op)
{
op->reply.hdr.magic = SECONDARY_OSD_REPLY_MAGIC;
op->reply.hdr.id = op->req.hdr.id;
op->reply.hdr.opcode = op->req.hdr.opcode;
if (op->req.hdr.opcode == OSD_OP_SEC_READ)
{
op->reply.hdr.retval = op->req.sec_rw.len;
op->buf = memalign_or_die(MEM_ALIGNMENT, op->req.sec_rw.len);
op->iov.push_back(op->buf, op->req.sec_rw.len);
op->reply.sec_rw.attr_len = 4;
op->bitmap = op->buf;
}
else if (op->req.hdr.opcode == OSD_OP_SEC_WRITE || op->req.hdr.opcode == OSD_OP_SEC_WRITE_STABLE)
{
op->reply.hdr.retval = op->req.sec_rw.len;
}
else if (op->req.hdr.opcode == OSD_OP_TEST_SYNC_STAB_ALL)
{
op->reply.hdr.retval = 0;
}
else
{
printf("client %d: unsupported stub opcode: %ju\n", op->peer_fd, op->req.hdr.opcode);
op->reply.hdr.retval = -EINVAL;
}
msgr->outbox_push(op);
}
src/test/test_allocator.cpp
+48
View File
@@ -0,0 +1,48 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
#include <stdio.h>
#include <stdlib.h>
#include "allocator.h"
void alloc_all(int size)
{
allocator *a = new allocator(size);
for (int i = 0; i < size; i++)
{
uint64_t x = a->find_free();
if (x == UINT64_MAX)
{
printf("ran out of space %d allocated=%d\n", size, i);
exit(1);
}
if (x != i)
{
printf("incorrect block allocated: expected %d, got %ju\n", i, x);
}
if (a->get(x))
{
printf("not free before set at %d\n", i);
}
a->set(x, true);
if (!a->get(x))
{
printf("free after set at %d\n", i);
}
}
uint64_t x = a->find_free();
if (x != UINT64_MAX)
{
printf("extra free space found: %jx (%d)\n", x, size);
exit(1);
}
delete a;
}
int main(int narg, char *args[])
{
alloc_all(8192);
alloc_all(8062);
alloc_all(4096);
return 0;
}
src/test/test_blockstore.cpp
+128
View File
@@ -0,0 +1,128 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
#include <malloc.h>
#include "blockstore.h"
#include "epoll_manager.h"
int main(int narg, char *args[])
{
blockstore_config_t config;
config["meta_device"] = "./test_meta.bin";
config["journal_device"] = "./test_journal.bin";
config["data_device"] = "./test_data.bin";
ring_loop_t *ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
epoll_manager_t *epmgr = new epoll_manager_t(ringloop);
blockstore_t *bs = new blockstore_t(config, ringloop, epmgr->tfd);
blockstore_op_t op;
int main_state = 0;
uint64_t version = 0;
ring_consumer_t main_cons;
op.callback = [&](blockstore_op_t *op)
{
printf("op completed %d\n", op->retval);
if (main_state == 1)
main_state = 2;
else if (main_state == 3)
main_state = 4;
else if (main_state == 5)
main_state = 6;
else if (main_state == 7)
main_state = 8;
else if (main_state == 9)
main_state = 10;
};
main_cons.loop = [&]()
{
if (main_state == 0)
{
if (bs->is_started())
{
printf("init completed\n");
op.opcode = BS_OP_WRITE;
op.oid = { .inode = 1, .stripe = 0 };
op.version = 0;
op.offset = 16384;
op.len = 4096;
op.buf = (uint8_t*)memalign(512, 128*1024);
memset(op.buf, 0xaa, 4096);
bs->enqueue_op(&op);
main_state = 1;
}
}
else if (main_state == 2)
{
printf("version %ju written, syncing\n", op.version);
version = op.version;
op.opcode = BS_OP_SYNC;
bs->enqueue_op(&op);
main_state = 3;
}
else if (main_state == 4)
{
printf("stabilizing version %ju\n", version);
op.opcode = BS_OP_STABLE;
op.len = 1;
*((obj_ver_id*)op.buf) = {
.oid = { .inode = 1, .stripe = 0 },
.version = version,
};
bs->enqueue_op(&op);
main_state = 5;
}
else if (main_state == 6)
{
printf("stabilizing version %ju\n", version);
op.opcode = BS_OP_STABLE;
op.len = 1;
*((obj_ver_id*)op.buf) = {
.oid = { .inode = 1, .stripe = 0 },
.version = version,
};
bs->enqueue_op(&op);
main_state = 7;
}
else if (main_state == 8)
{
printf("reading 0-128K\n");
op.opcode = BS_OP_READ;
op.oid = { .inode = 1, .stripe = 0 };
op.version = UINT64_MAX;
op.offset = 0;
op.len = 128*1024;
bs->enqueue_op(&op);
main_state = 9;
}
else if (main_state == 10)
{
void *cmp = memalign(512, 128*1024);
memset(cmp, 0, 128*1024);
memset(cmp+16384, 0xaa, 4096);
int ok = 1;
for (int i = 0; i < 128*1024; i += 4096)
{
if (memcmp(cmp+i, op.buf+i, 4096) != 0)
{
printf("bitmap works incorrectly, bytes %d - %d differ (%02x, should be %02x)\n", i, i+4096, ((uint8_t*)op.buf)[i], ((uint8_t*)cmp)[i]);
ok = 0;
}
}
if (ok)
printf("bitmap works correctly\n");
free(cmp);
main_state = 11;
}
};
ringloop->register_consumer(&main_cons);
while (1)
{
ringloop->loop();
ringloop->wait();
}
delete bs;
delete epmgr;
delete ringloop;
return 0;
}
src/test/test_cas.cpp
+135
View File
@@ -0,0 +1,135 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
#include <stdio.h>
#include <stdlib.h>
#include "epoll_manager.h"
#include "cluster_client.h"
void send_read(cluster_client_t *cli, uint64_t inode, std::function<void(int, uint64_t)> cb)
{
cluster_op_t *op = new cluster_op_t();
op->opcode = OSD_OP_READ;
op->inode = inode;
op->offset = 0;
op->len = 4096;
op->iov.push_back(malloc_or_die(op->len), op->len);
op->callback = [cb](cluster_op_t *op)
{
uint64_t version = op->version;
int retval = op->retval;
if (retval == op->len)
retval = 0;
free(op->iov.buf[0].iov_base);
delete op;
if (cb != NULL)
cb(retval, version);
};
cli->execute(op);
}
void send_write(cluster_client_t *cli, uint64_t inode, int byte, uint64_t version, std::function<void(int)> cb)
{
cluster_op_t *op = new cluster_op_t();
op->opcode = OSD_OP_WRITE;
op->inode = inode;
op->offset = 0;
op->len = 4096;
op->version = version;
op->iov.push_back(malloc_or_die(op->len), op->len);
memset(op->iov.buf[0].iov_base, byte, op->len);
op->callback = [cb](cluster_op_t *op)
{
int retval = op->retval;
if (retval == op->len)
retval = 0;
free(op->iov.buf[0].iov_base);
delete op;
if (cb != NULL)
cb(retval);
};
cli->execute(op);
}
int main(int narg, char *args[])
{
json11::Json::object cfgo;
for (int i = 1; i < narg; i++)
{
if (args[i][0] == '-' && args[i][1] == '-')
{
const char *opt = args[i]+2;
cfgo[opt] = i == narg-1 ? "1" : args[++i];
}
}
json11::Json cfg(cfgo);
uint64_t inode = (cfg["pool_id"].uint64_value() << (64-POOL_ID_BITS))
| cfg["inode_id"].uint64_value();
uint64_t base_ver = 0;
// Create client
auto ringloop = new ring_loop_t(RINGLOOP_DEFAULT_SIZE);
auto epmgr = new epoll_manager_t(ringloop);
auto cli = new cluster_client_t(ringloop, epmgr->tfd, cfg);
cli->on_ready([&]()
{
send_read(cli, inode, [&](int r, uint64_t v)
{
if (r < 0)
{
fprintf(stderr, "Initial read operation failed\n");
exit(1);
}
base_ver = v;
// CAS v=1 = compare with zero, non-existing object
send_write(cli, inode, 0x01, base_ver+1, [&](int r)
{
if (r < 0)
{
fprintf(stderr, "CAS for non-existing object failed\n");
exit(1);
}
// Check that read returns the new version
send_read(cli, inode, [&](int r, uint64_t v)
{
if (r < 0)
{
fprintf(stderr, "Read operation failed after write\n");
exit(1);
}
if (v != base_ver+1)
{
fprintf(stderr, "Read operation failed to return the new version number\n");
exit(1);
}
// CAS v=2 = compare with v=1, existing object
send_write(cli, inode, 0x02, base_ver+2, [&](int r)
{
if (r < 0)
{
fprintf(stderr, "CAS for existing object failed\n");
exit(1);
}
// CAS v=2 again = compare with v=1, but version is 2. Must fail with -EINTR
send_write(cli, inode, 0x03, base_ver+2, [&](int r)
{
if (r != -EINTR)
{
fprintf(stderr, "CAS conflict detection failed\n");
exit(1);
}
printf("Basic CAS test succeeded\n");
exit(0);
});
});
});
});
});
});
while (1)
{
ringloop->loop();
ringloop->wait();
}
return 0;
}
src/test/test_cluster_client.cpp
+491
View File
@@ -0,0 +1,491 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include "cluster_client_impl.h"
void configure_single_pg_pool(cluster_client_t *cli)
{
cli->st_cli.parse_state((etcd_kv_t){
.key = "/config/pools",
.value = json11::Json::object {
{ "1", json11::Json::object {
{ "name", "hddpool" },
{ "scheme", "replicated" },
{ "pg_size", 2 },
{ "pg_minsize", 1 },
{ "pg_count", 1 },
{ "failure_domain", "osd" },
} }
},
});
cli->st_cli.parse_state((etcd_kv_t){
.key = "/config/pgs",
.value = json11::Json::object {
{ "items", json11::Json::object {
{ "1", json11::Json::object {
{ "1", json11::Json::object {
{ "osd_set", json11::Json::array { 1, 2 } },
{ "primary", 1 },
} }
} }
} }
},
});
cli->st_cli.parse_state((etcd_kv_t){
.key = "/pg/state/1/1",
.value = json11::Json::object {
{ "peers", json11::Json::array { 1, 2 } },
{ "primary", 1 },
{ "state", json11::Json::array { "active" } },
},
});
cli->st_cli.on_load_pgs_hook(true);
std::map<std::string, etcd_kv_t> changes;
cli->st_cli.on_change_hook(changes);
}
int *test_write(cluster_client_t *cli, uint64_t offset, uint64_t len, uint8_t c, std::function<void()> cb = NULL, bool instant = false)
{
printf("Post write %jx+%jx\n", offset, len);
int *r = new int;
*r = instant ? -2 : -1;
cluster_op_t *op = new cluster_op_t();
op->opcode = OSD_OP_WRITE;
op->inode = 0x1000000000001;
op->offset = offset;
op->len = len;
op->iov.push_back(malloc_or_die(len), len);
memset(op->iov.buf[0].iov_base, c, len);
op->callback = [r, cb](cluster_op_t *op)
{
if (*r == -1)
printf("Error: Not allowed to complete yet\n");
assert(*r != -1);
*r = op->retval == op->len ? 1 : 0;
free(op->iov.buf[0].iov_base);
printf("Done write %jx+%jx r=%d\n", op->offset, op->len, op->retval);
delete op;
if (cb != NULL)
cb();
};
cli->execute(op);
if (instant)
{
long res = *r;
assert(*r >= 0);
delete r;
return (int*)res;
}
return r;
}
int *test_sync(cluster_client_t *cli)
{
printf("Post sync\n");
int *r = new int;
*r = -1;
cluster_op_t *op = new cluster_op_t();
op->opcode = OSD_OP_SYNC;
op->callback = [r](cluster_op_t *op)
{
if (*r == -1)
printf("Error: Not allowed to complete yet\n");
assert(*r != -1);
*r = op->retval == 0 ? 1 : 0;
printf("Done sync r=%d\n", op->retval);
delete op;
};
cli->execute(op);
return r;
}
void can_complete(int *r)
{
// Allow the operation to proceed so the test verifies
// that it doesn't complete earlier than expected
*r = -2;
}
void check_completed(int *r)
{
assert(*r == 1);
delete r;
}
void pretend_connected(cluster_client_t *cli, osd_num_t osd_num)
{
printf("OSD %ju connected\n", osd_num);
int peer_fd = cli->msgr.clients.size() ? std::prev(cli->msgr.clients.end())->first+1 : 10;
cli->msgr.osd_peer_fds[osd_num] = peer_fd;
cli->msgr.clients[peer_fd] = new osd_client_t();
cli->msgr.clients[peer_fd]->osd_num = osd_num;
cli->msgr.clients[peer_fd]->peer_state = PEER_CONNECTED;
cli->msgr.wanted_peers.erase(osd_num);
cli->msgr.repeer_pgs(osd_num);
}
void pretend_disconnected(cluster_client_t *cli, osd_num_t osd_num)
{
printf("OSD %ju disconnected\n", osd_num);
cli->msgr.stop_client(cli->msgr.osd_peer_fds.at(osd_num));
}
void check_disconnected(cluster_client_t *cli, osd_num_t osd_num)
{
if (cli->msgr.osd_peer_fds.find(osd_num) != cli->msgr.osd_peer_fds.end())
{
printf("OSD %ju not disconnected as it ought to be\n", osd_num);
assert(0);
}
}
void check_op_count(cluster_client_t *cli, osd_num_t osd_num, int ops)
{
int peer_fd = cli->msgr.osd_peer_fds.at(osd_num);
int real_ops = cli->msgr.clients[peer_fd]->sent_ops.size();
if (real_ops != ops)
{
printf("error: %d ops expected, but %d queued\n", ops, real_ops);
assert(0);
}
}
osd_op_t *find_op(cluster_client_t *cli, osd_num_t osd_num, uint64_t opcode, uint64_t offset, uint64_t len)
{
int peer_fd = cli->msgr.osd_peer_fds.at(osd_num);
auto op_it = cli->msgr.clients[peer_fd]->sent_ops.begin();
while (op_it != cli->msgr.clients[peer_fd]->sent_ops.end())
{
auto op = op_it->second;
if (op->req.hdr.opcode == opcode && (opcode == OSD_OP_SYNC ||
op->req.rw.inode == 0x1000000000001 && op->req.rw.offset == offset && op->req.rw.len == len))
{
return op;
}
op_it++;
}
op_it = cli->msgr.clients[peer_fd]->sent_ops.begin();
while (op_it != cli->msgr.clients[peer_fd]->sent_ops.end())
{
printf("Found opcode %ju offset %jx size %x\n", op_it->second->req.hdr.opcode, op_it->second->req.rw.offset, op_it->second->req.rw.len);
op_it++;
}
printf("Not found opcode %ju offset %jx size %jx\n", opcode, offset, len);
return NULL;
}
void pretend_op_completed(cluster_client_t *cli, osd_op_t *op, int64_t retval)
{
assert(op);
printf("Pretend completed %s %jx+%x\n", op->req.hdr.opcode == OSD_OP_SYNC
? "sync" : (op->req.hdr.opcode == OSD_OP_WRITE ? "write" : "read"), op->req.rw.offset, op->req.rw.len);
uint64_t op_id = op->req.hdr.id;
int peer_fd = op->peer_fd;
cli->msgr.clients[peer_fd]->sent_ops.erase(op_id);
op->reply.hdr.magic = SECONDARY_OSD_REPLY_MAGIC;
op->reply.hdr.id = op->req.hdr.id;
op->reply.hdr.opcode = op->req.hdr.opcode;
op->reply.hdr.retval = retval < 0 ? retval : (op->req.hdr.opcode == OSD_OP_SYNC ? 0 : op->req.rw.len);
// Copy lambda to be unaffected by `delete op`
std::function<void(osd_op_t*)>(op->callback)(op);
}
void test1()
{
json11::Json config;
timerfd_manager_t *tfd = new timerfd_manager_t([](int fd, bool wr, std::function<void(int, int)> callback){});
cluster_client_t *cli = new cluster_client_t(NULL, tfd, config);
int *r1 = test_write(cli, 0, 4096, 0x55);
configure_single_pg_pool(cli);
pretend_connected(cli, 1);
can_complete(r1);
check_op_count(cli, 1, 1);
pretend_op_completed(cli, find_op(cli, 1, OSD_OP_WRITE, 0, 4096), 0);
check_completed(r1);
r1 = test_write(cli, 4096, 4096, 0x56);
can_complete(r1);
check_op_count(cli, 1, 1);
pretend_op_completed(cli, find_op(cli, 1, OSD_OP_WRITE, 4096, 4096), 0);
check_completed(r1);
pretend_disconnected(cli, 1);
int *r2 = test_sync(cli);
pretend_connected(cli, 1);
check_op_count(cli, 1, 1);
pretend_op_completed(cli, find_op(cli, 1, OSD_OP_WRITE, 0, 8192), 0);
check_op_count(cli, 1, 1);
can_complete(r2);
pretend_op_completed(cli, find_op(cli, 1, OSD_OP_SYNC, 0, 0), 0);
check_completed(r2);
// Check that the client doesn't repeat operations once more
pretend_disconnected(cli, 1);
pretend_connected(cli, 1);
check_op_count(cli, 1, 0);
// Case:
// Write(1) -> Complete Write(1) -> Overwrite(2) -> Complete Write(2)
// -> Overwrite(3) -> Drop OSD connection -> Reestablish OSD connection
// -> Complete All Posted Writes -> Sync -> Complete Sync
// The resulting state of the block must be (3) over (2) over (1).
// I.e. the part overwritten by (3) must remain as in (3) and so on.
// More interesting case:
// Same, but both Write(2) and Write(3) must consist of two parts:
// one from an OSD 2 that drops connection and other from OSD 1 that doesn't.
// The idea is that if the whole Write(2) is repeated when OSD 2 drops connection
// then it may also overwrite a part in OSD 1 which shouldn't be overwritten.
// Another interesting case:
// A new operation added during replay (would also break with the previous implementation)
r1 = test_write(cli, 0, 0x10000, 0x56);
can_complete(r1);
check_op_count(cli, 1, 1);
pretend_op_completed(cli, find_op(cli, 1, OSD_OP_WRITE, 0, 0x10000), 0);
check_completed(r1);
r1 = test_write(cli, 0xE000, 0x4000, 0x57);
can_complete(r1);
check_op_count(cli, 1, 1);
pretend_op_completed(cli, find_op(cli, 1, OSD_OP_WRITE, 0xE000, 0x4000), 0);
check_completed(r1);
r1 = test_write(cli, 0x10000, 0x4000, 0x58);
pretend_disconnected(cli, 1);
pretend_connected(cli, 1);
cli->continue_ops(true);
// Check replay
{
uint64_t replay_start = UINT64_MAX;
uint64_t replay_end = 0;
std::vector<osd_op_t*> replay_ops;
auto osd_cl = cli->msgr.clients.at(cli->msgr.osd_peer_fds.at(1));
for (auto & op_p: osd_cl->sent_ops)
{
auto op = op_p.second;
assert(op->req.hdr.opcode == OSD_OP_WRITE);
uint64_t offset = op->req.rw.offset;
if (op->req.rw.offset < replay_start)
replay_start = op->req.rw.offset;
if (op->req.rw.offset+op->req.rw.len > replay_end)
replay_end = op->req.rw.offset+op->req.rw.len;
for (int buf_idx = 0; buf_idx < op->iov.count; buf_idx++)
{
for (int i = 0; i < op->iov.buf[buf_idx].iov_len; i++, offset++)
{
uint8_t c = offset < 0xE000 ? 0x56 : (offset < 0x10000 ? 0x57 : 0x58);
if (((uint8_t*)op->iov.buf[buf_idx].iov_base)[i] != c)
{
printf("Write replay: mismatch at %ju\n", offset-op->req.rw.offset);
goto fail;
}
}
}
fail:
assert(offset == op->req.rw.offset+op->req.rw.len);
replay_ops.push_back(op);
}
if (replay_start != 0 || replay_end != 0x14000)
{
printf("Write replay: range mismatch: %jx-%jx\n", replay_start, replay_end);
assert(0);
}
for (auto op: replay_ops)
{
pretend_op_completed(cli, op, 0);
}
}
// Check that the following write finally proceeds
check_op_count(cli, 1, 1);
can_complete(r1);
pretend_op_completed(cli, find_op(cli, 1, OSD_OP_WRITE, 0x10000, 0x4000), 0);
check_completed(r1);
check_op_count(cli, 1, 0);
// Check sync
r2 = test_sync(cli);
can_complete(r2);
pretend_op_completed(cli, find_op(cli, 1, OSD_OP_SYNC, 0, 0), 0);
check_completed(r2);
// Check disconnect during write
r1 = test_write(cli, 0, 4096, 0x59);
check_op_count(cli, 1, 1);
pretend_op_completed(cli, find_op(cli, 1, OSD_OP_WRITE, 0, 0x1000), -EPIPE);
check_disconnected(cli, 1);
pretend_connected(cli, 1);
check_op_count(cli, 1, 1);
pretend_op_completed(cli, find_op(cli, 1, OSD_OP_WRITE, 0, 0x1000), 0);
check_op_count(cli, 1, 1);
can_complete(r1);
pretend_op_completed(cli, find_op(cli, 1, OSD_OP_WRITE, 0, 0x1000), 0);
check_completed(r1);
// Check disconnect inside operation callback (reenterability)
// Probably doesn't happen too often, but possible in theory
r1 = test_write(cli, 0, 0x1000, 0x60, [cli]()
{
pretend_disconnected(cli, 1);
});
r2 = test_write(cli, 0x1000, 0x1000, 0x61);
check_op_count(cli, 1, 2);
can_complete(r1);
pretend_op_completed(cli, find_op(cli, 1, OSD_OP_WRITE, 0, 0x1000), 0);
check_completed(r1);
check_disconnected(cli, 1);
pretend_connected(cli, 1);
cli->continue_ops(true);
check_op_count(cli, 1, 1);
pretend_op_completed(cli, find_op(cli, 1, OSD_OP_WRITE, 0, 0x2000), 0);
check_op_count(cli, 1, 1);
can_complete(r2);
pretend_op_completed(cli, find_op(cli, 1, OSD_OP_WRITE, 0x1000, 0x1000), 0);
check_completed(r2);
// Free client
delete cli;
delete tfd;
printf("[ok] write replay test\n");
}
void test2()
{
writeback_cache_t *wb = new writeback_cache_t();
cluster_op_t *op = new cluster_op_t();
op->opcode = OSD_OP_WRITE;
op->inode = 1;
op->offset = 0;
op->len = 4096;
op->iov.push_back(malloc_or_die(4096*1024), 4096);
// 0-4k = 0x55
memset(op->iov.buf[0].iov_base, 0x55, op->iov.buf[0].iov_len);
wb->copy_write(op, CACHE_WRITTEN);
// 8k-12k = 0x66
op->offset = 8192;
memset(op->iov.buf[0].iov_base, 0x66, op->iov.buf[0].iov_len);
wb->copy_write(op, CACHE_WRITTEN);
// 4k-1M+4k = 0x77
op->len = op->iov.buf[0].iov_len = 1048576;
op->offset = 4096;
memset(op->iov.buf[0].iov_base, 0x77, op->iov.buf[0].iov_len);
wb->copy_write(op, CACHE_WRITTEN);
// check it
assert(wb->dirty_buffers.size() == 2);
auto uit = wb->dirty_buffers.begin();
int i;
assert(uit->first.inode == 1);
assert(uit->first.stripe == 0);
assert(uit->second.len == 4096);
for (i = 0; i < uit->second.len && ((uint8_t*)uit->second.buf)[i] == 0x55; i++) {}
assert(i == uit->second.len);
uit++;
assert(uit->first.inode == 1);
assert(uit->first.stripe == 4096);
assert(uit->second.len == 1048576);
for (i = 0; i < uit->second.len && ((uint8_t*)uit->second.buf)[i] == 0x77; i++) {}
assert(i == uit->second.len);
uit++;
// free memory
free(op->iov.buf[0].iov_base);
delete op;
delete wb;
printf("[ok] copy_write test\n");
}
void test_writeback()
{
json11::Json config = json11::Json::object {
{ "client_enable_writeback", true },
{ "client_writeback_allowed", true },
{ "client_max_buffered_bytes", 1024*1024 },
{ "client_max_buffered_ops", 2 },
{ "client_max_writeback_iodepth", 2 },
{ "client_max_dirty_bytes", 1024*1024 },
{ "client_max_dirty_ops", 2 },
};
timerfd_manager_t *tfd = new timerfd_manager_t([](int fd, bool wr, std::function<void(int, int)> callback){});
cluster_client_t *cli = new cluster_client_t(NULL, tfd, config);
configure_single_pg_pool(cli);
pretend_connected(cli, 1);
// Check that 3 consecutive writes are merged by writeback
assert((long)test_write(cli, 0, 4096, 0x55, NULL, true) == 1);
check_op_count(cli, 1, 0);
assert((long)test_write(cli, 4096, 4096, 0x55, NULL, true) == 1);
check_op_count(cli, 1, 0);
assert((long)test_write(cli, 8192, 4096, 0x55, NULL, true) == 1);
check_op_count(cli, 1, 0);
assert((long)test_write(cli, 1024*1024, 4096, 0x66, NULL, true) == 1);
check_op_count(cli, 1, 0);
// 3rd and 4th writes should trigger 1 writeback each
assert((long)test_write(cli, 2*1024*1024, 4096, 0x66, NULL, true) == 1);
check_op_count(cli, 1, 1);
assert((long)test_write(cli, 3*1024*1024, 4096, 0x66, NULL, true) == 1);
check_op_count(cli, 1, 2);
// 5th write should be postponed until at least 1 writeback is completed
int *r1 = test_write(cli, 4*1024*1024, 4096, 0x67, NULL);
check_op_count(cli, 1, 2);
can_complete(r1);
pretend_op_completed(cli, find_op(cli, 1, OSD_OP_WRITE, 0, 3*4096), 0);
check_completed(r1);
// autosync because max_dirty_ops=2, flush waits for sync
check_op_count(cli, 1, 1);
pretend_op_completed(cli, find_op(cli, 1, OSD_OP_WRITE, 1024*1024, 4096), 0);
check_op_count(cli, 1, 1);
pretend_op_completed(cli, find_op(cli, 1, OSD_OP_SYNC, 0, 0), 0);
check_op_count(cli, 1, 1);
pretend_op_completed(cli, find_op(cli, 1, OSD_OP_WRITE, 2*1024*1024, 4096), 0);
check_op_count(cli, 1, 0);
int *r2 = test_sync(cli);
check_op_count(cli, 1, 1);
pretend_op_completed(cli, find_op(cli, 1, OSD_OP_WRITE, 3*1024*1024, 4096), 0);
check_op_count(cli, 1, 1);
// autosync because max_dirty_ops=2, flush waits for sync
pretend_op_completed(cli, find_op(cli, 1, OSD_OP_SYNC, 0, 0), 0);
check_op_count(cli, 1, 1);
pretend_op_completed(cli, find_op(cli, 1, OSD_OP_WRITE, 4*1024*1024, 4096), 0);
check_op_count(cli, 1, 1);
can_complete(r2);
pretend_op_completed(cli, find_op(cli, 1, OSD_OP_SYNC, 0, 0), 0);
check_completed(r2);
// Check cutting of the beginning and end
assert((long)test_write(cli, 0, 32768, 0x55, NULL, true) == 1);
check_op_count(cli, 1, 0);
assert((long)test_write(cli, 32768, 32768, 0x56, NULL, true) == 1);
check_op_count(cli, 1, 0);
assert((long)test_write(cli, 16384, 32768, 0x57, NULL, true) == 1);
check_op_count(cli, 1, 0);
assert((long)test_write(cli, 16384+4096, 32768-4096, 0x58, NULL, true) == 1);
check_op_count(cli, 1, 0);
r2 = test_sync(cli);
check_op_count(cli, 1, 1);
pretend_op_completed(cli, find_op(cli, 1, OSD_OP_WRITE, 0, 65536), 0);
check_op_count(cli, 1, 1);
can_complete(r2);
pretend_op_completed(cli, find_op(cli, 1, OSD_OP_SYNC, 0, 0), 0);
check_completed(r2);
// Free client
delete cli;
delete tfd;
printf("[ok] writeback test\n");
}
int main(int narg, char *args[])
{
test1();
test2();
test_writeback();
return 0;
}
src/test/test_crc32.cpp
+28
View File
@@ -0,0 +1,28 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <stdint.h>
#include "malloc_or_die.h"
#include "errno.h"
#include "crc32c.h"
int main(int narg, char *args[])
{
int bufsize = 65536;
uint8_t *buf = (uint8_t*)malloc_or_die(bufsize);
uint32_t csum = 0;
while (1)
{
int r = read(0, buf, bufsize);
if (r <= 0 && errno != EAGAIN && errno != EINTR)
break;
csum = crc32c(csum, buf, r);
}
free(buf);
printf("%08x\n", csum);
return 0;
}
src/test/test_pattern.h
+15
View File
@@ -0,0 +1,15 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
#pragma once
#include <assert.h>
#include <stdint.h>
#define PATTERN0 0x8c4641acc762840e
#define PATTERN1 0x70a549add9a2280a
#define PATTERN2 0xffe3bad5f578a78e
#define PATTERN3 0x426bd7854eb08509
#define set_pattern(buf, len, pattern) for (uint64_t i = 0; i < len; i += 8) { *(uint64_t*)((uint8_t*)buf + i) = pattern; }
#define check_pattern(buf, len, pattern) { uint64_t bad = UINT64_MAX; for (uint64_t i = 0; i < len; i += 8) { if ((*(uint64_t*)((uint8_t*)buf + i)) != (pattern)) { bad = i; break; } } if (bad != UINT64_MAX) { printf("mismatch at %jx\n", bad); } assert(bad == UINT64_MAX); }
src/test/test_shit.cpp
+575
View File
@@ -0,0 +1,575 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
#define _LARGEFILE64_SOURCE
#include <sys/types.h>
#include <sys/ioctl.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdint.h>
#include <malloc.h>
#include <linux/fs.h>
#include <string.h>
#include <errno.h>
#include <assert.h>
#include <stdio.h>
#include <liburing.h>
#include <math.h>
#include <sys/socket.h>
#include <sys/epoll.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <map>
#include <vector>
#include <deque>
#include <algorithm>
#include "blockstore.h"
#include "blockstore_impl.h"
#include "osd_peering_pg.cpp"
//#include "cpp-btree/btree_map.h"
static int setup_context(unsigned entries, struct io_uring *ring)
{
int ret = io_uring_queue_init(entries, ring, 0);
if (ret < 0)
{
fprintf(stderr, "queue_init: %s\n", strerror(-ret));
return -1;
}
return 0;
}
static void test_write(struct io_uring *ring, int fd)
{
struct io_uring_sqe *sqe = io_uring_get_sqe(ring);
assert(sqe);
uint8_t *buf = (uint8_t*)memalign(512, 1024*1024*1024);
struct iovec iov = { buf, 1024*1024*1024 };
io_uring_prep_writev(sqe, fd, &iov, 1, 0);
io_uring_sqe_set_data(sqe, buf);
io_uring_submit_and_wait(ring, 1);
struct io_uring_cqe *cqe;
io_uring_peek_cqe(ring, &cqe);
int ret = cqe->res;
//int ret = writev(fd, &iov, 1);
if (ret < 0)
printf("cqe failed: %d %s\n", ret, strerror(-ret));
else
printf("result: %d user_data: %lld -> %lld\n", ret, sqe->user_data, cqe->user_data);
io_uring_cqe_seen(ring, cqe);
free(buf);
}
int main00(int argc, char *argv[])
{
// queue with random removal: vector is best :D
// deque: 8.1s
// vector: 6.6s
// list: 9.3s
for (int i = 0; i < 10000; i++)
{
std::list<int> q;
for (int i = 0; i < 20480; i++)
{
for (auto it = q.begin(); it != q.end();)
{
if (rand() < RAND_MAX/2)
{
//q.erase(it); -> for deque and vector
auto p = it++;
q.erase(p);
}
else
it++;
}
q.push_back(rand());
}
}
return 0;
}
int main01(int argc, char *argv[])
{
// deque: 2.091s
// vector: 18.733s
// list: 5.216s
// good, at least in this test deque is fine
for (int i = 0; i < 10000; i++)
{
std::deque<int> q;
for (int i = 0; i < 20480; i++)
{
int r = rand();
if (r < RAND_MAX/4 && q.size() > 0)
q.pop_front();
//q.erase(q.begin());
else
q.push_back(r);
}
}
return 0;
}
int main_vec(int argc, char *argv[])
{
// vector: 16 elements -> 0.047s, 256 elements -> 1.622s, 1024 elements -> 16.087s, 2048 elements -> 55.8s
for (int i = 0; i < 100000; i++)
{
std::vector<iovec> v;
for (int i = 0; i < 2048; i++)
{
int r = rand();
auto it = v.begin();
for (; it != v.end(); it++)
if (it->iov_len >= r)
break;
v.insert(it, (iovec){ .iov_base = 0, .iov_len = (size_t)r });
}
}
return 0;
}
int main_map(int argc, char *argv[])
{
// map: 16 elements -> 0.105s, 256 elements -> 2.634s, 1024 elements -> 12.55s, 2048 elements -> 27.475s
// conclustion: vector is better in fulfill_read
for (int i = 0; i < 100000; i++)
{
std::map<int,iovec> v;
for (int i = 0; i < 2048; i++)
{
int r = rand();
v[r] = (iovec){ .iov_base = 0, .iov_len = (size_t)r };
}
}
return 0;
}
int main0(int argc, char *argv[])
{
// std::map 5M entries monotone -> 2.115s, random -> 8.782s
// btree_map 5M entries monotone -> 0.458s, random -> 5.429s
// absl::btree_map 5M entries random -> 5.09s
// sparse_hash_map 5M entries -> 2.193s, random -> 2.586s
btree::btree_map<obj_ver_id, dirty_entry> dirty_db;
//std::map<obj_ver_id, dirty_entry> dirty_db;
//spp::sparse_hash_map<obj_ver_id, dirty_entry, obj_ver_hash> dirty_db;
for (int i = 0; i < 5000000; i++)
{
dirty_db[(obj_ver_id){
.oid = (object_id){
.inode = (uint64_t)rand(),
.stripe = (uint64_t)i,
},
.version = 1,
}] = (dirty_entry){
.state = BS_ST_SYNCED | BS_ST_BIG_WRITE,
.flags = 0,
.location = (uint64_t)i << 17,
.offset = 0,
.len = 1 << 17,
};
}
return 0;
}
int main1(int argc, char *argv[])
{
std::vector<uint64_t> v1, v2;
v1.reserve(10000);
v2.reserve(10000);
for (int i = 0; i < 10000; i++)
{
v1.push_back(i);
v2.push_back(i);
}
for (int i = 0; i < 100000; i++)
{
// haha (core i5-2500 | i7-6800HQ)
// vector 10000 items: 4.37/100000 | 3.66
// vector 100000 items: 9.68/10000 | 0.95
// deque 10000 items: 28.432/100000
// list 10000 items: 320.695/100000
std::vector<uint64_t> v3;
v3.insert(v3.end(), v1.begin(), v1.end());
v3.insert(v3.end(), v2.begin(), v2.end());
}
return 0;
}
int main02(int argc, char *argv[])
{
std::map<int, std::string> strs;
strs.emplace(12, "str");
auto it = strs.upper_bound(13);
//printf("s = %d %s %d\n", it->first, it->second.c_str(), it == strs.begin());
it--;
printf("%d\n", it == strs.end());
//printf("s = %d %s\n", it->first, it->second.c_str());
struct io_uring ring;
int fd = open("/dev/loop0", O_RDWR | O_DIRECT, 0644);
if (fd < 0)
{
perror("open infile");
return 1;
}
if (setup_context(32, &ring))
return 1;
test_write(&ring, fd);
close(fd);
io_uring_queue_exit(&ring);
return 0;
}
int main03(int argc, char *argv[])
{
int listen_fd = socket(AF_INET, SOCK_STREAM, 0), enable = 1;
assert(listen_fd >= 0);
setsockopt(listen_fd, SOL_SOCKET, SO_REUSEADDR, &enable, sizeof(enable));
struct sockaddr_in bind_addr;
assert(inet_pton(AF_INET, "0.0.0.0", &bind_addr.sin_addr) == 1);
bind_addr.sin_family = AF_INET;
bind_addr.sin_port = htons(13892);
int r = bind(listen_fd, (sockaddr*)&bind_addr, sizeof(bind_addr));
if (r)
{
perror("bind");
return 1;
}
assert(listen(listen_fd, 128) == 0);
struct sockaddr_in peer_addr;
socklen_t peer_addr_size = sizeof(peer_addr);
int peer_fd = accept(listen_fd, (sockaddr*)&peer_addr, &peer_addr_size);
assert(peer_fd >= 0);
//fcntl(peer_fd, F_SETFL, fcntl(listen_fd, F_GETFL, 0) | O_NONBLOCK);
struct io_uring ring;
assert(setup_context(32, &ring) == 0);
void *buf = memalign(512, 4096*1024);
struct io_uring_sqe *sqe = io_uring_get_sqe(&ring);
assert(sqe);
struct iovec iov = { buf, 4096*1024 };
struct msghdr msg = { 0 };
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
io_uring_prep_recvmsg(sqe, peer_fd, &msg, 0);
io_uring_sqe_set_data(sqe, buf);
io_uring_submit_and_wait(&ring, 1);
struct io_uring_cqe *cqe;
io_uring_peek_cqe(&ring, &cqe);
int ret = cqe->res;
printf("cqe result: %d\n", ret);
// ok, io_uring's sendmsg always reads as much data as is available and finishes
io_uring_cqe_seen(&ring, cqe);
close(peer_fd);
close(listen_fd);
io_uring_queue_exit(&ring);
return 0;
}
int main04(int argc, char *argv[])
{
/*spp::sparse_hash_set<obj_ver_id> osd1, osd2;
// fill takes 18.9 s
for (int i = 0; i < 1024*1024*8*2; i++)
{
obj_ver_id ovid = { { rand() % 500, rand() }, rand() };
osd1.insert(ovid);
osd2.insert(ovid);
}
for (int i = 0; i < 50000; i++)
{
obj_ver_id ovid = { { rand() % 500, rand() }, rand() };
osd1.insert(ovid);
ovid = { { rand() % 500, rand() }, rand() };
osd2.insert(ovid);
}
// diff takes only 2.3 s
spp::sparse_hash_set<obj_ver_id> osd1diff;
for (obj_ver_id e: osd1)
{
auto it = osd2.find(e);
if (it != osd2.end())
osd2.erase(it);
else
osd1diff.insert(e);
}*/
// fill vector takes 2 s
std::vector<obj_ver_role> to_sort;
to_sort.resize(1024*1024*8*2*3);
printf("Filling\n");
for (int i = 0; i < 1024*1024*8*2*3; i++)
{
to_sort[i] = {
.oid = (object_id){
.inode = (uint64_t)(rand() % 500),
.stripe = (uint64_t)rand(),
},
.version = (uint64_t)rand(),
.osd_num = (uint64_t)(rand() % 16),
};
}
printf("Sorting\n");
// sorting the whole array takes 7 s
// sorting in 3 parts... almost the same, 6 s
std::sort(to_sort.begin(), to_sort.end());
return 0;
}
uint64_t jumphash(uint64_t key, int count)
{
uint64_t b = 0;
uint64_t seed = key;
for (int j = 1; j < count; j++)
{
seed = 2862933555777941757ull*seed + 3037000493ull; // LCPRNG
if (seed < (UINT64_MAX / (j+1)))
{
b = j;
}
}
return b;
}
void jumphash_prepare(int count, uint64_t *out_weights, uint64_t *in_weights)
{
if (count <= 0)
{
return;
}
uint64_t total_weight = in_weights[0];
out_weights[0] = UINT64_MAX;
for (int j = 1; j < count; j++)
{
total_weight += in_weights[j];
out_weights[j] = UINT64_MAX / total_weight * in_weights[j];
}
}
uint64_t jumphash_weights(uint64_t key, int count, uint64_t *prepared_weights)
{
uint64_t b = 0;
uint64_t seed = key;
for (int j = 1; j < count; j++)
{
seed = 2862933555777941757ull*seed + 3037000493ull; // LCPRNG
if (seed < prepared_weights[j])
{
b = j;
}
}
return b;
}
void jumphash3(uint64_t key, int count, uint64_t *weights, uint64_t *r)
{
r[0] = 0;
r[1] = 1;
r[2] = 2;
uint64_t total_weight = weights[0]+weights[1]+weights[2];
uint64_t seed = key;
for (int j = 3; j < count; j++)
{
seed = 2862933555777941757ull*seed + 3037000493ull; // LCPRNG
total_weight += weights[j];
if (seed < UINT64_MAX*1.0*weights[j]/total_weight)
r[0] = j;
else
{
seed = 2862933555777941757ull*seed + 3037000493ull; // LCPRNG
if (seed < UINT64_MAX*1.0*weights[j]/total_weight)
r[1] = j;
else
{
seed = 2862933555777941757ull*seed + 3037000493ull; // LCPRNG
if (seed < UINT64_MAX*1.0*weights[j]/total_weight)
r[2] = j;
}
}
}
}
uint64_t crush(uint64_t key, int count, uint64_t *weights)
{
uint64_t b = 0;
uint64_t seed = 0;
uint64_t max = 0;
for (int j = 0; j < count; j++)
{
seed = (key + 0xc6a4a7935bd1e995 + (seed << 6) + (seed >> 2));
seed ^= (j + 0xc6a4a7935bd1e995 + (seed << 6) + (seed >> 2));
seed = 2862933555777941757ull*seed + 3037000493ull; // LCPRNG
seed = -log(((double)seed) / ((uint64_t)1 << 32) / ((uint64_t)1 << 32)) * weights[j];
if (seed > max)
{
max = seed;
b = j;
}
}
return b;
}
void crush3(uint64_t key, int count, uint64_t *weights, uint64_t *r, uint64_t total_weight)
{
uint64_t seed = 0;
uint64_t max = 0;
for (int k1 = 0; k1 < count; k1++)
{
for (int k2 = k1+1; k2 < count; k2++)
{
if (k2 == k1)
{
continue;
}
for (int k3 = k2+1; k3 < count; k3++)
{
if (k3 == k1 || k3 == k2)
{
continue;
}
seed = (key + 0xc6a4a7935bd1e995 + (seed << 6) + (seed >> 2));
seed ^= (k1 + 0xc6a4a7935bd1e995 + (seed << 6) + (seed >> 2));
seed ^= (k2 + 0xc6a4a7935bd1e995 + (seed << 6) + (seed >> 2));
seed ^= (k3 + 0xc6a4a7935bd1e995 + (seed << 6) + (seed >> 2));
seed = 2862933555777941757ull*seed + 3037000493ull; // LCPRNG
//seed = ((double)seed) / ((uint64_t)1 << 32) / ((uint64_t)1 << 32) * (weights[k1] + weights[k2] + weights[k3]);
seed = ((double)seed) / ((uint64_t)1 << 32) / ((uint64_t)1 << 32) * (1 -
(1 - 1.0*weights[k1]/total_weight)*
(1 - 1.0*weights[k2]/total_weight)*
(1 - 1.0*weights[k3]/total_weight)
) * UINT64_MAX;
if (seed > max)
{
r[0] = k1;
r[1] = k2;
r[2] = k3;
max = seed;
}
}
}
}
}
int main(int argc, char *argv[])
{
int host_count = 6;
uint64_t host_weights[] = {
34609*3,
34931*3,
35850+36387+35859,
36387,
36387*2,
36387,
};
/*int osd_count[] = { 3, 3, 3, 1, 2 };
uint64_t osd_weights[][3] = {
{ 34609, 34609, 34609 },
{ 34931, 34931, 34931 },
{ 35850, 36387, 35859 },
{ 36387 },
{ 36387, 36387 },
};*/
uint64_t total_weight = 0;
for (int i = 0; i < host_count; i++)
{
total_weight += host_weights[i];
}
uint64_t host_weights_prepared[host_count];
jumphash_prepare(host_count, host_weights_prepared, host_weights);
uint64_t total_pgs[host_count] = { 0 };
int pg_count = 256;
double uniformity[pg_count] = { 0 };
for (uint64_t pg = 1; pg <= pg_count; pg++)
{
uint64_t r[3];
/*
// Select first host
//r[0] = jumphash_weights(pg, host_count, host_weights_prepared);
r[0] = crush(pg, host_count, host_weights);
// Select second host
uint64_t seed = pg;
r[1] = r[0];
while (r[1] == r[0])
{
seed = 2862933555777941757ull*seed + 3037000493ull; // LCPRNG
//r[1] = jumphash_weights(seed, host_count, host_weights_prepared);
r[1] = crush(seed, host_count, host_weights);
}
// Select third host
seed = pg;
r[2] = r[0];
while (r[2] == r[0] || r[2] == r[1])
{
seed = 2862933555777941757ull*seed + 3037000493ull; // LCPRNG
//r[2] = jumphash_weights(seed, host_count, host_weights_prepared);
r[2] = crush(seed, host_count, host_weights);
}
*/
/*
// Select second host
uint64_t host_weights1[host_count];
for (int i = 0; i < r[0]; i++)
host_weights1[i] = host_weights[i];
for (int i = r[0]+1; i < host_count; i++)
host_weights1[i-1] = host_weights[i];
r[1] = crush(pg, host_count-1, host_weights1);
// Select third host
for (int i = r[1]+1; i < host_count-1; i++)
host_weights1[i-1] = host_weights[i];
r[2] = crush(pg, host_count-2, host_weights1);
// Transform numbers
r[2] = r[2] >= r[1] ? 1+r[2] : r[2];
r[2] = r[2] >= r[0] ? 1+r[2] : r[2];
r[1] = r[1] >= r[0] ? 1+r[1] : r[1];
*/
crush3(pg, host_count, host_weights, r, total_weight);
uint64_t shift = (2862933555777941757ull*pg + 3037000493ull) % host_count;
if (shift == 1)
{
uint64_t tmp;
tmp = r[0];
r[0] = r[1];
r[1] = r[2];
r[2] = tmp;
}
else if (shift == 2)
{
uint64_t tmp;
tmp = r[0];
r[0] = r[2];
r[2] = r[1];
r[1] = tmp;
}
total_pgs[r[0]]++;
total_pgs[r[1]]++;
total_pgs[r[2]]++;
double u = 0;
for (int i = 0; i < host_count; i++)
{
double d = abs(1 - total_pgs[i]/3.0/pg * total_weight/host_weights[i]);
u += d;
}
uniformity[pg-1] = u/host_count;
printf("pg %ju: hosts %ju, %ju, %ju ; avg deviation = %.2f\n", pg, r[0], r[1], r[2], u/host_count);
}
printf("total PGs: ");
for (int i = 0; i < host_count; i++)
{
printf(i > 0 ? ", %ju (%.2f)" : "%ju (%.2f)", total_pgs[i], total_pgs[i]/3.0/pg_count * total_weight/host_weights[i]);
}
printf("\n");
return 0;
}