Fix 2 bugs in NFS-RDMA allocator, add a test for both

1) alloc() could add the region start into freelist instead of its free part
2) free() was merging freed buffers incorrectly both forward and backward

src/nfs/CMakeLists.txt

@@ -35,3 +35,8 @@ target_link_libraries(vitastor-nfs
 	vitastor_kv
 	${RDMACM_LIBRARIES}
 )
+
+# test_rdma_alloc
+add_executable(test_rdma_alloc EXCLUDE_FROM_ALL test_rdma_alloc.cpp rdma_alloc.cpp)
+add_dependencies(build_tests test_rdma_alloc)
+add_test(NAME test_rdma_alloc COMMAND test_rdma_alloc)

src/nfs/rdma_alloc.cpp

@@ -136,9 +136,9 @@ void *rdma_malloc_alloc(rdma_allocator_t *self, size_t size)
     }
     else
     {
+        self->freelist.insert((rdma_free_t){ .len = frag.len-size, .buf = ptr });
         frag.len -= size;
         ptr = (uint8_t*)ptr + frag.len;
-        self->freelist.insert((rdma_free_t){ .len = frag.len, .buf = frag.rgn->buf });
         self->frags[ptr] = (rdma_frag_t){ .rgn = frag.rgn, .len = size, .is_free = false };
     }
     return ptr;
@@ -174,8 +174,10 @@ void rdma_malloc_free(rdma_allocator_t *self, void *buf)
     }
     else if (merge_back)
     {
+        self->freelist.erase((rdma_free_t){ .len = prev_it->second.len, .buf = prev_it->first });
         prev_it->second.len += frag_it->second.len;
         self->frags.erase(frag_it);
+        self->freelist.insert((rdma_free_t){ .len = prev_it->second.len, .buf = prev_it->first });
         frag_it = prev_it;
     }
     else if (merge_next)
@@ -184,6 +186,7 @@ void rdma_malloc_free(rdma_allocator_t *self, void *buf)
         frag_it->second.len += next_it->second.len;
         self->freelist.erase((rdma_free_t){ .len = next_it->second.len, .buf = next_it->first });
         self->frags.erase(next_it);
+        self->freelist.insert((rdma_free_t){ .len = frag_it->second.len, .buf = frag_it->first });
     }
     else
     {

src/nfs/test_rdma_alloc.cpp

@@ -0,0 +1,63 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 (see README.md for details)
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <set>
+#include "rdma_alloc.h"
+
+std::set<void*> mrs;
+
+extern "C" {
+
+ibv_mr *ibv_reg_mr_iova2(ibv_pd *pd, void *addr, size_t length, uint64_t, unsigned)
+{
+    bool added = mrs.insert(addr).second;
+    assert(added);
+    return (ibv_mr*)addr;
+}
+
+int ibv_dereg_mr(ibv_mr *mr)
+{
+    auto erased = mrs.erase((void*)mr);
+    assert(erased);
+    return 0;
+}
+
+}
+
+void test_split()
+{
+    rdma_allocator_t *alloc = rdma_malloc_create(NULL, 1048576, 64*1048576, IBV_ACCESS_LOCAL_WRITE);
+    auto buf0 = rdma_malloc_alloc(alloc, 1024*1024);
+    rdma_malloc_free(alloc, buf0);
+    auto buf1 = rdma_malloc_alloc(alloc, 224*1024);
+    auto buf2 = rdma_malloc_alloc(alloc, 500*1024);
+    auto buf3 = rdma_malloc_alloc(alloc, 300*1024);
+    rdma_malloc_free(alloc, buf2);
+    auto buf4 = rdma_malloc_alloc(alloc, 300*1024);
+    auto buf5 = rdma_malloc_alloc(alloc, 200*1024);
+    assert((buf5 == buf2 || buf4 == buf2) && buf5 != buf4 && buf5 != buf3 && buf5 != buf1 &&
+        buf4 != buf3 && buf4 != buf1 && buf3 != buf1);
+    rdma_malloc_free(alloc, buf5);
+    rdma_malloc_free(alloc, buf4);
+    rdma_malloc_free(alloc, buf3);
+    rdma_malloc_free(alloc, buf1);
+    // now check merge-both
+    buf1 = rdma_malloc_alloc(alloc, 224*1024);
+    buf2 = rdma_malloc_alloc(alloc, 500*1024);
+    buf3 = rdma_malloc_alloc(alloc, 300*1024);
+    rdma_malloc_free(alloc, buf3);
+    rdma_malloc_free(alloc, buf1);
+    rdma_malloc_free(alloc, buf2);
+    rdma_malloc_destroy(alloc);
+    printf("ok test_split\n");
+}
+
+int main(int narg, char *args[])
+{
+    test_split();
+    return 0;
+}