Run tests with writeback?

Should fix spurious client hangs during PG primary switchover

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "1.8.0")
+set(VITASTOR_VERSION "1.7.1")
 
 add_subdirectory(src)

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v1.8.0
+VITASTOR_VERSION ?= v1.7.1
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v1.8.0
+          image: vitalif/vitastor-csi:v1.7.1
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v1.8.0
+          image: vitalif/vitastor-csi:v1.7.1
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "1.8.0"
+    vitastorCSIDriverVersion = "1.7.1"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (1.8.0-1) unstable; urgency=medium
+vitastor (1.7.1-1) unstable; urgency=medium
 
   * Bugfixes
 

docs/installation/proxmox.en.md

@@ -17,10 +17,10 @@ To enable Vitastor support in Proxmox Virtual Environment (6.4-8.1 are supported
 - Restart pvedaemon: `systemctl restart pvedaemon`
 
 `/etc/pve/storage.cfg` example (the only required option is vitastor_pool, all others
-are listed below with their default values; `vitastor_ssd` is Proxmox storage pool id):
+are listed below with their default values):
 
 ```
-vitastor: vitastor_ssd
+vitastor: vitastor
     # pool to put new images into
     vitastor_pool testpool
     # path to the configuration file

docs/installation/proxmox.ru.md

@@ -16,10 +16,10 @@
 - Перезапустите демон Proxmox: `systemctl restart pvedaemon`
 
 Пример `/etc/pve/storage.cfg` (единственная обязательная опция - vitastor_pool, все остальные
-перечислены внизу для понимания значений по умолчанию; `vitastor_ssd` - имя хранилища в Proxmox):
+перечислены внизу для понимания значений по умолчанию):
 
 ```
-vitastor: vitastor_ssd
+vitastor: vitastor
     # Пул, в который будут помещаться образы дисков
     vitastor_pool testpool
     # Путь к файлу конфигурации

docs/usage/cli.en.md

@@ -307,7 +307,7 @@ OSD  PARENT            UP    SIZE  USED%    TAGS          WEIGHT  BLOCK  BITMAP
 
 `vitastor-cli modify-osd [--tags tag1,tag2,...] [--reweight <number>] [--noout true/false] <osd_number>`
 
-Set OSD reweight, tags or noout flag. See detail description in [OSD config documentation](../config/pool.en.md#osd-settings).
+Set OSD reweight, tags or noout flag. See detail description in [OSD config documentation](../config.pool.en.md#osd-settings).
 
 ## pg-list
 

docs/usage/cli.ru.md

@@ -324,7 +324,7 @@ OSD  PARENT            UP    SIZE  USED%    TAGS          WEIGHT  BLOCK  BITMAP
 
 `vitastor-cli modify-osd [--tags tag1,tag2,...] [--reweight <number>] [--noout true/false] <osd_number>`
 
-Установить вес OSD, теги или флаг noout. Смотрите подробное описание в [документации настроек OSD](../config/pool.ru.md#настройки-osd).
+Установить вес OSD, теги или флаг noout. Смотрите подробное описание в [документации настроек OSD](../config.pool.ru.md#настройки-osd).
 
 ## pg-list
 

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "1.8.0",
+  "version": "1.7.1",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {

node-binding/addon.cc

@@ -24,7 +24,7 @@ NAN_MODULE_INIT(InitAddon)
 
     tpl = Nan::New<v8::FunctionTemplate>(NodeVitastorImage::Create);
     tpl->SetClassName(Nan::New("Image").ToLocalChecked());
-    tpl->InstanceTemplate()->SetInternalFieldCount(1);
+    tpl->InstanceTemplate()->SetInternalFieldCount(2);
 
     Nan::SetPrototypeMethod(tpl, "read", NodeVitastorImage::Read);
     Nan::SetPrototypeMethod(tpl, "write", NodeVitastorImage::Write);
@@ -67,7 +67,7 @@ NAN_MODULE_INIT(InitAddon)
 
     tpl = Nan::New<v8::FunctionTemplate>(NodeVitastorKVListing::Create);
     tpl->SetClassName(Nan::New("KVListing").ToLocalChecked());
-    tpl->InstanceTemplate()->SetInternalFieldCount(1);
+    tpl->InstanceTemplate()->SetInternalFieldCount(2);
 
     Nan::SetPrototypeMethod(tpl, "next", NodeVitastorKVListing::Next);
     Nan::SetPrototypeMethod(tpl, "close", NodeVitastorKVListing::Close);

node-binding/binding.gyp

@@ -13,8 +13,7 @@
         '<!(pkg-config --cflags vitastor)'
       ],
       'libraries': [
-        '<!(pkg-config --libs vitastor)',
+        '<!(pkg-config --libs vitastor)'
-        '-lvitastor_kv'
       ]
     }
   ]

node-binding/client.cc

@@ -18,15 +18,12 @@
 class NodeVitastorRequest: public Nan::AsyncResource
 {
 public:
-    NodeVitastorRequest(NodeVitastor *cli, v8::Local<v8::Function> cb): Nan::AsyncResource("NodeVitastorRequest")
+    NodeVitastorRequest(v8::Local<v8::Function> cb): Nan::AsyncResource("NodeVitastorRequest")
     {
-        this->cli = cli;
         callback.Reset(cb);
     }
 
     iovec iov;
-    std::vector<iovec> iov_list;
-    NodeVitastor *cli = NULL;
     NodeVitastorImage *img = NULL;
     int op = 0;
     uint64_t offset = 0, len = 0, version = 0;
@@ -34,13 +31,6 @@ public:
     Nan::Persistent<v8::Function> callback;
 };
 
-static uint64_t get_ui64(const v8::Local<v8::Value> & val)
-{
-    if (val->IsBigInt())
-        return val->ToBigInt(Nan::GetCurrentContext()).ToLocalChecked()->Uint64Value();
-    return Nan::To<int64_t>(val).FromJust();
-}
-
 //////////////////////////////////////////////////
 // NodeVitastor
 //////////////////////////////////////////////////
@@ -53,7 +43,6 @@ NodeVitastor::NodeVitastor(): Nan::ObjectWrap()
 
 NodeVitastor::~NodeVitastor()
 {
-    TRACE("NodeVitastor: destructor");
     uv_poll_stop(&poll_watcher);
     vitastor_c_destroy(c);
     c = NULL;
@@ -85,10 +74,9 @@ NAN_METHOD(NodeVitastor::Create)
     if (res >= 0)
     {
         cli->eventfd = res;
-        res = uv_poll_init(uv_default_loop(), &cli->poll_watcher, cli->eventfd);
+        res = uv_poll_init_socket(uv_default_loop(), &cli->poll_watcher, cli->eventfd);
         if (res >= 0)
             res = uv_poll_start(&cli->poll_watcher, UV_READABLE, on_io_readable);
-        on_io_readable(&cli->poll_watcher, 0, UV_READABLE);
     }
     if (res < 0)
     {
@@ -109,14 +97,15 @@ void NodeVitastor::on_io_readable(uv_poll_t* handle, int status, int revents)
     if (revents & UV_READABLE)
     {
         NodeVitastor* self = (NodeVitastor*)handle->data;
+        std::unique_lock<std::mutex> lock(self->mu);
         vitastor_c_uring_handle_events(self->c);
     }
 }
 
-NodeVitastorRequest* NodeVitastor::get_read_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos)
+static NodeVitastorRequest* getReadRequest(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos)
 {
-    uint64_t offset = get_ui64(info[argpos+0]);
+    uint64_t offset = Nan::To<int64_t>(info[argpos+0]).FromJust();
-    uint64_t len = get_ui64(info[argpos+1]);
+    uint64_t len = Nan::To<int64_t>(info[argpos+1]).FromJust();
     uint8_t *buf = (uint8_t*)malloc(len);
     if (!buf)
     {
@@ -124,7 +113,7 @@ NodeVitastorRequest* NodeVitastor::get_read_request(const Nan::FunctionCallbackI
         return NULL;
     }
     v8::Local<v8::Function> callback = info[argpos+2].As<v8::Function>();
-    auto req = new NodeVitastorRequest(this, callback);
+    auto req = new NodeVitastorRequest(callback);
 
     req->offset = offset;
     req->len = len;
@@ -137,101 +126,73 @@ NodeVitastorRequest* NodeVitastor::get_read_request(const Nan::FunctionCallbackI
 NAN_METHOD(NodeVitastor::Read)
 {
     TRACE("NodeVitastor::Read");
-    if (info.Length() < 5)
-        Nan::ThrowError("Not enough arguments to read(pool, inode, offset, len, callback(err, buffer, version))");
 
     NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
 
-    uint64_t pool = get_ui64(info[0]);
+    uint64_t pool = Nan::To<int64_t>(info[0]).FromJust();
-    uint64_t inode = get_ui64(info[1]);
+    uint64_t inode = Nan::To<int64_t>(info[1]).FromJust();
 
-    auto req = self->get_read_request(info, 2);
+    auto req = getReadRequest(info, 2);
 
-    self->Ref();
+    std::unique_lock<std::mutex> lock(self->mu);
     vitastor_c_read(self->c, ((pool << (64-POOL_ID_BITS)) | inode), req->offset, req->len, &req->iov, 1, on_read_finish, req);
 }
 
-NodeVitastorRequest* NodeVitastor::get_write_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos)
+static NodeVitastorRequest* getWriteRequest(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos)
 {
-    uint64_t offset = get_ui64(info[argpos+0]);
+    uint64_t offset = Nan::To<int64_t>(info[argpos+0]).FromJust();
-    const auto & bufarg = info[argpos+1];
+    char *buf = node::Buffer::Data(info[argpos+1]);
+    uint64_t len = node::Buffer::Length(info[argpos+1]);
     uint64_t version = 0;
-    if (!info[argpos+2].IsEmpty() &&
+
-        !info[argpos+2]->IsFunction() &&
+    if (!info[argpos+2].IsEmpty() && info[argpos+2]->IsObject())
-        info[argpos+2]->IsObject())
     {
         auto key = Nan::New<v8::String>("version").ToLocalChecked();
         auto params = info[argpos+2].As<v8::Object>();
         auto versionObj = Nan::Get(params, key).ToLocalChecked();
         if (!versionObj.IsEmpty())
-            version = get_ui64(versionObj);
+            version = Nan::To<int64_t>(versionObj).FromJust();
         argpos++;
     }
 
     v8::Local<v8::Function> callback = info[argpos+2].As<v8::Function>();
-    auto req = new NodeVitastorRequest(this, callback);
+    auto req = new NodeVitastorRequest(callback);
 
     req->offset = offset;
+    req->len = len;
     req->version = version;
-
+    req->iov = { .iov_base = buf, .iov_len = req->len };
-    if (bufarg->IsArray())
-    {
-        auto buffers = bufarg.As<v8::Array>();
-        req->len = 0;
-        for (uint32_t i = 0; i < buffers->Length(); i++)
-        {
-            auto buffer_obj = Nan::Get(buffers, i).ToLocalChecked();
-            char *buf = node::Buffer::Data(buffer_obj);
-            uint64_t len = node::Buffer::Length(buffer_obj);
-            req->iov_list.push_back({ .iov_base = buf, .iov_len = len });
-            req->len += len;
-        }
-    }
-    else
-    {
-        char *buf = node::Buffer::Data(bufarg);
-        uint64_t len = node::Buffer::Length(bufarg);
-        req->iov = { .iov_base = buf, .iov_len = len };
-        req->len = len;
-    }
 
     return req;
 }
 
-// write(pool, inode, offset, buf: Buffer | Buffer[], { version }?, callback(err))
+// write(pool, inode, offset, buffer, { version }?, callback(err))
 NAN_METHOD(NodeVitastor::Write)
 {
     TRACE("NodeVitastor::Write");
-    if (info.Length() < 5)
-        Nan::ThrowError("Not enough arguments to write(pool, inode, offset, buf: Buffer | Buffer[], { version }?, callback(err))");
 
     NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
 
-    uint64_t pool = get_ui64(info[0]);
+    uint64_t pool = Nan::To<int64_t>(info[0]).FromJust();
-    uint64_t inode = get_ui64(info[1]);
+    uint64_t inode = Nan::To<int64_t>(info[1]).FromJust();
 
-    auto req = self->get_write_request(info, 2);
+    auto req = getWriteRequest(info, 2);
 
-    self->Ref();
+    std::unique_lock<std::mutex> lock(self->mu);
-    vitastor_c_write(self->c, ((pool << (64-POOL_ID_BITS)) | inode), req->offset, req->len, req->version,
+    vitastor_c_write(self->c, ((pool << (64-POOL_ID_BITS)) | inode), req->offset, req->len, req->version, &req->iov, 1, on_write_finish, req);
-        req->iov_list.size() ? req->iov_list.data() : &req->iov,
-        req->iov_list.size() ? req->iov_list.size() : 1,
-        on_write_finish, req);
 }
 
 // sync(callback(err))
 NAN_METHOD(NodeVitastor::Sync)
 {
     TRACE("NodeVitastor::Sync");
-    if (info.Length() < 1)
-        Nan::ThrowError("Not enough arguments to sync(callback(err))");
 
     NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
 
     v8::Local<v8::Function> callback = info[0].As<v8::Function>();
-    auto req = new NodeVitastorRequest(self, callback);
+    auto req = new NodeVitastorRequest(callback);
 
-    self->Ref();
+    std::unique_lock<std::mutex> lock(self->mu);
     vitastor_c_sync(self->c, on_write_finish, req);
 }
 
@@ -239,20 +200,17 @@ NAN_METHOD(NodeVitastor::Sync)
 NAN_METHOD(NodeVitastor::ReadBitmap)
 {
     TRACE("NodeVitastor::ReadBitmap");
-    if (info.Length() < 6)
-        Nan::ThrowError("Not enough arguments to read_bitmap(pool, inode, offset, len, with_parents, callback(err, bitmap_buffer))");
 
     NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
 
-    uint64_t pool = get_ui64(info[0]);
+    uint64_t pool = Nan::To<int64_t>(info[0]).FromJust();
-    uint64_t inode = get_ui64(info[1]);
+    uint64_t inode = Nan::To<int64_t>(info[1]).FromJust();
-    uint64_t offset = get_ui64(info[2]);
+    uint64_t offset = Nan::To<int64_t>(info[2]).FromJust();
-    uint64_t len = get_ui64(info[3]);
+    uint64_t len = Nan::To<int64_t>(info[3]).FromJust();
     bool with_parents = Nan::To<bool>(info[4]).FromJust();
     v8::Local<v8::Function> callback = info[5].As<v8::Function>();
 
-    auto req = new NodeVitastorRequest(self, callback);
+    auto req = new NodeVitastorRequest(callback);
-    self->Ref();
     vitastor_c_read_bitmap(self->c, ((pool << (64-POOL_ID_BITS)) | inode), offset, len, with_parents, on_read_bitmap_finish, req);
 }
 
@@ -269,7 +227,6 @@ static void on_error(NodeVitastorRequest *req, Nan::Callback & nanCallback, long
 
 void NodeVitastor::on_read_finish(void *opaque, long retval, uint64_t version)
 {
-    TRACE("NodeVitastor::on_read_finish");
     Nan::HandleScope scope;
     NodeVitastorRequest *req = (NodeVitastorRequest *)opaque;
     Nan::Callback nanCallback(Nan::New(req->callback));
@@ -278,7 +235,7 @@ void NodeVitastor::on_read_finish(void *opaque, long retval, uint64_t version)
         free(req->iov.iov_base);
         nanCallback.Call(0, NULL, req);
     }
-    else if (retval < 0 || (uint64_t)retval != req->len)
+    else if (retval < 0)
     {
         free(req->iov.iov_base);
         on_error(req, nanCallback, retval);
@@ -291,33 +248,20 @@ void NodeVitastor::on_read_finish(void *opaque, long retval, uint64_t version)
         args[2] = v8::BigInt::NewFromUnsigned(v8::Isolate::GetCurrent(), version);
         nanCallback.Call(3, args, req);
     }
-    req->cli->Unref();
     delete req;
 }
 
 void NodeVitastor::on_write_finish(void *opaque, long retval)
 {
-    TRACE("NodeVitastor::on_write_finish");
     Nan::HandleScope scope;
     NodeVitastorRequest *req = (NodeVitastorRequest *)opaque;
     Nan::Callback nanCallback(Nan::New(req->callback));
-    if (retval < 0 || (uint64_t)retval != req->len)
+    on_error(req, nanCallback, retval);
-    {
-        on_error(req, nanCallback, retval);
-    }
-    else
-    {
-        v8::Local<v8::Value> args[1];
-        args[0] = Nan::Null();
-        nanCallback.Call(1, args, req);
-    }
-    req->cli->Unref();
     delete req;
 }
 
 void NodeVitastor::on_read_bitmap_finish(void *opaque, long retval, uint8_t *bitmap)
 {
-    TRACE("NodeVitastor::on_read_bitmap_finish");
     Nan::HandleScope scope;
     NodeVitastorRequest *req = (NodeVitastorRequest *)opaque;
     Nan::Callback nanCallback(Nan::New(req->callback));
@@ -332,7 +276,6 @@ void NodeVitastor::on_read_bitmap_finish(void *opaque, long retval, uint8_t *bit
         args[1] = Nan::NewBuffer((char*)bitmap, (retval+7)/8).ToLocalChecked();
         nanCallback.Call(2, args, req);
     }
-    req->cli->Unref();
     delete req;
 }
 
@@ -348,23 +291,21 @@ void NodeVitastor::on_read_bitmap_finish(void *opaque, long retval, uint8_t *bit
 NAN_METHOD(NodeVitastorImage::Create)
 {
     TRACE("NodeVitastorImage::Create");
-    if (info.Length() < 2)
-        Nan::ThrowError("Not enough arguments to Image(client, name)");
 
     v8::Local<v8::Object> parent = info[0].As<v8::Object>();
     std::string name = std::string(*Nan::Utf8String(info[1].As<v8::String>()));
     NodeVitastor *cli = Nan::ObjectWrap::Unwrap<NodeVitastor>(parent);
 
     NodeVitastorImage *img = new NodeVitastorImage();
-    img->Wrap(info.This());
-
     img->cli = cli;
     img->name = name;
 
     img->Ref();
     cli->Ref();
+    std::unique_lock<std::mutex> lock(cli->mu);
     vitastor_c_watch_inode(cli->c, (char*)img->name.c_str(), on_watch_start, img);
 
+    img->Wrap(info.This());
     info.GetReturnValue().Set(info.This());
 }
 
@@ -382,12 +323,10 @@ NodeVitastorImage::~NodeVitastorImage()
 NAN_METHOD(NodeVitastorImage::Read)
 {
     TRACE("NodeVitastorImage::Read");
-    if (info.Length() < 3)
-        Nan::ThrowError("Not enough arguments to read(offset, len, callback(err, buffer, version))");
 
     NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
 
-    auto req = img->cli->get_read_request(info, 0);
+    auto req = getReadRequest(info, 0);
     req->img = img;
     req->op = NODE_VITASTOR_READ;
 
@@ -398,29 +337,24 @@ NAN_METHOD(NodeVitastorImage::Read)
 NAN_METHOD(NodeVitastorImage::Write)
 {
     TRACE("NodeVitastorImage::Write");
-    if (info.Length() < 3)
-        Nan::ThrowError("Not enough arguments to write(offset, buffer, { version }?, callback(err))");
 
     NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
 
-    auto req = img->cli->get_write_request(info, 0);
+    auto req = getWriteRequest(info, 0);
     req->img = img;
     req->op = NODE_VITASTOR_WRITE;
 
     img->exec_or_wait(req);
 }
 
-// sync(callback(err))
 NAN_METHOD(NodeVitastorImage::Sync)
 {
     TRACE("NodeVitastorImage::Sync");
-    if (info.Length() < 1)
-        Nan::ThrowError("Not enough arguments to sync(callback(err))");
 
     NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
 
     v8::Local<v8::Function> callback = info[0].As<v8::Function>();
-    auto req = new NodeVitastorRequest(img->cli, callback);
+    auto req = new NodeVitastorRequest(callback);
     req->img = img;
     req->op = NODE_VITASTOR_SYNC;
 
@@ -431,17 +365,15 @@ NAN_METHOD(NodeVitastorImage::Sync)
 NAN_METHOD(NodeVitastorImage::ReadBitmap)
 {
     TRACE("NodeVitastorImage::ReadBitmap");
-    if (info.Length() < 4)
-        Nan::ThrowError("Not enough arguments to read_bitmap(offset, len, with_parents, callback(err, bitmap_buffer))");
 
     NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
 
-    uint64_t offset = get_ui64(info[0]);
+    uint64_t offset = Nan::To<int64_t>(info[0]).FromJust();
-    uint64_t len = get_ui64(info[1]);
+    uint64_t len = Nan::To<int64_t>(info[1]).FromJust();
     bool with_parents = Nan::To<bool>(info[2]).FromJust();
     v8::Local<v8::Function> callback = info[3].As<v8::Function>();
 
-    auto req = new NodeVitastorRequest(img->cli, callback);
+    auto req = new NodeVitastorRequest(callback);
     req->img = img;
     req->op = NODE_VITASTOR_READ_BITMAP;
     req->offset = offset;
@@ -451,17 +383,14 @@ NAN_METHOD(NodeVitastorImage::ReadBitmap)
     img->exec_or_wait(req);
 }
 
-// get_info(callback({ num, name, size, parent_id?, readonly?, meta?, mod_revision, block_size, bitmap_granularity, immediate_commit }))
 NAN_METHOD(NodeVitastorImage::GetInfo)
 {
-    TRACE("NodeVitastorImage::GetInfo");
+    TRACE("NodeVitastorImage::Sync");
-    if (info.Length() < 1)
-        Nan::ThrowError("Not enough arguments to get_info(callback({ num, name, size, parent_id?, readonly?, meta?, mod_revision, block_size, bitmap_granularity, immediate_commit }))");
 
     NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
 
     v8::Local<v8::Function> callback = info[0].As<v8::Function>();
-    auto req = new NodeVitastorRequest(img->cli, callback);
+    auto req = new NodeVitastorRequest(callback);
     req->img = img;
     req->op = NODE_VITASTOR_GET_INFO;
 
@@ -483,26 +412,21 @@ void NodeVitastorImage::exec_or_wait(NodeVitastorRequest *req)
 
 void NodeVitastorImage::exec_request(NodeVitastorRequest *req)
 {
+    std::unique_lock<std::mutex> lock(cli->mu);
     if (req->op == NODE_VITASTOR_READ)
     {
         uint64_t ino = vitastor_c_inode_get_num(watch);
-        cli->Ref();
         vitastor_c_read(cli->c, ino, req->offset, req->len, &req->iov, 1, NodeVitastor::on_read_finish, req);
     }
     else if (req->op == NODE_VITASTOR_WRITE)
     {
         uint64_t ino = vitastor_c_inode_get_num(watch);
-        cli->Ref();
+        vitastor_c_write(cli->c, ino, req->offset, req->len, req->version, &req->iov, 1, NodeVitastor::on_write_finish, req);
-        vitastor_c_write(cli->c, ino, req->offset, req->len, req->version,
-            req->iov_list.size() ? req->iov_list.data() : &req->iov,
-            req->iov_list.size() ? req->iov_list.size() : 1,
-            NodeVitastor::on_write_finish, req);
     }
     else if (req->op == NODE_VITASTOR_SYNC)
     {
         uint64_t ino = vitastor_c_inode_get_num(watch);
         uint32_t imm = vitastor_c_inode_get_immediate_commit(cli->c, ino);
-        cli->Ref();
         if (imm != IMMEDIATE_ALL)
         {
             vitastor_c_sync(cli->c, NodeVitastor::on_write_finish, req);
@@ -515,7 +439,6 @@ void NodeVitastorImage::exec_request(NodeVitastorRequest *req)
     else if (req->op == NODE_VITASTOR_READ_BITMAP)
     {
         uint64_t ino = vitastor_c_inode_get_num(watch);
-        cli->Ref();
         vitastor_c_read_bitmap(cli->c, ino, req->offset, req->len, req->with_parents, NodeVitastor::on_read_bitmap_finish, req);
     }
     else if (req->op == NODE_VITASTOR_GET_INFO)
@@ -585,14 +508,10 @@ void NodeVitastorImage::on_watch_start(void *opaque, long retval)
 // NodeVitastorKV
 //////////////////////////////////////////////////
 
-Nan::Persistent<v8::Function> NodeVitastorKV::listing_class;
-
 // constructor(node_vitastor)
 NAN_METHOD(NodeVitastorKV::Create)
 {
     TRACE("NodeVitastorKV::Create");
-    if (info.Length() < 1)
-        Nan::ThrowError("Not enough arguments to new KV(client)");
 
     v8::Local<v8::Object> parent = info[0].As<v8::Object>();
     NodeVitastor *cli = Nan::ObjectWrap::Unwrap<NodeVitastor>(parent);
@@ -600,32 +519,29 @@ NAN_METHOD(NodeVitastorKV::Create)
     NodeVitastorKV *kv = new NodeVitastorKV();
     kv->cli = cli;
     {
+        std::unique_lock<std::mutex> lock(cli->mu);
         kv->dbw = new vitastorkv_dbw_t((cluster_client_t*)vitastor_c_get_internal_client(cli->c));
     }
 
     kv->Wrap(info.This());
-    cli->Ref();
     info.GetReturnValue().Set(info.This());
 }
 
 NodeVitastorKV::~NodeVitastorKV()
 {
     delete dbw;
-    cli->Unref();
 }
 
-// open(pool_id, inode_num, { ...config }, callback(err))
+// open(inode_id, { ...config }, callback(err))
 NAN_METHOD(NodeVitastorKV::Open)
 {
     TRACE("NodeVitastorKV::Open");
-    if (info.Length() < 4)
-        Nan::ThrowError("Not enough arguments to open(pool_id, inode_num, { ...config }, callback(err))");
 
     NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
 
-    uint64_t inode_id = INODE_WITH_POOL(get_ui64(info[0]), get_ui64(info[1]));
+    uint64_t inode_id = Nan::To<int64_t>(info[0]).FromJust();
 
-    v8::Local<v8::Object> jsParams = info[2].As<v8::Object>();
+    v8::Local<v8::Object> jsParams = info[1].As<v8::Object>();
     v8::Local<v8::Array> keys = Nan::GetOwnPropertyNames(jsParams).ToLocalChecked();
     std::map<std::string, std::string> cfg;
     for (uint32_t i = 0; i < keys->Length(); i++)
@@ -634,8 +550,8 @@ NAN_METHOD(NodeVitastorKV::Open)
         cfg[std::string(*Nan::Utf8String(key))] = std::string(*Nan::Utf8String(Nan::Get(jsParams, key).ToLocalChecked()));
     }
 
-    v8::Local<v8::Function> callback = info[3].As<v8::Function>();
+    v8::Local<v8::Function> callback = info[2].As<v8::Function>();
-    auto req = new NodeVitastorRequest(kv->cli, callback);
+    auto req = new NodeVitastorRequest(callback);
 
     kv->Ref();
     kv->dbw->open(inode_id, cfg, [kv, req](int res)
@@ -654,13 +570,11 @@ NAN_METHOD(NodeVitastorKV::Open)
 NAN_METHOD(NodeVitastorKV::Close)
 {
     TRACE("NodeVitastorKV::Close");
-    if (info.Length() < 1)
-        Nan::ThrowError("Not enough arguments to close(callback(err))");
 
     NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
 
     v8::Local<v8::Function> callback = info[0].As<v8::Function>();
-    auto req = new NodeVitastorRequest(kv->cli, callback);
+    auto req = new NodeVitastorRequest(callback);
 
     kv->Ref();
     kv->dbw->close([kv, req]()
@@ -677,8 +591,6 @@ NAN_METHOD(NodeVitastorKV::Close)
 NAN_METHOD(NodeVitastorKV::SetConfig)
 {
     TRACE("NodeVitastorKV::SetConfig");
-    if (info.Length() < 1)
-        Nan::ThrowError("Not enough arguments to set_config({ ...config })");
 
     NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
 
@@ -715,7 +627,7 @@ void NodeVitastorKV::get_impl(const Nan::FunctionCallbackInfo<v8::Value> & info,
     std::string key(*Nan::Utf8String(info[0].As<v8::String>()));
 
     v8::Local<v8::Function> callback = info[1].As<v8::Function>();
-    auto req = new NodeVitastorRequest(kv->cli, callback);
+    auto req = new NodeVitastorRequest(callback);
 
     kv->Ref();
     kv->dbw->get(key, [kv, req](int res, const std::string & value)
@@ -735,8 +647,6 @@ void NodeVitastorKV::get_impl(const Nan::FunctionCallbackInfo<v8::Value> & info,
 NAN_METHOD(NodeVitastorKV::Get)
 {
     TRACE("NodeVitastorKV::Get");
-    if (info.Length() < 2)
-        Nan::ThrowError("Not enough arguments to get(key, callback(err, value))");
     get_impl(info, false);
 }
 
@@ -744,8 +654,6 @@ NAN_METHOD(NodeVitastorKV::Get)
 NAN_METHOD(NodeVitastorKV::GetCached)
 {
     TRACE("NodeVitastorKV::GetCached");
-    if (info.Length() < 2)
-        Nan::ThrowError("Not enough arguments to get_cached(key, callback(err, value))");
     get_impl(info, true);
 }
 
@@ -764,12 +672,10 @@ static std::function<bool(int, const std::string &)> make_cas_callback(NodeVitas
     };
 }
 
-// set(key, value, callback(err), cas_compare(old_value)?)
+// set(key, value, callback(err), cas_compare(old_value))
 NAN_METHOD(NodeVitastorKV::Set)
 {
     TRACE("NodeVitastorKV::Set");
-    if (info.Length() < 3)
-        Nan::ThrowError("Not enough arguments to set(key, value, callback(err), cas_compare(old_value)?)");
 
     NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
 
@@ -778,13 +684,13 @@ NAN_METHOD(NodeVitastorKV::Set)
     std::string value(*Nan::Utf8String(info[1].As<v8::String>()));
 
     v8::Local<v8::Function> callback = info[2].As<v8::Function>();
-    NodeVitastorRequest *req = new NodeVitastorRequest(kv->cli, callback), *cas_req = NULL;
+    NodeVitastorRequest *req = new NodeVitastorRequest(callback), *cas_req = NULL;
 
     std::function<bool(int, const std::string &)> cas_cb;
     if (info.Length() > 3 && info[3]->IsObject())
     {
         v8::Local<v8::Function> cas_callback = info[3].As<v8::Function>();
-        cas_req = new NodeVitastorRequest(kv->cli, cas_callback);
+        cas_req = new NodeVitastorRequest(cas_callback);
         cas_cb = make_cas_callback(cas_req);
     }
 
@@ -803,12 +709,10 @@ NAN_METHOD(NodeVitastorKV::Set)
     }, cas_cb);
 }
 
-// del(key, callback(err), cas_compare(old_value)?)
+// del(key, callback(err), cas_compare(old_value))
 NAN_METHOD(NodeVitastorKV::Del)
 {
     TRACE("NodeVitastorKV::Del");
-    if (info.Length() < 2)
-        Nan::ThrowError("Not enough arguments to del(key, callback(err), cas_compare(old_value)?)");
 
     NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
 
@@ -816,13 +720,13 @@ NAN_METHOD(NodeVitastorKV::Del)
     std::string key(*Nan::Utf8String(info[0].As<v8::String>()));
 
     v8::Local<v8::Function> callback = info[1].As<v8::Function>();
-    NodeVitastorRequest *req = new NodeVitastorRequest(kv->cli, callback), *cas_req = NULL;
+    NodeVitastorRequest *req = new NodeVitastorRequest(callback), *cas_req = NULL;
 
     std::function<bool(int, const std::string &)> cas_cb;
     if (info.Length() > 2 && info[2]->IsObject())
     {
         v8::Local<v8::Function> cas_callback = info[2].As<v8::Function>();
-        cas_req = new NodeVitastorRequest(kv->cli, cas_callback);
+        cas_req = new NodeVitastorRequest(cas_callback);
         cas_cb = make_cas_callback(cas_req);
     }
 
@@ -858,14 +762,6 @@ NAN_METHOD(NodeVitastorKV::List)
     info.GetReturnValue().Set(Nan::NewInstance(cons, narg, args).ToLocalChecked());
 }
 
-/*NAN_METHOD(NodeVitastorKV::Destroy)
-{
-    TRACE("NodeVitastorKV::Destroy");
-    NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
-    if (!kv->dead)
-        kv->Unref();
-}*/
-
 //////////////////////////////////////////////////
 // NodeVitastorKVListing
 //////////////////////////////////////////////////
@@ -887,10 +783,12 @@ NAN_METHOD(NodeVitastorKVListing::Create)
 
     NodeVitastorKVListing *list = new NodeVitastorKVListing();
     list->kv = kv;
-    list->handle = list->kv->dbw->list_start(start_key);
+    {
+        std::unique_lock<std::mutex> lock(kv->cli->mu);
+        list->handle = list->kv->dbw->list_start(start_key);
+    }
 
     list->Wrap(info.This());
-    kv->Ref();
     info.GetReturnValue().Set(info.This());
 }
 
@@ -898,54 +796,35 @@ NodeVitastorKVListing::~NodeVitastorKVListing()
 {
     if (handle)
     {
+        std::unique_lock<std::mutex> lock(kv->cli->mu);
         kv->dbw->list_close(handle);
         handle = NULL;
     }
-    if (iter)
-    {
-        delete iter;
-        iter = NULL;
-    }
-    kv->Unref();
 }
 
-// next(callback(err, value)?)
+// next(callback(err, value))
 NAN_METHOD(NodeVitastorKVListing::Next)
 {
     TRACE("NodeVitastorKVListing::Next");
 
     NodeVitastorKVListing* list = Nan::ObjectWrap::Unwrap<NodeVitastorKVListing>(info.This());
 
-    if (info.Length() > 0)
+    v8::Local<v8::Function> callback = info[0].As<v8::Function>();
-    {
+    auto req = new NodeVitastorRequest(callback);
-        v8::Local<v8::Function> callback = info[0].As<v8::Function>();
-        if (list->iter)
-        {
-            delete list->iter;
-        }
-        list->iter = new NodeVitastorRequest(list->kv->cli, callback);
-    }
     if (!list->handle)
     {
         // Already closed
-        if (list->iter)
+        Nan::Callback nanCallback(Nan::New(req->callback));
-        {
+        v8::Local<v8::Value> args[1];
-            auto req = list->iter;
+        args[0] = Nan::New<v8::Int32>(-EINVAL);
-            list->iter = NULL;
+        nanCallback.Call(1, args, req);
-            Nan::Callback nanCallback(Nan::New(req->callback));
+        delete req;
-            v8::Local<v8::Value> args[1];
-            args[0] = Nan::New<v8::Int32>(-EINVAL);
-            nanCallback.Call(1, args, req);
-            delete req;
-        }
         return;
     }
 
     list->kv->Ref();
-    list->kv->dbw->list_next(list->handle, [list](int res, const std::string & key, const std::string & value)
+    list->kv->dbw->list_next(list->handle, [list, req](int res, const std::string & key, const std::string & value)
     {
-        auto req = list->iter;
-        list->iter = NULL;
         Nan::HandleScope scope;
         Nan::Callback nanCallback(Nan::New(req->callback));
         v8::Local<v8::Value> args[3];
@@ -953,10 +832,7 @@ NAN_METHOD(NodeVitastorKVListing::Next)
         args[1] = !res ? v8::Local<v8::Value>(Nan::New<v8::String>(key).ToLocalChecked()) : v8::Local<v8::Value>(Nan::Null());
         args[2] = !res ? v8::Local<v8::Value>(Nan::New<v8::String>(value).ToLocalChecked()) : v8::Local<v8::Value>(Nan::Null());
         nanCallback.Call(3, args, req);
-        if (list->iter)
+        delete req;
-            delete req;
-        else
-            list->iter = req;
         list->kv->Unref();
     });
 }
@@ -970,12 +846,8 @@ NAN_METHOD(NodeVitastorKVListing::Close)
 
     if (list->handle)
     {
+        std::unique_lock<std::mutex> lock(list->kv->cli->mu);
         list->kv->dbw->list_close(list->handle);
         list->handle = NULL;
     }
-    if (list->iter)
-    {
-        delete list->iter;
-        list->iter = NULL;
-    }
 }

node-binding/client.h

@@ -4,6 +4,8 @@
 #ifndef NODE_VITASTOR_CLIENT_H
 #define NODE_VITASTOR_CLIENT_H
 
+#include <mutex>
+
 #include <nan.h>
 #include <vitastor_c.h>
 #include <vitastor_kv.h>
@@ -17,7 +19,7 @@ public:
     static NAN_METHOD(Create);
     // read(pool, inode, offset, len, callback(err, buffer, version))
     static NAN_METHOD(Read);
-    // write(pool, inode, offset, buf: Buffer | Buffer[], { version }?, callback(err))
+    // write(pool, inode, offset, buffer, { version }?, callback(err))
     static NAN_METHOD(Write);
     // sync(callback(err))
     static NAN_METHOD(Sync);
@@ -32,6 +34,8 @@ private:
     vitastor_c *c = NULL;
     int eventfd = -1;
     uv_poll_t poll_watcher;
+    // FIXME: Is it really needed?
+    std::mutex mu;
 
     NodeVitastor();
 
@@ -40,9 +44,6 @@ private:
     static void on_write_finish(void *opaque, long retval);
     static void on_read_bitmap_finish(void *opaque, long retval, uint8_t *bitmap);
 
-    NodeVitastorRequest* get_read_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos);
-    NodeVitastorRequest* get_write_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos);
-
     friend class NodeVitastorImage;
     friend class NodeVitastorKV;
     friend class NodeVitastorKVListing;
@@ -55,7 +56,7 @@ public:
     static NAN_METHOD(Create);
     // read(offset, len, callback(err, buffer, version))
     static NAN_METHOD(Read);
-    // write(offset, buf: Buffer | Buffer[], { version }?, callback(err))
+    // write(offset, buffer, { version }?, callback(err))
     static NAN_METHOD(Write);
     // sync(callback(err))
     static NAN_METHOD(Sync);
@@ -73,6 +74,8 @@ private:
     std::vector<NodeVitastorRequest*> on_init;
     Nan::Persistent<v8::Object> cliObj;
 
+    NodeVitastorImage();
+
     static void on_watch_start(void *opaque, long retval);
     void exec_request(NodeVitastorRequest *req);
     void exec_or_wait(NodeVitastorRequest *req);
@@ -83,7 +86,7 @@ class NodeVitastorKV: public Nan::ObjectWrap
 public:
     // constructor(node_vitastor)
     static NAN_METHOD(Create);
-    // open(pool_id, inode_num, { ...config }, callback(err))
+    // open(inode_id, { ...config }, callback(err))
     static NAN_METHOD(Open);
     // set_config({ ...config })
     static NAN_METHOD(SetConfig);
@@ -95,9 +98,9 @@ public:
     static NAN_METHOD(Get);
     // get_cached(key, callback(err, value))
     static NAN_METHOD(GetCached);
-    // set(key, value, callback(err), cas_compare(old_value)?)
+    // set(key, value, callback(err), cas_compare(old_value))
     static NAN_METHOD(Set);
-    // del(key, callback(err), cas_compare(old_value)?)
+    // del(key, callback(err), cas_compare(old_value))
     static NAN_METHOD(Del);
     // list(start_key?)
     static NAN_METHOD(List);
@@ -110,6 +113,8 @@ private:
     NodeVitastor *cli = NULL;
     vitastorkv_dbw_t *dbw = NULL;
 
+    NodeVitastorKV();
+
     static void get_impl(const Nan::FunctionCallbackInfo<v8::Value> & info, bool allow_cache);
 
     friend class NodeVitastorKVListing;
@@ -120,7 +125,7 @@ class NodeVitastorKVListing: public Nan::ObjectWrap
 public:
     // constructor(node_vitastor_kv, start_key?)
     static NAN_METHOD(Create);
-    // next(callback(err, value)?)
+    // next(callback(err, value))
     static NAN_METHOD(Next);
     // close()
     static NAN_METHOD(Close);
@@ -130,7 +135,8 @@ public:
 private:
     NodeVitastorKV *kv = NULL;
     void *handle = NULL;
-    NodeVitastorRequest *iter = NULL;
+
+    NodeVitastorKVListing();
 };
 
 #endif

node-binding/index.js

@@ -1 +0,0 @@
-module.exports = require('bindings')('addon.node');

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '1.8.0'
+VITASTOR_VERSION = '1.7.1'
 
 LOG = logging.getLogger(__name__)
 

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.8.0
+Version:        1.7.1
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.8.0.el7.tar.gz
+Source0:        vitastor-1.7.1.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.8.0
+Version:        1.7.1
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.8.0.el8.tar.gz
+Source0:        vitastor-1.7.1.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.8.0
+Version:        1.7.1
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.8.0.el9.tar.gz
+Source0:        vitastor-1.7.1.el9.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -19,7 +19,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVITASTOR_VERSION="1.8.0")
+add_definitions(-DVITASTOR_VERSION="1.7.1")
 add_definitions(-D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/client/cluster_client.cpp

@@ -1286,11 +1286,7 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
         if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
         {
             copy_part_bitmap(op, part);
-            if (op->inode == op->cur_inode)
+            op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0;
-            {
-                // Read only returns the version of the uppermost layer
-                op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0;
-            }
         }
         else if (op->opcode == OSD_OP_WRITE)
         {

src/client/etcd_state_client.cpp

@@ -730,7 +730,7 @@ void etcd_state_client_t::clean_nonexistent_pgs()
             {
                 if (!pg_cfg.state_exists)
                 {
-                    if (this->log_level > 3 && (pg_cfg.cur_primary || pg_cfg.cur_state))
+                    if (this->log_level > 3)
                     {
                         fprintf(stderr, "PG %u/%u primary OSD disappeared after reload, forgetting it\n", pool_item.first, pg_it->first);
                     }
@@ -740,7 +740,7 @@ void etcd_state_client_t::clean_nonexistent_pgs()
                 }
                 if (!pg_cfg.history_exists)
                 {
-                    if (this->log_level > 3 && (pg_cfg.target_history.size() || pg_cfg.all_peers.size() || pg_cfg.epoch || pg_cfg.next_scrub))
+                    if (this->log_level > 3)
                     {
                         fprintf(stderr, "PG %u/%u history disappeared after reload, forgetting it\n", pool_item.first, pg_it->first);
                     }

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 1.8.0
+Version: 1.7.1
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/nfs/nfs_proxy.cpp

@@ -522,15 +522,17 @@ void nfs_proxy_t::do_accept(int listen_fd)
         {
             cli->proc_table.insert(fn);
         }
-        rpc_clients[nfs_fd] = cli;
         epmgr->tfd->set_fd_handler(nfs_fd, true, [cli](int nfs_fd, int epoll_events)
         {
             // Handle incoming event
             if (epoll_events & EPOLLRDHUP)
             {
-                if (cli->parent->trace)
+                auto parent = cli->parent;
+                if (parent->trace)
                     fprintf(stderr, "Client %d disconnected\n", nfs_fd);
                 cli->stop();
+                parent->active_connections--;
+                parent->check_exit();
                 return;
             }
             cli->epoll_events |= epoll_events;
@@ -693,8 +695,6 @@ void nfs_client_t::handle_read(int result)
                         frag_num++;
                     }
                 }
-                // Increase client refcount while the RPC call is being processed
-                refs++;
                 // Handle full message
                 int referenced = handle_rpc_message(cur_buffer.buf, data+4, wanted-4*fragments);
                 cur_buffer.refs += referenced ? 1 : 0;
@@ -779,13 +779,9 @@ void nfs_client_t::stop()
     stopped = true;
     if (refs <= 0)
     {
-        auto parent = this->parent;
-        parent->rpc_clients.erase(nfs_fd);
-        parent->active_connections--;
         parent->epmgr->tfd->set_fd_handler(nfs_fd, true, NULL);
         close(nfs_fd);
         delete this;
-        parent->check_exit();
     }
 }
 
@@ -838,10 +834,6 @@ void nfs_client_t::handle_send(int result)
                         }
                     }
                     free(rop);
-                    if (deref())
-                    {
-                        return;
-                    }
                 }
                 result -= iov.iov_len;
                 done++;
@@ -1064,11 +1056,6 @@ int nfs_client_t::handle_rpc_message(void *base_buf, void *msg_buf, uint32_t msg
 
 void nfs_proxy_t::daemonize()
 {
-    // Stop all clients because client I/O sometimes breaks during daemonize
-    // I.e. the new process stops receiving events on the old FD
-    // It doesn't happen if we call sleep(1) here, but we don't want to call sleep(1)...
-    for (auto & clp: rpc_clients)
-        clp.second->stop();
     if (fork())
         exit(0);
     setsid();

src/nfs/nfs_proxy.h

@@ -21,7 +21,6 @@ class cli_tool_t;
 
 struct kv_fs_state_t;
 struct block_fs_state_t;
-class nfs_client_t;
 
 class nfs_proxy_t
 {
@@ -55,7 +54,6 @@ public:
     vitastorkv_dbw_t *db = NULL;
     kv_fs_state_t *kvfs = NULL;
     block_fs_state_t *blockfs = NULL;
-    std::map<int, nfs_client_t*> rpc_clients;
 
     std::vector<XDR*> xdr_pool;
 

tests/run_3osds.sh

@@ -30,10 +30,10 @@ wait_etcd
 
 if [ "$IMMEDIATE_COMMIT" != "" ]; then
     NO_SAME="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all --log_level 10 --etcd_stats_interval 5"
-    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"immediate_commit":"all","client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}'
+    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"immediate_commit":"all","client_writeback_allowed":true,"client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}'
 else
     NO_SAME="--journal_sector_buffer_count 1024 --log_level 10 --etcd_stats_interval 5 --min_flusher_count 16"
-    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"immediate_commit":"none","client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}'
+    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"immediate_commit":"none","client_writeback_allowed":true,"client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}'
 fi
 
 start_osd_on()