Release 1.8.0

Bugfix release, would be 1.7.2, but etcd layout changes mandate it to be 1.8.0. :-)

- Change etcd layout: /config/pgs is now /pg/config, /pg/stats/* is now /pgstats/*.
  This is required to fix a rare PG history tracking issue caused by non-atomic
  delivery of etcd events sometimes resulting in `incomplete` objects in EC pools
  after mass OSD restarts. Upgrading can be performed freely, downgrade requires
  additional action: [1.8.0 to 1.7.1](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/docs/usage/admin.en.md#1-8-0-to-1-7-1)
- Fix a rare client hang on PG primary OSD switch
- Fix vitastor-nfs started using mount command sometimes not stopping automatically after unmount
- Fix vitastor-nfs mounts started using mount command sometimes hanging after daemonizing
- Fix merge/flatten into a pool with different object size (image migration between pools case)
- Do not print extra "PG disappeared after reload" verbose log messages for non-existing PGs
- Fix clustered Antietcd support and persistence filter
- Do not try to purge the same OSD multiple times if its multiple devices are passed to purge
- Various node.js binding fixes

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "1.7.1")
+set(VITASTOR_VERSION "1.8.0")
 
 add_subdirectory(src)

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v1.7.1
+VITASTOR_VERSION ?= v1.8.0
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v1.7.1
+          image: vitalif/vitastor-csi:v1.8.0
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v1.7.1
+          image: vitalif/vitastor-csi:v1.8.0
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "1.7.1"
+    vitastorCSIDriverVersion = "1.8.0"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (1.7.1-1) unstable; urgency=medium
+vitastor (1.8.0-1) unstable; urgency=medium
 
   * Bugfixes
 

docs/installation/proxmox.en.md

@@ -17,10 +17,10 @@ To enable Vitastor support in Proxmox Virtual Environment (6.4-8.1 are supported
 - Restart pvedaemon: `systemctl restart pvedaemon`
 
 `/etc/pve/storage.cfg` example (the only required option is vitastor_pool, all others
-are listed below with their default values):
+are listed below with their default values; `vitastor_ssd` is Proxmox storage pool id):
 
 ```
-vitastor: vitastor
+vitastor: vitastor_ssd
     # pool to put new images into
     vitastor_pool testpool
     # path to the configuration file

docs/installation/proxmox.ru.md

@@ -16,10 +16,10 @@
 - Перезапустите демон Proxmox: `systemctl restart pvedaemon`
 
 Пример `/etc/pve/storage.cfg` (единственная обязательная опция - vitastor_pool, все остальные
-перечислены внизу для понимания значений по умолчанию):
+перечислены внизу для понимания значений по умолчанию; `vitastor_ssd` - имя хранилища в Proxmox):
 
 ```
-vitastor: vitastor
+vitastor: vitastor_ssd
     # Пул, в который будут помещаться образы дисков
     vitastor_pool testpool
     # Путь к файлу конфигурации

docs/usage/cli.en.md

@@ -307,7 +307,7 @@ OSD  PARENT            UP    SIZE  USED%    TAGS          WEIGHT  BLOCK  BITMAP
 
 `vitastor-cli modify-osd [--tags tag1,tag2,...] [--reweight <number>] [--noout true/false] <osd_number>`
 
-Set OSD reweight, tags or noout flag. See detail description in [OSD config documentation](../config.pool.en.md#osd-settings).
+Set OSD reweight, tags or noout flag. See detail description in [OSD config documentation](../config/pool.en.md#osd-settings).
 
 ## pg-list
 

docs/usage/cli.ru.md

@@ -324,7 +324,7 @@ OSD  PARENT            UP    SIZE  USED%    TAGS          WEIGHT  BLOCK  BITMAP
 
 `vitastor-cli modify-osd [--tags tag1,tag2,...] [--reweight <number>] [--noout true/false] <osd_number>`
 
-Установить вес OSD, теги или флаг noout. Смотрите подробное описание в [документации настроек OSD](../config.pool.ru.md#настройки-osd).
+Установить вес OSD, теги или флаг noout. Смотрите подробное описание в [документации настроек OSD](../config/pool.ru.md#настройки-osd).
 
 ## pg-list
 

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "1.7.1",
+  "version": "1.8.0",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {

node-binding/addon.cc

@@ -24,7 +24,7 @@ NAN_MODULE_INIT(InitAddon)
 
     tpl = Nan::New<v8::FunctionTemplate>(NodeVitastorImage::Create);
     tpl->SetClassName(Nan::New("Image").ToLocalChecked());
-    tpl->InstanceTemplate()->SetInternalFieldCount(2);
+    tpl->InstanceTemplate()->SetInternalFieldCount(1);
 
     Nan::SetPrototypeMethod(tpl, "read", NodeVitastorImage::Read);
     Nan::SetPrototypeMethod(tpl, "write", NodeVitastorImage::Write);
@@ -67,7 +67,7 @@ NAN_MODULE_INIT(InitAddon)
 
     tpl = Nan::New<v8::FunctionTemplate>(NodeVitastorKVListing::Create);
     tpl->SetClassName(Nan::New("KVListing").ToLocalChecked());
-    tpl->InstanceTemplate()->SetInternalFieldCount(2);
+    tpl->InstanceTemplate()->SetInternalFieldCount(1);
 
     Nan::SetPrototypeMethod(tpl, "next", NodeVitastorKVListing::Next);
     Nan::SetPrototypeMethod(tpl, "close", NodeVitastorKVListing::Close);

node-binding/binding.gyp

@@ -13,7 +13,8 @@
         '<!(pkg-config --cflags vitastor)'
       ],
       'libraries': [
-        '<!(pkg-config --libs vitastor)'
+        '<!(pkg-config --libs vitastor)',
+        '-lvitastor_kv'
       ]
     }
   ]

node-binding/client.cc

@@ -18,12 +18,15 @@
 class NodeVitastorRequest: public Nan::AsyncResource
 {
 public:
-    NodeVitastorRequest(v8::Local<v8::Function> cb): Nan::AsyncResource("NodeVitastorRequest")
+    NodeVitastorRequest(NodeVitastor *cli, v8::Local<v8::Function> cb): Nan::AsyncResource("NodeVitastorRequest")
     {
+        this->cli = cli;
         callback.Reset(cb);
     }
 
     iovec iov;
+    std::vector<iovec> iov_list;
+    NodeVitastor *cli = NULL;
     NodeVitastorImage *img = NULL;
     int op = 0;
     uint64_t offset = 0, len = 0, version = 0;
@@ -31,6 +34,13 @@ public:
     Nan::Persistent<v8::Function> callback;
 };
 
+static uint64_t get_ui64(const v8::Local<v8::Value> & val)
+{
+    if (val->IsBigInt())
+        return val->ToBigInt(Nan::GetCurrentContext()).ToLocalChecked()->Uint64Value();
+    return Nan::To<int64_t>(val).FromJust();
+}
+
 //////////////////////////////////////////////////
 // NodeVitastor
 //////////////////////////////////////////////////
@@ -43,6 +53,7 @@ NodeVitastor::NodeVitastor(): Nan::ObjectWrap()
 
 NodeVitastor::~NodeVitastor()
 {
+    TRACE("NodeVitastor: destructor");
     uv_poll_stop(&poll_watcher);
     vitastor_c_destroy(c);
     c = NULL;
@@ -74,9 +85,10 @@ NAN_METHOD(NodeVitastor::Create)
     if (res >= 0)
     {
         cli->eventfd = res;
-        res = uv_poll_init_socket(uv_default_loop(), &cli->poll_watcher, cli->eventfd);
+        res = uv_poll_init(uv_default_loop(), &cli->poll_watcher, cli->eventfd);
         if (res >= 0)
             res = uv_poll_start(&cli->poll_watcher, UV_READABLE, on_io_readable);
+        on_io_readable(&cli->poll_watcher, 0, UV_READABLE);
     }
     if (res < 0)
     {
@@ -97,15 +109,14 @@ void NodeVitastor::on_io_readable(uv_poll_t* handle, int status, int revents)
     if (revents & UV_READABLE)
     {
         NodeVitastor* self = (NodeVitastor*)handle->data;
-        std::unique_lock<std::mutex> lock(self->mu);
         vitastor_c_uring_handle_events(self->c);
     }
 }
 
-static NodeVitastorRequest* getReadRequest(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos)
+NodeVitastorRequest* NodeVitastor::get_read_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos)
 {
-    uint64_t offset = Nan::To<int64_t>(info[argpos+0]).FromJust();
-    uint64_t len = Nan::To<int64_t>(info[argpos+1]).FromJust();
+    uint64_t offset = get_ui64(info[argpos+0]);
+    uint64_t len = get_ui64(info[argpos+1]);
     uint8_t *buf = (uint8_t*)malloc(len);
     if (!buf)
     {
@@ -113,7 +124,7 @@ static NodeVitastorRequest* getReadRequest(const Nan::FunctionCallbackInfo<v8::V
         return NULL;
     }
     v8::Local<v8::Function> callback = info[argpos+2].As<v8::Function>();
-    auto req = new NodeVitastorRequest(callback);
+    auto req = new NodeVitastorRequest(this, callback);
 
     req->offset = offset;
     req->len = len;
@@ -126,73 +137,101 @@ static NodeVitastorRequest* getReadRequest(const Nan::FunctionCallbackInfo<v8::V
 NAN_METHOD(NodeVitastor::Read)
 {
     TRACE("NodeVitastor::Read");
+    if (info.Length() < 5)
+        Nan::ThrowError("Not enough arguments to read(pool, inode, offset, len, callback(err, buffer, version))");
 
     NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
 
-    uint64_t pool = Nan::To<int64_t>(info[0]).FromJust();
-    uint64_t inode = Nan::To<int64_t>(info[1]).FromJust();
+    uint64_t pool = get_ui64(info[0]);
+    uint64_t inode = get_ui64(info[1]);
 
-    auto req = getReadRequest(info, 2);
+    auto req = self->get_read_request(info, 2);
 
-    std::unique_lock<std::mutex> lock(self->mu);
+    self->Ref();
     vitastor_c_read(self->c, ((pool << (64-POOL_ID_BITS)) | inode), req->offset, req->len, &req->iov, 1, on_read_finish, req);
 }
 
-static NodeVitastorRequest* getWriteRequest(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos)
+NodeVitastorRequest* NodeVitastor::get_write_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos)
 {
-    uint64_t offset = Nan::To<int64_t>(info[argpos+0]).FromJust();
-    char *buf = node::Buffer::Data(info[argpos+1]);
-    uint64_t len = node::Buffer::Length(info[argpos+1]);
+    uint64_t offset = get_ui64(info[argpos+0]);
+    const auto & bufarg = info[argpos+1];
     uint64_t version = 0;
-
-    if (!info[argpos+2].IsEmpty() && info[argpos+2]->IsObject())
+    if (!info[argpos+2].IsEmpty() &&
+        !info[argpos+2]->IsFunction() &&
+        info[argpos+2]->IsObject())
     {
         auto key = Nan::New<v8::String>("version").ToLocalChecked();
         auto params = info[argpos+2].As<v8::Object>();
         auto versionObj = Nan::Get(params, key).ToLocalChecked();
         if (!versionObj.IsEmpty())
-            version = Nan::To<int64_t>(versionObj).FromJust();
+            version = get_ui64(versionObj);
         argpos++;
     }
 
     v8::Local<v8::Function> callback = info[argpos+2].As<v8::Function>();
-    auto req = new NodeVitastorRequest(callback);
+    auto req = new NodeVitastorRequest(this, callback);
 
     req->offset = offset;
-    req->len = len;
     req->version = version;
-    req->iov = { .iov_base = buf, .iov_len = req->len };
+
+    if (bufarg->IsArray())
+    {
+        auto buffers = bufarg.As<v8::Array>();
+        req->len = 0;
+        for (uint32_t i = 0; i < buffers->Length(); i++)
+        {
+            auto buffer_obj = Nan::Get(buffers, i).ToLocalChecked();
+            char *buf = node::Buffer::Data(buffer_obj);
+            uint64_t len = node::Buffer::Length(buffer_obj);
+            req->iov_list.push_back({ .iov_base = buf, .iov_len = len });
+            req->len += len;
+        }
+    }
+    else
+    {
+        char *buf = node::Buffer::Data(bufarg);
+        uint64_t len = node::Buffer::Length(bufarg);
+        req->iov = { .iov_base = buf, .iov_len = len };
+        req->len = len;
+    }
 
     return req;
 }
 
-// write(pool, inode, offset, buffer, { version }?, callback(err))
+// write(pool, inode, offset, buf: Buffer | Buffer[], { version }?, callback(err))
 NAN_METHOD(NodeVitastor::Write)
 {
     TRACE("NodeVitastor::Write");
+    if (info.Length() < 5)
+        Nan::ThrowError("Not enough arguments to write(pool, inode, offset, buf: Buffer | Buffer[], { version }?, callback(err))");
 
     NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
 
-    uint64_t pool = Nan::To<int64_t>(info[0]).FromJust();
-    uint64_t inode = Nan::To<int64_t>(info[1]).FromJust();
+    uint64_t pool = get_ui64(info[0]);
+    uint64_t inode = get_ui64(info[1]);
 
-    auto req = getWriteRequest(info, 2);
+    auto req = self->get_write_request(info, 2);
 
-    std::unique_lock<std::mutex> lock(self->mu);
-    vitastor_c_write(self->c, ((pool << (64-POOL_ID_BITS)) | inode), req->offset, req->len, req->version, &req->iov, 1, on_write_finish, req);
+    self->Ref();
+    vitastor_c_write(self->c, ((pool << (64-POOL_ID_BITS)) | inode), req->offset, req->len, req->version,
+        req->iov_list.size() ? req->iov_list.data() : &req->iov,
+        req->iov_list.size() ? req->iov_list.size() : 1,
+        on_write_finish, req);
 }
 
 // sync(callback(err))
 NAN_METHOD(NodeVitastor::Sync)
 {
     TRACE("NodeVitastor::Sync");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to sync(callback(err))");
 
     NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
 
     v8::Local<v8::Function> callback = info[0].As<v8::Function>();
-    auto req = new NodeVitastorRequest(callback);
+    auto req = new NodeVitastorRequest(self, callback);
 
-    std::unique_lock<std::mutex> lock(self->mu);
+    self->Ref();
     vitastor_c_sync(self->c, on_write_finish, req);
 }
 
@@ -200,17 +239,20 @@ NAN_METHOD(NodeVitastor::Sync)
 NAN_METHOD(NodeVitastor::ReadBitmap)
 {
     TRACE("NodeVitastor::ReadBitmap");
+    if (info.Length() < 6)
+        Nan::ThrowError("Not enough arguments to read_bitmap(pool, inode, offset, len, with_parents, callback(err, bitmap_buffer))");
 
     NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
 
-    uint64_t pool = Nan::To<int64_t>(info[0]).FromJust();
-    uint64_t inode = Nan::To<int64_t>(info[1]).FromJust();
-    uint64_t offset = Nan::To<int64_t>(info[2]).FromJust();
-    uint64_t len = Nan::To<int64_t>(info[3]).FromJust();
+    uint64_t pool = get_ui64(info[0]);
+    uint64_t inode = get_ui64(info[1]);
+    uint64_t offset = get_ui64(info[2]);
+    uint64_t len = get_ui64(info[3]);
     bool with_parents = Nan::To<bool>(info[4]).FromJust();
     v8::Local<v8::Function> callback = info[5].As<v8::Function>();
 
-    auto req = new NodeVitastorRequest(callback);
+    auto req = new NodeVitastorRequest(self, callback);
+    self->Ref();
     vitastor_c_read_bitmap(self->c, ((pool << (64-POOL_ID_BITS)) | inode), offset, len, with_parents, on_read_bitmap_finish, req);
 }
 
@@ -227,6 +269,7 @@ static void on_error(NodeVitastorRequest *req, Nan::Callback & nanCallback, long
 
 void NodeVitastor::on_read_finish(void *opaque, long retval, uint64_t version)
 {
+    TRACE("NodeVitastor::on_read_finish");
     Nan::HandleScope scope;
     NodeVitastorRequest *req = (NodeVitastorRequest *)opaque;
     Nan::Callback nanCallback(Nan::New(req->callback));
@@ -235,7 +278,7 @@ void NodeVitastor::on_read_finish(void *opaque, long retval, uint64_t version)
         free(req->iov.iov_base);
         nanCallback.Call(0, NULL, req);
     }
-    else if (retval < 0)
+    else if (retval < 0 || (uint64_t)retval != req->len)
     {
         free(req->iov.iov_base);
         on_error(req, nanCallback, retval);
@@ -248,20 +291,33 @@ void NodeVitastor::on_read_finish(void *opaque, long retval, uint64_t version)
         args[2] = v8::BigInt::NewFromUnsigned(v8::Isolate::GetCurrent(), version);
         nanCallback.Call(3, args, req);
     }
+    req->cli->Unref();
     delete req;
 }
 
 void NodeVitastor::on_write_finish(void *opaque, long retval)
 {
+    TRACE("NodeVitastor::on_write_finish");
     Nan::HandleScope scope;
     NodeVitastorRequest *req = (NodeVitastorRequest *)opaque;
     Nan::Callback nanCallback(Nan::New(req->callback));
-    on_error(req, nanCallback, retval);
+    if (retval < 0 || (uint64_t)retval != req->len)
+    {
+        on_error(req, nanCallback, retval);
+    }
+    else
+    {
+        v8::Local<v8::Value> args[1];
+        args[0] = Nan::Null();
+        nanCallback.Call(1, args, req);
+    }
+    req->cli->Unref();
     delete req;
 }
 
 void NodeVitastor::on_read_bitmap_finish(void *opaque, long retval, uint8_t *bitmap)
 {
+    TRACE("NodeVitastor::on_read_bitmap_finish");
     Nan::HandleScope scope;
     NodeVitastorRequest *req = (NodeVitastorRequest *)opaque;
     Nan::Callback nanCallback(Nan::New(req->callback));
@@ -276,6 +332,7 @@ void NodeVitastor::on_read_bitmap_finish(void *opaque, long retval, uint8_t *bit
         args[1] = Nan::NewBuffer((char*)bitmap, (retval+7)/8).ToLocalChecked();
         nanCallback.Call(2, args, req);
     }
+    req->cli->Unref();
     delete req;
 }
 
@@ -291,21 +348,23 @@ void NodeVitastor::on_read_bitmap_finish(void *opaque, long retval, uint8_t *bit
 NAN_METHOD(NodeVitastorImage::Create)
 {
     TRACE("NodeVitastorImage::Create");
+    if (info.Length() < 2)
+        Nan::ThrowError("Not enough arguments to Image(client, name)");
 
     v8::Local<v8::Object> parent = info[0].As<v8::Object>();
     std::string name = std::string(*Nan::Utf8String(info[1].As<v8::String>()));
     NodeVitastor *cli = Nan::ObjectWrap::Unwrap<NodeVitastor>(parent);
 
     NodeVitastorImage *img = new NodeVitastorImage();
+    img->Wrap(info.This());
+
     img->cli = cli;
     img->name = name;
 
     img->Ref();
     cli->Ref();
-    std::unique_lock<std::mutex> lock(cli->mu);
     vitastor_c_watch_inode(cli->c, (char*)img->name.c_str(), on_watch_start, img);
 
-    img->Wrap(info.This());
     info.GetReturnValue().Set(info.This());
 }
 
@@ -323,10 +382,12 @@ NodeVitastorImage::~NodeVitastorImage()
 NAN_METHOD(NodeVitastorImage::Read)
 {
     TRACE("NodeVitastorImage::Read");
+    if (info.Length() < 3)
+        Nan::ThrowError("Not enough arguments to read(offset, len, callback(err, buffer, version))");
 
     NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
 
-    auto req = getReadRequest(info, 0);
+    auto req = img->cli->get_read_request(info, 0);
     req->img = img;
     req->op = NODE_VITASTOR_READ;
 
@@ -337,24 +398,29 @@ NAN_METHOD(NodeVitastorImage::Read)
 NAN_METHOD(NodeVitastorImage::Write)
 {
     TRACE("NodeVitastorImage::Write");
+    if (info.Length() < 3)
+        Nan::ThrowError("Not enough arguments to write(offset, buffer, { version }?, callback(err))");
 
     NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
 
-    auto req = getWriteRequest(info, 0);
+    auto req = img->cli->get_write_request(info, 0);
     req->img = img;
     req->op = NODE_VITASTOR_WRITE;
 
     img->exec_or_wait(req);
 }
 
+// sync(callback(err))
 NAN_METHOD(NodeVitastorImage::Sync)
 {
     TRACE("NodeVitastorImage::Sync");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to sync(callback(err))");
 
     NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
 
     v8::Local<v8::Function> callback = info[0].As<v8::Function>();
-    auto req = new NodeVitastorRequest(callback);
+    auto req = new NodeVitastorRequest(img->cli, callback);
     req->img = img;
     req->op = NODE_VITASTOR_SYNC;
 
@@ -365,15 +431,17 @@ NAN_METHOD(NodeVitastorImage::Sync)
 NAN_METHOD(NodeVitastorImage::ReadBitmap)
 {
     TRACE("NodeVitastorImage::ReadBitmap");
+    if (info.Length() < 4)
+        Nan::ThrowError("Not enough arguments to read_bitmap(offset, len, with_parents, callback(err, bitmap_buffer))");
 
     NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
 
-    uint64_t offset = Nan::To<int64_t>(info[0]).FromJust();
-    uint64_t len = Nan::To<int64_t>(info[1]).FromJust();
+    uint64_t offset = get_ui64(info[0]);
+    uint64_t len = get_ui64(info[1]);
     bool with_parents = Nan::To<bool>(info[2]).FromJust();
     v8::Local<v8::Function> callback = info[3].As<v8::Function>();
 
-    auto req = new NodeVitastorRequest(callback);
+    auto req = new NodeVitastorRequest(img->cli, callback);
     req->img = img;
     req->op = NODE_VITASTOR_READ_BITMAP;
     req->offset = offset;
@@ -383,14 +451,17 @@ NAN_METHOD(NodeVitastorImage::ReadBitmap)
     img->exec_or_wait(req);
 }
 
+// get_info(callback({ num, name, size, parent_id?, readonly?, meta?, mod_revision, block_size, bitmap_granularity, immediate_commit }))
 NAN_METHOD(NodeVitastorImage::GetInfo)
 {
-    TRACE("NodeVitastorImage::Sync");
+    TRACE("NodeVitastorImage::GetInfo");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to get_info(callback({ num, name, size, parent_id?, readonly?, meta?, mod_revision, block_size, bitmap_granularity, immediate_commit }))");
 
     NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
 
     v8::Local<v8::Function> callback = info[0].As<v8::Function>();
-    auto req = new NodeVitastorRequest(callback);
+    auto req = new NodeVitastorRequest(img->cli, callback);
     req->img = img;
     req->op = NODE_VITASTOR_GET_INFO;
 
@@ -412,21 +483,26 @@ void NodeVitastorImage::exec_or_wait(NodeVitastorRequest *req)
 
 void NodeVitastorImage::exec_request(NodeVitastorRequest *req)
 {
-    std::unique_lock<std::mutex> lock(cli->mu);
     if (req->op == NODE_VITASTOR_READ)
     {
         uint64_t ino = vitastor_c_inode_get_num(watch);
+        cli->Ref();
         vitastor_c_read(cli->c, ino, req->offset, req->len, &req->iov, 1, NodeVitastor::on_read_finish, req);
     }
     else if (req->op == NODE_VITASTOR_WRITE)
     {
         uint64_t ino = vitastor_c_inode_get_num(watch);
-        vitastor_c_write(cli->c, ino, req->offset, req->len, req->version, &req->iov, 1, NodeVitastor::on_write_finish, req);
+        cli->Ref();
+        vitastor_c_write(cli->c, ino, req->offset, req->len, req->version,
+            req->iov_list.size() ? req->iov_list.data() : &req->iov,
+            req->iov_list.size() ? req->iov_list.size() : 1,
+            NodeVitastor::on_write_finish, req);
     }
     else if (req->op == NODE_VITASTOR_SYNC)
     {
         uint64_t ino = vitastor_c_inode_get_num(watch);
         uint32_t imm = vitastor_c_inode_get_immediate_commit(cli->c, ino);
+        cli->Ref();
         if (imm != IMMEDIATE_ALL)
         {
             vitastor_c_sync(cli->c, NodeVitastor::on_write_finish, req);
@@ -439,6 +515,7 @@ void NodeVitastorImage::exec_request(NodeVitastorRequest *req)
     else if (req->op == NODE_VITASTOR_READ_BITMAP)
     {
         uint64_t ino = vitastor_c_inode_get_num(watch);
+        cli->Ref();
         vitastor_c_read_bitmap(cli->c, ino, req->offset, req->len, req->with_parents, NodeVitastor::on_read_bitmap_finish, req);
     }
     else if (req->op == NODE_VITASTOR_GET_INFO)
@@ -508,10 +585,14 @@ void NodeVitastorImage::on_watch_start(void *opaque, long retval)
 // NodeVitastorKV
 //////////////////////////////////////////////////
 
+Nan::Persistent<v8::Function> NodeVitastorKV::listing_class;
+
 // constructor(node_vitastor)
 NAN_METHOD(NodeVitastorKV::Create)
 {
     TRACE("NodeVitastorKV::Create");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to new KV(client)");
 
     v8::Local<v8::Object> parent = info[0].As<v8::Object>();
     NodeVitastor *cli = Nan::ObjectWrap::Unwrap<NodeVitastor>(parent);
@@ -519,29 +600,32 @@ NAN_METHOD(NodeVitastorKV::Create)
     NodeVitastorKV *kv = new NodeVitastorKV();
     kv->cli = cli;
     {
-        std::unique_lock<std::mutex> lock(cli->mu);
         kv->dbw = new vitastorkv_dbw_t((cluster_client_t*)vitastor_c_get_internal_client(cli->c));
     }
 
     kv->Wrap(info.This());
+    cli->Ref();
     info.GetReturnValue().Set(info.This());
 }
 
 NodeVitastorKV::~NodeVitastorKV()
 {
     delete dbw;
+    cli->Unref();
 }
 
-// open(inode_id, { ...config }, callback(err))
+// open(pool_id, inode_num, { ...config }, callback(err))
 NAN_METHOD(NodeVitastorKV::Open)
 {
     TRACE("NodeVitastorKV::Open");
+    if (info.Length() < 4)
+        Nan::ThrowError("Not enough arguments to open(pool_id, inode_num, { ...config }, callback(err))");
 
     NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
 
-    uint64_t inode_id = Nan::To<int64_t>(info[0]).FromJust();
+    uint64_t inode_id = INODE_WITH_POOL(get_ui64(info[0]), get_ui64(info[1]));
 
-    v8::Local<v8::Object> jsParams = info[1].As<v8::Object>();
+    v8::Local<v8::Object> jsParams = info[2].As<v8::Object>();
     v8::Local<v8::Array> keys = Nan::GetOwnPropertyNames(jsParams).ToLocalChecked();
     std::map<std::string, std::string> cfg;
     for (uint32_t i = 0; i < keys->Length(); i++)
@@ -550,8 +634,8 @@ NAN_METHOD(NodeVitastorKV::Open)
         cfg[std::string(*Nan::Utf8String(key))] = std::string(*Nan::Utf8String(Nan::Get(jsParams, key).ToLocalChecked()));
     }
 
-    v8::Local<v8::Function> callback = info[2].As<v8::Function>();
-    auto req = new NodeVitastorRequest(callback);
+    v8::Local<v8::Function> callback = info[3].As<v8::Function>();
+    auto req = new NodeVitastorRequest(kv->cli, callback);
 
     kv->Ref();
     kv->dbw->open(inode_id, cfg, [kv, req](int res)
@@ -570,11 +654,13 @@ NAN_METHOD(NodeVitastorKV::Open)
 NAN_METHOD(NodeVitastorKV::Close)
 {
     TRACE("NodeVitastorKV::Close");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to close(callback(err))");
 
     NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
 
     v8::Local<v8::Function> callback = info[0].As<v8::Function>();
-    auto req = new NodeVitastorRequest(callback);
+    auto req = new NodeVitastorRequest(kv->cli, callback);
 
     kv->Ref();
     kv->dbw->close([kv, req]()
@@ -591,6 +677,8 @@ NAN_METHOD(NodeVitastorKV::Close)
 NAN_METHOD(NodeVitastorKV::SetConfig)
 {
     TRACE("NodeVitastorKV::SetConfig");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to set_config({ ...config })");
 
     NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
 
@@ -627,7 +715,7 @@ void NodeVitastorKV::get_impl(const Nan::FunctionCallbackInfo<v8::Value> & info,
     std::string key(*Nan::Utf8String(info[0].As<v8::String>()));
 
     v8::Local<v8::Function> callback = info[1].As<v8::Function>();
-    auto req = new NodeVitastorRequest(callback);
+    auto req = new NodeVitastorRequest(kv->cli, callback);
 
     kv->Ref();
     kv->dbw->get(key, [kv, req](int res, const std::string & value)
@@ -647,6 +735,8 @@ void NodeVitastorKV::get_impl(const Nan::FunctionCallbackInfo<v8::Value> & info,
 NAN_METHOD(NodeVitastorKV::Get)
 {
     TRACE("NodeVitastorKV::Get");
+    if (info.Length() < 2)
+        Nan::ThrowError("Not enough arguments to get(key, callback(err, value))");
     get_impl(info, false);
 }
 
@@ -654,6 +744,8 @@ NAN_METHOD(NodeVitastorKV::Get)
 NAN_METHOD(NodeVitastorKV::GetCached)
 {
     TRACE("NodeVitastorKV::GetCached");
+    if (info.Length() < 2)
+        Nan::ThrowError("Not enough arguments to get_cached(key, callback(err, value))");
     get_impl(info, true);
 }
 
@@ -672,10 +764,12 @@ static std::function<bool(int, const std::string &)> make_cas_callback(NodeVitas
     };
 }
 
-// set(key, value, callback(err), cas_compare(old_value))
+// set(key, value, callback(err), cas_compare(old_value)?)
 NAN_METHOD(NodeVitastorKV::Set)
 {
     TRACE("NodeVitastorKV::Set");
+    if (info.Length() < 3)
+        Nan::ThrowError("Not enough arguments to set(key, value, callback(err), cas_compare(old_value)?)");
 
     NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
 
@@ -684,13 +778,13 @@ NAN_METHOD(NodeVitastorKV::Set)
     std::string value(*Nan::Utf8String(info[1].As<v8::String>()));
 
     v8::Local<v8::Function> callback = info[2].As<v8::Function>();
-    NodeVitastorRequest *req = new NodeVitastorRequest(callback), *cas_req = NULL;
+    NodeVitastorRequest *req = new NodeVitastorRequest(kv->cli, callback), *cas_req = NULL;
 
     std::function<bool(int, const std::string &)> cas_cb;
     if (info.Length() > 3 && info[3]->IsObject())
     {
         v8::Local<v8::Function> cas_callback = info[3].As<v8::Function>();
-        cas_req = new NodeVitastorRequest(cas_callback);
+        cas_req = new NodeVitastorRequest(kv->cli, cas_callback);
         cas_cb = make_cas_callback(cas_req);
     }
 
@@ -709,10 +803,12 @@ NAN_METHOD(NodeVitastorKV::Set)
     }, cas_cb);
 }
 
-// del(key, callback(err), cas_compare(old_value))
+// del(key, callback(err), cas_compare(old_value)?)
 NAN_METHOD(NodeVitastorKV::Del)
 {
     TRACE("NodeVitastorKV::Del");
+    if (info.Length() < 2)
+        Nan::ThrowError("Not enough arguments to del(key, callback(err), cas_compare(old_value)?)");
 
     NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
 
@@ -720,13 +816,13 @@ NAN_METHOD(NodeVitastorKV::Del)
     std::string key(*Nan::Utf8String(info[0].As<v8::String>()));
 
     v8::Local<v8::Function> callback = info[1].As<v8::Function>();
-    NodeVitastorRequest *req = new NodeVitastorRequest(callback), *cas_req = NULL;
+    NodeVitastorRequest *req = new NodeVitastorRequest(kv->cli, callback), *cas_req = NULL;
 
     std::function<bool(int, const std::string &)> cas_cb;
     if (info.Length() > 2 && info[2]->IsObject())
     {
         v8::Local<v8::Function> cas_callback = info[2].As<v8::Function>();
-        cas_req = new NodeVitastorRequest(cas_callback);
+        cas_req = new NodeVitastorRequest(kv->cli, cas_callback);
         cas_cb = make_cas_callback(cas_req);
     }
 
@@ -762,6 +858,14 @@ NAN_METHOD(NodeVitastorKV::List)
     info.GetReturnValue().Set(Nan::NewInstance(cons, narg, args).ToLocalChecked());
 }
 
+/*NAN_METHOD(NodeVitastorKV::Destroy)
+{
+    TRACE("NodeVitastorKV::Destroy");
+    NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
+    if (!kv->dead)
+        kv->Unref();
+}*/
+
 //////////////////////////////////////////////////
 // NodeVitastorKVListing
 //////////////////////////////////////////////////
@@ -783,12 +887,10 @@ NAN_METHOD(NodeVitastorKVListing::Create)
 
     NodeVitastorKVListing *list = new NodeVitastorKVListing();
     list->kv = kv;
-    {
-        std::unique_lock<std::mutex> lock(kv->cli->mu);
-        list->handle = list->kv->dbw->list_start(start_key);
-    }
+    list->handle = list->kv->dbw->list_start(start_key);
 
     list->Wrap(info.This());
+    kv->Ref();
     info.GetReturnValue().Set(info.This());
 }
 
@@ -796,35 +898,54 @@ NodeVitastorKVListing::~NodeVitastorKVListing()
 {
     if (handle)
     {
-        std::unique_lock<std::mutex> lock(kv->cli->mu);
         kv->dbw->list_close(handle);
         handle = NULL;
     }
+    if (iter)
+    {
+        delete iter;
+        iter = NULL;
+    }
+    kv->Unref();
 }
 
-// next(callback(err, value))
+// next(callback(err, value)?)
 NAN_METHOD(NodeVitastorKVListing::Next)
 {
     TRACE("NodeVitastorKVListing::Next");
 
     NodeVitastorKVListing* list = Nan::ObjectWrap::Unwrap<NodeVitastorKVListing>(info.This());
 
-    v8::Local<v8::Function> callback = info[0].As<v8::Function>();
-    auto req = new NodeVitastorRequest(callback);
+    if (info.Length() > 0)
+    {
+        v8::Local<v8::Function> callback = info[0].As<v8::Function>();
+        if (list->iter)
+        {
+            delete list->iter;
+        }
+        list->iter = new NodeVitastorRequest(list->kv->cli, callback);
+    }
     if (!list->handle)
     {
         // Already closed
-        Nan::Callback nanCallback(Nan::New(req->callback));
-        v8::Local<v8::Value> args[1];
-        args[0] = Nan::New<v8::Int32>(-EINVAL);
-        nanCallback.Call(1, args, req);
-        delete req;
+        if (list->iter)
+        {
+            auto req = list->iter;
+            list->iter = NULL;
+            Nan::Callback nanCallback(Nan::New(req->callback));
+            v8::Local<v8::Value> args[1];
+            args[0] = Nan::New<v8::Int32>(-EINVAL);
+            nanCallback.Call(1, args, req);
+            delete req;
+        }
         return;
     }
 
     list->kv->Ref();
-    list->kv->dbw->list_next(list->handle, [list, req](int res, const std::string & key, const std::string & value)
+    list->kv->dbw->list_next(list->handle, [list](int res, const std::string & key, const std::string & value)
     {
+        auto req = list->iter;
+        list->iter = NULL;
         Nan::HandleScope scope;
         Nan::Callback nanCallback(Nan::New(req->callback));
         v8::Local<v8::Value> args[3];
@@ -832,7 +953,10 @@ NAN_METHOD(NodeVitastorKVListing::Next)
         args[1] = !res ? v8::Local<v8::Value>(Nan::New<v8::String>(key).ToLocalChecked()) : v8::Local<v8::Value>(Nan::Null());
         args[2] = !res ? v8::Local<v8::Value>(Nan::New<v8::String>(value).ToLocalChecked()) : v8::Local<v8::Value>(Nan::Null());
         nanCallback.Call(3, args, req);
-        delete req;
+        if (list->iter)
+            delete req;
+        else
+            list->iter = req;
         list->kv->Unref();
     });
 }
@@ -846,8 +970,12 @@ NAN_METHOD(NodeVitastorKVListing::Close)
 
     if (list->handle)
     {
-        std::unique_lock<std::mutex> lock(list->kv->cli->mu);
         list->kv->dbw->list_close(list->handle);
         list->handle = NULL;
     }
+    if (list->iter)
+    {
+        delete list->iter;
+        list->iter = NULL;
+    }
 }

node-binding/client.h

@@ -4,8 +4,6 @@
 #ifndef NODE_VITASTOR_CLIENT_H
 #define NODE_VITASTOR_CLIENT_H
 
-#include <mutex>
-
 #include <nan.h>
 #include <vitastor_c.h>
 #include <vitastor_kv.h>
@@ -19,7 +17,7 @@ public:
     static NAN_METHOD(Create);
     // read(pool, inode, offset, len, callback(err, buffer, version))
     static NAN_METHOD(Read);
-    // write(pool, inode, offset, buffer, { version }?, callback(err))
+    // write(pool, inode, offset, buf: Buffer | Buffer[], { version }?, callback(err))
     static NAN_METHOD(Write);
     // sync(callback(err))
     static NAN_METHOD(Sync);
@@ -34,8 +32,6 @@ private:
     vitastor_c *c = NULL;
     int eventfd = -1;
     uv_poll_t poll_watcher;
-    // FIXME: Is it really needed?
-    std::mutex mu;
 
     NodeVitastor();
 
@@ -44,6 +40,9 @@ private:
     static void on_write_finish(void *opaque, long retval);
     static void on_read_bitmap_finish(void *opaque, long retval, uint8_t *bitmap);
 
+    NodeVitastorRequest* get_read_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos);
+    NodeVitastorRequest* get_write_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos);
+
     friend class NodeVitastorImage;
     friend class NodeVitastorKV;
     friend class NodeVitastorKVListing;
@@ -56,7 +55,7 @@ public:
     static NAN_METHOD(Create);
     // read(offset, len, callback(err, buffer, version))
     static NAN_METHOD(Read);
-    // write(offset, buffer, { version }?, callback(err))
+    // write(offset, buf: Buffer | Buffer[], { version }?, callback(err))
     static NAN_METHOD(Write);
     // sync(callback(err))
     static NAN_METHOD(Sync);
@@ -74,8 +73,6 @@ private:
     std::vector<NodeVitastorRequest*> on_init;
     Nan::Persistent<v8::Object> cliObj;
 
-    NodeVitastorImage();
-
     static void on_watch_start(void *opaque, long retval);
     void exec_request(NodeVitastorRequest *req);
     void exec_or_wait(NodeVitastorRequest *req);
@@ -86,7 +83,7 @@ class NodeVitastorKV: public Nan::ObjectWrap
 public:
     // constructor(node_vitastor)
     static NAN_METHOD(Create);
-    // open(inode_id, { ...config }, callback(err))
+    // open(pool_id, inode_num, { ...config }, callback(err))
     static NAN_METHOD(Open);
     // set_config({ ...config })
     static NAN_METHOD(SetConfig);
@@ -98,9 +95,9 @@ public:
     static NAN_METHOD(Get);
     // get_cached(key, callback(err, value))
     static NAN_METHOD(GetCached);
-    // set(key, value, callback(err), cas_compare(old_value))
+    // set(key, value, callback(err), cas_compare(old_value)?)
     static NAN_METHOD(Set);
-    // del(key, callback(err), cas_compare(old_value))
+    // del(key, callback(err), cas_compare(old_value)?)
     static NAN_METHOD(Del);
     // list(start_key?)
     static NAN_METHOD(List);
@@ -113,8 +110,6 @@ private:
     NodeVitastor *cli = NULL;
     vitastorkv_dbw_t *dbw = NULL;
 
-    NodeVitastorKV();
-
     static void get_impl(const Nan::FunctionCallbackInfo<v8::Value> & info, bool allow_cache);
 
     friend class NodeVitastorKVListing;
@@ -125,7 +120,7 @@ class NodeVitastorKVListing: public Nan::ObjectWrap
 public:
     // constructor(node_vitastor_kv, start_key?)
     static NAN_METHOD(Create);
-    // next(callback(err, value))
+    // next(callback(err, value)?)
     static NAN_METHOD(Next);
     // close()
     static NAN_METHOD(Close);
@@ -135,8 +130,7 @@ public:
 private:
     NodeVitastorKV *kv = NULL;
     void *handle = NULL;
-
-    NodeVitastorKVListing();
+    NodeVitastorRequest *iter = NULL;
 };
 
 #endif

node-binding/index.js

@@ -0,0 +1 @@
+module.exports = require('bindings')('addon.node');

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '1.7.1'
+VITASTOR_VERSION = '1.8.0'
 
 LOG = logging.getLogger(__name__)
 

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.7.1
+Version:        1.8.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.7.1.el7.tar.gz
+Source0:        vitastor-1.8.0.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.7.1
+Version:        1.8.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.7.1.el8.tar.gz
+Source0:        vitastor-1.8.0.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.7.1
+Version:        1.8.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.7.1.el9.tar.gz
+Source0:        vitastor-1.8.0.el9.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -19,7 +19,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVITASTOR_VERSION="1.7.1")
+add_definitions(-DVITASTOR_VERSION="1.8.0")
 add_definitions(-D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/client/cluster_client.cpp

@@ -1286,7 +1286,11 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
         if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
         {
             copy_part_bitmap(op, part);
-            op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0;
+            if (op->inode == op->cur_inode)
+            {
+                // Read only returns the version of the uppermost layer
+                op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0;
+            }
         }
         else if (op->opcode == OSD_OP_WRITE)
         {

src/client/etcd_state_client.cpp

@@ -730,7 +730,7 @@ void etcd_state_client_t::clean_nonexistent_pgs()
             {
                 if (!pg_cfg.state_exists)
                 {
-                    if (this->log_level > 3)
+                    if (this->log_level > 3 && (pg_cfg.cur_primary || pg_cfg.cur_state))
                     {
                         fprintf(stderr, "PG %u/%u primary OSD disappeared after reload, forgetting it\n", pool_item.first, pg_it->first);
                     }
@@ -740,7 +740,7 @@ void etcd_state_client_t::clean_nonexistent_pgs()
                 }
                 if (!pg_cfg.history_exists)
                 {
-                    if (this->log_level > 3)
+                    if (this->log_level > 3 && (pg_cfg.target_history.size() || pg_cfg.all_peers.size() || pg_cfg.epoch || pg_cfg.next_scrub))
                     {
                         fprintf(stderr, "PG %u/%u history disappeared after reload, forgetting it\n", pool_item.first, pg_it->first);
                     }

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 1.7.1
+Version: 1.8.0
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/nfs/nfs_proxy.cpp

@@ -522,17 +522,15 @@ void nfs_proxy_t::do_accept(int listen_fd)
         {
             cli->proc_table.insert(fn);
         }
+        rpc_clients[nfs_fd] = cli;
         epmgr->tfd->set_fd_handler(nfs_fd, true, [cli](int nfs_fd, int epoll_events)
         {
             // Handle incoming event
             if (epoll_events & EPOLLRDHUP)
             {
-                auto parent = cli->parent;
-                if (parent->trace)
+                if (cli->parent->trace)
                     fprintf(stderr, "Client %d disconnected\n", nfs_fd);
                 cli->stop();
-                parent->active_connections--;
-                parent->check_exit();
                 return;
             }
             cli->epoll_events |= epoll_events;
@@ -695,6 +693,8 @@ void nfs_client_t::handle_read(int result)
                         frag_num++;
                     }
                 }
+                // Increase client refcount while the RPC call is being processed
+                refs++;
                 // Handle full message
                 int referenced = handle_rpc_message(cur_buffer.buf, data+4, wanted-4*fragments);
                 cur_buffer.refs += referenced ? 1 : 0;
@@ -779,9 +779,13 @@ void nfs_client_t::stop()
     stopped = true;
     if (refs <= 0)
     {
+        auto parent = this->parent;
+        parent->rpc_clients.erase(nfs_fd);
+        parent->active_connections--;
         parent->epmgr->tfd->set_fd_handler(nfs_fd, true, NULL);
         close(nfs_fd);
         delete this;
+        parent->check_exit();
     }
 }
 
@@ -834,6 +838,10 @@ void nfs_client_t::handle_send(int result)
                         }
                     }
                     free(rop);
+                    if (deref())
+                    {
+                        return;
+                    }
                 }
                 result -= iov.iov_len;
                 done++;
@@ -1056,6 +1064,11 @@ int nfs_client_t::handle_rpc_message(void *base_buf, void *msg_buf, uint32_t msg
 
 void nfs_proxy_t::daemonize()
 {
+    // Stop all clients because client I/O sometimes breaks during daemonize
+    // I.e. the new process stops receiving events on the old FD
+    // It doesn't happen if we call sleep(1) here, but we don't want to call sleep(1)...
+    for (auto & clp: rpc_clients)
+        clp.second->stop();
     if (fork())
         exit(0);
     setsid();

src/nfs/nfs_proxy.h

@@ -21,6 +21,7 @@ class cli_tool_t;
 
 struct kv_fs_state_t;
 struct block_fs_state_t;
+class nfs_client_t;
 
 class nfs_proxy_t
 {
@@ -54,6 +55,7 @@ public:
     vitastorkv_dbw_t *db = NULL;
     kv_fs_state_t *kvfs = NULL;
     block_fs_state_t *blockfs = NULL;
+    std::map<int, nfs_client_t*> rpc_clients;
 
     std::vector<XDR*> xdr_pool;
 

tests/run_3osds.sh

@@ -30,10 +30,10 @@ wait_etcd
 
 if [ "$IMMEDIATE_COMMIT" != "" ]; then
     NO_SAME="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all --log_level 10 --etcd_stats_interval 5"
-    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"immediate_commit":"all","client_writeback_allowed":true,"client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}'
+    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"immediate_commit":"all","client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}'
 else
     NO_SAME="--journal_sector_buffer_count 1024 --log_level 10 --etcd_stats_interval 5 --min_flusher_count 16"
-    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"immediate_commit":"none","client_writeback_allowed":true,"client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}'
+    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"recovery_tune_util_low":1,"immediate_commit":"none","client_enable_writeback":true,"client_max_writeback_iodepth":32'$GLOBAL_CONFIG'}'
 fi
 
 start_osd_on()