WIP "Heap" metadata storage scheme

New features:

- [Localized read support](https://vitastor.io/docs/config/pool.html#local_reads) for multi-datacenter setups.
- io_uring-based [zero-copy send support](https://vitastor.io/docs/config/network.html#min_zerocopy_send_size) -
  read the instruction carefully for optimal performance!
- Improve and speedup data distribution, especially in cases of very large hosts (100 OSD+).
  Previously, PG optimization speed depended on the number of OSDs, now it only depends on
  the number of failure domains. Distribution over specific OSDs is now also more even and
  becomes strictly more even when you increase the number of PGs.
- Add a [very interesting instruction](https://vitastor.io/en/docs/usage/nfs.html#linux-nfs-write-size) to change NFS_MAX_FILE_IO_SIZE
- Check operation sequencing and stop connections when it breaks - should help catch some
  very rare RDMA packet loss problems.
- `vitastor-cli rm-osd` now refuses to remove OSDs which are still up and suggests to use `vitastor-disk purge`.
- Allow removal of direntries referring non-existent in VitastorFS.
- Change default vitastor-etcd data dir to /var/lib/etcd/vitastor.

Bug fixes:

- Fix compatibility with ISA-L 2.31+. ⚠️Very important: please upgrade Vitastor before upgrading ISA-L to 2.31+.
- Fix in-memory state cleanup for incomplete PGs.
- Fix monitor crash with non-existent node_placement nodes.
- Slightly speedup `vitastor-kv dump` command by adding output buffering.
- Fix theoretically possible slowdowns in OSD sub-operation failure handling code.
- Fix very rare stack overflows in vitastor-kv.
- Fix a possible crash in VitastorFS during handling of file creation race condition.
- Fix modify-pool -s PG_SIZE which didn't work without --pg_minsize.
- Fix marking peer OSDs as alive on receiving data from them via RDMA - in theory,
  the bug could result in instability with RDMA under high load with slow disks.
- Fix a rare OSD crash due to double handle_primary_subop() call.
- Fix latency aggregation in global stats (/vitastor/stats in etcd) - do not sum it.
- Hide "Ran out of journal space" log messages by default.
- Wait for RDMA-CM EVENT_ESTABLISHED after rdma_accept(), handle rdma_accept() before acking the event.
- Fix VitastorFS total & free numbers multiplied by extra 2.
- Fix systemd unit name in make-etcd.
- Do not allow reweight > 1 in vitastor-cli modify-osd.
- Fix docker build.

.gitea/workflows/test.yml

@@ -720,6 +720,24 @@ jobs:
           echo ""
         done
 
+  test_heal_local_read:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: TEST_NAME=local_read POOLCFG='"local_reads":"random",' /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_heal_ec:
     runs-on: ubuntu-latest
     needs: build

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "2.1.0")
+set(VITASTOR_VERSION "2.2.0")
 
 add_subdirectory(src)

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v2.1.0
+VITASTOR_VERSION ?= v2.2.0
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v2.1.0
+          image: vitalif/vitastor-csi:v2.2.0
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v2.1.0
+          image: vitalif/vitastor-csi:v2.2.0
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "2.1.0"
+    vitastorCSIDriverVersion = "2.2.0"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (2.1.0-1) unstable; urgency=medium
+vitastor (2.2.0-1) unstable; urgency=medium
 
   * Bugfixes
 

docker/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v2.1.0
+VITASTOR_VERSION ?= v2.2.0
 
 all: build push
 

docker/etc/vitastor/docker.conf

@@ -4,7 +4,7 @@
 #
 
 # Desired Vitastor version
-VITASTOR_VERSION=v2.1.0
+VITASTOR_VERSION=v2.2.0
 
 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here

docs/config/client.en.md

@@ -24,6 +24,7 @@ affect their interaction with the cluster.
 - [nbd_max_devices](#nbd_max_devices)
 - [nbd_max_part](#nbd_max_part)
 - [osd_nearfull_ratio](#osd_nearfull_ratio)
+- [hostname](#hostname)
 
 ## client_iothread_count
 
@@ -215,3 +216,12 @@ just one OSD becomes 100 % full!
 However, unlike in Ceph, 100 % full Vitastor OSDs don't crash (in Ceph they're
 unable to start at all), so you'll be able to recover from "out of space" errors
 without destroying and recreating OSDs.
+
+## hostname
+
+- Type: string
+- Can be changed online: yes
+
+Clients use host name to find their distance to OSDs when [localized reads](pool.en.md#local_reads)
+are enabled. By default, standard [gethostname](https://man7.org/linux/man-pages/man2/gethostname.2.html)
+function is used to determine host name, but you can also override it with this parameter.

docs/config/client.ru.md

@@ -24,6 +24,7 @@
 - [nbd_max_devices](#nbd_max_devices)
 - [nbd_max_part](#nbd_max_part)
 - [osd_nearfull_ratio](#osd_nearfull_ratio)
+- [hostname](#hostname)
 
 ## client_iothread_count
 
@@ -219,3 +220,13 @@ RDMA и хотите повысить пиковую производитель
 заполненные на 100% OSD вообще не могут стартовать), так что вы сможете
 восстановить работу кластера после ошибок отсутствия свободного места
 без уничтожения и пересоздания OSD.
+
+## hostname
+
+- Тип: строка
+- Можно менять на лету: да
+
+Клиенты используют имя хоста для определения расстояния до OSD, когда включены
+[локальные чтения](pool.ru.md#local_reads). По умолчанию для определения имени
+хоста используется стандартная функция [gethostname](https://man7.org/linux/man-pages/man2/gethostname.2.html),
+но вы также можете задать имя хоста вручную данным параметром.

docs/config/osd.en.md

@@ -63,6 +63,8 @@ with an OSD restart or, for some of them, even without restarting by updating co
 - [discard_on_start](#discard_on_start)
 - [min_discard_size](#min_discard_size)
 - [allow_net_split](#allow_net_split)
+- [enable_pg_locks](#enable_pg_locks)
+- [pg_lock_retry_interval_ms](#pg_lock_retry_interval_ms)
 
 ## bind_address
 
@@ -647,3 +649,20 @@ The downside is that it increases the probability of writing data into just pg_m
 OSDs during failover which can lead to PGs becoming incomplete after additional outages.
 
 The old behaviour in versions up to 2.0.0 was equal to enabled allow_net_split.
+
+## enable_pg_locks
+
+- Type: boolean
+
+Vitastor 2.2.0 introduces a new layer of split-brain prevention mechanism in
+addition to etcd: PG locks. They prevent split-brain even in abnormal theoretical cases
+when etcd is extremely laggy. As a new feature, by default, PG locks are only enabled
+for pools where they're required - pools with [localized reads](pool.en.md#local_reads).
+Use this parameter to enable or disable this function for all pools.
+
+## pg_lock_retry_interval_ms
+
+- Type: milliseconds
+- Default: 100
+
+Retry interval for failed PG lock attempts.

docs/config/osd.ru.md

@@ -64,6 +64,8 @@
 - [discard_on_start](#discard_on_start)
 - [min_discard_size](#min_discard_size)
 - [allow_net_split](#allow_net_split)
+- [enable_pg_locks](#enable_pg_locks)
+- [pg_lock_retry_interval_ms](#pg_lock_retry_interval_ms)
 
 ## bind_address
 
@@ -679,3 +681,21 @@ pg_minsize OSD во время переключений, что может по
 неполными (incomplete), если упадут ещё какие-то OSD.
 
 Старое поведение в версиях до 2.0.0 было идентично включённому allow_net_split.
+
+## enable_pg_locks
+
+- Тип: булево (да/нет)
+
+В Vitastor 2.2.0 появился новый слой защиты от сплитбрейна в дополнение к etcd -
+блокировки PG. Они гарантируют порядок даже в теоретических ненормальных случаях,
+когда etcd очень сильно тормозит. Так как функция новая, по умолчанию она включается
+только для пулов, в которых она необходима - а именно, в пулах с включёнными
+[локальными чтениями](pool.ru.md#local_reads). Ну а с помощью данного параметра
+можно включить блокировки PG для всех пулов.
+
+## pg_lock_retry_interval_ms
+
+- Тип: миллисекунды
+- Значение по умолчанию: 100
+
+Интервал повтора неудачных попыток блокировки PG.

docs/config/pool.en.md

@@ -34,6 +34,7 @@ Parameters:
 - [failure_domain](#failure_domain)
 - [level_placement](#level_placement)
 - [raw_placement](#raw_placement)
+- [local_reads](#local_reads)
 - [max_osd_combinations](#max_osd_combinations)
 - [block_size](#block_size)
 - [bitmap_granularity](#bitmap_granularity)
@@ -133,8 +134,8 @@ Pool name.
 ## scheme
 
 - Type: string
-- Required
 - One of: "replicated", "xor", "ec" or "jerasure"
+- Required
 
 Redundancy scheme used for data in this pool. "jerasure" is an alias for "ec",
 both use Reed-Solomon-Vandermonde codes based on ISA-L or jerasure libraries.
@@ -289,6 +290,30 @@ Examples:
 - EC 4+2 in 3 DC: `any, dc=1 host!=1, dc!=1, dc=3 host!=3, dc!=(1,3), dc=5 host!=5`
 - 1 replica in fixed DC + 2 in random DCs: `dc?=meow, dc!=1, dc!=(1,2)`
 
+## local_reads
+
+- Type: string
+- One of: "primary", "nearest" or "random"
+- Default: primary
+
+By default, Vitastor serves all read and write requests from the primary OSD of each PG.
+But it can also serve read requests for replicated pools from secondary OSDs in clean PGs
+(active or active+left_on_dead) which may be useful if you have OSDs with different network
+latency to the client - for example, if you have a cross-datacenter setup.
+
+If you set this parameter to "nearest", clients will try to read from the nearest OSD
+in the [Placement Tree](#placement-tree), i.e. from an OSD from the same host or datacenter.
+Distance to different OSDs will be calculated based on client hostname, determined
+automatically or set manually in the [hostname](client.en.md#hostname) parameter.
+
+If you set this parameter to "random", clients will try to distribute read requests over
+all available secondary OSDs. This mode is mainly useful for tests, but, probably, not
+really required in production setups.
+
+[PG locks](osd.en.md#enable_pg_locks) are required for local reads to function. However,
+PG locks are enabled automatically by default for pools with enabled local reads, so you
+don't have to enable them explicitly.
+
 ## max_osd_combinations
 
 - Type: integer
@@ -324,7 +349,8 @@ Read more about this parameter in [Cluster-Wide Disk Layout Parameters](layout-c
 
 ## immediate_commit
 
-- Type: string, one of "all", "small" and "none"
+- Type: string
+- One of: "all", "small" or "none"
 - Default: none
 
 Immediate commit setting for this pool. The value from /vitastor/config/global

docs/config/pool.ru.md

@@ -33,6 +33,7 @@
 - [failure_domain](#failure_domain)
 - [level_placement](#level_placement)
 - [raw_placement](#raw_placement)
+- [local_reads](#local_reads)
 - [max_osd_combinations](#max_osd_combinations)
 - [block_size](#block_size)
 - [bitmap_granularity](#bitmap_granularity)
@@ -133,8 +134,8 @@ OSD игнорируется и OSD не удаляется из распред
 ## scheme
 
 - Тип: строка
-- Обязательный
 - Возможные значения: "replicated", "xor", "ec" или "jerasure"
+- Обязательный
 
 Схема избыточности, используемая в данном пуле. "jerasure" - синоним для "ec",
 в обеих схемах используются коды Рида-Соломона-Вандермонда, реализованные на
@@ -287,6 +288,30 @@ meow недоступен".
 - EC 4+2 в 3 датацентрах: `any, dc=1 host!=1, dc!=1, dc=3 host!=3, dc!=(1,3), dc=5 host!=5`
 - 1 копия в фиксированном ДЦ + 2 в других ДЦ: `dc?=meow, dc!=1, dc!=(1,2)`
 
+## local_reads
+
+- Тип: строка
+- Возможные значения: "primary", "nearest" или "random"
+- По умолчанию: primary
+
+По умолчанию Vitastor обслуживает все запросы чтения и записи с первичного OSD каждой PG.
+Однако, в чистых PG (active или active+left_on_dead) реплицированных пулов также есть
+возможность обслуживать запросы чтения с вторичных OSD, что может быть полезно, если
+у вас сильно отличается время сетевого обращения от клиента к разным OSD - например,
+если у вас несколько дата-центров.
+
+Если данный параметр установлен в значение "nearest", клиенты будут стараться читать с
+ближайших по [Дереву размещения](#дерево-размещения) OSD, то есть, с OSD с того же хоста
+или датацентра. Расстояние до разных OSD будет рассчитываться с помощью имени хоста клиента,
+определяемого автоматически или заданного вручную параметром [hostname](client.ru.md#hostname).
+
+Если данный параметр установлен в значение "random", клиенты будут стараться распределять
+запросы чтения по всем доступным вторичным OSD. Этот режим в основном полезен для тестов,
+но, скорее всего, редко нужен в реальных инсталляциях.
+
+Для работы локальных чтений требуются [блокировки PG](osd.ru.md#enable_pg_locks). Включать
+их явно не нужно - они включаются автоматически для пулов с включёнными локальными чтениями.
+
 ## max_osd_combinations
 
 - Тип: целое число
@@ -324,7 +349,8 @@ meow недоступен".
 
 ## immediate_commit
 
-- Тип: строка "all", "small" или "none"
+- Тип: строка
+- Возможные значения: "all", "small" или "none"
 - По умолчанию: none
 
 Настройка мгновенного коммита для данного пула. Если не задана, используется

docs/config/src/client.yml

@@ -271,3 +271,15 @@
     заполненные на 100% OSD вообще не могут стартовать), так что вы сможете
     восстановить работу кластера после ошибок отсутствия свободного места
     без уничтожения и пересоздания OSD.
+- name: hostname
+  type: string
+  online: true
+  info: |
+    Clients use host name to find their distance to OSDs when [localized reads](pool.en.md#local_reads)
+    are enabled. By default, standard [gethostname](https://man7.org/linux/man-pages/man2/gethostname.2.html)
+    function is used to determine host name, but you can also override it with this parameter.
+  info_ru: |
+    Клиенты используют имя хоста для определения расстояния до OSD, когда включены
+    [локальные чтения](pool.ru.md#local_reads). По умолчанию для определения имени
+    хоста используется стандартная функция [gethostname](https://man7.org/linux/man-pages/man2/gethostname.2.html),
+    но вы также можете задать имя хоста вручную данным параметром.

docs/config/src/osd.yml

@@ -781,3 +781,23 @@
     неполными (incomplete), если упадут ещё какие-то OSD.
 
     Старое поведение в версиях до 2.0.0 было идентично включённому allow_net_split.
+- name: enable_pg_locks
+  type: bool
+  info: |
+    Vitastor 2.2.0 introduces a new layer of split-brain prevention mechanism in
+    addition to etcd: PG locks. They prevent split-brain even in abnormal theoretical cases
+    when etcd is extremely laggy. As a new feature, by default, PG locks are only enabled
+    for pools where they're required - pools with [localized reads](pool.en.md#local_reads).
+    Use this parameter to enable or disable this function for all pools.
+  info_ru: |
+    В Vitastor 2.2.0 появился новый слой защиты от сплитбрейна в дополнение к etcd -
+    блокировки PG. Они гарантируют порядок даже в теоретических ненормальных случаях,
+    когда etcd очень сильно тормозит. Так как функция новая, по умолчанию она включается
+    только для пулов, в которых она необходима - а именно, в пулах с включёнными
+    [локальными чтениями](pool.ru.md#local_reads). Ну а с помощью данного параметра
+    можно включить блокировки PG для всех пулов.
+- name: pg_lock_retry_interval_ms
+  type: ms
+  default: 100
+  info: Retry interval for failed PG lock attempts.
+  info_ru: Интервал повтора неудачных попыток блокировки PG.

docs/installation/docker.en.md

@@ -26,9 +26,9 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.
 
 1. Download a Docker image of the desired version: \
-   `docker pull vitastor:2.1.0`
+   `docker pull vitastor:v2.2.0`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:2.1.0 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:v2.2.0 install.sh`
 3. Reload udev rules: \
    `udevadm control --reload-rules`
 

docs/installation/docker.ru.md

@@ -25,9 +25,9 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.
 
 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitastor:2.1.0`
+   `docker pull vitastor:v2.2.0`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:2.1.0 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:v2.2.0 install.sh`
 3. Перезагрузите правила udev: \
    `udevadm control --reload-rules`
 

docs/intro/architecture.en.md

@@ -125,6 +125,13 @@ All other client-side components are based on the client library:
   all current read/write operations to it fail with EPIPE error and are retried by clients.
 - After completing all secondary read/write requests, primary OSD sends the response to
   the client.
+- When [localized reads](../config/pool.en.md#local_reads) are enabled for a PG in a
+  replicated pool, and the PG is in an active and clean state (active or
+  active+left_on_dead), the client can send the request to one of secondary OSDs instead
+  of the primary. Secondary OSD checks the [PG lock](../config/osd.en.md#enable_pg_locks)
+  and handles the request locally without communicating to the primary. PG lock is required
+  for the secondary OSD to know for sure that the PG is in clean state and not switching
+  primary at the moment.
 
 ### Nuances of request handling
 

docs/intro/architecture.ru.md

@@ -125,6 +125,12 @@
   и если любое из этих соединений отключается, PG перезапускается, а все текущие запросы чтения
   и записи в неё завершаются с ошибкой EPIPE, после чего повторяются клиентами.
 - После завершения всех вторичных операций чтения/записи первичный OSD отправляет ответ клиенту.
+- Если в реплицированном пуле включены [локализованные чтения](../config/pool.ru.md#local_reads),
+  а PG находится в чистом активном состоянии (active или active+left_on_dead), клиент может
+  послать запрос к одному из вторичных OSD вместо первичного. Вторичный OSD проверяет
+  [блокировку PG](../config/osd.ru.md#enable_pg_locks) и обрабатывает запрос локально, не
+  обращаясь к первичному. Блокировка PG здесь нужна, чтобы вторичный OSD мог точно знать,
+  что PG находится в чистом состоянии и не переключается на другой первичный OSD.
 
 ### Особенности обработки запросов
 

docs/intro/features.en.md

@@ -25,6 +25,7 @@
 - Recovery of degraded blocks
 - Rebalancing (data movement between OSDs)
 - [Lazy fsync support](../config/layout-cluster.en.md#immediate_commit)
+- [Localized read support](../config/pool.en.md#local_reads) for cross-datacenter setup optimization
 - Per-OSD and per-image I/O and space usage statistics in etcd
 - Snapshots and copy-on-write image clones
 - [Write throttling to smooth random write workloads in SSD+HDD configurations](../config/osd.en.md#throttle_small_writes)

docs/intro/features.ru.md

@@ -25,6 +25,7 @@
 - Восстановление деградированных блоков
 - Ребаланс, то есть перемещение данных между OSD (дисками)
 - [Поддержка "ленивого" fsync (fsync не на каждую операцию)](../config/layout-cluster.ru.md#immediate_commit)
+- [Локальные чтения](../config/pool.ru.md#local_reads) для оптимизации при нескольких датацентрах
 - Сбор статистики ввода/вывода в etcd
 - Статистика операций ввода/вывода и занятого места в разрезе инодов
 - Именование инодов через хранение их метаданных в etcd

docs/usage/admin.en.md

@@ -14,6 +14,7 @@
 - [Removing a failed disk](#removing-a-failed-disk)
 - [Adding a disk](#adding-a-disk)
 - [Restoring from lost pool configuration](#restoring-from-lost-pool-configuration)
+- [Incompatibility problems](#Incompatibility-problems)
 - [Upgrading Vitastor](#upgrading-vitastor)
 - [OSD memory usage](#osd-memory-usage)
 
@@ -166,6 +167,17 @@ done
 
 After that all PGs should peer and find all previous data.
 
+## Incompatibility problems
+
+### ISA-L 2.31
+
+⚠ It is FORBIDDEN to use Vitastor 2.1.0 and earlier versions with ISA-L 2.31 and newer if
+you use EC N+K pools and K > 1 on a CPU with GF-NI instruction support, because it WILL
+lead to **data loss** during EC recovery.
+
+If you accidentally upgraded ISA-L to 2.31 but didn't upgrade Vitastor and restarted OSDs,
+then stop them as soon as possible and either update Vitastor or roll back ISA-L.
+
 ## Upgrading Vitastor
 
 Every upcoming Vitastor version is usually compatible with previous both forward

docs/usage/admin.ru.md

@@ -14,6 +14,7 @@
 - [Удаление неисправного диска](#удаление-неисправного-диска)
 - [Добавление диска](#добавление-диска)
 - [Восстановление потерянной конфигурации пулов](#восстановление-потерянной-конфигурации-пулов)
+- [Проблемы несовместимости](#проблемы-несовместимости)
 - [Обновление Vitastor](#обновление-vitastor)
 - [Потребление памяти OSD](#потребление-памяти-osd)
 
@@ -163,6 +164,17 @@ done
 
 После этого все PG должны пройти peering и найти все предыдущие данные.
 
+## Проблемы несовместимости
+
+### ISA-L 2.31
+
+⚠ ЗАПРЕЩЕНО использовать Vitastor 2.1.0 и более ранних версий с библиотекой ISA-L версии 2.31
+или более новой, если вы используете EC-пулы N+K и K > 1 на CPU с поддержкой инструкций GF-NI,
+так как это приведёт к **потере данных** при восстановлении из EC.
+
+Если вы случайно обновили ISA-L до 2.31, но не обновили Vitastor, и успели перезапустить OSD,
+то как можно скорее остановите их все и либо обновите Vitastor, либо откатите ISA-L.
+
 ## Обновление Vitastor
 
 Обычно каждая следующая версия Vitastor совместима с предыдущими и "вперёд", и "назад"

docs/usage/cli.en.md

@@ -397,6 +397,7 @@ Optional parameters:
 | `--immediate_commit none`      | Put pool only on OSDs with this or larger immediate_commit (none < small < all) |
 | `--level_placement <rules>`    | Use additional failure domain rules (example: "dc=112233")                 |
 | `--raw_placement <rules>`      | Specify raw PG generation rules ([details](../config/pool.en.md#raw_placement)) |
+| `--local_reads primary`        | Local read policy for replicated pools: primary, nearest or random         |
 | `--primary_affinity_tags tags` | Prefer to put primary copies on OSDs with all specified tags               |
 | `--scrub_interval <time>`      | Enable regular scrubbing for this pool. Format: number + unit s/m/h/d/M/y  |
 | `--used_for_app fs:<name>`     | Mark pool as used for VitastorFS with metadata in image `<name>`           |

docs/usage/cli.ru.md

@@ -414,6 +414,7 @@ OSD  PARENT            UP    SIZE  USED%    TAGS          WEIGHT  BLOCK  BITMAP
 | `--immediate_commit none`      | ...только OSD с этим или большим immediate_commit (none < small < all)     |
 | `--level_placement <rules>`    | Задать правила дополнительных доменов отказа (пример: "dc=112233")         |
 | `--raw_placement <rules>`      | Задать низкоуровневые правила генерации PG ([детали](../config/pool.ru.md#raw_placement)) |
+| `--local_reads primary`        | Политика локальных чтений для реплик: primary, nearest или random          |
 | `--primary_affinity_tags tags` | Предпочитать OSD со всеми данными тегами для роли первичных                |
 | `--scrub_interval <time>`      | Включить скрабы с заданным интервалом времени (число + единица s/m/h/d/M/y) |
 | `--pg_stripe_size <number>`    | Увеличить блок группировки объектов по PG                                  |

mon/osd_tree.js

@@ -110,7 +110,7 @@ function make_hier_tree(global_config, tree)
             if (!(tree[node_id].children||[]).length && (tree[node_id].size||0) <= 0)
             {
                 const parent = tree[node_id].parent;
-                if (parent)
+                if (parent && tree[parent])
                 {
                     tree[parent].children = tree[parent].children.filter(c => c != tree[node_id]);
                 }

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "2.1.0",
+  "version": "2.2.0",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {
@@ -19,6 +19,6 @@
     "eslint-plugin-node": "^11.1.0"
   },
   "engines": {
-    "node": ">=12.1.0"
+    "node": ">=12.0.0"
   }
 }

node-binding/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor",
-  "version": "2.1.0",
+  "version": "2.2.0",
   "description": "Low-level native bindings to Vitastor client library",
   "main": "index.js",
   "keywords": [

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '2.1.0'
+VITASTOR_VERSION = '2.2.0'
 
 LOG = logging.getLogger(__name__)
 

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.1.0
+Version:        2.2.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.1.0.el7.tar.gz
+Source0:        vitastor-2.2.0.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.1.0
+Version:        2.2.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.1.0.el8.tar.gz
+Source0:        vitastor-2.2.0.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.1.0
+Version:        2.2.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.1.0.el9.tar.gz
+Source0:        vitastor-2.2.0.el9.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -19,7 +19,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVITASTOR_VERSION="2.1.0")
+add_definitions(-DVITASTOR_VERSION="2.2.0")
 add_definitions(-D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/blockstore/blockstore.h

@@ -31,6 +31,7 @@
 #define DEFAULT_DATA_BLOCK_ORDER 17
 #define MIN_DATA_BLOCK_SIZE 4*1024
 #define MAX_DATA_BLOCK_SIZE 128*1024*1024
+#define MAX_META_BLOCK_SIZE 64*1024
 #define DEFAULT_BITMAP_GRANULARITY 4096
 
 #define BS_OP_MIN 1

src/blockstore/blockstore_disk.cpp

@@ -127,9 +127,9 @@ void blockstore_disk_t::parse_config(std::map<std::string, std::string> & config
     {
         throw std::runtime_error("meta_block_size must be a multiple of "+std::to_string(DIRECT_IO_ALIGNMENT));
     }
-    else if (meta_block_size > MAX_DATA_BLOCK_SIZE)
+    else if (meta_block_size > MAX_META_BLOCK_SIZE)
     {
-        throw std::runtime_error("meta_block_size must not exceed "+std::to_string(MAX_DATA_BLOCK_SIZE));
+        throw std::runtime_error("meta_block_size must not exceed "+std::to_string(MAX_META_BLOCK_SIZE));
     }
     if (data_offset % disk_alignment)
     {

src/blockstore/blockstore_flush.cpp

@@ -427,13 +427,6 @@ stop_flusher:
         printf("Flushing %jx:%jx v%ju\n", cur.oid.inode, cur.oid.stripe, cur.version);
 #endif
         flusher->active_flushers++;
-        // Find it in clean_db
-        {
-            auto & clean_db = bs->clean_db_shard(cur.oid);
-            auto clean_it = clean_db.find(cur.oid);
-            old_clean_ver = (clean_it != clean_db.end() ? clean_it->second.version : 0);
-            old_clean_loc = (clean_it != clean_db.end() ? clean_it->second.location : UINT64_MAX);
-        }
         // Scan dirty versions of the object to determine what we need to read
         scan_dirty();
         // Writes and deletes shouldn't happen at the same time
@@ -538,7 +531,7 @@ resume_2:
         {
             // zero out old metadata entry
             {
-                clean_disk_entry *old_entry = (clean_disk_entry*)((uint8_t*)meta_old.buf + meta_old.pos*bs->dsk.clean_entry_size);
+                clean_disk_entry *old_entry = (clean_disk_entry*)((uint8_t*)meta_old.buf + meta_old.pos);
                 if (old_entry->oid.inode != 0 && old_entry->oid != cur.oid)
                 {
                     printf("Fatal error (metadata corruption or bug): tried to wipe metadata entry %ju (%jx:%jx v%ju) as old location of %jx:%jx\n",
@@ -547,7 +540,7 @@ resume_2:
                     exit(1);
                 }
             }
-            memset((uint8_t*)meta_old.buf + meta_old.pos*bs->dsk.clean_entry_size, 0, bs->dsk.clean_entry_size);
+            memset((uint8_t*)meta_old.buf + meta_old.pos, 0, bs->dsk.clean_entry_size);
     resume_20:
             if (meta_old.sector != meta_new.sector && !write_meta_block(meta_old, 20))
                 return false;
@@ -608,7 +601,7 @@ resume_2:
 
 void journal_flusher_co::update_metadata_entry()
 {
-    clean_disk_entry *new_entry = (clean_disk_entry*)((uint8_t*)meta_new.buf + meta_new.pos*bs->dsk.clean_entry_size);
+    clean_disk_entry *new_entry = (clean_disk_entry*)((uint8_t*)meta_new.buf + meta_new.pos);
     if (new_entry->oid.inode != 0 && new_entry->oid != cur.oid)
     {
         printf(
@@ -623,7 +616,7 @@ void journal_flusher_co::update_metadata_entry()
     if (has_delete)
     {
         // Zero out the new metadata entry
-        memset((uint8_t*)meta_new.buf + meta_new.pos*bs->dsk.clean_entry_size, 0, bs->dsk.clean_entry_size);
+        memset((uint8_t*)meta_new.buf + meta_new.pos, 0, bs->dsk.clean_entry_size);
     }
     else
     {
@@ -805,7 +798,7 @@ bool journal_flusher_co::clear_incomplete_csum_block_bits(int wait_base)
             }
         }
         {
-            clean_disk_entry *new_entry = (clean_disk_entry*)((uint8_t*)meta_new.buf + meta_new.pos*bs->dsk.clean_entry_size);
+            clean_disk_entry *new_entry = (clean_disk_entry*)((uint8_t*)meta_new.buf + meta_new.pos);
             if (new_entry->oid != cur.oid)
             {
                 printf(
@@ -912,6 +905,12 @@ void journal_flusher_co::calc_block_checksums(uint32_t *new_data_csums, bool ski
 
 void journal_flusher_co::scan_dirty()
 {
+    // Find it in clean_db
+    auto & clean_db = bs->clean_db_shard(cur.oid);
+    auto clean_it = clean_db.find(cur.oid);
+    old_clean_ver = (clean_it != clean_db.end() ? clean_it->second.version : 0);
+    old_clean_loc = (clean_it != clean_db.end() ? clean_it->second.location : UINT64_MAX);
+    auto old_clean_bitmap = (clean_it != clean_db.end() ? bs->get_clean_entry_bitmap(clean_it, 0) : NULL);
     dirty_it = dirty_start = dirty_end;
     v.clear();
     copy_count = 0;
@@ -1037,13 +1036,12 @@ void journal_flusher_co::scan_dirty()
             read_to_fill_incomplete = 0;
             return;
         }
-        uint8_t *bmp_ptr = bs->get_clean_entry_bitmap(old_clean_loc, 0);
         uint64_t fulfilled = 0;
         int last = v.size()-1;
         while (last >= 0 && (v[last].copy_flags & COPY_BUF_CSUM_FILL))
             last--;
         read_to_fill_incomplete = bs->fill_partial_checksum_blocks(
-            v, fulfilled, bmp_ptr, NULL, false, NULL, v[0].offset/bs->dsk.csum_block_size * bs->dsk.csum_block_size,
+            v, fulfilled, old_clean_bitmap, NULL, false, NULL, v[0].offset/bs->dsk.csum_block_size * bs->dsk.csum_block_size,
             ((v[last].offset+v[last].len-1) / bs->dsk.csum_block_size + 1) * bs->dsk.csum_block_size
         );
     }
@@ -1139,7 +1137,7 @@ bool journal_flusher_co::modify_meta_do_reads(int wait_base)
 resume_0:
     if (!modify_meta_read(clean_loc, meta_new, wait_base+0))
         return false;
-    new_clean_bitmap = (uint8_t*)meta_new.buf + meta_new.pos*bs->dsk.clean_entry_size + sizeof(clean_disk_entry);
+    new_clean_bitmap = (uint8_t*)meta_new.buf + meta_new.pos + sizeof(clean_disk_entry);
     if (old_clean_loc != UINT64_MAX && old_clean_loc != clean_loc)
     {
     resume_1:
@@ -1193,7 +1191,7 @@ bool journal_flusher_co::modify_meta_read(uint64_t meta_loc, flusher_meta_write_
     // so I'll avoid it as long as I can.
     wr.submitted = false;
     wr.sector = ((meta_loc >> bs->dsk.block_order) / (bs->dsk.meta_block_size / bs->dsk.clean_entry_size)) * bs->dsk.meta_block_size;
-    wr.pos = ((meta_loc >> bs->dsk.block_order) % (bs->dsk.meta_block_size / bs->dsk.clean_entry_size));
+    wr.pos = ((meta_loc >> bs->dsk.block_order) % (bs->dsk.meta_block_size / bs->dsk.clean_entry_size)) * bs->dsk.clean_entry_size;
     if (bs->inmemory_meta)
     {
         wr.buf = (uint8_t*)bs->metadata_buffer + wr.sector;

src/blockstore/blockstore_impl.cpp

@@ -42,6 +42,8 @@ blockstore_impl_t::~blockstore_impl_t()
         free(metadata_buffer);
     if (clean_bitmaps)
         free(clean_bitmaps);
+    if (heap_meta.blocks)
+        delete[] heap_meta.blocks;
 }
 
 bool blockstore_impl_t::is_started()
@@ -431,13 +433,29 @@ blockstore_clean_db_t& blockstore_impl_t::clean_db_shard(object_id oid)
 {
     uint64_t pg_num = 0;
     uint64_t pool_id = (oid.inode >> (64-POOL_ID_BITS));
-    auto sh_it = clean_db_settings.find(pool_id);
+    auto sett_it = clean_db_settings.find(pool_id);
-    if (sh_it != clean_db_settings.end())
+    if (sett_it != clean_db_settings.end())
     {
         // like map_to_pg()
-        pg_num = (oid.stripe / sh_it->second.pg_stripe_size) % sh_it->second.pg_count + 1;
+        pg_num = (oid.stripe / sett_it->second.pg_stripe_size) % sett_it->second.pg_count + 1;
     }
-    return clean_db_shards[(pool_id << (64-POOL_ID_BITS)) | pg_num];
+    auto shard_id = (pool_id << (64-POOL_ID_BITS)) | pg_num;
+    if (dsk.meta_format == BLOCKSTORE_META_FORMAT_HEAP)
+    {
+        auto sh_it = clean_db_shards.find(shard_id);
+        if (sh_it == clean_db_shards.end())
+        {
+            // clean_db_t stores larger entries with heap_meta, but we disguise it as smaller clean_entry :)
+            // patched cpp-btree with extra_data
+            clean_db_shards[shard_id] = blockstore_clean_db_t(
+                sizeof(clean_entry_heap_t) - sizeof(clean_entry)
+                + (inmemory_meta ? dsk.clean_dyn_size : 2*dsk.clean_entry_bitmap_size)
+            );
+            return clean_db_shards[shard_id];
+        }
+        return sh_it->second;
+    }
+    return clean_db_shards[shard_id];
 }
 
 void blockstore_impl_t::reshard_clean_db(pool_id_t pool, uint32_t pg_count, uint32_t pg_stripe_size)

src/blockstore/blockstore_impl.h

@@ -96,6 +96,9 @@
 #define BLOCKSTORE_META_MAGIC_V1 0x726F747341544956l
 #define BLOCKSTORE_META_FORMAT_V1 1
 #define BLOCKSTORE_META_FORMAT_V2 2
+#define BLOCKSTORE_META_FORMAT_HEAP 3
+#define BLOCKSTORE_META_HEADER_V1_SIZE 36
+#define BLOCKSTORE_META_HEADER_V2_SIZE 48
 
 // metadata header (superblock)
 struct __attribute__((__packed__)) blockstore_meta_header_v1_t
@@ -119,6 +122,7 @@ struct __attribute__((__packed__)) blockstore_meta_header_v2_t
     uint32_t data_csum_type;
     uint32_t csum_block_size;
     uint32_t header_csum;
+    uint32_t block_id_bits; // 32 by default in heap meta
 };
 
 // 32 bytes = 24 bytes + block bitmap (4 bytes by default) + external attributes (also bitmap, 4 bytes by default)
@@ -140,6 +144,62 @@ struct __attribute__((__packed__)) clean_entry
     uint64_t location;
 };
 
+typedef uint32_t heap_block_num_t;
+
+// 50 = 16 (key=object_id) + 26 (value) + 8 (bitmap) + N (checksum) bytes per "clean" entry in memory
+struct __attribute__((__packed__)) clean_entry_heap_t
+{
+    uint64_t version;
+    uint64_t location; // UINT64_MAX = deleted
+    // previous versions invalidated by this version
+    heap_block_num_t prev_versions;
+    // metadata block number
+    heap_block_num_t meta_block;
+    // offset within block
+    uint16_t block_offset;
+    uint8_t bitmap[];
+};
+
+struct __attribute__((__packed__)) heap_meta_block_header_t
+{
+    uint64_t magic;
+    uint64_t seq_num;
+    uint32_t invalidates_blocks;
+};
+
+// 48+checksums = (40+bitmap)+checksums bytes per on-disk "heap" entry
+// for 128 KB block without checksums, it's 48 bytes - 84 entries per 4 kb metadata block
+// for 128 KB block with 4k checksums, it's 176 bytes - 22 entries per 4 kb metadata block
+// for 1 MB block without checksums, it's 80 bytes - 50 entries per 4 kb metadata block
+// for 1 MB block with 4k checksums, it's 1104 bytes O_o - only 3 entries per 4 kb metadata block
+// for 1 MB block with 32k checksums, it's 176 bytes again
+struct __attribute__((__packed__)) heap_meta_entry_t
+{
+    object_id oid;
+    uint64_t version;
+    uint64_t location; // UINT64_MAX = deleted
+    uint64_t reserved;
+    uint8_t bitmap[];
+};
+
+struct heap_meta_block_t
+{
+    heap_block_num_t offset = 0;
+    uint64_t seq_num = 0;
+    uint32_t used_space = 0;
+    std::vector<uint64_t> invalidates_blocks;
+};
+
+struct heap_meta_t
+{
+    heap_block_num_t block_count = 0;
+    heap_meta_block_t *blocks = NULL;
+    // used space => block number
+    std::multimap<uint32_t, heap_block_num_t> used_space_map;
+    heap_block_num_t cur_written_block = 0;
+    uint8_t *written_block_buf = NULL;
+};
+
 // 64 = 24 + 40 bytes per dirty entry in memory (obj_ver_id => dirty_entry). Plus checksums
 struct __attribute__((__packed__)) dirty_entry
 {
@@ -272,6 +332,8 @@ class blockstore_impl_t
 
     struct ring_consumer_t ring_consumer;
 
+    heap_meta_t heap_meta;
+
     std::map<pool_id_t, pool_shard_settings_t> clean_db_settings;
     std::map<pool_pg_id_t, blockstore_clean_db_t> clean_db_shards;
     std::map<uint64_t, int> no_inode_stats;
@@ -317,7 +379,7 @@ class blockstore_impl_t
     void open_data();
     void open_meta();
     void open_journal();
-    uint8_t* get_clean_entry_bitmap(uint64_t block_loc, int offset);
+    uint8_t* get_clean_entry_bitmap(blockstore_clean_db_t::iterator clean_it, int offset);
 
     blockstore_clean_db_t& clean_db_shard(object_id oid);
     void reshard_clean_db(pool_id_t pool_id, uint32_t pg_count, uint32_t pg_stripe_size);
@@ -345,9 +407,9 @@ class blockstore_impl_t
         uint64_t &fulfilled, uint32_t item_start, uint32_t item_end,
         uint32_t item_state, uint64_t item_version, uint64_t item_location,
         uint64_t journal_sector, uint8_t *csum, int *dyn_data);
-    bool fulfill_clean_read(blockstore_op_t *read_op, uint64_t & fulfilled,
+    bool fulfill_clean_read_journal(blockstore_op_t *read_op, uint64_t & fulfilled,
-        uint8_t *clean_entry_bitmap, int *dyn_data,
+        uint8_t *clean_entry_bitmap, int *dyn_data, uint32_t item_start, uint32_t item_end, uint64_t clean_loc, uint64_t clean_ver);
-        uint32_t item_start, uint32_t item_end, uint64_t clean_loc, uint64_t clean_ver);
+    bool fulfill_clean_read_meta(blockstore_op_t *read_op, uint64_t & fulfilled, blockstore_clean_db_t::iterator clean_it);
     int fill_partial_checksum_blocks(std::vector<copy_buffer_t> & rv, uint64_t & fulfilled,
         uint8_t *clean_entry_bitmap, int *dyn_data, bool from_journal, uint8_t *read_buf, uint64_t read_offset, uint64_t read_end);
     int pad_journal_read(std::vector<copy_buffer_t> & rv, copy_buffer_t & cp,
@@ -356,7 +418,7 @@ class blockstore_impl_t
     bool read_range_fulfilled(std::vector<copy_buffer_t> & rv, uint64_t & fulfilled, uint8_t *read_buf,
         uint8_t *clean_entry_bitmap, uint32_t item_start, uint32_t item_end);
     bool read_checksum_block(blockstore_op_t *op, int rv_pos, uint64_t &fulfilled, uint64_t clean_loc);
-    uint8_t* read_clean_meta_block(blockstore_op_t *read_op, uint64_t clean_loc, int rv_pos);
+    uint8_t* read_clean_meta_block(blockstore_op_t *op, blockstore_clean_db_t::iterator clean_it, int rv_pos);
     bool verify_padded_checksums(uint8_t *clean_entry_bitmap, uint8_t *csum_buf, uint32_t offset,
         iovec *iov, int n_iov, std::function<void(uint32_t, uint32_t, uint32_t)> bad_block_cb);
     bool verify_journal_checksums(uint8_t *csums, uint32_t offset,

src/blockstore/blockstore_init.cpp

@@ -54,6 +54,7 @@ int blockstore_init_meta::loop()
     else if (wait_state == 4) goto resume_4;
     else if (wait_state == 5) goto resume_5;
     else if (wait_state == 6) goto resume_6;
+    else if (wait_state == 7) goto resume_7;
     printf("Reading blockstore metadata\n");
     if (bs->inmemory_meta)
         metadata_buffer = bs->metadata_buffer;
@@ -78,6 +79,7 @@ resume_1:
     if (iszero((uint64_t*)metadata_buffer, bs->dsk.meta_block_size / sizeof(uint64_t)))
     {
         {
+            memset(metadata_buffer, 0, bs->dsk.meta_block_size);
             blockstore_meta_header_v2_t *hdr = (blockstore_meta_header_v2_t *)metadata_buffer;
             hdr->zero = 0;
             hdr->magic = BLOCKSTORE_META_MAGIC_V1;
@@ -85,12 +87,19 @@ resume_1:
             hdr->meta_block_size = bs->dsk.meta_block_size;
             hdr->data_block_size = bs->dsk.data_block_size;
             hdr->bitmap_granularity = bs->dsk.bitmap_granularity;
+            if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_HEAP)
+            {
+                hdr->block_id_bits = sizeof(heap_block_num_t);
+            }
             if (bs->dsk.meta_format >= BLOCKSTORE_META_FORMAT_V2)
             {
                 hdr->data_csum_type = bs->dsk.data_csum_type;
                 hdr->csum_block_size = bs->dsk.csum_block_size;
                 hdr->header_csum = 0;
-                hdr->header_csum = crc32c(0, hdr, sizeof(*hdr));
+                hdr->header_csum = crc32c(0, hdr,
+                    bs->dsk.meta_format == BLOCKSTORE_META_FORMAT_V2
+                    ? BLOCKSTORE_META_HEADER_V2_SIZE
+                    : sizeof(*hdr));
             }
         }
         if (bs->readonly)
@@ -128,7 +137,7 @@ resume_1:
             );
             exit(1);
         }
-        if (hdr->version == BLOCKSTORE_META_FORMAT_V2)
+        if (hdr->version == BLOCKSTORE_META_FORMAT_HEAP)
         {
             uint32_t csum = hdr->header_csum;
             hdr->header_csum = 0;
@@ -138,6 +147,23 @@ resume_1:
                 exit(1);
             }
             hdr->header_csum = csum;
+            bs->dsk.meta_format = BLOCKSTORE_META_FORMAT_HEAP;
+            if (hdr->block_id_bits != sizeof(heap_block_num_t))
+            {
+                printf("Heap metadata block ID size (%u) is not supported by this build\n", hdr->block_id_bits);
+                exit(1);
+            }
+        }
+        else if (hdr->version == BLOCKSTORE_META_FORMAT_V2)
+        {
+            uint32_t csum = hdr->header_csum;
+            hdr->header_csum = 0;
+            if (crc32c(0, hdr, BLOCKSTORE_META_HEADER_V2_SIZE) != csum)
+            {
+                printf("Metadata header is corrupt (checksum mismatch).\n");
+                exit(1);
+            }
+            hdr->header_csum = csum;
             if (bs->dsk.meta_format != BLOCKSTORE_META_FORMAT_V2)
             {
                 bs->dsk.meta_format = BLOCKSTORE_META_FORMAT_V2;
@@ -160,11 +186,11 @@ resume_1:
                 printf("Warning: Starting with metadata in the old format without checksums, as stored on disk\n");
             }
         }
-        else if (hdr->version > BLOCKSTORE_META_FORMAT_V2)
+        else
         {
             printf(
                 "Metadata format is too new for me (stored version is %ju, max supported %u).\n",
-                hdr->version, BLOCKSTORE_META_FORMAT_V2
+                hdr->version, BLOCKSTORE_META_FORMAT_HEAP
             );
             exit(1);
         }
@@ -189,7 +215,12 @@ resume_1:
     // Skip superblock
     md_offset = bs->dsk.meta_block_size;
     next_offset = md_offset;
-    entries_per_block = bs->dsk.meta_block_size / bs->dsk.clean_entry_size;
+    entries_per_block = bs->dsk.meta_block_size / bs->dsk.clean_entry_size; // FIXME only array
+    if (bs->dsk.meta_format == BLOCKSTORE_META_FORMAT_HEAP)
+    {
+        bs->heap_meta.blocks = new heap_meta_block_t[bs->dsk.meta_len / bs->dsk.meta_block_size];
+        bs->heap_meta.block_count = bs->dsk.meta_len / bs->dsk.meta_block_size;
+    }
     // Read the rest of the metadata
 resume_2:
     if (next_offset < bs->dsk.meta_len && submitted == 0)
@@ -233,9 +264,10 @@ resume_2:
             bool changed = false;
             for (uint64_t sector = 0; sector < bufs[i].size; sector += bs->dsk.meta_block_size)
             {
-                // handle <count> entries
+                auto this_changed = bs->dsk.meta_format == BLOCKSTORE_META_FORMAT_HEAP
-                if (handle_meta_block(bufs[i].buf + sector, entries_per_block,
+                    ? handle_heap_meta_block(bufs[i].buf + sector, bufs[i].offset + sector - md_offset)
-                    ((bufs[i].offset + sector - md_offset) / bs->dsk.meta_block_size) * entries_per_block))
+                    : handle_array_meta_block(bufs[i].buf + sector, bufs[i].offset + sector - md_offset);
+                if (this_changed)
                     changed = true;
             }
             if (changed && !bs->inmemory_meta && !bs->readonly)
@@ -262,6 +294,41 @@ resume_2:
         wait_state = 2;
         return 1;
     }
+    if (bs->dsk.meta_format == BLOCKSTORE_META_FORMAT_HEAP)
+    {
+        // build used_space index
+        for (heap_block_num_t i = 0; i < bs->heap_meta.block_count; i++)
+        {
+            bs->heap_meta.used_space_map.emplace(std::pair<uint32_t, heap_block_num_t>(bs->heap_meta.blocks[i].used_space, i));
+        }
+    }
+    if (heap_invalidated_block_seq.size() && !bs->readonly)
+    {
+        // zero out invalidated blocks not zeroed during the previous OSD execution
+        for (auto inv_seq: heap_invalidated_block_seq)
+        {
+            auto num_it = heap_block_by_seq.find(inv_seq);
+            if (num_it != heap_block_by_seq.end())
+                heap_invalidated_block_nums.push_back(num_it->second);
+        }
+        memset(metadata_buffer, 0, bs->dsk.meta_block_size);
+        for (i = 0; i < heap_invalidated_block_nums.size(); i++)
+        {
+            GET_SQE();
+            last_read_offset = heap_invalidated_block_nums[i]*bs->dsk.meta_block_size;
+            data->iov = { metadata_buffer, (size_t)bs->dsk.meta_block_size };
+            data->callback = [this](ring_data_t *data) { handle_event(data, -1); };
+            my_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + last_read_offset);
+            bs->ringloop->submit();
+            submitted++;
+resume_7:
+            if (submitted > 0)
+            {
+                wait_state = 7;
+                return 1;
+            }
+        }
+    }
     if (entries_to_zero.size() && !bs->inmemory_meta && !bs->readonly)
     {
         std::sort(entries_to_zero.begin(), entries_to_zero.end());
@@ -329,8 +396,9 @@ resume_6:
     return 0;
 }
 
-bool blockstore_init_meta::handle_meta_block(uint8_t *buf, uint64_t entries_per_block, uint64_t done_cnt)
+bool blockstore_init_meta::handle_array_meta_block(uint8_t *buf, uint64_t block_offset)
 {
+    uint64_t done_cnt = (block_offset / bs->dsk.meta_block_size) * entries_per_block;
     bool updated = false;
     uint64_t max_i = entries_per_block;
     if (max_i > bs->dsk.block_count-done_cnt)
@@ -429,6 +497,132 @@ bool blockstore_init_meta::handle_meta_block(uint8_t *buf, uint64_t entries_per_
     return updated;
 }
 
+static int bitmap_count_ones(uint8_t *bitmap, int size)
+{
+    int n = 0, i = 0;
+    for (; i <= size-sizeof(unsigned); i += sizeof(unsigned))
+    {
+        n += __builtin_popcount(*(unsigned*)(bitmap+i));
+    }
+    for (; i < size; i++)
+    {
+        n += __builtin_popcount(*(unsigned char*)(bitmap+i));
+    }
+    return n;
+}
+
+// v3 / heap / "cow" metadata block
+bool blockstore_init_meta::handle_heap_meta_block(uint8_t *buf, uint64_t block_offset)
+{
+    if ((block_offset / bs->dsk.meta_block_size) > (heap_block_num_t)-1)
+    {
+        fprintf(stderr, "Metadata area too large\n");
+        exit(1);
+    }
+    // Validate block CRC
+    uint32_t block_crc = *(uint32_t*)(buf + bs->dsk.meta_block_size - 4);
+    if (crc32c(0, buf, bs->dsk.meta_block_size-4) != block_crc)
+    {
+        return false;
+    }
+    // Validate header
+    heap_meta_block_header_t *hdr = (heap_meta_block_header_t*)buf;
+    if (hdr->magic != BLOCKSTORE_META_MAGIC_V1)
+    {
+        return false;
+    }
+    if (hdr->invalidates_blocks > (bs->dsk.meta_block_size-4-sizeof(heap_meta_block_header_t))/sizeof(uint64_t))
+    {
+        fprintf(stderr, "Metadata block at %jx contains too large invalidates_blocks count: %x\n", block_offset, hdr->invalidates_blocks);
+        exit(1);
+    }
+    if (heap_invalidated_block_seq.find(hdr->seq_num) != heap_invalidated_block_seq.end())
+    {
+        // Check if the block is invalidated and handled after the block that invalidates it
+        return false;
+    }
+    uint64_t hdr_size = sizeof(heap_meta_block_header_t) + hdr->invalidates_blocks*8;
+    heap_meta_block_t & blk = bs->heap_meta.blocks[block_offset/bs->dsk.meta_block_size];
+    blk.offset = block_offset;
+    blk.seq_num = hdr->seq_num;
+    blk.used_space = hdr_size + 4;
+    uint64_t *hdr_inv = (uint64_t*)(hdr + 1);
+    for (int i = 0; i < hdr->invalidates_blocks; i++)
+    {
+        blk.invalidates_blocks.push_back(hdr_inv[i]);
+        heap_invalidated_block_seq.insert(hdr_inv[i]);
+    }
+    heap_block_by_seq[hdr->seq_num] = block_offset;
+    // Process sub-blocks
+    uint64_t heap_entry_size = sizeof(heap_meta_entry_t) + bs->dsk.clean_dyn_size;
+    for (uint64_t pos = sizeof(heap_meta_block_header_t); pos < bs->dsk.meta_block_size-4; pos += heap_entry_size)
+    {
+        heap_meta_entry_t *diskentry = (heap_meta_entry_t*)(buf + pos);
+        if (!diskentry->oid.inode || !diskentry->version)
+        {
+            continue;
+        }
+        auto & clean_db = bs->clean_db_shard(diskentry->oid);
+        auto mementry = (clean_entry_heap_t*)(&clean_db[diskentry->oid]);
+        bool exists = mementry->version != 0;
+        if (exists && mementry->version >= diskentry->version)
+        {
+            if (mementry->version == diskentry->version)
+            {
+                // Voluntarily allow duplicates of in-memory entries with different
+                // bitmaps to support checksum updates with hole-punching
+                int old_count = bitmap_count_ones(mementry->bitmap, bs->dsk.clean_entry_bitmap_size);
+                int new_count = bitmap_count_ones(diskentry->bitmap, bs->dsk.clean_entry_bitmap_size);
+                if (old_count < new_count)
+                {
+                    continue;
+                }
+            }
+            else
+            {
+                continue;
+            }
+        }
+        blk.used_space += heap_entry_size;
+        if (exists && mementry->location != UINT64_MAX)
+        {
+            // free the previous block
+            uint64_t old_clean_loc = mementry->location >> bs->dsk.block_order;
+#ifdef BLOCKSTORE_DEBUG
+            printf("Free block %ju from %jx:%jx v%ju\n", 1+old_clean_loc,
+                diskentry->oid.inode, diskentry->oid.stripe, mementry->version);
+#endif
+            bs->data_alloc->set(old_clean_loc, false);
+            bs->inode_space_stats[diskentry->oid.inode] -= bs->dsk.data_block_size;
+            bs->used_blocks--;
+            bs->heap_meta.blocks[mementry->meta_block].used_space -= heap_entry_size;
+        }
+        if (diskentry->location != UINT64_MAX)
+        {
+            bs->data_alloc->set(diskentry->location >> bs->dsk.block_order, true);
+            bs->inode_space_stats[diskentry->oid.inode] += bs->dsk.data_block_size;
+            bs->used_blocks++;
+#ifdef BLOCKSTORE_DEBUG
+            printf("Allocate block (heap entry) %ju: %jx:%jx v%ju\n", 1 + (diskentry->location >> bs->dsk.block_order),
+                diskentry->oid.inode, diskentry->oid.stripe, diskentry->version);
+#endif
+        }
+        mementry->version = diskentry->version;
+        mementry->location = diskentry->location;
+        mementry->meta_block = block_offset / bs->dsk.meta_block_size;
+        mementry->block_offset = block_offset % bs->dsk.meta_block_size;
+        if (exists)
+        {
+            mementry->prev_versions++;
+        }
+        // Extra data: 2 bitmaps + checksums or just 2 bitmaps if inmemory_meta is disabled
+        memcpy(&mementry->bitmap, &diskentry->bitmap, bs->inmemory_meta ? bs->dsk.clean_dyn_size : 2*bs->dsk.clean_entry_bitmap_size);
+        entries_loaded++;
+    }
+    // We have to zero out headers of invalidated blocks, but we'll do it later
+    return false;
+}
+
 blockstore_init_journal::blockstore_init_journal(blockstore_impl_t *bs)
 {
     this->bs = bs;

src/blockstore/blockstore_init.h

@@ -28,7 +28,13 @@ class blockstore_init_meta
     unsigned entries_per_block = 0;
     int i = 0, j = 0;
     std::vector<uint64_t> entries_to_zero;
-    bool handle_meta_block(uint8_t *buf, uint64_t count, uint64_t done_cnt);
+
+    std::map<uint64_t, heap_block_num_t> heap_block_by_seq;
+    std::set<uint64_t> heap_invalidated_block_seq;
+    std::vector<heap_block_num_t> heap_invalidated_block_nums;
+
+    bool handle_array_meta_block(uint8_t *buf, uint64_t block_offset);
+    bool handle_heap_meta_block(uint8_t *buf, uint64_t block_offset);
     void handle_event(ring_data_t *data, int buf_num);
 public:
     blockstore_init_meta(blockstore_impl_t *bs);

src/blockstore/blockstore_open.cpp

@@ -111,6 +111,10 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init)
     {
         metadata_buf_size = 4*1024*1024;
     }
+    if (metadata_buf_size % dsk.meta_block_size)
+    {
+        metadata_buf_size = ((metadata_buf_size+dsk.meta_block_size-1) / dsk.meta_block_size) * dsk.meta_block_size;
+    }
     if (dsk.meta_device == dsk.data_device)
     {
         disable_meta_fsync = disable_data_fsync;

src/blockstore/blockstore_read.cpp

@@ -148,10 +148,14 @@ int blockstore_impl_t::fulfill_read(blockstore_op_t *read_op,
     return r;
 }
 
-uint8_t* blockstore_impl_t::get_clean_entry_bitmap(uint64_t block_loc, int offset)
+uint8_t* blockstore_impl_t::get_clean_entry_bitmap(blockstore_clean_db_t::iterator clean_it, int offset)
 {
+    if (dsk.meta_format == BLOCKSTORE_META_FORMAT_HEAP)
+    {
+        return ((uint8_t*)&clean_it->second) + sizeof(clean_entry_heap_t) + offset;
+    }
     uint8_t *clean_entry_bitmap;
-    uint64_t meta_loc = block_loc >> dsk.block_order;
+    uint64_t meta_loc = clean_it->second.location >> dsk.block_order;
     if (inmemory_meta)
     {
         uint64_t sector = (meta_loc / (dsk.meta_block_size / dsk.clean_entry_size)) * dsk.meta_block_size;
@@ -159,7 +163,9 @@ uint8_t* blockstore_impl_t::get_clean_entry_bitmap(uint64_t block_loc, int offse
         clean_entry_bitmap = ((uint8_t*)metadata_buffer + sector + pos*dsk.clean_entry_size + sizeof(clean_disk_entry) + offset);
     }
     else
+    {
         clean_entry_bitmap = (uint8_t*)(clean_bitmaps + meta_loc*2*dsk.clean_entry_bitmap_size + offset);
+    }
     return clean_entry_bitmap;
 }
 
@@ -433,7 +439,7 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
                 if (!IS_JOURNAL(dirty.state))
                 {
                     // Read from data disk, possibly checking checksums
-                    if (!fulfill_clean_read(read_op, fulfilled, bmp_ptr, dyn_data,
+                    if (!fulfill_clean_read_journal(read_op, fulfilled, bmp_ptr, dyn_data,
                         dirty.offset, dirty.offset+dirty.len, dirty.location, dirty_it->first.version))
                     {
                         goto undo_read;
@@ -464,14 +470,13 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
             result_version = clean_it->second.version;
             if (read_op->bitmap)
             {
-                void *bmp_ptr = get_clean_entry_bitmap(clean_it->second.location, dsk.clean_entry_bitmap_size);
+                void *bmp_ptr = get_clean_entry_bitmap(clean_it, dsk.clean_entry_bitmap_size);
                 memcpy(read_op->bitmap, bmp_ptr, dsk.clean_entry_bitmap_size);
             }
         }
         if (fulfilled < read_op->len)
         {
-            if (!fulfill_clean_read(read_op, fulfilled, NULL, NULL, 0, dsk.data_block_size,
+            if (!fulfill_clean_read_meta(read_op, fulfilled, clean_it))
-                clean_it->second.location, clean_it->second.version))
             {
                 goto undo_read;
             }
@@ -581,40 +586,22 @@ int blockstore_impl_t::pad_journal_read(std::vector<copy_buffer_t> & rv, copy_bu
     return 0;
 }
 
-bool blockstore_impl_t::fulfill_clean_read(blockstore_op_t *read_op, uint64_t & fulfilled,
+bool blockstore_impl_t::fulfill_clean_read_journal(blockstore_op_t *read_op, uint64_t & fulfilled,
     uint8_t *clean_entry_bitmap, int *dyn_data, uint32_t item_start, uint32_t item_end, uint64_t clean_loc, uint64_t clean_ver)
 {
-    bool from_journal = clean_entry_bitmap != NULL;
-    if (!clean_entry_bitmap)
-    {
-        // NULL clean_entry_bitmap means we're reading from data, not from the journal,
-        // and the bitmap location is obvious
-        clean_entry_bitmap = get_clean_entry_bitmap(clean_loc, 0);
-    }
     if (dsk.csum_block_size > dsk.bitmap_granularity)
     {
         auto & rv = PRIV(read_op)->read_vec;
-        int req = fill_partial_checksum_blocks(rv, fulfilled, clean_entry_bitmap, dyn_data, from_journal,
+        int req = fill_partial_checksum_blocks(rv, fulfilled, clean_entry_bitmap, dyn_data, true,
             (uint8_t*)read_op->buf, read_op->offset, read_op->offset+read_op->len);
-        if (!inmemory_meta && !from_journal && req > 0)
-        {
-            // Read checksums from disk
-            uint8_t *csum_buf = read_clean_meta_block(read_op, clean_loc, rv.size()-req);
-            for (int i = req; i > 0; i--)
-            {
-                rv[rv.size()-i].csum_buf = csum_buf;
-            }
-        }
         for (int i = req; i > 0; i--)
         {
             if (!read_checksum_block(read_op, i, fulfilled, clean_loc))
-            {
                 return false;
-            }
         }
         PRIV(read_op)->clean_block_used = req > 0;
     }
-    else if (from_journal)
+    else
     {
         // Don't scan bitmap - journal writes don't have holes (internal bitmap)!
         uint8_t *csum = !dsk.csum_block_size ? 0 : (clean_entry_bitmap + dsk.clean_entry_bitmap_size +
@@ -635,6 +622,43 @@ bool blockstore_impl_t::fulfill_clean_read(blockstore_op_t *read_op, uint64_t &
             assert(fulfill_read(read_op, fulfilled, item_end, dsk.data_block_size, (BS_ST_DELETE | BS_ST_STABLE), 0, 0, 0, NULL, NULL));
         }
     }
+    // Increment reference counter if clean data is being read from the disk
+    if (PRIV(read_op)->clean_block_used)
+    {
+        auto & uo = used_clean_objects[clean_loc];
+        uo.refs++;
+        if (dsk.csum_block_size && flusher->is_mutated(clean_loc))
+            uo.was_changed = true;
+        PRIV(read_op)->clean_block_used = clean_loc;
+    }
+    return true;
+}
+
+bool blockstore_impl_t::fulfill_clean_read_meta(blockstore_op_t *read_op, uint64_t & fulfilled, blockstore_clean_db_t::iterator clean_it)
+{
+    uint8_t *clean_entry_bitmap = get_clean_entry_bitmap(clean_it, 0);
+    uint64_t clean_loc = clean_it->second.location;
+    if (dsk.csum_block_size > dsk.bitmap_granularity)
+    {
+        auto & rv = PRIV(read_op)->read_vec;
+        int req = fill_partial_checksum_blocks(rv, fulfilled, clean_entry_bitmap, NULL, false,
+            (uint8_t*)read_op->buf, read_op->offset, read_op->offset+read_op->len);
+        if (!inmemory_meta && req > 0)
+        {
+            // Read checksums from disk
+            uint8_t *csum_buf = read_clean_meta_block(read_op, clean_it, rv.size()-req);
+            for (int i = req; i > 0; i--)
+            {
+                rv[rv.size()-i].csum_buf = csum_buf;
+            }
+        }
+        for (int i = req; i > 0; i--)
+        {
+            if (!read_checksum_block(read_op, i, fulfilled, clean_loc))
+                return false;
+        }
+        PRIV(read_op)->clean_block_used = req > 0;
+    }
     else
     {
         bool csum_done = !dsk.csum_block_size || inmemory_meta;
@@ -662,13 +686,13 @@ bool blockstore_impl_t::fulfill_clean_read(blockstore_op_t *read_op, uint64_t &
                 if (!csum_done)
                 {
                     // Read checksums from disk
-                    csum_buf = read_clean_meta_block(read_op, clean_loc, PRIV(read_op)->read_vec.size());
+                    csum_buf = read_clean_meta_block(read_op, clean_it, PRIV(read_op)->read_vec.size());
                     csum_done = true;
                 }
                 uint8_t *csum = !dsk.csum_block_size ? 0 : (csum_buf + 2*dsk.clean_entry_bitmap_size + bmp_start*(dsk.data_csum_type & 0xFF));
                 if (!fulfill_read(read_op, fulfilled, bmp_start * dsk.bitmap_granularity,
                     bmp_end * dsk.bitmap_granularity, (BS_ST_BIG_WRITE | BS_ST_STABLE), 0,
-                    clean_loc + bmp_start * dsk.bitmap_granularity, 0, csum, dyn_data))
+                    clean_loc + bmp_start * dsk.bitmap_granularity, 0, csum, NULL))
                 {
                     return false;
                 }
@@ -688,11 +712,22 @@ bool blockstore_impl_t::fulfill_clean_read(blockstore_op_t *read_op, uint64_t &
     return true;
 }
 
-uint8_t* blockstore_impl_t::read_clean_meta_block(blockstore_op_t *op, uint64_t clean_loc, int rv_pos)
+uint8_t* blockstore_impl_t::read_clean_meta_block(blockstore_op_t *op, blockstore_clean_db_t::iterator clean_it, int rv_pos)
 {
+    uint64_t sector, pos;
     auto & rv = PRIV(op)->read_vec;
-    auto sector = ((clean_loc >> dsk.block_order) / (dsk.meta_block_size / dsk.clean_entry_size)) * dsk.meta_block_size;
+    if (dsk.meta_format == BLOCKSTORE_META_FORMAT_HEAP)
-    auto pos = ((clean_loc >> dsk.block_order) % (dsk.meta_block_size / dsk.clean_entry_size)) * dsk.clean_entry_size;
+    {
+        auto clean_heap_entry = (clean_entry_heap_t*)(&clean_it->second);
+        sector = clean_heap_entry->meta_block * dsk.meta_block_size;
+        pos = clean_heap_entry->block_offset;
+    }
+    else
+    {
+        auto clean_loc = clean_it->second.location;
+        sector = ((clean_loc >> dsk.block_order) / (dsk.meta_block_size / dsk.clean_entry_size)) * dsk.meta_block_size;
+        pos = ((clean_loc >> dsk.block_order) % (dsk.meta_block_size / dsk.clean_entry_size)) * dsk.clean_entry_size;
+    }
     uint8_t *buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.meta_block_size);
     rv.insert(rv.begin()+rv_pos, (copy_buffer_t){
         .copy_flags = COPY_BUF_META_BLOCK|COPY_BUF_CSUM_FILL,
@@ -807,11 +842,6 @@ bool blockstore_impl_t::verify_clean_padded_checksums(blockstore_op_t *op, uint6
     if (from_journal)
         return verify_padded_checksums(dyn_data, dyn_data + dsk.clean_entry_bitmap_size, offset, iov, n_iov, bad_block_cb);
     clean_loc = (clean_loc >> dsk.block_order) << dsk.block_order;
-    if (!dyn_data)
-    {
-        assert(inmemory_meta);
-        dyn_data = get_clean_entry_bitmap(clean_loc, 0);
-    }
     return verify_padded_checksums(dyn_data, dyn_data + 2*dsk.clean_entry_bitmap_size, offset, iov, n_iov, bad_block_cb);
 }
 
@@ -869,8 +899,18 @@ void blockstore_impl_t::handle_read_event(ring_data_t *data, blockstore_op_t *op
                         auto & uo = used_clean_objects.at((rv[i].disk_offset >> dsk.block_order) << dsk.block_order);
                         if (!uo.was_changed)
                         {
+                            bool from_journal = (rv[i].copy_flags & COPY_BUF_JOURNALED_BIG);
+                            auto csum_buf = rv[i].csum_buf;
+                            if (!from_journal && !csum_buf)
+                            {
+                                assert(inmemory_meta);
+                                auto & clean_db = clean_db_shard(op->oid);
+                                auto clean_it = clean_db.find(op->oid);
+                                assert(clean_it != clean_db.end());
+                                csum_buf = get_clean_entry_bitmap(clean_it, 0);
+                            }
                             verify_clean_padded_checksums(
-                                op, rv[i].disk_offset, rv[i].csum_buf, (rv[i].copy_flags & COPY_BUF_JOURNALED_BIG), iov, n_iov,
+                                op, rv[i].disk_offset, csum_buf, from_journal, iov, n_iov,
                                 [&](uint32_t bad_block, uint32_t calc_csum, uint32_t stored_csum)
                                 {
                                     ok = false;
@@ -1019,7 +1059,7 @@ int blockstore_impl_t::read_bitmap(object_id oid, uint64_t target_version, void
             *result_version = clean_it->second.version;
         if (bitmap)
         {
-            void *bmp_ptr = get_clean_entry_bitmap(clean_it->second.location, dsk.clean_entry_bitmap_size);
+            void *bmp_ptr = get_clean_entry_bitmap(clean_it, dsk.clean_entry_bitmap_size);
             memcpy(bitmap, bmp_ptr, dsk.clean_entry_bitmap_size);
         }
         return 0;

src/blockstore/blockstore_write.cpp

@@ -57,7 +57,7 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
             version = clean_it->second.version + 1;
             if (!is_del)
             {
-                void *bmp_ptr = get_clean_entry_bitmap(clean_it->second.location, dsk.clean_entry_bitmap_size);
+                void *bmp_ptr = get_clean_entry_bitmap(clean_it, dsk.clean_entry_bitmap_size);
                 memcpy(dyn_ptr, bmp_ptr, dsk.clean_entry_bitmap_size);
             }
         }
@@ -341,7 +341,7 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
             cancel_all_writes(op, dirty_it, -ENOSPC);
             return 2;
         }
-        if (inmemory_meta)
+        if (inmemory_meta && dsk.meta_format != BLOCKSTORE_META_FORMAT_HEAP)
         {
             // Check once more that metadata entry is zeroed (the reverse means a bug or corruption)
             uint64_t sector = (loc / (dsk.meta_block_size / dsk.clean_entry_size)) * dsk.meta_block_size;

src/client/CMakeLists.txt

@@ -93,7 +93,7 @@ add_executable(test_cluster_client
 	EXCLUDE_FROM_ALL
 	../test/test_cluster_client.cpp
 	pg_states.cpp osd_ops.cpp cluster_client.cpp cluster_client_list.cpp cluster_client_wb.cpp msgr_op.cpp ../test/mock/messenger.cpp msgr_stop.cpp
-	etcd_state_client.cpp ../util/timerfd_manager.cpp ../util/str_util.cpp ../util/json_util.cpp ../../json11/json11.cpp
+	etcd_state_client.cpp ../util/timerfd_manager.cpp ../util/addr_util.cpp ../util/str_util.cpp ../util/json_util.cpp ../../json11/json11.cpp
 )
 target_compile_definitions(test_cluster_client PUBLIC -D__MOCK__)
 target_include_directories(test_cluster_client BEFORE PUBLIC ${CMAKE_SOURCE_DIR}/src/test/mock)

src/client/cluster_client.cpp

@@ -3,6 +3,7 @@
 
 #include <stdexcept>
 #include <assert.h>
+#include "pg_states.h"
 #include "cluster_client_impl.h"
 #include "json_util.h"
 
@@ -57,6 +58,7 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
     st_cli.on_change_osd_state_hook = [this](uint64_t peer_osd) { on_change_osd_state_hook(peer_osd); };
     st_cli.on_change_pool_config_hook = [this]() { on_change_pool_config_hook(); };
     st_cli.on_change_pg_state_hook = [this](pool_id_t pool_id, pg_num_t pg_num, osd_num_t prev_primary) { on_change_pg_state_hook(pool_id, pg_num, prev_primary); };
+    st_cli.on_change_node_placement_hook = [this]() { on_change_node_placement_hook(); };
     st_cli.on_load_pgs_hook = [this](bool success) { on_load_pgs_hook(success); };
     st_cli.on_reload_hook = [this]() { st_cli.load_global_config(); };
 
@@ -470,11 +472,95 @@ void cluster_client_t::on_load_config_hook(json11::Json::object & etcd_global_co
     }
     // log_level
     log_level = config["log_level"].uint64_value();
+    // hostname
+    conf_hostname = config["hostname"].string_value();
+    auto new_hostname = conf_hostname != "" ? conf_hostname : gethostname_str();
+    if (new_hostname != client_hostname)
+    {
+        self_tree_metrics.clear();
+        client_hostname = new_hostname;
+    }
     msgr.parse_config(config);
     st_cli.parse_config(config);
     st_cli.load_pgs();
 }
 
+osd_num_t cluster_client_t::select_random_osd(const std::vector<osd_num_t> & osds)
+{
+    osd_num_t alive_set[osds.size()];
+    int alive_count = 0;
+    for (auto & osd_num: osds)
+    {
+        if (!st_cli.peer_states[osd_num].is_null())
+            alive_set[alive_count++] = osd_num;
+    }
+    if (!alive_count)
+        return 0;
+    return alive_set[lrand48() % alive_count];
+}
+
+osd_num_t cluster_client_t::select_nearest_osd(const std::vector<osd_num_t> & osds)
+{
+    if (!self_tree_metrics.size())
+    {
+        std::string cur_id = client_hostname;
+        int metric = 0;
+        while (self_tree_metrics.find(cur_id) == self_tree_metrics.end())
+        {
+            self_tree_metrics[cur_id] = metric++;
+            json11::Json cur_placement = st_cli.node_placement[cur_id];
+            cur_id = cur_placement["parent"].string_value();
+        }
+        if (cur_id != "")
+        {
+            self_tree_metrics[""] = metric++;
+        }
+    }
+    osd_num_t best_osd = 0;
+    int best_metric = -1;
+    for (auto & osd_num: osds)
+    {
+        int metric = -1;
+        auto met_it = osd_tree_metrics.find(osd_num);
+        if (met_it != osd_tree_metrics.end())
+        {
+            metric = met_it->second;
+        }
+        else
+        {
+            auto & peer_state = st_cli.peer_states[osd_num];
+            if (!peer_state.is_null())
+            {
+                metric = self_tree_metrics[""];
+                bool first = true;
+                std::string cur_id = std::to_string(osd_num);
+                std::set<std::string> seen;
+                while (seen.find(cur_id) == seen.end())
+                {
+                    seen.insert(cur_id);
+                    json11::Json cur_placement = st_cli.node_placement[cur_id];
+                    std::string cur_parent = cur_placement["parent"].string_value();
+                    cur_id = (!first || cur_parent != "" ? cur_parent : peer_state["host"].string_value());
+                    first = false;
+                    auto self_it = self_tree_metrics.find(cur_id);
+                    if (self_it != self_tree_metrics.end())
+                    {
+                        metric = self_it->second;
+                        break;
+                    }
+                }
+            }
+            osd_tree_metrics[osd_num] = metric;
+        }
+        if (metric >= 0 && (best_metric < 0 || metric < best_metric))
+        {
+            best_metric = metric;
+            best_osd = osd_num;
+        }
+    }
+    return best_osd;
+}
+
 void cluster_client_t::on_load_pgs_hook(bool success)
 {
     for (auto pool_item: st_cli.pool_config)
@@ -546,6 +632,7 @@ bool cluster_client_t::get_immediate_commit(uint64_t inode)
 
 void cluster_client_t::on_change_osd_state_hook(uint64_t peer_osd)
 {
+    osd_tree_metrics.erase(peer_osd);
     if (msgr.wanted_peers.find(peer_osd) != msgr.wanted_peers.end())
     {
         msgr.connect_peer(peer_osd, st_cli.peer_states[peer_osd]);
@@ -553,6 +640,12 @@ void cluster_client_t::on_change_osd_state_hook(uint64_t peer_osd)
     }
 }
 
+void cluster_client_t::on_change_node_placement_hook()
+{
+    osd_tree_metrics.clear();
+    self_tree_metrics.clear();
+}
+
 bool cluster_client_t::is_ready()
 {
     return pgs_loaded;
@@ -1221,6 +1314,17 @@ int cluster_client_t::try_send(cluster_op_t *op, int i)
         !pg_it->second.pause && pg_it->second.cur_primary)
     {
         osd_num_t primary_osd = pg_it->second.cur_primary;
+        if (pool_cfg.local_reads != POOL_LOCAL_READ_PRIMARY &&
+            pool_cfg.scheme == POOL_SCHEME_REPLICATED &&
+            (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP) &&
+            (pg_it->second.cur_state == PG_ACTIVE || pg_it->second.cur_state == (PG_ACTIVE|PG_LEFT_ON_DEAD)))
+        {
+            osd_num_t nearest_osd = pool_cfg.local_reads == POOL_LOCAL_READ_NEAREST
+                ? select_nearest_osd(pg_it->second.target_set)
+                : select_random_osd(pg_it->second.target_set);
+            if (nearest_osd)
+                primary_osd = nearest_osd;
+        }
         part->osd_num = primary_osd;
         auto peer_it = msgr.osd_peer_fds.find(primary_osd);
         if (peer_it != msgr.osd_peer_fds.end())

src/client/cluster_client.h

@@ -100,6 +100,7 @@ public:
     uint64_t client_max_buffered_bytes = 0;
     uint64_t client_max_buffered_ops = 0;
     uint64_t client_max_writeback_iodepth = 0;
+    std::string conf_hostname;
 
     int log_level = 0;
     int client_retry_interval = 50; // ms
@@ -107,6 +108,10 @@ public:
     bool client_retry_enospc = true;
     int client_wait_up_timeout = 16; // sec (for listings)
 
+    std::string client_hostname;
+    std::map<std::string, int> self_tree_metrics;
+    std::map<osd_num_t, int> osd_tree_metrics;
+
     int retry_timeout_id = -1;
     int retry_timeout_duration = 0;
     std::vector<cluster_op_t*> offline_ops;
@@ -161,11 +166,14 @@ protected:
 protected:
     bool affects_osd(uint64_t inode, uint64_t offset, uint64_t len, osd_num_t osd);
     bool affects_pg(uint64_t inode, uint64_t offset, uint64_t len, pool_id_t pool_id, pg_num_t pg_num);
+
     void on_load_config_hook(json11::Json::object & config);
     void on_load_pgs_hook(bool success);
     void on_change_pool_config_hook();
     void on_change_pg_state_hook(pool_id_t pool_id, pg_num_t pg_num, osd_num_t prev_primary);
     void on_change_osd_state_hook(uint64_t peer_osd);
+    void on_change_node_placement_hook();
+
     void execute_internal(cluster_op_t *op);
     void unshift_op(cluster_op_t *op);
     int continue_rw(cluster_op_t *op);
@@ -191,5 +199,8 @@ protected:
     bool check_finish_listing(inode_list_t *lst);
     void continue_raw_ops(osd_num_t peer_osd);
 
+    osd_num_t select_random_osd(const std::vector<osd_num_t> & osds);
+    osd_num_t select_nearest_osd(const std::vector<osd_num_t> & osds);
+
     friend class writeback_cache_t;
 };

src/client/etcd_state_client.cpp

@@ -922,6 +922,19 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
                 pc.used_for_app = "fs:"+pc.used_for_app;
             else
                 pc.used_for_app = pool_item.second["used_for_app"].as_string();
+            // Local Read Configuration
+            std::string local_reads = pool_item.second["local_reads"].string_value();
+            if (local_reads == "nearest")
+                pc.local_reads = POOL_LOCAL_READ_NEAREST;
+            else if (local_reads == "random")
+                pc.local_reads = POOL_LOCAL_READ_RANDOM;
+            else if (local_reads == "" || local_reads == "primary")
+                pc.local_reads = POOL_LOCAL_READ_PRIMARY;
+            else
+            {
+                pc.local_reads = POOL_LOCAL_READ_PRIMARY;
+                fprintf(stderr, "Warning: Pool %u has invalid local_reads, using 'primary'\n", pool_id);
+            }
             // Immediate Commit Mode
             pc.immediate_commit = pool_item.second["immediate_commit"].is_string()
                 ? parse_immediate_commit(pool_item.second["immediate_commit"].string_value(), IMMEDIATE_ALL)
@@ -1256,6 +1269,13 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
             }
         }
     }
+    else if (key == etcd_prefix+"/config/node_placement")
+    {
+        // <etcd_prefix>/config/node_placement
+        node_placement = value;
+        if (on_change_node_placement_hook)
+            on_change_node_placement_hook();
+    }
 }
 
 uint32_t etcd_state_client_t::parse_immediate_commit(const std::string & immediate_commit_str, uint32_t default_value)

src/client/etcd_state_client.h

@@ -25,6 +25,10 @@
 #define IMMEDIATE_ALL 2
 #endif
 
+#define POOL_LOCAL_READ_PRIMARY 0
+#define POOL_LOCAL_READ_NEAREST 1
+#define POOL_LOCAL_READ_RANDOM 2
+
 struct etcd_kv_t
 {
     std::string key;
@@ -48,21 +52,22 @@ struct pg_config_t
 
 struct pool_config_t
 {
-    bool exists;
+    bool exists = false;
-    pool_id_t id;
+    pool_id_t id = 0;
     std::string name;
-    uint64_t scheme;
+    uint64_t scheme = 0;
-    uint64_t pg_size, pg_minsize, parity_chunks;
+    uint64_t pg_size = 0, pg_minsize = 0, parity_chunks = 0;
-    uint32_t data_block_size, bitmap_granularity, immediate_commit;
+    uint32_t data_block_size = 0, bitmap_granularity = 0, immediate_commit = 0;
-    uint64_t pg_count;
+    uint64_t pg_count = 0;
-    uint64_t real_pg_count;
+    uint64_t real_pg_count = 0;
     std::string failure_domain;
-    uint64_t max_osd_combinations;
+    uint64_t max_osd_combinations = 0;
-    uint64_t pg_stripe_size;
+    uint64_t pg_stripe_size = 0;
     std::map<pg_num_t, pg_config_t> pg_config;
-    uint64_t scrub_interval;
+    uint64_t scrub_interval = 0;
     std::string used_for_app;
-    int backfillfull;
+    int backfillfull = 0;
+    int local_reads = 0;
 };
 
 struct inode_config_t
@@ -130,6 +135,7 @@ public:
     std::set<osd_num_t> seen_peers;
     std::map<inode_t, inode_config_t> inode_config;
     std::map<std::string, inode_t> inode_by_name;
+    json11::Json node_placement;
 
     std::function<void(std::map<std::string, etcd_kv_t> &)> on_change_hook;
     std::function<void(json11::Json::object &)> on_load_config_hook;
@@ -140,6 +146,7 @@ public:
     std::function<void(pool_id_t, pg_num_t, osd_num_t)> on_change_pg_state_hook;
     std::function<void(pool_id_t, pg_num_t)> on_change_pg_history_hook;
     std::function<void(osd_num_t)> on_change_osd_state_hook;
+    std::function<void()> on_change_node_placement_hook;
     std::function<void()> on_reload_hook;
     std::function<void(inode_t, bool)> on_inode_change_hook;
     std::function<void(http_co_t *)> on_start_watcher_hook;

src/client/messenger.cpp

@@ -638,7 +638,7 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl)
         // Inform that we're OSD <osd_num>
         payload["osd_num"] = osd_num;
     }
-    payload["check_sequencing"] = true;
+    payload["features"] = json11::Json::object{ { "check_sequencing", true } };
 #ifdef WITH_RDMA
     if (!use_rdmacm && rdma_contexts.size())
     {
@@ -773,12 +773,15 @@ void osd_messenger_t::accept_connections(int listen_fd)
         fcntl(peer_fd, F_SETFL, fcntl(peer_fd, F_GETFL, 0) | O_NONBLOCK);
         int one = 1;
         setsockopt(peer_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
-        clients[peer_fd] = new osd_client_t();
+        auto cl = new osd_client_t();
-        clients[peer_fd]->peer_addr = addr;
+        clients[peer_fd] = cl;
-        clients[peer_fd]->peer_port = ntohs(((sockaddr_in*)&addr)->sin_port);
+        cl->is_incoming = true;
-        clients[peer_fd]->peer_fd = peer_fd;
+        cl->peer_addr = addr;
-        clients[peer_fd]->peer_state = PEER_CONNECTED;
+        cl->peer_addr = addr;
-        clients[peer_fd]->in_buf = malloc_or_die(receive_buffer_size);
+        cl->peer_port = ntohs(((sockaddr_in*)&addr)->sin_port);
+        cl->peer_fd = peer_fd;
+        cl->peer_state = PEER_CONNECTED;
+        cl->in_buf = malloc_or_die(receive_buffer_size);
         // Add FD to epoll
         tfd->set_fd_handler(peer_fd, false, [this](int peer_fd, int epoll_events)
         {

src/client/messenger.h

@@ -60,6 +60,7 @@ struct osd_client_t
     int ping_time_remaining = 0;
     int idle_time_remaining = 0;
     osd_num_t osd_num = 0;
+    bool is_incoming = false;
 
     void *in_buf = NULL;
 
@@ -77,6 +78,7 @@ struct osd_client_t
     osd_op_buf_list_t recv_list;
     uint64_t read_op_id = 1;
     bool check_sequencing = false;
+    bool enable_pg_locks = false;
 
     // Incoming operations
     std::vector<osd_op_t*> received_ops;

src/client/msgr_rdmacm.cpp

@@ -510,6 +510,7 @@ void osd_messenger_t::rdmacm_established(rdma_cm_event *ev)
     rc->qp = conn->cmid->qp;
     // And an osd_client_t
     auto cl = new osd_client_t();
+    cl->is_incoming = true;
     cl->peer_addr = conn->parsed_addr;
     cl->peer_port = conn->rdmacm_port;
     cl->peer_fd = conn->peer_fd;

src/client/osd_ops.cpp

@@ -23,4 +23,5 @@ const char* osd_op_names[] = {
     "sec_read_bmp",
     "scrub",
     "describe",
+    "sec_lock",
 };

src/client/osd_ops.h

@@ -31,10 +31,13 @@
 #define OSD_OP_SEC_READ_BMP         16
 #define OSD_OP_SCRUB                17
 #define OSD_OP_DESCRIBE             18
-#define OSD_OP_MAX                  18
+#define OSD_OP_SEC_LOCK             19
+#define OSD_OP_MAX                  19
 #define OSD_RW_MAX                  64*1024*1024
 #define OSD_PROTOCOL_VERSION        1
+
 #define OSD_OP_RECOVERY_RELATED     (uint32_t)1
+#define OSD_OP_IGNORE_PG_LOCK       (uint32_t)2
 
 // Memory alignment for direct I/O (usually 512 bytes)
 #ifndef DIRECT_IO_ALIGNMENT
@@ -56,6 +59,9 @@
 #define OSD_DEL_SUPPORT_LEFT_ON_DEAD 1
 #define OSD_DEL_LEFT_ON_DEAD         2
 
+#define OSD_SEC_LOCK_PG 1
+#define OSD_SEC_UNLOCK_PG 2
+
 // common request and reply headers
 struct __attribute__((__packed__)) osd_op_header_t
 {
@@ -94,7 +100,7 @@ struct __attribute__((__packed__)) osd_op_sec_rw_t
     uint32_t len;
     // bitmap/attribute length - bitmap comes after header, but before data
     uint32_t attr_len;
-    // the only possible flag is OSD_OP_RECOVERY_RELATED
+    // OSD_OP_RECOVERY_RELATED, OSD_OP_IGNORE_PG_LOCK
     uint32_t flags;
 };
 
@@ -116,7 +122,7 @@ struct __attribute__((__packed__)) osd_op_sec_del_t
     object_id oid;
     // delete version (automatic or specific)
     uint64_t version;
-    // the only possible flag is OSD_OP_RECOVERY_RELATED
+    // OSD_OP_RECOVERY_RELATED, OSD_OP_IGNORE_PG_LOCK
     uint32_t flags;
     uint32_t pad0;
 };
@@ -131,7 +137,7 @@ struct __attribute__((__packed__)) osd_reply_sec_del_t
 struct __attribute__((__packed__)) osd_op_sec_sync_t
 {
     osd_op_header_t header;
-    // the only possible flag is OSD_OP_RECOVERY_RELATED
+    // OSD_OP_RECOVERY_RELATED, OSD_OP_IGNORE_PG_LOCK
     uint32_t flags;
     uint32_t pad0;
 };
@@ -147,7 +153,7 @@ struct __attribute__((__packed__)) osd_op_sec_stab_t
     osd_op_header_t header;
     // obj_ver_id array length in bytes
     uint64_t len;
-    // the only possible flag is OSD_OP_RECOVERY_RELATED
+    // OSD_OP_RECOVERY_RELATED, OSD_OP_IGNORE_PG_LOCK
     uint32_t flags;
     uint32_t pad0;
 };
@@ -165,6 +171,8 @@ struct __attribute__((__packed__)) osd_op_sec_read_bmp_t
     osd_op_header_t header;
     // obj_ver_id array length in bytes
     uint64_t len;
+    // OSD_OP_RECOVERY_RELATED, OSD_OP_IGNORE_PG_LOCK
+    uint32_t flags;
 };
 
 struct __attribute__((__packed__)) osd_reply_sec_read_bmp_t
@@ -173,7 +181,7 @@ struct __attribute__((__packed__)) osd_reply_sec_read_bmp_t
     osd_reply_header_t header;
 };
 
-// show configuration
+// show configuration and remember peer information
 struct __attribute__((__packed__)) osd_op_show_config_t
 {
     osd_op_header_t header;
@@ -303,6 +311,25 @@ struct __attribute__((__packed__)) osd_reply_describe_item_t
     osd_num_t osd_num;  // OSD number
 };
 
+// lock/unlock PG for use by a primary OSD
+struct __attribute__((__packed__)) osd_op_sec_lock_t
+{
+    osd_op_header_t header;
+    // OSD_SEC_LOCK_PG or OSD_SEC_UNLOCK_PG
+    uint64_t flags;
+    // Pool ID and PG number
+    uint64_t pool_id;
+    uint64_t pg_num;
+    // PG state as calculated by the primary OSD
+    uint64_t pg_state;
+};
+
+struct __attribute__((__packed__)) osd_reply_sec_lock_t
+{
+    osd_reply_header_t header;
+    uint64_t cur_primary;
+};
+
 // FIXME it would be interesting to try to unify blockstore_op and osd_op formats
 union osd_any_op_t
 {
@@ -313,6 +340,7 @@ union osd_any_op_t
     osd_op_sec_stab_t sec_stab;
     osd_op_sec_read_bmp_t sec_read_bmp;
     osd_op_sec_list_t sec_list;
+    osd_op_sec_lock_t sec_lock;
     osd_op_show_config_t show_conf;
     osd_op_rw_t rw;
     osd_op_sync_t sync;
@@ -329,6 +357,7 @@ union osd_any_reply_t
     osd_reply_sec_stab_t sec_stab;
     osd_reply_sec_read_bmp_t sec_read_bmp;
     osd_reply_sec_list_t sec_list;
+    osd_reply_sec_lock_t sec_lock;
     osd_reply_show_config_t show_conf;
     osd_reply_rw_t rw;
     osd_reply_del_t del;

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 2.1.0
+Version: 2.2.0
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/cmd/cli.cpp

@@ -185,6 +185,7 @@ static const char* help_text =
     "    --immediate_commit all        Put pool only on OSDs with this or larger immediate_commit (none < small < all)\n"
     "    --level_placement <rules>     Use additional failure domain rules (example: \"dc=112233\")\n"
     "    --raw_placement <rules>       Specify raw PG generation rules (see documentation for details)\n"
+    "    --local_reads primary         Local read policy for replicated pools: primary, nearest or random\n"
     "    --primary_affinity_tags tags  Prefer to put primary copies on OSDs with all specified tags\n"
     "    --scrub_interval <time>       Enable regular scrubbing for this pool. Format: number + unit s/m/h/d/M/y\n"
     "    --used_for_app fs:<name>      Mark pool as used for VitastorFS with metadata in image <name>\n"
@@ -282,6 +283,7 @@ static json11::Json::object parse_args(int narg, const char *args[])
                 !strcmp(opt, "readonly") || !strcmp(opt, "readwrite") ||
                 !strcmp(opt, "force") || !strcmp(opt, "reverse") ||
                 !strcmp(opt, "allow-data-loss") || !strcmp(opt, "allow_data_loss") ||
+                !strcmp(opt, "allow-up") || !strcmp(opt, "allow_up") ||
                 !strcmp(opt, "down-ok") || !strcmp(opt, "down_ok") ||
                 !strcmp(opt, "dry-run") || !strcmp(opt, "dry_run") ||
                 !strcmp(opt, "help") || !strcmp(opt, "all") ||

src/cmd/cli_fix.cpp

@@ -200,6 +200,7 @@ struct cli_fix_t
                                         .stripe = op->req.describe.min_offset | items[i].role,
                                     },
                                     .version = 0,
+                                    .flags = OSD_OP_IGNORE_PG_LOCK,
                                 },
                             };
                             rm_op->callback = [this, primary_osd, rm_osd_num, rm_count, &obj](osd_op_t *rm_op)

src/cmd/cli_pool_cfg.cpp

@@ -91,7 +91,7 @@ std::string validate_pool_config(json11::Json::object & new_cfg, json11::Json ol
         }
         else if (key == "name" || key == "scheme" || key == "immediate_commit" ||
             key == "failure_domain" || key == "root_node" || key == "scrub_interval" || key == "used_for_app" ||
-            key == "used_for_fs" || key == "raw_placement")
+            key == "used_for_fs" || key == "raw_placement" || key == "local_reads")
         {
             if (!value.is_string())
             {
@@ -165,6 +165,10 @@ std::string validate_pool_config(json11::Json::object & new_cfg, json11::Json ol
         new_cfg["used_for_app"] = "fs:"+new_cfg["used_for_fs"].string_value();
         new_cfg.erase("used_for_fs");
     }
+    if (new_cfg.find("local_reads") != new_cfg.end() && new_cfg["local_reads"].string_value() == "primary")
+    {
+        new_cfg.erase("local_reads");
+    }
 
     // Prevent autovivification of object keys. Now we don't modify the config, we just check it
     json11::Json cfg = new_cfg;
@@ -340,5 +344,19 @@ std::string validate_pool_config(json11::Json::object & new_cfg, json11::Json ol
         }
     }
 
+    // local_reads
+    if (!cfg["local_reads"].is_null())
+    {
+        auto lr = cfg["local_reads"].string_value();
+        if (lr != "" && lr != "primary" && lr != "nearest" && lr != "random")
+        {
+            return "local_reads must be '', 'primary', 'nearest' or 'random', but it is "+cfg["local_reads"].string_value();
+        }
+        if (lr != "" && lr != "primary" && scheme != POOL_SCHEME_REPLICATED)
+        {
+            return "EC pools don't support localized reads, please clear local_reads or set it to 'primary'";
+        }
+    }
+
     return "";
 }

src/cmd/cli_pool_ls.cpp

@@ -504,6 +504,7 @@ resume_3:
                 { "failure_domain", "Failure domain" },
                 { "root_node", "Root node" },
                 { "osd_tags_fmt", "OSD tags" },
+                { "local_reads", "Local read policy" },
                 { "primary_affinity_tags_fmt", "Primary affinity" },
                 { "block_size_fmt", "Block size" },
                 { "bitmap_granularity_fmt", "Bitmap granularity" },

src/cmd/cli_rm_osd.cpp

@@ -15,7 +15,7 @@ struct rm_osd_t
 {
     cli_tool_t *parent;
 
-    bool dry_run, force_warning, force_dataloss;
+    bool dry_run, force_warning, force_dataloss, allow_up;
     uint64_t etcd_tx_retry_ms = 500;
     uint64_t etcd_tx_retries = 10000;
     std::vector<uint64_t> osd_ids;
@@ -168,7 +168,7 @@ struct rm_osd_t
                                 : strtoupper(e["effect"].string_value())+" PGs"))
                 )+" after deleting OSD(s).\n";
             }
-            if (still_up.size())
+            if (still_up.size() && !allow_up)
                 error += (still_up.size() == 1 ? "OSD " : "OSDs ") + implode(", ", still_up) +
                     (still_up.size() == 1 ? "is" : "are") + " still up. Use `vitastor-disk purge` to delete them.\n";
             if (is_dataloss && !force_dataloss && !dry_run)
@@ -476,6 +476,7 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_rm_osd(json11::Json cfg)
     auto rm_osd = new rm_osd_t();
     rm_osd->parent = this;
     rm_osd->dry_run = cfg["dry_run"].bool_value();
+    rm_osd->allow_up = cfg["allow_up"].bool_value();
     rm_osd->force_dataloss = cfg["allow_data_loss"].bool_value();
     rm_osd->force_warning = rm_osd->force_dataloss || cfg["force"].bool_value();
     if (!cfg["etcd_tx_retries"].is_null())

src/disk_tool/disk_tool_udev.cpp

@@ -435,7 +435,7 @@ int disk_tool_t::purge_devices(const std::vector<std::string> & devices)
             printf("%s\n", json11::Json(result).dump().c_str());
         return 0;
     }
-    std::vector<std::string> rm_osd_cli = { "vitastor-cli", "rm-osd" };
+    std::vector<std::string> rm_osd_cli = { "vitastor-cli", "rm-osd", "--allow-up" };
     for (auto osd_num: osd_numbers)
     {
         rm_osd_cli.push_back(std::to_string(osd_num));

src/kv/kv_db.cpp

@@ -96,7 +96,7 @@ struct kv_block_t
 
     void set_data_size();
     static int kv_size(const std::string & key, const std::string & value);
-    int parse(uint64_t offset, uint8_t *data, int size, bool allow_empty = false);
+    int parse(uint64_t offset, uint8_t *data, int size);
     bool serialize(uint8_t *data, int size);
     void apply_change();
     void cancel_change();
@@ -243,13 +243,13 @@ static std::string read_string(uint8_t *data, int size, int *pos)
     return key;
 }
 
-int kv_block_t::parse(uint64_t offset, uint8_t *data, int size, bool allow_empty)
+int kv_block_t::parse(uint64_t offset, uint8_t *data, int size)
 {
     kv_stored_block_t *blk = (kv_stored_block_t *)data;
     if (blk->magic == 0 || blk->type == KV_EMPTY)
     {
         // empty block
-        if (offset != 0 && !allow_empty)
+        if (offset != 0)
             fprintf(stderr, "K/V: Block %ju is %s\n", offset, blk->magic == 0 ? "empty" : "cleared");
         return -ENOTBLK;
     }
@@ -884,10 +884,7 @@ static void get_block(kv_db_t *db, uint64_t offset, int cur_level, int recheck_p
     }
     else
     {
-        op->offset = (offset / db->ino_block_size) * db->ino_block_size;
+        op->len = db->kv_block_size;
-        op->len = db->ino_block_size;
-        //op->offset = offset;
-        //op->len = db->kv_block_size;
         op->iov.push_back(malloc_or_die(op->len), op->len);
     }
     op->callback = [=](cluster_op_t *op)
@@ -901,82 +898,59 @@ static void get_block(kv_db_t *db, uint64_t offset, int cur_level, int recheck_p
             delete op;
             return;
         }
-        int blk_err = 0;
-        int blk_status = 0;
         invalidate(db, op->offset, op->version);
-        for (uint64_t in_offset = op->offset; in_offset < (op->offset+op->len) || in_offset == offset; in_offset += db->kv_block_size)
+        auto blk_it = db->block_cache.find(op->offset);
+        if (blk_it != db->block_cache.end() &&
+            // read may start BEFORE update and end AFTER update, in this case known will be > returned version
+            (db->known_versions[op->offset/db->ino_block_size] >= op->version && !blk_it->second.invalidated || blk_it->second.updating > 0))
         {
-            auto blk_it = db->block_cache.find(in_offset);
+            auto blk = &db->block_cache.at(op->offset);
-            if (blk_it != db->block_cache.end() &&
+            if (blk->updating > 0 && recheck_policy == KV_RECHECK_WAIT)
-                // read may start BEFORE update and end AFTER update, in this case known will be > returned version
-                (db->known_versions[in_offset/db->ino_block_size] >= op->version && !blk_it->second.invalidated || blk_it->second.updating > 0))
             {
-                auto blk = &db->block_cache.at(in_offset);
+                // Wait until block update stops
-                if (blk->updating > 0 && recheck_policy == KV_RECHECK_WAIT)
+                if (op->len)
+                    free(op->iov.buf[0].iov_base);
+                delete op;
+                db->continue_update.emplace(blk->offset, [=, blk_offset = blk->offset]()
                 {
-                    // Wait until block update stops
+                    get_block(db, offset, cur_level, recheck_policy, cb);
-                    if (op->len)
+                    db->run_continue_update(blk_offset);
-                        free(op->iov.buf[0].iov_base);
+                });
-                    delete op;
+                return;
-                    db->continue_update.emplace(in_offset, [=, blk_offset = blk->offset]()
-                    {
-                        get_block(db, offset, cur_level, recheck_policy, cb);
-                        db->run_continue_update(blk_offset);
-                    });
-                    return;
-                }
             }
-            else if (!op->len)
+            blk->usage = db->usage_counter;
+            cb(0, blk->updating > 0 ? BLK_UPDATING : BLK_NOCHANGE);
+        }
+        else
+        {
+            if (!op->len)
             {
                 // Version check failed, re-read block
                 delete op;
                 get_block(db, offset, cur_level, recheck_policy, cb);
                 return;
             }
-        }
+            auto blk = &db->block_cache[op->offset];
-        for (uint64_t in_offset = op->offset; in_offset < (op->offset+op->len) || in_offset == offset; in_offset += db->kv_block_size)
+            if (blk_it != db->block_cache.end())
-        {
-            auto blk_it = db->block_cache.find(in_offset);
-            if (blk_it != db->block_cache.end() &&
-                // read may start BEFORE update and end AFTER update, in this case known will be > returned version
-                (db->known_versions[in_offset/db->ino_block_size] >= op->version && !blk_it->second.invalidated || blk_it->second.updating > 0))
             {
-                auto blk = &db->block_cache.at(in_offset);
+                del_block_level(db, blk);
+                *blk = {};
+            }
+            int err = blk->parse(op->offset, (uint8_t*)op->iov.buf[0].iov_base, op->len);
+            if (err == 0)
+            {
+                blk->level = cur_level;
                 blk->usage = db->usage_counter;
-                if (in_offset == offset)
+                add_block_level(db, blk);
-                {
+                cb(0, BLK_RELOADED);
-                    blk_err = 0;
-                    blk_status = (blk->updating > 0 ? BLK_UPDATING : BLK_NOCHANGE);
-                }
             }
             else
             {
-                auto blk = &db->block_cache[in_offset];
+                db->block_cache.erase(op->offset);
-                if (blk_it != db->block_cache.end())
+                cb(err, BLK_NOCHANGE);
-                {
-                    del_block_level(db, blk);
-                    *blk = {};
-                }
-                int err = blk->parse(in_offset, (uint8_t*)op->iov.buf[0].iov_base + (in_offset - op->offset), db->kv_block_size, in_offset != offset);
-                if (in_offset == offset)
-                {
-                    blk_err = err;
-                    blk_status = err == 0 ? BLK_RELOADED : BLK_NOCHANGE;
-                }
-                if (err == 0)
-                {
-                    blk->level = cur_level;
-                    blk->usage = db->usage_counter;
-                    add_block_level(db, blk);
-                }
-                else
-                {
-                    db->block_cache.erase(in_offset);
-                }
-                try_evict(db);
             }
+            try_evict(db);
         }
-        cb(blk_err, blk_status);
         if (op->len)
             free(op->iov.buf[0].iov_base);
         delete op;
@@ -1886,6 +1860,7 @@ void kv_op_t::next_handle_block(int res, int refresh)
     else
     {
         // OK, leaf block found
+        recheck_policy = KV_RECHECK_NONE;
         next_get();
     }
 }
@@ -1912,6 +1887,7 @@ void kv_op_t::next_get()
     else if (blk->type == KV_LEAF_SPLIT)
     {
         // Left half finished, go to the right
+        recheck_policy = KV_RECHECK_LEAF;
         key = blk->right_half;
         skip_equal = false;
         prev_key_ge = blk->right_half;
@@ -1945,6 +1921,7 @@ void kv_op_t::next_go_up()
         path.pop_back();
         cur_level--;
         cur_block = path[path.size()-1].offset;
+        recheck_policy = KV_RECHECK_LEAF;
         // Check if we can resume listing from the next key
         auto pb_it = db->block_cache.find(cur_block);
         if (pb_it == db->block_cache.end())

src/osd/osd.cpp

@@ -271,6 +271,12 @@ void osd_t::parse_config(bool init)
     inode_vanish_time = config["inode_vanish_time"].uint64_value();
     if (!inode_vanish_time)
         inode_vanish_time = 60;
+    enable_pg_locks = config["enable_pg_locks"].is_null() || json_is_true(config["enable_pg_locks"]);
+    bool old_pg_locks_localize_only = pg_locks_localize_only;
+    pg_locks_localize_only = config["enable_pg_locks"].is_null();
+    pg_lock_retry_interval_ms = config["pg_lock_retry_interval"].uint64_value();
+    if (pg_lock_retry_interval_ms <= 1)
+        pg_lock_retry_interval_ms = 100;
     auto old_auto_scrub = auto_scrub;
     auto_scrub = json_is_true(config["auto_scrub"]);
     global_scrub_interval = parse_time(config["scrub_interval"].string_value());
@@ -336,6 +342,10 @@ void osd_t::parse_config(bool init)
     {
         apply_recovery_tune_interval();
     }
+    if (old_pg_locks_localize_only != pg_locks_localize_only)
+    {
+        apply_pg_locks_localize_only();
+    }
 }
 
 void osd_t::bind_socket()
@@ -447,6 +457,7 @@ void osd_t::exec_op(osd_op_t *cur_op)
     }
     if (readonly &&
         cur_op->req.hdr.opcode != OSD_OP_SEC_READ &&
+        cur_op->req.hdr.opcode != OSD_OP_SEC_LOCK &&
         cur_op->req.hdr.opcode != OSD_OP_SEC_LIST &&
         cur_op->req.hdr.opcode != OSD_OP_READ &&
         cur_op->req.hdr.opcode != OSD_OP_SEC_READ_BMP &&

src/osd/osd.h

@@ -92,6 +92,12 @@ struct recovery_stat_t
     uint64_t count, usec, bytes;
 };
 
+struct osd_pg_lock_t
+{
+    osd_num_t primary_osd = 0;
+    uint64_t state = 0;
+};
+
 class osd_t
 {
     // config
@@ -140,6 +146,9 @@ class osd_t
     uint32_t scrub_list_limit = 1000;
     bool scrub_find_best = true;
     uint64_t scrub_ec_max_bruteforce = 100;
+    bool enable_pg_locks = false;
+    bool pg_locks_localize_only = false;
+    uint64_t pg_lock_retry_interval_ms = 100;
 
     // cluster state
 
@@ -159,6 +168,7 @@ class osd_t
 
     // peers and PGs
 
+    std::map<pool_pg_num_t, osd_pg_lock_t> pg_locks;
     std::map<pool_id_t, pg_num_t> pg_counts;
     std::map<pool_pg_num_t, pg_t> pgs;
     std::set<pool_pg_num_t> dirty_pgs;
@@ -239,6 +249,8 @@ class osd_t
     void on_change_etcd_state_hook(std::map<std::string, etcd_kv_t> & changes);
     void on_load_config_hook(json11::Json::object & changes);
     void on_reload_config_hook(json11::Json::object & changes);
+    void on_change_pool_config_hook();
+    void apply_pg_locks_localize_only();
     json11::Json on_load_pgs_checks_hook();
     void on_load_pgs_hook(bool success);
     void bind_socket();
@@ -268,11 +280,16 @@ class osd_t
     void repeer_pgs(osd_num_t osd_num);
     void start_pg_peering(pg_t & pg);
     void drop_dirty_pg_connections(pool_pg_num_t pg);
+    void record_pg_lock(pg_t & pg, osd_num_t peer_osd, uint64_t pg_state);
+    void relock_pg(pg_t & pg);
     void submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps);
     void discard_list_subop(osd_op_t *list_op);
     bool stop_pg(pg_t & pg);
     void reset_pg(pg_t & pg);
     void finish_stop_pg(pg_t & pg);
+    void rm_inflight(pg_t & pg);
+    void continue_pg(pg_t & pg);
+    bool continue_pg_peering(pg_t & pg);
 
     // flushing, recovery and backfill
     void submit_pg_flush_ops(pg_t & pg);
@@ -299,10 +316,13 @@ class osd_t
     void finish_op(osd_op_t *cur_op, int retval);
 
     // secondary ops
+    bool sec_check_pg_lock(osd_num_t primary_osd, const object_id &oid);
     void exec_sync_stab_all(osd_op_t *cur_op);
     void exec_show_config(osd_op_t *cur_op);
     void exec_secondary(osd_op_t *cur_op);
     void exec_secondary_real(osd_op_t *cur_op);
+    void exec_sec_read_bmp(osd_op_t *cur_op);
+    void exec_sec_lock(osd_op_t *cur_op);
     void secondary_op_callback(osd_op_t *cur_op);
 
     // primary ops
@@ -310,6 +330,7 @@ class osd_t
     bool prepare_primary_rw(osd_op_t *cur_op);
     void continue_primary_read(osd_op_t *cur_op);
     void continue_primary_scrub(osd_op_t *cur_op);
+    void continue_local_secondary_read(osd_op_t *cur_op);
     void continue_primary_describe(osd_op_t *cur_op);
     void continue_primary_list(osd_op_t *cur_op);
     void continue_primary_write(osd_op_t *cur_op);
@@ -347,13 +368,13 @@ class osd_t
     uint64_t* get_object_osd_set(pg_t &pg, object_id &oid, pg_osd_set_state_t **object_state);
 
     void continue_chained_read(osd_op_t *cur_op);
-    int submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op);
+    int submit_chained_read_requests(pg_t *pg, osd_op_t *cur_op);
     void check_corrupted_chained(pg_t & pg, osd_op_t *cur_op);
-    void send_chained_read_results(pg_t & pg, osd_op_t *cur_op);
+    void send_chained_read_results(pg_t *pg, osd_op_t *cur_op);
     std::vector<osd_chain_read_t> collect_chained_read_requests(osd_op_t *cur_op);
     int collect_bitmap_requests(osd_op_t *cur_op, pg_t & pg, std::vector<bitmap_request_t> & bitmap_requests);
     int submit_bitmap_subops(osd_op_t *cur_op, pg_t & pg);
-    int read_bitmaps(osd_op_t *cur_op, pg_t & pg, int base_state);
+    int read_bitmaps(osd_op_t *cur_op, pg_t *pg, int base_state);
 
     inline pg_num_t map_to_pg(object_id oid, uint64_t pg_stripe_size)
     {

src/osd/osd_cluster.cpp

@@ -65,6 +65,7 @@ void osd_t::init_cluster()
         st_cli.tfd = tfd;
         st_cli.log_level = log_level;
         st_cli.on_change_osd_state_hook = [this](osd_num_t peer_osd) { on_change_osd_state_hook(peer_osd); };
+        st_cli.on_change_pool_config_hook = [this]() { on_change_pool_config_hook(); };
         st_cli.on_change_backfillfull_hook = [this](pool_id_t pool_id) { on_change_backfillfull_hook(pool_id); };
         st_cli.on_change_pg_history_hook = [this](pool_id_t pool_id, pg_num_t pg_num) { on_change_pg_history_hook(pool_id, pg_num); };
         st_cli.on_change_hook = [this](std::map<std::string, etcd_kv_t> & changes) { on_change_etcd_state_hook(changes); };
@@ -153,23 +154,19 @@ bool osd_t::check_peer_config(osd_client_t *cl, json11::Json conf)
             return false;
         }
     }
+    cl->enable_pg_locks = conf["features"]["pg_locks"].bool_value();
     return true;
 }
 
 json11::Json osd_t::get_osd_state()
 {
-    std::vector<char> hostname;
-    hostname.resize(1024);
-    while (gethostname(hostname.data(), hostname.size()) < 0 && errno == ENAMETOOLONG)
-        hostname.resize(hostname.size()+1024);
-    hostname.resize(strnlen(hostname.data(), hostname.size()));
     json11::Json::object st;
     st["state"] = "up";
     if (bind_addresses.size() != 1 || bind_addresses[0] != "0.0.0.0")
         st["addresses"] = bind_addresses;
     else
         st["addresses"] = getifaddr_list();
-    st["host"] = std::string(hostname.data(), hostname.size());
+    st["host"] = gethostname_str();
     st["version"] = VITASTOR_VERSION;
     st["port"] = listening_port;
 #ifdef WITH_RDMACM
@@ -419,6 +416,28 @@ void osd_t::on_change_osd_state_hook(osd_num_t peer_osd)
     }
 }
 
+void osd_t::on_change_pool_config_hook()
+{
+    apply_pg_locks_localize_only();
+}
+
+void osd_t::apply_pg_locks_localize_only()
+{
+    for (auto & pp: pgs)
+    {
+        auto pool_it = st_cli.pool_config.find(pp.first.pool_id);
+        if (pool_it == st_cli.pool_config.end())
+        {
+            continue;
+        }
+        auto & pool_cfg = pool_it->second;
+        auto & pg = pp.second;
+        pg.disable_pg_locks = pg_locks_localize_only &&
+            pool_cfg.scheme == POOL_SCHEME_REPLICATED &&
+            pool_cfg.local_reads == POOL_LOCAL_READ_PRIMARY;
+    }
+}
+
 void osd_t::on_change_backfillfull_hook(pool_id_t pool_id)
 {
     if (!(peering_state & (OSD_RECOVERING | OSD_FLUSHING_PGS)))
@@ -696,20 +715,27 @@ void osd_t::apply_pg_count()
             // The external tool must wait for all PGs to come down before changing PG count
             // If it doesn't wait, a restarted OSD may apply the new count immediately which will lead to bugs
             // So an OSD just dies if it detects PG count change while there are active PGs
-            int still_active = 0;
+            int still_active_primary = 0;
             for (auto & kv: pgs)
             {
                 if (kv.first.pool_id == pool_item.first && (kv.second.state & PG_ACTIVE))
                 {
-                    still_active++;
+                    still_active_primary++;
                 }
             }
-            if (still_active > 0)
+            int still_active_secondary = 0;
+            for (auto lock_it = pg_locks.lower_bound((pool_pg_num_t){ .pool_id = pool_item.first, .pg_num = 0 });
+                lock_it != pg_locks.end() && lock_it->first.pool_id == pool_item.first; lock_it++)
+            {
+                still_active_secondary++;
+            }
+            if (still_active_primary > 0 || still_active_secondary > 0)
             {
                 printf(
                     "[OSD %ju] PG count change detected for pool %u (new is %ju, old is %u),"
-                    " but %u PG(s) are still active. This is not allowed. Exiting\n",
+                    " but %u PG(s) are still active as primary and %u as secondary. This is not allowed. Exiting\n",
-                    this->osd_num, pool_item.first, pool_item.second.real_pg_count, pg_counts[pool_item.first], still_active
+                    this->osd_num, pool_item.first, pool_item.second.real_pg_count, pg_counts[pool_item.first],
+                    still_active_primary, still_active_secondary
                 );
                 force_stop(1);
                 return;
@@ -836,22 +862,23 @@ void osd_t::apply_pg_config()
                     }
                 }
                 auto & pg = this->pgs[{ .pool_id = pool_id, .pg_num = pg_num }];
-                pg = (pg_t){
+                pg.state = pg_cfg.cur_primary == this->osd_num ? PG_PEERING : PG_STARTING;
-                    .state = pg_cfg.cur_primary == this->osd_num ? PG_PEERING : PG_STARTING,
+                pg.scheme = pool_item.second.scheme;
-                    .scheme = pool_item.second.scheme,
+                pg.pg_cursize = 0;
-                    .pg_cursize = 0,
+                pg.pg_size = pool_item.second.pg_size;
-                    .pg_size = pool_item.second.pg_size,
+                pg.pg_minsize = pool_item.second.pg_minsize;
-                    .pg_minsize = pool_item.second.pg_minsize,
+                pg.pg_data_size = pool_item.second.scheme == POOL_SCHEME_REPLICATED
-                    .pg_data_size = pool_item.second.scheme == POOL_SCHEME_REPLICATED
+                     ? 1 : pool_item.second.pg_size - pool_item.second.parity_chunks;
-                         ? 1 : pool_item.second.pg_size - pool_item.second.parity_chunks,
+                pg.pool_id = pool_id;
-                    .pool_id = pool_id,
+                pg.pg_num = pg_num;
-                    .pg_num = pg_num,
+                pg.reported_epoch = pg_cfg.epoch;
-                    .reported_epoch = pg_cfg.epoch,
+                pg.target_history = pg_cfg.target_history;
-                    .target_history = pg_cfg.target_history,
+                pg.all_peers = vec_all_peers;
-                    .all_peers = vec_all_peers,
+                pg.next_scrub = pg_cfg.next_scrub;
-                    .next_scrub = pg_cfg.next_scrub,
+                pg.target_set = pg_cfg.target_set;
-                    .target_set = pg_cfg.target_set,
+                pg.disable_pg_locks = pg_locks_localize_only &&
-                };
+                    pool_item.second.scheme == POOL_SCHEME_REPLICATED &&
+                    pool_item.second.local_reads == POOL_LOCAL_READ_PRIMARY;
                 if (pg.scheme == POOL_SCHEME_EC)
                 {
                     use_ec(pg.pg_size, pg.pg_data_size, true);

src/osd/osd_flush.cpp

@@ -150,14 +150,7 @@ void osd_t::handle_flush_op(bool rollback, pool_id_t pool_id, pg_num_t pg_num, p
         {
             continue_primary_write(op);
         }
-        if ((pg.state & PG_STOPPING) && pg.inflight == 0 && !pg.flush_batch)
+        continue_pg(pg);
-        {
-            finish_stop_pg(pg);
-        }
-        else if ((pg.state & PG_REPEERING) && pg.inflight == 0 && !pg.flush_batch)
-        {
-            start_pg_peering(pg);
-        }
     }
 }
 
@@ -254,7 +247,8 @@ bool osd_t::pick_next_recovery(osd_recovery_op_t &op)
         restart:
             for (auto pg_it = pgs.lower_bound(recovery_last_pg); pg_it != pgs.end(); pg_it++)
             {
-                if ((pg_it->second.state & mask) == check)
+                auto & src = recovery_last_degraded ? pg_it->second.degraded_objects : pg_it->second.misplaced_objects;
+                if ((pg_it->second.state & mask) == check && src.size() > 0)
                 {
                     auto pool_it = st_cli.pool_config.find(pg_it->first.pool_id);
                     if (pool_it != st_cli.pool_config.end() && pool_it->second.backfillfull)
@@ -263,8 +257,6 @@ bool osd_t::pick_next_recovery(osd_recovery_op_t &op)
                         recovery_last_pg.pool_id++;
                         goto restart;
                     }
-                    auto & src = recovery_last_degraded ? pg_it->second.degraded_objects : pg_it->second.misplaced_objects;
-                    assert(src.size() > 0);
                     // Restart scanning from the next object
                     for (auto obj_it = src.upper_bound(recovery_last_oid); obj_it != src.end(); obj_it++)
                     {

src/osd/osd_peering.cpp

@@ -21,28 +21,8 @@ void osd_t::handle_peers()
         {
             if (p.second.state == PG_PEERING)
             {
-                if (!p.second.peering_state->list_ops.size())
+                if (continue_pg_peering(p.second))
                 {
-                    p.second.calc_object_states(log_level);
-                    report_pg_state(p.second);
-                    schedule_scrub(p.second);
-                    incomplete_objects += p.second.incomplete_objects.size();
-                    misplaced_objects += p.second.misplaced_objects.size();
-                    // FIXME: degraded objects may currently include misplaced, too! Report them separately?
-                    degraded_objects += p.second.degraded_objects.size();
-                    if (p.second.state & PG_HAS_UNCLEAN)
-                        peering_state = peering_state | OSD_FLUSHING_PGS;
-                    else if (p.second.state & (PG_HAS_DEGRADED | PG_HAS_MISPLACED))
-                    {
-                        peering_state = peering_state | OSD_RECOVERING;
-                        if (p.second.state & PG_HAS_DEGRADED)
-                        {
-                            // Restart recovery from degraded objects
-                            recovery_last_degraded = true;
-                            recovery_last_pg = {};
-                            recovery_last_oid = {};
-                        }
-                    }
                     ringloop->wakeup();
                     return;
                 }
@@ -95,6 +75,16 @@ void osd_t::handle_peers()
 
 void osd_t::repeer_pgs(osd_num_t peer_osd)
 {
+    if (msgr.osd_peer_fds.find(peer_osd) == msgr.osd_peer_fds.end())
+    {
+        for (auto lock_it = pg_locks.begin(); lock_it != pg_locks.end(); )
+        {
+            if (lock_it->second.primary_osd == peer_osd)
+                pg_locks.erase(lock_it++);
+            else
+                lock_it++;
+        }
+    }
     // Re-peer affected PGs
     for (auto & p: pgs)
     {
@@ -114,7 +104,7 @@ void osd_t::repeer_pgs(osd_num_t peer_osd)
             {
                 // Repeer this pg
                 printf("[PG %u/%u] Repeer because of OSD %ju\n", pg.pool_id, pg.pg_num, peer_osd);
-                if (!(pg.state & (PG_ACTIVE | PG_REPEERING)) || pg.inflight == 0 && !pg.flush_batch)
+                if (!(pg.state & (PG_ACTIVE | PG_REPEERING)) || pg.can_repeer())
                 {
                     start_pg_peering(pg);
                 }
@@ -195,7 +185,6 @@ void osd_t::start_pg_peering(pg_t & pg)
     pg.state = PG_PEERING;
     this->peering_state |= OSD_PEERING_PGS;
     reset_pg(pg);
-    report_pg_state(pg);
     drop_dirty_pg_connections({ .pool_id = pg.pool_id, .pg_num = pg.pg_num });
     // Try to connect with current peers if they're up, but we don't have connections to them
     // Otherwise we may erroneously decide that the pg is incomplete :-)
@@ -215,8 +204,7 @@ void osd_t::start_pg_peering(pg_t & pg)
     {
         // Wait until all OSDs are either connected or their /osd/state disappears from etcd
         pg.state = PG_INCOMPLETE;
-        report_pg_state(pg);
+        // Fall through to cleanup list results
-        return;
     }
     // Calculate current write OSD set
     pg.pg_cursize = 0;
@@ -242,8 +230,6 @@ void osd_t::start_pg_peering(pg_t & pg)
         // because such PGs don't flush unstable entries on secondary OSDs so they can't remove these
         // entries from their journals...
         pg.state = PG_INCOMPLETE;
-        report_pg_state(pg);
-        return;
     }
     std::set<osd_num_t> cur_peers;
     std::set<osd_num_t> dead_peers;
@@ -278,8 +264,6 @@ void osd_t::start_pg_peering(pg_t & pg)
             if (nonzero >= pg.pg_data_size && found < pg.pg_data_size)
             {
                 pg.state = PG_INCOMPLETE;
-                report_pg_state(pg);
-                return;
             }
         }
     }
@@ -318,6 +302,7 @@ void osd_t::start_pg_peering(pg_t & pg)
             delete pg.peering_state;
             pg.peering_state = NULL;
         }
+        report_pg_state(pg);
         return;
     }
     if (!pg.peering_state)
@@ -326,16 +311,203 @@ void osd_t::start_pg_peering(pg_t & pg)
         pg.peering_state->pool_id = pg.pool_id;
         pg.peering_state->pg_num = pg.pg_num;
     }
-    for (osd_num_t peer_osd: cur_peers)
+    pg.peering_state->locked = false;
+    pg.peering_state->lists_done = false;
+    report_pg_state(pg);
+}
+
+bool osd_t::continue_pg_peering(pg_t & pg)
+{
+    if (pg.peering_state->locked)
     {
-        if (pg.peering_state->list_ops.find(peer_osd) != pg.peering_state->list_ops.end() ||
+        pg.peering_state->lists_done = true;
-            pg.peering_state->list_results.find(peer_osd) != pg.peering_state->list_results.end())
+        for (osd_num_t peer_osd: pg.cur_peers)
         {
+            if (pg.peering_state->list_results.find(peer_osd) == pg.peering_state->list_results.end())
+            {
+                pg.peering_state->lists_done = false;
+            }
+            if (pg.peering_state->list_ops.find(peer_osd) != pg.peering_state->list_ops.end() ||
+                pg.peering_state->list_results.find(peer_osd) != pg.peering_state->list_results.end())
+            {
+                continue;
+            }
+            submit_list_subop(peer_osd, pg.peering_state);
+        }
+    }
+    if (pg.peering_state->lists_done)
+    {
+        pg.calc_object_states(log_level);
+        report_pg_state(pg);
+        schedule_scrub(pg);
+        incomplete_objects += pg.incomplete_objects.size();
+        misplaced_objects += pg.misplaced_objects.size();
+        // FIXME: degraded objects may currently include misplaced, too! Report them separately?
+        degraded_objects += pg.degraded_objects.size();
+        if (pg.state & PG_HAS_UNCLEAN)
+            this->peering_state = peering_state | OSD_FLUSHING_PGS;
+        else if (pg.state & (PG_HAS_DEGRADED | PG_HAS_MISPLACED))
+        {
+            this->peering_state = peering_state | OSD_RECOVERING;
+            if (pg.state & PG_HAS_DEGRADED)
+            {
+                // Restart recovery from degraded objects
+                this->recovery_last_degraded = true;
+                this->recovery_last_pg = {};
+                this->recovery_last_oid = {};
+            }
+        }
+        return true;
+    }
+    return false;
+}
+
+void osd_t::record_pg_lock(pg_t & pg, osd_num_t peer_osd, uint64_t pg_state)
+{
+    if (!pg_state)
+        pg.lock_peers.erase(peer_osd);
+    else
+        pg.lock_peers[peer_osd] = pg_state;
+}
+
+void osd_t::relock_pg(pg_t & pg)
+{
+    if (!enable_pg_locks || pg.disable_pg_locks && !pg.lock_peers.size())
+    {
+        if (pg.state & PG_PEERING)
+            pg.peering_state->locked = true;
+        continue_pg(pg);
+        return;
+    }
+    if (pg.inflight_locks > 0 || pg.lock_waiting)
+    {
+        return;
+    }
+    // Check that lock_peers are equal to cur_peers and correct the difference, if any
+    uint64_t wanted_state = pg.state;
+    std::vector<osd_num_t> diff_osds;
+    if (!(pg.state & (PG_STOPPING | PG_OFFLINE | PG_INCOMPLETE)) && !pg.disable_pg_locks)
+    {
+        for (osd_num_t peer_osd: pg.cur_peers)
+        {
+            if (peer_osd != this->osd_num)
+            {
+                auto lock_it = pg.lock_peers.find(peer_osd);
+                if (lock_it == pg.lock_peers.end())
+                    diff_osds.push_back(peer_osd);
+                else
+                {
+                    if (lock_it->second != wanted_state)
+                        diff_osds.push_back(peer_osd);
+                    lock_it->second |= ((uint64_t)1 << 63);
+                }
+            }
+        }
+    }
+    int relock_osd_count = diff_osds.size();
+    for (auto & lp: pg.lock_peers)
+    {
+        if (!(lp.second & ((uint64_t)1 << 63)))
+            diff_osds.push_back(lp.first);
+        lp.second &= ~((uint64_t)1 << 63);
+    }
+    if (!diff_osds.size())
+    {
+        if (pg.state & PG_PEERING)
+            pg.peering_state->locked = true;
+        continue_pg(pg);
+        return;
+    }
+    pg.inflight_locks++;
+    for (int i = 0; i < diff_osds.size(); i++)
+    {
+        bool unlock_peer = (i >= relock_osd_count);
+        uint64_t new_state = unlock_peer ? 0 : pg.state;
+        auto peer_osd = diff_osds[i];
+        auto peer_fd_it = msgr.osd_peer_fds.find(peer_osd);
+        if (peer_fd_it == msgr.osd_peer_fds.end())
+        {
+            if (unlock_peer)
+            {
+                // Peer is dead - unlocked automatically
+                record_pg_lock(pg, peer_osd, new_state);
+                diff_osds.erase(diff_osds.begin()+(i--));
+            }
             continue;
         }
-        submit_list_subop(peer_osd, pg.peering_state);
+        int peer_fd = peer_fd_it->second;
+        auto cl = msgr.clients.at(peer_fd);
+        if (!cl->enable_pg_locks)
+        {
+            // Peer does not support locking - just instantly remember the lock as successful
+            record_pg_lock(pg, peer_osd, new_state);
+            diff_osds.erase(diff_osds.begin()+(i--));
+            continue;
+        }
+        pg.inflight_locks++;
+        osd_op_t *op = new osd_op_t();
+        op->op_type = OSD_OP_OUT;
+        op->peer_fd = peer_fd;
+        op->req = (osd_any_op_t){
+            .sec_lock = {
+                .header = {
+                    .magic = SECONDARY_OSD_OP_MAGIC,
+                    .opcode = OSD_OP_SEC_LOCK,
+                },
+                .flags = (uint64_t)(unlock_peer ? OSD_SEC_UNLOCK_PG : OSD_SEC_LOCK_PG),
+                .pool_id = pg.pool_id,
+                .pg_num = pg.pg_num,
+                .pg_state = new_state,
+            },
+        };
+        op->callback = [this, peer_osd](osd_op_t *op)
+        {
+            pool_pg_num_t pg_id = { .pool_id = (pool_id_t)op->req.sec_lock.pool_id, .pg_num = (pg_num_t)op->req.sec_lock.pg_num };
+            auto pg_it = pgs.find(pg_id);
+            if (pg_it == pgs.end())
+            {
+                return;
+            }
+            auto & pg = pg_it->second;
+            if (op->reply.hdr.retval == 0)
+            {
+                record_pg_lock(pg_it->second, peer_osd, op->req.sec_lock.pg_state);
+            }
+            else if (op->reply.hdr.retval != -EPIPE)
+            {
+                printf(
+                    (op->reply.hdr.retval == -ENOENT
+                        ? "Failed to %1$s PG %2$u/%3$u on OSD %4$ju - peer didn't load PG info yet\n"
+                        : (op->reply.sec_lock.cur_primary
+                            ? "Failed to %1$s PG %2$u/%3$u on OSD %4$ju - taken by OSD %6$ju (retval=%5$jd)\n"
+                            : "Failed to %1$s PG %2$u/%3$u on OSD %4$ju - retval=%5$jd\n")),
+                    op->req.sec_lock.flags == OSD_SEC_UNLOCK_PG ? "unlock" : "lock",
+                    pg_id.pool_id, pg_id.pg_num, peer_osd, op->reply.hdr.retval, op->reply.sec_lock.cur_primary
+                );
+                // Retry relocking/unlocking PG after a short time
+                pg.lock_waiting = true;
+                tfd->set_timer(pg_lock_retry_interval_ms, false, [this, pg_id](int)
+                {
+                    auto pg_it = pgs.find(pg_id);
+                    if (pg_it != pgs.end())
+                    {
+                        pg_it->second.lock_waiting = false;
+                        relock_pg(pg_it->second);
+                    }
+                });
+            }
+            pg.inflight_locks--;
+            relock_pg(pg);
+            delete op;
+        };
+        msgr.outbox_push(op);
     }
-    ringloop->wakeup();
+    if (pg.state & PG_PEERING)
+    {
+        pg.peering_state->locked = !diff_osds.size();
+    }
+    pg.inflight_locks--;
+    continue_pg(pg);
 }
 
 void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
@@ -383,10 +555,16 @@ void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
     }
     else
     {
+        auto role_fd_it = msgr.osd_peer_fds.find(role_osd);
+        if (role_fd_it == msgr.osd_peer_fds.end())
+        {
+            printf("Failed to get object list from OSD %ju because it is disconnected\n", role_osd);
+            return;
+        }
         // Peer
         osd_op_t *op = new osd_op_t();
         op->op_type = OSD_OP_OUT;
-        op->peer_fd = msgr.osd_peer_fds.at(role_osd);
+        op->peer_fd = role_fd_it->second;
         op->req = (osd_any_op_t){
             .sec_list = {
                 .header = {
@@ -478,13 +656,8 @@ bool osd_t::stop_pg(pg_t & pg)
         return false;
     }
     drop_dirty_pg_connections({ .pool_id = pg.pool_id, .pg_num = pg.pg_num });
-    if (!(pg.state & (PG_ACTIVE | PG_REPEERING)))
+    pg.state = pg.state & ~PG_STARTING & ~PG_PEERING & ~PG_INCOMPLETE & ~PG_ACTIVE & ~PG_REPEERING & ~PG_OFFLINE | PG_STOPPING;
-    {
+    if (pg.can_stop())
-        finish_stop_pg(pg);
-        return true;
-    }
-    pg.state = pg.state & ~PG_ACTIVE & ~PG_REPEERING | PG_STOPPING;
-    if (pg.inflight == 0 && !pg.flush_batch)
     {
         finish_stop_pg(pg);
     }
@@ -565,9 +738,33 @@ void osd_t::report_pg_state(pg_t & pg)
         pg_cfg.target_history = pg.target_history;
         pg_cfg.all_peers = pg.all_peers;
     }
+    relock_pg(pg);
     if (pg.state == PG_OFFLINE && !this->pg_config_applied)
     {
         apply_pg_config();
     }
     report_pg_states();
 }
+
+void osd_t::rm_inflight(pg_t & pg)
+{
+    pg.inflight--;
+    assert(pg.inflight >= 0);
+    continue_pg(pg);
+}
+
+void osd_t::continue_pg(pg_t & pg)
+{
+    if ((pg.state & PG_STOPPING) && pg.can_stop())
+    {
+        finish_stop_pg(pg);
+    }
+    else if ((pg.state & PG_REPEERING) && pg.can_repeer())
+    {
+        start_pg_peering(pg);
+    }
+    else if ((pg.state & PG_PEERING) && pg.peering_state->locked)
+    {
+        continue_pg_peering(pg);
+    }
+}

src/osd/osd_peering_pg.cpp

@@ -489,3 +489,13 @@ void pg_t::print_state()
         total_count
     );
 }
+
+bool pg_t::can_stop()
+{
+    return inflight == 0 && inflight_locks == 0 && !lock_peers.size() && !flush_batch;
+}
+
+bool pg_t::can_repeer()
+{
+    return inflight == 0 && !flush_batch;
+}

src/osd/osd_peering_pg.h

@@ -49,6 +49,8 @@ struct pg_peering_state_t
     std::map<osd_num_t, pg_list_result_t> list_results;
     pool_id_t pool_id = 0;
     pg_num_t pg_num = 0;
+    bool locked = false;
+    bool lists_done = false;
 };
 
 struct obj_piece_id_t
@@ -87,6 +89,7 @@ struct pg_t
     pool_id_t pool_id = 0;
     pg_num_t pg_num = 0;
     uint64_t clean_count = 0, total_count = 0;
+    bool disable_pg_locks = false;
     // epoch number - should increase with each non-clean activation of the PG
     uint64_t epoch = 0, reported_epoch = 0;
     // target history and all potential peers
@@ -104,6 +107,10 @@ struct pg_t
     // cur_set is the current set of connected peer OSDs for this PG
     // cur_set = (role => osd_num or UINT64_MAX if missing). role numbers begin with zero
     std::vector<osd_num_t> cur_set;
+    // locked peer list => pg state reported to the peer
+    std::map<osd_num_t, uint64_t> lock_peers;
+    int inflight_locks = 0;
+    bool lock_waiting = false;
     // same thing in state_dict-like format
     pg_osd_set_t cur_loc_set;
     // moved object map. by default, each object is considered to reside on cur_set.
@@ -125,6 +132,9 @@ struct pg_t
     pg_osd_set_state_t* add_object_to_state(const object_id oid, const uint64_t state, const pg_osd_set_t & osd_set);
     void calc_object_states(int log_level);
     void print_state();
+    bool can_stop();
+    bool can_repeer();
+    void rm_inflight();
 };
 
 inline bool operator < (const pg_obj_loc_t &a, const pg_obj_loc_t &b)

src/osd/osd_primary.cpp

@@ -37,7 +37,20 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
     };
     pg_num_t pg_num = (oid.stripe/pool_cfg.pg_stripe_size) % pg_counts[pool_id] + 1; // like map_to_pg()
     auto pg_it = pgs.find({ .pool_id = pool_id, .pg_num = pg_num });
-    if (pg_it == pgs.end() || !(pg_it->second.state & PG_ACTIVE))
+    if (pg_it == pgs.end() || pg_it->second.state == PG_OFFLINE)
+    {
+        // Check for a local replicated read from secondary OSD
+        auto lock_it = cur_op->req.hdr.opcode == OSD_OP_READ && pool_cfg.scheme == POOL_SCHEME_REPLICATED
+            ? pg_locks.find({ .pool_id = pool_id, .pg_num = pg_num })
+            : pg_locks.end();
+        if (lock_it == pg_locks.end() || lock_it->second.state != PG_ACTIVE && lock_it->second.state != (PG_ACTIVE|PG_LEFT_ON_DEAD))
+        {
+            // FIXME: Change EPIPE to something else
+            finish_op(cur_op, -EPIPE);
+            return false;
+        }
+    }
+    else if (!(pg_it->second.state & PG_ACTIVE))
     {
         // This OSD is not primary for this PG or the PG is inactive
         // FIXME: Allow reads from PGs degraded under pg_minsize, but don't allow writes
@@ -69,7 +82,7 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
         // Find parents from the same pool. Optimized reads only work within pools
         while (inode_it != st_cli.inode_config.end() &&
             inode_it->second.parent_id &&
-            INODE_POOL(inode_it->second.parent_id) == pg_it->second.pool_id)
+            INODE_POOL(inode_it->second.parent_id) == pool_cfg.id)
         {
             // Check for loops - FIXME check it in etcd_state_client
             if (inode_it->second.parent_id == cur_op->req.rw.inode ||
@@ -109,7 +122,7 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
     );
     void *data_buf = (uint8_t*)op_data + sizeof(osd_primary_op_data_t);
     op_data->pg_num = pg_num;
-    op_data->pg = &pg_it->second;
+    op_data->pg = pg_it == pgs.end() ? NULL : &pg_it->second;
     op_data->oid = oid;
     op_data->stripes = (osd_rmw_stripe_t*)data_buf;
     op_data->stripe_count = stripe_count;
@@ -144,7 +157,7 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
         chain_num++;
         auto inode_it = st_cli.inode_config.find(cur_op->req.rw.inode);
         while (inode_it != st_cli.inode_config.end() && inode_it->second.parent_id &&
-            INODE_POOL(inode_it->second.parent_id) == pg_it->second.pool_id &&
+            INODE_POOL(inode_it->second.parent_id) == pool_cfg.id &&
             // Check for loops
             inode_it->second.parent_id != cur_op->req.rw.inode)
         {
@@ -154,7 +167,10 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
             chain_num++;
         }
     }
-    pg_it->second.inflight++;
+    if (op_data->pg)
+    {
+        op_data->pg->inflight++;
+    }
     return true;
 }
 
@@ -194,6 +210,7 @@ void osd_t::continue_primary_read(osd_op_t *cur_op)
         return;
     }
     osd_primary_op_data_t *op_data = cur_op->op_data;
+    pg_t *pg = op_data->pg;
     if (op_data->chain_size)
     {
         continue_chained_read(cur_op);
@@ -206,11 +223,10 @@ void osd_t::continue_primary_read(osd_op_t *cur_op)
 resume_0:
     cur_op->reply.rw.bitmap_len = 0;
     {
-        auto & pg = *op_data->pg;
         if (cur_op->req.rw.len == 0)
         {
             // len=0 => bitmap read
-            for (int role = 0; role < pg.pg_data_size; role++)
+            for (int role = 0; role < (pg ? pg->pg_data_size : 1); role++)
             {
                 op_data->stripes[role].read_start = 0;
                 op_data->stripes[role].read_end = UINT32_MAX;
@@ -218,40 +234,48 @@ resume_0:
         }
         else
         {
-            for (int role = 0; role < pg.pg_data_size; role++)
+            for (int role = 0; role < (pg ? pg->pg_data_size : 1); role++)
             {
                 op_data->stripes[role].read_start = op_data->stripes[role].req_start;
                 op_data->stripes[role].read_end = op_data->stripes[role].req_end;
             }
         }
         // Determine version
-        auto vo_it = pg.ver_override.find(op_data->oid);
+        if (pg)
-        op_data->target_ver = vo_it != pg.ver_override.end() ? vo_it->second : UINT64_MAX;
+        {
-        // PG may have degraded or misplaced objects
+            auto vo_it = pg->ver_override.find(op_data->oid);
-        op_data->prev_set = get_object_osd_set(pg, op_data->oid, &op_data->object_state);
+            op_data->target_ver = vo_it != pg->ver_override.end() ? vo_it->second : UINT64_MAX;
-        if (pg.state == PG_ACTIVE || pg.scheme == POOL_SCHEME_REPLICATED)
+            // PG may have degraded or misplaced objects
+            op_data->prev_set = get_object_osd_set(*pg, op_data->oid, &op_data->object_state);
+        }
+        else
+        {
+            op_data->target_ver = UINT64_MAX;
+            op_data->prev_set = &this->osd_num;
+        }
+        if (!pg || pg->state == PG_ACTIVE || pg->scheme == POOL_SCHEME_REPLICATED)
         {
             // Fast happy-path
-            if (pg.scheme == POOL_SCHEME_REPLICATED &&
+            if (pg && pg->scheme == POOL_SCHEME_REPLICATED &&
                 op_data->object_state && (op_data->object_state->state & OBJ_INCOMPLETE))
             {
                 finish_op(cur_op, -EIO);
                 return;
             }
-            cur_op->buf = alloc_read_buffer(op_data->stripes, pg.pg_data_size, 0);
+            cur_op->buf = alloc_read_buffer(op_data->stripes, pg ? pg->pg_data_size : 1, 0);
             submit_primary_subops(SUBMIT_RMW_READ, op_data->target_ver, op_data->prev_set, cur_op);
             op_data->st = 1;
         }
         else
         {
-            if (extend_missing_stripes(op_data->stripes, op_data->prev_set, pg.pg_data_size, pg.pg_size) < 0)
+            if (extend_missing_stripes(op_data->stripes, op_data->prev_set, pg->pg_data_size, pg->pg_size) < 0)
             {
                 finish_op(cur_op, -EIO);
                 return;
             }
             // Submit reads
             op_data->degraded = 1;
-            cur_op->buf = alloc_read_buffer(op_data->stripes, pg.pg_size, 0);
+            cur_op->buf = alloc_read_buffer(op_data->stripes, pg->pg_size, 0);
             submit_primary_subops(SUBMIT_RMW_READ, op_data->target_ver, op_data->prev_set, cur_op);
             op_data->st = 1;
         }
@@ -261,32 +285,32 @@ resume_1:
 resume_2:
     if (op_data->errors > 0)
     {
-        if (op_data->errcode == -EIO || op_data->errcode == -EDOM)
+        if (pg && (op_data->errcode == -EIO || op_data->errcode == -EDOM))
         {
             // I/O or checksum error
             // FIXME: ref = true ideally... because new_state != state is not necessarily true if it's freed and recreated
-            op_data->object_state = mark_object_corrupted(*op_data->pg, op_data->oid, op_data->object_state, op_data->stripes, false);
+            op_data->object_state = mark_object_corrupted(*pg, op_data->oid, op_data->object_state, op_data->stripes, false);
             goto resume_0;
         }
         finish_op(cur_op, op_data->errcode);
         return;
     }
     cur_op->reply.rw.version = op_data->fact_ver;
-    cur_op->reply.rw.bitmap_len = op_data->pg->pg_data_size * clean_entry_bitmap_size;
+    cur_op->reply.rw.bitmap_len = (pg ? pg->pg_data_size : 1) * clean_entry_bitmap_size;
     if (op_data->degraded)
     {
         // Reconstruct missing stripes
         osd_rmw_stripe_t *stripes = op_data->stripes;
-        if (op_data->pg->scheme == POOL_SCHEME_XOR)
+        if (pg->scheme == POOL_SCHEME_XOR)
         {
-            reconstruct_stripes_xor(stripes, op_data->pg->pg_size, clean_entry_bitmap_size);
+            reconstruct_stripes_xor(stripes, pg->pg_size, clean_entry_bitmap_size);
         }
-        else if (op_data->pg->scheme == POOL_SCHEME_EC)
+        else if (pg->scheme == POOL_SCHEME_EC)
         {
-            reconstruct_stripes_ec(stripes, op_data->pg->pg_size, op_data->pg->pg_data_size, clean_entry_bitmap_size);
+            reconstruct_stripes_ec(stripes, pg->pg_size, pg->pg_data_size, clean_entry_bitmap_size);
         }
         cur_op->iov.push_back(op_data->stripes[0].bmp_buf, cur_op->reply.rw.bitmap_len);
-        for (int role = 0; role < op_data->pg->pg_size; role++)
+        for (int role = 0; role < pg->pg_size; role++)
         {
             if (stripes[role].req_end != 0)
             {
@@ -632,7 +656,7 @@ void osd_t::remove_object_from_state(object_id & oid, pg_osd_set_state_t **objec
     {
         this->misplaced_objects--;
         pg.misplaced_objects.erase(oid);
-        if (!pg.misplaced_objects.size())
+        if (!pg.misplaced_objects.size() && !pg.copies_to_delete_after_sync.size())
         {
             pg.state = pg.state & ~PG_HAS_MISPLACED;
             changed = true;

src/osd/osd_primary_chain.cpp

@@ -7,7 +7,7 @@
 void osd_t::continue_chained_read(osd_op_t *cur_op)
 {
     osd_primary_op_data_t *op_data = cur_op->op_data;
-    auto & pg = *op_data->pg;
+    auto pg = op_data->pg;
     if (op_data->st == 1)
         goto resume_1;
     else if (op_data->st == 2)
@@ -17,7 +17,7 @@ void osd_t::continue_chained_read(osd_op_t *cur_op)
     else if (op_data->st == 4)
         goto resume_4;
     cur_op->reply.rw.bitmap_len = 0;
-    for (int role = 0; role < pg.pg_data_size; role++)
+    for (int role = 0; role < (pg ? pg->pg_data_size : 1); role++)
     {
         op_data->stripes[role].read_start = op_data->stripes[role].req_start;
         op_data->stripes[role].read_end = op_data->stripes[role].req_end;
@@ -40,10 +40,10 @@ resume_3:
 resume_4:
     if (op_data->errors > 0)
     {
-        if (op_data->errcode == -EIO || op_data->errcode == -EDOM)
+        if (pg && (op_data->errcode == -EIO || op_data->errcode == -EDOM))
         {
             // Handle corrupted reads and retry...
-            check_corrupted_chained(pg, cur_op);
+            check_corrupted_chained(*pg, cur_op);
             free(cur_op->buf);
             cur_op->buf = NULL;
             free(op_data->chain_reads);
@@ -63,31 +63,30 @@ resume_4:
     finish_op(cur_op, cur_op->req.rw.len);
 }
 
-int osd_t::read_bitmaps(osd_op_t *cur_op, pg_t & pg, int base_state)
+int osd_t::read_bitmaps(osd_op_t *cur_op, pg_t *pg, int base_state)
 {
     osd_primary_op_data_t *op_data = cur_op->op_data;
     if (op_data->st == base_state)
         goto resume_0;
     else if (op_data->st == base_state+1)
         goto resume_1;
-    if (pg.state == PG_ACTIVE && pg.scheme == POOL_SCHEME_REPLICATED)
+    if (!pg || pg->state == PG_ACTIVE && pg->scheme == POOL_SCHEME_REPLICATED)
     {
         // Happy path for clean replicated PGs (all bitmaps are available locally)
+        osd_primary_op_data_t *op_data = cur_op->op_data;
         for (int chain_num = 0; chain_num < op_data->chain_size; chain_num++)
         {
             object_id cur_oid = { .inode = op_data->read_chain[chain_num], .stripe = op_data->oid.stripe };
-            auto vo_it = pg.ver_override.find(cur_oid);
-            auto read_version = (vo_it != pg.ver_override.end() ? vo_it->second : UINT64_MAX);
             // Read bitmap synchronously from the local database
             bs->read_bitmap(
-                cur_oid, read_version, (uint8_t*)op_data->snapshot_bitmaps + chain_num*clean_entry_bitmap_size,
+                cur_oid, UINT64_MAX, (uint8_t*)op_data->snapshot_bitmaps + chain_num*clean_entry_bitmap_size,
                 !chain_num ? &cur_op->reply.rw.version : NULL
             );
         }
     }
     else
     {
-        if (submit_bitmap_subops(cur_op, pg) < 0)
+        if (submit_bitmap_subops(cur_op, *pg) < 0)
         {
             // Failure
             finish_op(cur_op, -EIO);
@@ -101,32 +100,32 @@ resume_0:
             return 1;
         }
 resume_1:
-        if (pg.scheme != POOL_SCHEME_REPLICATED)
+        if (pg->scheme != POOL_SCHEME_REPLICATED)
         {
             for (int chain_num = 0; chain_num < op_data->chain_size; chain_num++)
             {
                 // Check if we need to reconstruct any bitmaps
-                for (int i = 0; i < pg.pg_size; i++)
+                for (int i = 0; i < pg->pg_size; i++)
                 {
-                    if (op_data->missing_flags[chain_num*pg.pg_size + i])
+                    if (op_data->missing_flags[chain_num*pg->pg_size + i])
                     {
-                        osd_rmw_stripe_t local_stripes[pg.pg_size];
+                        osd_rmw_stripe_t local_stripes[pg->pg_size];
-                        for (i = 0; i < pg.pg_size; i++)
+                        for (i = 0; i < pg->pg_size; i++)
                         {
                             local_stripes[i] = (osd_rmw_stripe_t){
-                                .bmp_buf = (uint8_t*)op_data->snapshot_bitmaps + (chain_num*pg.pg_size + i)*clean_entry_bitmap_size,
+                                .bmp_buf = (uint8_t*)op_data->snapshot_bitmaps + (chain_num*pg->pg_size + i)*clean_entry_bitmap_size,
                                 .read_start = 1,
                                 .read_end = 1,
-                                .missing = op_data->missing_flags[chain_num*pg.pg_size + i] && true,
+                                .missing = op_data->missing_flags[chain_num*pg->pg_size + i] && true,
                             };
                         }
-                        if (pg.scheme == POOL_SCHEME_XOR)
+                        if (pg->scheme == POOL_SCHEME_XOR)
                         {
-                            reconstruct_stripes_xor(local_stripes, pg.pg_size, clean_entry_bitmap_size);
+                            reconstruct_stripes_xor(local_stripes, pg->pg_size, clean_entry_bitmap_size);
                         }
-                        else if (pg.scheme == POOL_SCHEME_EC)
+                        else if (pg->scheme == POOL_SCHEME_EC)
                         {
-                            reconstruct_stripes_ec(local_stripes, pg.pg_size, pg.pg_data_size, clean_entry_bitmap_size);
+                            reconstruct_stripes_ec(local_stripes, pg->pg_size, pg->pg_data_size, clean_entry_bitmap_size);
                         }
                         break;
                     }
@@ -139,6 +138,7 @@ resume_1:
 
 int osd_t::collect_bitmap_requests(osd_op_t *cur_op, pg_t & pg, std::vector<bitmap_request_t> & bitmap_requests)
 {
+    assert(&pg);
     osd_primary_op_data_t *op_data = cur_op->op_data;
     for (int chain_num = 0; chain_num < op_data->chain_size; chain_num++)
     {
@@ -216,6 +216,7 @@ int osd_t::collect_bitmap_requests(osd_op_t *cur_op, pg_t & pg, std::vector<bitm
 
 int osd_t::submit_bitmap_subops(osd_op_t *cur_op, pg_t & pg)
 {
+    assert(&pg);
     osd_primary_op_data_t *op_data = cur_op->op_data;
     std::vector<bitmap_request_t> *bitmap_requests = new std::vector<bitmap_request_t>();
     if (collect_bitmap_requests(cur_op, pg, *bitmap_requests) < 0)
@@ -382,12 +383,12 @@ std::vector<osd_chain_read_t> osd_t::collect_chained_read_requests(osd_op_t *cur
     return chain_reads;
 }
 
-int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
+int osd_t::submit_chained_read_requests(pg_t *pg, osd_op_t *cur_op)
 {
     // Decide which parts of which objects we need to read based on bitmaps
     osd_primary_op_data_t *op_data = cur_op->op_data;
     auto chain_reads = collect_chained_read_requests(cur_op);
-    int stripe_count = (pg.scheme == POOL_SCHEME_REPLICATED ? 1 : pg.pg_size);
+    int stripe_count = (!pg || pg->scheme == POOL_SCHEME_REPLICATED ? 1 : pg->pg_size);
     op_data->chain_read_count = chain_reads.size();
     op_data->chain_reads = (osd_chain_read_t*)calloc_or_die(
         1, sizeof(osd_chain_read_t) * chain_reads.size()
@@ -408,23 +409,23 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
         object_id cur_oid = { .inode = chain_reads[cri].inode, .stripe = op_data->oid.stripe };
         // FIXME: maybe introduce split_read_stripes to shorten these lines and to remove read_start=req_start
         osd_rmw_stripe_t *stripes = chain_stripes + chain_reads[cri].chain_pos*stripe_count;
-        split_stripes(pg.pg_data_size, bs_block_size, chain_reads[cri].offset, chain_reads[cri].len, stripes);
+        split_stripes(pg ? pg->pg_data_size : 1, bs_block_size, chain_reads[cri].offset, chain_reads[cri].len, stripes);
-        if (pg.scheme == POOL_SCHEME_REPLICATED && !stripes[0].req_end)
+        if ((!pg || pg->scheme == POOL_SCHEME_REPLICATED) && !stripes[0].req_end)
         {
             continue;
         }
-        for (int role = 0; role < pg.pg_data_size; role++)
+        for (int role = 0; role < (pg ? pg->pg_data_size : 1); role++)
         {
             stripes[role].read_start = stripes[role].req_start;
             stripes[role].read_end = stripes[role].req_end;
         }
-        uint64_t *cur_set = pg.cur_set.data();
+        uint64_t *cur_set = pg ? pg->cur_set.data() : &this->osd_num;
-        if (pg.state != PG_ACTIVE)
+        if (pg && pg->state != PG_ACTIVE)
         {
-            cur_set = get_object_osd_set(pg, cur_oid, &op_data->chain_states[chain_reads[cri].chain_pos]);
+            cur_set = get_object_osd_set(*pg, cur_oid, &op_data->chain_states[chain_reads[cri].chain_pos]);
-            if (pg.scheme != POOL_SCHEME_REPLICATED)
+            if (pg->scheme != POOL_SCHEME_REPLICATED)
             {
-                if (extend_missing_stripes(stripes, cur_set, pg.pg_data_size, pg.pg_size) < 0)
+                if (extend_missing_stripes(stripes, cur_set, pg->pg_data_size, pg->pg_size) < 0)
                 {
                     free(op_data->chain_reads);
                     op_data->chain_reads = NULL;
@@ -445,14 +446,14 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
                 }
             }
         }
-        if (pg.scheme == POOL_SCHEME_REPLICATED)
+        if (!pg || pg->scheme == POOL_SCHEME_REPLICATED)
         {
             n_subops++;
             read_buffer_size += stripes[0].read_end - stripes[0].read_start;
         }
         else
         {
-            for (int role = 0; role < pg.pg_size; role++)
+            for (int role = 0; role < pg->pg_size; role++)
             {
                 if (stripes[role].read_end > 0 && cur_set[role] != 0)
                     n_subops++;
@@ -490,19 +491,23 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
     for (int cri = 0; cri < chain_reads.size(); cri++)
     {
         osd_rmw_stripe_t *stripes = chain_stripes + chain_reads[cri].chain_pos*stripe_count;
-        if (pg.scheme == POOL_SCHEME_REPLICATED && !stripes[0].req_end)
+        if ((!pg || pg->scheme == POOL_SCHEME_REPLICATED) && !stripes[0].req_end)
         {
             continue;
         }
         object_id cur_oid = { .inode = chain_reads[cri].inode, .stripe = op_data->oid.stripe };
-        auto vo_it = pg.ver_override.find(cur_oid);
+        uint64_t target_ver = UINT64_MAX;
-        uint64_t target_ver = vo_it != pg.ver_override.end() ? vo_it->second : UINT64_MAX;
+        if (pg)
-        auto cur_state = op_data->chain_states[chain_reads[cri].chain_pos];
-        uint64_t *cur_set = (pg.state != PG_ACTIVE && cur_state ? cur_state->read_target.data() : pg.cur_set.data());
-        int zero_read = -1;
-        if (pg.scheme == POOL_SCHEME_REPLICATED)
         {
-            for (int role = 0; role < pg.pg_size; role++)
+            auto vo_it = pg->ver_override.find(cur_oid);
+            target_ver = vo_it != pg->ver_override.end() ? vo_it->second : UINT64_MAX;
+        }
+        auto cur_state = op_data->chain_states[chain_reads[cri].chain_pos];
+        uint64_t *cur_set = (!pg ? &this->osd_num : (pg->state != PG_ACTIVE && cur_state ? cur_state->read_target.data() : pg->cur_set.data()));
+        int zero_read = -1;
+        if (!pg || pg->scheme == POOL_SCHEME_REPLICATED)
+        {
+            for (int role = 0; role < (pg ? pg->pg_size : 1); role++)
                 if (cur_set[role] == this->osd_num || zero_read == -1)
                     zero_read = role;
         }
@@ -514,6 +519,7 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
 
 void osd_t::check_corrupted_chained(pg_t & pg, osd_op_t *cur_op)
 {
+    assert(&pg);
     osd_primary_op_data_t *op_data = cur_op->op_data;
     int stripe_count = (pg.scheme == POOL_SCHEME_REPLICATED ? 1 : pg.pg_size);
     osd_rmw_stripe_t *chain_stripes = (osd_rmw_stripe_t*)(
@@ -539,33 +545,32 @@ void osd_t::check_corrupted_chained(pg_t & pg, osd_op_t *cur_op)
     }
 }
 
-void osd_t::send_chained_read_results(pg_t & pg, osd_op_t *cur_op)
+void osd_t::send_chained_read_results(pg_t *pg, osd_op_t *cur_op)
 {
     osd_primary_op_data_t *op_data = cur_op->op_data;
-    int stripe_count = (pg.scheme == POOL_SCHEME_REPLICATED ? 1 : pg.pg_size);
+    int stripe_count = (!pg || pg->scheme == POOL_SCHEME_REPLICATED ? 1 : pg->pg_size);
     osd_rmw_stripe_t *chain_stripes = (osd_rmw_stripe_t*)(
         (uint8_t*)op_data->chain_reads + sizeof(osd_chain_read_t) * op_data->chain_read_count
     );
     // Reconstruct parts if needed
     if (op_data->degraded)
     {
-        int stripe_count = (pg.scheme == POOL_SCHEME_REPLICATED ? 1 : pg.pg_size);
         for (int cri = 0; cri < op_data->chain_read_count; cri++)
         {
             // Reconstruct missing stripes
             osd_rmw_stripe_t *stripes = chain_stripes + op_data->chain_reads[cri].chain_pos*stripe_count;
-            if (pg.scheme == POOL_SCHEME_XOR)
+            if (pg->scheme == POOL_SCHEME_XOR)
             {
-                reconstruct_stripes_xor(stripes, pg.pg_size, clean_entry_bitmap_size);
+                reconstruct_stripes_xor(stripes, pg->pg_size, clean_entry_bitmap_size);
             }
-            else if (pg.scheme == POOL_SCHEME_EC)
+            else if (pg->scheme == POOL_SCHEME_EC)
             {
-                reconstruct_stripes_ec(stripes, pg.pg_size, pg.pg_data_size, clean_entry_bitmap_size);
+                reconstruct_stripes_ec(stripes, pg->pg_size, pg->pg_data_size, clean_entry_bitmap_size);
             }
         }
     }
     // Send bitmap
-    cur_op->reply.rw.bitmap_len = pg.pg_data_size * clean_entry_bitmap_size;
+    cur_op->reply.rw.bitmap_len = (pg ? pg->pg_data_size : 1) * clean_entry_bitmap_size;
     cur_op->iov.push_back(op_data->stripes[0].bmp_buf, cur_op->reply.rw.bitmap_len);
     // And finally compose the result
     uint64_t sent = 0;

src/osd/osd_primary_subops.cpp

@@ -67,26 +67,20 @@ void osd_t::finish_op(osd_op_t *cur_op, int retval)
         if (cur_op->req.hdr.opcode == OSD_OP_DELETE)
         {
             if (cur_op->op_data)
-                inode_stats[cur_op->req.rw.inode].op_bytes[inode_st_op] += cur_op->op_data->pg->pg_data_size * bs_block_size;
+            {
+                inode_stats[cur_op->req.rw.inode].op_bytes[inode_st_op] += (cur_op->op_data->pg
+                    ? cur_op->op_data->pg->pg_data_size : 1) * bs_block_size;
+            }
         }
         else
             inode_stats[cur_op->req.rw.inode].op_bytes[inode_st_op] += cur_op->req.rw.len;
     }
     if (cur_op->op_data)
     {
-        if (cur_op->op_data->pg_num > 0)
+        if (cur_op->op_data->pg)
         {
             auto & pg = *cur_op->op_data->pg;
-            pg.inflight--;
+            rm_inflight(pg);
-            assert(pg.inflight >= 0);
-            if ((pg.state & PG_STOPPING) && pg.inflight == 0 && !pg.flush_batch)
-            {
-                finish_stop_pg(pg);
-            }
-            else if ((pg.state & PG_REPEERING) && pg.inflight == 0 && !pg.flush_batch)
-            {
-                start_pg_peering(pg);
-            }
         }
         assert(!cur_op->op_data->subops);
         free(cur_op->op_data);
@@ -126,10 +120,10 @@ void osd_t::submit_primary_subops(int submit_type, uint64_t op_version, const ui
     bool wr = submit_type == SUBMIT_WRITE;
     osd_primary_op_data_t *op_data = cur_op->op_data;
     osd_rmw_stripe_t *stripes = op_data->stripes;
-    bool rep = op_data->pg->scheme == POOL_SCHEME_REPLICATED;
+    bool rep = !op_data->pg || op_data->pg->scheme == POOL_SCHEME_REPLICATED;
     // Allocate subops
     int n_subops = 0, zero_read = -1;
-    for (int role = 0; role < op_data->pg->pg_size; role++)
+    for (int role = 0; role < (op_data->pg ? op_data->pg->pg_size : 1); role++)
     {
         if (osd_set[role] == this->osd_num || osd_set[role] != 0 && zero_read == -1)
             zero_read = role;
@@ -152,11 +146,11 @@ void osd_t::submit_primary_subops(int submit_type, uint64_t op_version, const ui
 int osd_t::submit_primary_subop_batch(int submit_type, inode_t inode, uint64_t op_version,
     osd_rmw_stripe_t *stripes, const uint64_t* osd_set, osd_op_t *cur_op, int subop_idx, int zero_read)
 {
-    bool rep = cur_op->op_data->pg->scheme == POOL_SCHEME_REPLICATED;
+    bool rep = !cur_op->op_data->pg || cur_op->op_data->pg->scheme == POOL_SCHEME_REPLICATED;
     bool wr = submit_type == SUBMIT_WRITE;
     osd_primary_op_data_t *op_data = cur_op->op_data;
     int i = subop_idx;
-    for (int role = 0; role < op_data->pg->pg_size; role++)
+    for (int role = 0; role < (op_data->pg ? op_data->pg->pg_size : 1); role++)
     {
         // We always submit zero-length writes to all replicas, even if the stripe is not modified
         if (!(wr || !rep && stripes[role].read_end != 0 || zero_read == role || submit_type == SUBMIT_SCRUB_READ))
@@ -434,6 +428,14 @@ void osd_t::handle_primary_subop(osd_op_t *subop, osd_op_t *cur_op)
                 retval, expected, peer_osd
             );
         }
+        else if (opcode == OSD_OP_SEC_DELETE)
+        {
+            printf(
+                "delete subop to %jx:%jx v%ju failed on osd %jd: retval = %d (expected %d)\n",
+                subop->req.sec_del.oid.inode, subop->req.sec_del.oid.stripe, subop->req.sec_del.version,
+                peer_osd, retval, expected
+            );
+        }
         else
         {
             printf(

src/osd/osd_primary_sync.cpp

@@ -80,15 +80,17 @@ resume_2:
         this->unstable_writes.clear();
     }
     {
+        op_data->dirty_pg_count = dirty_pgs.size();
+        op_data->dirty_osd_count = dirty_osds.size();
         void *dirty_buf = malloc_or_die(
             sizeof(pool_pg_num_t)*dirty_pgs.size() +
+            sizeof(uint64_t)*dirty_pgs.size() +
             sizeof(osd_num_t)*dirty_osds.size() +
             sizeof(obj_ver_osd_t)*this->copies_to_delete_after_sync_count
         );
         op_data->dirty_pgs = (pool_pg_num_t*)dirty_buf;
-        op_data->dirty_osds = (osd_num_t*)((uint8_t*)dirty_buf + sizeof(pool_pg_num_t)*dirty_pgs.size());
+        uint64_t *pg_del_counts = (uint64_t*)((uint8_t*)op_data->dirty_pgs + (sizeof(pool_pg_num_t))*op_data->dirty_pg_count);
-        op_data->dirty_pg_count = dirty_pgs.size();
+        op_data->dirty_osds = (osd_num_t*)((uint8_t*)pg_del_counts + 8*op_data->dirty_pg_count);
-        op_data->dirty_osd_count = dirty_osds.size();
         if (this->copies_to_delete_after_sync_count)
         {
             op_data->copies_to_delete_count = 0;
@@ -103,16 +105,16 @@ resume_2:
                     sizeof(obj_ver_osd_t)*pg.copies_to_delete_after_sync.size()
                 );
                 op_data->copies_to_delete_count += pg.copies_to_delete_after_sync.size();
-                this->copies_to_delete_after_sync_count -= pg.copies_to_delete_after_sync.size();
-                pg.copies_to_delete_after_sync.clear();
             }
-            assert(this->copies_to_delete_after_sync_count == 0);
         }
         int dpg = 0;
         for (auto dirty_pg_num: dirty_pgs)
         {
-            pgs.at(dirty_pg_num).inflight++;
+            auto & pg = pgs.at(dirty_pg_num);
-            op_data->dirty_pgs[dpg++] = dirty_pg_num;
+            pg.inflight++;
+            op_data->dirty_pgs[dpg] = dirty_pg_num;
+            pg_del_counts[dpg] = pg.copies_to_delete_after_sync.size();
+            dpg++;
         }
         dirty_pgs.clear();
         dpg = 0;
@@ -183,23 +185,6 @@ resume_6:
                 }
             }
         }
-        if (op_data->copies_to_delete)
-        {
-            // Return 'copies to delete' back into respective PGs
-            for (int i = 0; i < op_data->copies_to_delete_count; i++)
-            {
-                auto & w = op_data->copies_to_delete[i];
-                auto & pg = pgs.at((pool_pg_num_t){
-                    .pool_id = INODE_POOL(w.oid.inode),
-                    .pg_num = map_to_pg(w.oid, st_cli.pool_config.at(INODE_POOL(w.oid.inode)).pg_stripe_size),
-                });
-                if (pg.state & PG_ACTIVE)
-                {
-                    pg.copies_to_delete_after_sync.push_back(w);
-                    copies_to_delete_after_sync_count++;
-                }
-            }
-        }
     }
     else if (op_data->copies_to_delete)
     {
@@ -213,6 +198,22 @@ resume_8:
         {
             goto resume_6;
         }
+        {
+            uint64_t *pg_del_counts = (uint64_t*)((uint8_t*)op_data->dirty_pgs + (sizeof(pool_pg_num_t))*op_data->dirty_pg_count);
+            for (int i = 0; i < op_data->dirty_pg_count; i++)
+            {
+                auto & pg = pgs.at(op_data->dirty_pgs[i]);
+                auto n = pg_del_counts[i];
+                assert(copies_to_delete_after_sync_count >= n);
+                copies_to_delete_after_sync_count -= n;
+                pg.copies_to_delete_after_sync.erase(pg.copies_to_delete_after_sync.begin(), pg.copies_to_delete_after_sync.begin()+n);
+                if (!pg.misplaced_objects.size() && !pg.copies_to_delete_after_sync.size() && (pg.state & PG_HAS_MISPLACED))
+                {
+                    pg.state = pg.state & ~PG_HAS_MISPLACED;
+                    report_pg_state(pg);
+                }
+            }
+        }
         if (immediate_commit == IMMEDIATE_NONE)
         {
             // Mark OSDs as dirty because deletions have to be synced too!
@@ -226,15 +227,7 @@ resume_8:
     for (int i = 0; i < op_data->dirty_pg_count; i++)
     {
         auto & pg = pgs.at(op_data->dirty_pgs[i]);
-        pg.inflight--;
+        rm_inflight(pg);
-        if ((pg.state & PG_STOPPING) && pg.inflight == 0 && !pg.flush_batch)
-        {
-            finish_stop_pg(pg);
-        }
-        else if ((pg.state & PG_REPEERING) && pg.inflight == 0 && !pg.flush_batch)
-        {
-            start_pg_peering(pg);
-        }
     }
     // FIXME: Free those in the destructor (not here)?
     free(op_data->dirty_pgs);

src/osd/osd_primary_write.cpp

@@ -301,6 +301,38 @@ resume_12:
     }
     if (op_data->object_state)
     {
+        // Any kind of a non-clean object can have extra chunks, because we don't record objects
+        // as degraded & misplaced or incomplete & misplaced at the same time. So try to remove extra chunks
+        if (immediate_commit != IMMEDIATE_ALL)
+        {
+            // We can't remove extra chunks yet if fsyncs are explicit, because
+            // new copies may not be committed to stable storage yet
+            // We can only remove extra chunks after a successful SYNC for this PG
+            for (auto & chunk: op_data->object_state->osd_set)
+            {
+                // Check is the same as in submit_primary_del_subops()
+                if (pg.scheme == POOL_SCHEME_REPLICATED
+                    ? !contains_osd(pg.cur_set.data(), pg.pg_size, chunk.osd_num)
+                    : (chunk.osd_num != pg.cur_set[chunk.role]))
+                {
+                    pg.copies_to_delete_after_sync.push_back((obj_ver_osd_t){
+                        .osd_num = chunk.osd_num,
+                        .oid = {
+                            .inode = op_data->oid.inode,
+                            .stripe = op_data->oid.stripe | (pg.scheme == POOL_SCHEME_REPLICATED ? 0 : chunk.role),
+                        },
+                        .version = op_data->fact_ver,
+                    });
+                    copies_to_delete_after_sync_count++;
+                }
+            }
+            if (pg.copies_to_delete_after_sync.size() && !(pg.state & PG_HAS_MISPLACED))
+            {
+                // PG can't be active+clean until extra copies aren't removed, so mark it as PG_HAS_MISPLACED
+                pg.state |= PG_HAS_MISPLACED;
+                //this->pg_state_dirty.insert({ .pool_id = pg.pool_id, .pg_num = pg.pg_num });
+            }
+        }
         // We must forget the unclean state of the object before deleting it
         // so the next reads don't accidentally read a deleted version
         // And it should be done at the same time as the removal of the version override
@@ -309,6 +341,7 @@ resume_12:
     }
 resume_6:
 resume_7:
+    op_data->n_subops = 0;
     if (!remember_unstable_write(cur_op, pg, pg.cur_loc_set, 6))
     {
         return;
@@ -344,48 +377,21 @@ resume_7:
             );
             recovery_stat[recovery_type].usec += usec;
         }
-        // Any kind of a non-clean object can have extra chunks, because we don't record objects
+        if (immediate_commit == IMMEDIATE_ALL)
-        // as degraded & misplaced or incomplete & misplaced at the same time. So try to remove extra chunks
-        if (immediate_commit != IMMEDIATE_ALL)
-        {
-            // We can't remove extra chunks yet if fsyncs are explicit, because
-            // new copies may not be committed to stable storage yet
-            // We can only remove extra chunks after a successful SYNC for this PG
-            for (auto & chunk: op_data->object_state->osd_set)
-            {
-                // Check is the same as in submit_primary_del_subops()
-                if (pg.scheme == POOL_SCHEME_REPLICATED
-                    ? !contains_osd(pg.cur_set.data(), pg.pg_size, chunk.osd_num)
-                    : (chunk.osd_num != pg.cur_set[chunk.role]))
-                {
-                    pg.copies_to_delete_after_sync.push_back((obj_ver_osd_t){
-                        .osd_num = chunk.osd_num,
-                        .oid = {
-                            .inode = op_data->oid.inode,
-                            .stripe = op_data->oid.stripe | (pg.scheme == POOL_SCHEME_REPLICATED ? 0 : chunk.role),
-                        },
-                        .version = op_data->fact_ver,
-                    });
-                    copies_to_delete_after_sync_count++;
-                }
-            }
-            deref_object_state(pg, &op_data->object_state, true);
-        }
-        else
         {
             submit_primary_del_subops(cur_op, pg.cur_set.data(), pg.pg_size, op_data->object_state->osd_set);
-            deref_object_state(pg, &op_data->object_state, true);
+        }
-            if (op_data->n_subops > 0)
+        deref_object_state(pg, &op_data->object_state, true);
-            {
+        if (op_data->n_subops > 0)
+        {
 resume_8:
-                op_data->st = 8;
+            op_data->st = 8;
-                return;
+            return;
 resume_9:
-                if (op_data->errors > 0)
+            if (op_data->errors > 0)
-                {
+            {
-                    pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->errcode);
+                pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->errcode);
-                    return;
+                return;
-                }
             }
         }
     }

src/osd/osd_rmw.cpp

@@ -162,6 +162,7 @@ struct reed_sol_matrix_t
     int refs = 0;
     int *je_data;
     uint8_t *isal_data;
+    int isal_item_size;
     // 32 bytes = 256/8 = max pg_size/8
     std::map<std::array<uint8_t, 32>, void*> subdata;
     std::map<reed_sol_erased_t, void*> decodings;
@@ -181,20 +182,42 @@ void use_ec(int pg_size, int pg_minsize, bool use)
         }
         int *matrix = reed_sol_vandermonde_coding_matrix(pg_minsize, pg_size-pg_minsize, OSD_JERASURE_W);
         uint8_t *isal_table = NULL;
+        int item_size = 8;
 #ifdef WITH_ISAL
         uint8_t *isal_matrix = (uint8_t*)malloc_or_die(pg_minsize*(pg_size-pg_minsize));
         for (int i = 0; i < pg_minsize*(pg_size-pg_minsize); i++)
         {
             isal_matrix[i] = matrix[i];
         }
-        isal_table = (uint8_t*)malloc_or_die(pg_minsize*(pg_size-pg_minsize)*32);
+        isal_table = (uint8_t*)calloc_or_die(1, pg_minsize*(pg_size-pg_minsize)*32);
         ec_init_tables(pg_minsize, pg_size-pg_minsize, isal_matrix, isal_table);
         free(isal_matrix);
+        for (int i = pg_minsize*(pg_size-pg_minsize)*8; i < pg_minsize*(pg_size-pg_minsize)*32; i++)
+        {
+            if (isal_table[i] != 0)
+            {
+                // ISA-L GF-NI version uses 8-byte table items
+                item_size = 32;
+                break;
+            }
+        }
+        // Sanity check: rows should never consist of all zeroes
+        uint8_t zero_row[pg_minsize*item_size];
+        memset(zero_row, 0, pg_minsize*item_size);
+        for (int i = 0; i < (pg_size-pg_minsize); i++)
+        {
+            if (memcmp(isal_table + i*pg_minsize*item_size, zero_row, pg_minsize*item_size) == 0)
+            {
+                fprintf(stderr, "BUG or ISA-L incompatibility: EC tables shouldn't have all-zero rows\n");
+                abort();
+            }
+        }
 #endif
         matrices[key] = (reed_sol_matrix_t){
             .refs = 0,
             .je_data = matrix,
             .isal_data = isal_table,
+            .isal_item_size = item_size,
         };
         rs_it = matrices.find(key);
     }
@@ -235,7 +258,7 @@ static reed_sol_matrix_t* get_ec_matrix(int pg_size, int pg_minsize)
 // we don't need it. also it makes an extra allocation of int *erased on every call and doesn't cache
 // the decoding matrix.
 // all these flaws are fixed in this function:
-static void* get_jerasure_decoding_matrix(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsize)
+static void* get_jerasure_decoding_matrix(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsize, int *item_size)
 {
     int edd = 0;
     int erased[pg_size];
@@ -292,6 +315,7 @@ static void* get_jerasure_decoding_matrix(osd_rmw_stripe_t *stripes, int pg_size
         int *erased_copy = (int*)(rectable + 32*smrow*pg_minsize);
         memcpy(erased_copy, erased, pg_size*sizeof(int));
         matrix->decodings.emplace((reed_sol_erased_t){ .data = erased_copy, .size = pg_size }, rectable);
+        *item_size = matrix->isal_item_size;
         return rectable;
 #else
         int *dm_ids = (int*)malloc_or_die(sizeof(int)*(pg_minsize + pg_minsize*pg_minsize + pg_size));
@@ -355,7 +379,8 @@ static void jerasure_matrix_encode_unaligned(int k, int m, int w, int *matrix, c
 #ifdef WITH_ISAL
 void reconstruct_stripes_ec(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsize, uint32_t bitmap_size)
 {
-    uint8_t *dectable = (uint8_t*)get_jerasure_decoding_matrix(stripes, pg_size, pg_minsize);
+    int item_size = 0;
+    uint8_t *dectable = (uint8_t*)get_jerasure_decoding_matrix(stripes, pg_size, pg_minsize, &item_size);
     if (!dectable)
     {
         return;
@@ -378,7 +403,7 @@ void reconstruct_stripes_ec(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsi
                 }
             }
             ec_encode_data(
-                read_end-read_start, pg_minsize, wanted, dectable + wanted_base*32*pg_minsize,
+                read_end-read_start, pg_minsize, wanted, dectable + wanted_base*item_size*pg_minsize,
                 data_ptrs, data_ptrs + pg_minsize
             );
         }
@@ -433,7 +458,7 @@ void reconstruct_stripes_ec(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsi
 #else
 void reconstruct_stripes_ec(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsize, uint32_t bitmap_size)
 {
-    int *dm_ids = (int*)get_jerasure_decoding_matrix(stripes, pg_size, pg_minsize);
+    int *dm_ids = (int*)get_jerasure_decoding_matrix(stripes, pg_size, pg_minsize, NULL);
     if (!dm_ids)
     {
         return;
@@ -980,7 +1005,7 @@ void calc_rmw_parity_ec(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsize,
                 {
                     int item_size =
 #ifdef WITH_ISAL
-                        32;
+                        matrix->isal_item_size;
 #else
                         sizeof(int);
 #endif

src/osd/osd_secondary.cpp

@@ -79,6 +79,32 @@ void osd_t::exec_secondary(osd_op_t *op)
     }
 }
 
+bool osd_t::sec_check_pg_lock(osd_num_t primary_osd, const object_id &oid)
+{
+    if (!enable_pg_locks)
+    {
+        return true;
+    }
+    pool_id_t pool_id = INODE_POOL(oid.inode);
+    auto pool_cfg_it = st_cli.pool_config.find(pool_id);
+    if (pool_cfg_it == st_cli.pool_config.end())
+    {
+        return false;
+    }
+    auto ppg = (pool_pg_num_t){ .pool_id = pool_id, .pg_num = map_to_pg(oid, pool_cfg_it->second.pg_stripe_size) };
+    auto pg_it = pgs.find(ppg);
+    if (pg_it != pgs.end() && pg_it->second.state != PG_OFFLINE)
+    {
+        return false;
+    }
+    if (pg_it->second.disable_pg_locks)
+    {
+        return true;
+    }
+    auto lock_it = pg_locks.find(ppg);
+    return lock_it != pg_locks.end() && lock_it->second.primary_osd == primary_osd;
+}
+
 void osd_t::exec_secondary_real(osd_op_t *cur_op)
 {
     if (cur_op->req.hdr.opcode == OSD_OP_SEC_LIST &&
@@ -89,23 +115,15 @@ void osd_t::exec_secondary_real(osd_op_t *cur_op)
     }
     if (cur_op->req.hdr.opcode == OSD_OP_SEC_READ_BMP)
     {
-        int n = cur_op->req.sec_read_bmp.len / sizeof(obj_ver_id);
+        exec_sec_read_bmp(cur_op);
-        if (n > 0)
-        {
-            obj_ver_id *ov = (obj_ver_id*)cur_op->buf;
-            void *reply_buf = malloc_or_die(n * (8 + clean_entry_bitmap_size));
-            void *cur_buf = reply_buf;
-            for (int i = 0; i < n; i++)
-            {
-                bs->read_bitmap(ov[i].oid, ov[i].version, (uint8_t*)cur_buf + sizeof(uint64_t), (uint64_t*)cur_buf);
-                cur_buf = (uint8_t*)cur_buf + (8 + clean_entry_bitmap_size);
-            }
-            free(cur_op->buf);
-            cur_op->buf = reply_buf;
-        }
-        finish_op(cur_op, n * (8 + clean_entry_bitmap_size));
         return;
     }
+    else if (cur_op->req.hdr.opcode == OSD_OP_SEC_LOCK)
+    {
+        exec_sec_lock(cur_op);
+        return;
+    }
+    auto cl = msgr.clients.at(cur_op->peer_fd);
     cur_op->bs_op = new blockstore_op_t();
     cur_op->bs_op->callback = [this, cur_op](blockstore_op_t* bs_op) { secondary_op_callback(cur_op); };
     cur_op->bs_op->opcode = (cur_op->req.hdr.opcode == OSD_OP_SEC_READ ? BS_OP_READ
@@ -121,6 +139,13 @@ void osd_t::exec_secondary_real(osd_op_t *cur_op)
         cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE ||
         cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE_STABLE)
     {
+        if (!(cur_op->req.sec_rw.flags & OSD_OP_IGNORE_PG_LOCK) &&
+            !sec_check_pg_lock(cl->osd_num, cur_op->req.sec_rw.oid))
+        {
+            cur_op->bs_op->retval = -EPIPE;
+            secondary_op_callback(cur_op);
+            return;
+        }
         if (cur_op->req.hdr.opcode == OSD_OP_SEC_READ)
         {
             // Allocate memory for the read operation
@@ -143,6 +168,13 @@ void osd_t::exec_secondary_real(osd_op_t *cur_op)
     }
     else if (cur_op->req.hdr.opcode == OSD_OP_SEC_DELETE)
     {
+        if (!(cur_op->req.sec_del.flags & OSD_OP_IGNORE_PG_LOCK) &&
+            !sec_check_pg_lock(cl->osd_num, cur_op->req.sec_del.oid))
+        {
+            cur_op->bs_op->retval = -EPIPE;
+            secondary_op_callback(cur_op);
+            return;
+        }
         cur_op->bs_op->oid = cur_op->req.sec_del.oid;
         cur_op->bs_op->version = cur_op->req.sec_del.version;
 #ifdef OSD_STUB
@@ -157,6 +189,18 @@ void osd_t::exec_secondary_real(osd_op_t *cur_op)
 #ifdef OSD_STUB
         cur_op->bs_op->retval = 0;
 #endif
+        if (enable_pg_locks && !(cur_op->req.sec_stab.flags & OSD_OP_IGNORE_PG_LOCK))
+        {
+            for (int i = 0; i < cur_op->bs_op->len; i++)
+            {
+                if (!sec_check_pg_lock(cl->osd_num, ((obj_ver_id*)cur_op->buf)[i].oid))
+                {
+                    cur_op->bs_op->retval = -EPIPE;
+                    secondary_op_callback(cur_op);
+                    return;
+                }
+            }
+        }
     }
     else if (cur_op->req.hdr.opcode == OSD_OP_SEC_LIST)
     {
@@ -192,15 +236,96 @@ void osd_t::exec_secondary_real(osd_op_t *cur_op)
 #endif
 }
 
+void osd_t::exec_sec_read_bmp(osd_op_t *cur_op)
+{
+    auto cl = msgr.clients.at(cur_op->peer_fd);
+    int n = cur_op->req.sec_read_bmp.len / sizeof(obj_ver_id);
+    if (n > 0)
+    {
+        obj_ver_id *ov = (obj_ver_id*)cur_op->buf;
+        void *reply_buf = malloc_or_die(n * (8 + clean_entry_bitmap_size));
+        void *cur_buf = reply_buf;
+        for (int i = 0; i < n; i++)
+        {
+            if (!sec_check_pg_lock(cl->osd_num, ov[i].oid) &&
+                !(cur_op->req.sec_read_bmp.flags & OSD_OP_IGNORE_PG_LOCK))
+            {
+                free(reply_buf);
+                cur_op->bs_op->retval = -EPIPE;
+                secondary_op_callback(cur_op);
+                return;
+            }
+            bs->read_bitmap(ov[i].oid, ov[i].version, (uint8_t*)cur_buf + sizeof(uint64_t), (uint64_t*)cur_buf);
+            cur_buf = (uint8_t*)cur_buf + (8 + clean_entry_bitmap_size);
+        }
+        free(cur_op->buf);
+        cur_op->buf = reply_buf;
+    }
+    finish_op(cur_op, n * (8 + clean_entry_bitmap_size));
+}
+
+// Lock/Unlock PG
+void osd_t::exec_sec_lock(osd_op_t *cur_op)
+{
+    cur_op->reply.sec_lock.cur_primary = 0;
+    auto cl = msgr.clients.at(cur_op->peer_fd);
+    if (!cl->osd_num ||
+        cur_op->req.sec_lock.flags != OSD_SEC_LOCK_PG &&
+        cur_op->req.sec_lock.flags != OSD_SEC_UNLOCK_PG ||
+        cur_op->req.sec_lock.pool_id > ((uint64_t)1<<POOL_ID_BITS) ||
+        !cur_op->req.sec_lock.pg_num ||
+        cur_op->req.sec_lock.pg_num > UINT32_MAX)
+    {
+        finish_op(cur_op, -EINVAL);
+        return;
+    }
+    auto ppg = (pool_pg_num_t){ .pool_id = (pool_id_t)cur_op->req.sec_lock.pool_id, .pg_num = (pg_num_t)cur_op->req.sec_lock.pg_num };
+    auto pool_cfg_it = st_cli.pool_config.find(ppg.pool_id);
+    if (pool_cfg_it == st_cli.pool_config.end() ||
+        pool_cfg_it->second.real_pg_count < cur_op->req.sec_lock.pg_num)
+    {
+        finish_op(cur_op, -ENOENT);
+        return;
+    }
+    auto lock_it = pg_locks.find(ppg);
+    if (cur_op->req.sec_lock.flags == OSD_SEC_LOCK_PG)
+    {
+        if (lock_it != pg_locks.end() && lock_it->second.primary_osd != cl->osd_num)
+        {
+            cur_op->reply.sec_lock.cur_primary = lock_it->second.primary_osd;
+            finish_op(cur_op, -EBUSY);
+            return;
+        }
+        auto primary_pg_it = pgs.find(ppg);
+        if (primary_pg_it != pgs.end() && primary_pg_it->second.state != PG_OFFLINE)
+        {
+            cur_op->reply.sec_lock.cur_primary = this->osd_num;
+            finish_op(cur_op, -EBUSY);
+            return;
+        }
+        pg_locks[ppg] = (osd_pg_lock_t){
+            .primary_osd = cl->osd_num,
+            .state = cur_op->req.sec_lock.pg_state,
+        };
+    }
+    else if (lock_it != pg_locks.end() && lock_it->second.primary_osd == cl->osd_num)
+    {
+        pg_locks.erase(lock_it);
+    }
+    finish_op(cur_op, 0);
+}
+
 void osd_t::exec_show_config(osd_op_t *cur_op)
 {
     std::string json_err;
     json11::Json req_json = cur_op->req.show_conf.json_len > 0
         ? json11::Json::parse(std::string((char *)cur_op->buf), json_err)
         : json11::Json();
-    if (req_json["check_sequencing"].bool_value())
+    auto peer_osd_num = req_json["osd_num"].uint64_value();
+    auto cl = msgr.clients.at(cur_op->peer_fd);
+    cl->osd_num = peer_osd_num;
+    if (req_json["features"]["check_sequencing"].bool_value())
     {
-        auto cl = msgr.clients.at(cur_op->peer_fd);
         cl->check_sequencing = true;
         cl->read_op_id = cur_op->req.hdr.id + 1;
     }
@@ -216,6 +341,7 @@ void osd_t::exec_show_config(osd_op_t *cur_op)
         { "immediate_commit", (immediate_commit == IMMEDIATE_ALL ? "all" :
             (immediate_commit == IMMEDIATE_SMALL ? "small" : "none")) },
         { "lease_timeout", etcd_report_interval+(st_cli.max_etcd_attempts*(2*st_cli.etcd_quick_timeout)+999)/1000 },
+        { "features", json11::Json::object{ { "pg_locks", true } } },
     };
 #ifdef WITH_RDMA
     if (msgr.is_rdma_enabled())
@@ -228,7 +354,7 @@ void osd_t::exec_show_config(osd_op_t *cur_op)
             bool ok = msgr.connect_rdma(cur_op->peer_fd, req_json["connect_rdma"].string_value(), req_json["rdma_max_msg"].uint64_value());
             if (ok)
             {
-                auto rc = msgr.clients.at(cur_op->peer_fd)->rdma_conn;
+                auto rc = cl->rdma_conn;
                 wire_config["rdma_address"] = rc->addr.to_string();
                 wire_config["rdma_max_msg"] = rc->max_msg;
             }

src/util/addr_util.cpp

@@ -245,3 +245,13 @@ int create_and_bind_socket(std::string bind_address, int bind_port, int listen_b
 
     return listen_fd;
 }
+
+std::string gethostname_str()
+{
+    std::string hostname;
+    hostname.resize(1024);
+    while (gethostname((char*)hostname.data(), hostname.size()) < 0 && errno == ENAMETOOLONG)
+        hostname.resize(hostname.size()+1024);
+    hostname.resize(strnlen(hostname.data(), hostname.size()));
+    return hostname;
+}

src/util/addr_util.h

@@ -21,3 +21,4 @@ bool cidr6_match(const in6_addr &address, const in6_addr &network, uint8_t bits)
 bool cidr_sockaddr_match(const sockaddr_storage &addr, const addr_mask_t &mask);
 std::vector<std::string> getifaddr_list(const std::vector<addr_mask_t> & masks = std::vector<addr_mask_t>(), bool include_v6 = false);
 int create_and_bind_socket(std::string bind_address, int bind_port, int listen_backlog, int *listening_port);
+std::string gethostname_str();

tests/run_3osds.sh

@@ -68,17 +68,17 @@ if [ "$SCHEME" = "ec" ]; then
     PG_SIZE=${PG_SIZE:-5}
     PG_MINSIZE=${PG_MINSIZE:-4}
     PG_DATA_SIZE=${PG_DATA_SIZE:-3}
-    POOLCFG='"scheme":"ec","parity_chunks":'$((PG_SIZE-PG_DATA_SIZE))
+    POOLCFG="$POOLCFG"'"scheme":"ec","parity_chunks":'$((PG_SIZE-PG_DATA_SIZE))
 elif [ "$SCHEME" = "xor" ]; then
     PG_SIZE=${PG_SIZE:-3}
     PG_MINSIZE=${PG_MINSIZE:-3}
     PG_DATA_SIZE=$((PG_SIZE-1))
-    POOLCFG='"scheme":"xor","parity_chunks":1'
+    POOLCFG="$POOLCFG"'"scheme":"xor","parity_chunks":1'
 else
     PG_SIZE=${PG_SIZE:-2}
     PG_MINSIZE=${PG_MINSIZE:-2}
     PG_DATA_SIZE=1
-    POOLCFG='"scheme":"replicated"'
+    POOLCFG="$POOLCFG"'"scheme":"replicated"'
 fi
 POOLCFG='"name":"testpool","failure_domain":"osd",'$POOLCFG
 $ETCDCTL put /vitastor/config/pools '{"1":{'$POOLCFG',"pg_size":'$PG_SIZE',"pg_minsize":'$PG_MINSIZE',"pg_count":'$PG_COUNT'}}'

tests/run_tests.sh

@@ -63,6 +63,7 @@ SCHEME=xor ./test_write.sh
 ./test_write_no_same.sh
 
 PG_SIZE=2 ./test_heal.sh
+TEST_NAME=local_read POOLCFG='"local_reads":"random",' ./test_heal.sh
 SCHEME=ec ./test_heal.sh
 ANTIETCD=1 ./test_heal.sh