Support containerized Vitastor installations

2025-02-27 20:06:15 +03:00
parent 124162ad38
commit 59f0b0427c
32 changed files with 331 additions and 14 deletions
@@ -41,6 +41,7 @@ Vitastor поддерживает QEMU-драйвер, протоколы NBD и
  - [Автор и лицензия](docs/intro/author.ru.md)
 - Установка
  - [Пакеты](docs/installation/packages.ru.md)
+  - [Docker](docs/installation/docker.ru.md)
  - [Proxmox](docs/installation/proxmox.ru.md)
  - [OpenNebula](docs/installation/opennebula.ru.md)
  - [OpenStack](docs/installation/openstack.ru.md)
@@ -41,6 +41,7 @@ Read more details in the documentation. You can start from here: [Quick Start](d
  - [Author and license](docs/intro/author.en.md)
 - Installation
  - [Packages](docs/installation/packages.en.md)
+  - [Docker](docs/installation/docker.en.md)
  - [Proxmox](docs/installation/proxmox.en.md)
  - [OpenNebula](docs/installation/opennebula.en.md)
  - [OpenStack](docs/installation/openstack.en.md)
@@ -2,8 +2,10 @@

 FROM debian:bookworm

-ADD vitastor.list /etc/apt/sources.list.d
-ADD vitastor.gpg /etc/apt/trusted.gpg.d
-ADD vitastor.pref /etc/apt/preferences.d
-ADD apt.conf /etc/apt/
-RUN apt-get update && apt-get -y install vitastor qemu-system-x86 qemu-system-common qemu-block-extra qemu-utils && apt-get clean
+ADD etc/apt /etc/apt/
+RUN apt-get update && apt-get -y install vitastor qemu-system-x86 qemu-system-common qemu-block-extra qemu-utils jq nfs-common && apt-get clean
+ADD sleep.sh /usr/bin/
+ADD install.sh /usr/bin/
+ADD scripts /opt/scripts/
+ADD etc /etc/
+RUN ln -s /usr/lib/vitastor/mon/make-etcd /usr/bin/make-etcd
@@ -0,0 +1,9 @@
+VITASTOR_VERSION ?= v1.10.1
+
+all: build push
+
+build:
+	@docker build --rm -t vitalif/vitastor:$(VITASTOR_VERSION) .
+
+push:
+	@docker push vitalif/vitastor:$(VITASTOR_VERSION)
@@ -0,0 +1,27 @@
+[Unit]
+Description=Containerized etcd for Vitastor
+After=network-online.target local-fs.target time-sync.target docker.service vitastor-host.service
+Wants=network-online.target local-fs.target time-sync.target docker.service vitastor-host.service
+PartOf=vitastor.target
+
+[Service]
+Restart=always
+Environment=GOGC=50
+EnvironmentFile=/etc/vitastor/docker.conf
+EnvironmentFile=/etc/vitastor/etcd.conf
+SyslogIdentifier=etcd
+ExecStart=bash -c 'docker run --rm -i -v /var/lib/vitastor/etcd:/data \
+    --log-driver none --network host $CONTAINER_OPTIONS --name vitastor-etcd \
+    $ETCD_IMAGE /usr/local/bin/etcd --name "$ETCD_NAME" --data-dir /data \
+    --snapshot-count 10000 --advertise-client-urls http://$ETCD_IP:2379 --listen-client-urls http://$ETCD_IP:2379 \
+    --initial-advertise-peer-urls http://$ETCD_IP:2380 --listen-peer-urls http://$ETCD_IP:2380 \
+    --initial-cluster-token vitastor-etcd-1 --initial-cluster "$ETCD_INITIAL_CLUSTER" \
+    --initial-cluster-state new --max-txn-ops=100000 --max-request-bytes=104857600 \
+    --auto-compaction-retention=10 --auto-compaction-mode=revision'
+ExecStop=docker stop vitastor-etcd
+Restart=always
+StartLimitInterval=0
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,23 @@
+[Unit]
+Description=Empty container for running Vitastor commands
+After=network-online.target local-fs.target time-sync.target docker.service
+Wants=network-online.target local-fs.target time-sync.target docker.service
+PartOf=vitastor.target
+
+[Service]
+Restart=always
+EnvironmentFile=/etc/vitastor/docker.conf
+ExecStart=bash -c 'docker run --rm -i -v /etc/vitastor:/etc/vitastor -v /dev:/dev \
+    --privileged --log-driver none --network host --name vitastor vitastor:$VITASTOR_VERSION \
+    sleep.sh'
+ExecStartPost=udevadm trigger
+ExecStop=docker stop vitastor
+WorkingDirectory=/
+PrivateTmp=false
+TasksMax=infinity
+Restart=always
+StartLimitInterval=0
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,23 @@
+[Unit]
+Description=Containerized Vitastor monitor
+After=network-online.target local-fs.target time-sync.target docker.service
+Wants=network-online.target local-fs.target time-sync.target docker.service
+PartOf=vitastor.target
+
+[Service]
+Restart=always
+EnvironmentFile=/etc/vitastor/docker.conf
+SyslogIdentifier=vitastor-mon
+ExecStart=bash -c 'docker run --rm -i -v /etc/vitastor:/etc/vitastor -v /var/lib/vitastor:/var/lib/vitastor -v /dev:/dev \
+    --log-driver none --network host $CONTAINER_OPTIONS --name vitastor-mon vitastor:$VITASTOR_VERSION \
+    node /usr/lib/vitastor/mon/mon-main.js'
+ExecStop=docker stop vitastor-mon
+WorkingDirectory=/
+PrivateTmp=false
+TasksMax=infinity
+Restart=always
+StartLimitInterval=0
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,27 @@
+[Unit]
+Description=Containerized Vitastor object storage daemon osd.%i
+After=network-online.target local-fs.target time-sync.target docker.service vitastor-host.service
+Wants=network-online.target local-fs.target time-sync.target docker.service vitastor-host.service
+PartOf=vitastor.target
+
+[Service]
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitMEMLOCK=infinity
+EnvironmentFile=/etc/vitastor/docker.conf
+SyslogIdentifier=vitastor-osd%i
+ExecStart=bash -c 'docker run --rm -i -v /etc/vitastor:/etc/vitastor -v /dev:/dev \
+    $(for i in $(ls /dev/vitastor/osd%i-*); do echo --device $i:$i; done) \
+    --log-driver none --network host --ulimit nofile=1048576 --ulimit memlock=-1 $CONTAINER_OPTIONS --name vitastor-osd%i \
+    vitastor:$VITASTOR_VERSION vitastor-disk exec-osd /dev/vitastor/osd%i-data'
+ExecStartPre=+docker exec vitastor vitastor-disk pre-exec /dev/vitastor/osd%i-data
+ExecStop=docker stop vitastor-etcd%i
+WorkingDirectory=/
+PrivateTmp=false
+TasksMax=infinity
+Restart=always
+StartLimitInterval=0
+RestartSec=10
+
+[Install]
+WantedBy=vitastor.target
@@ -0,0 +1,4 @@
+[Unit]
+Description=vitastor target
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,7 @@
+SUBSYSTEM=="block", ENV{ID_PART_ENTRY_TYPE}=="e7009fac-a5a1-4d72-af72-53de13059903", \
+    OWNER="vitastor", GROUP="vitastor", \
+    IMPORT{program}="/usr/bin/docker exec vitastor vitastor-disk udev $devnode", \
+    SYMLINK+="vitastor/$env{VITASTOR_ALIAS}"
+
+ENV{VITASTOR_OSD_NUM}!="", ACTION=="add", RUN{program}+="/usr/bin/systemctl enable --now --no-block vitastor-osd@$env{VITASTOR_OSD_NUM}"
+ENV{VITASTOR_OSD_NUM}!="", ACTION=="remove", RUN{program}+="/usr/bin/systemctl disable --now --no-block vitastor-osd@$env{VITASTOR_OSD_NUM}"
@@ -0,0 +1,11 @@
+#
+# Configuration file for containerized Vitastor installation
+# (non-Kubernetes, with systemd and udev-based orchestration)
+#
+
+# Desired Vitastor version
+VITASTOR_VERSION=1.10.1
+
+# Additional arguments for all containers
+# For example, you may want to specify a custom logging driver here
+CONTAINER_OPTIONS=""
@@ -0,0 +1,4 @@
+ETCD_IMAGE=quay.io/coreos/etcd:v3.5.18
+ETCD_NAME=""
+ETCD_IP=""
+ETCD_INITIAL_CLUSTER=""
@@ -0,0 +1,2 @@
+{
+}
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+cp -urv /etc/default /host-etc/
+cp -urv /etc/systemd /host-etc/
+cp -urv /etc/udev /host-etc/
+cp -urnv /etc/vitastor /host-etc/
+cp -urnv /opt/scripts/* /host-bin/
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker exec -it vitastor vitastor-cli "$@"
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker exec -it vitastor vitastor-disk "$@"
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker exec -it vitastor fio "$@"
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker exec -it vitastor vitastor-nbd "$@"
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+while :; do sleep infinity; done
@@ -14,8 +14,12 @@

 {{../../installation/packages.en.md}}

+{{../../installation/docker.en.md}}
+
 {{../../installation/proxmox.en.md}}

+{{../../installation/opennebula.en.md}}
+
 {{../../installation/openstack.en.md}}

 {{../../installation/kubernetes.en.md}}
@@ -14,8 +14,12 @@

 {{../../installation/packages.ru.md}}

+{{../../installation/docker.ru.md}}
+
 {{../../installation/proxmox.ru.md}}

+{{../../installation/opennebula.ru.md}}
+
 {{../../installation/openstack.ru.md}}

 {{../../installation/kubernetes.ru.md}}
@@ -0,0 +1,60 @@
+[Documentation](../../README.md#documentation) → Installation → Dockerized Installation
+
+-----
+
+[Читать на русском](docker.ru.md)
+
+# Dockerized Installation
+
+Vitastor may be installed in Docker/Podman. In such setups etcd, monitors and OSD
+all run in containers, but everything else looks as close as possible to a usual
+setup with packages:
+- host network is used
+- auto-start is implemented through udev and systemd
+- logs are written to journald (not docker json log files)
+- command-line wrapper scripts are installed to the host system to call vitastor-disk,
+  vitastor-cli and others through the container
+
+Such installations may be useful when it's impossible or inconvenient to install
+Vitastor from packages, for example, in exotic Linux distributions.
+
+If you don't want just a simple containerized installation, you can also take a look
+at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operator
+
+## Installing Containers
+
+The instruction is very simple.
+
+1. Download a Docker image of the desired version: \
+   `docker pull vitastor:1.10.2`
+2. Install scripts to the host system: \
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:1.10.2 install.sh`
+3. Reload udev rules: \
+   `udevadm control --reload-rules`
+
+And you can return to [Quick Start](../intro/quickstart.en.md).
+
+## Upgrading Containers
+
+First make sure to check the topic [Upgrading Vitastor](../usage/admin.en.md#upgrading-vitastor)
+to figure out if you need any additional steps.
+
+Then, to upgrade a containerized installation, you just need to change the `VITASTOR_VERSION`
+option in `/etc/vitastor/docker.conf` and restart all Vitastor services:
+
+`systemctl restart vitastor.target`
+
+## QEMU
+
+Vitastor Docker image also contains QEMU, qemu-img and qemu-storage-daemon built with Vitastor support.
+
+However, running QEMU in Docker is harder to setup and it depends on the used virtualization UI
+(OpenNebula, Proxmox and so on). Some of them also required patched Libvirt.
+
+That's why containerized installation of Vitastor doesn't contain a ready-made QEMU setup and it's
+recommended to install QEMU from packages or build it manually.
+
+## fio
+
+Vitastor Docker image also contains fio and installs a wrapper called `vitastor-fio` to use it from
+the host system.
@@ -0,0 +1,60 @@
+[Документация](../../README-ru.md#документация) → Установка → Установка в Docker
+
+-----
+
+[Read in English](docker.en.md)
+
+# Установка в Docker
+
+Vitastor можно установить в Docker/Podman. При этом etcd, мониторы и OSD запускаются
+в контейнерах, но всё остальное выглядит максимально приближенно к установке из пакетов:
+- используется сеть хост-системы
+- для автозапуска используются udev и systemd
+- журналы записываются в journald (не в json-файлы журналов docker)
+- в хост-систему устанавливаются обёртки для вызова консольных инструментов vitastor-disk,
+  vitastor-cli и других через контейнер
+
+Такая установка полезна тогда, когда установка из пакетов невозможна или неудобна,
+например, в нестандартных Linux-дистрибутивах.
+
+Если вам нужна не просто контейнеризованная инсталляция, вы также можете обратить внимание
+на Vitastor Kubernetes-оператор: https://github.com/Antilles7227/vitastor-operator
+
+## Установка контейнеров
+
+Инструкция по установке максимально простая.
+
+1. Скачайте Docker-образ желаемой версии: \
+   `docker pull vitastor:1.10.2`
+2. Установите скрипты в хост-систему командой: \
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:1.10.2 install.sh`
+3. Перезагрузите правила udev: \
+   `udevadm control --reload-rules`
+
+После этого вы можете возвращаться к разделу [Быстрый старт](../intro/quickstart.ru.md).
+
+## Обновление контейнеров
+
+Сначала обязательно проверьте раздел [Обновление Vitastor](../usage/admin.ru.md#обновление-vitastor),
+чтобы понять, не требуются ли вам какие-то дополнительные действия.
+
+После этого для обновления Docker-инсталляции вам нужно просто поменять опцию `VITASTOR_VERSION`
+в файле `/etc/vitastor/docker.conf` и перезапустить все сервисы Vitastor командой:
+
+`systemctl restart vitastor.target`
+
+## QEMU
+
+В Docker-образ также входят QEMU, qemu-img и qemu-storage-daemon, собранные с поддержкой Vitastor.
+
+Однако настроить запуск QEMU в Docker сложнее и способ запуска зависит от используемого интерфейса
+виртуализации (OpenNebula, Proxmox и т.п.). Также для OpenNebula, например, требуется патченый
+Libvirt.
+
+Поэтому по умолчанию Docker-сборка пока что не включает в себя готового способа запуска QEMU
+и QEMU рекомендуется устанавливать из пакетов или собирать самостоятельно.
+
+## fio
+
+fio также входит в Docker-контейнер vitastor, и в хост-систему устанавливается обёртка `vitastor-fio`
+для запуска fio в контейнер.
@@ -26,7 +26,7 @@
  you also need small SSDs for journal and metadata (even 2 GB per 1 TB of HDD space is enough).
 - Get a fast network (at least 10 Gbit/s). Something like Mellanox ConnectX-4 with RoCEv2 is ideal.
 - Disable CPU powersaving: `cpupower idle-set -D 0 && cpupower frequency-set -g performance`.
- [Install Vitastor packages](../installation/packages.en.md).
+- Either [install Vitastor packages](../installation/packages.en.md) or [install Vitastor in Docker](../installation/docker.en.md).

 ## Recommended drives

@@ -45,7 +45,8 @@ On the monitor hosts:
  }
  ```
 - Create systemd units for etcd by running: `/usr/lib/vitastor/mon/make-etcd`
- Start etcd and monitors: `systemctl enable --now etcd vitastor-mon`
+  Or, if you installed Vitastor in Docker, run `systemctl start vitastor-host; docker exec vitastor make-etcd`.
+- Start etcd and monitors: `systemctl enable --now vitastor-etcd vitastor-mon`

 ## Configure OSDs

@@ -26,7 +26,7 @@
  обязательно возьмите SSD под метаданные и журнал (маленькие, буквально 2 ГБ на 1 ТБ HDD-места).
 - Возьмите быструю сеть, минимум 10 гбит/с. Идеал - что-то вроде Mellanox ConnectX-4 с RoCEv2.
 - Для лучшей производительности отключите энергосбережение CPU: `cpupower idle-set -D 0 && cpupower frequency-set -g performance`.
- [Установите пакеты Vitastor](../installation/packages.ru.md).
+- Либо [установите пакеты Vitastor](../installation/packages.ru.md), либо [установите Vitastor в Docker](../installation/docker.ru.md).

 ## Рекомендуемые диски

@@ -44,8 +44,9 @@
    "etcd_address": ["10.200.1.10:2379","10.200.1.11:2379","10.200.1.12:2379"]
  }
  ```
- Инициализируйте сервисы etcd, запустив `/usr/lib/vitastor/mon/make-etcd`
- Запустите etcd и мониторы: `systemctl enable --now etcd vitastor-mon`
+- Инициализируйте сервисы etcd, запустив `/usr/lib/vitastor/mon/make-etcd`.\
+  Либо, если вы установили Vitastor в Docker, запустите `systemctl start vitastor-host; docker exec vitastor make-etcd`.
+- Запустите etcd и мониторы: `systemctl enable --now vitastor-etcd vitastor-mon`

 ## Настройте OSD

@@ -33,9 +33,11 @@ async function run()
        console.log(config_path+' is missing');
        process.exit(1);
    }
-    if (fs.existsSync("/etc/systemd/system/etcd.service"))
+    const in_docker = fs.existsSync("/etc/vitastor/etcd.conf") &&
+        fs.existsSync("/etc/vitastor/docker.conf");
+    if (!in_docker && fs.existsSync("/etc/systemd/system/vitastor-etcd.service"))
    {
-        console.log("/etc/systemd/system/etcd.service already exists");
+        console.log("/etc/systemd/system/vitastor-etcd.service already exists");
        process.exit(1);
    }
    const config = JSON.parse(fs.readFileSync(config_path, { encoding: 'utf-8' }));
@@ -52,10 +54,21 @@ async function run()
        console.log('No matching IPs in etcd_address from '+config_path);
        process.exit(0);
    }
+    const etcd_name = 'etcd'+etcds[num].replace(/[^0-9a-z_]/ig, '_');
    const etcd_cluster = etcds.map(e => `etcd${e.replace(/[^0-9a-z_]/ig, '_')}=http://${e}:2380`).join(',');
+    if (in_docker)
+    {
+        let etcd_conf = fs.readFileSync("/etc/vitastor/etcd.conf", { encoding: 'utf-8' });
+        etcd_conf = replace_env(etcd_conf, 'ETCD_NAME', etcd_name);
+        etcd_conf = replace_env(etcd_conf, 'ETCD_IP', etcds[num]);
+        etcd_conf = replace_env(etcd_conf, 'ETCD_INITIAL_CLUSTER', etcd_cluster);
+        fs.writeFileSync("/etc/vitastor/etcd.conf", etcd_conf);
+        console.log('etcd for Vitastor configured. Run `systemctl enable --now vitastor-etcd` to start etcd');
+        process.exit(0);
+    }
    await system(`mkdir -p /var/lib/etcd`);
    fs.writeFileSync(
-        "/etc/systemd/system/etcd.service",
+        "/etc/systemd/system/vitastor-etcd.service",
 `[Unit]
 Description=etcd for vitastor
 After=network-online.target local-fs.target time-sync.target
@@ -64,7 +77,7 @@ Wants=network-online.target local-fs.target time-sync.target
 [Service]
 Restart=always
 Environment=GOGC=50
-ExecStart=etcd --name etcd${etcds[num].replace(/[^0-9a-z_]/ig, '_')} --data-dir /var/lib/etcd \\
+ExecStart=etcd --name ${etcd_name} --data-dir /var/lib/etcd \\
    --snapshot-count 10000 --advertise-client-urls http://${etcds[num]}:2379 --listen-client-urls http://${etcds[num]}:2379 \\
    --initial-advertise-peer-urls http://${etcds[num]}:2380 --listen-peer-urls http://${etcds[num]}:2380 \\
    --initial-cluster-token vitastor-etcd-1 --initial-cluster ${etcd_cluster} \\
@@ -89,6 +102,13 @@ WantedBy=multi-user.target
    process.exit(0);
 }

+function replace_env(text, key, value)
+{
+    let found = false;
+    text = text.replace(new RegExp('^'+key+'\\s*=.*', 'm'), () => { found = true; return key+'='+value; });
+    return found ? text : text.replace(/\s*$/, '\n')+key+'='+value+'\n';
+}
+
 function select_local_etcd(etcds)
 {
    const ifaces = os.networkInterfaces();
@@ -5,6 +5,7 @@ Wants=network-online.target local-fs.target time-sync.target

 [Service]
 Restart=always
+SyslogIdentifier=vitastor-mon
 ExecStart=node /usr/lib/vitastor/mon/mon-main.js
 WorkingDirectory=/
 User=vitastor
@@ -8,6 +8,7 @@ PartOf=vitastor.target
 LimitNOFILE=1048576
 LimitNPROC=1048576
 LimitMEMLOCK=infinity
+SyslogIdentifier=vitastor-osd%i
 # Use the following for direct logs to files
 #ExecStart=bash -c 'exec vitastor-disk exec-osd /dev/vitastor/osd%i-data >>/var/log/vitastor/osd%i.log 2>&1'
 ExecStart=vitastor-disk exec-osd /dev/vitastor/osd%i-data