From 59f0b0427c89c9b5dfbec028fb413dff16dfae52 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Tue, 25 Feb 2025 02:22:10 +0300 Subject: [PATCH] Support containerized Vitastor installations --- README-ru.md | 1 + README.md | 1 + docker/Dockerfile | 12 ++-- docker/Makefile | 9 +++ docker/{ => etc/apt}/apt.conf | 0 .../{ => etc/apt/preferences.d}/vitastor.pref | 0 .../apt/sources.list.d}/vitastor.list | 0 .../{ => etc/apt/trusted.gpg.d}/vitastor.gpg | Bin .../etc/systemd/system/vitastor-etcd.service | 27 ++++++++ .../etc/systemd/system/vitastor-host.service | 23 +++++++ .../etc/systemd/system/vitastor-mon.service | 23 +++++++ .../etc/systemd/system/vitastor-osd@.service | 27 ++++++++ docker/etc/systemd/system/vitastor.target | 4 ++ docker/etc/udev/rules.d/90-vitastor.rules | 7 ++ docker/etc/vitastor/docker.conf | 11 ++++ docker/etc/vitastor/etcd.conf | 4 ++ docker/etc/vitastor/vitastor.conf | 2 + docker/install.sh | 9 +++ docker/scripts/vitastor-cli | 3 + docker/scripts/vitastor-disk | 3 + docker/scripts/vitastor-fio | 3 + docker/scripts/vitastor-nbd | 3 + docker/sleep.sh | 3 + docs/config/src/included.en.md | 4 ++ docs/config/src/included.ru.md | 4 ++ docs/installation/docker.en.md | 60 ++++++++++++++++++ docs/installation/docker.ru.md | 60 ++++++++++++++++++ docs/intro/quickstart.en.md | 5 +- docs/intro/quickstart.ru.md | 7 +- mon/scripts/make-etcd | 28 ++++++-- mon/scripts/vitastor-mon.service | 1 + mon/scripts/vitastor-osd@.service | 1 + 32 files changed, 331 insertions(+), 14 deletions(-) create mode 100644 docker/Makefile rename docker/{ => etc/apt}/apt.conf (100%) rename docker/{ => etc/apt/preferences.d}/vitastor.pref (100%) rename docker/{ => etc/apt/sources.list.d}/vitastor.list (100%) rename docker/{ => etc/apt/trusted.gpg.d}/vitastor.gpg (100%) create mode 100644 docker/etc/systemd/system/vitastor-etcd.service create mode 100644 docker/etc/systemd/system/vitastor-host.service create mode 100644 docker/etc/systemd/system/vitastor-mon.service create mode 100644 docker/etc/systemd/system/vitastor-osd@.service create mode 100644 docker/etc/systemd/system/vitastor.target create mode 100644 docker/etc/udev/rules.d/90-vitastor.rules create mode 100644 docker/etc/vitastor/docker.conf create mode 100644 docker/etc/vitastor/etcd.conf create mode 100644 docker/etc/vitastor/vitastor.conf create mode 100755 docker/install.sh create mode 100755 docker/scripts/vitastor-cli create mode 100755 docker/scripts/vitastor-disk create mode 100755 docker/scripts/vitastor-fio create mode 100755 docker/scripts/vitastor-nbd create mode 100755 docker/sleep.sh create mode 100644 docs/installation/docker.en.md create mode 100644 docs/installation/docker.ru.md diff --git a/README-ru.md b/README-ru.md index 1a3f3aed..9657a4d6 100644 --- a/README-ru.md +++ b/README-ru.md @@ -41,6 +41,7 @@ Vitastor поддерживает QEMU-драйвер, протоколы NBD и - [Автор и лицензия](docs/intro/author.ru.md) - Установка - [Пакеты](docs/installation/packages.ru.md) + - [Docker](docs/installation/docker.ru.md) - [Proxmox](docs/installation/proxmox.ru.md) - [OpenNebula](docs/installation/opennebula.ru.md) - [OpenStack](docs/installation/openstack.ru.md) diff --git a/README.md b/README.md index 1aa087cf..c167afaf 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ Read more details in the documentation. You can start from here: [Quick Start](d - [Author and license](docs/intro/author.en.md) - Installation - [Packages](docs/installation/packages.en.md) + - [Docker](docs/installation/docker.en.md) - [Proxmox](docs/installation/proxmox.en.md) - [OpenNebula](docs/installation/opennebula.en.md) - [OpenStack](docs/installation/openstack.en.md) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0efbce7f..471270da 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,8 +2,10 @@ FROM debian:bookworm -ADD vitastor.list /etc/apt/sources.list.d -ADD vitastor.gpg /etc/apt/trusted.gpg.d -ADD vitastor.pref /etc/apt/preferences.d -ADD apt.conf /etc/apt/ -RUN apt-get update && apt-get -y install vitastor qemu-system-x86 qemu-system-common qemu-block-extra qemu-utils && apt-get clean +ADD etc/apt /etc/apt/ +RUN apt-get update && apt-get -y install vitastor qemu-system-x86 qemu-system-common qemu-block-extra qemu-utils jq nfs-common && apt-get clean +ADD sleep.sh /usr/bin/ +ADD install.sh /usr/bin/ +ADD scripts /opt/scripts/ +ADD etc /etc/ +RUN ln -s /usr/lib/vitastor/mon/make-etcd /usr/bin/make-etcd diff --git a/docker/Makefile b/docker/Makefile new file mode 100644 index 00000000..8ea324e9 --- /dev/null +++ b/docker/Makefile @@ -0,0 +1,9 @@ +VITASTOR_VERSION ?= v1.10.1 + +all: build push + +build: + @docker build --rm -t vitalif/vitastor:$(VITASTOR_VERSION) . + +push: + @docker push vitalif/vitastor:$(VITASTOR_VERSION) diff --git a/docker/apt.conf b/docker/etc/apt/apt.conf similarity index 100% rename from docker/apt.conf rename to docker/etc/apt/apt.conf diff --git a/docker/vitastor.pref b/docker/etc/apt/preferences.d/vitastor.pref similarity index 100% rename from docker/vitastor.pref rename to docker/etc/apt/preferences.d/vitastor.pref diff --git a/docker/vitastor.list b/docker/etc/apt/sources.list.d/vitastor.list similarity index 100% rename from docker/vitastor.list rename to docker/etc/apt/sources.list.d/vitastor.list diff --git a/docker/vitastor.gpg b/docker/etc/apt/trusted.gpg.d/vitastor.gpg similarity index 100% rename from docker/vitastor.gpg rename to docker/etc/apt/trusted.gpg.d/vitastor.gpg diff --git a/docker/etc/systemd/system/vitastor-etcd.service b/docker/etc/systemd/system/vitastor-etcd.service new file mode 100644 index 00000000..8827c4f7 --- /dev/null +++ b/docker/etc/systemd/system/vitastor-etcd.service @@ -0,0 +1,27 @@ +[Unit] +Description=Containerized etcd for Vitastor +After=network-online.target local-fs.target time-sync.target docker.service vitastor-host.service +Wants=network-online.target local-fs.target time-sync.target docker.service vitastor-host.service +PartOf=vitastor.target + +[Service] +Restart=always +Environment=GOGC=50 +EnvironmentFile=/etc/vitastor/docker.conf +EnvironmentFile=/etc/vitastor/etcd.conf +SyslogIdentifier=etcd +ExecStart=bash -c 'docker run --rm -i -v /var/lib/vitastor/etcd:/data \ + --log-driver none --network host $CONTAINER_OPTIONS --name vitastor-etcd \ + $ETCD_IMAGE /usr/local/bin/etcd --name "$ETCD_NAME" --data-dir /data \ + --snapshot-count 10000 --advertise-client-urls http://$ETCD_IP:2379 --listen-client-urls http://$ETCD_IP:2379 \ + --initial-advertise-peer-urls http://$ETCD_IP:2380 --listen-peer-urls http://$ETCD_IP:2380 \ + --initial-cluster-token vitastor-etcd-1 --initial-cluster "$ETCD_INITIAL_CLUSTER" \ + --initial-cluster-state new --max-txn-ops=100000 --max-request-bytes=104857600 \ + --auto-compaction-retention=10 --auto-compaction-mode=revision' +ExecStop=docker stop vitastor-etcd +Restart=always +StartLimitInterval=0 +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/docker/etc/systemd/system/vitastor-host.service b/docker/etc/systemd/system/vitastor-host.service new file mode 100644 index 00000000..b02c71ac --- /dev/null +++ b/docker/etc/systemd/system/vitastor-host.service @@ -0,0 +1,23 @@ +[Unit] +Description=Empty container for running Vitastor commands +After=network-online.target local-fs.target time-sync.target docker.service +Wants=network-online.target local-fs.target time-sync.target docker.service +PartOf=vitastor.target + +[Service] +Restart=always +EnvironmentFile=/etc/vitastor/docker.conf +ExecStart=bash -c 'docker run --rm -i -v /etc/vitastor:/etc/vitastor -v /dev:/dev \ + --privileged --log-driver none --network host --name vitastor vitastor:$VITASTOR_VERSION \ + sleep.sh' +ExecStartPost=udevadm trigger +ExecStop=docker stop vitastor +WorkingDirectory=/ +PrivateTmp=false +TasksMax=infinity +Restart=always +StartLimitInterval=0 +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/docker/etc/systemd/system/vitastor-mon.service b/docker/etc/systemd/system/vitastor-mon.service new file mode 100644 index 00000000..959b4052 --- /dev/null +++ b/docker/etc/systemd/system/vitastor-mon.service @@ -0,0 +1,23 @@ +[Unit] +Description=Containerized Vitastor monitor +After=network-online.target local-fs.target time-sync.target docker.service +Wants=network-online.target local-fs.target time-sync.target docker.service +PartOf=vitastor.target + +[Service] +Restart=always +EnvironmentFile=/etc/vitastor/docker.conf +SyslogIdentifier=vitastor-mon +ExecStart=bash -c 'docker run --rm -i -v /etc/vitastor:/etc/vitastor -v /var/lib/vitastor:/var/lib/vitastor -v /dev:/dev \ + --log-driver none --network host $CONTAINER_OPTIONS --name vitastor-mon vitastor:$VITASTOR_VERSION \ + node /usr/lib/vitastor/mon/mon-main.js' +ExecStop=docker stop vitastor-mon +WorkingDirectory=/ +PrivateTmp=false +TasksMax=infinity +Restart=always +StartLimitInterval=0 +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/docker/etc/systemd/system/vitastor-osd@.service b/docker/etc/systemd/system/vitastor-osd@.service new file mode 100644 index 00000000..c9fcfdf9 --- /dev/null +++ b/docker/etc/systemd/system/vitastor-osd@.service @@ -0,0 +1,27 @@ +[Unit] +Description=Containerized Vitastor object storage daemon osd.%i +After=network-online.target local-fs.target time-sync.target docker.service vitastor-host.service +Wants=network-online.target local-fs.target time-sync.target docker.service vitastor-host.service +PartOf=vitastor.target + +[Service] +LimitNOFILE=1048576 +LimitNPROC=1048576 +LimitMEMLOCK=infinity +EnvironmentFile=/etc/vitastor/docker.conf +SyslogIdentifier=vitastor-osd%i +ExecStart=bash -c 'docker run --rm -i -v /etc/vitastor:/etc/vitastor -v /dev:/dev \ + $(for i in $(ls /dev/vitastor/osd%i-*); do echo --device $i:$i; done) \ + --log-driver none --network host --ulimit nofile=1048576 --ulimit memlock=-1 $CONTAINER_OPTIONS --name vitastor-osd%i \ + vitastor:$VITASTOR_VERSION vitastor-disk exec-osd /dev/vitastor/osd%i-data' +ExecStartPre=+docker exec vitastor vitastor-disk pre-exec /dev/vitastor/osd%i-data +ExecStop=docker stop vitastor-etcd%i +WorkingDirectory=/ +PrivateTmp=false +TasksMax=infinity +Restart=always +StartLimitInterval=0 +RestartSec=10 + +[Install] +WantedBy=vitastor.target diff --git a/docker/etc/systemd/system/vitastor.target b/docker/etc/systemd/system/vitastor.target new file mode 100644 index 00000000..f85ac6a6 --- /dev/null +++ b/docker/etc/systemd/system/vitastor.target @@ -0,0 +1,4 @@ +[Unit] +Description=vitastor target +[Install] +WantedBy=multi-user.target diff --git a/docker/etc/udev/rules.d/90-vitastor.rules b/docker/etc/udev/rules.d/90-vitastor.rules new file mode 100644 index 00000000..aba25da4 --- /dev/null +++ b/docker/etc/udev/rules.d/90-vitastor.rules @@ -0,0 +1,7 @@ +SUBSYSTEM=="block", ENV{ID_PART_ENTRY_TYPE}=="e7009fac-a5a1-4d72-af72-53de13059903", \ + OWNER="vitastor", GROUP="vitastor", \ + IMPORT{program}="/usr/bin/docker exec vitastor vitastor-disk udev $devnode", \ + SYMLINK+="vitastor/$env{VITASTOR_ALIAS}" + +ENV{VITASTOR_OSD_NUM}!="", ACTION=="add", RUN{program}+="/usr/bin/systemctl enable --now --no-block vitastor-osd@$env{VITASTOR_OSD_NUM}" +ENV{VITASTOR_OSD_NUM}!="", ACTION=="remove", RUN{program}+="/usr/bin/systemctl disable --now --no-block vitastor-osd@$env{VITASTOR_OSD_NUM}" diff --git a/docker/etc/vitastor/docker.conf b/docker/etc/vitastor/docker.conf new file mode 100644 index 00000000..645292f2 --- /dev/null +++ b/docker/etc/vitastor/docker.conf @@ -0,0 +1,11 @@ +# +# Configuration file for containerized Vitastor installation +# (non-Kubernetes, with systemd and udev-based orchestration) +# + +# Desired Vitastor version +VITASTOR_VERSION=1.10.1 + +# Additional arguments for all containers +# For example, you may want to specify a custom logging driver here +CONTAINER_OPTIONS="" diff --git a/docker/etc/vitastor/etcd.conf b/docker/etc/vitastor/etcd.conf new file mode 100644 index 00000000..82bc1303 --- /dev/null +++ b/docker/etc/vitastor/etcd.conf @@ -0,0 +1,4 @@ +ETCD_IMAGE=quay.io/coreos/etcd:v3.5.18 +ETCD_NAME="" +ETCD_IP="" +ETCD_INITIAL_CLUSTER="" diff --git a/docker/etc/vitastor/vitastor.conf b/docker/etc/vitastor/vitastor.conf new file mode 100644 index 00000000..2c63c085 --- /dev/null +++ b/docker/etc/vitastor/vitastor.conf @@ -0,0 +1,2 @@ +{ +} diff --git a/docker/install.sh b/docker/install.sh new file mode 100755 index 00000000..df23f878 --- /dev/null +++ b/docker/install.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e + +cp -urv /etc/default /host-etc/ +cp -urv /etc/systemd /host-etc/ +cp -urv /etc/udev /host-etc/ +cp -urnv /etc/vitastor /host-etc/ +cp -urnv /opt/scripts/* /host-bin/ diff --git a/docker/scripts/vitastor-cli b/docker/scripts/vitastor-cli new file mode 100755 index 00000000..ebfe9916 --- /dev/null +++ b/docker/scripts/vitastor-cli @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec -it vitastor vitastor-cli "$@" diff --git a/docker/scripts/vitastor-disk b/docker/scripts/vitastor-disk new file mode 100755 index 00000000..8097809d --- /dev/null +++ b/docker/scripts/vitastor-disk @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec -it vitastor vitastor-disk "$@" diff --git a/docker/scripts/vitastor-fio b/docker/scripts/vitastor-fio new file mode 100755 index 00000000..413e7a5f --- /dev/null +++ b/docker/scripts/vitastor-fio @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec -it vitastor fio "$@" diff --git a/docker/scripts/vitastor-nbd b/docker/scripts/vitastor-nbd new file mode 100755 index 00000000..2f3edde5 --- /dev/null +++ b/docker/scripts/vitastor-nbd @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec -it vitastor vitastor-nbd "$@" diff --git a/docker/sleep.sh b/docker/sleep.sh new file mode 100755 index 00000000..d86b1f54 --- /dev/null +++ b/docker/sleep.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +while :; do sleep infinity; done diff --git a/docs/config/src/included.en.md b/docs/config/src/included.en.md index a9827894..bfce4d26 100644 --- a/docs/config/src/included.en.md +++ b/docs/config/src/included.en.md @@ -14,8 +14,12 @@ {{../../installation/packages.en.md}} +{{../../installation/docker.en.md}} + {{../../installation/proxmox.en.md}} +{{../../installation/opennebula.en.md}} + {{../../installation/openstack.en.md}} {{../../installation/kubernetes.en.md}} diff --git a/docs/config/src/included.ru.md b/docs/config/src/included.ru.md index b89059d9..f724aec7 100644 --- a/docs/config/src/included.ru.md +++ b/docs/config/src/included.ru.md @@ -14,8 +14,12 @@ {{../../installation/packages.ru.md}} +{{../../installation/docker.ru.md}} + {{../../installation/proxmox.ru.md}} +{{../../installation/opennebula.ru.md}} + {{../../installation/openstack.ru.md}} {{../../installation/kubernetes.ru.md}} diff --git a/docs/installation/docker.en.md b/docs/installation/docker.en.md new file mode 100644 index 00000000..fc49d01d --- /dev/null +++ b/docs/installation/docker.en.md @@ -0,0 +1,60 @@ +[Documentation](../../README.md#documentation) → Installation → Dockerized Installation + +----- + +[Читать на русском](docker.ru.md) + +# Dockerized Installation + +Vitastor may be installed in Docker/Podman. In such setups etcd, monitors and OSD +all run in containers, but everything else looks as close as possible to a usual +setup with packages: +- host network is used +- auto-start is implemented through udev and systemd +- logs are written to journald (not docker json log files) +- command-line wrapper scripts are installed to the host system to call vitastor-disk, + vitastor-cli and others through the container + +Such installations may be useful when it's impossible or inconvenient to install +Vitastor from packages, for example, in exotic Linux distributions. + +If you don't want just a simple containerized installation, you can also take a look +at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operator + +## Installing Containers + +The instruction is very simple. + +1. Download a Docker image of the desired version: \ + `docker pull vitastor:1.10.2` +2. Install scripts to the host system: \ + `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:1.10.2 install.sh` +3. Reload udev rules: \ + `udevadm control --reload-rules` + +And you can return to [Quick Start](../intro/quickstart.en.md). + +## Upgrading Containers + +First make sure to check the topic [Upgrading Vitastor](../usage/admin.en.md#upgrading-vitastor) +to figure out if you need any additional steps. + +Then, to upgrade a containerized installation, you just need to change the `VITASTOR_VERSION` +option in `/etc/vitastor/docker.conf` and restart all Vitastor services: + +`systemctl restart vitastor.target` + +## QEMU + +Vitastor Docker image also contains QEMU, qemu-img and qemu-storage-daemon built with Vitastor support. + +However, running QEMU in Docker is harder to setup and it depends on the used virtualization UI +(OpenNebula, Proxmox and so on). Some of them also required patched Libvirt. + +That's why containerized installation of Vitastor doesn't contain a ready-made QEMU setup and it's +recommended to install QEMU from packages or build it manually. + +## fio + +Vitastor Docker image also contains fio and installs a wrapper called `vitastor-fio` to use it from +the host system. diff --git a/docs/installation/docker.ru.md b/docs/installation/docker.ru.md new file mode 100644 index 00000000..53cb07f0 --- /dev/null +++ b/docs/installation/docker.ru.md @@ -0,0 +1,60 @@ +[Документация](../../README-ru.md#документация) → Установка → Установка в Docker + +----- + +[Read in English](docker.en.md) + +# Установка в Docker + +Vitastor можно установить в Docker/Podman. При этом etcd, мониторы и OSD запускаются +в контейнерах, но всё остальное выглядит максимально приближенно к установке из пакетов: +- используется сеть хост-системы +- для автозапуска используются udev и systemd +- журналы записываются в journald (не в json-файлы журналов docker) +- в хост-систему устанавливаются обёртки для вызова консольных инструментов vitastor-disk, + vitastor-cli и других через контейнер + +Такая установка полезна тогда, когда установка из пакетов невозможна или неудобна, +например, в нестандартных Linux-дистрибутивах. + +Если вам нужна не просто контейнеризованная инсталляция, вы также можете обратить внимание +на Vitastor Kubernetes-оператор: https://github.com/Antilles7227/vitastor-operator + +## Установка контейнеров + +Инструкция по установке максимально простая. + +1. Скачайте Docker-образ желаемой версии: \ + `docker pull vitastor:1.10.2` +2. Установите скрипты в хост-систему командой: \ + `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:1.10.2 install.sh` +3. Перезагрузите правила udev: \ + `udevadm control --reload-rules` + +После этого вы можете возвращаться к разделу [Быстрый старт](../intro/quickstart.ru.md). + +## Обновление контейнеров + +Сначала обязательно проверьте раздел [Обновление Vitastor](../usage/admin.ru.md#обновление-vitastor), +чтобы понять, не требуются ли вам какие-то дополнительные действия. + +После этого для обновления Docker-инсталляции вам нужно просто поменять опцию `VITASTOR_VERSION` +в файле `/etc/vitastor/docker.conf` и перезапустить все сервисы Vitastor командой: + +`systemctl restart vitastor.target` + +## QEMU + +В Docker-образ также входят QEMU, qemu-img и qemu-storage-daemon, собранные с поддержкой Vitastor. + +Однако настроить запуск QEMU в Docker сложнее и способ запуска зависит от используемого интерфейса +виртуализации (OpenNebula, Proxmox и т.п.). Также для OpenNebula, например, требуется патченый +Libvirt. + +Поэтому по умолчанию Docker-сборка пока что не включает в себя готового способа запуска QEMU +и QEMU рекомендуется устанавливать из пакетов или собирать самостоятельно. + +## fio + +fio также входит в Docker-контейнер vitastor, и в хост-систему устанавливается обёртка `vitastor-fio` +для запуска fio в контейнер. diff --git a/docs/intro/quickstart.en.md b/docs/intro/quickstart.en.md index efe2a65a..765dda65 100644 --- a/docs/intro/quickstart.en.md +++ b/docs/intro/quickstart.en.md @@ -26,7 +26,7 @@ you also need small SSDs for journal and metadata (even 2 GB per 1 TB of HDD space is enough). - Get a fast network (at least 10 Gbit/s). Something like Mellanox ConnectX-4 with RoCEv2 is ideal. - Disable CPU powersaving: `cpupower idle-set -D 0 && cpupower frequency-set -g performance`. -- [Install Vitastor packages](../installation/packages.en.md). +- Either [install Vitastor packages](../installation/packages.en.md) or [install Vitastor in Docker](../installation/docker.en.md). ## Recommended drives @@ -45,7 +45,8 @@ On the monitor hosts: } ``` - Create systemd units for etcd by running: `/usr/lib/vitastor/mon/make-etcd` -- Start etcd and monitors: `systemctl enable --now etcd vitastor-mon` + Or, if you installed Vitastor in Docker, run `systemctl start vitastor-host; docker exec vitastor make-etcd`. +- Start etcd and monitors: `systemctl enable --now vitastor-etcd vitastor-mon` ## Configure OSDs diff --git a/docs/intro/quickstart.ru.md b/docs/intro/quickstart.ru.md index 4e6e628d..068e9c02 100644 --- a/docs/intro/quickstart.ru.md +++ b/docs/intro/quickstart.ru.md @@ -26,7 +26,7 @@ обязательно возьмите SSD под метаданные и журнал (маленькие, буквально 2 ГБ на 1 ТБ HDD-места). - Возьмите быструю сеть, минимум 10 гбит/с. Идеал - что-то вроде Mellanox ConnectX-4 с RoCEv2. - Для лучшей производительности отключите энергосбережение CPU: `cpupower idle-set -D 0 && cpupower frequency-set -g performance`. -- [Установите пакеты Vitastor](../installation/packages.ru.md). +- Либо [установите пакеты Vitastor](../installation/packages.ru.md), либо [установите Vitastor в Docker](../installation/docker.ru.md). ## Рекомендуемые диски @@ -44,8 +44,9 @@ "etcd_address": ["10.200.1.10:2379","10.200.1.11:2379","10.200.1.12:2379"] } ``` -- Инициализируйте сервисы etcd, запустив `/usr/lib/vitastor/mon/make-etcd` -- Запустите etcd и мониторы: `systemctl enable --now etcd vitastor-mon` +- Инициализируйте сервисы etcd, запустив `/usr/lib/vitastor/mon/make-etcd`.\ + Либо, если вы установили Vitastor в Docker, запустите `systemctl start vitastor-host; docker exec vitastor make-etcd`. +- Запустите etcd и мониторы: `systemctl enable --now vitastor-etcd vitastor-mon` ## Настройте OSD diff --git a/mon/scripts/make-etcd b/mon/scripts/make-etcd index 9ff789aa..2655a03d 100755 --- a/mon/scripts/make-etcd +++ b/mon/scripts/make-etcd @@ -33,9 +33,11 @@ async function run() console.log(config_path+' is missing'); process.exit(1); } - if (fs.existsSync("/etc/systemd/system/etcd.service")) + const in_docker = fs.existsSync("/etc/vitastor/etcd.conf") && + fs.existsSync("/etc/vitastor/docker.conf"); + if (!in_docker && fs.existsSync("/etc/systemd/system/vitastor-etcd.service")) { - console.log("/etc/systemd/system/etcd.service already exists"); + console.log("/etc/systemd/system/vitastor-etcd.service already exists"); process.exit(1); } const config = JSON.parse(fs.readFileSync(config_path, { encoding: 'utf-8' })); @@ -52,10 +54,21 @@ async function run() console.log('No matching IPs in etcd_address from '+config_path); process.exit(0); } + const etcd_name = 'etcd'+etcds[num].replace(/[^0-9a-z_]/ig, '_'); const etcd_cluster = etcds.map(e => `etcd${e.replace(/[^0-9a-z_]/ig, '_')}=http://${e}:2380`).join(','); + if (in_docker) + { + let etcd_conf = fs.readFileSync("/etc/vitastor/etcd.conf", { encoding: 'utf-8' }); + etcd_conf = replace_env(etcd_conf, 'ETCD_NAME', etcd_name); + etcd_conf = replace_env(etcd_conf, 'ETCD_IP', etcds[num]); + etcd_conf = replace_env(etcd_conf, 'ETCD_INITIAL_CLUSTER', etcd_cluster); + fs.writeFileSync("/etc/vitastor/etcd.conf", etcd_conf); + console.log('etcd for Vitastor configured. Run `systemctl enable --now vitastor-etcd` to start etcd'); + process.exit(0); + } await system(`mkdir -p /var/lib/etcd`); fs.writeFileSync( - "/etc/systemd/system/etcd.service", + "/etc/systemd/system/vitastor-etcd.service", `[Unit] Description=etcd for vitastor After=network-online.target local-fs.target time-sync.target @@ -64,7 +77,7 @@ Wants=network-online.target local-fs.target time-sync.target [Service] Restart=always Environment=GOGC=50 -ExecStart=etcd --name etcd${etcds[num].replace(/[^0-9a-z_]/ig, '_')} --data-dir /var/lib/etcd \\ +ExecStart=etcd --name ${etcd_name} --data-dir /var/lib/etcd \\ --snapshot-count 10000 --advertise-client-urls http://${etcds[num]}:2379 --listen-client-urls http://${etcds[num]}:2379 \\ --initial-advertise-peer-urls http://${etcds[num]}:2380 --listen-peer-urls http://${etcds[num]}:2380 \\ --initial-cluster-token vitastor-etcd-1 --initial-cluster ${etcd_cluster} \\ @@ -89,6 +102,13 @@ WantedBy=multi-user.target process.exit(0); } +function replace_env(text, key, value) +{ + let found = false; + text = text.replace(new RegExp('^'+key+'\\s*=.*', 'm'), () => { found = true; return key+'='+value; }); + return found ? text : text.replace(/\s*$/, '\n')+key+'='+value+'\n'; +} + function select_local_etcd(etcds) { const ifaces = os.networkInterfaces(); diff --git a/mon/scripts/vitastor-mon.service b/mon/scripts/vitastor-mon.service index 9e70e824..1eb40a5c 100644 --- a/mon/scripts/vitastor-mon.service +++ b/mon/scripts/vitastor-mon.service @@ -5,6 +5,7 @@ Wants=network-online.target local-fs.target time-sync.target [Service] Restart=always +SyslogIdentifier=vitastor-mon ExecStart=node /usr/lib/vitastor/mon/mon-main.js WorkingDirectory=/ User=vitastor diff --git a/mon/scripts/vitastor-osd@.service b/mon/scripts/vitastor-osd@.service index 1930752e..44999beb 100644 --- a/mon/scripts/vitastor-osd@.service +++ b/mon/scripts/vitastor-osd@.service @@ -8,6 +8,7 @@ PartOf=vitastor.target LimitNOFILE=1048576 LimitNPROC=1048576 LimitMEMLOCK=infinity +SyslogIdentifier=vitastor-osd%i # Use the following for direct logs to files #ExecStart=bash -c 'exec vitastor-disk exec-osd /dev/vitastor/osd%i-data >>/var/log/vitastor/osd%i.log 2>&1' ExecStart=vitastor-disk exec-osd /dev/vitastor/osd%i-data