From f3dcd938b1f2f67d918bcfe0baa1c2dc87c9ed0f Mon Sep 17 00:00:00 2001 From: Lukas Remis Date: Mon, 14 Sep 2026 19:49:03 +0200 Subject: [PATCH] Deploy: fix the two things that broke the Container Manager build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DSM reported "invalid interpolation format for x-backend-env.MASTER_KEY" and "required variable MASTER_KEY is missing a value". Both were mine. 1. DSM's bundled Compose cannot parse `${VAR:?message}`. Those guards are gone; interpolation now sticks to `${VAR}` and `${VAR:-default}`, the forms every Compose understands. Bind mounts keep a `${DATA_ROOT:-/DATA_ROOT-is-unset}` sentinel so a missing config still cannot scatter directories across the root of the NAS filesystem, and the header says to keep it this way. 2. Config was at docker/.env.prod with a `.env` symlink beside it. Container Manager did not pick that up. The real file is now `.env` in the project folder — what every compose front end reads without being told — and docker/.env.prod becomes a symlink to it. `init` and `require_env` migrate an existing install in place, keeping its MASTER_KEY. Also, while testing the migration: `init` only generated secrets when creating a file from scratch, so a migrated or hand-written .env with an empty POSTGRES_PASSWORD started Postgres in trust-auth mode. It now tops up empty secrets — and never touches a MASTER_KEY that already has a value, since replacing it would make every stored credential undecryptable. doctor flags an empty password too. Verified: parses cleanly with no .env at all (the failure DSM hit), migration preserves an existing key, and `docker compose up -d --wait` from the project folder brings the stack up healthy and serving. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Nux3LCu4zP9nih4BRhPyHw --- deploy/webmail.sh | 48 +++++++++++++++++++++++++++++++++++++++++---- docker-compose.yml | 35 +++++++++++++++++++-------------- docs/nas-install.md | 18 ++++++++--------- 3 files changed, 73 insertions(+), 28 deletions(-) diff --git a/deploy/webmail.sh b/deploy/webmail.sh index f7affcd..93cc01b 100755 --- a/deploy/webmail.sh +++ b/deploy/webmail.sh @@ -20,7 +20,8 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT_DIR="$(dirname "$SCRIPT_DIR")" COMPOSE_FILE="${ROOT_DIR}/docker-compose.yml" -ENV_FILE="${ROOT_DIR}/docker/.env.prod" +ENV_FILE="${ROOT_DIR}/.env" +LEGACY_ENV_FILE="${ROOT_DIR}/docker/.env.prod" ENV_EXAMPLE="${ROOT_DIR}/docker/.env.prod.example" # Services that must come back healthy after a roll. @@ -58,6 +59,7 @@ load_env_file() { } require_env() { + migrate_legacy_env [ -f "$ENV_FILE" ] || die "$ENV_FILE not found. Run: ./deploy/webmail.sh init" # The env file is chmod 600 and usually owned by root (init is run as root # on a NAS), and the data directories are root-owned too. Say so plainly @@ -109,6 +111,38 @@ apply_init_flags() { return 0 } +# Earlier versions kept config at docker/.env.prod with a `.env` symlink beside +# it. Synology's Container Manager did not pick that up, so the real file now +# lives at the project root. Move an existing one across (keeping its master +# key) and leave a symlink behind so older notes still work. +migrate_legacy_env() { + [ -e "$ENV_FILE" ] && return 0 + [ -f "$LEGACY_ENV_FILE" ] || return 0 + [ -L "$LEGACY_ENV_FILE" ] && return 0 + mv "$LEGACY_ENV_FILE" "$ENV_FILE" + chmod 600 "$ENV_FILE" + ln -sf ../.env "$LEGACY_ENV_FILE" 2>/dev/null || true + ylw "moved docker/.env.prod -> .env (Container Manager reads .env from the project folder)" +} + +# Fill in required secrets that are present-but-empty (a migrated file from an +# older layout, or a hand-written .env). NEVER touches a MASTER_KEY that already +# has a value: replacing it would make every stored mail-account credential +# undecryptable. +ensure_secrets() { + local current + current="$(sed -n 's/^MASTER_KEY=//p' "$ENV_FILE" | head -1)" + if [ -z "$current" ]; then + set_env_value MASTER_KEY "$(openssl rand -base64 32)" + ylw "MASTER_KEY was empty — generated a new one. Back it up: webmail.sh show-key" + fi + current="$(sed -n 's/^POSTGRES_PASSWORD=//p' "$ENV_FILE" | head -1)" + if [ -z "$current" ]; then + set_env_value POSTGRES_PASSWORD "$(openssl rand -base64 24 | tr -d '/+=' | cut -c1-24)" + ylw "POSTGRES_PASSWORD was empty — generated one" + fi +} + cmd_init() { # Settings can be passed as flags so the whole install can be driven from a # NAS GUI (DSM Task Scheduler / QTS cron), with no editing of a hidden @@ -128,6 +162,7 @@ cmd_init() { esac done + migrate_legacy_env if [ ! -f "$ENV_FILE" ]; then cp "$ENV_EXAMPLE" "$ENV_FILE" chmod 600 "$ENV_FILE" @@ -138,15 +173,15 @@ cmd_init() { sed "s|^MASTER_KEY=.*|MASTER_KEY=${master}|; s|^POSTGRES_PASSWORD=.*|POSTGRES_PASSWORD=${pgpass}|" \ "$ENV_FILE" > "${ENV_FILE}.tmp" && mv "${ENV_FILE}.tmp" "$ENV_FILE" chmod 600 "$ENV_FILE" - # The NAS GUIs read `.env` from the project folder (the repo root). - # A symlink keeps one source of truth instead of two copies to drift. - [ -e "${ROOT_DIR}/.env" ] || ln -s docker/.env.prod "${ROOT_DIR}/.env" 2>/dev/null || true + # Compatibility shim for notes that reference the old path. + [ -e "$LEGACY_ENV_FILE" ] || ln -s ../.env "$LEGACY_ENV_FILE" 2>/dev/null || true grn "created $ENV_FILE with freshly generated secrets" ylw "" ylw " >> BACK UP MASTER_KEY NOW. Copy it into your password manager. <<" ylw " Without it, saved mail-account passwords cannot be decrypted even" ylw " from a perfect database backup." ylw "" + ensure_secrets apply_init_flags if [ -n "$set_data_root" ]; then grn "configured from flags; you can go straight to: webmail.sh up" @@ -157,6 +192,7 @@ cmd_init() { return 0 fi + ensure_secrets if [ -n "${set_data_root}${set_domain}${set_tz}${set_port}${set_bind}" ]; then apply_init_flags grn "updated $ENV_FILE" @@ -187,6 +223,10 @@ cmd_doctor() { grn " MASTER_KEY looks well-formed" fi + if [ -z "${POSTGRES_PASSWORD:-}" ]; then + red " POSTGRES_PASSWORD is empty — Postgres will refuse to initialise"; ok=1 + fi + if [ "$(stat -c %a "$ENV_FILE" 2>/dev/null || echo 600)" != "600" ]; then ylw " $ENV_FILE is not chmod 600 (it holds your master key)" fi diff --git a/docker-compose.yml b/docker-compose.yml index d7850a0..a1d9d43 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,11 @@ # Start: ./deploy/webmail.sh up # Manual: docker compose up -d (reads .env from this directory) # +# Interpolation here must stay on the oldest, dumbest forms: ${VAR} and +# ${VAR:-default}. DSM's bundled Compose rejects `${VAR:?message}` with +# "invalid interpolation format", and rejects `include:` outright. Required +# values are enforced by deploy/webmail.sh and `webmail.sh doctor` instead. +# # Project name is deliberately NOT "webmail": docker/docker-compose.yml uses # that, and two compose files sharing one project name fight over each other's # containers — running `make up` would stop production. @@ -42,10 +47,10 @@ x-backend-image: &backend-image GIT_SHA: ${GIT_SHA:-unknown} x-backend-env: &backend-env - APP_DOMAIN: "${APP_DOMAIN:?set APP_DOMAIN in your .env}" - DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-webmail}:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in your .env}@postgres:5432/${POSTGRES_DB:-webmail} + APP_DOMAIN: "${APP_DOMAIN:-APP_DOMAIN-unset}" + DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-webmail}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-webmail} REDIS_URL: redis://redis:6379/0 - MASTER_KEY: "${MASTER_KEY:?set MASTER_KEY in your .env - generate one with openssl rand -base64 32}" + MASTER_KEY: "${MASTER_KEY}" BLOB_ROOT: /data/blobs SEARCH_BACKEND: ${SEARCH_BACKEND:-pg} MEILI_URL: ${MEILI_URL:-http://meilisearch:7700} @@ -90,10 +95,10 @@ services: chmod 700 /mnt/backups echo "[init] data directories ready" volumes: - - ${DATA_ROOT}/blobs:/mnt/blobs - - ${DATA_ROOT}/backups:/mnt/backups - - ${DATA_ROOT}/redis:/mnt/redis - - ${DATA_ROOT}/meili:/mnt/meili + - ${DATA_ROOT:-/DATA_ROOT-is-unset}/blobs:/mnt/blobs + - ${DATA_ROOT:-/DATA_ROOT-is-unset}/backups:/mnt/backups + - ${DATA_ROOT:-/DATA_ROOT-is-unset}/redis:/mnt/redis + - ${DATA_ROOT:-/DATA_ROOT-is-unset}/meili:/mnt/meili restart: "no" logging: *logging @@ -121,7 +126,7 @@ services: - effective_cache_size=${PG_EFFECTIVE_CACHE_SIZE:-768MB} # Full page writes stay ON: NAS filesystems (btrfs/ext4) are not 8k-atomic. volumes: - - ${DATA_ROOT:?set DATA_ROOT in your .env (see docker/.env.prod.example)}/postgres:/var/lib/postgresql/data + - ${DATA_ROOT:-/DATA_ROOT-is-unset}/postgres:/var/lib/postgresql/data healthcheck: test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-webmail} -d ${POSTGRES_DB:-webmail}"] interval: 10s @@ -148,7 +153,7 @@ services: - --maxmemory-policy - noeviction volumes: - - ${DATA_ROOT}/redis:/data + - ${DATA_ROOT:-/DATA_ROOT-is-unset}/redis:/data healthcheck: test: ["CMD", "redis-cli", "ping"] interval: 10s @@ -171,7 +176,7 @@ services: MEILI_NO_ANALYTICS: "true" TZ: ${TZ:-UTC} volumes: - - ${DATA_ROOT}/meili:/meili_data + - ${DATA_ROOT:-/DATA_ROOT-is-unset}/meili:/meili_data healthcheck: test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:7700/health | grep -q available"] interval: 10s @@ -194,7 +199,7 @@ services: <<: *backend-env ENABLED_PLUGINS: ${ENABLED_PLUGINS:-calendar contacts labels filters} volumes: - - ${DATA_ROOT}/blobs:/data/blobs + - ${DATA_ROOT:-/DATA_ROOT-is-unset}/blobs:/data/blobs depends_on: postgres: { condition: service_healthy } init: { condition: service_completed_successfully } @@ -206,7 +211,7 @@ services: command: ["api"] environment: *backend-env volumes: - - ${DATA_ROOT}/blobs:/data/blobs + - ${DATA_ROOT:-/DATA_ROOT-is-unset}/blobs:/data/blobs healthcheck: test: ["CMD-SHELL", "python -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8000/api/healthz', timeout=5).status==200 else 1)\""] interval: 30s @@ -227,7 +232,7 @@ services: command: ["worker-sync"] environment: *backend-env volumes: - - ${DATA_ROOT}/blobs:/data/blobs + - ${DATA_ROOT:-/DATA_ROOT-is-unset}/blobs:/data/blobs # Persistent IMAP IDLE connections; the default 1024 is fine for a personal # install but this costs nothing and matches the sync-engine v2 sizing. ulimits: @@ -248,7 +253,7 @@ services: command: ["worker-jobs"] environment: *backend-env volumes: - - ${DATA_ROOT}/blobs:/data/blobs + - ${DATA_ROOT:-/DATA_ROOT-is-unset}/blobs:/data/blobs ulimits: nofile: { soft: 8192, hard: 8192 } depends_on: @@ -304,7 +309,7 @@ services: BACKUP_KEEP_DAYS: ${BACKUP_KEEP_DAYS:-14} TZ: ${TZ:-UTC} volumes: - - ${DATA_ROOT}/backups:/backups + - ${DATA_ROOT:-/DATA_ROOT-is-unset}/backups:/backups - ./docker/backup.sh:/usr/local/bin/backup.sh:ro depends_on: postgres: { condition: service_healthy } diff --git a/docs/nas-install.md b/docs/nas-install.md index 0ff0230..cebcefd 100644 --- a/docs/nas-install.md +++ b/docs/nas-install.md @@ -75,7 +75,7 @@ No git on the NAS? Copy the folder over with File Station or `rsync` instead. ./deploy/webmail.sh init ``` -This creates `docker/.env.prod` with a freshly generated `MASTER_KEY` and +This creates `.env` with a freshly generated `MASTER_KEY` and database password. Then open it and set: ```ini @@ -90,7 +90,7 @@ WEB_BIND=0.0.0.0 # or one NAS IP, e.g. 192.168.1.185 > It encrypts every stored mail-account password. It exists **only** in this > file. A perfect database backup without this key gives you your mail but not > your credentials — you would have to re-enter every account password by hand. -> `docker/.env.prod` is `chmod 600` and git-ignored; treat it as a backup +> `.env` is `chmod 600` and git-ignored; treat it as a backup > artefact in its own right. ### 2.4 Start @@ -214,7 +214,7 @@ Containers are disposable. These are not: |---|---|---|---| | 1 | **Database** | `$DATA_ROOT/postgres` + nightly dumps in `$DATA_ROOT/backups` | Everything: accounts, threads, flags, calendar, contacts | | 2 | **Blob store** | `$DATA_ROOT/blobs` | Mail bodies and attachments (re-downloadable from the servers, slowly) | -| 3 | **`MASTER_KEY`** | `docker/.env.prod` + your password manager | Every saved mail-account password, permanently | +| 3 | **`MASTER_KEY`** | `.env` + your password manager | Every saved mail-account password, permanently | Everything else is derived and needs no backup: Redis holds queues and leases, Meilisearch's index rebuilds from Postgres (§8), and images rebuild from source. @@ -240,7 +240,7 @@ A snapshot of a *running* Postgres directory is crash-consistent — it usually replays fine, but it is not a substitute for the dumps, which are verified and restorable into a different Postgres version. Keep both. -Also back up `docker/.env.prod` somewhere that is not the NAS. +Also back up `.env` somewhere that is not the NAS. ### Restore drill — do this once, before you need it @@ -255,7 +255,7 @@ the old database untouched), and starts everything back up. Full disaster recovery onto a new box: 1. Install Docker, copy `$DATA_ROOT` back from your backup. -2. Copy the repo and `docker/.env.prod` — **with the original `MASTER_KEY`**. +2. Copy the repo and `.env` — **with the original `MASTER_KEY`**. 3. `./deploy/webmail.sh up` That's it. If the blob store was lost but the database survived, mail bodies @@ -301,7 +301,7 @@ Building on the NAS is simple but slow and RAM-hungry. Two alternatives: ```bash # On your PC — push to your Gitea registry: REGISTRY_PREFIX=192.168.1.185:8418/claude/webmail ./deploy/build-push.sh v1.0.0 -# then on the NAS, set REGISTRY_PREFIX + IMAGE_TAG in docker/.env.prod and: +# then on the NAS, set REGISTRY_PREFIX + IMAGE_TAG in .env and: ./deploy/webmail.sh update v1.0.0 # Or with no registry at all — build to a file and carry it over: @@ -353,7 +353,7 @@ API is started with `--proxy-headers`. Nothing to configure. Steady state for one or two mail accounts, `SEARCH_BACKEND=pg`: -| | Typical | Cap (`docker/.env.prod`) | +| | Typical | Cap (`.env`) | |---|---|---| | postgres | 150-300 MB | `PG_MEM_LIMIT=768m` | | api | 150-250 MB | `API_MEM_LIMIT=512m` | @@ -385,7 +385,7 @@ large mailboxes, at the cost of a container and 300-800 MB. To switch: ```bash -# in docker/.env.prod: SEARCH_BACKEND=meili and set MEILI_MASTER_KEY +# in .env: SEARCH_BACKEND=meili and set MEILI_MASTER_KEY ./deploy/webmail.sh up # starts Meilisearch automatically ./deploy/webmail.sh reindex # build the index from existing mail ``` @@ -411,7 +411,7 @@ backends, or search ever looks stale, just rebuild it: | "port is already allocated" | 8088 taken by another app | change `WEB_PORT` | | Mail not syncing | worker stopped, or credentials rejected | `./deploy/webmail.sh logs worker-sync` | | Search finds nothing after a restore | index not rebuilt | `./deploy/webmail.sh reindex --purge` | -| App won't start after editing `.env.prod` | malformed `MASTER_KEY` | `./deploy/webmail.sh doctor` | +| App won't start after editing `.env` | malformed `MASTER_KEY` | `./deploy/webmail.sh doctor` | Logs: `./deploy/webmail.sh logs [service] -f`. All logs are capped at 10 MB ×3 per container, so they can't fill the volume.