Deploy: fix the two things that broke the Container Manager build

DSM reported "invalid interpolation format for x-backend-env.MASTER_KEY" and "required variable MASTER_KEY is missing a value". Both were mine. 1. DSM's bundled Compose cannot parse `${VAR:?message}`. Those guards are gone; interpolation now sticks to `${VAR}` and `${VAR:-default}`, the forms every Compose understands. Bind mounts keep a `${DATA_ROOT:-/DATA_ROOT-is-unset}` sentinel so a missing config still cannot scatter directories across the root of the NAS filesystem, and the header says to keep it this way. 2. Config was at docker/.env.prod with a `.env` symlink beside it. Container Manager did not pick that up. The real file is now `.env` in the project folder — what every compose front end reads without being told — and docker/.env.prod becomes a symlink to it. `init` and `require_env` migrate an existing install in place, keeping its MASTER_KEY. Also, while testing the migration: `init` only generated secrets when creating a file from scratch, so a migrated or hand-written .env with an empty POSTGRES_PASSWORD started Postgres in trust-auth mode. It now tops up empty secrets — and never touches a MASTER_KEY that already has a value, since replacing it would make every stored credential undecryptable. doctor flags an empty password too. Verified: parses cleanly with no .env at all (the failure DSM hit), migration preserves an existing key, and `docker compose up -d --wait` from the project folder brings the stack up healthy and serving. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Nux3LCu4zP9nih4BRhPyHw
2026-09-14 19:49:03 +02:00
parent 5c444a6fd3
commit f3dcd938b1
3 changed files with 73 additions and 28 deletions
@@ -20,7 +20,8 @@ set -euo pipefail
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 ROOT_DIR="$(dirname "$SCRIPT_DIR")"
 COMPOSE_FILE="${ROOT_DIR}/docker-compose.yml"
-ENV_FILE="${ROOT_DIR}/docker/.env.prod"
+ENV_FILE="${ROOT_DIR}/.env"
+LEGACY_ENV_FILE="${ROOT_DIR}/docker/.env.prod"
 ENV_EXAMPLE="${ROOT_DIR}/docker/.env.prod.example"

 # Services that must come back healthy after a roll.
@@ -58,6 +59,7 @@ load_env_file() {
 }

 require_env() {
+    migrate_legacy_env
    [ -f "$ENV_FILE" ] || die "$ENV_FILE not found. Run: ./deploy/webmail.sh init"
    # The env file is chmod 600 and usually owned by root (init is run as root
    # on a NAS), and the data directories are root-owned too. Say so plainly
@@ -109,6 +111,38 @@ apply_init_flags() {
    return 0
 }

+# Earlier versions kept config at docker/.env.prod with a `.env` symlink beside
+# it. Synology's Container Manager did not pick that up, so the real file now
+# lives at the project root. Move an existing one across (keeping its master
+# key) and leave a symlink behind so older notes still work.
+migrate_legacy_env() {
+    [ -e "$ENV_FILE" ] && return 0
+    [ -f "$LEGACY_ENV_FILE" ] || return 0
+    [ -L "$LEGACY_ENV_FILE" ] && return 0
+    mv "$LEGACY_ENV_FILE" "$ENV_FILE"
+    chmod 600 "$ENV_FILE"
+    ln -sf ../.env "$LEGACY_ENV_FILE" 2>/dev/null || true
+    ylw "moved docker/.env.prod -> .env (Container Manager reads .env from the project folder)"
+}
+
+# Fill in required secrets that are present-but-empty (a migrated file from an
+# older layout, or a hand-written .env). NEVER touches a MASTER_KEY that already
+# has a value: replacing it would make every stored mail-account credential
+# undecryptable.
+ensure_secrets() {
+    local current
+    current="$(sed -n 's/^MASTER_KEY=//p' "$ENV_FILE" | head -1)"
+    if [ -z "$current" ]; then
+        set_env_value MASTER_KEY "$(openssl rand -base64 32)"
+        ylw "MASTER_KEY was empty — generated a new one. Back it up: webmail.sh show-key"
+    fi
+    current="$(sed -n 's/^POSTGRES_PASSWORD=//p' "$ENV_FILE" | head -1)"
+    if [ -z "$current" ]; then
+        set_env_value POSTGRES_PASSWORD "$(openssl rand -base64 24 | tr -d '/+=' | cut -c1-24)"
+        ylw "POSTGRES_PASSWORD was empty — generated one"
+    fi
+}
+
 cmd_init() {
    # Settings can be passed as flags so the whole install can be driven from a
    # NAS GUI (DSM Task Scheduler / QTS cron), with no editing of a hidden
@@ -128,6 +162,7 @@ cmd_init() {
        esac
    done

+    migrate_legacy_env
    if [ ! -f "$ENV_FILE" ]; then
        cp "$ENV_EXAMPLE" "$ENV_FILE"
        chmod 600 "$ENV_FILE"
@@ -138,15 +173,15 @@ cmd_init() {
        sed "s|^MASTER_KEY=.*|MASTER_KEY=${master}|; s|^POSTGRES_PASSWORD=.*|POSTGRES_PASSWORD=${pgpass}|" \
            "$ENV_FILE" > "${ENV_FILE}.tmp" && mv "${ENV_FILE}.tmp" "$ENV_FILE"
        chmod 600 "$ENV_FILE"
-        # The NAS GUIs read `.env` from the project folder (the repo root).
-        # A symlink keeps one source of truth instead of two copies to drift.
-        [ -e "${ROOT_DIR}/.env" ] || ln -s docker/.env.prod "${ROOT_DIR}/.env" 2>/dev/null || true
+        # Compatibility shim for notes that reference the old path.
+        [ -e "$LEGACY_ENV_FILE" ] || ln -s ../.env "$LEGACY_ENV_FILE" 2>/dev/null || true
        grn "created $ENV_FILE with freshly generated secrets"
        ylw ""
        ylw "  >> BACK UP MASTER_KEY NOW. Copy it into your password manager. <<"
        ylw "  Without it, saved mail-account passwords cannot be decrypted even"
        ylw "  from a perfect database backup."
        ylw ""
+        ensure_secrets
        apply_init_flags
        if [ -n "$set_data_root" ]; then
            grn "configured from flags; you can go straight to: webmail.sh up"
@@ -157,6 +192,7 @@ cmd_init() {
        return 0
    fi

+    ensure_secrets
    if [ -n "${set_data_root}${set_domain}${set_tz}${set_port}${set_bind}" ]; then
        apply_init_flags
        grn "updated $ENV_FILE"
@@ -187,6 +223,10 @@ cmd_doctor() {
        grn "  MASTER_KEY looks well-formed"
    fi

+    if [ -z "${POSTGRES_PASSWORD:-}" ]; then
+        red "  POSTGRES_PASSWORD is empty — Postgres will refuse to initialise"; ok=1
+    fi
+
    if [ "$(stat -c %a "$ENV_FILE" 2>/dev/null || echo 600)" != "600" ]; then
        ylw "  $ENV_FILE is not chmod 600 (it holds your master key)"
    fi
@@ -21,6 +21,11 @@
 #   Start:  ./deploy/webmail.sh up
 #   Manual: docker compose up -d          (reads .env from this directory)
 #
+#   Interpolation here must stay on the oldest, dumbest forms: ${VAR} and
+#   ${VAR:-default}. DSM's bundled Compose rejects `${VAR:?message}` with
+#   "invalid interpolation format", and rejects `include:` outright. Required
+#   values are enforced by deploy/webmail.sh and `webmail.sh doctor` instead.
+#
 #   Project name is deliberately NOT "webmail": docker/docker-compose.yml uses
 #   that, and two compose files sharing one project name fight over each other's
 #   containers — running `make up` would stop production.
@@ -42,10 +47,10 @@ x-backend-image: &backend-image
      GIT_SHA: ${GIT_SHA:-unknown}

 x-backend-env: &backend-env
-  APP_DOMAIN: "${APP_DOMAIN:?set APP_DOMAIN in your .env}"
-  DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-webmail}:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in your .env}@postgres:5432/${POSTGRES_DB:-webmail}
+  APP_DOMAIN: "${APP_DOMAIN:-APP_DOMAIN-unset}"
+  DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-webmail}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-webmail}
  REDIS_URL: redis://redis:6379/0
-  MASTER_KEY: "${MASTER_KEY:?set MASTER_KEY in your .env - generate one with openssl rand -base64 32}"
+  MASTER_KEY: "${MASTER_KEY}"
  BLOB_ROOT: /data/blobs
  SEARCH_BACKEND: ${SEARCH_BACKEND:-pg}
  MEILI_URL: ${MEILI_URL:-http://meilisearch:7700}
@@ -90,10 +95,10 @@ services:
        chmod 700 /mnt/backups
        echo "[init] data directories ready"
    volumes:
-      - ${DATA_ROOT}/blobs:/mnt/blobs
-      - ${DATA_ROOT}/backups:/mnt/backups
-      - ${DATA_ROOT}/redis:/mnt/redis
-      - ${DATA_ROOT}/meili:/mnt/meili
+      - ${DATA_ROOT:-/DATA_ROOT-is-unset}/blobs:/mnt/blobs
+      - ${DATA_ROOT:-/DATA_ROOT-is-unset}/backups:/mnt/backups
+      - ${DATA_ROOT:-/DATA_ROOT-is-unset}/redis:/mnt/redis
+      - ${DATA_ROOT:-/DATA_ROOT-is-unset}/meili:/mnt/meili
    restart: "no"
    logging: *logging

@@ -121,7 +126,7 @@ services:
      - effective_cache_size=${PG_EFFECTIVE_CACHE_SIZE:-768MB}
      # Full page writes stay ON: NAS filesystems (btrfs/ext4) are not 8k-atomic.
    volumes:
-      - ${DATA_ROOT:?set DATA_ROOT in your .env (see docker/.env.prod.example)}/postgres:/var/lib/postgresql/data
+      - ${DATA_ROOT:-/DATA_ROOT-is-unset}/postgres:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-webmail} -d ${POSTGRES_DB:-webmail}"]
      interval: 10s
@@ -148,7 +153,7 @@ services:
      - --maxmemory-policy
      - noeviction
    volumes:
-      - ${DATA_ROOT}/redis:/data
+      - ${DATA_ROOT:-/DATA_ROOT-is-unset}/redis:/data
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 10s
@@ -171,7 +176,7 @@ services:
      MEILI_NO_ANALYTICS: "true"
      TZ: ${TZ:-UTC}
    volumes:
-      - ${DATA_ROOT}/meili:/meili_data
+      - ${DATA_ROOT:-/DATA_ROOT-is-unset}/meili:/meili_data
    healthcheck:
      test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:7700/health | grep -q available"]
      interval: 10s
@@ -194,7 +199,7 @@ services:
      <<: *backend-env
      ENABLED_PLUGINS: ${ENABLED_PLUGINS:-calendar contacts labels filters}
    volumes:
-      - ${DATA_ROOT}/blobs:/data/blobs
+      - ${DATA_ROOT:-/DATA_ROOT-is-unset}/blobs:/data/blobs
    depends_on:
      postgres: { condition: service_healthy }
      init: { condition: service_completed_successfully }
@@ -206,7 +211,7 @@ services:
    command: ["api"]
    environment: *backend-env
    volumes:
-      - ${DATA_ROOT}/blobs:/data/blobs
+      - ${DATA_ROOT:-/DATA_ROOT-is-unset}/blobs:/data/blobs
    healthcheck:
      test: ["CMD-SHELL", "python -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8000/api/healthz', timeout=5).status==200 else 1)\""]
      interval: 30s
@@ -227,7 +232,7 @@ services:
    command: ["worker-sync"]
    environment: *backend-env
    volumes:
-      - ${DATA_ROOT}/blobs:/data/blobs
+      - ${DATA_ROOT:-/DATA_ROOT-is-unset}/blobs:/data/blobs
    # Persistent IMAP IDLE connections; the default 1024 is fine for a personal
    # install but this costs nothing and matches the sync-engine v2 sizing.
    ulimits:
@@ -248,7 +253,7 @@ services:
    command: ["worker-jobs"]
    environment: *backend-env
    volumes:
-      - ${DATA_ROOT}/blobs:/data/blobs
+      - ${DATA_ROOT:-/DATA_ROOT-is-unset}/blobs:/data/blobs
    ulimits:
      nofile: { soft: 8192, hard: 8192 }
    depends_on:
@@ -304,7 +309,7 @@ services:
      BACKUP_KEEP_DAYS: ${BACKUP_KEEP_DAYS:-14}
      TZ: ${TZ:-UTC}
    volumes:
-      - ${DATA_ROOT}/backups:/backups
+      - ${DATA_ROOT:-/DATA_ROOT-is-unset}/backups:/backups
      - ./docker/backup.sh:/usr/local/bin/backup.sh:ro
    depends_on:
      postgres: { condition: service_healthy }
@@ -75,7 +75,7 @@ No git on the NAS? Copy the folder over with File Station or `rsync` instead.
 ./deploy/webmail.sh init
 ```

-This creates `docker/.env.prod` with a freshly generated `MASTER_KEY` and
+This creates `.env` with a freshly generated `MASTER_KEY` and
 database password. Then open it and set:

 ```ini
@@ -90,7 +90,7 @@ WEB_BIND=0.0.0.0                         # or one NAS IP, e.g. 192.168.1.185
 > It encrypts every stored mail-account password. It exists **only** in this
 > file. A perfect database backup without this key gives you your mail but not
 > your credentials — you would have to re-enter every account password by hand.
-> `docker/.env.prod` is `chmod 600` and git-ignored; treat it as a backup
+> `.env` is `chmod 600` and git-ignored; treat it as a backup
 > artefact in its own right.

 ### 2.4 Start
@@ -214,7 +214,7 @@ Containers are disposable. These are not:
 |---|---|---|---|
 | 1 | **Database** | `$DATA_ROOT/postgres` + nightly dumps in `$DATA_ROOT/backups` | Everything: accounts, threads, flags, calendar, contacts |
 | 2 | **Blob store** | `$DATA_ROOT/blobs` | Mail bodies and attachments (re-downloadable from the servers, slowly) |
-| 3 | **`MASTER_KEY`** | `docker/.env.prod` + your password manager | Every saved mail-account password, permanently |
+| 3 | **`MASTER_KEY`** | `.env` + your password manager | Every saved mail-account password, permanently |

 Everything else is derived and needs no backup: Redis holds queues and leases,
 Meilisearch's index rebuilds from Postgres (§8), and images rebuild from source.
@@ -240,7 +240,7 @@ A snapshot of a *running* Postgres directory is crash-consistent — it usually
 replays fine, but it is not a substitute for the dumps, which are verified and
 restorable into a different Postgres version. Keep both.

-Also back up `docker/.env.prod` somewhere that is not the NAS.
+Also back up `.env` somewhere that is not the NAS.

 ### Restore drill — do this once, before you need it

@@ -255,7 +255,7 @@ the old database untouched), and starts everything back up.
 Full disaster recovery onto a new box:

 1. Install Docker, copy `$DATA_ROOT` back from your backup.
-2. Copy the repo and `docker/.env.prod` — **with the original `MASTER_KEY`**.
+2. Copy the repo and `.env` — **with the original `MASTER_KEY`**.
 3. `./deploy/webmail.sh up`

 That's it. If the blob store was lost but the database survived, mail bodies
@@ -301,7 +301,7 @@ Building on the NAS is simple but slow and RAM-hungry. Two alternatives:
 ```bash
 # On your PC — push to your Gitea registry:
 REGISTRY_PREFIX=192.168.1.185:8418/claude/webmail ./deploy/build-push.sh v1.0.0
-# then on the NAS, set REGISTRY_PREFIX + IMAGE_TAG in docker/.env.prod and:
+# then on the NAS, set REGISTRY_PREFIX + IMAGE_TAG in .env and:
 ./deploy/webmail.sh update v1.0.0

 # Or with no registry at all — build to a file and carry it over:
@@ -353,7 +353,7 @@ API is started with `--proxy-headers`. Nothing to configure.

 Steady state for one or two mail accounts, `SEARCH_BACKEND=pg`:

-| | Typical | Cap (`docker/.env.prod`) |
+| | Typical | Cap (`.env`) |
 |---|---|---|
 | postgres | 150-300 MB | `PG_MEM_LIMIT=768m` |
 | api | 150-250 MB | `API_MEM_LIMIT=512m` |
@@ -385,7 +385,7 @@ large mailboxes, at the cost of a container and 300-800 MB.
 To switch:

 ```bash
-# in docker/.env.prod:  SEARCH_BACKEND=meili  and set MEILI_MASTER_KEY
+# in .env:  SEARCH_BACKEND=meili  and set MEILI_MASTER_KEY
 ./deploy/webmail.sh up          # starts Meilisearch automatically
 ./deploy/webmail.sh reindex     # build the index from existing mail
 ```
@@ -411,7 +411,7 @@ backends, or search ever looks stale, just rebuild it:
 | "port is already allocated" | 8088 taken by another app | change `WEB_PORT` |
 | Mail not syncing | worker stopped, or credentials rejected | `./deploy/webmail.sh logs worker-sync` |
 | Search finds nothing after a restore | index not rebuilt | `./deploy/webmail.sh reindex --purge` |
-| App won't start after editing `.env.prod` | malformed `MASTER_KEY` | `./deploy/webmail.sh doctor` |
+| App won't start after editing `.env` | malformed `MASTER_KEY` | `./deploy/webmail.sh doctor` |

 Logs: `./deploy/webmail.sh logs [service] -f`. All logs are capped at 10 MB ×3
 per container, so they can't fill the volume.