webmail-client/deploy/webmail.sh

#!/usr/bin/env bash
# Webmail operator control script. One command per thing you might need to do.
#
#   ./deploy/webmail.sh init            create data dirs + a filled-in .env.prod
#   ./deploy/webmail.sh pull [tag]      fetch prebuilt images from your registry
#   ./deploy/webmail.sh up [--meili]    start (builds images if missing)
#   ./deploy/webmail.sh down            stop (data is untouched)
#   ./deploy/webmail.sh status          what is running, and which version
#   ./deploy/webmail.sh logs [svc] [-f] tail logs
#   ./deploy/webmail.sh update [tag]    backup -> new images -> migrate -> roll -> health-gate
#   ./deploy/webmail.sh rollback [tag]  go back to the previous version
#   ./deploy/webmail.sh backup          take a verified database dump now
#   ./deploy/webmail.sh restore <file>  DESTRUCTIVE: restore a dump
#   ./deploy/webmail.sh reindex [args]  rebuild the search index from Postgres
#   ./deploy/webmail.sh ngrams [langs]  download spell-check n-gram data (default: en de fr es)
#   ./deploy/webmail.sh psql            open a database shell
#   ./deploy/webmail.sh shell [svc]     shell inside a container (default: api)
#   ./deploy/webmail.sh show-key        print MASTER_KEY so you can back it up
#   ./deploy/webmail.sh doctor          check the install before it bites you
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT_DIR="$(dirname "$SCRIPT_DIR")"
COMPOSE_FILE="${ROOT_DIR}/docker-compose.yml"
ENV_FILE="${ROOT_DIR}/.env"
LEGACY_ENV_FILE="${ROOT_DIR}/docker/.env.prod"
ENV_EXAMPLE="${ROOT_DIR}/docker/.env.prod.example"

# Services that must come back healthy after a roll.
HEALTH_SERVICES=(postgres redis api web)
HEALTH_TIMEOUT=120

red()  { printf '\033[31m%s\033[0m\n' "$*"; }
grn()  { printf '\033[32m%s\033[0m\n' "$*"; }
ylw()  { printf '\033[33m%s\033[0m\n' "$*"; }
die()  { red "error: $*" >&2; exit 1; }

# QNAP keeps Container Station's `docker` and `docker-compose` inside the QPKG
# directory, which is NOT on the PATH of an ordinary SSH session — the script
# would otherwise die with "no compose found" on a box that has both. The volume
# name differs between QTS (CACHEDEV1_DATA) and QuTS hero (the ZFS pool name),
# so glob rather than hard-code it. No-op everywhere else.
ensure_docker_on_path() {
    command -v docker >/dev/null 2>&1 && return 0
    local dir
    for dir in /share/*/.qpkg/container-station/bin /share/.qpkg/container-station/bin; do
        if [ -x "$dir/docker" ]; then
            PATH="$dir:$PATH"
            export PATH
            ylw "using Container Station's docker from $dir"
            return 0
        fi
    done
    return 0   # not found: let the caller fail with its own, clearer message
}

# Which compose is on this box. Synology DSM and QNAP QTS/QuTS hero ship the
# standalone `docker-compose` binary and may have no `docker compose`
# subcommand, so detect rather than assume. COMPOSE_WAIT is empty on v1, which
# has no `--wait`; the script's own health_gate covers that case.
COMPOSE_BIN=()
COMPOSE_WAIT=""
detect_compose() {
    [ "${#COMPOSE_BIN[@]}" -gt 0 ] && return 0
    ensure_docker_on_path
    if docker compose version >/dev/null 2>&1; then
        COMPOSE_BIN=(docker compose)
    elif command -v docker-compose >/dev/null 2>&1; then
        COMPOSE_BIN=(docker-compose)
    else
        die "no compose found: neither 'docker compose' nor 'docker-compose' is available"
    fi
    local ver
    ver="$("${COMPOSE_BIN[@]}" version 2>/dev/null | head -1)"
    case "$ver" in
        *v2.*|*" 2."*) COMPOSE_WAIT="--wait" ;;
        *) COMPOSE_WAIT=""
           ylw "compose v1 detected (${ver:-unknown}) — using the script's own health checks" ;;
    esac
}

compose() {
    detect_compose
    "${COMPOSE_BIN[@]}" --env-file "$ENV_FILE" -f "$COMPOSE_FILE" "$@"
}

# Read KEY=VALUE pairs the way compose's .env parser does, WITHOUT sourcing.
# Sourcing would execute the file as shell — it would choke on an ordinary
# value containing spaces (ENABLED_PLUGINS) and would run anything else that
# found its way into the file that holds the master key.
load_env_file() {
    local line key val
    while IFS= read -r line || [ -n "$line" ]; do
        case "$line" in ''|'#'*) continue ;; esac
        case "$line" in *=*) ;; *) continue ;; esac
        key="${line%%=*}"
        val="${line#*=}"
        # Trim whitespace around the key; strip one layer of surrounding quotes.
        key="$(printf '%s' "$key" | tr -d '[:space:]')"
        case "$val" in
            \"*\") val="${val#\"}"; val="${val%\"}" ;;
            \'*\') val="${val#\'}"; val="${val%\'}" ;;
        esac
        [ -n "$key" ] && export "${key}=${val}"
    done < "$1"
}

require_env() {
    migrate_legacy_env
    [ -f "$ENV_FILE" ] || die "$ENV_FILE not found. Run: ./deploy/webmail.sh init"
    # The env file is chmod 600 and usually owned by root (init is run as root
    # on a NAS), and the data directories are root-owned too. Say so plainly
    # instead of letting a bare "Permission denied" escape from a redirect.
    if [ ! -r "$ENV_FILE" ]; then
        red "error: cannot read $ENV_FILE (it holds the master key, so it is chmod 600)." >&2
        red "You are running as $(id -un). Re-run this with sudo:" >&2
        red "    sudo $0 ${SUBCOMMAND:-<command>}" >&2
        red "Or start a root shell for the session: sudo -i" >&2
        exit 1
    fi
    load_env_file "$ENV_FILE"
    [ -n "${DATA_ROOT:-}" ]         || die "DATA_ROOT is not set in $ENV_FILE"
    [ -n "${MASTER_KEY:-}" ]        || die "MASTER_KEY is not set in $ENV_FILE"
    [ -n "${POSTGRES_PASSWORD:-}" ] || die "POSTGRES_PASSWORD is not set in $ENV_FILE"
}

state_dir() { echo "${DATA_ROOT}/state"; }

# busybox sed (what you may get on DSM/QTS) has no -i suffix argument, so every
# in-place edit goes through a temp file.
set_env_value() {
    local key="$1" value="$2"
    if grep -q "^${key}=" "$ENV_FILE" 2>/dev/null; then
        sed "s|^${key}=.*|${key}=${value}|" "$ENV_FILE" > "${ENV_FILE}.tmp" \
            && mv "${ENV_FILE}.tmp" "$ENV_FILE"
    else
        # A key introduced by a later release is simply absent from an older
        # .env, and `sed s|^KEY=.*|` substitutes nothing while reporting success
        # — so a generated secret was announced and then silently dropped. Append
        # instead, after making sure the file ends in a newline so the new key
        # cannot be glued onto the last line.
        if [ -s "$ENV_FILE" ] && [ -n "$(tail -c1 "$ENV_FILE")" ]; then
            printf '\n' >> "$ENV_FILE"
        fi
        printf '%s=%s\n' "$key" "$value" >> "$ENV_FILE"
    fi
    chmod 600 "$ENV_FILE"
    # Never report success for a value that did not land: this function silently
    # dropped a generated secret once, and the install only failed much later.
    grep -q "^${key}=" "$ENV_FILE" || die "failed to write ${key} to ${ENV_FILE}"
}

record_version() {
    local new="$1" cur
    mkdir -p "$(state_dir)" 2>/dev/null || true
    if ! { : > "$(state_dir)/.probe"; } 2>/dev/null; then
        ylw "cannot write ${DATA_ROOT}/state — version history not recorded (run as the owner of DATA_ROOT)"
        return 0
    fi
    rm -f "$(state_dir)/.probe"
    cur="$(cat "$(state_dir)/version" 2>/dev/null || echo "")"
    [ -n "$cur" ] && [ "$cur" != "$new" ] && echo "$cur" > "$(state_dir)/previous"
    echo "$new" > "$(state_dir)/version"
}

# --- commands ----------------------------------------------------------------

apply_init_flags() {
    [ -n "$set_data_root" ] && set_env_value DATA_ROOT  "$set_data_root" && echo "  DATA_ROOT  = $set_data_root"
    [ -n "$set_domain" ]    && set_env_value APP_DOMAIN "$set_domain"    && echo "  APP_DOMAIN = $set_domain"
    [ -n "$set_tz" ]        && set_env_value TZ         "$set_tz"        && echo "  TZ         = $set_tz"
    [ -n "$set_port" ]      && set_env_value WEB_PORT   "$set_port"      && echo "  WEB_PORT   = $set_port"
    [ -n "$set_bind" ]      && set_env_value WEB_BIND   "$set_bind"      && echo "  WEB_BIND   = $set_bind"
    return 0
}

# Earlier versions kept config at docker/.env.prod with a `.env` symlink beside
# it. Synology's Container Manager did not pick that up, so the real file now
# lives at the project root. Move an existing one across (keeping its master
# key) and leave a symlink behind so older notes still work.
migrate_legacy_env() {
    [ -e "$ENV_FILE" ] && return 0
    [ -f "$LEGACY_ENV_FILE" ] || return 0
    [ -L "$LEGACY_ENV_FILE" ] && return 0
    mv "$LEGACY_ENV_FILE" "$ENV_FILE"
    chmod 600 "$ENV_FILE"
    ln -sf ../.env "$LEGACY_ENV_FILE" 2>/dev/null || true
    ylw "moved docker/.env.prod -> .env (Container Manager reads .env from the project folder)"
}

# Fill in required secrets that are present-but-empty (a migrated file from an
# older layout, or a hand-written .env). NEVER touches a MASTER_KEY that already
# has a value: replacing it would make every stored mail-account credential
# undecryptable.
ensure_secrets() {
    local current
    current="$(sed -n 's/^MASTER_KEY=//p' "$ENV_FILE" | head -1)"
    if [ -z "$current" ]; then
        set_env_value MASTER_KEY "$(openssl rand -base64 32)"
        ylw "MASTER_KEY was empty — generated a new one. Back it up: webmail.sh show-key"
    fi
    current="$(sed -n 's/^POSTGRES_PASSWORD=//p' "$ENV_FILE" | head -1)"
    if [ -z "$current" ]; then
        set_env_value POSTGRES_PASSWORD "$(openssl rand -base64 24 | tr -d '/+=' | cut -c1-24)"
        ylw "POSTGRES_PASSWORD was empty — generated one"
    fi
    # Internal only: the antispam engine's controller. Nobody types this — the
    # admin panel authenticates humans with the ordinary administrator account
    # and holds this to talk to Rspamd. Nothing is lost if it is regenerated.
    current="$(sed -n 's/^RSPAMD_CONTROLLER_PASSWORD=//p' "$ENV_FILE" | head -1)"
    if [ -z "$current" ]; then
        set_env_value RSPAMD_CONTROLLER_PASSWORD \
            "$(openssl rand -base64 24 | tr -d '/+=' | cut -c1-24)"
        ylw "RSPAMD_CONTROLLER_PASSWORD was empty — generated one"
    fi
}

cmd_init() {
    # Settings can be passed as flags so the whole install can be driven from a
    # NAS GUI (DSM Task Scheduler / QTS cron), with no editing of a hidden
    # dotfile in File Station. Re-running with flags updates those values.
    local set_data_root="" set_domain="" set_tz="" set_port="" set_bind=""
    while [ "$#" -gt 0 ]; do
        case "$1" in
            --data-root) set_data_root="${2:-}"; shift 2 ;;
            --domain)    set_domain="${2:-}";    shift 2 ;;
            --tz)        set_tz="${2:-}";        shift 2 ;;
            --port)      set_port="${2:-}";      shift 2 ;;
            --bind)      set_bind="${2:-}";      shift 2 ;;
            -h|--help)
                echo "usage: webmail.sh init [--data-root PATH] [--domain DOMAIN] [--tz ZONE] [--port N] [--bind IP]"
                return 0 ;;
            *) die "init: unknown option $1" ;;
        esac
    done

    migrate_legacy_env
    if [ ! -f "$ENV_FILE" ]; then
        cp "$ENV_EXAMPLE" "$ENV_FILE"
        chmod 600 "$ENV_FILE"
        local master pgpass
        master="$(openssl rand -base64 32)"
        pgpass="$(openssl rand -base64 24 | tr -d '/+=' | cut -c1-24)"
        # Portable in-place edit (BSD and GNU sed disagree about -i).
        sed "s|^MASTER_KEY=.*|MASTER_KEY=${master}|; s|^POSTGRES_PASSWORD=.*|POSTGRES_PASSWORD=${pgpass}|" \
            "$ENV_FILE" > "${ENV_FILE}.tmp" && mv "${ENV_FILE}.tmp" "$ENV_FILE"
        chmod 600 "$ENV_FILE"
        # Compatibility shim for notes that reference the old path.
        [ -e "$LEGACY_ENV_FILE" ] || ln -s ../.env "$LEGACY_ENV_FILE" 2>/dev/null || true
        grn "created $ENV_FILE with freshly generated secrets"
        ylw ""
        ylw "  >> BACK UP MASTER_KEY NOW. Copy it into your password manager. <<"
        ylw "  Without it, saved mail-account passwords cannot be decrypted even"
        ylw "  from a perfect database backup."
        ylw ""
        ensure_secrets
        apply_init_flags
        if [ -n "$set_data_root" ]; then
            grn "configured from flags; you can go straight to: webmail.sh up"
        else
            echo "Then edit $ENV_FILE and set at least: DATA_ROOT, APP_DOMAIN, TZ, WEB_PORT."
            echo "(or re-run: webmail.sh init --data-root ... --domain ... --tz ... --port ...)"
        fi
        return 0
    fi

    ensure_secrets
    if [ -n "${set_data_root}${set_domain}${set_tz}${set_port}${set_bind}" ]; then
        apply_init_flags
        grn "updated $ENV_FILE"
    else
        ylw "$ENV_FILE already exists; leaving it alone."
    fi
    require_env
    mkdir -p "${DATA_ROOT}"/{postgres,blobs,redis,backups,meili,state} 2>/dev/null \
        || ylw "could not create every directory under ${DATA_ROOT} (the init container will)"
    grn "data directories ready under ${DATA_ROOT}"
}

cmd_doctor() {
    require_env
    local ok=0

    if [ ! -s "$ENV_FILE" ]; then
        red "  $ENV_FILE is missing or empty — nothing will start. Run: webmail.sh init" >&2
    fi

    echo "config:"
    echo "  DATA_ROOT      ${DATA_ROOT}"
    echo "  APP_DOMAIN     ${APP_DOMAIN:-<unset>}"
    echo "  WEB_PORT       ${WEB_PORT:-8088}"
    echo "  ADMIN_PORT     ${ADMIN_PORT:-8089}  (keep this one LAN-only)"
    echo "  SEARCH_BACKEND ${SEARCH_BACKEND:-pg}"
    echo "  IMAGE_TAG      ${IMAGE_TAG:-local}"
    echo

    if [ "$(printf '%s' "${MASTER_KEY}" | base64 -d 2>/dev/null | wc -c)" -ne 32 ]; then
        red "  MASTER_KEY is not 32 bytes of base64 — the app will refuse to start"; ok=1
    else
        grn "  MASTER_KEY looks well-formed"
    fi

    if [ -z "${POSTGRES_PASSWORD:-}" ]; then
        red "  POSTGRES_PASSWORD is empty — Postgres will refuse to initialise"; ok=1
    fi

    if [ "$(stat -c %a "$ENV_FILE" 2>/dev/null || echo 600)" != "600" ]; then
        ylw "  $ENV_FILE is not chmod 600 (it holds your master key)"
    fi

    # Missing directories are not a failure before the first start: `up` and the
    # `init` container create them. Only say so, so a pre-install doctor is green.
    local missing=0
    for d in postgres blobs redis backups state; do
        [ -d "${DATA_ROOT}/${d}" ] || missing=1
    done
    if [ "$missing" -eq 1 ]; then
        ylw "  data directories not created yet — 'up' will create them"
    else
        grn "  data directories present"
    fi

    # Probe the nearest existing ancestor: the NFS/SMB check has to work BEFORE
    # the first install, which is the only time it can still save you.
    local fstype probe="${DATA_ROOT}"
    while [ ! -d "$probe" ] && [ "$probe" != "/" ] && [ -n "$probe" ]; do
        probe="$(dirname "$probe")"
    done
    # `|| true`: with `set -e -o pipefail`, df failing on a not-yet-created
    # DATA_ROOT would abort doctor on exactly the run where you need it most.
    fstype="$(df -PT "$probe" 2>/dev/null | awk 'NR==2{print $2}' || true)"
    [ -n "$fstype" ] || fstype="$(stat -f -c %T "$probe" 2>/dev/null || echo "")"
    case "$fstype" in
        nfs*|cifs|smbfs|fuse*)
            red "  ${DATA_ROOT} is on a network filesystem — Postgres WILL corrupt there."
            red "  Move DATA_ROOT to a local volume (btrfs/ext4)."; ok=1 ;;
        "") ylw "  could not determine the filesystem of ${DATA_ROOT}" ;;
        *)  grn "  ${DATA_ROOT} is on a local filesystem" ;;
    esac

    local avail
    avail="$(df -Pm "$probe" 2>/dev/null | awk 'NR==2{print $4}' || true)"
    [ -n "$avail" ] && {
        if [ "$avail" -lt 5120 ]; then ylw "  only ${avail} MB free on ${DATA_ROOT}"
        else grn "  ${avail} MB free on ${DATA_ROOT}"; fi
    }

    # A plugin shipped in the image but absent from ENABLED_PLUGINS never gets its
    # tables created, and then fails at runtime rather than at install. The
    # entrypoint's default is the list of what ships, so compare against that
    # instead of duplicating it here.
    local shipped missing=""
    shipped="$(sed -n 's/^: "${ENABLED_PLUGINS:=\(.*\)}"/\1/p' "${ROOT_DIR}/docker/entrypoint.sh")"
    if [ -n "${ENABLED_PLUGINS:-}" ] && [ -n "$shipped" ]; then
        local p
        for p in $shipped; do
            case " ${ENABLED_PLUGINS} " in *" $p "*) ;; *) missing="${missing} $p" ;; esac
        done
        if [ -n "$missing" ]; then
            ylw "  ENABLED_PLUGINS is missing:${missing}"
            echo "    these ship in the image but will have no tables created."
            echo "    fix: add them to ENABLED_PLUGINS in ${ENV_FILE}, then: webmail.sh update"
        else
            grn "  ENABLED_PLUGINS covers every plugin in the image"
        fi
    fi

    # Antispam splits over two ports and only one of them needs a password, so a
    # missing password looks like a working filter: mail still gets classified by
    # the scan worker while every learn is refused by the controller. Prove the
    # app containers can actually reach it rather than trusting that the variable
    # is set somewhere.
    #
    # Every path below says something. The first cut of this check hid its own
    # result behind an `elif` with no `else` and a `docker compose ps --status`
    # that older Compose does not accept, so on the install it was written for it
    # printed nothing at all — the same silence it exists to break.
    case " ${ENABLED_PLUGINS:-} " in
      *" antispam "*)
        if [ -z "${RSPAMD_CONTROLLER_PASSWORD:-}" ]; then
            red "  RSPAMD_CONTROLLER_PASSWORD is empty — antispam will classify but never learn"
            echo "    fix: webmail.sh init regenerates it, then: webmail.sh update"; ok=1
        else
            # The call the learning loop makes, from the container that makes it,
            # with the password that container actually holds.
            probe="$(compose exec -T api python -c "
import os, urllib.request
req = urllib.request.Request(
    os.environ.get('RSPAMD_CONTROLLER_URL', 'http://rspamd:11334') + '/stat',
    headers={'Password': os.environ.get('RSPAMD_CONTROLLER_PASSWORD', ''),
             'Deliver-To': 'doctor'})
urllib.request.urlopen(req, timeout=5).read()
print('AUTHORISED')
" 2>&1)" || true
            case "$probe" in
                *AUTHORISED*)
                    grn "  antispam engine reachable and learning is authorised" ;;
                *401*|*Unauthorized*)
                    red "  the api container cannot authenticate to the antispam controller."
                    red "  Mail will still be classified, but NOTHING will be learned."
                    echo "    The api and rspamd containers hold different controller passwords. Either:"
                    if [ "$(grep -c RSPAMD_CONTROLLER_PASSWORD "$COMPOSE_FILE")" -lt 3 ]; then
                        echo "    - ${COMPOSE_FILE} is older than v1.3.2 and does not give the password"
                        echo "      to api/worker-*. fix: refresh the source tree, keeping .env"
                        echo "      (docs/nas-install.md §4, 'Refreshing the source tree')"
                    fi
                    echo "    - rspamd was started with an older value from .env. fix: webmail.sh update"
                    echo "      (it recreates rspamd), or just that container:"
                    echo "      docker compose --env-file .env -f docker-compose.yml up -d --no-deps --force-recreate rspamd"
                    ok=1 ;;
                *"not running"*|*"No such service"*|*"no such service"*|*"is not running"*)
                    ylw "  antispam not checked — the stack is not running. Start it, then re-run doctor." ;;
                *)
                    ylw "  antispam check inconclusive; the engine may be starting. Raw result:"
                    printf '    %s\n' "$(printf '%s' "$probe" | tail -3)" ;;
            esac
        fi ;;
      *)
        ylw "  antispam is not in ENABLED_PLUGINS — no spam filtering on this install" ;;
    esac

    # Spelling/grammar check: is the engine answering, and with which n-grams?
    lt="$(compose exec -T api python -c "
import os, urllib.request
url = os.environ.get('LANGUAGETOOL_URL', '')
print('OFF' if not url else urllib.request.urlopen(url + '/v2/languages', timeout=5).status)
" 2>&1 | tail -1)" || true
    ngrams=""
    for d in "${DATA_ROOT}/languagetool-ngrams"/*/; do
        [ -d "${d}3grams" ] && ngrams="${ngrams} $(basename "$d")"
    done
    case "$lt" in
        200) grn "  spell check engine answering; n-grams:${ngrams:- none (optional: webmail.sh ngrams)}" ;;
        OFF) ylw "  spell check is off (LANGUAGETOOL_URL is empty)" ;;
        *)   ylw "  spell check engine not answering — the compose window falls back to the browser's checker"
             printf '    %s\n' "$lt" ;;
    esac

    # Email translation: is the translator answering, with which languages?
    tr="$(compose exec -T api python -c "
import json, os, urllib.request
url = os.environ.get('TRANSLATOR_URL', '')
if not url:
    print('OFF')
else:
    d = json.load(urllib.request.urlopen(url + '/languages', timeout=5))
    busy = ', '.join(d.get('installing') or {})
    print('OK ' + ' '.join(d['installed']) + (' (installing: ' + busy + ')' if busy else ''))
" 2>&1 | tail -1)" || true
    case "$tr" in
        OK*) grn "  translation answering; languages: ${tr#OK }" ;;
        OFF) ylw "  translation is off (TRANSLATOR_URL is empty)" ;;
        *)   ylw "  translation not answering — the Translate button stays hidden"
             printf '    %s\n' "$tr" ;;
    esac

    if command -v ss >/dev/null 2>&1; then
        local p
        for p in "WEB_PORT:${WEB_PORT:-8088}" "ADMIN_PORT:${ADMIN_PORT:-8089}"; do
            if ss -ltn 2>/dev/null | grep -q ":${p#*:} "; then
                ylw "  port ${p#*:} already has a listener — change ${p%%:*} if that is not us"
            fi
        done
    fi

    # chmod 700 and owned by uid 999, so count them as root in a container.
    local recent
    recent="$(docker run --rm -v "${DATA_ROOT}/backups:/b:ro" alpine:3.20 \
        sh -c 'find /b -maxdepth 1 -name "*.dump" -mtime -2 2>/dev/null | wc -l' 2>/dev/null || echo 0)"
    if [ "${recent:-0}" -gt 0 ]; then grn "  ${recent} database backup(s) from the last 48 h"
    else ylw "  no database backup in the last 48 h"; fi

    [ "$ok" -eq 0 ] && grn "doctor: no blocking problems" || red "doctor: problems found (above)"
    return "$ok"
}

cmd_up() {
    require_env
    local profile=() build=()
    for arg in "$@"; do
        case "$arg" in
            --meili) profile=(--profile meili) ;;
            --build) build=(--build) ;;
            *) die "unknown option: $arg" ;;
        esac
    done
    [ "${SEARCH_BACKEND:-pg}" = "meili" ] && [ ${#profile[@]} -eq 0 ] && {
        ylw "SEARCH_BACKEND=meili — starting the Meilisearch container too"
        profile=(--profile meili)
    }
    # DATA_ROOT must exist before compose runs: Synology's Docker daemon will
    # not create a missing bind-mount source, it just fails the container. The
    # `init` service creates the subdirectories from there.
    mkdir -p "${DATA_ROOT}" 2>/dev/null || true
    if [ ! -d "${DATA_ROOT}" ]; then
        die "cannot create ${DATA_ROOT} (running as $(id -un)). Create it first:
    sudo mkdir -p ${DATA_ROOT}
then re-run this command."
    fi
    detect_compose
    # ${arr[@]+"${arr[@]}"} rather than plain "${arr[@]}": under `set -u`, bash
    # 4.3 and older — which is what QuTS hero ships — treats expanding an EMPTY
    # array as an unbound variable and aborts. The +-form expands to nothing at
    # all when the array is empty, and is identical otherwise.
    # shellcheck disable=SC2086
    compose ${profile[@]+"${profile[@]}"} up -d ${build[@]+"${build[@]}"} $COMPOSE_WAIT
    health_gate || die "services did not come up healthy — check: ./deploy/webmail.sh logs"
    record_version "${IMAGE_TAG:-local}"
    grn "up. open http://<this-host>:${WEB_PORT:-8088}/"
    echo "    admin: http://<this-host>:${ADMIN_PORT:-8089}/ — restrict this port to your LAN"
}

# First install from a registry: `up` only builds what is missing locally, and
# `update` refuses to run without an existing database to back up. This is the
# gap between them — fetch the images, then `up` starts them instead of building.
cmd_pull() {
    require_env
    local tag="${1:-${IMAGE_TAG:-local}}"
    [ "${REGISTRY_PREFIX:-webmail}" = "webmail" ] && die \
        "REGISTRY_PREFIX is still the local default — nothing to pull from.
    Set REGISTRY_PREFIX and IMAGE_TAG in ${ENV_FILE}, or use: webmail.sh up --build"
    detect_compose
    IMAGE_TAG="$tag" compose pull || die "pull failed — is the registry reachable and logged in?"
    grn "pulled ${REGISTRY_PREFIX}/{backend,web,translator}:${tag}. Next: ./deploy/webmail.sh up"
}

cmd_down()    { require_env; compose --profile meili down; grn "stopped (data kept in ${DATA_ROOT})"; }
cmd_restart() { require_env; compose restart "${@:-}"; }

cmd_status() {
    require_env
    compose --profile meili ps
    echo
    echo "recorded version : $(cat "$(state_dir)/version" 2>/dev/null || echo unknown)"
    echo "previous version : $(cat "$(state_dir)/previous" 2>/dev/null || echo none)"
    local running
    running="$(compose exec -T api python -c \
        "import urllib.request,json;print(json.load(urllib.request.urlopen('http://127.0.0.1:8000/api/version'))['version'])" \
        2>/dev/null || echo unreachable)"
    echo "running version  : ${running}"
    echo
    # The data directories are owned by the container users (postgres 999,
    # backend 10001) and backups are chmod 700, so read them as root inside a
    # throwaway container rather than reporting misleading zeroes.
    docker run --rm -v "${DATA_ROOT}:/d:ro" alpine:3.20 sh -c '
        echo "data usage:"
        du -sh /d/postgres /d/blobs /d/backups 2>/dev/null | sed "s|/d/|  |"
        echo
        echo "latest backups:"
        ls -lh /d/backups/*.dump 2>/dev/null | tail -3 | sed "s|/d/backups/|  |" || echo "  none yet"
    '
}

cmd_logs() {
    require_env
    compose --profile meili logs --tail 200 "$@"
}

health_gate() {
    local deadline=$(( $(date +%s) + HEALTH_TIMEOUT )) svc cid state
    while [ "$(date +%s)" -lt "$deadline" ]; do
        local all_ok=1
        for svc in "${HEALTH_SERVICES[@]}"; do
            cid="$(compose ps -q "$svc" 2>/dev/null || true)"
            [ -n "$cid" ] || { all_ok=0; break; }
            state="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$cid" 2>/dev/null || echo missing)"
            case "$state" in
                healthy|running) ;;
                *) all_ok=0; break ;;
            esac
        done
        [ "$all_ok" -eq 1 ] && return 0
        sleep 5
    done
    return 1
}

cmd_backup() {
    require_env
    compose run --rm backup once
}

cmd_update() {
    require_env
    local target="${1:-${IMAGE_TAG:-local}}"
    local previous="${IMAGE_TAG:-local}"

    echo "== 1/5 backing up the database before anything changes"
    cmd_backup || die "backup failed — refusing to update. Fix the backup first."

    echo "== 2/5 obtaining images for tag '${target}'"
    if [ "${REGISTRY_PREFIX:-webmail}" = "webmail" ] || [ "$target" = "local" ]; then
        IMAGE_TAG="$target" compose build
    else
        IMAGE_TAG="$target" compose pull || die "pull failed — nothing has changed, you are still on ${previous}"
    fi

    echo "== 3/5 applying migrations"
    IMAGE_TAG="$target" compose run --rm migrate \
        || die "migration failed — services untouched, still on ${previous}. Restore from ${DATA_ROOT}/backups if the schema is damaged."

    echo "== 4/5 rolling services"
    # Workers first (leases transfer to the replacement within ~90 s; the lease
    # protocol and idempotent (mailbox_id, uid) upserts make this lossless),
    # then the API, then the web front end.
    IMAGE_TAG="$target" compose up -d worker-sync worker-jobs
    IMAGE_TAG="$target" compose up -d api web
    # The translator is ours too, so it follows the tag. Named here (not left
    # to --no-recreate below) so it is replaced on every update, and so init,
    # which it depends on, runs again when a release adds a data directory.
    IMAGE_TAG="$target" compose up -d translator
    # Converge anything the ordered roll does not name — a release that adds a
    # service (the antispam engine did) would otherwise never start it, and the
    # stack would look updated while missing a container. --no-recreate so this
    # only creates what is absent and leaves the rolled services alone.
    IMAGE_TAG="$target" compose up -d --no-recreate
    # rspamd holds no state of its own (its Bayes data lives in redis-rspamd), and
    # it reads RSPAMD_CONTROLLER_PASSWORD only when its container is created. Left
    # out of the roll it kept whatever password it first started with, so a later
    # change in .env reached the app but not the engine, and every lesson was
    # refused with 401 while mail still got classified. Recreating it is a few
    # seconds of "deliver unclassified", which the filter is built to tolerate.
    IMAGE_TAG="$target" compose up -d --no-deps --force-recreate rspamd

    echo "== 5/5 health gate (${HEALTH_TIMEOUT}s)"
    if health_gate; then
        record_version "$target"
        set_env_value IMAGE_TAG "$target"
        grn "updated to ${target}"
    else
        red "health gate failed — rolling back to ${previous}"
        # shellcheck disable=SC2086
        IMAGE_TAG="$previous" compose up -d $COMPOSE_WAIT || true
        die "update to ${target} failed; production is back on ${previous}"
    fi
}

cmd_rollback() {
    require_env
    local target="${1:-$(cat "$(state_dir)/previous" 2>/dev/null || echo "")}"
    [ -n "$target" ] || die "no previous version recorded; pass a tag: ./deploy/webmail.sh rollback v1.0.0"
    ylw "rolling back to ${target}"
    # shellcheck disable=SC2086
    IMAGE_TAG="$target" compose up -d $COMPOSE_WAIT
    if health_gate; then
        record_version "$target"
        set_env_value IMAGE_TAG "$target"
        grn "rolled back to ${target}"
    else
        die "rollback to ${target} did not come up healthy — check: ./deploy/webmail.sh logs"
    fi
}

cmd_restore() {
    require_env
    local arg="${1:-}" dir base
    [ -n "$arg" ] || die "usage: ./deploy/webmail.sh restore <file.dump>   (run \`status\` to list them)"

    # Backups are chmod 700 owned by uid 999, so a plain `[ -f ]` from an
    # unprivileged shell says "missing" for a file that is right there. Resolve
    # and verify inside a container instead.
    if [ -f "$arg" ]; then
        dir="$(cd "$(dirname "$arg")" && pwd)"; base="$(basename "$arg")"
    else
        dir="${DATA_ROOT}/backups"; base="$(basename "$arg")"
    fi
    if ! docker run --rm -v "${dir}:/b:ro" alpine:3.20 test -f "/b/${base}" 2>/dev/null; then
        red "no such dump: ${base}"
        echo "available in ${DATA_ROOT}/backups:"
        docker run --rm -v "${DATA_ROOT}/backups:/b:ro" alpine:3.20 \
            sh -c 'ls -1 /b/*.dump 2>/dev/null | sed "s|/b/|  |"' || true
        exit 1
    fi
    local dump="${dir}/${base}"

    red "THIS REPLACES THE CURRENT DATABASE with ${base}."
    red "Mail synced since that dump will be gone (it re-syncs from the servers)."
    printf 'Type RESTORE to continue: '
    read -r confirm
    [ "$confirm" = "RESTORE" ] || die "aborted"

    echo "stopping application containers (postgres stays up)"
    compose stop api worker-sync worker-jobs web backup

    echo "restoring $(basename "$dump")"
    # --clean --if-exists drops and recreates objects; a single transaction so a
    # failure leaves the old database intact.
    docker run --rm -i --network webmail-prod \
        -e PGPASSWORD="${POSTGRES_PASSWORD}" \
        -v "$(dirname "$dump")":/restore:ro \
        postgres:16 \
        pg_restore --clean --if-exists --no-owner --single-transaction \
            -h postgres -U "${POSTGRES_USER:-webmail}" -d "${POSTGRES_DB:-webmail}" \
            "/restore/$(basename "$dump")" \
        || die "restore failed; the database was left as it was"

    echo "starting application containers"
    # shellcheck disable=SC2086
    compose up -d $COMPOSE_WAIT
    health_gate || ylw "services are still starting; check: ./deploy/webmail.sh status"
    grn "restored. If the blob store was lost too, mail bodies will refill on next sync."
}

# Print the master key so it can be backed up without opening a hidden dotfile
# in File Station. Intended for a one-off DSM Task Scheduler job:
#   bash deploy/webmail.sh show-key > /volume1/docker/webmail/MASTER_KEY.txt
# copy it into your password manager, then delete that file.
cmd_show_key() {
    require_env
    echo "MASTER_KEY=${MASTER_KEY}"
    echo
    echo "Copy this into your password manager, then DELETE the file you wrote"
    echo "it to. Without this key, saved mail-account passwords cannot be"
    echo "decrypted from any backup."
}

cmd_reindex() { require_env; compose run --rm api reindex "$@"; }

# Optional n-gram data for the spell checker (~13 GB download for en de fr es):
# lets LanguageTool catch real words used wrongly ("loose weight", "das er
# kommt"). Downloaded and unpacked by deploy/ngrams.py inside the backend image
# (it resumes an interrupted download — just run it again), then the engine is
# recreated to load it. Polish has no n-gram data; it is checked as before.
cmd_ngrams() {
    require_env
    local dir="${DATA_ROOT}/languagetool-ngrams"
    mkdir -p "$dir" || die "cannot create ${dir} (run as the owner of DATA_ROOT, or as root)"
    echo "== downloading n-gram data into ${dir} (${*:-en de fr es})"
    compose run --rm --no-deps -T --user 0 -v "${dir}:/ngrams" --entrypoint python api - "$@" \
        < "${SCRIPT_DIR}/ngrams.py" || die "n-gram download failed — run the same command again to resume"
    echo "== restarting the spell checker to load it"
    compose up -d --no-deps --force-recreate languagetool
    grn "n-grams installed. It reads them into memory in the background over the next minutes."
}
cmd_psql()    { require_env; compose exec postgres psql -U "${POSTGRES_USER:-webmail}" -d "${POSTGRES_DB:-webmail}"; }
cmd_shell()   { require_env; compose exec "${1:-api}" /bin/sh; }

# Print the header comment block, however long it grows.
usage() { awk 'NR>1 { if ($0 ~ /^#/) { sub(/^# ?/, ""); print } else exit }' "${BASH_SOURCE[0]}"; }

cmd="${1:-}"; shift || true
SUBCOMMAND="$cmd"
# Before anything touches Docker: on QNAP the binaries are off-PATH (see above).
# doctor/backup/restore call `docker` directly, not only through compose.
[ -n "$cmd" ] && ensure_docker_on_path
case "$cmd" in
    init)     cmd_init "$@" ;;
    doctor)   cmd_doctor "$@" ;;
    pull)     cmd_pull "$@" ;;
    up)       cmd_up "$@" ;;
    down)     cmd_down "$@" ;;
    restart)  cmd_restart "$@" ;;
    status)   cmd_status "$@" ;;
    logs)     cmd_logs "$@" ;;
    update)   cmd_update "$@" ;;
    rollback) cmd_rollback "$@" ;;
    backup)   cmd_backup "$@" ;;
    restore)  cmd_restore "$@" ;;
    reindex)  cmd_reindex "$@" ;;
    ngrams)   cmd_ngrams "$@" ;;
    show-key) cmd_show_key "$@" ;;
    psql)     cmd_psql "$@" ;;
    shell)    cmd_shell "$@" ;;
    ""|-h|--help|help) usage ;;
    *) die "unknown command: $cmd (try --help)" ;;
esac