webmail-client/deploy/webmail.sh

#!/usr/bin/env bash
# Webmail operator control script. One command per thing you might need to do.
#
#   ./deploy/webmail.sh init            create data dirs + a filled-in .env.prod
#   ./deploy/webmail.sh up [--meili]    start (builds images if missing)
#   ./deploy/webmail.sh down            stop (data is untouched)
#   ./deploy/webmail.sh status          what is running, and which version
#   ./deploy/webmail.sh logs [svc] [-f] tail logs
#   ./deploy/webmail.sh update [tag]    backup -> new images -> migrate -> roll -> health-gate
#   ./deploy/webmail.sh rollback [tag]  go back to the previous version
#   ./deploy/webmail.sh backup          take a verified database dump now
#   ./deploy/webmail.sh restore <file>  DESTRUCTIVE: restore a dump
#   ./deploy/webmail.sh reindex [args]  rebuild the search index from Postgres
#   ./deploy/webmail.sh psql            open a database shell
#   ./deploy/webmail.sh shell [svc]     shell inside a container (default: api)
#   ./deploy/webmail.sh doctor          check the install before it bites you
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT_DIR="$(dirname "$SCRIPT_DIR")"
COMPOSE_FILE="${ROOT_DIR}/docker/docker-compose.prod.yml"
ENV_FILE="${ROOT_DIR}/docker/.env.prod"
ENV_EXAMPLE="${ROOT_DIR}/docker/.env.prod.example"

# Services that must come back healthy after a roll.
HEALTH_SERVICES=(postgres redis api web)
HEALTH_TIMEOUT=120

red()  { printf '\033[31m%s\033[0m\n' "$*"; }
grn()  { printf '\033[32m%s\033[0m\n' "$*"; }
ylw()  { printf '\033[33m%s\033[0m\n' "$*"; }
die()  { red "error: $*" >&2; exit 1; }

compose() {
    docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" "$@"
}

require_env() {
    [ -f "$ENV_FILE" ] || die "$ENV_FILE not found. Run: ./deploy/webmail.sh init"
    # shellcheck disable=SC1090
    set -a; . "$ENV_FILE"; set +a
    [ -n "${DATA_ROOT:-}" ]         || die "DATA_ROOT is not set in $ENV_FILE"
    [ -n "${MASTER_KEY:-}" ]        || die "MASTER_KEY is not set in $ENV_FILE"
    [ -n "${POSTGRES_PASSWORD:-}" ] || die "POSTGRES_PASSWORD is not set in $ENV_FILE"
}

state_dir() { echo "${DATA_ROOT}/state"; }

# busybox sed (what you may get on DSM/QTS) has no -i suffix argument, so every
# in-place edit goes through a temp file.
set_env_value() {
    local key="$1" value="$2"
    sed "s|^${key}=.*|${key}=${value}|" "$ENV_FILE" > "${ENV_FILE}.tmp" \
        && mv "${ENV_FILE}.tmp" "$ENV_FILE" && chmod 600 "$ENV_FILE"
}

record_version() {
    local new="$1" cur
    mkdir -p "$(state_dir)" 2>/dev/null || true
    if ! { : > "$(state_dir)/.probe"; } 2>/dev/null; then
        ylw "cannot write ${DATA_ROOT}/state — version history not recorded (run as the owner of DATA_ROOT)"
        return 0
    fi
    rm -f "$(state_dir)/.probe"
    cur="$(cat "$(state_dir)/version" 2>/dev/null || echo "")"
    [ -n "$cur" ] && [ "$cur" != "$new" ] && echo "$cur" > "$(state_dir)/previous"
    echo "$new" > "$(state_dir)/version"
}

# --- commands ----------------------------------------------------------------

cmd_init() {
    if [ ! -f "$ENV_FILE" ]; then
        cp "$ENV_EXAMPLE" "$ENV_FILE"
        chmod 600 "$ENV_FILE"
        local master pgpass
        master="$(openssl rand -base64 32)"
        pgpass="$(openssl rand -base64 24 | tr -d '/+=' | cut -c1-24)"
        # Portable in-place edit (BSD and GNU sed disagree about -i).
        sed "s|^MASTER_KEY=.*|MASTER_KEY=${master}|; s|^POSTGRES_PASSWORD=.*|POSTGRES_PASSWORD=${pgpass}|" \
            "$ENV_FILE" > "${ENV_FILE}.tmp" && mv "${ENV_FILE}.tmp" "$ENV_FILE"
        chmod 600 "$ENV_FILE"
        grn "created $ENV_FILE with freshly generated secrets"
        ylw ""
        ylw "  >> BACK UP MASTER_KEY NOW. Copy it into your password manager. <<"
        ylw "  Without it, saved mail-account passwords cannot be decrypted even"
        ylw "  from a perfect database backup."
        ylw ""
        echo "Then edit $ENV_FILE and set at least: DATA_ROOT, APP_DOMAIN, TZ, WEB_PORT."
        return 0
    fi

    ylw "$ENV_FILE already exists; leaving it alone."
    require_env
    mkdir -p "${DATA_ROOT}"/{postgres,blobs,redis,backups,meili,state} 2>/dev/null \
        || ylw "could not create every directory under ${DATA_ROOT} (the init container will)"
    grn "data directories ready under ${DATA_ROOT}"
}

cmd_doctor() {
    require_env
    local ok=0

    echo "config:"
    echo "  DATA_ROOT      ${DATA_ROOT}"
    echo "  APP_DOMAIN     ${APP_DOMAIN:-<unset>}"
    echo "  WEB_PORT       ${WEB_PORT:-8088}"
    echo "  SEARCH_BACKEND ${SEARCH_BACKEND:-pg}"
    echo "  IMAGE_TAG      ${IMAGE_TAG:-local}"
    echo

    if [ "$(printf '%s' "${MASTER_KEY}" | base64 -d 2>/dev/null | wc -c)" -ne 32 ]; then
        red "  MASTER_KEY is not 32 bytes of base64 — the app will refuse to start"; ok=1
    else
        grn "  MASTER_KEY looks well-formed"
    fi

    if [ "$(stat -c %a "$ENV_FILE" 2>/dev/null || echo 600)" != "600" ]; then
        ylw "  $ENV_FILE is not chmod 600 (it holds your master key)"
    fi

    # Missing directories are not a failure before the first start: `up` and the
    # `init` container create them. Only say so, so a pre-install doctor is green.
    local missing=0
    for d in postgres blobs redis backups state; do
        [ -d "${DATA_ROOT}/${d}" ] || missing=1
    done
    if [ "$missing" -eq 1 ]; then
        ylw "  data directories not created yet — 'up' will create them"
    else
        grn "  data directories present"
    fi

    # Probe the nearest existing ancestor: the NFS/SMB check has to work BEFORE
    # the first install, which is the only time it can still save you.
    local fstype probe="${DATA_ROOT}"
    while [ ! -d "$probe" ] && [ "$probe" != "/" ] && [ -n "$probe" ]; do
        probe="$(dirname "$probe")"
    done
    # `|| true`: with `set -e -o pipefail`, df failing on a not-yet-created
    # DATA_ROOT would abort doctor on exactly the run where you need it most.
    fstype="$(df -PT "$probe" 2>/dev/null | awk 'NR==2{print $2}' || true)"
    [ -n "$fstype" ] || fstype="$(stat -f -c %T "$probe" 2>/dev/null || echo "")"
    case "$fstype" in
        nfs*|cifs|smbfs|fuse*)
            red "  ${DATA_ROOT} is on a network filesystem — Postgres WILL corrupt there."
            red "  Move DATA_ROOT to a local volume (btrfs/ext4)."; ok=1 ;;
        "") ylw "  could not determine the filesystem of ${DATA_ROOT}" ;;
        *)  grn "  ${DATA_ROOT} is on a local filesystem" ;;
    esac

    local avail
    avail="$(df -Pm "$probe" 2>/dev/null | awk 'NR==2{print $4}' || true)"
    [ -n "$avail" ] && {
        if [ "$avail" -lt 5120 ]; then ylw "  only ${avail} MB free on ${DATA_ROOT}"
        else grn "  ${avail} MB free on ${DATA_ROOT}"; fi
    }

    if command -v ss >/dev/null 2>&1 && ss -ltn 2>/dev/null | grep -q ":${WEB_PORT:-8088} "; then
        ylw "  port ${WEB_PORT:-8088} already has a listener — change WEB_PORT if that is not us"
    fi

    # chmod 700 and owned by uid 999, so count them as root in a container.
    local recent
    recent="$(docker run --rm -v "${DATA_ROOT}/backups:/b:ro" alpine:3.20 \
        sh -c 'find /b -maxdepth 1 -name "*.dump" -mtime -2 2>/dev/null | wc -l' 2>/dev/null || echo 0)"
    if [ "${recent:-0}" -gt 0 ]; then grn "  ${recent} database backup(s) from the last 48 h"
    else ylw "  no database backup in the last 48 h"; fi

    [ "$ok" -eq 0 ] && grn "doctor: no blocking problems" || red "doctor: problems found (above)"
    return "$ok"
}

cmd_up() {
    require_env
    local profile=() build=()
    for arg in "$@"; do
        case "$arg" in
            --meili) profile=(--profile meili) ;;
            --build) build=(--build) ;;
            *) die "unknown option: $arg" ;;
        esac
    done
    [ "${SEARCH_BACKEND:-pg}" = "meili" ] && [ ${#profile[@]} -eq 0 ] && {
        ylw "SEARCH_BACKEND=meili — starting the Meilisearch container too"
        profile=(--profile meili)
    }
    # Best-effort only: Docker creates any missing bind-mount path itself, and
    # the `init` service fixes ownership. This line must not be fatal, because
    # DATA_ROOT is typically root-owned while you may be running as yourself.
    mkdir -p "${DATA_ROOT}"/{postgres,blobs,redis,backups,meili,state} 2>/dev/null || true
    compose "${profile[@]}" up -d "${build[@]}" --wait
    record_version "${IMAGE_TAG:-local}"
    grn "up. open http://<this-host>:${WEB_PORT:-8088}/"
}

cmd_down()    { require_env; compose --profile meili down; grn "stopped (data kept in ${DATA_ROOT})"; }
cmd_restart() { require_env; compose restart "${@:-}"; }

cmd_status() {
    require_env
    compose --profile meili ps
    echo
    echo "recorded version : $(cat "$(state_dir)/version" 2>/dev/null || echo unknown)"
    echo "previous version : $(cat "$(state_dir)/previous" 2>/dev/null || echo none)"
    local running
    running="$(compose exec -T api python -c \
        "import urllib.request,json;print(json.load(urllib.request.urlopen('http://127.0.0.1:8000/api/version'))['version'])" \
        2>/dev/null || echo unreachable)"
    echo "running version  : ${running}"
    echo
    # The data directories are owned by the container users (postgres 999,
    # backend 10001) and backups are chmod 700, so read them as root inside a
    # throwaway container rather than reporting misleading zeroes.
    docker run --rm -v "${DATA_ROOT}:/d:ro" alpine:3.20 sh -c '
        echo "data usage:"
        du -sh /d/postgres /d/blobs /d/backups 2>/dev/null | sed "s|/d/|  |"
        echo
        echo "latest backups:"
        ls -lh /d/backups/*.dump 2>/dev/null | tail -3 | sed "s|/d/backups/|  |" || echo "  none yet"
    '
}

cmd_logs() {
    require_env
    compose --profile meili logs --tail 200 "$@"
}

health_gate() {
    local deadline=$(( $(date +%s) + HEALTH_TIMEOUT )) svc cid state
    while [ "$(date +%s)" -lt "$deadline" ]; do
        local all_ok=1
        for svc in "${HEALTH_SERVICES[@]}"; do
            cid="$(compose ps -q "$svc" 2>/dev/null || true)"
            [ -n "$cid" ] || { all_ok=0; break; }
            state="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$cid" 2>/dev/null || echo missing)"
            case "$state" in
                healthy|running) ;;
                *) all_ok=0; break ;;
            esac
        done
        [ "$all_ok" -eq 1 ] && return 0
        sleep 5
    done
    return 1
}

cmd_backup() {
    require_env
    compose run --rm backup once
}

cmd_update() {
    require_env
    local target="${1:-${IMAGE_TAG:-local}}"
    local previous="${IMAGE_TAG:-local}"

    echo "== 1/5 backing up the database before anything changes"
    cmd_backup || die "backup failed — refusing to update. Fix the backup first."

    echo "== 2/5 obtaining images for tag '${target}'"
    if [ "${REGISTRY_PREFIX:-webmail}" = "webmail" ] || [ "$target" = "local" ]; then
        IMAGE_TAG="$target" compose build
    else
        IMAGE_TAG="$target" compose pull || die "pull failed — nothing has changed, you are still on ${previous}"
    fi

    echo "== 3/5 applying migrations"
    IMAGE_TAG="$target" compose run --rm migrate \
        || die "migration failed — services untouched, still on ${previous}. Restore from ${DATA_ROOT}/backups if the schema is damaged."

    echo "== 4/5 rolling services"
    # Workers first (leases transfer to the replacement within ~90 s; the lease
    # protocol and idempotent (mailbox_id, uid) upserts make this lossless),
    # then the API, then the web front end.
    IMAGE_TAG="$target" compose up -d worker-sync worker-jobs
    IMAGE_TAG="$target" compose up -d api web

    echo "== 5/5 health gate (${HEALTH_TIMEOUT}s)"
    if health_gate; then
        record_version "$target"
        set_env_value IMAGE_TAG "$target"
        grn "updated to ${target}"
    else
        red "health gate failed — rolling back to ${previous}"
        IMAGE_TAG="$previous" compose up -d --wait || true
        die "update to ${target} failed; production is back on ${previous}"
    fi
}

cmd_rollback() {
    require_env
    local target="${1:-$(cat "$(state_dir)/previous" 2>/dev/null || echo "")}"
    [ -n "$target" ] || die "no previous version recorded; pass a tag: ./deploy/webmail.sh rollback v1.0.0"
    ylw "rolling back to ${target}"
    IMAGE_TAG="$target" compose up -d --wait
    if health_gate; then
        record_version "$target"
        set_env_value IMAGE_TAG "$target"
        grn "rolled back to ${target}"
    else
        die "rollback to ${target} did not come up healthy — check: ./deploy/webmail.sh logs"
    fi
}

cmd_restore() {
    require_env
    local arg="${1:-}" dir base
    [ -n "$arg" ] || die "usage: ./deploy/webmail.sh restore <file.dump>   (run \`status\` to list them)"

    # Backups are chmod 700 owned by uid 999, so a plain `[ -f ]` from an
    # unprivileged shell says "missing" for a file that is right there. Resolve
    # and verify inside a container instead.
    if [ -f "$arg" ]; then
        dir="$(cd "$(dirname "$arg")" && pwd)"; base="$(basename "$arg")"
    else
        dir="${DATA_ROOT}/backups"; base="$(basename "$arg")"
    fi
    if ! docker run --rm -v "${dir}:/b:ro" alpine:3.20 test -f "/b/${base}" 2>/dev/null; then
        red "no such dump: ${base}"
        echo "available in ${DATA_ROOT}/backups:"
        docker run --rm -v "${DATA_ROOT}/backups:/b:ro" alpine:3.20 \
            sh -c 'ls -1 /b/*.dump 2>/dev/null | sed "s|/b/|  |"' || true
        exit 1
    fi
    local dump="${dir}/${base}"

    red "THIS REPLACES THE CURRENT DATABASE with ${base}."
    red "Mail synced since that dump will be gone (it re-syncs from the servers)."
    printf 'Type RESTORE to continue: '
    read -r confirm
    [ "$confirm" = "RESTORE" ] || die "aborted"

    echo "stopping application containers (postgres stays up)"
    compose stop api worker-sync worker-jobs web backup

    echo "restoring $(basename "$dump")"
    # --clean --if-exists drops and recreates objects; a single transaction so a
    # failure leaves the old database intact.
    docker run --rm -i --network webmail-prod \
        -e PGPASSWORD="${POSTGRES_PASSWORD}" \
        -v "$(dirname "$dump")":/restore:ro \
        postgres:16 \
        pg_restore --clean --if-exists --no-owner --single-transaction \
            -h postgres -U "${POSTGRES_USER:-webmail}" -d "${POSTGRES_DB:-webmail}" \
            "/restore/$(basename "$dump")" \
        || die "restore failed; the database was left as it was"

    echo "starting application containers"
    compose up -d --wait
    grn "restored. If the blob store was lost too, mail bodies will refill on next sync."
}

cmd_reindex() { require_env; compose run --rm api reindex "$@"; }
cmd_psql()    { require_env; compose exec postgres psql -U "${POSTGRES_USER:-webmail}" -d "${POSTGRES_DB:-webmail}"; }
cmd_shell()   { require_env; compose exec "${1:-api}" /bin/sh; }

usage() { sed -n '2,20p' "${BASH_SOURCE[0]}" | sed 's|^# \?||'; }

cmd="${1:-}"; shift || true
case "$cmd" in
    init)     cmd_init "$@" ;;
    doctor)   cmd_doctor "$@" ;;
    up)       cmd_up "$@" ;;
    down)     cmd_down "$@" ;;
    restart)  cmd_restart "$@" ;;
    status)   cmd_status "$@" ;;
    logs)     cmd_logs "$@" ;;
    update)   cmd_update "$@" ;;
    rollback) cmd_rollback "$@" ;;
    backup)   cmd_backup "$@" ;;
    restore)  cmd_restore "$@" ;;
    reindex)  cmd_reindex "$@" ;;
    psql)     cmd_psql "$@" ;;
    shell)    cmd_shell "$@" ;;
    ""|-h|--help|help) usage ;;
    *) die "unknown command: $cmd (try --help)" ;;
esac