#!/usr/bin/env bash # Webmail operator control script. One command per thing you might need to do. # # ./deploy/webmail.sh init create data dirs + a filled-in .env.prod # ./deploy/webmail.sh up [--meili] start (builds images if missing) # ./deploy/webmail.sh down stop (data is untouched) # ./deploy/webmail.sh status what is running, and which version # ./deploy/webmail.sh logs [svc] [-f] tail logs # ./deploy/webmail.sh update [tag] backup -> new images -> migrate -> roll -> health-gate # ./deploy/webmail.sh rollback [tag] go back to the previous version # ./deploy/webmail.sh backup take a verified database dump now # ./deploy/webmail.sh restore DESTRUCTIVE: restore a dump # ./deploy/webmail.sh reindex [args] rebuild the search index from Postgres # ./deploy/webmail.sh psql open a database shell # ./deploy/webmail.sh shell [svc] shell inside a container (default: api) # ./deploy/webmail.sh doctor check the install before it bites you set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT_DIR="$(dirname "$SCRIPT_DIR")" COMPOSE_FILE="${ROOT_DIR}/docker/docker-compose.prod.yml" ENV_FILE="${ROOT_DIR}/docker/.env.prod" ENV_EXAMPLE="${ROOT_DIR}/docker/.env.prod.example" # Services that must come back healthy after a roll. HEALTH_SERVICES=(postgres redis api web) HEALTH_TIMEOUT=120 red() { printf '\033[31m%s\033[0m\n' "$*"; } grn() { printf '\033[32m%s\033[0m\n' "$*"; } ylw() { printf '\033[33m%s\033[0m\n' "$*"; } die() { red "error: $*" >&2; exit 1; } compose() { docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" "$@" } require_env() { [ -f "$ENV_FILE" ] || die "$ENV_FILE not found. Run: ./deploy/webmail.sh init" # shellcheck disable=SC1090 set -a; . "$ENV_FILE"; set +a [ -n "${DATA_ROOT:-}" ] || die "DATA_ROOT is not set in $ENV_FILE" [ -n "${MASTER_KEY:-}" ] || die "MASTER_KEY is not set in $ENV_FILE" [ -n "${POSTGRES_PASSWORD:-}" ] || die "POSTGRES_PASSWORD is not set in $ENV_FILE" } state_dir() { echo "${DATA_ROOT}/state"; } # busybox sed (what you may get on DSM/QTS) has no -i suffix argument, so every # in-place edit goes through a temp file. set_env_value() { local key="$1" value="$2" sed "s|^${key}=.*|${key}=${value}|" "$ENV_FILE" > "${ENV_FILE}.tmp" \ && mv "${ENV_FILE}.tmp" "$ENV_FILE" && chmod 600 "$ENV_FILE" } record_version() { local new="$1" cur mkdir -p "$(state_dir)" 2>/dev/null || true if ! { : > "$(state_dir)/.probe"; } 2>/dev/null; then ylw "cannot write ${DATA_ROOT}/state — version history not recorded (run as the owner of DATA_ROOT)" return 0 fi rm -f "$(state_dir)/.probe" cur="$(cat "$(state_dir)/version" 2>/dev/null || echo "")" [ -n "$cur" ] && [ "$cur" != "$new" ] && echo "$cur" > "$(state_dir)/previous" echo "$new" > "$(state_dir)/version" } # --- commands ---------------------------------------------------------------- cmd_init() { if [ ! -f "$ENV_FILE" ]; then cp "$ENV_EXAMPLE" "$ENV_FILE" chmod 600 "$ENV_FILE" local master pgpass master="$(openssl rand -base64 32)" pgpass="$(openssl rand -base64 24 | tr -d '/+=' | cut -c1-24)" # Portable in-place edit (BSD and GNU sed disagree about -i). sed "s|^MASTER_KEY=.*|MASTER_KEY=${master}|; s|^POSTGRES_PASSWORD=.*|POSTGRES_PASSWORD=${pgpass}|" \ "$ENV_FILE" > "${ENV_FILE}.tmp" && mv "${ENV_FILE}.tmp" "$ENV_FILE" chmod 600 "$ENV_FILE" grn "created $ENV_FILE with freshly generated secrets" ylw "" ylw " >> BACK UP MASTER_KEY NOW. Copy it into your password manager. <<" ylw " Without it, saved mail-account passwords cannot be decrypted even" ylw " from a perfect database backup." ylw "" echo "Then edit $ENV_FILE and set at least: DATA_ROOT, APP_DOMAIN, TZ, WEB_PORT." return 0 fi ylw "$ENV_FILE already exists; leaving it alone." require_env mkdir -p "${DATA_ROOT}"/{postgres,blobs,redis,backups,meili,state} 2>/dev/null \ || ylw "could not create every directory under ${DATA_ROOT} (the init container will)" grn "data directories ready under ${DATA_ROOT}" } cmd_doctor() { require_env local ok=0 echo "config:" echo " DATA_ROOT ${DATA_ROOT}" echo " APP_DOMAIN ${APP_DOMAIN:-}" echo " WEB_PORT ${WEB_PORT:-8088}" echo " SEARCH_BACKEND ${SEARCH_BACKEND:-pg}" echo " IMAGE_TAG ${IMAGE_TAG:-local}" echo if [ "$(printf '%s' "${MASTER_KEY}" | base64 -d 2>/dev/null | wc -c)" -ne 32 ]; then red " MASTER_KEY is not 32 bytes of base64 — the app will refuse to start"; ok=1 else grn " MASTER_KEY looks well-formed" fi if [ "$(stat -c %a "$ENV_FILE" 2>/dev/null || echo 600)" != "600" ]; then ylw " $ENV_FILE is not chmod 600 (it holds your master key)" fi # Missing directories are not a failure before the first start: `up` and the # `init` container create them. Only say so, so a pre-install doctor is green. local missing=0 for d in postgres blobs redis backups state; do [ -d "${DATA_ROOT}/${d}" ] || missing=1 done if [ "$missing" -eq 1 ]; then ylw " data directories not created yet — 'up' will create them" else grn " data directories present" fi # Probe the nearest existing ancestor: the NFS/SMB check has to work BEFORE # the first install, which is the only time it can still save you. local fstype probe="${DATA_ROOT}" while [ ! -d "$probe" ] && [ "$probe" != "/" ] && [ -n "$probe" ]; do probe="$(dirname "$probe")" done # `|| true`: with `set -e -o pipefail`, df failing on a not-yet-created # DATA_ROOT would abort doctor on exactly the run where you need it most. fstype="$(df -PT "$probe" 2>/dev/null | awk 'NR==2{print $2}' || true)" [ -n "$fstype" ] || fstype="$(stat -f -c %T "$probe" 2>/dev/null || echo "")" case "$fstype" in nfs*|cifs|smbfs|fuse*) red " ${DATA_ROOT} is on a network filesystem — Postgres WILL corrupt there." red " Move DATA_ROOT to a local volume (btrfs/ext4)."; ok=1 ;; "") ylw " could not determine the filesystem of ${DATA_ROOT}" ;; *) grn " ${DATA_ROOT} is on a local filesystem" ;; esac local avail avail="$(df -Pm "$probe" 2>/dev/null | awk 'NR==2{print $4}' || true)" [ -n "$avail" ] && { if [ "$avail" -lt 5120 ]; then ylw " only ${avail} MB free on ${DATA_ROOT}" else grn " ${avail} MB free on ${DATA_ROOT}"; fi } if command -v ss >/dev/null 2>&1 && ss -ltn 2>/dev/null | grep -q ":${WEB_PORT:-8088} "; then ylw " port ${WEB_PORT:-8088} already has a listener — change WEB_PORT if that is not us" fi # chmod 700 and owned by uid 999, so count them as root in a container. local recent recent="$(docker run --rm -v "${DATA_ROOT}/backups:/b:ro" alpine:3.20 \ sh -c 'find /b -maxdepth 1 -name "*.dump" -mtime -2 2>/dev/null | wc -l' 2>/dev/null || echo 0)" if [ "${recent:-0}" -gt 0 ]; then grn " ${recent} database backup(s) from the last 48 h" else ylw " no database backup in the last 48 h"; fi [ "$ok" -eq 0 ] && grn "doctor: no blocking problems" || red "doctor: problems found (above)" return "$ok" } cmd_up() { require_env local profile=() build=() for arg in "$@"; do case "$arg" in --meili) profile=(--profile meili) ;; --build) build=(--build) ;; *) die "unknown option: $arg" ;; esac done [ "${SEARCH_BACKEND:-pg}" = "meili" ] && [ ${#profile[@]} -eq 0 ] && { ylw "SEARCH_BACKEND=meili — starting the Meilisearch container too" profile=(--profile meili) } # Best-effort only: Docker creates any missing bind-mount path itself, and # the `init` service fixes ownership. This line must not be fatal, because # DATA_ROOT is typically root-owned while you may be running as yourself. mkdir -p "${DATA_ROOT}"/{postgres,blobs,redis,backups,meili,state} 2>/dev/null || true compose "${profile[@]}" up -d "${build[@]}" --wait record_version "${IMAGE_TAG:-local}" grn "up. open http://:${WEB_PORT:-8088}/" } cmd_down() { require_env; compose --profile meili down; grn "stopped (data kept in ${DATA_ROOT})"; } cmd_restart() { require_env; compose restart "${@:-}"; } cmd_status() { require_env compose --profile meili ps echo echo "recorded version : $(cat "$(state_dir)/version" 2>/dev/null || echo unknown)" echo "previous version : $(cat "$(state_dir)/previous" 2>/dev/null || echo none)" local running running="$(compose exec -T api python -c \ "import urllib.request,json;print(json.load(urllib.request.urlopen('http://127.0.0.1:8000/api/version'))['version'])" \ 2>/dev/null || echo unreachable)" echo "running version : ${running}" echo # The data directories are owned by the container users (postgres 999, # backend 10001) and backups are chmod 700, so read them as root inside a # throwaway container rather than reporting misleading zeroes. docker run --rm -v "${DATA_ROOT}:/d:ro" alpine:3.20 sh -c ' echo "data usage:" du -sh /d/postgres /d/blobs /d/backups 2>/dev/null | sed "s|/d/| |" echo echo "latest backups:" ls -lh /d/backups/*.dump 2>/dev/null | tail -3 | sed "s|/d/backups/| |" || echo " none yet" ' } cmd_logs() { require_env compose --profile meili logs --tail 200 "$@" } health_gate() { local deadline=$(( $(date +%s) + HEALTH_TIMEOUT )) svc cid state while [ "$(date +%s)" -lt "$deadline" ]; do local all_ok=1 for svc in "${HEALTH_SERVICES[@]}"; do cid="$(compose ps -q "$svc" 2>/dev/null || true)" [ -n "$cid" ] || { all_ok=0; break; } state="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$cid" 2>/dev/null || echo missing)" case "$state" in healthy|running) ;; *) all_ok=0; break ;; esac done [ "$all_ok" -eq 1 ] && return 0 sleep 5 done return 1 } cmd_backup() { require_env compose run --rm backup once } cmd_update() { require_env local target="${1:-${IMAGE_TAG:-local}}" local previous="${IMAGE_TAG:-local}" echo "== 1/5 backing up the database before anything changes" cmd_backup || die "backup failed — refusing to update. Fix the backup first." echo "== 2/5 obtaining images for tag '${target}'" if [ "${REGISTRY_PREFIX:-webmail}" = "webmail" ] || [ "$target" = "local" ]; then IMAGE_TAG="$target" compose build else IMAGE_TAG="$target" compose pull || die "pull failed — nothing has changed, you are still on ${previous}" fi echo "== 3/5 applying migrations" IMAGE_TAG="$target" compose run --rm migrate \ || die "migration failed — services untouched, still on ${previous}. Restore from ${DATA_ROOT}/backups if the schema is damaged." echo "== 4/5 rolling services" # Workers first (leases transfer to the replacement within ~90 s; the lease # protocol and idempotent (mailbox_id, uid) upserts make this lossless), # then the API, then the web front end. IMAGE_TAG="$target" compose up -d worker-sync worker-jobs IMAGE_TAG="$target" compose up -d api web echo "== 5/5 health gate (${HEALTH_TIMEOUT}s)" if health_gate; then record_version "$target" set_env_value IMAGE_TAG "$target" grn "updated to ${target}" else red "health gate failed — rolling back to ${previous}" IMAGE_TAG="$previous" compose up -d --wait || true die "update to ${target} failed; production is back on ${previous}" fi } cmd_rollback() { require_env local target="${1:-$(cat "$(state_dir)/previous" 2>/dev/null || echo "")}" [ -n "$target" ] || die "no previous version recorded; pass a tag: ./deploy/webmail.sh rollback v1.0.0" ylw "rolling back to ${target}" IMAGE_TAG="$target" compose up -d --wait if health_gate; then record_version "$target" set_env_value IMAGE_TAG "$target" grn "rolled back to ${target}" else die "rollback to ${target} did not come up healthy — check: ./deploy/webmail.sh logs" fi } cmd_restore() { require_env local arg="${1:-}" dir base [ -n "$arg" ] || die "usage: ./deploy/webmail.sh restore (run \`status\` to list them)" # Backups are chmod 700 owned by uid 999, so a plain `[ -f ]` from an # unprivileged shell says "missing" for a file that is right there. Resolve # and verify inside a container instead. if [ -f "$arg" ]; then dir="$(cd "$(dirname "$arg")" && pwd)"; base="$(basename "$arg")" else dir="${DATA_ROOT}/backups"; base="$(basename "$arg")" fi if ! docker run --rm -v "${dir}:/b:ro" alpine:3.20 test -f "/b/${base}" 2>/dev/null; then red "no such dump: ${base}" echo "available in ${DATA_ROOT}/backups:" docker run --rm -v "${DATA_ROOT}/backups:/b:ro" alpine:3.20 \ sh -c 'ls -1 /b/*.dump 2>/dev/null | sed "s|/b/| |"' || true exit 1 fi local dump="${dir}/${base}" red "THIS REPLACES THE CURRENT DATABASE with ${base}." red "Mail synced since that dump will be gone (it re-syncs from the servers)." printf 'Type RESTORE to continue: ' read -r confirm [ "$confirm" = "RESTORE" ] || die "aborted" echo "stopping application containers (postgres stays up)" compose stop api worker-sync worker-jobs web backup echo "restoring $(basename "$dump")" # --clean --if-exists drops and recreates objects; a single transaction so a # failure leaves the old database intact. docker run --rm -i --network webmail-prod \ -e PGPASSWORD="${POSTGRES_PASSWORD}" \ -v "$(dirname "$dump")":/restore:ro \ postgres:16 \ pg_restore --clean --if-exists --no-owner --single-transaction \ -h postgres -U "${POSTGRES_USER:-webmail}" -d "${POSTGRES_DB:-webmail}" \ "/restore/$(basename "$dump")" \ || die "restore failed; the database was left as it was" echo "starting application containers" compose up -d --wait grn "restored. If the blob store was lost too, mail bodies will refill on next sync." } cmd_reindex() { require_env; compose run --rm api reindex "$@"; } cmd_psql() { require_env; compose exec postgres psql -U "${POSTGRES_USER:-webmail}" -d "${POSTGRES_DB:-webmail}"; } cmd_shell() { require_env; compose exec "${1:-api}" /bin/sh; } usage() { sed -n '2,20p' "${BASH_SOURCE[0]}" | sed 's|^# \?||'; } cmd="${1:-}"; shift || true case "$cmd" in init) cmd_init "$@" ;; doctor) cmd_doctor "$@" ;; up) cmd_up "$@" ;; down) cmd_down "$@" ;; restart) cmd_restart "$@" ;; status) cmd_status "$@" ;; logs) cmd_logs "$@" ;; update) cmd_update "$@" ;; rollback) cmd_rollback "$@" ;; backup) cmd_backup "$@" ;; restore) cmd_restore "$@" ;; reindex) cmd_reindex "$@" ;; psql) cmd_psql "$@" ;; shell) cmd_shell "$@" ;; ""|-h|--help|help) usage ;; *) die "unknown command: $cmd (try --help)" ;; esac