Compare commits
4
Commits
a27755a4d3
...
c472404486
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c472404486 | ||
|
|
07fd0a7adf | ||
|
|
cc26ae7eaf | ||
|
|
5553b5febe |
@@ -0,0 +1,30 @@
|
||||
# Build context hygiene: the backend image builds from the repo root (it needs
|
||||
# backend/ *and* plugins/), so everything heavy or secret is excluded here.
|
||||
.git
|
||||
.github
|
||||
.gitea
|
||||
.claude
|
||||
**/node_modules
|
||||
**/dist
|
||||
**/__pycache__
|
||||
**/*.py[cod]
|
||||
**/.mypy_cache
|
||||
**/.ruff_cache
|
||||
**/.pytest_cache
|
||||
**/*.egg-info
|
||||
# Runtime data: mail content and local blobs must never enter an image layer.
|
||||
var/
|
||||
backend/var/
|
||||
**/var/blobs
|
||||
# Secrets
|
||||
.env.dev
|
||||
.env.*.local
|
||||
gitea-server-credentials.md
|
||||
docker/.env.prod
|
||||
deploy/*.env
|
||||
# Docs / screenshots / scratch
|
||||
screenshots/
|
||||
docs/
|
||||
*.md
|
||||
!README.md
|
||||
claude-session.txt
|
||||
@@ -1,5 +1,8 @@
|
||||
# Secrets / local env
|
||||
.env.dev
|
||||
docker/.env.prod
|
||||
docker/.env
|
||||
/.env
|
||||
.env.*.local
|
||||
gitea-server-credentials.md
|
||||
|
||||
|
||||
@@ -21,3 +21,22 @@ test:
|
||||
cd backend && pytest -q && cd ../frontend && npx vitest run
|
||||
mail-test: ## start throwaway IMAP/SMTP server
|
||||
docker compose -f docker/docker-compose.yml up -d greenmail
|
||||
|
||||
# --- production (NAS / single-host Docker deploy; see docs/nas-install.md) ---
|
||||
prod-init: ## generate docker/.env.prod + data dirs
|
||||
./deploy/webmail.sh init
|
||||
prod-up: ## start the production stack
|
||||
./deploy/webmail.sh up
|
||||
prod-down: ## stop the production stack (data kept)
|
||||
./deploy/webmail.sh down
|
||||
prod-status: ## what is running, versions, disk, backups
|
||||
./deploy/webmail.sh status
|
||||
prod-update: ## backup -> migrate -> roll -> health-gate (auto-rollback on failure)
|
||||
./deploy/webmail.sh update
|
||||
prod-backup: ## take a verified database dump now
|
||||
./deploy/webmail.sh backup
|
||||
prod-doctor: ## pre-flight checks on a deployment
|
||||
./deploy/webmail.sh doctor
|
||||
prod-images: ## build production images locally (no push)
|
||||
docker build -f docker/Dockerfile.backend -t webmail/backend:local .
|
||||
docker build -f docker/Dockerfile.frontend -t webmail/web:local .
|
||||
|
||||
@@ -32,6 +32,9 @@ class Settings(BaseSettings):
|
||||
session_ttl_days: int = Field(default=30, alias="SESSION_TTL_DAYS")
|
||||
max_attachment_mb: int = Field(default=25, alias="MAX_ATTACHMENT_MB")
|
||||
log_level: str = Field(default="INFO", alias="LOG_LEVEL")
|
||||
# Release identifier baked into the container image at build time; surfaced
|
||||
# by GET /api/version so a deploy can verify what is actually running.
|
||||
app_version: str = Field(default="dev", alias="APP_VERSION")
|
||||
|
||||
# --- Sync engine v2 (workers layer) --------------------------------------
|
||||
# Worker role: "sync" (lease-owned account runners) | "jobs" (arq consumer)
|
||||
|
||||
@@ -45,6 +45,11 @@ def create_app() -> FastAPI:
|
||||
await get_redis().ping()
|
||||
return {"status": "ok"}
|
||||
|
||||
@app.get("/api/version")
|
||||
async def version() -> dict[str, str]:
|
||||
"""Report the running release, for deploy/rollback verification."""
|
||||
return {"version": get_settings().app_version}
|
||||
|
||||
from app.api import register_routers
|
||||
|
||||
register_routers(app)
|
||||
|
||||
@@ -52,6 +52,13 @@ dev = [
|
||||
where = ["."]
|
||||
include = ["app*", "workers*"]
|
||||
|
||||
# Non-Python files that a *non-editable* install (the Docker image) must carry:
|
||||
# the account presets config (SPEC §6.2) and the shared Alembic env used by the
|
||||
# per-plugin migration runner (app/plugins/migrate.py). Without these, a
|
||||
# `pip install ./backend` produces a package that fails at runtime.
|
||||
[tool.setuptools.package-data]
|
||||
app = ["accounts/presets.yaml", "plugins/alembic/*"]
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 100
|
||||
target-version = "py311"
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Rebuild the search index for every message, from Postgres alone.
|
||||
|
||||
Both backends index the same :class:`SearchDoc` (SPEC §5.2), and every field of
|
||||
it is already stored in the database — subject/addresses/flags on ``messages``,
|
||||
the plain-text body on ``body_parts``, filenames on ``attachments``. So the
|
||||
index is fully derivable and never needs its own backup: if the Meilisearch
|
||||
volume is lost, or you switch SEARCH_BACKEND, run this.
|
||||
|
||||
python scripts/reindex_search.py # everything
|
||||
python scripts/reindex_search.py --user <uuid> # one user
|
||||
python scripts/reindex_search.py --purge # drop stale docs first
|
||||
|
||||
In Docker: ./deploy/webmail.sh reindex [--purge]
|
||||
|
||||
``--purge`` clears each user's existing documents before rebuilding, which is
|
||||
what you want after deleting mail outside the app or when the index has drifted;
|
||||
without it the rebuild is a pure upsert (safe to re-run, safe to interrupt).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import uuid
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.db import get_sessionmaker
|
||||
from app.models import Attachment, BodyPart, Mailbox, Message
|
||||
from app.search.base import SearchBackend, SearchDoc
|
||||
from app.search.factory import get_search_backend
|
||||
|
||||
# Rows pulled per round-trip, and how many documents are pushed to the backend
|
||||
# concurrently. Deliberately modest: this runs on a NAS alongside live sync.
|
||||
BATCH = 500
|
||||
CONCURRENCY = 8
|
||||
|
||||
|
||||
def _epoch(value: datetime) -> int:
|
||||
return int(value.timestamp())
|
||||
|
||||
|
||||
async def _index_batch(backend: SearchBackend, docs: list[SearchDoc]) -> None:
|
||||
sem = asyncio.Semaphore(CONCURRENCY)
|
||||
|
||||
async def one(doc: SearchDoc) -> None:
|
||||
async with sem:
|
||||
await backend.index(doc)
|
||||
|
||||
await asyncio.gather(*(one(d) for d in docs))
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Rebuild the search index from Postgres")
|
||||
parser.add_argument("--user", help="only this user_id (UUID)")
|
||||
parser.add_argument(
|
||||
"--purge",
|
||||
action="store_true",
|
||||
help="delete each user's existing documents before rebuilding",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
only_user = uuid.UUID(args.user) if args.user else None
|
||||
backend = get_search_backend()
|
||||
sm = get_sessionmaker()
|
||||
|
||||
if args.purge:
|
||||
async with sm() as session:
|
||||
users = (
|
||||
[only_user]
|
||||
if only_user
|
||||
else list(await session.scalars(select(Message.user_id).distinct()))
|
||||
)
|
||||
for user_id in users:
|
||||
await backend.delete_user(user_id)
|
||||
print(f"purged existing documents for {len(users)} user(s)")
|
||||
|
||||
total = 0
|
||||
after: uuid.UUID | None = None
|
||||
while True:
|
||||
async with sm() as session:
|
||||
stmt = (
|
||||
select(Message, Mailbox.special_use)
|
||||
.join(Mailbox, Mailbox.id == Message.mailbox_id)
|
||||
.order_by(Message.id)
|
||||
.limit(BATCH)
|
||||
)
|
||||
if after is not None:
|
||||
stmt = stmt.where(Message.id > after)
|
||||
if only_user is not None:
|
||||
stmt = stmt.where(Message.user_id == only_user)
|
||||
rows = list(await session.execute(stmt))
|
||||
if not rows:
|
||||
break
|
||||
|
||||
ids = [row[0].id for row in rows]
|
||||
after = ids[-1]
|
||||
|
||||
# Two batched lookups instead of 2N per-message queries.
|
||||
bodies: dict[uuid.UUID, str] = {}
|
||||
for message_id, content in await session.execute(
|
||||
select(BodyPart.message_id, BodyPart.content).where(
|
||||
BodyPart.message_id.in_(ids), BodyPart.kind == "text"
|
||||
)
|
||||
):
|
||||
bodies.setdefault(message_id, content)
|
||||
|
||||
names: dict[uuid.UUID, list[str]] = defaultdict(list)
|
||||
for message_id, filename in await session.execute(
|
||||
select(Attachment.message_id, Attachment.filename).where(
|
||||
Attachment.message_id.in_(ids),
|
||||
Attachment.content_id.is_(None), # real attachments, not inline images
|
||||
)
|
||||
):
|
||||
names[message_id].append(filename)
|
||||
|
||||
docs = [
|
||||
SearchDoc(
|
||||
message_id=msg.id,
|
||||
thread_id=msg.thread_id or msg.id,
|
||||
user_id=msg.user_id,
|
||||
account_id=msg.account_id,
|
||||
mailbox_special_use=special_use,
|
||||
from_=msg.from_addr,
|
||||
to=msg.to_addr,
|
||||
cc=msg.cc_addr,
|
||||
subject=msg.subject,
|
||||
body_text=bodies.get(msg.id, ""),
|
||||
attachment_names=names.get(msg.id, []),
|
||||
has_attachments=msg.has_attachments,
|
||||
is_unread="\\Seen" not in msg.flags,
|
||||
is_starred="\\Flagged" in msg.flags,
|
||||
date=_epoch(msg.internal_date),
|
||||
size_bytes=msg.size_bytes,
|
||||
)
|
||||
for msg, special_use in rows
|
||||
]
|
||||
|
||||
await _index_batch(backend, docs)
|
||||
total += len(docs)
|
||||
print(f"indexed {total} messages", flush=True)
|
||||
|
||||
print(f"done: {total} messages reindexed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Executable
+77
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build the two production images on THIS machine and either push them to a
|
||||
# registry or export them as a file you can copy to the NAS.
|
||||
#
|
||||
# Why you'd use this instead of building on the NAS: the frontend build (Vite +
|
||||
# tsc) wants ~2 GB of RAM and a lot of CPU. On a 2-4 GB NAS that is slow at best
|
||||
# and an OOM at worst, and it competes with everything else the box is doing.
|
||||
#
|
||||
# ./deploy/build-push.sh v1.0.0 # build + push to $REGISTRY_PREFIX
|
||||
# REGISTRY_PREFIX=192.168.1.185:8418/claude/webmail \
|
||||
# ./deploy/build-push.sh v1.0.0
|
||||
# ./deploy/build-push.sh v1.0.0 --save # build + write a .tar to load on the NAS
|
||||
#
|
||||
# On the NAS afterwards:
|
||||
# push mode: set REGISTRY_PREFIX and IMAGE_TAG in docker/.env.prod, then
|
||||
# ./deploy/webmail.sh update v1.0.0
|
||||
# save mode: docker load -i webmail-v1.0.0.tar && ./deploy/webmail.sh update v1.0.0
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
TAG="${1:-}"
|
||||
MODE="${2:-push}"
|
||||
REGISTRY_PREFIX="${REGISTRY_PREFIX:-webmail}"
|
||||
PLATFORM="${PLATFORM:-linux/amd64}" # Synology and QNAP Intel boxes
|
||||
|
||||
[ -n "$TAG" ] || { echo "usage: $0 <version-tag> [--save]" >&2; exit 1; }
|
||||
|
||||
GIT_SHA="$(git -C "$ROOT_DIR" rev-parse --short HEAD 2>/dev/null || echo unknown)"
|
||||
if ! git -C "$ROOT_DIR" diff --quiet 2>/dev/null; then
|
||||
echo "warning: working tree has uncommitted changes; tagging ${TAG} anyway" >&2
|
||||
fi
|
||||
|
||||
build() {
|
||||
local name="$1" dockerfile="$2"
|
||||
echo "== building ${REGISTRY_PREFIX}/${name}:${TAG}"
|
||||
docker build \
|
||||
--platform "$PLATFORM" \
|
||||
-f "${ROOT_DIR}/docker/${dockerfile}" \
|
||||
--build-arg "APP_VERSION=${TAG}" \
|
||||
--build-arg "GIT_SHA=${GIT_SHA}" \
|
||||
-t "${REGISTRY_PREFIX}/${name}:${TAG}" \
|
||||
-t "${REGISTRY_PREFIX}/${name}:latest" \
|
||||
"$ROOT_DIR"
|
||||
}
|
||||
|
||||
build backend Dockerfile.backend
|
||||
build web Dockerfile.frontend
|
||||
|
||||
if [ "$MODE" = "--save" ]; then
|
||||
out="${ROOT_DIR}/webmail-${TAG}.tar"
|
||||
echo "== saving both images to ${out}"
|
||||
docker save -o "$out" \
|
||||
"${REGISTRY_PREFIX}/backend:${TAG}" \
|
||||
"${REGISTRY_PREFIX}/web:${TAG}"
|
||||
echo "done. Copy it to the NAS and run: docker load -i $(basename "$out")"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$REGISTRY_PREFIX" = "webmail" ]; then
|
||||
cat >&2 <<'MSG'
|
||||
REGISTRY_PREFIX is still the local default ("webmail"), so there is nothing to
|
||||
push to. Either set it to your registry path, e.g.
|
||||
|
||||
REGISTRY_PREFIX=192.168.1.185:8418/claude/webmail ./deploy/build-push.sh v1.0.0
|
||||
|
||||
(run `docker login 192.168.1.185:8418` once first), or re-run with --save to
|
||||
produce a tar file instead.
|
||||
MSG
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "== pushing"
|
||||
docker push "${REGISTRY_PREFIX}/backend:${TAG}"
|
||||
docker push "${REGISTRY_PREFIX}/web:${TAG}"
|
||||
docker push "${REGISTRY_PREFIX}/backend:latest"
|
||||
docker push "${REGISTRY_PREFIX}/web:latest"
|
||||
echo "pushed ${TAG}. On the NAS: ./deploy/webmail.sh update ${TAG}"
|
||||
Executable
+401
@@ -0,0 +1,401 @@
|
||||
#!/usr/bin/env bash
|
||||
# Webmail operator control script. One command per thing you might need to do.
|
||||
#
|
||||
# ./deploy/webmail.sh init create data dirs + a filled-in .env.prod
|
||||
# ./deploy/webmail.sh up [--meili] start (builds images if missing)
|
||||
# ./deploy/webmail.sh down stop (data is untouched)
|
||||
# ./deploy/webmail.sh status what is running, and which version
|
||||
# ./deploy/webmail.sh logs [svc] [-f] tail logs
|
||||
# ./deploy/webmail.sh update [tag] backup -> new images -> migrate -> roll -> health-gate
|
||||
# ./deploy/webmail.sh rollback [tag] go back to the previous version
|
||||
# ./deploy/webmail.sh backup take a verified database dump now
|
||||
# ./deploy/webmail.sh restore <file> DESTRUCTIVE: restore a dump
|
||||
# ./deploy/webmail.sh reindex [args] rebuild the search index from Postgres
|
||||
# ./deploy/webmail.sh psql open a database shell
|
||||
# ./deploy/webmail.sh shell [svc] shell inside a container (default: api)
|
||||
# ./deploy/webmail.sh doctor check the install before it bites you
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
COMPOSE_FILE="${ROOT_DIR}/docker/docker-compose.prod.yml"
|
||||
ENV_FILE="${ROOT_DIR}/docker/.env.prod"
|
||||
ENV_EXAMPLE="${ROOT_DIR}/docker/.env.prod.example"
|
||||
|
||||
# Services that must come back healthy after a roll.
|
||||
HEALTH_SERVICES=(postgres redis api web)
|
||||
HEALTH_TIMEOUT=120
|
||||
|
||||
red() { printf '\033[31m%s\033[0m\n' "$*"; }
|
||||
grn() { printf '\033[32m%s\033[0m\n' "$*"; }
|
||||
ylw() { printf '\033[33m%s\033[0m\n' "$*"; }
|
||||
die() { red "error: $*" >&2; exit 1; }
|
||||
|
||||
compose() {
|
||||
docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" "$@"
|
||||
}
|
||||
|
||||
# Read KEY=VALUE pairs the way compose's .env parser does, WITHOUT sourcing.
|
||||
# Sourcing would execute the file as shell — it would choke on an ordinary
|
||||
# value containing spaces (ENABLED_PLUGINS) and would run anything else that
|
||||
# found its way into the file that holds the master key.
|
||||
load_env_file() {
|
||||
local line key val
|
||||
while IFS= read -r line || [ -n "$line" ]; do
|
||||
case "$line" in ''|'#'*) continue ;; esac
|
||||
case "$line" in *=*) ;; *) continue ;; esac
|
||||
key="${line%%=*}"
|
||||
val="${line#*=}"
|
||||
# Trim whitespace around the key; strip one layer of surrounding quotes.
|
||||
key="$(printf '%s' "$key" | tr -d '[:space:]')"
|
||||
case "$val" in
|
||||
\"*\") val="${val#\"}"; val="${val%\"}" ;;
|
||||
\'*\') val="${val#\'}"; val="${val%\'}" ;;
|
||||
esac
|
||||
[ -n "$key" ] && export "${key}=${val}"
|
||||
done < "$1"
|
||||
}
|
||||
|
||||
require_env() {
|
||||
[ -f "$ENV_FILE" ] || die "$ENV_FILE not found. Run: ./deploy/webmail.sh init"
|
||||
load_env_file "$ENV_FILE"
|
||||
[ -n "${DATA_ROOT:-}" ] || die "DATA_ROOT is not set in $ENV_FILE"
|
||||
[ -n "${MASTER_KEY:-}" ] || die "MASTER_KEY is not set in $ENV_FILE"
|
||||
[ -n "${POSTGRES_PASSWORD:-}" ] || die "POSTGRES_PASSWORD is not set in $ENV_FILE"
|
||||
}
|
||||
|
||||
state_dir() { echo "${DATA_ROOT}/state"; }
|
||||
|
||||
# busybox sed (what you may get on DSM/QTS) has no -i suffix argument, so every
|
||||
# in-place edit goes through a temp file.
|
||||
set_env_value() {
|
||||
local key="$1" value="$2"
|
||||
sed "s|^${key}=.*|${key}=${value}|" "$ENV_FILE" > "${ENV_FILE}.tmp" \
|
||||
&& mv "${ENV_FILE}.tmp" "$ENV_FILE" && chmod 600 "$ENV_FILE"
|
||||
}
|
||||
|
||||
record_version() {
|
||||
local new="$1" cur
|
||||
mkdir -p "$(state_dir)" 2>/dev/null || true
|
||||
if ! { : > "$(state_dir)/.probe"; } 2>/dev/null; then
|
||||
ylw "cannot write ${DATA_ROOT}/state — version history not recorded (run as the owner of DATA_ROOT)"
|
||||
return 0
|
||||
fi
|
||||
rm -f "$(state_dir)/.probe"
|
||||
cur="$(cat "$(state_dir)/version" 2>/dev/null || echo "")"
|
||||
[ -n "$cur" ] && [ "$cur" != "$new" ] && echo "$cur" > "$(state_dir)/previous"
|
||||
echo "$new" > "$(state_dir)/version"
|
||||
}
|
||||
|
||||
# --- commands ----------------------------------------------------------------
|
||||
|
||||
cmd_init() {
|
||||
if [ ! -f "$ENV_FILE" ]; then
|
||||
cp "$ENV_EXAMPLE" "$ENV_FILE"
|
||||
chmod 600 "$ENV_FILE"
|
||||
local master pgpass
|
||||
master="$(openssl rand -base64 32)"
|
||||
pgpass="$(openssl rand -base64 24 | tr -d '/+=' | cut -c1-24)"
|
||||
# Portable in-place edit (BSD and GNU sed disagree about -i).
|
||||
sed "s|^MASTER_KEY=.*|MASTER_KEY=${master}|; s|^POSTGRES_PASSWORD=.*|POSTGRES_PASSWORD=${pgpass}|" \
|
||||
"$ENV_FILE" > "${ENV_FILE}.tmp" && mv "${ENV_FILE}.tmp" "$ENV_FILE"
|
||||
chmod 600 "$ENV_FILE"
|
||||
# The NAS GUIs read `.env` from the project folder (the repo root).
|
||||
# A symlink keeps one source of truth instead of two copies to drift.
|
||||
[ -e "${ROOT_DIR}/.env" ] || ln -s docker/.env.prod "${ROOT_DIR}/.env" 2>/dev/null || true
|
||||
grn "created $ENV_FILE with freshly generated secrets"
|
||||
ylw ""
|
||||
ylw " >> BACK UP MASTER_KEY NOW. Copy it into your password manager. <<"
|
||||
ylw " Without it, saved mail-account passwords cannot be decrypted even"
|
||||
ylw " from a perfect database backup."
|
||||
ylw ""
|
||||
echo "Then edit $ENV_FILE and set at least: DATA_ROOT, APP_DOMAIN, TZ, WEB_PORT."
|
||||
return 0
|
||||
fi
|
||||
|
||||
ylw "$ENV_FILE already exists; leaving it alone."
|
||||
require_env
|
||||
mkdir -p "${DATA_ROOT}"/{postgres,blobs,redis,backups,meili,state} 2>/dev/null \
|
||||
|| ylw "could not create every directory under ${DATA_ROOT} (the init container will)"
|
||||
grn "data directories ready under ${DATA_ROOT}"
|
||||
}
|
||||
|
||||
cmd_doctor() {
|
||||
require_env
|
||||
local ok=0
|
||||
|
||||
echo "config:"
|
||||
echo " DATA_ROOT ${DATA_ROOT}"
|
||||
echo " APP_DOMAIN ${APP_DOMAIN:-<unset>}"
|
||||
echo " WEB_PORT ${WEB_PORT:-8088}"
|
||||
echo " SEARCH_BACKEND ${SEARCH_BACKEND:-pg}"
|
||||
echo " IMAGE_TAG ${IMAGE_TAG:-local}"
|
||||
echo
|
||||
|
||||
if [ "$(printf '%s' "${MASTER_KEY}" | base64 -d 2>/dev/null | wc -c)" -ne 32 ]; then
|
||||
red " MASTER_KEY is not 32 bytes of base64 — the app will refuse to start"; ok=1
|
||||
else
|
||||
grn " MASTER_KEY looks well-formed"
|
||||
fi
|
||||
|
||||
if [ "$(stat -c %a "$ENV_FILE" 2>/dev/null || echo 600)" != "600" ]; then
|
||||
ylw " $ENV_FILE is not chmod 600 (it holds your master key)"
|
||||
fi
|
||||
|
||||
# Missing directories are not a failure before the first start: `up` and the
|
||||
# `init` container create them. Only say so, so a pre-install doctor is green.
|
||||
local missing=0
|
||||
for d in postgres blobs redis backups state; do
|
||||
[ -d "${DATA_ROOT}/${d}" ] || missing=1
|
||||
done
|
||||
if [ "$missing" -eq 1 ]; then
|
||||
ylw " data directories not created yet — 'up' will create them"
|
||||
else
|
||||
grn " data directories present"
|
||||
fi
|
||||
|
||||
# Probe the nearest existing ancestor: the NFS/SMB check has to work BEFORE
|
||||
# the first install, which is the only time it can still save you.
|
||||
local fstype probe="${DATA_ROOT}"
|
||||
while [ ! -d "$probe" ] && [ "$probe" != "/" ] && [ -n "$probe" ]; do
|
||||
probe="$(dirname "$probe")"
|
||||
done
|
||||
# `|| true`: with `set -e -o pipefail`, df failing on a not-yet-created
|
||||
# DATA_ROOT would abort doctor on exactly the run where you need it most.
|
||||
fstype="$(df -PT "$probe" 2>/dev/null | awk 'NR==2{print $2}' || true)"
|
||||
[ -n "$fstype" ] || fstype="$(stat -f -c %T "$probe" 2>/dev/null || echo "")"
|
||||
case "$fstype" in
|
||||
nfs*|cifs|smbfs|fuse*)
|
||||
red " ${DATA_ROOT} is on a network filesystem — Postgres WILL corrupt there."
|
||||
red " Move DATA_ROOT to a local volume (btrfs/ext4)."; ok=1 ;;
|
||||
"") ylw " could not determine the filesystem of ${DATA_ROOT}" ;;
|
||||
*) grn " ${DATA_ROOT} is on a local filesystem" ;;
|
||||
esac
|
||||
|
||||
local avail
|
||||
avail="$(df -Pm "$probe" 2>/dev/null | awk 'NR==2{print $4}' || true)"
|
||||
[ -n "$avail" ] && {
|
||||
if [ "$avail" -lt 5120 ]; then ylw " only ${avail} MB free on ${DATA_ROOT}"
|
||||
else grn " ${avail} MB free on ${DATA_ROOT}"; fi
|
||||
}
|
||||
|
||||
if command -v ss >/dev/null 2>&1 && ss -ltn 2>/dev/null | grep -q ":${WEB_PORT:-8088} "; then
|
||||
ylw " port ${WEB_PORT:-8088} already has a listener — change WEB_PORT if that is not us"
|
||||
fi
|
||||
|
||||
# chmod 700 and owned by uid 999, so count them as root in a container.
|
||||
local recent
|
||||
recent="$(docker run --rm -v "${DATA_ROOT}/backups:/b:ro" alpine:3.20 \
|
||||
sh -c 'find /b -maxdepth 1 -name "*.dump" -mtime -2 2>/dev/null | wc -l' 2>/dev/null || echo 0)"
|
||||
if [ "${recent:-0}" -gt 0 ]; then grn " ${recent} database backup(s) from the last 48 h"
|
||||
else ylw " no database backup in the last 48 h"; fi
|
||||
|
||||
[ "$ok" -eq 0 ] && grn "doctor: no blocking problems" || red "doctor: problems found (above)"
|
||||
return "$ok"
|
||||
}
|
||||
|
||||
cmd_up() {
|
||||
require_env
|
||||
local profile=() build=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--meili) profile=(--profile meili) ;;
|
||||
--build) build=(--build) ;;
|
||||
*) die "unknown option: $arg" ;;
|
||||
esac
|
||||
done
|
||||
[ "${SEARCH_BACKEND:-pg}" = "meili" ] && [ ${#profile[@]} -eq 0 ] && {
|
||||
ylw "SEARCH_BACKEND=meili — starting the Meilisearch container too"
|
||||
profile=(--profile meili)
|
||||
}
|
||||
# Best-effort only: Docker creates any missing bind-mount path itself, and
|
||||
# the `init` service fixes ownership. This line must not be fatal, because
|
||||
# DATA_ROOT is typically root-owned while you may be running as yourself.
|
||||
mkdir -p "${DATA_ROOT}"/{postgres,blobs,redis,backups,meili,state} 2>/dev/null || true
|
||||
compose "${profile[@]}" up -d "${build[@]}" --wait
|
||||
record_version "${IMAGE_TAG:-local}"
|
||||
grn "up. open http://<this-host>:${WEB_PORT:-8088}/"
|
||||
}
|
||||
|
||||
cmd_down() { require_env; compose --profile meili down; grn "stopped (data kept in ${DATA_ROOT})"; }
|
||||
cmd_restart() { require_env; compose restart "${@:-}"; }
|
||||
|
||||
cmd_status() {
|
||||
require_env
|
||||
compose --profile meili ps
|
||||
echo
|
||||
echo "recorded version : $(cat "$(state_dir)/version" 2>/dev/null || echo unknown)"
|
||||
echo "previous version : $(cat "$(state_dir)/previous" 2>/dev/null || echo none)"
|
||||
local running
|
||||
running="$(compose exec -T api python -c \
|
||||
"import urllib.request,json;print(json.load(urllib.request.urlopen('http://127.0.0.1:8000/api/version'))['version'])" \
|
||||
2>/dev/null || echo unreachable)"
|
||||
echo "running version : ${running}"
|
||||
echo
|
||||
# The data directories are owned by the container users (postgres 999,
|
||||
# backend 10001) and backups are chmod 700, so read them as root inside a
|
||||
# throwaway container rather than reporting misleading zeroes.
|
||||
docker run --rm -v "${DATA_ROOT}:/d:ro" alpine:3.20 sh -c '
|
||||
echo "data usage:"
|
||||
du -sh /d/postgres /d/blobs /d/backups 2>/dev/null | sed "s|/d/| |"
|
||||
echo
|
||||
echo "latest backups:"
|
||||
ls -lh /d/backups/*.dump 2>/dev/null | tail -3 | sed "s|/d/backups/| |" || echo " none yet"
|
||||
'
|
||||
}
|
||||
|
||||
cmd_logs() {
|
||||
require_env
|
||||
compose --profile meili logs --tail 200 "$@"
|
||||
}
|
||||
|
||||
health_gate() {
|
||||
local deadline=$(( $(date +%s) + HEALTH_TIMEOUT )) svc cid state
|
||||
while [ "$(date +%s)" -lt "$deadline" ]; do
|
||||
local all_ok=1
|
||||
for svc in "${HEALTH_SERVICES[@]}"; do
|
||||
cid="$(compose ps -q "$svc" 2>/dev/null || true)"
|
||||
[ -n "$cid" ] || { all_ok=0; break; }
|
||||
state="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$cid" 2>/dev/null || echo missing)"
|
||||
case "$state" in
|
||||
healthy|running) ;;
|
||||
*) all_ok=0; break ;;
|
||||
esac
|
||||
done
|
||||
[ "$all_ok" -eq 1 ] && return 0
|
||||
sleep 5
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
cmd_backup() {
|
||||
require_env
|
||||
compose run --rm backup once
|
||||
}
|
||||
|
||||
cmd_update() {
|
||||
require_env
|
||||
local target="${1:-${IMAGE_TAG:-local}}"
|
||||
local previous="${IMAGE_TAG:-local}"
|
||||
|
||||
echo "== 1/5 backing up the database before anything changes"
|
||||
cmd_backup || die "backup failed — refusing to update. Fix the backup first."
|
||||
|
||||
echo "== 2/5 obtaining images for tag '${target}'"
|
||||
if [ "${REGISTRY_PREFIX:-webmail}" = "webmail" ] || [ "$target" = "local" ]; then
|
||||
IMAGE_TAG="$target" compose build
|
||||
else
|
||||
IMAGE_TAG="$target" compose pull || die "pull failed — nothing has changed, you are still on ${previous}"
|
||||
fi
|
||||
|
||||
echo "== 3/5 applying migrations"
|
||||
IMAGE_TAG="$target" compose run --rm migrate \
|
||||
|| die "migration failed — services untouched, still on ${previous}. Restore from ${DATA_ROOT}/backups if the schema is damaged."
|
||||
|
||||
echo "== 4/5 rolling services"
|
||||
# Workers first (leases transfer to the replacement within ~90 s; the lease
|
||||
# protocol and idempotent (mailbox_id, uid) upserts make this lossless),
|
||||
# then the API, then the web front end.
|
||||
IMAGE_TAG="$target" compose up -d worker-sync worker-jobs
|
||||
IMAGE_TAG="$target" compose up -d api web
|
||||
|
||||
echo "== 5/5 health gate (${HEALTH_TIMEOUT}s)"
|
||||
if health_gate; then
|
||||
record_version "$target"
|
||||
set_env_value IMAGE_TAG "$target"
|
||||
grn "updated to ${target}"
|
||||
else
|
||||
red "health gate failed — rolling back to ${previous}"
|
||||
IMAGE_TAG="$previous" compose up -d --wait || true
|
||||
die "update to ${target} failed; production is back on ${previous}"
|
||||
fi
|
||||
}
|
||||
|
||||
cmd_rollback() {
|
||||
require_env
|
||||
local target="${1:-$(cat "$(state_dir)/previous" 2>/dev/null || echo "")}"
|
||||
[ -n "$target" ] || die "no previous version recorded; pass a tag: ./deploy/webmail.sh rollback v1.0.0"
|
||||
ylw "rolling back to ${target}"
|
||||
IMAGE_TAG="$target" compose up -d --wait
|
||||
if health_gate; then
|
||||
record_version "$target"
|
||||
set_env_value IMAGE_TAG "$target"
|
||||
grn "rolled back to ${target}"
|
||||
else
|
||||
die "rollback to ${target} did not come up healthy — check: ./deploy/webmail.sh logs"
|
||||
fi
|
||||
}
|
||||
|
||||
cmd_restore() {
|
||||
require_env
|
||||
local arg="${1:-}" dir base
|
||||
[ -n "$arg" ] || die "usage: ./deploy/webmail.sh restore <file.dump> (run \`status\` to list them)"
|
||||
|
||||
# Backups are chmod 700 owned by uid 999, so a plain `[ -f ]` from an
|
||||
# unprivileged shell says "missing" for a file that is right there. Resolve
|
||||
# and verify inside a container instead.
|
||||
if [ -f "$arg" ]; then
|
||||
dir="$(cd "$(dirname "$arg")" && pwd)"; base="$(basename "$arg")"
|
||||
else
|
||||
dir="${DATA_ROOT}/backups"; base="$(basename "$arg")"
|
||||
fi
|
||||
if ! docker run --rm -v "${dir}:/b:ro" alpine:3.20 test -f "/b/${base}" 2>/dev/null; then
|
||||
red "no such dump: ${base}"
|
||||
echo "available in ${DATA_ROOT}/backups:"
|
||||
docker run --rm -v "${DATA_ROOT}/backups:/b:ro" alpine:3.20 \
|
||||
sh -c 'ls -1 /b/*.dump 2>/dev/null | sed "s|/b/| |"' || true
|
||||
exit 1
|
||||
fi
|
||||
local dump="${dir}/${base}"
|
||||
|
||||
red "THIS REPLACES THE CURRENT DATABASE with ${base}."
|
||||
red "Mail synced since that dump will be gone (it re-syncs from the servers)."
|
||||
printf 'Type RESTORE to continue: '
|
||||
read -r confirm
|
||||
[ "$confirm" = "RESTORE" ] || die "aborted"
|
||||
|
||||
echo "stopping application containers (postgres stays up)"
|
||||
compose stop api worker-sync worker-jobs web backup
|
||||
|
||||
echo "restoring $(basename "$dump")"
|
||||
# --clean --if-exists drops and recreates objects; a single transaction so a
|
||||
# failure leaves the old database intact.
|
||||
docker run --rm -i --network webmail-prod \
|
||||
-e PGPASSWORD="${POSTGRES_PASSWORD}" \
|
||||
-v "$(dirname "$dump")":/restore:ro \
|
||||
postgres:16 \
|
||||
pg_restore --clean --if-exists --no-owner --single-transaction \
|
||||
-h postgres -U "${POSTGRES_USER:-webmail}" -d "${POSTGRES_DB:-webmail}" \
|
||||
"/restore/$(basename "$dump")" \
|
||||
|| die "restore failed; the database was left as it was"
|
||||
|
||||
echo "starting application containers"
|
||||
compose up -d --wait
|
||||
grn "restored. If the blob store was lost too, mail bodies will refill on next sync."
|
||||
}
|
||||
|
||||
cmd_reindex() { require_env; compose run --rm api reindex "$@"; }
|
||||
cmd_psql() { require_env; compose exec postgres psql -U "${POSTGRES_USER:-webmail}" -d "${POSTGRES_DB:-webmail}"; }
|
||||
cmd_shell() { require_env; compose exec "${1:-api}" /bin/sh; }
|
||||
|
||||
usage() { sed -n '2,20p' "${BASH_SOURCE[0]}" | sed 's|^# \?||'; }
|
||||
|
||||
cmd="${1:-}"; shift || true
|
||||
case "$cmd" in
|
||||
init) cmd_init "$@" ;;
|
||||
doctor) cmd_doctor "$@" ;;
|
||||
up) cmd_up "$@" ;;
|
||||
down) cmd_down "$@" ;;
|
||||
restart) cmd_restart "$@" ;;
|
||||
status) cmd_status "$@" ;;
|
||||
logs) cmd_logs "$@" ;;
|
||||
update) cmd_update "$@" ;;
|
||||
rollback) cmd_rollback "$@" ;;
|
||||
backup) cmd_backup "$@" ;;
|
||||
restore) cmd_restore "$@" ;;
|
||||
reindex) cmd_reindex "$@" ;;
|
||||
psql) cmd_psql "$@" ;;
|
||||
shell) cmd_shell "$@" ;;
|
||||
""|-h|--help|help) usage ;;
|
||||
*) die "unknown command: $cmd (try --help)" ;;
|
||||
esac
|
||||
@@ -0,0 +1,131 @@
|
||||
# Deployment & Release System — Agent Build Instructions
|
||||
|
||||
Companion to `SPEC.md` and `DEV_ENVIRONMENT.md`. This document tells you (the implementing agent) to **build the complete versioning, CI/CD, and production-update system** for this project. The human operator is not a DevOps specialist — your job is to make deployment a one-command, safe, reversible operation and to document the few manual steps clearly.
|
||||
|
||||
Everything lives inside the operator's own **Gitea** server (source, CI, and container registry). No GitHub, no external CI, no external registry.
|
||||
|
||||
---
|
||||
|
||||
## 0. What you are building (deliverables checklist)
|
||||
|
||||
Create all of the following, committed to the repo:
|
||||
|
||||
1. `.gitea/workflows/ci.yml` — test + build + push pipeline (runs on every push and PR).
|
||||
2. `.gitea/workflows/release.yml` — on a version tag: test, build images, push to Gitea registry, deploy to production.
|
||||
3. `docker/Dockerfile.backend` and `docker/Dockerfile.frontend` — production images (multi-stage, slim).
|
||||
4. `docker/docker-compose.prod.yml` — the production stack, referencing images by **pinned tag** from the Gitea registry.
|
||||
5. `deploy/deploy.sh` — the script that runs ON the production server to apply an update safely.
|
||||
6. `deploy/rollback.sh` — one-command rollback to the previous version.
|
||||
7. `backend/app/api/health.py` — `/api/healthz` and `/api/version` endpoints.
|
||||
8. `deploy/VERSION` — plain-text current version, written by the release pipeline.
|
||||
9. `docs/deployment.md` — a short **operator runbook** in plain language (how to release, how to roll back, what to do if it breaks). Write this for a non-expert.
|
||||
10. `docs/gitea-setup.md` — the **one-time manual setup** the operator must do in Gitea's UI (documented step by step, see §7).
|
||||
|
||||
Do not require any tool the operator must learn. The whole system is: **push a tag → it deploys. Run one script → it rolls back.**
|
||||
|
||||
---
|
||||
|
||||
## 1. Versioning rules (implement, don't deviate)
|
||||
|
||||
- **SemVer git tags** are the single source of truth: `v{MAJOR}.{MINOR}.{PATCH}` (e.g. `v1.4.2`).
|
||||
- The tag name becomes the image tag. Images are **never** tagged `latest` in production compose. `latest` may exist as a convenience alias but production always pins an explicit version.
|
||||
- `/api/version` returns the running version (read from a build-time env `APP_VERSION` baked into the image). `/api/healthz` returns 200 only when DB + Redis are reachable.
|
||||
- Every release is atomic (roll the whole tag) and reversible (redeploy the previous tag). Never hand-edit files on the production server to "patch" — always go through a tag.
|
||||
- Tag creation is the human's explicit release action. Do NOT auto-tag from routine commits — routine commits run CI (test+build) but do not deploy. Only a `v*` tag deploys.
|
||||
|
||||
## 2. Images (`docker/Dockerfile.backend`, `docker/Dockerfile.frontend`)
|
||||
|
||||
- Multi-stage builds. Backend: build deps in a builder stage, copy only the venv + app into a slim `python:3.11-slim` runtime; run as a non-root user; `ENV APP_VERSION` set via build arg. Frontend: build with Node 20, serve the static bundle from nginx-alpine (or have the API serve it — pick API-served only if simpler for a single node; default to a separate `web` nginx image).
|
||||
- Backend image entrypoint supports three roles via command: API (`uvicorn`), worker-jobs, worker-sync (matches SPEC §6.3 `WORKER_ROLE`). One image, three commands.
|
||||
- Label images with the version and git SHA (`org.opencontainers.image.version`, `.revision`).
|
||||
- Registry path: `{GITEA_HOST}/{OWNER}/webmail/backend` and `.../web`. Read `GITEA_HOST`/`OWNER` from CI variables, never hardcode.
|
||||
|
||||
## 3. CI pipeline (`.gitea/workflows/ci.yml`)
|
||||
|
||||
Trigger: push to any branch, and pull requests. Steps:
|
||||
1. Spin up service containers (postgres:16, redis:7, meilisearch:v1.8, greenmail) — reuse `docker/docker-compose.ci.yml` from DEV_ENVIRONMENT.
|
||||
2. Backend gates: `ruff check`, `ruff format --check`, `mypy --strict app`, `pytest -q` (unit + integration, coverage floors per SPEC §13).
|
||||
3. Frontend gates: `tsc --noEmit`, `eslint`, `vitest run`.
|
||||
4. Build both Docker images (validates the Dockerfiles) — but on plain branch pushes, **do not push** to the registry (build-only smoke).
|
||||
5. Fail the whole workflow if any gate fails.
|
||||
|
||||
This gives every commit a green/red signal without deploying anything.
|
||||
|
||||
## 4. Release pipeline (`.gitea/workflows/release.yml`)
|
||||
|
||||
Trigger: push of a tag matching `v*`. Steps, in order, **stopping on any failure**:
|
||||
1. Run the full CI gate set (§3) against the tagged commit. No shortcuts — a tag that fails tests must not deploy.
|
||||
2. Build backend + `web` images with `APP_VERSION={tag}` and the git SHA.
|
||||
3. Log in to the Gitea container registry (using a CI secret token) and push both images tagged `{tag}` and `latest`.
|
||||
4. Deploy to production by connecting to the production host over SSH (key from a CI secret) and running `deploy/deploy.sh {tag}`. See §5.
|
||||
5. On deploy success, write `{tag}` to `deploy/VERSION` and update `/api/version` expectation. On deploy failure, the pipeline must surface the error loudly and NOT leave a half-applied state (deploy.sh handles atomicity; see §5).
|
||||
|
||||
Keep the runner OFF the production box (run it on the dev/build machine or a spare fleet node) so a build can't starve the mail server.
|
||||
|
||||
## 5. The deploy script (`deploy/deploy.sh`) — safety is the whole point
|
||||
|
||||
Runs on the production host. Takes one argument: the target version tag. It MUST do, in this exact order:
|
||||
|
||||
1. **Record current version** (read running `/api/version` or `deploy/VERSION`) → save as the rollback target.
|
||||
2. **Snapshot the database BEFORE touching anything** — trigger a `pgBackRest` backup (or a ZFS snapshot of `tank/pg` if pgBackRest isn't configured yet). Abort the deploy if the snapshot fails. This is non-negotiable: a bad migration must always be recoverable.
|
||||
3. **Pull the new images** for `{tag}` from the Gitea registry. Abort if pull fails (nothing has changed yet — safe).
|
||||
4. **Run database migrations** as a one-shot container: `alembic upgrade head` (this includes plugin migration branches per SPEC §11.1). Migrations MUST be backward-compatible / expand-then-contract so the brief mixed-version window is safe. Abort + alert if migration fails (DB snapshot from step 2 is the recovery path).
|
||||
5. **Roll services in dependency order:** workers first (`worker-sync`, `worker-jobs`), then the API, then `web` — `docker compose -f docker-compose.prod.yml up -d --wait` with the new tag. `--wait` blocks on healthchecks.
|
||||
6. **Health-gate:** poll `/api/healthz` for up to 60 s. If it doesn't go green, **automatically invoke `rollback.sh`** to the saved previous version and exit non-zero. Never leave production unhealthy.
|
||||
7. On success: write `{tag}` to `deploy/VERSION`, log the old→new transition, done.
|
||||
|
||||
The script is idempotent and logs every step with timestamps. It reads all config (registry host, compose file path, healthcheck URL, ret(ry counts) from `deploy/deploy.env`, not hardcoded.
|
||||
|
||||
Because SPEC §6.3 sync workers are lease-owned and stateless, rolling them mid-operation is safe: leases transfer to the new worker within ~90 s, no mail lost, no duplicate rows (idempotent `(mailbox_id, uid)` upserts). Note this in the script comments so no future change breaks the assumption.
|
||||
|
||||
## 6. Rollback (`deploy/rollback.sh`)
|
||||
|
||||
One argument (target version) or, with no argument, the version recorded as "previous" by the last deploy. Steps: pull the old images (should be cached), roll services back to that tag, health-gate. **Migration note:** because migrations are expand-then-contract and backward-compatible, rolling the *code* back does not require rolling the *schema* back in the common case — document this clearly, and for the rare breaking migration, instruct the operator to restore the pre-deploy DB snapshot from step 5.2 (give the exact pgBackRest/ZFS restore command in `docs/deployment.md`).
|
||||
|
||||
## 7. One-time manual setup (`docs/gitea-setup.md`) — write this for a non-expert
|
||||
|
||||
The operator must do these once in Gitea's web UI; document each with exact click paths and screenshots-in-words:
|
||||
1. **Enable Gitea Actions** for the repo (Settings → Actions).
|
||||
2. **Register a runner:** provide the exact commands to run the `act_runner` container on the build machine, register it against the Gitea instance with a registration token, and label it (`self-hosted`, `linux`). Include the docker-compose snippet for the runner.
|
||||
3. **Enable the Gitea container registry** (it's built in) and create a **package-registry access token**; store it as an Actions secret `REGISTRY_TOKEN`.
|
||||
4. **Add Actions secrets:** `REGISTRY_TOKEN`, `PROD_SSH_KEY` (private key for the deploy user on the production host), `PROD_HOST`, `PROD_USER`. List exactly where in the UI (Settings → Actions → Secrets).
|
||||
5. **On the production host:** create a low-privilege `deploy` user, add it to the `docker` group, install its public key, and `docker login` to the Gitea registry once (or bake it into deploy.sh via the token). Give copy-paste commands.
|
||||
6. **First deploy:** how to cut the first release — `git tag v1.0.0 && git push origin v1.0.0` — and what success looks like (`/api/version` returns it).
|
||||
|
||||
Keep this doc short, sequential, and copy-pasteable. Assume the reader will follow it literally.
|
||||
|
||||
## 8. Operator runbook (`docs/deployment.md`) — plain language
|
||||
|
||||
Cover, in a few short sections:
|
||||
- **How to release:** "Tag and push. That's it." — `git tag v1.4.3 && git push origin v1.4.3`, then watch the Actions tab.
|
||||
- **How to see what's running in production:** `curl https://{host}/api/version`.
|
||||
- **How to roll back:** `ssh {prod} 'cd /opt/webmail && ./deploy/rollback.sh'`.
|
||||
- **If a deploy fails:** it auto-rolls-back and production stays on the old version; where to read the logs; when to restore a DB snapshot (rare) with the exact command.
|
||||
- **What NOT to do:** never edit files directly on the server, never use `latest` in the compose file, never skip the tag.
|
||||
|
||||
## 9. Migration discipline (encode as a rule)
|
||||
|
||||
Add a short `docs/migrations.md`: all Alembic migrations must be **expand-then-contract** — add new columns/tables in one release, backfill, switch code, drop old columns only in a *later* release once no running code needs them. This is what makes zero-downtime rolls and easy rollback possible. Add a CI check or PR-checklist item reminding of this; never generate a destructive migration (DROP/rename) in the same release that stops using the column.
|
||||
|
||||
## 10. Acceptance criteria
|
||||
|
||||
1. Pushing a normal commit runs CI (tests + image build) and does **not** deploy.
|
||||
2. Pushing a `v*` tag whose tests pass builds images, pushes them to the Gitea registry, and deploys to production with no manual step beyond the tag push.
|
||||
3. Pushing a `v*` tag whose tests **fail** produces no image and no deploy.
|
||||
4. `/api/version` on production returns the deployed tag after a release.
|
||||
5. A deploy whose new version fails its healthcheck auto-rolls-back and leaves production healthy on the prior version (test with a deliberately broken image).
|
||||
6. `rollback.sh` returns production to the prior version in one command.
|
||||
7. A DB snapshot exists (pgBackRest or ZFS) timestamped immediately before every migration — verify one is created on a test deploy.
|
||||
8. Rolling sync workers during a deploy loses no mail and creates no duplicate message rows (ties to SPEC §6.3 acceptance tests).
|
||||
9. `docs/gitea-setup.md` and `docs/deployment.md` are followable by a non-DevOps operator: every manual step has an exact command or click path.
|
||||
|
||||
## 11. Build order
|
||||
|
||||
1. Health/version endpoints (§1) + Dockerfiles (§2) + `docker-compose.prod.yml`.
|
||||
2. `ci.yml` (get green tests + building images first).
|
||||
3. Registry push + `release.yml`.
|
||||
4. `deploy.sh` + `rollback.sh` + `deploy.env` (test against a staging target — the same compose on a different port on the dev box — before pointing at the real host).
|
||||
5. The two docs (§7, §8) + `migrations.md`.
|
||||
6. Prove all §10 acceptance criteria, including the deliberately-broken-image auto-rollback test.
|
||||
|
||||
Keep it boring and legible. The operator should never have to think about the mechanics again after the one-time Gitea setup.
|
||||
@@ -0,0 +1,23 @@
|
||||
# Production stack, at the repo root so the NAS GUIs find it.
|
||||
#
|
||||
# Synology Container Manager and QNAP Container Station create a project from a
|
||||
# folder and look for `docker-compose.yml` in it. Pointing them at `docker/`
|
||||
# would find the DEV service stack (docker/docker-compose.yml: throwaway
|
||||
# Postgres/Redis/Meilisearch/GreenMail on published dev ports) and start the
|
||||
# wrong thing entirely — so the production entry point lives here instead.
|
||||
#
|
||||
# Everything real is in docker/docker-compose.prod.yml; `include` resolves that
|
||||
# file's relative paths against its own directory, so the build context and
|
||||
# backup.sh still resolve correctly from here.
|
||||
#
|
||||
# Config comes from `.env` in this directory — `deploy/webmail.sh init` creates
|
||||
# it as a symlink to docker/.env.prod so there is only ever one copy to edit.
|
||||
#
|
||||
# GUI: create a project on this folder, name it `webmail-prod`
|
||||
# CLI: ./deploy/webmail.sh up (preferred: adds backup, health-gated
|
||||
# updates, rollback and restore)
|
||||
|
||||
name: webmail-prod
|
||||
|
||||
include:
|
||||
- docker/docker-compose.prod.yml
|
||||
@@ -0,0 +1,111 @@
|
||||
# ============================================================================
|
||||
# Webmail production configuration.
|
||||
#
|
||||
# Copy to docker/.env.prod, fill in the three REQUIRED values, keep it safe:
|
||||
# cp docker/.env.prod.example docker/.env.prod && chmod 600 docker/.env.prod
|
||||
#
|
||||
# THIS FILE IS A BACKUP ARTEFACT. MASTER_KEY is not stored anywhere else: lose
|
||||
# it and every saved mail-account password becomes permanently undecryptable,
|
||||
# even from a perfect database backup. Keep a copy in a password manager.
|
||||
# ============================================================================
|
||||
|
||||
# --- REQUIRED ---------------------------------------------------------------
|
||||
|
||||
# Where all persistent data lives on the host. Use a real shared folder your NAS
|
||||
# backup tool can see — not a Docker named volume.
|
||||
# Synology: /volume1/docker/webmail/data
|
||||
# QNAP: /share/Container/webmail/data
|
||||
DATA_ROOT=/volume1/docker/webmail/data
|
||||
|
||||
# 32 random bytes, base64. Generate ONCE, then never change it:
|
||||
# openssl rand -base64 32
|
||||
MASTER_KEY=
|
||||
|
||||
# Database password. Generate once:
|
||||
# openssl rand -base64 24 | tr -d '/+=' | cut -c1-24
|
||||
POSTGRES_PASSWORD=
|
||||
|
||||
# --- Identity ---------------------------------------------------------------
|
||||
# The domain your user handles are minted under (user@APP_DOMAIN) and the issuer
|
||||
# shown in 2FA authenticator apps. It does NOT have to be publicly resolvable.
|
||||
APP_DOMAIN=mail.home.lan
|
||||
|
||||
# --- Network ----------------------------------------------------------------
|
||||
# The single published port. Avoid 5000/5001 (Synology DSM) and 8080 (QNAP QTS).
|
||||
WEB_PORT=8088
|
||||
# Bind to one NAS interface instead of every network, e.g. WEB_BIND=192.168.1.185
|
||||
WEB_BIND=0.0.0.0
|
||||
|
||||
TZ=Europe/Warsaw
|
||||
|
||||
# --- Search -----------------------------------------------------------------
|
||||
# pg = Postgres full-text. No extra container, no extra RAM. Default.
|
||||
# meili = Meilisearch: typo tolerance, better ranking, faster on large mailboxes.
|
||||
# Costs a container and ~0.7-1 GB RAM. Start it with:
|
||||
# ./deploy/webmail.sh up --meili
|
||||
# and after switching an existing install, rebuild the index:
|
||||
# ./deploy/webmail.sh reindex
|
||||
SEARCH_BACKEND=pg
|
||||
# Required if SEARCH_BACKEND=meili — generate with: openssl rand -base64 32
|
||||
MEILI_MASTER_KEY=
|
||||
|
||||
# --- Images -----------------------------------------------------------------
|
||||
# `local` = built on this host from the repo. To deploy prebuilt images from
|
||||
# your Gitea registry instead, set both of these (see deploy/build-push.sh):
|
||||
# REGISTRY_PREFIX=192.168.1.185:8418/claude/webmail
|
||||
# IMAGE_TAG=v1.0.0
|
||||
REGISTRY_PREFIX=webmail
|
||||
IMAGE_TAG=local
|
||||
|
||||
# --- Application ------------------------------------------------------------
|
||||
LOG_LEVEL=INFO
|
||||
UNDO_SECONDS=10
|
||||
SESSION_TTL_DAYS=30
|
||||
MAX_ATTACHMENT_MB=25
|
||||
# Auto-purge Trash/Spam older than N days (0 disables).
|
||||
TRASH_RETENTION_DAYS=30
|
||||
# Server-side sender avatars (Gravatar + favicon), fetched by the server so
|
||||
# senders never see your IP. Set false to make zero outbound web requests.
|
||||
SENDER_AVATARS_ENABLED=true
|
||||
|
||||
GOOGLE_OAUTH_ENABLED=false
|
||||
OAUTH_GOOGLE_CLIENT_ID=
|
||||
OAUTH_GOOGLE_CLIENT_SECRET=
|
||||
OAUTH_MS_CLIENT_ID=
|
||||
OAUTH_MS_CLIENT_SECRET=
|
||||
|
||||
# Plugins whose schema is migrated on every start. Remove one to stop shipping
|
||||
# its tables; `hello` is the SDK demo and is intentionally not listed.
|
||||
ENABLED_PLUGINS="calendar contacts labels filters"
|
||||
|
||||
# --- Backups ----------------------------------------------------------------
|
||||
BACKUP_AT=03:30
|
||||
BACKUP_KEEP_DAYS=14
|
||||
|
||||
# --- Sizing (defaults suit a personal install on a shared NAS) --------------
|
||||
# Peak worker RAM is roughly PARSE_CONCURRENCY x STREAM_PARSE_MB plus overhead.
|
||||
INTERACTIVE_CONCURRENCY=8
|
||||
BULK_CONCURRENCY=1
|
||||
PARSE_CONCURRENCY=2
|
||||
STREAM_PARSE_MB=8
|
||||
PER_HOST_CONN_CAP=10
|
||||
HOT_BUDGET_PER_WORKER=50
|
||||
|
||||
# Container caps. Total without Meilisearch: ~2.2 GB of *limits*; real steady
|
||||
# usage for one or two mail accounts is around 700 MB-1 GB.
|
||||
PG_MEM_LIMIT=768m
|
||||
PG_SHARED_BUFFERS=256MB
|
||||
PG_MAX_CONNECTIONS=100
|
||||
PG_WORK_MEM=8MB
|
||||
PG_MAINTENANCE_WORK_MEM=128MB
|
||||
PG_EFFECTIVE_CACHE_SIZE=768MB
|
||||
REDIS_MEM_LIMIT=160m
|
||||
REDIS_MAXMEMORY=96mb
|
||||
API_MEM_LIMIT=512m
|
||||
API_CPUS=1.0
|
||||
WORKER_SYNC_MEM_LIMIT=512m
|
||||
WORKER_JOBS_MEM_LIMIT=640m
|
||||
WORKER_CPUS=1.5
|
||||
WEB_MEM_LIMIT=96m
|
||||
BACKUP_MEM_LIMIT=256m
|
||||
MEILI_MEM_LIMIT=1g
|
||||
@@ -0,0 +1,79 @@
|
||||
# Production backend image: API, both worker roles, and the migration/admin CLI.
|
||||
#
|
||||
# One image, several commands (see docker/entrypoint.sh) — matches SPEC §6.3's
|
||||
# WORKER_ROLE split. Build context is the REPO ROOT, because the image needs
|
||||
# backend/ *and* plugins/ (the five plugins are separate distributions that the
|
||||
# loader discovers through the `webmail.plugins` entry-point group, SPEC §11).
|
||||
#
|
||||
# docker build -f docker/Dockerfile.backend -t webmail/backend .
|
||||
|
||||
# ---------- builder: resolve wheels into a self-contained venv ----------------
|
||||
FROM python:3.11-slim AS builder
|
||||
|
||||
ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1
|
||||
|
||||
# Only needed if a pinned dep has no manylinux wheel for this platform; the
|
||||
# whole stage is discarded, so it costs nothing in the runtime image.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN python -m venv /opt/venv
|
||||
ENV PATH="/opt/venv/bin:$PATH"
|
||||
|
||||
# Backend first: its dependency set is the slowest layer and changes least often.
|
||||
COPY backend/pyproject.toml /src/backend/pyproject.toml
|
||||
COPY backend/app /src/backend/app
|
||||
COPY backend/workers /src/backend/workers
|
||||
RUN pip install /src/backend
|
||||
|
||||
# Plugins are ordinary distributions installed on top (each owns its own deps).
|
||||
COPY plugins /src/plugins
|
||||
RUN pip install \
|
||||
/src/plugins/webmail_plugin_calendar \
|
||||
/src/plugins/webmail_plugin_contacts \
|
||||
/src/plugins/webmail_plugin_labels \
|
||||
/src/plugins/webmail_plugin_filters
|
||||
|
||||
# ---------- runtime -----------------------------------------------------------
|
||||
FROM python:3.11-slim
|
||||
|
||||
ARG APP_VERSION=dev
|
||||
ARG GIT_SHA=unknown
|
||||
LABEL org.opencontainers.image.title="webmail-backend" \
|
||||
org.opencontainers.image.version="${APP_VERSION}" \
|
||||
org.opencontainers.image.revision="${GIT_SHA}"
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
PATH="/opt/venv/bin:$PATH" \
|
||||
APP_VERSION="${APP_VERSION}" \
|
||||
GIT_SHA="${GIT_SHA}"
|
||||
|
||||
# tini reaps zombies and forwards SIGTERM, which the workers need for the <10 s
|
||||
# graceful lease release in workers/run.py.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends tini \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& useradd --system --uid 10001 --create-home --home-dir /home/webmail webmail
|
||||
|
||||
COPY --from=builder /opt/venv /opt/venv
|
||||
|
||||
WORKDIR /app
|
||||
# Alembic's core tree lives outside the installed package, so it is copied in.
|
||||
COPY backend/alembic.ini ./alembic.ini
|
||||
COPY backend/migrations ./migrations
|
||||
COPY backend/scripts ./scripts
|
||||
COPY docker/entrypoint.sh /usr/local/bin/entrypoint
|
||||
RUN chmod +x /usr/local/bin/entrypoint
|
||||
|
||||
# Blob store mount point (BLOB_ROOT). Ownership is set by the compose `init`
|
||||
# service, which runs before anything else touches the bind mount.
|
||||
RUN mkdir -p /data/blobs && chown webmail:webmail /data/blobs
|
||||
VOLUME ["/data/blobs"]
|
||||
|
||||
USER webmail
|
||||
ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/entrypoint"]
|
||||
CMD ["api"]
|
||||
@@ -0,0 +1,35 @@
|
||||
# Production web image: the built SPA served by nginx, which also reverse-proxies
|
||||
# /api (including the /api/ws WebSocket) to the API container. Build context is
|
||||
# the REPO ROOT so the build stage can see frontend/.
|
||||
#
|
||||
# docker build -f docker/Dockerfile.frontend -t webmail/web .
|
||||
|
||||
# ---------- builder -----------------------------------------------------------
|
||||
FROM node:20-slim AS builder
|
||||
|
||||
WORKDIR /src
|
||||
COPY frontend/package.json frontend/package-lock.json* ./
|
||||
# `npm ci` when a lockfile is present (reproducible), `npm install` otherwise.
|
||||
RUN if [ -f package-lock.json ]; then npm ci; else npm install; fi
|
||||
|
||||
COPY frontend/ ./
|
||||
# Type-check + bundle. `npm run build` runs `tsc --noEmit && vite build`, so a
|
||||
# type error fails the image build rather than shipping a broken bundle.
|
||||
RUN npm run build
|
||||
|
||||
# ---------- runtime -----------------------------------------------------------
|
||||
FROM nginx:1.27-alpine
|
||||
|
||||
ARG APP_VERSION=dev
|
||||
ARG GIT_SHA=unknown
|
||||
LABEL org.opencontainers.image.title="webmail-web" \
|
||||
org.opencontainers.image.version="${APP_VERSION}" \
|
||||
org.opencontainers.image.revision="${GIT_SHA}"
|
||||
|
||||
COPY docker/nginx.conf /etc/nginx/conf.d/default.conf
|
||||
COPY --from=builder /src/dist /usr/share/nginx/html
|
||||
|
||||
# Surface the running version to `deploy/webmail.sh status` without needing the API.
|
||||
RUN printf '%s\n' "${APP_VERSION}" > /usr/share/nginx/html/version.txt
|
||||
|
||||
EXPOSE 80
|
||||
Executable
+88
@@ -0,0 +1,88 @@
|
||||
#!/bin/bash
|
||||
# Nightly logical backup of the webmail database.
|
||||
#
|
||||
# Runs as a long-lived sidecar: sleeps until BACKUP_AT, dumps, verifies, prunes,
|
||||
# repeats. `docker compose run --rm backup once` takes a single backup now (this
|
||||
# is what `deploy/webmail.sh backup` does, e.g. immediately before an update).
|
||||
#
|
||||
# Why logical dumps and not just a NAS snapshot: a btrfs/Hyper Backup snapshot of
|
||||
# a *running* Postgres data directory is crash-consistent, which usually replays
|
||||
# fine but is not guaranteed and cannot be restored into a different Postgres
|
||||
# version. A pg_dump is version-portable and verifiable. Keep both.
|
||||
set -euo pipefail
|
||||
|
||||
BACKUP_DIR="${BACKUP_DIR:-/backups}"
|
||||
BACKUP_AT="${BACKUP_AT:-03:30}"
|
||||
BACKUP_KEEP_DAYS="${BACKUP_KEEP_DAYS:-14}"
|
||||
DB="${PGDATABASE:-webmail}"
|
||||
|
||||
log() { echo "[backup $(date -Is)] $*"; }
|
||||
|
||||
take_backup() {
|
||||
local stamp out globals
|
||||
stamp="$(date +%Y%m%d-%H%M%S)"
|
||||
out="${BACKUP_DIR}/${DB}-${stamp}.dump"
|
||||
globals="${BACKUP_DIR}/globals-${stamp}.sql"
|
||||
|
||||
log "dumping ${DB} -> $(basename "$out")"
|
||||
# -Fc: compressed custom format, restorable selectively with pg_restore.
|
||||
if ! pg_dump --format=custom --compress=6 --file="${out}.partial" "$DB"; then
|
||||
log "ERROR: pg_dump failed; leaving no partial file"
|
||||
rm -f "${out}.partial"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Verify the archive is readable before it counts as a backup.
|
||||
if ! pg_restore --list "${out}.partial" >/dev/null 2>&1; then
|
||||
log "ERROR: dump failed verification (pg_restore --list); discarding"
|
||||
rm -f "${out}.partial"
|
||||
return 1
|
||||
fi
|
||||
|
||||
mv "${out}.partial" "$out"
|
||||
chmod 600 "$out"
|
||||
|
||||
# Roles/passwords live outside the database; tiny, and makes a restore onto a
|
||||
# bare Postgres one step shorter.
|
||||
pg_dumpall --globals-only --file="$globals" 2>/dev/null || true
|
||||
[ -f "$globals" ] && chmod 600 "$globals"
|
||||
|
||||
ln -sfn "$(basename "$out")" "${BACKUP_DIR}/latest.dump"
|
||||
log "ok: $(du -h "$out" | cut -f1) $(basename "$out")"
|
||||
|
||||
log "pruning dumps older than ${BACKUP_KEEP_DAYS} days"
|
||||
find "$BACKUP_DIR" -maxdepth 1 -name "${DB}-*.dump" -mtime "+${BACKUP_KEEP_DAYS}" -print -delete || true
|
||||
find "$BACKUP_DIR" -maxdepth 1 -name "globals-*.sql" -mtime "+${BACKUP_KEEP_DAYS}" -print -delete || true
|
||||
}
|
||||
|
||||
seconds_until() {
|
||||
local target now next
|
||||
target="$1"
|
||||
now=$(date +%s)
|
||||
next=$(date -d "today ${target}" +%s 2>/dev/null || echo 0)
|
||||
if [ "$next" -le "$now" ]; then
|
||||
next=$(date -d "tomorrow ${target}" +%s)
|
||||
fi
|
||||
echo $(( next - now ))
|
||||
}
|
||||
|
||||
if [ "${1:-}" = "once" ]; then
|
||||
take_backup
|
||||
exit $?
|
||||
fi
|
||||
|
||||
log "backup sidecar started; schedule ${BACKUP_AT} daily, retention ${BACKUP_KEEP_DAYS} days"
|
||||
|
||||
# Baseline on first start: if there is no dump from the last 24 h, take one now
|
||||
# rather than leaving a fresh install unprotected until the first scheduled run.
|
||||
if ! find "$BACKUP_DIR" -maxdepth 1 -name "${DB}-*.dump" -mtime -1 | grep -q .; then
|
||||
log "no recent dump found; taking a baseline backup"
|
||||
take_backup || log "baseline backup failed; will retry on schedule"
|
||||
fi
|
||||
|
||||
while true; do
|
||||
wait_s="$(seconds_until "$BACKUP_AT")"
|
||||
log "next backup in ${wait_s}s (at ${BACKUP_AT})"
|
||||
sleep "$wait_s"
|
||||
take_backup || log "scheduled backup failed; continuing"
|
||||
done
|
||||
@@ -0,0 +1,307 @@
|
||||
# Production stack for a single box (Synology / QNAP / any Docker host).
|
||||
#
|
||||
# Data lives in bind mounts under ${DATA_ROOT} — NOT in named volumes — so it
|
||||
# survives `docker compose down`, image updates, project deletion in the NAS
|
||||
# UI, and a full reinstall, and so the NAS backup tools (Hyper Backup,
|
||||
# Snapshot Replication, HBS3) can actually see it.
|
||||
#
|
||||
# Nothing is published except ${WEB_PORT}. Postgres, Redis, Meilisearch and the
|
||||
# API are reachable only on the internal `webmail` network.
|
||||
#
|
||||
# Start: ./deploy/webmail.sh up
|
||||
# Manual: docker compose --env-file docker/.env.prod -f docker/docker-compose.prod.yml up -d
|
||||
|
||||
# Deliberately NOT "webmail": docker/docker-compose.yml (the dev service stack)
|
||||
# uses that project name, and two compose files sharing one project name fight
|
||||
# over each other's containers — running `make up` would stop production.
|
||||
name: webmail-prod
|
||||
|
||||
x-logging: &logging
|
||||
# A shared NAS must never have its volume filled by container logs.
|
||||
driver: json-file
|
||||
options: { max-size: "10m", max-file: "3" }
|
||||
|
||||
x-backend-image: &backend-image
|
||||
image: ${REGISTRY_PREFIX:-webmail}/backend:${IMAGE_TAG:-local}
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: docker/Dockerfile.backend
|
||||
args:
|
||||
APP_VERSION: ${IMAGE_TAG:-local}
|
||||
GIT_SHA: ${GIT_SHA:-unknown}
|
||||
|
||||
x-backend-env: &backend-env
|
||||
APP_DOMAIN: "${APP_DOMAIN:?set APP_DOMAIN in your .env}"
|
||||
DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-webmail}:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in your .env}@postgres:5432/${POSTGRES_DB:-webmail}
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
MASTER_KEY: "${MASTER_KEY:?set MASTER_KEY in your .env - generate one with openssl rand -base64 32}"
|
||||
BLOB_ROOT: /data/blobs
|
||||
SEARCH_BACKEND: ${SEARCH_BACKEND:-pg}
|
||||
MEILI_URL: ${MEILI_URL:-http://meilisearch:7700}
|
||||
MEILI_KEY: ${MEILI_MASTER_KEY:-}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
UNDO_SECONDS: ${UNDO_SECONDS:-10}
|
||||
SESSION_TTL_DAYS: ${SESSION_TTL_DAYS:-30}
|
||||
MAX_ATTACHMENT_MB: ${MAX_ATTACHMENT_MB:-25}
|
||||
TRASH_RETENTION_DAYS: ${TRASH_RETENTION_DAYS:-30}
|
||||
SENDER_AVATARS_ENABLED: ${SENDER_AVATARS_ENABLED:-true}
|
||||
GOOGLE_OAUTH_ENABLED: ${GOOGLE_OAUTH_ENABLED:-false}
|
||||
OAUTH_GOOGLE_CLIENT_ID: ${OAUTH_GOOGLE_CLIENT_ID:-}
|
||||
OAUTH_GOOGLE_CLIENT_SECRET: ${OAUTH_GOOGLE_CLIENT_SECRET:-}
|
||||
OAUTH_MS_CLIENT_ID: ${OAUTH_MS_CLIENT_ID:-}
|
||||
OAUTH_MS_CLIENT_SECRET: ${OAUTH_MS_CLIENT_SECRET:-}
|
||||
# Worker sizing — see docker/.env.prod.example for what these cost in RAM.
|
||||
INTERACTIVE_CONCURRENCY: ${INTERACTIVE_CONCURRENCY:-8}
|
||||
BULK_CONCURRENCY: ${BULK_CONCURRENCY:-1}
|
||||
PARSE_CONCURRENCY: ${PARSE_CONCURRENCY:-2}
|
||||
STREAM_PARSE_MB: ${STREAM_PARSE_MB:-8}
|
||||
PER_HOST_CONN_CAP: ${PER_HOST_CONN_CAP:-10}
|
||||
MAX_ACCOUNTS_PER_WORKER: ${MAX_ACCOUNTS_PER_WORKER:-400}
|
||||
HOT_BUDGET_PER_WORKER: ${HOT_BUDGET_PER_WORKER:-50}
|
||||
APP_VERSION: ${IMAGE_TAG:-local}
|
||||
TZ: ${TZ:-UTC}
|
||||
|
||||
services:
|
||||
# One-shot: make the bind-mounted host folders writable by the unprivileged
|
||||
# container users before anything tries to use them. Runs as root by design.
|
||||
init:
|
||||
image: alpine:3.20
|
||||
user: root
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -e
|
||||
mkdir -p /mnt/blobs /mnt/backups /mnt/redis /mnt/meili
|
||||
chown -R 10001:10001 /mnt/blobs # backend runs as uid 10001
|
||||
chown -R 999:999 /mnt/backups # pg_dump runs as postgres (999)
|
||||
chown -R 999:999 /mnt/redis # redis runs as uid 999
|
||||
chmod 700 /mnt/backups
|
||||
echo "[init] data directories ready"
|
||||
volumes:
|
||||
- ${DATA_ROOT}/blobs:/mnt/blobs
|
||||
- ${DATA_ROOT}/backups:/mnt/backups
|
||||
- ${DATA_ROOT}/redis:/mnt/redis
|
||||
- ${DATA_ROOT}/meili:/mnt/meili
|
||||
restart: "no"
|
||||
logging: *logging
|
||||
|
||||
postgres:
|
||||
image: postgres:16
|
||||
environment:
|
||||
POSTGRES_USER: ${POSTGRES_USER:-webmail}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
POSTGRES_DB: ${POSTGRES_DB:-webmail}
|
||||
# Keeps the data directory one level down, so the bind mount can hold lost+found
|
||||
# and NAS metadata files without initdb refusing to start.
|
||||
PGDATA: /var/lib/postgresql/data/pgdata
|
||||
TZ: ${TZ:-UTC}
|
||||
command:
|
||||
- postgres
|
||||
- -c
|
||||
- shared_buffers=${PG_SHARED_BUFFERS:-256MB}
|
||||
- -c
|
||||
- max_connections=${PG_MAX_CONNECTIONS:-100}
|
||||
- -c
|
||||
- work_mem=${PG_WORK_MEM:-8MB}
|
||||
- -c
|
||||
- maintenance_work_mem=${PG_MAINTENANCE_WORK_MEM:-128MB}
|
||||
- -c
|
||||
- effective_cache_size=${PG_EFFECTIVE_CACHE_SIZE:-768MB}
|
||||
# Full page writes stay ON: NAS filesystems (btrfs/ext4) are not 8k-atomic.
|
||||
volumes:
|
||||
- ${DATA_ROOT:?set DATA_ROOT in your .env (see docker/.env.prod.example)}/postgres:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-webmail} -d ${POSTGRES_DB:-webmail}"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 12
|
||||
start_period: 30s
|
||||
restart: unless-stopped
|
||||
mem_limit: ${PG_MEM_LIMIT:-768m}
|
||||
stop_grace_period: 60s
|
||||
logging: *logging
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
# Leases and the arq queues live here. AOF everysec means a NAS power cut
|
||||
# costs at most a second of queue state instead of the whole queue.
|
||||
command:
|
||||
- redis-server
|
||||
- --appendonly
|
||||
- "yes"
|
||||
- --appendfsync
|
||||
- everysec
|
||||
- --maxmemory
|
||||
- ${REDIS_MAXMEMORY:-96mb}
|
||||
- --maxmemory-policy
|
||||
- noeviction
|
||||
volumes:
|
||||
- ${DATA_ROOT}/redis:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
depends_on:
|
||||
init: { condition: service_completed_successfully }
|
||||
restart: unless-stopped
|
||||
mem_limit: ${REDIS_MEM_LIMIT:-160m}
|
||||
logging: *logging
|
||||
|
||||
# Opt-in: `./deploy/webmail.sh up --meili` (or COMPOSE_PROFILES=meili).
|
||||
# Costs ~700MB-1GB RSS; with SEARCH_BACKEND=pg the stack runs without it.
|
||||
meilisearch:
|
||||
image: getmeili/meilisearch:v1.8
|
||||
profiles: ["meili"]
|
||||
environment:
|
||||
MEILI_MASTER_KEY: ${MEILI_MASTER_KEY:-}
|
||||
MEILI_ENV: production
|
||||
MEILI_NO_ANALYTICS: "true"
|
||||
TZ: ${TZ:-UTC}
|
||||
volumes:
|
||||
- ${DATA_ROOT}/meili:/meili_data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:7700/health | grep -q available"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 12
|
||||
start_period: 20s
|
||||
depends_on:
|
||||
init: { condition: service_completed_successfully }
|
||||
restart: unless-stopped
|
||||
mem_limit: ${MEILI_MEM_LIMIT:-1g}
|
||||
logging: *logging
|
||||
|
||||
# One-shot schema upgrade: core Alembic head + every plugin's own branch.
|
||||
# Everything else waits for this to exit 0, so a failed migration stops the
|
||||
# roll instead of starting code against an old schema.
|
||||
migrate:
|
||||
<<: *backend-image
|
||||
command: ["migrate"]
|
||||
environment:
|
||||
<<: *backend-env
|
||||
ENABLED_PLUGINS: ${ENABLED_PLUGINS:-calendar contacts labels filters}
|
||||
volumes:
|
||||
- ${DATA_ROOT}/blobs:/data/blobs
|
||||
depends_on:
|
||||
postgres: { condition: service_healthy }
|
||||
init: { condition: service_completed_successfully }
|
||||
restart: "no"
|
||||
logging: *logging
|
||||
|
||||
api:
|
||||
<<: *backend-image
|
||||
command: ["api"]
|
||||
environment: *backend-env
|
||||
volumes:
|
||||
- ${DATA_ROOT}/blobs:/data/blobs
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "python -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8000/api/healthz', timeout=5).status==200 else 1)\""]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
start_period: 30s
|
||||
depends_on:
|
||||
postgres: { condition: service_healthy }
|
||||
redis: { condition: service_healthy }
|
||||
migrate: { condition: service_completed_successfully }
|
||||
restart: unless-stopped
|
||||
mem_limit: ${API_MEM_LIMIT:-512m}
|
||||
cpus: ${API_CPUS:-1.0}
|
||||
logging: *logging
|
||||
|
||||
worker-sync:
|
||||
<<: *backend-image
|
||||
command: ["worker-sync"]
|
||||
environment: *backend-env
|
||||
volumes:
|
||||
- ${DATA_ROOT}/blobs:/data/blobs
|
||||
# Persistent IMAP IDLE connections; the default 1024 is fine for a personal
|
||||
# install but this costs nothing and matches the sync-engine v2 sizing.
|
||||
ulimits:
|
||||
nofile: { soft: 8192, hard: 8192 }
|
||||
depends_on:
|
||||
postgres: { condition: service_healthy }
|
||||
redis: { condition: service_healthy }
|
||||
migrate: { condition: service_completed_successfully }
|
||||
restart: unless-stopped
|
||||
mem_limit: ${WORKER_SYNC_MEM_LIMIT:-512m}
|
||||
cpus: ${WORKER_CPUS:-1.5}
|
||||
# Leases are released on SIGTERM (workers/run.py targets <10 s); give it room.
|
||||
stop_grace_period: 30s
|
||||
logging: *logging
|
||||
|
||||
worker-jobs:
|
||||
<<: *backend-image
|
||||
command: ["worker-jobs"]
|
||||
environment: *backend-env
|
||||
volumes:
|
||||
- ${DATA_ROOT}/blobs:/data/blobs
|
||||
ulimits:
|
||||
nofile: { soft: 8192, hard: 8192 }
|
||||
depends_on:
|
||||
postgres: { condition: service_healthy }
|
||||
redis: { condition: service_healthy }
|
||||
migrate: { condition: service_completed_successfully }
|
||||
restart: unless-stopped
|
||||
mem_limit: ${WORKER_JOBS_MEM_LIMIT:-640m}
|
||||
cpus: ${WORKER_CPUS:-1.5}
|
||||
stop_grace_period: 30s
|
||||
logging: *logging
|
||||
|
||||
web:
|
||||
image: ${REGISTRY_PREFIX:-webmail}/web:${IMAGE_TAG:-local}
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: docker/Dockerfile.frontend
|
||||
args:
|
||||
APP_VERSION: ${IMAGE_TAG:-local}
|
||||
GIT_SHA: ${GIT_SHA:-unknown}
|
||||
environment:
|
||||
TZ: ${TZ:-UTC}
|
||||
ports:
|
||||
# The ONLY published port. Bind to a specific NAS interface by setting
|
||||
# WEB_BIND (e.g. 192.168.1.185) to keep it off other networks.
|
||||
- "${WEB_BIND:-0.0.0.0}:${WEB_PORT:-8088}:80"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -qO- http://127.0.0.1/healthz | grep -q ok"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
depends_on:
|
||||
api: { condition: service_started }
|
||||
restart: unless-stopped
|
||||
mem_limit: ${WEB_MEM_LIMIT:-96m}
|
||||
logging: *logging
|
||||
|
||||
# Nightly logical backup of Postgres into ${DATA_ROOT}/backups, pruned to
|
||||
# BACKUP_KEEP_DAYS. Point Hyper Backup / HBS3 at ${DATA_ROOT} and this plus the
|
||||
# blob store is the whole recoverable state (the other half is MASTER_KEY —
|
||||
# see docs/nas-install.md §"The three things that matter").
|
||||
backup:
|
||||
image: postgres:16
|
||||
user: "999:999"
|
||||
entrypoint: ["/bin/bash", "/usr/local/bin/backup.sh"]
|
||||
environment:
|
||||
PGHOST: postgres
|
||||
PGUSER: ${POSTGRES_USER:-webmail}
|
||||
PGPASSWORD: ${POSTGRES_PASSWORD}
|
||||
PGDATABASE: ${POSTGRES_DB:-webmail}
|
||||
BACKUP_DIR: /backups
|
||||
BACKUP_AT: ${BACKUP_AT:-03:30}
|
||||
BACKUP_KEEP_DAYS: ${BACKUP_KEEP_DAYS:-14}
|
||||
TZ: ${TZ:-UTC}
|
||||
volumes:
|
||||
- ${DATA_ROOT}/backups:/backups
|
||||
- ./backup.sh:/usr/local/bin/backup.sh:ro
|
||||
depends_on:
|
||||
postgres: { condition: service_healthy }
|
||||
init: { condition: service_completed_successfully }
|
||||
restart: unless-stopped
|
||||
mem_limit: ${BACKUP_MEM_LIMIT:-256m}
|
||||
logging: *logging
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: webmail-prod
|
||||
Executable
+62
@@ -0,0 +1,62 @@
|
||||
#!/bin/sh
|
||||
# One image, several roles. Called as: entrypoint <command> [args...]
|
||||
#
|
||||
# api uvicorn, behind the `web` nginx container
|
||||
# worker-sync lease-owned account sync supervisor (SPEC §6.3)
|
||||
# worker-jobs arq consumer (interactive + bulk queues)
|
||||
# worker-all both in one process (small installs; halves RAM use)
|
||||
# migrate core Alembic upgrade + per-plugin install; exits 0 when done
|
||||
# reindex rebuild the search index from Postgres
|
||||
# plugins `python -m app.plugins.cli ...` passthrough
|
||||
# shell /bin/sh (debugging)
|
||||
#
|
||||
# Any other argument is exec'd verbatim, so `docker compose run api python -c ...`
|
||||
# still works.
|
||||
set -eu
|
||||
|
||||
: "${API_HOST:=0.0.0.0}"
|
||||
: "${API_PORT:=8000}"
|
||||
: "${API_WORKERS:=1}"
|
||||
# Plugins whose migrations are applied on every `migrate` run. Space-separated so
|
||||
# an operator can add a third-party plugin without editing the image.
|
||||
: "${ENABLED_PLUGINS:=calendar contacts labels filters}"
|
||||
|
||||
cmd="${1:-api}"
|
||||
[ "$#" -gt 0 ] && shift || true
|
||||
|
||||
run_migrations() {
|
||||
echo "[entrypoint] core schema: alembic upgrade head"
|
||||
alembic upgrade head
|
||||
for plugin in $ENABLED_PLUGINS; do
|
||||
echo "[entrypoint] plugin schema: $plugin"
|
||||
python -m app.plugins.cli install "$plugin"
|
||||
done
|
||||
echo "[entrypoint] migrations complete"
|
||||
}
|
||||
|
||||
case "$cmd" in
|
||||
api)
|
||||
# --proxy-headers + --forwarded-allow-ips are what let the app see the
|
||||
# real scheme through nginx/DSM. Without them every request looks like
|
||||
# plain http and the session cookie loses its __Host- prefix and Secure
|
||||
# flag (app/identity/cookies.py:22). The allow-list is the internal
|
||||
# docker network only; this port is never published directly.
|
||||
exec uvicorn app.main:create_app \
|
||||
--factory \
|
||||
--host "$API_HOST" \
|
||||
--port "$API_PORT" \
|
||||
--workers "$API_WORKERS" \
|
||||
--proxy-headers \
|
||||
--forwarded-allow-ips "${FORWARDED_ALLOW_IPS:-*}" \
|
||||
--no-server-header \
|
||||
"$@"
|
||||
;;
|
||||
worker-sync) WORKER_ROLE=sync exec python -m workers.run "$@" ;;
|
||||
worker-jobs) WORKER_ROLE=jobs exec python -m workers.run "$@" ;;
|
||||
worker-all) WORKER_ROLE=all exec python -m workers.run "$@" ;;
|
||||
migrate) run_migrations ;;
|
||||
reindex) exec python scripts/reindex_search.py "$@" ;;
|
||||
plugins) exec python -m app.plugins.cli "$@" ;;
|
||||
shell) exec /bin/sh "$@" ;;
|
||||
*) exec "$cmd" "$@" ;;
|
||||
esac
|
||||
@@ -0,0 +1,104 @@
|
||||
# Webmail front door: static SPA + reverse proxy to the API container.
|
||||
#
|
||||
# This file is dropped into /etc/nginx/conf.d/, i.e. it is already inside the
|
||||
# `http` block — hence the bare `map` below.
|
||||
|
||||
# Preserve the original scheme when something else terminates TLS in front of us
|
||||
# (Synology's Login Portal reverse proxy, QNAP's, Caddy, Cloudflare Tunnel...).
|
||||
# The API only emits Secure/__Host- cookies when it believes the request was
|
||||
# https, so losing this header silently downgrades session cookies.
|
||||
map $http_x_forwarded_proto $fwd_proto {
|
||||
default $scheme;
|
||||
https https;
|
||||
http http;
|
||||
}
|
||||
|
||||
# Connection header for the WebSocket upgrade dance on /api/ws.
|
||||
map $http_upgrade $connection_upgrade {
|
||||
default upgrade;
|
||||
'' close;
|
||||
}
|
||||
|
||||
upstream webmail_api {
|
||||
server api:8000;
|
||||
keepalive 16;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name _;
|
||||
|
||||
root /usr/share/nginx/html;
|
||||
index index.html;
|
||||
|
||||
# Attachment uploads. Keep this comfortably above MAX_ATTACHMENT_MB (default
|
||||
# 25) to leave room for multipart/base64 overhead.
|
||||
client_max_body_size 64m;
|
||||
client_body_timeout 300s;
|
||||
|
||||
# The NAS is doing other jobs too — don't buffer big uploads to disk twice.
|
||||
proxy_request_buffering off;
|
||||
|
||||
gzip on;
|
||||
gzip_vary on;
|
||||
gzip_min_length 1024;
|
||||
gzip_types text/plain text/css application/javascript application/json
|
||||
image/svg+xml application/manifest+json;
|
||||
|
||||
# Clickjacking / sniffing basics. The app's own CSP for mail HTML is applied
|
||||
# server-side in app/indexer/sanitize.py; this is only the shell document.
|
||||
add_header X-Content-Type-Options nosniff always;
|
||||
add_header X-Frame-Options SAMEORIGIN always;
|
||||
add_header Referrer-Policy no-referrer always;
|
||||
|
||||
# ---- API ----------------------------------------------------------------
|
||||
location /api/ {
|
||||
proxy_pass http://webmail_api;
|
||||
proxy_http_version 1.1;
|
||||
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $fwd_proto;
|
||||
proxy_set_header X-Forwarded-Host $host;
|
||||
|
||||
# WebSocket (/api/ws) and any long-poll: no upgrade means no live mail.
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection $connection_upgrade;
|
||||
|
||||
# Live updates hold the connection open; don't cut them at 60 s.
|
||||
proxy_read_timeout 3600s;
|
||||
proxy_send_timeout 3600s;
|
||||
proxy_buffering off;
|
||||
}
|
||||
|
||||
# ---- static -------------------------------------------------------------
|
||||
# Vite emits content-hashed filenames into /assets, so they are immutable.
|
||||
location /assets/ {
|
||||
expires 1y;
|
||||
add_header Cache-Control "public, immutable";
|
||||
try_files $uri =404;
|
||||
}
|
||||
|
||||
location /fonts/ {
|
||||
expires 30d;
|
||||
add_header Cache-Control "public";
|
||||
try_files $uri =404;
|
||||
}
|
||||
|
||||
location = /healthz {
|
||||
access_log off;
|
||||
return 200 "ok\n";
|
||||
add_header Content-Type text/plain;
|
||||
}
|
||||
|
||||
# SPA fallback: every non-file route renders index.html (react-router).
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
|
||||
# index.html itself must never be cached, or a deploy won't be picked up.
|
||||
location = /index.html {
|
||||
add_header Cache-Control "no-store, must-revalidate";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,402 @@
|
||||
# Running Webmail on a Synology or QNAP NAS
|
||||
|
||||
This is the operator runbook: how to install it, how to keep your mail safe from
|
||||
anything Docker does to itself, and how to update without holding your breath.
|
||||
It assumes the NAS is doing other work too, so the stack is deliberately modest:
|
||||
**one published port, no host networking, memory caps on every container.**
|
||||
|
||||
Everything here works on any Intel/AMD Docker host — the NAS parts are just the
|
||||
specifics for DSM and QTS.
|
||||
|
||||
---
|
||||
|
||||
## 1. What gets installed
|
||||
|
||||
| Container | What it does | Published? |
|
||||
|---|---|---|
|
||||
| `web` | nginx: serves the UI, proxies `/api` (incl. the WebSocket) | **yes — the only one** |
|
||||
| `api` | FastAPI app | no, internal network only |
|
||||
| `worker-sync` | keeps IMAP connections open, syncs mail | no |
|
||||
| `worker-jobs` | background jobs (send, backfill, purge, plugin jobs) | no |
|
||||
| `postgres` | the database | no |
|
||||
| `redis` | job queues + sync leases | no |
|
||||
| `backup` | nightly verified database dump | no |
|
||||
| `meilisearch` | optional better search (see §8) | no |
|
||||
| `migrate`, `init` | run once at start, then exit | — |
|
||||
|
||||
Plugins (calendar, contacts, labels, filters) are baked into the backend image
|
||||
and their database schema is applied automatically on every start.
|
||||
|
||||
**Requirements:** an Intel/AMD NAS with Docker (Synology *Container Manager*,
|
||||
QNAP *Container Station*), ~2 GB RAM free, and 10 GB+ of disk for a typical
|
||||
personal mailbox. Nothing is needed from the internet except your mail servers.
|
||||
|
||||
---
|
||||
|
||||
## 2. Install
|
||||
|
||||
### 2.1 Pick where the data lives
|
||||
|
||||
This is the single most important decision. Use a **shared folder**, never a
|
||||
Docker named volume:
|
||||
|
||||
| NAS | Use |
|
||||
|---|---|
|
||||
| Synology | `/volume1/docker/webmail/data` |
|
||||
| QNAP | `/share/Container/webmail/data` |
|
||||
|
||||
Why: named volumes live inside Docker's internal storage, where Hyper Backup,
|
||||
Snapshot Replication and HBS3 cannot see them, and where "delete the project" in
|
||||
the NAS UI can take your mail with it. A shared folder is backed up by the tools
|
||||
you already run, survives every container operation, and can be copied to
|
||||
another machine wholesale.
|
||||
|
||||
> **Never put this on an NFS or SMB mount.** Postgres on a network filesystem
|
||||
> corrupts. `./deploy/webmail.sh doctor` checks this and refuses to be quiet
|
||||
> about it.
|
||||
|
||||
### 2.2 Get the code onto the NAS
|
||||
|
||||
Enable SSH on the NAS (DSM: *Control Panel → Terminal & SNMP → Enable SSH*;
|
||||
QTS: *Control Panel → Telnet/SSH*), then:
|
||||
|
||||
```bash
|
||||
ssh admin@your-nas
|
||||
sudo -i
|
||||
mkdir -p /volume1/docker/webmail && cd /volume1/docker/webmail
|
||||
git clone <your-gitea-url>/webmail.git src && cd src
|
||||
```
|
||||
|
||||
No git on the NAS? Copy the folder over with File Station or `rsync` instead.
|
||||
|
||||
### 2.3 Configure
|
||||
|
||||
```bash
|
||||
./deploy/webmail.sh init
|
||||
```
|
||||
|
||||
This creates `docker/.env.prod` with a freshly generated `MASTER_KEY` and
|
||||
database password. Then open it and set:
|
||||
|
||||
```ini
|
||||
DATA_ROOT=/volume1/docker/webmail/data # or /share/Container/webmail/data
|
||||
APP_DOMAIN=mail.home.lan # your handles become you@this
|
||||
TZ=Europe/Warsaw
|
||||
WEB_PORT=8088 # avoid 5000/5001 (DSM), 8080 (QTS)
|
||||
WEB_BIND=0.0.0.0 # or one NAS IP, e.g. 192.168.1.185
|
||||
```
|
||||
|
||||
> ### Copy `MASTER_KEY` into your password manager. Now.
|
||||
> It encrypts every stored mail-account password. It exists **only** in this
|
||||
> file. A perfect database backup without this key gives you your mail but not
|
||||
> your credentials — you would have to re-enter every account password by hand.
|
||||
> `docker/.env.prod` is `chmod 600` and git-ignored; treat it as a backup
|
||||
> artefact in its own right.
|
||||
|
||||
### 2.4 Start
|
||||
|
||||
```bash
|
||||
./deploy/webmail.sh doctor # sanity-checks config, disk, filesystem, ports
|
||||
./deploy/webmail.sh up # builds the images on first run, then starts
|
||||
```
|
||||
|
||||
First run builds both images. The frontend build wants ~2 GB of RAM: if your NAS
|
||||
is small or you'd rather not tie it up, build on your PC instead — see §6.3.
|
||||
|
||||
Open `http://your-nas:8088/`, create your account, and add a mail account
|
||||
through the wizard.
|
||||
|
||||
### 2.5 Synology Container Manager (GUI instead of SSH)
|
||||
|
||||
Yes, this runs from the GUI. *Container Manager → Project → Create*, point it at
|
||||
the **repo root** (`/volume1/docker/webmail/src`), and name the project
|
||||
`webmail-prod`.
|
||||
|
||||
Point it at the repo root, **not** at `docker/`. Both folders contain a
|
||||
`docker-compose.yml`: the one in `docker/` is the throwaway *dev* service stack
|
||||
(Postgres/Redis/Meilisearch/GreenMail on dev ports) and starting it does not
|
||||
give you a webmail server. The root one is production.
|
||||
|
||||
Name the project `webmail-prod` too — the name you type in the UI overrides the
|
||||
one in the compose file, and keeping them the same means `deploy/webmail.sh`
|
||||
and the GUI manage the same stack instead of two rival copies.
|
||||
|
||||
**You still need a shell once**, to create `docker/.env.prod` with a generated
|
||||
`MASTER_KEY` — there is no way to generate a random key in the GUI. Either SSH
|
||||
in and run `./deploy/webmail.sh init`, or, if you would rather not enable SSH at
|
||||
all, use *Control Panel → Task Scheduler → Create → Scheduled Task → User-defined
|
||||
script*, run as `root`, with:
|
||||
|
||||
```bash
|
||||
cd /volume1/docker/webmail/src && ./deploy/webmail.sh init
|
||||
```
|
||||
|
||||
Run it once, then edit `docker/.env.prod` in DSM's Text Editor (`init` also puts
|
||||
a `.env` symlink in the repo root, which is what Container Manager reads — one
|
||||
file, one place to edit).
|
||||
|
||||
**What you get in the GUI:** start/stop, live logs, per-container resource use,
|
||||
automatic restart after reboot, and the nightly database backups (the `backup`
|
||||
service is an ordinary container, so it runs regardless of how you started the
|
||||
stack).
|
||||
|
||||
**What the GUI cannot do:**
|
||||
|
||||
| | Why | Instead |
|
||||
|---|---|---|
|
||||
| Start Meilisearch | compose profiles aren't exposed in the UI | `./deploy/webmail.sh up` — or delete the `profiles:` line from `docker/docker-compose.prod.yml` |
|
||||
| Health-gated updates | it just pulls and restarts; no gate, no rollback | `./deploy/webmail.sh update` |
|
||||
| Backup before migrating | — | `./deploy/webmail.sh update` does it automatically |
|
||||
| Restore a dump | — | `./deploy/webmail.sh restore <file>` |
|
||||
| Pre-flight checks, reindex | — | `./deploy/webmail.sh doctor` / `reindex` |
|
||||
|
||||
**Two things that look wrong but aren't:** `init` and `migrate` sit there as
|
||||
**Exited (0)** — they are one-shot jobs that did their work and stopped, and
|
||||
every other service waited for them. And the first start builds both images on
|
||||
the NAS, which is slow and wants ~2 GB of RAM; §6.3 shows how to build them on
|
||||
your PC instead.
|
||||
|
||||
After editing `.env`, containers keep their old values until they are recreated
|
||||
— `./deploy/webmail.sh up` does that cleanly.
|
||||
|
||||
The honest recommendation: install and update over SSH, and use the GUI for
|
||||
day-to-day watching, starting and stopping. The update path is where all the
|
||||
safety lives, and it is the one thing the GUI has no equivalent for.
|
||||
|
||||
### 2.6 Starting automatically after a reboot
|
||||
|
||||
Every service is `restart: unless-stopped`, so Docker brings the stack back on
|
||||
boot by itself. On Synology, make sure Container Manager itself is set to start
|
||||
on boot (it is by default).
|
||||
|
||||
---
|
||||
|
||||
## 3. The three things that matter
|
||||
|
||||
Containers are disposable. These are not:
|
||||
|
||||
| # | What | Where | If you lose it |
|
||||
|---|---|---|---|
|
||||
| 1 | **Database** | `$DATA_ROOT/postgres` + nightly dumps in `$DATA_ROOT/backups` | Everything: accounts, threads, flags, calendar, contacts |
|
||||
| 2 | **Blob store** | `$DATA_ROOT/blobs` | Mail bodies and attachments (re-downloadable from the servers, slowly) |
|
||||
| 3 | **`MASTER_KEY`** | `docker/.env.prod` + your password manager | Every saved mail-account password, permanently |
|
||||
|
||||
Everything else is derived and needs no backup: Redis holds queues and leases,
|
||||
Meilisearch's index rebuilds from Postgres (§8), and images rebuild from source.
|
||||
|
||||
### Backups
|
||||
|
||||
Two layers, and you want both:
|
||||
|
||||
**Nightly dumps (automatic).** The `backup` container writes a compressed
|
||||
`pg_dump` to `$DATA_ROOT/backups` at `BACKUP_AT` (default 03:30) and keeps
|
||||
`BACKUP_KEEP_DAYS` (14) of them. Every dump is verified with `pg_restore --list`
|
||||
before it counts, so you never find out at restore time that it was truncated.
|
||||
|
||||
**Off-box copies (you set this up, once).** Point your NAS backup tool at
|
||||
`$DATA_ROOT`:
|
||||
|
||||
- Synology: *Hyper Backup* → local/remote/cloud destination → include the
|
||||
`docker/webmail` shared folder. Add *Snapshot Replication* on a Btrfs volume
|
||||
for fast local rollback.
|
||||
- QNAP: *HBS 3* → the `Container/webmail` folder. Add snapshots on QuTS hero.
|
||||
|
||||
A snapshot of a *running* Postgres directory is crash-consistent — it usually
|
||||
replays fine, but it is not a substitute for the dumps, which are verified and
|
||||
restorable into a different Postgres version. Keep both.
|
||||
|
||||
Also back up `docker/.env.prod` somewhere that is not the NAS.
|
||||
|
||||
### Restore drill — do this once, before you need it
|
||||
|
||||
```bash
|
||||
./deploy/webmail.sh status # see available dumps
|
||||
./deploy/webmail.sh restore webmail-20260914-033000.dump
|
||||
```
|
||||
|
||||
It stops the app containers, restores in a single transaction (a failure leaves
|
||||
the old database untouched), and starts everything back up.
|
||||
|
||||
Full disaster recovery onto a new box:
|
||||
|
||||
1. Install Docker, copy `$DATA_ROOT` back from your backup.
|
||||
2. Copy the repo and `docker/.env.prod` — **with the original `MASTER_KEY`**.
|
||||
3. `./deploy/webmail.sh up`
|
||||
|
||||
That's it. If the blob store was lost but the database survived, mail bodies
|
||||
refill from the mail servers as accounts re-sync.
|
||||
|
||||
---
|
||||
|
||||
## 4. Updating
|
||||
|
||||
```bash
|
||||
git pull
|
||||
./deploy/webmail.sh update
|
||||
```
|
||||
|
||||
`update` does, in this order, stopping the moment anything fails:
|
||||
|
||||
1. **Takes a verified database backup.** No backup, no update.
|
||||
2. Builds (or pulls) the new images. A failure here changes nothing.
|
||||
3. Runs migrations as a one-shot container — **before** any service restarts, so
|
||||
a bad migration never leaves new code running against an old schema.
|
||||
4. Rolls workers first, then API, then web. Sync leases transfer to the
|
||||
replacement worker within ~90 s and message upserts are idempotent, so no
|
||||
mail is lost or duplicated by a roll.
|
||||
5. Health-gates for 120 s. **If the new version doesn't come up healthy it rolls
|
||||
itself back automatically** and leaves you on the previous version.
|
||||
|
||||
To go back manually:
|
||||
|
||||
```bash
|
||||
./deploy/webmail.sh rollback # previous version
|
||||
./deploy/webmail.sh rollback v1.2.0 # a specific one
|
||||
```
|
||||
|
||||
Rolling *code* back does not require rolling the *schema* back, as long as
|
||||
migrations stay expand-then-contract (see `docs/migrations.md` discipline in
|
||||
`deployment-setup-instructions.md` §9). For the rare breaking migration, restore
|
||||
the dump that `update` took in step 1.
|
||||
|
||||
### Building images elsewhere
|
||||
|
||||
Building on the NAS is simple but slow and RAM-hungry. Two alternatives:
|
||||
|
||||
```bash
|
||||
# On your PC — push to your Gitea registry:
|
||||
REGISTRY_PREFIX=192.168.1.185:8418/claude/webmail ./deploy/build-push.sh v1.0.0
|
||||
# then on the NAS, set REGISTRY_PREFIX + IMAGE_TAG in docker/.env.prod and:
|
||||
./deploy/webmail.sh update v1.0.0
|
||||
|
||||
# Or with no registry at all — build to a file and carry it over:
|
||||
./deploy/build-push.sh v1.0.0 --save
|
||||
# copy webmail-v1.0.0.tar to the NAS, then:
|
||||
docker load -i webmail-v1.0.0.tar && ./deploy/webmail.sh update v1.0.0
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. Reaching it from outside the LAN
|
||||
|
||||
### 5.1 Read this first
|
||||
|
||||
**Anyone who can open the page can create an account on it.** Registration is
|
||||
open by design (it's a personal server), so do not put this straight on the
|
||||
public internet. Pick one:
|
||||
|
||||
- **VPN** (Tailscale, WireGuard, or the NAS's own VPN Server) — simplest and
|
||||
safest. The port stays LAN-only.
|
||||
- **NAS reverse proxy with HTTPS**, restricted by firewall rules to the networks
|
||||
you trust.
|
||||
|
||||
### 5.2 HTTPS through the NAS reverse proxy
|
||||
|
||||
DSM owns ports 80/443, which is why the stack publishes 8088. To put DSM's
|
||||
certificate in front of it:
|
||||
|
||||
*Control Panel → Login Portal → Advanced → Reverse Proxy → Create*
|
||||
|
||||
| Field | Value |
|
||||
|---|---|
|
||||
| Source protocol / hostname / port | HTTPS / `mail.yourdomain` / 443 |
|
||||
| Destination protocol / hostname / port | HTTP / `localhost` / `8088` |
|
||||
|
||||
Then open **Custom Header → Create → WebSocket** and save. Without that,
|
||||
`/api/ws` is blocked and the UI stops updating live.
|
||||
|
||||
QNAP: *Network & Virtual Switch → Reverse Proxy* — same idea, and likewise
|
||||
enable WebSocket support.
|
||||
|
||||
Session cookies get the `Secure` and `__Host-` treatment automatically as soon
|
||||
as the app sees an HTTPS request: nginx forwards `X-Forwarded-Proto`, and the
|
||||
API is started with `--proxy-headers`. Nothing to configure.
|
||||
|
||||
---
|
||||
|
||||
## 6. Resource usage
|
||||
|
||||
Steady state for one or two mail accounts, `SEARCH_BACKEND=pg`:
|
||||
|
||||
| | Typical | Cap (`docker/.env.prod`) |
|
||||
|---|---|---|
|
||||
| postgres | 150-300 MB | `PG_MEM_LIMIT=768m` |
|
||||
| api | 150-250 MB | `API_MEM_LIMIT=512m` |
|
||||
| worker-sync | 120-250 MB | `WORKER_SYNC_MEM_LIMIT=512m` |
|
||||
| worker-jobs | 150-350 MB | `WORKER_JOBS_MEM_LIMIT=640m` |
|
||||
| redis / web / backup | < 100 MB total | — |
|
||||
| **total** | **~0.7-1.2 GB** | ~2.2 GB of limits |
|
||||
| meilisearch (optional) | 300-800 MB | `MEILI_MEM_LIMIT=1g` |
|
||||
|
||||
On a 2 GB NAS: stay on `SEARCH_BACKEND=pg` and drop `PG_SHARED_BUFFERS` to
|
||||
`128MB`. Tight but workable. On 4 GB+ you have room for Meilisearch.
|
||||
|
||||
Two NAS-specific realities worth knowing:
|
||||
|
||||
- **Disk hibernation won't happen.** Active accounts hold IMAP IDLE connections
|
||||
open and write as mail arrives. That is the feature working, not a bug.
|
||||
- **Keep NTP on.** Two-factor codes fail on a drifting clock.
|
||||
|
||||
---
|
||||
|
||||
## 7. Search: Postgres or Meilisearch
|
||||
|
||||
`SEARCH_BACKEND=pg` (default) uses Postgres full-text search. No extra
|
||||
container, no extra RAM, and it is perfectly good for a personal mailbox.
|
||||
|
||||
`SEARCH_BACKEND=meili` adds typo tolerance, better ranking and stays fast on
|
||||
large mailboxes, at the cost of a container and 300-800 MB.
|
||||
|
||||
To switch:
|
||||
|
||||
```bash
|
||||
# in docker/.env.prod: SEARCH_BACKEND=meili and set MEILI_MASTER_KEY
|
||||
./deploy/webmail.sh up # starts Meilisearch automatically
|
||||
./deploy/webmail.sh reindex # build the index from existing mail
|
||||
```
|
||||
|
||||
The index is **derived data** — every field it holds also lives in Postgres — so
|
||||
it never needs backing up. If the Meilisearch volume is lost, or you switch
|
||||
backends, or search ever looks stale, just rebuild it:
|
||||
|
||||
```bash
|
||||
./deploy/webmail.sh reindex # upsert everything (safe to re-run)
|
||||
./deploy/webmail.sh reindex --purge # drop stale documents first
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 8. Troubleshooting
|
||||
|
||||
| Symptom | Cause | Fix |
|
||||
|---|---|---|
|
||||
| `up` hangs on `postgres` | `DATA_ROOT` on NFS/SMB, or wrong permissions | `./deploy/webmail.sh doctor` |
|
||||
| UI loads, no live updates | WebSocket not enabled in the NAS reverse proxy | §5.2 Custom Header → WebSocket |
|
||||
| Login works, logs out on reload | HTTPS terminated upstream but `X-Forwarded-Proto` stripped | Use the NAS reverse proxy as documented; don't strip headers |
|
||||
| "port is already allocated" | 8088 taken by another app | change `WEB_PORT` |
|
||||
| Mail not syncing | worker stopped, or credentials rejected | `./deploy/webmail.sh logs worker-sync` |
|
||||
| Search finds nothing after a restore | index not rebuilt | `./deploy/webmail.sh reindex --purge` |
|
||||
| App won't start after editing `.env.prod` | malformed `MASTER_KEY` | `./deploy/webmail.sh doctor` |
|
||||
|
||||
Logs: `./deploy/webmail.sh logs [service] -f`. All logs are capped at 10 MB ×3
|
||||
per container, so they can't fill the volume.
|
||||
|
||||
---
|
||||
|
||||
## 9. Command reference
|
||||
|
||||
```
|
||||
./deploy/webmail.sh init create data dirs + generate secrets
|
||||
./deploy/webmail.sh doctor pre-flight checks
|
||||
./deploy/webmail.sh up [--meili] start everything
|
||||
./deploy/webmail.sh down stop (data untouched)
|
||||
./deploy/webmail.sh status what's running, versions, disk, backups
|
||||
./deploy/webmail.sh logs [svc] [-f] logs
|
||||
./deploy/webmail.sh update [tag] backup -> migrate -> roll -> health-gate
|
||||
./deploy/webmail.sh rollback [tag] back to the previous version
|
||||
./deploy/webmail.sh backup verified dump, right now
|
||||
./deploy/webmail.sh restore <dump> restore a dump (asks for confirmation)
|
||||
./deploy/webmail.sh reindex [--purge] rebuild the search index
|
||||
./deploy/webmail.sh psql database shell
|
||||
./deploy/webmail.sh shell [svc] shell in a container
|
||||
```
|
||||
@@ -0,0 +1,50 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<svg
|
||||
width="26"
|
||||
height="26"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
version="1.1"
|
||||
id="svg3"
|
||||
sodipodi:docname="document.svg"
|
||||
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
|
||||
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
xmlns:svg="http://www.w3.org/2000/svg">
|
||||
<defs
|
||||
id="defs3" />
|
||||
<sodipodi:namedview
|
||||
id="namedview3"
|
||||
pagecolor="#ffffff"
|
||||
bordercolor="#000000"
|
||||
borderopacity="0.25"
|
||||
inkscape:showpageshadow="2"
|
||||
inkscape:pageopacity="0.0"
|
||||
inkscape:pagecheckerboard="0"
|
||||
inkscape:deskcolor="#d1d1d1" />
|
||||
<rect
|
||||
style="fill:#333333;stroke-width:2.53985;stroke-linecap:round"
|
||||
id="rect3"
|
||||
width="13.26046"
|
||||
height="4.0990605"
|
||||
x="2.8693423"
|
||||
y="1.8650727" />
|
||||
<path
|
||||
d="m 6,2.4795047 h 7 l 6,5.5 V 20.979505 a 1.5,1.5 0 0 1 -1.5,1.5 h -11 a 1.5,1.5 0 0 1 -1.5,-1.5 V 3.9795047 a 1.5,1.5 0 0 1 1,-1.5 z"
|
||||
fill="#2563eb"
|
||||
opacity="0.15"
|
||||
id="path1"
|
||||
style="fill:#25eb33;fill-opacity:1" />
|
||||
<path
|
||||
d="M13 2.5L19 8h-4.5A1.5 1.5 0 0 1 13 6.5V2.5z"
|
||||
fill="#2563eb"
|
||||
opacity="0.4"
|
||||
id="path2"
|
||||
style="fill:#2beb25;fill-opacity:1" />
|
||||
<path
|
||||
d="M8 12.5h8M8 15.5h8M8 18.5h5"
|
||||
stroke="#2563eb"
|
||||
stroke-width="1.3"
|
||||
stroke-linecap="round"
|
||||
id="path3" />
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 1.4 KiB |
Reference in New Issue
Block a user