lukas/webmail-client
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Attachments stored once; nightly clean-up of unused files

Browse Source 
Attachments are no longer stored apart from their message (they were kept
twice: base64 inside the raw message and decoded on their own). The indexer
records the MIME part (attachments.part_index, migration 0023); downloads,
zip, inline images, thumbnails and forwarding take it out of the message;
videos stream from a short-lived copy (kept a day, 1 GiB per user). A
background pass on worker start removes the copies stored before, each only
after checking it can be taken out again (spec App. D §D.6 item 1, criterion
corrected to <= 60%).

A nightly job (03:47, after the Trash/Spam purge) removes files nothing
refers to any more — deleted or purged messages, previews of attachments
gone, uploads no draft has used for a week, expired streaming copies. Files
younger than an hour are left alone; a sent message waiting for its Sent copy
is held until its job is done. Account removal keeps files a draft still uses.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
lukas
2026-10-08 20:20:21 +02:00
co-authored by Claude Opus 5.5
parent 0514053252
commit 3957dd540a
17 changed files with 1063 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/indexer/attachments.py
+237
View File
@@ -0,0 +1,237 @@
"""Attachments, kept once: inside their message (SPEC App. D §D.6 item 1).
An attachment is not stored apart from its message any more — it was, which
kept most of a mailbox's bytes twice (once base64 in the raw message, once
decoded on its own). It is taken out of the raw message when wanted:
* :func:`load` — its bytes (downloads, zip, inline images, forwarding, thumbnails);
* :func:`file` — a file holding it, for streaming with HTTP Range (videos):
taken out into ``{user}/cache/`` and kept a day after its last use;
* :func:`dedupe_existing` — the one-time pass removing the separate copies
stored before, each only after checking it can be taken out of its message.
Blocking (parsing, disk): call these in a thread.
"""
from __future__ import annotations
import contextlib
import hashlib
import logging
import time
import uuid
from collections import defaultdict
from pathlib import Path
from app.indexer import blobstore
from app.indexer.mime import load_message, part_payload
logger = logging.getLogger(__name__)
CACHE_KEEP_S = 24 * 3600 # a taken-out file is kept this long after its last use
CACHE_MAX_BYTES = 1024**3 # per user; the least recently used go first beyond it
# part_index of a row whose content is not in its message (kept as its own file).
NOT_IN_MESSAGE = -1
def _legacy(user_id: uuid.UUID, sha: str) -> Path | None:
"""The separate copy stored before attachments were kept once, if still there."""
path = blobstore.blob_path(user_id, sha)
return path if path.exists() else None
def load(user_id: uuid.UUID, message_sha: str, sha: str, part_index: int | None) -> bytes:
"""An attachment's bytes. Raises FileNotFoundError when they cannot be had."""
legacy = _legacy(user_id, sha)
if legacy is not None:
return legacy.read_bytes()
raw = blobstore.blob_path(user_id, message_sha)
if part_index == NOT_IN_MESSAGE or not raw.exists():
raise FileNotFoundError(sha)
payload = part_payload(load_message(path=raw), part_index, sha)
if payload is None:
raise FileNotFoundError(sha)
return payload
def materialize(user_id: uuid.UUID, message_sha: str, sha: str, part_index: int | None) -> bool:
"""Make sure the attachment exists as a blob of its own — for a compose
attachment (forward, a draft reopened), which is read by its hash. False
when its bytes cannot be had."""
if blobstore.exists(user_id, sha):
return True
try:
blobstore.store(user_id, load(user_id, message_sha, sha, part_index))
except FileNotFoundError:
return False
return True
def _cache_dir(user_id: uuid.UUID) -> Path:
return blobstore.blob_path(user_id, "00").parent.parent / "cache"
def prune_cache(cache: Path) -> None:
"""Drop streaming copies past their keep time, and the oldest beyond the cap."""
files = [(p, p.stat()) for p in cache.iterdir() if p.is_file()]
now = time.time()
total = 0
for path, st in sorted(files, key=lambda f: f[1].st_mtime, reverse=True):
total += st.st_size
if now - st.st_mtime > CACHE_KEEP_S or total > CACHE_MAX_BYTES:
path.unlink(missing_ok=True)
def file(user_id: uuid.UUID, message_sha: str, sha: str, part_index: int | None) -> Path:
"""A file holding the attachment, to stream from. Raises FileNotFoundError."""
legacy = _legacy(user_id, sha)
if legacy is not None:
return legacy
cache = _cache_dir(user_id)
path = cache / sha
if path.exists():
path.touch() # used again: kept longer
return path
data = load(user_id, message_sha, sha, part_index)
cache.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(f".{uuid.uuid4().hex}.tmp")
tmp.write_bytes(data)
tmp.replace(path)
with contextlib.suppress(OSError): # a file removed meanwhile by another request
prune_cache(cache)
return path
def drop_cached(user_id: uuid.UUID) -> None:
"""Forget every taken-out file of a user (account removed)."""
cache = _cache_dir(user_id)
if cache.exists():
for path in cache.iterdir():
path.unlink(missing_ok=True)
# ---- removing the copies stored before ------------------------------------------------
# A file this recent may belong to work in progress (a sent message waiting
# for its Sent copy is referenced only by its job): left for the next pass.
_MIN_AGE_S = 3600
def _part_indexes(raw_path: Path) -> dict[str, int]:
"""Content hash → MIME part position, for every leaf part of a message."""
out: dict[str, int] = {}
for index, part in enumerate(load_message(path=raw_path).walk()):
if part.is_multipart():
continue
payload = part.get_payload(decode=True)
if isinstance(payload, bytes):
out.setdefault(hashlib.sha256(payload).hexdigest(), index)
return out
async def dedupe_existing(batch: int = 200, *, user_id: uuid.UUID | None = None) -> dict[str, int]:
"""Record where each older attachment sits in its message, and delete its
separate copy once it is known to be there. Idempotent; resumable."""
import asyncio
from sqlalchemy import select, update
from app.db import session_scope
from app.models import Attachment, ComposeAttachment, Message
stats = {"rows": 0, "found": 0, "not_found": 0, "files_removed": 0, "bytes_freed": 0}
while True:
async with session_scope() as db:
rows = (
await db.execute(
select(
Attachment.id,
Attachment.user_id,
Attachment.blob_sha256,
Message.blob_sha256,
)
.join(Message, Message.id == Attachment.message_id)
.where(
Attachment.part_index.is_(None),
*([Attachment.user_id == user_id] if user_id is not None else []),
)
.order_by(Attachment.message_id)
.limit(batch)
)
).all()
if not rows:
return stats
by_message: dict[tuple[uuid.UUID, str], list[tuple[uuid.UUID, str]]] = defaultdict(list)
for att_id, owner, sha, msg_sha in rows:
by_message[(owner, msg_sha)].append((att_id, sha))
located: dict[uuid.UUID, int] = {}
for (owner, msg_sha), atts in by_message.items():
raw = blobstore.blob_path(owner, msg_sha)
try:
indexes = await asyncio.to_thread(_part_indexes, raw) if raw.exists() else {}
except Exception: # noqa: BLE001 - unreadable message: keep the copies
logger.warning("attachments: cannot parse %s", raw, exc_info=True)
indexes = {}
for att_id, sha in atts:
located[att_id] = indexes.get(sha, NOT_IN_MESSAGE)
async with session_scope() as db:
for att_id, index in located.items():
await db.execute(
update(Attachment).where(Attachment.id == att_id).values(part_index=index)
)
stats["rows"] += len(located)
stats["found"] += sum(1 for i in located.values() if i != NOT_IN_MESSAGE)
stats["not_found"] += sum(1 for i in located.values() if i == NOT_IN_MESSAGE)
# A separate copy goes only when every attachment row with that content
# is found in its message and nothing else uses the file.
candidates = {
(owner, sha)
for att_id, owner, sha, _ in rows
if located.get(att_id, NOT_IN_MESSAGE) != NOT_IN_MESSAGE
}
async with session_scope() as db:
for owner, sha in candidates:
path = blobstore.blob_path(owner, sha)
if not path.exists() or time.time() - path.stat().st_mtime < _MIN_AGE_S:
continue
in_use = (
await db.scalar(
select(Message.id)
.where(Message.user_id == owner, Message.blob_sha256 == sha)
.limit(1)
)
or await db.scalar(
select(ComposeAttachment.id)
.where(
ComposeAttachment.user_id == owner,
ComposeAttachment.blob_sha256 == sha,
)
.limit(1)
)
or await db.scalar(
select(Attachment.id)
.where(
Attachment.user_id == owner,
Attachment.blob_sha256 == sha,
(Attachment.part_index.is_(None))
| (Attachment.part_index == NOT_IN_MESSAGE),
)
.limit(1)
)
)
if in_use:
continue
size = path.stat().st_size
path.unlink(missing_ok=True)
stats["files_removed"] += 1
stats["bytes_freed"] += size
async def dedupe_existing_job(ctx: dict[str, object]) -> dict[str, int]:
"""arq job (bulk queue): :func:`dedupe_existing`, run once after an update."""
stats = await dedupe_existing()
logger.info("attachments kept once: %s", stats)
return stats
backend/app/indexer/gc.py
+198
View File
@@ -0,0 +1,198 @@
"""The nightly clean-up of stored files nothing uses any more.
Messages leave in many ways — Delete forever, Empty Trash, the Trash/Spam
retention purge, an expunge on the server, a folder re-synced — and none of
them removes the message's file: files are stored by content, so one file can
serve several rows (the same email in two folders, a forwarded file in a
draft), and removing it is only safe once nothing refers to it. This pass does
that for every user, whatever the way the rows went:
* uploads (compose attachments) no draft has referred to for a week — left by
a sent message (its draft is gone) or a composer closed without saving;
* message/attachment files no row refers to;
* attachment previews (thumbnails) of attachments that are gone;
* streaming copies past their keep time (app/indexer/attachments.py).
A file younger than an hour is left alone: it may belong to work in progress
(a message being stored, a sent message waiting for its Sent copy).
"""
from __future__ import annotations
import datetime as dt
import logging
import re
import time
import uuid
from pathlib import Path
from typing import Any
from sqlalchemy import delete, or_, select
from app.config import get_settings
from app.indexer import attachments as att_store
logger = logging.getLogger(__name__)
MIN_AGE_S = 3600
UPLOAD_KEEP = dt.timedelta(days=7)
_SHA = re.compile(r"^[0-9a-f]{64}$")
# ---- files a job still needs ---------------------------------------------------------
#
# A sent message whose Sent copy could not be filed yet is referenced only by
# its retrying job (app/outbox/jobs.py), for up to a day: held here meanwhile.
# Entries older than HOLD_MAX are ignored, in case a job died without letting go.
HOLD_MAX_S = 3 * 24 * 3600
def _hold_key(user_id: uuid.UUID | str) -> str:
return f"blob:held:{user_id}"
async def hold(user_id: uuid.UUID | str, sha: str) -> None:
from app.redis_client import get_redis
await get_redis().zadd(_hold_key(user_id), {sha: time.time()})
async def release(user_id: uuid.UUID | str, sha: str) -> None:
from app.redis_client import get_redis
await get_redis().zrem(_hold_key(user_id), sha)
async def _held(user_id: uuid.UUID) -> set[str]:
from app.redis_client import get_redis
redis = get_redis()
oldest = time.time() - HOLD_MAX_S
await redis.zremrangebyscore(_hold_key(user_id), "-inf", oldest)
return {
m.decode() if isinstance(m, bytes) else str(m)
for m in await redis.zrangebyscore(_hold_key(user_id), oldest, "+inf")
}
def _draft_upload_ids(payloads: list[dict[str, Any]]) -> set[uuid.UUID]:
out: set[uuid.UUID] = set()
for payload in payloads:
for value in payload.get("attachment_ids") or []:
try:
out.add(uuid.UUID(str(value)))
except ValueError:
continue
return out
def _old(path: Path, now: float) -> bool:
return now - path.stat().st_mtime >= MIN_AGE_S
async def sweep_user(user_id: uuid.UUID) -> dict[str, int]:
"""Clean one user's stored files; returns what was removed."""
from app.db import session_scope
from app.models import Attachment, ComposeAttachment, Draft, Message
stats = {"uploads": 0, "files": 0, "bytes": 0, "thumbs": 0}
async with session_scope() as db:
# Uploads no draft refers to, once a week old.
used = _draft_upload_ids(
list(await db.scalars(select(Draft.payload).where(Draft.user_id == user_id)))
)
stale = select(ComposeAttachment.id).where(
ComposeAttachment.user_id == user_id,
ComposeAttachment.created_at < dt.datetime.now(dt.UTC) - UPLOAD_KEEP,
)
if used:
stale = stale.where(ComposeAttachment.id.not_in(used))
gone = list(await db.scalars(stale))
if gone:
await db.execute(delete(ComposeAttachment).where(ComposeAttachment.id.in_(gone)))
stats["uploads"] = len(gone)
async with session_scope() as db:
# Every file still referred to. Attachments count only where their file
# is their only copy (not found in the message, or not yet looked for).
referenced = set(
await db.scalars(select(Message.blob_sha256).where(Message.user_id == user_id))
)
referenced |= set(
await db.scalars(
select(ComposeAttachment.blob_sha256).where(ComposeAttachment.user_id == user_id)
)
)
referenced |= set(
await db.scalars(
select(Attachment.blob_sha256).where(
Attachment.user_id == user_id,
or_(
Attachment.part_index.is_(None),
Attachment.part_index == att_store.NOT_IN_MESSAGE,
),
)
)
)
referenced |= await _held(user_id)
attachment_shas = set(
await db.scalars(select(Attachment.blob_sha256).where(Attachment.user_id == user_id))
)
root = Path(get_settings().blob_root) / str(user_id)
if not root.is_dir():
return stats
now = time.time()
for path in root.glob("??/*"):
if not path.is_file() or not _SHA.match(path.name) or path.name in referenced:
continue
if not _old(path, now):
continue
size = path.stat().st_size
path.unlink(missing_ok=True)
stats["files"] += 1
stats["bytes"] += size
thumbs = root / "thumbs"
if thumbs.is_dir():
for path in thumbs.iterdir():
sha = path.name.split("-", 1)[0]
if (
path.is_file()
and _SHA.match(sha)
and sha not in attachment_shas
and _old(path, now)
):
path.unlink(missing_ok=True)
stats["thumbs"] += 1
cache = root / "cache"
if cache.is_dir():
att_store.prune_cache(cache)
return stats
async def sweep(user_id: uuid.UUID | None = None) -> dict[str, int]:
""":func:`sweep_user` for one user, or every user."""
from app.db import session_scope
from app.models import User
async with session_scope() as db:
users = [user_id] if user_id is not None else list(await db.scalars(select(User.id)))
total = {"users": 0, "uploads": 0, "files": 0, "bytes": 0, "thumbs": 0}
for uid in users:
try:
stats = await sweep_user(uid)
except Exception: # noqa: BLE001 - one user's trouble must not stop the rest
logger.warning("stored-file clean-up failed for user %s", uid, exc_info=True)
continue
total["users"] += 1
for key, value in stats.items():
total[key] += value
return total
async def sweep_job(ctx: dict[str, Any]) -> dict[str, int]:
"""arq cron job: the nightly :func:`sweep`."""
stats = await sweep()
logger.info("stored-file clean-up: %s", stats)
return stats
backend/app/indexer/indexer.py
+4 -2
View File
@@ -216,8 +216,9 @@ async def persist_message(
)
)
# Attachments are not stored apart: they stay inside the raw blob, and are
# taken out of it when wanted (app/indexer/attachments.py).
for att in parsed.attachments:
att_sha = blobstore.store(user_id, att.payload)
session.add(
Attachment(
message_id=message.id,
@@ -226,7 +227,8 @@ async def persist_message(
mime_type=att.mime_type,
size_bytes=len(att.payload),
content_id=att.content_id,
blob_sha256=att_sha,
blob_sha256=blobstore.sha256_hex(att.payload),
part_index=att.part_index,
)
)
backend/app/indexer/mime.py
+40 -2
View File
@@ -41,6 +41,9 @@ class ParsedAttachment:
mime_type: str
content_id: str | None
payload: bytes
# The part's position in ``msg.walk()``: where to find it again in the raw
# message, which is the only copy kept (attachments are not stored apart).
part_index: int = -1
@dataclass
@@ -114,7 +117,7 @@ def _extract_attachments(msg: EmailMessage) -> list[ParsedAttachment]:
# or inline resource if it has a filename, a Content-ID, or attachment disposition;
# the body text/plain and text/html parts have none of these and are skipped.
out: list[ParsedAttachment] = []
for part in msg.walk():
for index, part in enumerate(msg.walk()):
if part.is_multipart() or not isinstance(part, EmailMessage):
continue
cid = part.get("Content-ID")
@@ -137,13 +140,14 @@ def _extract_attachments(msg: EmailMessage) -> list[ParsedAttachment]:
mime_type=part.get_content_type(),
content_id=cid.strip("<>") if (inline and cid is not None) else None,
payload=payload,
part_index=index,
)
)
# An invitation sent only as the text/calendar alternative (Outlook, often) has
# no file to show: keep it as invite.ics, so the event can be seen, answered
# and downloaded like a Gmail one. Not when the message attaches its own .ics.
if not any(a.filename.lower().endswith(".ics") for a in out):
for part in msg.walk():
for index, part in enumerate(msg.walk()):
if part.get_content_type() != "text/calendar" or part.get_filename():
continue
payload = part.get_payload(decode=True)
@@ -154,6 +158,7 @@ def _extract_attachments(msg: EmailMessage) -> list[ParsedAttachment]:
mime_type="text/calendar",
content_id=None,
payload=payload,
part_index=index,
)
)
break
@@ -215,3 +220,36 @@ def _parse_email(msg: EmailMessage) -> ParsedEmail:
html_body=_scrub(html_body) if html_body is not None else None,
attachments=_extract_attachments(msg),
)
def load_message(
raw: bytes | None = None, path: str | os.PathLike[str] | None = None
) -> EmailMessage:
"""The parsed tree of a raw message, from memory or straight from its file."""
if path is not None:
with open(path, "rb") as handle:
msg = email.message_from_binary_file(handle, policy=default_policy)
else:
msg = email.message_from_bytes(raw or b"", policy=default_policy)
assert isinstance(msg, EmailMessage)
return msg
def part_payload(msg: EmailMessage, part_index: int | None, sha256: str) -> bytes | None:
"""The decoded bytes of the attachment whose content hashes to ``sha256``:
the part at ``part_index`` when that is it, else whichever part is (rows
stored before the index was kept). None when the message has no such part."""
import hashlib
parts = list(msg.walk())
if part_index is not None and 0 <= part_index < len(parts):
payload = parts[part_index].get_payload(decode=True)
if isinstance(payload, bytes) and hashlib.sha256(payload).hexdigest() == sha256:
return payload
for part in parts:
if part.is_multipart():
continue
payload = part.get_payload(decode=True)
if isinstance(payload, bytes) and hashlib.sha256(payload).hexdigest() == sha256:
return payload
return None
backend/app/indexer/thumbs.py
+17 -5
View File
@@ -15,6 +15,7 @@ from __future__ import annotations
import io
import logging
import uuid
from collections.abc import Callable
from pathlib import Path
from app.indexer import blobstore
@@ -86,8 +87,16 @@ def _render(data: bytes, mime_type: str) -> bytes | None:
return out.getvalue()
def thumbnail(user_id: uuid.UUID, sha: str, mime_type: str) -> bytes | None:
"""The thumbnail of a stored attachment, or None if it cannot have one.
def thumbnail(
user_id: uuid.UUID,
sha: str,
mime_type: str,
size: int,
load: Callable[[], bytes],
) -> bytes | None:
"""The thumbnail of an attachment (content hash ``sha``, ``size`` bytes,
read with ``load`` only when it is not made yet), or None if it cannot
have one.
Blocking (decoding is CPU work): call it in a thread.
"""
@@ -97,11 +106,14 @@ def thumbnail(user_id: uuid.UUID, sha: str, mime_type: str) -> bytes | None:
none = _path(user_id, sha, ".none")
if none.exists() or not can_thumbnail(mime_type):
return None
path = blobstore.blob_path(user_id, sha)
if not path.exists() or path.stat().st_size > _MAX_INPUT_BYTES:
if size > _MAX_INPUT_BYTES:
return None
try:
thumb = _render(path.read_bytes(), mime_type)
data = load()
except FileNotFoundError:
return None
try:
thumb = _render(data, mime_type)
except Exception as exc: # noqa: BLE001 - any undecodable file: no thumbnail
logger.info("no thumbnail for blob %s (%s): %s", sha, mime_type, exc)
thumb = None