Attachments stored once; nightly clean-up of unused files

Attachments are no longer stored apart from their message (they were kept
twice: base64 inside the raw message and decoded on their own). The indexer
records the MIME part (attachments.part_index, migration 0023); downloads,
zip, inline images, thumbnails and forwarding take it out of the message;
videos stream from a short-lived copy (kept a day, 1 GiB per user). A
background pass on worker start removes the copies stored before, each only
after checking it can be taken out again (spec App. D §D.6 item 1, criterion
corrected to <= 60%).

A nightly job (03:47, after the Trash/Spam purge) removes files nothing
refers to any more — deleted or purged messages, previews of attachments
gone, uploads no draft has used for a week, expired streaming copies. Files
younger than an hour are left alone; a sent message waiting for its Sent copy
is held until its job is done. Account removal keeps files a draft still uses.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

backend/workers/run.py

@@ -25,14 +25,19 @@ from arq import cron
 from arq.worker import Worker
 
 from app.config import get_settings
+from app.indexer.gc import sweep_job
 from app.queue import QUEUE_BULK, QUEUE_INTERACTIVE, redis_settings
 from app.sync.actions import autopurge_trash_spam
 from app.sync.coordinator import SyncCoordinator
 from workers.worker import BULK_FUNCTIONS, INTERACTIVE_FUNCTIONS
 
 # Core scheduled jobs. Daily Trash/Spam retention purge at 03:17 (off-peak, and a
-# minute that won't collide with typical top-of-hour crons).
-_CORE_CRON_JOBS = [cron(autopurge_trash_spam, hour=3, minute=17)]
+# minute that won't collide with typical top-of-hour crons); then the clean-up of
+# stored files nothing uses any more (the purge's among them) at 03:47.
+_CORE_CRON_JOBS = [
+    cron(autopurge_trash_spam, hour=3, minute=17),
+    cron(sweep_job, hour=3, minute=47, timeout=3600),
+]
 
 logger = logging.getLogger(__name__)
 
@@ -95,6 +100,15 @@ async def _run() -> None:
         await coordinator.start()
     arq_tasks = [asyncio.create_task(w.async_run()) for w in build_arq_workers(role)]
     logger.info("worker starting: role=%s (%d arq worker(s))", role, len(arq_tasks))
+    if arq_tasks:
+        # Attachments stored apart before they were kept once: remove those
+        # copies in the background (quick when there are none; one at a time).
+        from app.queue import enqueue_job
+
+        try:
+            await enqueue_job("dedupe_existing_job", _job_id="dedupe-attachments")
+        except Exception:  # noqa: BLE001 - tried again on the next start
+            logger.warning("could not enqueue the attachment clean-up", exc_info=True)
 
     stop = asyncio.Event()
     loop = asyncio.get_running_loop()

backend/workers/worker.py

@@ -14,6 +14,7 @@ from typing import Any
 
 from arq import func
 
+from app.indexer.attachments import dedupe_existing_job
 from app.outbox.draft_sync import push_draft_to_imap_job, remove_imap_draft_job
 from app.outbox.jobs import file_sent_copy_job, send_job
 from app.push.service import push_new_mail_job
@@ -62,4 +63,8 @@ INTERACTIVE_FUNCTIONS: list[Any] = [
 ]
 
 # Bulk work (§6): long timeout, low concurrency, isolated from interactive jobs.
-BULK_FUNCTIONS = [sync_bootstrap_account]
+BULK_FUNCTIONS = [
+    sync_bootstrap_account,
+    # Resumable (each batch is saved), so a long first run may span timeouts.
+    func(dedupe_existing_job, timeout=6 * 3600),
+]