Unreachable mail servers: send retries in Scheduled, real action retries, account health

Sending: a send the server could not take (unreachable, timeout, 4xx) is no
longer failed at once. It waits in Scheduled ("retrying") and is tried again
after 1, 2, 5, 10, 30 min, then hourly, for 24 hours — on the server, so it
works with the browser closed. A permanent refusal (5xx, wrong password) is
"failed" at once, with the server's reason. GET /outbox lists both;
POST /outbox/{id}/retry (Retry now) and /cancel (Edit, Delete) act on them,
and they are left out of Drafts. Mail delivered but not filed in Sent now
counts as sent, and a conditional claim keeps two jobs from sending one
message twice. The app says when a send failed and when it finally went.

Actions: arq retries a job only on Retry, never on a plain exception, so the
"retry x5, then revert" for flags and moves never happened. Flag, move,
archive, restore, delete and empty-folder jobs now raise Retry while the
server is unreachable (~24 hours of growing gaps), skip a change a later one
replaced, stop at the first connection failure instead of 20 s per message,
and delete/empty keep the rows until the server has the messages gone.

Account health: 3 failed connections in a row mark an account unreachable
(amber in the sidebar and Settings, cleared on the next success); 3 refused
sign-ins stop its syncing as auth_failed until the password is entered
again, so the provider does not lock the account. Sync loops back off 30 s
doubling to 10 min and log one line per expected failure; a refused
connection fails at once with its reason (the connect task is kept), and a
silently dead IDLE connection is noticed within ~11 min instead of ~29.

Also: the empty reading pane is clipped to its width when dragged very
narrow; tests no longer leave live accounts and outbox rows in the database.

Migration 0021: outbox attempts / next_attempt_at / retry_until, accounts
sync_error / sync_error_since.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

backend/app/outbox/jobs.py

@@ -8,14 +8,13 @@ to Sent (skipped for Gmail, which self-appends), harvest recipients into
 
 from __future__ import annotations
 
-import contextlib
 import logging
 import uuid
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
 from email.utils import getaddresses
 from typing import Any
 
-from sqlalchemy import select
+from sqlalchemy import select, update
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.ext.asyncio import AsyncSession
 
@@ -27,7 +26,9 @@ from app.outbox.attachments import load_attachments
 from app.outbox.mime_build import build_mime
 from app.outbox.service import delete_mirrored_draft_row
 from app.providers.base import RemoteMessage
+from app.queue import enqueue_job
 from app.search.factory import get_search_backend
+from app.sync.health import is_temporary
 from app.sync.provider_factory import build_provider
 
 logger = logging.getLogger(__name__)
@@ -111,14 +112,80 @@ async def _persist_sent_copy(user_id: uuid.UUID, account: Account, raw: bytes, u
         await bus.publish("thread.updated", {"user_id": str(user_id), "thread_id": str(thread_id)})
 
 
+# A send the mail server could not take (unreachable, a temporary 4xx) is tried
+# again after these gaps, then hourly, until RETRY_WINDOW after the first
+# failure. Meanwhile the message waits in the Scheduled folder.
+_RETRY_GAPS_S = (60, 120, 300, 600, 1800)
+_RETRY_HOURLY_S = 3600
+RETRY_WINDOW = timedelta(hours=24)
+
+
+def _retry_gap(attempts: int) -> int:
+    return _RETRY_GAPS_S[attempts - 1] if attempts <= len(_RETRY_GAPS_S) else _RETRY_HOURLY_S
+
+
+async def _send_failed(
+    oid: uuid.UUID, user_id: uuid.UUID, subject: str, exc: BaseException
+) -> None:
+    """Record a failed send: "retrying" if it may work later, else "failed"."""
+    now = datetime.now(UTC)
+    async with session_scope() as session:
+        ob = await session.get(Outbox, oid)
+        if ob is None:
+            return
+        ob.attempts += 1
+        ob.error = (getattr(exc, "detail", None) or str(exc) or type(exc).__name__)[:500]
+        ob.retry_until = ob.retry_until or now + RETRY_WINDOW
+        if is_temporary(exc) and now < ob.retry_until:
+            ob.state = "retrying"
+            ob.next_attempt_at = now + timedelta(seconds=_retry_gap(ob.attempts))
+        else:
+            ob.state = "failed"
+            ob.next_attempt_at = None
+        state, attempts, next_at, error = ob.state, ob.attempts, ob.next_attempt_at, ob.error
+    if state == "retrying" and next_at is not None:
+        await enqueue_job(
+            "send_job", str(oid), _defer_until=next_at, _job_id=f"send:{oid}:{attempts}"
+        )
+    await get_event_bus().publish(
+        "outbox.state_changed",
+        {
+            "user_id": str(user_id),
+            "outbox_id": str(oid),
+            "state": state,
+            "attempts": attempts,
+            "subject": subject,
+            "error": error,
+            "next_attempt_at": next_at.isoformat() if next_at else None,
+        },
+    )
+    if state == "failed":
+        logger.warning("send failed for outbox %s: %s", oid, error)
+    else:
+        logger.info(
+            "send of outbox %s failed (try %d), retrying at %s: %s", oid, attempts, next_at, error
+        )
+
+
 async def send_job(ctx: dict[str, Any], outbox_id: str) -> None:
-    """Send a queued message unless it was cancelled during the undo window."""
+    """Send a queued message unless it was cancelled during the undo window.
+
+    Also runs each retry of a send that failed for a temporary reason. The row is
+    claimed with one conditional UPDATE, so two jobs for the same send (a planned
+    retry and "Retry now") can never both send it.
+    """
     oid = uuid.UUID(outbox_id)
     async with session_scope() as session:
-        outbox = await session.get(Outbox, oid)
-        if outbox is None or outbox.state != "undo_window":
+        claimed = await session.scalar(
+            update(Outbox)
+            .where(Outbox.id == oid, Outbox.state.in_(("undo_window", "retrying")))
+            .values(state="sending")
+            .returning(Outbox.id)
+        )
+        if claimed is None:
             return  # cancelled, or already handled -> no SMTP traffic
-        outbox.state = "sending"
+        outbox = await session.get(Outbox, oid)
+        assert outbox is not None
         draft = await session.get(Draft, outbox.draft_id)
         if draft is None:
             outbox.state = "failed"
@@ -153,28 +220,25 @@ async def send_job(ctx: dict[str, Any], outbox_id: str) -> None:
         gmail = _is_gmail(account)
         account_obj = account
 
+    subject = str(draft.payload.get("subject") or "")
     provider = build_provider(account_obj)
     appended_uid = 0
     try:
-        await provider.send(transmit_raw, envelope_from, envelope_to)
-        if sent_path and not gmail:
-            appended_uid = await provider.append(sent_path, sent_raw, frozenset({"\\Seen"}))
-        # Remove the mirrored Drafts-folder copy now that it has been sent.
-        if drafts_path and draft_imap_uid:
-            with contextlib.suppress(Exception):
+        try:
+            await provider.send(transmit_raw, envelope_from, envelope_to)
+        except Exception as exc:  # noqa: BLE001 - retry or record; the draft is kept
+            await _send_failed(oid, user_id, subject, exc)
+            return
+        # Delivered. Filing it is housekeeping: if the IMAP server is not there
+        # right now, the send still succeeded (the Sent poll files it later).
+        try:
+            if sent_path and not gmail:
+                appended_uid = await provider.append(sent_path, sent_raw, frozenset({"\\Seen"}))
+            # Remove the mirrored Drafts-folder copy now that it has been sent.
+            if drafts_path and draft_imap_uid:
                 await provider.delete_messages(drafts_path, [draft_imap_uid])
-    except Exception as exc:  # noqa: BLE001 - record and notify, keep the draft
-        async with session_scope() as session:
-            ob = await session.get(Outbox, oid)
-            if ob is not None:
-                ob.state = "failed"
-                ob.error = str(exc)[:500]
-        await get_event_bus().publish(
-            "outbox.state_changed",
-            {"user_id": str(user_id), "outbox_id": str(oid), "state": "failed"},
-        )
-        logger.exception("send failed for outbox %s", oid)
-        return
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("outbox %s sent, but filing it on the server failed: %s", oid, exc)
     finally:
         await provider.close()