Unreachable mail servers: send retries in Scheduled, real action retries, account health

Sending: a send the server could not take (unreachable, timeout, 4xx) is no
longer failed at once. It waits in Scheduled ("retrying") and is tried again
after 1, 2, 5, 10, 30 min, then hourly, for 24 hours — on the server, so it
works with the browser closed. A permanent refusal (5xx, wrong password) is
"failed" at once, with the server's reason. GET /outbox lists both;
POST /outbox/{id}/retry (Retry now) and /cancel (Edit, Delete) act on them,
and they are left out of Drafts. Mail delivered but not filed in Sent now
counts as sent, and a conditional claim keeps two jobs from sending one
message twice. The app says when a send failed and when it finally went.

Actions: arq retries a job only on Retry, never on a plain exception, so the
"retry x5, then revert" for flags and moves never happened. Flag, move,
archive, restore, delete and empty-folder jobs now raise Retry while the
server is unreachable (~24 hours of growing gaps), skip a change a later one
replaced, stop at the first connection failure instead of 20 s per message,
and delete/empty keep the rows until the server has the messages gone.

Account health: 3 failed connections in a row mark an account unreachable
(amber in the sidebar and Settings, cleared on the next success); 3 refused
sign-ins stop its syncing as auth_failed until the password is entered
again, so the provider does not lock the account. Sync loops back off 30 s
doubling to 10 min and log one line per expected failure; a refused
connection fails at once with its reason (the connect task is kept), and a
silently dead IDLE connection is noticed within ~11 min instead of ~29.

Also: the empty reading pane is clipped to its width when dragged very
narrow; tests no longer leave live accounts and outbox rows in the database.

Migration 0021: outbox attempts / next_attempt_at / retry_until, accounts
sync_error / sync_error_since.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

backend/tests/outbox/test_outbox.py

@@ -2,11 +2,12 @@
 
 from __future__ import annotations
 
+import asyncio
 import email
 import uuid
 from collections.abc import AsyncIterator
 from dataclasses import dataclass, field
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
 from email.policy import default as default_policy
 
 import httpx
@@ -25,6 +26,7 @@ from app.main import create_app
 from app.models import Account, Draft, KnownRecipient, Mailbox, Message, Outbox, User
 from app.outbox import jobs as outbox_jobs
 from app.outbox.mime_build import build_mime
+from app.providers.base import ProviderConnectError, ProviderRejectedError
 
 CSRF = "csrf"
 
@@ -193,6 +195,8 @@ async def ctx(monkeypatch) -> AsyncIterator[Ctx]:  # type: ignore[no-untyped-def
         yield c
     finally:
         await c.client.aclose()
+        # outbox.user_id has no foreign key: drop the rows with the user.
+        await session.execute(delete(Outbox).where(Outbox.user_id == user.id))
         await session.execute(delete(User).where(User.id == user.id))
         await session.commit()
         await session.close()
@@ -425,13 +429,19 @@ class FakeProvider:
         self.sent: list = []
         self.appended: list = []
         self.raise_on_send = False
+        self.send_error: Exception | None = None
+        self.append_error: Exception | None = None
 
     async def send(self, raw: bytes, env_from: str, env_to: list) -> None:
         if self.raise_on_send:
             raise RuntimeError("smtp down")
+        if self.send_error is not None:
+            raise self.send_error
         self.sent.append((env_from, env_to))
 
     async def append(self, mailbox: str, raw: bytes, flags: frozenset) -> int:
+        if self.append_error is not None:
+            raise self.append_error
         self.appended.append(mailbox)
         return 99
 
@@ -516,3 +526,124 @@ async def test_send_job_failure_marks_failed(ctx: Ctx, monkeypatch) -> None:  #
     ob = await ctx.session.get(Outbox, oid)
     await ctx.session.refresh(ob)
     assert ob is not None and ob.state == "failed" and ob.error
+
+
+# --- a send the server could not take: Scheduled, retried for 24 hours ------------
+
+
+async def _outbox(ctx: Ctx, oid: uuid.UUID) -> Outbox:
+    ob = await ctx.session.get(Outbox, oid)
+    assert ob is not None
+    await ctx.session.refresh(ob)
+    return ob
+
+
+async def test_unreachable_smtp_is_retried_and_waits_in_scheduled(ctx: Ctx, monkeypatch) -> None:  # type: ignore[no-untyped-def]
+    oid = await _make_outbox(ctx)
+    fake = FakeProvider()
+    fake.send_error = ProviderConnectError("cannot connect to smtp.example.com")
+    monkeypatch.setattr(outbox_jobs, "build_provider", lambda _a: fake)
+
+    before = datetime.now(UTC)
+    await outbox_jobs.send_job({"job_try": 1}, str(oid))
+
+    ob = await _outbox(ctx, oid)
+    assert ob.state == "retrying" and ob.attempts == 1 and "cannot connect" in (ob.error or "")
+    assert ob.retry_until is not None
+    assert ob.retry_until >= before + timedelta(hours=23, minutes=59)
+    assert ob.next_attempt_at is not None
+    assert timedelta(seconds=55) < ob.next_attempt_at - before < timedelta(seconds=70)
+    args, kwargs = ctx.enqueued[-1]
+    assert args == ("send_job", str(oid))
+    assert kwargs["_defer_until"] == ob.next_attempt_at and kwargs["_job_id"] == f"send:{oid}:1"
+
+    # Waiting in Scheduled, not in Drafts.
+    drafts = (await ctx.client.get("/api/drafts")).json()
+    assert all(d["id"] != str(ob.draft_id) for d in drafts)
+    waiting = (await ctx.client.get("/api/outbox")).json()
+    assert [(w["outbox_id"], w["state"], w["subject"]) for w in waiting] == [
+        (str(oid), "retrying", "Re: Hi")
+    ]
+
+    # The server is back: the planned retry sends it.
+    fake.send_error = None
+    await outbox_jobs.send_job({"job_try": 1}, str(oid))
+    assert (await _outbox(ctx, oid)).state == "sent" and len(fake.sent) == 1
+    assert (await ctx.client.get("/api/outbox")).json() == []
+
+
+def test_retry_gaps_grow_then_hourly() -> None:
+    assert [outbox_jobs._retry_gap(n) for n in range(1, 8)] == [60, 120, 300, 600, 1800, 3600, 3600]
+
+
+async def test_send_gives_up_after_24_hours(ctx: Ctx, monkeypatch) -> None:  # type: ignore[no-untyped-def]
+    oid = await _make_outbox(ctx, state="retrying")
+    ob = await _outbox(ctx, oid)
+    ob.retry_until = datetime.now(UTC) - timedelta(seconds=1)
+    await ctx.session.commit()
+    fake = FakeProvider()
+    fake.send_error = ProviderConnectError("still down")
+    monkeypatch.setattr(outbox_jobs, "build_provider", lambda _a: fake)
+
+    await outbox_jobs.send_job({"job_try": 1}, str(oid))
+    ob = await _outbox(ctx, oid)
+    assert ob.state == "failed" and ob.next_attempt_at is None
+    assert not any(a[0] == "send_job" for a, _ in ctx.enqueued)
+
+
+async def test_permanent_refusal_fails_at_once(ctx: Ctx, monkeypatch) -> None:  # type: ignore[no-untyped-def]
+    oid = await _make_outbox(ctx)
+    fake = FakeProvider()
+    fake.send_error = ProviderRejectedError("550 no such user")
+    monkeypatch.setattr(outbox_jobs, "build_provider", lambda _a: fake)
+
+    await outbox_jobs.send_job({"job_try": 1}, str(oid))
+    ob = await _outbox(ctx, oid)
+    assert ob.state == "failed" and "550" in (ob.error or "")
+    assert not any(a[0] == "send_job" for a, _ in ctx.enqueued)
+    # Shown in Scheduled as failed, with the reason.
+    waiting = (await ctx.client.get("/api/outbox")).json()
+    assert waiting[0]["state"] == "failed" and "550" in waiting[0]["error"]
+
+
+async def test_delivered_but_not_filed_counts_as_sent(ctx: Ctx, monkeypatch) -> None:  # type: ignore[no-untyped-def]
+    oid = await _make_outbox(ctx)
+    fake = FakeProvider()
+    fake.append_error = ProviderConnectError("imap unreachable")
+    monkeypatch.setattr(outbox_jobs, "build_provider", lambda _a: fake)
+
+    await outbox_jobs.send_job({"job_try": 1}, str(oid))
+    assert (await _outbox(ctx, oid)).state == "sent" and len(fake.sent) == 1
+
+
+async def test_two_jobs_for_one_send_send_once(ctx: Ctx, monkeypatch) -> None:  # type: ignore[no-untyped-def]
+    oid = await _make_outbox(ctx, state="retrying")
+    fake = FakeProvider()
+    monkeypatch.setattr(outbox_jobs, "build_provider", lambda _a: fake)
+
+    await asyncio.gather(
+        outbox_jobs.send_job({"job_try": 1}, str(oid)),
+        outbox_jobs.send_job({"job_try": 1}, str(oid)),
+    )
+    assert len(fake.sent) == 1
+
+
+async def test_retry_now_and_cancel(ctx: Ctx) -> None:
+    oid = await _make_outbox(ctx, state="failed")
+    resp = await ctx.client.post(f"/api/outbox/{oid}/retry", headers=_hdr())
+    assert resp.status_code == 200
+    ob = await _outbox(ctx, oid)
+    assert ob.state == "retrying"
+    assert ob.retry_until is not None and ob.retry_until > datetime.now(UTC) + timedelta(hours=23)
+    assert ctx.enqueued[-1][0] == ("send_job", str(oid))
+
+    resp = await ctx.client.post(f"/api/outbox/{oid}/cancel", headers=_hdr())
+    assert resp.status_code == 200 and resp.json()["draft_id"] == str(ob.draft_id)
+    assert (await _outbox(ctx, oid)).state == "cancelled"
+    # Back in Drafts, gone from Scheduled.
+    drafts = (await ctx.client.get("/api/drafts")).json()
+    assert any(d["id"] == str(ob.draft_id) for d in drafts)
+    assert (await ctx.client.get("/api/outbox")).json() == []
+    # Only a waiting send can be retried or cancelled.
+    again = await ctx.client.post(f"/api/outbox/{oid}/retry", headers=_hdr())
+    assert again.status_code == 409