Unreachable mail servers: send retries in Scheduled, real action retries, account health

Sending: a send the server could not take (unreachable, timeout, 4xx) is no
longer failed at once. It waits in Scheduled ("retrying") and is tried again
after 1, 2, 5, 10, 30 min, then hourly, for 24 hours — on the server, so it
works with the browser closed. A permanent refusal (5xx, wrong password) is
"failed" at once, with the server's reason. GET /outbox lists both;
POST /outbox/{id}/retry (Retry now) and /cancel (Edit, Delete) act on them,
and they are left out of Drafts. Mail delivered but not filed in Sent now
counts as sent, and a conditional claim keeps two jobs from sending one
message twice. The app says when a send failed and when it finally went.

Actions: arq retries a job only on Retry, never on a plain exception, so the
"retry x5, then revert" for flags and moves never happened. Flag, move,
archive, restore, delete and empty-folder jobs now raise Retry while the
server is unreachable (~24 hours of growing gaps), skip a change a later one
replaced, stop at the first connection failure instead of 20 s per message,
and delete/empty keep the rows until the server has the messages gone.

Account health: 3 failed connections in a row mark an account unreachable
(amber in the sidebar and Settings, cleared on the next success); 3 refused
sign-ins stop its syncing as auth_failed until the password is entered
again, so the provider does not lock the account. Sync loops back off 30 s
doubling to 10 min and log one line per expected failure; a refused
connection fails at once with its reason (the connect task is kept), and a
silently dead IDLE connection is noticed within ~11 min instead of ~29.

Also: the empty reading pane is clipped to its width when dragged very
narrow; tests no longer leave live accounts and outbox rows in the database.

Migration 0021: outbox attempts / next_attempt_at / retry_until, accounts
sync_error / sync_error_since.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

backend/tests/mail/test_actions.py

@@ -9,7 +9,9 @@ from datetime import UTC, datetime
 from email.message import EmailMessage
 
 import httpx
+import pytest
 import pytest_asyncio
+from arq import Retry
 from httpx import ASGITransport
 from sqlalchemy import delete, func, select
 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
@@ -23,7 +25,7 @@ from app.identity.sessions import create_session
 from app.indexer.indexer import persist_message
 from app.main import create_app
 from app.models import Account, Mailbox, Message, Thread, User
-from app.providers.base import RemoteMessage
+from app.providers.base import ProviderConnectError, RemoteMessage
 from app.sync import actions as sync_actions
 
 CSRF = "csrf"
@@ -319,3 +321,103 @@ async def test_push_flags_job_reverts_on_failure(ctx: Ctx, monkeypatch) -> None:
     reverted = await ctx.session.get(Message, mid)
     await ctx.session.refresh(reverted)
     assert reverted is not None and "\\Seen" not in reverted.flags
+
+
+# --- the mail server unreachable: real retries (arq Retry), ~24 hours -------------
+
+
+class _Unreachable:
+    """A provider whose server cannot be reached."""
+
+    def __init__(self) -> None:
+        self.calls = 0
+
+    async def push_flags(self, *a: object, **k: object) -> None:
+        self.calls += 1
+        raise ProviderConnectError("cannot connect to imap.example.com")
+
+    async def move(self, *a: object, **k: object) -> int:
+        self.calls += 1
+        raise ProviderConnectError("cannot connect to imap.example.com")
+
+    async def delete_messages(self, *a: object, **k: object) -> None:
+        self.calls += 1
+        raise ProviderConnectError("cannot connect to imap.example.com")
+
+    async def close(self) -> None:
+        return None
+
+
+async def _mark_read(ctx: Ctx) -> uuid.UUID:
+    mid = ctx.message_ids[0]
+    msg = await ctx.session.get(Message, mid)
+    assert msg is not None
+    msg.flags = ["\\Seen"]
+    await ctx.session.commit()
+    return mid
+
+
+async def test_flag_push_retries_while_unreachable(ctx: Ctx, monkeypatch) -> None:  # type: ignore[no-untyped-def]
+    mid = await _mark_read(ctx)
+    monkeypatch.setattr(sync_actions, "build_provider", lambda _a: _Unreachable())
+
+    with pytest.raises(Retry) as caught:
+        await sync_actions.push_flags_job({"job_try": 1}, str(mid), ["\\Seen"], [])
+    assert caught.value.defer_score == 30_000  # first gap: 30 s
+    msg = await ctx.session.get(Message, mid)
+    await ctx.session.refresh(msg)
+    assert msg is not None and "\\Seen" in msg.flags  # kept while retrying
+
+    # The last try still cannot reach it: the change is rolled back.
+    await sync_actions.push_flags_job(
+        {"job_try": sync_actions.ACTION_MAX_TRIES}, str(mid), ["\\Seen"], []
+    )
+    await ctx.session.refresh(msg)
+    assert "\\Seen" not in msg.flags
+
+
+def test_retry_gaps_cover_about_a_day() -> None:
+    total = sum(sync_actions._RETRY_DEFERS_S)
+    assert 23 * 3600 < total <= 24 * 3600
+
+
+async def test_superseded_flag_change_is_not_pushed(ctx: Ctx, monkeypatch) -> None:  # type: ignore[no-untyped-def]
+    # "Read" was queued, then the message was marked unread again: the stale
+    # "read" must not reach the server after the newer change.
+    mid = ctx.message_ids[0]
+    provider = _Unreachable()
+    monkeypatch.setattr(sync_actions, "build_provider", lambda _a: provider)
+    await sync_actions.push_flags_job({"job_try": 3}, str(mid), ["\\Seen"], [])
+    assert provider.calls == 0
+
+
+async def test_archive_retries_while_unreachable(ctx: Ctx, monkeypatch) -> None:  # type: ignore[no-untyped-def]
+    provider = _Unreachable()
+    monkeypatch.setattr(sync_actions, "build_provider", lambda _a: provider)
+    with pytest.raises(Retry):
+        await sync_actions.move_thread_to_folder_job({"job_try": 1}, str(ctx.thread_id), "archive")
+    assert provider.calls == 1  # stopped at the first failure, not 20 s per message
+    ctx.session.expunge_all()
+    async with session_scope() as verify:
+        inbox = await verify.scalar(
+            select(Mailbox).where(Mailbox.user_id == ctx.user_id, Mailbox.special_use == "inbox")
+        )
+        assert inbox is not None
+        moved = await verify.scalar(
+            select(func.count())
+            .select_from(Message)
+            .where(Message.thread_id == ctx.thread_id, Message.mailbox_id != inbox.id)
+        )
+        assert moved == 0  # nothing recorded as moved
+
+
+async def test_delete_keeps_rows_while_unreachable(ctx: Ctx, monkeypatch) -> None:  # type: ignore[no-untyped-def]
+    monkeypatch.setattr(sync_actions, "build_provider", lambda _a: _Unreachable())
+    with pytest.raises(Retry):
+        await sync_actions.delete_thread_job({"job_try": 1}, str(ctx.thread_id))
+    ctx.session.expunge_all()
+    async with session_scope() as verify:
+        left = await verify.scalar(
+            select(func.count()).select_from(Message).where(Message.thread_id == ctx.thread_id)
+        )
+        assert left == len(ctx.message_ids)  # kept: the server still has them