Unreachable mail servers: send retries in Scheduled, real action retries, account health

Sending: a send the server could not take (unreachable, timeout, 4xx) is no
longer failed at once. It waits in Scheduled ("retrying") and is tried again
after 1, 2, 5, 10, 30 min, then hourly, for 24 hours — on the server, so it
works with the browser closed. A permanent refusal (5xx, wrong password) is
"failed" at once, with the server's reason. GET /outbox lists both;
POST /outbox/{id}/retry (Retry now) and /cancel (Edit, Delete) act on them,
and they are left out of Drafts. Mail delivered but not filed in Sent now
counts as sent, and a conditional claim keeps two jobs from sending one
message twice. The app says when a send failed and when it finally went.

Actions: arq retries a job only on Retry, never on a plain exception, so the
"retry x5, then revert" for flags and moves never happened. Flag, move,
archive, restore, delete and empty-folder jobs now raise Retry while the
server is unreachable (~24 hours of growing gaps), skip a change a later one
replaced, stop at the first connection failure instead of 20 s per message,
and delete/empty keep the rows until the server has the messages gone.

Account health: 3 failed connections in a row mark an account unreachable
(amber in the sidebar and Settings, cleared on the next success); 3 refused
sign-ins stop its syncing as auth_failed until the password is entered
again, so the provider does not lock the account. Sync loops back off 30 s
doubling to 10 min and log one line per expected failure; a refused
connection fails at once with its reason (the connect task is kept), and a
silently dead IDLE connection is noticed within ~11 min instead of ~29.

Also: the empty reading pane is clipped to its width when dragged very
narrow; tests no longer leave live accounts and outbox rows in the database.

Migration 0021: outbox attempts / next_attempt_at / retry_until, accounts
sync_error / sync_error_since.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-05 22:08:42 +02:00
co-authored by Claude Opus 5.5
parent e1bda37188
commit 609e7c58f9
31 changed files with 1375 additions and 151 deletions
+103 -1
View File
@@ -9,7 +9,9 @@ from datetime import UTC, datetime
from email.message import EmailMessage
import httpx
import pytest
import pytest_asyncio
from arq import Retry
from httpx import ASGITransport
from sqlalchemy import delete, func, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
@@ -23,7 +25,7 @@ from app.identity.sessions import create_session
from app.indexer.indexer import persist_message
from app.main import create_app
from app.models import Account, Mailbox, Message, Thread, User
from app.providers.base import RemoteMessage
from app.providers.base import ProviderConnectError, RemoteMessage
from app.sync import actions as sync_actions
CSRF = "csrf"
@@ -319,3 +321,103 @@ async def test_push_flags_job_reverts_on_failure(ctx: Ctx, monkeypatch) -> None:
reverted = await ctx.session.get(Message, mid)
await ctx.session.refresh(reverted)
assert reverted is not None and "\\Seen" not in reverted.flags
# --- the mail server unreachable: real retries (arq Retry), ~24 hours -------------
class _Unreachable:
"""A provider whose server cannot be reached."""
def __init__(self) -> None:
self.calls = 0
async def push_flags(self, *a: object, **k: object) -> None:
self.calls += 1
raise ProviderConnectError("cannot connect to imap.example.com")
async def move(self, *a: object, **k: object) -> int:
self.calls += 1
raise ProviderConnectError("cannot connect to imap.example.com")
async def delete_messages(self, *a: object, **k: object) -> None:
self.calls += 1
raise ProviderConnectError("cannot connect to imap.example.com")
async def close(self) -> None:
return None
async def _mark_read(ctx: Ctx) -> uuid.UUID:
mid = ctx.message_ids[0]
msg = await ctx.session.get(Message, mid)
assert msg is not None
msg.flags = ["\\Seen"]
await ctx.session.commit()
return mid
async def test_flag_push_retries_while_unreachable(ctx: Ctx, monkeypatch) -> None: # type: ignore[no-untyped-def]
mid = await _mark_read(ctx)
monkeypatch.setattr(sync_actions, "build_provider", lambda _a: _Unreachable())
with pytest.raises(Retry) as caught:
await sync_actions.push_flags_job({"job_try": 1}, str(mid), ["\\Seen"], [])
assert caught.value.defer_score == 30_000 # first gap: 30 s
msg = await ctx.session.get(Message, mid)
await ctx.session.refresh(msg)
assert msg is not None and "\\Seen" in msg.flags # kept while retrying
# The last try still cannot reach it: the change is rolled back.
await sync_actions.push_flags_job(
{"job_try": sync_actions.ACTION_MAX_TRIES}, str(mid), ["\\Seen"], []
)
await ctx.session.refresh(msg)
assert "\\Seen" not in msg.flags
def test_retry_gaps_cover_about_a_day() -> None:
total = sum(sync_actions._RETRY_DEFERS_S)
assert 23 * 3600 < total <= 24 * 3600
async def test_superseded_flag_change_is_not_pushed(ctx: Ctx, monkeypatch) -> None: # type: ignore[no-untyped-def]
# "Read" was queued, then the message was marked unread again: the stale
# "read" must not reach the server after the newer change.
mid = ctx.message_ids[0]
provider = _Unreachable()
monkeypatch.setattr(sync_actions, "build_provider", lambda _a: provider)
await sync_actions.push_flags_job({"job_try": 3}, str(mid), ["\\Seen"], [])
assert provider.calls == 0
async def test_archive_retries_while_unreachable(ctx: Ctx, monkeypatch) -> None: # type: ignore[no-untyped-def]
provider = _Unreachable()
monkeypatch.setattr(sync_actions, "build_provider", lambda _a: provider)
with pytest.raises(Retry):
await sync_actions.move_thread_to_folder_job({"job_try": 1}, str(ctx.thread_id), "archive")
assert provider.calls == 1 # stopped at the first failure, not 20 s per message
ctx.session.expunge_all()
async with session_scope() as verify:
inbox = await verify.scalar(
select(Mailbox).where(Mailbox.user_id == ctx.user_id, Mailbox.special_use == "inbox")
)
assert inbox is not None
moved = await verify.scalar(
select(func.count())
.select_from(Message)
.where(Message.thread_id == ctx.thread_id, Message.mailbox_id != inbox.id)
)
assert moved == 0 # nothing recorded as moved
async def test_delete_keeps_rows_while_unreachable(ctx: Ctx, monkeypatch) -> None: # type: ignore[no-untyped-def]
monkeypatch.setattr(sync_actions, "build_provider", lambda _a: _Unreachable())
with pytest.raises(Retry):
await sync_actions.delete_thread_job({"job_try": 1}, str(ctx.thread_id))
ctx.session.expunge_all()
async with session_scope() as verify:
left = await verify.scalar(
select(func.count()).select_from(Message).where(Message.thread_id == ctx.thread_id)
)
assert left == len(ctx.message_ids) # kept: the server still has them
+132 -1
View File
@@ -2,11 +2,12 @@
from __future__ import annotations
import asyncio
import email
import uuid
from collections.abc import AsyncIterator
from dataclasses import dataclass, field
from datetime import UTC, datetime
from datetime import UTC, datetime, timedelta
from email.policy import default as default_policy
import httpx
@@ -25,6 +26,7 @@ from app.main import create_app
from app.models import Account, Draft, KnownRecipient, Mailbox, Message, Outbox, User
from app.outbox import jobs as outbox_jobs
from app.outbox.mime_build import build_mime
from app.providers.base import ProviderConnectError, ProviderRejectedError
CSRF = "csrf"
@@ -193,6 +195,8 @@ async def ctx(monkeypatch) -> AsyncIterator[Ctx]: # type: ignore[no-untyped-def
yield c
finally:
await c.client.aclose()
# outbox.user_id has no foreign key: drop the rows with the user.
await session.execute(delete(Outbox).where(Outbox.user_id == user.id))
await session.execute(delete(User).where(User.id == user.id))
await session.commit()
await session.close()
@@ -425,13 +429,19 @@ class FakeProvider:
self.sent: list = []
self.appended: list = []
self.raise_on_send = False
self.send_error: Exception | None = None
self.append_error: Exception | None = None
async def send(self, raw: bytes, env_from: str, env_to: list) -> None:
if self.raise_on_send:
raise RuntimeError("smtp down")
if self.send_error is not None:
raise self.send_error
self.sent.append((env_from, env_to))
async def append(self, mailbox: str, raw: bytes, flags: frozenset) -> int:
if self.append_error is not None:
raise self.append_error
self.appended.append(mailbox)
return 99
@@ -516,3 +526,124 @@ async def test_send_job_failure_marks_failed(ctx: Ctx, monkeypatch) -> None: #
ob = await ctx.session.get(Outbox, oid)
await ctx.session.refresh(ob)
assert ob is not None and ob.state == "failed" and ob.error
# --- a send the server could not take: Scheduled, retried for 24 hours ------------
async def _outbox(ctx: Ctx, oid: uuid.UUID) -> Outbox:
ob = await ctx.session.get(Outbox, oid)
assert ob is not None
await ctx.session.refresh(ob)
return ob
async def test_unreachable_smtp_is_retried_and_waits_in_scheduled(ctx: Ctx, monkeypatch) -> None: # type: ignore[no-untyped-def]
oid = await _make_outbox(ctx)
fake = FakeProvider()
fake.send_error = ProviderConnectError("cannot connect to smtp.example.com")
monkeypatch.setattr(outbox_jobs, "build_provider", lambda _a: fake)
before = datetime.now(UTC)
await outbox_jobs.send_job({"job_try": 1}, str(oid))
ob = await _outbox(ctx, oid)
assert ob.state == "retrying" and ob.attempts == 1 and "cannot connect" in (ob.error or "")
assert ob.retry_until is not None
assert ob.retry_until >= before + timedelta(hours=23, minutes=59)
assert ob.next_attempt_at is not None
assert timedelta(seconds=55) < ob.next_attempt_at - before < timedelta(seconds=70)
args, kwargs = ctx.enqueued[-1]
assert args == ("send_job", str(oid))
assert kwargs["_defer_until"] == ob.next_attempt_at and kwargs["_job_id"] == f"send:{oid}:1"
# Waiting in Scheduled, not in Drafts.
drafts = (await ctx.client.get("/api/drafts")).json()
assert all(d["id"] != str(ob.draft_id) for d in drafts)
waiting = (await ctx.client.get("/api/outbox")).json()
assert [(w["outbox_id"], w["state"], w["subject"]) for w in waiting] == [
(str(oid), "retrying", "Re: Hi")
]
# The server is back: the planned retry sends it.
fake.send_error = None
await outbox_jobs.send_job({"job_try": 1}, str(oid))
assert (await _outbox(ctx, oid)).state == "sent" and len(fake.sent) == 1
assert (await ctx.client.get("/api/outbox")).json() == []
def test_retry_gaps_grow_then_hourly() -> None:
assert [outbox_jobs._retry_gap(n) for n in range(1, 8)] == [60, 120, 300, 600, 1800, 3600, 3600]
async def test_send_gives_up_after_24_hours(ctx: Ctx, monkeypatch) -> None: # type: ignore[no-untyped-def]
oid = await _make_outbox(ctx, state="retrying")
ob = await _outbox(ctx, oid)
ob.retry_until = datetime.now(UTC) - timedelta(seconds=1)
await ctx.session.commit()
fake = FakeProvider()
fake.send_error = ProviderConnectError("still down")
monkeypatch.setattr(outbox_jobs, "build_provider", lambda _a: fake)
await outbox_jobs.send_job({"job_try": 1}, str(oid))
ob = await _outbox(ctx, oid)
assert ob.state == "failed" and ob.next_attempt_at is None
assert not any(a[0] == "send_job" for a, _ in ctx.enqueued)
async def test_permanent_refusal_fails_at_once(ctx: Ctx, monkeypatch) -> None: # type: ignore[no-untyped-def]
oid = await _make_outbox(ctx)
fake = FakeProvider()
fake.send_error = ProviderRejectedError("550 no such user")
monkeypatch.setattr(outbox_jobs, "build_provider", lambda _a: fake)
await outbox_jobs.send_job({"job_try": 1}, str(oid))
ob = await _outbox(ctx, oid)
assert ob.state == "failed" and "550" in (ob.error or "")
assert not any(a[0] == "send_job" for a, _ in ctx.enqueued)
# Shown in Scheduled as failed, with the reason.
waiting = (await ctx.client.get("/api/outbox")).json()
assert waiting[0]["state"] == "failed" and "550" in waiting[0]["error"]
async def test_delivered_but_not_filed_counts_as_sent(ctx: Ctx, monkeypatch) -> None: # type: ignore[no-untyped-def]
oid = await _make_outbox(ctx)
fake = FakeProvider()
fake.append_error = ProviderConnectError("imap unreachable")
monkeypatch.setattr(outbox_jobs, "build_provider", lambda _a: fake)
await outbox_jobs.send_job({"job_try": 1}, str(oid))
assert (await _outbox(ctx, oid)).state == "sent" and len(fake.sent) == 1
async def test_two_jobs_for_one_send_send_once(ctx: Ctx, monkeypatch) -> None: # type: ignore[no-untyped-def]
oid = await _make_outbox(ctx, state="retrying")
fake = FakeProvider()
monkeypatch.setattr(outbox_jobs, "build_provider", lambda _a: fake)
await asyncio.gather(
outbox_jobs.send_job({"job_try": 1}, str(oid)),
outbox_jobs.send_job({"job_try": 1}, str(oid)),
)
assert len(fake.sent) == 1
async def test_retry_now_and_cancel(ctx: Ctx) -> None:
oid = await _make_outbox(ctx, state="failed")
resp = await ctx.client.post(f"/api/outbox/{oid}/retry", headers=_hdr())
assert resp.status_code == 200
ob = await _outbox(ctx, oid)
assert ob.state == "retrying"
assert ob.retry_until is not None and ob.retry_until > datetime.now(UTC) + timedelta(hours=23)
assert ctx.enqueued[-1][0] == ("send_job", str(oid))
resp = await ctx.client.post(f"/api/outbox/{oid}/cancel", headers=_hdr())
assert resp.status_code == 200 and resp.json()["draft_id"] == str(ob.draft_id)
assert (await _outbox(ctx, oid)).state == "cancelled"
# Back in Drafts, gone from Scheduled.
drafts = (await ctx.client.get("/api/drafts")).json()
assert any(d["id"] == str(ob.draft_id) for d in drafts)
assert (await ctx.client.get("/api/outbox")).json() == []
# Only a waiting send can be retried or cancelled.
again = await ctx.client.post(f"/api/outbox/{oid}/retry", headers=_hdr())
assert again.status_code == 409
+11 -3
View File
@@ -44,7 +44,9 @@ async def connections(monkeypatch) -> AsyncIterator[dict[uuid.UUID, int]]: # ty
"""Patch the runner's loops; yield a live (hot-tier IDLE) connection count."""
conns: dict[uuid.UUID, int] = defaultdict(int)
async def fake_live_loop(account_id: uuid.UUID, folder: str, stop: asyncio.Event) -> None:
async def fake_live_loop(
account_id: uuid.UUID, folder: str, stop: asyncio.Event, health: object = None
) -> None:
conns[account_id] += 1 # one persistent IDLE connection while alive (hot)
try:
await stop.wait()
@@ -52,13 +54,19 @@ async def connections(monkeypatch) -> AsyncIterator[dict[uuid.UUID, int]]: # ty
conns[account_id] -= 1
async def fake_secondary(
account_id: uuid.UUID, stop: asyncio.Event | None = None, interval: float | None = None
account_id: uuid.UUID,
stop: asyncio.Event | None = None,
interval: float | None = None,
health: object = None,
) -> None:
assert stop is not None
await stop.wait()
async def fake_poll_all(
account_id: uuid.UUID, interval: float, stop: asyncio.Event | None = None
account_id: uuid.UUID,
interval: float,
stop: asyncio.Event | None = None,
health: object = None,
) -> None:
assert stop is not None # warm/cold: no persistent connection, just wait
await stop.wait()
+127
View File
@@ -0,0 +1,127 @@
"""An account whose mail server is unreachable, or refuses the password."""
from __future__ import annotations
import uuid
from collections.abc import AsyncIterator
import aiosmtplib
import pytest_asyncio
from aioimaplib import aioimaplib
from sqlalchemy import delete
from app.db import session_scope
from app.identity.passwords import hash_password
from app.models import Account, User
from app.providers.base import (
ProviderAuthError,
ProviderConnectError,
ProviderRejectedError,
)
from app.providers.imap_smtp.provider import _smtp_error
from app.sync.health import AUTH_GIVE_UP_AFTER, UNREACHABLE_AFTER, AccountHealth, is_temporary
@pytest_asyncio.fixture
async def account_id() -> AsyncIterator[uuid.UUID]:
async with session_scope() as session:
user = User(
handle=f"hl{uuid.uuid4().hex[:8]}@test.local", password_hash=hash_password("pw")
)
session.add(user)
await session.flush()
account = Account(
user_id=user.id,
email="me@example.com",
imap_config={"host": "imap.example.com", "port": 993, "tls": "ssl"},
smtp_config={"host": "smtp.example.com", "port": 465, "tls": "ssl"},
secret_ct=b"",
discovery_src="manual",
state="live",
)
session.add(account)
await session.flush()
user_id, aid = user.id, account.id
try:
yield aid
finally:
async with session_scope() as session:
await session.execute(delete(User).where(User.id == user_id))
async def _account(aid: uuid.UUID) -> Account:
async with session_scope() as session:
account = await session.get(Account, aid)
assert account is not None
return account
def test_what_counts_as_temporary() -> None:
assert is_temporary(ProviderConnectError("cannot connect"))
assert is_temporary(TimeoutError())
assert is_temporary(ConnectionResetError())
assert is_temporary(aioimaplib.CommandTimeout("uid move"))
# After the connection drops, aioimaplib refuses commands this way.
assert is_temporary(aioimaplib.Abort("command UID illegal in state LOGOUT"))
assert not is_temporary(aioimaplib.Abort("server has not MOVE capability"))
assert not is_temporary(ProviderAuthError("bad password"))
assert not is_temporary(ProviderRejectedError("550 no such user"))
assert not is_temporary(ValueError("bug"))
def test_smtp_errors_say_whether_to_retry() -> None:
assert isinstance(_smtp_error(aiosmtplib.SMTPConnectError("refused")), ProviderConnectError)
assert isinstance(_smtp_error(aiosmtplib.SMTPServerDisconnected("gone")), ProviderConnectError)
assert isinstance(
_smtp_error(aiosmtplib.SMTPResponseException(451, "try later")), ProviderConnectError
)
assert isinstance(
_smtp_error(aiosmtplib.SMTPResponseException(554, "rejected")), ProviderRejectedError
)
assert isinstance(
_smtp_error(aiosmtplib.SMTPAuthenticationError(535, "bad credentials")), ProviderAuthError
)
refused = aiosmtplib.SMTPRecipientsRefused(
[aiosmtplib.SMTPRecipientRefused(550, "no such user", "x@example.com")]
)
assert isinstance(_smtp_error(refused), ProviderRejectedError)
greylisted = aiosmtplib.SMTPRecipientsRefused(
[aiosmtplib.SMTPRecipientRefused(450, "greylisted", "x@example.com")]
)
assert isinstance(_smtp_error(greylisted), ProviderConnectError)
async def test_unreachable_after_failures_in_a_row_then_clears(account_id: uuid.UUID) -> None:
health = AccountHealth(account_id)
for _ in range(UNREACHABLE_AFTER - 1):
await health.failed(ProviderConnectError("cannot connect"))
assert (await _account(account_id)).sync_error is None # a blip is not an outage
await health.failed(ProviderConnectError("cannot connect"))
account = await _account(account_id)
assert account.sync_error == "unreachable" and account.sync_error_since is not None
assert account.state == "live" # still synced: retried until it answers
await health.ok()
account = await _account(account_id)
assert account.sync_error is None and account.sync_error_since is None
async def test_mark_survives_a_restart_and_clears_on_success(account_id: uuid.UUID) -> None:
first = AccountHealth(account_id)
for _ in range(UNREACHABLE_AFTER):
await first.failed(ProviderConnectError("cannot connect"))
# A new worker knows nothing of the mark, yet its first success clears it.
await AccountHealth(account_id).ok()
assert (await _account(account_id)).sync_error is None
async def test_refused_sign_in_stops_syncing(account_id: uuid.UUID) -> None:
health = AccountHealth(account_id)
for _ in range(AUTH_GIVE_UP_AFTER - 1):
await health.failed(ProviderAuthError("AUTHENTICATIONFAILED"))
assert (await _account(account_id)).state == "live"
await health.failed(ProviderAuthError("AUTHENTICATIONFAILED"))
account = await _account(account_id)
assert account.state == "auth_failed" and "AUTHENTICATIONFAILED" in account.state_detail
+17 -3
View File
@@ -8,7 +8,7 @@ from collections.abc import AsyncIterator
from datetime import UTC, date, datetime, timedelta
from email.message import EmailMessage
from sqlalchemy import func, select
from sqlalchemy import delete, func, select
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from sqlalchemy.pool import NullPool
@@ -16,7 +16,9 @@ from app.config import get_settings
from app.identity.passwords import hash_password
from app.models import Account, Mailbox, Message, User
from app.providers.base import RemoteMailbox, RemoteMessage
from app.redis_client import get_redis
from app.sync import engine as sync_engine
from app.sync import leases
def _raw(mid: str) -> bytes:
@@ -89,6 +91,16 @@ async def _seed(sm: async_sessionmaker, window: str) -> tuple[uuid.UUID, uuid.UU
return user.id, aid
async def _cleanup(sm: async_sessionmaker, user_id: uuid.UUID, account_id: uuid.UUID) -> None: # type: ignore[type-arg]
"""Remove the seeded user — backfill sets its account live, and left behind a
running dev worker would keep trying to sync it."""
await leases.mark_not_live(get_redis(), account_id)
session = sm()
await session.execute(delete(User).where(User.id == user_id))
await session.commit()
await session.close()
async def test_window_new_sets_watermark_and_skips_backfill(monkeypatch) -> None: # type: ignore[no-untyped-def]
engine = create_async_engine(get_settings().database_url, poolclass=NullPool)
sm = async_sessionmaker(engine, expire_on_commit=False)
@@ -112,6 +124,7 @@ async def test_window_new_sets_watermark_and_skips_backfill(monkeypatch) -> None
assert msgs == 0 # nothing imported
await session.close()
finally:
await _cleanup(sm, _uid, account_id)
await engine.dispose()
@@ -142,6 +155,7 @@ async def test_window_all_imports_recent_then_remainder(monkeypatch) -> None: #
assert msgs == 2 # one from each pass
await session.close()
finally:
await _cleanup(sm, _uid, account_id)
await engine.dispose()
@@ -157,6 +171,6 @@ async def test_window_3months_single_dated_pass(monkeypatch) -> None: # type: i
since = date.today() - timedelta(days=sync_engine._WINDOW_DAYS["3months"])
assert fake.backfill_calls == [("INBOX", since, None)] # single windowed pass
assert fake.count_calls == [("INBOX", since)] # denominator counts the window
await engine.dispose()
finally:
pass
await _cleanup(sm, _uid, account_id)
await engine.dispose()