lukasandClaude Opus 5.5
609e7c58f9
Sending: a send the server could not take (unreachable, timeout, 4xx) is no
longer failed at once. It waits in Scheduled ("retrying") and is tried again
after 1, 2, 5, 10, 30 min, then hourly, for 24 hours — on the server, so it
works with the browser closed. A permanent refusal (5xx, wrong password) is
"failed" at once, with the server's reason. GET /outbox lists both;
POST /outbox/{id}/retry (Retry now) and /cancel (Edit, Delete) act on them,
and they are left out of Drafts. Mail delivered but not filed in Sent now
counts as sent, and a conditional claim keeps two jobs from sending one
message twice. The app says when a send failed and when it finally went.
Actions: arq retries a job only on Retry, never on a plain exception, so the
"retry x5, then revert" for flags and moves never happened. Flag, move,
archive, restore, delete and empty-folder jobs now raise Retry while the
server is unreachable (~24 hours of growing gaps), skip a change a later one
replaced, stop at the first connection failure instead of 20 s per message,
and delete/empty keep the rows until the server has the messages gone.
Account health: 3 failed connections in a row mark an account unreachable
(amber in the sidebar and Settings, cleared on the next success); 3 refused
sign-ins stop its syncing as auth_failed until the password is entered
again, so the provider does not lock the account. Sync loops back off 30 s
doubling to 10 min and log one line per expected failure; a refused
connection fails at once with its reason (the connect task is kept), and a
silently dead IDLE connection is noticed within ~11 min instead of ~29.
Also: the empty reading pane is clipped to its width when dragged very
narrow; tests no longer leave live accounts and outbox rows in the database.
Migration 0021: outbox attempts / next_attempt_at / retry_until, accounts
sync_error / sync_error_since.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
110 lines
4.2 KiB
Python
110 lines
4.2 KiB
Python
"""An account's connection health.
|
|
|
|
Telling "the mail server is not there right now" from everything else.
|
|
|
|
``is_temporary`` decides whether a failure is worth another try later: the
|
|
server could not be reached, timed out, dropped the connection, or answered
|
|
"not now" (SMTP 4xx). Sends, user actions and the sync loops all retry on it,
|
|
and only on it — a refused password or a permanent 5xx is not retried.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import uuid
|
|
from datetime import UTC, datetime
|
|
|
|
from aioimaplib import aioimaplib
|
|
from sqlalchemy import update
|
|
|
|
from app.providers.base import ProviderAuthError, ProviderConnectError
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def is_temporary(exc: BaseException) -> bool:
|
|
if isinstance(exc, (ProviderConnectError, TimeoutError, OSError)):
|
|
return True
|
|
# aioimaplib's own: a command that timed out, or a reply cut off.
|
|
if isinstance(exc, (aioimaplib.CommandTimeout, aioimaplib.IncompleteRead)):
|
|
return True
|
|
# After the connection drops, aioimaplib refuses every further command as
|
|
# "illegal in state LOGOUT" — the connection is gone, not the command wrong.
|
|
return isinstance(exc, aioimaplib.Abort) and "illegal in state" in str(exc)
|
|
|
|
|
|
# Failed connections in a row before an account shows as unreachable, and
|
|
# refused sign-ins in a row before its syncing stops ("auth_failed": retrying a
|
|
# wrong password only gets the account locked by the provider). Changing the
|
|
# password in Settings re-runs the initial sync, which sets it live again.
|
|
UNREACHABLE_AFTER = 3
|
|
AUTH_GIVE_UP_AFTER = 3
|
|
|
|
|
|
class AccountHealth:
|
|
"""One account's run of failures, shared by its sync loops."""
|
|
|
|
def __init__(self, account_id: uuid.UUID) -> None:
|
|
self.account_id = account_id
|
|
self.failures = 0
|
|
self.auth_failures = 0
|
|
# Whether the account is marked unreachable; None = not known yet (a
|
|
# mark can survive a worker restart), so the first success checks.
|
|
self.marked: bool | None = None
|
|
# Set once the sign-in was refused too often: the account is no longer
|
|
# synced (state auth_failed), so its loops should stop now.
|
|
self.gave_up = False
|
|
|
|
async def ok(self) -> None:
|
|
"""The server answered: reset the counts, and clear an unreachable mark."""
|
|
self.failures = 0
|
|
self.auth_failures = 0
|
|
if self.marked is not False:
|
|
await _set_sync_error(self.account_id, None)
|
|
self.marked = False
|
|
|
|
async def failed(self, exc: BaseException) -> None:
|
|
if isinstance(exc, ProviderAuthError):
|
|
self.auth_failures += 1
|
|
if self.auth_failures >= AUTH_GIVE_UP_AFTER:
|
|
from app.sync.engine import _set_state
|
|
|
|
logger.warning(
|
|
"account %s: sign-in refused %d times; stopping sync",
|
|
self.account_id,
|
|
self.auth_failures,
|
|
)
|
|
await _set_state(self.account_id, "auth_failed", exc.detail[:500])
|
|
self.gave_up = True
|
|
return
|
|
if not is_temporary(exc):
|
|
return
|
|
self.failures += 1
|
|
if self.failures >= UNREACHABLE_AFTER and self.marked is not True:
|
|
await _set_sync_error(self.account_id, "unreachable")
|
|
self.marked = True
|
|
|
|
|
|
async def _set_sync_error(account_id: uuid.UUID, error: str | None) -> None:
|
|
"""Mark (or clear) an account's sync problem; tell the user's open apps."""
|
|
from app.db import session_scope
|
|
from app.events import get_event_bus
|
|
from app.models import Account
|
|
|
|
now = datetime.now(UTC)
|
|
async with session_scope() as session:
|
|
cond = Account.sync_error.isnot(None) if error is None else Account.sync_error.is_(None)
|
|
user_id = await session.scalar(
|
|
update(Account)
|
|
.where(Account.id == account_id, cond)
|
|
.values(sync_error=error, sync_error_since=None if error is None else now)
|
|
.returning(Account.user_id)
|
|
)
|
|
if user_id is None:
|
|
return # nothing changed
|
|
logger.warning("account %s: %s", account_id, error or "reachable again")
|
|
await get_event_bus().publish(
|
|
"account.state_changed",
|
|
{"user_id": str(user_id), "account_id": str(account_id), "sync_error": error},
|
|
)
|