110 lines
4.2 KiB
Python
110 lines
4.2 KiB
Python
 
|
"""An account's connection health.
|
||
|
|
|
||
|
|
Telling "the mail server is not there right now" from everything else.
|
||
|
|
|
||
|
|
``is_temporary`` decides whether a failure is worth another try later: the
|
||
|
|
server could not be reached, timed out, dropped the connection, or answered
|
||
|
|
"not now" (SMTP 4xx). Sends, user actions and the sync loops all retry on it,
|
||
|
|
and only on it — a refused password or a permanent 5xx is not retried.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import logging
|
||
|
|
import uuid
|
||
|
|
from datetime import UTC, datetime
|
||
|
|
|
||
|
|
from aioimaplib import aioimaplib
|
||
|
|
from sqlalchemy import update
|
||
|
|
|
||
|
|
from app.providers.base import ProviderAuthError, ProviderConnectError
|
||
|
|
|
||
|
|
logger = logging.getLogger(__name__)
|
||
|
|
|
||
|
|
|
||
|
|
def is_temporary(exc: BaseException) -> bool:
|
||
|
|
if isinstance(exc, (ProviderConnectError, TimeoutError, OSError)):
|
||
|
|
return True
|
||
|
|
# aioimaplib's own: a command that timed out, or a reply cut off.
|
||
|
|
if isinstance(exc, (aioimaplib.CommandTimeout, aioimaplib.IncompleteRead)):
|
||
|
|
return True
|
||
|
|
# After the connection drops, aioimaplib refuses every further command as
|
||
|
|
# "illegal in state LOGOUT" — the connection is gone, not the command wrong.
|
||
|
|
return isinstance(exc, aioimaplib.Abort) and "illegal in state" in str(exc)
|
||
|
|
|
||
|
|
|
||
|
|
# Failed connections in a row before an account shows as unreachable, and
|
||
|
|
# refused sign-ins in a row before its syncing stops ("auth_failed": retrying a
|
||
|
|
# wrong password only gets the account locked by the provider). Changing the
|
||
|
|
# password in Settings re-runs the initial sync, which sets it live again.
|
||
|
|
UNREACHABLE_AFTER = 3
|
||
|
|
AUTH_GIVE_UP_AFTER = 3
|
||
|
|
|
||
|
|
|
||
|
|
class AccountHealth:
|
||
|
|
"""One account's run of failures, shared by its sync loops."""
|
||
|
|
|
||
|
|
def __init__(self, account_id: uuid.UUID) -> None:
|
||
|
|
self.account_id = account_id
|
||
|
|
self.failures = 0
|
||
|
|
self.auth_failures = 0
|
||
|
|
# Whether the account is marked unreachable; None = not known yet (a
|
||
|
|
# mark can survive a worker restart), so the first success checks.
|
||
|
|
self.marked: bool | None = None
|
||
|
|
# Set once the sign-in was refused too often: the account is no longer
|
||
|
|
# synced (state auth_failed), so its loops should stop now.
|
||
|
|
self.gave_up = False
|
||
|
|
|
||
|
|
async def ok(self) -> None:
|
||
|
|
"""The server answered: reset the counts, and clear an unreachable mark."""
|
||
|
|
self.failures = 0
|
||
|
|
self.auth_failures = 0
|
||
|
|
if self.marked is not False:
|
||
|
|
await _set_sync_error(self.account_id, None)
|
||
|
|
self.marked = False
|
||
|
|
|
||
|
|
async def failed(self, exc: BaseException) -> None:
|
||
|
|
if isinstance(exc, ProviderAuthError):
|
||
|
|
self.auth_failures += 1
|
||
|
|
if self.auth_failures >= AUTH_GIVE_UP_AFTER:
|
||
|
|
from app.sync.engine import _set_state
|
||
|
|
|
||
|
|
logger.warning(
|
||
|
|
"account %s: sign-in refused %d times; stopping sync",
|
||
|
|
self.account_id,
|
||
|
|
self.auth_failures,
|
||
|
|
)
|
||
|
|
await _set_state(self.account_id, "auth_failed", exc.detail[:500])
|
||
|
|
self.gave_up = True
|
||
|
|
return
|
||
|
|
if not is_temporary(exc):
|
||
|
|
return
|
||
|
|
self.failures += 1
|
||
|
|
if self.failures >= UNREACHABLE_AFTER and self.marked is not True:
|
||
|
|
await _set_sync_error(self.account_id, "unreachable")
|
||
|
|
self.marked = True
|
||
|
|
|
||
|
|
|
||
|
|
async def _set_sync_error(account_id: uuid.UUID, error: str | None) -> None:
|
||
|
|
"""Mark (or clear) an account's sync problem; tell the user's open apps."""
|
||
|
|
from app.db import session_scope
|
||
|
|
from app.events import get_event_bus
|
||
|
|
from app.models import Account
|
||
|
|
|
||
|
|
now = datetime.now(UTC)
|
||
|
|
async with session_scope() as session:
|
||
|
|
cond = Account.sync_error.isnot(None) if error is None else Account.sync_error.is_(None)
|
||
|
|
user_id = await session.scalar(
|
||
|
|
update(Account)
|
||
|
|
.where(Account.id == account_id, cond)
|
||
|
|
.values(sync_error=error, sync_error_since=None if error is None else now)
|
||
|
|
.returning(Account.user_id)
|
||
|
|
)
|
||
|
|
if user_id is None:
|
||
|
|
return # nothing changed
|
||
|
|
logger.warning("account %s: %s", account_id, error or "reachable again")
|
||
|
|
await get_event_bus().publish(
|
||
|
|
"account.state_changed",
|
||
|
|
{"user_id": str(user_id), "account_id": str(account_id), "sync_error": error},
|
||
|
|
)
|