Unreachable mail servers: send retries in Scheduled, real action retries, account health
Sending: a send the server could not take (unreachable, timeout, 4xx) is no
longer failed at once. It waits in Scheduled ("retrying") and is tried again
after 1, 2, 5, 10, 30 min, then hourly, for 24 hours — on the server, so it
works with the browser closed. A permanent refusal (5xx, wrong password) is
"failed" at once, with the server's reason. GET /outbox lists both;
POST /outbox/{id}/retry (Retry now) and /cancel (Edit, Delete) act on them,
and they are left out of Drafts. Mail delivered but not filed in Sent now
counts as sent, and a conditional claim keeps two jobs from sending one
message twice. The app says when a send failed and when it finally went.
Actions: arq retries a job only on Retry, never on a plain exception, so the
"retry x5, then revert" for flags and moves never happened. Flag, move,
archive, restore, delete and empty-folder jobs now raise Retry while the
server is unreachable (~24 hours of growing gaps), skip a change a later one
replaced, stop at the first connection failure instead of 20 s per message,
and delete/empty keep the rows until the server has the messages gone.
Account health: 3 failed connections in a row mark an account unreachable
(amber in the sidebar and Settings, cleared on the next success); 3 refused
sign-ins stop its syncing as auth_failed until the password is entered
again, so the provider does not lock the account. Sync loops back off 30 s
doubling to 10 min and log one line per expected failure; a refused
connection fails at once with its reason (the connect task is kept), and a
silently dead IDLE connection is noticed within ~11 min instead of ~29.
Also: the empty reading pane is clipped to its width when dragged very
narrow; tests no longer leave live accounts and outbox rows in the database.
Migration 0021: outbox attempts / next_attempt_at / retry_until, accounts
sync_error / sync_error_since.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -9,7 +9,9 @@ from datetime import UTC, datetime
|
||||
from email.message import EmailMessage
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from arq import Retry
|
||||
from httpx import ASGITransport
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
@@ -23,7 +25,7 @@ from app.identity.sessions import create_session
|
||||
from app.indexer.indexer import persist_message
|
||||
from app.main import create_app
|
||||
from app.models import Account, Mailbox, Message, Thread, User
|
||||
from app.providers.base import RemoteMessage
|
||||
from app.providers.base import ProviderConnectError, RemoteMessage
|
||||
from app.sync import actions as sync_actions
|
||||
|
||||
CSRF = "csrf"
|
||||
@@ -319,3 +321,103 @@ async def test_push_flags_job_reverts_on_failure(ctx: Ctx, monkeypatch) -> None:
|
||||
reverted = await ctx.session.get(Message, mid)
|
||||
await ctx.session.refresh(reverted)
|
||||
assert reverted is not None and "\\Seen" not in reverted.flags
|
||||
|
||||
|
||||
# --- the mail server unreachable: real retries (arq Retry), ~24 hours -------------
|
||||
|
||||
|
||||
class _Unreachable:
|
||||
"""A provider whose server cannot be reached."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls = 0
|
||||
|
||||
async def push_flags(self, *a: object, **k: object) -> None:
|
||||
self.calls += 1
|
||||
raise ProviderConnectError("cannot connect to imap.example.com")
|
||||
|
||||
async def move(self, *a: object, **k: object) -> int:
|
||||
self.calls += 1
|
||||
raise ProviderConnectError("cannot connect to imap.example.com")
|
||||
|
||||
async def delete_messages(self, *a: object, **k: object) -> None:
|
||||
self.calls += 1
|
||||
raise ProviderConnectError("cannot connect to imap.example.com")
|
||||
|
||||
async def close(self) -> None:
|
||||
return None
|
||||
|
||||
|
||||
async def _mark_read(ctx: Ctx) -> uuid.UUID:
|
||||
mid = ctx.message_ids[0]
|
||||
msg = await ctx.session.get(Message, mid)
|
||||
assert msg is not None
|
||||
msg.flags = ["\\Seen"]
|
||||
await ctx.session.commit()
|
||||
return mid
|
||||
|
||||
|
||||
async def test_flag_push_retries_while_unreachable(ctx: Ctx, monkeypatch) -> None: # type: ignore[no-untyped-def]
|
||||
mid = await _mark_read(ctx)
|
||||
monkeypatch.setattr(sync_actions, "build_provider", lambda _a: _Unreachable())
|
||||
|
||||
with pytest.raises(Retry) as caught:
|
||||
await sync_actions.push_flags_job({"job_try": 1}, str(mid), ["\\Seen"], [])
|
||||
assert caught.value.defer_score == 30_000 # first gap: 30 s
|
||||
msg = await ctx.session.get(Message, mid)
|
||||
await ctx.session.refresh(msg)
|
||||
assert msg is not None and "\\Seen" in msg.flags # kept while retrying
|
||||
|
||||
# The last try still cannot reach it: the change is rolled back.
|
||||
await sync_actions.push_flags_job(
|
||||
{"job_try": sync_actions.ACTION_MAX_TRIES}, str(mid), ["\\Seen"], []
|
||||
)
|
||||
await ctx.session.refresh(msg)
|
||||
assert "\\Seen" not in msg.flags
|
||||
|
||||
|
||||
def test_retry_gaps_cover_about_a_day() -> None:
|
||||
total = sum(sync_actions._RETRY_DEFERS_S)
|
||||
assert 23 * 3600 < total <= 24 * 3600
|
||||
|
||||
|
||||
async def test_superseded_flag_change_is_not_pushed(ctx: Ctx, monkeypatch) -> None: # type: ignore[no-untyped-def]
|
||||
# "Read" was queued, then the message was marked unread again: the stale
|
||||
# "read" must not reach the server after the newer change.
|
||||
mid = ctx.message_ids[0]
|
||||
provider = _Unreachable()
|
||||
monkeypatch.setattr(sync_actions, "build_provider", lambda _a: provider)
|
||||
await sync_actions.push_flags_job({"job_try": 3}, str(mid), ["\\Seen"], [])
|
||||
assert provider.calls == 0
|
||||
|
||||
|
||||
async def test_archive_retries_while_unreachable(ctx: Ctx, monkeypatch) -> None: # type: ignore[no-untyped-def]
|
||||
provider = _Unreachable()
|
||||
monkeypatch.setattr(sync_actions, "build_provider", lambda _a: provider)
|
||||
with pytest.raises(Retry):
|
||||
await sync_actions.move_thread_to_folder_job({"job_try": 1}, str(ctx.thread_id), "archive")
|
||||
assert provider.calls == 1 # stopped at the first failure, not 20 s per message
|
||||
ctx.session.expunge_all()
|
||||
async with session_scope() as verify:
|
||||
inbox = await verify.scalar(
|
||||
select(Mailbox).where(Mailbox.user_id == ctx.user_id, Mailbox.special_use == "inbox")
|
||||
)
|
||||
assert inbox is not None
|
||||
moved = await verify.scalar(
|
||||
select(func.count())
|
||||
.select_from(Message)
|
||||
.where(Message.thread_id == ctx.thread_id, Message.mailbox_id != inbox.id)
|
||||
)
|
||||
assert moved == 0 # nothing recorded as moved
|
||||
|
||||
|
||||
async def test_delete_keeps_rows_while_unreachable(ctx: Ctx, monkeypatch) -> None: # type: ignore[no-untyped-def]
|
||||
monkeypatch.setattr(sync_actions, "build_provider", lambda _a: _Unreachable())
|
||||
with pytest.raises(Retry):
|
||||
await sync_actions.delete_thread_job({"job_try": 1}, str(ctx.thread_id))
|
||||
ctx.session.expunge_all()
|
||||
async with session_scope() as verify:
|
||||
left = await verify.scalar(
|
||||
select(func.count()).select_from(Message).where(Message.thread_id == ctx.thread_id)
|
||||
)
|
||||
assert left == len(ctx.message_ids) # kept: the server still has them
|
||||
|
||||
@@ -2,11 +2,12 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import email
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from email.policy import default as default_policy
|
||||
|
||||
import httpx
|
||||
@@ -25,6 +26,7 @@ from app.main import create_app
|
||||
from app.models import Account, Draft, KnownRecipient, Mailbox, Message, Outbox, User
|
||||
from app.outbox import jobs as outbox_jobs
|
||||
from app.outbox.mime_build import build_mime
|
||||
from app.providers.base import ProviderConnectError, ProviderRejectedError
|
||||
|
||||
CSRF = "csrf"
|
||||
|
||||
@@ -193,6 +195,8 @@ async def ctx(monkeypatch) -> AsyncIterator[Ctx]: # type: ignore[no-untyped-def
|
||||
yield c
|
||||
finally:
|
||||
await c.client.aclose()
|
||||
# outbox.user_id has no foreign key: drop the rows with the user.
|
||||
await session.execute(delete(Outbox).where(Outbox.user_id == user.id))
|
||||
await session.execute(delete(User).where(User.id == user.id))
|
||||
await session.commit()
|
||||
await session.close()
|
||||
@@ -425,13 +429,19 @@ class FakeProvider:
|
||||
self.sent: list = []
|
||||
self.appended: list = []
|
||||
self.raise_on_send = False
|
||||
self.send_error: Exception | None = None
|
||||
self.append_error: Exception | None = None
|
||||
|
||||
async def send(self, raw: bytes, env_from: str, env_to: list) -> None:
|
||||
if self.raise_on_send:
|
||||
raise RuntimeError("smtp down")
|
||||
if self.send_error is not None:
|
||||
raise self.send_error
|
||||
self.sent.append((env_from, env_to))
|
||||
|
||||
async def append(self, mailbox: str, raw: bytes, flags: frozenset) -> int:
|
||||
if self.append_error is not None:
|
||||
raise self.append_error
|
||||
self.appended.append(mailbox)
|
||||
return 99
|
||||
|
||||
@@ -516,3 +526,124 @@ async def test_send_job_failure_marks_failed(ctx: Ctx, monkeypatch) -> None: #
|
||||
ob = await ctx.session.get(Outbox, oid)
|
||||
await ctx.session.refresh(ob)
|
||||
assert ob is not None and ob.state == "failed" and ob.error
|
||||
|
||||
|
||||
# --- a send the server could not take: Scheduled, retried for 24 hours ------------
|
||||
|
||||
|
||||
async def _outbox(ctx: Ctx, oid: uuid.UUID) -> Outbox:
|
||||
ob = await ctx.session.get(Outbox, oid)
|
||||
assert ob is not None
|
||||
await ctx.session.refresh(ob)
|
||||
return ob
|
||||
|
||||
|
||||
async def test_unreachable_smtp_is_retried_and_waits_in_scheduled(ctx: Ctx, monkeypatch) -> None: # type: ignore[no-untyped-def]
|
||||
oid = await _make_outbox(ctx)
|
||||
fake = FakeProvider()
|
||||
fake.send_error = ProviderConnectError("cannot connect to smtp.example.com")
|
||||
monkeypatch.setattr(outbox_jobs, "build_provider", lambda _a: fake)
|
||||
|
||||
before = datetime.now(UTC)
|
||||
await outbox_jobs.send_job({"job_try": 1}, str(oid))
|
||||
|
||||
ob = await _outbox(ctx, oid)
|
||||
assert ob.state == "retrying" and ob.attempts == 1 and "cannot connect" in (ob.error or "")
|
||||
assert ob.retry_until is not None
|
||||
assert ob.retry_until >= before + timedelta(hours=23, minutes=59)
|
||||
assert ob.next_attempt_at is not None
|
||||
assert timedelta(seconds=55) < ob.next_attempt_at - before < timedelta(seconds=70)
|
||||
args, kwargs = ctx.enqueued[-1]
|
||||
assert args == ("send_job", str(oid))
|
||||
assert kwargs["_defer_until"] == ob.next_attempt_at and kwargs["_job_id"] == f"send:{oid}:1"
|
||||
|
||||
# Waiting in Scheduled, not in Drafts.
|
||||
drafts = (await ctx.client.get("/api/drafts")).json()
|
||||
assert all(d["id"] != str(ob.draft_id) for d in drafts)
|
||||
waiting = (await ctx.client.get("/api/outbox")).json()
|
||||
assert [(w["outbox_id"], w["state"], w["subject"]) for w in waiting] == [
|
||||
(str(oid), "retrying", "Re: Hi")
|
||||
]
|
||||
|
||||
# The server is back: the planned retry sends it.
|
||||
fake.send_error = None
|
||||
await outbox_jobs.send_job({"job_try": 1}, str(oid))
|
||||
assert (await _outbox(ctx, oid)).state == "sent" and len(fake.sent) == 1
|
||||
assert (await ctx.client.get("/api/outbox")).json() == []
|
||||
|
||||
|
||||
def test_retry_gaps_grow_then_hourly() -> None:
|
||||
assert [outbox_jobs._retry_gap(n) for n in range(1, 8)] == [60, 120, 300, 600, 1800, 3600, 3600]
|
||||
|
||||
|
||||
async def test_send_gives_up_after_24_hours(ctx: Ctx, monkeypatch) -> None: # type: ignore[no-untyped-def]
|
||||
oid = await _make_outbox(ctx, state="retrying")
|
||||
ob = await _outbox(ctx, oid)
|
||||
ob.retry_until = datetime.now(UTC) - timedelta(seconds=1)
|
||||
await ctx.session.commit()
|
||||
fake = FakeProvider()
|
||||
fake.send_error = ProviderConnectError("still down")
|
||||
monkeypatch.setattr(outbox_jobs, "build_provider", lambda _a: fake)
|
||||
|
||||
await outbox_jobs.send_job({"job_try": 1}, str(oid))
|
||||
ob = await _outbox(ctx, oid)
|
||||
assert ob.state == "failed" and ob.next_attempt_at is None
|
||||
assert not any(a[0] == "send_job" for a, _ in ctx.enqueued)
|
||||
|
||||
|
||||
async def test_permanent_refusal_fails_at_once(ctx: Ctx, monkeypatch) -> None: # type: ignore[no-untyped-def]
|
||||
oid = await _make_outbox(ctx)
|
||||
fake = FakeProvider()
|
||||
fake.send_error = ProviderRejectedError("550 no such user")
|
||||
monkeypatch.setattr(outbox_jobs, "build_provider", lambda _a: fake)
|
||||
|
||||
await outbox_jobs.send_job({"job_try": 1}, str(oid))
|
||||
ob = await _outbox(ctx, oid)
|
||||
assert ob.state == "failed" and "550" in (ob.error or "")
|
||||
assert not any(a[0] == "send_job" for a, _ in ctx.enqueued)
|
||||
# Shown in Scheduled as failed, with the reason.
|
||||
waiting = (await ctx.client.get("/api/outbox")).json()
|
||||
assert waiting[0]["state"] == "failed" and "550" in waiting[0]["error"]
|
||||
|
||||
|
||||
async def test_delivered_but_not_filed_counts_as_sent(ctx: Ctx, monkeypatch) -> None: # type: ignore[no-untyped-def]
|
||||
oid = await _make_outbox(ctx)
|
||||
fake = FakeProvider()
|
||||
fake.append_error = ProviderConnectError("imap unreachable")
|
||||
monkeypatch.setattr(outbox_jobs, "build_provider", lambda _a: fake)
|
||||
|
||||
await outbox_jobs.send_job({"job_try": 1}, str(oid))
|
||||
assert (await _outbox(ctx, oid)).state == "sent" and len(fake.sent) == 1
|
||||
|
||||
|
||||
async def test_two_jobs_for_one_send_send_once(ctx: Ctx, monkeypatch) -> None: # type: ignore[no-untyped-def]
|
||||
oid = await _make_outbox(ctx, state="retrying")
|
||||
fake = FakeProvider()
|
||||
monkeypatch.setattr(outbox_jobs, "build_provider", lambda _a: fake)
|
||||
|
||||
await asyncio.gather(
|
||||
outbox_jobs.send_job({"job_try": 1}, str(oid)),
|
||||
outbox_jobs.send_job({"job_try": 1}, str(oid)),
|
||||
)
|
||||
assert len(fake.sent) == 1
|
||||
|
||||
|
||||
async def test_retry_now_and_cancel(ctx: Ctx) -> None:
|
||||
oid = await _make_outbox(ctx, state="failed")
|
||||
resp = await ctx.client.post(f"/api/outbox/{oid}/retry", headers=_hdr())
|
||||
assert resp.status_code == 200
|
||||
ob = await _outbox(ctx, oid)
|
||||
assert ob.state == "retrying"
|
||||
assert ob.retry_until is not None and ob.retry_until > datetime.now(UTC) + timedelta(hours=23)
|
||||
assert ctx.enqueued[-1][0] == ("send_job", str(oid))
|
||||
|
||||
resp = await ctx.client.post(f"/api/outbox/{oid}/cancel", headers=_hdr())
|
||||
assert resp.status_code == 200 and resp.json()["draft_id"] == str(ob.draft_id)
|
||||
assert (await _outbox(ctx, oid)).state == "cancelled"
|
||||
# Back in Drafts, gone from Scheduled.
|
||||
drafts = (await ctx.client.get("/api/drafts")).json()
|
||||
assert any(d["id"] == str(ob.draft_id) for d in drafts)
|
||||
assert (await ctx.client.get("/api/outbox")).json() == []
|
||||
# Only a waiting send can be retried or cancelled.
|
||||
again = await ctx.client.post(f"/api/outbox/{oid}/retry", headers=_hdr())
|
||||
assert again.status_code == 409
|
||||
|
||||
@@ -44,7 +44,9 @@ async def connections(monkeypatch) -> AsyncIterator[dict[uuid.UUID, int]]: # ty
|
||||
"""Patch the runner's loops; yield a live (hot-tier IDLE) connection count."""
|
||||
conns: dict[uuid.UUID, int] = defaultdict(int)
|
||||
|
||||
async def fake_live_loop(account_id: uuid.UUID, folder: str, stop: asyncio.Event) -> None:
|
||||
async def fake_live_loop(
|
||||
account_id: uuid.UUID, folder: str, stop: asyncio.Event, health: object = None
|
||||
) -> None:
|
||||
conns[account_id] += 1 # one persistent IDLE connection while alive (hot)
|
||||
try:
|
||||
await stop.wait()
|
||||
@@ -52,13 +54,19 @@ async def connections(monkeypatch) -> AsyncIterator[dict[uuid.UUID, int]]: # ty
|
||||
conns[account_id] -= 1
|
||||
|
||||
async def fake_secondary(
|
||||
account_id: uuid.UUID, stop: asyncio.Event | None = None, interval: float | None = None
|
||||
account_id: uuid.UUID,
|
||||
stop: asyncio.Event | None = None,
|
||||
interval: float | None = None,
|
||||
health: object = None,
|
||||
) -> None:
|
||||
assert stop is not None
|
||||
await stop.wait()
|
||||
|
||||
async def fake_poll_all(
|
||||
account_id: uuid.UUID, interval: float, stop: asyncio.Event | None = None
|
||||
account_id: uuid.UUID,
|
||||
interval: float,
|
||||
stop: asyncio.Event | None = None,
|
||||
health: object = None,
|
||||
) -> None:
|
||||
assert stop is not None # warm/cold: no persistent connection, just wait
|
||||
await stop.wait()
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
"""An account whose mail server is unreachable, or refuses the password."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
import aiosmtplib
|
||||
import pytest_asyncio
|
||||
from aioimaplib import aioimaplib
|
||||
from sqlalchemy import delete
|
||||
|
||||
from app.db import session_scope
|
||||
from app.identity.passwords import hash_password
|
||||
from app.models import Account, User
|
||||
from app.providers.base import (
|
||||
ProviderAuthError,
|
||||
ProviderConnectError,
|
||||
ProviderRejectedError,
|
||||
)
|
||||
from app.providers.imap_smtp.provider import _smtp_error
|
||||
from app.sync.health import AUTH_GIVE_UP_AFTER, UNREACHABLE_AFTER, AccountHealth, is_temporary
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def account_id() -> AsyncIterator[uuid.UUID]:
|
||||
async with session_scope() as session:
|
||||
user = User(
|
||||
handle=f"hl{uuid.uuid4().hex[:8]}@test.local", password_hash=hash_password("pw")
|
||||
)
|
||||
session.add(user)
|
||||
await session.flush()
|
||||
account = Account(
|
||||
user_id=user.id,
|
||||
email="me@example.com",
|
||||
imap_config={"host": "imap.example.com", "port": 993, "tls": "ssl"},
|
||||
smtp_config={"host": "smtp.example.com", "port": 465, "tls": "ssl"},
|
||||
secret_ct=b"",
|
||||
discovery_src="manual",
|
||||
state="live",
|
||||
)
|
||||
session.add(account)
|
||||
await session.flush()
|
||||
user_id, aid = user.id, account.id
|
||||
try:
|
||||
yield aid
|
||||
finally:
|
||||
async with session_scope() as session:
|
||||
await session.execute(delete(User).where(User.id == user_id))
|
||||
|
||||
|
||||
async def _account(aid: uuid.UUID) -> Account:
|
||||
async with session_scope() as session:
|
||||
account = await session.get(Account, aid)
|
||||
assert account is not None
|
||||
return account
|
||||
|
||||
|
||||
def test_what_counts_as_temporary() -> None:
|
||||
assert is_temporary(ProviderConnectError("cannot connect"))
|
||||
assert is_temporary(TimeoutError())
|
||||
assert is_temporary(ConnectionResetError())
|
||||
assert is_temporary(aioimaplib.CommandTimeout("uid move"))
|
||||
# After the connection drops, aioimaplib refuses commands this way.
|
||||
assert is_temporary(aioimaplib.Abort("command UID illegal in state LOGOUT"))
|
||||
assert not is_temporary(aioimaplib.Abort("server has not MOVE capability"))
|
||||
assert not is_temporary(ProviderAuthError("bad password"))
|
||||
assert not is_temporary(ProviderRejectedError("550 no such user"))
|
||||
assert not is_temporary(ValueError("bug"))
|
||||
|
||||
|
||||
def test_smtp_errors_say_whether_to_retry() -> None:
|
||||
assert isinstance(_smtp_error(aiosmtplib.SMTPConnectError("refused")), ProviderConnectError)
|
||||
assert isinstance(_smtp_error(aiosmtplib.SMTPServerDisconnected("gone")), ProviderConnectError)
|
||||
assert isinstance(
|
||||
_smtp_error(aiosmtplib.SMTPResponseException(451, "try later")), ProviderConnectError
|
||||
)
|
||||
assert isinstance(
|
||||
_smtp_error(aiosmtplib.SMTPResponseException(554, "rejected")), ProviderRejectedError
|
||||
)
|
||||
assert isinstance(
|
||||
_smtp_error(aiosmtplib.SMTPAuthenticationError(535, "bad credentials")), ProviderAuthError
|
||||
)
|
||||
refused = aiosmtplib.SMTPRecipientsRefused(
|
||||
[aiosmtplib.SMTPRecipientRefused(550, "no such user", "x@example.com")]
|
||||
)
|
||||
assert isinstance(_smtp_error(refused), ProviderRejectedError)
|
||||
greylisted = aiosmtplib.SMTPRecipientsRefused(
|
||||
[aiosmtplib.SMTPRecipientRefused(450, "greylisted", "x@example.com")]
|
||||
)
|
||||
assert isinstance(_smtp_error(greylisted), ProviderConnectError)
|
||||
|
||||
|
||||
async def test_unreachable_after_failures_in_a_row_then_clears(account_id: uuid.UUID) -> None:
|
||||
health = AccountHealth(account_id)
|
||||
for _ in range(UNREACHABLE_AFTER - 1):
|
||||
await health.failed(ProviderConnectError("cannot connect"))
|
||||
assert (await _account(account_id)).sync_error is None # a blip is not an outage
|
||||
|
||||
await health.failed(ProviderConnectError("cannot connect"))
|
||||
account = await _account(account_id)
|
||||
assert account.sync_error == "unreachable" and account.sync_error_since is not None
|
||||
assert account.state == "live" # still synced: retried until it answers
|
||||
|
||||
await health.ok()
|
||||
account = await _account(account_id)
|
||||
assert account.sync_error is None and account.sync_error_since is None
|
||||
|
||||
|
||||
async def test_mark_survives_a_restart_and_clears_on_success(account_id: uuid.UUID) -> None:
|
||||
first = AccountHealth(account_id)
|
||||
for _ in range(UNREACHABLE_AFTER):
|
||||
await first.failed(ProviderConnectError("cannot connect"))
|
||||
# A new worker knows nothing of the mark, yet its first success clears it.
|
||||
await AccountHealth(account_id).ok()
|
||||
assert (await _account(account_id)).sync_error is None
|
||||
|
||||
|
||||
async def test_refused_sign_in_stops_syncing(account_id: uuid.UUID) -> None:
|
||||
health = AccountHealth(account_id)
|
||||
for _ in range(AUTH_GIVE_UP_AFTER - 1):
|
||||
await health.failed(ProviderAuthError("AUTHENTICATIONFAILED"))
|
||||
assert (await _account(account_id)).state == "live"
|
||||
|
||||
await health.failed(ProviderAuthError("AUTHENTICATIONFAILED"))
|
||||
account = await _account(account_id)
|
||||
assert account.state == "auth_failed" and "AUTHENTICATIONFAILED" in account.state_detail
|
||||
@@ -8,7 +8,7 @@ from collections.abc import AsyncIterator
|
||||
from datetime import UTC, date, datetime, timedelta
|
||||
from email.message import EmailMessage
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.pool import NullPool
|
||||
|
||||
@@ -16,7 +16,9 @@ from app.config import get_settings
|
||||
from app.identity.passwords import hash_password
|
||||
from app.models import Account, Mailbox, Message, User
|
||||
from app.providers.base import RemoteMailbox, RemoteMessage
|
||||
from app.redis_client import get_redis
|
||||
from app.sync import engine as sync_engine
|
||||
from app.sync import leases
|
||||
|
||||
|
||||
def _raw(mid: str) -> bytes:
|
||||
@@ -89,6 +91,16 @@ async def _seed(sm: async_sessionmaker, window: str) -> tuple[uuid.UUID, uuid.UU
|
||||
return user.id, aid
|
||||
|
||||
|
||||
async def _cleanup(sm: async_sessionmaker, user_id: uuid.UUID, account_id: uuid.UUID) -> None: # type: ignore[type-arg]
|
||||
"""Remove the seeded user — backfill sets its account live, and left behind a
|
||||
running dev worker would keep trying to sync it."""
|
||||
await leases.mark_not_live(get_redis(), account_id)
|
||||
session = sm()
|
||||
await session.execute(delete(User).where(User.id == user_id))
|
||||
await session.commit()
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_window_new_sets_watermark_and_skips_backfill(monkeypatch) -> None: # type: ignore[no-untyped-def]
|
||||
engine = create_async_engine(get_settings().database_url, poolclass=NullPool)
|
||||
sm = async_sessionmaker(engine, expire_on_commit=False)
|
||||
@@ -112,6 +124,7 @@ async def test_window_new_sets_watermark_and_skips_backfill(monkeypatch) -> None
|
||||
assert msgs == 0 # nothing imported
|
||||
await session.close()
|
||||
finally:
|
||||
await _cleanup(sm, _uid, account_id)
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@@ -142,6 +155,7 @@ async def test_window_all_imports_recent_then_remainder(monkeypatch) -> None: #
|
||||
assert msgs == 2 # one from each pass
|
||||
await session.close()
|
||||
finally:
|
||||
await _cleanup(sm, _uid, account_id)
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@@ -157,6 +171,6 @@ async def test_window_3months_single_dated_pass(monkeypatch) -> None: # type: i
|
||||
since = date.today() - timedelta(days=sync_engine._WINDOW_DAYS["3months"])
|
||||
assert fake.backfill_calls == [("INBOX", since, None)] # single windowed pass
|
||||
assert fake.count_calls == [("INBOX", since)] # denominator counts the window
|
||||
await engine.dispose()
|
||||
finally:
|
||||
pass
|
||||
await _cleanup(sm, _uid, account_id)
|
||||
await engine.dispose()
|
||||
|
||||
Reference in New Issue
Block a user