Productivity features, UI translations, local email translation
- Snooze (with a Snoozed folder), keyboard shortcuts with a Settings section, mail templates (Compose > More > Templates), a Subscriptions folder with bulk unsubscribe/delete, and tracker blocking with a Privacy toggle (on by default) - Own dialogs instead of browser popups; emoji picker in compose; compose windows survive a visit to Settings - Sentence completion learned from the user's sent mail (no AI) - Interface in English and Polish (one file per language, English as the fallback), picked from the browser on first run, then kept - Email translation on the server via LibreTranslate: Translate button and "Translate to...", messages in several languages translated part by part, names/addresses/emoji/file names/phone numbers kept, an animated line while translating - Antispam "Mark as spam" loads with the message, not when the menu opens Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,149 @@
|
||||
"""A signed-in user with an API client, for endpoint tests in this package."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass
|
||||
from email.message import EmailMessage
|
||||
|
||||
import httpx
|
||||
import pytest_asyncio
|
||||
from httpx import ASGITransport
|
||||
from sqlalchemy import delete
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.pool import NullPool
|
||||
|
||||
from app.config import get_settings
|
||||
from app.identity.passwords import hash_password
|
||||
from app.identity.sessions import create_session
|
||||
from app.indexer.indexer import persist_message
|
||||
from app.main import create_app
|
||||
from app.models import Account, Mailbox, Message, User
|
||||
from app.providers.base import RemoteMessage
|
||||
|
||||
CSRF = "csrf"
|
||||
HDR = {"X-CSRF": CSRF}
|
||||
|
||||
|
||||
@dataclass
|
||||
class UserCtx:
|
||||
client: httpx.AsyncClient
|
||||
user_id: uuid.UUID
|
||||
session: AsyncSession # for arranging rows directly
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def user_ctx() -> AsyncIterator[UserCtx]:
|
||||
engine = create_async_engine(get_settings().database_url, poolclass=NullPool)
|
||||
sm = async_sessionmaker(engine, expire_on_commit=False)
|
||||
session = sm()
|
||||
user = User(handle=f"t{uuid.uuid4().hex[:10]}@test.local", password_hash=hash_password("pw"))
|
||||
session.add(user)
|
||||
await session.commit()
|
||||
user_id = user.id # kept: a test may expire the session's objects
|
||||
token = await create_session(user_id, "pytest")
|
||||
client = httpx.AsyncClient(
|
||||
transport=ASGITransport(app=create_app()),
|
||||
base_url="http://testserver",
|
||||
cookies={"session": token, "csrf": CSRF},
|
||||
)
|
||||
try:
|
||||
yield UserCtx(client=client, user_id=user_id, session=session)
|
||||
finally:
|
||||
await client.aclose()
|
||||
await session.rollback()
|
||||
await session.execute(delete(User).where(User.id == user_id))
|
||||
await session.commit()
|
||||
await session.close()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@dataclass
|
||||
class MailCtx(UserCtx):
|
||||
account_id: uuid.UUID
|
||||
inbox_id: uuid.UUID
|
||||
sent_id: uuid.UUID
|
||||
|
||||
async def deliver(
|
||||
self,
|
||||
raw: bytes,
|
||||
uid: int,
|
||||
*,
|
||||
seen: bool = False,
|
||||
at: dt.datetime | None = None,
|
||||
folder: str = "inbox",
|
||||
) -> Message:
|
||||
"""Store a message the way sync does (indexer.persist_message), in the
|
||||
Inbox or (``folder="sent"``) the Sent folder."""
|
||||
remote = RemoteMessage(
|
||||
uid=uid,
|
||||
raw=raw,
|
||||
flags=frozenset({"\\Seen"}) if seen else frozenset(),
|
||||
internal_date=at or dt.datetime.now(dt.UTC),
|
||||
)
|
||||
result = await persist_message(
|
||||
self.session,
|
||||
user_id=self.user_id,
|
||||
account_id=self.account_id,
|
||||
mailbox_id=self.sent_id if folder == "sent" else self.inbox_id,
|
||||
mailbox_special_use=folder,
|
||||
remote=remote,
|
||||
)
|
||||
await self.session.commit()
|
||||
assert result is not None
|
||||
return result.message
|
||||
|
||||
|
||||
def raw_mail(
|
||||
subject: str,
|
||||
*,
|
||||
sender: str = "News <news@shop.example>",
|
||||
headers: dict[str, str] | None = None,
|
||||
mid: str | None = None,
|
||||
html: str | None = None,
|
||||
) -> bytes:
|
||||
m = EmailMessage()
|
||||
m["From"] = sender
|
||||
m["To"] = "me@example.com"
|
||||
m["Subject"] = subject
|
||||
m["Message-ID"] = mid or f"<{uuid.uuid4().hex}@shop.example>"
|
||||
for name, value in (headers or {}).items():
|
||||
m[name] = value
|
||||
m.set_content(f"{subject} body")
|
||||
if html is not None:
|
||||
m.add_alternative(html, subtype="html")
|
||||
return m.as_bytes()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def mail_ctx(user_ctx: UserCtx) -> MailCtx:
|
||||
s = user_ctx.session
|
||||
account = Account(
|
||||
user_id=user_ctx.user_id,
|
||||
email="me@example.com",
|
||||
imap_config={},
|
||||
smtp_config={},
|
||||
secret_ct=b"",
|
||||
discovery_src="manual",
|
||||
)
|
||||
s.add(account)
|
||||
await s.flush()
|
||||
inbox = Mailbox(
|
||||
account_id=account.id, user_id=user_ctx.user_id, path="INBOX", special_use="inbox"
|
||||
)
|
||||
trash = Mailbox(
|
||||
account_id=account.id, user_id=user_ctx.user_id, path="Trash", special_use="trash"
|
||||
)
|
||||
sent = Mailbox(account_id=account.id, user_id=user_ctx.user_id, path="Sent", special_use="sent")
|
||||
s.add_all([inbox, trash, sent])
|
||||
await s.commit()
|
||||
return MailCtx(
|
||||
client=user_ctx.client,
|
||||
user_id=user_ctx.user_id,
|
||||
session=s,
|
||||
account_id=account.id,
|
||||
inbox_id=inbox.id,
|
||||
sent_id=sent.id,
|
||||
)
|
||||
@@ -391,7 +391,7 @@ async def test_folder_counts(api: ApiCtx) -> None:
|
||||
# All three seeded inbox messages are unread; no drafts yet.
|
||||
resp = await api.client.get("/api/mailboxes/counts")
|
||||
assert resp.status_code == 200
|
||||
assert resp.json() == {"inbox_unread": 3, "drafts": 0}
|
||||
assert resp.json() == {"inbox_unread": 3, "drafts": 0, "snoozed": 0}
|
||||
|
||||
# Marking one message \Seen lowers the unread count.
|
||||
msg = await api.session.get(Message, api.message_ids[0])
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Sentence completion over the API, from the user's sent mail."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from app.api import completion as api_completion
|
||||
from app.completion import build_index_job
|
||||
from tests.api.conftest import HDR, MailCtx, raw_mail
|
||||
|
||||
|
||||
async def _suggest(ctx: MailCtx, before: str) -> str | None:
|
||||
resp = await ctx.client.post("/api/completion", json={"before": before}, headers=HDR)
|
||||
assert resp.status_code == 200, resp.text
|
||||
return resp.json()["suggestion"]
|
||||
|
||||
|
||||
async def test_suggests_from_sent_mail(mail_ctx: MailCtx, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
queued: list[tuple[Any, ...]] = []
|
||||
|
||||
async def fake_enqueue(name: str, *args: Any, **kwargs: Any) -> None:
|
||||
queued.append((name, *args))
|
||||
|
||||
monkeypatch.setattr(api_completion, "enqueue_job", fake_enqueue)
|
||||
body = "Hi Anna,\nThe renders will be ready by Thursday evening.\n\nOn Mon Anna wrote:\n> ok"
|
||||
for uid in (1, 2):
|
||||
await mail_ctx.deliver(
|
||||
raw_mail("Renders", sender="me@example.com").replace(b"Renders body", body.encode()),
|
||||
uid,
|
||||
folder="sent",
|
||||
)
|
||||
|
||||
# No index yet: a build is asked for, and the starter phrases still answer.
|
||||
assert await _suggest(mail_ctx, "I look forward ") == "to hearing from you."
|
||||
assert queued and queued[0][0] == "build_index_job"
|
||||
|
||||
assert await build_index_job({}, str(mail_ctx.user_id)) > 0
|
||||
assert await _suggest(mail_ctx, "The renders will be re") == "ady by Thursday evening."
|
||||
assert await _suggest(mail_ctx, "> ok") is None # quoted text was not learned
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Snooze: a conversation leaves the Inbox until a set time, then comes back."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.models import Message, Thread
|
||||
from app.snooze import wake_due_job
|
||||
from tests.api.conftest import HDR, MailCtx, raw_mail
|
||||
|
||||
|
||||
def _in(days: float = 1) -> str:
|
||||
return (dt.datetime.now(dt.UTC) + dt.timedelta(days=days)).isoformat()
|
||||
|
||||
|
||||
async def _inbox_ids(ctx: MailCtx) -> list[str]:
|
||||
resp = await ctx.client.get("/api/threads", params={"view": "folder:inbox"})
|
||||
return [t["id"] for t in resp.json()["threads"]]
|
||||
|
||||
|
||||
async def test_snoozed_thread_leaves_the_inbox(mail_ctx: MailCtx) -> None:
|
||||
old = await mail_ctx.deliver(
|
||||
raw_mail("Old"), 1, at=dt.datetime.now(dt.UTC) - dt.timedelta(days=2)
|
||||
)
|
||||
new = await mail_ctx.deliver(raw_mail("New"), 2)
|
||||
c = mail_ctx.client
|
||||
resp = await c.post(f"/api/threads/{new.thread_id}/snooze", json={"until": _in()}, headers=HDR)
|
||||
assert resp.status_code == 200, resp.text
|
||||
|
||||
assert await _inbox_ids(mail_ctx) == [str(old.thread_id)]
|
||||
snoozed = (await c.get("/api/threads", params={"view": "snoozed"})).json()["threads"]
|
||||
assert [t["id"] for t in snoozed] == [str(new.thread_id)]
|
||||
assert snoozed[0]["snoozed_until"] is not None
|
||||
counts = (await c.get("/api/mailboxes/counts")).json()
|
||||
assert counts["snoozed"] == 1 and counts["inbox_unread"] == 1
|
||||
|
||||
# Unsnooze: straight back, at the top.
|
||||
await c.post(f"/api/threads/{new.thread_id}/unsnooze", headers=HDR)
|
||||
assert await _inbox_ids(mail_ctx) == [str(new.thread_id), str(old.thread_id)]
|
||||
|
||||
|
||||
async def test_snooze_needs_a_future_time(mail_ctx: MailCtx) -> None:
|
||||
msg = await mail_ctx.deliver(raw_mail("x"), 1)
|
||||
for until in (_in(-1), _in(400), "2030-01-01T09:00:00"):
|
||||
resp = await mail_ctx.client.post(
|
||||
f"/api/threads/{msg.thread_id}/snooze", json={"until": until}, headers=HDR
|
||||
)
|
||||
assert resp.status_code == 400, until
|
||||
|
||||
|
||||
async def test_due_thread_wakes_on_top_and_unread(mail_ctx: MailCtx) -> None:
|
||||
week_ago = dt.datetime.now(dt.UTC) - dt.timedelta(days=7)
|
||||
snoozed = await mail_ctx.deliver(raw_mail("Snoozed"), 1, seen=True, at=week_ago)
|
||||
newer = await mail_ctx.deliver(raw_mail("Newer"), 2, seen=True)
|
||||
thread = await mail_ctx.session.get(Thread, snoozed.thread_id)
|
||||
assert thread is not None
|
||||
thread.snoozed_until = dt.datetime.now(dt.UTC) - dt.timedelta(minutes=1)
|
||||
thread.snoozed_at = week_ago
|
||||
await mail_ctx.session.commit()
|
||||
|
||||
assert await wake_due_job({}) >= 1
|
||||
assert await _inbox_ids(mail_ctx) == [str(snoozed.thread_id), str(newer.thread_id)]
|
||||
snoozed_id = snoozed.id
|
||||
mail_ctx.session.expire_all()
|
||||
flags = await mail_ctx.session.scalar(select(Message.flags).where(Message.id == snoozed_id))
|
||||
assert flags is not None and "\\Seen" not in flags
|
||||
|
||||
|
||||
async def test_a_reply_wakes_a_snoozed_thread(mail_ctx: MailCtx) -> None:
|
||||
first = await mail_ctx.deliver(raw_mail("Question", mid="<q@x>"), 1)
|
||||
await mail_ctx.client.post(
|
||||
f"/api/threads/{first.thread_id}/snooze", json={"until": _in(3)}, headers=HDR
|
||||
)
|
||||
m = raw_mail("Re: Question", headers={"In-Reply-To": "<q@x>", "References": "<q@x>"})
|
||||
await mail_ctx.deliver(m, 2, at=dt.datetime.now(dt.UTC) + dt.timedelta(seconds=5))
|
||||
|
||||
await wake_due_job({})
|
||||
assert await _inbox_ids(mail_ctx) == [str(first.thread_id)]
|
||||
@@ -0,0 +1,239 @@
|
||||
"""Subscriptions: bulk senders worked out from List-* headers, and unsubscribing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select, update
|
||||
|
||||
from app.api import subscriptions as api_subs
|
||||
from app.models import Draft, Mailbox, Message
|
||||
from app.subscriptions import backfill_list_headers_job, read_list_headers, unsubscribe_targets
|
||||
from tests.api.conftest import HDR, MailCtx, raw_mail
|
||||
|
||||
NEWS = {
|
||||
"List-Id": '"Weekly news" <weekly.shop.example>',
|
||||
"List-Unsubscribe": "<https://shop.example/u/1>, <mailto:leave@shop.example?subject=stop>",
|
||||
"List-Unsubscribe-Post": "List-Unsubscribe=One-Click",
|
||||
}
|
||||
|
||||
|
||||
def test_read_list_headers() -> None:
|
||||
h = read_list_headers(raw_mail("x", headers=NEWS))
|
||||
assert h.list_id == "weekly.shop.example"
|
||||
assert h.unsubscribe.startswith("<https://shop.example/u/1>")
|
||||
assert h.one_click is True
|
||||
plain = read_list_headers(raw_mail("hi"))
|
||||
assert (plain.list_id, plain.unsubscribe, plain.one_click) == (None, "", False)
|
||||
|
||||
|
||||
def test_unsubscribe_targets() -> None:
|
||||
t = unsubscribe_targets(NEWS["List-Unsubscribe"])
|
||||
assert t.https == "https://shop.example/u/1"
|
||||
assert t.mailto == "leave@shop.example" and t.mailto_subject == "stop"
|
||||
assert unsubscribe_targets("<http://insecure.example/u>").https is None
|
||||
assert unsubscribe_targets("<mailto:a@b.c%0d%0aBcc:x@y.z>").mailto is None
|
||||
|
||||
|
||||
async def _list(ctx: MailCtx) -> list[dict[str, Any]]:
|
||||
resp = await ctx.client.get("/api/subscriptions")
|
||||
assert resp.status_code == 200, resp.text
|
||||
return list(resp.json()["subscriptions"])
|
||||
|
||||
|
||||
async def test_groups_by_list_then_sender(mail_ctx: MailCtx) -> None:
|
||||
await mail_ctx.deliver(raw_mail("Issue 1", headers=NEWS), 1, seen=True)
|
||||
await mail_ctx.deliver(raw_mail("Issue 2", headers=NEWS), 2)
|
||||
await mail_ctx.deliver(
|
||||
raw_mail(
|
||||
"Sale",
|
||||
sender="Deals <deals@other.example>",
|
||||
headers={"List-Unsubscribe": "<mailto:u@other.example>"},
|
||||
),
|
||||
3,
|
||||
)
|
||||
await mail_ctx.deliver(raw_mail("Hi from a friend", sender="ann@friend.example"), 4)
|
||||
|
||||
subs = {s["key"]: s for s in await _list(mail_ctx)}
|
||||
assert set(subs) == {"list:weekly.shop.example", "from:deals@other.example"}
|
||||
news = subs["list:weekly.shop.example"]
|
||||
assert (news["total"], news["unread"], news["name"]) == (2, 1, "News")
|
||||
assert news["one_click"] and news["has_mailto"] and news["has_web"]
|
||||
assert news["last_subject"] == "Issue 2"
|
||||
threads = await mail_ctx.client.get(
|
||||
"/api/subscriptions/threads", params={"key": "list:weekly.shop.example"}
|
||||
)
|
||||
assert len(threads.json()) == 2
|
||||
|
||||
|
||||
async def test_one_click_unsubscribe(mail_ctx: MailCtx, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
await mail_ctx.deliver(raw_mail("Issue", headers=NEWS), 1)
|
||||
posted: list[str] = []
|
||||
|
||||
async def fake_one_click(url: str) -> bool:
|
||||
posted.append(url)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(api_subs, "_one_click", fake_one_click)
|
||||
resp = await mail_ctx.client.post(
|
||||
"/api/subscriptions/unsubscribe", json={"key": "list:weekly.shop.example"}, headers=HDR
|
||||
)
|
||||
assert resp.json() == {"status": "done", "method": "one_click", "url": None}
|
||||
assert posted == ["https://shop.example/u/1"]
|
||||
(news,) = await _list(mail_ctx)
|
||||
assert news["unsubscribed_at"] is not None and news["still_sending"] is False
|
||||
|
||||
|
||||
async def test_mailto_unsubscribe_and_still_sending(
|
||||
mail_ctx: MailCtx, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
enqueued: list[tuple[Any, ...]] = []
|
||||
|
||||
async def fake_enqueue(name: str, *args: Any, **kwargs: Any) -> None:
|
||||
enqueued.append((name, *args))
|
||||
|
||||
monkeypatch.setattr(api_subs, "enqueue_job", fake_enqueue)
|
||||
headers = {"List-Unsubscribe": "<mailto:leave@shop.example?subject=stop>"}
|
||||
await mail_ctx.deliver(raw_mail("Issue", headers=headers), 1)
|
||||
resp = await mail_ctx.client.post(
|
||||
"/api/subscriptions/unsubscribe", json={"key": "from:news@shop.example"}, headers=HDR
|
||||
)
|
||||
assert resp.json()["method"] == "mailto"
|
||||
assert [e[0] for e in enqueued] == ["send_job"]
|
||||
draft = await mail_ctx.session.scalar(select(Draft).where(Draft.user_id == mail_ctx.user_id))
|
||||
assert draft is not None
|
||||
assert draft.payload["to"] == ["leave@shop.example"] and draft.payload["subject"] == "stop"
|
||||
|
||||
# Mail still arriving after the grace period is flagged.
|
||||
later = dt.datetime.now(dt.UTC) + dt.timedelta(days=3)
|
||||
await mail_ctx.deliver(raw_mail("Issue again", headers=headers), 2, at=later)
|
||||
(sub,) = await _list(mail_ctx)
|
||||
assert sub["still_sending"] is True
|
||||
|
||||
|
||||
async def test_web_only_unsubscribe_hands_back_the_page(mail_ctx: MailCtx) -> None:
|
||||
headers = {"List-Unsubscribe": "<https://shop.example/prefs>"}
|
||||
await mail_ctx.deliver(raw_mail("Issue", headers=headers), 1)
|
||||
resp = await mail_ctx.client.post(
|
||||
"/api/subscriptions/unsubscribe", json={"key": "from:news@shop.example"}, headers=HDR
|
||||
)
|
||||
assert resp.json() == {"status": "open", "method": "web", "url": "https://shop.example/prefs"}
|
||||
|
||||
|
||||
async def test_backfill_reads_older_mail(mail_ctx: MailCtx) -> None:
|
||||
msg = await mail_ctx.deliver(raw_mail("Old issue", headers=NEWS), 1)
|
||||
await mail_ctx.session.execute(
|
||||
update(Message).where(Message.id == msg.id).values(list_unsubscribe=None, list_id=None)
|
||||
)
|
||||
await mail_ctx.session.commit()
|
||||
resp = await mail_ctx.client.get("/api/subscriptions")
|
||||
assert resp.json()["pending"] == 1
|
||||
|
||||
assert await backfill_list_headers_job({}, str(mail_ctx.user_id)) == 1
|
||||
resp = await mail_ctx.client.get("/api/subscriptions")
|
||||
assert resp.json()["pending"] == 0
|
||||
assert [s["key"] for s in resp.json()["subscriptions"]] == ["list:weekly.shop.example"]
|
||||
|
||||
|
||||
async def test_trash_all_leaves_mixed_conversations(
|
||||
mail_ctx: MailCtx, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
moved: list[str] = []
|
||||
|
||||
async def fake_enqueue(name: str, *args: Any, **kwargs: Any) -> None:
|
||||
moved.append(args[0])
|
||||
|
||||
monkeypatch.setattr(api_subs, "enqueue_job", fake_enqueue)
|
||||
only = await mail_ctx.deliver(raw_mail("Issue", headers=NEWS, mid="<i1@shop>"), 1)
|
||||
await mail_ctx.deliver(raw_mail("Issue 2", headers=NEWS, mid="<i2@shop>"), 2)
|
||||
reply = raw_mail(
|
||||
"Re: Issue 2",
|
||||
sender="me@example.com",
|
||||
headers={"In-Reply-To": "<i2@shop>", "References": "<i2@shop>"},
|
||||
)
|
||||
await mail_ctx.deliver(reply, 3)
|
||||
resp = await mail_ctx.client.post(
|
||||
"/api/subscriptions/trash", json={"key": "list:weekly.shop.example"}, headers=HDR
|
||||
)
|
||||
assert resp.json() == {"moved": 1}
|
||||
assert moved == [str(only.thread_id)]
|
||||
|
||||
|
||||
async def test_deleted_mail_is_left_out(mail_ctx: MailCtx) -> None:
|
||||
msg = await mail_ctx.deliver(raw_mail("Issue", headers=NEWS), 1)
|
||||
trash = await mail_ctx.session.scalar(
|
||||
select(Mailbox.id).where(
|
||||
Mailbox.account_id == mail_ctx.account_id, Mailbox.special_use == "trash"
|
||||
)
|
||||
)
|
||||
await mail_ctx.session.execute(
|
||||
update(Message).where(Message.id == msg.id).values(mailbox_id=trash)
|
||||
)
|
||||
await mail_ctx.session.commit()
|
||||
assert await _list(mail_ctx) == []
|
||||
|
||||
|
||||
async def test_reading_pane_offers_unsubscribe(
|
||||
mail_ctx: MailCtx, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
async def fake_one_click(url: str) -> bool:
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(api_subs, "_one_click", fake_one_click)
|
||||
news = await mail_ctx.deliver(raw_mail("Issue", headers=NEWS), 1)
|
||||
plain = await mail_ctx.deliver(raw_mail("Hi", sender="ann@friend.example"), 2)
|
||||
c = mail_ctx.client
|
||||
|
||||
(env,) = (await c.get(f"/api/threads/{news.thread_id}")).json()["messages"]
|
||||
assert env["subscription_key"] == "list:weekly.shop.example"
|
||||
assert env["unsubscribe_via"] == "one_click" and env["unsubscribed"] is False
|
||||
(other,) = (await c.get(f"/api/threads/{plain.thread_id}")).json()["messages"]
|
||||
assert other["subscription_key"] is None and other["unsubscribe_via"] is None
|
||||
|
||||
resp = await c.post(
|
||||
"/api/subscriptions/unsubscribe",
|
||||
json={"key": env["subscription_key"], "message_id": env["id"]},
|
||||
headers=HDR,
|
||||
)
|
||||
assert resp.json()["status"] == "done"
|
||||
(env,) = (await c.get(f"/api/threads/{news.thread_id}")).json()["messages"]
|
||||
assert env["unsubscribed"] is True
|
||||
|
||||
# A message without the header can't be used.
|
||||
resp = await c.post(
|
||||
"/api/subscriptions/unsubscribe",
|
||||
json={"key": "from:ann@friend.example", "message_id": str(plain.id)},
|
||||
headers=HDR,
|
||||
)
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
async def test_senders_come_in_pages_newest_first(mail_ctx: MailCtx) -> None:
|
||||
now = dt.datetime.now(dt.UTC)
|
||||
for i, name in enumerate(["old", "mid", "new"]):
|
||||
await mail_ctx.deliver(
|
||||
raw_mail(
|
||||
f"From {name}",
|
||||
sender=f"{name}@lists.example",
|
||||
headers={"List-Unsubscribe": f"<mailto:u@{name}.example>"},
|
||||
),
|
||||
i + 1,
|
||||
at=now - dt.timedelta(days=10 - i),
|
||||
)
|
||||
c = mail_ctx.client
|
||||
first = (await c.get("/api/subscriptions", params={"limit": 2})).json()
|
||||
assert first["total"] == 3
|
||||
assert [s["email"] for s in first["subscriptions"]] == [
|
||||
"new@lists.example",
|
||||
"mid@lists.example",
|
||||
]
|
||||
rest = (await c.get("/api/subscriptions", params={"limit": 2, "offset": 2})).json()
|
||||
assert [s["email"] for s in rest["subscriptions"]] == ["old@lists.example"]
|
||||
assert (await c.get("/api/subscriptions", params={"offset": 9})).json()["subscriptions"] == []
|
||||
oldest = (await c.get("/api/subscriptions", params={"limit": 2, "order": "asc"})).json()
|
||||
assert [s["email"] for s in oldest["subscriptions"]] == [
|
||||
"old@lists.example",
|
||||
"mid@lists.example",
|
||||
]
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Templates: saved replies from the compose window's More menu."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from tests.api.conftest import HDR, UserCtx
|
||||
|
||||
|
||||
async def test_save_list_replace_delete(user_ctx: UserCtx) -> None:
|
||||
c = user_ctx.client
|
||||
assert (await c.get("/api/templates")).json() == []
|
||||
|
||||
resp = await c.post(
|
||||
"/api/templates",
|
||||
json={"name": " Thanks ", "subject": "Re: offer", "body_html": "<p>Thank <b>you</b></p>"},
|
||||
headers=HDR,
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
first = resp.json()
|
||||
assert first["name"] == "Thanks" and first["subject"] == "Re: offer"
|
||||
|
||||
# Same name, any case: replaces rather than adding a second one.
|
||||
resp = await c.post(
|
||||
"/api/templates", json={"name": "thanks", "body_html": "<p>Cheers</p>"}, headers=HDR
|
||||
)
|
||||
assert resp.json()["id"] == first["id"]
|
||||
await c.post("/api/templates", json={"name": "Apology", "body_html": "x"}, headers=HDR)
|
||||
listed = (await c.get("/api/templates")).json()
|
||||
assert [t["name"] for t in listed] == ["Apology", "thanks"]
|
||||
assert listed[1]["body_html"] == "<p>Cheers</p>"
|
||||
|
||||
assert (await c.delete(f"/api/templates/{first['id']}", headers=HDR)).status_code == 200
|
||||
assert [t["name"] for t in (await c.get("/api/templates")).json()] == ["Apology"]
|
||||
|
||||
|
||||
async def test_template_html_is_cleaned(user_ctx: UserCtx) -> None:
|
||||
resp = await user_ctx.client.post(
|
||||
"/api/templates",
|
||||
json={
|
||||
"name": "x",
|
||||
"body_html": '<p onclick="evil()">Hi<script>alert(1)</script>'
|
||||
'<img src="x" onerror="evil()"><a href="javascript:evil()">l</a></p>',
|
||||
},
|
||||
headers=HDR,
|
||||
)
|
||||
html = resp.json()["body_html"]
|
||||
assert "script" not in html and "onclick" not in html and "onerror" not in html
|
||||
assert "javascript:" not in html and "Hi" in html
|
||||
|
||||
|
||||
async def test_template_needs_a_name(user_ctx: UserCtx) -> None:
|
||||
resp = await user_ctx.client.post(
|
||||
"/api/templates", json={"name": " ", "body_html": "x"}, headers=HDR
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Tracker blocking (Settings → Privacy): on by default, and it can be switched off."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from tests.api.conftest import HDR, MailCtx, raw_mail
|
||||
|
||||
PIXEL = "https://t.example.com/open.gif?u=42"
|
||||
HTML = f'<p>Hello</p><img src="{PIXEL}" width="1" height="1">'
|
||||
|
||||
|
||||
async def _html(ctx: MailCtx, message_id: object, images: bool = False) -> tuple[str, str]:
|
||||
resp = await ctx.client.get(
|
||||
f"/api/messages/{message_id}/html", params={"images": "1"} if images else {}
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
return resp.text, resp.headers["X-Trackers-Blocked"]
|
||||
|
||||
|
||||
async def test_trackers_blocked_by_default(mail_ctx: MailCtx) -> None:
|
||||
msg = await mail_ctx.deliver(raw_mail("News", html=HTML), 1)
|
||||
assert (await mail_ctx.client.get("/api/settings")).json()["block_trackers"] is True
|
||||
body, count = await _html(mail_ctx, msg.id, images=True)
|
||||
assert count == "1" and PIXEL not in body and "Hello" in body
|
||||
(env,) = (await mail_ctx.client.get(f"/api/threads/{msg.thread_id}")).json()["messages"]
|
||||
assert env["has_remote_images"] is False # nothing left worth loading
|
||||
|
||||
|
||||
async def test_switched_off_trackers_are_ordinary_remote_images(mail_ctx: MailCtx) -> None:
|
||||
msg = await mail_ctx.deliver(raw_mail("News", html=HTML), 1)
|
||||
resp = await mail_ctx.client.patch("/api/settings", json={"block_trackers": False}, headers=HDR)
|
||||
assert resp.json()["block_trackers"] is False
|
||||
|
||||
# Still a remote image: not loaded until images are allowed, but offered.
|
||||
body, count = await _html(mail_ctx, msg.id)
|
||||
assert count == "0" and PIXEL not in body
|
||||
(env,) = (await mail_ctx.client.get(f"/api/threads/{msg.thread_id}")).json()["messages"]
|
||||
assert env["has_remote_images"] is True
|
||||
|
||||
body, count = await _html(mail_ctx, msg.id, images=True)
|
||||
assert count == "0" and PIXEL in body.replace("&", "&")
|
||||
|
||||
# Back on: gone again.
|
||||
await mail_ctx.client.patch("/api/settings", json={"block_trackers": True}, headers=HDR)
|
||||
body, count = await _html(mail_ctx, msg.id, images=True)
|
||||
assert count == "1" and PIXEL not in body
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Email translation over the API, with LibreTranslate replaced by a stand-in."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app import translation
|
||||
from tests.api.conftest import MailCtx, raw_mail
|
||||
|
||||
TEXT = "Hello Lukas, I would like to ask about the project schedule for next month."
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_service(monkeypatch: pytest.MonkeyPatch) -> dict[str, object]:
|
||||
calls: dict[str, object] = {"detect": 0, "translate": 0}
|
||||
|
||||
async def detect(text: str) -> str:
|
||||
calls["detect"] = int(calls["detect"]) + 1 # type: ignore[call-overload]
|
||||
return "en"
|
||||
|
||||
async def translate_html(
|
||||
html: str, source: str, target: str, keep: object = (), names: object = ()
|
||||
) -> str:
|
||||
calls["translate"] = int(calls["translate"]) + 1 # type: ignore[call-overload]
|
||||
calls["keep"] = list(keep) # type: ignore[call-overload]
|
||||
calls["names"] = list(names) # type: ignore[call-overload]
|
||||
assert source == "en" and target == "pl"
|
||||
return "<p>Cześć Łukasz</p><script>alert(1)</script>"
|
||||
|
||||
monkeypatch.setattr(translation, "enabled", lambda: True)
|
||||
monkeypatch.setattr(translation, "detect", detect)
|
||||
monkeypatch.setattr(translation, "translate_html", translate_html)
|
||||
return calls
|
||||
|
||||
|
||||
async def test_language_is_found_once_and_kept(
|
||||
mail_ctx: MailCtx, fake_service: dict[str, object]
|
||||
) -> None:
|
||||
msg = await mail_ctx.deliver(raw_mail("Project", html=f"<p>{TEXT}</p>"), 1)
|
||||
for _ in range(2):
|
||||
resp = await mail_ctx.client.get(f"/api/messages/{msg.id}/html")
|
||||
assert resp.headers["X-Content-Language"] == "en"
|
||||
assert fake_service["detect"] == 1
|
||||
(env,) = (await mail_ctx.client.get(f"/api/threads/{msg.thread_id}")).json()["messages"]
|
||||
assert env["content_language"] == "en"
|
||||
|
||||
|
||||
async def test_translation_is_cleaned_and_kept(
|
||||
mail_ctx: MailCtx, fake_service: dict[str, object]
|
||||
) -> None:
|
||||
msg = await mail_ctx.deliver(raw_mail("Project", html=f"<p>{TEXT}</p>"), 1)
|
||||
for _ in range(2):
|
||||
resp = await mail_ctx.client.get(f"/api/messages/{msg.id}/html", params={"translate": "pl"})
|
||||
assert resp.status_code == 200
|
||||
assert "Cześć Łukasz" in resp.text and "alert(1)" not in resp.text
|
||||
assert resp.headers["X-Translated-From"] == "en"
|
||||
assert resp.headers["X-Translated-To"] == "pl"
|
||||
assert fake_service["translate"] == 1 # the second came from the cache
|
||||
# The people on the message are names to keep if the model would change them.
|
||||
assert "News" in fake_service["names"] # type: ignore[operator]
|
||||
|
||||
|
||||
async def test_service_trouble_is_a_clear_error(
|
||||
mail_ctx: MailCtx, monkeypatch: pytest.MonkeyPatch, fake_service: dict[str, object]
|
||||
) -> None:
|
||||
async def broken(
|
||||
html: str, source: str, target: str, keep: object = (), names: object = ()
|
||||
) -> str:
|
||||
raise translation.TranslationError("the translation service is unreachable")
|
||||
|
||||
monkeypatch.setattr(translation, "translate_html", broken)
|
||||
msg = await mail_ctx.deliver(raw_mail("Project", html=f"<p>{TEXT}</p>"), 1)
|
||||
resp = await mail_ctx.client.get(f"/api/messages/{msg.id}/html", params={"translate": "pl"})
|
||||
assert resp.status_code == 503
|
||||
assert resp.json()["error"]["code"] == "translation_failed"
|
||||
# The message itself still shows.
|
||||
assert (await mail_ctx.client.get(f"/api/messages/{msg.id}/html")).status_code == 200
|
||||
|
||||
|
||||
async def test_off_without_a_service(mail_ctx: MailCtx, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(translation, "enabled", lambda: False)
|
||||
msg = await mail_ctx.deliver(raw_mail("Project", html=f"<p>{TEXT}</p>"), 1)
|
||||
resp = await mail_ctx.client.get(f"/api/messages/{msg.id}/html")
|
||||
assert "X-Content-Language" not in resp.headers
|
||||
info = (await mail_ctx.client.get("/api/translation/languages")).json()
|
||||
assert info == {"available": False, "languages": []}
|
||||
@@ -4,6 +4,8 @@ from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
|
||||
from app.indexer.sanitize import sanitize_html
|
||||
|
||||
MID = uuid.uuid4()
|
||||
@@ -99,3 +101,72 @@ def test_script_content_still_stripped_with_style_allowed() -> None:
|
||||
result = sanitize_html(html, MID)
|
||||
assert "alert(1)" not in result.html
|
||||
assert ".a{color:red}" in result.html
|
||||
|
||||
|
||||
# --- Tracking pixels ---------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tag",
|
||||
[
|
||||
'<img src="https://t.example.com/o.gif?u=1" width="1" height="1">',
|
||||
'<img src="https://t.example.com/o.gif" width="1px" height="1px" border="0">',
|
||||
"<img src='https://t.example.com/x' style='width:1px;height:1px'>",
|
||||
'<img src="https://t.example.com/x" style="display:none">',
|
||||
'<img src="https://x.list-manage.com/track/open.php?u=1&id=2">',
|
||||
'<img src="https://u1.ct.sendgrid.net/wf/open?upn=abc" alt="">',
|
||||
'<img src="https://mailtrack.io/trace/mail/abc.png">',
|
||||
# Squeezed to a pixel by max-width, its height following.
|
||||
'<img src="https://x.example/p.heic" width="1" height="1" '
|
||||
'style="display:block; height:auto; max-width:1px !important; width:100%">',
|
||||
'<img src="https://x.example/p" width="0" height="0" style="border:0;width:0;height:0;">',
|
||||
],
|
||||
)
|
||||
def test_tracking_pixels_are_removed_and_counted(tag: str) -> None:
|
||||
html = f"<p>Hello</p>{tag}<p>Bye</p>"
|
||||
for allow in (False, True):
|
||||
result = sanitize_html(html, MID, allow_remote_images=allow)
|
||||
assert result.trackers_blocked == 1
|
||||
assert "<img" not in result.html
|
||||
assert "Hello" in result.html and "Bye" in result.html
|
||||
# A pixel alone is no reason to offer "Load images".
|
||||
assert sanitize_html(html, MID).has_remote_images is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tag",
|
||||
[
|
||||
'<img src="https://cdn.example.com/logo.png" width="180" height="47">',
|
||||
'<img src="https://cdn.example.com/photo.jpg">',
|
||||
'<img src="https://cdn.example.com/spacer.gif" width="1" height="40">',
|
||||
'<img src="cid:logo" width="1" height="1">',
|
||||
# Outlook signature logos: sizes in inches, not pixels.
|
||||
'<img width="30" height="30" style="width:0.3125in; height:0.3125in" '
|
||||
'src="https://pfx.tv/img/sig/logo.png">',
|
||||
'<img style="width:1.5em;height:1.5em" src="https://cdn.example.com/icon.png">',
|
||||
'<img style="width:1%;height:1%" src="https://cdn.example.com/bar.png">',
|
||||
'<img style="width:1pt;height:1pt" src="https://cdn.example.com/dot.png">',
|
||||
# Attributes say 1x1, the style draws it full width.
|
||||
'<img src="https://cdn.example.com/photo.jpg" width="1" height="1" '
|
||||
'style="width:100%;height:auto">',
|
||||
],
|
||||
)
|
||||
def test_real_images_are_kept(tag: str) -> None:
|
||||
result = sanitize_html(f"<p>x</p>{tag}", MID, allow_remote_images=True)
|
||||
assert result.trackers_blocked == 0
|
||||
assert "<img" in result.html
|
||||
|
||||
|
||||
@pytest.mark.parametrize("size", [".", "1.2.3", "..", "", "px", "1e3", "-1"])
|
||||
def test_malformed_sizes_never_break_sanitizing(size: str) -> None:
|
||||
tag = f'<img src="https://cdn.example.com/a.png" width="{size}" height="{size}">'
|
||||
result = sanitize_html(f"<p>x</p>{tag}", MID, allow_remote_images=True)
|
||||
assert result.trackers_blocked == 0
|
||||
css = f'<img src="https://cdn.example.com/a.png" style="width:{size}px;height:{size}px">'
|
||||
assert sanitize_html(css, MID, allow_remote_images=True).trackers_blocked == 0
|
||||
|
||||
|
||||
def test_tracker_blocking_can_be_switched_off() -> None:
|
||||
tag = '<img src="https://t.example.com/o.gif" width="1" height="1">'
|
||||
kept = sanitize_html(f"<p>x</p>{tag}", MID, allow_remote_images=True, block_trackers=False)
|
||||
assert kept.trackers_blocked == 0 and "t.example.com/o.gif" in kept.html
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Sentence completion: phrases from the user's own sent mail, plus starters."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
|
||||
from app.completion import build_phrases, choose, complete, html_to_text, own_text
|
||||
|
||||
|
||||
def personal(*texts: str) -> dict[str, Counter[str]]:
|
||||
out: dict[str, Counter[str]] = {}
|
||||
for (ctx, cont), n in build_phrases(texts).items():
|
||||
out.setdefault(ctx, Counter())[cont] = n
|
||||
return out
|
||||
|
||||
|
||||
def test_own_text_drops_quotes_and_signature() -> None:
|
||||
for quote in (
|
||||
"On Mon, 5 Oct 2026 Anna wrote:",
|
||||
"W dniu 5.10.2026 Anna pisze:",
|
||||
"Am 05.10.2026 um 10:00 schrieb Anna:",
|
||||
"-----Original Message-----",
|
||||
"From: Anna <anna@x.example>",
|
||||
"--",
|
||||
):
|
||||
text = f"Sure, I'll send it today.\n\n{quote}\nOld text here."
|
||||
assert own_text(text).strip() == "Sure, I'll send it today.", quote
|
||||
assert own_text("Yes.\n> quoted line\nAgreed.") == "Yes.\nAgreed."
|
||||
|
||||
|
||||
def test_only_repeated_phrases_are_kept() -> None:
|
||||
phrases = build_phrases(
|
||||
["I'll send the drawings by Friday.", "I'll send the drawings by Friday."]
|
||||
)
|
||||
assert phrases[("the", "drawings by Friday.")] == 2
|
||||
once = build_phrases(["A private sentence written once."])
|
||||
assert once == {}
|
||||
|
||||
|
||||
def test_personal_phrases_complete_in_any_language() -> None:
|
||||
mine = personal(
|
||||
"I'll send the drawings by Friday.",
|
||||
"I'll send the drawings by Friday.",
|
||||
"Prześlę rysunki do piątku.",
|
||||
"Prześlę rysunki do piątku.",
|
||||
)
|
||||
assert complete(mine, "I'll send the dr") == "awings by Friday."
|
||||
assert complete(mine, "Prześlę rys") == "unki do piątku."
|
||||
assert complete(mine, "Prześlę rysunki ") == "do piątku."
|
||||
|
||||
|
||||
def test_starter_phrases_and_spacing() -> None:
|
||||
assert complete({}, "I look forward ") == "to hearing from you."
|
||||
assert complete({}, "I look forward") == " to hearing from you." # space added
|
||||
assert complete({}, "Z pow") == "ażaniem,"
|
||||
assert complete({}, "Mit freundlichen ") == "Grüßen"
|
||||
|
||||
|
||||
def test_nothing_after_punctuation_or_on_an_empty_line() -> None:
|
||||
assert complete({}, "Thanks,") is None
|
||||
assert complete({}, "Done. ") is None
|
||||
assert complete({}, "") is None
|
||||
assert complete({}, " ") is None
|
||||
|
||||
|
||||
def test_suggestion_stops_where_habits_part() -> None:
|
||||
# "see you" is followed by "soon" or "tomorrow" equally: suggest only "you".
|
||||
assert choose(Counter({"you": 4, "you soon": 2, "you tomorrow": 2})) == "you"
|
||||
assert choose(Counter({"you": 4, "you soon": 4, "you soon then": 3})) == "you soon then"
|
||||
|
||||
|
||||
def test_polish_gmail_quotes_and_own_signature_are_left_out() -> None:
|
||||
text = (
|
||||
"Dziękuję, prześlę jutro.\n"
|
||||
"Imaginary Pixels – visual effects studio\n"
|
||||
"pon., 5 paź 2026 o 10:00 Anna Nowak <anna@x.example> napisał(a):\n"
|
||||
"> stary tekst"
|
||||
)
|
||||
sig = html_to_text(
|
||||
"<div>Imaginary Pixels – visual effects studio</div><div>+48 602 691 231</div>"
|
||||
)
|
||||
assert own_text(text, [sig]).strip() == "Dziękuję, prześlę jutro."
|
||||
|
||||
|
||||
def test_numbers_addresses_and_links_are_never_learned() -> None:
|
||||
texts = ["Call me at +48 602 691 231 or write to me@x.example today."] * 3
|
||||
learned = " ".join(f"{c} {n}" for (c, n) in build_phrases(texts))
|
||||
assert "602" not in learned and "@" not in learned
|
||||
assert ("call me", "at") in build_phrases(texts)
|
||||
@@ -0,0 +1,128 @@
|
||||
"""Getting a message ready for translation (app/translation_prep.py)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.translation_prep import (
|
||||
find_names,
|
||||
find_names_in_signature,
|
||||
join_wrapped_lines,
|
||||
keep_symbols,
|
||||
keep_terms,
|
||||
split_pieces,
|
||||
tidy,
|
||||
translate_phrases,
|
||||
)
|
||||
|
||||
KEPT = '<span translate="no">'
|
||||
|
||||
|
||||
def test_wrapped_lines_are_joined_but_paragraphs_kept() -> None:
|
||||
assert join_wrapped_lines("<p>ask you about<br>the schedule.</p>") == (
|
||||
"<p>ask you about the schedule.</p>"
|
||||
)
|
||||
assert join_wrapped_lines("<div>ask you about</div><div>the schedule.</div>") == (
|
||||
"<div>ask you about the schedule.</div>"
|
||||
)
|
||||
text = '<div class="wm-plaintext">ask you about\nthe schedule.\n\nThanks,\nAnna</div>'
|
||||
assert join_wrapped_lines(text) == (
|
||||
'<div class="wm-plaintext">ask you about the schedule.\n\nThanks,\nAnna</div>'
|
||||
)
|
||||
# A new sentence, or a capital after the break: left alone.
|
||||
for html in ("<p>Done.<br>then more</p>", "<p>Hi Lukas,<br>Natalia</p>"):
|
||||
assert join_wrapped_lines(html) == html
|
||||
|
||||
|
||||
def test_sign_offs_come_from_the_phrase_table() -> None:
|
||||
assert translate_phrases("<p>Best,<br>Natalia</p>", "en", "pl") == (
|
||||
f"<p>{KEPT}Pozdrawiam,</span><br>Natalia</p>"
|
||||
)
|
||||
assert translate_phrases("<div>Mit freundlichen Grüßen</div>", "de", "en") == (
|
||||
f"<div>{KEPT}Kind regards</span></div>"
|
||||
)
|
||||
assert "Pozdrawiam" not in translate_phrases("<p>Best results so far</p>", "en", "pl")
|
||||
assert translate_phrases("<p>Best,</p>", "fr", "pl") == "<p>Best,</p>" # no table
|
||||
|
||||
|
||||
def test_names_mid_sentence_but_not_headlines_or_german() -> None:
|
||||
html = "<p>I came across Imaginary Pixels and Trusted Partner Network, in Spain.</p>"
|
||||
assert find_names(html, "en") == {"Imaginary Pixels", "Trusted Partner Network"}
|
||||
assert find_names("<p>Why Matchmoving Matters, Editing Digger And More</p>", "en") == set()
|
||||
assert find_names("<p>Wir haben das Neue Angebot gesehen.</p>", "de") == set()
|
||||
# Not across a sentence's end.
|
||||
assert find_names("<p>working mostly in NukeX. My work includes paint.</p>", "en") == set()
|
||||
assert find_names_in_signature("Imaginary Pixels – visual effects studio") == [
|
||||
"Imaginary Pixels"
|
||||
]
|
||||
|
||||
|
||||
def test_kept_terms_and_spacing() -> None:
|
||||
out = keep_terms(
|
||||
"<p>Hi Natalia Raz, see Imaginary Pixels.</p>", ["Natalia Raz", "Imaginary Pixels"]
|
||||
)
|
||||
assert out == f"<p>Hi {KEPT}Natalia Raz</span>, see {KEPT}Imaginary Pixels</span>.</p>"
|
||||
assert keep_terms("<style>.Natalia{}</style>", ["Natalia"]) == "<style>.Natalia{}</style>"
|
||||
assert tidy(f"nad {KEPT}Bruton</span>i dalej") == f"nad {KEPT}Bruton</span> i dalej"
|
||||
assert tidy("<span>a</span>b") == "<span>a</span>b" # the sender's own spans
|
||||
|
||||
|
||||
def test_emoji_kept() -> None:
|
||||
html = (
|
||||
'<p>nice to e-meet you <span style="font-family:Segoe UI Emoji">\n😊</span> </p>'
|
||||
"<p>Great 👍🏽 work ❤️</p>"
|
||||
)
|
||||
out = keep_symbols(html)
|
||||
assert f"{KEPT}😊</span>" in out
|
||||
assert f"{KEPT}👍🏽</span>" in out
|
||||
assert f"{KEPT}❤️</span>" in out
|
||||
assert "nice to e-meet you" in out # the words still go to translation
|
||||
|
||||
|
||||
def test_file_names_kept() -> None:
|
||||
html = (
|
||||
"<li>A_0023C017_260521_130157_p1CAC.mov</li>"
|
||||
"<p>See final_v2 and plate-03.exr, e.g. the U.S cut.</p>"
|
||||
)
|
||||
out = keep_symbols(html)
|
||||
assert f"{KEPT}A_0023C017_260521_130157_p1CAC.mov</span>" in out
|
||||
assert f"{KEPT}final_v2</span>" in out
|
||||
assert f"{KEPT}plate-03.exr</span>" in out
|
||||
assert "e.g. the U.S cut" in out # no file names there
|
||||
|
||||
|
||||
def test_symbols_not_marked_twice_or_in_attributes() -> None:
|
||||
html = (
|
||||
f'<a href="https://x.io/a_b.mov">{KEPT}https://x.io/a_b.mov</span></a>'
|
||||
"<style>.a_b{}</style>"
|
||||
)
|
||||
assert keep_symbols(html) == html
|
||||
|
||||
|
||||
def test_text_without_letters_is_kept() -> None:
|
||||
# A byte-order mark alone between tags came back as "Uzasadnienie".
|
||||
out = keep_symbols(
|
||||
'<div dir="ltr">\ufeff<div>Dear Lukas,</div><div> </div><p> – </p></div>'
|
||||
)
|
||||
assert f"{KEPT}\ufeff</span>" in out
|
||||
assert "<div> </div>" in out # only spaces: nothing to keep
|
||||
assert f"<p> {KEPT}–</span> </p>" in out
|
||||
assert "<div>Dear Lukas,</div>" in out
|
||||
|
||||
|
||||
def test_handles_and_phone_numbers_kept() -> None:
|
||||
out = keep_symbols(
|
||||
"<p>WeChat: Jacquelineeeee_ WhatsApp: +48 780 549 877, for 1300 - 2500 Euro</p>"
|
||||
)
|
||||
assert f"{KEPT}Jacquelineeeee_</span>" in out
|
||||
assert f"{KEPT}+48 780 549 877</span>" in out
|
||||
assert "1300 - 2500 Euro" in out # plain numbers stay in their sentence
|
||||
|
||||
|
||||
def test_paragraphs_split_at_block_tags() -> None:
|
||||
html = (
|
||||
f"<style>p{{}}</style><div>Dear <b>Anna</b>,<br>hello {KEPT}x.mov</span> there</div>"
|
||||
"<p> </p><li>Item</li>"
|
||||
)
|
||||
pieces, paragraphs = split_pieces(html)
|
||||
assert [p.text for p in paragraphs] == ["Dear Anna,", "hello x.mov there", "Item"]
|
||||
# The kept file name counts for the language but is not translated.
|
||||
assert [pieces[i] for i in paragraphs[1].pieces] == ["hello ", " there"]
|
||||
@@ -0,0 +1,123 @@
|
||||
"""What is sent to the translation service (app/translation.py)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app import translation
|
||||
from app.translation import detect, languages_of, names_to_keep, protect_addresses, translate_html
|
||||
|
||||
|
||||
def test_addresses_in_text_are_kept_from_translation() -> None:
|
||||
html = (
|
||||
'<p>My reel: <a href="https://vimeo.com/1">https://vimeo.com/1</a>, '
|
||||
"write to anna@x.example or see www.example.com today.</p>"
|
||||
)
|
||||
out = protect_addresses(html)
|
||||
assert '<span translate="no">https://vimeo.com/1</span>' in out
|
||||
assert '<span translate="no">anna@x.example</span>' in out
|
||||
assert '<span translate="no">www.example.com</span>' in out
|
||||
assert '<a href="https://vimeo.com/1">' in out # attributes untouched
|
||||
|
||||
|
||||
def test_style_blocks_are_left_alone() -> None:
|
||||
html = "<style>.a{background:url(https://x.example/a.png)}</style><p>Hi</p>"
|
||||
assert protect_addresses(html) == html
|
||||
|
||||
|
||||
async def test_only_names_the_model_changes_are_kept(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
sent: list[object] = []
|
||||
|
||||
async def post(path: str, payload: dict[str, object]) -> object:
|
||||
sent.append(payload["q"])
|
||||
alone = {
|
||||
"Lukas Remis": "Lukas Remis.",
|
||||
"Milan": "Mediolan",
|
||||
"Imaginary Pixels": "Imaginanal",
|
||||
}
|
||||
return {"translatedText": [alone[q] for q in payload["q"]]} # type: ignore[attr-defined]
|
||||
|
||||
monkeypatch.setattr(translation, "_post", post)
|
||||
kept = await names_to_keep(
|
||||
["Milan", "Lukas Remis", "Imaginary Pixels", "Milan", "x"], "en", "pl"
|
||||
)
|
||||
assert kept == {"Milan", "Imaginary Pixels"} # "Lukas Remis" stays in its sentence
|
||||
assert len(sent) == 1 # one request for all
|
||||
|
||||
|
||||
async def test_all_names_kept_when_the_check_fails(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
async def post(path: str, payload: dict[str, object]) -> object:
|
||||
raise translation.TranslationError("down")
|
||||
|
||||
monkeypatch.setattr(translation, "_post", post)
|
||||
assert await names_to_keep(["Milan", "Lukas Remis"], "en", "pl") == {"Milan", "Lukas Remis"}
|
||||
|
||||
|
||||
def _fake_detector(monkeypatch: pytest.MonkeyPatch) -> list[str]:
|
||||
"""A detector that knows Chinese signs, Polish letters, else English."""
|
||||
asked: list[str] = []
|
||||
|
||||
async def detect_one(text: str) -> str:
|
||||
asked.append(text)
|
||||
if any("\u4e00" <= c <= "\u9fff" for c in text):
|
||||
return "zh-Hans"
|
||||
return "pl" if any(c in "ąćęłńóśźż" for c in text.lower()) else "en"
|
||||
|
||||
monkeypatch.setattr(translation, "_detect_one", detect_one)
|
||||
return asked
|
||||
|
||||
|
||||
async def test_short_parts_take_the_next_language(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
asked = _fake_detector(monkeypatch)
|
||||
parts = [
|
||||
"您好！",
|
||||
"Szanowny Kliencie,", # short: goes with the Polish that follows
|
||||
"Widzę, że złożyłeś wniosek o otwarcie sklepu na platformie.",
|
||||
"-----",
|
||||
"Hello, my name is Jacqueline, the manager for Europe.",
|
||||
"Jacqueline Lian", # short, last: goes with the one before
|
||||
]
|
||||
assert await languages_of(parts) == ["zh-Hans", "pl", "pl", "", "en", "en"]
|
||||
assert "Jacqueline Lian" not in asked # too short to ask about
|
||||
|
||||
|
||||
async def test_a_message_in_three_languages(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_fake_detector(monkeypatch)
|
||||
text = (
|
||||
"尊敬的客户，我是速卖通的官方客户经理，我从后台看到您申请入驻速卖通平台开店。\n"
|
||||
"Szanowny Kliencie,\n"
|
||||
"Widzę, że złożyłeś wniosek o otwarcie sklepu na platformie AliExpress.\n"
|
||||
"Dears,\nI noticed from the backend that you have applied to join the platform.\n"
|
||||
)
|
||||
assert set((await detect(text)).split(",")) == {"zh-Hans", "pl", "en"}
|
||||
# One language: one code.
|
||||
assert await detect("I noticed from the backend that you applied, thanks a lot.") == "en"
|
||||
|
||||
|
||||
async def test_each_paragraph_from_its_own_language(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_fake_detector(monkeypatch)
|
||||
calls: list[tuple[str, list[str]]] = []
|
||||
|
||||
async def post(path: str, payload: dict[str, object]) -> object:
|
||||
q = payload["q"]
|
||||
if path == "/translate" and isinstance(q, list) and payload.get("source") != "auto":
|
||||
calls.append((str(payload["source"]), list(q)))
|
||||
return {"translatedText": [f"PL({x})" for x in q]}
|
||||
return {"translatedText": list(q) if isinstance(q, list) else q} # names: unchanged
|
||||
|
||||
monkeypatch.setattr(translation, "_post", post)
|
||||
html = (
|
||||
"<div><span>我是速卖通的官方客户经理奎琳，</span>我从后台看到您申请入驻速卖通平台开店。</div>"
|
||||
"<div>Widzę, że złożyłeś wniosek o otwarcie sklepu & więcej.</div>"
|
||||
"<div>I noticed that you applied to join the platform & more.</div>"
|
||||
"<div>WhatsApp: +48 780 549 877</div>"
|
||||
)
|
||||
out = await translate_html(html, "zh-Hans,pl,en", "pl")
|
||||
assert [c[0] for c in calls] == ["zh-Hans", "en"] # the Polish is left alone
|
||||
assert "<div>Widzę, że złożyłeś wniosek o otwarcie sklepu & więcej.</div>" in out
|
||||
# Two pieces of one Chinese sentence get a space between them.
|
||||
assert "PL(我是速卖通的官方客户经理奎琳，) </span>PL(" in out
|
||||
assert "PL(I noticed that you applied to join the platform & more.)" in out.replace(
|
||||
"&", "&"
|
||||
)
|
||||
assert '<span translate="no">+48 780 549 877</span>' in out
|
||||
Reference in New Issue
Block a user