373 lines
13 KiB
Python
373 lines
13 KiB
Python
 
|
"""Subscriptions page: bulk senders mailing the user (app/subscriptions.py)."""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import datetime as dt
|
||
|
|
import logging
|
||
|
|
import uuid
|
||
|
|
from typing import Annotated, Any
|
||
|
|
|
||
|
|
from fastapi import APIRouter, Depends, Query
|
||
|
|
from pydantic import BaseModel
|
||
|
|
from sqlalchemy import case, func, literal, or_, select
|
||
|
|
from sqlalchemy.dialects.postgresql import insert
|
||
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
|
from sqlalchemy.sql.elements import ColumnElement
|
||
|
|
|
||
|
|
from app import subscriptions as subs
|
||
|
|
from app.accounts.ssrf import guarded_request
|
||
|
|
from app.api.errors import raise_api_error
|
||
|
|
from app.config import get_settings
|
||
|
|
from app.db import get_session
|
||
|
|
from app.identity.deps import CurrentUser, require_csrf
|
||
|
|
from app.models import Mailbox, Message, SubscriptionState
|
||
|
|
from app.outbox.schemas import DraftIn, DraftPayload
|
||
|
|
from app.outbox.service import DraftError, create_outbox, upsert_draft
|
||
|
|
from app.queue import enqueue_job
|
||
|
|
|
||
|
|
logger = logging.getLogger(__name__)
|
||
|
|
|
||
|
|
router = APIRouter(prefix="/subscriptions", tags=["mail"], dependencies=[Depends(require_csrf)])
|
||
|
|
|
||
|
|
DbSession = Annotated[AsyncSession, Depends(get_session)]
|
||
|
|
|
||
|
|
# RFC 8058 senders should stop within two days of a one-click request.
|
||
|
|
GRACE = dt.timedelta(days=2)
|
||
|
|
# Not counted: mail the user sent, and mail already thrown away (Trash, Spam) —
|
||
|
|
# a sender whose mail was all deleted drops off the list.
|
||
|
|
_LEFT_OUT = ("sent", "drafts", "trash", "junk")
|
||
|
|
|
||
|
|
|
||
|
|
class SubscriptionOut(BaseModel):
|
||
|
|
key: str
|
||
|
|
name: str
|
||
|
|
email: str
|
||
|
|
list_id: str | None
|
||
|
|
total: int
|
||
|
|
unread: int
|
||
|
|
last_30_days: int
|
||
|
|
per_month: float
|
||
|
|
first_at: dt.datetime
|
||
|
|
last_at: dt.datetime
|
||
|
|
last_subject: str
|
||
|
|
one_click: bool
|
||
|
|
has_mailto: bool
|
||
|
|
has_web: bool
|
||
|
|
unsubscribed_at: dt.datetime | None
|
||
|
|
method: str | None
|
||
|
|
still_sending: bool
|
||
|
|
|
||
|
|
|
||
|
|
class SubscriptionListOut(BaseModel):
|
||
|
|
subscriptions: list[SubscriptionOut] # one page
|
||
|
|
total: int # senders in all
|
||
|
|
pending: int # messages whose headers are still being read
|
||
|
|
|
||
|
|
|
||
|
|
class KeyIn(BaseModel):
|
||
|
|
key: str
|
||
|
|
|
||
|
|
|
||
|
|
class UnsubscribeIn(KeyIn):
|
||
|
|
# From the reading pane: this message's own header, wherever it is filed
|
||
|
|
# (Spam included, which the page's groups leave out).
|
||
|
|
message_id: uuid.UUID | None = None
|
||
|
|
|
||
|
|
|
||
|
|
class UnsubscribeOut(BaseModel):
|
||
|
|
status: str # done | open
|
||
|
|
method: str # one_click | mailto | web
|
||
|
|
url: str | None = None
|
||
|
|
|
||
|
|
|
||
|
|
def _sender() -> ColumnElement[Any]:
|
||
|
|
return func.lower(func.coalesce(Message.from_addr[0]["email"].astext, ""))
|
||
|
|
|
||
|
|
|
||
|
|
def _key() -> ColumnElement[Any]:
|
||
|
|
return case(
|
||
|
|
(Message.list_id.isnot(None), literal("list:") + Message.list_id),
|
||
|
|
else_=literal("from:") + _sender(),
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def _bulk(user_id: uuid.UUID) -> list[ColumnElement[bool]]:
|
||
|
|
return [
|
||
|
|
Message.user_id == user_id,
|
||
|
|
Message.list_unsubscribe.isnot(None),
|
||
|
|
Message.list_unsubscribe != "",
|
||
|
|
or_(Mailbox.special_use.is_(None), Mailbox.special_use.notin_(_LEFT_OUT)),
|
||
|
|
]
|
||
|
|
|
||
|
|
|
||
|
|
async def _start_backfill(user_id: uuid.UUID) -> None:
|
||
|
|
# One job per user at a time: arq refuses a second while the first exists.
|
||
|
|
await enqueue_job("backfill_list_headers_job", str(user_id), _job_id=f"listhdr:{user_id}")
|
||
|
|
|
||
|
|
|
||
|
|
@router.get("", response_model=SubscriptionListOut)
|
||
|
|
async def list_subscriptions(
|
||
|
|
user: CurrentUser,
|
||
|
|
db: DbSession,
|
||
|
|
offset: int = Query(0, ge=0),
|
||
|
|
limit: int = Query(50, ge=1, le=200),
|
||
|
|
order: str = Query("desc"), # desc: heard from most recently first | asc: longest ago
|
||
|
|
) -> SubscriptionListOut:
|
||
|
|
"""One page of senders, by when their latest mail came."""
|
||
|
|
pending = await subs.pending_count(db, user.id)
|
||
|
|
if pending:
|
||
|
|
await _start_backfill(user.id)
|
||
|
|
|
||
|
|
now = dt.datetime.now(dt.UTC)
|
||
|
|
key = _key().label("key")
|
||
|
|
seen = Message.flags.contains(["\\Seen"])
|
||
|
|
groups = (
|
||
|
|
select(
|
||
|
|
key,
|
||
|
|
func.count().label("total"),
|
||
|
|
func.count().filter(~seen).label("unread"),
|
||
|
|
func.count()
|
||
|
|
.filter(Message.internal_date > now - dt.timedelta(days=30))
|
||
|
|
.label("recent"),
|
||
|
|
func.min(Message.internal_date).label("first_at"),
|
||
|
|
func.max(Message.internal_date).label("last_at"),
|
||
|
|
)
|
||
|
|
.join(Mailbox, Mailbox.id == Message.mailbox_id)
|
||
|
|
.where(*_bulk(user.id))
|
||
|
|
.group_by(key)
|
||
|
|
)
|
||
|
|
total = await db.scalar(select(func.count()).select_from(groups.subquery())) or 0
|
||
|
|
agg_rows = (
|
||
|
|
await db.execute(
|
||
|
|
groups.order_by(
|
||
|
|
func.max(Message.internal_date).asc()
|
||
|
|
if order == "asc"
|
||
|
|
else func.max(Message.internal_date).desc(),
|
||
|
|
key,
|
||
|
|
)
|
||
|
|
.offset(offset)
|
||
|
|
.limit(limit)
|
||
|
|
)
|
||
|
|
).all()
|
||
|
|
page_keys = [r.key for r in agg_rows]
|
||
|
|
if not page_keys:
|
||
|
|
return SubscriptionListOut(subscriptions=[], total=total, pending=pending)
|
||
|
|
|
||
|
|
in_page = _key().in_(page_keys)
|
||
|
|
latest_rows = (
|
||
|
|
await db.execute(
|
||
|
|
select(
|
||
|
|
key,
|
||
|
|
Message.from_addr,
|
||
|
|
Message.subject,
|
||
|
|
Message.list_id,
|
||
|
|
Message.list_unsubscribe,
|
||
|
|
Message.list_unsubscribe_post,
|
||
|
|
)
|
||
|
|
.join(Mailbox, Mailbox.id == Message.mailbox_id)
|
||
|
|
.where(*_bulk(user.id), in_page)
|
||
|
|
.distinct(key)
|
||
|
|
.order_by(key, Message.internal_date.desc())
|
||
|
|
)
|
||
|
|
).all()
|
||
|
|
latest = {r.key: r for r in latest_rows}
|
||
|
|
states = {
|
||
|
|
s.key: s
|
||
|
|
for s in await db.scalars(
|
||
|
|
select(SubscriptionState).where(
|
||
|
|
SubscriptionState.user_id == user.id, SubscriptionState.key.in_(page_keys)
|
||
|
|
)
|
||
|
|
)
|
||
|
|
}
|
||
|
|
# Senders still mailing after the grace period, for the whole page at once.
|
||
|
|
still_sending = set(
|
||
|
|
await db.scalars(
|
||
|
|
select(key)
|
||
|
|
.select_from(Message)
|
||
|
|
.join(Mailbox, Mailbox.id == Message.mailbox_id)
|
||
|
|
.join(
|
||
|
|
SubscriptionState,
|
||
|
|
(SubscriptionState.user_id == user.id) & (SubscriptionState.key == _key()),
|
||
|
|
)
|
||
|
|
.where(
|
||
|
|
*_bulk(user.id),
|
||
|
|
in_page,
|
||
|
|
Message.internal_date > SubscriptionState.unsubscribed_at + GRACE,
|
||
|
|
)
|
||
|
|
.distinct()
|
||
|
|
)
|
||
|
|
)
|
||
|
|
|
||
|
|
out: list[SubscriptionOut] = []
|
||
|
|
for row in agg_rows:
|
||
|
|
last = latest.get(row.key)
|
||
|
|
if last is None:
|
||
|
|
continue
|
||
|
|
sender = last.from_addr[0] if last.from_addr else {}
|
||
|
|
targets = subs.unsubscribe_targets(last.list_unsubscribe or "")
|
||
|
|
state = states.get(row.key)
|
||
|
|
months = max(1.0, (row.last_at - row.first_at).days / 30.44)
|
||
|
|
out.append(
|
||
|
|
SubscriptionOut(
|
||
|
|
key=row.key,
|
||
|
|
name=sender.get("name") or sender.get("email", ""),
|
||
|
|
email=sender.get("email", ""),
|
||
|
|
list_id=last.list_id,
|
||
|
|
total=row.total,
|
||
|
|
unread=row.unread,
|
||
|
|
last_30_days=row.recent,
|
||
|
|
per_month=round(row.total / months, 1),
|
||
|
|
first_at=row.first_at,
|
||
|
|
last_at=row.last_at,
|
||
|
|
last_subject=last.subject,
|
||
|
|
one_click=bool(last.list_unsubscribe_post and targets.https),
|
||
|
|
has_mailto=targets.mailto is not None,
|
||
|
|
has_web=targets.https is not None,
|
||
|
|
unsubscribed_at=state.unsubscribed_at if state else None,
|
||
|
|
method=state.method if state else None,
|
||
|
|
still_sending=row.key in still_sending,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
return SubscriptionListOut(subscriptions=out, total=total, pending=pending)
|
||
|
|
|
||
|
|
|
||
|
|
@router.get("/threads", response_model=list[uuid.UUID])
|
||
|
|
async def subscription_threads(
|
||
|
|
user: CurrentUser, db: DbSession, key: str = Query(...), limit: int = Query(50, le=200)
|
||
|
|
) -> list[uuid.UUID]:
|
||
|
|
"""The group's conversations, newest first (for the reading pane)."""
|
||
|
|
rows = await db.execute(
|
||
|
|
select(Message.thread_id, func.max(Message.internal_date).label("at"))
|
||
|
|
.join(Mailbox, Mailbox.id == Message.mailbox_id)
|
||
|
|
.where(*_bulk(user.id), _key() == key, Message.thread_id.isnot(None))
|
||
|
|
.group_by(Message.thread_id)
|
||
|
|
.order_by(func.max(Message.internal_date).desc())
|
||
|
|
.limit(limit)
|
||
|
|
)
|
||
|
|
return [r.thread_id for r in rows if r.thread_id is not None]
|
||
|
|
|
||
|
|
|
||
|
|
async def _latest_with_header(db: AsyncSession, user_id: uuid.UUID, key: str) -> Message:
|
||
|
|
message = await db.scalar(
|
||
|
|
select(Message)
|
||
|
|
.join(Mailbox, Mailbox.id == Message.mailbox_id)
|
||
|
|
.where(*_bulk(user_id), _key() == key)
|
||
|
|
.order_by(Message.internal_date.desc())
|
||
|
|
.limit(1)
|
||
|
|
)
|
||
|
|
if message is None:
|
||
|
|
raise_api_error(404, "not_found", "no mail from this sender")
|
||
|
|
return message
|
||
|
|
|
||
|
|
|
||
|
|
async def _record(db: AsyncSession, user_id: uuid.UUID, key: str, method: str) -> None:
|
||
|
|
stmt = insert(SubscriptionState).values(user_id=user_id, key=key, method=method)
|
||
|
|
await db.execute(
|
||
|
|
stmt.on_conflict_do_update(
|
||
|
|
index_elements=["user_id", "key"],
|
||
|
|
set_={"method": method, "unsubscribed_at": func.now()},
|
||
|
|
)
|
||
|
|
)
|
||
|
|
await db.commit()
|
||
|
|
|
||
|
|
|
||
|
|
async def _one_click(url: str) -> bool:
|
||
|
|
try:
|
||
|
|
resp = await guarded_request(
|
||
|
|
"POST",
|
||
|
|
url,
|
||
|
|
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
||
|
|
content="List-Unsubscribe=One-Click",
|
||
|
|
max_bytes=256 * 1024,
|
||
|
|
)
|
||
|
|
except Exception as exc: # SSRF refusal, network, TLS: any failure tries the next way
|
||
|
|
logger.info("one-click unsubscribe failed: %s", exc)
|
||
|
|
return False
|
||
|
|
return 200 <= resp.status < 400
|
||
|
|
|
||
|
|
|
||
|
|
async def _send_mailto(
|
||
|
|
db: AsyncSession, user_id: uuid.UUID, message: Message, targets: subs.UnsubscribeTargets
|
||
|
|
) -> bool:
|
||
|
|
assert targets.mailto is not None
|
||
|
|
body = DraftIn(
|
||
|
|
account_id=message.account_id,
|
||
|
|
kind="new",
|
||
|
|
payload=DraftPayload(
|
||
|
|
to=[targets.mailto], subject=targets.mailto_subject, text=targets.mailto_body
|
||
|
|
),
|
||
|
|
)
|
||
|
|
try:
|
||
|
|
draft = await upsert_draft(db, user_id=user_id, body=body, draft_id=None)
|
||
|
|
except DraftError:
|
||
|
|
return False
|
||
|
|
outbox = await create_outbox(db, user_id=user_id, draft=draft)
|
||
|
|
await db.commit()
|
||
|
|
await enqueue_job(
|
||
|
|
"send_job", str(outbox.id), _defer_by=dt.timedelta(seconds=get_settings().undo_seconds)
|
||
|
|
)
|
||
|
|
return True
|
||
|
|
|
||
|
|
|
||
|
|
@router.post("/unsubscribe", response_model=UnsubscribeOut)
|
||
|
|
async def unsubscribe(body: UnsubscribeIn, user: CurrentUser, db: DbSession) -> UnsubscribeOut:
|
||
|
|
"""Unsubscribe from a sender: one-click if offered, else by mail, else the web page."""
|
||
|
|
if body.message_id is not None:
|
||
|
|
found = await db.scalar(
|
||
|
|
select(Message).where(
|
||
|
|
Message.id == body.message_id,
|
||
|
|
Message.user_id == user.id,
|
||
|
|
Message.list_unsubscribe.isnot(None),
|
||
|
|
Message.list_unsubscribe != "",
|
||
|
|
)
|
||
|
|
)
|
||
|
|
if found is None:
|
||
|
|
raise_api_error(404, "not_found", "this message offers no way to unsubscribe")
|
||
|
|
message = found
|
||
|
|
else:
|
||
|
|
message = await _latest_with_header(db, user.id, body.key)
|
||
|
|
targets = subs.unsubscribe_targets(message.list_unsubscribe or "")
|
||
|
|
if message.list_unsubscribe_post and targets.https and await _one_click(targets.https):
|
||
|
|
await _record(db, user.id, body.key, "one_click")
|
||
|
|
return UnsubscribeOut(status="done", method="one_click")
|
||
|
|
if targets.mailto and await _send_mailto(db, user.id, message, targets):
|
||
|
|
await _record(db, user.id, body.key, "mailto")
|
||
|
|
return UnsubscribeOut(status="done", method="mailto")
|
||
|
|
if targets.https:
|
||
|
|
# The sender wants it done on their page: the user opens it.
|
||
|
|
await _record(db, user.id, body.key, "web")
|
||
|
|
return UnsubscribeOut(status="open", method="web", url=targets.https)
|
||
|
|
raise_api_error(409, "no_unsubscribe", "this sender offers no way to unsubscribe")
|
||
|
|
|
||
|
|
|
||
|
|
@router.post("/trash")
|
||
|
|
async def trash_all(body: KeyIn, user: CurrentUser, db: DbSession) -> dict[str, int]:
|
||
|
|
"""Move every conversation from this sender to Trash.
|
||
|
|
|
||
|
|
Only conversations made up entirely of this sender's mail: one where the
|
||
|
|
user replied, or someone else wrote, stays where it is.
|
||
|
|
"""
|
||
|
|
rows = (
|
||
|
|
await db.execute(
|
||
|
|
select(Message.thread_id, _key().label("key"), Mailbox.special_use)
|
||
|
|
.join(Mailbox, Mailbox.id == Message.mailbox_id)
|
||
|
|
.where(
|
||
|
|
Message.user_id == user.id,
|
||
|
|
Message.thread_id.in_(
|
||
|
|
select(Message.thread_id)
|
||
|
|
.join(Mailbox, Mailbox.id == Message.mailbox_id)
|
||
|
|
.where(*_bulk(user.id), _key() == body.key)
|
||
|
|
),
|
||
|
|
)
|
||
|
|
)
|
||
|
|
).all()
|
||
|
|
own: dict[uuid.UUID, bool] = {}
|
||
|
|
for thread_id, key, special in rows:
|
||
|
|
if thread_id is None or special == "trash":
|
||
|
|
continue
|
||
|
|
own[thread_id] = own.get(thread_id, True) and key == body.key
|
||
|
|
moved = [tid for tid, only_theirs in own.items() if only_theirs]
|
||
|
|
for tid in moved:
|
||
|
|
await enqueue_job("move_thread_to_folder_job", str(tid), "trash")
|
||
|
|
return {"moved": len(moved)}
|