"""Subscriptions page: bulk senders mailing the user (app/subscriptions.py).""" from __future__ import annotations import datetime as dt import logging import uuid from typing import Annotated, Any from fastapi import APIRouter, Depends, Query from pydantic import BaseModel from sqlalchemy import case, func, literal, or_, select from sqlalchemy.dialects.postgresql import insert from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.sql.elements import ColumnElement from app import subscriptions as subs from app.accounts.ssrf import guarded_request from app.api.errors import raise_api_error from app.config import get_settings from app.db import get_session from app.identity.deps import CurrentUser, require_csrf from app.models import Mailbox, Message, SubscriptionState from app.outbox.schemas import DraftIn, DraftPayload from app.outbox.service import DraftError, create_outbox, upsert_draft from app.queue import enqueue_job logger = logging.getLogger(__name__) router = APIRouter(prefix="/subscriptions", tags=["mail"], dependencies=[Depends(require_csrf)]) DbSession = Annotated[AsyncSession, Depends(get_session)] # RFC 8058 senders should stop within two days of a one-click request. GRACE = dt.timedelta(days=2) # Not counted: mail the user sent, and mail already thrown away (Trash, Spam) — # a sender whose mail was all deleted drops off the list. _LEFT_OUT = ("sent", "drafts", "trash", "junk") class SubscriptionOut(BaseModel): key: str name: str email: str list_id: str | None total: int unread: int last_30_days: int per_month: float first_at: dt.datetime last_at: dt.datetime last_subject: str one_click: bool has_mailto: bool has_web: bool unsubscribed_at: dt.datetime | None method: str | None still_sending: bool class SubscriptionListOut(BaseModel): subscriptions: list[SubscriptionOut] # one page total: int # senders in all pending: int # messages whose headers are still being read class KeyIn(BaseModel): key: str class UnsubscribeIn(KeyIn): # From the reading pane: this message's own header, wherever it is filed # (Spam included, which the page's groups leave out). message_id: uuid.UUID | None = None class UnsubscribeOut(BaseModel): status: str # done | open method: str # one_click | mailto | web url: str | None = None def _sender() -> ColumnElement[Any]: return func.lower(func.coalesce(Message.from_addr[0]["email"].astext, "")) def _key() -> ColumnElement[Any]: return case( (Message.list_id.isnot(None), literal("list:") + Message.list_id), else_=literal("from:") + _sender(), ) def _bulk(user_id: uuid.UUID) -> list[ColumnElement[bool]]: return [ Message.user_id == user_id, Message.list_unsubscribe.isnot(None), Message.list_unsubscribe != "", or_(Mailbox.special_use.is_(None), Mailbox.special_use.notin_(_LEFT_OUT)), ] async def _start_backfill(user_id: uuid.UUID) -> None: # One job per user at a time: arq refuses a second while the first exists. await enqueue_job("backfill_list_headers_job", str(user_id), _job_id=f"listhdr:{user_id}") @router.get("", response_model=SubscriptionListOut) async def list_subscriptions( user: CurrentUser, db: DbSession, offset: int = Query(0, ge=0), limit: int = Query(50, ge=1, le=200), order: str = Query("desc"), # desc: heard from most recently first | asc: longest ago ) -> SubscriptionListOut: """One page of senders, by when their latest mail came.""" pending = await subs.pending_count(db, user.id) if pending: await _start_backfill(user.id) now = dt.datetime.now(dt.UTC) key = _key().label("key") seen = Message.flags.contains(["\\Seen"]) groups = ( select( key, func.count().label("total"), func.count().filter(~seen).label("unread"), func.count() .filter(Message.internal_date > now - dt.timedelta(days=30)) .label("recent"), func.min(Message.internal_date).label("first_at"), func.max(Message.internal_date).label("last_at"), ) .join(Mailbox, Mailbox.id == Message.mailbox_id) .where(*_bulk(user.id)) .group_by(key) ) total = await db.scalar(select(func.count()).select_from(groups.subquery())) or 0 agg_rows = ( await db.execute( groups.order_by( func.max(Message.internal_date).asc() if order == "asc" else func.max(Message.internal_date).desc(), key, ) .offset(offset) .limit(limit) ) ).all() page_keys = [r.key for r in agg_rows] if not page_keys: return SubscriptionListOut(subscriptions=[], total=total, pending=pending) in_page = _key().in_(page_keys) latest_rows = ( await db.execute( select( key, Message.from_addr, Message.subject, Message.list_id, Message.list_unsubscribe, Message.list_unsubscribe_post, ) .join(Mailbox, Mailbox.id == Message.mailbox_id) .where(*_bulk(user.id), in_page) .distinct(key) .order_by(key, Message.internal_date.desc()) ) ).all() latest = {r.key: r for r in latest_rows} states = { s.key: s for s in await db.scalars( select(SubscriptionState).where( SubscriptionState.user_id == user.id, SubscriptionState.key.in_(page_keys) ) ) } # Senders still mailing after the grace period, for the whole page at once. still_sending = set( await db.scalars( select(key) .select_from(Message) .join(Mailbox, Mailbox.id == Message.mailbox_id) .join( SubscriptionState, (SubscriptionState.user_id == user.id) & (SubscriptionState.key == _key()), ) .where( *_bulk(user.id), in_page, Message.internal_date > SubscriptionState.unsubscribed_at + GRACE, ) .distinct() ) ) out: list[SubscriptionOut] = [] for row in agg_rows: last = latest.get(row.key) if last is None: continue sender = last.from_addr[0] if last.from_addr else {} targets = subs.unsubscribe_targets(last.list_unsubscribe or "") state = states.get(row.key) months = max(1.0, (row.last_at - row.first_at).days / 30.44) out.append( SubscriptionOut( key=row.key, name=sender.get("name") or sender.get("email", ""), email=sender.get("email", ""), list_id=last.list_id, total=row.total, unread=row.unread, last_30_days=row.recent, per_month=round(row.total / months, 1), first_at=row.first_at, last_at=row.last_at, last_subject=last.subject, one_click=bool(last.list_unsubscribe_post and targets.https), has_mailto=targets.mailto is not None, has_web=targets.https is not None, unsubscribed_at=state.unsubscribed_at if state else None, method=state.method if state else None, still_sending=row.key in still_sending, ) ) return SubscriptionListOut(subscriptions=out, total=total, pending=pending) @router.get("/threads", response_model=list[uuid.UUID]) async def subscription_threads( user: CurrentUser, db: DbSession, key: str = Query(...), limit: int = Query(50, le=200) ) -> list[uuid.UUID]: """The group's conversations, newest first (for the reading pane).""" rows = await db.execute( select(Message.thread_id, func.max(Message.internal_date).label("at")) .join(Mailbox, Mailbox.id == Message.mailbox_id) .where(*_bulk(user.id), _key() == key, Message.thread_id.isnot(None)) .group_by(Message.thread_id) .order_by(func.max(Message.internal_date).desc()) .limit(limit) ) return [r.thread_id for r in rows if r.thread_id is not None] async def _latest_with_header(db: AsyncSession, user_id: uuid.UUID, key: str) -> Message: message = await db.scalar( select(Message) .join(Mailbox, Mailbox.id == Message.mailbox_id) .where(*_bulk(user_id), _key() == key) .order_by(Message.internal_date.desc()) .limit(1) ) if message is None: raise_api_error(404, "not_found", "no mail from this sender") return message async def _record(db: AsyncSession, user_id: uuid.UUID, key: str, method: str) -> None: stmt = insert(SubscriptionState).values(user_id=user_id, key=key, method=method) await db.execute( stmt.on_conflict_do_update( index_elements=["user_id", "key"], set_={"method": method, "unsubscribed_at": func.now()}, ) ) await db.commit() async def _one_click(url: str) -> bool: try: resp = await guarded_request( "POST", url, headers={"Content-Type": "application/x-www-form-urlencoded"}, content="List-Unsubscribe=One-Click", max_bytes=256 * 1024, ) except Exception as exc: # SSRF refusal, network, TLS: any failure tries the next way logger.info("one-click unsubscribe failed: %s", exc) return False return 200 <= resp.status < 400 async def _send_mailto( db: AsyncSession, user_id: uuid.UUID, message: Message, targets: subs.UnsubscribeTargets ) -> bool: assert targets.mailto is not None body = DraftIn( account_id=message.account_id, kind="new", payload=DraftPayload( to=[targets.mailto], subject=targets.mailto_subject, text=targets.mailto_body ), ) try: draft = await upsert_draft(db, user_id=user_id, body=body, draft_id=None) except DraftError: return False outbox = await create_outbox(db, user_id=user_id, draft=draft) await db.commit() await enqueue_job( "send_job", str(outbox.id), _defer_by=dt.timedelta(seconds=get_settings().undo_seconds) ) return True @router.post("/unsubscribe", response_model=UnsubscribeOut) async def unsubscribe(body: UnsubscribeIn, user: CurrentUser, db: DbSession) -> UnsubscribeOut: """Unsubscribe from a sender: one-click if offered, else by mail, else the web page.""" if body.message_id is not None: found = await db.scalar( select(Message).where( Message.id == body.message_id, Message.user_id == user.id, Message.list_unsubscribe.isnot(None), Message.list_unsubscribe != "", ) ) if found is None: raise_api_error(404, "not_found", "this message offers no way to unsubscribe") message = found else: message = await _latest_with_header(db, user.id, body.key) targets = subs.unsubscribe_targets(message.list_unsubscribe or "") if message.list_unsubscribe_post and targets.https and await _one_click(targets.https): await _record(db, user.id, body.key, "one_click") return UnsubscribeOut(status="done", method="one_click") if targets.mailto and await _send_mailto(db, user.id, message, targets): await _record(db, user.id, body.key, "mailto") return UnsubscribeOut(status="done", method="mailto") if targets.https: # The sender wants it done on their page: the user opens it. await _record(db, user.id, body.key, "web") return UnsubscribeOut(status="open", method="web", url=targets.https) raise_api_error(409, "no_unsubscribe", "this sender offers no way to unsubscribe") @router.post("/trash") async def trash_all(body: KeyIn, user: CurrentUser, db: DbSession) -> dict[str, int]: """Move every conversation from this sender to Trash. Only conversations made up entirely of this sender's mail: one where the user replied, or someone else wrote, stays where it is. """ rows = ( await db.execute( select(Message.thread_id, _key().label("key"), Mailbox.special_use) .join(Mailbox, Mailbox.id == Message.mailbox_id) .where( Message.user_id == user.id, Message.thread_id.in_( select(Message.thread_id) .join(Mailbox, Mailbox.id == Message.mailbox_id) .where(*_bulk(user.id), _key() == body.key) ), ) ) ).all() own: dict[uuid.UUID, bool] = {} for thread_id, key, special in rows: if thread_id is None or special == "trash": continue own[thread_id] = own.get(thread_id, True) and key == body.key moved = [tid for tid, only_theirs in own.items() if only_theirs] for tid in moved: await enqueue_job("move_thread_to_folder_job", str(tid), "trash") return {"moved": len(moved)}