diff --git a/backend/app/api/schemas.py b/backend/app/api/schemas.py index 00058a8..827e861 100644 --- a/backend/app/api/schemas.py +++ b/backend/app/api/schemas.py @@ -112,3 +112,5 @@ class SearchThreadHit(BaseModel): class SearchOut(BaseModel): threads: list[SearchThreadHit] cursor: str | None = None + # Conversations matching in all (first page only; None where not counted). + total: int | None = None diff --git a/backend/app/api/search.py b/backend/app/api/search.py index 6f5cef5..9f9ae07 100644 --- a/backend/app/api/search.py +++ b/backend/app/api/search.py @@ -29,13 +29,22 @@ async def search( q: str = Query(""), cursor: str | None = None, limit: int = Query(50, ge=1, le=200), + filter: str | None = Query(None, pattern="^(unread|pinned)$"), ) -> SearchOut: + """One page of matching messages, grouped into conversations. ``filter`` + keeps conversations with unread mail / pinned ones (the list's Unread and + Pinned tabs). The first page also says how many conversations match in + all, where the search engine can count them.""" if not q.strip(): return SearchOut(threads=[], cursor=None) backend = get_search_backend() query = to_search_query(q, user.id) hits, next_cursor = await backend.query(query, limit=limit, cursor=cursor) + total: int | None = None + count = getattr(backend, "count_threads", None) + if cursor is None and count is not None: + total = await count(query, thread_filter=filter) # Group hits by thread, preserving order and keeping the first highlight. order: list[uuid.UUID] = [] @@ -48,7 +57,7 @@ async def search( highlights[hit.thread_id] = hit.highlights if not order: - return SearchOut(threads=[], cursor=next_cursor) + return SearchOut(threads=[], cursor=next_cursor, total=total) threads = { t.id: t @@ -64,5 +73,9 @@ async def search( ) for tid in order if tid in threads + and ( + filter is None + or (threads[tid].unread_count > 0 if filter == "unread" else threads[tid].pinned) + ) ] - return SearchOut(threads=results, cursor=next_cursor) + return SearchOut(threads=results, cursor=next_cursor, total=total) diff --git a/backend/app/search/pg.py b/backend/app/search/pg.py index c5e4ee4..3f2f8ab 100644 --- a/backend/app/search/pg.py +++ b/backend/app/search/pg.py @@ -13,6 +13,7 @@ import base64 import datetime as dt import html import re +from typing import Any from uuid import UUID from sqlalchemy import ( @@ -20,6 +21,7 @@ from sqlalchemy import ( any_, case, cast, + distinct, exists, func, literal, @@ -32,7 +34,7 @@ from sqlalchemy import ( from sqlalchemy.sql.elements import ColumnElement from app.db import get_sessionmaker -from app.models import Attachment, Mailbox, Message +from app.models import Attachment, Mailbox, Message, Thread from app.search.base import SearchBackend, SearchDoc, SearchHit, SearchQuery _TS_CONFIG = "simple" # language-neutral; avoids wrong-language stemming (M0) @@ -52,6 +54,8 @@ def _prefix_tsquery(value: str) -> ColumnElement[object]: lexemes = cast(func.plainto_tsquery(_TS_CONFIG, value), Text) prefixed = func.regexp_replace(lexemes, r"'([^']+)'", r"'\1':*", "g") return func.to_tsquery(_TS_CONFIG, prefixed) + + _RELATIVE_MULT = {"d": 1, "w": 7, "m": 30, "y": 365} @@ -254,20 +258,9 @@ class PgSearchBackend(SearchBackend): .join(Mailbox, Mailbox.id == Message.mailbox_id) .where(Message.user_id == q.user_id, Message.thread_id.is_not(None)) ) - for cond in conditions: + for cond in self._scope(groups, conditions): stmt = stmt.where(cond) - # Exclude Trash/Junk from search by default (Gmail-style), so deleting a - # message from search results removes it. An explicit `in:trash`/`in:junk` - # overrides this to search there on purpose. - if not _targets_removal_folder(groups): - stmt = stmt.where( - or_( - Mailbox.special_use.is_(None), - Mailbox.special_use.notin_(_REMOVAL_FOLDERS), - ) - ) - decoded = _decode_cursor(cursor) if decoded is not None: cur_hit, cur_date, cur_id = decoded @@ -301,6 +294,41 @@ class PgSearchBackend(SearchBackend): ) return hits, next_cursor + @staticmethod + def _scope(groups: list[list[dict[str, object]]], conditions: list[Any]) -> list[Any]: + """The query's conditions, plus leaving out Trash/Junk by default + (Gmail-style), so deleting a message from search results removes it. An + explicit `in:trash`/`in:junk` overrides this to search there on purpose.""" + out = list(conditions) + if not _targets_removal_folder(groups): + out.append( + or_( + Mailbox.special_use.is_(None), + Mailbox.special_use.notin_(_REMOVAL_FOLDERS), + ) + ) + return out + + async def count_threads(self, q: SearchQuery, *, thread_filter: str | None = None) -> int: + """How many conversations match (``thread_filter``: "unread" / "pinned" + keeps only conversations with unread mail / pinned ones).""" + groups = q.filters.get("groups", []) if isinstance(q.filters, dict) else [] + assert isinstance(groups, list) + stmt = ( + select(func.count(distinct(Message.thread_id))) + .select_from(Message) + .join(Mailbox, Mailbox.id == Message.mailbox_id) + .where(Message.user_id == q.user_id, Message.thread_id.is_not(None)) + ) + for cond in self._scope(groups, self._build_conditions(groups)): + stmt = stmt.where(cond) + if thread_filter in ("unread", "pinned"): + stmt = stmt.join(Thread, Thread.id == Message.thread_id).where( + Thread.unread_count > 0 if thread_filter == "unread" else Thread.pinned.is_(True) + ) + async with get_sessionmaker()() as session: + return int(await session.scalar(stmt) or 0) + _REMOVAL_FOLDERS = ("trash", "junk") diff --git a/backend/tests/search/test_search_api.py b/backend/tests/search/test_search_api.py index 9a2d0df..bac8839 100644 --- a/backend/tests/search/test_search_api.py +++ b/backend/tests/search/test_search_api.py @@ -245,3 +245,44 @@ async def test_negation(seeded: Ctx) -> None: async def test_combined_operators(seeded: Ctx) -> None: assert await _search(seeded, "from:dave has:attachment") == 1 assert await _search(seeded, "from:bob has:attachment") == 0 + + +async def test_more_than_one_page_counts_and_pages(ctx: Ctx) -> None: + """Beyond the first page: the first page says how many conversations match + in all, the cursor brings the rest, and Unread/Pinned filter conversations.""" + for i in range(7): + await _ingest( + ctx, + frm=f"Sender {i} ", + subject=f"Quarterly report {i}", + body="numbers inside", + flags=frozenset({"\\Seen"}) if i % 2 else frozenset(), + ) + await _ingest(ctx, frm="Other ", subject="Lunch", body="nothing") + + first = (await ctx.client.get("/api/search", params={"q": "quarterly", "limit": 3})).json() + assert first["total"] == 7 and len(first["threads"]) == 3 and first["cursor"] + seen = {t["thread"]["id"] for t in first["threads"]} + cursor = first["cursor"] + while cursor: + page = ( + await ctx.client.get( + "/api/search", params={"q": "quarterly", "limit": 3, "cursor": cursor} + ) + ).json() + assert page["total"] is None # counted once, on the first page + seen |= {t["thread"]["id"] for t in page["threads"]} + cursor = page["cursor"] + assert len(seen) == 7 + + unread = ( + await ctx.client.get("/api/search", params={"q": "quarterly", "filter": "unread"}) + ).json() + assert unread["total"] == 4 and len(unread["threads"]) == 4 + assert all(t["thread"]["unread_count"] > 0 for t in unread["threads"]) + pinned = ( + await ctx.client.get("/api/search", params={"q": "quarterly", "filter": "pinned"}) + ).json() + assert pinned["total"] == 0 and pinned["threads"] == [] + bad = await ctx.client.get("/api/search", params={"q": "quarterly", "filter": "nope"}) + assert bad.status_code == 422 diff --git a/frontend/src/i18n/en.ts b/frontend/src/i18n/en.ts index e8e6674..4a933c2 100644 --- a/frontend/src/i18n/en.ts +++ b/frontend/src/i18n/en.ts @@ -599,7 +599,7 @@ export const en = { "Not available here: browsers allow offline access only when the app is opened over HTTPS.", "push.settingTitle": "Notifications", "push.settingHelp": - "Get a notification on this device when new mail arrives in an Inbox, even when Ultimo isn't open. Shows the sender and subject. Turned off when you sign out.", + "Get a notification on this device when new mail arrives in an Inbox, even when Ultimo isn't open.", "push.needsInstall": "On iPhone and iPad, add Ultimo to your Home Screen first (Share → Add to Home Screen), then turn this on from the app.", "push.blocked": diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index bdc2a24..bdd3a34 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -184,9 +184,10 @@ export const api = { thread: (id: string, sort: "asc" | "desc" = "asc", view = "unified") => apiFetch(threadPath(id, sort, view)), - search: (q: string, cursor?: string) => { + search: (q: string, cursor?: string | null, filter?: string) => { const params = new URLSearchParams({ q }); if (cursor) params.set("cursor", cursor); + if (filter === "unread" || filter === "pinned") params.set("filter", filter); return apiFetch(`/search?${params.toString()}`); }, diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index 6255e99..e21dece 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -159,6 +159,8 @@ export interface SearchThreadHit { export interface SearchOut { threads: SearchThreadHit[]; cursor: string | null; + /** Conversations matching in all (first page; null where not counted). */ + total?: number | null; } // --- M1: account wizard ------------------------------------------------------ diff --git a/frontend/src/lib/ws.ts b/frontend/src/lib/ws.ts index f82a922..6d977ee 100644 --- a/frontend/src/lib/ws.ts +++ b/frontend/src/lib/ws.ts @@ -107,6 +107,9 @@ export function connectWs(queryClient: QueryClient): () => void { if (typeof payload.thread_id === "string") { queryClient.invalidateQueries({ queryKey: ["thread", payload.thread_id] }); } + // Search results too (a deleted conversation leaves them) — not on every + // backfill tick, which would refetch an open search over and over. + if (name !== "sync.progress") queryClient.invalidateQueries({ queryKey: ["search"] }); } if (name === "flags.changed") { // Read/unread changes affect the Inbox unread badge. diff --git a/frontend/src/routes/MailLayout.tsx b/frontend/src/routes/MailLayout.tsx index f9d3a48..906a087 100644 --- a/frontend/src/routes/MailLayout.tsx +++ b/frontend/src/routes/MailLayout.tsx @@ -1,5 +1,11 @@ -import { keepPreviousData, useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { Fragment, useCallback, useEffect, useRef, useState } from "react"; +import { + keepPreviousData, + useInfiniteQuery, + useMutation, + useQuery, + useQueryClient, +} from "@tanstack/react-query"; +import { Fragment, useCallback, useEffect, useMemo, useRef, useState } from "react"; import { Link, useMatch, useNavigate, useSearchParams } from "react-router-dom"; import signetLogo from "../../icons/ultimo-logo-signet.svg"; @@ -42,6 +48,7 @@ import type { AccountOut, QuotaOut, SavedSearch, ThreadSummary } from "../lib/ty import { useReadingPane } from "../lib/readingPane"; import { useIsMobile } from "../lib/useIsMobile"; import { useKeyboardShortcuts } from "../lib/useKeyboardShortcuts"; +import { subscribeWs } from "../lib/ws"; import { PluginSlot } from "../plugins/PluginSlot"; function quotaLabel(q: QuotaOut): string { @@ -57,6 +64,9 @@ const PAGE_SIZE = 50; // How long a row the list hid for a queued move waits for the move to land // before it is shown again. const HIDE_FALLBACK_MS = 30_000; +// Search results load 50 matching messages at a time; this caps how many +// pages an oldest-first search (which needs them all) fetches. +const MAX_SEARCH_PAGES = 40; // Collapsed sidebar width: just wide enough for the icons. const COLLAPSED_W = 60; @@ -271,11 +281,28 @@ export function MailLayout() { const effectiveQuery = searchScope ? `${searchScope} ${debouncedTerm}`.trim() : debouncedTerm; - const searchQuery = useQuery({ - queryKey: ["search", effectiveQuery], - queryFn: () => api.search(effectiveQuery), + // Search comes in pages of matching messages (a conversation can span two); + // more are fetched as the list is paged through (effect below). + const searchFilter = filter === "unread" || filter === "pinned" ? filter : undefined; + const searchQuery = useInfiniteQuery({ + queryKey: ["search", effectiveQuery, searchFilter], + queryFn: ({ pageParam }) => api.search(effectiveQuery, pageParam, searchFilter), + initialPageParam: null as string | null, + getNextPageParam: (last) => last.cursor, enabled: searching, }); + const searchThreads = useMemo(() => { + const seen = new Set(); + const out: ThreadSummary[] = []; + for (const pg of searchQuery.data?.pages ?? []) { + for (const h of pg.threads) { + if (seen.has(h.thread.id)) continue; + seen.add(h.thread.id); + out.push(h.thread); + } + } + return out; + }, [searchQuery.data]); const savedSearchesQuery = useQuery({ queryKey: ["saved-searches"], queryFn: api.savedSearches }); const settingsQuery = useQuery({ queryKey: ["settings"], queryFn: api.settings }); const threadOrder: "asc" | "desc" = settingsQuery.data?.thread_order ?? "asc"; @@ -296,12 +323,15 @@ export function MailLayout() { let pageThreads: ThreadSummary[]; let total: number; if (searching) { - const hits = (searchQuery.data?.threads ?? []).map((h) => h.thread); - const f = hits.filter((th) => - filter === "unread" ? th.unread_count > 0 : filter === "pinned" ? th.pinned : true, - ); - const ordered = listOrder === "newest" ? f : [...f].reverse(); - total = ordered.length; + const shown = searchThreads.filter((th) => !hiddenIds.has(th.id)); + const ordered = listOrder === "newest" ? shown : [...shown].reverse(); + // The server's count of matching conversations; without one (or while + // still loading pages), what has been loaded. + const counted = searchQuery.data?.pages[0]?.total; + total = + counted != null + ? Math.max(ordered.length, counted - (searchThreads.length - shown.length)) + : ordered.length; pageThreads = ordered.slice(page * PAGE_SIZE, page * PAGE_SIZE + PAGE_SIZE); } else { const all = threadsQuery.data?.threads ?? []; @@ -310,6 +340,21 @@ export function MailLayout() { } const pageStart = page * PAGE_SIZE; const pageEnd = pageStart + pageThreads.length; + // Search: fetch further pages of results as far as the page shown needs (one + // conversation more, to know whether a next page exists) — or all of them + // for oldest-first, whose first page is the end of the results. + const { + hasNextPage: searchHasMore, + isFetchingNextPage: searchFetchingMore, + fetchNextPage: fetchMoreSearch, + } = searchQuery; + const searchLoaded = searchThreads.length; + const searchPages = searchQuery.data?.pages.length ?? 0; + useEffect(() => { + if (!searching || !searchHasMore || searchFetchingMore) return; + const wanted = listOrder === "newest" ? (page + 1) * PAGE_SIZE : Infinity; + if (searchLoaded <= wanted && searchPages < MAX_SEARCH_PAGES) void fetchMoreSearch(); + }, [searching, searchHasMore, searchFetchingMore, fetchMoreSearch, searchLoaded, searchPages, page, listOrder]); // Reset to the first page whenever the list identity (not the page) changes. useEffect(() => setPage(0), [view, filter, debouncedTerm, listOrder]); @@ -550,6 +595,28 @@ export function MailLayout() { // row is hidden straight away (it is leaving the folder's scope); in search it // may still match a global query, so it is greyed out instead. A failed // request puts the row back. + // Moves are queued on the server and land one by one (each announced as + // thread.updated). A hidden row comes back only when nothing has landed for + // a while — the move failed — not while a long batch is still going. + const lastMoveLanded = useRef(0); + useEffect( + () => + subscribeWs((name) => { + if (name === "thread.updated") lastMoveLanded.current = Date.now(); + }), + [], + ); + const unhideIfStuck = (unmark: () => void) => { + const check = () => { + if (Date.now() - lastMoveLanded.current < HIDE_FALLBACK_MS) { + window.setTimeout(check, HIDE_FALLBACK_MS); + } else { + unmark(); + } + }; + window.setTimeout(check, HIDE_FALLBACK_MS); + }; + const listMove = async (id: string, action: (id: string) => Promise) => { const mark = searching ? setDimmedIds : setHiddenIds; const unmark = () => @@ -573,7 +640,7 @@ export function MailLayout() { // The server only queues the move, so the refetch above can still carry the // row; the effect below un-hides it once a list arrives without it. Should // the move never land, the row comes back rather than staying lost. - if (!searching) window.setTimeout(unmark, HIDE_FALLBACK_MS); + if (!searching) unhideIfStuck(unmark); }; // A row stays hidden until a fetched list no longer contains it. useEffect(() => { @@ -586,6 +653,16 @@ export function MailLayout() { return next.size === prev.size ? prev : next; }); }, [threadsQuery.data]); + // The same for search results (deleted conversations drop out of them). + useEffect(() => { + if (!searchQuery.data || !searching) return; + setHiddenIds((prev) => { + if (prev.size === 0) return prev; + const present = new Set(searchThreads.map((th) => th.id)); + const next = new Set([...prev].filter((id) => present.has(id))); + return next.size === prev.size ? prev : next; + }); + }, [searchQuery.data, searchThreads, searching]); const listArchive = (id: string) => listMove(id, api.archiveThread); const listTrash = (id: string) => listMove(id, api.trashThread); // Phone: a conversation swiped right goes to Trash — after a short Undo @@ -648,26 +725,52 @@ export function MailLayout() { // Bulk actions run the given operation over every selected thread, then clear the // selection and refresh (navigating away if the open thread was among them). - const runBulk = async (action: (id: string) => Promise) => { + // + // Like the row buttons, the result shows at once: conversations leaving the + // view (deleted; archived out of a folder) are hidden — in search too, which + // leaves out Trash — and ones still matching a search (archived) are greyed. + // The requests run behind; a failed one puts its row back. + const runBulk = async ( + action: (id: string) => Promise, + effect: "hide" | "dim" | null = null, + ) => { const ids = [...checkedIds]; + const mark = effect === "hide" ? setHiddenIds : effect === "dim" ? setDimmedIds : null; + const unmark = (gone: string[]) => + mark?.((prev) => { + const next = new Set(prev); + for (const id of gone) next.delete(id); + return next; + }); + mark?.((prev) => new Set([...prev, ...ids])); + setCheckedIds(new Set()); + // The conversations are gone from the list: the reading pane goes back to + // "Select a conversation" rather than to whatever was open before selecting. + if (threadId && (effect !== null || ids.includes(threadId))) navigate("/mail"); + const failed: string[] = []; for (const id of ids) { try { await action(id); } catch { - /* keep going */ + failed.push(id); } } - setCheckedIds(new Set()); + if (failed.length) unmark(failed); queryClient.invalidateQueries({ queryKey: ["threads"] }); + queryClient.invalidateQueries({ queryKey: ["search"] }); queryClient.invalidateQueries({ queryKey: ["quota"] }); queryClient.invalidateQueries({ queryKey: ["folder-counts"] }); - if (threadId && ids.includes(threadId)) navigate("/mail"); + // The server only queues the moves, so a refetch can still carry the rows; + // they stay hidden until a list arrives without them — or come back should + // the moves never land. + if (effect === "hide") unhideIfStuck(() => unmark(ids)); }; - const bulkArchive = () => runBulk((id) => api.archiveThread(id)); + const leaving = searching ? "dim" : "hide"; // archived: gone from a folder, still found by a search + const bulkArchive = () => runBulk((id) => api.archiveThread(id), leaving); const bulkPin = () => runBulk((id) => api.threadFlags(id, ["\\Flagged"], [])); - const bulkTrash = () => runBulk((id) => api.trashThread(id)); - const bulkRestore = () => runBulk((id) => api.restoreThread(id)); - const bulkDeletePermanent = () => runBulk((id) => api.deleteThreadPermanent(id)); + const bulkTrash = () => runBulk((id) => api.trashThread(id), "hide"); + const bulkRestore = () => runBulk((id) => api.restoreThread(id), "hide"); + const bulkDeletePermanent = () => runBulk((id) => api.deleteThreadPermanent(id), "hide"); // Bulk-action buttons over the selected conversations: in the reading pane on // the desktop, above the list on a phone.