Search: all results with a real count; bulk actions on results stay done

Search showed at most the first 50 matching messages ("1–50 of 50"). The first page now carries the number of matching conversations, further pages load as the list is paged through (a conversation spanning two appears once; oldest-first loads them all, up to 40 pages), and the Unread/Pinned tabs filter on the server, so their count covers every result. Bulk Delete / Archive / Restore over selected results acts at once: the rows leave the list (greyed instead when archived from a search, which still finds them), the search results are refreshed, and the reading pane returns to "Select a conversation". Each move landing on the server refreshes them again, and a hidden row only comes back when no move has landed for 30 seconds — a long batch no longer brings deleted results back. Settings → Accounts → Notifications: shorter description. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 21:15:06 +02:00
parent 9c08cbb140
commit 5bf8fc81f6
9 changed files with 230 additions and 37 deletions
@@ -112,3 +112,5 @@ class SearchThreadHit(BaseModel):
 class SearchOut(BaseModel):
    threads: list[SearchThreadHit]
    cursor: str | None = None
+    # Conversations matching in all (first page only; None where not counted).
+    total: int | None = None
@@ -29,13 +29,22 @@ async def search(
    q: str = Query(""),
    cursor: str | None = None,
    limit: int = Query(50, ge=1, le=200),
+    filter: str | None = Query(None, pattern="^(unread|pinned)$"),
 ) -> SearchOut:
+    """One page of matching messages, grouped into conversations. ``filter``
+    keeps conversations with unread mail / pinned ones (the list's Unread and
+    Pinned tabs). The first page also says how many conversations match in
+    all, where the search engine can count them."""
    if not q.strip():
        return SearchOut(threads=[], cursor=None)

    backend = get_search_backend()
    query = to_search_query(q, user.id)
    hits, next_cursor = await backend.query(query, limit=limit, cursor=cursor)
+    total: int | None = None
+    count = getattr(backend, "count_threads", None)
+    if cursor is None and count is not None:
+        total = await count(query, thread_filter=filter)

    # Group hits by thread, preserving order and keeping the first highlight.
    order: list[uuid.UUID] = []
@@ -48,7 +57,7 @@ async def search(
            highlights[hit.thread_id] = hit.highlights

    if not order:
-        return SearchOut(threads=[], cursor=next_cursor)
+        return SearchOut(threads=[], cursor=next_cursor, total=total)

    threads = {
        t.id: t
@@ -64,5 +73,9 @@ async def search(
        )
        for tid in order
        if tid in threads
+        and (
+            filter is None
+            or (threads[tid].unread_count > 0 if filter == "unread" else threads[tid].pinned)
+        )
    ]
-    return SearchOut(threads=results, cursor=next_cursor)
+    return SearchOut(threads=results, cursor=next_cursor, total=total)
@@ -13,6 +13,7 @@ import base64
 import datetime as dt
 import html
 import re
+from typing import Any
 from uuid import UUID

 from sqlalchemy import (
@@ -20,6 +21,7 @@ from sqlalchemy import (
    any_,
    case,
    cast,
+    distinct,
    exists,
    func,
    literal,
@@ -32,7 +34,7 @@ from sqlalchemy import (
 from sqlalchemy.sql.elements import ColumnElement

 from app.db import get_sessionmaker
-from app.models import Attachment, Mailbox, Message
+from app.models import Attachment, Mailbox, Message, Thread
 from app.search.base import SearchBackend, SearchDoc, SearchHit, SearchQuery

 _TS_CONFIG = "simple"  # language-neutral; avoids wrong-language stemming (M0)
@@ -52,6 +54,8 @@ def _prefix_tsquery(value: str) -> ColumnElement[object]:
    lexemes = cast(func.plainto_tsquery(_TS_CONFIG, value), Text)
    prefixed = func.regexp_replace(lexemes, r"'([^']+)'", r"'\1':*", "g")
    return func.to_tsquery(_TS_CONFIG, prefixed)
+
+
 _RELATIVE_MULT = {"d": 1, "w": 7, "m": 30, "y": 365}


@@ -254,20 +258,9 @@ class PgSearchBackend(SearchBackend):
            .join(Mailbox, Mailbox.id == Message.mailbox_id)
            .where(Message.user_id == q.user_id, Message.thread_id.is_not(None))
        )
-        for cond in conditions:
+        for cond in self._scope(groups, conditions):
            stmt = stmt.where(cond)

-        # Exclude Trash/Junk from search by default (Gmail-style), so deleting a
-        # message from search results removes it. An explicit `in:trash`/`in:junk`
-        # overrides this to search there on purpose.
-        if not _targets_removal_folder(groups):
-            stmt = stmt.where(
-                or_(
-                    Mailbox.special_use.is_(None),
-                    Mailbox.special_use.notin_(_REMOVAL_FOLDERS),
-                )
-            )
-
        decoded = _decode_cursor(cursor)
        if decoded is not None:
            cur_hit, cur_date, cur_id = decoded
@@ -301,6 +294,41 @@ class PgSearchBackend(SearchBackend):
        )
        return hits, next_cursor

+    @staticmethod
+    def _scope(groups: list[list[dict[str, object]]], conditions: list[Any]) -> list[Any]:
+        """The query's conditions, plus leaving out Trash/Junk by default
+        (Gmail-style), so deleting a message from search results removes it. An
+        explicit `in:trash`/`in:junk` overrides this to search there on purpose."""
+        out = list(conditions)
+        if not _targets_removal_folder(groups):
+            out.append(
+                or_(
+                    Mailbox.special_use.is_(None),
+                    Mailbox.special_use.notin_(_REMOVAL_FOLDERS),
+                )
+            )
+        return out
+
+    async def count_threads(self, q: SearchQuery, *, thread_filter: str | None = None) -> int:
+        """How many conversations match (``thread_filter``: "unread" / "pinned"
+        keeps only conversations with unread mail / pinned ones)."""
+        groups = q.filters.get("groups", []) if isinstance(q.filters, dict) else []
+        assert isinstance(groups, list)
+        stmt = (
+            select(func.count(distinct(Message.thread_id)))
+            .select_from(Message)
+            .join(Mailbox, Mailbox.id == Message.mailbox_id)
+            .where(Message.user_id == q.user_id, Message.thread_id.is_not(None))
+        )
+        for cond in self._scope(groups, self._build_conditions(groups)):
+            stmt = stmt.where(cond)
+        if thread_filter in ("unread", "pinned"):
+            stmt = stmt.join(Thread, Thread.id == Message.thread_id).where(
+                Thread.unread_count > 0 if thread_filter == "unread" else Thread.pinned.is_(True)
+            )
+        async with get_sessionmaker()() as session:
+            return int(await session.scalar(stmt) or 0)
+

 _REMOVAL_FOLDERS = ("trash", "junk")

@@ -245,3 +245,44 @@ async def test_negation(seeded: Ctx) -> None:
 async def test_combined_operators(seeded: Ctx) -> None:
    assert await _search(seeded, "from:dave has:attachment") == 1
    assert await _search(seeded, "from:bob has:attachment") == 0
+
+
+async def test_more_than_one_page_counts_and_pages(ctx: Ctx) -> None:
+    """Beyond the first page: the first page says how many conversations match
+    in all, the cursor brings the rest, and Unread/Pinned filter conversations."""
+    for i in range(7):
+        await _ingest(
+            ctx,
+            frm=f"Sender {i} <s{i}@example.org>",
+            subject=f"Quarterly report {i}",
+            body="numbers inside",
+            flags=frozenset({"\\Seen"}) if i % 2 else frozenset(),
+        )
+    await _ingest(ctx, frm="Other <o@example.org>", subject="Lunch", body="nothing")
+
+    first = (await ctx.client.get("/api/search", params={"q": "quarterly", "limit": 3})).json()
+    assert first["total"] == 7 and len(first["threads"]) == 3 and first["cursor"]
+    seen = {t["thread"]["id"] for t in first["threads"]}
+    cursor = first["cursor"]
+    while cursor:
+        page = (
+            await ctx.client.get(
+                "/api/search", params={"q": "quarterly", "limit": 3, "cursor": cursor}
+            )
+        ).json()
+        assert page["total"] is None  # counted once, on the first page
+        seen |= {t["thread"]["id"] for t in page["threads"]}
+        cursor = page["cursor"]
+    assert len(seen) == 7
+
+    unread = (
+        await ctx.client.get("/api/search", params={"q": "quarterly", "filter": "unread"})
+    ).json()
+    assert unread["total"] == 4 and len(unread["threads"]) == 4
+    assert all(t["thread"]["unread_count"] > 0 for t in unread["threads"])
+    pinned = (
+        await ctx.client.get("/api/search", params={"q": "quarterly", "filter": "pinned"})
+    ).json()
+    assert pinned["total"] == 0 and pinned["threads"] == []
+    bad = await ctx.client.get("/api/search", params={"q": "quarterly", "filter": "nope"})
+    assert bad.status_code == 422
@@ -599,7 +599,7 @@ export const en = {
    "Not available here: browsers allow offline access only when the app is opened over HTTPS.",
  "push.settingTitle": "Notifications",
  "push.settingHelp":
-    "Get a notification on this device when new mail arrives in an Inbox, even when Ultimo isn't open. Shows the sender and subject. Turned off when you sign out.",
+    "Get a notification on this device when new mail arrives in an Inbox, even when Ultimo isn't open.",
  "push.needsInstall":
    "On iPhone and iPad, add Ultimo to your Home Screen first (Share → Add to Home Screen), then turn this on from the app.",
  "push.blocked":
@@ -184,9 +184,10 @@ export const api = {
  thread: (id: string, sort: "asc" | "desc" = "asc", view = "unified") =>
    apiFetch<ThreadDetail>(threadPath(id, sort, view)),

-  search: (q: string, cursor?: string) => {
+  search: (q: string, cursor?: string | null, filter?: string) => {
    const params = new URLSearchParams({ q });
    if (cursor) params.set("cursor", cursor);
+    if (filter === "unread" || filter === "pinned") params.set("filter", filter);
    return apiFetch<SearchOut>(`/search?${params.toString()}`);
  },

@@ -159,6 +159,8 @@ export interface SearchThreadHit {
 export interface SearchOut {
  threads: SearchThreadHit[];
  cursor: string | null;
+  /** Conversations matching in all (first page; null where not counted). */
+  total?: number | null;
 }

 // --- M1: account wizard ------------------------------------------------------
@@ -107,6 +107,9 @@ export function connectWs(queryClient: QueryClient): () => void {
      if (typeof payload.thread_id === "string") {
        queryClient.invalidateQueries({ queryKey: ["thread", payload.thread_id] });
      }
+      // Search results too (a deleted conversation leaves them) — not on every
+      // backfill tick, which would refetch an open search over and over.
+      if (name !== "sync.progress") queryClient.invalidateQueries({ queryKey: ["search"] });
    }
    if (name === "flags.changed") {
      // Read/unread changes affect the Inbox unread badge.
@@ -1,5 +1,11 @@
-import { keepPreviousData, useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
-import { Fragment, useCallback, useEffect, useRef, useState } from "react";
+import {
+  keepPreviousData,
+  useInfiniteQuery,
+  useMutation,
+  useQuery,
+  useQueryClient,
+} from "@tanstack/react-query";
+import { Fragment, useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { Link, useMatch, useNavigate, useSearchParams } from "react-router-dom";

 import signetLogo from "../../icons/ultimo-logo-signet.svg";
@@ -42,6 +48,7 @@ import type { AccountOut, QuotaOut, SavedSearch, ThreadSummary } from "../lib/ty
 import { useReadingPane } from "../lib/readingPane";
 import { useIsMobile } from "../lib/useIsMobile";
 import { useKeyboardShortcuts } from "../lib/useKeyboardShortcuts";
+import { subscribeWs } from "../lib/ws";
 import { PluginSlot } from "../plugins/PluginSlot";

 function quotaLabel(q: QuotaOut): string {
@@ -57,6 +64,9 @@ const PAGE_SIZE = 50;
 // How long a row the list hid for a queued move waits for the move to land
 // before it is shown again.
 const HIDE_FALLBACK_MS = 30_000;
+// Search results load 50 matching messages at a time; this caps how many
+// pages an oldest-first search (which needs them all) fetches.
+const MAX_SEARCH_PAGES = 40;

 // Collapsed sidebar width: just wide enough for the icons.
 const COLLAPSED_W = 60;
@@ -271,11 +281,28 @@ export function MailLayout() {
  const effectiveQuery = searchScope
    ? `${searchScope} ${debouncedTerm}`.trim()
    : debouncedTerm;
-  const searchQuery = useQuery({
-    queryKey: ["search", effectiveQuery],
-    queryFn: () => api.search(effectiveQuery),
+  // Search comes in pages of matching messages (a conversation can span two);
+  // more are fetched as the list is paged through (effect below).
+  const searchFilter = filter === "unread" || filter === "pinned" ? filter : undefined;
+  const searchQuery = useInfiniteQuery({
+    queryKey: ["search", effectiveQuery, searchFilter],
+    queryFn: ({ pageParam }) => api.search(effectiveQuery, pageParam, searchFilter),
+    initialPageParam: null as string | null,
+    getNextPageParam: (last) => last.cursor,
    enabled: searching,
  });
+  const searchThreads = useMemo(() => {
+    const seen = new Set<string>();
+    const out: ThreadSummary[] = [];
+    for (const pg of searchQuery.data?.pages ?? []) {
+      for (const h of pg.threads) {
+        if (seen.has(h.thread.id)) continue;
+        seen.add(h.thread.id);
+        out.push(h.thread);
+      }
+    }
+    return out;
+  }, [searchQuery.data]);
  const savedSearchesQuery = useQuery({ queryKey: ["saved-searches"], queryFn: api.savedSearches });
  const settingsQuery = useQuery({ queryKey: ["settings"], queryFn: api.settings });
  const threadOrder: "asc" | "desc" = settingsQuery.data?.thread_order ?? "asc";
@@ -296,12 +323,15 @@ export function MailLayout() {
  let pageThreads: ThreadSummary[];
  let total: number;
  if (searching) {
-    const hits = (searchQuery.data?.threads ?? []).map((h) => h.thread);
-    const f = hits.filter((th) =>
-      filter === "unread" ? th.unread_count > 0 : filter === "pinned" ? th.pinned : true,
-    );
-    const ordered = listOrder === "newest" ? f : [...f].reverse();
-    total = ordered.length;
+    const shown = searchThreads.filter((th) => !hiddenIds.has(th.id));
+    const ordered = listOrder === "newest" ? shown : [...shown].reverse();
+    // The server's count of matching conversations; without one (or while
+    // still loading pages), what has been loaded.
+    const counted = searchQuery.data?.pages[0]?.total;
+    total =
+      counted != null
+        ? Math.max(ordered.length, counted - (searchThreads.length - shown.length))
+        : ordered.length;
    pageThreads = ordered.slice(page * PAGE_SIZE, page * PAGE_SIZE + PAGE_SIZE);
  } else {
    const all = threadsQuery.data?.threads ?? [];
@@ -310,6 +340,21 @@ export function MailLayout() {
  }
  const pageStart = page * PAGE_SIZE;
  const pageEnd = pageStart + pageThreads.length;
+  // Search: fetch further pages of results as far as the page shown needs (one
+  // conversation more, to know whether a next page exists) — or all of them
+  // for oldest-first, whose first page is the end of the results.
+  const {
+    hasNextPage: searchHasMore,
+    isFetchingNextPage: searchFetchingMore,
+    fetchNextPage: fetchMoreSearch,
+  } = searchQuery;
+  const searchLoaded = searchThreads.length;
+  const searchPages = searchQuery.data?.pages.length ?? 0;
+  useEffect(() => {
+    if (!searching || !searchHasMore || searchFetchingMore) return;
+    const wanted = listOrder === "newest" ? (page + 1) * PAGE_SIZE : Infinity;
+    if (searchLoaded <= wanted && searchPages < MAX_SEARCH_PAGES) void fetchMoreSearch();
+  }, [searching, searchHasMore, searchFetchingMore, fetchMoreSearch, searchLoaded, searchPages, page, listOrder]);
  // Reset to the first page whenever the list identity (not the page) changes.
  useEffect(() => setPage(0), [view, filter, debouncedTerm, listOrder]);

@@ -550,6 +595,28 @@ export function MailLayout() {
  // row is hidden straight away (it is leaving the folder's scope); in search it
  // may still match a global query, so it is greyed out instead. A failed
  // request puts the row back.
+  // Moves are queued on the server and land one by one (each announced as
+  // thread.updated). A hidden row comes back only when nothing has landed for
+  // a while — the move failed — not while a long batch is still going.
+  const lastMoveLanded = useRef(0);
+  useEffect(
+    () =>
+      subscribeWs((name) => {
+        if (name === "thread.updated") lastMoveLanded.current = Date.now();
+      }),
+    [],
+  );
+  const unhideIfStuck = (unmark: () => void) => {
+    const check = () => {
+      if (Date.now() - lastMoveLanded.current < HIDE_FALLBACK_MS) {
+        window.setTimeout(check, HIDE_FALLBACK_MS);
+      } else {
+        unmark();
+      }
+    };
+    window.setTimeout(check, HIDE_FALLBACK_MS);
+  };
+
  const listMove = async (id: string, action: (id: string) => Promise<unknown>) => {
    const mark = searching ? setDimmedIds : setHiddenIds;
    const unmark = () =>
@@ -573,7 +640,7 @@ export function MailLayout() {
    // The server only queues the move, so the refetch above can still carry the
    // row; the effect below un-hides it once a list arrives without it. Should
    // the move never land, the row comes back rather than staying lost.
-    if (!searching) window.setTimeout(unmark, HIDE_FALLBACK_MS);
+    if (!searching) unhideIfStuck(unmark);
  };
  // A row stays hidden until a fetched list no longer contains it.
  useEffect(() => {
@@ -586,6 +653,16 @@ export function MailLayout() {
      return next.size === prev.size ? prev : next;
    });
  }, [threadsQuery.data]);
+  // The same for search results (deleted conversations drop out of them).
+  useEffect(() => {
+    if (!searchQuery.data || !searching) return;
+    setHiddenIds((prev) => {
+      if (prev.size === 0) return prev;
+      const present = new Set(searchThreads.map((th) => th.id));
+      const next = new Set([...prev].filter((id) => present.has(id)));
+      return next.size === prev.size ? prev : next;
+    });
+  }, [searchQuery.data, searchThreads, searching]);
  const listArchive = (id: string) => listMove(id, api.archiveThread);
  const listTrash = (id: string) => listMove(id, api.trashThread);
  // Phone: a conversation swiped right goes to Trash — after a short Undo
@@ -648,26 +725,52 @@ export function MailLayout() {

  // Bulk actions run the given operation over every selected thread, then clear the
  // selection and refresh (navigating away if the open thread was among them).
-  const runBulk = async (action: (id: string) => Promise<unknown>) => {
+  //
+  // Like the row buttons, the result shows at once: conversations leaving the
+  // view (deleted; archived out of a folder) are hidden — in search too, which
+  // leaves out Trash — and ones still matching a search (archived) are greyed.
+  // The requests run behind; a failed one puts its row back.
+  const runBulk = async (
+    action: (id: string) => Promise<unknown>,
+    effect: "hide" | "dim" | null = null,
+  ) => {
    const ids = [...checkedIds];
+    const mark = effect === "hide" ? setHiddenIds : effect === "dim" ? setDimmedIds : null;
+    const unmark = (gone: string[]) =>
+      mark?.((prev) => {
+        const next = new Set(prev);
+        for (const id of gone) next.delete(id);
+        return next;
+      });
+    mark?.((prev) => new Set([...prev, ...ids]));
+    setCheckedIds(new Set());
+    // The conversations are gone from the list: the reading pane goes back to
+    // "Select a conversation" rather than to whatever was open before selecting.
+    if (threadId && (effect !== null || ids.includes(threadId))) navigate("/mail");
+    const failed: string[] = [];
    for (const id of ids) {
      try {
        await action(id);
      } catch {
-        /* keep going */
+        failed.push(id);
      }
    }
-    setCheckedIds(new Set());
+    if (failed.length) unmark(failed);
    queryClient.invalidateQueries({ queryKey: ["threads"] });
+    queryClient.invalidateQueries({ queryKey: ["search"] });
    queryClient.invalidateQueries({ queryKey: ["quota"] });
    queryClient.invalidateQueries({ queryKey: ["folder-counts"] });
-    if (threadId && ids.includes(threadId)) navigate("/mail");
+    // The server only queues the moves, so a refetch can still carry the rows;
+    // they stay hidden until a list arrives without them — or come back should
+    // the moves never land.
+    if (effect === "hide") unhideIfStuck(() => unmark(ids));
  };
-  const bulkArchive = () => runBulk((id) => api.archiveThread(id));
+  const leaving = searching ? "dim" : "hide"; // archived: gone from a folder, still found by a search
+  const bulkArchive = () => runBulk((id) => api.archiveThread(id), leaving);
  const bulkPin = () => runBulk((id) => api.threadFlags(id, ["\\Flagged"], []));
-  const bulkTrash = () => runBulk((id) => api.trashThread(id));
-  const bulkRestore = () => runBulk((id) => api.restoreThread(id));
-  const bulkDeletePermanent = () => runBulk((id) => api.deleteThreadPermanent(id));
+  const bulkTrash = () => runBulk((id) => api.trashThread(id), "hide");
+  const bulkRestore = () => runBulk((id) => api.restoreThread(id), "hide");
+  const bulkDeletePermanent = () => runBulk((id) => api.deleteThreadPermanent(id), "hide");

  // Bulk-action buttons over the selected conversations: in the reading pane on
  // the desktop, above the list on a phone.