shared hashes: separate background job with visible state

- compute duplicate-hash statistics in their own job (startup, hourly,
  on demand) so they no longer block the table stats
- single-pass query over DISTINCT (hash, addon_id), 15 min timeout
- record errors and panics; never stay stuck in a computing state
- whitelist page polls while computing, shows errors and a Recompute
  button; tab is linkable via ?tab=shared

web/src/lib/types.ts

@@ -60,7 +60,6 @@ export interface HeavyStats {
   dbDriver: string;
   dbSizeBytes: number;
   tables: TableInfo[];
-  duplicatesAsOf: number;
   error?: string;
 }
 
@@ -213,6 +212,19 @@ export interface DupHash {
   whitelisted: boolean;
 }
 
+/** State and result of the background job that finds shared hashes. */
+export interface DupResponse {
+  running: boolean;
+  startedAt: number;
+  asOf: number;
+  tookMs: number;
+  error: string;
+  errorAt: number;
+  distinctHashes: number;
+  sharedHashes: number;
+  items: DupHash[];
+}
+
 export interface ServiceInstance {
   id: number;
   service: string;

web/src/pages/Whitelist.tsx

@@ -1,10 +1,10 @@
 import { useState } from "react";
 import { useMutation, useQuery, useQueryClient, keepPreviousData } from "@tanstack/react-query";
-import { useNavigate } from "react-router";
-import { Plus, Search, ShieldCheck, Trash2 } from "lucide-react";
+import { useNavigate, useSearchParams } from "react-router";
+import { Plus, RefreshCw, Search, ShieldCheck, Trash2 } from "lucide-react";
 import { api, qs } from "../lib/api";
-import type { DupHash, Paged, WhitelistRow } from "../lib/types";
-import { int, relative, shortHash } from "../lib/format";
+import type { DupResponse, Paged, WhitelistRow } from "../lib/types";
+import { int, ms, relative, seconds, shortHash } from "../lib/format";
 import {
   Badge,
   Button,
@@ -18,6 +18,7 @@ import {
   PageHeader,
   Pager,
   Segmented,
+  Spinner,
   Table,
   Td,
   Textarea,
@@ -33,7 +34,9 @@ interface HashUsage {
 }
 
 export default function Whitelist() {
-  const [tab, setTab] = useState<"list" | "candidates">("list");
+  const [sp, setSp] = useSearchParams();
+  const tab: "list" | "candidates" = sp.get("tab") === "shared" ? "candidates" : "list";
+  const setTab = (t: "list" | "candidates") => setSp(t === "candidates" ? { tab: "shared" } : {}, { replace: true });
   const [inspect, setInspect] = useState<string | null>(null);
   return (
     <div className="space-y-4">
@@ -190,7 +193,20 @@ function Candidates({ onInspect }: { onInspect: (h: string) => void }) {
   const [selected, setSelected] = useState<Set<string>>(new Set());
   const dups = useQuery({
     queryKey: ["dups", hide],
-    queryFn: () => api<{ asOf: number; items: DupHash[]; pending?: boolean }>(`/overview/duplicates${hide ? "?hideWhitelisted=1" : ""}`),
+    queryFn: () => api<DupResponse>(`/overview/duplicates${hide ? "?hideWhitelisted=1" : ""}`),
+    // poll while the background job is running or has not produced anything yet
+    refetchInterval: (q) => {
+      const d = q.state.data;
+      return !d || d.running || (d.asOf === 0 && !d.error) ? 5000 : false;
+    },
+  });
+  const recompute = useMutation({
+    mutationFn: () => api("/overview/duplicates/refresh", { method: "POST" }),
+    onSuccess: () => {
+      toast("Recomputing shared hashes", "neutral");
+      setTimeout(() => qc.invalidateQueries({ queryKey: ["dups"] }), 500);
+    },
+    onError: (e) => toast((e as Error).message, "critical"),
   });
   const add = useMutation({
     mutationFn: (hashes: string[]) => api<{ added: number }>("/whitelist", { method: "POST", body: { hashes, note: "from most shared hashes" } }),
@@ -205,25 +221,54 @@ function Candidates({ onInspect }: { onInspect: (h: string) => void }) {
 
   if (dups.isLoading) return <Loading />;
   if (dups.error) return <ErrorBox error={dups.error} />;
-  const items = dups.data!.items;
+  const d = dups.data!;
+  const items = d.items;
+
+  let subtitle: string;
+  if (d.running) subtitle = `Computing… started ${relative(d.startedAt)}`;
+  else if (d.asOf)
+    subtitle = `Computed ${relative(d.asOf)} in ${d.tookMs < 60000 ? ms(d.tookMs) : seconds(d.tookMs / 1000)} · ${int(d.sharedHashes)} shared of ${int(d.distinctHashes)} distinct hashes · refreshed hourly`;
+  else if (d.error) subtitle = "Not computed yet";
+  else subtitle = "Waiting for the first run…";
 
   return (
     <Card
       title="Hashes shared by the most addons"
-      subtitle={dups.data!.pending ? "Statistics are being computed…" : `Computed ${relative(dups.data!.asOf)} · refreshed hourly`}
+      subtitle={
+        <span className="inline-flex items-center gap-1.5">
+          {d.running && <Spinner className="size-3" />}
+          {subtitle}
+        </span>
+      }
       actions={
         <>
           <label className="flex items-center gap-1.5 text-xs text-ink-2">
             <Checkbox checked={hide} onChange={setHide} label="Hide whitelisted" /> Hide whitelisted
           </label>
+          <Button size="sm" icon={<RefreshCw className="size-3.5" />} disabled={d.running} loading={recompute.isPending} onClick={() => recompute.mutate()}>
+            Recompute
+          </Button>
           <Button size="sm" variant="primary" icon={<ShieldCheck className="size-3.5" />} disabled={selected.size === 0} loading={add.isPending} onClick={() => add.mutate([...selected])}>
             Whitelist {selected.size || ""}
           </Button>
         </>
       }
     >
+      {d.error && (
+        <div className="mb-3">
+          <ErrorBox error={`Last run failed ${relative(d.errorAt)}: ${d.error}${d.asOf ? " — showing the previous result" : ""}`} />
+        </div>
+      )}
       {items.length === 0 ? (
-        <Empty>No shared hashes</Empty>
+        <Empty>
+          {d.asOf
+            ? hide && d.sharedHashes > 0
+              ? "All of the most shared hashes are whitelisted"
+              : "No hashes are shared between addons"
+            : d.running || !d.error
+              ? "Computing shared hashes — this scans every indexed file and can take a few minutes"
+              : "No result yet"}
+        </Empty>
       ) : (
         <Table>
           <thead>