- embedded React admin UI at /admin/ui (go:embed), login with ADMIN_SECRET via signed HttpOnly session cookie, X-ADMIN-KEY still works for scripts - admin API: overview stats, indexing queue, addon management (priority, reindex, delete/restore/purge, bulk), whitelist CRUD, duplicate hashes, traffic, metrics explorer, services, events/audit log, system info - built-in metrics: request latency per route, visitors, top addons and searches, indexer/scraper counters; minute buckets in the DB with hourly rollups and retention; optional Prometheus /metrics (METRICS_TOKEN) - POST /back/telemetry for indexers/scrapers (metrics, events, heartbeat) and a stdlib-only Go client in telemetry/ - auth hardening: empty secrets no longer allow access, constant-time compare, CORS not applied to /admin and /back, TRUSTED_PROXIES setting - Dockerfile builds the UI and no longer bakes config.yaml into the image; CI builds the UI, runs tests and builds arm64 with CGO
This commit is contained in:
ilbinek committed
1 parent
0ef27b0072
commit
56d9e10621
79 files changed
+13146
-118
No files matched your search
@@ -0,0 +1,208 @@
|
||||
import { useMutation, useQuery, useQueryClient, keepPreviousData } from "@tanstack/react-query";
|
||||
import { useNavigate } from "react-router";
|
||||
import { RotateCcw, Star, StarOff } from "lucide-react";
|
||||
import { api } from "../lib/api";
|
||||
import { useMetric, useRangeStep } from "../lib/queries";
|
||||
import { useRefetch } from "../lib/settings";
|
||||
import type { QueueData, QueueRow } from "../lib/types";
|
||||
import { bytes, compact, duration, int, num, relative, seconds } from "../lib/format";
|
||||
import { Badge, Button, Card, ErrorBox, Loading, PageHeader, StatTile, Table, Td, Th, useFeedback } from "../components/ui";
|
||||
import { fromQuery, TimeChart } from "../components/charts/TimeChart";
|
||||
|
||||
export default function Queue() {
|
||||
const refetchInterval = useRefetch();
|
||||
const { step } = useRangeStep();
|
||||
const qc = useQueryClient();
|
||||
const { toast, confirm } = useFeedback();
|
||||
const navigate = useNavigate();
|
||||
const q = useQuery({ queryKey: ["queue"], queryFn: () => api<QueueData>("/queue"), refetchInterval: refetchInterval || 15000, placeholderData: keepPreviousData });
|
||||
|
||||
const depth = useMetric({ name: "queue_depth", agg: "last", groupBy: ["kind"], step });
|
||||
const claims = useMetric({ name: "indexer_claims_total", agg: "sum", groupBy: ["queue"], step });
|
||||
const jobDuration = useMetric({ name: "indexer_job_duration_seconds", agg: "p50", step });
|
||||
|
||||
const resetStuck = useMutation({
|
||||
mutationFn: () => api<{ reset: number }>("/queue/reset-stuck", { method: "POST", body: {} }),
|
||||
onSuccess: (r) => {
|
||||
toast(`Released ${r.reset} timed-out claim${r.reset === 1 ? "" : "s"}`);
|
||||
qc.invalidateQueries({ queryKey: ["queue"] });
|
||||
},
|
||||
onError: (e) => toast(String((e as Error).message), "critical"),
|
||||
});
|
||||
|
||||
const action = useMutation({
|
||||
mutationFn: ({ id, act, value }: { id: string; act: string; value?: boolean }) =>
|
||||
api(`/addons/${encodeURIComponent(id)}/${act}`, { method: "POST", body: { value: value ?? true } }),
|
||||
onSuccess: () => {
|
||||
toast("Updated");
|
||||
qc.invalidateQueries({ queryKey: ["queue"] });
|
||||
},
|
||||
onError: (e) => toast(String((e as Error).message), "critical"),
|
||||
});
|
||||
|
||||
if (q.isLoading) return <Loading />;
|
||||
if (q.error) return <ErrorBox error={q.error} />;
|
||||
const d = q.data!;
|
||||
const stuck = d.indexing.filter((r) => r.stuck).length;
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<PageHeader
|
||||
title="Indexing queue"
|
||||
description={<>Jobs are claimed priority-first, then by oldest update · claims time out after {d.timeoutMinutes} min · updated {relative(d.asOf)}</>}
|
||||
actions={
|
||||
<Button
|
||||
icon={<RotateCcw className="size-3.5" />}
|
||||
loading={resetStuck.isPending}
|
||||
onClick={async () => {
|
||||
if (await confirm({ title: "Release timed-out claims?", body: `Addons claimed more than ${d.timeoutMinutes} minutes ago go back to the queue.` }))
|
||||
resetStuck.mutate();
|
||||
}}
|
||||
>
|
||||
Release stuck claims
|
||||
</Button>
|
||||
}
|
||||
/>
|
||||
|
||||
<div className="grid grid-cols-1 gap-3 sm:grid-cols-2 xl:grid-cols-4">
|
||||
<StatTile label="Waiting" value={int(d.counts.queued)} sub={`${int(d.counts.priority)} priority · ${int(d.blockedQueued)} blocked (never claimed)`} />
|
||||
<StatTile label="In progress" value={int(d.counts.indexing)} sub={stuck ? `${stuck} past the timeout` : "none stuck"} />
|
||||
<StatTile
|
||||
label="Throughput"
|
||||
value={`${num(d.throughput.perHour6h, 1)}/h`}
|
||||
sub={`${num(d.throughput.perHour1h, 0)} last hour · ${int(d.throughput.indexed24h)} in 24h`}
|
||||
/>
|
||||
<StatTile
|
||||
label="Queue drains in"
|
||||
value={d.throughput.etaHours === null ? "—" : duration(d.throughput.etaHours * 3600)}
|
||||
sub={d.throughput.duration24h ? `job p50 ${duration(d.throughput.duration24h.p50)} · p95 ${duration(d.throughput.duration24h.p95)}` : "at the 6h average rate"}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-1 gap-4 xl:grid-cols-3">
|
||||
<Card title="Queue depth">
|
||||
<TimeChart data={fromQuery(depth.data, (l) => (l.kind === "priority" ? "Priority" : "All queued"), 1, (l) => (l.kind === "priority" ? 2 : 1))} />
|
||||
</Card>
|
||||
<Card title="Claims per interval" subtitle="What indexers got when asking for work">
|
||||
<TimeChart data={fromQuery(claims.data, (l) => l.queue)} />
|
||||
</Card>
|
||||
<Card title="Job duration p50" subtitle="Claim to submitted result">
|
||||
<TimeChart data={fromQuery(jobDuration.data, () => "p50")} format={seconds} />
|
||||
</Card>
|
||||
</div>
|
||||
|
||||
<QueueTable
|
||||
title={`In progress (${d.indexing.length})`}
|
||||
rows={d.indexing}
|
||||
empty="No addon is being indexed right now"
|
||||
extra={(r) => (
|
||||
<>
|
||||
<Td className="tnum whitespace-nowrap">{duration(r.elapsedSec)}</Td>
|
||||
<Td>{r.stuck ? <Badge tone="critical">past timeout</Badge> : <Badge tone="good">running</Badge>}</Td>
|
||||
</>
|
||||
)}
|
||||
extraHead={
|
||||
<>
|
||||
<Th>Elapsed</Th>
|
||||
<Th>State</Th>
|
||||
</>
|
||||
}
|
||||
actions={(r) => (
|
||||
<Button size="sm" variant="ghost" onClick={() => action.mutate({ id: r.id, act: "reset-indexing" })}>
|
||||
Release
|
||||
</Button>
|
||||
)}
|
||||
onOpen={(id) => navigate(`/addons/${id}`)}
|
||||
/>
|
||||
|
||||
<QueueTable
|
||||
title={`Priority (${d.counts.priority})`}
|
||||
rows={d.priority}
|
||||
empty="No priority jobs. Mark addons as priority from the addon list."
|
||||
actions={(r) => (
|
||||
<Button size="sm" variant="ghost" icon={<StarOff className="size-3.5" />} onClick={() => action.mutate({ id: r.id, act: "priority", value: false })}>
|
||||
Unprioritize
|
||||
</Button>
|
||||
)}
|
||||
onOpen={(id) => navigate(`/addons/${id}`)}
|
||||
/>
|
||||
|
||||
<QueueTable
|
||||
title="Next up"
|
||||
subtitle="The next jobs in claim order (an indexer may skip addons above its size limit)"
|
||||
rows={d.next}
|
||||
empty="The queue is empty"
|
||||
actions={(r) =>
|
||||
r.priorityIndexing ? null : (
|
||||
<Button size="sm" variant="ghost" icon={<Star className="size-3.5" />} onClick={() => action.mutate({ id: r.id, act: "priority", value: true })}>
|
||||
Prioritize
|
||||
</Button>
|
||||
)
|
||||
}
|
||||
onOpen={(id) => navigate(`/addons/${id}`)}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function QueueTable({
|
||||
title,
|
||||
subtitle,
|
||||
rows,
|
||||
empty,
|
||||
extra,
|
||||
extraHead,
|
||||
actions,
|
||||
onOpen,
|
||||
}: {
|
||||
title: string;
|
||||
subtitle?: string;
|
||||
rows: QueueRow[];
|
||||
empty: string;
|
||||
extra?: (r: QueueRow) => React.ReactNode;
|
||||
extraHead?: React.ReactNode;
|
||||
actions?: (r: QueueRow) => React.ReactNode;
|
||||
onOpen: (id: string) => void;
|
||||
}) {
|
||||
return (
|
||||
<Card title={title} subtitle={subtitle}>
|
||||
{rows.length === 0 ? (
|
||||
<div className="py-4 text-sm text-muted">{empty}</div>
|
||||
) : (
|
||||
<Table>
|
||||
<thead>
|
||||
<tr>
|
||||
<Th>Addon</Th>
|
||||
<Th className="text-right">Size</Th>
|
||||
<Th className="text-right">Subscribers</Th>
|
||||
<Th>Waiting since</Th>
|
||||
{extraHead}
|
||||
<Th />
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{rows.map((r) => (
|
||||
<tr key={r.id} className="cursor-pointer hover:bg-surface-2/40" onClick={() => onOpen(r.id)}>
|
||||
<Td className="text-ink">
|
||||
<div className="flex items-center gap-2">
|
||||
{r.name || r.id}
|
||||
{r.priorityIndexing && <Badge tone="accent">priority</Badge>}
|
||||
{r.blocked && <Badge tone="critical">blocked</Badge>}
|
||||
</div>
|
||||
<div className="font-mono text-[11px] text-muted">{r.id}</div>
|
||||
</Td>
|
||||
<Td className="tnum text-right">{bytes(r.currentVersionSize)}</Td>
|
||||
<Td className="tnum text-right">{compact(r.subscriberCount)}</Td>
|
||||
<Td className="whitespace-nowrap">{relative(r.updatedAt)}</Td>
|
||||
{extra?.(r)}
|
||||
<Td className="text-right" >
|
||||
<span onClick={(e) => e.stopPropagation()}>{actions?.(r)}</span>
|
||||
</Td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</Table>
|
||||
)}
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
Reference in new issue
Block a user