admin dashboard, metrics and telemetry

- embedded React admin UI at /admin/ui (go:embed), login with ADMIN_SECRET
  via signed HttpOnly session cookie, X-ADMIN-KEY still works for scripts
- admin API: overview stats, indexing queue, addon management (priority,
  reindex, delete/restore/purge, bulk), whitelist CRUD, duplicate hashes,
  traffic, metrics explorer, services, events/audit log, system info
- built-in metrics: request latency per route, visitors, top addons and
  searches, indexer/scraper counters; minute buckets in the DB with hourly
  rollups and retention; optional Prometheus /metrics (METRICS_TOKEN)
- POST /back/telemetry for indexers/scrapers (metrics, events, heartbeat)
  and a stdlib-only Go client in telemetry/
- auth hardening: empty secrets no longer allow access, constant-time
  compare, CORS not applied to /admin and /back, TRUSTED_PROXIES setting
- Dockerfile builds the UI and no longer bakes config.yaml into the image;
  CI builds the UI, runs tests and builds arm64 with CGO

.dockerignore

@@ -9,3 +9,5 @@ api
 .env
 .env.*
 README.md
+web/node_modules
+web/dist

.env.dokploy.example

@@ -26,3 +26,26 @@ POSTGRES_USER=postgres
 POSTGRES_PASSWORD=change-me-password
 POSTGRES_DB_NAME=dev
 POSTGRES_SSLMODE=disable
+
+# --- Admin dashboard (/admin/ui/) ---------------------------------------------
+# Log in with ADMIN_SECRET (use 16+ random characters). Rotating it signs
+# everyone out. The X-ADMIN-KEY header keeps working for scripts.
+ADMIN_SESSION_TTL_HOURS=168
+# auto = Secure cookie when the request arrived over HTTPS (X-Forwarded-Proto)
+ADMIN_COOKIE_SECURE=auto
+# Proxies whose X-Forwarded-For is trusted (comma separated IPs/CIDRs), e.g. the
+# Traefik network. Without it, client IPs (visitor counts, login throttling)
+# can be spoofed.
+TRUSTED_PROXIES=10.0.0.0/8,172.16.0.0/12
+
+# --- Metrics & telemetry ---------------------------------------------------------
+METRICS_ENABLED=true
+# Enables GET /metrics (Prometheus) with "Authorization: Bearer <token>".
+# Leave empty to disable the endpoint.
+METRICS_TOKEN=
+METRICS_MINUTE_RETENTION_DAYS=7
+METRICS_HOUR_RETENTION_DAYS=180
+EVENTS_RETENTION_DAYS=30
+TOP_ITEMS_RETENTION_DAYS=90
+# Indexing claims older than this go back to the queue
+INDEX_TIMEOUT_MINUTES=30

.gitea/workflows/build.yaml

@@ -13,18 +13,40 @@ jobs:
       - name: Checkout Code
         uses: actions/checkout@v4
 
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+          cache: npm
+          cache-dependency-path: web/package-lock.json
+
+      # The admin dashboard is embedded into the binary, so build it first
+      - name: Build admin UI
+        run: |
+          npm ci --prefix web
+          npm run build --prefix web
+
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: '1.23'
+          go-version: '1.25'
+
+      - name: Vet and test
+        run: |
+          go vet ./...
+          go test ./...
+
+      # SQLite needs CGO, so cross-compile ARM64 with the aarch64 C toolchain
+      - name: Install ARM64 C toolchain
+        run: sudo apt-get update && sudo apt-get install -y gcc-aarch64-linux-gnu
 
       # Build ARM64
       - name: Build ARM64
-        run: GOOS=linux GOARCH=arm64 go build -v -o crawler_main-linux-arm64 .
+        run: CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc GOOS=linux GOARCH=arm64 go build -v -ldflags "-s -w -X main.buildCommit=${{ github.sha }}" -o crawler_main-linux-arm64 .
 
       # Build AMD64
       - name: Build AMD64
-        run: GOOS=linux GOARCH=amd64 go build -v -o crawler_main-linux-amd64 .
+        run: CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -v -ldflags "-s -w -X main.buildCommit=${{ github.sha }}" -o crawler_main-linux-amd64 .
 
       # Create organized archives for download
       - name: Bundle Artifacts
@@ -52,4 +74,4 @@ jobs:
         with:
           name: linux-amd64
           path: release-amd64.tar.gz
-          if-no-files-found: error
+          if-no-files-found: error

.gitignore

@@ -5,3 +5,8 @@ crawler.db
 .env
 .env.*
 !.env.dokploy.example
+
+# admin UI
+web/node_modules/
+web/dist/*
+!web/dist/.gitkeep

Dockerfile

@@ -1,3 +1,14 @@
+# Admin dashboard (React) - built once, embedded into the Go binary via go:embed.
+FROM --platform=$BUILDPLATFORM node:22-bookworm-slim AS ui
+
+WORKDIR /web
+
+COPY web/package.json web/package-lock.json ./
+RUN npm ci
+
+COPY web/ ./
+RUN npm run build
+
 FROM golang:1.25-bookworm AS builder
 
 WORKDIR /app
@@ -11,7 +22,10 @@ COPY go.mod go.sum ./
 RUN go mod download
 
 COPY . .
-RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o /out/reforger_crawler_main .
+COPY --from=ui /web/dist ./web/dist
+
+ARG GIT_SHA=""
+RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -ldflags "-s -w -X main.buildCommit=${GIT_SHA}" -o /out/reforger_crawler_main .
 
 FROM debian:bookworm-slim AS runtime
 
@@ -22,7 +36,9 @@ RUN apt-get update \
     && rm -rf /var/lib/apt/lists/*
 
 COPY --from=builder /out/reforger_crawler_main /app/reforger_crawler_main
-COPY config.yaml /app/config.yaml
+
+# Configuration comes from environment variables (see .env.dokploy.example).
+# Mount a config.yaml into /app if you prefer a file.
 
 EXPOSE 8083
 

admin/addons.go

@@ -0,0 +1,510 @@
+package admin
+
+import (
+	"errors"
+	"net/http"
+	"strings"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/controllers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
+	"github.com/gin-gonic/gin"
+	"gorm.io/gorm"
+)
+
+// addonRow is the admin view of an addon (no files).
+type addonRow struct {
+	ID                   string     `json:"id"`
+	Name                 string     `json:"name"`
+	Type                 string     `json:"type"`
+	Author               string     `json:"author"`
+	Preview              string     `json:"preview"`
+	SubscriberCount      int64      `json:"subscriberCount"`
+	CurrentVersionNumber string     `json:"currentVersionNumber"`
+	CurrentVersionSize   int64      `json:"currentVersionSize"`
+	Unlisted             bool       `json:"unlisted"`
+	Private              bool       `json:"private"`
+	Blocked              bool       `json:"blocked"`
+	ToBeIndexed          bool       `json:"toBeIndexed"`
+	IsBeingIndexed       bool       `json:"isBeingIndexed"`
+	PriorityIndexing     bool       `json:"priorityIndexing"`
+	Recheck              bool       `json:"recheck"`
+	IndexStartTime       *time.Time `json:"indexStartTime"`
+	LastIndexedAt        *time.Time `json:"lastIndexedAt"`
+	CreatedAt            time.Time  `json:"createdAt"`
+	UpdatedAt            time.Time  `json:"updatedAt"`
+	DeletedAt            *time.Time `json:"deletedAt"`
+}
+
+const addonColumns = `id, name, type, author, preview, subscriber_count, current_version_number,
+	current_version_size, unlisted, private, blocked, to_be_indexed, is_being_indexed, priority_indexing,
+	recheck, index_start_time, last_indexed_at, created_at, updated_at, deleted_at`
+
+var addonSorts = map[string]string{
+	"name":        "name",
+	"subscribers": "subscriber_count",
+	"size":        "current_version_size",
+	"created":     "created_at",
+	"updated":     "updated_at",
+	"indexed":     "last_indexed_at",
+	"deleted":     "deleted_at",
+}
+
+func listAddons(c *gin.Context) {
+	limit := queryInt(c, "limit", 50, 1, 200)
+	offset := queryInt(c, "offset", 0, 0, 1<<30)
+	status := c.DefaultQuery("status", "all")
+
+	q := initializers.DB.Model(&models.Addon{})
+	switch status {
+	case "all":
+	case "indexed":
+		q = q.Where("to_be_indexed = ?", false)
+	case "queued":
+		q = q.Where("to_be_indexed = ? AND is_being_indexed = ?", true, false)
+	case "indexing":
+		q = q.Where("is_being_indexed = ?", true)
+	case "priority":
+		q = q.Where("priority_indexing = ?", true)
+	case "recheck":
+		q = q.Where("recheck = ?", true)
+	case "blocked":
+		q = q.Where("blocked = ?", true)
+	case "unlisted":
+		q = q.Where("unlisted = ?", true)
+	case "private":
+		q = q.Where("private = ?", true)
+	case "deleted":
+		q = q.Unscoped().Where("deleted_at IS NOT NULL")
+	default:
+		fail(c, http.StatusBadRequest, "unknown status")
+		return
+	}
+	if t := c.Query("type"); t != "" {
+		q = q.Where("type = ?", t)
+	}
+	if s := strings.TrimSpace(c.Query("q")); s != "" {
+		like := "%" + likeEscape(strings.ToLower(s)) + "%"
+		q = q.Where(`id = ? OR LOWER(name) LIKE ? ESCAPE '\' OR LOWER(author) LIKE ? ESCAPE '\'`, s, like, like)
+	}
+
+	var total int64
+	if err := q.Session(&gorm.Session{}).Count(&total).Error; err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+
+	sortCol, ok := addonSorts[c.DefaultQuery("sort", "subscribers")]
+	if !ok {
+		sortCol = "subscriber_count"
+	}
+	dir := "DESC"
+	if c.Query("order") == "asc" {
+		dir = "ASC"
+	}
+	nulls := ""
+	if isPostgres() && dir == "DESC" {
+		nulls = " NULLS LAST"
+	}
+
+	rows := []addonRow{}
+	err := q.Select(addonColumns).Order(sortCol + " " + dir + nulls + ", id ASC").Limit(limit).Offset(offset).Scan(&rows).Error
+	if err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+	c.JSON(http.StatusOK, gin.H{"items": rows, "total": total, "limit": limit, "offset": offset})
+}
+
+type sharedAddon struct {
+	AddonID     string `json:"addonId"`
+	Name        string `json:"name"`
+	SharedFiles int64  `json:"sharedFiles"`
+}
+
+func addonDetail(c *gin.Context) {
+	id := c.Param("id")
+	var row addonRow
+	res := initializers.DB.Unscoped().Model(&models.Addon{}).Select(addonColumns).Where("id = ?", id).Limit(1).Scan(&row)
+	if res.Error != nil {
+		fail(c, http.StatusInternalServerError, res.Error.Error())
+		return
+	}
+	if res.RowsAffected == 0 {
+		fail(c, http.StatusNotFound, "addon not found")
+		return
+	}
+	var extra struct {
+		Summary          string
+		CurrentVersionID int64
+	}
+	initializers.DB.Unscoped().Model(&models.Addon{}).Select("summary, current_version_id").Where("id = ?", id).Limit(1).Scan(&extra)
+
+	var fileCount, deletedFiles int64
+	initializers.DB.Model(&models.AddonFile{}).Where("addon_id = ?", id).Count(&fileCount)
+	initializers.DB.Unscoped().Model(&models.AddonFile{}).Where("addon_id = ? AND deleted_at IS NOT NULL", id).Count(&deletedFiles)
+
+	deps, depsTotal, _ := controllers.QueryRelatedAddons(id, "ad.dependency_id", "ad.addon_id", 50, 0)
+	dependents, dependentsTotal, _ := controllers.QueryRelatedAddons(id, "ad.addon_id", "ad.dependency_id", 50, 0)
+
+	// declared dependencies that are not (or no longer) in the addons table
+	var missingDeps []struct {
+		DependencyID   string `json:"id"`
+		DependencyName string `json:"name"`
+		Version        string `json:"version"`
+	}
+	initializers.DB.Raw(`SELECT ad.dependency_id, ad.dependency_name, ad.version FROM addon_dependencies ad
+		LEFT JOIN addons a ON a.id = ad.dependency_id AND a.deleted_at IS NULL
+		WHERE ad.addon_id = ? AND a.id IS NULL LIMIT 50`, id).Scan(&missingDeps)
+
+	shared := []sharedAddon{}
+	err := initializers.DB.Raw(`SELECT f.addon_id AS addon_id, COALESCE(MAX(a.name), '') AS name, COUNT(*) AS shared_files
+		FROM addon_files f
+		LEFT JOIN addons a ON a.id = f.addon_id
+		WHERE f.deleted_at IS NULL AND f.addon_id <> ?
+		  AND f.hash IN (SELECT s.hash FROM addon_files s WHERE s.addon_id = ? AND s.deleted_at IS NULL)
+		  AND NOT EXISTS (SELECT 1 FROM whitelisted_hashes w WHERE w.hash = f.hash AND w.deleted_at IS NULL)
+		GROUP BY f.addon_id ORDER BY shared_files DESC LIMIT 25`, id, id).Scan(&shared).Error
+	if err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"addon":            row,
+		"summary":          extra.Summary,
+		"currentVersionId": extra.CurrentVersionID,
+		"fileCount":        fileCount,
+		"deletedFileRows":  deletedFiles,
+		"dependencies":     gin.H{"total": depsTotal, "items": deps, "missing": missingDeps},
+		"dependents":       gin.H{"total": dependentsTotal, "items": dependents},
+		"duplicates":       shared,
+	})
+}
+
+type fileRow struct {
+	ID          uint   `json:"id"`
+	Path        string `json:"path"`
+	Hash        string `json:"hash"`
+	Version     string `json:"version"`
+	Whitelisted bool   `json:"whitelisted"`
+	OtherAddons int64  `json:"otherAddons"`
+}
+
+func addonFiles(c *gin.Context) {
+	id := c.Param("id")
+	limit := queryInt(c, "limit", 100, 1, 500)
+	offset := queryInt(c, "offset", 0, 0, 1<<30)
+	q := initializers.DB.Model(&models.AddonFile{}).Where("addon_id = ?", id)
+	if s := strings.TrimSpace(c.Query("q")); s != "" {
+		like := "%" + likeEscape(strings.ToLower(s)) + "%"
+		q = q.Where(`LOWER(path) LIKE ? ESCAPE '\' OR hash = ?`, like, strings.ToLower(s))
+	}
+	var total int64
+	q.Session(&gorm.Session{}).Count(&total)
+
+	rows := []fileRow{}
+	if err := q.Select("id, path, hash, version").Order("path ASC").Limit(limit).Offset(offset).Scan(&rows).Error; err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+
+	if len(rows) > 0 && c.Query("shared") != "0" {
+		hashes := make([]string, 0, len(rows))
+		for _, r := range rows {
+			hashes = append(hashes, r.Hash)
+		}
+		var wl []string
+		initializers.DB.Model(&models.WhitelistedHash{}).Where("hash IN ?", hashes).Pluck("hash", &wl)
+		wlSet := map[string]bool{}
+		for _, h := range wl {
+			wlSet[h] = true
+		}
+		var counts []struct {
+			Hash   string
+			Addons int64
+		}
+		initializers.DB.Raw(`SELECT hash, COUNT(DISTINCT addon_id) AS addons FROM addon_files
+			WHERE deleted_at IS NULL AND hash IN ? GROUP BY hash`, hashes).Scan(&counts)
+		cnt := map[string]int64{}
+		for _, x := range counts {
+			cnt[x.Hash] = x.Addons
+		}
+		for i := range rows {
+			rows[i].Whitelisted = wlSet[rows[i].Hash]
+			rows[i].OtherAddons = max(0, cnt[rows[i].Hash]-1)
+		}
+	}
+	c.JSON(http.StatusOK, gin.H{"items": rows, "total": total, "limit": limit, "offset": offset})
+}
+
+var errNotFound = errors.New("addon not found")
+
+// applyAction changes queue flags of the given addons. UpdateColumns is used
+// so updated_at (the queue order) is left alone.
+func applyAction(tx *gorm.DB, ids []string, action string, value bool) (int64, error) {
+	base := tx.Model(&models.Addon{}).Where("id IN ?", ids)
+	var res *gorm.DB
+	switch action {
+	case "priority":
+		cols := map[string]any{"priority_indexing": value}
+		if value {
+			cols["to_be_indexed"] = true
+		}
+		res = base.UpdateColumns(cols)
+	case "to-be-indexed":
+		res = base.UpdateColumns(map[string]any{"to_be_indexed": value})
+	case "reindex":
+		cols := map[string]any{"to_be_indexed": true, "is_being_indexed": false}
+		if value {
+			cols["priority_indexing"] = true
+		}
+		res = base.UpdateColumns(cols)
+	case "reset-indexing":
+		res = base.UpdateColumns(map[string]any{"is_being_indexed": false})
+	case "recheck":
+		res = base.UpdateColumns(map[string]any{"recheck": value})
+	case "restore":
+		res = tx.Unscoped().Model(&models.Addon{}).Where("id IN ? AND deleted_at IS NOT NULL", ids).
+			UpdateColumns(map[string]any{"deleted_at": nil, "to_be_indexed": true, "is_being_indexed": false})
+	case "delete":
+		var addons []models.Addon
+		if err := tx.Where("id IN ?", ids).Find(&addons).Error; err != nil {
+			return 0, err
+		}
+		for i := range addons {
+			if err := controllers.DeleteAddonCascade(tx, &addons[i]); err != nil {
+				return 0, err
+			}
+		}
+		return int64(len(addons)), nil
+	default:
+		return 0, errors.New("unknown action")
+	}
+	return res.RowsAffected, res.Error
+}
+
+func addonAction(c *gin.Context) {
+	id, action := c.Param("id"), c.Param("action")
+	var body struct {
+		Value *bool `json:"value"`
+	}
+	_ = c.ShouldBindJSON(&body)
+	value := body.Value == nil || *body.Value
+	if action == "delete" {
+		fail(c, http.StatusBadRequest, "use DELETE /addons/:id")
+		return
+	}
+	n, err := applyAction(initializers.DB, []string{id}, action, value)
+	if err != nil {
+		fail(c, http.StatusBadRequest, err.Error())
+		return
+	}
+	if n == 0 {
+		fail(c, http.StatusNotFound, "addon not found or nothing to change")
+		return
+	}
+	audit(c, "addon_"+strings.ReplaceAll(action, "-", "_"), map[string]any{"addon": id, "value": value})
+	invalidate("overview", "queue")
+	c.JSON(http.StatusOK, gin.H{"updated": n})
+}
+
+func deleteAddon(c *gin.Context) {
+	id := c.Param("id")
+	err := initializers.DB.Transaction(func(tx *gorm.DB) error {
+		var addon models.Addon
+		if err := tx.Where("id = ?", id).First(&addon).Error; err != nil {
+			return errNotFound
+		}
+		return controllers.DeleteAddonCascade(tx, &addon)
+	})
+	if errors.Is(err, errNotFound) {
+		fail(c, http.StatusNotFound, err.Error())
+		return
+	}
+	if err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+	audit(c, "addon_delete", map[string]any{"addon": id})
+	invalidate("overview", "queue")
+	c.JSON(http.StatusOK, gin.H{"deleted": id})
+}
+
+// purgeAddon permanently removes an already soft-deleted addon and its rows.
+func purgeAddon(c *gin.Context) {
+	id := c.Param("id")
+	if c.Query("confirm") != id {
+		fail(c, http.StatusBadRequest, "pass ?confirm=<addon id> to purge")
+		return
+	}
+	var files int64
+	err := initializers.DB.Transaction(func(tx *gorm.DB) error {
+		var addon models.Addon
+		if err := tx.Unscoped().Where("id = ? AND deleted_at IS NOT NULL", id).First(&addon).Error; err != nil {
+			return errNotFound
+		}
+		res := tx.Unscoped().Where("addon_id = ?", id).Delete(&models.AddonFile{})
+		if res.Error != nil {
+			return res.Error
+		}
+		files = res.RowsAffected
+		if err := tx.Where("addon_id = ?", id).Delete(&models.AddonDependency{}).Error; err != nil {
+			return err
+		}
+		return tx.Unscoped().Delete(&addon).Error
+	})
+	if errors.Is(err, errNotFound) {
+		fail(c, http.StatusNotFound, "addon not found or not deleted")
+		return
+	}
+	if err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+	audit(c, "addon_purge", map[string]any{"addon": id, "files": files})
+	invalidate("overview")
+	c.JSON(http.StatusOK, gin.H{"purged": id, "files": files})
+}
+
+func bulkAddons(c *gin.Context) {
+	var body struct {
+		IDs    []string `json:"ids"`
+		Action string   `json:"action"`
+		Value  *bool    `json:"value"`
+	}
+	if err := c.ShouldBindJSON(&body); err != nil || len(body.IDs) == 0 {
+		fail(c, http.StatusBadRequest, "ids and action are required")
+		return
+	}
+	if len(body.IDs) > 500 {
+		fail(c, http.StatusBadRequest, "at most 500 ids per request")
+		return
+	}
+	value := body.Value == nil || *body.Value
+	var n int64
+	err := initializers.DB.Transaction(func(tx *gorm.DB) error {
+		var err error
+		n, err = applyAction(tx, body.IDs, body.Action, value)
+		return err
+	})
+	if err != nil {
+		fail(c, http.StatusBadRequest, err.Error())
+		return
+	}
+	audit(c, "bulk_"+strings.ReplaceAll(body.Action, "-", "_"), map[string]any{"count": len(body.IDs), "updated": n, "value": value})
+	invalidate("overview", "queue")
+	c.JSON(http.StatusOK, gin.H{"updated": n})
+}
+
+// ---- queue ---------------------------------------------------------------
+
+type queueRow struct {
+	ID                 string     `json:"id"`
+	Name               string     `json:"name"`
+	CurrentVersionSize int64      `json:"currentVersionSize"`
+	SubscriberCount    int64      `json:"subscriberCount"`
+	PriorityIndexing   bool       `json:"priorityIndexing"`
+	Blocked            bool       `json:"blocked"`
+	IndexStartTime     *time.Time `json:"indexStartTime"`
+	UpdatedAt          time.Time  `json:"updatedAt"`
+	ElapsedSec         int64      `json:"elapsedSec"`
+	Stuck              bool       `json:"stuck"`
+}
+
+const queueColumns = "id, name, current_version_size, subscriber_count, priority_indexing, blocked, index_start_time, updated_at"
+
+func buildQueue() (gin.H, error) {
+	counts, err := metrics.LoadQueueCounts()
+	if err != nil {
+		return nil, err
+	}
+	var blockedQueued int64
+	initializers.DB.Model(&models.Addon{}).Where("to_be_indexed = ? AND blocked = ?", true, true).Count(&blockedQueued)
+
+	now := time.Now()
+	indexing := []queueRow{}
+	initializers.DB.Model(&models.Addon{}).Select(queueColumns).Where("is_being_indexed = ?", true).
+		Order("index_start_time ASC").Limit(100).Scan(&indexing)
+	for i := range indexing {
+		if t := indexing[i].IndexStartTime; t != nil && !t.IsZero() {
+			indexing[i].ElapsedSec = int64(now.Sub(*t).Seconds())
+			indexing[i].Stuck = now.Sub(*t) > initializers.IndexTimeout
+		}
+	}
+
+	priority := []queueRow{}
+	initializers.DB.Model(&models.Addon{}).Select(queueColumns).
+		Where("to_be_indexed = ? AND is_being_indexed = ? AND priority_indexing = ?", true, false, true).
+		Order("updated_at ASC").Limit(50).Scan(&priority)
+
+	// same order the indexers claim jobs in
+	next := []queueRow{}
+	initializers.DB.Model(&models.Addon{}).Select(queueColumns).
+		Where("to_be_indexed = ? AND is_being_indexed = ? AND blocked = ?", true, false, false).
+		Order("priority_indexing DESC, updated_at ASC").Limit(25).Scan(&next)
+
+	ts := now.Unix()
+	ok := map[string]string{"result": "ok"}
+	sel := metrics.Selector{Service: metrics.MainService, Name: "indexer_submissions_total", Match: ok}
+	perHour1h := metrics.Sum(sel, ts-3600, ts)
+	perHour6h := metrics.Sum(sel, ts-6*3600, ts) / 6
+	indexed24h := metrics.Sum(sel, ts-86400, ts)
+	var etaHours *float64
+	if perHour6h > 0 {
+		v := float64(counts.Queued) / perHour6h
+		etaHours = &v
+	}
+	duration, _ := metrics.Totals(metrics.Selector{Service: metrics.MainService, Name: "indexer_job_duration_seconds"}, ts-86400, ts)
+	var durationStats any
+	if len(duration) > 0 {
+		durationStats = duration[0]
+	}
+
+	return gin.H{
+		"counts":         counts,
+		"blockedQueued":  blockedQueued,
+		"timeoutMinutes": int(initializers.IndexTimeout.Minutes()),
+		"indexing":       indexing,
+		"priority":       priority,
+		"next":           next,
+		"throughput": gin.H{
+			"perHour1h":   perHour1h,
+			"perHour6h":   perHour6h,
+			"indexed24h":  indexed24h,
+			"etaHours":    etaHours,
+			"duration24h": durationStats,
+		},
+	}, nil
+}
+
+func queue(c *gin.Context) {
+	data, at, err := cached("queue", 5*time.Second, buildQueue)
+	if err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+	out := gin.H{"asOf": at.Unix()}
+	for k, v := range data {
+		out[k] = v
+	}
+	c.JSON(http.StatusOK, out)
+}
+
+func resetStuck(c *gin.Context) {
+	var body struct {
+		OlderThanMinutes int `json:"olderThanMinutes"`
+	}
+	_ = c.ShouldBindJSON(&body)
+	older := initializers.IndexTimeout
+	if body.OlderThanMinutes > 0 {
+		older = time.Duration(body.OlderThanMinutes) * time.Minute
+	}
+	n := controllers.ResetTimedOutIndexing(older)
+	metrics.IndexerTimeoutsReset.Add(float64(n))
+	audit(c, "queue_reset_stuck", map[string]any{"olderThanMinutes": older.Minutes(), "reset": n})
+	invalidate("queue", "overview")
+	c.JSON(http.StatusOK, gin.H{"reset": n})
+}

admin/authapi.go

@@ -0,0 +1,71 @@
+package admin
+
+import (
+	"net/http"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/auth"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
+	"github.com/gin-gonic/gin"
+)
+
+var limiter = auth.NewLoginLimiter(5, 10*time.Minute)
+
+func login(c *gin.Context) {
+	if initializers.AdminSecret == "" {
+		fail(c, http.StatusServiceUnavailable, "admin disabled: ADMIN_SECRET not set")
+		return
+	}
+	if c.GetHeader(auth.CSRFHeader) != "1" {
+		fail(c, http.StatusForbidden, "csrf check failed")
+		return
+	}
+	ip := c.ClientIP()
+	if limiter.Blocked(ip) {
+		fail(c, http.StatusTooManyRequests, "too many failed attempts, try again later")
+		return
+	}
+	var body struct {
+		Secret string `json:"secret"`
+	}
+	if err := c.ShouldBindJSON(&body); err != nil {
+		fail(c, http.StatusBadRequest, "invalid JSON")
+		return
+	}
+	if !auth.SecretEqual(body.Secret, initializers.AdminSecret) {
+		limiter.Fail(ip)
+		time.Sleep(300 * time.Millisecond)
+		metrics.AdminActions.Inc("login_failed")
+		metrics.Event("warn", "admin: login failed", map[string]any{"kind": "audit", "ip": ip})
+		fail(c, http.StatusUnauthorized, "invalid secret")
+		return
+	}
+	limiter.Reset(ip)
+	token, exp := auth.IssueToken(time.Now())
+	auth.SetSessionCookie(c, token, exp)
+	c.Set("adminAuth", "login")
+	audit(c, "login", nil)
+	c.JSON(http.StatusOK, gin.H{"expiresAt": exp.Unix()})
+}
+
+func logout(c *gin.Context) {
+	auth.ClearSessionCookie(c)
+	c.Status(http.StatusNoContent)
+}
+
+func me(c *gin.Context) {
+	method := c.GetString("adminAuth")
+	resp := gin.H{"method": method}
+	if v, ok := c.Get("adminSessionExp"); ok {
+		exp := v.(time.Time)
+		// sliding renewal: extend sessions that are about to expire
+		if time.Until(exp) < 24*time.Hour {
+			token, newExp := auth.IssueToken(time.Now())
+			auth.SetSessionCookie(c, token, newExp)
+			exp = newExp
+		}
+		resp["expiresAt"] = exp.Unix()
+	}
+	c.JSON(http.StatusOK, resp)
+}

admin/metricsapi.go

@@ -0,0 +1,259 @@
+package admin
+
+import (
+	"encoding/json"
+	"net/http"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
+	"github.com/gin-gonic/gin"
+)
+
+func metricsCatalog(c *gin.Context) {
+	service := c.Query("service")
+	data, _, err := cached("catalog:"+service, 15*time.Second, func() ([]metrics.CatalogEntry, error) {
+		return metrics.Catalog(service)
+	})
+	if err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+	c.JSON(http.StatusOK, gin.H{"items": data, "aggregations": metrics.Aggregations})
+}
+
+func selectorFrom(c *gin.Context) (metrics.Selector, bool) {
+	sel := metrics.Selector{Service: c.Query("service"), Name: c.Query("name")}
+	if m := c.Query("match"); m != "" {
+		if err := json.Unmarshal([]byte(m), &sel.Match); err != nil {
+			fail(c, http.StatusBadRequest, "match must be a JSON object of label values")
+			return sel, false
+		}
+	}
+	if g := c.Query("groupBy"); g != "" {
+		for _, part := range strings.Split(g, ",") {
+			if part = strings.TrimSpace(part); part != "" {
+				sel.GroupBy = append(sel.GroupBy, part)
+			}
+		}
+	}
+	if sel.Name == "" {
+		fail(c, http.StatusBadRequest, "name is required")
+		return sel, false
+	}
+	return sel, true
+}
+
+func metricsQuery(c *gin.Context) {
+	sel, ok := selectorFrom(c)
+	if !ok {
+		return
+	}
+	from, to := timeRange(c, time.Hour)
+	res, err := metrics.Query(sel, c.Query("agg"), from, to, queryInt64(c, "step", 0))
+	if err != nil {
+		fail(c, http.StatusBadRequest, err.Error())
+		return
+	}
+	c.JSON(http.StatusOK, res)
+}
+
+func autoStep(from, to int64) int64 {
+	span := to - from
+	switch {
+	case span <= 3*3600:
+		return 60
+	case span <= 12*3600:
+		return 300
+	case span <= 48*3600:
+		return 900
+	case span <= 8*86400:
+		return 3600
+	default:
+		return 6 * 3600
+	}
+}
+
+type routeStat struct {
+	Method  string   `json:"method"`
+	Route   string   `json:"route"`
+	Group   string   `json:"group"`
+	Count   int64    `json:"count"`
+	Errors  int64    `json:"errors"`
+	Client4 int64    `json:"clientErrors"`
+	P50     *float64 `json:"p50"`
+	P95     *float64 `json:"p95"`
+	P99     *float64 `json:"p99"`
+	Max     *float64 `json:"max"`
+	Avg     *float64 `json:"avg"`
+}
+
+// traffic returns pre-composed request, latency and error data for a range.
+func traffic(c *gin.Context) {
+	from, to := timeRange(c, 24*time.Hour)
+	step := queryInt64(c, "step", 0)
+	if step <= 0 {
+		step = autoStep(from, to)
+	}
+	key := "traffic:" + fmtInt64(from/30) + ":" + fmtInt64(to/30) + ":" + fmtInt64(step)
+	data, at, err := cached(key, 30*time.Second, func() (gin.H, error) {
+		return buildTraffic(from, to, step)
+	})
+	if err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+	out := gin.H{"asOf": at.Unix()}
+	for k, v := range data {
+		out[k] = v
+	}
+	c.JSON(http.StatusOK, out)
+}
+
+func fmtInt64(v int64) string { return strconv.FormatInt(v, 10) }
+
+func buildTraffic(from, to, step int64) (gin.H, error) {
+	const name = "http_request_duration_ms"
+	sel := func(match map[string]string, groupBy ...string) metrics.Selector {
+		return metrics.Selector{Service: metrics.MainService, Name: name, Match: match, GroupBy: groupBy}
+	}
+	v1 := map[string]string{"group": "v1"}
+
+	byGroup, err := metrics.Query(sel(nil, "group"), "rate", from, to, step)
+	if err != nil {
+		return nil, err
+	}
+	latency := gin.H{}
+	for _, q := range []string{"p50", "p95", "p99"} {
+		r, err := metrics.Query(sel(v1), q, from, to, step)
+		if err != nil {
+			return nil, err
+		}
+		latency[q] = r
+	}
+	errors5xx, _ := metrics.Query(sel(map[string]string{"status": "5xx"}, "group"), "rate", from, to, step)
+	errors4xx, _ := metrics.Query(sel(map[string]string{"status": "4xx", "group": "v1"}), "rate", from, to, step)
+
+	statusTotals, _ := metrics.Totals(sel(nil, "status"), from, to)
+	clientTotals, _ := metrics.Totals(sel(v1, "client"), from, to)
+	groupTotals, _ := metrics.Totals(sel(nil, "group"), from, to)
+
+	routeTotals, err := metrics.Totals(sel(nil, "method", "route", "group"), from, to)
+	if err != nil {
+		return nil, err
+	}
+	statusByRoute, _ := metrics.Totals(sel(nil, "method", "route", "status"), from, to)
+	errs := map[string][2]int64{}
+	for _, t := range statusByRoute {
+		k := t.Labels["method"] + " " + t.Labels["route"]
+		e := errs[k]
+		switch t.Labels["status"] {
+		case "5xx":
+			e[0] += t.Count
+		case "4xx":
+			e[1] += t.Count
+		}
+		errs[k] = e
+	}
+	routes := make([]routeStat, 0, len(routeTotals))
+	for _, t := range routeTotals {
+		k := t.Labels["method"] + " " + t.Labels["route"]
+		routes = append(routes, routeStat{
+			Method: t.Labels["method"], Route: t.Labels["route"], Group: t.Labels["group"],
+			Count: t.Count, Errors: errs[k][0], Client4: errs[k][1],
+			P50: t.P50, P95: t.P95, P99: t.P99, Max: t.Max, Avg: t.Avg,
+		})
+	}
+	sort.Slice(routes, func(i, j int) bool { return routes[i].Count > routes[j].Count })
+
+	now := time.Now().Unix()
+	visitorsDaily, _ := metrics.Query(metrics.Selector{Service: metrics.MainService, Name: "visitors_unique", GroupBy: []string{"kind"}},
+		"max", now-30*86400, now, 86400)
+	humans, bots := metrics.Visitors()
+
+	var v1Total *metrics.Total
+	if all, _ := metrics.Totals(sel(v1), from, to); len(all) > 0 {
+		v1Total = &all[0]
+	}
+
+	return gin.H{
+		"from": from, "to": to, "step": step,
+		"requestsByGroup": byGroup,
+		"latency":         latency,
+		"errors5xx":       errors5xx,
+		"errors4xx":       errors4xx,
+		"statusTotals":    statusTotals,
+		"clientTotals":    clientTotals,
+		"groupTotals":     groupTotals,
+		"website":         v1Total,
+		"routes":          routes,
+		"visitorsDaily":   visitorsDaily,
+		"visitorsToday":   gin.H{"human": humans, "bot": bots},
+	}, nil
+}
+
+type topEntry struct {
+	Item  string `json:"item"`
+	Count int64  `json:"count"`
+	Name  string `json:"name,omitempty"`
+}
+
+// topItems returns the most looked-up addons or search queries.
+func topItems(c *gin.Context) {
+	kind := c.DefaultQuery("kind", "addon")
+	if kind != "addon" && kind != "search" {
+		fail(c, http.StatusBadRequest, "kind must be addon or search")
+		return
+	}
+	days := queryInt(c, "days", 1, 1, 90)
+	limit := queryInt(c, "limit", 20, 1, 200)
+
+	today, counts := metrics.Top.Today(kind)
+	if today == "" {
+		today = time.Now().UTC().Format("2006-01-02")
+	}
+	if days > 1 {
+		since := time.Now().UTC().AddDate(0, 0, -(days - 1)).Format("2006-01-02")
+		var rows []models.DailyTopItem
+		initializers.DB.Where("kind = ? AND day >= ? AND day < ?", kind, since, today).Find(&rows)
+		for _, r := range rows {
+			counts[r.Item] += r.Count
+		}
+	}
+	list := make([]topEntry, 0, len(counts))
+	for k, v := range counts {
+		list = append(list, topEntry{Item: k, Count: v})
+	}
+	sort.Slice(list, func(i, j int) bool {
+		if list[i].Count != list[j].Count {
+			return list[i].Count > list[j].Count
+		}
+		return list[i].Item < list[j].Item
+	})
+	if len(list) > limit {
+		list = list[:limit]
+	}
+	if kind == "addon" && len(list) > 0 {
+		ids := make([]string, len(list))
+		for i, e := range list {
+			ids[i] = e.Item
+		}
+		var names []struct {
+			ID   string
+			Name string
+		}
+		initializers.DB.Unscoped().Model(&models.Addon{}).Select("id, name").Where("id IN ?", ids).Scan(&names)
+		byID := map[string]string{}
+		for _, n := range names {
+			byID[n.ID] = n.Name
+		}
+		for i := range list {
+			list[i].Name = byID[list[i].Item]
+		}
+	}
+	c.JSON(http.StatusOK, gin.H{"kind": kind, "days": days, "items": list})
+}

admin/overview.go

@@ -0,0 +1,443 @@
+package admin
+
+import (
+	"context"
+	"log"
+	"net/http"
+	"os"
+	"strconv"
+	"sync"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
+	"github.com/gin-gonic/gin"
+	"gorm.io/gorm"
+)
+
+type addonCounts struct {
+	Total      int64 `json:"total"`
+	Indexed    int64 `json:"indexed"`
+	Queued     int64 `json:"queued"`
+	Indexing   int64 `json:"indexing"`
+	Priority   int64 `json:"priority"`
+	Recheck    int64 `json:"recheck"`
+	Blocked    int64 `json:"blocked"`
+	Unlisted   int64 `json:"unlisted"`
+	Private    int64 `json:"private"`
+	TotalSize  int64 `json:"totalSizeBytes"`
+	Added24h   int64 `json:"added24h"`
+	Added7d    int64 `json:"added7d"`
+	Indexed24h int64 `json:"indexed24h"`
+
+	SoftDeleted int64 `json:"softDeleted"`
+}
+
+type typeCount struct {
+	Type  string `json:"type"`
+	Count int64  `json:"count"`
+}
+
+type miniAddon struct {
+	ID                 string     `json:"id"`
+	Name               string     `json:"name"`
+	Author             string     `json:"author"`
+	SubscriberCount    int64      `json:"subscriberCount"`
+	CurrentVersionSize int64      `json:"currentVersionSize"`
+	LastIndexedAt      *time.Time `json:"lastIndexedAt,omitempty"`
+	CreatedAt          time.Time  `json:"createdAt"`
+}
+
+type overviewData struct {
+	Addons          addonCounts    `json:"addons"`
+	Types           []typeCount    `json:"types"`
+	Whitelist       int64          `json:"whitelistCount"`
+	Dependencies    map[string]any `json:"dependencies"`
+	Traffic         map[string]any `json:"traffic"`
+	Activity        map[string]any `json:"activity"`
+	Services        map[string]any `json:"services"`
+	TopSubscribed   []miniAddon    `json:"topSubscribed"`
+	Largest         []miniAddon    `json:"largest"`
+	RecentlyAdded   []miniAddon    `json:"recentlyAdded"`
+	RecentlyIndexed []miniAddon    `json:"recentlyIndexed"`
+	Heavy           *heavyStats    `json:"heavy"`
+}
+
+func loadAddonCounts() (addonCounts, error) {
+	var ac addonCounts
+	now := time.Now()
+	err := initializers.DB.Raw(`SELECT COUNT(*) AS total,
+		COALESCE(SUM(CASE WHEN to_be_indexed = ? THEN 1 ELSE 0 END), 0) AS indexed,
+		COALESCE(SUM(CASE WHEN to_be_indexed = ? AND is_being_indexed = ? THEN 1 ELSE 0 END), 0) AS queued,
+		COALESCE(SUM(CASE WHEN is_being_indexed = ? THEN 1 ELSE 0 END), 0) AS indexing,
+		COALESCE(SUM(CASE WHEN priority_indexing = ? AND to_be_indexed = ? THEN 1 ELSE 0 END), 0) AS priority,
+		COALESCE(SUM(CASE WHEN recheck = ? THEN 1 ELSE 0 END), 0) AS recheck,
+		COALESCE(SUM(CASE WHEN blocked = ? THEN 1 ELSE 0 END), 0) AS blocked,
+		COALESCE(SUM(CASE WHEN unlisted = ? THEN 1 ELSE 0 END), 0) AS unlisted,
+		COALESCE(SUM(CASE WHEN private = ? THEN 1 ELSE 0 END), 0) AS private,
+		CAST(COALESCE(SUM(current_version_size), 0) AS BIGINT) AS total_size,
+		COALESCE(SUM(CASE WHEN created_at >= ? THEN 1 ELSE 0 END), 0) AS added24h,
+		COALESCE(SUM(CASE WHEN created_at >= ? THEN 1 ELSE 0 END), 0) AS added7d,
+		COALESCE(SUM(CASE WHEN last_indexed_at >= ? THEN 1 ELSE 0 END), 0) AS indexed24h
+		FROM addons WHERE deleted_at IS NULL`,
+		false, true, false, true, true, true, true, true, true, true,
+		now.Add(-24*time.Hour), now.Add(-7*24*time.Hour), now.Add(-24*time.Hour),
+	).Scan(&ac).Error
+	if err != nil {
+		return ac, err
+	}
+	err = initializers.DB.Unscoped().Model(&models.Addon{}).Where("deleted_at IS NOT NULL").Count(&ac.SoftDeleted).Error
+	return ac, err
+}
+
+func loadMiniAddons(order string, where string, args ...any) []miniAddon {
+	out := []miniAddon{}
+	q := initializers.DB.Model(&models.Addon{}).
+		Select("id, name, author, subscriber_count, current_version_size, last_indexed_at, created_at")
+	if where != "" {
+		q = q.Where(where, args...)
+	}
+	q.Order(order).Limit(8).Scan(&out)
+	return out
+}
+
+func buildOverview() (*overviewData, error) {
+	ac, err := loadAddonCounts()
+	if err != nil {
+		return nil, err
+	}
+	d := &overviewData{Addons: ac}
+
+	d.Types = []typeCount{}
+	initializers.DB.Model(&models.Addon{}).Select("type, COUNT(*) AS count").Group("type").Order("count DESC").Limit(12).Scan(&d.Types)
+
+	initializers.DB.Model(&models.WhitelistedHash{}).Count(&d.Whitelist)
+
+	var deps struct {
+		Edges      int64
+		WithDeps   int64
+		Dependents int64
+	}
+	initializers.DB.Raw(`SELECT COUNT(*) AS edges, COUNT(DISTINCT addon_id) AS with_deps,
+		COUNT(DISTINCT dependency_id) AS dependents FROM addon_dependencies`).Scan(&deps)
+	d.Dependencies = map[string]any{"edges": deps.Edges, "addonsWithDependencies": deps.WithDeps, "addonsDependedOn": deps.Dependents}
+
+	now := time.Now().Unix()
+	dayAgo := now - 86400
+	httpSel := metrics.Selector{Service: metrics.MainService, Name: "http_request_duration_ms"}
+	traffic := map[string]any{}
+	if t, err := metrics.Totals(metrics.Selector{Service: httpSel.Service, Name: httpSel.Name, GroupBy: []string{"group"}}, dayAgo, now); err == nil {
+		var total int64
+		for _, g := range t {
+			total += g.Count
+			if g.Labels["group"] == "v1" {
+				traffic["websiteRequests24h"] = g.Count
+				traffic["websiteP95Ms"] = g.P95
+				traffic["websiteP50Ms"] = g.P50
+			}
+			if g.Labels["group"] == "back" {
+				traffic["workerRequests24h"] = g.Count
+			}
+		}
+		traffic["requests24h"] = total
+	}
+	traffic["errors5xx24h"] = metrics.Sum(metrics.Selector{Service: httpSel.Service, Name: httpSel.Name, Match: map[string]string{"status": "5xx"}}, dayAgo, now)
+	humans, bots := metrics.Visitors()
+	traffic["visitorsToday"] = humans
+	traffic["botsToday"] = bots
+	d.Traffic = traffic
+
+	sum := func(name string, match map[string]string) float64 {
+		return metrics.Sum(metrics.Selector{Service: metrics.MainService, Name: name, Match: match}, dayAgo, now)
+	}
+	d.Activity = map[string]any{
+		"scraperCreated24h":      sum("scraper_addons_total", map[string]string{"action": "created"}),
+		"scraperUpdated24h":      sum("scraper_addons_total", map[string]string{"action": "updated"}),
+		"scraperCreateFailed24h": sum("scraper_addons_total", map[string]string{"action": "create_failed"}),
+		"deleted24h":             sum("scraper_addons_deleted_total", map[string]string{"result": "deleted"}),
+		"indexed24h":             sum("indexer_submissions_total", map[string]string{"result": "ok"}),
+		"claims24h":              sum("indexer_claims_total", nil),
+		"timeoutsReset24h":       sum("indexer_timeouts_reset_total", nil),
+		"dependencyReports24h":   sum("dependency_reports_total", nil),
+	}
+
+	var instances []models.ServiceInstance
+	initializers.DB.Find(&instances)
+	online := 0
+	for i := range instances {
+		if metrics.Online(&instances[i], time.Now()) {
+			online++
+		}
+	}
+	d.Services = map[string]any{"online": online, "total": len(instances)}
+
+	d.TopSubscribed = loadMiniAddons("subscriber_count DESC", "")
+	d.Largest = loadMiniAddons("current_version_size DESC", "")
+	d.RecentlyAdded = loadMiniAddons("created_at DESC", "")
+	d.RecentlyIndexed = loadMiniAddons("last_indexed_at DESC", "last_indexed_at IS NOT NULL")
+
+	d.Heavy = currentHeavy()
+	return d, nil
+}
+
+func overview(c *gin.Context) {
+	data, at, err := cached("overview", 20*time.Second, buildOverview)
+	if err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+	c.JSON(http.StatusOK, gin.H{"asOf": at.Unix(), "data": data})
+}
+
+// ---- heavy statistics (background) -----------------------------------------
+
+type tableInfo struct {
+	Name      string `json:"name"`
+	Rows      int64  `json:"rows"`
+	DeadRows  int64  `json:"deadRows"`
+	SizeBytes int64  `json:"sizeBytes"`
+}
+
+type dupHash struct {
+	Hash        string `json:"hash"`
+	AddonCount  int64  `json:"addonCount"`
+	FileCount   int64  `json:"fileCount"`
+	SamplePath  string `json:"samplePath"`
+	Whitelisted bool   `json:"whitelisted"`
+}
+
+type heavyStats struct {
+	AsOf             int64       `json:"asOf"`
+	FilesLive        int64       `json:"filesLive"`
+	FilesSoftDeleted int64       `json:"filesSoftDeleted"`
+	FilesApprox      bool        `json:"filesApprox"`
+	DistinctHashes   int64       `json:"distinctHashes"`
+	DistinctAsOf     int64       `json:"distinctAsOf"`
+	DBDriver         string      `json:"dbDriver"`
+	DBSizeBytes      int64       `json:"dbSizeBytes"`
+	Tables           []tableInfo `json:"tables"`
+	Duplicates       []dupHash   `json:"-"`
+	DuplicatesAsOf   int64       `json:"duplicatesAsOf"`
+	Error            string      `json:"error,omitempty"`
+}
+
+var (
+	heavyMu   sync.Mutex
+	heavy     *heavyStats
+	refreshCh = make(chan bool, 1)
+)
+
+func currentHeavy() *heavyStats {
+	heavyMu.Lock()
+	defer heavyMu.Unlock()
+	return heavy
+}
+
+// requestHeavyRefresh triggers a refresh; full=true also recomputes the
+// expensive duplicate statistics.
+func requestHeavyRefresh(full bool) {
+	select {
+	case refreshCh <- full:
+	default:
+	}
+}
+
+func heavyStatsLoop(ctx context.Context) {
+	lastFull := time.Time{}
+	run := func(full bool) {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Printf("admin: heavy stats panicked: %v", r)
+			}
+		}()
+		full = full || time.Since(lastFull) > time.Hour
+		computeHeavy(full)
+		if full {
+			lastFull = time.Now()
+		}
+	}
+	// give the server a moment to start before the first scan
+	select {
+	case <-ctx.Done():
+		return
+	case <-time.After(5 * time.Second):
+	}
+	run(true)
+	ticker := time.NewTicker(10 * time.Minute)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			run(false)
+		case full := <-refreshCh:
+			run(full)
+		}
+	}
+}
+
+// withTimeout runs fn in a transaction with a statement timeout on Postgres.
+func withTimeout(seconds int, fn func(tx *gorm.DB) error) error {
+	return initializers.DB.Transaction(func(tx *gorm.DB) error {
+		if isPostgres() {
+			if err := tx.Exec("SET LOCAL statement_timeout = " + strconv.Itoa(seconds*1000)).Error; err != nil {
+				return err
+			}
+		}
+		return fn(tx)
+	})
+}
+
+var ourTables = []string{"addons", "addon_files", "addon_dependencies", "whitelisted_hashes",
+	"metric_series", "metric_points", "service_instances", "service_events", "daily_top_items"}
+
+func computeHeavy(full bool) {
+	started := time.Now()
+	h := &heavyStats{AsOf: time.Now().Unix(), DBDriver: initializers.DB.Dialector.Name(), Tables: []tableInfo{}}
+	prev := currentHeavy()
+	if prev != nil {
+		h.DistinctHashes, h.DistinctAsOf = prev.DistinctHashes, prev.DistinctAsOf
+		h.Duplicates, h.DuplicatesAsOf = prev.Duplicates, prev.DuplicatesAsOf
+	}
+
+	var errs []string
+	note := func(what string, err error) {
+		if err != nil {
+			errs = append(errs, what+": "+err.Error())
+		}
+	}
+
+	if isPostgres() {
+		note("tables", initializers.DB.Raw(`SELECT relname AS name, n_live_tup AS rows, n_dead_tup AS dead_rows,
+			pg_total_relation_size(relid) AS size_bytes FROM pg_stat_user_tables ORDER BY size_bytes DESC`).Scan(&h.Tables).Error)
+		note("db size", initializers.DB.Raw(`SELECT pg_database_size(current_database())`).Scan(&h.DBSizeBytes).Error)
+		note("soft-deleted files", withTimeout(60, func(tx *gorm.DB) error {
+			return tx.Raw(`SELECT COUNT(*) FROM addon_files WHERE deleted_at IS NOT NULL`).Scan(&h.FilesSoftDeleted).Error
+		}))
+		for _, t := range h.Tables {
+			if t.Name == "addon_files" {
+				h.FilesLive = max(0, t.Rows-h.FilesSoftDeleted)
+				h.FilesApprox = true
+			}
+		}
+	} else {
+		var pageCount, pageSize int64
+		initializers.DB.Raw("PRAGMA page_count").Scan(&pageCount)
+		initializers.DB.Raw("PRAGMA page_size").Scan(&pageSize)
+		h.DBSizeBytes = pageCount * pageSize
+		if st, err := os.Stat(sqlitePath() + "-wal"); err == nil {
+			h.DBSizeBytes += st.Size()
+		}
+		for _, name := range ourTables {
+			var n int64
+			if err := initializers.DB.Table(name).Count(&n).Error; err == nil {
+				h.Tables = append(h.Tables, tableInfo{Name: name, Rows: n})
+			}
+		}
+		note("files", initializers.DB.Raw(`SELECT
+			COALESCE(SUM(CASE WHEN deleted_at IS NULL THEN 1 ELSE 0 END), 0),
+			COALESCE(SUM(CASE WHEN deleted_at IS NOT NULL THEN 1 ELSE 0 END), 0) FROM addon_files`).Row().
+			Scan(&h.FilesLive, &h.FilesSoftDeleted))
+	}
+
+	if full {
+		var distinct int64
+		err := withTimeout(120, func(tx *gorm.DB) error {
+			return tx.Raw(`SELECT COUNT(DISTINCT hash) FROM addon_files WHERE deleted_at IS NULL`).Scan(&distinct).Error
+		})
+		note("distinct hashes", err)
+		if err == nil {
+			h.DistinctHashes, h.DistinctAsOf = distinct, time.Now().Unix()
+		}
+
+		var dups []dupHash
+		err = withTimeout(120, func(tx *gorm.DB) error {
+			return tx.Raw(`SELECT hash, COUNT(DISTINCT addon_id) AS addon_count, COUNT(*) AS file_count
+				FROM addon_files WHERE deleted_at IS NULL
+				GROUP BY hash HAVING COUNT(DISTINCT addon_id) > 1
+				ORDER BY addon_count DESC, file_count DESC LIMIT 200`).Scan(&dups).Error
+		})
+		note("duplicates", err)
+		if err == nil {
+			h.Duplicates, h.DuplicatesAsOf = dups, time.Now().Unix()
+			if len(dups) > 0 {
+				hashes := make([]string, len(dups))
+				for i, d := range dups {
+					hashes[i] = d.Hash
+				}
+				var samples []struct {
+					Hash string
+					Path string
+				}
+				initializers.DB.Raw(`SELECT hash, MIN(path) AS path FROM addon_files
+					WHERE deleted_at IS NULL AND hash IN ? GROUP BY hash`, hashes).Scan(&samples)
+				byHash := map[string]string{}
+				for _, s := range samples {
+					byHash[s.Hash] = s.Path
+				}
+				for i := range h.Duplicates {
+					h.Duplicates[i].SamplePath = byHash[h.Duplicates[i].Hash]
+				}
+			}
+		}
+	}
+
+	if len(errs) > 0 {
+		h.Error = errs[0]
+		for _, e := range errs {
+			log.Printf("admin: heavy stats: %s", e)
+		}
+	}
+	heavyMu.Lock()
+	heavy = h
+	heavyMu.Unlock()
+	invalidate("overview")
+	if time.Since(started) > 10*time.Second {
+		log.Printf("admin: heavy stats took %s", time.Since(started).Round(time.Millisecond))
+	}
+}
+
+func sqlitePath() string {
+	if initializers.SQLitePath != "" {
+		return initializers.SQLitePath
+	}
+	return "crawler.db"
+}
+
+// duplicateHashes lists the hashes shared by the most addons.
+func duplicateHashes(c *gin.Context) {
+	h := currentHeavy()
+	if h == nil {
+		c.JSON(http.StatusOK, gin.H{"asOf": 0, "items": []dupHash{}, "pending": true})
+		return
+	}
+	items := make([]dupHash, len(h.Duplicates))
+	copy(items, h.Duplicates)
+	if len(items) > 0 {
+		hashes := make([]string, len(items))
+		for i, d := range items {
+			hashes[i] = d.Hash
+		}
+		var wl []string
+		initializers.DB.Model(&models.WhitelistedHash{}).Where("hash IN ?", hashes).Pluck("hash", &wl)
+		set := map[string]bool{}
+		for _, w := range wl {
+			set[w] = true
+		}
+		for i := range items {
+			items[i].Whitelisted = set[items[i].Hash]
+		}
+	}
+	if c.Query("hideWhitelisted") == "1" {
+		kept := items[:0]
+		for _, it := range items {
+			if !it.Whitelisted {
+				kept = append(kept, it)
+			}
+		}
+		items = kept
+	}
+	c.JSON(http.StatusOK, gin.H{"asOf": h.DuplicatesAsOf, "items": items})
+}

admin/routes.go

@@ -0,0 +1,57 @@
+package admin
+
+import (
+	"context"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/auth"
+	"github.com/gin-gonic/gin"
+)
+
+// Register mounts the admin JSON API under /admin/api and the legacy
+// header-authenticated /admin/addWhitelistedHash endpoint.
+func Register(r *gin.Engine) {
+	r.POST("/admin/addWhitelistedHash", auth.RequireAdminHeader, legacyAddWhitelistedHash)
+
+	api := r.Group("/admin/api")
+	api.POST("/auth/login", login)
+	api.POST("/auth/logout", logout)
+
+	p := api.Group("", auth.RequireAdmin)
+	p.GET("/auth/me", me)
+
+	p.GET("/overview", overview)
+	p.GET("/overview/duplicates", duplicateHashes)
+
+	p.GET("/queue", queue)
+	p.POST("/queue/reset-stuck", resetStuck)
+
+	p.GET("/addons", listAddons)
+	p.GET("/addons/:id", addonDetail)
+	p.GET("/addons/:id/files", addonFiles)
+	p.POST("/addons/:id/:action", addonAction)
+	p.DELETE("/addons/:id", deleteAddon)
+	p.DELETE("/addons/:id/purge", purgeAddon)
+	p.POST("/bulk/addons", bulkAddons)
+
+	p.GET("/whitelist", listWhitelist)
+	p.POST("/whitelist", addWhitelist)
+	p.DELETE("/whitelist/:id", deleteWhitelist)
+	p.GET("/hashes/:hash", hashUsage)
+
+	p.GET("/metrics/catalog", metricsCatalog)
+	p.GET("/metrics/query", metricsQuery)
+	p.GET("/traffic", traffic)
+	p.GET("/top", topItems)
+
+	p.GET("/services", listServices)
+	p.GET("/services/events", listEvents)
+	p.DELETE("/services/:id", deleteService)
+
+	p.GET("/system", system)
+	p.POST("/system/maintenance", maintenance)
+}
+
+// StartBackground runs the periodic heavy statistics job.
+func StartBackground(ctx context.Context) {
+	go heavyStatsLoop(ctx)
+}

admin/system.go

@@ -0,0 +1,244 @@
+package admin
+
+import (
+	"encoding/json"
+	"net/http"
+	"os"
+	"runtime"
+	"runtime/debug"
+	"strconv"
+	"strings"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
+	"github.com/gin-gonic/gin"
+)
+
+// BuildCommit is set by main from -ldflags.
+var BuildCommit = ""
+
+type serviceRow struct {
+	models.ServiceInstance
+	Online    bool           `json:"online"`
+	UptimeSec int64          `json:"uptimeSec"`
+	MetaMap   map[string]any `json:"metaData,omitempty"`
+}
+
+func listServices(c *gin.Context) {
+	var rows []models.ServiceInstance
+	if err := initializers.DB.Order("service ASC, instance ASC").Find(&rows).Error; err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+	now := time.Now()
+	out := make([]serviceRow, 0, len(rows))
+	for _, r := range rows {
+		s := serviceRow{ServiceInstance: r, Online: metrics.Online(&r, now)}
+		if r.StartedAt > 0 && s.Online {
+			s.UptimeSec = now.Unix() - r.StartedAt
+		}
+		if r.Meta != "" {
+			_ = json.Unmarshal([]byte(r.Meta), &s.MetaMap)
+		}
+		out = append(out, s)
+	}
+	c.JSON(http.StatusOK, gin.H{"items": out})
+}
+
+func deleteService(c *gin.Context) {
+	id, err := strconv.ParseUint(c.Param("id"), 10, 64)
+	if err != nil {
+		fail(c, http.StatusBadRequest, "invalid id")
+		return
+	}
+	var row models.ServiceInstance
+	if err := initializers.DB.First(&row, id).Error; err != nil {
+		fail(c, http.StatusNotFound, "not found")
+		return
+	}
+	initializers.DB.Delete(&row)
+	audit(c, "service_forget", map[string]any{"service": row.Service, "instance": row.Instance})
+	c.JSON(http.StatusOK, gin.H{"deleted": id})
+}
+
+type eventRow struct {
+	models.ServiceEvent
+	FieldsMap map[string]any `json:"fieldsData,omitempty"`
+}
+
+func listEvents(c *gin.Context) {
+	limit := queryInt(c, "limit", 100, 1, 500)
+	q := initializers.DB.Model(&models.ServiceEvent{})
+	if v := c.Query("service"); v != "" {
+		q = q.Where("service = ?", v)
+	}
+	if v := c.Query("instance"); v != "" {
+		q = q.Where("instance = ?", v)
+	}
+	if v := c.Query("level"); v != "" {
+		levels := strings.Split(v, ",")
+		q = q.Where("level IN ?", levels)
+	}
+	if c.Query("audit") == "1" {
+		q = q.Where("fields LIKE ?", `%"kind":"audit"%`)
+	}
+	if v := strings.TrimSpace(c.Query("q")); v != "" {
+		q = q.Where(`LOWER(message) LIKE ? ESCAPE '\'`, "%"+likeEscape(strings.ToLower(v))+"%")
+	}
+	if before := queryInt64(c, "before", 0); before > 0 {
+		q = q.Where("id < ?", before)
+	}
+	var rows []models.ServiceEvent
+	if err := q.Order("id DESC").Limit(limit).Find(&rows).Error; err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+	out := make([]eventRow, 0, len(rows))
+	for _, r := range rows {
+		e := eventRow{ServiceEvent: r}
+		if r.Fields != "" {
+			_ = json.Unmarshal([]byte(r.Fields), &e.FieldsMap)
+		}
+		out = append(out, e)
+	}
+	var next uint
+	if len(rows) == limit {
+		next = rows[len(rows)-1].ID
+	}
+	c.JSON(http.StatusOK, gin.H{"items": out, "next": next})
+}
+
+func buildInfo() gin.H {
+	info := gin.H{"commit": BuildCommit}
+	if bi, ok := debug.ReadBuildInfo(); ok {
+		info["module"] = bi.Main.Path
+		for _, s := range bi.Settings {
+			switch s.Key {
+			case "vcs.revision":
+				if BuildCommit == "" {
+					info["commit"] = s.Value
+				}
+			case "vcs.time":
+				info["time"] = s.Value
+			case "vcs.modified":
+				info["modified"] = s.Value == "true"
+			}
+		}
+	}
+	return info
+}
+
+func system(c *gin.Context) {
+	var ms runtime.MemStats
+	runtime.ReadMemStats(&ms)
+	host, _ := os.Hostname()
+	started := metrics.StartedAt()
+
+	resp := gin.H{
+		"go": gin.H{
+			"version": runtime.Version(), "os": runtime.GOOS, "arch": runtime.GOARCH,
+			"cpus": runtime.NumCPU(), "maxProcs": runtime.GOMAXPROCS(0), "goroutines": runtime.NumGoroutine(),
+		},
+		"process": gin.H{
+			"pid": os.Getpid(), "hostname": host, "startedAt": started.Unix(),
+			"uptimeSec": int64(time.Since(started).Seconds()),
+		},
+		"memory": gin.H{
+			"heapAlloc": ms.HeapAlloc, "heapInuse": ms.HeapInuse, "sys": ms.Sys,
+			"numGC": ms.NumGC, "pauseTotalNs": ms.PauseTotalNs, "lastGC": int64(ms.LastGC / 1e9),
+		},
+		"build":   buildInfo(),
+		"metrics": metrics.Status(),
+		"config": gin.H{
+			"dbDriver":                   initializers.DB.Dialector.Name(),
+			"trustedProxies":             initializers.TrustedProxies,
+			"cookieSecure":               initializers.AdminCookieSecure,
+			"sessionTtlHours":            initializers.AdminSessionTTL.Hours(),
+			"metricsEnabled":             initializers.MetricsEnabled,
+			"prometheusEndpoint":         initializers.MetricsToken != "",
+			"metricsMinuteRetentionDays": initializers.MetricsMinuteRetentionDays,
+			"metricsHourRetentionDays":   initializers.MetricsHourRetentionDays,
+			"eventsRetentionDays":        initializers.EventsRetentionDays,
+			"topItemsRetentionDays":      initializers.TopItemsRetentionDays,
+			"indexTimeoutMinutes":        initializers.IndexTimeout.Minutes(),
+			"adminSecretStrong":          len(initializers.AdminSecret) >= 16,
+			"secretKeySet":               initializers.SecretKey != "",
+			"discordWebhook":             initializers.DiscordWebhookURL != "",
+			"scraperWebhook":             initializers.ScraperWebhookURL != "",
+		},
+	}
+	if sqlDB, err := initializers.DB.DB(); err == nil {
+		st := sqlDB.Stats()
+		resp["dbPool"] = gin.H{
+			"open": st.OpenConnections, "inUse": st.InUse, "idle": st.Idle,
+			"maxOpen": st.MaxOpenConnections, "waitCount": st.WaitCount, "waitMs": st.WaitDuration.Milliseconds(),
+		}
+		start := time.Now()
+		if err := sqlDB.Ping(); err == nil {
+			resp["dbPingMs"] = float64(time.Since(start).Microseconds()) / 1000
+		} else {
+			resp["dbPingError"] = err.Error()
+		}
+	}
+	if h := currentHeavy(); h != nil {
+		resp["db"] = h
+	}
+	c.JSON(http.StatusOK, resp)
+}
+
+func maintenance(c *gin.Context) {
+	var body struct {
+		Action    string `json:"action"`
+		OlderDays int    `json:"olderThanDays"`
+	}
+	if err := c.ShouldBindJSON(&body); err != nil {
+		fail(c, http.StatusBadRequest, "invalid JSON")
+		return
+	}
+	switch body.Action {
+	case "refresh-stats":
+		requestHeavyRefresh(true)
+	case "flush-metrics":
+		metrics.FlushAll()
+	case "rollup":
+		go metrics.RunMaintenance(time.Now())
+	case "purge-deleted-files":
+		days := body.OlderDays
+		if days <= 0 {
+			days = 7
+		}
+		go purgeDeletedFiles(days)
+	default:
+		fail(c, http.StatusBadRequest, "unknown action")
+		return
+	}
+	audit(c, "maintenance_"+strings.ReplaceAll(body.Action, "-", "_"), map[string]any{"olderThanDays": body.OlderDays})
+	c.JSON(http.StatusAccepted, gin.H{"started": body.Action})
+}
+
+// purgeDeletedFiles hard-deletes file rows that were soft-deleted (replaced by
+// a newer index) more than `days` ago, in batches.
+func purgeDeletedFiles(days int) {
+	cutoff := time.Now().AddDate(0, 0, -days)
+	started := time.Now()
+	var total int64
+	for {
+		res := initializers.DB.Exec(`DELETE FROM addon_files WHERE id IN (
+			SELECT id FROM addon_files WHERE deleted_at IS NOT NULL AND deleted_at < ? LIMIT 5000)`, cutoff)
+		if res.Error != nil {
+			metrics.Event("error", "purge of deleted file rows failed", map[string]any{"kind": "maintenance", "error": res.Error.Error(), "deleted": total})
+			return
+		}
+		total += res.RowsAffected
+		if res.RowsAffected == 0 {
+			break
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	metrics.Event("info", "purged deleted file rows", map[string]any{
+		"kind": "maintenance", "deleted": total, "olderThanDays": days, "tookSec": int64(time.Since(started).Seconds()),
+	})
+	requestHeavyRefresh(false)
+}

admin/util.go

@@ -0,0 +1,136 @@
+package admin
+
+import (
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
+	"github.com/gin-gonic/gin"
+)
+
+// ---- TTL cache with request coalescing -------------------------------------
+
+type cacheEntry struct {
+	val any
+	at  time.Time
+}
+
+type inflight struct {
+	wg  sync.WaitGroup
+	val any
+	err error
+}
+
+var (
+	cacheMu   sync.Mutex
+	cacheData = map[string]cacheEntry{}
+	cacheRun  = map[string]*inflight{}
+)
+
+// cached returns a cached value younger than ttl, or computes it once even if
+// many requests ask at the same time. It also returns when the value was made.
+func cached[T any](key string, ttl time.Duration, fn func() (T, error)) (T, time.Time, error) {
+	cacheMu.Lock()
+	if e, ok := cacheData[key]; ok && time.Since(e.at) < ttl {
+		cacheMu.Unlock()
+		return e.val.(T), e.at, nil
+	}
+	if call, ok := cacheRun[key]; ok {
+		cacheMu.Unlock()
+		call.wg.Wait()
+		var zero T
+		if call.err != nil {
+			return zero, time.Time{}, call.err
+		}
+		return call.val.(T), time.Now(), nil
+	}
+	call := &inflight{}
+	call.wg.Add(1)
+	cacheRun[key] = call
+	cacheMu.Unlock()
+
+	val, err := fn()
+	call.val, call.err = val, err
+	now := time.Now()
+	cacheMu.Lock()
+	delete(cacheRun, key)
+	if err == nil {
+		cacheData[key] = cacheEntry{val: val, at: now}
+	}
+	cacheMu.Unlock()
+	call.wg.Done()
+	return val, now, err
+}
+
+// invalidate drops cached values whose key starts with one of the prefixes.
+func invalidate(prefixes ...string) {
+	cacheMu.Lock()
+	defer cacheMu.Unlock()
+	for k := range cacheData {
+		for _, p := range prefixes {
+			if strings.HasPrefix(k, p) {
+				delete(cacheData, k)
+			}
+		}
+	}
+}
+
+// ---- request helpers -----------------------------------------------------
+
+func fail(c *gin.Context, code int, msg string) {
+	c.AbortWithStatusJSON(code, gin.H{"error": msg})
+}
+
+func queryInt(c *gin.Context, name string, def, lo, hi int) int {
+	v, err := strconv.Atoi(c.Query(name))
+	if err != nil {
+		return def
+	}
+	return max(lo, min(hi, v))
+}
+
+func queryInt64(c *gin.Context, name string, def int64) int64 {
+	v, err := strconv.ParseInt(c.Query(name), 10, 64)
+	if err != nil {
+		return def
+	}
+	return v
+}
+
+// timeRange reads from/to (unix seconds) with a default window ending now.
+func timeRange(c *gin.Context, def time.Duration) (int64, int64) {
+	now := time.Now().Unix()
+	to := queryInt64(c, "to", now)
+	if to > now {
+		to = now
+	}
+	from := queryInt64(c, "from", to-int64(def.Seconds()))
+	if from >= to {
+		from = to - int64(def.Seconds())
+	}
+	return from, to
+}
+
+func isPostgres() bool {
+	return initializers.DB.Dialector.Name() == "postgres"
+}
+
+// likeEscape escapes LIKE wildcards in user input.
+func likeEscape(s string) string {
+	return strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(s)
+}
+
+// audit records an admin action as a metric and an event.
+func audit(c *gin.Context, action string, fields map[string]any) {
+	if fields == nil {
+		fields = map[string]any{}
+	}
+	fields["kind"] = "audit"
+	fields["ip"] = c.ClientIP()
+	fields["auth"] = c.GetString("adminAuth")
+	metrics.AdminActions.Inc(action)
+	metrics.Event("info", "admin: "+action, fields)
+}

admin/whitelist.go

@@ -0,0 +1,227 @@
+package admin
+
+import (
+	"net/http"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
+	"github.com/gin-gonic/gin"
+	"gorm.io/gorm"
+)
+
+var sha512Re = regexp.MustCompile(`^[0-9a-f]{128}$`)
+
+// legacyAddWhitelistedHash keeps the original form-based endpoint working.
+func legacyAddWhitelistedHash(c *gin.Context) {
+	hash := c.PostForm("hash")
+	if hash == "" {
+		c.JSON(400, gin.H{"error": "Hash is required"})
+		return
+	}
+
+	ret := initializers.DB.Find(&models.WhitelistedHash{}, "hash = ?", hash)
+	if ret.RowsAffected > 0 {
+		c.JSON(400, gin.H{"error": "Hash already whitelisted"})
+		return
+	}
+
+	if err := initializers.DB.Create(&models.WhitelistedHash{Hash: hash}).Error; err != nil {
+		c.JSON(500, gin.H{"error": "Failed to add whitelisted hash"})
+		return
+	}
+	audit(c, "whitelist_add", map[string]any{"added": 1, "legacy": true})
+	invalidate("overview")
+	c.JSON(200, gin.H{"message": "Whitelisted hash added successfully"})
+}
+
+type whitelistRow struct {
+	ID         uint      `json:"id"`
+	Hash       string    `json:"hash"`
+	Note       string    `json:"note"`
+	CreatedAt  time.Time `json:"createdAt"`
+	Files      int64     `json:"files"`
+	Addons     int64     `json:"addons"`
+	SamplePath string    `json:"samplePath"`
+}
+
+func listWhitelist(c *gin.Context) {
+	limit := queryInt(c, "limit", 50, 1, 500)
+	offset := queryInt(c, "offset", 0, 0, 1<<30)
+	q := initializers.DB.Model(&models.WhitelistedHash{})
+	if s := strings.TrimSpace(strings.ToLower(c.Query("q"))); s != "" {
+		like := "%" + likeEscape(s) + "%"
+		q = q.Where(`hash LIKE ? ESCAPE '\' OR LOWER(note) LIKE ? ESCAPE '\'`, likeEscape(s)+"%", like)
+	}
+	var total int64
+	q.Session(&gorm.Session{}).Count(&total)
+
+	rows := []whitelistRow{}
+	if err := q.Select("id, hash, note, created_at").Order("id DESC").Limit(limit).Offset(offset).Scan(&rows).Error; err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+	if len(rows) > 0 {
+		hashes := make([]string, len(rows))
+		for i, r := range rows {
+			hashes[i] = r.Hash
+		}
+		var usage []struct {
+			Hash   string
+			Files  int64
+			Addons int64
+			Sample string
+		}
+		initializers.DB.Raw(`SELECT hash, COUNT(*) AS files, COUNT(DISTINCT addon_id) AS addons, MIN(path) AS sample
+			FROM addon_files WHERE deleted_at IS NULL AND hash IN ? GROUP BY hash`, hashes).Scan(&usage)
+		byHash := map[string]int{}
+		for i, u := range usage {
+			byHash[u.Hash] = i
+		}
+		for i := range rows {
+			if j, ok := byHash[rows[i].Hash]; ok {
+				rows[i].Files, rows[i].Addons, rows[i].SamplePath = usage[j].Files, usage[j].Addons, usage[j].Sample
+			}
+		}
+	}
+	c.JSON(http.StatusOK, gin.H{"items": rows, "total": total, "limit": limit, "offset": offset})
+}
+
+type skipped struct {
+	Hash   string `json:"hash"`
+	Reason string `json:"reason"`
+}
+
+func addWhitelist(c *gin.Context) {
+	var body struct {
+		Hashes []string `json:"hashes"`
+		Text   string   `json:"text"`
+		Note   string   `json:"note"`
+	}
+	if err := c.ShouldBindJSON(&body); err != nil {
+		fail(c, http.StatusBadRequest, "invalid JSON")
+		return
+	}
+	candidates := append([]string{}, body.Hashes...)
+	candidates = append(candidates, strings.FieldsFunc(body.Text, func(r rune) bool {
+		return r == ',' || r == ';' || r == ' ' || r == '\n' || r == '\r' || r == '\t'
+	})...)
+	if len(candidates) == 0 {
+		fail(c, http.StatusBadRequest, "no hashes given")
+		return
+	}
+	if len(candidates) > 5000 {
+		fail(c, http.StatusBadRequest, "at most 5000 hashes per request")
+		return
+	}
+	note := body.Note
+	if len(note) > 256 {
+		note = note[:256]
+	}
+
+	var skips []skipped
+	seen := map[string]bool{}
+	valid := []string{}
+	for _, raw := range candidates {
+		h := strings.ToLower(strings.TrimSpace(raw))
+		switch {
+		case h == "":
+			continue
+		case !sha512Re.MatchString(h):
+			skips = append(skips, skipped{raw, "not a SHA-512 hex hash"})
+		case seen[h]:
+			skips = append(skips, skipped{h, "duplicate in request"})
+		default:
+			seen[h] = true
+			valid = append(valid, h)
+		}
+	}
+
+	added := 0
+	err := initializers.DB.Transaction(func(tx *gorm.DB) error {
+		existing := map[string]bool{}
+		for start := 0; start < len(valid); start += 500 {
+			end := min(start+500, len(valid))
+			var found []string
+			if err := tx.Model(&models.WhitelistedHash{}).Where("hash IN ?", valid[start:end]).Pluck("hash", &found).Error; err != nil {
+				return err
+			}
+			for _, h := range found {
+				existing[h] = true
+			}
+		}
+		rows := []models.WhitelistedHash{}
+		for _, h := range valid {
+			if existing[h] {
+				skips = append(skips, skipped{h, "already whitelisted"})
+				continue
+			}
+			rows = append(rows, models.WhitelistedHash{Hash: h, Note: note})
+		}
+		if len(rows) == 0 {
+			return nil
+		}
+		added = len(rows)
+		return tx.CreateInBatches(&rows, 200).Error
+	})
+	if err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+	if added > 0 {
+		audit(c, "whitelist_add", map[string]any{"added": added, "note": note})
+		invalidate("overview")
+	}
+	if skips == nil {
+		skips = []skipped{}
+	}
+	c.JSON(http.StatusOK, gin.H{"added": added, "skipped": skips})
+}
+
+func deleteWhitelist(c *gin.Context) {
+	id, err := strconv.ParseUint(c.Param("id"), 10, 64)
+	if err != nil {
+		fail(c, http.StatusBadRequest, "invalid id")
+		return
+	}
+	var row models.WhitelistedHash
+	if err := initializers.DB.Unscoped().First(&row, id).Error; err != nil {
+		fail(c, http.StatusNotFound, "not found")
+		return
+	}
+	// remove every row of this hash, the table has no unique constraint
+	res := initializers.DB.Unscoped().Where("hash = ?", row.Hash).Delete(&models.WhitelistedHash{})
+	if res.Error != nil {
+		fail(c, http.StatusInternalServerError, res.Error.Error())
+		return
+	}
+	audit(c, "whitelist_remove", map[string]any{"hash": row.Hash})
+	invalidate("overview")
+	c.JSON(http.StatusOK, gin.H{"deleted": res.RowsAffected})
+}
+
+// hashUsage lists the addons that contain a file with the given hash.
+func hashUsage(c *gin.Context) {
+	hash := strings.ToLower(c.Param("hash"))
+	var total int64
+	initializers.DB.Model(&models.AddonFile{}).Where("hash = ?", hash).Count(&total)
+	items := []struct {
+		AddonID string `json:"addonId"`
+		Name    string `json:"name"`
+		Path    string `json:"path"`
+		Version string `json:"version"`
+	}{}
+	err := initializers.DB.Raw(`SELECT f.addon_id, COALESCE(a.name, '') AS name, f.path, f.version
+		FROM addon_files f LEFT JOIN addons a ON a.id = f.addon_id
+		WHERE f.hash = ? AND f.deleted_at IS NULL ORDER BY a.subscriber_count DESC LIMIT 200`, hash).Scan(&items).Error
+	if err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+	var wl int64
+	initializers.DB.Model(&models.WhitelistedHash{}).Where("hash = ?", hash).Count(&wl)
+	c.JSON(http.StatusOK, gin.H{"hash": hash, "whitelisted": wl > 0, "total": total, "items": items})
+}

auth/auth_test.go

@@ -0,0 +1,80 @@
+package auth
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+)
+
+func TestSecretEqual(t *testing.T) {
+	if SecretEqual("", "") {
+		t.Fatal("empty configured secret must never match")
+	}
+	if SecretEqual("anything", "") {
+		t.Fatal("empty configured secret must never match")
+	}
+	if !SecretEqual("abc", "abc") {
+		t.Fatal("equal secrets must match")
+	}
+	if SecretEqual("abd", "abc") || SecretEqual("ab", "abc") {
+		t.Fatal("different secrets must not match")
+	}
+}
+
+func TestSessionToken(t *testing.T) {
+	initializers.AdminSecret = "first-secret-value"
+	initializers.AdminSessionTTL = time.Hour
+	now := time.Now()
+
+	tok, exp := IssueToken(now)
+	got, ok := VerifyToken(tok, now)
+	if !ok || got.Unix() != exp.Unix() {
+		t.Fatalf("fresh token should verify, ok=%v", ok)
+	}
+
+	if _, ok := VerifyToken(tok, now.Add(2*time.Hour)); ok {
+		t.Fatal("expired token must not verify")
+	}
+
+	parts := strings.Split(tok, ".")
+	parts[1] = "9999999999"
+	if _, ok := VerifyToken(strings.Join(parts, "."), now); ok {
+		t.Fatal("tampered token must not verify")
+	}
+
+	if _, ok := VerifyToken("garbage", now); ok {
+		t.Fatal("garbage must not verify")
+	}
+
+	initializers.AdminSecret = "rotated-secret-value"
+	if _, ok := VerifyToken(tok, now); ok {
+		t.Fatal("rotating the secret must invalidate sessions")
+	}
+
+	initializers.AdminSecret = ""
+	if _, ok := VerifyToken(tok, now); ok {
+		t.Fatal("no secret configured must reject all sessions")
+	}
+}
+
+func TestLoginLimiter(t *testing.T) {
+	l := NewLoginLimiter(3, time.Minute)
+	for range 3 {
+		if l.Blocked("1.2.3.4") {
+			t.Fatal("blocked too early")
+		}
+		l.Fail("1.2.3.4")
+	}
+	if !l.Blocked("1.2.3.4") {
+		t.Fatal("should be blocked after max failures")
+	}
+	if l.Blocked("5.6.7.8") {
+		t.Fatal("other IPs must not be affected")
+	}
+	l.Reset("1.2.3.4")
+	if l.Blocked("1.2.3.4") {
+		t.Fatal("reset should unblock")
+	}
+}

auth/middleware.go

@@ -0,0 +1,142 @@
+package auth
+
+import (
+	"net/http"
+	"net/url"
+	"sync"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"github.com/gin-gonic/gin"
+)
+
+// CSRFHeader must be sent with every state-changing cookie-authenticated
+// request. Cross-site forms cannot set custom headers.
+const CSRFHeader = "X-Admin-CSRF"
+
+// RequireSecret guards the /back endpoints used by scrapers and indexers.
+func RequireSecret(c *gin.Context) {
+	if initializers.SecretKey == "" {
+		c.AbortWithStatusJSON(http.StatusServiceUnavailable, gin.H{"error": "backend auth not configured"})
+		return
+	}
+	if !SecretEqual(c.GetHeader("X-SECRET-KEY"), initializers.SecretKey) {
+		c.AbortWithStatusJSON(http.StatusForbidden, gin.H{"error": "Forbidden"})
+		return
+	}
+}
+
+// RequireAdminHeader is the legacy admin check: X-ADMIN-KEY only, 403 on failure.
+func RequireAdminHeader(c *gin.Context) {
+	if !SecretEqual(c.GetHeader("X-ADMIN-KEY"), initializers.AdminSecret) {
+		c.AbortWithStatusJSON(http.StatusForbidden, gin.H{"error": "Forbidden"})
+		return
+	}
+	c.Set("adminAuth", "header")
+}
+
+// RequireAdmin accepts either the X-ADMIN-KEY header (scripts) or a valid
+// session cookie (dashboard). Cookie requests that change state must also carry
+// the CSRF header and, when present, a same-host Origin.
+func RequireAdmin(c *gin.Context) {
+	if initializers.AdminSecret == "" {
+		c.AbortWithStatusJSON(http.StatusServiceUnavailable, gin.H{"error": "admin disabled: ADMIN_SECRET not set"})
+		return
+	}
+
+	if key := c.GetHeader("X-ADMIN-KEY"); key != "" {
+		if !SecretEqual(key, initializers.AdminSecret) {
+			c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "invalid admin key"})
+			return
+		}
+		c.Set("adminAuth", "header")
+		return
+	}
+
+	cookie, err := c.Cookie(CookieName)
+	if err != nil || cookie == "" {
+		c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "not authenticated"})
+		return
+	}
+	exp, ok := VerifyToken(cookie, time.Now())
+	if !ok {
+		ClearSessionCookie(c)
+		c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "session expired"})
+		return
+	}
+
+	if c.Request.Method != http.MethodGet && c.Request.Method != http.MethodHead {
+		if c.GetHeader(CSRFHeader) != "1" || !sameOrigin(c) {
+			c.AbortWithStatusJSON(http.StatusForbidden, gin.H{"error": "csrf check failed"})
+			return
+		}
+	}
+
+	c.Set("adminAuth", "cookie")
+	c.Set("adminSessionExp", exp)
+}
+
+func sameOrigin(c *gin.Context) bool {
+	origin := c.GetHeader("Origin")
+	if origin == "" {
+		return true
+	}
+	u, err := url.Parse(origin)
+	if err != nil {
+		return false
+	}
+	return u.Host == c.Request.Host
+}
+
+// LoginLimiter throttles failed logins per client IP.
+type LoginLimiter struct {
+	mu       sync.Mutex
+	failures map[string][]time.Time
+	max      int
+	window   time.Duration
+}
+
+func NewLoginLimiter(max int, window time.Duration) *LoginLimiter {
+	return &LoginLimiter{failures: make(map[string][]time.Time), max: max, window: window}
+}
+
+func (l *LoginLimiter) prune(ip string, now time.Time) []time.Time {
+	kept := l.failures[ip][:0]
+	for _, t := range l.failures[ip] {
+		if now.Sub(t) < l.window {
+			kept = append(kept, t)
+		}
+	}
+	if len(kept) == 0 {
+		delete(l.failures, ip)
+		return nil
+	}
+	l.failures[ip] = kept
+	return kept
+}
+
+// Blocked reports whether the IP has used up its failed attempts.
+func (l *LoginLimiter) Blocked(ip string) bool {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return len(l.prune(ip, time.Now())) >= l.max
+}
+
+func (l *LoginLimiter) Fail(ip string) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	now := time.Now()
+	l.failures[ip] = append(l.prune(ip, now), now)
+	// keep the map bounded if someone sprays from many addresses
+	if len(l.failures) > 10000 {
+		for k := range l.failures {
+			l.prune(k, now)
+		}
+	}
+}
+
+func (l *LoginLimiter) Reset(ip string) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	delete(l.failures, ip)
+}

auth/secret.go

@@ -0,0 +1,19 @@
+package auth
+
+import (
+	"crypto/sha256"
+	"crypto/subtle"
+)
+
+// SecretEqual compares a client-supplied secret with the configured one in
+// constant time. An empty configured secret never matches, so an unset secret
+// disables access instead of letting requests without the header through.
+func SecretEqual(given, want string) bool {
+	if want == "" {
+		return false
+	}
+	// hashing first keeps the comparison length-independent
+	g := sha256.Sum256([]byte(given))
+	w := sha256.Sum256([]byte(want))
+	return subtle.ConstantTimeCompare(g[:], w[:]) == 1
+}

auth/session.go

@@ -0,0 +1,108 @@
+package auth
+
+import (
+	"crypto/hmac"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"github.com/gin-gonic/gin"
+)
+
+// CookieName is the admin session cookie. It is scoped to /admin so it is never
+// sent to the public API.
+const CookieName = "rc_admin"
+
+var b64 = base64.RawURLEncoding
+
+// The signing key is derived from the admin secret, so rotating ADMIN_SECRET
+// invalidates every issued session.
+func sessionKey() []byte {
+	m := hmac.New(sha256.New, []byte(initializers.AdminSecret))
+	m.Write([]byte("rc-admin-session/v1"))
+	return m.Sum(nil)
+}
+
+func sign(payload string) string {
+	m := hmac.New(sha256.New, sessionKey())
+	m.Write([]byte(payload))
+	return b64.EncodeToString(m.Sum(nil))
+}
+
+// IssueToken creates a stateless session token: v1.<expiry>.<nonce>.<mac>
+func IssueToken(now time.Time) (string, time.Time) {
+	exp := now.Add(initializers.AdminSessionTTL)
+	nonce := make([]byte, 16)
+	_, _ = rand.Read(nonce)
+	payload := fmt.Sprintf("v1.%d.%s", exp.Unix(), b64.EncodeToString(nonce))
+	return payload + "." + sign(payload), exp
+}
+
+// VerifyToken checks the signature and expiry and returns the expiry time.
+func VerifyToken(token string, now time.Time) (time.Time, bool) {
+	if initializers.AdminSecret == "" {
+		return time.Time{}, false
+	}
+	idx := strings.LastIndexByte(token, '.')
+	if idx <= 0 {
+		return time.Time{}, false
+	}
+	payload, mac := token[:idx], token[idx+1:]
+	if !hmac.Equal([]byte(mac), []byte(sign(payload))) {
+		return time.Time{}, false
+	}
+	parts := strings.Split(payload, ".")
+	if len(parts) != 3 || parts[0] != "v1" {
+		return time.Time{}, false
+	}
+	expUnix, err := strconv.ParseInt(parts[1], 10, 64)
+	if err != nil {
+		return time.Time{}, false
+	}
+	exp := time.Unix(expUnix, 0)
+	if !now.Before(exp) {
+		return time.Time{}, false
+	}
+	return exp, true
+}
+
+func cookieSecure(c *gin.Context) bool {
+	switch initializers.AdminCookieSecure {
+	case "true", "1", "yes":
+		return true
+	case "false", "0", "no":
+		return false
+	}
+	return c.Request.TLS != nil || strings.EqualFold(c.GetHeader("X-Forwarded-Proto"), "https")
+}
+
+func SetSessionCookie(c *gin.Context, token string, exp time.Time) {
+	http.SetCookie(c.Writer, &http.Cookie{
+		Name:     CookieName,
+		Value:    token,
+		Path:     "/admin",
+		Expires:  exp,
+		MaxAge:   int(time.Until(exp).Seconds()),
+		HttpOnly: true,
+		Secure:   cookieSecure(c),
+		SameSite: http.SameSiteStrictMode,
+	})
+}
+
+func ClearSessionCookie(c *gin.Context) {
+	http.SetCookie(c.Writer, &http.Cookie{
+		Name:     CookieName,
+		Value:    "",
+		Path:     "/admin",
+		MaxAge:   -1,
+		HttpOnly: true,
+		Secure:   cookieSecure(c),
+		SameSite: http.SameSiteStrictMode,
+	})
+}

controllers/adminController.go

@@ -1,30 +0,0 @@
-package controllers
-
-import (
-	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
-	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
-	"github.com/gin-gonic/gin"
-)
-
-func AddWhitelistedHash(c *gin.Context) {
-	hash := c.PostForm("hash")
-	if hash == "" {
-		c.JSON(400, gin.H{"error": "Hash is required"})
-		return
-	}
-
-	hashAddon := models.WhitelistedHash{Hash: hash}
-
-	ret := initializers.DB.Find(&models.WhitelistedHash{}, "hash = ?", hash)
-	if ret.RowsAffected > 0 {
-		c.JSON(400, gin.H{"error": "Hash already whitelisted"})
-		return
-	}
-
-	if err := initializers.DB.Create(&hashAddon).Error; err != nil {
-		c.JSON(500, gin.H{"error": "Failed to add whitelisted hash"})
-		return
-	}
-
-	c.JSON(200, gin.H{"message": "Whitelisted hash added successfully"})
-}

controllers/authController.go

@@ -1,26 +1,16 @@
 package controllers
 
 import (
-	"net/http"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/auth"
-
-	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
 	"github.com/gin-gonic/gin"
 )
 
+// CheckAllowed guards the scraper/indexer endpoints (X-SECRET-KEY).
 func CheckAllowed(c *gin.Context) {
-	secret := c.GetHeader("X-SECRET-KEY")
+	auth.RequireSecret(c)
-	if secret != initializers.SecretKey {
-		c.JSON(http.StatusForbidden, gin.H{"error": "Forbidden"})
-		c.Abort()
-		return
-	}
 }
 
+// CheckAdmin is the legacy header-only admin check (X-ADMIN-KEY).
 func CheckAdmin(c *gin.Context) {
-	adminSecret := c.GetHeader("X-ADMIN-KEY")
+	auth.RequireAdminHeader(c)
-	if adminSecret != initializers.AdminSecret {
-		c.JSON(http.StatusForbidden, gin.H{"error": "Forbidden"})
-		c.Abort()
-		return
-	}
 }

controllers/dependencyController.go

@@ -4,6 +4,7 @@ import (
 	"strconv"
 
 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
 	"github.com/gin-gonic/gin"
 	"gorm.io/gorm"
@@ -53,6 +54,8 @@ func SetAddonDependencies(c *gin.Context) {
 		return
 	}
 
+	metrics.DependencyReports.Inc()
+	metrics.DependencyEdges.Add(float64(len(edges)))
 	c.JSON(200, gin.H{"status": "success", "stored": len(edges)})
 }
 
@@ -69,7 +72,7 @@ func GetDependents(c *gin.Context) {
 	limit := parseLimit(c.Query("limit"))
 	offset := parseOffset(c.Query("offset"))
 
-	dependents, total, err := queryRelatedAddons(addonID, "ad.addon_id", "ad.dependency_id", limit, offset)
+	dependents, total, err := QueryRelatedAddons(addonID, "ad.addon_id", "ad.dependency_id", limit, offset)
 	if err != nil {
 		c.JSON(500, gin.H{"error": err.Error()})
 		return
@@ -97,7 +100,7 @@ func GetDependencies(c *gin.Context) {
 	limit := parseLimit(c.Query("limit"))
 	offset := parseOffset(c.Query("offset"))
 
-	dependencies, total, err := queryRelatedAddons(addonID, "ad.dependency_id", "ad.addon_id", limit, offset)
+	dependencies, total, err := QueryRelatedAddons(addonID, "ad.dependency_id", "ad.addon_id", limit, offset)
 	if err != nil {
 		c.JSON(500, gin.H{"error": err.Error()})
 		return
@@ -112,7 +115,7 @@ func GetDependencies(c *gin.Context) {
 	})
 }
 
-// queryRelatedAddons returns one page of addons linked to addonID through the
+// QueryRelatedAddons returns one page of addons linked to addonID through the
 // addon_dependencies edge table, plus the total number of matches.
 //
 // joinCol is the edge column joined to addons.id; filterCol is the edge column
@@ -120,7 +123,7 @@ func GetDependencies(c *gin.Context) {
 //
 // The addons.id tiebreaker in the ORDER BY is required: subscriber_count alone
 // is not a unique ordering, so without it paging could repeat or skip rows.
-func queryRelatedAddons(addonID, joinCol, filterCol string, limit, offset int) ([]models.AddonSearchResult, int, error) {
+func QueryRelatedAddons(addonID, joinCol, filterCol string, limit, offset int) ([]models.AddonSearchResult, int, error) {
 	base := func() *gorm.DB {
 		return initializers.DB.Model(&models.Addon{}).
 			Joins("JOIN addon_dependencies ad ON "+joinCol+" = addons.id").

controllers/feController.go

@@ -6,6 +6,7 @@ import (
 	"strconv"
 
 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
 	"github.com/gin-gonic/gin"
 )
@@ -114,6 +115,7 @@ func GetDuplicates(c *gin.Context) {
 	for _, file := range duplicateFiles {
 		duplicatesByAddonID[file.AddonID] = append(duplicatesByAddonID[file.AddonID], file)
 	}
+	metrics.DuplicatesFound.Observe(float64(len(duplicatesByAddonID)))
 
 	// Fetch addon information for each addon that has duplicates
 	var addonIDs []string

controllers/indexerController.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
 	"github.com/gin-gonic/gin"
 	"gorm.io/gorm"
@@ -16,7 +17,7 @@ import (
 
 func GetNextToBeIndexed(c *gin.Context) {
 
-	checkIndexingTimeout()
+	metrics.IndexerTimeoutsReset.Add(float64(ResetTimedOutIndexing(initializers.IndexTimeout)))
 
 	maxFileSize := c.Query("maxFileSize")
 	maxSize := 10_000_000_000
@@ -26,25 +27,31 @@ func GetNextToBeIndexed(c *gin.Context) {
 		}
 	}
 
+	queue := "priority"
 	addon, err := claimNextAddon(maxSize, true)
 	if err != nil {
+		metrics.IndexerClaims.Inc("error")
 		c.JSON(500, gin.H{"error": err.Error()})
 		return
 	}
 	if addon == nil {
+		queue = "normal"
 		addon, err = claimNextAddon(maxSize, false)
 		if err != nil {
+			metrics.IndexerClaims.Inc("error")
 			c.JSON(500, gin.H{"error": err.Error()})
 			return
 		}
 	}
 
 	if addon != nil {
+		metrics.IndexerClaims.Inc(queue)
 		c.JSON(200, gin.H{"guid": addon.ID, "currentVersion": addon.CurrentVersionNumber})
 		return
 	}
 
 	// nothing to index
+	metrics.IndexerClaims.Inc("empty")
 	c.JSON(200, gin.H{"guid": "", "currentVersion": ""})
 }
 
@@ -96,6 +103,7 @@ func SaveIndexingResult(c *gin.Context) {
 	err := c.ShouldBindJSON(&result)
 
 	if err != nil {
+		metrics.IndexerSubmissions.Inc("invalid")
 		c.JSON(400, gin.H{"error": "Invalid JSON"})
 		return
 	}
@@ -103,6 +111,7 @@ func SaveIndexingResult(c *gin.Context) {
 	var addon models.Addon
 	ret := initializers.DB.Where("id = ?", result.GUID).First(&addon)
 	if ret.Error != nil {
+		metrics.IndexerSubmissions.Inc("not_found")
 		c.JSON(404, gin.H{"error": "Addon not found"})
 		return
 	}
@@ -110,25 +119,42 @@ func SaveIndexingResult(c *gin.Context) {
 	// get all files for this addon and delete them - old version
 	ret = initializers.DB.Where("addon_id = ?", addon.ID).Delete(&models.AddonFile{})
 	if ret.Error != nil {
+		metrics.IndexerSubmissions.Inc("error")
 		c.JSON(500, gin.H{"error": ret.Error.Error()})
 		return
 	}
 
 	ret = initializers.DB.CreateInBatches(&result.Files, 100)
 	if ret.Error != nil {
+		metrics.IndexerSubmissions.Inc("error")
 		c.JSON(500, gin.H{"error": ret.Error.Error()})
 		return
 	}
 
+	now := time.Now()
+	wasClaimed := addon.IsBeingIndexed && !addon.IndexStartTime.IsZero()
 	addon.CurrentVersionNumber = result.CurrentVersion
 	addon.IsBeingIndexed = false
 	addon.ToBeIndexed = false
+	addon.LastIndexedAt = &now
 	ret = initializers.DB.Save(&addon)
 	if ret.Error != nil {
+		metrics.IndexerSubmissions.Inc("error")
 		c.JSON(500, gin.H{"error": ret.Error.Error()})
 		return
 	}
 
+	outcome := "ok"
+	if !result.Success {
+		outcome = "unsuccessful"
+	}
+	metrics.IndexerSubmissions.Inc(outcome)
+	if wasClaimed {
+		metrics.IndexerJobDuration.Observe(now.Sub(addon.IndexStartTime).Seconds())
+	}
+	metrics.IndexerFilesPerSubmission.Observe(float64(len(result.Files)))
+	metrics.IndexerBytesIndexed.Add(float64(addon.CurrentVersionSize))
+
 	if initializers.DiscordWebhookURL != "" {
 		text := "Indexing of addon " + addon.Name + " (" + addon.ID + ") was successful."
 		colour := 2228479
@@ -179,7 +205,7 @@ func SaveIndexingResult(c *gin.Context) {
 			Embeds:   []models.CustomEmbed{myEmbed},
 		}
 
-		err := SendCustomWebhook(initializers.DiscordWebhookURL, myHook)
+		err := sendWebhook("index", initializers.DiscordWebhookURL, myHook)
 		if err != nil {
 			fmt.Println("Error sending webhook:", err)
 		}
@@ -198,40 +224,60 @@ func DeleteAddon(c *gin.Context) {
 	var addon models.Addon
 	ret := initializers.DB.Where("id = ?", guid).First(&addon)
 	if ret.Error != nil {
+		metrics.ScraperDeletes.Inc("not_found")
 		c.JSON(404, gin.H{"error": "Addon not found"})
 		return
 	}
 
-	// Delete associated files
+	err := initializers.DB.Transaction(func(tx *gorm.DB) error {
-	ret = initializers.DB.Where("addon_id = ?", addon.ID).Delete(&models.AddonFile{})
+		return DeleteAddonCascade(tx, &addon)
-	if ret.Error != nil {
+	})
-		c.JSON(500, gin.H{"error": ret.Error.Error()})
+	if err != nil {
+		metrics.ScraperDeletes.Inc("error")
+		c.JSON(500, gin.H{"error": err.Error()})
 		return
 	}
 
+	metrics.ScraperDeletes.Inc("deleted")
+	c.JSON(200, gin.H{"status": "addon deleted"})
+}
+
+// DeleteAddonCascade soft-deletes an addon and its files and removes its
+// outgoing dependency edges.
+func DeleteAddonCascade(tx *gorm.DB, addon *models.Addon) error {
+	// Delete associated files
+	if err := tx.Where("addon_id = ?", addon.ID).Delete(&models.AddonFile{}).Error; err != nil {
+		return err
+	}
+
 	// Delete outgoing dependency edges; incoming edges stay - other addons
 	// still declare this dependency even if the addon row is gone
-	ret = initializers.DB.Where("addon_id = ?", addon.ID).Delete(&models.AddonDependency{})
+	if err := tx.Where("addon_id = ?", addon.ID).Delete(&models.AddonDependency{}).Error; err != nil {
-	if ret.Error != nil {
+		return err
-		c.JSON(500, gin.H{"error": ret.Error.Error()})
-		return
 	}
 
 	// Delete the addon
-	ret = initializers.DB.Delete(&addon)
+	return tx.Delete(addon).Error
-	if ret.Error != nil {
-		c.JSON(500, gin.H{"error": ret.Error.Error()})
-		return
-	}
-
-	c.JSON(200, gin.H{"status": "addon deleted"})
 }
 
-func checkIndexingTimeout() {
+// ResetTimedOutIndexing releases claims older than olderThan so another
-	initializers.DB.Model(&models.Addon{}).
+// indexer can pick the addon up. Returns the number of released claims.
+func ResetTimedOutIndexing(olderThan time.Duration) int64 {
+	return initializers.DB.Model(&models.Addon{}).
 		Where("is_being_indexed = ?", true).
-		Where("index_start_time < ?", time.Now().Add(-30*time.Minute)).
+		Where("index_start_time < ?", time.Now().Add(-olderThan)).
-		Update("is_being_indexed", false)
+		Update("is_being_indexed", false).RowsAffected
+}
+
+// sendWebhook sends a Discord webhook and records the outcome.
+func sendWebhook(kind, webhookURL string, hook models.CustomHook) error {
+	err := SendCustomWebhook(webhookURL, hook)
+	result := "ok"
+	if err != nil {
+		result = "error"
+	}
+	metrics.DiscordWebhooks.Inc(kind, result)
+	return err
 }
 
 func SendCustomWebhook(webhookURL string, hook models.CustomHook) error {

controllers/scraperController.go

@@ -1,31 +1,26 @@
 package controllers
 
 import (
-	"bytes"
 	"fmt"
-	"io"
 	"strconv"
+	"sync"
 	"time"
 
 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
 	"github.com/gin-gonic/gin"
 )
 
+var webhookMu sync.Mutex
 var lastWebhookTime time.Time
 var nbrOfUnsentWebhooks int
 
 func CreateAddon(c *gin.Context) {
 	var addonR models.Addon
-	body, err := io.ReadAll(c.Request.Body)
+	err := c.ShouldBindJSON(&addonR)
-	if err != nil {
-		c.JSON(400, gin.H{"error": "Failed to read request body"})
-		return
-	}
-	fmt.Println(string(body))
-	c.Request.Body = io.NopCloser(bytes.NewBuffer(body))
-	err = c.ShouldBindJSON(&addonR)
 	if err != nil {
+		metrics.ScraperAddons.Inc("invalid")
 		c.JSON(400, gin.H{"error": "Invalid JSON"})
 		return
 	}
@@ -37,17 +32,26 @@ func CreateAddon(c *gin.Context) {
 	result := initializers.DB.Where(&addon).First(&addon)
 	if result.Error != nil {
 		// addon does not exist, create it
-		initializers.DB.Create(&addonR)
+		if err := initializers.DB.Create(&addonR).Error; err != nil {
+			// usually a soft-deleted addon with the same ID; it can be restored from the admin dashboard
+			metrics.ScraperAddons.Inc("create_failed")
+			fmt.Println("Failed to create addon", addonR.ID, err)
+		} else {
+			metrics.ScraperAddons.Inc("created")
+		}
 
 		// send webhook about creation
+		webhookMu.Lock()
 		if time.Since(lastWebhookTime) < time.Second*10 {
 			nbrOfUnsentWebhooks++
+			webhookMu.Unlock()
 			return
 		}
 
 		lastWebhookTime = time.Now()
 		oldNbr := nbrOfUnsentWebhooks
 		nbrOfUnsentWebhooks = 0
+		webhookMu.Unlock()
 
 		go func() {
 			webhookURL := initializers.ScraperWebhookURL
@@ -82,7 +86,7 @@ func CreateAddon(c *gin.Context) {
 					Embeds:   []models.CustomEmbed{myEmbed},
 				}
 
-				err := SendCustomWebhook(webhookURL, myHook)
+				err := sendWebhook("scraper", webhookURL, myHook)
 				if err != nil {
 					fmt.Println("Error sending webhook:", err)
 				}
@@ -105,6 +109,7 @@ func CreateAddon(c *gin.Context) {
 			addon.Blocked = addonR.Blocked
 			addon.ToBeIndexed = true
 			initializers.DB.Save(&addon)
+			metrics.ScraperAddons.Inc("updated")
 
 			// send webhook about update
 			go func() {
@@ -140,12 +145,14 @@ func CreateAddon(c *gin.Context) {
 						Embeds:   []models.CustomEmbed{myEmbed},
 					}
 
-					err := SendCustomWebhook(webhookURL, myHook)
+					err := sendWebhook("scraper", webhookURL, myHook)
 					if err != nil {
 						fmt.Println("Error sending webhook:", err)
 					}
 				}
 			}()
+		} else {
+			metrics.ScraperAddons.Inc("unchanged")
 		}
 	}
 	c.JSON(200, gin.H{"status": "success"})

controllers/telemetryController.go

@@ -0,0 +1,53 @@
+package controllers
+
+import (
+	"net/http"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
+	"github.com/gin-gonic/gin"
+)
+
+const (
+	maxTelemetryBody    = 1 << 20
+	maxTelemetryMetrics = 1000
+	maxTelemetryEvents  = 200
+)
+
+// SubmitTelemetry receives metrics, events and a heartbeat from a service
+// (indexer, scraper, ...). See models.TelemetryBatch for the payload.
+func SubmitTelemetry(c *gin.Context) {
+	c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxTelemetryBody)
+
+	var batch models.TelemetryBatch
+	if err := c.ShouldBindJSON(&batch); err != nil {
+		c.JSON(400, gin.H{"error": "Invalid JSON"})
+		return
+	}
+	if !metrics.ValidName(batch.Service) || batch.Service == metrics.MainService {
+		c.JSON(400, gin.H{"error": "service must match ^[a-z_][a-z0-9_]{0,63}$ and not be \"main\""})
+		return
+	}
+	if batch.Instance == "" || len(batch.Instance) > 128 {
+		c.JSON(400, gin.H{"error": "instance is required (max 128 characters)"})
+		return
+	}
+	if len(batch.Metrics) > maxTelemetryMetrics || len(batch.Events) > maxTelemetryEvents {
+		c.JSON(413, gin.H{"error": "too many metrics or events in one batch"})
+		return
+	}
+
+	now := time.Now()
+	if err := metrics.Heartbeat(&batch, c.ClientIP(), now); err != nil {
+		c.JSON(500, gin.H{"error": "failed to store heartbeat"})
+		return
+	}
+	result := metrics.Ingest(batch.Service, batch.Instance, batch.Metrics, now)
+	if err := metrics.RecordEvents(batch.Service, batch.Instance, batch.Events, now); err != nil {
+		c.JSON(500, gin.H{"error": "failed to store events"})
+		return
+	}
+
+	c.JSON(200, result)
+}

docs/admin-dashboard.md

@@ -0,0 +1,124 @@
+# Admin dashboard, metrics and telemetry
+
+The API binary serves an admin dashboard at **`/admin/ui/`** (`/admin` redirects there).
+It is a React app in `web/`, built into `web/dist` and embedded with `go:embed`.
+
+## Access
+
+- Sign in with `ADMIN_SECRET`. The server sets an HttpOnly, `SameSite=Strict`
+  session cookie (`ADMIN_SESSION_TTL_HOURS`, default 7 days).
+- Rotating `ADMIN_SECRET` signs every session out.
+- If `ADMIN_SECRET` is empty, the dashboard and admin API are disabled.
+- Set `TRUSTED_PROXIES` behind Traefik/Dokploy. Without it, client IPs can be
+  spoofed via `X-Forwarded-For`, which affects visitor counts and login throttling.
+
+## Admin API
+
+The JSON API lives under `/admin/api`. Scripts can call it with the
+`X-ADMIN-KEY: <ADMIN_SECRET>` header instead of a cookie. Cookie requests that
+change data must also send `X-Admin-CSRF: 1`.
+
+| Area | Endpoints |
+|---|---|
+| Stats | `GET overview`, `GET overview/duplicates`, `GET traffic?from&to`, `GET top?kind=addon\|search&days` |
+| Queue | `GET queue`, `POST queue/reset-stuck` |
+| Addons | `GET addons?q&status&type&sort&order&limit&offset`, `GET addons/:id`, `GET addons/:id/files` |
+| Addon actions | `POST addons/:id/{priority\|to-be-indexed\|reindex\|reset-indexing\|recheck\|restore}` with body `{"value": bool}`, `DELETE addons/:id`, `DELETE addons/:id/purge?confirm=<id>`, `POST bulk/addons` |
+| Whitelist | `GET whitelist`, `POST whitelist` with body `{"hashes":[], "text":"", "note":""}`, `DELETE whitelist/:id`, `GET hashes/:hash` |
+| Metrics | `GET metrics/catalog`, `GET metrics/query?service&name&agg&groupBy&match&from&to&step` |
+| Services | `GET services`, `GET services/events`, `DELETE services/:id` |
+| System | `GET system`, `POST system/maintenance` with body `{"action": "refresh-stats\|flush-metrics\|rollup\|purge-deleted-files"}` |
+
+The legacy endpoint `POST /admin/addWhitelistedHash` (form field `hash`, `X-ADMIN-KEY`) still works.
+
+## Metrics
+
+- Every request is timed. Latency is recorded per route as a histogram, plus
+  status class, caller group and web vs direct client.
+- The handlers also emit metrics for indexer claims and submissions, job
+  duration, scraper creates/updates/deletes, Discord webhooks and admin actions.
+- Every minute, gauges sample the queue depth and total addon count.
+
+Storage:
+
+- Data is aggregated in memory per minute and flushed to the database every
+  minute (`metric_series` and `metric_points` tables).
+- Minute data is rolled up hourly.
+- Retention is configurable: `METRICS_MINUTE_RETENTION_DAYS` (default 7) and
+  `METRICS_HOUR_RETENTION_DAYS` (default 180).
+
+Website visitors are counted as a salted hash of IP and user agent. The salt
+rotates daily and only lives in memory, so the count is approximate. The most
+looked-up addons and search queries are stored per day in `daily_top_items`.
+
+Prometheus:
+
+- Set `METRICS_TOKEN` to expose `GET /metrics` in the Prometheus text format.
+- Authenticate with `Authorization: Bearer <token>`.
+- This lets you add an external Prometheus/Grafana later.
+
+## Telemetry from indexers and scrapers
+
+Services report to `POST /back/telemetry` with `X-SECRET-KEY`. Every push is
+also a heartbeat. An instance is shown as offline after three missed
+`heartbeatIntervalSec`.
+
+```json
+{
+  "service": "indexer", "instance": "indexer-1", "version": "1.4.0",
+  "startedAt": 1727950000, "heartbeatIntervalSec": 15, "status": "ok",
+  "metrics": [
+    {"name": "jobs_total", "type": "counter", "value": 3, "labels": {"result": "ok"}},
+    {"name": "data_window_bytes", "type": "gauge", "value": 7.1e9},
+    {"name": "download_seconds", "type": "summary", "count": 3, "sum": 41.2, "min": 8.1, "max": 20.3}
+  ],
+  "events": [{"level": "warn", "message": "data limit reached", "fields": {"limitGB": 50}}]
+}
+```
+
+Metric types:
+
+- `counter`: the increase since the last push.
+- `gauge`: the current value.
+- `summary`: pre-aggregated values from the client.
+
+Limits:
+
+- Names must match `^[a-z_][a-z0-9_]{0,63}$`.
+- At most 8 labels per metric.
+- At most 1000 metrics and 200 events per push.
+- At most 200 series per metric and 5000 external series in total. Anything
+  over these limits is dropped and counted.
+
+A stdlib-only Go client is in `telemetry/`, as its own module:
+
+```go
+import "gitea.tbdevent.eu/TBD/reforger_crawler_main/telemetry"
+
+c := telemetry.New(telemetry.Config{BaseURL: apiURL, SecretKey: secret, Service: "indexer", Version: version})
+c.Start(ctx)
+defer c.Close(context.Background())
+
+c.Counter("addons_indexed_total", "result", "ok").Inc()
+c.Summary("download_seconds").Observe(took.Seconds())
+c.Gauge("data_window_bytes").Set(float64(used))
+c.Event(telemetry.Warn, "data limit reached", nil)
+```
+
+Reported metrics show up in the dashboard under **Services** and in the **Metrics explorer**.
+
+## Development
+
+```sh
+# API on SQLite (overrides config.yaml)
+DB_DRIVER=sqlite SQLITE_PATH=dev.db SECRET_KEY=dev ADMIN_SECRET=devadminsecret1234 go run .
+
+# UI with hot reload, proxied to :8083
+npm --prefix web install
+npm --prefix web run dev     # http://localhost:5173/admin/ui/
+
+# production build (needed before `go build` to embed the UI)
+npm --prefix web run build
+```
+
+If the UI was not built, the binary still compiles and serves a placeholder page.

go.mod

@@ -3,8 +3,8 @@ module gitea.tbdevent.eu/TBD/reforger_crawler_main
 go 1.25.0
 
 require (
-	github.com/bensch777/discord-webhook-golang v0.0.6
 	gopkg.in/yaml.v3 v3.0.1
+	gorm.io/driver/postgres v1.6.0
 	gorm.io/driver/sqlite v1.6.0
 	gorm.io/gorm v1.30.3
 )
@@ -26,11 +26,13 @@ require (
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
+	github.com/rogpeppe/go-internal v1.16.0 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
 	golang.org/x/arch v0.8.0 // indirect
@@ -39,7 +41,6 @@ require (
 	golang.org/x/sync v0.16.0 // indirect
 	golang.org/x/sys v0.28.0 // indirect
 	google.golang.org/protobuf v1.34.1 // indirect
-	gorm.io/driver/postgres v1.6.0 // indirect
 )
 
 require (
@@ -49,3 +50,6 @@ require (
 	github.com/mattn/go-sqlite3 v1.14.32 // indirect
 	golang.org/x/text v0.28.0 // indirect
 )
+
+// keep the admin UI dependencies out of ./... package patterns
+ignore ./web/node_modules

go.sum

@@ -1,5 +1,3 @@
-github.com/bensch777/discord-webhook-golang v0.0.6 h1:91BMU6vKgymAMfRwtXPMUrKX+SUoPPHTDJHTFA/1Kgk=
-github.com/bensch777/discord-webhook-golang v0.0.6/go.mod h1:GcIorMZAZaHZyQJkjNoYKvZ6VpZo8XLib/eD51xN7Is=
 github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0=
 github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
 github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
@@ -8,6 +6,7 @@ github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/
 github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
 github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
 github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -48,6 +47,10 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02
 github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
 github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
+github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
@@ -63,6 +66,8 @@ github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6
 github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g=
+github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -82,8 +87,6 @@ github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZ
 golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc=
 golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
-golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
-golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
 golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
@@ -92,8 +95,6 @@ golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
 golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
-golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
 golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
@@ -102,8 +103,9 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IV
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
 google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

initializers/initialize.go

@@ -5,7 +5,9 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"strconv"
 	"strings"
+	"time"
 
 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
 	"gopkg.in/yaml.v3"
@@ -25,6 +27,18 @@ var DatabaseDriver string
 var SQLitePath string
 var PostgresDSN string
 
+// Admin dashboard, metrics and telemetry settings
+var TrustedProxies []string
+var AdminSessionTTL time.Duration
+var AdminCookieSecure string // auto|true|false
+var MetricsEnabled bool
+var MetricsToken string
+var MetricsMinuteRetentionDays int
+var MetricsHourRetentionDays int
+var EventsRetentionDays int
+var TopItemsRetentionDays int
+var IndexTimeout time.Duration
+
 func ConnectToDB() {
 	var (
 		db  *gorm.DB
@@ -75,7 +89,10 @@ func ConnectToDB() {
 
 	DB = db
 
-	if err := DB.AutoMigrate(&models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{}, &models.AddonDependency{}); err != nil {
+	if err := DB.AutoMigrate(
+		&models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{}, &models.AddonDependency{},
+		&models.MetricSeries{}, &models.MetricPoint{}, &models.ServiceInstance{}, &models.ServiceEvent{}, &models.DailyTopItem{},
+	); err != nil {
 		log.Fatalf("Failed to run migrations: %v", err)
 	}
 
@@ -98,6 +115,21 @@ func ConnectToDB() {
 		ON addon_files(addon_id, hash, path, version) WHERE deleted_at IS NULL`).Error; err != nil {
 		log.Fatalf("Failed to create idx_addon_files_covering: %v", err)
 	}
+
+	// Indexes used by the indexing queue and the admin dashboard; failures are
+	// not fatal - they only cost performance
+	optionalIndexes := []string{
+		`CREATE INDEX IF NOT EXISTS idx_addons_queue ON addons(priority_indexing, updated_at)
+			WHERE to_be_indexed = true AND is_being_indexed = false AND deleted_at IS NULL`,
+		`CREATE INDEX IF NOT EXISTS idx_addons_created_at ON addons(created_at)`,
+		`CREATE INDEX IF NOT EXISTS idx_addons_subscriber_count ON addons(subscriber_count)`,
+		`CREATE INDEX IF NOT EXISTS idx_addons_current_version_size ON addons(current_version_size)`,
+	}
+	for _, stmt := range optionalIndexes {
+		if err := DB.Exec(stmt).Error; err != nil {
+			log.Printf("Warning: failed to create index: %v", err)
+		}
+	}
 }
 
 type Configuration struct {
@@ -116,6 +148,16 @@ type Configuration struct {
 	PostgresPassword  string `yaml:"postgres_password"`
 	PostgresDBName    string `yaml:"postgres_db_name"`
 	PostgresSSLMode   string `yaml:"postgres_sslmode"`
+	TrustedProxies    string `yaml:"trusted_proxies"`
+	AdminSessionTTLH  int    `yaml:"admin_session_ttl_hours"`
+	AdminCookieSecure string `yaml:"admin_cookie_secure"`
+	MetricsEnabled    *bool  `yaml:"metrics_enabled"`
+	MetricsToken      string `yaml:"metrics_token"`
+	MetricsMinuteDays int    `yaml:"metrics_minute_retention_days"`
+	MetricsHourDays   int    `yaml:"metrics_hour_retention_days"`
+	EventsDays        int    `yaml:"events_retention_days"`
+	TopItemsDays      int    `yaml:"top_items_retention_days"`
+	IndexTimeoutMin   int    `yaml:"index_timeout_minutes"`
 	LegacyPort        string `yaml:"port"`
 	LegacyIP          string `yaml:"ip"`
 	LegacySecret      string `yaml:"secret"`
@@ -132,6 +174,14 @@ func Load() {
 		DatabaseDriver:  "sqlite",
 		SQLitePath:      "crawler.db",
 		PostgresSSLMode: "disable",
+
+		AdminSessionTTLH:  168,
+		AdminCookieSecure: "auto",
+		MetricsMinuteDays: 7,
+		MetricsHourDays:   180,
+		EventsDays:        30,
+		TopItemsDays:      90,
+		IndexTimeoutMin:   30,
 	}
 
 	file, err := os.ReadFile("config.yaml")
@@ -155,6 +205,17 @@ func Load() {
 	DatabaseDriver = strings.ToLower(configuration.DatabaseDriver)
 	SQLitePath = configuration.SQLitePath
 
+	TrustedProxies = splitList(configuration.TrustedProxies)
+	AdminSessionTTL = time.Duration(positiveOr(configuration.AdminSessionTTLH, 168)) * time.Hour
+	AdminCookieSecure = strings.ToLower(configuration.AdminCookieSecure)
+	MetricsEnabled = configuration.MetricsEnabled == nil || *configuration.MetricsEnabled
+	MetricsToken = configuration.MetricsToken
+	MetricsMinuteRetentionDays = positiveOr(configuration.MetricsMinuteDays, 7)
+	MetricsHourRetentionDays = positiveOr(configuration.MetricsHourDays, 180)
+	EventsRetentionDays = positiveOr(configuration.EventsDays, 30)
+	TopItemsRetentionDays = positiveOr(configuration.TopItemsDays, 90)
+	IndexTimeout = time.Duration(positiveOr(configuration.IndexTimeoutMin, 30)) * time.Minute
+
 	if strings.TrimSpace(configuration.PostgresDSN) != "" {
 		PostgresDSN = configuration.PostgresDSN
 	} else {
@@ -202,6 +263,49 @@ func applyEnvOverrides(c *Configuration) {
 	c.PostgresPassword = firstEnv([]string{"POSTGRES_PASSWORD"}, c.PostgresPassword)
 	c.PostgresDBName = firstEnv([]string{"POSTGRES_DB_NAME"}, c.PostgresDBName)
 	c.PostgresSSLMode = firstEnv([]string{"POSTGRES_SSLMODE"}, c.PostgresSSLMode)
+	c.TrustedProxies = firstEnv([]string{"TRUSTED_PROXIES"}, c.TrustedProxies)
+	c.AdminSessionTTLH = firstEnvInt([]string{"ADMIN_SESSION_TTL_HOURS"}, c.AdminSessionTTLH)
+	c.AdminCookieSecure = firstEnv([]string{"ADMIN_COOKIE_SECURE"}, c.AdminCookieSecure)
+	if v := firstEnv([]string{"METRICS_ENABLED"}, ""); v != "" {
+		enabled := v == "1" || strings.EqualFold(v, "true") || strings.EqualFold(v, "yes")
+		c.MetricsEnabled = &enabled
+	}
+	c.MetricsToken = firstEnv([]string{"METRICS_TOKEN"}, c.MetricsToken)
+	c.MetricsMinuteDays = firstEnvInt([]string{"METRICS_MINUTE_RETENTION_DAYS"}, c.MetricsMinuteDays)
+	c.MetricsHourDays = firstEnvInt([]string{"METRICS_HOUR_RETENTION_DAYS"}, c.MetricsHourDays)
+	c.EventsDays = firstEnvInt([]string{"EVENTS_RETENTION_DAYS"}, c.EventsDays)
+	c.TopItemsDays = firstEnvInt([]string{"TOP_ITEMS_RETENTION_DAYS"}, c.TopItemsDays)
+	c.IndexTimeoutMin = firstEnvInt([]string{"INDEX_TIMEOUT_MINUTES"}, c.IndexTimeoutMin)
+}
+
+func firstEnvInt(keys []string, fallback int) int {
+	raw := firstEnv(keys, "")
+	if raw == "" {
+		return fallback
+	}
+	val, err := strconv.Atoi(strings.TrimSpace(raw))
+	if err != nil {
+		log.Printf("Warning: ignoring invalid integer %q for %s", raw, strings.Join(keys, "/"))
+		return fallback
+	}
+	return val
+}
+
+func positiveOr(val, fallback int) int {
+	if val > 0 {
+		return val
+	}
+	return fallback
+}
+
+func splitList(raw string) []string {
+	var out []string
+	for _, part := range strings.Split(raw, ",") {
+		if part = strings.TrimSpace(part); part != "" {
+			out = append(out, part)
+		}
+	}
+	return out
 }
 
 func firstEnv(keys []string, fallback string) string {

main.go

@@ -1,22 +1,49 @@
 package main
 
 import (
+	"context"
+	"errors"
+	"log"
 	"net/http"
+	"os/signal"
+	"strings"
+	"syscall"
+	"time"
 
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/admin"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/auth"
 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/controllers"
 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/web"
 
 	"github.com/gin-gonic/gin"
 )
 
+// buildCommit is set at build time: -ldflags "-X main.buildCommit=<sha>"
+var buildCommit = ""
+
 func init() {
 	initializers.Load()
 	initializers.ConnectToDB()
 }
 
 func main() {
-	r := gin.Default()
+	warnAboutConfig()
+
+	r := gin.New()
+	r.Use(gin.LoggerWithConfig(gin.LoggerConfig{Skip: skipLog}), gin.Recovery())
+	if len(initializers.TrustedProxies) > 0 {
+		if err := r.SetTrustedProxies(initializers.TrustedProxies); err != nil {
+			log.Fatalf("Invalid TRUSTED_PROXIES: %v", err)
+		}
+	}
+	metrics.SetEnabled(initializers.MetricsEnabled)
+	if initializers.MetricsEnabled {
+		r.Use(metrics.Middleware())
+	}
 	r.Use(CORSMiddleware())
+
 	r.GET("/health", func(c *gin.Context) {
 		c.JSON(http.StatusOK, gin.H{"status": "ok"})
 	})
@@ -29,14 +56,18 @@ func main() {
 		back.POST("/submitAddon", controllers.CheckAllowed, controllers.SaveIndexingResult)
 		back.DELETE("/deleteAddon", controllers.CheckAllowed, controllers.DeleteAddon)
 		back.POST("/addonDependencies", controllers.CheckAllowed, controllers.SetAddonDependencies)
+		back.POST("/telemetry", controllers.CheckAllowed, controllers.SubmitTelemetry)
 	}
 
-	admin := r.Group("/admin")
+	// admin JSON API, legacy admin endpoint and the embedded dashboard
-	{
+	admin.Register(r)
-		admin.POST("/addWhitelistedHash", controllers.CheckAdmin, controllers.AddWhitelistedHash)
+	web.Register(r)
-		admin.GET("/duplicates", controllers.CheckAdmin)
+
-		admin.POST("/setPriority", controllers.CheckAdmin)
+	if initializers.MetricsToken != "" {
-		admin.POST("/setToBeIndexed", controllers.CheckAdmin)
+		r.GET("/metrics", requireMetricsToken, func(c *gin.Context) {
+			c.Header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
+			metrics.WritePrometheus(c.Writer)
+		})
 	}
 
 	v1 := r.Group("/v1")
@@ -47,11 +78,95 @@ func main() {
 		v1.GET("/getPossible", controllers.GetPossibleAddons)
 	}
 
-	r.Run(initializers.ServerHost + ":" + initializers.ServerPort)
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+
+	version := buildCommit
+	if version == "" {
+		version = "dev"
+	}
+	admin.BuildCommit = buildCommit
+	metrics.SetVersion(version)
+	if initializers.MetricsEnabled {
+		metrics.Start(ctx)
+	}
+	admin.StartBackground(ctx)
+
+	srv := &http.Server{
+		Addr:              initializers.ServerHost + ":" + initializers.ServerPort,
+		Handler:           r,
+		ReadHeaderTimeout: 10 * time.Second,
+	}
+	go func() {
+		log.Printf("Listening on %s (admin dashboard at /admin/ui/)", srv.Addr)
+		if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+			log.Fatalf("Server error: %v", err)
+		}
+	}()
+
+	<-ctx.Done()
+	log.Println("Shutting down...")
+	shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	if err := srv.Shutdown(shutdownCtx); err != nil {
+		log.Printf("Shutdown error: %v", err)
+	}
+	if initializers.MetricsEnabled {
+		metrics.FlushAll()
+	}
+}
+
+// skipLog keeps health checks, dashboard assets and dashboard polling out of
+// the access log.
+func skipLog(c *gin.Context) bool {
+	p := c.Request.URL.Path
+	if p == "/health" || strings.HasPrefix(p, "/admin/ui") {
+		return true
+	}
+	return strings.HasPrefix(p, "/admin/api") && c.Request.Method == http.MethodGet && c.Writer.Status() < 400
+}
+
+func requireMetricsToken(c *gin.Context) {
+	token := strings.TrimPrefix(c.GetHeader("Authorization"), "Bearer ")
+	if auth.SecretEqual(token, initializers.MetricsToken) {
+		return
+	}
+	// also allow a logged-in admin to look at the raw exposition
+	if auth.SecretEqual(c.GetHeader("X-ADMIN-KEY"), initializers.AdminSecret) {
+		return
+	}
+	if cookie, err := c.Cookie(auth.CookieName); err == nil {
+		if _, ok := auth.VerifyToken(cookie, time.Now()); ok {
+			return
+		}
+	}
+	c.Header("WWW-Authenticate", `Bearer realm="metrics"`)
+	c.AbortWithStatus(http.StatusUnauthorized)
+}
+
+func warnAboutConfig() {
+	if initializers.SecretKey == "" {
+		log.Println("Warning: SECRET_KEY is not set - /back endpoints are disabled")
+	}
+	if initializers.AdminSecret == "" {
+		log.Println("Warning: ADMIN_SECRET is not set - the admin dashboard is disabled")
+	} else if len(initializers.AdminSecret) < 16 {
+		log.Println("Warning: ADMIN_SECRET is shorter than 16 characters")
+	}
+	if len(initializers.TrustedProxies) == 0 {
+		log.Println("Warning: TRUSTED_PROXIES is not set - client IPs from X-Forwarded-For are trusted from any source")
+	}
 }
 
 func CORSMiddleware() gin.HandlerFunc {
 	return func(c *gin.Context) {
+		// the admin API and worker endpoints are never called cross-origin
+		p := c.Request.URL.Path
+		if strings.HasPrefix(p, "/admin") || strings.HasPrefix(p, "/back") || p == "/metrics" {
+			c.Next()
+			return
+		}
+
 		orig := c.Request.Header.Clone().Get("Origin")
 		c.Writer.Header().Set("Access-Control-Allow-Origin", orig)
 		c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
@@ -66,13 +181,3 @@ func CORSMiddleware() gin.HandlerFunc {
 		c.Next()
 	}
 }
-
-func OptionsMiddleware() gin.HandlerFunc {
-	return func(c *gin.Context) {
-		if c.Request.Method != "OPTIONS" {
-			c.Next()
-		} else {
-			c.AbortWithStatus(http.StatusOK)
-		}
-	}
-}

metrics/aggregator.go

@@ -0,0 +1,286 @@
+package metrics
+
+import (
+	"encoding/json"
+	"math"
+	"sort"
+	"sync"
+	"sync/atomic"
+)
+
+// disabled turns all recording into no-ops (METRICS_ENABLED=false).
+var disabled atomic.Bool
+
+// SetEnabled switches metric recording on or off.
+func SetEnabled(on bool) { disabled.Store(!on) }
+
+// Agg is the aggregate of all samples of one series inside one bucket.
+type Agg struct {
+	Count  int64
+	Sum    float64
+	Min    float64
+	Max    float64
+	Last   float64
+	LastTs int64
+	Hist   []uint64 // len(bounds)+1, last element is +Inf
+}
+
+func (a *Agg) add(v float64, ts int64, bounds []float64) {
+	if a.Count == 0 {
+		a.Min, a.Max = v, v
+	} else {
+		a.Min = math.Min(a.Min, v)
+		a.Max = math.Max(a.Max, v)
+	}
+	a.Count++
+	a.Sum += v
+	if ts >= a.LastTs {
+		a.Last, a.LastTs = v, ts
+	}
+	if len(bounds) > 0 {
+		if a.Hist == nil {
+			a.Hist = make([]uint64, len(bounds)+1)
+		}
+		a.Hist[bucketIndex(bounds, v)]++
+	}
+}
+
+func (a *Agg) addSummary(count int64, sum, min, max float64, ts int64) {
+	if count <= 0 {
+		return
+	}
+	if a.Count == 0 {
+		a.Min, a.Max = min, max
+	} else {
+		a.Min = math.Min(a.Min, min)
+		a.Max = math.Max(a.Max, max)
+	}
+	a.Count += count
+	a.Sum += sum
+	if ts >= a.LastTs {
+		a.Last, a.LastTs = sum/float64(count), ts
+	}
+}
+
+// merge folds b into a.
+func (a *Agg) merge(b *Agg) {
+	if b == nil || b.Count == 0 {
+		return
+	}
+	if a.Count == 0 {
+		a.Min, a.Max = b.Min, b.Max
+	} else {
+		a.Min = math.Min(a.Min, b.Min)
+		a.Max = math.Max(a.Max, b.Max)
+	}
+	a.Count += b.Count
+	a.Sum += b.Sum
+	if b.LastTs >= a.LastTs {
+		a.Last, a.LastTs = b.Last, b.LastTs
+	}
+	if len(b.Hist) > 0 {
+		if len(a.Hist) == 0 {
+			a.Hist = make([]uint64, len(b.Hist))
+		}
+		if len(a.Hist) == len(b.Hist) {
+			for i, v := range b.Hist {
+				a.Hist[i] += v
+			}
+		}
+	}
+}
+
+func (a *Agg) clone() *Agg {
+	c := *a
+	if a.Hist != nil {
+		c.Hist = append([]uint64(nil), a.Hist...)
+	}
+	return &c
+}
+
+func (a *Agg) histJSON() string {
+	if len(a.Hist) == 0 {
+		return ""
+	}
+	b, _ := json.Marshal(a.Hist)
+	return string(b)
+}
+
+func bucketIndex(bounds []float64, v float64) int {
+	return sort.SearchFloat64s(bounds, v) // first bound >= v; len(bounds) = +Inf
+}
+
+// quantile estimates the q-quantile from histogram buckets with linear
+// interpolation inside the bucket (like Prometheus), clamped to [min, max].
+func quantile(a *Agg, bounds []float64, q float64) (float64, bool) {
+	if a == nil || a.Count == 0 || len(a.Hist) != len(bounds)+1 {
+		return 0, false
+	}
+	var total uint64
+	for _, c := range a.Hist {
+		total += c
+	}
+	if total == 0 {
+		return 0, false
+	}
+	rank := q * float64(total)
+	var cum uint64
+	for i, c := range a.Hist {
+		if c == 0 {
+			continue
+		}
+		if float64(cum+c) >= rank {
+			lower := a.Min
+			if i > 0 {
+				lower = math.Max(bounds[i-1], a.Min)
+			}
+			upper := a.Max
+			if i < len(bounds) {
+				upper = math.Min(bounds[i], a.Max)
+			}
+			if upper < lower {
+				upper = lower
+			}
+			frac := (rank - float64(cum)) / float64(c)
+			return lower + (upper-lower)*frac, true
+		}
+		cum += c
+	}
+	return a.Max, true
+}
+
+type pointKey struct {
+	s      *Series
+	bucket int64
+}
+
+// cumulative is the since-start state of a series, used for /metrics.
+type cumulative struct {
+	count int64
+	sum   float64
+	last  float64
+	hist  []uint64
+}
+
+// aggregator holds the not yet persisted minute buckets and cumulative state.
+type aggregator struct {
+	mu   sync.Mutex
+	open map[pointKey]*Agg
+	cum  map[*Series]*cumulative
+}
+
+var defaultAgg = &aggregator{
+	open: make(map[pointKey]*Agg),
+	cum:  make(map[*Series]*cumulative),
+}
+
+func minuteOf(ts int64) int64 { return ts - ts%60 }
+
+func (g *aggregator) entry(s *Series, ts int64) (*Agg, *cumulative) {
+	k := pointKey{s, minuteOf(ts)}
+	a := g.open[k]
+	if a == nil {
+		a = &Agg{}
+		g.open[k] = a
+	}
+	c := g.cum[s]
+	if c == nil {
+		c = &cumulative{}
+		g.cum[s] = c
+	}
+	return a, c
+}
+
+func (g *aggregator) observe(s *Series, ts int64, v float64) {
+	if disabled.Load() {
+		return
+	}
+	g.mu.Lock()
+	a, c := g.entry(s, ts)
+	a.add(v, ts, s.Bounds)
+	c.count++
+	c.sum += v
+	c.last = v
+	if len(s.Bounds) > 0 {
+		if c.hist == nil {
+			c.hist = make([]uint64, len(s.Bounds)+1)
+		}
+		c.hist[bucketIndex(s.Bounds, v)]++
+	}
+	g.mu.Unlock()
+	reg.touch(s, ts)
+}
+
+func (g *aggregator) observeSummary(s *Series, ts int64, count int64, sum, min, max float64) {
+	if disabled.Load() {
+		return
+	}
+	g.mu.Lock()
+	a, c := g.entry(s, ts)
+	a.addSummary(count, sum, min, max, ts)
+	c.count += count
+	c.sum += sum
+	c.last = sum / float64(count)
+	g.mu.Unlock()
+	reg.touch(s, ts)
+}
+
+// takeClosed removes and returns all buckets that started before `before`.
+func (g *aggregator) takeClosed(before int64) map[pointKey]*Agg {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	out := make(map[pointKey]*Agg)
+	for k, a := range g.open {
+		if k.bucket < before {
+			out[k] = a
+			delete(g.open, k)
+		}
+	}
+	return out
+}
+
+// putBack re-queues buckets after a failed flush.
+func (g *aggregator) putBack(points map[pointKey]*Agg) {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	for k, a := range points {
+		if cur := g.open[k]; cur != nil {
+			cur.merge(a)
+		} else {
+			g.open[k] = a
+		}
+	}
+}
+
+// snapshotOpen copies the open buckets of the given series.
+func (g *aggregator) snapshotOpen(want map[*Series]bool) map[pointKey]*Agg {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	out := make(map[pointKey]*Agg)
+	for k, a := range g.open {
+		if want[k.s] {
+			out[k] = a.clone()
+		}
+	}
+	return out
+}
+
+func (g *aggregator) openCount() int {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	return len(g.open)
+}
+
+func (g *aggregator) snapshotCum() map[*Series]cumulative {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	out := make(map[*Series]cumulative, len(g.cum))
+	for s, c := range g.cum {
+		cc := *c
+		if c.hist != nil {
+			cc.hist = append([]uint64(nil), c.hist...)
+		}
+		out[s] = cc
+	}
+	return out
+}

metrics/internal.go

@@ -0,0 +1,38 @@
+package metrics
+
+// Metrics emitted by the API itself (service "main").
+
+// HTTPBoundsMs are the latency histogram buckets in milliseconds.
+var HTTPBoundsMs = []float64{1, 2, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000}
+
+var (
+	HTTPDuration = NewHistogram("http_request_duration_ms", "HTTP request latency in milliseconds", HTTPBoundsMs,
+		"method", "route", "status", "group", "client")
+	HTTPResponseBytes = NewCounter("http_response_bytes_total", "HTTP response body bytes", "group")
+
+	IndexerClaims        = NewCounter("indexer_claims_total", "Indexing jobs requested by indexers, by outcome", "queue")
+	IndexerTimeoutsReset = NewCounter("indexer_timeouts_reset_total", "Indexing claims released after the timeout")
+	IndexerSubmissions   = NewCounter("indexer_submissions_total", "Indexing results submitted", "result")
+	IndexerJobDuration   = NewHistogram("indexer_job_duration_seconds", "Time from claim to submitted result",
+		[]float64{1, 5, 15, 30, 60, 120, 300, 600, 1200, 1800, 3600})
+	IndexerFilesPerSubmission = NewHistogram("indexer_files_per_submission", "Files per indexing result",
+		[]float64{1, 10, 50, 100, 500, 1000, 5000, 10000, 50000})
+	IndexerBytesIndexed = NewCounter("indexer_bytes_indexed_total", "Addon bytes indexed")
+
+	ScraperAddons     = NewCounter("scraper_addons_total", "Addon reports from scrapers", "action")
+	ScraperDeletes    = NewCounter("scraper_addons_deleted_total", "Addon deletions requested by scrapers", "result")
+	DependencyReports = NewCounter("dependency_reports_total", "Dependency reports received")
+	DependencyEdges   = NewCounter("dependency_edges_reported_total", "Dependency edges stored")
+	DuplicatesFound   = NewHistogram("duplicates_found_addons", "Addons sharing files with a looked-up addon",
+		[]float64{0, 1, 2, 5, 10, 25, 50, 100})
+	DiscordWebhooks = NewCounter("discord_webhook_total", "Discord webhook sends", "kind", "result")
+	AdminActions    = NewCounter("admin_actions_total", "Admin dashboard actions", "action")
+
+	AddonsTotal    = NewGauge("addons_total", "Addons in the database (not deleted)")
+	QueueDepth     = NewGauge("queue_depth", "Addons waiting to be indexed", "kind")
+	BeingIndexed   = NewGauge("being_indexed", "Addons currently claimed by an indexer")
+	VisitorsUnique = NewGauge("visitors_unique", "Unique website visitors so far today (approximate)", "kind")
+
+	TelemetrySamples = NewCounter("telemetry_samples_total", "Telemetry samples received", "result")
+	PointsDropped    = NewCounter("metrics_points_dropped_total", "Metric points dropped after failed writes")
+)

metrics/metrics_test.go

@@ -0,0 +1,216 @@
+package metrics
+
+import (
+	"math"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+)
+
+func setupDB(t *testing.T) {
+	t.Helper()
+	db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "m.db")), &gorm.Config{Logger: logger.Discard})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := db.AutoMigrate(&models.Addon{}, &models.MetricSeries{}, &models.MetricPoint{},
+		&models.ServiceInstance{}, &models.ServiceEvent{}, &models.DailyTopItem{}); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		if sqlDB, err := db.DB(); err == nil {
+			sqlDB.Close()
+		}
+	})
+	initializers.DB = db
+	initializers.MetricsMinuteRetentionDays = 7
+	initializers.MetricsHourRetentionDays = 180
+
+	// fresh in-memory state per test
+	reg = &registry{series: map[seriesKey]*Series{}, perName: map[string]int{}, lastSeen: map[*Series]int64{}}
+	defaultAgg = &aggregator{open: map[pointKey]*Agg{}, cum: map[*Series]*cumulative{}}
+	seriesID = map[*Series]uint{}
+	rolledUntil.Store(0)
+	for _, f := range families {
+		f.cache = map[string]*Series{}
+	}
+}
+
+func TestQuantileInterpolation(t *testing.T) {
+	bounds := []float64{10, 20, 30}
+	a := &Agg{}
+	for i := 1; i <= 100; i++ {
+		a.add(float64(i%30)+0.5, 0, bounds)
+	}
+	p50, ok := quantile(a, bounds, 0.5)
+	if !ok || p50 < 10 || p50 > 20 {
+		t.Fatalf("p50 = %v, want within (10, 20]", p50)
+	}
+	p99, _ := quantile(a, bounds, 0.99)
+	if p99 > a.Max || p99 < 20 {
+		t.Fatalf("p99 = %v, want in [20, max=%v]", p99, a.Max)
+	}
+	if _, ok := quantile(&Agg{}, bounds, 0.5); ok {
+		t.Fatal("empty histogram must not produce a quantile")
+	}
+}
+
+func TestAggMerge(t *testing.T) {
+	a, b := &Agg{}, &Agg{}
+	a.add(5, 100, nil)
+	a.add(1, 101, nil)
+	b.add(9, 99, nil)
+	a.merge(b)
+	if a.Count != 3 || a.Sum != 15 || a.Min != 1 || a.Max != 9 || a.Last != 1 {
+		t.Fatalf("unexpected merge result %+v", a)
+	}
+}
+
+func TestConcurrentObserve(t *testing.T) {
+	setupDB(t)
+	f := NewCounter("test_concurrent_total", "test", "worker")
+	var wg sync.WaitGroup
+	for w := range 8 {
+		wg.Add(1)
+		go func(w int) {
+			defer wg.Done()
+			for range 1000 {
+				f.Inc(string(rune('a' + w%2)))
+			}
+		}(w)
+	}
+	wg.Wait()
+	var total float64
+	for s, c := range defaultAgg.snapshotCum() {
+		if s.Name == "test_concurrent_total" {
+			total += c.sum
+		}
+	}
+	if total != 8000 {
+		t.Fatalf("total = %v, want 8000", total)
+	}
+}
+
+func TestFlushQueryRollup(t *testing.T) {
+	setupDB(t)
+	h := NewHistogram("test_latency_ms", "test", []float64{1, 10, 100}, "route")
+	s := h.series([]string{"/a"})
+
+	// two hours ago, spread over three minutes, plus one point in the current minute
+	base := hourOf(time.Now().Unix()) - 2*3600
+	for i := range 30 {
+		defaultAgg.observe(s, base+int64(i%3)*60, float64(i))
+	}
+	now := time.Now().Unix()
+	defaultAgg.observe(s, now, 50)
+
+	if err := flush(minuteOf(now)); err != nil {
+		t.Fatal(err)
+	}
+	var rows int64
+	initializers.DB.Model(&models.MetricPoint{}).Where("resolution = ?", resMinute).Count(&rows)
+	if rows != 3 {
+		t.Fatalf("minute rows = %d, want 3 (current minute stays in memory)", rows)
+	}
+
+	// flushing the same bucket again must merge, not overwrite
+	defaultAgg.observe(s, base, 1000)
+	if err := flush(minuteOf(now)); err != nil {
+		t.Fatal(err)
+	}
+	var p models.MetricPoint
+	initializers.DB.Where("resolution = ? AND bucket = ?", resMinute, base).First(&p)
+	if p.Count != 11 || p.Max != 1000 {
+		t.Fatalf("merged point = %+v, want count 11 and max 1000", p)
+	}
+
+	// query sees DB rows plus the unflushed current minute
+	sel := Selector{Service: MainService, Name: "test_latency_ms"}
+	res, err := Query(sel, "count", now-3*3600, now, 60)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var count float64
+	for _, pt := range res.Series[0].Points {
+		if pt.V != nil {
+			count += *pt.V
+		}
+	}
+	if count != 32 {
+		t.Fatalf("queried count = %v, want 32", count)
+	}
+
+	if err := rollup(now); err != nil {
+		t.Fatal(err)
+	}
+	var hourly models.MetricPoint
+	if err := initializers.DB.Where("resolution = ? AND bucket = ?", resHour, base).First(&hourly).Error; err != nil {
+		t.Fatalf("missing hourly rollup: %v", err)
+	}
+	if hourly.Count != 31 || hourly.Max != 1000 {
+		t.Fatalf("hourly = %+v, want count 31 max 1000", hourly)
+	}
+	// rolling up again is idempotent
+	rolledUntil.Store(0)
+	if err := rollup(now); err != nil {
+		t.Fatal(err)
+	}
+	initializers.DB.Where("resolution = ? AND bucket = ?", resHour, base).First(&hourly)
+	if hourly.Count != 31 {
+		t.Fatalf("re-rollup changed count to %d", hourly.Count)
+	}
+
+	totals, err := Totals(sel, now-3*3600, now)
+	if err != nil || len(totals) != 1 || totals[0].P95 == nil || math.IsNaN(*totals[0].P95) {
+		t.Fatalf("totals = %+v, err %v", totals, err)
+	}
+}
+
+func TestIngestValidation(t *testing.T) {
+	setupDB(t)
+	now := time.Now()
+	res := Ingest("indexer", "i-1", []models.TelemetryMetric{
+		{Name: "ok_total", Type: "counter", Value: 2},
+		{Name: "Bad-Name", Type: "gauge", Value: 1},
+		{Name: "neg_total", Type: "counter", Value: -1},
+		{Name: "lat", Type: "summary", Count: 2, Sum: 3, Min: 1, Max: 2},
+		{Name: "ok_total", Type: "gauge", Value: 1},
+		{Name: "weird", Type: "nope"},
+	}, now)
+	if res.Accepted != 2 || res.Dropped != 4 {
+		t.Fatalf("accepted %d dropped %d (%v)", res.Accepted, res.Dropped, res.Reasons)
+	}
+	if res.Reasons["type_mismatch"] != 1 || res.Reasons["invalid_name"] != 1 {
+		t.Fatalf("unexpected reasons %v", res.Reasons)
+	}
+}
+
+func TestPrometheusOutput(t *testing.T) {
+	setupDB(t)
+	c := NewCounter("test_prom_total", "a \"quoted\" help", "kind")
+	c.Add(3, `va"l`)
+	h := NewHistogram("test_prom_ms", "hist", []float64{1, 10})
+	h.Observe(5)
+	var b strings.Builder
+	WritePrometheus(&b)
+	out := b.String()
+	for _, want := range []string{
+		`test_prom_total{kind="va\"l"} 3`,
+		`test_prom_ms_bucket{le="1"} 0`,
+		`test_prom_ms_bucket{le="10"} 1`,
+		`test_prom_ms_bucket{le="+Inf"} 1`,
+		`# TYPE test_prom_ms histogram`,
+	} {
+		if !strings.Contains(out, want) {
+			t.Errorf("missing %q in output", want)
+		}
+	}
+}

metrics/pipeline.go

@@ -0,0 +1,156 @@
+package metrics
+
+import (
+	"context"
+	"log"
+	"math"
+	"sync"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+)
+
+var startOnce sync.Once
+
+// Start launches the background jobs: minute flush and samplers, top-N
+// persistence, hourly rollup and retention, and service heartbeat checks.
+func Start(ctx context.Context) {
+	startOnce.Do(func() {
+		Top.seed()
+		go every(ctx, "flush", time.Minute, 5*time.Second, func(now time.Time) {
+			sample()
+			if err := flush(minuteOf(now.Unix())); err != nil {
+				log.Printf("metrics: flush failed: %v", err)
+			}
+		})
+		go every(ctx, "top", 5*time.Minute, 0, func(time.Time) {
+			if err := Top.flush(); err != nil {
+				log.Printf("metrics: top items flush failed: %v", err)
+			}
+		})
+		go every(ctx, "rollup", time.Hour, 2*time.Minute, func(now time.Time) {
+			RunMaintenance(now)
+		})
+		go every(ctx, "services", 30*time.Second, 0, func(now time.Time) {
+			selfHeartbeat(now)
+			checkServices(now)
+		})
+		// initialize the rollup watermark and service state right away
+		go func() {
+			selfHeartbeat(time.Now())
+			checkServices(time.Now())
+			if err := rollup(time.Now().Unix()); err != nil {
+				log.Printf("metrics: rollup failed: %v", err)
+			}
+		}()
+	})
+}
+
+// RunMaintenance runs the rollup and retention jobs.
+func RunMaintenance(now time.Time) {
+	if err := rollup(now.Unix()); err != nil {
+		log.Printf("metrics: rollup failed: %v", err)
+	}
+	if err := retention(now); err != nil {
+		log.Printf("metrics: retention failed: %v", err)
+	}
+}
+
+// every runs fn on a fixed period aligned to wall clock (+offset).
+func every(ctx context.Context, name string, period, offset time.Duration, fn func(time.Time)) {
+	for {
+		now := time.Now()
+		next := now.Truncate(period).Add(offset)
+		if !next.After(now) {
+			next = next.Add(period)
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case t := <-time.After(time.Until(next)):
+			func() {
+				defer func() {
+					if r := recover(); r != nil {
+						log.Printf("metrics: %s job panicked: %v", name, r)
+					}
+				}()
+				fn(t)
+			}()
+		}
+	}
+}
+
+// FlushAll writes everything still in memory; used on shutdown.
+func FlushAll() {
+	sample()
+	if err := flush(math.MaxInt64); err != nil {
+		log.Printf("metrics: final flush failed: %v", err)
+	}
+	if err := Top.flush(); err != nil {
+		log.Printf("metrics: final top items flush failed: %v", err)
+	}
+}
+
+// QueueCounts is a cheap snapshot of the indexing queue.
+type QueueCounts struct {
+	Total    int64 `json:"total"`
+	Queued   int64 `json:"queued"`
+	Priority int64 `json:"priority"`
+	Indexing int64 `json:"indexing"`
+}
+
+// LoadQueueCounts counts addons by queue state with one query.
+func LoadQueueCounts() (QueueCounts, error) {
+	var qc QueueCounts
+	err := initializers.DB.Raw(`SELECT COUNT(*) AS total,
+		COALESCE(SUM(CASE WHEN to_be_indexed = ? AND is_being_indexed = ? THEN 1 ELSE 0 END), 0) AS queued,
+		COALESCE(SUM(CASE WHEN to_be_indexed = ? AND is_being_indexed = ? AND priority_indexing = ? THEN 1 ELSE 0 END), 0) AS priority,
+		COALESCE(SUM(CASE WHEN is_being_indexed = ? THEN 1 ELSE 0 END), 0) AS indexing
+		FROM addons WHERE deleted_at IS NULL`,
+		true, false, true, false, true, true).Scan(&qc).Error
+	return qc, err
+}
+
+// sample records gauges that are read from the database or memory.
+func sample() {
+	if qc, err := LoadQueueCounts(); err == nil {
+		AddonsTotal.Set(float64(qc.Total))
+		QueueDepth.Set(float64(qc.Queued), "total")
+		QueueDepth.Set(float64(qc.Priority), "priority")
+		BeingIndexed.Set(float64(qc.Indexing))
+	}
+	humans, bots := liveVisitors()
+	VisitorsUnique.Set(float64(humans), "human")
+	VisitorsUnique.Set(float64(bots), "bot")
+}
+
+// PipelineStatus describes the health of the metrics pipeline.
+type PipelineStatus struct {
+	Enabled       bool   `json:"enabled"`
+	LastFlush     int64  `json:"lastFlush"`
+	LastFlushMs   int64  `json:"lastFlushMs"`
+	LastFlushErr  string `json:"lastFlushError"`
+	LastRollup    int64  `json:"lastRollup"`
+	LastRetention int64  `json:"lastRetention"`
+	RolledUntil   int64  `json:"rolledUntil"`
+	OpenPoints    int    `json:"openPoints"`
+	LiveSeries    int    `json:"liveSeries"`
+	DroppedPoints int64  `json:"droppedPoints"`
+}
+
+func Status() PipelineStatus {
+	status.mu.Lock()
+	defer status.mu.Unlock()
+	return PipelineStatus{
+		Enabled:       initializers.MetricsEnabled,
+		LastFlush:     status.LastFlush,
+		LastFlushMs:   status.LastFlushMs,
+		LastFlushErr:  status.LastFlushErr,
+		LastRollup:    status.LastRollup,
+		LastRetention: status.LastRetention,
+		RolledUntil:   rolledUntil.Load(),
+		OpenPoints:    defaultAgg.openCount(),
+		LiveSeries:    len(reg.all()),
+		DroppedPoints: status.Dropped,
+	}
+}

metrics/prometheus.go

@@ -0,0 +1,181 @@
+package metrics
+
+import (
+	"bufio"
+	"io"
+	"runtime"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+)
+
+// WritePrometheus writes all cumulative series in the Prometheus text format
+// (version 0.0.4). External metrics are exposed as <service>_<name> with an
+// "instance" label; counters are totals since this process started.
+func WritePrometheus(w io.Writer) {
+	bw := bufio.NewWriter(w)
+	defer bw.Flush()
+
+	cum := defaultAgg.snapshotCum()
+	type entry struct {
+		s *Series
+		c cumulative
+	}
+	byName := map[string][]entry{}
+	meta := map[string]*Series{}
+	for s, c := range cum {
+		name := s.Name
+		if s.Service != MainService {
+			name = s.Service + "_" + s.Name
+		}
+		byName[name] = append(byName[name], entry{s, c})
+		if meta[name] == nil {
+			meta[name] = s
+		}
+	}
+	names := make([]string, 0, len(byName))
+	for n := range byName {
+		names = append(names, n)
+	}
+	sort.Strings(names)
+
+	for _, name := range names {
+		s := meta[name]
+		writeHeader(bw, name, s.Help, promType(s.Kind))
+		entries := byName[name]
+		sort.Slice(entries, func(i, j int) bool {
+			if entries[i].s.Instance != entries[j].s.Instance {
+				return entries[i].s.Instance < entries[j].s.Instance
+			}
+			return entries[i].s.LabelsJSON < entries[j].s.LabelsJSON
+		})
+		for _, e := range entries {
+			labels := e.s.Labels
+			if e.s.Service != MainService {
+				labels = withLabel(labels, "instance", e.s.Instance)
+			}
+			switch e.s.Kind {
+			case Counter:
+				writeSample(bw, name, labels, e.c.sum)
+			case Gauge:
+				writeSample(bw, name, labels, e.c.last)
+			case Summary:
+				writeSample(bw, name+"_sum", labels, e.c.sum)
+				writeSample(bw, name+"_count", labels, float64(e.c.count))
+			case Histogram:
+				var acc uint64
+				for i, b := range e.s.Bounds {
+					if i < len(e.c.hist) {
+						acc += e.c.hist[i]
+					}
+					writeSample(bw, name+"_bucket", withLabel(labels, "le", formatFloat(b)), float64(acc))
+				}
+				writeSample(bw, name+"_bucket", withLabel(labels, "le", "+Inf"), float64(e.c.count))
+				writeSample(bw, name+"_sum", labels, e.c.sum)
+				writeSample(bw, name+"_count", labels, float64(e.c.count))
+			}
+		}
+	}
+
+	up := ServiceUp()
+	if len(up) > 0 {
+		writeHeader(bw, "reforger_service_up", "Whether a reporting service instance is sending heartbeats", "gauge")
+		keys := make([][2]string, 0, len(up))
+		for k := range up {
+			keys = append(keys, k)
+		}
+		sort.Slice(keys, func(i, j int) bool { return keys[i][0]+keys[i][1] < keys[j][0]+keys[j][1] })
+		for _, k := range keys {
+			v := 0.0
+			if up[k] {
+				v = 1
+			}
+			writeSample(bw, "reforger_service_up", map[string]string{"service": k[0], "instance": k[1]}, v)
+		}
+	}
+
+	var ms runtime.MemStats
+	runtime.ReadMemStats(&ms)
+	writeHeader(bw, "go_goroutines", "Number of goroutines", "gauge")
+	writeSample(bw, "go_goroutines", nil, float64(runtime.NumGoroutine()))
+	writeHeader(bw, "go_memstats_heap_alloc_bytes", "Heap bytes allocated and in use", "gauge")
+	writeSample(bw, "go_memstats_heap_alloc_bytes", nil, float64(ms.HeapAlloc))
+	writeHeader(bw, "go_memstats_sys_bytes", "Bytes obtained from the OS", "gauge")
+	writeSample(bw, "go_memstats_sys_bytes", nil, float64(ms.Sys))
+	writeHeader(bw, "go_gc_cycles_total", "Completed GC cycles", "counter")
+	writeSample(bw, "go_gc_cycles_total", nil, float64(ms.NumGC))
+	writeHeader(bw, "process_start_time_seconds", "Process start time", "gauge")
+	writeSample(bw, "process_start_time_seconds", nil, float64(startedAt.Unix()))
+	writeHeader(bw, "process_uptime_seconds", "Process uptime", "gauge")
+	writeSample(bw, "process_uptime_seconds", nil, time.Since(startedAt).Seconds())
+
+	if sqlDB, err := initializers.DB.DB(); err == nil {
+		st := sqlDB.Stats()
+		writeHeader(bw, "db_open_connections", "Open database connections", "gauge")
+		writeSample(bw, "db_open_connections", nil, float64(st.OpenConnections))
+		writeHeader(bw, "db_in_use_connections", "Database connections in use", "gauge")
+		writeSample(bw, "db_in_use_connections", nil, float64(st.InUse))
+		writeHeader(bw, "db_wait_count_total", "Waits for a database connection", "counter")
+		writeSample(bw, "db_wait_count_total", nil, float64(st.WaitCount))
+		writeHeader(bw, "db_wait_seconds_total", "Time spent waiting for a database connection", "counter")
+		writeSample(bw, "db_wait_seconds_total", nil, st.WaitDuration.Seconds())
+	}
+}
+
+func promType(k Kind) string {
+	switch k {
+	case Counter:
+		return "counter"
+	case Histogram:
+		return "histogram"
+	case Summary:
+		return "summary"
+	}
+	return "gauge"
+}
+
+func withLabel(labels map[string]string, k, v string) map[string]string {
+	out := make(map[string]string, len(labels)+1)
+	for lk, lv := range labels {
+		out[lk] = lv
+	}
+	out[k] = v
+	return out
+}
+
+func formatFloat(v float64) string { return strconv.FormatFloat(v, 'g', -1, 64) }
+
+var helpEscaper = strings.NewReplacer(`\`, `\\`, "\n", `\n`)
+var valueEscaper = strings.NewReplacer(`\`, `\\`, "\n", `\n`, `"`, `\"`)
+
+func writeHeader(w *bufio.Writer, name, help, typ string) {
+	if help != "" {
+		w.WriteString("# HELP " + name + " " + helpEscaper.Replace(help) + "\n")
+	}
+	w.WriteString("# TYPE " + name + " " + typ + "\n")
+}
+
+func writeSample(w *bufio.Writer, name string, labels map[string]string, v float64) {
+	w.WriteString(name)
+	if len(labels) > 0 {
+		keys := make([]string, 0, len(labels))
+		for k := range labels {
+			keys = append(keys, k)
+		}
+		sort.Strings(keys)
+		w.WriteByte('{')
+		for i, k := range keys {
+			if i > 0 {
+				w.WriteByte(',')
+			}
+			w.WriteString(k + `="` + valueEscaper.Replace(labels[k]) + `"`)
+		}
+		w.WriteByte('}')
+	}
+	w.WriteByte(' ')
+	w.WriteString(formatFloat(v))
+	w.WriteByte('\n')
+}

metrics/query.go

@@ -0,0 +1,526 @@
+package metrics
+
+import (
+	"encoding/json"
+	"fmt"
+	"math"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
+)
+
+const (
+	maxQuerySeries = 50
+	maxQueryPoints = 1500
+)
+
+// Selector picks series by service, name and exact label matches, and groups
+// them by the GroupBy labels ("service" and "instance" work as labels too).
+type Selector struct {
+	Service string
+	Name    string
+	Match   map[string]string
+	GroupBy []string
+}
+
+// Aggregations supported by Query and Totals.
+var Aggregations = []string{"sum", "count", "rate", "avg", "min", "max", "last", "p50", "p90", "p95", "p99"}
+
+func validAgg(agg string) bool {
+	for _, a := range Aggregations {
+		if a == agg {
+			return true
+		}
+	}
+	return false
+}
+
+// DefaultAgg picks a sensible aggregation for a metric kind.
+func DefaultAgg(kind Kind) string {
+	switch kind {
+	case Counter:
+		return "sum"
+	case Gauge:
+		return "last"
+	case Histogram:
+		return "p95"
+	default:
+		return "avg"
+	}
+}
+
+// Point is [unix seconds, value or null] in JSON.
+type Point struct {
+	T int64
+	V *float64
+}
+
+func (p Point) MarshalJSON() ([]byte, error) {
+	if p.V == nil {
+		return []byte("[" + strconv.FormatInt(p.T, 10) + ",null]"), nil
+	}
+	return []byte("[" + strconv.FormatInt(p.T, 10) + "," + strconv.FormatFloat(*p.V, 'g', 10, 64) + "]"), nil
+}
+
+type SeriesResult struct {
+	Labels map[string]string `json:"labels"`
+	Points []Point           `json:"points"`
+}
+
+type QueryResult struct {
+	Name       string         `json:"name"`
+	Kind       Kind           `json:"kind"`
+	Agg        string         `json:"agg"`
+	Resolution int64          `json:"resolution"`
+	Step       int64          `json:"step"`
+	From       int64          `json:"from"`
+	To         int64          `json:"to"`
+	Series     []SeriesResult `json:"series"`
+	Truncated  bool           `json:"truncated"`
+}
+
+// matched is a series selected for a query, possibly only known from the DB.
+type matched struct {
+	id     uint
+	live   *Series // nil when this process never saw the series
+	labels map[string]string
+	svc    string
+	inst   string
+	kind   Kind
+	bounds []float64
+}
+
+func (m *matched) label(name string) string {
+	switch name {
+	case "service":
+		if v, ok := m.labels["service"]; ok {
+			return v
+		}
+		return m.svc
+	case "instance":
+		if v, ok := m.labels["instance"]; ok {
+			return v
+		}
+		return m.inst
+	}
+	return m.labels[name]
+}
+
+func (sel Selector) matches(m *matched) bool {
+	if sel.Service != "" && m.svc != sel.Service {
+		return false
+	}
+	for k, v := range sel.Match {
+		if m.label(k) != v {
+			return false
+		}
+	}
+	return true
+}
+
+func findSeries(sel Selector) ([]*matched, error) {
+	byKey := map[seriesKey]*matched{}
+
+	q := initializers.DB.Model(&models.MetricSeries{}).Where("name = ?", sel.Name)
+	if sel.Service != "" {
+		q = q.Where("service = ?", sel.Service)
+	}
+	var rows []models.MetricSeries
+	if err := q.Find(&rows).Error; err != nil {
+		return nil, err
+	}
+	for _, r := range rows {
+		m := &matched{id: r.ID, svc: r.Service, inst: r.Instance, kind: Kind(r.Kind), labels: map[string]string{}}
+		_ = json.Unmarshal([]byte(r.Labels), &m.labels)
+		if r.Bounds != "" {
+			_ = json.Unmarshal([]byte(r.Bounds), &m.bounds)
+		}
+		byKey[seriesKey{r.Service, r.Instance, r.Name, r.Labels}] = m
+	}
+
+	for _, s := range reg.all() {
+		if s.Name != sel.Name || (sel.Service != "" && s.Service != sel.Service) {
+			continue
+		}
+		k := s.key()
+		if m := byKey[k]; m != nil {
+			m.live = s
+			continue
+		}
+		byKey[k] = &matched{live: s, svc: s.Service, inst: s.Instance, kind: s.Kind, bounds: s.Bounds, labels: s.Labels}
+	}
+
+	out := make([]*matched, 0, len(byKey))
+	for _, m := range byKey {
+		if sel.matches(m) {
+			out = append(out, m)
+		}
+	}
+	return out, nil
+}
+
+type groupAcc struct {
+	labels  map[string]string
+	steps   map[int64]*Agg
+	lastSum map[int64]float64
+}
+
+// collect loads all matching data in [from, to] and folds it into groups and
+// step buckets chosen by bucketFn. Gauge "last" values are summed across the
+// series of a group (e.g. queue depth over all instances).
+func collect(sel Selector, from, to, res int64, bucketFn func(int64) int64) (map[string]*groupAcc, Kind, []float64, error) {
+	series, err := findSeries(sel)
+	if err != nil {
+		return nil, "", nil, err
+	}
+	if len(series) == 0 {
+		return map[string]*groupAcc{}, "", nil, nil
+	}
+	kind, bounds := series[0].kind, series[0].bounds
+
+	perSeries := map[*matched]map[int64]*Agg{}
+	add := func(m *matched, ts int64, a *Agg) {
+		b := bucketFn(ts)
+		steps := perSeries[m]
+		if steps == nil {
+			steps = map[int64]*Agg{}
+			perSeries[m] = steps
+		}
+		if cur := steps[b]; cur != nil {
+			cur.merge(a)
+		} else {
+			steps[b] = a.clone()
+		}
+	}
+
+	byID := map[uint]*matched{}
+	var ids []uint
+	for _, m := range series {
+		if m.id != 0 {
+			byID[m.id] = m
+			ids = append(ids, m.id)
+		}
+	}
+
+	load := func(resolution int, lo, hi int64) error {
+		if lo > hi || len(ids) == 0 {
+			return nil
+		}
+		for start := 0; start < len(ids); start += 300 {
+			end := min(start+300, len(ids))
+			var rows []models.MetricPoint
+			err := initializers.DB.
+				Where("resolution = ? AND series_id IN ? AND bucket >= ? AND bucket <= ?", resolution, ids[start:end], lo, hi).
+				Find(&rows).Error
+			if err != nil {
+				return err
+			}
+			for i := range rows {
+				add(byID[rows[i].SeriesID], rows[i].Bucket, pointToAgg(&rows[i]))
+			}
+		}
+		return nil
+	}
+
+	if res == resHour {
+		cut := rolledUntil.Load()
+		if err := load(resHour, hourOf(from), cut-1); err != nil {
+			return nil, "", nil, err
+		}
+		if err := load(resMinute, max(cut, from), to); err != nil {
+			return nil, "", nil, err
+		}
+	} else if err := load(resMinute, minuteOf(from), to); err != nil {
+		return nil, "", nil, err
+	}
+
+	// not yet flushed buckets
+	live := map[*Series]*matched{}
+	want := map[*Series]bool{}
+	for _, m := range series {
+		if m.live != nil {
+			live[m.live] = m
+			want[m.live] = true
+		}
+	}
+	for k, a := range defaultAgg.snapshotOpen(want) {
+		if k.bucket >= minuteOf(from) && k.bucket <= to {
+			add(live[k.s], k.bucket, a)
+		}
+	}
+
+	groups := map[string]*groupAcc{}
+	for m, steps := range perSeries {
+		labels := make(map[string]string, len(sel.GroupBy))
+		parts := make([]string, len(sel.GroupBy))
+		for i, g := range sel.GroupBy {
+			labels[g] = m.label(g)
+			parts[i] = labels[g]
+		}
+		gk := strings.Join(parts, "\x00")
+		g := groups[gk]
+		if g == nil {
+			g = &groupAcc{labels: labels, steps: map[int64]*Agg{}, lastSum: map[int64]float64{}}
+			groups[gk] = g
+		}
+		for b, a := range steps {
+			if cur := g.steps[b]; cur != nil {
+				cur.merge(a)
+			} else {
+				g.steps[b] = a.clone()
+			}
+			g.lastSum[b] += a.Last
+		}
+	}
+	return groups, kind, bounds, nil
+}
+
+func evaluate(a *Agg, lastSum float64, agg string, kind Kind, bounds []float64, span int64) (float64, bool) {
+	if a == nil || a.Count == 0 {
+		return 0, false
+	}
+	switch agg {
+	case "count":
+		return float64(a.Count), true
+	case "sum":
+		return a.Sum, true
+	case "rate":
+		if span <= 0 {
+			return 0, false
+		}
+		if kind == Counter {
+			return a.Sum / float64(span), true
+		}
+		return float64(a.Count) / float64(span), true
+	case "avg":
+		return a.Sum / float64(a.Count), true
+	case "min":
+		return a.Min, true
+	case "max":
+		return a.Max, true
+	case "last":
+		return lastSum, true
+	case "p50", "p90", "p95", "p99":
+		q, _ := strconv.ParseFloat(agg[1:], 64)
+		return quantile(a, bounds, q/100)
+	}
+	return 0, false
+}
+
+func pickResolution(from, to, step int64) int64 {
+	minuteHorizon := time.Now().Unix() - int64(initializers.MetricsMinuteRetentionDays)*86400
+	if to-from > 48*3600 || step >= resHour || from < minuteHorizon {
+		return resHour
+	}
+	return resMinute
+}
+
+// Query returns one time series per group with values at every step.
+func Query(sel Selector, agg string, from, to, step int64) (*QueryResult, error) {
+	if sel.Name == "" {
+		return nil, fmt.Errorf("name is required")
+	}
+	if to <= from {
+		return nil, fmt.Errorf("to must be after from")
+	}
+	res := pickResolution(from, to, step)
+	if step < res {
+		step = res
+	}
+	if (to-from)/step > maxQueryPoints {
+		step = (to - from) / maxQueryPoints
+	}
+	step = int64(math.Ceil(float64(step)/float64(res))) * res
+	align := func(ts int64) int64 { return ts - ((ts%step)+step)%step }
+
+	groups, kind, bounds, err := collect(sel, from, to, res, align)
+	if err != nil {
+		return nil, err
+	}
+	if agg == "" {
+		agg = DefaultAgg(kind)
+	}
+	if !validAgg(agg) {
+		return nil, fmt.Errorf("unknown aggregation %q", agg)
+	}
+
+	result := &QueryResult{Name: sel.Name, Kind: kind, Agg: agg, Resolution: res, Step: step, From: from, To: to, Series: []SeriesResult{}}
+	keys := make([]string, 0, len(groups))
+	for k := range groups {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	if len(keys) > maxQuerySeries {
+		keys = keys[:maxQuerySeries]
+		result.Truncated = true
+	}
+	for _, k := range keys {
+		g := groups[k]
+		sr := SeriesResult{Labels: g.labels}
+		for b := align(from); b <= to; b += step {
+			p := Point{T: b}
+			if v, ok := evaluate(g.steps[b], g.lastSum[b], agg, kind, bounds, step); ok && !math.IsNaN(v) && !math.IsInf(v, 0) {
+				p.V = &v
+			}
+			sr.Points = append(sr.Points, p)
+		}
+		result.Series = append(result.Series, sr)
+	}
+	return result, nil
+}
+
+// Total is the whole-range aggregate of one group.
+type Total struct {
+	Labels map[string]string `json:"labels"`
+	Count  int64             `json:"count"`
+	Sum    float64           `json:"sum"`
+	Min    *float64          `json:"min"`
+	Max    *float64          `json:"max"`
+	Avg    *float64          `json:"avg"`
+	Last   *float64          `json:"last"`
+	Rate   *float64          `json:"rate"`
+	P50    *float64          `json:"p50"`
+	P95    *float64          `json:"p95"`
+	P99    *float64          `json:"p99"`
+}
+
+// Totals aggregates [from, to] into one value set per group.
+func Totals(sel Selector, from, to int64) ([]Total, error) {
+	res := pickResolution(from, to, 0)
+	groups, kind, bounds, err := collect(sel, from, to, res, func(int64) int64 { return 0 })
+	if err != nil {
+		return nil, err
+	}
+	span := to - from
+	out := make([]Total, 0, len(groups))
+	for _, g := range groups {
+		a := g.steps[0]
+		if a == nil {
+			continue
+		}
+		t := Total{Labels: g.labels, Count: a.Count, Sum: a.Sum}
+		val := func(agg string) *float64 {
+			if v, ok := evaluate(a, g.lastSum[0], agg, kind, bounds, span); ok && !math.IsNaN(v) && !math.IsInf(v, 0) {
+				return &v
+			}
+			return nil
+		}
+		t.Min, t.Max, t.Avg, t.Last, t.Rate = val("min"), val("max"), val("avg"), val("last"), val("rate")
+		t.P50, t.P95, t.P99 = val("p50"), val("p95"), val("p99")
+		out = append(out, t)
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Count > out[j].Count })
+	return out, nil
+}
+
+// Sum returns the summed value of a counter (or count of observations for
+// other kinds) over the range, across all matching series.
+func Sum(sel Selector, from, to int64) float64 {
+	sel.GroupBy = nil
+	totals, err := Totals(sel, from, to)
+	if err != nil || len(totals) == 0 {
+		return 0
+	}
+	if f := familyByName(sel.Name); f != nil && f.Kind != Counter {
+		return float64(totals[0].Count)
+	}
+	return totals[0].Sum
+}
+
+// CatalogEntry describes one metric name of one service.
+type CatalogEntry struct {
+	Service   string              `json:"service"`
+	Name      string              `json:"name"`
+	Kind      Kind                `json:"kind"`
+	Help      string              `json:"help"`
+	Labels    map[string][]string `json:"labels"`
+	Instances []string            `json:"instances"`
+	Series    int                 `json:"series"`
+	LastSeen  int64               `json:"lastSeen"`
+}
+
+// Catalog lists every metric known in the DB or in memory.
+func Catalog(service string) ([]CatalogEntry, error) {
+	type acc struct {
+		e      CatalogEntry
+		labels map[string]map[string]bool
+		inst   map[string]bool
+		keys   map[seriesKey]bool
+	}
+	entries := map[string]*acc{}
+	addSeries := func(svc, inst, name string, kind Kind, help, labelsJSON string, labels map[string]string, lastSeen int64) {
+		k := svc + "/" + name
+		a := entries[k]
+		if a == nil {
+			a = &acc{e: CatalogEntry{Service: svc, Name: name, Kind: kind, Help: help}, labels: map[string]map[string]bool{}, inst: map[string]bool{}, keys: map[seriesKey]bool{}}
+			entries[k] = a
+		}
+		if a.e.Help == "" {
+			a.e.Help = help
+		}
+		sk := seriesKey{svc, inst, name, labelsJSON}
+		if !a.keys[sk] {
+			a.keys[sk] = true
+			a.e.Series++
+		}
+		a.inst[inst] = true
+		for lk, lv := range labels {
+			if a.labels[lk] == nil {
+				a.labels[lk] = map[string]bool{}
+			}
+			a.labels[lk][lv] = true
+		}
+		if lastSeen > a.e.LastSeen {
+			a.e.LastSeen = lastSeen
+		}
+	}
+
+	q := initializers.DB.Model(&models.MetricSeries{})
+	if service != "" {
+		q = q.Where("service = ?", service)
+	}
+	var rows []models.MetricSeries
+	if err := q.Find(&rows).Error; err != nil {
+		return nil, err
+	}
+	for _, r := range rows {
+		labels := map[string]string{}
+		_ = json.Unmarshal([]byte(r.Labels), &labels)
+		addSeries(r.Service, r.Instance, r.Name, Kind(r.Kind), r.Help, r.Labels, labels, r.LastSeen)
+	}
+	for _, s := range reg.all() {
+		if service != "" && s.Service != service {
+			continue
+		}
+		addSeries(s.Service, s.Instance, s.Name, s.Kind, s.Help, s.LabelsJSON, s.Labels, reg.seen(s))
+	}
+
+	out := make([]CatalogEntry, 0, len(entries))
+	for _, a := range entries {
+		a.e.Labels = map[string][]string{}
+		for lk, vals := range a.labels {
+			list := make([]string, 0, len(vals))
+			for v := range vals {
+				list = append(list, v)
+			}
+			sort.Strings(list)
+			a.e.Labels[lk] = list
+		}
+		for inst := range a.inst {
+			a.e.Instances = append(a.e.Instances, inst)
+		}
+		sort.Strings(a.e.Instances)
+		out = append(out, a.e)
+	}
+	sort.Slice(out, func(i, j int) bool {
+		if out[i].Service != out[j].Service {
+			return out[i].Service < out[j].Service
+		}
+		return out[i].Name < out[j].Name
+	})
+	return out, nil
+}

metrics/registry.go

@@ -0,0 +1,284 @@
+package metrics
+
+import (
+	"encoding/json"
+	"errors"
+	"os"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+)
+
+type Kind string
+
+const (
+	Counter   Kind = "counter"   // value is an increment; Sum holds the total
+	Gauge     Kind = "gauge"     // value is a sample; Last holds the latest
+	Summary   Kind = "summary"   // pre-aggregated count/sum/min/max from a client
+	Histogram Kind = "histogram" // observations bucketed by Bounds
+)
+
+// MainService is the service name the API uses for its own metrics.
+const MainService = "main"
+
+var (
+	errSeriesLimit  = errors.New("series limit reached")
+	errKindMismatch = errors.New("metric reported with a different type")
+)
+
+// Limits for series created from external telemetry.
+const (
+	maxSeriesPerName = 200
+	maxExternal      = 5000
+)
+
+var hostname = func() string {
+	h, err := os.Hostname()
+	if err != nil || h == "" {
+		return "api"
+	}
+	return h
+}()
+
+// Instance returns the instance name used for this process' own metrics.
+func Instance() string { return hostname }
+
+// Series is one metric name + label set from one service instance. Series are
+// interned, so pointer equality means same series.
+type Series struct {
+	Service    string
+	Instance   string
+	Name       string
+	Labels     map[string]string
+	LabelsJSON string
+	Kind       Kind
+	Bounds     []float64
+	Help       string
+}
+
+type seriesKey struct{ service, instance, name, labels string }
+
+func (s *Series) key() seriesKey {
+	return seriesKey{s.Service, s.Instance, s.Name, s.LabelsJSON}
+}
+
+// Label returns a label value, including the service/instance pseudo labels.
+func (s *Series) Label(name string) string {
+	switch name {
+	case "service":
+		if v, ok := s.Labels["service"]; ok {
+			return v
+		}
+		return s.Service
+	case "instance":
+		if v, ok := s.Labels["instance"]; ok {
+			return v
+		}
+		return s.Instance
+	}
+	return s.Labels[name]
+}
+
+// canonicalLabels serializes labels deterministically (encoding/json sorts map keys).
+func canonicalLabels(labels map[string]string) string {
+	if len(labels) == 0 {
+		return "{}"
+	}
+	b, _ := json.Marshal(labels)
+	return string(b)
+}
+
+type registry struct {
+	mu       sync.RWMutex
+	series   map[seriesKey]*Series
+	perName  map[string]int // external series per service/name
+	external int
+	lastSeen map[*Series]int64
+}
+
+var reg = &registry{
+	series:   make(map[seriesKey]*Series),
+	perName:  make(map[string]int),
+	lastSeen: make(map[*Series]int64),
+}
+
+func (r *registry) get(service, instance, name string, labels map[string]string, kind Kind, bounds []float64, help string, external bool) (*Series, error) {
+	lj := canonicalLabels(labels)
+	k := seriesKey{service, instance, name, lj}
+
+	r.mu.RLock()
+	s := r.series[k]
+	r.mu.RUnlock()
+	if s != nil {
+		if s.Kind != kind {
+			return nil, errKindMismatch
+		}
+		return s, nil
+	}
+
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if s = r.series[k]; s != nil {
+		return s, nil
+	}
+	nameKey := service + "/" + name
+	if external {
+		if r.external >= maxExternal || r.perName[nameKey] >= maxSeriesPerName {
+			return nil, errSeriesLimit
+		}
+		r.external++
+		r.perName[nameKey]++
+	}
+	copied := make(map[string]string, len(labels))
+	for k, v := range labels {
+		copied[k] = v
+	}
+	s = &Series{
+		Service:    service,
+		Instance:   instance,
+		Name:       name,
+		Labels:     copied,
+		LabelsJSON: lj,
+		Kind:       kind,
+		Bounds:     bounds,
+		Help:       help,
+	}
+	r.series[k] = s
+	return s, nil
+}
+
+func (r *registry) lookup(k seriesKey) *Series {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	return r.series[k]
+}
+
+// all returns every known series, sorted by name then labels.
+func (r *registry) all() []*Series {
+	r.mu.RLock()
+	out := make([]*Series, 0, len(r.series))
+	for _, s := range r.series {
+		out = append(out, s)
+	}
+	r.mu.RUnlock()
+	sort.Slice(out, func(i, j int) bool {
+		if out[i].Name != out[j].Name {
+			return out[i].Name < out[j].Name
+		}
+		if out[i].Service != out[j].Service {
+			return out[i].Service < out[j].Service
+		}
+		if out[i].Instance != out[j].Instance {
+			return out[i].Instance < out[j].Instance
+		}
+		return out[i].LabelsJSON < out[j].LabelsJSON
+	})
+	return out
+}
+
+func (r *registry) touch(s *Series, ts int64) {
+	r.mu.Lock()
+	if ts > r.lastSeen[s] {
+		r.lastSeen[s] = ts
+	}
+	r.mu.Unlock()
+}
+
+func (r *registry) seen(s *Series) int64 {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	return r.lastSeen[s]
+}
+
+// Family is a metric emitted by this process, with a fixed set of label names.
+type Family struct {
+	Name       string
+	Help       string
+	Kind       Kind
+	Bounds     []float64
+	LabelNames []string
+
+	mu    sync.Mutex
+	cache map[string]*Series
+}
+
+var (
+	familiesMu sync.Mutex
+	families   = map[string]*Family{}
+)
+
+func newFamily(name, help string, kind Kind, bounds []float64, labels []string) *Family {
+	f := &Family{Name: name, Help: help, Kind: kind, Bounds: bounds, LabelNames: labels, cache: map[string]*Series{}}
+	familiesMu.Lock()
+	families[name] = f
+	familiesMu.Unlock()
+	return f
+}
+
+func familyByName(name string) *Family {
+	familiesMu.Lock()
+	defer familiesMu.Unlock()
+	return families[name]
+}
+
+func NewCounter(name, help string, labels ...string) *Family {
+	return newFamily(name, help, Counter, nil, labels)
+}
+
+func NewGauge(name, help string, labels ...string) *Family {
+	return newFamily(name, help, Gauge, nil, labels)
+}
+
+func NewHistogram(name, help string, bounds []float64, labels ...string) *Family {
+	return newFamily(name, help, Histogram, bounds, labels)
+}
+
+func (f *Family) series(values []string) *Series {
+	key := strings.Join(values, "\x00")
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if s := f.cache[key]; s != nil {
+		return s
+	}
+	labels := make(map[string]string, len(f.LabelNames))
+	for i, n := range f.LabelNames {
+		if i < len(values) {
+			labels[n] = values[i]
+		} else {
+			labels[n] = ""
+		}
+	}
+	s, err := reg.get(MainService, Instance(), f.Name, labels, f.Kind, f.Bounds, f.Help, false)
+	if err != nil {
+		return nil
+	}
+	f.cache[key] = s
+	return s
+}
+
+// Add increments a counter. Zero increments are ignored.
+func (f *Family) Add(v float64, labelValues ...string) {
+	if v == 0 {
+		return
+	}
+	if s := f.series(labelValues); s != nil {
+		defaultAgg.observe(s, time.Now().Unix(), v)
+	}
+}
+
+func (f *Family) Inc(labelValues ...string) { f.Add(1, labelValues...) }
+
+// Set records a gauge sample.
+func (f *Family) Set(v float64, labelValues ...string) {
+	if s := f.series(labelValues); s != nil {
+		defaultAgg.observe(s, time.Now().Unix(), v)
+	}
+}
+
+// Observe records a histogram observation.
+func (f *Family) Observe(v float64, labelValues ...string) {
+	if s := f.series(labelValues); s != nil {
+		defaultAgg.observe(s, time.Now().Unix(), v)
+	}
+}

metrics/services.go

@@ -0,0 +1,325 @@
+package metrics
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"math"
+	"regexp"
+	"sync"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
+	"gorm.io/gorm/clause"
+)
+
+var nameRe = regexp.MustCompile(`^[a-z_][a-z0-9_]{0,63}$`)
+
+// ValidName reports whether a service, metric or label name is acceptable.
+func ValidName(s string) bool { return nameRe.MatchString(s) }
+
+const (
+	defaultHeartbeat = 30
+	maxLabels        = 8
+	maxLabelValue    = 128
+)
+
+// normalizeTs converts unix seconds or milliseconds to seconds and clamps it
+// to [now-1h, now+1m]; anything outside becomes now.
+func normalizeTs(ts int64, now int64) int64 {
+	if ts <= 0 {
+		return now
+	}
+	if ts > 1e12 {
+		ts /= 1000
+	}
+	if ts < now-3600 || ts > now+60 {
+		return now
+	}
+	return ts
+}
+
+// IngestResult reports what happened to a telemetry batch.
+type IngestResult struct {
+	Accepted int            `json:"accepted"`
+	Dropped  int            `json:"dropped"`
+	Reasons  map[string]int `json:"reasons,omitempty"`
+}
+
+// Ingest records external metric samples for a service instance.
+func Ingest(service, instance string, samples []models.TelemetryMetric, now time.Time) IngestResult {
+	res := IngestResult{Reasons: map[string]int{}}
+	drop := func(reason string) {
+		res.Dropped++
+		res.Reasons[reason]++
+	}
+	nowTs := now.Unix()
+
+	for _, m := range samples {
+		if !ValidName(m.Name) {
+			drop("invalid_name")
+			continue
+		}
+		if len(m.Labels) > maxLabels {
+			drop("too_many_labels")
+			continue
+		}
+		badLabel := false
+		for k, v := range m.Labels {
+			if !ValidName(k) || len(v) > maxLabelValue {
+				badLabel = true
+				break
+			}
+		}
+		if badLabel {
+			drop("invalid_label")
+			continue
+		}
+		var kind Kind
+		switch m.Type {
+		case "counter":
+			kind = Counter
+		case "gauge", "":
+			kind = Gauge
+		case "summary":
+			kind = Summary
+		default:
+			drop("invalid_type")
+			continue
+		}
+		if math.IsNaN(m.Value) || math.IsInf(m.Value, 0) || math.IsNaN(m.Sum) || math.IsInf(m.Sum, 0) {
+			drop("invalid_value")
+			continue
+		}
+		if kind == Counter && m.Value < 0 {
+			drop("negative_counter")
+			continue
+		}
+		if kind == Summary && m.Count <= 0 {
+			drop("empty_summary")
+			continue
+		}
+		help := m.Help
+		if len(help) > 256 {
+			help = help[:256]
+		}
+		s, err := reg.get(service, instance, m.Name, m.Labels, kind, nil, help, true)
+		if err != nil {
+			if errors.Is(err, errKindMismatch) {
+				drop("type_mismatch")
+			} else {
+				drop("series_limit")
+			}
+			continue
+		}
+		ts := normalizeTs(m.Ts, nowTs)
+		if kind == Summary {
+			lo, hi := m.Min, m.Max
+			if lo == 0 && hi == 0 {
+				avg := m.Sum / float64(m.Count)
+				lo, hi = avg, avg
+			}
+			defaultAgg.observeSummary(s, ts, m.Count, m.Sum, lo, hi)
+		} else {
+			defaultAgg.observe(s, ts, m.Value)
+		}
+		res.Accepted++
+	}
+	TelemetrySamples.Add(float64(res.Accepted), "accepted")
+	TelemetrySamples.Add(float64(res.Dropped), "dropped")
+	return res
+}
+
+func jsonString(v map[string]any) string {
+	if len(v) == 0 {
+		return ""
+	}
+	b, err := json.Marshal(v)
+	if err != nil {
+		return ""
+	}
+	return string(b)
+}
+
+func truncate(s string, n int) string {
+	if len(s) > n {
+		return s[:n]
+	}
+	return s
+}
+
+// Heartbeat upserts the service instance row.
+func Heartbeat(b *models.TelemetryBatch, remoteAddr string, now time.Time) error {
+	interval := b.HeartbeatIntervalSec
+	if interval <= 0 {
+		interval = defaultHeartbeat
+	}
+	status := b.Status
+	if status == "" {
+		status = "ok"
+	}
+	row := models.ServiceInstance{
+		Service:              b.Service,
+		Instance:             b.Instance,
+		Version:              truncate(b.Version, 64),
+		Status:               truncate(status, 16),
+		StatusMessage:        truncate(b.StatusMessage, 512),
+		StartedAt:            normalizeStarted(b.StartedAt),
+		LastSeen:             now.Unix(),
+		HeartbeatIntervalSec: interval,
+		RemoteAddr:           truncate(remoteAddr, 64),
+		Meta:                 jsonString(b.Meta),
+	}
+	return initializers.DB.Clauses(clause.OnConflict{
+		Columns: []clause.Column{{Name: "service"}, {Name: "instance"}},
+		DoUpdates: clause.AssignmentColumns([]string{
+			"version", "status", "status_message", "started_at", "last_seen",
+			"heartbeat_interval_sec", "remote_addr", "meta", "updated_at",
+		}),
+	}).Create(&row).Error
+}
+
+func normalizeStarted(ts int64) int64 {
+	if ts > 1e12 {
+		return ts / 1000
+	}
+	return ts
+}
+
+// RecordEvents stores service events.
+func RecordEvents(service, instance string, events []models.TelemetryEvent, now time.Time) error {
+	if len(events) == 0 {
+		return nil
+	}
+	rows := make([]models.ServiceEvent, 0, len(events))
+	for _, e := range events {
+		level := e.Level
+		switch level {
+		case "debug", "info", "warn", "error":
+		case "warning":
+			level = "warn"
+		default:
+			level = "info"
+		}
+		ts := e.Ts
+		if ts <= 0 {
+			ts = now.UnixMilli()
+		} else if ts < 1e12 {
+			ts *= 1000
+		}
+		rows = append(rows, models.ServiceEvent{
+			Service: service, Instance: instance, Level: level,
+			Message: truncate(e.Message, 1024), Fields: jsonString(e.Fields), TS: ts,
+		})
+	}
+	return initializers.DB.CreateInBatches(&rows, 100).Error
+}
+
+// Event records a single event from the API itself.
+func Event(level, message string, fields map[string]any) {
+	err := RecordEvents(MainService, Instance(), []models.TelemetryEvent{{Level: level, Message: message, Fields: fields}}, time.Now())
+	if err != nil {
+		fmt.Printf("metrics: failed to record event: %v\n", err)
+	}
+}
+
+// Online reports whether an instance has sent a heartbeat recently enough.
+func Online(inst *models.ServiceInstance, now time.Time) bool {
+	interval := inst.HeartbeatIntervalSec
+	if interval <= 0 {
+		interval = defaultHeartbeat
+	}
+	return now.Unix()-inst.LastSeen < int64(3*interval)
+}
+
+var (
+	upMu    sync.Mutex
+	upState = map[string]bool{} // "service/instance" -> online
+)
+
+// ServiceUp returns the last known online state of every instance.
+func ServiceUp() map[[2]string]bool {
+	upMu.Lock()
+	defer upMu.Unlock()
+	out := make(map[[2]string]bool, len(upState))
+	for k, v := range upState {
+		var svc, inst string
+		for i := 0; i < len(k); i++ {
+			if k[i] == '/' {
+				svc, inst = k[:i], k[i+1:]
+				break
+			}
+		}
+		out[[2]string{svc, inst}] = v
+	}
+	return out
+}
+
+// checkServices records online/offline transitions as events.
+func checkServices(now time.Time) {
+	var rows []models.ServiceInstance
+	if err := initializers.DB.Find(&rows).Error; err != nil {
+		return
+	}
+	upMu.Lock()
+	known := make(map[string]bool, len(rows))
+	type change struct {
+		inst   models.ServiceInstance
+		online bool
+	}
+	var changes []change
+	for _, r := range rows {
+		k := r.Service + "/" + r.Instance
+		known[k] = true
+		online := Online(&r, now)
+		prev, seen := upState[k]
+		upState[k] = online
+		if seen && prev != online {
+			changes = append(changes, change{r, online})
+		}
+	}
+	for k := range upState {
+		if !known[k] {
+			delete(upState, k)
+		}
+	}
+	upMu.Unlock()
+
+	for _, ch := range changes {
+		level, msg := "info", "back online"
+		if !ch.online {
+			level, msg = "warn", "heartbeat lost"
+		}
+		_ = RecordEvents(ch.inst.Service, ch.inst.Instance, []models.TelemetryEvent{{
+			Level: level, Message: msg,
+			Fields: map[string]any{"kind": "status", "lastSeen": ch.inst.LastSeen},
+		}}, now)
+	}
+}
+
+var (
+	buildMu      sync.Mutex
+	buildVersion = "dev"
+	startedAt    = time.Now()
+)
+
+// SetVersion sets the version reported in the API's own heartbeat.
+func SetVersion(v string) {
+	buildMu.Lock()
+	buildVersion = v
+	buildMu.Unlock()
+}
+
+// StartedAt is when this process started.
+func StartedAt() time.Time { return startedAt }
+
+func selfHeartbeat(now time.Time) {
+	buildMu.Lock()
+	v := buildVersion
+	buildMu.Unlock()
+	_ = Heartbeat(&models.TelemetryBatch{
+		Service: MainService, Instance: Instance(), Version: v,
+		StartedAt: startedAt.Unix(), HeartbeatIntervalSec: 30, Status: "ok",
+	}, "", now)
+}

metrics/store.go

@@ -0,0 +1,371 @@
+package metrics
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"sort"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+)
+
+const (
+	resMinute = 60
+	resHour   = 3600
+	// points that cannot be written for this long are dropped
+	maxRetryAge = 15 * 60
+)
+
+var (
+	idMu     sync.Mutex
+	seriesID = map[*Series]uint{}
+
+	// rolledUntil is the start of the first hour without an hourly rollup.
+	rolledUntil atomic.Int64
+	// flushMu serializes writers of metric_points (flush, rollup, retention).
+	flushMu sync.Mutex
+)
+
+func pointToAgg(p *models.MetricPoint) *Agg {
+	a := &Agg{Count: p.Count, Sum: p.Sum, Min: p.Min, Max: p.Max, Last: p.Last, LastTs: p.Bucket}
+	if p.Hist != "" {
+		_ = json.Unmarshal([]byte(p.Hist), &a.Hist)
+	}
+	return a
+}
+
+func aggToPoint(id uint, res int, bucket int64, a *Agg) models.MetricPoint {
+	return models.MetricPoint{
+		SeriesID: id, Resolution: res, Bucket: bucket,
+		Count: a.Count, Sum: a.Sum, Min: a.Min, Max: a.Max, Last: a.Last, Hist: a.histJSON(),
+	}
+}
+
+// resolveIDs makes sure every series has a metric_series row.
+func resolveIDs(list []*Series) (map[*Series]uint, error) {
+	out := make(map[*Series]uint, len(list))
+	var missing []*Series
+	idMu.Lock()
+	for _, s := range list {
+		if id, ok := seriesID[s]; ok {
+			out[s] = id
+		} else {
+			missing = append(missing, s)
+		}
+	}
+	idMu.Unlock()
+
+	for _, s := range missing {
+		bounds := ""
+		if len(s.Bounds) > 0 {
+			b, _ := json.Marshal(s.Bounds)
+			bounds = string(b)
+		}
+		row := models.MetricSeries{
+			Service: s.Service, Instance: s.Instance, Name: s.Name, Labels: s.LabelsJSON,
+			Kind: string(s.Kind), Bounds: bounds, Help: s.Help, LastSeen: time.Now().Unix(),
+		}
+		if err := initializers.DB.Clauses(clause.OnConflict{DoNothing: true}).Create(&row).Error; err != nil {
+			return nil, err
+		}
+		if row.ID == 0 {
+			if err := initializers.DB.
+				Where("service = ? AND instance = ? AND name = ? AND labels = ?", s.Service, s.Instance, s.Name, s.LabelsJSON).
+				First(&row).Error; err != nil {
+				return nil, err
+			}
+		}
+		idMu.Lock()
+		seriesID[s] = row.ID
+		idMu.Unlock()
+		out[s] = row.ID
+	}
+	return out, nil
+}
+
+type pointID struct {
+	id     uint
+	bucket int64
+}
+
+// persist merges the given buckets into existing rows and upserts them. The
+// read-merge-write runs in one transaction; this process is the only writer.
+func persist(tx *gorm.DB, res int, points map[pointID]*Agg) error {
+	if len(points) == 0 {
+		return nil
+	}
+	keys := make([]pointID, 0, len(points))
+	for k := range points {
+		keys = append(keys, k)
+	}
+	sort.Slice(keys, func(i, j int) bool {
+		if keys[i].id != keys[j].id {
+			return keys[i].id < keys[j].id
+		}
+		return keys[i].bucket < keys[j].bucket
+	})
+
+	for start := 0; start < len(keys); start += 400 {
+		end := min(start+400, len(keys))
+		chunk := keys[start:end]
+		idSet, bucketSet := map[uint]bool{}, map[int64]bool{}
+		for _, k := range chunk {
+			idSet[k.id] = true
+			bucketSet[k.bucket] = true
+		}
+		ids := make([]uint, 0, len(idSet))
+		for id := range idSet {
+			ids = append(ids, id)
+		}
+		buckets := make([]int64, 0, len(bucketSet))
+		for b := range bucketSet {
+			buckets = append(buckets, b)
+		}
+
+		var existing []models.MetricPoint
+		if err := tx.Where("resolution = ? AND series_id IN ? AND bucket IN ?", res, ids, buckets).Find(&existing).Error; err != nil {
+			return err
+		}
+		prev := make(map[pointID]*Agg, len(existing))
+		for i := range existing {
+			prev[pointID{existing[i].SeriesID, existing[i].Bucket}] = pointToAgg(&existing[i])
+		}
+
+		rows := make([]models.MetricPoint, 0, len(chunk))
+		for _, k := range chunk {
+			a := points[k]
+			if old := prev[k]; old != nil {
+				old.merge(a)
+				a = old
+			}
+			rows = append(rows, aggToPoint(k.id, res, k.bucket, a))
+		}
+		err := tx.Clauses(clause.OnConflict{
+			Columns:   []clause.Column{{Name: "series_id"}, {Name: "resolution"}, {Name: "bucket"}},
+			DoUpdates: clause.AssignmentColumns([]string{"count", "sum", "min", "max", "last", "hist"}),
+		}).CreateInBatches(&rows, 200).Error
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+type pipelineStatus struct {
+	mu            sync.Mutex
+	LastFlush     int64  `json:"lastFlush"`
+	LastFlushMs   int64  `json:"lastFlushMs"`
+	LastFlushErr  string `json:"lastFlushError"`
+	LastRollup    int64  `json:"lastRollup"`
+	LastRetention int64  `json:"lastRetention"`
+	Dropped       int64  `json:"droppedPoints"`
+}
+
+var status pipelineStatus
+
+// flush writes every bucket that started before `before`.
+func flush(before int64) error {
+	flushMu.Lock()
+	defer flushMu.Unlock()
+
+	started := time.Now()
+	points := defaultAgg.takeClosed(before)
+	if len(points) == 0 {
+		return nil
+	}
+
+	fail := func(err error) error {
+		now := time.Now().Unix()
+		keep := make(map[pointKey]*Agg)
+		var dropped int64
+		for k, a := range points {
+			if now-k.bucket <= maxRetryAge {
+				keep[k] = a
+			} else {
+				dropped++
+			}
+		}
+		defaultAgg.putBack(keep)
+		status.mu.Lock()
+		status.LastFlushErr = err.Error()
+		status.Dropped += dropped
+		status.mu.Unlock()
+		if dropped > 0 {
+			PointsDropped.Add(float64(dropped))
+		}
+		return err
+	}
+
+	set := map[*Series]bool{}
+	for k := range points {
+		set[k.s] = true
+	}
+	list := make([]*Series, 0, len(set))
+	for s := range set {
+		list = append(list, s)
+	}
+	ids, err := resolveIDs(list)
+	if err != nil {
+		return fail(fmt.Errorf("resolve series: %w", err))
+	}
+
+	rows := make(map[pointID]*Agg, len(points))
+	for k, a := range points {
+		rows[pointID{ids[k.s], k.bucket}] = a
+	}
+	err = initializers.DB.Transaction(func(tx *gorm.DB) error {
+		return persist(tx, resMinute, rows)
+	})
+	if err != nil {
+		return fail(fmt.Errorf("write points: %w", err))
+	}
+
+	// last_seen lets the catalog hide series that stopped reporting
+	idList := make([]uint, 0, len(ids))
+	for _, id := range ids {
+		idList = append(idList, id)
+	}
+	for start := 0; start < len(idList); start += 500 {
+		end := min(start+500, len(idList))
+		initializers.DB.Model(&models.MetricSeries{}).Where("id IN ?", idList[start:end]).
+			Update("last_seen", time.Now().Unix())
+	}
+
+	status.mu.Lock()
+	status.LastFlush = time.Now().Unix()
+	status.LastFlushMs = time.Since(started).Milliseconds()
+	status.LastFlushErr = ""
+	status.mu.Unlock()
+	return nil
+}
+
+func hourOf(ts int64) int64 { return ts - ts%resHour }
+
+// rollup aggregates complete hours of minute data into hourly points. The last
+// rolled hour is redone each time to pick up late telemetry; it is idempotent.
+func rollup(now int64) error {
+	flushMu.Lock()
+	defer flushMu.Unlock()
+
+	// the minute data of an hour is complete ~1 flush after the hour ends
+	lastFull := hourOf(now - 2*resMinute)
+	start := rolledUntil.Load()
+	if start == 0 {
+		var maxHour, minMinute *int64
+		initializers.DB.Model(&models.MetricPoint{}).Where("resolution = ?", resHour).Select("MAX(bucket)").Scan(&maxHour)
+		if maxHour != nil {
+			start = *maxHour + resHour
+		} else {
+			initializers.DB.Model(&models.MetricPoint{}).Where("resolution = ?", resMinute).Select("MIN(bucket)").Scan(&minMinute)
+			if minMinute == nil {
+				status.mu.Lock()
+				status.LastRollup = now
+				status.mu.Unlock()
+				return nil
+			}
+			start = hourOf(*minMinute)
+		}
+	}
+	from := start - resHour
+	if earliest := hourOf(now - int64(initializers.MetricsMinuteRetentionDays)*86400); from < earliest {
+		from = earliest
+	}
+
+	for h := from; h < lastFull; h += resHour {
+		var rows []models.MetricPoint
+		err := initializers.DB.Where("resolution = ? AND bucket >= ? AND bucket < ?", resMinute, h, h+resHour).
+			Order("bucket asc").Find(&rows).Error
+		if err != nil {
+			return err
+		}
+		if len(rows) == 0 {
+			continue
+		}
+		merged := map[pointID]*Agg{}
+		for i := range rows {
+			k := pointID{rows[i].SeriesID, h}
+			a := merged[k]
+			if a == nil {
+				a = &Agg{}
+				merged[k] = a
+			}
+			a.merge(pointToAgg(&rows[i]))
+		}
+		// replace (not merge) so re-rolling an hour is idempotent
+		out := make([]models.MetricPoint, 0, len(merged))
+		for k, a := range merged {
+			out = append(out, aggToPoint(k.id, resHour, k.bucket, a))
+		}
+		err = initializers.DB.Clauses(clause.OnConflict{
+			Columns:   []clause.Column{{Name: "series_id"}, {Name: "resolution"}, {Name: "bucket"}},
+			DoUpdates: clause.AssignmentColumns([]string{"count", "sum", "min", "max", "last", "hist"}),
+		}).CreateInBatches(&out, 200).Error
+		if err != nil {
+			return err
+		}
+	}
+	if lastFull > rolledUntil.Load() {
+		rolledUntil.Store(lastFull)
+	}
+	status.mu.Lock()
+	status.LastRollup = now
+	status.mu.Unlock()
+	return nil
+}
+
+// retention deletes data older than the configured windows.
+func retention(now time.Time) error {
+	flushMu.Lock()
+	defer flushMu.Unlock()
+
+	day := int64(86400)
+	ts := now.Unix()
+	steps := []struct {
+		what string
+		run  func() error
+	}{
+		{"minute points", func() error {
+			return initializers.DB.Where("resolution = ? AND bucket < ?", resMinute, ts-int64(initializers.MetricsMinuteRetentionDays)*day).
+				Delete(&models.MetricPoint{}).Error
+		}},
+		{"hour points", func() error {
+			return initializers.DB.Where("resolution = ? AND bucket < ?", resHour, ts-int64(initializers.MetricsHourRetentionDays)*day).
+				Delete(&models.MetricPoint{}).Error
+		}},
+		{"events", func() error {
+			return initializers.DB.Where("ts < ?", (ts-int64(initializers.EventsRetentionDays)*day)*1000).
+				Delete(&models.ServiceEvent{}).Error
+		}},
+		{"top items", func() error {
+			cutoff := now.UTC().AddDate(0, 0, -initializers.TopItemsRetentionDays).Format("2006-01-02")
+			return initializers.DB.Where("day < ?", cutoff).Delete(&models.DailyTopItem{}).Error
+		}},
+		{"stale series", func() error {
+			stale := ts - 30*day
+			res := initializers.DB.Where("last_seen < ? AND id NOT IN (SELECT DISTINCT series_id FROM metric_points)", stale).
+				Delete(&models.MetricSeries{})
+			if res.Error == nil && res.RowsAffected > 0 {
+				// forget cached ids so a revived series gets a new row
+				idMu.Lock()
+				seriesID = map[*Series]uint{}
+				idMu.Unlock()
+			}
+			return res.Error
+		}},
+	}
+	for _, step := range steps {
+		if err := step.run(); err != nil {
+			log.Printf("metrics retention (%s): %v", step.what, err)
+		}
+	}
+	status.mu.Lock()
+	status.LastRetention = now.Unix()
+	status.mu.Unlock()
+	return nil
+}

metrics/traffic.go

@@ -0,0 +1,328 @@
+package metrics
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/binary"
+	"log"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
+	"github.com/gin-gonic/gin"
+	"gorm.io/gorm/clause"
+)
+
+// Middleware records latency and traffic for every request except static admin
+// UI files and health checks.
+func Middleware() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		path := c.Request.URL.Path
+		if c.Request.Method == "OPTIONS" || path == "/health" || strings.HasPrefix(path, "/admin/ui") {
+			c.Next()
+			return
+		}
+		start := time.Now()
+		c.Next()
+		ms := float64(time.Since(start).Microseconds()) / 1000
+
+		route := c.FullPath()
+		if route == "" {
+			route = "(unmatched)"
+		}
+		status := c.Writer.Status()
+		group := routeGroup(path)
+		client := "direct"
+		if c.GetHeader("Origin") != "" || c.GetHeader("Referer") != "" {
+			client = "web"
+		}
+		HTTPDuration.Observe(ms, c.Request.Method, route, strconv.Itoa(status/100)+"xx", group, client)
+		if size := c.Writer.Size(); size > 0 {
+			HTTPResponseBytes.Add(float64(size), group)
+		}
+
+		if group == "v1" && status < 500 {
+			visitors.add(c.ClientIP(), c.Request.UserAgent())
+		}
+		if status == 200 {
+			switch route {
+			case "/v1/addon/:id":
+				Top.Add("addon", c.Param("id"))
+			case "/v1/getPossible":
+				Top.Add("search", normalizeSearch(c.Query("q")))
+			}
+		}
+	}
+}
+
+func routeGroup(path string) string {
+	for _, g := range []string{"v1", "back", "admin", "metrics"} {
+		if path == "/"+g || strings.HasPrefix(path, "/"+g+"/") {
+			return g
+		}
+	}
+	return "other"
+}
+
+func normalizeSearch(q string) string {
+	q = strings.Join(strings.Fields(strings.ToLower(q)), " ")
+	if len(q) > 64 {
+		q = q[:64]
+	}
+	return q
+}
+
+// visitorSet counts distinct (IP, user agent) pairs per UTC day. Only salted
+// 64-bit hashes are kept and the salt never leaves memory.
+type visitorSet struct {
+	mu     sync.Mutex
+	day    string
+	salt   []byte
+	humans map[uint64]struct{}
+	bots   map[uint64]struct{}
+}
+
+var visitors = &visitorSet{}
+
+const maxVisitors = 2_000_000
+
+func (v *visitorSet) rotate(day string) {
+	v.day = day
+	v.salt = make([]byte, 32)
+	_, _ = rand.Read(v.salt)
+	v.humans = make(map[uint64]struct{})
+	v.bots = make(map[uint64]struct{})
+}
+
+func isBot(ua string) bool {
+	ua = strings.ToLower(ua)
+	if ua == "" {
+		return true
+	}
+	for _, s := range []string{"bot", "spider", "crawl", "curl", "wget", "python", "go-http", "headless", "preview", "monitor"} {
+		if strings.Contains(ua, s) {
+			return true
+		}
+	}
+	return false
+}
+
+func (v *visitorSet) add(ip, ua string) {
+	day := time.Now().UTC().Format("2006-01-02")
+	v.mu.Lock()
+	defer v.mu.Unlock()
+	if v.day != day {
+		v.rotate(day)
+	}
+	h := sha256.New()
+	h.Write(v.salt)
+	h.Write([]byte(ip))
+	h.Write([]byte{0})
+	h.Write([]byte(ua))
+	key := binary.BigEndian.Uint64(h.Sum(nil)[:8])
+	set := v.humans
+	if isBot(ua) {
+		set = v.bots
+	}
+	if len(set) < maxVisitors {
+		set[key] = struct{}{}
+	}
+}
+
+// Visitors returns today's approximate unique human and bot visitors. The set
+// lives in memory, so after a restart the day's highest recorded value is used
+// until the new set catches up.
+func Visitors() (humans, bots int) {
+	now := time.Now().UTC()
+	day := now.Format("2006-01-02")
+	visitors.mu.Lock()
+	if visitors.day == day {
+		humans, bots = len(visitors.humans), len(visitors.bots)
+	}
+	visitors.mu.Unlock()
+
+	dayStart := now.Truncate(24 * time.Hour).Unix()
+	if dayStart < startedAt.Unix() {
+		if t, err := Totals(Selector{Service: MainService, Name: "visitors_unique", GroupBy: []string{"kind"}}, dayStart, now.Unix()); err == nil {
+			for _, g := range t {
+				if g.Max == nil {
+					continue
+				}
+				switch g.Labels["kind"] {
+				case "human":
+					humans = max(humans, int(*g.Max))
+				case "bot":
+					bots = max(bots, int(*g.Max))
+				}
+			}
+		}
+	}
+	return humans, bots
+}
+
+// liveVisitors is the in-memory count only, used when recording the gauge.
+func liveVisitors() (humans, bots int) {
+	day := time.Now().UTC().Format("2006-01-02")
+	visitors.mu.Lock()
+	defer visitors.mu.Unlock()
+	if visitors.day != day {
+		return 0, 0
+	}
+	return len(visitors.humans), len(visitors.bots)
+}
+
+// topCounter counts requested items per UTC day and persists the most popular.
+type topCounter struct {
+	mu      sync.Mutex
+	day     string
+	counts  map[string]map[string]int64
+	pending []topSnapshot // previous days not yet written
+}
+
+type topSnapshot struct {
+	day    string
+	counts map[string]map[string]int64
+}
+
+// Top tracks the most looked-up addons and search queries.
+var Top = &topCounter{counts: map[string]map[string]int64{}}
+
+const (
+	maxTopKeys    = 50_000
+	topPersistMax = 500
+)
+
+func (t *topCounter) Add(kind, item string) {
+	if item == "" {
+		return
+	}
+	if len(item) > 128 {
+		item = item[:128]
+	}
+	day := time.Now().UTC().Format("2006-01-02")
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	if t.day != day {
+		if t.day != "" {
+			t.pending = append(t.pending, topSnapshot{t.day, t.counts})
+		}
+		t.day = day
+		t.counts = map[string]map[string]int64{}
+	}
+	m := t.counts[kind]
+	if m == nil {
+		m = map[string]int64{}
+		t.counts[kind] = m
+	}
+	if _, ok := m[item]; !ok && len(m) >= maxTopKeys {
+		for k, v := range m {
+			if v <= 1 {
+				delete(m, k)
+			}
+		}
+		if len(m) >= maxTopKeys {
+			return
+		}
+	}
+	m[item]++
+}
+
+// Today returns today's in-memory counts for a kind.
+func (t *topCounter) Today(kind string) (string, map[string]int64) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	out := make(map[string]int64, len(t.counts[kind]))
+	for k, v := range t.counts[kind] {
+		out[k] = v
+	}
+	return t.day, out
+}
+
+// seed loads today's persisted counts so a restart does not reset them.
+func (t *topCounter) seed() {
+	day := time.Now().UTC().Format("2006-01-02")
+	var rows []models.DailyTopItem
+	if err := initializers.DB.Where("day = ?", day).Find(&rows).Error; err != nil {
+		log.Printf("metrics: failed to load top items: %v", err)
+		return
+	}
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	if t.day != day {
+		t.day = day
+		t.counts = map[string]map[string]int64{}
+	}
+	for _, r := range rows {
+		if t.counts[r.Kind] == nil {
+			t.counts[r.Kind] = map[string]int64{}
+		}
+		t.counts[r.Kind][r.Item] += r.Count
+	}
+}
+
+func topRows(snap topSnapshot) []models.DailyTopItem {
+	var rows []models.DailyTopItem
+	for kind, m := range snap.counts {
+		type kv struct {
+			k string
+			v int64
+		}
+		list := make([]kv, 0, len(m))
+		for k, v := range m {
+			list = append(list, kv{k, v})
+		}
+		sort.Slice(list, func(i, j int) bool { return list[i].v > list[j].v })
+		if len(list) > topPersistMax {
+			list = list[:topPersistMax]
+		}
+		for _, e := range list {
+			rows = append(rows, models.DailyTopItem{Day: snap.day, Kind: kind, Item: e.k, Count: e.v})
+		}
+	}
+	return rows
+}
+
+func (t *topCounter) flush() error {
+	t.mu.Lock()
+	snaps := append([]topSnapshot{}, t.pending...)
+	t.pending = nil
+	cur := topSnapshot{day: t.day, counts: map[string]map[string]int64{}}
+	for kind, m := range t.counts {
+		cp := make(map[string]int64, len(m))
+		for k, v := range m {
+			cp[k] = v
+		}
+		cur.counts[kind] = cp
+	}
+	t.mu.Unlock()
+	if cur.day != "" {
+		snaps = append(snaps, cur)
+	}
+
+	for i, snap := range snaps {
+		rows := topRows(snap)
+		if len(rows) == 0 {
+			continue
+		}
+		err := initializers.DB.Clauses(clause.OnConflict{
+			Columns:   []clause.Column{{Name: "day"}, {Name: "kind"}, {Name: "item"}},
+			DoUpdates: clause.AssignmentColumns([]string{"count"}),
+		}).CreateInBatches(&rows, 200).Error
+		if err != nil {
+			// keep finished days for the next attempt; today is re-read from memory
+			t.mu.Lock()
+			for _, old := range snaps[i:] {
+				if old.day != cur.day {
+					t.pending = append(t.pending, old)
+				}
+			}
+			t.mu.Unlock()
+			return err
+		}
+	}
+	return nil
+}

models/addon.go

@@ -27,6 +27,7 @@ type Addon struct {
 	PriorityIndexing     bool           `json:"priorityIndexing" gorm:"default:false"`
 	AddonFiles           []AddonFile    `json:"addonFiles"`
 	Recheck              bool           `json:"recheck" gorm:"default:false"`
+	LastIndexedAt        *time.Time     `json:"lastIndexedAt"`
 
 	Preview string `json:"preview"`
 	Author  string `json:"author"`

models/metrics.go

@@ -0,0 +1,74 @@
+package models
+
+import "time"
+
+// All timestamps in the metrics tables are int64 unix seconds (events: unix
+// milliseconds) so range queries behave the same on SQLite and Postgres.
+
+// MetricSeries identifies one time series: a metric name plus a fixed label set
+// reported by one service instance.
+type MetricSeries struct {
+	ID        uint   `gorm:"primaryKey" json:"id"`
+	Service   string `gorm:"size:64;not null;uniqueIndex:ux_metric_series,priority:1" json:"service"`
+	Instance  string `gorm:"size:128;not null;uniqueIndex:ux_metric_series,priority:2" json:"instance"`
+	Name      string `gorm:"size:96;not null;uniqueIndex:ux_metric_series,priority:3;index" json:"name"`
+	Labels    string `gorm:"size:512;not null;uniqueIndex:ux_metric_series,priority:4" json:"labels"` // canonical JSON object
+	Kind      string `gorm:"size:16;not null" json:"kind"`                                            // counter|gauge|summary|histogram
+	Bounds    string `gorm:"type:text" json:"bounds"`                                                 // JSON []float64, histograms only
+	Help      string `gorm:"size:256" json:"help"`
+	CreatedAt time.Time
+	LastSeen  int64 `gorm:"index" json:"lastSeen"`
+}
+
+// MetricPoint is one aggregated bucket of a series. Resolution is the bucket
+// width in seconds (60 = raw minute data, 3600 = hourly rollup).
+type MetricPoint struct {
+	SeriesID   uint    `gorm:"primaryKey;autoIncrement:false"`
+	Resolution int     `gorm:"primaryKey;autoIncrement:false;index:idx_metric_points_res_bucket,priority:1"`
+	Bucket     int64   `gorm:"primaryKey;autoIncrement:false;index:idx_metric_points_res_bucket,priority:2"`
+	Count      int64   `gorm:"not null;default:0"`
+	Sum        float64 `gorm:"not null;default:0"`
+	Min        float64 `gorm:"not null;default:0"`
+	Max        float64 `gorm:"not null;default:0"`
+	Last       float64 `gorm:"not null;default:0"`
+	Hist       string  `gorm:"type:text"` // JSON []uint64, histograms only
+}
+
+// ServiceInstance is one running process (indexer, scraper, the API itself)
+// that reports telemetry. Every telemetry push counts as a heartbeat.
+type ServiceInstance struct {
+	ID                   uint      `gorm:"primaryKey" json:"id"`
+	Service              string    `gorm:"size:64;not null;uniqueIndex:ux_service_instance,priority:1" json:"service"`
+	Instance             string    `gorm:"size:128;not null;uniqueIndex:ux_service_instance,priority:2" json:"instance"`
+	Version              string    `gorm:"size:64" json:"version"`
+	Status               string    `gorm:"size:16" json:"status"` // self-reported: ok|degraded|error
+	StatusMessage        string    `gorm:"size:512" json:"statusMessage"`
+	StartedAt            int64     `json:"startedAt"`
+	LastSeen             int64     `gorm:"index" json:"lastSeen"`
+	HeartbeatIntervalSec int       `json:"heartbeatIntervalSec"`
+	RemoteAddr           string    `gorm:"size:64" json:"remoteAddr"`
+	Meta                 string    `gorm:"type:text" json:"meta"` // JSON object
+	CreatedAt            time.Time `json:"createdAt"`
+	UpdatedAt            time.Time `json:"updatedAt"`
+}
+
+// ServiceEvent is a log line reported by a service. Admin actions are recorded
+// here too (service "main", fields.kind "audit").
+type ServiceEvent struct {
+	ID       uint   `gorm:"primaryKey" json:"id"`
+	Service  string `gorm:"size:64;index:idx_service_events_svc_ts,priority:1" json:"service"`
+	Instance string `gorm:"size:128" json:"instance"`
+	Level    string `gorm:"size:8;index" json:"level"` // debug|info|warn|error
+	Message  string `gorm:"size:1024" json:"message"`
+	Fields   string `gorm:"type:text" json:"fields"`                                    // JSON object
+	TS       int64  `gorm:"index:idx_service_events_svc_ts,priority:2;index" json:"ts"` // unix ms
+}
+
+// DailyTopItem counts how often something was requested on one UTC day
+// (Kind "addon": addon lookups, Kind "search": search queries).
+type DailyTopItem struct {
+	Day   string `gorm:"primaryKey;size:10"` // YYYY-MM-DD
+	Kind  string `gorm:"primaryKey;size:16"`
+	Item  string `gorm:"primaryKey;size:128"`
+	Count int64
+}

models/telemetry.go

@@ -0,0 +1,39 @@
+package models
+
+// TelemetryBatch is the payload of POST /back/telemetry. The client lives in
+// the telemetry/ module; keep both definitions in sync.
+type TelemetryBatch struct {
+	Service              string            `json:"service"`
+	Instance             string            `json:"instance"`
+	Version              string            `json:"version"`
+	StartedAt            int64             `json:"startedAt"`
+	HeartbeatIntervalSec int               `json:"heartbeatIntervalSec"`
+	Status               string            `json:"status"`
+	StatusMessage        string            `json:"statusMessage"`
+	Meta                 map[string]any    `json:"meta"`
+	Metrics              []TelemetryMetric `json:"metrics"`
+	Events               []TelemetryEvent  `json:"events"`
+}
+
+// TelemetryMetric is one sample. Type "counter" carries a delta since the last
+// push, "gauge" the current value, "summary" client-side pre-aggregated
+// Count/Sum/Min/Max. Ts is unix seconds or milliseconds, optional.
+type TelemetryMetric struct {
+	Name   string            `json:"name"`
+	Type   string            `json:"type"`
+	Help   string            `json:"help"`
+	Value  float64           `json:"value"`
+	Count  int64             `json:"count"`
+	Sum    float64           `json:"sum"`
+	Min    float64           `json:"min"`
+	Max    float64           `json:"max"`
+	Labels map[string]string `json:"labels"`
+	Ts     int64             `json:"ts"`
+}
+
+type TelemetryEvent struct {
+	Level   string         `json:"level"`
+	Message string         `json:"message"`
+	Ts      int64          `json:"ts"`
+	Fields  map[string]any `json:"fields"`
+}

models/whitelisted.go

@@ -4,5 +4,6 @@ import "gorm.io/gorm"
 
 type WhitelistedHash struct {
 	gorm.Model
-	Hash    string `json:"hash"`
+	Hash string `json:"hash"`
+	Note string `json:"note" gorm:"size:256"`
 }

telemetry/client.go

@@ -0,0 +1,430 @@
+// Package telemetry reports metrics, events and heartbeats from a crawler
+// service (indexer, scraper, ...) to the main API's POST /back/telemetry.
+//
+// It has no dependencies outside the standard library, so it can be imported
+// by any service, or the file can simply be copied.
+//
+//	c := telemetry.New(telemetry.Config{
+//		BaseURL:   "https://crawler.example.com",
+//		SecretKey: os.Getenv("SECRET_KEY"),
+//		Service:   "indexer",
+//		Version:   "1.4.0",
+//	})
+//	c.Start(ctx)
+//	defer c.Close(context.Background())
+//
+//	c.Counter("jobs_total", "result", "ok").Add(1)
+//	c.Gauge("data_window_bytes").Set(float64(used))
+//	c.Summary("download_seconds").Observe(took.Seconds())
+//	c.Event(telemetry.Warn, "data limit reached", map[string]any{"limitGB": 50})
+//	c.SetStatus(telemetry.Degraded, "rate limited by workshop")
+//
+// Recording never blocks on the network. A push that fails keeps the counter
+// increments for the next attempt; a few failed batches of events and samples
+// are buffered and the oldest are dropped first.
+package telemetry
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"os"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+)
+
+// Level of an event.
+type Level string
+
+const (
+	Debug Level = "debug"
+	Info  Level = "info"
+	Warn  Level = "warn"
+	Error Level = "error"
+)
+
+// Status is the self-reported health shown on the dashboard.
+type Status string
+
+const (
+	OK       Status = "ok"
+	Degraded Status = "degraded"
+	Failing  Status = "error"
+)
+
+// Config configures a Client. BaseURL, SecretKey and Service are required.
+type Config struct {
+	BaseURL   string        // e.g. https://crawler.example.com (no trailing /back)
+	SecretKey string        // the API's SECRET_KEY
+	Service   string        // [a-z0-9_], e.g. "indexer"
+	Instance  string        // defaults to the hostname
+	Version   string        // shown on the dashboard
+	Interval  time.Duration // push interval, default 15s (also the heartbeat)
+	Meta      map[string]any
+	// HTTPClient defaults to a client with a 10s timeout.
+	HTTPClient *http.Client
+	// OnError is called when a push fails (optional).
+	OnError func(error)
+}
+
+// Client buffers metrics and events and pushes them periodically.
+type Client struct {
+	cfg     Config
+	started time.Time
+
+	mu        sync.Mutex
+	counters  map[string]*Counter
+	gauges    map[string]*Gauge
+	summaries map[string]*Summary
+	events    []event
+	status    Status
+	statusMsg string
+	pending   [][]byte // failed batches waiting for a retry
+
+	stop chan struct{}
+	done chan struct{}
+}
+
+type metric struct {
+	Name   string            `json:"name"`
+	Type   string            `json:"type"`
+	Value  float64           `json:"value,omitempty"`
+	Count  int64             `json:"count,omitempty"`
+	Sum    float64           `json:"sum,omitempty"`
+	Min    float64           `json:"min,omitempty"`
+	Max    float64           `json:"max,omitempty"`
+	Labels map[string]string `json:"labels,omitempty"`
+	Ts     int64             `json:"ts"`
+}
+
+type event struct {
+	Level   Level          `json:"level"`
+	Message string         `json:"message"`
+	Ts      int64          `json:"ts"`
+	Fields  map[string]any `json:"fields,omitempty"`
+}
+
+type batch struct {
+	Service              string         `json:"service"`
+	Instance             string         `json:"instance"`
+	Version              string         `json:"version"`
+	StartedAt            int64          `json:"startedAt"`
+	HeartbeatIntervalSec int            `json:"heartbeatIntervalSec"`
+	Status               Status         `json:"status"`
+	StatusMessage        string         `json:"statusMessage"`
+	Meta                 map[string]any `json:"meta,omitempty"`
+	Metrics              []metric       `json:"metrics"`
+	Events               []event        `json:"events"`
+}
+
+const (
+	maxPending       = 20
+	maxEvents        = 200
+	maxMetricsPerReq = 1000
+)
+
+// New creates a client. Call Start to begin pushing.
+func New(cfg Config) *Client {
+	if cfg.Instance == "" {
+		cfg.Instance, _ = os.Hostname()
+	}
+	if cfg.Interval <= 0 {
+		cfg.Interval = 15 * time.Second
+	}
+	if cfg.HTTPClient == nil {
+		cfg.HTTPClient = &http.Client{Timeout: 10 * time.Second}
+	}
+	cfg.BaseURL = strings.TrimRight(cfg.BaseURL, "/")
+	return &Client{
+		cfg:       cfg,
+		started:   time.Now(),
+		counters:  map[string]*Counter{},
+		gauges:    map[string]*Gauge{},
+		summaries: map[string]*Summary{},
+		status:    OK,
+	}
+}
+
+func key(name string, labels []string) string {
+	return name + "\x00" + strings.Join(labels, "\x00")
+}
+
+func labelMap(kv []string) map[string]string {
+	if len(kv) < 2 {
+		return nil
+	}
+	m := make(map[string]string, len(kv)/2)
+	for i := 0; i+1 < len(kv); i += 2 {
+		m[kv[i]] = kv[i+1]
+	}
+	return m
+}
+
+// Counter returns a counter; labels are key/value pairs.
+func (c *Client) Counter(name string, labels ...string) *Counter {
+	k := key(name, labels)
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	ctr := c.counters[k]
+	if ctr == nil {
+		ctr = &Counter{name: name, labels: labelMap(labels)}
+		c.counters[k] = ctr
+	}
+	return ctr
+}
+
+// Gauge returns a gauge; labels are key/value pairs.
+func (c *Client) Gauge(name string, labels ...string) *Gauge {
+	k := key(name, labels)
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	g := c.gauges[k]
+	if g == nil {
+		g = &Gauge{name: name, labels: labelMap(labels)}
+		c.gauges[k] = g
+	}
+	return g
+}
+
+// Summary returns a summary (count/sum/min/max per push); labels are key/value pairs.
+func (c *Client) Summary(name string, labels ...string) *Summary {
+	k := key(name, labels)
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	s := c.summaries[k]
+	if s == nil {
+		s = &Summary{name: name, labels: labelMap(labels)}
+		c.summaries[k] = s
+	}
+	return s
+}
+
+// Event queues a log event for the dashboard.
+func (c *Client) Event(level Level, message string, fields map[string]any) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if len(c.events) >= maxEvents {
+		c.events = c.events[1:]
+	}
+	c.events = append(c.events, event{Level: level, Message: message, Ts: time.Now().UnixMilli(), Fields: fields})
+}
+
+// SetStatus sets the self-reported health shown on the dashboard.
+func (c *Client) SetStatus(s Status, message string) {
+	c.mu.Lock()
+	c.status, c.statusMsg = s, message
+	c.mu.Unlock()
+}
+
+// Counter accumulates increments between pushes.
+type Counter struct {
+	mu     sync.Mutex
+	name   string
+	labels map[string]string
+	delta  float64
+}
+
+func (m *Counter) Add(v float64) {
+	if v <= 0 || math.IsNaN(v) || math.IsInf(v, 0) {
+		return
+	}
+	m.mu.Lock()
+	m.delta += v
+	m.mu.Unlock()
+}
+
+func (m *Counter) Inc() { m.Add(1) }
+
+// Gauge holds the latest value.
+type Gauge struct {
+	mu     sync.Mutex
+	name   string
+	labels map[string]string
+	value  float64
+	set    bool
+}
+
+func (m *Gauge) Set(v float64) {
+	if math.IsNaN(v) || math.IsInf(v, 0) {
+		return
+	}
+	m.mu.Lock()
+	m.value, m.set = v, true
+	m.mu.Unlock()
+}
+
+// Summary aggregates observations between pushes.
+type Summary struct {
+	mu       sync.Mutex
+	name     string
+	labels   map[string]string
+	count    int64
+	sum      float64
+	min, max float64
+}
+
+func (m *Summary) Observe(v float64) {
+	if math.IsNaN(v) || math.IsInf(v, 0) {
+		return
+	}
+	m.mu.Lock()
+	if m.count == 0 || v < m.min {
+		m.min = v
+	}
+	if m.count == 0 || v > m.max {
+		m.max = v
+	}
+	m.count++
+	m.sum += v
+	m.mu.Unlock()
+}
+
+// collect drains the current values into a batch payload.
+func (c *Client) collect() []byte {
+	now := time.Now()
+	c.mu.Lock()
+	b := batch{
+		Service:              c.cfg.Service,
+		Instance:             c.cfg.Instance,
+		Version:              c.cfg.Version,
+		StartedAt:            c.started.Unix(),
+		HeartbeatIntervalSec: int(c.cfg.Interval.Seconds()),
+		Status:               c.status,
+		StatusMessage:        c.statusMsg,
+		Meta:                 c.cfg.Meta,
+		Events:               c.events,
+	}
+	c.events = nil
+	counters := make([]*Counter, 0, len(c.counters))
+	for _, m := range c.counters {
+		counters = append(counters, m)
+	}
+	gauges := make([]*Gauge, 0, len(c.gauges))
+	for _, m := range c.gauges {
+		gauges = append(gauges, m)
+	}
+	summaries := make([]*Summary, 0, len(c.summaries))
+	for _, m := range c.summaries {
+		summaries = append(summaries, m)
+	}
+	c.mu.Unlock()
+
+	ts := now.UnixMilli()
+	for _, m := range counters {
+		m.mu.Lock()
+		if m.delta > 0 {
+			b.Metrics = append(b.Metrics, metric{Name: m.name, Type: "counter", Value: m.delta, Labels: m.labels, Ts: ts})
+			m.delta = 0
+		}
+		m.mu.Unlock()
+	}
+	for _, m := range gauges {
+		m.mu.Lock()
+		if m.set {
+			b.Metrics = append(b.Metrics, metric{Name: m.name, Type: "gauge", Value: m.value, Labels: m.labels, Ts: ts})
+		}
+		m.mu.Unlock()
+	}
+	for _, m := range summaries {
+		m.mu.Lock()
+		if m.count > 0 {
+			b.Metrics = append(b.Metrics, metric{Name: m.name, Type: "summary", Count: m.count, Sum: m.sum, Min: m.min, Max: m.max, Labels: m.labels, Ts: ts})
+			m.count, m.sum, m.min, m.max = 0, 0, 0, 0
+		}
+		m.mu.Unlock()
+	}
+	sort.Slice(b.Metrics, func(i, j int) bool { return b.Metrics[i].Name < b.Metrics[j].Name })
+	if len(b.Metrics) > maxMetricsPerReq {
+		b.Metrics = b.Metrics[:maxMetricsPerReq]
+	}
+	if b.Metrics == nil {
+		b.Metrics = []metric{}
+	}
+	if b.Events == nil {
+		b.Events = []event{}
+	}
+	payload, _ := json.Marshal(b)
+	return payload
+}
+
+func (c *Client) post(ctx context.Context, payload []byte) error {
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.cfg.BaseURL+"/back/telemetry", bytes.NewReader(payload))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("X-SECRET-KEY", c.cfg.SecretKey)
+	resp, err := c.cfg.HTTPClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+	if resp.StatusCode/100 != 2 {
+		return fmt.Errorf("telemetry push: %s: %s", resp.Status, strings.TrimSpace(string(body)))
+	}
+	return nil
+}
+
+// Flush pushes buffered batches and the current values now.
+func (c *Client) Flush(ctx context.Context) error {
+	payload := c.collect()
+	c.mu.Lock()
+	queue := append(c.pending, payload)
+	c.pending = nil
+	c.mu.Unlock()
+
+	for i, p := range queue {
+		if err := c.post(ctx, p); err != nil {
+			c.mu.Lock()
+			c.pending = append(queue[i:], c.pending...)
+			if len(c.pending) > maxPending {
+				c.pending = c.pending[len(c.pending)-maxPending:]
+			}
+			c.mu.Unlock()
+			if c.cfg.OnError != nil {
+				c.cfg.OnError(err)
+			}
+			return err
+		}
+	}
+	return nil
+}
+
+// Start pushes in the background every Interval until ctx ends or Close is called.
+func (c *Client) Start(ctx context.Context) {
+	c.stop = make(chan struct{})
+	c.done = make(chan struct{})
+	go func() {
+		defer close(c.done)
+		t := time.NewTicker(c.cfg.Interval)
+		defer t.Stop()
+		_ = c.Flush(ctx)
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-c.stop:
+				return
+			case <-t.C:
+				_ = c.Flush(ctx)
+			}
+		}
+	}()
+}
+
+// Close stops the background loop and makes a final push.
+func (c *Client) Close(ctx context.Context) error {
+	if c.stop != nil {
+		close(c.stop)
+		<-c.done
+		c.stop = nil
+	}
+	return c.Flush(ctx)
+}

telemetry/client_test.go

@@ -0,0 +1,73 @@
+package telemetry
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+)
+
+func TestFlushAndRetry(t *testing.T) {
+	var fail atomic.Bool
+	var got []batch
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/back/telemetry" || r.Header.Get("X-SECRET-KEY") != "k" {
+			w.WriteHeader(http.StatusForbidden)
+			return
+		}
+		if fail.Load() {
+			w.WriteHeader(http.StatusServiceUnavailable)
+			return
+		}
+		body, _ := io.ReadAll(r.Body)
+		var b batch
+		if err := json.Unmarshal(body, &b); err != nil {
+			t.Error(err)
+		}
+		got = append(got, b)
+		w.Write([]byte(`{"accepted":1}`))
+	}))
+	defer srv.Close()
+
+	c := New(Config{BaseURL: srv.URL + "/", SecretKey: "k", Service: "indexer", Instance: "i1"})
+	c.Counter("jobs_total", "result", "ok").Add(2)
+	c.Gauge("queue").Set(7)
+	c.Summary("took_seconds").Observe(1)
+	c.Summary("took_seconds").Observe(3)
+	c.Event(Warn, "hello", nil)
+
+	fail.Store(true)
+	if err := c.Flush(context.Background()); err == nil {
+		t.Fatal("expected an error while the server fails")
+	}
+	c.Counter("jobs_total", "result", "ok").Add(1)
+
+	fail.Store(false)
+	if err := c.Flush(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 2 {
+		t.Fatalf("got %d batches, want the retried one plus the new one", len(got))
+	}
+	first := got[0]
+	if first.Service != "indexer" || first.Instance != "i1" || len(first.Events) != 1 {
+		t.Fatalf("unexpected first batch %+v", first)
+	}
+	var sum float64
+	for _, b := range got {
+		for _, m := range b.Metrics {
+			if m.Name == "jobs_total" {
+				sum += m.Value
+			}
+			if m.Name == "took_seconds" && (m.Count != 2 || m.Min != 1 || m.Max != 3) {
+				t.Fatalf("bad summary %+v", m)
+			}
+		}
+	}
+	if sum != 3 {
+		t.Fatalf("counter total = %v, want 3", sum)
+	}
+}

telemetry/go.mod

@@ -0,0 +1,3 @@
+module gitea.tbdevent.eu/TBD/reforger_crawler_main/telemetry
+
+go 1.22

web/embed.go

@@ -0,0 +1,13 @@
+// Package web serves the admin dashboard (a React app built into web/dist).
+//
+// Build the UI before building the Go binary:
+//
+//	npm --prefix web ci && npm --prefix web run build
+//
+// Without a build the binary still compiles and serves a placeholder page.
+package web
+
+import "embed"
+
+//go:embed all:dist
+var dist embed.FS

web/index.html

@@ -0,0 +1,15 @@
+<!doctype html>
+<html lang="en" data-theme="dark">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <meta name="robots" content="noindex" />
+    <link rel="icon" type="image/svg+xml" href="/admin/ui/favicon.svg" />
+    <title>Crawler Admin</title>
+    <script src="/admin/ui/theme-init.js"></script>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>

web/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "reforger-crawler-admin",
+  "private": true,
+  "version": "1.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "clean": "node -e \"const fs=require('fs');fs.rmSync('dist/assets',{recursive:true,force:true});fs.rmSync('dist/index.html',{force:true})\"",
+    "build": "npm run clean && tsc -b && vite build",
+    "typecheck": "tsc -b"
+  },
+  "dependencies": {
+    "@tanstack/react-query": "^5.80.0",
+    "lucide-react": "^0.511.0",
+    "react": "^19.1.0",
+    "react-dom": "^19.1.0",
+    "react-router": "^7.6.0",
+    "uplot": "^1.6.32"
+  },
+  "devDependencies": {
+    "@tailwindcss/vite": "^4.1.8",
+    "@types/react": "^19.1.6",
+    "@types/react-dom": "^19.1.5",
+    "@vitejs/plugin-react": "^4.5.0",
+    "tailwindcss": "^4.1.8",
+    "typescript": "^5.8.3",
+    "vite": "^6.3.5"
+  }
+}

web/public/favicon.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32"><rect width="32" height="32" rx="8" fill="#1a1a19"/><path d="M8 21l5-6 4 4 7-9" fill="none" stroke="#3987e5" stroke-width="3" stroke-linecap="round" stroke-linejoin="round"/></svg>

web/public/theme-init.js

@@ -0,0 +1,5 @@
+// Applies the saved theme before first paint (external file: the CSP blocks inline scripts).
+try {
+  var t = localStorage.getItem("rc-theme");
+  document.documentElement.setAttribute("data-theme", t === "light" ? "light" : "dark");
+} catch (e) {}

web/spa.go

@@ -0,0 +1,75 @@
+package web
+
+import (
+	"io/fs"
+	"net/http"
+	"strings"
+
+	"github.com/gin-gonic/gin"
+)
+
+const notBuiltHTML = `<!doctype html><html><head><meta charset="utf-8"><title>Admin UI not built</title>
+<style>body{font-family:system-ui,sans-serif;background:#0b0d10;color:#c9d1d9;display:grid;place-items:center;height:100vh;margin:0}
+code{background:#161b22;padding:2px 6px;border-radius:4px}</style></head>
+<body><div><h1>Admin UI not built</h1><p>Run <code>npm --prefix web ci &amp;&amp; npm --prefix web run build</code> and rebuild the binary.</p>
+<p>The JSON API is available under <code>/admin/api</code>.</p></div></body></html>`
+
+const csp = "default-src 'self'; img-src 'self' https: data:; style-src 'self' 'unsafe-inline'; " +
+	"script-src 'self'; connect-src 'self'; font-src 'self' data:; frame-ancestors 'none'; base-uri 'self'; form-action 'self'"
+
+func securityHeaders(c *gin.Context) {
+	c.Header("Content-Security-Policy", csp)
+	c.Header("X-Content-Type-Options", "nosniff")
+	c.Header("Referrer-Policy", "no-referrer")
+	c.Header("X-Frame-Options", "DENY")
+}
+
+// Handler returns the SPA handler: real files are served from the embedded
+// build, every other path gets index.html so client-side routes deep-link.
+func Handler() gin.HandlerFunc {
+	sub, err := fs.Sub(dist, "dist")
+	if err != nil {
+		panic(err)
+	}
+	index, err := fs.ReadFile(sub, "index.html")
+	if err != nil {
+		index = []byte(notBuiltHTML)
+	}
+	files := http.FileServer(http.FS(sub))
+
+	return func(c *gin.Context) {
+		securityHeaders(c)
+		p := strings.TrimPrefix(c.Param("filepath"), "/")
+		if p != "" && p != "index.html" {
+			if st, err := fs.Stat(sub, p); err == nil && !st.IsDir() {
+				if strings.HasPrefix(p, "assets/") {
+					c.Header("Cache-Control", "public, max-age=31536000, immutable")
+				} else {
+					c.Header("Cache-Control", "no-cache")
+				}
+				req := c.Request.Clone(c.Request.Context())
+				req.URL.Path = "/" + p
+				files.ServeHTTP(c.Writer, req)
+				return
+			}
+			// a missing hashed asset must not turn into an HTML page
+			if strings.HasPrefix(p, "assets/") {
+				c.Status(http.StatusNotFound)
+				return
+			}
+		}
+		// serve the bytes directly; FileServer would redirect /index.html to ./
+		c.Header("Cache-Control", "no-cache")
+		c.Data(http.StatusOK, "text/html; charset=utf-8", index)
+	}
+}
+
+// Register mounts the dashboard at /admin/ui and redirects /admin there.
+func Register(r *gin.Engine) {
+	h := Handler()
+	r.GET("/admin/ui/*filepath", h)
+	r.HEAD("/admin/ui/*filepath", h)
+	redirect := func(c *gin.Context) { c.Redirect(http.StatusFound, "/admin/ui/") }
+	r.GET("/admin", redirect)
+	r.GET("/admin/ui", redirect)
+}

web/spa_test.go

@@ -0,0 +1,51 @@
+package web
+
+import (
+	"io/fs"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+)
+
+func TestSPA(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+	r := gin.New()
+	Register(r)
+
+	get := func(path string) *httptest.ResponseRecorder {
+		w := httptest.NewRecorder()
+		r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil))
+		return w
+	}
+
+	for _, p := range []string{"/admin/ui/", "/admin/ui/addons/123", "/admin/ui/index.html"} {
+		w := get(p)
+		if w.Code != http.StatusOK || !strings.Contains(w.Header().Get("Content-Type"), "text/html") {
+			t.Errorf("%s: got %d %s", p, w.Code, w.Header().Get("Content-Type"))
+		}
+		if w.Header().Get("Content-Security-Policy") == "" {
+			t.Errorf("%s: missing CSP", p)
+		}
+	}
+
+	if w := get("/admin/ui/assets/missing.js"); w.Code != http.StatusNotFound {
+		t.Errorf("missing asset: got %d, want 404", w.Code)
+	}
+	if w := get("/admin"); w.Code != http.StatusFound || w.Header().Get("Location") != "/admin/ui/" {
+		t.Errorf("/admin: got %d -> %s", w.Code, w.Header().Get("Location"))
+	}
+
+	// when the UI is built, hashed assets are served with long caching
+	sub, _ := fs.Sub(dist, "dist")
+	entries, _ := fs.ReadDir(sub, "assets")
+	for _, e := range entries {
+		w := get("/admin/ui/assets/" + e.Name())
+		if w.Code != http.StatusOK || !strings.Contains(w.Header().Get("Cache-Control"), "immutable") {
+			t.Errorf("asset %s: got %d cache %q", e.Name(), w.Code, w.Header().Get("Cache-Control"))
+		}
+		break
+	}
+}

web/src/components/Shell.tsx

@@ -0,0 +1,160 @@
+import { useEffect, type ReactNode } from "react";
+import { NavLink, Navigate, Outlet, useLocation, useNavigate } from "react-router";
+import { useIsFetching, useQuery, useQueryClient } from "@tanstack/react-query";
+import {
+  Activity,
+  Cpu,
+  LayoutDashboard,
+  LineChart,
+  ListOrdered,
+  LogOut,
+  Moon,
+  Package,
+  RefreshCw,
+  ScrollText,
+  Server,
+  ShieldCheck,
+  Sun,
+} from "lucide-react";
+import { api, ApiError, UNAUTHORIZED_EVENT } from "../lib/api";
+import { RANGES, REFRESH, useSettings } from "../lib/settings";
+import { Loading, Segmented, Select, cn } from "./ui";
+
+const NAV: { to: string; label: string; icon: ReactNode }[] = [
+  { to: "/", label: "Overview", icon: <LayoutDashboard className="size-4" /> },
+  { to: "/traffic", label: "Traffic", icon: <Activity className="size-4" /> },
+  { to: "/queue", label: "Indexing queue", icon: <ListOrdered className="size-4" /> },
+  { to: "/addons", label: "Addons", icon: <Package className="size-4" /> },
+  { to: "/whitelist", label: "Whitelist", icon: <ShieldCheck className="size-4" /> },
+  { to: "/services", label: "Services", icon: <Server className="size-4" /> },
+  { to: "/explorer", label: "Metrics explorer", icon: <LineChart className="size-4" /> },
+  { to: "/events", label: "Events", icon: <ScrollText className="size-4" /> },
+  { to: "/system", label: "System", icon: <Cpu className="size-4" /> },
+];
+
+/** Redirects to /login when the session is missing or expired. */
+export function RequireAuth() {
+  const navigate = useNavigate();
+  const qc = useQueryClient();
+  const location = useLocation();
+  const me = useQuery({ queryKey: ["me"], queryFn: () => api<{ method: string }>("/auth/me"), retry: false, staleTime: 60_000 });
+
+  useEffect(() => {
+    const onUnauthorized = () => {
+      qc.clear();
+      navigate("/login", { replace: true, state: { from: location.pathname } });
+    };
+    window.addEventListener(UNAUTHORIZED_EVENT, onUnauthorized);
+    return () => window.removeEventListener(UNAUTHORIZED_EVENT, onUnauthorized);
+  }, [navigate, qc, location.pathname]);
+
+  if (me.isLoading) return <div className="p-8"><Loading /></div>;
+  if (me.error instanceof ApiError && (me.error.status === 401 || me.error.status === 503)) {
+    return <Navigate to="/login" replace state={{ from: location.pathname }} />;
+  }
+  return <Shell />;
+}
+
+function Shell() {
+  const { theme, setTheme, range, setRange, refresh, setRefresh } = useSettings();
+  const fetching = useIsFetching();
+  const navigate = useNavigate();
+  const qc = useQueryClient();
+
+  const logout = async () => {
+    await api("/auth/logout", { method: "POST" }).catch(() => undefined);
+    qc.clear();
+    navigate("/login", { replace: true });
+  };
+
+  return (
+    <div className="flex min-h-screen">
+      <aside className="sticky top-0 hidden h-screen w-56 shrink-0 flex-col border-r border-line bg-surface md:flex">
+        <div className="flex items-center gap-2.5 px-4 py-4">
+          <div className="flex size-7 items-center justify-center rounded-lg bg-accent/15">
+            <svg viewBox="0 0 32 32" className="size-4"><path d="M6 22l6-7 5 5 9-11" fill="none" stroke="var(--accent)" strokeWidth="3.5" strokeLinecap="round" strokeLinejoin="round" /></svg>
+          </div>
+          <div className="leading-tight">
+            <div className="text-[13px] font-semibold text-ink">Reforger Crawler</div>
+            <div className="text-[11px] text-muted">Admin</div>
+          </div>
+        </div>
+        <nav className="flex-1 space-y-0.5 overflow-y-auto px-2 py-2">
+          {NAV.map((n) => (
+            <NavLink
+              key={n.to}
+              to={n.to}
+              end={n.to === "/"}
+              className={({ isActive }) =>
+                cn(
+                  "flex items-center gap-2.5 rounded-lg px-2.5 py-1.5 text-[13px] transition-colors",
+                  isActive ? "bg-surface-2 font-medium text-ink" : "text-ink-2 hover:bg-surface-2/60 hover:text-ink",
+                )
+              }
+            >
+              {n.icon}
+              {n.label}
+            </NavLink>
+          ))}
+        </nav>
+        <div className="space-y-0.5 border-t border-line p-2">
+          <button
+            onClick={() => setTheme(theme === "dark" ? "light" : "dark")}
+            className="flex w-full items-center gap-2.5 rounded-lg px-2.5 py-1.5 text-[13px] text-ink-2 hover:bg-surface-2/60 hover:text-ink"
+          >
+            {theme === "dark" ? <Sun className="size-4" /> : <Moon className="size-4" />}
+            {theme === "dark" ? "Light mode" : "Dark mode"}
+          </button>
+          <button
+            onClick={logout}
+            className="flex w-full items-center gap-2.5 rounded-lg px-2.5 py-1.5 text-[13px] text-ink-2 hover:bg-surface-2/60 hover:text-ink"
+          >
+            <LogOut className="size-4" />
+            Sign out
+          </button>
+        </div>
+      </aside>
+
+      <div className="flex min-w-0 flex-1 flex-col">
+        <div className="sticky top-0 z-20 border-b border-line bg-page/85 backdrop-blur">
+          <div className="flex items-center gap-2 overflow-x-auto px-4 py-2 md:hidden">
+            {NAV.map((n) => (
+              <NavLink
+                key={n.to}
+                to={n.to}
+                end={n.to === "/"}
+                className={({ isActive }) =>
+                  cn("flex shrink-0 items-center gap-1.5 rounded-lg px-2 py-1 text-xs", isActive ? "bg-surface-2 text-ink" : "text-ink-2")
+                }
+              >
+                {n.icon}
+                {n.label}
+              </NavLink>
+            ))}
+          </div>
+          <div className="flex flex-wrap items-center justify-end gap-2 px-4 py-2 md:px-6">
+            <RefreshCw className={cn("size-3.5 text-muted", fetching > 0 && "animate-spin text-accent")} aria-label={fetching ? "Refreshing" : "Idle"} />
+            <Select value={refresh} onChange={(e) => setRefresh(Number(e.target.value))} className="h-7.5 text-xs" aria-label="Auto refresh">
+              {REFRESH.map((r) => (
+                <option key={r.key} value={r.key}>
+                  Refresh: {r.label}
+                </option>
+              ))}
+            </Select>
+            <Segmented size="sm" options={RANGES.map((r) => ({ key: r.key, label: r.label }))} value={range} onChange={setRange} />
+            <button
+              onClick={() => setTheme(theme === "dark" ? "light" : "dark")}
+              className="inline-flex size-7 items-center justify-center rounded-lg text-ink-2 hover:bg-surface-2 md:hidden"
+              aria-label="Toggle theme"
+            >
+              {theme === "dark" ? <Sun className="size-4" /> : <Moon className="size-4" />}
+            </button>
+          </div>
+        </div>
+        <main className="mx-auto w-full max-w-[1400px] flex-1 px-4 py-6 md:px-6">
+          <Outlet />
+        </main>
+      </div>
+    </div>
+  );
+}

web/src/components/charts/BarList.tsx

@@ -0,0 +1,37 @@
+import type { ReactNode } from "react";
+import { compact } from "../../lib/format";
+import { cn } from "../ui";
+
+export interface BarItem {
+  key: string;
+  label: ReactNode;
+  value: number;
+  hint?: string;
+  onClick?: () => void;
+}
+
+/** Ranked list with a thin magnitude bar under each label. */
+export function BarList({ items, format = compact, empty = "Nothing yet" }: { items: BarItem[]; format?: (v: number) => string; empty?: string }) {
+  if (items.length === 0) return <div className="py-6 text-center text-sm text-muted">{empty}</div>;
+  const max = Math.max(...items.map((i) => i.value), 1);
+  return (
+    <ul className="space-y-2.5">
+      {items.map((it) => (
+        <li
+          key={it.key}
+          title={it.hint}
+          onClick={it.onClick}
+          className={cn("group", it.onClick && "cursor-pointer")}
+        >
+          <div className="flex items-baseline justify-between gap-3 text-[13px]">
+            <span className={cn("min-w-0 truncate text-ink-2", it.onClick && "group-hover:text-ink")}>{it.label}</span>
+            <span className="tnum shrink-0 text-ink">{format(it.value)}</span>
+          </div>
+          <div className="mt-1 h-1 w-full rounded-full bg-surface-2">
+            <div className="h-1 rounded-full bg-accent" style={{ width: `${Math.max(2, (it.value / max) * 100)}%` }} />
+          </div>
+        </li>
+      ))}
+    </ul>
+  );
+}

web/src/components/charts/Sparkline.tsx

@@ -0,0 +1,36 @@
+/** Tiny trend line: de-emphasised line, latest value marked in the accent. */
+export function Sparkline({ data, className }: { data: (number | null)[]; className?: string }) {
+  const vals = data.map((v) => (v === null || Number.isNaN(v) ? null : v));
+  const nums = vals.filter((v): v is number => v !== null);
+  if (nums.length < 2) return null;
+  const min = Math.min(...nums, 0);
+  const max = Math.max(...nums);
+  const W = 96;
+  const H = 32;
+  const pad = 4;
+  const x = (i: number) => (i / (vals.length - 1)) * (W - pad * 2) + pad;
+  const y = (v: number) => H - pad - (max === min ? 0.5 : (v - min) / (max - min)) * (H - pad * 2);
+
+  const segments: string[] = [];
+  let cur = "";
+  vals.forEach((v, i) => {
+    if (v === null) {
+      if (cur) segments.push(cur);
+      cur = "";
+      return;
+    }
+    cur += `${cur ? "L" : "M"}${x(i).toFixed(1)},${y(v).toFixed(1)}`;
+  });
+  if (cur) segments.push(cur);
+  let lastIdx = vals.length - 1;
+  while (lastIdx >= 0 && vals[lastIdx] === null) lastIdx--;
+
+  return (
+    <svg viewBox={`0 0 ${W} ${H}`} className={className} preserveAspectRatio="none" aria-hidden>
+      {segments.map((d, i) => (
+        <path key={i} d={d} fill="none" stroke="var(--muted)" strokeWidth={1.5} strokeLinejoin="round" strokeLinecap="round" vectorEffect="non-scaling-stroke" />
+      ))}
+      {lastIdx >= 0 && <circle cx={x(lastIdx)} cy={y(vals[lastIdx] as number)} r={2.5} fill="var(--accent)" stroke="var(--surface)" strokeWidth={1.5} />}
+    </svg>
+  );
+}

web/src/components/charts/TimeChart.tsx

@@ -0,0 +1,233 @@
+import { useEffect, useMemo, useRef, useState } from "react";
+import uPlot from "uplot";
+import { useSettings } from "../../lib/settings";
+import type { QueryResult } from "../../lib/types";
+import { compact } from "../../lib/format";
+import { cn } from "../ui";
+
+export interface ChartSeries {
+  label: string;
+  data: (number | null)[];
+  /** fixed categorical slot (1-8) so an entity keeps its colour across filters */
+  slot?: number;
+}
+
+export interface ChartData {
+  x: number[];
+  series: ChartSeries[];
+}
+
+/** Converts a metrics query result into chart data. */
+export function fromQuery(
+  q: QueryResult | null | undefined,
+  label: (labels: Record<string, string>, i: number) => string = (l) => Object.values(l).join(" · ") || q?.name || "value",
+  scale = 1,
+  slotOf?: (labels: Record<string, string>) => number | undefined,
+): ChartData {
+  if (!q || q.series.length === 0) return { x: [], series: [] };
+  const x = q.series[0].points.map((p) => p[0]);
+  return {
+    x,
+    series: q.series.map((s, i) => ({
+      label: label(s.labels, i),
+      slot: slotOf?.(s.labels),
+      data: s.points.map((p) => (p[1] === null ? null : p[1] * scale)),
+    })),
+  };
+}
+
+function cssVar(name: string): string {
+  return getComputedStyle(document.documentElement).getPropertyValue(name).trim();
+}
+
+function seriesColor(i: number, slot?: number): string {
+  const n = slot ?? i + 1;
+  return cssVar(`--series-${((n - 1) % 8) + 1}`);
+}
+
+function fmtTick(ts: number, span: number): string {
+  const d = new Date(ts * 1000);
+  if (span >= 2 * 86400) return d.toLocaleDateString("en-GB", { day: "2-digit", month: "short" });
+  return d.toLocaleTimeString("en-GB", { hour: "2-digit", minute: "2-digit" });
+}
+
+function fmtTime(ts: number, span: number): string {
+  const d = new Date(ts * 1000);
+  if (span > 2 * 86400) return d.toLocaleString("en-GB", { month: "short", day: "2-digit", hour: "2-digit", minute: "2-digit" });
+  return d.toLocaleTimeString("en-GB", { hour: "2-digit", minute: "2-digit" });
+}
+
+function tooltipPlugin(format: (v: number) => string, labels: () => ChartSeries[]): uPlot.Plugin {
+  let tip: HTMLDivElement;
+  return {
+    hooks: {
+      init: (u) => {
+        tip = document.createElement("div");
+        tip.className =
+          "pointer-events-none absolute z-10 hidden min-w-36 rounded-lg border border-line bg-surface px-2.5 py-2 text-xs text-ink shadow-lg";
+        u.over.appendChild(tip);
+        u.over.addEventListener("mouseleave", () => tip.classList.add("hidden"));
+      },
+      setCursor: (u) => {
+        const idx = u.cursor.idx;
+        if (idx === null || idx === undefined || u.cursor.left === undefined || u.cursor.left < 0) {
+          tip.classList.add("hidden");
+          return;
+        }
+        const ts = u.data[0][idx];
+        const span = (u.data[0][u.data[0].length - 1] ?? 0) - (u.data[0][0] ?? 0);
+        const rows = labels()
+          .map((s, i) => {
+            const v = u.data[i + 1][idx];
+            const color = (u.series[i + 1].stroke as () => string)?.() ?? "";
+            return `<div class="flex items-center justify-between gap-4 py-0.5"><span class="flex items-center gap-1.5 text-ink-2"><span style="background:${color}" class="inline-block h-0.5 w-3 rounded"></span>${escapeHtml(s.label)}</span><span class="tnum text-ink">${v === null || v === undefined ? "—" : format(v)}</span></div>`;
+          })
+          .join("");
+        tip.innerHTML = `<div class="mb-1 text-muted">${fmtTime(ts, span)}${span > 2 * 86400 ? "" : " · " + new Date(ts * 1000).toLocaleDateString("en-GB", { day: "2-digit", month: "short" })}</div>${rows}`;
+        tip.classList.remove("hidden");
+        const left = u.cursor.left;
+        const w = tip.offsetWidth;
+        const overW = u.over.clientWidth;
+        tip.style.left = `${left + 12 + w > overW ? left - w - 12 : left + 12}px`;
+        tip.style.top = `4px`;
+      },
+    },
+  };
+}
+
+function escapeHtml(s: string): string {
+  return s.replace(/[&<>"]/g, (c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" })[c] as string);
+}
+
+export function TimeChart({
+  data,
+  height = 220,
+  format = compact,
+  area,
+  bars,
+  yMin = 0,
+  className,
+  empty = "No data for this range yet",
+}: {
+  data: ChartData;
+  height?: number;
+  format?: (v: number) => string;
+  area?: boolean;
+  bars?: boolean;
+  yMin?: number | null;
+  className?: string;
+  empty?: string;
+}) {
+  const { theme } = useSettings();
+  const wrapRef = useRef<HTMLDivElement>(null);
+  const plotRef = useRef<uPlot | null>(null);
+  const seriesRef = useRef<ChartSeries[]>(data.series);
+  seriesRef.current = data.series;
+  const [width, setWidth] = useState(0);
+
+  const hasData = data.x.length > 0 && data.series.some((s) => s.data.some((v) => v !== null));
+  const shape = `${data.series.map((s) => `${s.label}#${s.slot ?? ""}`).join("|")}`;
+  const aligned = useMemo<uPlot.AlignedData>(() => [data.x, ...data.series.map((s) => s.data)] as uPlot.AlignedData, [data]);
+
+  useEffect(() => {
+    const el = wrapRef.current;
+    if (!el) return;
+    const ro = new ResizeObserver((entries) => setWidth(Math.floor(entries[0].contentRect.width)));
+    ro.observe(el);
+    return () => ro.disconnect();
+  }, []);
+
+  // (re)create when the theme or the set of series changes
+  useEffect(() => {
+    const el = wrapRef.current;
+    if (!el || !hasData || width === 0) return;
+    const muted = cssVar("--muted");
+    const grid = cssVar("--grid");
+    const font = "11px system-ui, -apple-system, Segoe UI, sans-serif";
+    const single = data.series.length === 1;
+    const opts: uPlot.Options = {
+      width,
+      height,
+      padding: [8, 14, 0, 0],
+      legend: { show: false },
+      cursor: { points: { size: 8, width: 2 }, drag: { x: false, y: false } },
+      scales: { x: { time: true }, y: { range: (_u, min, max) => [yMin === null ? min : Math.min(yMin, min), max <= 0 ? 1 : max * 1.08] } },
+      axes: [
+        {
+          stroke: muted,
+          font,
+          grid: { show: false },
+          ticks: { stroke: grid, width: 1, size: 4 },
+          space: 70,
+          values: (_u, splits) => {
+            const span = (splits[splits.length - 1] ?? 0) - (splits[0] ?? 0);
+            return splits.map((ts) => fmtTick(ts, span));
+          },
+        },
+        {
+          stroke: muted,
+          font,
+          size: 48,
+          space: 36,
+          grid: { stroke: grid, width: 1 },
+          ticks: { show: false },
+          values: (_u, vals) => vals.map((v) => format(v)),
+        },
+      ],
+      series: [
+        {},
+        ...data.series.map((s, i) => {
+          const color = seriesColor(i, s.slot);
+          return {
+            label: s.label,
+            stroke: color,
+            width: 2,
+            spanGaps: false,
+            // uPlot shows markers only when samples are far apart, so sparse data stays visible
+            points: { size: 7, width: 2, stroke: cssVar("--surface"), fill: color },
+            fill: bars ? color : area || single ? `color-mix(in srgb, ${color} 12%, transparent)` : undefined,
+            paths: bars ? uPlot.paths.bars!({ size: [0.6, 24], radius: 0.15 }) : undefined,
+          } as uPlot.Series;
+        }),
+      ],
+      plugins: [tooltipPlugin(format, () => seriesRef.current)],
+    };
+    plotRef.current?.destroy();
+    plotRef.current = new uPlot(opts, aligned, el);
+    return () => {
+      plotRef.current?.destroy();
+      plotRef.current = null;
+    };
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [theme, shape, hasData, height, bars, area, width === 0]);
+
+  useEffect(() => {
+    if (plotRef.current && width > 0) plotRef.current.setSize({ width, height });
+  }, [width, height]);
+
+  useEffect(() => {
+    plotRef.current?.setData(aligned);
+  }, [aligned]);
+
+  return (
+    <div className={cn("min-w-0", className)}>
+      <div ref={wrapRef} style={{ height }} className="relative w-full">
+        {!hasData && <div className="flex h-full items-center justify-center text-sm text-muted">{empty}</div>}
+      </div>
+      {hasData && data.series.length > 1 && <Legend series={data.series} />}
+    </div>
+  );
+}
+
+export function Legend({ series }: { series: ChartSeries[] }) {
+  return (
+    <div className="mt-2 flex flex-wrap gap-x-4 gap-y-1 text-xs text-ink-2">
+      {series.map((s, i) => (
+        <span key={s.label + i} className="inline-flex items-center gap-1.5">
+          <span className="inline-block h-0.5 w-3 rounded" style={{ background: `var(--series-${(((s.slot ?? i + 1) - 1) % 8) + 1})` }} />
+          {s.label}
+        </span>
+      ))}
+    </div>
+  );
+}

web/src/components/ui.tsx

@@ -0,0 +1,484 @@
+import {
+  createContext,
+  useCallback,
+  useContext,
+  useEffect,
+  useRef,
+  useState,
+  type ButtonHTMLAttributes,
+  type InputHTMLAttributes,
+  type ReactNode,
+  type SelectHTMLAttributes,
+  type TextareaHTMLAttributes,
+} from "react";
+import { AlertTriangle, Check, CheckCircle2, ChevronLeft, ChevronRight, Copy, Info, Loader2, X, XCircle } from "lucide-react";
+import { Sparkline } from "./charts/Sparkline";
+
+export function cn(...parts: (string | false | null | undefined)[]): string {
+  return parts.filter(Boolean).join(" ");
+}
+
+// ---- containers -------------------------------------------------------------
+
+export function Card({
+  title,
+  subtitle,
+  actions,
+  children,
+  className,
+  bodyClass,
+}: {
+  title?: ReactNode;
+  subtitle?: ReactNode;
+  actions?: ReactNode;
+  children: ReactNode;
+  className?: string;
+  bodyClass?: string;
+}) {
+  return (
+    <section className={cn("rounded-xl border border-line bg-surface", className)}>
+      {(title || actions) && (
+        <header className="flex items-start justify-between gap-3 px-4 pt-3.5">
+          <div className="min-w-0">
+            {title && <h2 className="text-[13px] font-medium text-ink">{title}</h2>}
+            {subtitle && <p className="mt-0.5 text-xs text-muted">{subtitle}</p>}
+          </div>
+          {actions && <div className="flex shrink-0 items-center gap-2">{actions}</div>}
+        </header>
+      )}
+      <div className={cn("p-4", bodyClass)}>{children}</div>
+    </section>
+  );
+}
+
+export function PageHeader({ title, description, actions }: { title: string; description?: ReactNode; actions?: ReactNode }) {
+  return (
+    <div className="mb-5 flex flex-wrap items-end justify-between gap-3">
+      <div>
+        <h1 className="text-xl font-semibold tracking-tight text-ink">{title}</h1>
+        {description && <p className="mt-1 text-sm text-ink-2">{description}</p>}
+      </div>
+      {actions && <div className="flex flex-wrap items-center gap-2">{actions}</div>}
+    </div>
+  );
+}
+
+// ---- controls ---------------------------------------------------------------
+
+type ButtonVariant = "primary" | "secondary" | "ghost" | "danger";
+
+export function Button({
+  variant = "secondary",
+  size = "md",
+  icon,
+  loading,
+  className,
+  children,
+  ...rest
+}: ButtonHTMLAttributes<HTMLButtonElement> & {
+  variant?: ButtonVariant;
+  size?: "sm" | "md";
+  icon?: ReactNode;
+  loading?: boolean;
+}) {
+  const styles: Record<ButtonVariant, string> = {
+    primary: "bg-accent text-accent-ink hover:brightness-110 border-transparent",
+    secondary: "bg-surface-2 text-ink border-line hover:border-ink/25",
+    ghost: "bg-transparent text-ink-2 border-transparent hover:bg-surface-2 hover:text-ink",
+    danger: "bg-transparent text-critical border-critical/40 hover:bg-critical/10",
+  };
+  return (
+    <button
+      {...rest}
+      disabled={rest.disabled || loading}
+      className={cn(
+        "inline-flex items-center justify-center gap-1.5 whitespace-nowrap rounded-lg border font-medium transition-colors",
+        "disabled:cursor-not-allowed disabled:opacity-50 focus-visible:outline-2 focus-visible:outline-accent",
+        size === "sm" ? "h-7 px-2.5 text-xs" : "h-8.5 px-3 text-[13px]",
+        styles[variant],
+        className,
+      )}
+    >
+      {loading ? <Loader2 className="size-3.5 animate-spin" /> : icon}
+      {children}
+    </button>
+  );
+}
+
+const fieldClass =
+  "h-8.5 rounded-lg border border-line bg-surface px-2.5 text-[13px] text-ink placeholder:text-muted " +
+  "focus:border-accent focus:outline-none";
+
+export function Input({ className, ...rest }: InputHTMLAttributes<HTMLInputElement>) {
+  return <input {...rest} className={cn(fieldClass, className)} />;
+}
+
+export function Select({ className, children, ...rest }: SelectHTMLAttributes<HTMLSelectElement>) {
+  return (
+    <select {...rest} className={cn(fieldClass, "pr-7", className)}>
+      {children}
+    </select>
+  );
+}
+
+export function Textarea({ className, ...rest }: TextareaHTMLAttributes<HTMLTextAreaElement>) {
+  return <textarea {...rest} className={cn(fieldClass, "h-auto py-2 font-mono text-xs", className)} />;
+}
+
+export function Segmented<T extends string | number>({
+  options,
+  value,
+  onChange,
+  size = "md",
+}: {
+  options: readonly { key: T; label: ReactNode }[];
+  value: T;
+  onChange: (v: T) => void;
+  size?: "sm" | "md";
+}) {
+  return (
+    <div className="inline-flex rounded-lg border border-line bg-surface p-0.5">
+      {options.map((o) => (
+        <button
+          key={String(o.key)}
+          onClick={() => onChange(o.key)}
+          className={cn(
+            "rounded-md px-2.5 font-medium transition-colors",
+            size === "sm" ? "h-6 text-xs" : "h-7 text-[13px]",
+            o.key === value ? "bg-surface-2 text-ink shadow-sm" : "text-muted hover:text-ink",
+          )}
+        >
+          {o.label}
+        </button>
+      ))}
+    </div>
+  );
+}
+
+export function Checkbox({ checked, onChange, label }: { checked: boolean; onChange: (v: boolean) => void; label?: string }) {
+  return (
+    <input
+      type="checkbox"
+      aria-label={label}
+      checked={checked}
+      onChange={(e) => onChange(e.target.checked)}
+      onClick={(e) => e.stopPropagation()}
+      className="size-3.5 cursor-pointer accent-[var(--accent)]"
+    />
+  );
+}
+
+// ---- status -----------------------------------------------------------------
+
+export type Tone = "neutral" | "accent" | "good" | "warning" | "serious" | "critical";
+
+const toneDot: Record<Tone, string> = {
+  neutral: "bg-muted",
+  accent: "bg-accent",
+  good: "bg-good",
+  warning: "bg-warning",
+  serious: "bg-serious",
+  critical: "bg-critical",
+};
+
+/** A status label: always a dot plus text, never colour alone. */
+export function Badge({ tone = "neutral", children, title }: { tone?: Tone; children: ReactNode; title?: string }) {
+  return (
+    <span
+      title={title}
+      className="inline-flex items-center gap-1.5 whitespace-nowrap rounded-full border border-line bg-surface-2 px-2 py-0.5 text-[11px] font-medium text-ink-2"
+    >
+      <span className={cn("size-1.5 rounded-full", toneDot[tone])} />
+      {children}
+    </span>
+  );
+}
+
+export function StatusDot({ online }: { online: boolean }) {
+  return (
+    <span className="relative inline-flex size-2">
+      {online && <span className="absolute inline-flex size-full animate-ping rounded-full bg-good opacity-50" />}
+      <span className={cn("relative inline-flex size-2 rounded-full", online ? "bg-good" : "bg-critical")} />
+    </span>
+  );
+}
+
+// ---- figures ----------------------------------------------------------------
+
+export function StatTile({
+  label,
+  value,
+  sub,
+  trend,
+  hero,
+  onClick,
+}: {
+  label: string;
+  value: ReactNode;
+  sub?: ReactNode;
+  trend?: (number | null)[];
+  hero?: boolean;
+  onClick?: () => void;
+}) {
+  return (
+    <div
+      onClick={onClick}
+      className={cn(
+        "flex min-w-0 flex-col justify-between rounded-xl border border-line bg-surface p-4",
+        onClick && "cursor-pointer transition-colors hover:border-ink/25",
+      )}
+    >
+      <div className="text-xs text-muted">{label}</div>
+      <div className="mt-2 flex items-end justify-between gap-3">
+        <div className="min-w-0">
+          <div className={cn("truncate font-semibold tracking-tight text-ink", hero ? "text-[40px] leading-none" : "text-2xl")}>{value}</div>
+          {sub && <div className="mt-1.5 truncate text-xs text-ink-2">{sub}</div>}
+        </div>
+        {trend && trend.length > 1 && <Sparkline data={trend} className="h-8 w-24 shrink-0" />}
+      </div>
+    </div>
+  );
+}
+
+/** Progress meter: unfilled track is a lighter step of the fill. */
+export function Meter({ value, max, tone = "accent" }: { value: number; max: number; tone?: "accent" | "good" | "warning" | "critical" }) {
+  const p = max > 0 ? Math.min(100, (value / max) * 100) : 0;
+  const color = { accent: "var(--accent)", good: "var(--good)", warning: "var(--warning)", critical: "var(--critical)" }[tone];
+  return (
+    <div className="h-1.5 w-full overflow-hidden rounded-full" style={{ background: `color-mix(in srgb, ${color} 18%, transparent)` }}>
+      <div className="h-full rounded-full" style={{ width: `${p}%`, background: color }} />
+    </div>
+  );
+}
+
+// ---- feedback -----------------------------------------------------------------
+
+export function Spinner({ className }: { className?: string }) {
+  return <Loader2 className={cn("size-4 animate-spin text-muted", className)} />;
+}
+
+export function Loading({ label = "Loading…" }: { label?: string }) {
+  return (
+    <div className="flex items-center gap-2 py-8 text-sm text-muted">
+      <Spinner /> {label}
+    </div>
+  );
+}
+
+export function Empty({ children }: { children: ReactNode }) {
+  return <div className="py-8 text-center text-sm text-muted">{children}</div>;
+}
+
+export function ErrorBox({ error }: { error: unknown }) {
+  const msg = error instanceof Error ? error.message : String(error);
+  return (
+    <div className="flex items-center gap-2 rounded-lg border border-critical/30 bg-critical/10 px-3 py-2 text-sm text-ink">
+      <XCircle className="size-4 shrink-0 text-critical" />
+      {msg}
+    </div>
+  );
+}
+
+export function CopyButton({ value, className }: { value: string; className?: string }) {
+  const [done, setDone] = useState(false);
+  return (
+    <button
+      title="Copy"
+      onClick={(e) => {
+        e.stopPropagation();
+        navigator.clipboard?.writeText(value).then(() => {
+          setDone(true);
+          setTimeout(() => setDone(false), 1200);
+        });
+      }}
+      className={cn("inline-flex size-6 items-center justify-center rounded-md text-muted hover:bg-surface-2 hover:text-ink", className)}
+    >
+      {done ? <Check className="size-3.5 text-good" /> : <Copy className="size-3.5" />}
+    </button>
+  );
+}
+
+export function Pager({ total, limit, offset, onChange }: { total: number; limit: number; offset: number; onChange: (offset: number) => void }) {
+  const page = Math.floor(offset / limit) + 1;
+  const pages = Math.max(1, Math.ceil(total / limit));
+  return (
+    <div className="flex items-center justify-between gap-3 text-xs text-muted">
+      <span className="tnum">
+        {total === 0 ? "No results" : `${(offset + 1).toLocaleString()}–${Math.min(offset + limit, total).toLocaleString()} of ${total.toLocaleString()}`}
+      </span>
+      <div className="flex items-center gap-1">
+        <Button size="sm" variant="ghost" disabled={page <= 1} onClick={() => onChange(Math.max(0, offset - limit))} icon={<ChevronLeft className="size-3.5" />} />
+        <span className="tnum px-1">
+          {page} / {pages}
+        </span>
+        <Button size="sm" variant="ghost" disabled={page >= pages} onClick={() => onChange(offset + limit)} icon={<ChevronRight className="size-3.5" />} />
+      </div>
+    </div>
+  );
+}
+
+// ---- table --------------------------------------------------------------------
+
+export function Table({ children, className }: { children: ReactNode; className?: string }) {
+  return (
+    <div className={cn("-mx-4 overflow-x-auto", className)}>
+      <table className="w-full border-collapse text-[13px]">{children}</table>
+    </div>
+  );
+}
+
+export function Th({ children, className, onClick }: { children?: ReactNode; className?: string; onClick?: () => void }) {
+  return (
+    <th
+      onClick={onClick}
+      className={cn(
+        "border-b border-line px-4 py-2 text-left text-[11px] font-medium uppercase tracking-wide text-muted first:pl-4 whitespace-nowrap",
+        onClick && "cursor-pointer select-none hover:text-ink",
+        className,
+      )}
+    >
+      {children}
+    </th>
+  );
+}
+
+export function Td({ children, className, title }: { children?: ReactNode; className?: string; title?: string }) {
+  return (
+    <td title={title} className={cn("border-b border-line/60 px-4 py-2 align-middle text-ink-2", className)}>
+      {children}
+    </td>
+  );
+}
+
+// ---- toasts & confirm ---------------------------------------------------------
+
+interface ToastItem {
+  id: number;
+  tone: "good" | "critical" | "neutral";
+  text: string;
+}
+
+interface Feedback {
+  toast: (text: string, tone?: ToastItem["tone"]) => void;
+  confirm: (opts: { title: string; body?: ReactNode; confirmLabel?: string; danger?: boolean; requireText?: string }) => Promise<boolean>;
+}
+
+const FeedbackCtx = createContext<Feedback | null>(null);
+
+export function useFeedback(): Feedback {
+  const f = useContext(FeedbackCtx);
+  if (!f) throw new Error("FeedbackProvider missing");
+  return f;
+}
+
+interface ConfirmState {
+  title: string;
+  body?: ReactNode;
+  confirmLabel?: string;
+  danger?: boolean;
+  requireText?: string;
+  resolve: (ok: boolean) => void;
+}
+
+export function FeedbackProvider({ children }: { children: ReactNode }) {
+  const [toasts, setToasts] = useState<ToastItem[]>([]);
+  const [confirmState, setConfirmState] = useState<ConfirmState | null>(null);
+  const [typed, setTyped] = useState("");
+  const dialogRef = useRef<HTMLDialogElement>(null);
+  const nextId = useRef(1);
+
+  const toast = useCallback((text: string, tone: ToastItem["tone"] = "good") => {
+    const id = nextId.current++;
+    setToasts((t) => [...t, { id, tone, text }]);
+    setTimeout(() => setToasts((t) => t.filter((x) => x.id !== id)), tone === "critical" ? 6000 : 3500);
+  }, []);
+
+  const confirm = useCallback<Feedback["confirm"]>(
+    (opts) =>
+      new Promise<boolean>((resolve) => {
+        setTyped("");
+        setConfirmState({ ...opts, resolve });
+      }),
+    [],
+  );
+
+  useEffect(() => {
+    const d = dialogRef.current;
+    if (!d) return;
+    if (confirmState && !d.open) d.showModal();
+    if (!confirmState && d.open) d.close();
+  }, [confirmState]);
+
+  const close = (ok: boolean) => {
+    confirmState?.resolve(ok);
+    setConfirmState(null);
+  };
+  const blocked = !!confirmState?.requireText && typed !== confirmState.requireText;
+
+  return (
+    <FeedbackCtx.Provider value={{ toast, confirm }}>
+      {children}
+      <dialog
+        ref={dialogRef}
+        onCancel={(e) => {
+          e.preventDefault();
+          close(false);
+        }}
+        className="m-auto w-[min(440px,calc(100vw-32px))] rounded-xl border border-line bg-surface p-0 text-ink shadow-2xl"
+      >
+        {confirmState && (
+          <div className="p-5">
+            <div className="flex items-start gap-3">
+              {confirmState.danger ? (
+                <AlertTriangle className="mt-0.5 size-5 shrink-0 text-critical" />
+              ) : (
+                <Info className="mt-0.5 size-5 shrink-0 text-accent" />
+              )}
+              <div className="min-w-0">
+                <h3 className="font-medium">{confirmState.title}</h3>
+                {confirmState.body && <div className="mt-1.5 text-sm text-ink-2">{confirmState.body}</div>}
+              </div>
+            </div>
+            {confirmState.requireText && (
+              <div className="mt-4">
+                <label className="text-xs text-muted">
+                  Type <span className="font-mono text-ink">{confirmState.requireText}</span> to confirm
+                </label>
+                <Input autoFocus className="mt-1 w-full font-mono" value={typed} onChange={(e) => setTyped(e.target.value)} />
+              </div>
+            )}
+            <div className="mt-5 flex justify-end gap-2">
+              <Button variant="ghost" onClick={() => close(false)}>
+                Cancel
+              </Button>
+              <Button variant={confirmState.danger ? "danger" : "primary"} disabled={blocked} onClick={() => close(true)} autoFocus={!confirmState.requireText}>
+                {confirmState.confirmLabel ?? "Confirm"}
+              </Button>
+            </div>
+          </div>
+        )}
+      </dialog>
+      <div className="pointer-events-none fixed right-4 bottom-4 z-50 flex flex-col gap-2">
+        {toasts.map((t) => (
+          <div
+            key={t.id}
+            className="pointer-events-auto flex max-w-sm items-center gap-2 rounded-lg border border-line bg-surface px-3 py-2 text-sm text-ink shadow-lg"
+          >
+            {t.tone === "good" && <CheckCircle2 className="size-4 shrink-0 text-good" />}
+            {t.tone === "critical" && <XCircle className="size-4 shrink-0 text-critical" />}
+            {t.tone === "neutral" && <Info className="size-4 shrink-0 text-accent" />}
+            <span className="min-w-0 break-words">{t.text}</span>
+            <button className="ml-1 text-muted hover:text-ink" onClick={() => setToasts((x) => x.filter((y) => y.id !== t.id))}>
+              <X className="size-3.5" />
+            </button>
+          </div>
+        ))}
+      </div>
+    </FeedbackCtx.Provider>
+  );
+}
+
+export function LevelBadge({ level }: { level: string }) {
+  const tone: Tone = level === "error" ? "critical" : level === "warn" ? "warning" : level === "debug" ? "neutral" : "accent";
+  return <Badge tone={tone}>{level}</Badge>;
+}

web/src/index.css

@@ -0,0 +1,176 @@
+@import "tailwindcss";
+
+@custom-variant dark (&:where([data-theme="dark"], [data-theme="dark"] *));
+
+/* Colour tokens. Chart colours follow the validated reference palette
+   (categorical slots in fixed order, separate light/dark steps). */
+:root {
+  color-scheme: light;
+  --page: #f9f9f7;
+  --surface: #fcfcfb;
+  --surface-2: #f2f2ef;
+  --ink: #0b0b0b;
+  --ink-2: #52514e;
+  --muted: #898781;
+  --line: rgba(11, 11, 11, 0.1);
+  --grid: #e1e0d9;
+  --axis: #c3c2b7;
+  --accent: #2a78d6;
+  --accent-ink: #ffffff;
+  --good: #0ca30c;
+  --good-ink: #006300;
+  --warning: #fab219;
+  --serious: #ec835a;
+  --critical: #d03b3b;
+
+  --series-1: #2a78d6;
+  --series-2: #eb6834;
+  --series-3: #1baf7a;
+  --series-4: #eda100;
+  --series-5: #e87ba4;
+  --series-6: #008300;
+  --series-7: #4a3aa7;
+  --series-8: #e34948;
+}
+
+:root[data-theme="dark"] {
+  color-scheme: dark;
+  --page: #0d0d0d;
+  --surface: #1a1a19;
+  --surface-2: #232321;
+  --ink: #ffffff;
+  --ink-2: #c3c2b7;
+  --muted: #898781;
+  --line: rgba(255, 255, 255, 0.1);
+  --grid: #2c2c2a;
+  --axis: #383835;
+  --accent: #3987e5;
+  --accent-ink: #ffffff;
+  --good-ink: #0ca30c;
+
+  --series-1: #3987e5;
+  --series-2: #d95926;
+  --series-3: #199e70;
+  --series-4: #c98500;
+  --series-5: #d55181;
+  --series-6: #008300;
+  --series-7: #9085e9;
+  --series-8: #e66767;
+}
+
+@theme inline {
+  --color-page: var(--page);
+  --color-surface: var(--surface);
+  --color-surface-2: var(--surface-2);
+  --color-ink: var(--ink);
+  --color-ink-2: var(--ink-2);
+  --color-muted: var(--muted);
+  --color-line: var(--line);
+  --color-grid: var(--grid);
+  --color-accent: var(--accent);
+  --color-accent-ink: var(--accent-ink);
+  --color-good: var(--good);
+  --color-good-ink: var(--good-ink);
+  --color-warning: var(--warning);
+  --color-serious: var(--serious);
+  --color-critical: var(--critical);
+  --font-sans: system-ui, -apple-system, "Segoe UI", Roboto, sans-serif;
+  --font-mono: ui-monospace, "SF Mono", "Cascadia Code", Consolas, monospace;
+}
+
+html,
+body {
+  background: var(--page);
+  color: var(--ink);
+}
+
+body {
+  font-family: var(--font-sans);
+  font-size: 14px;
+  -webkit-font-smoothing: antialiased;
+}
+
+.tnum {
+  font-variant-numeric: tabular-nums;
+}
+
+/* uPlot base styles (trimmed from uplot.min.css) */
+.uplot,
+.uplot *,
+.uplot *::before,
+.uplot *::after {
+  box-sizing: border-box;
+}
+.uplot {
+  font-family: var(--font-sans);
+  line-height: 1.5;
+  width: min-content;
+}
+.u-wrap {
+  position: relative;
+  user-select: none;
+}
+.u-over,
+.u-under {
+  position: absolute;
+}
+.u-under {
+  overflow: hidden;
+}
+.uplot canvas {
+  display: block;
+  position: relative;
+  width: 100%;
+  height: 100%;
+}
+.u-axis {
+  position: absolute;
+}
+.u-select {
+  background: color-mix(in srgb, var(--accent) 12%, transparent);
+  position: absolute;
+  pointer-events: none;
+}
+.u-cursor-x,
+.u-cursor-y {
+  position: absolute;
+  left: 0;
+  top: 0;
+  pointer-events: none;
+  will-change: transform;
+}
+.u-cursor-x {
+  height: 100%;
+  border-right: 1px solid var(--axis);
+}
+.u-cursor-y {
+  width: 100%;
+  border-bottom: 0;
+}
+.u-cursor-pt {
+  position: absolute;
+  top: 0;
+  left: 0;
+  border-radius: 50%;
+  border: 2px solid var(--surface);
+  pointer-events: none;
+  will-change: transform;
+  background-clip: padding-box !important;
+}
+.u-hz .u-cursor-y {
+  display: none;
+}
+
+dialog::backdrop {
+  background: rgba(0, 0, 0, 0.5);
+}
+
+::-webkit-scrollbar {
+  width: 10px;
+  height: 10px;
+}
+::-webkit-scrollbar-thumb {
+  background: var(--grid);
+  border-radius: 10px;
+  border: 2px solid var(--page);
+}

web/src/lib/api.ts

@@ -0,0 +1,53 @@
+// Thin fetch wrapper for the admin JSON API. Auth is the HttpOnly session
+// cookie; every request carries the CSRF header the server requires.
+
+export class ApiError extends Error {
+  status: number;
+  constructor(status: number, message: string) {
+    super(message);
+    this.status = status;
+  }
+}
+
+export const UNAUTHORIZED_EVENT = "rc:unauthorized";
+
+type Method = "GET" | "POST" | "DELETE";
+
+export async function api<T>(path: string, opts: { method?: Method; body?: unknown } = {}): Promise<T> {
+  const headers: Record<string, string> = { "X-Admin-CSRF": "1", Accept: "application/json" };
+  if (opts.body !== undefined) headers["Content-Type"] = "application/json";
+  const res = await fetch(`/admin/api${path}`, {
+    method: opts.method ?? "GET",
+    credentials: "same-origin",
+    headers,
+    body: opts.body !== undefined ? JSON.stringify(opts.body) : undefined,
+  });
+  if (res.status === 204) return undefined as T;
+  let data: unknown = null;
+  const text = await res.text();
+  if (text) {
+    try {
+      data = JSON.parse(text);
+    } catch {
+      data = { error: text };
+    }
+  }
+  if (!res.ok) {
+    if (res.status === 401 && !path.startsWith("/auth/")) {
+      window.dispatchEvent(new Event(UNAUTHORIZED_EVENT));
+    }
+    const msg = (data as { error?: string } | null)?.error ?? `${res.status} ${res.statusText}`;
+    throw new ApiError(res.status, msg);
+  }
+  return data as T;
+}
+
+export function qs(params: Record<string, string | number | boolean | undefined | null>): string {
+  const p = new URLSearchParams();
+  for (const [k, v] of Object.entries(params)) {
+    if (v === undefined || v === null || v === "") continue;
+    p.set(k, String(v));
+  }
+  const s = p.toString();
+  return s ? `?${s}` : "";
+}

web/src/lib/format.ts

@@ -0,0 +1,94 @@
+const compactFmt = new Intl.NumberFormat("en", { notation: "compact", maximumFractionDigits: 1 });
+const intFmt = new Intl.NumberFormat("en");
+
+export function compact(n: number | null | undefined): string {
+  if (n === null || n === undefined || Number.isNaN(n)) return "—";
+  if (Math.abs(n) < 1000) return intFmt.format(Math.round(n * 10) / 10);
+  return compactFmt.format(n);
+}
+
+export function int(n: number | null | undefined): string {
+  if (n === null || n === undefined || Number.isNaN(n)) return "—";
+  return intFmt.format(Math.round(n));
+}
+
+export function num(n: number | null | undefined, digits = 1): string {
+  if (n === null || n === undefined || Number.isNaN(n)) return "—";
+  return n.toLocaleString("en", { maximumFractionDigits: digits });
+}
+
+export function pct(part: number, total: number, digits = 1): string {
+  if (!total) return "—";
+  return `${((part / total) * 100).toFixed(digits)}%`;
+}
+
+export function bytes(n: number | null | undefined): string {
+  if (n === null || n === undefined || Number.isNaN(n)) return "—";
+  const units = ["B", "KB", "MB", "GB", "TB", "PB"];
+  let v = n;
+  let i = 0;
+  while (Math.abs(v) >= 1000 && i < units.length - 1) {
+    v /= 1000;
+    i++;
+  }
+  return `${v.toFixed(v < 10 && i > 0 ? 1 : 0)} ${units[i]}`;
+}
+
+export function ms(n: number | null | undefined): string {
+  if (n === null || n === undefined || Number.isNaN(n)) return "—";
+  if (n < 1) return `${(n * 1000).toFixed(0)} µs`;
+  if (n < 1000) return `${n.toFixed(n < 10 ? 1 : 0)} ms`;
+  return `${(n / 1000).toFixed(2)} s`;
+}
+
+export function duration(sec: number | null | undefined): string {
+  if (sec === null || sec === undefined || Number.isNaN(sec)) return "—";
+  sec = Math.max(0, Math.round(sec));
+  const d = Math.floor(sec / 86400);
+  const h = Math.floor((sec % 86400) / 3600);
+  const m = Math.floor((sec % 3600) / 60);
+  const s = sec % 60;
+  if (d) return `${d}d ${h}h`;
+  if (h) return `${h}h ${m}m`;
+  if (m) return `${m}m ${s}s`;
+  return `${s}s`;
+}
+
+/** Parses an ISO string or unix seconds/ms; returns null for zero/empty times. */
+export function toDate(v: string | number | null | undefined): Date | null {
+  if (v === null || v === undefined || v === "" || v === 0) return null;
+  const d = typeof v === "number" ? new Date(v > 1e12 ? v : v * 1000) : new Date(v);
+  if (Number.isNaN(d.getTime()) || d.getFullYear() < 2000) return null;
+  return d;
+}
+
+export function relative(v: string | number | null | undefined): string {
+  const d = toDate(v);
+  if (!d) return "—";
+  const diff = (Date.now() - d.getTime()) / 1000;
+  if (Math.abs(diff) < 45) return "just now";
+  const future = diff < 0;
+  const a = Math.abs(diff);
+  let out: string;
+  if (a < 3600) out = `${Math.round(a / 60)}m`;
+  else if (a < 86400) out = `${Math.round(a / 3600)}h`;
+  else if (a < 86400 * 60) out = `${Math.round(a / 86400)}d`;
+  else out = `${Math.round(a / (86400 * 30))}mo`;
+  return future ? `in ${out}` : `${out} ago`;
+}
+
+export function dateTime(v: string | number | null | undefined): string {
+  const d = toDate(v);
+  if (!d) return "—";
+  return d.toLocaleString("en-GB", { year: "numeric", month: "short", day: "2-digit", hour: "2-digit", minute: "2-digit" });
+}
+
+export function shortHash(h: string, n = 10): string {
+  return h.length > n * 2 ? `${h.slice(0, n)}…${h.slice(-4)}` : h;
+}
+
+/** Seconds for chart axes: keeps one decimal below a minute so ticks stay distinct. */
+export function seconds(v: number): string {
+  if (v < 60) return `${num(v, v < 10 ? 1 : 0)}s`;
+  return duration(v);
+}

web/src/lib/queries.ts

@@ -0,0 +1,59 @@
+import { useQuery, keepPreviousData } from "@tanstack/react-query";
+import { api, qs } from "./api";
+import { rangeSeconds, useRefetch, useSettings } from "./settings";
+import type { QueryResult } from "./types";
+
+export interface MetricParams {
+  service?: string;
+  name: string;
+  agg?: string;
+  groupBy?: string[];
+  match?: Record<string, string>;
+  step?: number;
+  /** override the global range (seconds) */
+  rangeSec?: number;
+}
+
+/** Fetches a metric time series for the global time range. */
+export function useMetric(p: MetricParams, enabled = true) {
+  const { range } = useSettings();
+  const refetchInterval = useRefetch();
+  const span = p.rangeSec ?? rangeSeconds(range);
+  return useQuery({
+    queryKey: ["metric", p, span],
+    enabled,
+    refetchInterval,
+    placeholderData: keepPreviousData,
+    queryFn: () => {
+      const to = Math.floor(Date.now() / 1000);
+      return api<QueryResult>(
+        "/metrics/query" +
+          qs({
+            service: p.service ?? "main",
+            name: p.name,
+            agg: p.agg,
+            groupBy: p.groupBy?.join(","),
+            match: p.match ? JSON.stringify(p.match) : undefined,
+            step: p.step,
+            from: to - span,
+            to,
+          }),
+      );
+    },
+  });
+}
+
+/** Default step for a range so charts stay readable. */
+export function stepFor(seconds: number): number {
+  if (seconds <= 3 * 3600) return 60;
+  if (seconds <= 12 * 3600) return 300;
+  if (seconds <= 48 * 3600) return 900;
+  if (seconds <= 8 * 86400) return 3600;
+  return 6 * 3600;
+}
+
+export function useRangeStep() {
+  const { range } = useSettings();
+  const seconds = rangeSeconds(range);
+  return { seconds, step: stepFor(seconds) };
+}

web/src/lib/settings.tsx

@@ -0,0 +1,102 @@
+import { createContext, useCallback, useContext, useEffect, useMemo, useState, type ReactNode } from "react";
+
+export type Theme = "dark" | "light";
+
+export const RANGES = [
+  { key: "1h", label: "1h", seconds: 3600 },
+  { key: "6h", label: "6h", seconds: 6 * 3600 },
+  { key: "24h", label: "24h", seconds: 86400 },
+  { key: "7d", label: "7d", seconds: 7 * 86400 },
+  { key: "30d", label: "30d", seconds: 30 * 86400 },
+] as const;
+export type RangeKey = (typeof RANGES)[number]["key"];
+
+export const REFRESH = [
+  { key: 0, label: "Off" },
+  { key: 10, label: "10s" },
+  { key: 30, label: "30s" },
+  { key: 60, label: "1m" },
+] as const;
+
+interface Settings {
+  theme: Theme;
+  setTheme: (t: Theme) => void;
+  range: RangeKey;
+  setRange: (r: RangeKey) => void;
+  refresh: number;
+  setRefresh: (s: number) => void;
+}
+
+const Ctx = createContext<Settings | null>(null);
+
+function load<T>(key: string, fallback: T, valid: (v: unknown) => boolean): T {
+  try {
+    const raw = localStorage.getItem(key);
+    if (raw === null) return fallback;
+    const v = JSON.parse(raw);
+    return valid(v) ? (v as T) : fallback;
+  } catch {
+    return fallback;
+  }
+}
+
+function save(key: string, v: unknown) {
+  try {
+    localStorage.setItem(key, JSON.stringify(v));
+  } catch {
+    /* storage unavailable */
+  }
+}
+
+export function SettingsProvider({ children }: { children: ReactNode }) {
+  const [theme, setThemeState] = useState<Theme>(() =>
+    document.documentElement.getAttribute("data-theme") === "light" ? "light" : "dark",
+  );
+  const [range, setRangeState] = useState<RangeKey>(() =>
+    load("rc-range", "24h", (v) => RANGES.some((r) => r.key === v)),
+  );
+  const [refresh, setRefreshState] = useState<number>(() =>
+    load("rc-refresh", 30, (v) => REFRESH.some((r) => r.key === v)),
+  );
+
+  useEffect(() => {
+    document.documentElement.setAttribute("data-theme", theme);
+    try {
+      localStorage.setItem("rc-theme", theme);
+    } catch {
+      /* ignore */
+    }
+  }, [theme]);
+
+  const setTheme = useCallback((t: Theme) => setThemeState(t), []);
+  const setRange = useCallback((r: RangeKey) => {
+    setRangeState(r);
+    save("rc-range", r);
+  }, []);
+  const setRefresh = useCallback((s: number) => {
+    setRefreshState(s);
+    save("rc-refresh", s);
+  }, []);
+
+  const value = useMemo(
+    () => ({ theme, setTheme, range, setRange, refresh, setRefresh }),
+    [theme, setTheme, range, setRange, refresh, setRefresh],
+  );
+  return <Ctx.Provider value={value}>{children}</Ctx.Provider>;
+}
+
+export function useSettings(): Settings {
+  const s = useContext(Ctx);
+  if (!s) throw new Error("SettingsProvider missing");
+  return s;
+}
+
+export function rangeSeconds(key: RangeKey): number {
+  return RANGES.find((r) => r.key === key)?.seconds ?? 86400;
+}
+
+/** Refetch interval in ms for TanStack Query (false when auto-refresh is off). */
+export function useRefetch(): number | false {
+  const { refresh } = useSettings();
+  return refresh > 0 ? refresh * 1000 : false;
+}

web/src/lib/types.ts

@@ -0,0 +1,289 @@
+// Response shapes of the admin API (see admin/*.go).
+
+export type Pt = [number, number | null];
+
+export interface SeriesResult {
+  labels: Record<string, string>;
+  points: Pt[];
+}
+
+export interface QueryResult {
+  name: string;
+  kind: string;
+  agg: string;
+  resolution: number;
+  step: number;
+  from: number;
+  to: number;
+  series: SeriesResult[];
+  truncated: boolean;
+}
+
+export interface Total {
+  labels: Record<string, string>;
+  count: number;
+  sum: number;
+  min: number | null;
+  max: number | null;
+  avg: number | null;
+  last: number | null;
+  rate: number | null;
+  p50: number | null;
+  p95: number | null;
+  p99: number | null;
+}
+
+export interface MiniAddon {
+  id: string;
+  name: string;
+  author: string;
+  subscriberCount: number;
+  currentVersionSize: number;
+  lastIndexedAt?: string | null;
+  createdAt: string;
+}
+
+export interface TableInfo {
+  name: string;
+  rows: number;
+  deadRows: number;
+  sizeBytes: number;
+}
+
+export interface HeavyStats {
+  asOf: number;
+  filesLive: number;
+  filesSoftDeleted: number;
+  filesApprox: boolean;
+  distinctHashes: number;
+  distinctAsOf: number;
+  dbDriver: string;
+  dbSizeBytes: number;
+  tables: TableInfo[];
+  duplicatesAsOf: number;
+  error?: string;
+}
+
+export interface Overview {
+  asOf: number;
+  data: {
+    addons: {
+      total: number;
+      indexed: number;
+      queued: number;
+      indexing: number;
+      priority: number;
+      recheck: number;
+      blocked: number;
+      unlisted: number;
+      private: number;
+      totalSizeBytes: number;
+      added24h: number;
+      added7d: number;
+      indexed24h: number;
+      softDeleted: number;
+    };
+    types: { type: string; count: number }[];
+    whitelistCount: number;
+    dependencies: { edges: number; addonsWithDependencies: number; addonsDependedOn: number };
+    traffic: {
+      requests24h?: number;
+      websiteRequests24h?: number;
+      workerRequests24h?: number;
+      websiteP95Ms?: number | null;
+      websiteP50Ms?: number | null;
+      errors5xx24h: number;
+      visitorsToday: number;
+      botsToday: number;
+    };
+    activity: Record<string, number>;
+    services: { online: number; total: number };
+    topSubscribed: MiniAddon[];
+    largest: MiniAddon[];
+    recentlyAdded: MiniAddon[];
+    recentlyIndexed: MiniAddon[];
+    heavy: HeavyStats | null;
+  };
+}
+
+export interface AddonRow {
+  id: string;
+  name: string;
+  type: string;
+  author: string;
+  preview: string;
+  subscriberCount: number;
+  currentVersionNumber: string;
+  currentVersionSize: number;
+  unlisted: boolean;
+  private: boolean;
+  blocked: boolean;
+  toBeIndexed: boolean;
+  isBeingIndexed: boolean;
+  priorityIndexing: boolean;
+  recheck: boolean;
+  indexStartTime: string | null;
+  lastIndexedAt: string | null;
+  createdAt: string;
+  updatedAt: string;
+  deletedAt: string | null;
+}
+
+export interface Paged<T> {
+  items: T[];
+  total: number;
+  limit: number;
+  offset: number;
+}
+
+export interface RelatedAddon {
+  id: string;
+  name: string;
+  type: string;
+  author: string;
+  subscriberCount: number;
+  currentVersionNumber: string;
+}
+
+export interface AddonDetail {
+  addon: AddonRow;
+  summary: string;
+  currentVersionId: number;
+  fileCount: number;
+  deletedFileRows: number;
+  dependencies: { total: number; items: RelatedAddon[] | null; missing: { id: string; name: string; version: string }[] | null };
+  dependents: { total: number; items: RelatedAddon[] | null };
+  duplicates: { addonId: string; name: string; sharedFiles: number }[];
+}
+
+export interface FileRow {
+  id: number;
+  path: string;
+  hash: string;
+  version: string;
+  whitelisted: boolean;
+  otherAddons: number;
+}
+
+export interface QueueRow {
+  id: string;
+  name: string;
+  currentVersionSize: number;
+  subscriberCount: number;
+  priorityIndexing: boolean;
+  blocked: boolean;
+  indexStartTime: string | null;
+  updatedAt: string;
+  elapsedSec: number;
+  stuck: boolean;
+}
+
+export interface QueueData {
+  asOf: number;
+  counts: { total: number; queued: number; priority: number; indexing: number };
+  blockedQueued: number;
+  timeoutMinutes: number;
+  indexing: QueueRow[];
+  priority: QueueRow[];
+  next: QueueRow[];
+  throughput: {
+    perHour1h: number;
+    perHour6h: number;
+    indexed24h: number;
+    etaHours: number | null;
+    duration24h: Total | null;
+  };
+}
+
+export interface WhitelistRow {
+  id: number;
+  hash: string;
+  note: string;
+  createdAt: string;
+  files: number;
+  addons: number;
+  samplePath: string;
+}
+
+export interface DupHash {
+  hash: string;
+  addonCount: number;
+  fileCount: number;
+  samplePath: string;
+  whitelisted: boolean;
+}
+
+export interface ServiceInstance {
+  id: number;
+  service: string;
+  instance: string;
+  version: string;
+  status: string;
+  statusMessage: string;
+  startedAt: number;
+  lastSeen: number;
+  heartbeatIntervalSec: number;
+  remoteAddr: string;
+  online: boolean;
+  uptimeSec: number;
+  metaData?: Record<string, unknown>;
+}
+
+export interface ServiceEvent {
+  id: number;
+  service: string;
+  instance: string;
+  level: "debug" | "info" | "warn" | "error";
+  message: string;
+  ts: number;
+  fieldsData?: Record<string, unknown>;
+}
+
+export interface CatalogEntry {
+  service: string;
+  name: string;
+  kind: string;
+  help: string;
+  labels: Record<string, string[]>;
+  instances: string[];
+  series: number;
+  lastSeen: number;
+}
+
+export interface RouteStat {
+  method: string;
+  route: string;
+  group: string;
+  count: number;
+  errors: number;
+  clientErrors: number;
+  p50: number | null;
+  p95: number | null;
+  p99: number | null;
+  max: number | null;
+  avg: number | null;
+}
+
+export interface TrafficData {
+  asOf: number;
+  from: number;
+  to: number;
+  step: number;
+  requestsByGroup: QueryResult;
+  latency: { p50: QueryResult; p95: QueryResult; p99: QueryResult };
+  errors5xx: QueryResult | null;
+  errors4xx: QueryResult | null;
+  statusTotals: Total[] | null;
+  clientTotals: Total[] | null;
+  groupTotals: Total[] | null;
+  website: Total | null;
+  routes: RouteStat[];
+  visitorsDaily: QueryResult | null;
+  visitorsToday: { human: number; bot: number };
+}
+
+export interface TopEntry {
+  item: string;
+  count: number;
+  name?: string;
+}

web/src/main.tsx

@@ -0,0 +1,67 @@
+import { StrictMode, lazy, Suspense } from "react";
+import { createRoot } from "react-dom/client";
+import { createBrowserRouter, RouterProvider } from "react-router";
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
+import "./index.css";
+import { SettingsProvider } from "./lib/settings";
+import { FeedbackProvider, Loading } from "./components/ui";
+import { RequireAuth } from "./components/Shell";
+import { ApiError } from "./lib/api";
+import Login from "./pages/Login";
+import Overview from "./pages/Overview";
+
+const Traffic = lazy(() => import("./pages/Traffic"));
+const Queue = lazy(() => import("./pages/Queue"));
+const Addons = lazy(() => import("./pages/Addons"));
+const Whitelist = lazy(() => import("./pages/Whitelist"));
+const Services = lazy(() => import("./pages/Services"));
+const Explorer = lazy(() => import("./pages/Explorer"));
+const Events = lazy(() => import("./pages/Events"));
+const System = lazy(() => import("./pages/System"));
+
+const queryClient = new QueryClient({
+  defaultOptions: {
+    queries: {
+      staleTime: 10_000,
+      refetchOnWindowFocus: false,
+      retry: (count, err) => !(err instanceof ApiError && err.status < 500) && count < 2,
+    },
+  },
+});
+
+const page = (el: React.ReactNode) => <Suspense fallback={<Loading />}>{el}</Suspense>;
+
+const router = createBrowserRouter(
+  [
+    { path: "/login", element: <Login /> },
+    {
+      element: <RequireAuth />,
+      children: [
+        { index: true, element: <Overview /> },
+        { path: "traffic", element: page(<Traffic />) },
+        { path: "queue", element: page(<Queue />) },
+        { path: "addons", element: page(<Addons />) },
+        { path: "addons/:id", element: page(<Addons />) },
+        { path: "whitelist", element: page(<Whitelist />) },
+        { path: "services", element: page(<Services />) },
+        { path: "explorer", element: page(<Explorer />) },
+        { path: "events", element: page(<Events />) },
+        { path: "system", element: page(<System />) },
+        { path: "*", element: <Overview /> },
+      ],
+    },
+  ],
+  { basename: "/admin/ui" },
+);
+
+createRoot(document.getElementById("root")!).render(
+  <StrictMode>
+    <SettingsProvider>
+      <QueryClientProvider client={queryClient}>
+        <FeedbackProvider>
+          <RouterProvider router={router} />
+        </FeedbackProvider>
+      </QueryClientProvider>
+    </SettingsProvider>
+  </StrictMode>,
+);

web/src/pages/Addons.tsx

@@ -0,0 +1,606 @@
+import { useEffect, useState } from "react";
+import { useMutation, useQuery, useQueryClient, keepPreviousData } from "@tanstack/react-query";
+import { useNavigate, useParams, useSearchParams } from "react-router";
+import { ExternalLink, RefreshCw, RotateCcw, Search, ShieldCheck, Star, StarOff, Trash2, Undo2, X } from "lucide-react";
+import { api, qs } from "../lib/api";
+import { useRefetch } from "../lib/settings";
+import type { AddonDetail, AddonRow, FileRow, Overview, Paged } from "../lib/types";
+import { bytes, compact, dateTime, int, relative, shortHash } from "../lib/format";
+import {
+  Badge,
+  Button,
+  Card,
+  Checkbox,
+  CopyButton,
+  Empty,
+  ErrorBox,
+  Input,
+  Loading,
+  PageHeader,
+  Pager,
+  Segmented,
+  Select,
+  Table,
+  Td,
+  Th,
+  useFeedback,
+  type Tone,
+} from "../components/ui";
+
+const STATUSES = [
+  ["all", "All"],
+  ["indexed", "Indexed"],
+  ["queued", "Queued"],
+  ["indexing", "Indexing"],
+  ["priority", "Priority"],
+  ["recheck", "Recheck"],
+  ["blocked", "Blocked"],
+  ["unlisted", "Unlisted"],
+  ["private", "Private"],
+  ["deleted", "Deleted"],
+] as const;
+
+const SORTS = [
+  ["subscribers", "Subscribers"],
+  ["size", "Size"],
+  ["name", "Name"],
+  ["created", "Created"],
+  ["updated", "Updated"],
+  ["indexed", "Last indexed"],
+  ["deleted", "Deleted at"],
+] as const;
+
+export function addonBadges(a: AddonRow): { tone: Tone; label: string }[] {
+  const out: { tone: Tone; label: string }[] = [];
+  if (a.deletedAt) out.push({ tone: "serious", label: "deleted" });
+  if (a.blocked) out.push({ tone: "critical", label: "blocked" });
+  if (a.isBeingIndexed) out.push({ tone: "good", label: "indexing" });
+  else if (a.toBeIndexed) out.push({ tone: "warning", label: "queued" });
+  if (a.priorityIndexing) out.push({ tone: "accent", label: "priority" });
+  if (a.unlisted) out.push({ tone: "neutral", label: "unlisted" });
+  if (a.private) out.push({ tone: "neutral", label: "private" });
+  return out;
+}
+
+const workshopUrl = (id: string) => `https://reforger.armaplatform.com/workshop/${id}`;
+
+export default function Addons() {
+  const { id } = useParams();
+  const navigate = useNavigate();
+  const [sp, setSp] = useSearchParams();
+  const refetchInterval = useRefetch();
+  const qc = useQueryClient();
+  const { toast, confirm } = useFeedback();
+  const [selected, setSelected] = useState<Set<string>>(new Set());
+
+  const status = sp.get("status") ?? "all";
+  const search = sp.get("q") ?? "";
+  const type = sp.get("type") ?? "";
+  const sort = sp.get("sort") ?? (status === "deleted" ? "deleted" : "subscribers");
+  const order = sp.get("order") ?? "desc";
+  const offset = Number(sp.get("offset") ?? 0);
+  const limit = 50;
+  const [draft, setDraft] = useState(search);
+  useEffect(() => setDraft(search), [search]);
+
+  const set = (patch: Record<string, string | null>) => {
+    const next = new URLSearchParams(sp);
+    for (const [k, v] of Object.entries(patch)) {
+      if (v === null || v === "") next.delete(k);
+      else next.set(k, v);
+    }
+    if (!("offset" in patch)) next.delete("offset");
+    setSp(next, { replace: true });
+    setSelected(new Set());
+  };
+
+  const list = useQuery({
+    queryKey: ["addons", status, search, type, sort, order, offset],
+    queryFn: () => api<Paged<AddonRow>>("/addons" + qs({ status, q: search, type, sort, order, limit, offset })),
+    placeholderData: keepPreviousData,
+    refetchInterval,
+  });
+  const overview = useQuery({ queryKey: ["overview"], queryFn: () => api<Overview>("/overview"), staleTime: 60_000 });
+
+  const bulk = useMutation({
+    mutationFn: (body: { action: string; value?: boolean }) =>
+      api<{ updated: number }>("/bulk/addons", { method: "POST", body: { ids: [...selected], ...body } }),
+    onSuccess: (r, v) => {
+      toast(`${v.action}: ${r.updated} addon${r.updated === 1 ? "" : "s"} updated`);
+      setSelected(new Set());
+      qc.invalidateQueries({ queryKey: ["addons"] });
+      qc.invalidateQueries({ queryKey: ["overview"] });
+    },
+    onError: (e) => toast((e as Error).message, "critical"),
+  });
+
+  const items = list.data?.items ?? [];
+  const allSelected = items.length > 0 && items.every((a) => selected.has(a.id));
+  const query = sp.toString() ? `?${sp.toString()}` : "";
+
+  return (
+    <div className="space-y-4">
+      <PageHeader title="Addons" description="Search, inspect and manage every addon the crawler knows about" />
+
+      <Card bodyClass="p-0">
+        <div className="flex flex-wrap items-center gap-2 border-b border-line p-3">
+          <form
+            className="relative min-w-56 flex-1"
+            onSubmit={(e) => {
+              e.preventDefault();
+              set({ q: draft.trim() });
+            }}
+          >
+            <Search className="pointer-events-none absolute top-1/2 left-2.5 size-3.5 -translate-y-1/2 text-muted" />
+            <Input value={draft} onChange={(e) => setDraft(e.target.value)} placeholder="Search name, author or exact ID…" className="w-full pl-8" />
+          </form>
+          <Select value={status} onChange={(e) => set({ status: e.target.value, sort: null })}>
+            {STATUSES.map(([k, l]) => (
+              <option key={k} value={k}>
+                {l}
+              </option>
+            ))}
+          </Select>
+          <Select value={type} onChange={(e) => set({ type: e.target.value })}>
+            <option value="">All types</option>
+            {(overview.data?.data.types ?? []).map((t) => (
+              <option key={t.type} value={t.type}>
+                {t.type || "(none)"}
+              </option>
+            ))}
+          </Select>
+          <Select value={sort} onChange={(e) => set({ sort: e.target.value })}>
+            {SORTS.map(([k, l]) => (
+              <option key={k} value={k}>
+                Sort: {l}
+              </option>
+            ))}
+          </Select>
+          <Segmented size="sm" options={[{ key: "desc", label: "Desc" }, { key: "asc", label: "Asc" }]} value={order} onChange={(v) => set({ order: v })} />
+        </div>
+
+        {selected.size > 0 && (
+          <div className="flex flex-wrap items-center gap-2 border-b border-line bg-surface-2/50 px-3 py-2 text-xs">
+            <span className="text-ink">{selected.size} selected</span>
+            <Button size="sm" icon={<Star className="size-3.5" />} onClick={() => bulk.mutate({ action: "priority", value: true })}>Prioritize</Button>
+            <Button size="sm" icon={<StarOff className="size-3.5" />} onClick={() => bulk.mutate({ action: "priority", value: false })}>Unprioritize</Button>
+            <Button size="sm" icon={<RefreshCw className="size-3.5" />} onClick={() => bulk.mutate({ action: "reindex", value: false })}>Re-index</Button>
+            {status === "deleted" ? (
+              <Button size="sm" icon={<Undo2 className="size-3.5" />} onClick={() => bulk.mutate({ action: "restore" })}>Restore</Button>
+            ) : (
+              <Button
+                size="sm"
+                variant="danger"
+                icon={<Trash2 className="size-3.5" />}
+                onClick={async () => {
+                  if (await confirm({ title: `Delete ${selected.size} addons?`, body: "They are soft-deleted with their files and can be restored later.", danger: true, confirmLabel: "Delete" }))
+                    bulk.mutate({ action: "delete" });
+                }}
+              >
+                Delete
+              </Button>
+            )}
+            <Button size="sm" variant="ghost" onClick={() => setSelected(new Set())}>Clear</Button>
+          </div>
+        )}
+
+        {list.error ? (
+          <div className="p-4"><ErrorBox error={list.error} /></div>
+        ) : list.isLoading ? (
+          <div className="px-4"><Loading /></div>
+        ) : items.length === 0 ? (
+          <Empty>No addons match these filters</Empty>
+        ) : (
+          <div className="px-4">
+            <Table>
+              <thead>
+                <tr>
+                  <Th className="w-8">
+                    <Checkbox
+                      label="Select all"
+                      checked={allSelected}
+                      onChange={(v) => setSelected(v ? new Set(items.map((a) => a.id)) : new Set())}
+                    />
+                  </Th>
+                  <Th>Addon</Th>
+                  <Th>State</Th>
+                  <Th className="text-right">Subscribers</Th>
+                  <Th className="text-right">Size</Th>
+                  <Th>Version</Th>
+                  <Th>{status === "deleted" ? "Deleted" : "Last indexed"}</Th>
+                </tr>
+              </thead>
+              <tbody>
+                {items.map((a) => (
+                  <tr key={a.id} onClick={() => navigate(`/addons/${a.id}${query}`)} className="cursor-pointer hover:bg-surface-2/40">
+                    <Td>
+                      <Checkbox
+                        label={`Select ${a.name}`}
+                        checked={selected.has(a.id)}
+                        onChange={(v) => {
+                          const next = new Set(selected);
+                          if (v) next.add(a.id);
+                          else next.delete(a.id);
+                          setSelected(next);
+                        }}
+                      />
+                    </Td>
+                    <Td className="max-w-[340px]">
+                      <div className="truncate text-ink" title={a.name}>{a.name || "(unnamed)"}</div>
+                      <div className="truncate text-xs text-muted">
+                        <span className="font-mono">{a.id}</span> · {a.author || "unknown"} · {a.type}
+                      </div>
+                    </Td>
+                    <Td>
+                      <div className="flex flex-wrap gap-1">
+                        {addonBadges(a).map((b) => (
+                          <Badge key={b.label} tone={b.tone}>{b.label}</Badge>
+                        ))}
+                      </div>
+                    </Td>
+                    <Td className="tnum text-right">{compact(a.subscriberCount)}</Td>
+                    <Td className="tnum text-right">{bytes(a.currentVersionSize)}</Td>
+                    <Td className="font-mono text-xs">{a.currentVersionNumber}</Td>
+                    <Td className="whitespace-nowrap">{relative(status === "deleted" ? a.deletedAt : a.lastIndexedAt)}</Td>
+                  </tr>
+                ))}
+              </tbody>
+            </Table>
+          </div>
+        )}
+        {list.data && (
+          <div className="border-t border-line px-4 py-2.5">
+            <Pager total={list.data.total} limit={limit} offset={offset} onChange={(o) => set({ offset: String(o) })} />
+          </div>
+        )}
+      </Card>
+
+      {id && <AddonDrawer id={id} onClose={() => navigate(`/addons${query}`)} />}
+    </div>
+  );
+}
+
+function AddonDrawer({ id, onClose }: { id: string; onClose: () => void }) {
+  const qc = useQueryClient();
+  const navigate = useNavigate();
+  const { toast, confirm } = useFeedback();
+  const [tab, setTab] = useState<"overview" | "files">("overview");
+  const detail = useQuery({ queryKey: ["addon", id], queryFn: () => api<AddonDetail>(`/addons/${encodeURIComponent(id)}`) });
+
+  useEffect(() => {
+    const onKey = (e: KeyboardEvent) => e.key === "Escape" && onClose();
+    window.addEventListener("keydown", onKey);
+    return () => window.removeEventListener("keydown", onKey);
+  }, [onClose]);
+  useEffect(() => setTab("overview"), [id]);
+
+  const refresh = () => {
+    qc.invalidateQueries({ queryKey: ["addon", id] });
+    qc.invalidateQueries({ queryKey: ["addons"] });
+    qc.invalidateQueries({ queryKey: ["overview"] });
+    qc.invalidateQueries({ queryKey: ["queue"] });
+  };
+
+  const act = useMutation({
+    mutationFn: ({ action, value, method }: { action: string; value?: boolean; method?: "POST" | "DELETE" }) =>
+      method === "DELETE"
+        ? api(`/addons/${encodeURIComponent(id)}${action ? `/${action}` : ""}${action === "purge" ? `?confirm=${encodeURIComponent(id)}` : ""}`, { method: "DELETE" })
+        : api(`/addons/${encodeURIComponent(id)}/${action}`, { method: "POST", body: { value: value ?? true } }),
+    onSuccess: (_r, v) => {
+      toast(v.action === "purge" ? "Addon purged" : "Saved");
+      if (v.action === "purge") onClose();
+      refresh();
+    },
+    onError: (e) => toast((e as Error).message, "critical"),
+  });
+
+  const a = detail.data?.addon;
+
+  return (
+    <div className="fixed inset-0 z-40 flex justify-end">
+      <div className="absolute inset-0 bg-black/40" onClick={onClose} />
+      <aside className="relative flex h-full w-[min(760px,100vw)] flex-col border-l border-line bg-page shadow-2xl">
+        <header className="flex items-start justify-between gap-3 border-b border-line bg-surface px-5 py-4">
+          <div className="min-w-0">
+            <div className="flex items-center gap-2">
+              <h2 className="truncate text-lg font-semibold text-ink">{a?.name || id}</h2>
+              <a href={workshopUrl(id)} target="_blank" rel="noreferrer" className="text-muted hover:text-accent" title="Open on the workshop">
+                <ExternalLink className="size-4" />
+              </a>
+            </div>
+            <div className="mt-0.5 flex items-center gap-1 font-mono text-xs text-muted">
+              {id} <CopyButton value={id} />
+            </div>
+            {a && (
+              <div className="mt-2 flex flex-wrap gap-1">
+                {addonBadges(a).map((b) => <Badge key={b.label} tone={b.tone}>{b.label}</Badge>)}
+                {a.recheck && <Badge tone="warning">recheck</Badge>}
+              </div>
+            )}
+          </div>
+          <Button variant="ghost" size="sm" onClick={onClose} icon={<X className="size-4" />} aria-label="Close" />
+        </header>
+
+        {detail.isLoading ? (
+          <div className="px-5"><Loading /></div>
+        ) : detail.error ? (
+          <div className="p-5"><ErrorBox error={detail.error} /></div>
+        ) : (
+          a && (
+            <>
+              <div className="flex flex-wrap gap-2 border-b border-line bg-surface px-5 py-3">
+                {a.deletedAt ? (
+                  <>
+                    <Button size="sm" variant="primary" icon={<Undo2 className="size-3.5" />} onClick={() => act.mutate({ action: "restore" })}>
+                      Restore
+                    </Button>
+                    <Button
+                      size="sm"
+                      variant="danger"
+                      icon={<Trash2 className="size-3.5" />}
+                      onClick={async () => {
+                        if (
+                          await confirm({
+                            title: "Permanently purge this addon?",
+                            body: "Removes the addon row, all its file rows and dependency edges. This cannot be undone.",
+                            danger: true,
+                            confirmLabel: "Purge",
+                            requireText: id,
+                          })
+                        )
+                          act.mutate({ action: "purge", method: "DELETE" });
+                      }}
+                    >
+                      Purge permanently
+                    </Button>
+                  </>
+                ) : (
+                  <>
+                    <Button
+                      size="sm"
+                      icon={a.priorityIndexing ? <StarOff className="size-3.5" /> : <Star className="size-3.5" />}
+                      onClick={() => act.mutate({ action: "priority", value: !a.priorityIndexing })}
+                    >
+                      {a.priorityIndexing ? "Remove priority" : "Prioritize"}
+                    </Button>
+                    <Button size="sm" icon={<RefreshCw className="size-3.5" />} onClick={() => act.mutate({ action: "reindex", value: false })}>
+                      Re-index
+                    </Button>
+                    {a.toBeIndexed && !a.isBeingIndexed && (
+                      <Button size="sm" onClick={() => act.mutate({ action: "to-be-indexed", value: false })}>
+                        Remove from queue
+                      </Button>
+                    )}
+                    {a.isBeingIndexed && (
+                      <Button size="sm" icon={<RotateCcw className="size-3.5" />} onClick={() => act.mutate({ action: "reset-indexing" })}>
+                        Release claim
+                      </Button>
+                    )}
+                    {a.recheck && (
+                      <Button size="sm" onClick={() => act.mutate({ action: "recheck", value: false })}>
+                        Clear recheck
+                      </Button>
+                    )}
+                    <Button
+                      size="sm"
+                      variant="danger"
+                      icon={<Trash2 className="size-3.5" />}
+                      onClick={async () => {
+                        if (await confirm({ title: "Delete this addon?", body: "Soft-deletes the addon and its files; it can be restored later.", danger: true, confirmLabel: "Delete" }))
+                          act.mutate({ action: "", method: "DELETE" });
+                      }}
+                    >
+                      Delete
+                    </Button>
+                  </>
+                )}
+              </div>
+
+              <div className="border-b border-line bg-surface px-5 py-2">
+                <Segmented
+                  size="sm"
+                  options={[
+                    { key: "overview", label: "Overview" },
+                    { key: "files", label: `Files (${int(detail.data!.fileCount)})` },
+                  ]}
+                  value={tab}
+                  onChange={setTab}
+                />
+              </div>
+
+              <div className="flex-1 space-y-4 overflow-y-auto p-5">
+                {tab === "overview" ? (
+                  <>
+                    <Card>
+                      <dl className="grid grid-cols-2 gap-x-6 gap-y-3 text-[13px] sm:grid-cols-3">
+                        {[
+                          ["Author", a.author || "—"],
+                          ["Type", a.type || "—"],
+                          ["Subscribers", int(a.subscriberCount)],
+                          ["Version", a.currentVersionNumber || "—"],
+                          ["Size", bytes(a.currentVersionSize)],
+                          ["Files", `${int(detail.data!.fileCount)}${detail.data!.deletedFileRows ? ` (+${int(detail.data!.deletedFileRows)} old rows)` : ""}`],
+                          ["Added", dateTime(a.createdAt)],
+                          ["Updated", dateTime(a.updatedAt)],
+                          ["Last indexed", dateTime(a.lastIndexedAt)],
+                          ...(a.isBeingIndexed ? [["Claimed", relative(a.indexStartTime)]] : []),
+                          ...(a.deletedAt ? [["Deleted", dateTime(a.deletedAt)]] : []),
+                        ].map(([k, v]) => (
+                          <div key={k}>
+                            <dt className="text-xs text-muted">{k}</dt>
+                            <dd className="mt-0.5 truncate text-ink">{v}</dd>
+                          </div>
+                        ))}
+                      </dl>
+                      {detail.data!.summary && <p className="mt-4 border-t border-line pt-3 text-[13px] text-ink-2">{detail.data!.summary}</p>}
+                    </Card>
+
+                    <Card title="Shares files with" subtitle="Other addons containing identical files (whitelisted hashes excluded)">
+                      {detail.data!.duplicates.length === 0 ? (
+                        <div className="text-sm text-muted">No shared files</div>
+                      ) : (
+                        <Table>
+                          <tbody>
+                            {detail.data!.duplicates.map((d) => (
+                              <tr key={d.addonId} className="cursor-pointer hover:bg-surface-2/40" onClick={() => navigate(`/addons/${d.addonId}`)}>
+                                <Td className="text-ink">
+                                  {d.name || d.addonId}
+                                  <div className="font-mono text-[11px] text-muted">{d.addonId}</div>
+                                </Td>
+                                <Td className="tnum text-right">{int(d.sharedFiles)} {d.sharedFiles === 1 ? "file" : "files"}</Td>
+                              </tr>
+                            ))}
+                          </tbody>
+                        </Table>
+                      )}
+                    </Card>
+
+                    <div className="grid grid-cols-1 gap-4 sm:grid-cols-2">
+                      <Card title={`Depends on (${detail.data!.dependencies.total})`}>
+                        <Related items={detail.data!.dependencies.items ?? []} onOpen={(x) => navigate(`/addons/${x}`)} />
+                        {(detail.data!.dependencies.missing ?? []).length > 0 && (
+                          <div className="mt-3 border-t border-line pt-3">
+                            <div className="mb-1.5 text-xs text-muted">Not in the database</div>
+                            {detail.data!.dependencies.missing!.map((m) => (
+                              <div key={m.id} className="flex items-center justify-between gap-2 py-0.5 text-[13px]">
+                                <span className="truncate text-ink-2">{m.name || m.id}</span>
+                                <span className="font-mono text-[11px] text-muted">{m.id}</span>
+                              </div>
+                            ))}
+                          </div>
+                        )}
+                      </Card>
+                      <Card title={`Used by (${detail.data!.dependents.total})`}>
+                        <Related items={detail.data!.dependents.items ?? []} onOpen={(x) => navigate(`/addons/${x}`)} />
+                      </Card>
+                    </div>
+                  </>
+                ) : (
+                  <Files id={id} />
+                )}
+              </div>
+            </>
+          )
+        )}
+      </aside>
+    </div>
+  );
+}
+
+function Related({ items, onOpen }: { items: { id: string; name: string; subscriberCount: number }[]; onOpen: (id: string) => void }) {
+  if (items.length === 0) return <div className="text-sm text-muted">None</div>;
+  return (
+    <ul className="space-y-1">
+      {items.map((x) => (
+        <li key={x.id}>
+          <button onClick={() => onOpen(x.id)} className="flex w-full items-center justify-between gap-2 rounded px-1 py-0.5 text-left text-[13px] hover:bg-surface-2">
+            <span className="truncate text-ink-2">{x.name || x.id}</span>
+            <span className="tnum shrink-0 text-xs text-muted">{compact(x.subscriberCount)}</span>
+          </button>
+        </li>
+      ))}
+    </ul>
+  );
+}
+
+function Files({ id }: { id: string }) {
+  const [q, setQ] = useState("");
+  const [draft, setDraft] = useState("");
+  const [offset, setOffset] = useState(0);
+  const qc = useQueryClient();
+  const { toast, confirm } = useFeedback();
+  const limit = 100;
+  const files = useQuery({
+    queryKey: ["addon-files", id, q, offset],
+    queryFn: () => api<Paged<FileRow>>(`/addons/${encodeURIComponent(id)}/files` + qs({ q, offset, limit })),
+    placeholderData: keepPreviousData,
+  });
+  const whitelist = useMutation({
+    mutationFn: (f: FileRow) => api<{ added: number }>("/whitelist", { method: "POST", body: { hashes: [f.hash], note: f.path } }),
+    onSuccess: (r) => {
+      toast(r.added ? "Hash whitelisted" : "Already whitelisted");
+      qc.invalidateQueries({ queryKey: ["addon-files", id] });
+      qc.invalidateQueries({ queryKey: ["addon", id] });
+    },
+    onError: (e) => toast((e as Error).message, "critical"),
+  });
+
+  return (
+    <Card bodyClass="p-0">
+      <form
+        className="border-b border-line p-3"
+        onSubmit={(e) => {
+          e.preventDefault();
+          setOffset(0);
+          setQ(draft.trim());
+        }}
+      >
+        <Input value={draft} onChange={(e) => setDraft(e.target.value)} placeholder="Filter by path or exact hash…" className="w-full" />
+      </form>
+      {files.isLoading ? (
+        <div className="px-4"><Loading /></div>
+      ) : files.error ? (
+        <div className="p-4"><ErrorBox error={files.error} /></div>
+      ) : (files.data?.items.length ?? 0) === 0 ? (
+        <Empty>No files</Empty>
+      ) : (
+        <div className="px-4">
+          <Table>
+            <thead>
+              <tr>
+                <Th>Path</Th>
+                <Th>Hash</Th>
+                <Th className="text-right">Also in</Th>
+                <Th />
+              </tr>
+            </thead>
+            <tbody>
+              {files.data!.items.map((f) => (
+                <tr key={f.id}>
+                  <Td className="max-w-[300px] truncate font-mono text-xs text-ink" title={f.path}>{f.path}</Td>
+                  <Td className="whitespace-nowrap font-mono text-xs">
+                    {shortHash(f.hash, 8)} <CopyButton value={f.hash} />
+                  </Td>
+                  <Td className="tnum text-right">{f.otherAddons ? `${int(f.otherAddons)} addons` : "—"}</Td>
+                  <Td className="text-right">
+                    {f.whitelisted ? (
+                      <Badge tone="good">whitelisted</Badge>
+                    ) : (
+                      <Button
+                        size="sm"
+                        variant="ghost"
+                        icon={<ShieldCheck className="size-3.5" />}
+                        onClick={async () => {
+                          if (
+                            await confirm({
+                              title: "Whitelist this hash?",
+                              body: (
+                                <>
+                                  Files with this content will be ignored by duplicate detection for <b>every</b> addon.
+                                  <div className="mt-2 font-mono text-xs break-all text-muted">{f.path}</div>
+                                </>
+                              ),
+                              confirmLabel: "Whitelist",
+                            })
+                          )
+                            whitelist.mutate(f);
+                        }}
+                      >
+                        Whitelist
+                      </Button>
+                    )}
+                  </Td>
+                </tr>
+              ))}
+            </tbody>
+          </Table>
+        </div>
+      )}
+      {files.data && files.data.total > limit && (
+        <div className="border-t border-line px-4 py-2.5">
+          <Pager total={files.data.total} limit={limit} offset={offset} onChange={setOffset} />
+        </div>
+      )}
+    </Card>
+  );
+}

web/src/pages/Events.tsx

@@ -0,0 +1,117 @@
+import { useState } from "react";
+import { useInfiniteQuery, useQuery } from "@tanstack/react-query";
+import { api, qs } from "../lib/api";
+import { useRefetch } from "../lib/settings";
+import type { ServiceEvent, ServiceInstance } from "../lib/types";
+import { dateTime, relative } from "../lib/format";
+import { Button, Card, Checkbox, Empty, ErrorBox, Input, LevelBadge, Loading, PageHeader, Select, Table, Td, Th } from "../components/ui";
+
+export default function Events() {
+  const refetchInterval = useRefetch();
+  const [service, setService] = useState("");
+  const [level, setLevel] = useState("");
+  const [audit, setAudit] = useState(false);
+  const [search, setSearch] = useState("");
+  const [draft, setDraft] = useState("");
+  const services = useQuery({ queryKey: ["services"], queryFn: () => api<{ items: ServiceInstance[] }>("/services") });
+
+  const events = useInfiniteQuery({
+    queryKey: ["events", "log", service, level, audit, search],
+    initialPageParam: 0,
+    refetchInterval,
+    queryFn: ({ pageParam }) =>
+      api<{ items: ServiceEvent[]; next: number }>(
+        "/services/events" + qs({ service, level, audit: audit ? "1" : undefined, q: search, before: pageParam || undefined, limit: 100 }),
+      ),
+    getNextPageParam: (last) => last.next || undefined,
+  });
+
+  const rows = events.data?.pages.flatMap((p) => p.items) ?? [];
+  const names = [...new Set((services.data?.items ?? []).map((s) => s.service))];
+
+  return (
+    <div className="space-y-4">
+      <PageHeader title="Events" description="Logs reported by services, status changes and the admin audit trail" />
+      <Card bodyClass="p-0">
+        <div className="flex flex-wrap items-center gap-2 border-b border-line p-3">
+          <form
+            className="min-w-56 flex-1"
+            onSubmit={(e) => {
+              e.preventDefault();
+              setSearch(draft.trim());
+            }}
+          >
+            <Input value={draft} onChange={(e) => setDraft(e.target.value)} placeholder="Search messages…" className="w-full" />
+          </form>
+          <Select value={service} onChange={(e) => setService(e.target.value)}>
+            <option value="">All services</option>
+            {names.map((n) => <option key={n} value={n}>{n}</option>)}
+          </Select>
+          <Select value={level} onChange={(e) => setLevel(e.target.value)}>
+            <option value="">All levels</option>
+            <option value="error">Errors</option>
+            <option value="warn,error">Warnings & errors</option>
+            <option value="info">Info</option>
+            <option value="debug">Debug</option>
+          </Select>
+          <label className="flex items-center gap-1.5 text-xs text-ink-2">
+            <Checkbox checked={audit} onChange={setAudit} label="Admin audit only" /> Admin audit only
+          </label>
+        </div>
+        {events.isLoading ? (
+          <div className="px-4"><Loading /></div>
+        ) : events.error ? (
+          <div className="p-4"><ErrorBox error={events.error} /></div>
+        ) : rows.length === 0 ? (
+          <Empty>No events</Empty>
+        ) : (
+          <div className="px-4">
+            <Table>
+              <thead>
+                <tr>
+                  <Th>Time</Th>
+                  <Th>Level</Th>
+                  <Th>Service</Th>
+                  <Th>Message</Th>
+                </tr>
+              </thead>
+              <tbody>
+                {rows.map((e) => (
+                  <tr key={e.id} className="align-top">
+                    <Td className="whitespace-nowrap text-xs" title={relative(e.ts)}>{dateTime(e.ts)}</Td>
+                    <Td><LevelBadge level={e.level} /></Td>
+                    <Td className="whitespace-nowrap text-xs">
+                      {e.service}
+                      <div className="text-muted">{e.instance}</div>
+                    </Td>
+                    <Td className="text-ink">
+                      {e.message}
+                      {e.fieldsData && Object.keys(e.fieldsData).length > 0 && (
+                        <div className="mt-0.5 flex flex-wrap gap-x-3 font-mono text-[11px] text-muted">
+                          {Object.entries(e.fieldsData)
+                            .filter(([k]) => k !== "kind")
+                            .map(([k, v]) => (
+                              <span key={k}>
+                                {k}={typeof v === "object" ? JSON.stringify(v) : String(v)}
+                              </span>
+                            ))}
+                        </div>
+                      )}
+                    </Td>
+                  </tr>
+                ))}
+              </tbody>
+            </Table>
+          </div>
+        )}
+        {events.hasNextPage && (
+          <div className="border-t border-line p-3 text-center">
+            <Button size="sm" loading={events.isFetchingNextPage} onClick={() => events.fetchNextPage()}>
+              Load older events
+            </Button>
+          </div>
+        )}
+      </Card>
+    </div>
+  );
+}

web/src/pages/Explorer.tsx

@@ -0,0 +1,232 @@
+import { useMemo } from "react";
+import { useQuery } from "@tanstack/react-query";
+import { useSearchParams } from "react-router";
+import { X } from "lucide-react";
+import { api } from "../lib/api";
+import { stepFor, useMetric } from "../lib/queries";
+import { rangeSeconds, useSettings } from "../lib/settings";
+import type { CatalogEntry } from "../lib/types";
+import { compact, num, relative } from "../lib/format";
+import { Card, Empty, ErrorBox, Loading, PageHeader, Select, Table, Td, Th } from "../components/ui";
+import { fromQuery, TimeChart } from "../components/charts/TimeChart";
+
+const AGG_HELP: Record<string, string> = {
+  sum: "Total of values (counters: increase per interval)",
+  count: "Number of samples / observations",
+  rate: "Per second (counters: increase, others: observations)",
+  avg: "Average value",
+  min: "Minimum",
+  max: "Maximum",
+  last: "Latest value (summed over series)",
+  p50: "Median (histograms)",
+  p90: "90th percentile (histograms)",
+  p95: "95th percentile (histograms)",
+  p99: "99th percentile (histograms)",
+};
+
+const STEPS = [
+  { key: 0, label: "Auto" },
+  { key: 60, label: "1 min" },
+  { key: 300, label: "5 min" },
+  { key: 900, label: "15 min" },
+  { key: 3600, label: "1 hour" },
+  { key: 86400, label: "1 day" },
+];
+
+export default function Explorer() {
+  const [sp, setSp] = useSearchParams();
+  const { range } = useSettings();
+  const catalog = useQuery({ queryKey: ["catalog", "all"], queryFn: () => api<{ items: CatalogEntry[]; aggregations: string[] }>("/metrics/catalog") });
+
+  const service = sp.get("service") ?? "main";
+  const name = sp.get("name") ?? "";
+  const groupBy = (sp.get("groupBy") ?? "").split(",").filter(Boolean);
+  const match: Record<string, string> = useMemo(() => {
+    try {
+      return JSON.parse(sp.get("match") ?? "{}");
+    } catch {
+      return {};
+    }
+  }, [sp]);
+  const stepParam = Number(sp.get("step") ?? 0);
+
+  const update = (patch: Record<string, string | null>) => {
+    const next = new URLSearchParams(sp);
+    for (const [k, v] of Object.entries(patch)) {
+      if (v === null || v === "") next.delete(k);
+      else next.set(k, v);
+    }
+    setSp(next, { replace: true });
+  };
+
+  const services = useMemo(() => [...new Set((catalog.data?.items ?? []).map((c) => c.service))], [catalog.data]);
+  const metrics = (catalog.data?.items ?? []).filter((c) => c.service === service);
+  const entry = metrics.find((m) => m.name === name) ?? null;
+  const defaultAgg = entry ? (entry.kind === "counter" ? "sum" : entry.kind === "gauge" ? "last" : entry.kind === "histogram" ? "p95" : "avg") : "";
+  const agg = sp.get("agg") ?? defaultAgg;
+  const labelKeys = entry ? [...Object.keys(entry.labels), ...(entry.instances.length > 1 ? ["instance"] : [])] : [];
+  const step = stepParam || stepFor(rangeSeconds(range));
+
+  const q = useMetric({ service, name, agg, groupBy, match, step }, !!entry);
+  // colour only the 8 largest series (fixed slots, never cycled); the table lists all of them
+  const MAX_CHART_SERIES = 8;
+  const full = fromQuery(q.data, (l) => (Object.keys(l).length ? Object.entries(l).map(([k, v]) => `${k}=${v}`).join(", ") : name));
+  const ranked = full.series
+    .map((s) => ({ s, w: s.data.reduce<number>((a, v) => a + (v ?? 0), 0) }))
+    .sort((a, b) => b.w - a.w)
+    .slice(0, MAX_CHART_SERIES)
+    .map((x, i) => ({ ...x.s, slot: i + 1 }));
+  const data = { x: full.x, series: ranked };
+  const hidden = full.series.length - ranked.length;
+  const fmt = agg === "rate" ? (v: number) => num(v, 3) : compact;
+
+  const summary = (q.data?.series ?? []).map((s) => {
+    const vals = s.points.map((p) => p[1]).filter((v): v is number => v !== null);
+    return {
+      labels: s.labels,
+      last: vals.at(-1) ?? null,
+      min: vals.length ? Math.min(...vals) : null,
+      max: vals.length ? Math.max(...vals) : null,
+      avg: vals.length ? vals.reduce((a, b) => a + b, 0) / vals.length : null,
+    };
+  });
+
+  return (
+    <div className="space-y-4">
+      <PageHeader title="Metrics explorer" description="Query any metric from the API or a reporting service. The URL keeps the query, so views can be bookmarked." />
+
+      <Card>
+        {catalog.isLoading ? (
+          <Loading />
+        ) : catalog.error ? (
+          <ErrorBox error={catalog.error} />
+        ) : (
+          <div className="flex flex-wrap items-end gap-3">
+            <Field label="Service">
+              <Select value={service} onChange={(e) => update({ service: e.target.value, name: null, agg: null, groupBy: null, match: null })}>
+                {services.map((s) => <option key={s} value={s}>{s}</option>)}
+              </Select>
+            </Field>
+            <Field label="Metric">
+              <Select value={name} onChange={(e) => update({ name: e.target.value, agg: null, groupBy: null, match: null })} className="min-w-64">
+                <option value="">Choose a metric…</option>
+                {metrics.map((m) => <option key={m.name} value={m.name}>{m.name} ({m.kind})</option>)}
+              </Select>
+            </Field>
+            {entry && (
+              <>
+                <Field label="Aggregation">
+                  <Select value={agg} onChange={(e) => update({ agg: e.target.value })} title={AGG_HELP[agg]}>
+                    {(catalog.data?.aggregations ?? []).map((a) => <option key={a} value={a}>{a}</option>)}
+                  </Select>
+                </Field>
+                <Field label="Interval">
+                  <Select value={stepParam} onChange={(e) => update({ step: e.target.value === "0" ? null : e.target.value })}>
+                    {STEPS.map((s) => <option key={s.key} value={s.key}>{s.label}</option>)}
+                  </Select>
+                </Field>
+                {labelKeys.length > 0 && (
+                  <Field label="Split by">
+                    <div className="flex flex-wrap gap-1">
+                      {labelKeys.map((k) => {
+                        const on = groupBy.includes(k);
+                        return (
+                          <button
+                            key={k}
+                            onClick={() => update({ groupBy: (on ? groupBy.filter((g) => g !== k) : [...groupBy, k]).join(",") })}
+                            className={`h-8.5 rounded-lg border px-2.5 text-xs ${on ? "border-accent bg-accent/15 text-ink" : "border-line text-ink-2 hover:text-ink"}`}
+                          >
+                            {k}
+                          </button>
+                        );
+                      })}
+                    </div>
+                  </Field>
+                )}
+              </>
+            )}
+          </div>
+        )}
+
+        {entry && Object.keys(entry.labels).length > 0 && (
+          <div className="mt-3 flex flex-wrap items-center gap-2 border-t border-line pt-3">
+            <span className="text-xs text-muted">Filter</span>
+            {Object.entries(entry.labels).map(([k, vals]) => (
+              <Select
+                key={k}
+                value={match[k] ?? ""}
+                onChange={(e) => {
+                  const next = { ...match };
+                  if (e.target.value) next[k] = e.target.value;
+                  else delete next[k];
+                  update({ match: Object.keys(next).length ? JSON.stringify(next) : null });
+                }}
+                className="h-7.5 text-xs"
+              >
+                <option value="">{k}: any</option>
+                {vals.map((v) => <option key={v} value={v}>{k}={v || "(empty)"}</option>)}
+              </Select>
+            ))}
+            {Object.keys(match).length > 0 && (
+              <button className="inline-flex items-center gap-1 text-xs text-muted hover:text-ink" onClick={() => update({ match: null })}>
+                <X className="size-3" /> clear
+              </button>
+            )}
+          </div>
+        )}
+      </Card>
+
+      {!entry ? (
+        <Card><Empty>Pick a service and a metric to plot it</Empty></Card>
+      ) : (
+        <>
+          <Card
+            title={<span className="font-mono">{name}</span>}
+            subtitle={`${entry.help || entry.kind} · ${agg}: ${AGG_HELP[agg] ?? ""}${q.data ? ` · ${q.data.resolution === 3600 ? "hourly" : "minute"} data, ${q.data.step / 60} min interval` : ""}`}
+          >
+            {q.error ? <ErrorBox error={q.error} /> : <TimeChart data={data} height={320} format={fmt} />}
+            {hidden > 0 && (
+              <p className="mt-2 text-xs text-muted">
+                Charting the {MAX_CHART_SERIES} largest of {full.series.length} series — use a filter or fewer split labels to see the rest.
+              </p>
+            )}
+            {q.data?.truncated && <p className="mt-2 text-xs text-muted">The query returned the first 50 series only.</p>}
+          </Card>
+          <Card title="Series" subtitle={`${entry.series} series · ${entry.instances.length} instance(s) · last sample ${relative(entry.lastSeen)}`}>
+            <Table>
+              <thead>
+                <tr>
+                  <Th>Labels</Th>
+                  <Th className="text-right">Last</Th>
+                  <Th className="text-right">Min</Th>
+                  <Th className="text-right">Avg</Th>
+                  <Th className="text-right">Max</Th>
+                </tr>
+              </thead>
+              <tbody>
+                {summary.map((s, i) => (
+                  <tr key={i}>
+                    <Td className="font-mono text-xs text-ink">{Object.entries(s.labels).map(([k, v]) => `${k}=${v}`).join(", ") || name}</Td>
+                    <Td className="tnum text-right">{s.last === null ? "—" : fmt(s.last)}</Td>
+                    <Td className="tnum text-right">{s.min === null ? "—" : fmt(s.min)}</Td>
+                    <Td className="tnum text-right">{s.avg === null ? "—" : fmt(s.avg)}</Td>
+                    <Td className="tnum text-right">{s.max === null ? "—" : fmt(s.max)}</Td>
+                  </tr>
+                ))}
+              </tbody>
+            </Table>
+          </Card>
+        </>
+      )}
+    </div>
+  );
+}
+
+function Field({ label, children }: { label: string; children: React.ReactNode }) {
+  return (
+    <label className="flex flex-col gap-1">
+      <span className="text-xs text-muted">{label}</span>
+      {children}
+    </label>
+  );
+}

web/src/pages/Login.tsx

@@ -0,0 +1,64 @@
+import { useState, type FormEvent } from "react";
+import { useLocation, useNavigate } from "react-router";
+import { useQueryClient } from "@tanstack/react-query";
+import { KeyRound } from "lucide-react";
+import { api } from "../lib/api";
+import { Button, ErrorBox, Input } from "../components/ui";
+
+export default function Login() {
+  const [secret, setSecret] = useState("");
+  const [error, setError] = useState<unknown>(null);
+  const [busy, setBusy] = useState(false);
+  const navigate = useNavigate();
+  const location = useLocation();
+  const qc = useQueryClient();
+
+  const submit = async (e: FormEvent) => {
+    e.preventDefault();
+    setBusy(true);
+    setError(null);
+    try {
+      await api("/auth/login", { method: "POST", body: { secret } });
+      qc.clear();
+      const from = (location.state as { from?: string } | null)?.from;
+      navigate(from && from !== "/login" ? from : "/", { replace: true });
+    } catch (err) {
+      setError(err);
+    } finally {
+      setBusy(false);
+    }
+  };
+
+  return (
+    <div className="flex min-h-screen items-center justify-center px-4">
+      <form onSubmit={submit} className="w-full max-w-sm rounded-2xl border border-line bg-surface p-6 shadow-sm">
+        <div className="mb-6 flex items-center gap-3">
+          <div className="flex size-10 items-center justify-center rounded-xl bg-accent/15">
+            <KeyRound className="size-5 text-accent" />
+          </div>
+          <div>
+            <h1 className="font-semibold text-ink">Reforger Crawler admin</h1>
+            <p className="text-xs text-muted">Sign in with the admin secret</p>
+          </div>
+        </div>
+        <label className="text-xs text-ink-2" htmlFor="secret">
+          Admin secret
+        </label>
+        <Input
+          id="secret"
+          type="password"
+          autoFocus
+          autoComplete="current-password"
+          value={secret}
+          onChange={(e) => setSecret(e.target.value)}
+          className="mt-1 w-full"
+        />
+        {error !== null && <div className="mt-3"><ErrorBox error={error} /></div>}
+        <Button type="submit" variant="primary" className="mt-4 w-full" loading={busy} disabled={!secret}>
+          Sign in
+        </Button>
+        <p className="mt-4 text-center text-[11px] text-muted">The session is kept in an HttpOnly cookie for 7 days.</p>
+      </form>
+    </div>
+  );
+}

web/src/pages/Overview.tsx

@@ -0,0 +1,209 @@
+import { useQuery, keepPreviousData } from "@tanstack/react-query";
+import { useNavigate } from "react-router";
+import { api } from "../lib/api";
+import { useMetric, useRangeStep } from "../lib/queries";
+import { useRefetch } from "../lib/settings";
+import type { MiniAddon, Overview as OverviewT, ServiceEvent, TopEntry } from "../lib/types";
+import { bytes, compact, int, ms, num, pct, relative } from "../lib/format";
+import { Badge, Card, ErrorBox, LevelBadge, Loading, Meter, PageHeader, StatTile, Table, Td } from "../components/ui";
+import { fromQuery, TimeChart } from "../components/charts/TimeChart";
+import { BarList } from "../components/charts/BarList";
+
+const GROUP_SLOT: Record<string, number> = { v1: 1, back: 2, admin: 3, metrics: 4, other: 5 };
+const GROUP_LABEL: Record<string, string> = { v1: "Website API", back: "Workers", admin: "Admin", metrics: "Prometheus", other: "Other" };
+
+function values(q: { series: { points: [number, number | null][] }[] } | undefined): (number | null)[] {
+  return q?.series[0]?.points.map((p) => p[1]) ?? [];
+}
+
+export default function Overview() {
+  const navigate = useNavigate();
+  const refetchInterval = useRefetch();
+  const { step } = useRangeStep();
+  const ov = useQuery({ queryKey: ["overview"], queryFn: () => api<OverviewT>("/overview"), refetchInterval, placeholderData: keepPreviousData });
+  const topAddons = useQuery({ queryKey: ["top", "addon"], queryFn: () => api<{ items: TopEntry[] }>("/top?kind=addon&days=1&limit=8"), refetchInterval });
+  const topSearch = useQuery({ queryKey: ["top", "search"], queryFn: () => api<{ items: TopEntry[] }>("/top?kind=search&days=1&limit=8"), refetchInterval });
+  const events = useQuery({ queryKey: ["events", "recent"], queryFn: () => api<{ items: ServiceEvent[] }>("/services/events?limit=6"), refetchInterval });
+
+  const requests = useMetric({ name: "http_request_duration_ms", agg: "rate", groupBy: ["group"], step });
+  const p95 = useMetric({ name: "http_request_duration_ms", agg: "p95", match: { group: "v1" }, step });
+  const queueDepth = useMetric({ name: "queue_depth", agg: "last", groupBy: ["kind"], step });
+  const indexed = useMetric({ name: "indexer_submissions_total", agg: "sum", groupBy: ["result"], step });
+  const addonsTotal = useMetric({ name: "addons_total", agg: "last", step });
+  const visitors = useMetric({ name: "visitors_unique", agg: "max", match: { kind: "human" }, step: 86400, rangeSec: 14 * 86400 });
+
+  if (ov.isLoading) return <Loading />;
+  if (ov.error) return <ErrorBox error={ov.error} />;
+  const d = ov.data!.data;
+  const a = d.addons;
+  const heavy = d.heavy;
+  const openAddon = (id: string) => navigate(`/addons/${id}`);
+
+  return (
+    <div className="space-y-4">
+      <PageHeader title="Overview" description={<>Snapshot of the crawler, its workers and the public website · updated {relative(ov.data!.asOf)}</>} />
+
+      <div className="grid grid-cols-1 gap-3 sm:grid-cols-2 xl:grid-cols-4">
+        <StatTile hero label="Addons tracked" value={int(a.total)} sub={`+${int(a.added24h)} today · +${int(a.added7d)} this week`} trend={values(addonsTotal.data)} onClick={() => navigate("/addons")} />
+        <div className="rounded-xl border border-line bg-surface p-4">
+          <div className="text-xs text-muted">Indexed</div>
+          <div className="mt-2 text-2xl font-semibold tracking-tight text-ink">{pct(a.indexed, a.total)}</div>
+          <div className="mt-1.5 text-xs text-ink-2">
+            {int(a.indexed)} of {int(a.total)} · {int(a.indexed24h)} in last 24h
+          </div>
+          <div className="mt-3">
+            <Meter value={a.indexed} max={a.total} />
+          </div>
+        </div>
+        <StatTile
+          label="Website visitors today"
+          value={int(d.traffic.visitorsToday)}
+          sub={`${int(d.traffic.botsToday)} bots / scripts · approx.`}
+          trend={values(visitors.data)}
+          onClick={() => navigate("/traffic")}
+        />
+        <StatTile
+          label="Website latency p95 (24h)"
+          value={ms(d.traffic.websiteP95Ms)}
+          sub={`p50 ${ms(d.traffic.websiteP50Ms)} · ${compact(d.traffic.websiteRequests24h ?? 0)} requests`}
+          trend={values(p95.data)}
+          onClick={() => navigate("/traffic")}
+        />
+        <StatTile label="Indexing queue" value={int(a.queued)} sub={`${int(a.priority)} priority · ${int(a.indexing)} in progress`} onClick={() => navigate("/queue")} />
+        <StatTile
+          label="Indexed files"
+          value={compact(heavy?.filesLive)}
+          sub={heavy ? `${compact(heavy.distinctHashes)} distinct hashes${heavy.filesApprox ? " · approx." : ""}` : "computing…"}
+        />
+        <StatTile label="Server errors (24h)" value={int(d.traffic.errors5xx24h)} sub={`${compact(d.traffic.requests24h ?? 0)} requests in total`} onClick={() => navigate("/traffic")} />
+        <StatTile label="Services online" value={`${d.services.online} / ${d.services.total}`} sub="indexers, scrapers, API" onClick={() => navigate("/services")} />
+      </div>
+
+      <div className="grid grid-cols-1 gap-4 xl:grid-cols-3">
+        <Card title="Requests per minute" subtitle="All HTTP traffic by caller" className="xl:col-span-2">
+          <TimeChart
+            data={fromQuery(requests.data, (l) => GROUP_LABEL[l.group] ?? l.group, 60, (l) => GROUP_SLOT[l.group])}
+            format={(v) => num(v, 1)}
+          />
+        </Card>
+        <Card title="Queue depth" subtitle="Addons waiting to be indexed">
+          <TimeChart
+            data={fromQuery(queueDepth.data, (l) => (l.kind === "priority" ? "Priority" : "All queued"), 1, (l) => (l.kind === "priority" ? 2 : 1))}
+            format={compact}
+          />
+        </Card>
+      </div>
+
+      <div className="grid grid-cols-1 gap-4 xl:grid-cols-3">
+        <Card title="Indexing results" subtitle="Submissions from indexers per interval" className="xl:col-span-2">
+          <TimeChart data={fromQuery(indexed.data, (l) => l.result)} format={compact} />
+        </Card>
+        <Card title="Last 24 hours">
+          <dl className="space-y-2 text-[13px]">
+            {[
+              ["New addons from scrapers", d.activity.scraperCreated24h],
+              ["Updated addons", d.activity.scraperUpdated24h],
+              ["Failed creates (soft-deleted IDs)", d.activity.scraperCreateFailed24h],
+              ["Deleted by scrapers", d.activity.deleted24h],
+              ["Indexed", d.activity.indexed24h],
+              ["Indexing claims", d.activity.claims24h],
+              ["Timed-out claims reset", d.activity.timeoutsReset24h],
+              ["Dependency reports", d.activity.dependencyReports24h],
+            ].map(([k, v]) => (
+              <div key={k as string} className="flex justify-between gap-3">
+                <dt className="text-ink-2">{k}</dt>
+                <dd className="tnum text-ink">{int(v as number)}</dd>
+              </div>
+            ))}
+          </dl>
+        </Card>
+      </div>
+
+      <div className="grid grid-cols-1 gap-4 md:grid-cols-2 xl:grid-cols-3">
+        <Card title="Most viewed addons today">
+          <BarList
+            items={(topAddons.data?.items ?? []).map((t) => ({ key: t.item, label: t.name || t.item, value: t.count, onClick: () => openAddon(t.item) }))}
+            empty="No lookups yet today"
+          />
+        </Card>
+        <Card title="Top searches today">
+          <BarList items={(topSearch.data?.items ?? []).map((t) => ({ key: t.item, label: t.item, value: t.count }))} empty="No searches yet today" />
+        </Card>
+        <Card title="Addon states">
+          <div className="space-y-2.5 text-[13px]">
+            {[
+              { label: "Blocked on workshop", value: a.blocked, tone: "critical" as const, status: "blocked" },
+              { label: "Soft-deleted", value: a.softDeleted, tone: "serious" as const, status: "deleted" },
+              { label: "Unlisted", value: a.unlisted, tone: "neutral" as const, status: "unlisted" },
+              { label: "Private", value: a.private, tone: "neutral" as const, status: "private" },
+              { label: "Priority indexing", value: a.priority, tone: "accent" as const, status: "priority" },
+              { label: "Marked for recheck", value: a.recheck, tone: "warning" as const, status: "recheck" },
+            ].map((s) => (
+              <button key={s.label} onClick={() => navigate(`/addons?status=${s.status}`)} className="flex w-full items-center justify-between gap-3 text-left hover:text-ink">
+                <Badge tone={s.tone}>{s.label}</Badge>
+                <span className="tnum text-ink">{int(s.value)}</span>
+              </button>
+            ))}
+            <div className="border-t border-line pt-2.5 text-xs text-ink-2">
+              {int(d.whitelistCount)} whitelisted hashes · {int(d.dependencies.edges)} dependency edges · {bytes(a.totalSizeBytes)} total size
+            </div>
+          </div>
+        </Card>
+      </div>
+
+      <div className="grid grid-cols-1 gap-4 md:grid-cols-2 xl:grid-cols-3">
+        <AddonList title="Most subscribed" items={d.topSubscribed} metric={(x) => compact(x.subscriberCount)} onOpen={openAddon} />
+        <AddonList title="Largest" items={d.largest} metric={(x) => bytes(x.currentVersionSize)} onOpen={openAddon} />
+        <AddonList title="Recently indexed" items={d.recentlyIndexed} metric={(x) => relative(x.lastIndexedAt)} onOpen={openAddon} />
+      </div>
+
+      <div className="grid grid-cols-1 gap-4 xl:grid-cols-3">
+        <Card title="Addon types">
+          <BarList items={d.types.map((t) => ({ key: t.type || "(none)", label: t.type || "(none)", value: t.count }))} />
+        </Card>
+        <Card title="Recent events" className="xl:col-span-2" actions={<button onClick={() => navigate("/events")} className="text-xs text-accent hover:underline">All events</button>}>
+          {(events.data?.items ?? []).length === 0 ? (
+            <div className="py-6 text-center text-sm text-muted">No events yet</div>
+          ) : (
+            <Table>
+              <tbody>
+                {events.data!.items.map((e) => (
+                  <tr key={e.id}>
+                    <Td className="w-24"><LevelBadge level={e.level} /></Td>
+                    <Td className="text-ink">{e.message}</Td>
+                    <Td className="whitespace-nowrap">{e.service}</Td>
+                    <Td className="whitespace-nowrap text-right text-muted">{relative(e.ts)}</Td>
+                  </tr>
+                ))}
+              </tbody>
+            </Table>
+          )}
+        </Card>
+      </div>
+    </div>
+  );
+}
+
+function AddonList({ title, items, metric, onOpen }: { title: string; items: MiniAddon[]; metric: (a: MiniAddon) => string; onOpen: (id: string) => void }) {
+  return (
+    <Card title={title}>
+      {items.length === 0 ? (
+        <div className="py-6 text-center text-sm text-muted">Nothing yet</div>
+      ) : (
+        <ul className="-mx-2">
+          {items.map((x) => (
+            <li key={x.id}>
+              <button onClick={() => onOpen(x.id)} className="flex w-full items-center justify-between gap-3 rounded-lg px-2 py-1.5 text-left hover:bg-surface-2/60">
+                <span className="min-w-0">
+                  <span className="block truncate text-[13px] text-ink" title={x.name}>{x.name || x.id}</span>
+                  <span className="block truncate text-xs text-muted">{x.author || "unknown author"}</span>
+                </span>
+                <span className="tnum shrink-0 text-[13px] text-ink-2">{metric(x)}</span>
+              </button>
+            </li>
+          ))}
+        </ul>
+      )}
+    </Card>
+  );
+}

web/src/pages/Queue.tsx

@@ -0,0 +1,208 @@
+import { useMutation, useQuery, useQueryClient, keepPreviousData } from "@tanstack/react-query";
+import { useNavigate } from "react-router";
+import { RotateCcw, Star, StarOff } from "lucide-react";
+import { api } from "../lib/api";
+import { useMetric, useRangeStep } from "../lib/queries";
+import { useRefetch } from "../lib/settings";
+import type { QueueData, QueueRow } from "../lib/types";
+import { bytes, compact, duration, int, num, relative, seconds } from "../lib/format";
+import { Badge, Button, Card, ErrorBox, Loading, PageHeader, StatTile, Table, Td, Th, useFeedback } from "../components/ui";
+import { fromQuery, TimeChart } from "../components/charts/TimeChart";
+
+export default function Queue() {
+  const refetchInterval = useRefetch();
+  const { step } = useRangeStep();
+  const qc = useQueryClient();
+  const { toast, confirm } = useFeedback();
+  const navigate = useNavigate();
+  const q = useQuery({ queryKey: ["queue"], queryFn: () => api<QueueData>("/queue"), refetchInterval: refetchInterval || 15000, placeholderData: keepPreviousData });
+
+  const depth = useMetric({ name: "queue_depth", agg: "last", groupBy: ["kind"], step });
+  const claims = useMetric({ name: "indexer_claims_total", agg: "sum", groupBy: ["queue"], step });
+  const jobDuration = useMetric({ name: "indexer_job_duration_seconds", agg: "p50", step });
+
+  const resetStuck = useMutation({
+    mutationFn: () => api<{ reset: number }>("/queue/reset-stuck", { method: "POST", body: {} }),
+    onSuccess: (r) => {
+      toast(`Released ${r.reset} timed-out claim${r.reset === 1 ? "" : "s"}`);
+      qc.invalidateQueries({ queryKey: ["queue"] });
+    },
+    onError: (e) => toast(String((e as Error).message), "critical"),
+  });
+
+  const action = useMutation({
+    mutationFn: ({ id, act, value }: { id: string; act: string; value?: boolean }) =>
+      api(`/addons/${encodeURIComponent(id)}/${act}`, { method: "POST", body: { value: value ?? true } }),
+    onSuccess: () => {
+      toast("Updated");
+      qc.invalidateQueries({ queryKey: ["queue"] });
+    },
+    onError: (e) => toast(String((e as Error).message), "critical"),
+  });
+
+  if (q.isLoading) return <Loading />;
+  if (q.error) return <ErrorBox error={q.error} />;
+  const d = q.data!;
+  const stuck = d.indexing.filter((r) => r.stuck).length;
+
+  return (
+    <div className="space-y-4">
+      <PageHeader
+        title="Indexing queue"
+        description={<>Jobs are claimed priority-first, then by oldest update · claims time out after {d.timeoutMinutes} min · updated {relative(d.asOf)}</>}
+        actions={
+          <Button
+            icon={<RotateCcw className="size-3.5" />}
+            loading={resetStuck.isPending}
+            onClick={async () => {
+              if (await confirm({ title: "Release timed-out claims?", body: `Addons claimed more than ${d.timeoutMinutes} minutes ago go back to the queue.` }))
+                resetStuck.mutate();
+            }}
+          >
+            Release stuck claims
+          </Button>
+        }
+      />
+
+      <div className="grid grid-cols-1 gap-3 sm:grid-cols-2 xl:grid-cols-4">
+        <StatTile label="Waiting" value={int(d.counts.queued)} sub={`${int(d.counts.priority)} priority · ${int(d.blockedQueued)} blocked (never claimed)`} />
+        <StatTile label="In progress" value={int(d.counts.indexing)} sub={stuck ? `${stuck} past the timeout` : "none stuck"} />
+        <StatTile
+          label="Throughput"
+          value={`${num(d.throughput.perHour6h, 1)}/h`}
+          sub={`${num(d.throughput.perHour1h, 0)} last hour · ${int(d.throughput.indexed24h)} in 24h`}
+        />
+        <StatTile
+          label="Queue drains in"
+          value={d.throughput.etaHours === null ? "—" : duration(d.throughput.etaHours * 3600)}
+          sub={d.throughput.duration24h ? `job p50 ${duration(d.throughput.duration24h.p50)} · p95 ${duration(d.throughput.duration24h.p95)}` : "at the 6h average rate"}
+        />
+      </div>
+
+      <div className="grid grid-cols-1 gap-4 xl:grid-cols-3">
+        <Card title="Queue depth">
+          <TimeChart data={fromQuery(depth.data, (l) => (l.kind === "priority" ? "Priority" : "All queued"), 1, (l) => (l.kind === "priority" ? 2 : 1))} />
+        </Card>
+        <Card title="Claims per interval" subtitle="What indexers got when asking for work">
+          <TimeChart data={fromQuery(claims.data, (l) => l.queue)} />
+        </Card>
+        <Card title="Job duration p50" subtitle="Claim to submitted result">
+          <TimeChart data={fromQuery(jobDuration.data, () => "p50")} format={seconds} />
+        </Card>
+      </div>
+
+      <QueueTable
+        title={`In progress (${d.indexing.length})`}
+        rows={d.indexing}
+        empty="No addon is being indexed right now"
+        extra={(r) => (
+          <>
+            <Td className="tnum whitespace-nowrap">{duration(r.elapsedSec)}</Td>
+            <Td>{r.stuck ? <Badge tone="critical">past timeout</Badge> : <Badge tone="good">running</Badge>}</Td>
+          </>
+        )}
+        extraHead={
+          <>
+            <Th>Elapsed</Th>
+            <Th>State</Th>
+          </>
+        }
+        actions={(r) => (
+          <Button size="sm" variant="ghost" onClick={() => action.mutate({ id: r.id, act: "reset-indexing" })}>
+            Release
+          </Button>
+        )}
+        onOpen={(id) => navigate(`/addons/${id}`)}
+      />
+
+      <QueueTable
+        title={`Priority (${d.counts.priority})`}
+        rows={d.priority}
+        empty="No priority jobs. Mark addons as priority from the addon list."
+        actions={(r) => (
+          <Button size="sm" variant="ghost" icon={<StarOff className="size-3.5" />} onClick={() => action.mutate({ id: r.id, act: "priority", value: false })}>
+            Unprioritize
+          </Button>
+        )}
+        onOpen={(id) => navigate(`/addons/${id}`)}
+      />
+
+      <QueueTable
+        title="Next up"
+        subtitle="The next jobs in claim order (an indexer may skip addons above its size limit)"
+        rows={d.next}
+        empty="The queue is empty"
+        actions={(r) =>
+          r.priorityIndexing ? null : (
+            <Button size="sm" variant="ghost" icon={<Star className="size-3.5" />} onClick={() => action.mutate({ id: r.id, act: "priority", value: true })}>
+              Prioritize
+            </Button>
+          )
+        }
+        onOpen={(id) => navigate(`/addons/${id}`)}
+      />
+    </div>
+  );
+}
+
+function QueueTable({
+  title,
+  subtitle,
+  rows,
+  empty,
+  extra,
+  extraHead,
+  actions,
+  onOpen,
+}: {
+  title: string;
+  subtitle?: string;
+  rows: QueueRow[];
+  empty: string;
+  extra?: (r: QueueRow) => React.ReactNode;
+  extraHead?: React.ReactNode;
+  actions?: (r: QueueRow) => React.ReactNode;
+  onOpen: (id: string) => void;
+}) {
+  return (
+    <Card title={title} subtitle={subtitle}>
+      {rows.length === 0 ? (
+        <div className="py-4 text-sm text-muted">{empty}</div>
+      ) : (
+        <Table>
+          <thead>
+            <tr>
+              <Th>Addon</Th>
+              <Th className="text-right">Size</Th>
+              <Th className="text-right">Subscribers</Th>
+              <Th>Waiting since</Th>
+              {extraHead}
+              <Th />
+            </tr>
+          </thead>
+          <tbody>
+            {rows.map((r) => (
+              <tr key={r.id} className="cursor-pointer hover:bg-surface-2/40" onClick={() => onOpen(r.id)}>
+                <Td className="text-ink">
+                  <div className="flex items-center gap-2">
+                    {r.name || r.id}
+                    {r.priorityIndexing && <Badge tone="accent">priority</Badge>}
+                    {r.blocked && <Badge tone="critical">blocked</Badge>}
+                  </div>
+                  <div className="font-mono text-[11px] text-muted">{r.id}</div>
+                </Td>
+                <Td className="tnum text-right">{bytes(r.currentVersionSize)}</Td>
+                <Td className="tnum text-right">{compact(r.subscriberCount)}</Td>
+                <Td className="whitespace-nowrap">{relative(r.updatedAt)}</Td>
+                {extra?.(r)}
+                <Td className="text-right" >
+                  <span onClick={(e) => e.stopPropagation()}>{actions?.(r)}</span>
+                </Td>
+              </tr>
+            ))}
+          </tbody>
+        </Table>
+      )}
+    </Card>
+  );
+}

web/src/pages/Services.tsx

@@ -0,0 +1,212 @@
+import { useState } from "react";
+import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
+import { useNavigate } from "react-router";
+import { LineChart, Trash2 } from "lucide-react";
+import { api, qs } from "../lib/api";
+import { useMetric, useRangeStep } from "../lib/queries";
+import { useRefetch } from "../lib/settings";
+import type { CatalogEntry, ServiceEvent, ServiceInstance } from "../lib/types";
+import { compact, dateTime, duration, relative } from "../lib/format";
+import { Badge, Button, Card, Empty, ErrorBox, LevelBadge, Loading, PageHeader, StatusDot, Table, Td, cn, useFeedback } from "../components/ui";
+import { fromQuery, TimeChart } from "../components/charts/TimeChart";
+
+const EXAMPLE = `POST /back/telemetry
+X-SECRET-KEY: <SECRET_KEY>
+Content-Type: application/json
+
+{
+  "service": "indexer",            // [a-z0-9_], not "main"
+  "instance": "indexer-1",
+  "version": "1.4.0",
+  "startedAt": 1727950000,
+  "heartbeatIntervalSec": 15,      // offline after 3 missed beats
+  "status": "ok",                  // ok | degraded | error
+  "metrics": [
+    {"name": "jobs_total", "type": "counter", "value": 3, "labels": {"result": "ok"}},
+    {"name": "data_window_bytes", "type": "gauge", "value": 7.1e9},
+    {"name": "download_seconds", "type": "summary", "count": 3, "sum": 41.2, "min": 8.1, "max": 20.3}
+  ],
+  "events": [{"level": "warn", "message": "data limit reached", "fields": {"limitGB": 50}}]
+}`;
+
+export default function Services() {
+  const refetchInterval = useRefetch();
+  const qc = useQueryClient();
+  const { toast, confirm } = useFeedback();
+  const [selected, setSelected] = useState<string | null>(null);
+  const services = useQuery({ queryKey: ["services"], queryFn: () => api<{ items: ServiceInstance[] }>("/services"), refetchInterval: refetchInterval || 30000 });
+
+  const forget = useMutation({
+    mutationFn: (id: number) => api(`/services/${id}`, { method: "DELETE" }),
+    onSuccess: () => {
+      toast("Instance removed");
+      qc.invalidateQueries({ queryKey: ["services"] });
+    },
+  });
+
+  if (services.isLoading) return <Loading />;
+  if (services.error) return <ErrorBox error={services.error} />;
+  const items = services.data!.items;
+  const names = [...new Set(items.map((s) => s.service))];
+  const current = selected ?? names.find((n) => n !== "main") ?? names[0] ?? null;
+
+  return (
+    <div className="space-y-4">
+      <PageHeader title="Services" description="Indexers, scrapers and the API report heartbeats, metrics and events here" />
+
+      {items.length === 0 ? (
+        <Card><Empty>No service has reported yet</Empty></Card>
+      ) : (
+        <div className="grid grid-cols-1 gap-3 md:grid-cols-2 xl:grid-cols-3">
+          {items.map((s) => (
+            <div
+              key={s.id}
+              onClick={() => setSelected(s.service)}
+              className={cn(
+                "cursor-pointer rounded-xl border bg-surface p-4 transition-colors",
+                current === s.service ? "border-accent/60" : "border-line hover:border-ink/25",
+              )}
+            >
+              <div className="flex items-start justify-between gap-3">
+                <div className="min-w-0">
+                  <div className="flex items-center gap-2">
+                    <StatusDot online={s.online} />
+                    <span className="font-medium text-ink">{s.service}</span>
+                    <span className="truncate text-xs text-muted">{s.instance}</span>
+                  </div>
+                  <div className="mt-2 flex flex-wrap gap-1">
+                    <Badge tone={s.online ? "good" : "critical"}>{s.online ? "online" : "offline"}</Badge>
+                    {s.status && s.status !== "ok" && <Badge tone={s.status === "error" ? "critical" : "warning"}>{s.status}</Badge>}
+                    {s.version && <Badge>v{s.version}</Badge>}
+                  </div>
+                </div>
+                {!s.online && s.service !== "main" && (
+                  <Button
+                    size="sm"
+                    variant="ghost"
+                    icon={<Trash2 className="size-3.5" />}
+                    aria-label="Forget instance"
+                    onClick={async (e) => {
+                      e.stopPropagation();
+                      if (await confirm({ title: `Forget ${s.service}/${s.instance}?`, body: "Its metric history stays; the instance will reappear if it reports again." }))
+                        forget.mutate(s.id);
+                    }}
+                  />
+                )}
+              </div>
+              {s.statusMessage && <div className="mt-2 text-xs text-ink-2">{s.statusMessage}</div>}
+              <dl className="mt-3 grid grid-cols-2 gap-2 text-xs">
+                <div>
+                  <dt className="text-muted">Last seen</dt>
+                  <dd className="text-ink-2">{relative(s.lastSeen)}</dd>
+                </div>
+                <div>
+                  <dt className="text-muted">Uptime</dt>
+                  <dd className="text-ink-2">{s.online && s.uptimeSec ? duration(s.uptimeSec) : "—"}</dd>
+                </div>
+                <div>
+                  <dt className="text-muted">Heartbeat</dt>
+                  <dd className="text-ink-2">every {s.heartbeatIntervalSec}s</dd>
+                </div>
+                <div>
+                  <dt className="text-muted">Address</dt>
+                  <dd className="truncate text-ink-2">{s.remoteAddr || "local"}</dd>
+                </div>
+                {s.metaData &&
+                  Object.entries(s.metaData).slice(0, 4).map(([k, v]) => (
+                    <div key={k}>
+                      <dt className="text-muted">{k}</dt>
+                      <dd className="truncate text-ink-2">{String(v)}</dd>
+                    </div>
+                  ))}
+              </dl>
+            </div>
+          ))}
+        </div>
+      )}
+
+      {current && <ServiceDetail service={current} />}
+
+      <Card title="Sending telemetry" subtitle="Any process can report; a stdlib-only Go client lives in the telemetry/ module of this repo">
+        <pre className="overflow-x-auto rounded-lg bg-surface-2 p-3 font-mono text-xs leading-relaxed text-ink-2">{EXAMPLE}</pre>
+        <p className="mt-3 text-xs text-ink-2">
+          Counters carry the increase since the previous push, gauges the current value, summaries a pre-aggregated count/sum/min/max. Every push counts as a heartbeat.
+        </p>
+      </Card>
+    </div>
+  );
+}
+
+function ServiceDetail({ service }: { service: string }) {
+  const navigate = useNavigate();
+  const refetchInterval = useRefetch();
+  const catalog = useQuery({ queryKey: ["catalog", service], queryFn: () => api<{ items: CatalogEntry[] }>("/metrics/catalog" + qs({ service })) });
+  const events = useQuery({
+    queryKey: ["events", service],
+    queryFn: () => api<{ items: ServiceEvent[] }>("/services/events" + qs({ service, limit: 15 })),
+    refetchInterval,
+  });
+  const metrics = (catalog.data?.items ?? []).slice(0, service === "main" ? 12 : 24);
+
+  return (
+    <div className="space-y-4">
+      <div className="flex items-center justify-between">
+        <h2 className="text-base font-semibold text-ink">{service} metrics</h2>
+        <Button size="sm" icon={<LineChart className="size-3.5" />} onClick={() => navigate(`/explorer?service=${service}`)}>
+          Open in explorer
+        </Button>
+      </div>
+      {catalog.isLoading ? (
+        <Loading />
+      ) : metrics.length === 0 ? (
+        <Card><Empty>This service has not sent any metrics yet</Empty></Card>
+      ) : (
+        <div className="grid grid-cols-1 gap-4 md:grid-cols-2 xl:grid-cols-3">
+          {metrics.map((m) => (
+            <MiniMetric key={m.name} entry={m} onOpen={() => navigate(`/explorer?service=${service}&name=${m.name}`)} />
+          ))}
+        </div>
+      )}
+      <Card title="Recent events">
+        {(events.data?.items ?? []).length === 0 ? (
+          <div className="text-sm text-muted">No events</div>
+        ) : (
+          <Table>
+            <tbody>
+              {events.data!.items.map((e) => (
+                <tr key={e.id}>
+                  <Td className="w-20"><LevelBadge level={e.level} /></Td>
+                  <Td className="text-ink">
+                    {e.message}
+                    {e.fieldsData && <div className="truncate font-mono text-[11px] text-muted">{JSON.stringify(e.fieldsData)}</div>}
+                  </Td>
+                  <Td className="whitespace-nowrap text-xs">{e.instance}</Td>
+                  <Td className="whitespace-nowrap text-right text-xs text-muted" title={dateTime(e.ts)}>{relative(e.ts)}</Td>
+                </tr>
+              ))}
+            </tbody>
+          </Table>
+        )}
+      </Card>
+    </div>
+  );
+}
+
+function aggFor(kind: string): string {
+  return kind === "counter" ? "sum" : kind === "gauge" ? "last" : kind === "histogram" ? "p95" : "avg";
+}
+
+function MiniMetric({ entry, onOpen }: { entry: CatalogEntry; onOpen: () => void }) {
+  const { step } = useRangeStep();
+  const groupBy = entry.instances.length > 1 ? ["instance"] : undefined;
+  const q = useMetric({ service: entry.service, name: entry.name, agg: aggFor(entry.kind), groupBy, step });
+  return (
+    <Card
+      title={<span className="font-mono text-xs">{entry.name}</span>}
+      subtitle={`${entry.kind} · ${aggFor(entry.kind)}${entry.help ? ` · ${entry.help}` : ""}`}
+      actions={<button className="text-xs text-accent hover:underline" onClick={onOpen}>Explore</button>}
+    >
+      <TimeChart data={fromQuery(q.data, (l) => l.instance ?? entry.name)} height={140} format={compact} />
+    </Card>
+  );
+}

web/src/pages/System.tsx

@@ -0,0 +1,248 @@
+import { useState } from "react";
+import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
+import { Database, Eraser, Gauge, RefreshCw } from "lucide-react";
+import { api } from "../lib/api";
+import { useRefetch } from "../lib/settings";
+import type { HeavyStats } from "../lib/types";
+import { bytes, dateTime, duration, int, num, relative } from "../lib/format";
+import { Badge, Button, Card, ErrorBox, Input, Loading, PageHeader, StatTile, Table, Td, Th, useFeedback } from "../components/ui";
+
+interface SystemData {
+  go: { version: string; os: string; arch: string; cpus: number; maxProcs: number; goroutines: number };
+  process: { pid: number; hostname: string; startedAt: number; uptimeSec: number };
+  memory: { heapAlloc: number; heapInuse: number; sys: number; numGC: number; pauseTotalNs: number; lastGC: number };
+  build: { commit?: string; time?: string; modified?: boolean; module?: string };
+  metrics: {
+    enabled: boolean;
+    lastFlush: number;
+    lastFlushMs: number;
+    lastFlushError: string;
+    lastRollup: number;
+    lastRetention: number;
+    rolledUntil: number;
+    openPoints: number;
+    liveSeries: number;
+    droppedPoints: number;
+  };
+  config: Record<string, unknown>;
+  dbPool?: { open: number; inUse: number; idle: number; maxOpen: number; waitCount: number; waitMs: number };
+  dbPingMs?: number;
+  dbPingError?: string;
+  db?: HeavyStats;
+}
+
+export default function System() {
+  const refetchInterval = useRefetch();
+  const qc = useQueryClient();
+  const { toast, confirm } = useFeedback();
+  const [days, setDays] = useState("7");
+  const sys = useQuery({ queryKey: ["system"], queryFn: () => api<SystemData>("/system"), refetchInterval });
+
+  const run = useMutation({
+    mutationFn: (body: { action: string; olderThanDays?: number }) => api("/system/maintenance", { method: "POST", body }),
+    onSuccess: (_r, v) => {
+      toast(`Started: ${v.action}`);
+      setTimeout(() => qc.invalidateQueries({ queryKey: ["system"] }), 1500);
+    },
+    onError: (e) => toast((e as Error).message, "critical"),
+  });
+
+  if (sys.isLoading) return <Loading />;
+  if (sys.error) return <ErrorBox error={sys.error} />;
+  const s = sys.data!;
+  const db = s.db;
+
+  return (
+    <div className="space-y-4">
+      <PageHeader title="System" description={<>{s.process.hostname} · pid {s.process.pid} · build {s.build.commit ? s.build.commit.slice(0, 10) : "dev"}{s.build.modified ? " (modified)" : ""}</>} />
+
+      <div className="grid grid-cols-1 gap-3 sm:grid-cols-2 xl:grid-cols-4">
+        <StatTile label="Uptime" value={duration(s.process.uptimeSec)} sub={`since ${dateTime(s.process.startedAt)}`} />
+        <StatTile label="Heap in use" value={bytes(s.memory.heapAlloc)} sub={`${bytes(s.memory.sys)} from OS · ${int(s.memory.numGC)} GCs`} />
+        <StatTile label="Goroutines" value={int(s.go.goroutines)} sub={`${s.go.version} · ${s.go.os}/${s.go.arch} · ${s.go.cpus} CPUs`} />
+        <StatTile
+          label="Database"
+          value={db ? bytes(db.dbSizeBytes) : "—"}
+          sub={`${db?.dbDriver ?? ""} · ping ${s.dbPingMs !== undefined ? `${num(s.dbPingMs, 1)} ms` : s.dbPingError ?? "—"}`}
+        />
+      </div>
+
+      <div className="grid grid-cols-1 gap-4 xl:grid-cols-3">
+        <Card
+          title="Tables"
+          subtitle={db ? `Updated ${relative(db.asOf)}${db.dbDriver === "postgres" ? " · row counts are estimates" : ""}` : "Computing…"}
+          className="xl:col-span-2"
+          actions={
+            <Button size="sm" icon={<RefreshCw className="size-3.5" />} onClick={() => run.mutate({ action: "refresh-stats" })}>
+              Refresh
+            </Button>
+          }
+        >
+          {db?.error && <div className="mb-3"><ErrorBox error={db.error} /></div>}
+          <Table>
+            <thead>
+              <tr>
+                <Th>Table</Th>
+                <Th className="text-right">Rows</Th>
+                <Th className="text-right">Dead rows</Th>
+                <Th className="text-right">Size</Th>
+              </tr>
+            </thead>
+            <tbody>
+              {(db?.tables ?? []).map((t) => (
+                <tr key={t.name}>
+                  <Td className="font-mono text-xs text-ink">{t.name}</Td>
+                  <Td className="tnum text-right">{int(t.rows)}</Td>
+                  <Td className="tnum text-right">{t.deadRows ? int(t.deadRows) : "—"}</Td>
+                  <Td className="tnum text-right">{t.sizeBytes ? bytes(t.sizeBytes) : "—"}</Td>
+                </tr>
+              ))}
+            </tbody>
+          </Table>
+          {db && (
+            <div className="mt-3 grid grid-cols-2 gap-3 text-xs text-ink-2 sm:grid-cols-3">
+              <div>
+                <div className="text-muted">Live file rows</div>
+                <div className="text-ink">{int(db.filesLive)}{db.filesApprox ? " (est.)" : ""}</div>
+              </div>
+              <div>
+                <div className="text-muted">Soft-deleted file rows</div>
+                <div className="text-ink">{int(db.filesSoftDeleted)}</div>
+              </div>
+              <div>
+                <div className="text-muted">Distinct hashes</div>
+                <div className="text-ink">{int(db.distinctHashes)} <span className="text-muted">· {relative(db.distinctAsOf)}</span></div>
+              </div>
+            </div>
+          )}
+        </Card>
+
+        <Card title="Connection pool">
+          {s.dbPool ? (
+            <dl className="space-y-2 text-[13px]">
+              {[
+                ["Open", s.dbPool.open],
+                ["In use", s.dbPool.inUse],
+                ["Idle", s.dbPool.idle],
+                ["Max open", s.dbPool.maxOpen || "unlimited"],
+                ["Waits", s.dbPool.waitCount],
+                ["Wait time", `${int(s.dbPool.waitMs)} ms`],
+              ].map(([k, v]) => (
+                <div key={k as string} className="flex justify-between">
+                  <dt className="text-ink-2">{k}</dt>
+                  <dd className="tnum text-ink">{typeof v === "number" ? int(v) : v}</dd>
+                </div>
+              ))}
+            </dl>
+          ) : (
+            <div className="text-sm text-muted">Unavailable</div>
+          )}
+        </Card>
+      </div>
+
+      <div className="grid grid-cols-1 gap-4 xl:grid-cols-3">
+        <Card title="Metrics pipeline" subtitle="Minute buckets are flushed to the database every minute and rolled up hourly">
+          <dl className="space-y-2 text-[13px]">
+            <Row k="Status" v={s.metrics.enabled ? <Badge tone="good">enabled</Badge> : <Badge tone="warning">disabled</Badge>} />
+            <Row k="Last flush" v={`${relative(s.metrics.lastFlush)} · ${s.metrics.lastFlushMs} ms`} />
+            {s.metrics.lastFlushError && <Row k="Flush error" v={<span className="text-critical">{s.metrics.lastFlushError}</span>} />}
+            <Row k="Last rollup" v={relative(s.metrics.lastRollup)} />
+            <Row k="Rolled up until" v={dateTime(s.metrics.rolledUntil)} />
+            <Row k="Last retention run" v={relative(s.metrics.lastRetention)} />
+            <Row k="Unflushed buckets" v={int(s.metrics.openPoints)} />
+            <Row k="Series in memory" v={int(s.metrics.liveSeries)} />
+            <Row k="Dropped points" v={int(s.metrics.droppedPoints)} />
+          </dl>
+        </Card>
+
+        <Card title="Configuration" subtitle="Effective settings (secrets are never shown)">
+          <dl className="space-y-2 text-[13px]">
+            {Object.entries(s.config).map(([k, v]) => (
+              <Row
+                key={k}
+                k={k}
+                v={
+                  typeof v === "boolean" ? (
+                    <Badge tone={v ? "good" : k === "adminSecretStrong" || k === "secretKeySet" ? "critical" : "neutral"}>{v ? "yes" : "no"}</Badge>
+                  ) : Array.isArray(v) ? (
+                    v.join(", ") || "—"
+                  ) : v === null || v === "" ? (
+                    "—"
+                  ) : (
+                    String(v)
+                  )
+                }
+              />
+            ))}
+          </dl>
+        </Card>
+
+        <Card title="Maintenance">
+          <div className="space-y-4 text-[13px]">
+            <Action
+              icon={<Gauge className="size-4" />}
+              title="Flush metrics now"
+              text="Writes all buffered metric buckets to the database."
+              button={<Button size="sm" onClick={() => run.mutate({ action: "flush-metrics" })}>Flush</Button>}
+            />
+            <Action
+              icon={<Database className="size-4" />}
+              title="Roll up & apply retention"
+              text="Builds hourly rollups and deletes data past the retention windows."
+              button={<Button size="sm" onClick={() => run.mutate({ action: "rollup" })}>Run</Button>}
+            />
+            <Action
+              icon={<Eraser className="size-4" />}
+              title="Purge replaced file rows"
+              text={`Every re-index soft-deletes the previous file rows (${int(db?.filesSoftDeleted ?? 0)} now). Permanently remove rows replaced more than N days ago.`}
+              button={
+                <div className="flex items-center gap-1.5">
+                  <Input value={days} onChange={(e) => setDays(e.target.value.replace(/\D/g, ""))} className="h-7 w-14 text-xs" aria-label="Days" />
+                  <Button
+                    size="sm"
+                    variant="danger"
+                    onClick={async () => {
+                      if (
+                        await confirm({
+                          title: "Purge old file rows?",
+                          body: `Permanently deletes soft-deleted addon_files rows older than ${days} days. Runs in the background in batches.`,
+                          danger: true,
+                          confirmLabel: "Purge",
+                        })
+                      )
+                        run.mutate({ action: "purge-deleted-files", olderThanDays: Number(days) || 7 });
+                    }}
+                  >
+                    Purge
+                  </Button>
+                </div>
+              }
+            />
+          </div>
+        </Card>
+      </div>
+    </div>
+  );
+}
+
+function Row({ k, v }: { k: string; v: React.ReactNode }) {
+  return (
+    <div className="flex items-center justify-between gap-3">
+      <dt className="truncate text-ink-2">{k}</dt>
+      <dd className="tnum shrink-0 text-right text-ink">{v}</dd>
+    </div>
+  );
+}
+
+function Action({ icon, title, text, button }: { icon: React.ReactNode; title: string; text: string; button: React.ReactNode }) {
+  return (
+    <div className="flex items-start gap-3">
+      <div className="mt-0.5 text-muted">{icon}</div>
+      <div className="min-w-0 flex-1">
+        <div className="text-ink">{title}</div>
+        <div className="mt-0.5 text-xs text-ink-2">{text}</div>
+      </div>
+      <div className="shrink-0">{button}</div>
+    </div>
+  );
+}

web/src/pages/Traffic.tsx

@@ -0,0 +1,156 @@
+import { useMemo, useState } from "react";
+import { useQuery, keepPreviousData } from "@tanstack/react-query";
+import { api, qs } from "../lib/api";
+import { rangeSeconds, useRefetch, useSettings } from "../lib/settings";
+import type { RouteStat, TrafficData } from "../lib/types";
+import { compact, int, ms, num, pct, relative } from "../lib/format";
+import { Badge, Card, ErrorBox, Loading, PageHeader, Segmented, StatTile, Table, Td, Th } from "../components/ui";
+import { fromQuery, TimeChart, type ChartData } from "../components/charts/TimeChart";
+import { BarList } from "../components/charts/BarList";
+
+const GROUP_SLOT: Record<string, number> = { v1: 1, back: 2, admin: 3, metrics: 4, other: 5 };
+const GROUP_LABEL: Record<string, string> = { v1: "Website API", back: "Workers", admin: "Admin", metrics: "Prometheus", other: "Other" };
+
+type SortKey = "count" | "errors" | "p50" | "p95" | "p99" | "max";
+
+export default function Traffic() {
+  const { range } = useSettings();
+  const refetchInterval = useRefetch();
+  const [groupFilter, setGroupFilter] = useState<string>("all");
+  const [sort, setSort] = useState<SortKey>("count");
+
+  const q = useQuery({
+    queryKey: ["traffic", range],
+    refetchInterval,
+    placeholderData: keepPreviousData,
+    queryFn: () => {
+      const to = Math.floor(Date.now() / 1000);
+      return api<TrafficData>("/traffic" + qs({ from: to - rangeSeconds(range), to }));
+    },
+  });
+
+  const latency = useMemo<ChartData>(() => {
+    if (!q.data) return { x: [], series: [] };
+    const parts = (["p50", "p95", "p99"] as const).map((k) => fromQuery(q.data!.latency[k], () => k));
+    return { x: parts[0].x, series: parts.flatMap((p, i) => p.series.map((s) => ({ ...s, slot: i + 1 }))) };
+  }, [q.data]);
+
+  const errors = useMemo<ChartData>(() => {
+    if (!q.data) return { x: [], series: [] };
+    const e5 = fromQuery(q.data.errors5xx, (l) => `5xx · ${GROUP_LABEL[l.group] ?? l.group}`, 60);
+    const e4 = fromQuery(q.data.errors4xx, () => "4xx · Website API", 60);
+    const x = e5.x.length ? e5.x : e4.x;
+    return { x, series: [...e5.series, ...e4.series].map((s, i) => ({ ...s, slot: i + 1 })) };
+  }, [q.data]);
+
+  if (q.isLoading) return <Loading />;
+  if (q.error) return <ErrorBox error={q.error} />;
+  const t = q.data!;
+  const totalReq = (t.groupTotals ?? []).reduce((s, g) => s + g.count, 0);
+  const status = Object.fromEntries((t.statusTotals ?? []).map((s) => [s.labels.status, s.count]));
+  const web = t.website;
+  const visitors = fromQuery(t.visitorsDaily, (l) => (l.kind === "human" ? "Visitors" : "Bots & scripts"), 1, (l) => (l.kind === "human" ? 1 : 2));
+
+  const routes = t.routes
+    .filter((r) => groupFilter === "all" || r.group === groupFilter)
+    .sort((a, b) => ((b[sort] as number | null) ?? -1) - ((a[sort] as number | null) ?? -1));
+
+  return (
+    <div className="space-y-4">
+      <PageHeader title="Traffic & performance" description={<>Request rates, latency and errors for the API · updated {relative(t.asOf)}</>} />
+
+      <div className="grid grid-cols-1 gap-3 sm:grid-cols-2 xl:grid-cols-4">
+        <StatTile label="Requests in range" value={compact(totalReq)} sub={`${compact(web?.count ?? 0)} from the website API`} />
+        <StatTile label="Website latency p95" value={ms(web?.p95)} sub={`p50 ${ms(web?.p50)} · p99 ${ms(web?.p99)}`} />
+        <StatTile
+          label="Error rate"
+          value={pct(status["5xx"] ?? 0, totalReq, 2)}
+          sub={`${int(status["5xx"] ?? 0)} server · ${int(status["4xx"] ?? 0)} client errors`}
+        />
+        <StatTile label="Visitors today" value={int(t.visitorsToday.human)} sub={`${int(t.visitorsToday.bot)} bots & scripts · approx.`} />
+      </div>
+
+      <div className="grid grid-cols-1 gap-4 xl:grid-cols-2">
+        <Card title="Requests per minute" subtitle="By caller">
+          <TimeChart data={fromQuery(t.requestsByGroup, (l) => GROUP_LABEL[l.group] ?? l.group, 60, (l) => GROUP_SLOT[l.group])} format={(v) => num(v, 1)} />
+        </Card>
+        <Card title="Website API latency" subtitle="Percentiles per interval, milliseconds">
+          <TimeChart data={latency} format={(v) => ms(v)} />
+        </Card>
+        <Card title="Errors per minute" subtitle="Server errors by caller, client errors on the website API">
+          <TimeChart data={errors} format={(v) => num(v, 2)} empty="No errors in this range" />
+        </Card>
+        <Card title="Unique visitors per day" subtitle="Last 30 days · salted hash of IP + user agent, approximate">
+          <TimeChart data={visitors} bars format={compact} />
+        </Card>
+      </div>
+
+      <div className="grid grid-cols-1 gap-4 md:grid-cols-3">
+        <Card title="Status codes">
+          <BarList items={(t.statusTotals ?? []).map((s) => ({ key: s.labels.status, label: s.labels.status, value: s.count }))} />
+        </Card>
+        <Card title="Callers">
+          <BarList items={(t.groupTotals ?? []).map((g) => ({ key: g.labels.group, label: GROUP_LABEL[g.labels.group] ?? g.labels.group, value: g.count }))} />
+        </Card>
+        <Card title="Website API clients" subtitle="Browser (Origin/Referer set) vs direct calls">
+          <BarList items={(t.clientTotals ?? []).map((g) => ({ key: g.labels.client, label: g.labels.client === "web" ? "Browser" : "Direct / scripts", value: g.count }))} />
+        </Card>
+      </div>
+
+      <Card
+        title="Routes"
+        subtitle="Latency in milliseconds over the selected range"
+        actions={
+          <Segmented
+            size="sm"
+            options={[
+              { key: "all", label: "All" },
+              { key: "v1", label: "Website" },
+              { key: "back", label: "Workers" },
+              { key: "admin", label: "Admin" },
+            ]}
+            value={groupFilter}
+            onChange={setGroupFilter}
+          />
+        }
+      >
+        <Table>
+          <thead>
+            <tr>
+              <Th>Route</Th>
+              <Th>Caller</Th>
+              {(["count", "errors", "p50", "p95", "p99", "max"] as SortKey[]).map((k) => (
+                <Th key={k} className="text-right" onClick={() => setSort(k)}>
+                  {k === "count" ? "Requests" : k === "errors" ? "5xx" : k}
+                  {sort === k ? " ↓" : ""}
+                </Th>
+              ))}
+            </tr>
+          </thead>
+          <tbody>
+            {routes.map((r: RouteStat) => (
+              <tr key={`${r.method} ${r.route} ${r.group}`} className="hover:bg-surface-2/40">
+                <Td className="font-mono text-xs text-ink">
+                  <span className="mr-2 inline-block w-12 text-muted">{r.method}</span>
+                  {r.route}
+                </Td>
+                <Td><Badge tone={r.group === "v1" ? "accent" : "neutral"}>{GROUP_LABEL[r.group] ?? r.group}</Badge></Td>
+                <Td className="tnum text-right text-ink">{int(r.count)}</Td>
+                <Td className="tnum text-right">{r.errors ? <span className="text-critical">{int(r.errors)}</span> : "0"}</Td>
+                <Td className="tnum text-right">{ms(r.p50)}</Td>
+                <Td className="tnum text-right">{ms(r.p95)}</Td>
+                <Td className="tnum text-right">{ms(r.p99)}</Td>
+                <Td className="tnum text-right">{ms(r.max)}</Td>
+              </tr>
+            ))}
+            {routes.length === 0 && (
+              <tr>
+                <Td className="py-8 text-center text-muted">No requests in this range</Td>
+              </tr>
+            )}
+          </tbody>
+        </Table>
+      </Card>
+    </div>
+  );
+}

web/src/pages/Whitelist.tsx

@@ -0,0 +1,315 @@
+import { useState } from "react";
+import { useMutation, useQuery, useQueryClient, keepPreviousData } from "@tanstack/react-query";
+import { useNavigate } from "react-router";
+import { Plus, Search, ShieldCheck, Trash2 } from "lucide-react";
+import { api, qs } from "../lib/api";
+import type { DupHash, Paged, WhitelistRow } from "../lib/types";
+import { int, relative, shortHash } from "../lib/format";
+import {
+  Badge,
+  Button,
+  Card,
+  Checkbox,
+  CopyButton,
+  Empty,
+  ErrorBox,
+  Input,
+  Loading,
+  PageHeader,
+  Pager,
+  Segmented,
+  Table,
+  Td,
+  Textarea,
+  Th,
+  useFeedback,
+} from "../components/ui";
+
+interface HashUsage {
+  hash: string;
+  whitelisted: boolean;
+  total: number;
+  items: { addonId: string; name: string; path: string; version: string }[];
+}
+
+export default function Whitelist() {
+  const [tab, setTab] = useState<"list" | "candidates">("list");
+  const [inspect, setInspect] = useState<string | null>(null);
+  return (
+    <div className="space-y-4">
+      <PageHeader
+        title="Whitelisted hashes"
+        description="Files whose hash is whitelisted are ignored by duplicate detection for every addon — use it for shared engine or template files."
+        actions={
+          <Segmented
+            options={[
+              { key: "list", label: "Whitelist" },
+              { key: "candidates", label: "Most shared hashes" },
+            ]}
+            value={tab}
+            onChange={setTab}
+          />
+        }
+      />
+      {tab === "list" ? <List onInspect={setInspect} /> : <Candidates onInspect={setInspect} />}
+      {inspect && <UsageDialog hash={inspect} onClose={() => setInspect(null)} />}
+    </div>
+  );
+}
+
+function List({ onInspect }: { onInspect: (h: string) => void }) {
+  const qc = useQueryClient();
+  const { toast, confirm } = useFeedback();
+  const [q, setQ] = useState("");
+  const [draft, setDraft] = useState("");
+  const [offset, setOffset] = useState(0);
+  const [text, setText] = useState("");
+  const [note, setNote] = useState("");
+  const limit = 50;
+
+  const list = useQuery({
+    queryKey: ["whitelist", q, offset],
+    queryFn: () => api<Paged<WhitelistRow>>("/whitelist" + qs({ q, offset, limit })),
+    placeholderData: keepPreviousData,
+  });
+
+  const add = useMutation({
+    mutationFn: () => api<{ added: number; skipped: { hash: string; reason: string }[] }>("/whitelist", { method: "POST", body: { text, note } }),
+    onSuccess: (r) => {
+      const reasons = r.skipped.length ? ` · ${r.skipped.length} skipped (${[...new Set(r.skipped.map((s) => s.reason))].join(", ")})` : "";
+      toast(`${r.added} hash${r.added === 1 ? "" : "es"} added${reasons}`, r.added ? "good" : "neutral");
+      if (r.added) {
+        setText("");
+        setNote("");
+      }
+      qc.invalidateQueries({ queryKey: ["whitelist"] });
+      qc.invalidateQueries({ queryKey: ["dups"] });
+    },
+    onError: (e) => toast((e as Error).message, "critical"),
+  });
+
+  const remove = useMutation({
+    mutationFn: (id: number) => api(`/whitelist/${id}`, { method: "DELETE" }),
+    onSuccess: () => {
+      toast("Removed from whitelist");
+      qc.invalidateQueries({ queryKey: ["whitelist"] });
+    },
+    onError: (e) => toast((e as Error).message, "critical"),
+  });
+
+  const count = text.split(/[\s,;]+/).filter(Boolean).length;
+
+  return (
+    <div className="grid grid-cols-1 gap-4 xl:grid-cols-3">
+      <Card title="Add hashes" subtitle="SHA-512 hex, one per line (or separated by commas/spaces)" className="xl:order-2">
+        <Textarea rows={8} value={text} onChange={(e) => setText(e.target.value)} placeholder="9f86d081884c7d659a2feaa0c55ad015…" className="w-full" />
+        <Input value={note} onChange={(e) => setNote(e.target.value)} placeholder="Note (optional) — e.g. vanilla UI layout" className="mt-2 w-full" />
+        <Button variant="primary" className="mt-3 w-full" icon={<Plus className="size-4" />} disabled={!count} loading={add.isPending} onClick={() => add.mutate()}>
+          Add {count ? int(count) : ""} hash{count === 1 ? "" : "es"}
+        </Button>
+      </Card>
+
+      <Card bodyClass="p-0" className="xl:col-span-2">
+        <form
+          className="relative border-b border-line p-3"
+          onSubmit={(e) => {
+            e.preventDefault();
+            setOffset(0);
+            setQ(draft.trim());
+          }}
+        >
+          <Search className="pointer-events-none absolute top-1/2 left-5.5 size-3.5 -translate-y-1/2 text-muted" />
+          <Input value={draft} onChange={(e) => setDraft(e.target.value)} placeholder="Search by hash prefix or note…" className="w-full pl-8" />
+        </form>
+        {list.isLoading ? (
+          <div className="px-4"><Loading /></div>
+        ) : list.error ? (
+          <div className="p-4"><ErrorBox error={list.error} /></div>
+        ) : list.data!.items.length === 0 ? (
+          <Empty>{q ? "No matching hashes" : "No whitelisted hashes yet"}</Empty>
+        ) : (
+          <div className="px-4">
+            <Table>
+              <thead>
+                <tr>
+                  <Th>Hash</Th>
+                  <Th>Example file</Th>
+                  <Th className="text-right">Used by</Th>
+                  <Th>Added</Th>
+                  <Th />
+                </tr>
+              </thead>
+              <tbody>
+                {list.data!.items.map((w) => (
+                  <tr key={w.id} className="hover:bg-surface-2/40">
+                    <Td className="whitespace-nowrap">
+                      <button className="font-mono text-xs text-ink hover:text-accent" onClick={() => onInspect(w.hash)}>
+                        {shortHash(w.hash, 10)}
+                      </button>
+                      <CopyButton value={w.hash} />
+                      {w.note && <div className="max-w-[240px] truncate text-xs text-muted" title={w.note}>{w.note}</div>}
+                    </Td>
+                    <Td className="max-w-[260px] truncate font-mono text-xs" title={w.samplePath}>{w.samplePath || "—"}</Td>
+                    <Td className="tnum whitespace-nowrap text-right">
+                      {int(w.addons)} addons
+                      <div className="text-xs text-muted">{int(w.files)} files</div>
+                    </Td>
+                    <Td className="whitespace-nowrap">{relative(w.createdAt)}</Td>
+                    <Td className="text-right">
+                      <Button
+                        size="sm"
+                        variant="ghost"
+                        icon={<Trash2 className="size-3.5" />}
+                        aria-label="Remove"
+                        onClick={async () => {
+                          if (await confirm({ title: "Remove from whitelist?", body: "Files with this hash will count as duplicates again.", danger: true, confirmLabel: "Remove" }))
+                            remove.mutate(w.id);
+                        }}
+                      />
+                    </Td>
+                  </tr>
+                ))}
+              </tbody>
+            </Table>
+          </div>
+        )}
+        {list.data && (
+          <div className="border-t border-line px-4 py-2.5">
+            <Pager total={list.data.total} limit={limit} offset={offset} onChange={setOffset} />
+          </div>
+        )}
+      </Card>
+    </div>
+  );
+}
+
+function Candidates({ onInspect }: { onInspect: (h: string) => void }) {
+  const qc = useQueryClient();
+  const { toast } = useFeedback();
+  const [hide, setHide] = useState(true);
+  const [selected, setSelected] = useState<Set<string>>(new Set());
+  const dups = useQuery({
+    queryKey: ["dups", hide],
+    queryFn: () => api<{ asOf: number; items: DupHash[]; pending?: boolean }>(`/overview/duplicates${hide ? "?hideWhitelisted=1" : ""}`),
+  });
+  const add = useMutation({
+    mutationFn: (hashes: string[]) => api<{ added: number }>("/whitelist", { method: "POST", body: { hashes, note: "from most shared hashes" } }),
+    onSuccess: (r) => {
+      toast(`${r.added} hash${r.added === 1 ? "" : "es"} whitelisted`);
+      setSelected(new Set());
+      qc.invalidateQueries({ queryKey: ["dups"] });
+      qc.invalidateQueries({ queryKey: ["whitelist"] });
+    },
+    onError: (e) => toast((e as Error).message, "critical"),
+  });
+
+  if (dups.isLoading) return <Loading />;
+  if (dups.error) return <ErrorBox error={dups.error} />;
+  const items = dups.data!.items;
+
+  return (
+    <Card
+      title="Hashes shared by the most addons"
+      subtitle={dups.data!.pending ? "Statistics are being computed…" : `Computed ${relative(dups.data!.asOf)} · refreshed hourly`}
+      actions={
+        <>
+          <label className="flex items-center gap-1.5 text-xs text-ink-2">
+            <Checkbox checked={hide} onChange={setHide} label="Hide whitelisted" /> Hide whitelisted
+          </label>
+          <Button size="sm" variant="primary" icon={<ShieldCheck className="size-3.5" />} disabled={selected.size === 0} loading={add.isPending} onClick={() => add.mutate([...selected])}>
+            Whitelist {selected.size || ""}
+          </Button>
+        </>
+      }
+    >
+      {items.length === 0 ? (
+        <Empty>No shared hashes</Empty>
+      ) : (
+        <Table>
+          <thead>
+            <tr>
+              <Th className="w-8" />
+              <Th>Hash</Th>
+              <Th>Example file</Th>
+              <Th className="text-right">Addons</Th>
+              <Th className="text-right">Files</Th>
+              <Th />
+            </tr>
+          </thead>
+          <tbody>
+            {items.map((d) => (
+              <tr key={d.hash} className="hover:bg-surface-2/40">
+                <Td>
+                  {!d.whitelisted && (
+                    <Checkbox
+                      label="Select"
+                      checked={selected.has(d.hash)}
+                      onChange={(v) => {
+                        const next = new Set(selected);
+                        if (v) next.add(d.hash);
+                        else next.delete(d.hash);
+                        setSelected(next);
+                      }}
+                    />
+                  )}
+                </Td>
+                <Td className="whitespace-nowrap">
+                  <button className="font-mono text-xs text-ink hover:text-accent" onClick={() => onInspect(d.hash)}>
+                    {shortHash(d.hash, 10)}
+                  </button>
+                  <CopyButton value={d.hash} />
+                </Td>
+                <Td className="max-w-[340px] truncate font-mono text-xs" title={d.samplePath}>{d.samplePath}</Td>
+                <Td className="tnum text-right text-ink">{int(d.addonCount)}</Td>
+                <Td className="tnum text-right">{int(d.fileCount)}</Td>
+                <Td className="text-right">{d.whitelisted && <Badge tone="good">whitelisted</Badge>}</Td>
+              </tr>
+            ))}
+          </tbody>
+        </Table>
+      )}
+    </Card>
+  );
+}
+
+function UsageDialog({ hash, onClose }: { hash: string; onClose: () => void }) {
+  const navigate = useNavigate();
+  const usage = useQuery({ queryKey: ["hash", hash], queryFn: () => api<HashUsage>(`/hashes/${hash}`) });
+  return (
+    <div className="fixed inset-0 z-40 flex items-center justify-center p-4">
+      <div className="absolute inset-0 bg-black/50" onClick={onClose} />
+      <div className="relative max-h-[80vh] w-[min(720px,100%)] overflow-hidden rounded-xl border border-line bg-surface shadow-2xl">
+        <div className="flex items-start justify-between gap-3 border-b border-line px-5 py-4">
+          <div className="min-w-0">
+            <h3 className="font-medium text-ink">Files with this hash</h3>
+            <div className="mt-1 font-mono text-[11px] break-all text-muted">{hash}</div>
+          </div>
+          <Button size="sm" variant="ghost" onClick={onClose}>Close</Button>
+        </div>
+        <div className="max-h-[60vh] overflow-y-auto px-5 py-3">
+          {usage.isLoading ? (
+            <Loading />
+          ) : usage.error ? (
+            <ErrorBox error={usage.error} />
+          ) : (
+            <>
+              <div className="mb-2 flex items-center gap-2 text-xs text-ink-2">
+                {int(usage.data!.total)} files {usage.data!.whitelisted ? <Badge tone="good">whitelisted</Badge> : <Badge>not whitelisted</Badge>}
+              </div>
+              <Table>
+                <tbody>
+                  {usage.data!.items.map((u, i) => (
+                    <tr key={u.addonId + i} className="cursor-pointer hover:bg-surface-2/40" onClick={() => navigate(`/addons/${u.addonId}`)}>
+                      <Td className="text-ink">{u.name || u.addonId}</Td>
+                      <Td className="max-w-[320px] truncate font-mono text-xs" title={u.path}>{u.path}</Td>
+                    </tr>
+                  ))}
+                </tbody>
+              </Table>
+            </>
+          )}
+        </div>
+      </div>
+    </div>
+  );
+}

web/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "jsx": "react-jsx",
+    "strict": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noFallthroughCasesInSwitch": true,
+    "skipLibCheck": true,
+    "isolatedModules": true,
+    "resolveJsonModule": true,
+    "noEmit": true,
+    "types": ["vite/client"],
+    "tsBuildInfoFile": "node_modules/.tmp/tsconfig.tsbuildinfo"
+  },
+  "include": ["src"]
+}

web/vite.config.ts

@@ -0,0 +1,22 @@
+import { defineConfig } from "vite";
+import react from "@vitejs/plugin-react";
+import tailwindcss from "@tailwindcss/vite";
+
+// The build is embedded into the Go binary (web/embed.go) and served at /admin/ui/.
+// In dev, run the Go server on :8083 and `npm run dev`, then open
+// http://localhost:5173/admin/ui/
+export default defineConfig({
+  base: "/admin/ui/",
+  plugins: [react(), tailwindcss()],
+  build: {
+    outDir: "dist",
+    // keep dist/.gitkeep so go:embed always has a directory
+    emptyOutDir: false,
+    chunkSizeWarningLimit: 600,
+  },
+  server: {
+    proxy: {
+      "/admin/api": "http://localhost:8083",
+    },
+  },
+});