admin dashboard, metrics and telemetry

- embedded React admin UI at /admin/ui (go:embed), login with ADMIN_SECRET
  via signed HttpOnly session cookie, X-ADMIN-KEY still works for scripts
- admin API: overview stats, indexing queue, addon management (priority,
  reindex, delete/restore/purge, bulk), whitelist CRUD, duplicate hashes,
  traffic, metrics explorer, services, events/audit log, system info
- built-in metrics: request latency per route, visitors, top addons and
  searches, indexer/scraper counters; minute buckets in the DB with hourly
  rollups and retention; optional Prometheus /metrics (METRICS_TOKEN)
- POST /back/telemetry for indexers/scrapers (metrics, events, heartbeat)
  and a stdlib-only Go client in telemetry/
- auth hardening: empty secrets no longer allow access, constant-time
  compare, CORS not applied to /admin and /back, TRUSTED_PROXIES setting
- Dockerfile builds the UI and no longer bakes config.yaml into the image;
  CI builds the UI, runs tests and builds arm64 with CGO

admin/addons.go

@@ -0,0 +1,510 @@
+package admin
+
+import (
+	"errors"
+	"net/http"
+	"strings"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/controllers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
+	"github.com/gin-gonic/gin"
+	"gorm.io/gorm"
+)
+
+// addonRow is the admin view of an addon (no files).
+type addonRow struct {
+	ID                   string     `json:"id"`
+	Name                 string     `json:"name"`
+	Type                 string     `json:"type"`
+	Author               string     `json:"author"`
+	Preview              string     `json:"preview"`
+	SubscriberCount      int64      `json:"subscriberCount"`
+	CurrentVersionNumber string     `json:"currentVersionNumber"`
+	CurrentVersionSize   int64      `json:"currentVersionSize"`
+	Unlisted             bool       `json:"unlisted"`
+	Private              bool       `json:"private"`
+	Blocked              bool       `json:"blocked"`
+	ToBeIndexed          bool       `json:"toBeIndexed"`
+	IsBeingIndexed       bool       `json:"isBeingIndexed"`
+	PriorityIndexing     bool       `json:"priorityIndexing"`
+	Recheck              bool       `json:"recheck"`
+	IndexStartTime       *time.Time `json:"indexStartTime"`
+	LastIndexedAt        *time.Time `json:"lastIndexedAt"`
+	CreatedAt            time.Time  `json:"createdAt"`
+	UpdatedAt            time.Time  `json:"updatedAt"`
+	DeletedAt            *time.Time `json:"deletedAt"`
+}
+
+const addonColumns = `id, name, type, author, preview, subscriber_count, current_version_number,
+	current_version_size, unlisted, private, blocked, to_be_indexed, is_being_indexed, priority_indexing,
+	recheck, index_start_time, last_indexed_at, created_at, updated_at, deleted_at`
+
+var addonSorts = map[string]string{
+	"name":        "name",
+	"subscribers": "subscriber_count",
+	"size":        "current_version_size",
+	"created":     "created_at",
+	"updated":     "updated_at",
+	"indexed":     "last_indexed_at",
+	"deleted":     "deleted_at",
+}
+
+func listAddons(c *gin.Context) {
+	limit := queryInt(c, "limit", 50, 1, 200)
+	offset := queryInt(c, "offset", 0, 0, 1<<30)
+	status := c.DefaultQuery("status", "all")
+
+	q := initializers.DB.Model(&models.Addon{})
+	switch status {
+	case "all":
+	case "indexed":
+		q = q.Where("to_be_indexed = ?", false)
+	case "queued":
+		q = q.Where("to_be_indexed = ? AND is_being_indexed = ?", true, false)
+	case "indexing":
+		q = q.Where("is_being_indexed = ?", true)
+	case "priority":
+		q = q.Where("priority_indexing = ?", true)
+	case "recheck":
+		q = q.Where("recheck = ?", true)
+	case "blocked":
+		q = q.Where("blocked = ?", true)
+	case "unlisted":
+		q = q.Where("unlisted = ?", true)
+	case "private":
+		q = q.Where("private = ?", true)
+	case "deleted":
+		q = q.Unscoped().Where("deleted_at IS NOT NULL")
+	default:
+		fail(c, http.StatusBadRequest, "unknown status")
+		return
+	}
+	if t := c.Query("type"); t != "" {
+		q = q.Where("type = ?", t)
+	}
+	if s := strings.TrimSpace(c.Query("q")); s != "" {
+		like := "%" + likeEscape(strings.ToLower(s)) + "%"
+		q = q.Where(`id = ? OR LOWER(name) LIKE ? ESCAPE '\' OR LOWER(author) LIKE ? ESCAPE '\'`, s, like, like)
+	}
+
+	var total int64
+	if err := q.Session(&gorm.Session{}).Count(&total).Error; err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+
+	sortCol, ok := addonSorts[c.DefaultQuery("sort", "subscribers")]
+	if !ok {
+		sortCol = "subscriber_count"
+	}
+	dir := "DESC"
+	if c.Query("order") == "asc" {
+		dir = "ASC"
+	}
+	nulls := ""
+	if isPostgres() && dir == "DESC" {
+		nulls = " NULLS LAST"
+	}
+
+	rows := []addonRow{}
+	err := q.Select(addonColumns).Order(sortCol + " " + dir + nulls + ", id ASC").Limit(limit).Offset(offset).Scan(&rows).Error
+	if err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+	c.JSON(http.StatusOK, gin.H{"items": rows, "total": total, "limit": limit, "offset": offset})
+}
+
+type sharedAddon struct {
+	AddonID     string `json:"addonId"`
+	Name        string `json:"name"`
+	SharedFiles int64  `json:"sharedFiles"`
+}
+
+func addonDetail(c *gin.Context) {
+	id := c.Param("id")
+	var row addonRow
+	res := initializers.DB.Unscoped().Model(&models.Addon{}).Select(addonColumns).Where("id = ?", id).Limit(1).Scan(&row)
+	if res.Error != nil {
+		fail(c, http.StatusInternalServerError, res.Error.Error())
+		return
+	}
+	if res.RowsAffected == 0 {
+		fail(c, http.StatusNotFound, "addon not found")
+		return
+	}
+	var extra struct {
+		Summary          string
+		CurrentVersionID int64
+	}
+	initializers.DB.Unscoped().Model(&models.Addon{}).Select("summary, current_version_id").Where("id = ?", id).Limit(1).Scan(&extra)
+
+	var fileCount, deletedFiles int64
+	initializers.DB.Model(&models.AddonFile{}).Where("addon_id = ?", id).Count(&fileCount)
+	initializers.DB.Unscoped().Model(&models.AddonFile{}).Where("addon_id = ? AND deleted_at IS NOT NULL", id).Count(&deletedFiles)
+
+	deps, depsTotal, _ := controllers.QueryRelatedAddons(id, "ad.dependency_id", "ad.addon_id", 50, 0)
+	dependents, dependentsTotal, _ := controllers.QueryRelatedAddons(id, "ad.addon_id", "ad.dependency_id", 50, 0)
+
+	// declared dependencies that are not (or no longer) in the addons table
+	var missingDeps []struct {
+		DependencyID   string `json:"id"`
+		DependencyName string `json:"name"`
+		Version        string `json:"version"`
+	}
+	initializers.DB.Raw(`SELECT ad.dependency_id, ad.dependency_name, ad.version FROM addon_dependencies ad
+		LEFT JOIN addons a ON a.id = ad.dependency_id AND a.deleted_at IS NULL
+		WHERE ad.addon_id = ? AND a.id IS NULL LIMIT 50`, id).Scan(&missingDeps)
+
+	shared := []sharedAddon{}
+	err := initializers.DB.Raw(`SELECT f.addon_id AS addon_id, COALESCE(MAX(a.name), '') AS name, COUNT(*) AS shared_files
+		FROM addon_files f
+		LEFT JOIN addons a ON a.id = f.addon_id
+		WHERE f.deleted_at IS NULL AND f.addon_id <> ?
+		  AND f.hash IN (SELECT s.hash FROM addon_files s WHERE s.addon_id = ? AND s.deleted_at IS NULL)
+		  AND NOT EXISTS (SELECT 1 FROM whitelisted_hashes w WHERE w.hash = f.hash AND w.deleted_at IS NULL)
+		GROUP BY f.addon_id ORDER BY shared_files DESC LIMIT 25`, id, id).Scan(&shared).Error
+	if err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"addon":            row,
+		"summary":          extra.Summary,
+		"currentVersionId": extra.CurrentVersionID,
+		"fileCount":        fileCount,
+		"deletedFileRows":  deletedFiles,
+		"dependencies":     gin.H{"total": depsTotal, "items": deps, "missing": missingDeps},
+		"dependents":       gin.H{"total": dependentsTotal, "items": dependents},
+		"duplicates":       shared,
+	})
+}
+
+type fileRow struct {
+	ID          uint   `json:"id"`
+	Path        string `json:"path"`
+	Hash        string `json:"hash"`
+	Version     string `json:"version"`
+	Whitelisted bool   `json:"whitelisted"`
+	OtherAddons int64  `json:"otherAddons"`
+}
+
+func addonFiles(c *gin.Context) {
+	id := c.Param("id")
+	limit := queryInt(c, "limit", 100, 1, 500)
+	offset := queryInt(c, "offset", 0, 0, 1<<30)
+	q := initializers.DB.Model(&models.AddonFile{}).Where("addon_id = ?", id)
+	if s := strings.TrimSpace(c.Query("q")); s != "" {
+		like := "%" + likeEscape(strings.ToLower(s)) + "%"
+		q = q.Where(`LOWER(path) LIKE ? ESCAPE '\' OR hash = ?`, like, strings.ToLower(s))
+	}
+	var total int64
+	q.Session(&gorm.Session{}).Count(&total)
+
+	rows := []fileRow{}
+	if err := q.Select("id, path, hash, version").Order("path ASC").Limit(limit).Offset(offset).Scan(&rows).Error; err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+
+	if len(rows) > 0 && c.Query("shared") != "0" {
+		hashes := make([]string, 0, len(rows))
+		for _, r := range rows {
+			hashes = append(hashes, r.Hash)
+		}
+		var wl []string
+		initializers.DB.Model(&models.WhitelistedHash{}).Where("hash IN ?", hashes).Pluck("hash", &wl)
+		wlSet := map[string]bool{}
+		for _, h := range wl {
+			wlSet[h] = true
+		}
+		var counts []struct {
+			Hash   string
+			Addons int64
+		}
+		initializers.DB.Raw(`SELECT hash, COUNT(DISTINCT addon_id) AS addons FROM addon_files
+			WHERE deleted_at IS NULL AND hash IN ? GROUP BY hash`, hashes).Scan(&counts)
+		cnt := map[string]int64{}
+		for _, x := range counts {
+			cnt[x.Hash] = x.Addons
+		}
+		for i := range rows {
+			rows[i].Whitelisted = wlSet[rows[i].Hash]
+			rows[i].OtherAddons = max(0, cnt[rows[i].Hash]-1)
+		}
+	}
+	c.JSON(http.StatusOK, gin.H{"items": rows, "total": total, "limit": limit, "offset": offset})
+}
+
+var errNotFound = errors.New("addon not found")
+
+// applyAction changes queue flags of the given addons. UpdateColumns is used
+// so updated_at (the queue order) is left alone.
+func applyAction(tx *gorm.DB, ids []string, action string, value bool) (int64, error) {
+	base := tx.Model(&models.Addon{}).Where("id IN ?", ids)
+	var res *gorm.DB
+	switch action {
+	case "priority":
+		cols := map[string]any{"priority_indexing": value}
+		if value {
+			cols["to_be_indexed"] = true
+		}
+		res = base.UpdateColumns(cols)
+	case "to-be-indexed":
+		res = base.UpdateColumns(map[string]any{"to_be_indexed": value})
+	case "reindex":
+		cols := map[string]any{"to_be_indexed": true, "is_being_indexed": false}
+		if value {
+			cols["priority_indexing"] = true
+		}
+		res = base.UpdateColumns(cols)
+	case "reset-indexing":
+		res = base.UpdateColumns(map[string]any{"is_being_indexed": false})
+	case "recheck":
+		res = base.UpdateColumns(map[string]any{"recheck": value})
+	case "restore":
+		res = tx.Unscoped().Model(&models.Addon{}).Where("id IN ? AND deleted_at IS NOT NULL", ids).
+			UpdateColumns(map[string]any{"deleted_at": nil, "to_be_indexed": true, "is_being_indexed": false})
+	case "delete":
+		var addons []models.Addon
+		if err := tx.Where("id IN ?", ids).Find(&addons).Error; err != nil {
+			return 0, err
+		}
+		for i := range addons {
+			if err := controllers.DeleteAddonCascade(tx, &addons[i]); err != nil {
+				return 0, err
+			}
+		}
+		return int64(len(addons)), nil
+	default:
+		return 0, errors.New("unknown action")
+	}
+	return res.RowsAffected, res.Error
+}
+
+func addonAction(c *gin.Context) {
+	id, action := c.Param("id"), c.Param("action")
+	var body struct {
+		Value *bool `json:"value"`
+	}
+	_ = c.ShouldBindJSON(&body)
+	value := body.Value == nil || *body.Value
+	if action == "delete" {
+		fail(c, http.StatusBadRequest, "use DELETE /addons/:id")
+		return
+	}
+	n, err := applyAction(initializers.DB, []string{id}, action, value)
+	if err != nil {
+		fail(c, http.StatusBadRequest, err.Error())
+		return
+	}
+	if n == 0 {
+		fail(c, http.StatusNotFound, "addon not found or nothing to change")
+		return
+	}
+	audit(c, "addon_"+strings.ReplaceAll(action, "-", "_"), map[string]any{"addon": id, "value": value})
+	invalidate("overview", "queue")
+	c.JSON(http.StatusOK, gin.H{"updated": n})
+}
+
+func deleteAddon(c *gin.Context) {
+	id := c.Param("id")
+	err := initializers.DB.Transaction(func(tx *gorm.DB) error {
+		var addon models.Addon
+		if err := tx.Where("id = ?", id).First(&addon).Error; err != nil {
+			return errNotFound
+		}
+		return controllers.DeleteAddonCascade(tx, &addon)
+	})
+	if errors.Is(err, errNotFound) {
+		fail(c, http.StatusNotFound, err.Error())
+		return
+	}
+	if err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+	audit(c, "addon_delete", map[string]any{"addon": id})
+	invalidate("overview", "queue")
+	c.JSON(http.StatusOK, gin.H{"deleted": id})
+}
+
+// purgeAddon permanently removes an already soft-deleted addon and its rows.
+func purgeAddon(c *gin.Context) {
+	id := c.Param("id")
+	if c.Query("confirm") != id {
+		fail(c, http.StatusBadRequest, "pass ?confirm=<addon id> to purge")
+		return
+	}
+	var files int64
+	err := initializers.DB.Transaction(func(tx *gorm.DB) error {
+		var addon models.Addon
+		if err := tx.Unscoped().Where("id = ? AND deleted_at IS NOT NULL", id).First(&addon).Error; err != nil {
+			return errNotFound
+		}
+		res := tx.Unscoped().Where("addon_id = ?", id).Delete(&models.AddonFile{})
+		if res.Error != nil {
+			return res.Error
+		}
+		files = res.RowsAffected
+		if err := tx.Where("addon_id = ?", id).Delete(&models.AddonDependency{}).Error; err != nil {
+			return err
+		}
+		return tx.Unscoped().Delete(&addon).Error
+	})
+	if errors.Is(err, errNotFound) {
+		fail(c, http.StatusNotFound, "addon not found or not deleted")
+		return
+	}
+	if err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+	audit(c, "addon_purge", map[string]any{"addon": id, "files": files})
+	invalidate("overview")
+	c.JSON(http.StatusOK, gin.H{"purged": id, "files": files})
+}
+
+func bulkAddons(c *gin.Context) {
+	var body struct {
+		IDs    []string `json:"ids"`
+		Action string   `json:"action"`
+		Value  *bool    `json:"value"`
+	}
+	if err := c.ShouldBindJSON(&body); err != nil || len(body.IDs) == 0 {
+		fail(c, http.StatusBadRequest, "ids and action are required")
+		return
+	}
+	if len(body.IDs) > 500 {
+		fail(c, http.StatusBadRequest, "at most 500 ids per request")
+		return
+	}
+	value := body.Value == nil || *body.Value
+	var n int64
+	err := initializers.DB.Transaction(func(tx *gorm.DB) error {
+		var err error
+		n, err = applyAction(tx, body.IDs, body.Action, value)
+		return err
+	})
+	if err != nil {
+		fail(c, http.StatusBadRequest, err.Error())
+		return
+	}
+	audit(c, "bulk_"+strings.ReplaceAll(body.Action, "-", "_"), map[string]any{"count": len(body.IDs), "updated": n, "value": value})
+	invalidate("overview", "queue")
+	c.JSON(http.StatusOK, gin.H{"updated": n})
+}
+
+// ---- queue ---------------------------------------------------------------
+
+type queueRow struct {
+	ID                 string     `json:"id"`
+	Name               string     `json:"name"`
+	CurrentVersionSize int64      `json:"currentVersionSize"`
+	SubscriberCount    int64      `json:"subscriberCount"`
+	PriorityIndexing   bool       `json:"priorityIndexing"`
+	Blocked            bool       `json:"blocked"`
+	IndexStartTime     *time.Time `json:"indexStartTime"`
+	UpdatedAt          time.Time  `json:"updatedAt"`
+	ElapsedSec         int64      `json:"elapsedSec"`
+	Stuck              bool       `json:"stuck"`
+}
+
+const queueColumns = "id, name, current_version_size, subscriber_count, priority_indexing, blocked, index_start_time, updated_at"
+
+func buildQueue() (gin.H, error) {
+	counts, err := metrics.LoadQueueCounts()
+	if err != nil {
+		return nil, err
+	}
+	var blockedQueued int64
+	initializers.DB.Model(&models.Addon{}).Where("to_be_indexed = ? AND blocked = ?", true, true).Count(&blockedQueued)
+
+	now := time.Now()
+	indexing := []queueRow{}
+	initializers.DB.Model(&models.Addon{}).Select(queueColumns).Where("is_being_indexed = ?", true).
+		Order("index_start_time ASC").Limit(100).Scan(&indexing)
+	for i := range indexing {
+		if t := indexing[i].IndexStartTime; t != nil && !t.IsZero() {
+			indexing[i].ElapsedSec = int64(now.Sub(*t).Seconds())
+			indexing[i].Stuck = now.Sub(*t) > initializers.IndexTimeout
+		}
+	}
+
+	priority := []queueRow{}
+	initializers.DB.Model(&models.Addon{}).Select(queueColumns).
+		Where("to_be_indexed = ? AND is_being_indexed = ? AND priority_indexing = ?", true, false, true).
+		Order("updated_at ASC").Limit(50).Scan(&priority)
+
+	// same order the indexers claim jobs in
+	next := []queueRow{}
+	initializers.DB.Model(&models.Addon{}).Select(queueColumns).
+		Where("to_be_indexed = ? AND is_being_indexed = ? AND blocked = ?", true, false, false).
+		Order("priority_indexing DESC, updated_at ASC").Limit(25).Scan(&next)
+
+	ts := now.Unix()
+	ok := map[string]string{"result": "ok"}
+	sel := metrics.Selector{Service: metrics.MainService, Name: "indexer_submissions_total", Match: ok}
+	perHour1h := metrics.Sum(sel, ts-3600, ts)
+	perHour6h := metrics.Sum(sel, ts-6*3600, ts) / 6
+	indexed24h := metrics.Sum(sel, ts-86400, ts)
+	var etaHours *float64
+	if perHour6h > 0 {
+		v := float64(counts.Queued) / perHour6h
+		etaHours = &v
+	}
+	duration, _ := metrics.Totals(metrics.Selector{Service: metrics.MainService, Name: "indexer_job_duration_seconds"}, ts-86400, ts)
+	var durationStats any
+	if len(duration) > 0 {
+		durationStats = duration[0]
+	}
+
+	return gin.H{
+		"counts":         counts,
+		"blockedQueued":  blockedQueued,
+		"timeoutMinutes": int(initializers.IndexTimeout.Minutes()),
+		"indexing":       indexing,
+		"priority":       priority,
+		"next":           next,
+		"throughput": gin.H{
+			"perHour1h":   perHour1h,
+			"perHour6h":   perHour6h,
+			"indexed24h":  indexed24h,
+			"etaHours":    etaHours,
+			"duration24h": durationStats,
+		},
+	}, nil
+}
+
+func queue(c *gin.Context) {
+	data, at, err := cached("queue", 5*time.Second, buildQueue)
+	if err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+	out := gin.H{"asOf": at.Unix()}
+	for k, v := range data {
+		out[k] = v
+	}
+	c.JSON(http.StatusOK, out)
+}
+
+func resetStuck(c *gin.Context) {
+	var body struct {
+		OlderThanMinutes int `json:"olderThanMinutes"`
+	}
+	_ = c.ShouldBindJSON(&body)
+	older := initializers.IndexTimeout
+	if body.OlderThanMinutes > 0 {
+		older = time.Duration(body.OlderThanMinutes) * time.Minute
+	}
+	n := controllers.ResetTimedOutIndexing(older)
+	metrics.IndexerTimeoutsReset.Add(float64(n))
+	audit(c, "queue_reset_stuck", map[string]any{"olderThanMinutes": older.Minutes(), "reset": n})
+	invalidate("queue", "overview")
+	c.JSON(http.StatusOK, gin.H{"reset": n})
+}

admin/authapi.go

@@ -0,0 +1,71 @@
+package admin
+
+import (
+	"net/http"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/auth"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
+	"github.com/gin-gonic/gin"
+)
+
+var limiter = auth.NewLoginLimiter(5, 10*time.Minute)
+
+func login(c *gin.Context) {
+	if initializers.AdminSecret == "" {
+		fail(c, http.StatusServiceUnavailable, "admin disabled: ADMIN_SECRET not set")
+		return
+	}
+	if c.GetHeader(auth.CSRFHeader) != "1" {
+		fail(c, http.StatusForbidden, "csrf check failed")
+		return
+	}
+	ip := c.ClientIP()
+	if limiter.Blocked(ip) {
+		fail(c, http.StatusTooManyRequests, "too many failed attempts, try again later")
+		return
+	}
+	var body struct {
+		Secret string `json:"secret"`
+	}
+	if err := c.ShouldBindJSON(&body); err != nil {
+		fail(c, http.StatusBadRequest, "invalid JSON")
+		return
+	}
+	if !auth.SecretEqual(body.Secret, initializers.AdminSecret) {
+		limiter.Fail(ip)
+		time.Sleep(300 * time.Millisecond)
+		metrics.AdminActions.Inc("login_failed")
+		metrics.Event("warn", "admin: login failed", map[string]any{"kind": "audit", "ip": ip})
+		fail(c, http.StatusUnauthorized, "invalid secret")
+		return
+	}
+	limiter.Reset(ip)
+	token, exp := auth.IssueToken(time.Now())
+	auth.SetSessionCookie(c, token, exp)
+	c.Set("adminAuth", "login")
+	audit(c, "login", nil)
+	c.JSON(http.StatusOK, gin.H{"expiresAt": exp.Unix()})
+}
+
+func logout(c *gin.Context) {
+	auth.ClearSessionCookie(c)
+	c.Status(http.StatusNoContent)
+}
+
+func me(c *gin.Context) {
+	method := c.GetString("adminAuth")
+	resp := gin.H{"method": method}
+	if v, ok := c.Get("adminSessionExp"); ok {
+		exp := v.(time.Time)
+		// sliding renewal: extend sessions that are about to expire
+		if time.Until(exp) < 24*time.Hour {
+			token, newExp := auth.IssueToken(time.Now())
+			auth.SetSessionCookie(c, token, newExp)
+			exp = newExp
+		}
+		resp["expiresAt"] = exp.Unix()
+	}
+	c.JSON(http.StatusOK, resp)
+}

admin/metricsapi.go

@@ -0,0 +1,259 @@
+package admin
+
+import (
+	"encoding/json"
+	"net/http"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
+	"github.com/gin-gonic/gin"
+)
+
+func metricsCatalog(c *gin.Context) {
+	service := c.Query("service")
+	data, _, err := cached("catalog:"+service, 15*time.Second, func() ([]metrics.CatalogEntry, error) {
+		return metrics.Catalog(service)
+	})
+	if err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+	c.JSON(http.StatusOK, gin.H{"items": data, "aggregations": metrics.Aggregations})
+}
+
+func selectorFrom(c *gin.Context) (metrics.Selector, bool) {
+	sel := metrics.Selector{Service: c.Query("service"), Name: c.Query("name")}
+	if m := c.Query("match"); m != "" {
+		if err := json.Unmarshal([]byte(m), &sel.Match); err != nil {
+			fail(c, http.StatusBadRequest, "match must be a JSON object of label values")
+			return sel, false
+		}
+	}
+	if g := c.Query("groupBy"); g != "" {
+		for _, part := range strings.Split(g, ",") {
+			if part = strings.TrimSpace(part); part != "" {
+				sel.GroupBy = append(sel.GroupBy, part)
+			}
+		}
+	}
+	if sel.Name == "" {
+		fail(c, http.StatusBadRequest, "name is required")
+		return sel, false
+	}
+	return sel, true
+}
+
+func metricsQuery(c *gin.Context) {
+	sel, ok := selectorFrom(c)
+	if !ok {
+		return
+	}
+	from, to := timeRange(c, time.Hour)
+	res, err := metrics.Query(sel, c.Query("agg"), from, to, queryInt64(c, "step", 0))
+	if err != nil {
+		fail(c, http.StatusBadRequest, err.Error())
+		return
+	}
+	c.JSON(http.StatusOK, res)
+}
+
+func autoStep(from, to int64) int64 {
+	span := to - from
+	switch {
+	case span <= 3*3600:
+		return 60
+	case span <= 12*3600:
+		return 300
+	case span <= 48*3600:
+		return 900
+	case span <= 8*86400:
+		return 3600
+	default:
+		return 6 * 3600
+	}
+}
+
+type routeStat struct {
+	Method  string   `json:"method"`
+	Route   string   `json:"route"`
+	Group   string   `json:"group"`
+	Count   int64    `json:"count"`
+	Errors  int64    `json:"errors"`
+	Client4 int64    `json:"clientErrors"`
+	P50     *float64 `json:"p50"`
+	P95     *float64 `json:"p95"`
+	P99     *float64 `json:"p99"`
+	Max     *float64 `json:"max"`
+	Avg     *float64 `json:"avg"`
+}
+
+// traffic returns pre-composed request, latency and error data for a range.
+func traffic(c *gin.Context) {
+	from, to := timeRange(c, 24*time.Hour)
+	step := queryInt64(c, "step", 0)
+	if step <= 0 {
+		step = autoStep(from, to)
+	}
+	key := "traffic:" + fmtInt64(from/30) + ":" + fmtInt64(to/30) + ":" + fmtInt64(step)
+	data, at, err := cached(key, 30*time.Second, func() (gin.H, error) {
+		return buildTraffic(from, to, step)
+	})
+	if err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+	out := gin.H{"asOf": at.Unix()}
+	for k, v := range data {
+		out[k] = v
+	}
+	c.JSON(http.StatusOK, out)
+}
+
+func fmtInt64(v int64) string { return strconv.FormatInt(v, 10) }
+
+func buildTraffic(from, to, step int64) (gin.H, error) {
+	const name = "http_request_duration_ms"
+	sel := func(match map[string]string, groupBy ...string) metrics.Selector {
+		return metrics.Selector{Service: metrics.MainService, Name: name, Match: match, GroupBy: groupBy}
+	}
+	v1 := map[string]string{"group": "v1"}
+
+	byGroup, err := metrics.Query(sel(nil, "group"), "rate", from, to, step)
+	if err != nil {
+		return nil, err
+	}
+	latency := gin.H{}
+	for _, q := range []string{"p50", "p95", "p99"} {
+		r, err := metrics.Query(sel(v1), q, from, to, step)
+		if err != nil {
+			return nil, err
+		}
+		latency[q] = r
+	}
+	errors5xx, _ := metrics.Query(sel(map[string]string{"status": "5xx"}, "group"), "rate", from, to, step)
+	errors4xx, _ := metrics.Query(sel(map[string]string{"status": "4xx", "group": "v1"}), "rate", from, to, step)
+
+	statusTotals, _ := metrics.Totals(sel(nil, "status"), from, to)
+	clientTotals, _ := metrics.Totals(sel(v1, "client"), from, to)
+	groupTotals, _ := metrics.Totals(sel(nil, "group"), from, to)
+
+	routeTotals, err := metrics.Totals(sel(nil, "method", "route", "group"), from, to)
+	if err != nil {
+		return nil, err
+	}
+	statusByRoute, _ := metrics.Totals(sel(nil, "method", "route", "status"), from, to)
+	errs := map[string][2]int64{}
+	for _, t := range statusByRoute {
+		k := t.Labels["method"] + " " + t.Labels["route"]
+		e := errs[k]
+		switch t.Labels["status"] {
+		case "5xx":
+			e[0] += t.Count
+		case "4xx":
+			e[1] += t.Count
+		}
+		errs[k] = e
+	}
+	routes := make([]routeStat, 0, len(routeTotals))
+	for _, t := range routeTotals {
+		k := t.Labels["method"] + " " + t.Labels["route"]
+		routes = append(routes, routeStat{
+			Method: t.Labels["method"], Route: t.Labels["route"], Group: t.Labels["group"],
+			Count: t.Count, Errors: errs[k][0], Client4: errs[k][1],
+			P50: t.P50, P95: t.P95, P99: t.P99, Max: t.Max, Avg: t.Avg,
+		})
+	}
+	sort.Slice(routes, func(i, j int) bool { return routes[i].Count > routes[j].Count })
+
+	now := time.Now().Unix()
+	visitorsDaily, _ := metrics.Query(metrics.Selector{Service: metrics.MainService, Name: "visitors_unique", GroupBy: []string{"kind"}},
+		"max", now-30*86400, now, 86400)
+	humans, bots := metrics.Visitors()
+
+	var v1Total *metrics.Total
+	if all, _ := metrics.Totals(sel(v1), from, to); len(all) > 0 {
+		v1Total = &all[0]
+	}
+
+	return gin.H{
+		"from": from, "to": to, "step": step,
+		"requestsByGroup": byGroup,
+		"latency":         latency,
+		"errors5xx":       errors5xx,
+		"errors4xx":       errors4xx,
+		"statusTotals":    statusTotals,
+		"clientTotals":    clientTotals,
+		"groupTotals":     groupTotals,
+		"website":         v1Total,
+		"routes":          routes,
+		"visitorsDaily":   visitorsDaily,
+		"visitorsToday":   gin.H{"human": humans, "bot": bots},
+	}, nil
+}
+
+type topEntry struct {
+	Item  string `json:"item"`
+	Count int64  `json:"count"`
+	Name  string `json:"name,omitempty"`
+}
+
+// topItems returns the most looked-up addons or search queries.
+func topItems(c *gin.Context) {
+	kind := c.DefaultQuery("kind", "addon")
+	if kind != "addon" && kind != "search" {
+		fail(c, http.StatusBadRequest, "kind must be addon or search")
+		return
+	}
+	days := queryInt(c, "days", 1, 1, 90)
+	limit := queryInt(c, "limit", 20, 1, 200)
+
+	today, counts := metrics.Top.Today(kind)
+	if today == "" {
+		today = time.Now().UTC().Format("2006-01-02")
+	}
+	if days > 1 {
+		since := time.Now().UTC().AddDate(0, 0, -(days - 1)).Format("2006-01-02")
+		var rows []models.DailyTopItem
+		initializers.DB.Where("kind = ? AND day >= ? AND day < ?", kind, since, today).Find(&rows)
+		for _, r := range rows {
+			counts[r.Item] += r.Count
+		}
+	}
+	list := make([]topEntry, 0, len(counts))
+	for k, v := range counts {
+		list = append(list, topEntry{Item: k, Count: v})
+	}
+	sort.Slice(list, func(i, j int) bool {
+		if list[i].Count != list[j].Count {
+			return list[i].Count > list[j].Count
+		}
+		return list[i].Item < list[j].Item
+	})
+	if len(list) > limit {
+		list = list[:limit]
+	}
+	if kind == "addon" && len(list) > 0 {
+		ids := make([]string, len(list))
+		for i, e := range list {
+			ids[i] = e.Item
+		}
+		var names []struct {
+			ID   string
+			Name string
+		}
+		initializers.DB.Unscoped().Model(&models.Addon{}).Select("id, name").Where("id IN ?", ids).Scan(&names)
+		byID := map[string]string{}
+		for _, n := range names {
+			byID[n.ID] = n.Name
+		}
+		for i := range list {
+			list[i].Name = byID[list[i].Item]
+		}
+	}
+	c.JSON(http.StatusOK, gin.H{"kind": kind, "days": days, "items": list})
+}

admin/overview.go

@@ -0,0 +1,443 @@
+package admin
+
+import (
+	"context"
+	"log"
+	"net/http"
+	"os"
+	"strconv"
+	"sync"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
+	"github.com/gin-gonic/gin"
+	"gorm.io/gorm"
+)
+
+type addonCounts struct {
+	Total      int64 `json:"total"`
+	Indexed    int64 `json:"indexed"`
+	Queued     int64 `json:"queued"`
+	Indexing   int64 `json:"indexing"`
+	Priority   int64 `json:"priority"`
+	Recheck    int64 `json:"recheck"`
+	Blocked    int64 `json:"blocked"`
+	Unlisted   int64 `json:"unlisted"`
+	Private    int64 `json:"private"`
+	TotalSize  int64 `json:"totalSizeBytes"`
+	Added24h   int64 `json:"added24h"`
+	Added7d    int64 `json:"added7d"`
+	Indexed24h int64 `json:"indexed24h"`
+
+	SoftDeleted int64 `json:"softDeleted"`
+}
+
+type typeCount struct {
+	Type  string `json:"type"`
+	Count int64  `json:"count"`
+}
+
+type miniAddon struct {
+	ID                 string     `json:"id"`
+	Name               string     `json:"name"`
+	Author             string     `json:"author"`
+	SubscriberCount    int64      `json:"subscriberCount"`
+	CurrentVersionSize int64      `json:"currentVersionSize"`
+	LastIndexedAt      *time.Time `json:"lastIndexedAt,omitempty"`
+	CreatedAt          time.Time  `json:"createdAt"`
+}
+
+type overviewData struct {
+	Addons          addonCounts    `json:"addons"`
+	Types           []typeCount    `json:"types"`
+	Whitelist       int64          `json:"whitelistCount"`
+	Dependencies    map[string]any `json:"dependencies"`
+	Traffic         map[string]any `json:"traffic"`
+	Activity        map[string]any `json:"activity"`
+	Services        map[string]any `json:"services"`
+	TopSubscribed   []miniAddon    `json:"topSubscribed"`
+	Largest         []miniAddon    `json:"largest"`
+	RecentlyAdded   []miniAddon    `json:"recentlyAdded"`
+	RecentlyIndexed []miniAddon    `json:"recentlyIndexed"`
+	Heavy           *heavyStats    `json:"heavy"`
+}
+
+func loadAddonCounts() (addonCounts, error) {
+	var ac addonCounts
+	now := time.Now()
+	err := initializers.DB.Raw(`SELECT COUNT(*) AS total,
+		COALESCE(SUM(CASE WHEN to_be_indexed = ? THEN 1 ELSE 0 END), 0) AS indexed,
+		COALESCE(SUM(CASE WHEN to_be_indexed = ? AND is_being_indexed = ? THEN 1 ELSE 0 END), 0) AS queued,
+		COALESCE(SUM(CASE WHEN is_being_indexed = ? THEN 1 ELSE 0 END), 0) AS indexing,
+		COALESCE(SUM(CASE WHEN priority_indexing = ? AND to_be_indexed = ? THEN 1 ELSE 0 END), 0) AS priority,
+		COALESCE(SUM(CASE WHEN recheck = ? THEN 1 ELSE 0 END), 0) AS recheck,
+		COALESCE(SUM(CASE WHEN blocked = ? THEN 1 ELSE 0 END), 0) AS blocked,
+		COALESCE(SUM(CASE WHEN unlisted = ? THEN 1 ELSE 0 END), 0) AS unlisted,
+		COALESCE(SUM(CASE WHEN private = ? THEN 1 ELSE 0 END), 0) AS private,
+		CAST(COALESCE(SUM(current_version_size), 0) AS BIGINT) AS total_size,
+		COALESCE(SUM(CASE WHEN created_at >= ? THEN 1 ELSE 0 END), 0) AS added24h,
+		COALESCE(SUM(CASE WHEN created_at >= ? THEN 1 ELSE 0 END), 0) AS added7d,
+		COALESCE(SUM(CASE WHEN last_indexed_at >= ? THEN 1 ELSE 0 END), 0) AS indexed24h
+		FROM addons WHERE deleted_at IS NULL`,
+		false, true, false, true, true, true, true, true, true, true,
+		now.Add(-24*time.Hour), now.Add(-7*24*time.Hour), now.Add(-24*time.Hour),
+	).Scan(&ac).Error
+	if err != nil {
+		return ac, err
+	}
+	err = initializers.DB.Unscoped().Model(&models.Addon{}).Where("deleted_at IS NOT NULL").Count(&ac.SoftDeleted).Error
+	return ac, err
+}
+
+func loadMiniAddons(order string, where string, args ...any) []miniAddon {
+	out := []miniAddon{}
+	q := initializers.DB.Model(&models.Addon{}).
+		Select("id, name, author, subscriber_count, current_version_size, last_indexed_at, created_at")
+	if where != "" {
+		q = q.Where(where, args...)
+	}
+	q.Order(order).Limit(8).Scan(&out)
+	return out
+}
+
+func buildOverview() (*overviewData, error) {
+	ac, err := loadAddonCounts()
+	if err != nil {
+		return nil, err
+	}
+	d := &overviewData{Addons: ac}
+
+	d.Types = []typeCount{}
+	initializers.DB.Model(&models.Addon{}).Select("type, COUNT(*) AS count").Group("type").Order("count DESC").Limit(12).Scan(&d.Types)
+
+	initializers.DB.Model(&models.WhitelistedHash{}).Count(&d.Whitelist)
+
+	var deps struct {
+		Edges      int64
+		WithDeps   int64
+		Dependents int64
+	}
+	initializers.DB.Raw(`SELECT COUNT(*) AS edges, COUNT(DISTINCT addon_id) AS with_deps,
+		COUNT(DISTINCT dependency_id) AS dependents FROM addon_dependencies`).Scan(&deps)
+	d.Dependencies = map[string]any{"edges": deps.Edges, "addonsWithDependencies": deps.WithDeps, "addonsDependedOn": deps.Dependents}
+
+	now := time.Now().Unix()
+	dayAgo := now - 86400
+	httpSel := metrics.Selector{Service: metrics.MainService, Name: "http_request_duration_ms"}
+	traffic := map[string]any{}
+	if t, err := metrics.Totals(metrics.Selector{Service: httpSel.Service, Name: httpSel.Name, GroupBy: []string{"group"}}, dayAgo, now); err == nil {
+		var total int64
+		for _, g := range t {
+			total += g.Count
+			if g.Labels["group"] == "v1" {
+				traffic["websiteRequests24h"] = g.Count
+				traffic["websiteP95Ms"] = g.P95
+				traffic["websiteP50Ms"] = g.P50
+			}
+			if g.Labels["group"] == "back" {
+				traffic["workerRequests24h"] = g.Count
+			}
+		}
+		traffic["requests24h"] = total
+	}
+	traffic["errors5xx24h"] = metrics.Sum(metrics.Selector{Service: httpSel.Service, Name: httpSel.Name, Match: map[string]string{"status": "5xx"}}, dayAgo, now)
+	humans, bots := metrics.Visitors()
+	traffic["visitorsToday"] = humans
+	traffic["botsToday"] = bots
+	d.Traffic = traffic
+
+	sum := func(name string, match map[string]string) float64 {
+		return metrics.Sum(metrics.Selector{Service: metrics.MainService, Name: name, Match: match}, dayAgo, now)
+	}
+	d.Activity = map[string]any{
+		"scraperCreated24h":      sum("scraper_addons_total", map[string]string{"action": "created"}),
+		"scraperUpdated24h":      sum("scraper_addons_total", map[string]string{"action": "updated"}),
+		"scraperCreateFailed24h": sum("scraper_addons_total", map[string]string{"action": "create_failed"}),
+		"deleted24h":             sum("scraper_addons_deleted_total", map[string]string{"result": "deleted"}),
+		"indexed24h":             sum("indexer_submissions_total", map[string]string{"result": "ok"}),
+		"claims24h":              sum("indexer_claims_total", nil),
+		"timeoutsReset24h":       sum("indexer_timeouts_reset_total", nil),
+		"dependencyReports24h":   sum("dependency_reports_total", nil),
+	}
+
+	var instances []models.ServiceInstance
+	initializers.DB.Find(&instances)
+	online := 0
+	for i := range instances {
+		if metrics.Online(&instances[i], time.Now()) {
+			online++
+		}
+	}
+	d.Services = map[string]any{"online": online, "total": len(instances)}
+
+	d.TopSubscribed = loadMiniAddons("subscriber_count DESC", "")
+	d.Largest = loadMiniAddons("current_version_size DESC", "")
+	d.RecentlyAdded = loadMiniAddons("created_at DESC", "")
+	d.RecentlyIndexed = loadMiniAddons("last_indexed_at DESC", "last_indexed_at IS NOT NULL")
+
+	d.Heavy = currentHeavy()
+	return d, nil
+}
+
+func overview(c *gin.Context) {
+	data, at, err := cached("overview", 20*time.Second, buildOverview)
+	if err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+	c.JSON(http.StatusOK, gin.H{"asOf": at.Unix(), "data": data})
+}
+
+// ---- heavy statistics (background) -----------------------------------------
+
+type tableInfo struct {
+	Name      string `json:"name"`
+	Rows      int64  `json:"rows"`
+	DeadRows  int64  `json:"deadRows"`
+	SizeBytes int64  `json:"sizeBytes"`
+}
+
+type dupHash struct {
+	Hash        string `json:"hash"`
+	AddonCount  int64  `json:"addonCount"`
+	FileCount   int64  `json:"fileCount"`
+	SamplePath  string `json:"samplePath"`
+	Whitelisted bool   `json:"whitelisted"`
+}
+
+type heavyStats struct {
+	AsOf             int64       `json:"asOf"`
+	FilesLive        int64       `json:"filesLive"`
+	FilesSoftDeleted int64       `json:"filesSoftDeleted"`
+	FilesApprox      bool        `json:"filesApprox"`
+	DistinctHashes   int64       `json:"distinctHashes"`
+	DistinctAsOf     int64       `json:"distinctAsOf"`
+	DBDriver         string      `json:"dbDriver"`
+	DBSizeBytes      int64       `json:"dbSizeBytes"`
+	Tables           []tableInfo `json:"tables"`
+	Duplicates       []dupHash   `json:"-"`
+	DuplicatesAsOf   int64       `json:"duplicatesAsOf"`
+	Error            string      `json:"error,omitempty"`
+}
+
+var (
+	heavyMu   sync.Mutex
+	heavy     *heavyStats
+	refreshCh = make(chan bool, 1)
+)
+
+func currentHeavy() *heavyStats {
+	heavyMu.Lock()
+	defer heavyMu.Unlock()
+	return heavy
+}
+
+// requestHeavyRefresh triggers a refresh; full=true also recomputes the
+// expensive duplicate statistics.
+func requestHeavyRefresh(full bool) {
+	select {
+	case refreshCh <- full:
+	default:
+	}
+}
+
+func heavyStatsLoop(ctx context.Context) {
+	lastFull := time.Time{}
+	run := func(full bool) {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Printf("admin: heavy stats panicked: %v", r)
+			}
+		}()
+		full = full || time.Since(lastFull) > time.Hour
+		computeHeavy(full)
+		if full {
+			lastFull = time.Now()
+		}
+	}
+	// give the server a moment to start before the first scan
+	select {
+	case <-ctx.Done():
+		return
+	case <-time.After(5 * time.Second):
+	}
+	run(true)
+	ticker := time.NewTicker(10 * time.Minute)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			run(false)
+		case full := <-refreshCh:
+			run(full)
+		}
+	}
+}
+
+// withTimeout runs fn in a transaction with a statement timeout on Postgres.
+func withTimeout(seconds int, fn func(tx *gorm.DB) error) error {
+	return initializers.DB.Transaction(func(tx *gorm.DB) error {
+		if isPostgres() {
+			if err := tx.Exec("SET LOCAL statement_timeout = " + strconv.Itoa(seconds*1000)).Error; err != nil {
+				return err
+			}
+		}
+		return fn(tx)
+	})
+}
+
+var ourTables = []string{"addons", "addon_files", "addon_dependencies", "whitelisted_hashes",
+	"metric_series", "metric_points", "service_instances", "service_events", "daily_top_items"}
+
+func computeHeavy(full bool) {
+	started := time.Now()
+	h := &heavyStats{AsOf: time.Now().Unix(), DBDriver: initializers.DB.Dialector.Name(), Tables: []tableInfo{}}
+	prev := currentHeavy()
+	if prev != nil {
+		h.DistinctHashes, h.DistinctAsOf = prev.DistinctHashes, prev.DistinctAsOf
+		h.Duplicates, h.DuplicatesAsOf = prev.Duplicates, prev.DuplicatesAsOf
+	}
+
+	var errs []string
+	note := func(what string, err error) {
+		if err != nil {
+			errs = append(errs, what+": "+err.Error())
+		}
+	}
+
+	if isPostgres() {
+		note("tables", initializers.DB.Raw(`SELECT relname AS name, n_live_tup AS rows, n_dead_tup AS dead_rows,
+			pg_total_relation_size(relid) AS size_bytes FROM pg_stat_user_tables ORDER BY size_bytes DESC`).Scan(&h.Tables).Error)
+		note("db size", initializers.DB.Raw(`SELECT pg_database_size(current_database())`).Scan(&h.DBSizeBytes).Error)
+		note("soft-deleted files", withTimeout(60, func(tx *gorm.DB) error {
+			return tx.Raw(`SELECT COUNT(*) FROM addon_files WHERE deleted_at IS NOT NULL`).Scan(&h.FilesSoftDeleted).Error
+		}))
+		for _, t := range h.Tables {
+			if t.Name == "addon_files" {
+				h.FilesLive = max(0, t.Rows-h.FilesSoftDeleted)
+				h.FilesApprox = true
+			}
+		}
+	} else {
+		var pageCount, pageSize int64
+		initializers.DB.Raw("PRAGMA page_count").Scan(&pageCount)
+		initializers.DB.Raw("PRAGMA page_size").Scan(&pageSize)
+		h.DBSizeBytes = pageCount * pageSize
+		if st, err := os.Stat(sqlitePath() + "-wal"); err == nil {
+			h.DBSizeBytes += st.Size()
+		}
+		for _, name := range ourTables {
+			var n int64
+			if err := initializers.DB.Table(name).Count(&n).Error; err == nil {
+				h.Tables = append(h.Tables, tableInfo{Name: name, Rows: n})
+			}
+		}
+		note("files", initializers.DB.Raw(`SELECT
+			COALESCE(SUM(CASE WHEN deleted_at IS NULL THEN 1 ELSE 0 END), 0),
+			COALESCE(SUM(CASE WHEN deleted_at IS NOT NULL THEN 1 ELSE 0 END), 0) FROM addon_files`).Row().
+			Scan(&h.FilesLive, &h.FilesSoftDeleted))
+	}
+
+	if full {
+		var distinct int64
+		err := withTimeout(120, func(tx *gorm.DB) error {
+			return tx.Raw(`SELECT COUNT(DISTINCT hash) FROM addon_files WHERE deleted_at IS NULL`).Scan(&distinct).Error
+		})
+		note("distinct hashes", err)
+		if err == nil {
+			h.DistinctHashes, h.DistinctAsOf = distinct, time.Now().Unix()
+		}
+
+		var dups []dupHash
+		err = withTimeout(120, func(tx *gorm.DB) error {
+			return tx.Raw(`SELECT hash, COUNT(DISTINCT addon_id) AS addon_count, COUNT(*) AS file_count
+				FROM addon_files WHERE deleted_at IS NULL
+				GROUP BY hash HAVING COUNT(DISTINCT addon_id) > 1
+				ORDER BY addon_count DESC, file_count DESC LIMIT 200`).Scan(&dups).Error
+		})
+		note("duplicates", err)
+		if err == nil {
+			h.Duplicates, h.DuplicatesAsOf = dups, time.Now().Unix()
+			if len(dups) > 0 {
+				hashes := make([]string, len(dups))
+				for i, d := range dups {
+					hashes[i] = d.Hash
+				}
+				var samples []struct {
+					Hash string
+					Path string
+				}
+				initializers.DB.Raw(`SELECT hash, MIN(path) AS path FROM addon_files
+					WHERE deleted_at IS NULL AND hash IN ? GROUP BY hash`, hashes).Scan(&samples)
+				byHash := map[string]string{}
+				for _, s := range samples {
+					byHash[s.Hash] = s.Path
+				}
+				for i := range h.Duplicates {
+					h.Duplicates[i].SamplePath = byHash[h.Duplicates[i].Hash]
+				}
+			}
+		}
+	}
+
+	if len(errs) > 0 {
+		h.Error = errs[0]
+		for _, e := range errs {
+			log.Printf("admin: heavy stats: %s", e)
+		}
+	}
+	heavyMu.Lock()
+	heavy = h
+	heavyMu.Unlock()
+	invalidate("overview")
+	if time.Since(started) > 10*time.Second {
+		log.Printf("admin: heavy stats took %s", time.Since(started).Round(time.Millisecond))
+	}
+}
+
+func sqlitePath() string {
+	if initializers.SQLitePath != "" {
+		return initializers.SQLitePath
+	}
+	return "crawler.db"
+}
+
+// duplicateHashes lists the hashes shared by the most addons.
+func duplicateHashes(c *gin.Context) {
+	h := currentHeavy()
+	if h == nil {
+		c.JSON(http.StatusOK, gin.H{"asOf": 0, "items": []dupHash{}, "pending": true})
+		return
+	}
+	items := make([]dupHash, len(h.Duplicates))
+	copy(items, h.Duplicates)
+	if len(items) > 0 {
+		hashes := make([]string, len(items))
+		for i, d := range items {
+			hashes[i] = d.Hash
+		}
+		var wl []string
+		initializers.DB.Model(&models.WhitelistedHash{}).Where("hash IN ?", hashes).Pluck("hash", &wl)
+		set := map[string]bool{}
+		for _, w := range wl {
+			set[w] = true
+		}
+		for i := range items {
+			items[i].Whitelisted = set[items[i].Hash]
+		}
+	}
+	if c.Query("hideWhitelisted") == "1" {
+		kept := items[:0]
+		for _, it := range items {
+			if !it.Whitelisted {
+				kept = append(kept, it)
+			}
+		}
+		items = kept
+	}
+	c.JSON(http.StatusOK, gin.H{"asOf": h.DuplicatesAsOf, "items": items})
+}

admin/routes.go

@@ -0,0 +1,57 @@
+package admin
+
+import (
+	"context"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/auth"
+	"github.com/gin-gonic/gin"
+)
+
+// Register mounts the admin JSON API under /admin/api and the legacy
+// header-authenticated /admin/addWhitelistedHash endpoint.
+func Register(r *gin.Engine) {
+	r.POST("/admin/addWhitelistedHash", auth.RequireAdminHeader, legacyAddWhitelistedHash)
+
+	api := r.Group("/admin/api")
+	api.POST("/auth/login", login)
+	api.POST("/auth/logout", logout)
+
+	p := api.Group("", auth.RequireAdmin)
+	p.GET("/auth/me", me)
+
+	p.GET("/overview", overview)
+	p.GET("/overview/duplicates", duplicateHashes)
+
+	p.GET("/queue", queue)
+	p.POST("/queue/reset-stuck", resetStuck)
+
+	p.GET("/addons", listAddons)
+	p.GET("/addons/:id", addonDetail)
+	p.GET("/addons/:id/files", addonFiles)
+	p.POST("/addons/:id/:action", addonAction)
+	p.DELETE("/addons/:id", deleteAddon)
+	p.DELETE("/addons/:id/purge", purgeAddon)
+	p.POST("/bulk/addons", bulkAddons)
+
+	p.GET("/whitelist", listWhitelist)
+	p.POST("/whitelist", addWhitelist)
+	p.DELETE("/whitelist/:id", deleteWhitelist)
+	p.GET("/hashes/:hash", hashUsage)
+
+	p.GET("/metrics/catalog", metricsCatalog)
+	p.GET("/metrics/query", metricsQuery)
+	p.GET("/traffic", traffic)
+	p.GET("/top", topItems)
+
+	p.GET("/services", listServices)
+	p.GET("/services/events", listEvents)
+	p.DELETE("/services/:id", deleteService)
+
+	p.GET("/system", system)
+	p.POST("/system/maintenance", maintenance)
+}
+
+// StartBackground runs the periodic heavy statistics job.
+func StartBackground(ctx context.Context) {
+	go heavyStatsLoop(ctx)
+}

admin/system.go

@@ -0,0 +1,244 @@
+package admin
+
+import (
+	"encoding/json"
+	"net/http"
+	"os"
+	"runtime"
+	"runtime/debug"
+	"strconv"
+	"strings"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
+	"github.com/gin-gonic/gin"
+)
+
+// BuildCommit is set by main from -ldflags.
+var BuildCommit = ""
+
+type serviceRow struct {
+	models.ServiceInstance
+	Online    bool           `json:"online"`
+	UptimeSec int64          `json:"uptimeSec"`
+	MetaMap   map[string]any `json:"metaData,omitempty"`
+}
+
+func listServices(c *gin.Context) {
+	var rows []models.ServiceInstance
+	if err := initializers.DB.Order("service ASC, instance ASC").Find(&rows).Error; err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+	now := time.Now()
+	out := make([]serviceRow, 0, len(rows))
+	for _, r := range rows {
+		s := serviceRow{ServiceInstance: r, Online: metrics.Online(&r, now)}
+		if r.StartedAt > 0 && s.Online {
+			s.UptimeSec = now.Unix() - r.StartedAt
+		}
+		if r.Meta != "" {
+			_ = json.Unmarshal([]byte(r.Meta), &s.MetaMap)
+		}
+		out = append(out, s)
+	}
+	c.JSON(http.StatusOK, gin.H{"items": out})
+}
+
+func deleteService(c *gin.Context) {
+	id, err := strconv.ParseUint(c.Param("id"), 10, 64)
+	if err != nil {
+		fail(c, http.StatusBadRequest, "invalid id")
+		return
+	}
+	var row models.ServiceInstance
+	if err := initializers.DB.First(&row, id).Error; err != nil {
+		fail(c, http.StatusNotFound, "not found")
+		return
+	}
+	initializers.DB.Delete(&row)
+	audit(c, "service_forget", map[string]any{"service": row.Service, "instance": row.Instance})
+	c.JSON(http.StatusOK, gin.H{"deleted": id})
+}
+
+type eventRow struct {
+	models.ServiceEvent
+	FieldsMap map[string]any `json:"fieldsData,omitempty"`
+}
+
+func listEvents(c *gin.Context) {
+	limit := queryInt(c, "limit", 100, 1, 500)
+	q := initializers.DB.Model(&models.ServiceEvent{})
+	if v := c.Query("service"); v != "" {
+		q = q.Where("service = ?", v)
+	}
+	if v := c.Query("instance"); v != "" {
+		q = q.Where("instance = ?", v)
+	}
+	if v := c.Query("level"); v != "" {
+		levels := strings.Split(v, ",")
+		q = q.Where("level IN ?", levels)
+	}
+	if c.Query("audit") == "1" {
+		q = q.Where("fields LIKE ?", `%"kind":"audit"%`)
+	}
+	if v := strings.TrimSpace(c.Query("q")); v != "" {
+		q = q.Where(`LOWER(message) LIKE ? ESCAPE '\'`, "%"+likeEscape(strings.ToLower(v))+"%")
+	}
+	if before := queryInt64(c, "before", 0); before > 0 {
+		q = q.Where("id < ?", before)
+	}
+	var rows []models.ServiceEvent
+	if err := q.Order("id DESC").Limit(limit).Find(&rows).Error; err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+	out := make([]eventRow, 0, len(rows))
+	for _, r := range rows {
+		e := eventRow{ServiceEvent: r}
+		if r.Fields != "" {
+			_ = json.Unmarshal([]byte(r.Fields), &e.FieldsMap)
+		}
+		out = append(out, e)
+	}
+	var next uint
+	if len(rows) == limit {
+		next = rows[len(rows)-1].ID
+	}
+	c.JSON(http.StatusOK, gin.H{"items": out, "next": next})
+}
+
+func buildInfo() gin.H {
+	info := gin.H{"commit": BuildCommit}
+	if bi, ok := debug.ReadBuildInfo(); ok {
+		info["module"] = bi.Main.Path
+		for _, s := range bi.Settings {
+			switch s.Key {
+			case "vcs.revision":
+				if BuildCommit == "" {
+					info["commit"] = s.Value
+				}
+			case "vcs.time":
+				info["time"] = s.Value
+			case "vcs.modified":
+				info["modified"] = s.Value == "true"
+			}
+		}
+	}
+	return info
+}
+
+func system(c *gin.Context) {
+	var ms runtime.MemStats
+	runtime.ReadMemStats(&ms)
+	host, _ := os.Hostname()
+	started := metrics.StartedAt()
+
+	resp := gin.H{
+		"go": gin.H{
+			"version": runtime.Version(), "os": runtime.GOOS, "arch": runtime.GOARCH,
+			"cpus": runtime.NumCPU(), "maxProcs": runtime.GOMAXPROCS(0), "goroutines": runtime.NumGoroutine(),
+		},
+		"process": gin.H{
+			"pid": os.Getpid(), "hostname": host, "startedAt": started.Unix(),
+			"uptimeSec": int64(time.Since(started).Seconds()),
+		},
+		"memory": gin.H{
+			"heapAlloc": ms.HeapAlloc, "heapInuse": ms.HeapInuse, "sys": ms.Sys,
+			"numGC": ms.NumGC, "pauseTotalNs": ms.PauseTotalNs, "lastGC": int64(ms.LastGC / 1e9),
+		},
+		"build":   buildInfo(),
+		"metrics": metrics.Status(),
+		"config": gin.H{
+			"dbDriver":                   initializers.DB.Dialector.Name(),
+			"trustedProxies":             initializers.TrustedProxies,
+			"cookieSecure":               initializers.AdminCookieSecure,
+			"sessionTtlHours":            initializers.AdminSessionTTL.Hours(),
+			"metricsEnabled":             initializers.MetricsEnabled,
+			"prometheusEndpoint":         initializers.MetricsToken != "",
+			"metricsMinuteRetentionDays": initializers.MetricsMinuteRetentionDays,
+			"metricsHourRetentionDays":   initializers.MetricsHourRetentionDays,
+			"eventsRetentionDays":        initializers.EventsRetentionDays,
+			"topItemsRetentionDays":      initializers.TopItemsRetentionDays,
+			"indexTimeoutMinutes":        initializers.IndexTimeout.Minutes(),
+			"adminSecretStrong":          len(initializers.AdminSecret) >= 16,
+			"secretKeySet":               initializers.SecretKey != "",
+			"discordWebhook":             initializers.DiscordWebhookURL != "",
+			"scraperWebhook":             initializers.ScraperWebhookURL != "",
+		},
+	}
+	if sqlDB, err := initializers.DB.DB(); err == nil {
+		st := sqlDB.Stats()
+		resp["dbPool"] = gin.H{
+			"open": st.OpenConnections, "inUse": st.InUse, "idle": st.Idle,
+			"maxOpen": st.MaxOpenConnections, "waitCount": st.WaitCount, "waitMs": st.WaitDuration.Milliseconds(),
+		}
+		start := time.Now()
+		if err := sqlDB.Ping(); err == nil {
+			resp["dbPingMs"] = float64(time.Since(start).Microseconds()) / 1000
+		} else {
+			resp["dbPingError"] = err.Error()
+		}
+	}
+	if h := currentHeavy(); h != nil {
+		resp["db"] = h
+	}
+	c.JSON(http.StatusOK, resp)
+}
+
+func maintenance(c *gin.Context) {
+	var body struct {
+		Action    string `json:"action"`
+		OlderDays int    `json:"olderThanDays"`
+	}
+	if err := c.ShouldBindJSON(&body); err != nil {
+		fail(c, http.StatusBadRequest, "invalid JSON")
+		return
+	}
+	switch body.Action {
+	case "refresh-stats":
+		requestHeavyRefresh(true)
+	case "flush-metrics":
+		metrics.FlushAll()
+	case "rollup":
+		go metrics.RunMaintenance(time.Now())
+	case "purge-deleted-files":
+		days := body.OlderDays
+		if days <= 0 {
+			days = 7
+		}
+		go purgeDeletedFiles(days)
+	default:
+		fail(c, http.StatusBadRequest, "unknown action")
+		return
+	}
+	audit(c, "maintenance_"+strings.ReplaceAll(body.Action, "-", "_"), map[string]any{"olderThanDays": body.OlderDays})
+	c.JSON(http.StatusAccepted, gin.H{"started": body.Action})
+}
+
+// purgeDeletedFiles hard-deletes file rows that were soft-deleted (replaced by
+// a newer index) more than `days` ago, in batches.
+func purgeDeletedFiles(days int) {
+	cutoff := time.Now().AddDate(0, 0, -days)
+	started := time.Now()
+	var total int64
+	for {
+		res := initializers.DB.Exec(`DELETE FROM addon_files WHERE id IN (
+			SELECT id FROM addon_files WHERE deleted_at IS NOT NULL AND deleted_at < ? LIMIT 5000)`, cutoff)
+		if res.Error != nil {
+			metrics.Event("error", "purge of deleted file rows failed", map[string]any{"kind": "maintenance", "error": res.Error.Error(), "deleted": total})
+			return
+		}
+		total += res.RowsAffected
+		if res.RowsAffected == 0 {
+			break
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	metrics.Event("info", "purged deleted file rows", map[string]any{
+		"kind": "maintenance", "deleted": total, "olderThanDays": days, "tookSec": int64(time.Since(started).Seconds()),
+	})
+	requestHeavyRefresh(false)
+}

admin/util.go

@@ -0,0 +1,136 @@
+package admin
+
+import (
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
+	"github.com/gin-gonic/gin"
+)
+
+// ---- TTL cache with request coalescing -------------------------------------
+
+type cacheEntry struct {
+	val any
+	at  time.Time
+}
+
+type inflight struct {
+	wg  sync.WaitGroup
+	val any
+	err error
+}
+
+var (
+	cacheMu   sync.Mutex
+	cacheData = map[string]cacheEntry{}
+	cacheRun  = map[string]*inflight{}
+)
+
+// cached returns a cached value younger than ttl, or computes it once even if
+// many requests ask at the same time. It also returns when the value was made.
+func cached[T any](key string, ttl time.Duration, fn func() (T, error)) (T, time.Time, error) {
+	cacheMu.Lock()
+	if e, ok := cacheData[key]; ok && time.Since(e.at) < ttl {
+		cacheMu.Unlock()
+		return e.val.(T), e.at, nil
+	}
+	if call, ok := cacheRun[key]; ok {
+		cacheMu.Unlock()
+		call.wg.Wait()
+		var zero T
+		if call.err != nil {
+			return zero, time.Time{}, call.err
+		}
+		return call.val.(T), time.Now(), nil
+	}
+	call := &inflight{}
+	call.wg.Add(1)
+	cacheRun[key] = call
+	cacheMu.Unlock()
+
+	val, err := fn()
+	call.val, call.err = val, err
+	now := time.Now()
+	cacheMu.Lock()
+	delete(cacheRun, key)
+	if err == nil {
+		cacheData[key] = cacheEntry{val: val, at: now}
+	}
+	cacheMu.Unlock()
+	call.wg.Done()
+	return val, now, err
+}
+
+// invalidate drops cached values whose key starts with one of the prefixes.
+func invalidate(prefixes ...string) {
+	cacheMu.Lock()
+	defer cacheMu.Unlock()
+	for k := range cacheData {
+		for _, p := range prefixes {
+			if strings.HasPrefix(k, p) {
+				delete(cacheData, k)
+			}
+		}
+	}
+}
+
+// ---- request helpers -----------------------------------------------------
+
+func fail(c *gin.Context, code int, msg string) {
+	c.AbortWithStatusJSON(code, gin.H{"error": msg})
+}
+
+func queryInt(c *gin.Context, name string, def, lo, hi int) int {
+	v, err := strconv.Atoi(c.Query(name))
+	if err != nil {
+		return def
+	}
+	return max(lo, min(hi, v))
+}
+
+func queryInt64(c *gin.Context, name string, def int64) int64 {
+	v, err := strconv.ParseInt(c.Query(name), 10, 64)
+	if err != nil {
+		return def
+	}
+	return v
+}
+
+// timeRange reads from/to (unix seconds) with a default window ending now.
+func timeRange(c *gin.Context, def time.Duration) (int64, int64) {
+	now := time.Now().Unix()
+	to := queryInt64(c, "to", now)
+	if to > now {
+		to = now
+	}
+	from := queryInt64(c, "from", to-int64(def.Seconds()))
+	if from >= to {
+		from = to - int64(def.Seconds())
+	}
+	return from, to
+}
+
+func isPostgres() bool {
+	return initializers.DB.Dialector.Name() == "postgres"
+}
+
+// likeEscape escapes LIKE wildcards in user input.
+func likeEscape(s string) string {
+	return strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(s)
+}
+
+// audit records an admin action as a metric and an event.
+func audit(c *gin.Context, action string, fields map[string]any) {
+	if fields == nil {
+		fields = map[string]any{}
+	}
+	fields["kind"] = "audit"
+	fields["ip"] = c.ClientIP()
+	fields["auth"] = c.GetString("adminAuth")
+	metrics.AdminActions.Inc(action)
+	metrics.Event("info", "admin: "+action, fields)
+}

admin/whitelist.go

@@ -0,0 +1,227 @@
+package admin
+
+import (
+	"net/http"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
+	"github.com/gin-gonic/gin"
+	"gorm.io/gorm"
+)
+
+var sha512Re = regexp.MustCompile(`^[0-9a-f]{128}$`)
+
+// legacyAddWhitelistedHash keeps the original form-based endpoint working.
+func legacyAddWhitelistedHash(c *gin.Context) {
+	hash := c.PostForm("hash")
+	if hash == "" {
+		c.JSON(400, gin.H{"error": "Hash is required"})
+		return
+	}
+
+	ret := initializers.DB.Find(&models.WhitelistedHash{}, "hash = ?", hash)
+	if ret.RowsAffected > 0 {
+		c.JSON(400, gin.H{"error": "Hash already whitelisted"})
+		return
+	}
+
+	if err := initializers.DB.Create(&models.WhitelistedHash{Hash: hash}).Error; err != nil {
+		c.JSON(500, gin.H{"error": "Failed to add whitelisted hash"})
+		return
+	}
+	audit(c, "whitelist_add", map[string]any{"added": 1, "legacy": true})
+	invalidate("overview")
+	c.JSON(200, gin.H{"message": "Whitelisted hash added successfully"})
+}
+
+type whitelistRow struct {
+	ID         uint      `json:"id"`
+	Hash       string    `json:"hash"`
+	Note       string    `json:"note"`
+	CreatedAt  time.Time `json:"createdAt"`
+	Files      int64     `json:"files"`
+	Addons     int64     `json:"addons"`
+	SamplePath string    `json:"samplePath"`
+}
+
+func listWhitelist(c *gin.Context) {
+	limit := queryInt(c, "limit", 50, 1, 500)
+	offset := queryInt(c, "offset", 0, 0, 1<<30)
+	q := initializers.DB.Model(&models.WhitelistedHash{})
+	if s := strings.TrimSpace(strings.ToLower(c.Query("q"))); s != "" {
+		like := "%" + likeEscape(s) + "%"
+		q = q.Where(`hash LIKE ? ESCAPE '\' OR LOWER(note) LIKE ? ESCAPE '\'`, likeEscape(s)+"%", like)
+	}
+	var total int64
+	q.Session(&gorm.Session{}).Count(&total)
+
+	rows := []whitelistRow{}
+	if err := q.Select("id, hash, note, created_at").Order("id DESC").Limit(limit).Offset(offset).Scan(&rows).Error; err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+	if len(rows) > 0 {
+		hashes := make([]string, len(rows))
+		for i, r := range rows {
+			hashes[i] = r.Hash
+		}
+		var usage []struct {
+			Hash   string
+			Files  int64
+			Addons int64
+			Sample string
+		}
+		initializers.DB.Raw(`SELECT hash, COUNT(*) AS files, COUNT(DISTINCT addon_id) AS addons, MIN(path) AS sample
+			FROM addon_files WHERE deleted_at IS NULL AND hash IN ? GROUP BY hash`, hashes).Scan(&usage)
+		byHash := map[string]int{}
+		for i, u := range usage {
+			byHash[u.Hash] = i
+		}
+		for i := range rows {
+			if j, ok := byHash[rows[i].Hash]; ok {
+				rows[i].Files, rows[i].Addons, rows[i].SamplePath = usage[j].Files, usage[j].Addons, usage[j].Sample
+			}
+		}
+	}
+	c.JSON(http.StatusOK, gin.H{"items": rows, "total": total, "limit": limit, "offset": offset})
+}
+
+type skipped struct {
+	Hash   string `json:"hash"`
+	Reason string `json:"reason"`
+}
+
+func addWhitelist(c *gin.Context) {
+	var body struct {
+		Hashes []string `json:"hashes"`
+		Text   string   `json:"text"`
+		Note   string   `json:"note"`
+	}
+	if err := c.ShouldBindJSON(&body); err != nil {
+		fail(c, http.StatusBadRequest, "invalid JSON")
+		return
+	}
+	candidates := append([]string{}, body.Hashes...)
+	candidates = append(candidates, strings.FieldsFunc(body.Text, func(r rune) bool {
+		return r == ',' || r == ';' || r == ' ' || r == '\n' || r == '\r' || r == '\t'
+	})...)
+	if len(candidates) == 0 {
+		fail(c, http.StatusBadRequest, "no hashes given")
+		return
+	}
+	if len(candidates) > 5000 {
+		fail(c, http.StatusBadRequest, "at most 5000 hashes per request")
+		return
+	}
+	note := body.Note
+	if len(note) > 256 {
+		note = note[:256]
+	}
+
+	var skips []skipped
+	seen := map[string]bool{}
+	valid := []string{}
+	for _, raw := range candidates {
+		h := strings.ToLower(strings.TrimSpace(raw))
+		switch {
+		case h == "":
+			continue
+		case !sha512Re.MatchString(h):
+			skips = append(skips, skipped{raw, "not a SHA-512 hex hash"})
+		case seen[h]:
+			skips = append(skips, skipped{h, "duplicate in request"})
+		default:
+			seen[h] = true
+			valid = append(valid, h)
+		}
+	}
+
+	added := 0
+	err := initializers.DB.Transaction(func(tx *gorm.DB) error {
+		existing := map[string]bool{}
+		for start := 0; start < len(valid); start += 500 {
+			end := min(start+500, len(valid))
+			var found []string
+			if err := tx.Model(&models.WhitelistedHash{}).Where("hash IN ?", valid[start:end]).Pluck("hash", &found).Error; err != nil {
+				return err
+			}
+			for _, h := range found {
+				existing[h] = true
+			}
+		}
+		rows := []models.WhitelistedHash{}
+		for _, h := range valid {
+			if existing[h] {
+				skips = append(skips, skipped{h, "already whitelisted"})
+				continue
+			}
+			rows = append(rows, models.WhitelistedHash{Hash: h, Note: note})
+		}
+		if len(rows) == 0 {
+			return nil
+		}
+		added = len(rows)
+		return tx.CreateInBatches(&rows, 200).Error
+	})
+	if err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+	if added > 0 {
+		audit(c, "whitelist_add", map[string]any{"added": added, "note": note})
+		invalidate("overview")
+	}
+	if skips == nil {
+		skips = []skipped{}
+	}
+	c.JSON(http.StatusOK, gin.H{"added": added, "skipped": skips})
+}
+
+func deleteWhitelist(c *gin.Context) {
+	id, err := strconv.ParseUint(c.Param("id"), 10, 64)
+	if err != nil {
+		fail(c, http.StatusBadRequest, "invalid id")
+		return
+	}
+	var row models.WhitelistedHash
+	if err := initializers.DB.Unscoped().First(&row, id).Error; err != nil {
+		fail(c, http.StatusNotFound, "not found")
+		return
+	}
+	// remove every row of this hash, the table has no unique constraint
+	res := initializers.DB.Unscoped().Where("hash = ?", row.Hash).Delete(&models.WhitelistedHash{})
+	if res.Error != nil {
+		fail(c, http.StatusInternalServerError, res.Error.Error())
+		return
+	}
+	audit(c, "whitelist_remove", map[string]any{"hash": row.Hash})
+	invalidate("overview")
+	c.JSON(http.StatusOK, gin.H{"deleted": res.RowsAffected})
+}
+
+// hashUsage lists the addons that contain a file with the given hash.
+func hashUsage(c *gin.Context) {
+	hash := strings.ToLower(c.Param("hash"))
+	var total int64
+	initializers.DB.Model(&models.AddonFile{}).Where("hash = ?", hash).Count(&total)
+	items := []struct {
+		AddonID string `json:"addonId"`
+		Name    string `json:"name"`
+		Path    string `json:"path"`
+		Version string `json:"version"`
+	}{}
+	err := initializers.DB.Raw(`SELECT f.addon_id, COALESCE(a.name, '') AS name, f.path, f.version
+		FROM addon_files f LEFT JOIN addons a ON a.id = f.addon_id
+		WHERE f.hash = ? AND f.deleted_at IS NULL ORDER BY a.subscriber_count DESC LIMIT 200`, hash).Scan(&items).Error
+	if err != nil {
+		fail(c, http.StatusInternalServerError, err.Error())
+		return
+	}
+	var wl int64
+	initializers.DB.Model(&models.WhitelistedHash{}).Where("hash = ?", hash).Count(&wl)
+	c.JSON(http.StatusOK, gin.H{"hash": hash, "whitelisted": wl > 0, "total": total, "items": items})
+}