admin dashboard, metrics and telemetry

- embedded React admin UI at /admin/ui (go:embed), login with ADMIN_SECRET
  via signed HttpOnly session cookie, X-ADMIN-KEY still works for scripts
- admin API: overview stats, indexing queue, addon management (priority,
  reindex, delete/restore/purge, bulk), whitelist CRUD, duplicate hashes,
  traffic, metrics explorer, services, events/audit log, system info
- built-in metrics: request latency per route, visitors, top addons and
  searches, indexer/scraper counters; minute buckets in the DB with hourly
  rollups and retention; optional Prometheus /metrics (METRICS_TOKEN)
- POST /back/telemetry for indexers/scrapers (metrics, events, heartbeat)
  and a stdlib-only Go client in telemetry/
- auth hardening: empty secrets no longer allow access, constant-time
  compare, CORS not applied to /admin and /back, TRUSTED_PROXIES setting
- Dockerfile builds the UI and no longer bakes config.yaml into the image;
  CI builds the UI, runs tests and builds arm64 with CGO

controllers/adminController.go

@@ -1,30 +0,0 @@
-package controllers
-
-import (
-	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
-	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
-	"github.com/gin-gonic/gin"
-)
-
-func AddWhitelistedHash(c *gin.Context) {
-	hash := c.PostForm("hash")
-	if hash == "" {
-		c.JSON(400, gin.H{"error": "Hash is required"})
-		return
-	}
-
-	hashAddon := models.WhitelistedHash{Hash: hash}
-
-	ret := initializers.DB.Find(&models.WhitelistedHash{}, "hash = ?", hash)
-	if ret.RowsAffected > 0 {
-		c.JSON(400, gin.H{"error": "Hash already whitelisted"})
-		return
-	}
-
-	if err := initializers.DB.Create(&hashAddon).Error; err != nil {
-		c.JSON(500, gin.H{"error": "Failed to add whitelisted hash"})
-		return
-	}
-
-	c.JSON(200, gin.H{"message": "Whitelisted hash added successfully"})
-}

controllers/authController.go

@@ -1,26 +1,16 @@
 package controllers
 
 import (
-	"net/http"
-
-	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/auth"
 	"github.com/gin-gonic/gin"
 )
 
+// CheckAllowed guards the scraper/indexer endpoints (X-SECRET-KEY).
 func CheckAllowed(c *gin.Context) {
-	secret := c.GetHeader("X-SECRET-KEY")
-	if secret != initializers.SecretKey {
-		c.JSON(http.StatusForbidden, gin.H{"error": "Forbidden"})
-		c.Abort()
-		return
-	}
+	auth.RequireSecret(c)
 }
 
+// CheckAdmin is the legacy header-only admin check (X-ADMIN-KEY).
 func CheckAdmin(c *gin.Context) {
-	adminSecret := c.GetHeader("X-ADMIN-KEY")
-	if adminSecret != initializers.AdminSecret {
-		c.JSON(http.StatusForbidden, gin.H{"error": "Forbidden"})
-		c.Abort()
-		return
-	}
+	auth.RequireAdminHeader(c)
 }

controllers/dependencyController.go

@@ -4,6 +4,7 @@ import (
 	"strconv"
 
 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
 	"github.com/gin-gonic/gin"
 	"gorm.io/gorm"
@@ -53,6 +54,8 @@ func SetAddonDependencies(c *gin.Context) {
 		return
 	}
 
+	metrics.DependencyReports.Inc()
+	metrics.DependencyEdges.Add(float64(len(edges)))
 	c.JSON(200, gin.H{"status": "success", "stored": len(edges)})
 }
 
@@ -69,7 +72,7 @@ func GetDependents(c *gin.Context) {
 	limit := parseLimit(c.Query("limit"))
 	offset := parseOffset(c.Query("offset"))
 
-	dependents, total, err := queryRelatedAddons(addonID, "ad.addon_id", "ad.dependency_id", limit, offset)
+	dependents, total, err := QueryRelatedAddons(addonID, "ad.addon_id", "ad.dependency_id", limit, offset)
 	if err != nil {
 		c.JSON(500, gin.H{"error": err.Error()})
 		return
@@ -97,7 +100,7 @@ func GetDependencies(c *gin.Context) {
 	limit := parseLimit(c.Query("limit"))
 	offset := parseOffset(c.Query("offset"))
 
-	dependencies, total, err := queryRelatedAddons(addonID, "ad.dependency_id", "ad.addon_id", limit, offset)
+	dependencies, total, err := QueryRelatedAddons(addonID, "ad.dependency_id", "ad.addon_id", limit, offset)
 	if err != nil {
 		c.JSON(500, gin.H{"error": err.Error()})
 		return
@@ -112,7 +115,7 @@ func GetDependencies(c *gin.Context) {
 	})
 }
 
-// queryRelatedAddons returns one page of addons linked to addonID through the
+// QueryRelatedAddons returns one page of addons linked to addonID through the
 // addon_dependencies edge table, plus the total number of matches.
 //
 // joinCol is the edge column joined to addons.id; filterCol is the edge column
@@ -120,7 +123,7 @@ func GetDependencies(c *gin.Context) {
 //
 // The addons.id tiebreaker in the ORDER BY is required: subscriber_count alone
 // is not a unique ordering, so without it paging could repeat or skip rows.
-func queryRelatedAddons(addonID, joinCol, filterCol string, limit, offset int) ([]models.AddonSearchResult, int, error) {
+func QueryRelatedAddons(addonID, joinCol, filterCol string, limit, offset int) ([]models.AddonSearchResult, int, error) {
 	base := func() *gorm.DB {
 		return initializers.DB.Model(&models.Addon{}).
 			Joins("JOIN addon_dependencies ad ON "+joinCol+" = addons.id").

controllers/feController.go

@@ -6,6 +6,7 @@ import (
 	"strconv"
 
 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
 	"github.com/gin-gonic/gin"
 )
@@ -114,6 +115,7 @@ func GetDuplicates(c *gin.Context) {
 	for _, file := range duplicateFiles {
 		duplicatesByAddonID[file.AddonID] = append(duplicatesByAddonID[file.AddonID], file)
 	}
+	metrics.DuplicatesFound.Observe(float64(len(duplicatesByAddonID)))
 
 	// Fetch addon information for each addon that has duplicates
 	var addonIDs []string

controllers/indexerController.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
 	"github.com/gin-gonic/gin"
 	"gorm.io/gorm"
@@ -16,7 +17,7 @@ import (
 
 func GetNextToBeIndexed(c *gin.Context) {
 
-	checkIndexingTimeout()
+	metrics.IndexerTimeoutsReset.Add(float64(ResetTimedOutIndexing(initializers.IndexTimeout)))
 
 	maxFileSize := c.Query("maxFileSize")
 	maxSize := 10_000_000_000
@@ -26,25 +27,31 @@ func GetNextToBeIndexed(c *gin.Context) {
 		}
 	}
 
+	queue := "priority"
 	addon, err := claimNextAddon(maxSize, true)
 	if err != nil {
+		metrics.IndexerClaims.Inc("error")
 		c.JSON(500, gin.H{"error": err.Error()})
 		return
 	}
 	if addon == nil {
+		queue = "normal"
 		addon, err = claimNextAddon(maxSize, false)
 		if err != nil {
+			metrics.IndexerClaims.Inc("error")
 			c.JSON(500, gin.H{"error": err.Error()})
 			return
 		}
 	}
 
 	if addon != nil {
+		metrics.IndexerClaims.Inc(queue)
 		c.JSON(200, gin.H{"guid": addon.ID, "currentVersion": addon.CurrentVersionNumber})
 		return
 	}
 
 	// nothing to index
+	metrics.IndexerClaims.Inc("empty")
 	c.JSON(200, gin.H{"guid": "", "currentVersion": ""})
 }
 
@@ -96,6 +103,7 @@ func SaveIndexingResult(c *gin.Context) {
 	err := c.ShouldBindJSON(&result)
 
 	if err != nil {
+		metrics.IndexerSubmissions.Inc("invalid")
 		c.JSON(400, gin.H{"error": "Invalid JSON"})
 		return
 	}
@@ -103,6 +111,7 @@ func SaveIndexingResult(c *gin.Context) {
 	var addon models.Addon
 	ret := initializers.DB.Where("id = ?", result.GUID).First(&addon)
 	if ret.Error != nil {
+		metrics.IndexerSubmissions.Inc("not_found")
 		c.JSON(404, gin.H{"error": "Addon not found"})
 		return
 	}
@@ -110,25 +119,42 @@ func SaveIndexingResult(c *gin.Context) {
 	// get all files for this addon and delete them - old version
 	ret = initializers.DB.Where("addon_id = ?", addon.ID).Delete(&models.AddonFile{})
 	if ret.Error != nil {
+		metrics.IndexerSubmissions.Inc("error")
 		c.JSON(500, gin.H{"error": ret.Error.Error()})
 		return
 	}
 
 	ret = initializers.DB.CreateInBatches(&result.Files, 100)
 	if ret.Error != nil {
+		metrics.IndexerSubmissions.Inc("error")
 		c.JSON(500, gin.H{"error": ret.Error.Error()})
 		return
 	}
 
+	now := time.Now()
+	wasClaimed := addon.IsBeingIndexed && !addon.IndexStartTime.IsZero()
 	addon.CurrentVersionNumber = result.CurrentVersion
 	addon.IsBeingIndexed = false
 	addon.ToBeIndexed = false
+	addon.LastIndexedAt = &now
 	ret = initializers.DB.Save(&addon)
 	if ret.Error != nil {
+		metrics.IndexerSubmissions.Inc("error")
 		c.JSON(500, gin.H{"error": ret.Error.Error()})
 		return
 	}
 
+	outcome := "ok"
+	if !result.Success {
+		outcome = "unsuccessful"
+	}
+	metrics.IndexerSubmissions.Inc(outcome)
+	if wasClaimed {
+		metrics.IndexerJobDuration.Observe(now.Sub(addon.IndexStartTime).Seconds())
+	}
+	metrics.IndexerFilesPerSubmission.Observe(float64(len(result.Files)))
+	metrics.IndexerBytesIndexed.Add(float64(addon.CurrentVersionSize))
+
 	if initializers.DiscordWebhookURL != "" {
 		text := "Indexing of addon " + addon.Name + " (" + addon.ID + ") was successful."
 		colour := 2228479
@@ -179,7 +205,7 @@ func SaveIndexingResult(c *gin.Context) {
 			Embeds:   []models.CustomEmbed{myEmbed},
 		}
 
-		err := SendCustomWebhook(initializers.DiscordWebhookURL, myHook)
+		err := sendWebhook("index", initializers.DiscordWebhookURL, myHook)
 		if err != nil {
 			fmt.Println("Error sending webhook:", err)
 		}
@@ -198,40 +224,60 @@ func DeleteAddon(c *gin.Context) {
 	var addon models.Addon
 	ret := initializers.DB.Where("id = ?", guid).First(&addon)
 	if ret.Error != nil {
+		metrics.ScraperDeletes.Inc("not_found")
 		c.JSON(404, gin.H{"error": "Addon not found"})
 		return
 	}
 
-	// Delete associated files
-	ret = initializers.DB.Where("addon_id = ?", addon.ID).Delete(&models.AddonFile{})
-	if ret.Error != nil {
-		c.JSON(500, gin.H{"error": ret.Error.Error()})
+	err := initializers.DB.Transaction(func(tx *gorm.DB) error {
+		return DeleteAddonCascade(tx, &addon)
+	})
+	if err != nil {
+		metrics.ScraperDeletes.Inc("error")
+		c.JSON(500, gin.H{"error": err.Error()})
 		return
 	}
 
+	metrics.ScraperDeletes.Inc("deleted")
+	c.JSON(200, gin.H{"status": "addon deleted"})
+}
+
+// DeleteAddonCascade soft-deletes an addon and its files and removes its
+// outgoing dependency edges.
+func DeleteAddonCascade(tx *gorm.DB, addon *models.Addon) error {
+	// Delete associated files
+	if err := tx.Where("addon_id = ?", addon.ID).Delete(&models.AddonFile{}).Error; err != nil {
+		return err
+	}
+
 	// Delete outgoing dependency edges; incoming edges stay - other addons
 	// still declare this dependency even if the addon row is gone
-	ret = initializers.DB.Where("addon_id = ?", addon.ID).Delete(&models.AddonDependency{})
-	if ret.Error != nil {
-		c.JSON(500, gin.H{"error": ret.Error.Error()})
-		return
+	if err := tx.Where("addon_id = ?", addon.ID).Delete(&models.AddonDependency{}).Error; err != nil {
+		return err
 	}
 
 	// Delete the addon
-	ret = initializers.DB.Delete(&addon)
-	if ret.Error != nil {
-		c.JSON(500, gin.H{"error": ret.Error.Error()})
-		return
-	}
-
-	c.JSON(200, gin.H{"status": "addon deleted"})
+	return tx.Delete(addon).Error
 }
 
-func checkIndexingTimeout() {
-	initializers.DB.Model(&models.Addon{}).
+// ResetTimedOutIndexing releases claims older than olderThan so another
+// indexer can pick the addon up. Returns the number of released claims.
+func ResetTimedOutIndexing(olderThan time.Duration) int64 {
+	return initializers.DB.Model(&models.Addon{}).
 		Where("is_being_indexed = ?", true).
-		Where("index_start_time < ?", time.Now().Add(-30*time.Minute)).
-		Update("is_being_indexed", false)
+		Where("index_start_time < ?", time.Now().Add(-olderThan)).
+		Update("is_being_indexed", false).RowsAffected
+}
+
+// sendWebhook sends a Discord webhook and records the outcome.
+func sendWebhook(kind, webhookURL string, hook models.CustomHook) error {
+	err := SendCustomWebhook(webhookURL, hook)
+	result := "ok"
+	if err != nil {
+		result = "error"
+	}
+	metrics.DiscordWebhooks.Inc(kind, result)
+	return err
 }
 
 func SendCustomWebhook(webhookURL string, hook models.CustomHook) error {

controllers/scraperController.go

@@ -1,31 +1,26 @@
 package controllers
 
 import (
-	"bytes"
 	"fmt"
-	"io"
 	"strconv"
+	"sync"
 	"time"
 
 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
 	"github.com/gin-gonic/gin"
 )
 
+var webhookMu sync.Mutex
 var lastWebhookTime time.Time
 var nbrOfUnsentWebhooks int
 
 func CreateAddon(c *gin.Context) {
 	var addonR models.Addon
-	body, err := io.ReadAll(c.Request.Body)
-	if err != nil {
-		c.JSON(400, gin.H{"error": "Failed to read request body"})
-		return
-	}
-	fmt.Println(string(body))
-	c.Request.Body = io.NopCloser(bytes.NewBuffer(body))
-	err = c.ShouldBindJSON(&addonR)
+	err := c.ShouldBindJSON(&addonR)
 	if err != nil {
+		metrics.ScraperAddons.Inc("invalid")
 		c.JSON(400, gin.H{"error": "Invalid JSON"})
 		return
 	}
@@ -37,17 +32,26 @@ func CreateAddon(c *gin.Context) {
 	result := initializers.DB.Where(&addon).First(&addon)
 	if result.Error != nil {
 		// addon does not exist, create it
-		initializers.DB.Create(&addonR)
+		if err := initializers.DB.Create(&addonR).Error; err != nil {
+			// usually a soft-deleted addon with the same ID; it can be restored from the admin dashboard
+			metrics.ScraperAddons.Inc("create_failed")
+			fmt.Println("Failed to create addon", addonR.ID, err)
+		} else {
+			metrics.ScraperAddons.Inc("created")
+		}
 
 		// send webhook about creation
+		webhookMu.Lock()
 		if time.Since(lastWebhookTime) < time.Second*10 {
 			nbrOfUnsentWebhooks++
+			webhookMu.Unlock()
 			return
 		}
 
 		lastWebhookTime = time.Now()
 		oldNbr := nbrOfUnsentWebhooks
 		nbrOfUnsentWebhooks = 0
+		webhookMu.Unlock()
 
 		go func() {
 			webhookURL := initializers.ScraperWebhookURL
@@ -82,7 +86,7 @@ func CreateAddon(c *gin.Context) {
 					Embeds:   []models.CustomEmbed{myEmbed},
 				}
 
-				err := SendCustomWebhook(webhookURL, myHook)
+				err := sendWebhook("scraper", webhookURL, myHook)
 				if err != nil {
 					fmt.Println("Error sending webhook:", err)
 				}
@@ -105,6 +109,7 @@ func CreateAddon(c *gin.Context) {
 			addon.Blocked = addonR.Blocked
 			addon.ToBeIndexed = true
 			initializers.DB.Save(&addon)
+			metrics.ScraperAddons.Inc("updated")
 
 			// send webhook about update
 			go func() {
@@ -140,12 +145,14 @@ func CreateAddon(c *gin.Context) {
 						Embeds:   []models.CustomEmbed{myEmbed},
 					}
 
-					err := SendCustomWebhook(webhookURL, myHook)
+					err := sendWebhook("scraper", webhookURL, myHook)
 					if err != nil {
 						fmt.Println("Error sending webhook:", err)
 					}
 				}
 			}()
+		} else {
+			metrics.ScraperAddons.Inc("unchanged")
 		}
 	}
 	c.JSON(200, gin.H{"status": "success"})

controllers/telemetryController.go

@@ -0,0 +1,53 @@
+package controllers
+
+import (
+	"net/http"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
+	"github.com/gin-gonic/gin"
+)
+
+const (
+	maxTelemetryBody    = 1 << 20
+	maxTelemetryMetrics = 1000
+	maxTelemetryEvents  = 200
+)
+
+// SubmitTelemetry receives metrics, events and a heartbeat from a service
+// (indexer, scraper, ...). See models.TelemetryBatch for the payload.
+func SubmitTelemetry(c *gin.Context) {
+	c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxTelemetryBody)
+
+	var batch models.TelemetryBatch
+	if err := c.ShouldBindJSON(&batch); err != nil {
+		c.JSON(400, gin.H{"error": "Invalid JSON"})
+		return
+	}
+	if !metrics.ValidName(batch.Service) || batch.Service == metrics.MainService {
+		c.JSON(400, gin.H{"error": "service must match ^[a-z_][a-z0-9_]{0,63}$ and not be \"main\""})
+		return
+	}
+	if batch.Instance == "" || len(batch.Instance) > 128 {
+		c.JSON(400, gin.H{"error": "instance is required (max 128 characters)"})
+		return
+	}
+	if len(batch.Metrics) > maxTelemetryMetrics || len(batch.Events) > maxTelemetryEvents {
+		c.JSON(413, gin.H{"error": "too many metrics or events in one batch"})
+		return
+	}
+
+	now := time.Now()
+	if err := metrics.Heartbeat(&batch, c.ClientIP(), now); err != nil {
+		c.JSON(500, gin.H{"error": "failed to store heartbeat"})
+		return
+	}
+	result := metrics.Ingest(batch.Service, batch.Instance, batch.Metrics, now)
+	if err := metrics.RecordEvents(batch.Service, batch.Instance, batch.Events, now); err != nil {
+		c.JSON(500, gin.H{"error": "failed to store events"})
+		return
+	}
+
+	c.JSON(200, result)
+}