admin dashboard, metrics and telemetry

- embedded React admin UI at /admin/ui (go:embed), login with ADMIN_SECRET
  via signed HttpOnly session cookie, X-ADMIN-KEY still works for scripts
- admin API: overview stats, indexing queue, addon management (priority,
  reindex, delete/restore/purge, bulk), whitelist CRUD, duplicate hashes,
  traffic, metrics explorer, services, events/audit log, system info
- built-in metrics: request latency per route, visitors, top addons and
  searches, indexer/scraper counters; minute buckets in the DB with hourly
  rollups and retention; optional Prometheus /metrics (METRICS_TOKEN)
- POST /back/telemetry for indexers/scrapers (metrics, events, heartbeat)
  and a stdlib-only Go client in telemetry/
- auth hardening: empty secrets no longer allow access, constant-time
  compare, CORS not applied to /admin and /back, TRUSTED_PROXIES setting
- Dockerfile builds the UI and no longer bakes config.yaml into the image;
  CI builds the UI, runs tests and builds arm64 with CGO

controllers/scraperController.go

@@ -1,31 +1,26 @@
 package controllers
 
 import (
-	"bytes"
 	"fmt"
-	"io"
 	"strconv"
+	"sync"
 	"time"
 
 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
 	"github.com/gin-gonic/gin"
 )
 
+var webhookMu sync.Mutex
 var lastWebhookTime time.Time
 var nbrOfUnsentWebhooks int
 
 func CreateAddon(c *gin.Context) {
 	var addonR models.Addon
-	body, err := io.ReadAll(c.Request.Body)
-	if err != nil {
-		c.JSON(400, gin.H{"error": "Failed to read request body"})
-		return
-	}
-	fmt.Println(string(body))
-	c.Request.Body = io.NopCloser(bytes.NewBuffer(body))
-	err = c.ShouldBindJSON(&addonR)
+	err := c.ShouldBindJSON(&addonR)
 	if err != nil {
+		metrics.ScraperAddons.Inc("invalid")
 		c.JSON(400, gin.H{"error": "Invalid JSON"})
 		return
 	}
@@ -37,17 +32,26 @@ func CreateAddon(c *gin.Context) {
 	result := initializers.DB.Where(&addon).First(&addon)
 	if result.Error != nil {
 		// addon does not exist, create it
-		initializers.DB.Create(&addonR)
+		if err := initializers.DB.Create(&addonR).Error; err != nil {
+			// usually a soft-deleted addon with the same ID; it can be restored from the admin dashboard
+			metrics.ScraperAddons.Inc("create_failed")
+			fmt.Println("Failed to create addon", addonR.ID, err)
+		} else {
+			metrics.ScraperAddons.Inc("created")
+		}
 
 		// send webhook about creation
+		webhookMu.Lock()
 		if time.Since(lastWebhookTime) < time.Second*10 {
 			nbrOfUnsentWebhooks++
+			webhookMu.Unlock()
 			return
 		}
 
 		lastWebhookTime = time.Now()
 		oldNbr := nbrOfUnsentWebhooks
 		nbrOfUnsentWebhooks = 0
+		webhookMu.Unlock()
 
 		go func() {
 			webhookURL := initializers.ScraperWebhookURL
@@ -82,7 +86,7 @@ func CreateAddon(c *gin.Context) {
 					Embeds:   []models.CustomEmbed{myEmbed},
 				}
 
-				err := SendCustomWebhook(webhookURL, myHook)
+				err := sendWebhook("scraper", webhookURL, myHook)
 				if err != nil {
 					fmt.Println("Error sending webhook:", err)
 				}
@@ -105,6 +109,7 @@ func CreateAddon(c *gin.Context) {
 			addon.Blocked = addonR.Blocked
 			addon.ToBeIndexed = true
 			initializers.DB.Save(&addon)
+			metrics.ScraperAddons.Inc("updated")
 
 			// send webhook about update
 			go func() {
@@ -140,12 +145,14 @@ func CreateAddon(c *gin.Context) {
 						Embeds:   []models.CustomEmbed{myEmbed},
 					}
 
-					err := SendCustomWebhook(webhookURL, myHook)
+					err := sendWebhook("scraper", webhookURL, myHook)
 					if err != nil {
 						fmt.Println("Error sending webhook:", err)
 					}
 				}
 			}()
+		} else {
+			metrics.ScraperAddons.Inc("unchanged")
 		}
 	}
 	c.JSON(200, gin.H{"status": "success"})