admin dashboard, metrics and telemetry

- embedded React admin UI at /admin/ui (go:embed), login with ADMIN_SECRET
  via signed HttpOnly session cookie, X-ADMIN-KEY still works for scripts
- admin API: overview stats, indexing queue, addon management (priority,
  reindex, delete/restore/purge, bulk), whitelist CRUD, duplicate hashes,
  traffic, metrics explorer, services, events/audit log, system info
- built-in metrics: request latency per route, visitors, top addons and
  searches, indexer/scraper counters; minute buckets in the DB with hourly
  rollups and retention; optional Prometheus /metrics (METRICS_TOKEN)
- POST /back/telemetry for indexers/scrapers (metrics, events, heartbeat)
  and a stdlib-only Go client in telemetry/
- auth hardening: empty secrets no longer allow access, constant-time
  compare, CORS not applied to /admin and /back, TRUSTED_PROXIES setting
- Dockerfile builds the UI and no longer bakes config.yaml into the image;
  CI builds the UI, runs tests and builds arm64 with CGO

main.go

@@ -1,22 +1,49 @@
 package main
 
 import (
+	"context"
+	"errors"
+	"log"
 	"net/http"
+	"os/signal"
+	"strings"
+	"syscall"
+	"time"
 
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/admin"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/auth"
 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/controllers"
 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/web"
 
 	"github.com/gin-gonic/gin"
 )
 
+// buildCommit is set at build time: -ldflags "-X main.buildCommit=<sha>"
+var buildCommit = ""
+
 func init() {
 	initializers.Load()
 	initializers.ConnectToDB()
 }
 
 func main() {
-	r := gin.Default()
+	warnAboutConfig()
+
+	r := gin.New()
+	r.Use(gin.LoggerWithConfig(gin.LoggerConfig{Skip: skipLog}), gin.Recovery())
+	if len(initializers.TrustedProxies) > 0 {
+		if err := r.SetTrustedProxies(initializers.TrustedProxies); err != nil {
+			log.Fatalf("Invalid TRUSTED_PROXIES: %v", err)
+		}
+	}
+	metrics.SetEnabled(initializers.MetricsEnabled)
+	if initializers.MetricsEnabled {
+		r.Use(metrics.Middleware())
+	}
 	r.Use(CORSMiddleware())
+
 	r.GET("/health", func(c *gin.Context) {
 		c.JSON(http.StatusOK, gin.H{"status": "ok"})
 	})
@@ -29,14 +56,18 @@ func main() {
 		back.POST("/submitAddon", controllers.CheckAllowed, controllers.SaveIndexingResult)
 		back.DELETE("/deleteAddon", controllers.CheckAllowed, controllers.DeleteAddon)
 		back.POST("/addonDependencies", controllers.CheckAllowed, controllers.SetAddonDependencies)
+		back.POST("/telemetry", controllers.CheckAllowed, controllers.SubmitTelemetry)
 	}
 
-	admin := r.Group("/admin")
-	{
-		admin.POST("/addWhitelistedHash", controllers.CheckAdmin, controllers.AddWhitelistedHash)
-		admin.GET("/duplicates", controllers.CheckAdmin)
-		admin.POST("/setPriority", controllers.CheckAdmin)
-		admin.POST("/setToBeIndexed", controllers.CheckAdmin)
+	// admin JSON API, legacy admin endpoint and the embedded dashboard
+	admin.Register(r)
+	web.Register(r)
+
+	if initializers.MetricsToken != "" {
+		r.GET("/metrics", requireMetricsToken, func(c *gin.Context) {
+			c.Header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
+			metrics.WritePrometheus(c.Writer)
+		})
 	}
 
 	v1 := r.Group("/v1")
@@ -47,11 +78,95 @@ func main() {
 		v1.GET("/getPossible", controllers.GetPossibleAddons)
 	}
 
-	r.Run(initializers.ServerHost + ":" + initializers.ServerPort)
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+
+	version := buildCommit
+	if version == "" {
+		version = "dev"
+	}
+	admin.BuildCommit = buildCommit
+	metrics.SetVersion(version)
+	if initializers.MetricsEnabled {
+		metrics.Start(ctx)
+	}
+	admin.StartBackground(ctx)
+
+	srv := &http.Server{
+		Addr:              initializers.ServerHost + ":" + initializers.ServerPort,
+		Handler:           r,
+		ReadHeaderTimeout: 10 * time.Second,
+	}
+	go func() {
+		log.Printf("Listening on %s (admin dashboard at /admin/ui/)", srv.Addr)
+		if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+			log.Fatalf("Server error: %v", err)
+		}
+	}()
+
+	<-ctx.Done()
+	log.Println("Shutting down...")
+	shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	if err := srv.Shutdown(shutdownCtx); err != nil {
+		log.Printf("Shutdown error: %v", err)
+	}
+	if initializers.MetricsEnabled {
+		metrics.FlushAll()
+	}
+}
+
+// skipLog keeps health checks, dashboard assets and dashboard polling out of
+// the access log.
+func skipLog(c *gin.Context) bool {
+	p := c.Request.URL.Path
+	if p == "/health" || strings.HasPrefix(p, "/admin/ui") {
+		return true
+	}
+	return strings.HasPrefix(p, "/admin/api") && c.Request.Method == http.MethodGet && c.Writer.Status() < 400
+}
+
+func requireMetricsToken(c *gin.Context) {
+	token := strings.TrimPrefix(c.GetHeader("Authorization"), "Bearer ")
+	if auth.SecretEqual(token, initializers.MetricsToken) {
+		return
+	}
+	// also allow a logged-in admin to look at the raw exposition
+	if auth.SecretEqual(c.GetHeader("X-ADMIN-KEY"), initializers.AdminSecret) {
+		return
+	}
+	if cookie, err := c.Cookie(auth.CookieName); err == nil {
+		if _, ok := auth.VerifyToken(cookie, time.Now()); ok {
+			return
+		}
+	}
+	c.Header("WWW-Authenticate", `Bearer realm="metrics"`)
+	c.AbortWithStatus(http.StatusUnauthorized)
+}
+
+func warnAboutConfig() {
+	if initializers.SecretKey == "" {
+		log.Println("Warning: SECRET_KEY is not set - /back endpoints are disabled")
+	}
+	if initializers.AdminSecret == "" {
+		log.Println("Warning: ADMIN_SECRET is not set - the admin dashboard is disabled")
+	} else if len(initializers.AdminSecret) < 16 {
+		log.Println("Warning: ADMIN_SECRET is shorter than 16 characters")
+	}
+	if len(initializers.TrustedProxies) == 0 {
+		log.Println("Warning: TRUSTED_PROXIES is not set - client IPs from X-Forwarded-For are trusted from any source")
+	}
 }
 
 func CORSMiddleware() gin.HandlerFunc {
 	return func(c *gin.Context) {
+		// the admin API and worker endpoints are never called cross-origin
+		p := c.Request.URL.Path
+		if strings.HasPrefix(p, "/admin") || strings.HasPrefix(p, "/back") || p == "/metrics" {
+			c.Next()
+			return
+		}
+
 		orig := c.Request.Header.Clone().Get("Origin")
 		c.Writer.Header().Set("Access-Control-Allow-Origin", orig)
 		c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
@@ -66,13 +181,3 @@ func CORSMiddleware() gin.HandlerFunc {
 		c.Next()
 	}
 }
-
-func OptionsMiddleware() gin.HandlerFunc {
-	return func(c *gin.Context) {
-		if c.Request.Method != "OPTIONS" {
-			c.Next()
-		} else {
-			c.AbortWithStatus(http.StatusOK)
-		}
-	}
-}