- embedded React admin UI at /admin/ui (go:embed), login with ADMIN_SECRET via signed HttpOnly session cookie, X-ADMIN-KEY still works for scripts - admin API: overview stats, indexing queue, addon management (priority, reindex, delete/restore/purge, bulk), whitelist CRUD, duplicate hashes, traffic, metrics explorer, services, events/audit log, system info - built-in metrics: request latency per route, visitors, top addons and searches, indexer/scraper counters; minute buckets in the DB with hourly rollups and retention; optional Prometheus /metrics (METRICS_TOKEN) - POST /back/telemetry for indexers/scrapers (metrics, events, heartbeat) and a stdlib-only Go client in telemetry/ - auth hardening: empty secrets no longer allow access, constant-time compare, CORS not applied to /admin and /back, TRUSTED_PROXIES setting - Dockerfile builds the UI and no longer bakes config.yaml into the image; CI builds the UI, runs tests and builds arm64 with CGO
This commit is contained in:
ilbinek committed
1 parent
0ef27b0072
commit
56d9e10621
79 files changed
+13146
-118
No files matched your search
+244
@@ -0,0 +1,244 @@
|
||||
package admin
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"os"
|
||||
"runtime"
|
||||
"runtime/debug"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
|
||||
"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
|
||||
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// BuildCommit is set by main from -ldflags.
|
||||
var BuildCommit = ""
|
||||
|
||||
type serviceRow struct {
|
||||
models.ServiceInstance
|
||||
Online bool `json:"online"`
|
||||
UptimeSec int64 `json:"uptimeSec"`
|
||||
MetaMap map[string]any `json:"metaData,omitempty"`
|
||||
}
|
||||
|
||||
func listServices(c *gin.Context) {
|
||||
var rows []models.ServiceInstance
|
||||
if err := initializers.DB.Order("service ASC, instance ASC").Find(&rows).Error; err != nil {
|
||||
fail(c, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
out := make([]serviceRow, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
s := serviceRow{ServiceInstance: r, Online: metrics.Online(&r, now)}
|
||||
if r.StartedAt > 0 && s.Online {
|
||||
s.UptimeSec = now.Unix() - r.StartedAt
|
||||
}
|
||||
if r.Meta != "" {
|
||||
_ = json.Unmarshal([]byte(r.Meta), &s.MetaMap)
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"items": out})
|
||||
}
|
||||
|
||||
func deleteService(c *gin.Context) {
|
||||
id, err := strconv.ParseUint(c.Param("id"), 10, 64)
|
||||
if err != nil {
|
||||
fail(c, http.StatusBadRequest, "invalid id")
|
||||
return
|
||||
}
|
||||
var row models.ServiceInstance
|
||||
if err := initializers.DB.First(&row, id).Error; err != nil {
|
||||
fail(c, http.StatusNotFound, "not found")
|
||||
return
|
||||
}
|
||||
initializers.DB.Delete(&row)
|
||||
audit(c, "service_forget", map[string]any{"service": row.Service, "instance": row.Instance})
|
||||
c.JSON(http.StatusOK, gin.H{"deleted": id})
|
||||
}
|
||||
|
||||
type eventRow struct {
|
||||
models.ServiceEvent
|
||||
FieldsMap map[string]any `json:"fieldsData,omitempty"`
|
||||
}
|
||||
|
||||
func listEvents(c *gin.Context) {
|
||||
limit := queryInt(c, "limit", 100, 1, 500)
|
||||
q := initializers.DB.Model(&models.ServiceEvent{})
|
||||
if v := c.Query("service"); v != "" {
|
||||
q = q.Where("service = ?", v)
|
||||
}
|
||||
if v := c.Query("instance"); v != "" {
|
||||
q = q.Where("instance = ?", v)
|
||||
}
|
||||
if v := c.Query("level"); v != "" {
|
||||
levels := strings.Split(v, ",")
|
||||
q = q.Where("level IN ?", levels)
|
||||
}
|
||||
if c.Query("audit") == "1" {
|
||||
q = q.Where("fields LIKE ?", `%"kind":"audit"%`)
|
||||
}
|
||||
if v := strings.TrimSpace(c.Query("q")); v != "" {
|
||||
q = q.Where(`LOWER(message) LIKE ? ESCAPE '\'`, "%"+likeEscape(strings.ToLower(v))+"%")
|
||||
}
|
||||
if before := queryInt64(c, "before", 0); before > 0 {
|
||||
q = q.Where("id < ?", before)
|
||||
}
|
||||
var rows []models.ServiceEvent
|
||||
if err := q.Order("id DESC").Limit(limit).Find(&rows).Error; err != nil {
|
||||
fail(c, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
out := make([]eventRow, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
e := eventRow{ServiceEvent: r}
|
||||
if r.Fields != "" {
|
||||
_ = json.Unmarshal([]byte(r.Fields), &e.FieldsMap)
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
var next uint
|
||||
if len(rows) == limit {
|
||||
next = rows[len(rows)-1].ID
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"items": out, "next": next})
|
||||
}
|
||||
|
||||
func buildInfo() gin.H {
|
||||
info := gin.H{"commit": BuildCommit}
|
||||
if bi, ok := debug.ReadBuildInfo(); ok {
|
||||
info["module"] = bi.Main.Path
|
||||
for _, s := range bi.Settings {
|
||||
switch s.Key {
|
||||
case "vcs.revision":
|
||||
if BuildCommit == "" {
|
||||
info["commit"] = s.Value
|
||||
}
|
||||
case "vcs.time":
|
||||
info["time"] = s.Value
|
||||
case "vcs.modified":
|
||||
info["modified"] = s.Value == "true"
|
||||
}
|
||||
}
|
||||
}
|
||||
return info
|
||||
}
|
||||
|
||||
func system(c *gin.Context) {
|
||||
var ms runtime.MemStats
|
||||
runtime.ReadMemStats(&ms)
|
||||
host, _ := os.Hostname()
|
||||
started := metrics.StartedAt()
|
||||
|
||||
resp := gin.H{
|
||||
"go": gin.H{
|
||||
"version": runtime.Version(), "os": runtime.GOOS, "arch": runtime.GOARCH,
|
||||
"cpus": runtime.NumCPU(), "maxProcs": runtime.GOMAXPROCS(0), "goroutines": runtime.NumGoroutine(),
|
||||
},
|
||||
"process": gin.H{
|
||||
"pid": os.Getpid(), "hostname": host, "startedAt": started.Unix(),
|
||||
"uptimeSec": int64(time.Since(started).Seconds()),
|
||||
},
|
||||
"memory": gin.H{
|
||||
"heapAlloc": ms.HeapAlloc, "heapInuse": ms.HeapInuse, "sys": ms.Sys,
|
||||
"numGC": ms.NumGC, "pauseTotalNs": ms.PauseTotalNs, "lastGC": int64(ms.LastGC / 1e9),
|
||||
},
|
||||
"build": buildInfo(),
|
||||
"metrics": metrics.Status(),
|
||||
"config": gin.H{
|
||||
"dbDriver": initializers.DB.Dialector.Name(),
|
||||
"trustedProxies": initializers.TrustedProxies,
|
||||
"cookieSecure": initializers.AdminCookieSecure,
|
||||
"sessionTtlHours": initializers.AdminSessionTTL.Hours(),
|
||||
"metricsEnabled": initializers.MetricsEnabled,
|
||||
"prometheusEndpoint": initializers.MetricsToken != "",
|
||||
"metricsMinuteRetentionDays": initializers.MetricsMinuteRetentionDays,
|
||||
"metricsHourRetentionDays": initializers.MetricsHourRetentionDays,
|
||||
"eventsRetentionDays": initializers.EventsRetentionDays,
|
||||
"topItemsRetentionDays": initializers.TopItemsRetentionDays,
|
||||
"indexTimeoutMinutes": initializers.IndexTimeout.Minutes(),
|
||||
"adminSecretStrong": len(initializers.AdminSecret) >= 16,
|
||||
"secretKeySet": initializers.SecretKey != "",
|
||||
"discordWebhook": initializers.DiscordWebhookURL != "",
|
||||
"scraperWebhook": initializers.ScraperWebhookURL != "",
|
||||
},
|
||||
}
|
||||
if sqlDB, err := initializers.DB.DB(); err == nil {
|
||||
st := sqlDB.Stats()
|
||||
resp["dbPool"] = gin.H{
|
||||
"open": st.OpenConnections, "inUse": st.InUse, "idle": st.Idle,
|
||||
"maxOpen": st.MaxOpenConnections, "waitCount": st.WaitCount, "waitMs": st.WaitDuration.Milliseconds(),
|
||||
}
|
||||
start := time.Now()
|
||||
if err := sqlDB.Ping(); err == nil {
|
||||
resp["dbPingMs"] = float64(time.Since(start).Microseconds()) / 1000
|
||||
} else {
|
||||
resp["dbPingError"] = err.Error()
|
||||
}
|
||||
}
|
||||
if h := currentHeavy(); h != nil {
|
||||
resp["db"] = h
|
||||
}
|
||||
c.JSON(http.StatusOK, resp)
|
||||
}
|
||||
|
||||
func maintenance(c *gin.Context) {
|
||||
var body struct {
|
||||
Action string `json:"action"`
|
||||
OlderDays int `json:"olderThanDays"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&body); err != nil {
|
||||
fail(c, http.StatusBadRequest, "invalid JSON")
|
||||
return
|
||||
}
|
||||
switch body.Action {
|
||||
case "refresh-stats":
|
||||
requestHeavyRefresh(true)
|
||||
case "flush-metrics":
|
||||
metrics.FlushAll()
|
||||
case "rollup":
|
||||
go metrics.RunMaintenance(time.Now())
|
||||
case "purge-deleted-files":
|
||||
days := body.OlderDays
|
||||
if days <= 0 {
|
||||
days = 7
|
||||
}
|
||||
go purgeDeletedFiles(days)
|
||||
default:
|
||||
fail(c, http.StatusBadRequest, "unknown action")
|
||||
return
|
||||
}
|
||||
audit(c, "maintenance_"+strings.ReplaceAll(body.Action, "-", "_"), map[string]any{"olderThanDays": body.OlderDays})
|
||||
c.JSON(http.StatusAccepted, gin.H{"started": body.Action})
|
||||
}
|
||||
|
||||
// purgeDeletedFiles hard-deletes file rows that were soft-deleted (replaced by
|
||||
// a newer index) more than `days` ago, in batches.
|
||||
func purgeDeletedFiles(days int) {
|
||||
cutoff := time.Now().AddDate(0, 0, -days)
|
||||
started := time.Now()
|
||||
var total int64
|
||||
for {
|
||||
res := initializers.DB.Exec(`DELETE FROM addon_files WHERE id IN (
|
||||
SELECT id FROM addon_files WHERE deleted_at IS NOT NULL AND deleted_at < ? LIMIT 5000)`, cutoff)
|
||||
if res.Error != nil {
|
||||
metrics.Event("error", "purge of deleted file rows failed", map[string]any{"kind": "maintenance", "error": res.Error.Error(), "deleted": total})
|
||||
return
|
||||
}
|
||||
total += res.RowsAffected
|
||||
if res.RowsAffected == 0 {
|
||||
break
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
metrics.Event("info", "purged deleted file rows", map[string]any{
|
||||
"kind": "maintenance", "deleted": total, "olderThanDays": days, "tookSec": int64(time.Since(started).Seconds()),
|
||||
})
|
||||
requestHeavyRefresh(false)
|
||||
}
|
||||
Reference in new issue
Block a user