- embedded React admin UI at /admin/ui (go:embed), login with ADMIN_SECRET via signed HttpOnly session cookie, X-ADMIN-KEY still works for scripts - admin API: overview stats, indexing queue, addon management (priority, reindex, delete/restore/purge, bulk), whitelist CRUD, duplicate hashes, traffic, metrics explorer, services, events/audit log, system info - built-in metrics: request latency per route, visitors, top addons and searches, indexer/scraper counters; minute buckets in the DB with hourly rollups and retention; optional Prometheus /metrics (METRICS_TOKEN) - POST /back/telemetry for indexers/scrapers (metrics, events, heartbeat) and a stdlib-only Go client in telemetry/ - auth hardening: empty secrets no longer allow access, constant-time compare, CORS not applied to /admin and /back, TRUSTED_PROXIES setting - Dockerfile builds the UI and no longer bakes config.yaml into the image; CI builds the UI, runs tests and builds arm64 with CGO
This commit is contained in:
ilbinek committed
1 parent
0ef27b0072
commit
56d9e10621
79 files changed
+13146
-118
No files matched your search
@@ -0,0 +1,443 @@
|
||||
package admin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
|
||||
"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
|
||||
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
|
||||
"github.com/gin-gonic/gin"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type addonCounts struct {
|
||||
Total int64 `json:"total"`
|
||||
Indexed int64 `json:"indexed"`
|
||||
Queued int64 `json:"queued"`
|
||||
Indexing int64 `json:"indexing"`
|
||||
Priority int64 `json:"priority"`
|
||||
Recheck int64 `json:"recheck"`
|
||||
Blocked int64 `json:"blocked"`
|
||||
Unlisted int64 `json:"unlisted"`
|
||||
Private int64 `json:"private"`
|
||||
TotalSize int64 `json:"totalSizeBytes"`
|
||||
Added24h int64 `json:"added24h"`
|
||||
Added7d int64 `json:"added7d"`
|
||||
Indexed24h int64 `json:"indexed24h"`
|
||||
|
||||
SoftDeleted int64 `json:"softDeleted"`
|
||||
}
|
||||
|
||||
type typeCount struct {
|
||||
Type string `json:"type"`
|
||||
Count int64 `json:"count"`
|
||||
}
|
||||
|
||||
type miniAddon struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Author string `json:"author"`
|
||||
SubscriberCount int64 `json:"subscriberCount"`
|
||||
CurrentVersionSize int64 `json:"currentVersionSize"`
|
||||
LastIndexedAt *time.Time `json:"lastIndexedAt,omitempty"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
}
|
||||
|
||||
type overviewData struct {
|
||||
Addons addonCounts `json:"addons"`
|
||||
Types []typeCount `json:"types"`
|
||||
Whitelist int64 `json:"whitelistCount"`
|
||||
Dependencies map[string]any `json:"dependencies"`
|
||||
Traffic map[string]any `json:"traffic"`
|
||||
Activity map[string]any `json:"activity"`
|
||||
Services map[string]any `json:"services"`
|
||||
TopSubscribed []miniAddon `json:"topSubscribed"`
|
||||
Largest []miniAddon `json:"largest"`
|
||||
RecentlyAdded []miniAddon `json:"recentlyAdded"`
|
||||
RecentlyIndexed []miniAddon `json:"recentlyIndexed"`
|
||||
Heavy *heavyStats `json:"heavy"`
|
||||
}
|
||||
|
||||
func loadAddonCounts() (addonCounts, error) {
|
||||
var ac addonCounts
|
||||
now := time.Now()
|
||||
err := initializers.DB.Raw(`SELECT COUNT(*) AS total,
|
||||
COALESCE(SUM(CASE WHEN to_be_indexed = ? THEN 1 ELSE 0 END), 0) AS indexed,
|
||||
COALESCE(SUM(CASE WHEN to_be_indexed = ? AND is_being_indexed = ? THEN 1 ELSE 0 END), 0) AS queued,
|
||||
COALESCE(SUM(CASE WHEN is_being_indexed = ? THEN 1 ELSE 0 END), 0) AS indexing,
|
||||
COALESCE(SUM(CASE WHEN priority_indexing = ? AND to_be_indexed = ? THEN 1 ELSE 0 END), 0) AS priority,
|
||||
COALESCE(SUM(CASE WHEN recheck = ? THEN 1 ELSE 0 END), 0) AS recheck,
|
||||
COALESCE(SUM(CASE WHEN blocked = ? THEN 1 ELSE 0 END), 0) AS blocked,
|
||||
COALESCE(SUM(CASE WHEN unlisted = ? THEN 1 ELSE 0 END), 0) AS unlisted,
|
||||
COALESCE(SUM(CASE WHEN private = ? THEN 1 ELSE 0 END), 0) AS private,
|
||||
CAST(COALESCE(SUM(current_version_size), 0) AS BIGINT) AS total_size,
|
||||
COALESCE(SUM(CASE WHEN created_at >= ? THEN 1 ELSE 0 END), 0) AS added24h,
|
||||
COALESCE(SUM(CASE WHEN created_at >= ? THEN 1 ELSE 0 END), 0) AS added7d,
|
||||
COALESCE(SUM(CASE WHEN last_indexed_at >= ? THEN 1 ELSE 0 END), 0) AS indexed24h
|
||||
FROM addons WHERE deleted_at IS NULL`,
|
||||
false, true, false, true, true, true, true, true, true, true,
|
||||
now.Add(-24*time.Hour), now.Add(-7*24*time.Hour), now.Add(-24*time.Hour),
|
||||
).Scan(&ac).Error
|
||||
if err != nil {
|
||||
return ac, err
|
||||
}
|
||||
err = initializers.DB.Unscoped().Model(&models.Addon{}).Where("deleted_at IS NOT NULL").Count(&ac.SoftDeleted).Error
|
||||
return ac, err
|
||||
}
|
||||
|
||||
func loadMiniAddons(order string, where string, args ...any) []miniAddon {
|
||||
out := []miniAddon{}
|
||||
q := initializers.DB.Model(&models.Addon{}).
|
||||
Select("id, name, author, subscriber_count, current_version_size, last_indexed_at, created_at")
|
||||
if where != "" {
|
||||
q = q.Where(where, args...)
|
||||
}
|
||||
q.Order(order).Limit(8).Scan(&out)
|
||||
return out
|
||||
}
|
||||
|
||||
func buildOverview() (*overviewData, error) {
|
||||
ac, err := loadAddonCounts()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d := &overviewData{Addons: ac}
|
||||
|
||||
d.Types = []typeCount{}
|
||||
initializers.DB.Model(&models.Addon{}).Select("type, COUNT(*) AS count").Group("type").Order("count DESC").Limit(12).Scan(&d.Types)
|
||||
|
||||
initializers.DB.Model(&models.WhitelistedHash{}).Count(&d.Whitelist)
|
||||
|
||||
var deps struct {
|
||||
Edges int64
|
||||
WithDeps int64
|
||||
Dependents int64
|
||||
}
|
||||
initializers.DB.Raw(`SELECT COUNT(*) AS edges, COUNT(DISTINCT addon_id) AS with_deps,
|
||||
COUNT(DISTINCT dependency_id) AS dependents FROM addon_dependencies`).Scan(&deps)
|
||||
d.Dependencies = map[string]any{"edges": deps.Edges, "addonsWithDependencies": deps.WithDeps, "addonsDependedOn": deps.Dependents}
|
||||
|
||||
now := time.Now().Unix()
|
||||
dayAgo := now - 86400
|
||||
httpSel := metrics.Selector{Service: metrics.MainService, Name: "http_request_duration_ms"}
|
||||
traffic := map[string]any{}
|
||||
if t, err := metrics.Totals(metrics.Selector{Service: httpSel.Service, Name: httpSel.Name, GroupBy: []string{"group"}}, dayAgo, now); err == nil {
|
||||
var total int64
|
||||
for _, g := range t {
|
||||
total += g.Count
|
||||
if g.Labels["group"] == "v1" {
|
||||
traffic["websiteRequests24h"] = g.Count
|
||||
traffic["websiteP95Ms"] = g.P95
|
||||
traffic["websiteP50Ms"] = g.P50
|
||||
}
|
||||
if g.Labels["group"] == "back" {
|
||||
traffic["workerRequests24h"] = g.Count
|
||||
}
|
||||
}
|
||||
traffic["requests24h"] = total
|
||||
}
|
||||
traffic["errors5xx24h"] = metrics.Sum(metrics.Selector{Service: httpSel.Service, Name: httpSel.Name, Match: map[string]string{"status": "5xx"}}, dayAgo, now)
|
||||
humans, bots := metrics.Visitors()
|
||||
traffic["visitorsToday"] = humans
|
||||
traffic["botsToday"] = bots
|
||||
d.Traffic = traffic
|
||||
|
||||
sum := func(name string, match map[string]string) float64 {
|
||||
return metrics.Sum(metrics.Selector{Service: metrics.MainService, Name: name, Match: match}, dayAgo, now)
|
||||
}
|
||||
d.Activity = map[string]any{
|
||||
"scraperCreated24h": sum("scraper_addons_total", map[string]string{"action": "created"}),
|
||||
"scraperUpdated24h": sum("scraper_addons_total", map[string]string{"action": "updated"}),
|
||||
"scraperCreateFailed24h": sum("scraper_addons_total", map[string]string{"action": "create_failed"}),
|
||||
"deleted24h": sum("scraper_addons_deleted_total", map[string]string{"result": "deleted"}),
|
||||
"indexed24h": sum("indexer_submissions_total", map[string]string{"result": "ok"}),
|
||||
"claims24h": sum("indexer_claims_total", nil),
|
||||
"timeoutsReset24h": sum("indexer_timeouts_reset_total", nil),
|
||||
"dependencyReports24h": sum("dependency_reports_total", nil),
|
||||
}
|
||||
|
||||
var instances []models.ServiceInstance
|
||||
initializers.DB.Find(&instances)
|
||||
online := 0
|
||||
for i := range instances {
|
||||
if metrics.Online(&instances[i], time.Now()) {
|
||||
online++
|
||||
}
|
||||
}
|
||||
d.Services = map[string]any{"online": online, "total": len(instances)}
|
||||
|
||||
d.TopSubscribed = loadMiniAddons("subscriber_count DESC", "")
|
||||
d.Largest = loadMiniAddons("current_version_size DESC", "")
|
||||
d.RecentlyAdded = loadMiniAddons("created_at DESC", "")
|
||||
d.RecentlyIndexed = loadMiniAddons("last_indexed_at DESC", "last_indexed_at IS NOT NULL")
|
||||
|
||||
d.Heavy = currentHeavy()
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func overview(c *gin.Context) {
|
||||
data, at, err := cached("overview", 20*time.Second, buildOverview)
|
||||
if err != nil {
|
||||
fail(c, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"asOf": at.Unix(), "data": data})
|
||||
}
|
||||
|
||||
// ---- heavy statistics (background) -----------------------------------------
|
||||
|
||||
type tableInfo struct {
|
||||
Name string `json:"name"`
|
||||
Rows int64 `json:"rows"`
|
||||
DeadRows int64 `json:"deadRows"`
|
||||
SizeBytes int64 `json:"sizeBytes"`
|
||||
}
|
||||
|
||||
type dupHash struct {
|
||||
Hash string `json:"hash"`
|
||||
AddonCount int64 `json:"addonCount"`
|
||||
FileCount int64 `json:"fileCount"`
|
||||
SamplePath string `json:"samplePath"`
|
||||
Whitelisted bool `json:"whitelisted"`
|
||||
}
|
||||
|
||||
type heavyStats struct {
|
||||
AsOf int64 `json:"asOf"`
|
||||
FilesLive int64 `json:"filesLive"`
|
||||
FilesSoftDeleted int64 `json:"filesSoftDeleted"`
|
||||
FilesApprox bool `json:"filesApprox"`
|
||||
DistinctHashes int64 `json:"distinctHashes"`
|
||||
DistinctAsOf int64 `json:"distinctAsOf"`
|
||||
DBDriver string `json:"dbDriver"`
|
||||
DBSizeBytes int64 `json:"dbSizeBytes"`
|
||||
Tables []tableInfo `json:"tables"`
|
||||
Duplicates []dupHash `json:"-"`
|
||||
DuplicatesAsOf int64 `json:"duplicatesAsOf"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
var (
|
||||
heavyMu sync.Mutex
|
||||
heavy *heavyStats
|
||||
refreshCh = make(chan bool, 1)
|
||||
)
|
||||
|
||||
func currentHeavy() *heavyStats {
|
||||
heavyMu.Lock()
|
||||
defer heavyMu.Unlock()
|
||||
return heavy
|
||||
}
|
||||
|
||||
// requestHeavyRefresh triggers a refresh; full=true also recomputes the
|
||||
// expensive duplicate statistics.
|
||||
func requestHeavyRefresh(full bool) {
|
||||
select {
|
||||
case refreshCh <- full:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func heavyStatsLoop(ctx context.Context) {
|
||||
lastFull := time.Time{}
|
||||
run := func(full bool) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("admin: heavy stats panicked: %v", r)
|
||||
}
|
||||
}()
|
||||
full = full || time.Since(lastFull) > time.Hour
|
||||
computeHeavy(full)
|
||||
if full {
|
||||
lastFull = time.Now()
|
||||
}
|
||||
}
|
||||
// give the server a moment to start before the first scan
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(5 * time.Second):
|
||||
}
|
||||
run(true)
|
||||
ticker := time.NewTicker(10 * time.Minute)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
run(false)
|
||||
case full := <-refreshCh:
|
||||
run(full)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// withTimeout runs fn in a transaction with a statement timeout on Postgres.
|
||||
func withTimeout(seconds int, fn func(tx *gorm.DB) error) error {
|
||||
return initializers.DB.Transaction(func(tx *gorm.DB) error {
|
||||
if isPostgres() {
|
||||
if err := tx.Exec("SET LOCAL statement_timeout = " + strconv.Itoa(seconds*1000)).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return fn(tx)
|
||||
})
|
||||
}
|
||||
|
||||
var ourTables = []string{"addons", "addon_files", "addon_dependencies", "whitelisted_hashes",
|
||||
"metric_series", "metric_points", "service_instances", "service_events", "daily_top_items"}
|
||||
|
||||
func computeHeavy(full bool) {
|
||||
started := time.Now()
|
||||
h := &heavyStats{AsOf: time.Now().Unix(), DBDriver: initializers.DB.Dialector.Name(), Tables: []tableInfo{}}
|
||||
prev := currentHeavy()
|
||||
if prev != nil {
|
||||
h.DistinctHashes, h.DistinctAsOf = prev.DistinctHashes, prev.DistinctAsOf
|
||||
h.Duplicates, h.DuplicatesAsOf = prev.Duplicates, prev.DuplicatesAsOf
|
||||
}
|
||||
|
||||
var errs []string
|
||||
note := func(what string, err error) {
|
||||
if err != nil {
|
||||
errs = append(errs, what+": "+err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
if isPostgres() {
|
||||
note("tables", initializers.DB.Raw(`SELECT relname AS name, n_live_tup AS rows, n_dead_tup AS dead_rows,
|
||||
pg_total_relation_size(relid) AS size_bytes FROM pg_stat_user_tables ORDER BY size_bytes DESC`).Scan(&h.Tables).Error)
|
||||
note("db size", initializers.DB.Raw(`SELECT pg_database_size(current_database())`).Scan(&h.DBSizeBytes).Error)
|
||||
note("soft-deleted files", withTimeout(60, func(tx *gorm.DB) error {
|
||||
return tx.Raw(`SELECT COUNT(*) FROM addon_files WHERE deleted_at IS NOT NULL`).Scan(&h.FilesSoftDeleted).Error
|
||||
}))
|
||||
for _, t := range h.Tables {
|
||||
if t.Name == "addon_files" {
|
||||
h.FilesLive = max(0, t.Rows-h.FilesSoftDeleted)
|
||||
h.FilesApprox = true
|
||||
}
|
||||
}
|
||||
} else {
|
||||
var pageCount, pageSize int64
|
||||
initializers.DB.Raw("PRAGMA page_count").Scan(&pageCount)
|
||||
initializers.DB.Raw("PRAGMA page_size").Scan(&pageSize)
|
||||
h.DBSizeBytes = pageCount * pageSize
|
||||
if st, err := os.Stat(sqlitePath() + "-wal"); err == nil {
|
||||
h.DBSizeBytes += st.Size()
|
||||
}
|
||||
for _, name := range ourTables {
|
||||
var n int64
|
||||
if err := initializers.DB.Table(name).Count(&n).Error; err == nil {
|
||||
h.Tables = append(h.Tables, tableInfo{Name: name, Rows: n})
|
||||
}
|
||||
}
|
||||
note("files", initializers.DB.Raw(`SELECT
|
||||
COALESCE(SUM(CASE WHEN deleted_at IS NULL THEN 1 ELSE 0 END), 0),
|
||||
COALESCE(SUM(CASE WHEN deleted_at IS NOT NULL THEN 1 ELSE 0 END), 0) FROM addon_files`).Row().
|
||||
Scan(&h.FilesLive, &h.FilesSoftDeleted))
|
||||
}
|
||||
|
||||
if full {
|
||||
var distinct int64
|
||||
err := withTimeout(120, func(tx *gorm.DB) error {
|
||||
return tx.Raw(`SELECT COUNT(DISTINCT hash) FROM addon_files WHERE deleted_at IS NULL`).Scan(&distinct).Error
|
||||
})
|
||||
note("distinct hashes", err)
|
||||
if err == nil {
|
||||
h.DistinctHashes, h.DistinctAsOf = distinct, time.Now().Unix()
|
||||
}
|
||||
|
||||
var dups []dupHash
|
||||
err = withTimeout(120, func(tx *gorm.DB) error {
|
||||
return tx.Raw(`SELECT hash, COUNT(DISTINCT addon_id) AS addon_count, COUNT(*) AS file_count
|
||||
FROM addon_files WHERE deleted_at IS NULL
|
||||
GROUP BY hash HAVING COUNT(DISTINCT addon_id) > 1
|
||||
ORDER BY addon_count DESC, file_count DESC LIMIT 200`).Scan(&dups).Error
|
||||
})
|
||||
note("duplicates", err)
|
||||
if err == nil {
|
||||
h.Duplicates, h.DuplicatesAsOf = dups, time.Now().Unix()
|
||||
if len(dups) > 0 {
|
||||
hashes := make([]string, len(dups))
|
||||
for i, d := range dups {
|
||||
hashes[i] = d.Hash
|
||||
}
|
||||
var samples []struct {
|
||||
Hash string
|
||||
Path string
|
||||
}
|
||||
initializers.DB.Raw(`SELECT hash, MIN(path) AS path FROM addon_files
|
||||
WHERE deleted_at IS NULL AND hash IN ? GROUP BY hash`, hashes).Scan(&samples)
|
||||
byHash := map[string]string{}
|
||||
for _, s := range samples {
|
||||
byHash[s.Hash] = s.Path
|
||||
}
|
||||
for i := range h.Duplicates {
|
||||
h.Duplicates[i].SamplePath = byHash[h.Duplicates[i].Hash]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(errs) > 0 {
|
||||
h.Error = errs[0]
|
||||
for _, e := range errs {
|
||||
log.Printf("admin: heavy stats: %s", e)
|
||||
}
|
||||
}
|
||||
heavyMu.Lock()
|
||||
heavy = h
|
||||
heavyMu.Unlock()
|
||||
invalidate("overview")
|
||||
if time.Since(started) > 10*time.Second {
|
||||
log.Printf("admin: heavy stats took %s", time.Since(started).Round(time.Millisecond))
|
||||
}
|
||||
}
|
||||
|
||||
func sqlitePath() string {
|
||||
if initializers.SQLitePath != "" {
|
||||
return initializers.SQLitePath
|
||||
}
|
||||
return "crawler.db"
|
||||
}
|
||||
|
||||
// duplicateHashes lists the hashes shared by the most addons.
|
||||
func duplicateHashes(c *gin.Context) {
|
||||
h := currentHeavy()
|
||||
if h == nil {
|
||||
c.JSON(http.StatusOK, gin.H{"asOf": 0, "items": []dupHash{}, "pending": true})
|
||||
return
|
||||
}
|
||||
items := make([]dupHash, len(h.Duplicates))
|
||||
copy(items, h.Duplicates)
|
||||
if len(items) > 0 {
|
||||
hashes := make([]string, len(items))
|
||||
for i, d := range items {
|
||||
hashes[i] = d.Hash
|
||||
}
|
||||
var wl []string
|
||||
initializers.DB.Model(&models.WhitelistedHash{}).Where("hash IN ?", hashes).Pluck("hash", &wl)
|
||||
set := map[string]bool{}
|
||||
for _, w := range wl {
|
||||
set[w] = true
|
||||
}
|
||||
for i := range items {
|
||||
items[i].Whitelisted = set[items[i].Hash]
|
||||
}
|
||||
}
|
||||
if c.Query("hideWhitelisted") == "1" {
|
||||
kept := items[:0]
|
||||
for _, it := range items {
|
||||
if !it.Whitelisted {
|
||||
kept = append(kept, it)
|
||||
}
|
||||
}
|
||||
items = kept
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"asOf": h.DuplicatesAsOf, "items": items})
|
||||
}
|
||||
Reference in new issue
Block a user