- embedded React admin UI at /admin/ui (go:embed), login with ADMIN_SECRET via signed HttpOnly session cookie, X-ADMIN-KEY still works for scripts - admin API: overview stats, indexing queue, addon management (priority, reindex, delete/restore/purge, bulk), whitelist CRUD, duplicate hashes, traffic, metrics explorer, services, events/audit log, system info - built-in metrics: request latency per route, visitors, top addons and searches, indexer/scraper counters; minute buckets in the DB with hourly rollups and retention; optional Prometheus /metrics (METRICS_TOKEN) - POST /back/telemetry for indexers/scrapers (metrics, events, heartbeat) and a stdlib-only Go client in telemetry/ - auth hardening: empty secrets no longer allow access, constant-time compare, CORS not applied to /admin and /back, TRUSTED_PROXIES setting - Dockerfile builds the UI and no longer bakes config.yaml into the image; CI builds the UI, runs tests and builds arm64 with CGO
This commit is contained in:
ilbinek committed
1 parent
0ef27b0072
commit
56d9e10621
79 files changed
+13146
-118
No files matched your search
@@ -0,0 +1,227 @@
|
||||
package admin
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
|
||||
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
|
||||
"github.com/gin-gonic/gin"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
var sha512Re = regexp.MustCompile(`^[0-9a-f]{128}$`)
|
||||
|
||||
// legacyAddWhitelistedHash keeps the original form-based endpoint working.
|
||||
func legacyAddWhitelistedHash(c *gin.Context) {
|
||||
hash := c.PostForm("hash")
|
||||
if hash == "" {
|
||||
c.JSON(400, gin.H{"error": "Hash is required"})
|
||||
return
|
||||
}
|
||||
|
||||
ret := initializers.DB.Find(&models.WhitelistedHash{}, "hash = ?", hash)
|
||||
if ret.RowsAffected > 0 {
|
||||
c.JSON(400, gin.H{"error": "Hash already whitelisted"})
|
||||
return
|
||||
}
|
||||
|
||||
if err := initializers.DB.Create(&models.WhitelistedHash{Hash: hash}).Error; err != nil {
|
||||
c.JSON(500, gin.H{"error": "Failed to add whitelisted hash"})
|
||||
return
|
||||
}
|
||||
audit(c, "whitelist_add", map[string]any{"added": 1, "legacy": true})
|
||||
invalidate("overview")
|
||||
c.JSON(200, gin.H{"message": "Whitelisted hash added successfully"})
|
||||
}
|
||||
|
||||
type whitelistRow struct {
|
||||
ID uint `json:"id"`
|
||||
Hash string `json:"hash"`
|
||||
Note string `json:"note"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
Files int64 `json:"files"`
|
||||
Addons int64 `json:"addons"`
|
||||
SamplePath string `json:"samplePath"`
|
||||
}
|
||||
|
||||
func listWhitelist(c *gin.Context) {
|
||||
limit := queryInt(c, "limit", 50, 1, 500)
|
||||
offset := queryInt(c, "offset", 0, 0, 1<<30)
|
||||
q := initializers.DB.Model(&models.WhitelistedHash{})
|
||||
if s := strings.TrimSpace(strings.ToLower(c.Query("q"))); s != "" {
|
||||
like := "%" + likeEscape(s) + "%"
|
||||
q = q.Where(`hash LIKE ? ESCAPE '\' OR LOWER(note) LIKE ? ESCAPE '\'`, likeEscape(s)+"%", like)
|
||||
}
|
||||
var total int64
|
||||
q.Session(&gorm.Session{}).Count(&total)
|
||||
|
||||
rows := []whitelistRow{}
|
||||
if err := q.Select("id, hash, note, created_at").Order("id DESC").Limit(limit).Offset(offset).Scan(&rows).Error; err != nil {
|
||||
fail(c, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if len(rows) > 0 {
|
||||
hashes := make([]string, len(rows))
|
||||
for i, r := range rows {
|
||||
hashes[i] = r.Hash
|
||||
}
|
||||
var usage []struct {
|
||||
Hash string
|
||||
Files int64
|
||||
Addons int64
|
||||
Sample string
|
||||
}
|
||||
initializers.DB.Raw(`SELECT hash, COUNT(*) AS files, COUNT(DISTINCT addon_id) AS addons, MIN(path) AS sample
|
||||
FROM addon_files WHERE deleted_at IS NULL AND hash IN ? GROUP BY hash`, hashes).Scan(&usage)
|
||||
byHash := map[string]int{}
|
||||
for i, u := range usage {
|
||||
byHash[u.Hash] = i
|
||||
}
|
||||
for i := range rows {
|
||||
if j, ok := byHash[rows[i].Hash]; ok {
|
||||
rows[i].Files, rows[i].Addons, rows[i].SamplePath = usage[j].Files, usage[j].Addons, usage[j].Sample
|
||||
}
|
||||
}
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"items": rows, "total": total, "limit": limit, "offset": offset})
|
||||
}
|
||||
|
||||
type skipped struct {
|
||||
Hash string `json:"hash"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
func addWhitelist(c *gin.Context) {
|
||||
var body struct {
|
||||
Hashes []string `json:"hashes"`
|
||||
Text string `json:"text"`
|
||||
Note string `json:"note"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&body); err != nil {
|
||||
fail(c, http.StatusBadRequest, "invalid JSON")
|
||||
return
|
||||
}
|
||||
candidates := append([]string{}, body.Hashes...)
|
||||
candidates = append(candidates, strings.FieldsFunc(body.Text, func(r rune) bool {
|
||||
return r == ',' || r == ';' || r == ' ' || r == '\n' || r == '\r' || r == '\t'
|
||||
})...)
|
||||
if len(candidates) == 0 {
|
||||
fail(c, http.StatusBadRequest, "no hashes given")
|
||||
return
|
||||
}
|
||||
if len(candidates) > 5000 {
|
||||
fail(c, http.StatusBadRequest, "at most 5000 hashes per request")
|
||||
return
|
||||
}
|
||||
note := body.Note
|
||||
if len(note) > 256 {
|
||||
note = note[:256]
|
||||
}
|
||||
|
||||
var skips []skipped
|
||||
seen := map[string]bool{}
|
||||
valid := []string{}
|
||||
for _, raw := range candidates {
|
||||
h := strings.ToLower(strings.TrimSpace(raw))
|
||||
switch {
|
||||
case h == "":
|
||||
continue
|
||||
case !sha512Re.MatchString(h):
|
||||
skips = append(skips, skipped{raw, "not a SHA-512 hex hash"})
|
||||
case seen[h]:
|
||||
skips = append(skips, skipped{h, "duplicate in request"})
|
||||
default:
|
||||
seen[h] = true
|
||||
valid = append(valid, h)
|
||||
}
|
||||
}
|
||||
|
||||
added := 0
|
||||
err := initializers.DB.Transaction(func(tx *gorm.DB) error {
|
||||
existing := map[string]bool{}
|
||||
for start := 0; start < len(valid); start += 500 {
|
||||
end := min(start+500, len(valid))
|
||||
var found []string
|
||||
if err := tx.Model(&models.WhitelistedHash{}).Where("hash IN ?", valid[start:end]).Pluck("hash", &found).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for _, h := range found {
|
||||
existing[h] = true
|
||||
}
|
||||
}
|
||||
rows := []models.WhitelistedHash{}
|
||||
for _, h := range valid {
|
||||
if existing[h] {
|
||||
skips = append(skips, skipped{h, "already whitelisted"})
|
||||
continue
|
||||
}
|
||||
rows = append(rows, models.WhitelistedHash{Hash: h, Note: note})
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
added = len(rows)
|
||||
return tx.CreateInBatches(&rows, 200).Error
|
||||
})
|
||||
if err != nil {
|
||||
fail(c, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if added > 0 {
|
||||
audit(c, "whitelist_add", map[string]any{"added": added, "note": note})
|
||||
invalidate("overview")
|
||||
}
|
||||
if skips == nil {
|
||||
skips = []skipped{}
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"added": added, "skipped": skips})
|
||||
}
|
||||
|
||||
func deleteWhitelist(c *gin.Context) {
|
||||
id, err := strconv.ParseUint(c.Param("id"), 10, 64)
|
||||
if err != nil {
|
||||
fail(c, http.StatusBadRequest, "invalid id")
|
||||
return
|
||||
}
|
||||
var row models.WhitelistedHash
|
||||
if err := initializers.DB.Unscoped().First(&row, id).Error; err != nil {
|
||||
fail(c, http.StatusNotFound, "not found")
|
||||
return
|
||||
}
|
||||
// remove every row of this hash, the table has no unique constraint
|
||||
res := initializers.DB.Unscoped().Where("hash = ?", row.Hash).Delete(&models.WhitelistedHash{})
|
||||
if res.Error != nil {
|
||||
fail(c, http.StatusInternalServerError, res.Error.Error())
|
||||
return
|
||||
}
|
||||
audit(c, "whitelist_remove", map[string]any{"hash": row.Hash})
|
||||
invalidate("overview")
|
||||
c.JSON(http.StatusOK, gin.H{"deleted": res.RowsAffected})
|
||||
}
|
||||
|
||||
// hashUsage lists the addons that contain a file with the given hash.
|
||||
func hashUsage(c *gin.Context) {
|
||||
hash := strings.ToLower(c.Param("hash"))
|
||||
var total int64
|
||||
initializers.DB.Model(&models.AddonFile{}).Where("hash = ?", hash).Count(&total)
|
||||
items := []struct {
|
||||
AddonID string `json:"addonId"`
|
||||
Name string `json:"name"`
|
||||
Path string `json:"path"`
|
||||
Version string `json:"version"`
|
||||
}{}
|
||||
err := initializers.DB.Raw(`SELECT f.addon_id, COALESCE(a.name, '') AS name, f.path, f.version
|
||||
FROM addon_files f LEFT JOIN addons a ON a.id = f.addon_id
|
||||
WHERE f.hash = ? AND f.deleted_at IS NULL ORDER BY a.subscriber_count DESC LIMIT 200`, hash).Scan(&items).Error
|
||||
if err != nil {
|
||||
fail(c, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
var wl int64
|
||||
initializers.DB.Model(&models.WhitelistedHash{}).Where("hash = ?", hash).Count(&wl)
|
||||
c.JSON(http.StatusOK, gin.H{"hash": hash, "whitelisted": wl > 0, "total": total, "items": items})
|
||||
}
|
||||
Reference in new issue
Block a user