- embedded React admin UI at /admin/ui (go:embed), login with ADMIN_SECRET via signed HttpOnly session cookie, X-ADMIN-KEY still works for scripts - admin API: overview stats, indexing queue, addon management (priority, reindex, delete/restore/purge, bulk), whitelist CRUD, duplicate hashes, traffic, metrics explorer, services, events/audit log, system info - built-in metrics: request latency per route, visitors, top addons and searches, indexer/scraper counters; minute buckets in the DB with hourly rollups and retention; optional Prometheus /metrics (METRICS_TOKEN) - POST /back/telemetry for indexers/scrapers (metrics, events, heartbeat) and a stdlib-only Go client in telemetry/ - auth hardening: empty secrets no longer allow access, constant-time compare, CORS not applied to /admin and /back, TRUSTED_PROXIES setting - Dockerfile builds the UI and no longer bakes config.yaml into the image; CI builds the UI, runs tests and builds arm64 with CGO
This commit is contained in:
ilbinek committed
1 parent
0ef27b0072
commit
56d9e10621
79 files changed
+13146
-118
No files matched your search
@@ -9,6 +9,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
|
||||
"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
|
||||
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
|
||||
"github.com/gin-gonic/gin"
|
||||
"gorm.io/gorm"
|
||||
@@ -16,7 +17,7 @@ import (
|
||||
|
||||
func GetNextToBeIndexed(c *gin.Context) {
|
||||
|
||||
checkIndexingTimeout()
|
||||
metrics.IndexerTimeoutsReset.Add(float64(ResetTimedOutIndexing(initializers.IndexTimeout)))
|
||||
|
||||
maxFileSize := c.Query("maxFileSize")
|
||||
maxSize := 10_000_000_000
|
||||
@@ -26,25 +27,31 @@ func GetNextToBeIndexed(c *gin.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
queue := "priority"
|
||||
addon, err := claimNextAddon(maxSize, true)
|
||||
if err != nil {
|
||||
metrics.IndexerClaims.Inc("error")
|
||||
c.JSON(500, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
if addon == nil {
|
||||
queue = "normal"
|
||||
addon, err = claimNextAddon(maxSize, false)
|
||||
if err != nil {
|
||||
metrics.IndexerClaims.Inc("error")
|
||||
c.JSON(500, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if addon != nil {
|
||||
metrics.IndexerClaims.Inc(queue)
|
||||
c.JSON(200, gin.H{"guid": addon.ID, "currentVersion": addon.CurrentVersionNumber})
|
||||
return
|
||||
}
|
||||
|
||||
// nothing to index
|
||||
metrics.IndexerClaims.Inc("empty")
|
||||
c.JSON(200, gin.H{"guid": "", "currentVersion": ""})
|
||||
}
|
||||
|
||||
@@ -96,6 +103,7 @@ func SaveIndexingResult(c *gin.Context) {
|
||||
err := c.ShouldBindJSON(&result)
|
||||
|
||||
if err != nil {
|
||||
metrics.IndexerSubmissions.Inc("invalid")
|
||||
c.JSON(400, gin.H{"error": "Invalid JSON"})
|
||||
return
|
||||
}
|
||||
@@ -103,6 +111,7 @@ func SaveIndexingResult(c *gin.Context) {
|
||||
var addon models.Addon
|
||||
ret := initializers.DB.Where("id = ?", result.GUID).First(&addon)
|
||||
if ret.Error != nil {
|
||||
metrics.IndexerSubmissions.Inc("not_found")
|
||||
c.JSON(404, gin.H{"error": "Addon not found"})
|
||||
return
|
||||
}
|
||||
@@ -110,25 +119,42 @@ func SaveIndexingResult(c *gin.Context) {
|
||||
// get all files for this addon and delete them - old version
|
||||
ret = initializers.DB.Where("addon_id = ?", addon.ID).Delete(&models.AddonFile{})
|
||||
if ret.Error != nil {
|
||||
metrics.IndexerSubmissions.Inc("error")
|
||||
c.JSON(500, gin.H{"error": ret.Error.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
ret = initializers.DB.CreateInBatches(&result.Files, 100)
|
||||
if ret.Error != nil {
|
||||
metrics.IndexerSubmissions.Inc("error")
|
||||
c.JSON(500, gin.H{"error": ret.Error.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
wasClaimed := addon.IsBeingIndexed && !addon.IndexStartTime.IsZero()
|
||||
addon.CurrentVersionNumber = result.CurrentVersion
|
||||
addon.IsBeingIndexed = false
|
||||
addon.ToBeIndexed = false
|
||||
addon.LastIndexedAt = &now
|
||||
ret = initializers.DB.Save(&addon)
|
||||
if ret.Error != nil {
|
||||
metrics.IndexerSubmissions.Inc("error")
|
||||
c.JSON(500, gin.H{"error": ret.Error.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
outcome := "ok"
|
||||
if !result.Success {
|
||||
outcome = "unsuccessful"
|
||||
}
|
||||
metrics.IndexerSubmissions.Inc(outcome)
|
||||
if wasClaimed {
|
||||
metrics.IndexerJobDuration.Observe(now.Sub(addon.IndexStartTime).Seconds())
|
||||
}
|
||||
metrics.IndexerFilesPerSubmission.Observe(float64(len(result.Files)))
|
||||
metrics.IndexerBytesIndexed.Add(float64(addon.CurrentVersionSize))
|
||||
|
||||
if initializers.DiscordWebhookURL != "" {
|
||||
text := "Indexing of addon " + addon.Name + " (" + addon.ID + ") was successful."
|
||||
colour := 2228479
|
||||
@@ -179,7 +205,7 @@ func SaveIndexingResult(c *gin.Context) {
|
||||
Embeds: []models.CustomEmbed{myEmbed},
|
||||
}
|
||||
|
||||
err := SendCustomWebhook(initializers.DiscordWebhookURL, myHook)
|
||||
err := sendWebhook("index", initializers.DiscordWebhookURL, myHook)
|
||||
if err != nil {
|
||||
fmt.Println("Error sending webhook:", err)
|
||||
}
|
||||
@@ -198,40 +224,60 @@ func DeleteAddon(c *gin.Context) {
|
||||
var addon models.Addon
|
||||
ret := initializers.DB.Where("id = ?", guid).First(&addon)
|
||||
if ret.Error != nil {
|
||||
metrics.ScraperDeletes.Inc("not_found")
|
||||
c.JSON(404, gin.H{"error": "Addon not found"})
|
||||
return
|
||||
}
|
||||
|
||||
// Delete associated files
|
||||
ret = initializers.DB.Where("addon_id = ?", addon.ID).Delete(&models.AddonFile{})
|
||||
if ret.Error != nil {
|
||||
c.JSON(500, gin.H{"error": ret.Error.Error()})
|
||||
err := initializers.DB.Transaction(func(tx *gorm.DB) error {
|
||||
return DeleteAddonCascade(tx, &addon)
|
||||
})
|
||||
if err != nil {
|
||||
metrics.ScraperDeletes.Inc("error")
|
||||
c.JSON(500, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
metrics.ScraperDeletes.Inc("deleted")
|
||||
c.JSON(200, gin.H{"status": "addon deleted"})
|
||||
}
|
||||
|
||||
// DeleteAddonCascade soft-deletes an addon and its files and removes its
|
||||
// outgoing dependency edges.
|
||||
func DeleteAddonCascade(tx *gorm.DB, addon *models.Addon) error {
|
||||
// Delete associated files
|
||||
if err := tx.Where("addon_id = ?", addon.ID).Delete(&models.AddonFile{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete outgoing dependency edges; incoming edges stay - other addons
|
||||
// still declare this dependency even if the addon row is gone
|
||||
ret = initializers.DB.Where("addon_id = ?", addon.ID).Delete(&models.AddonDependency{})
|
||||
if ret.Error != nil {
|
||||
c.JSON(500, gin.H{"error": ret.Error.Error()})
|
||||
return
|
||||
if err := tx.Where("addon_id = ?", addon.ID).Delete(&models.AddonDependency{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete the addon
|
||||
ret = initializers.DB.Delete(&addon)
|
||||
if ret.Error != nil {
|
||||
c.JSON(500, gin.H{"error": ret.Error.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(200, gin.H{"status": "addon deleted"})
|
||||
return tx.Delete(addon).Error
|
||||
}
|
||||
|
||||
func checkIndexingTimeout() {
|
||||
initializers.DB.Model(&models.Addon{}).
|
||||
// ResetTimedOutIndexing releases claims older than olderThan so another
|
||||
// indexer can pick the addon up. Returns the number of released claims.
|
||||
func ResetTimedOutIndexing(olderThan time.Duration) int64 {
|
||||
return initializers.DB.Model(&models.Addon{}).
|
||||
Where("is_being_indexed = ?", true).
|
||||
Where("index_start_time < ?", time.Now().Add(-30*time.Minute)).
|
||||
Update("is_being_indexed", false)
|
||||
Where("index_start_time < ?", time.Now().Add(-olderThan)).
|
||||
Update("is_being_indexed", false).RowsAffected
|
||||
}
|
||||
|
||||
// sendWebhook sends a Discord webhook and records the outcome.
|
||||
func sendWebhook(kind, webhookURL string, hook models.CustomHook) error {
|
||||
err := SendCustomWebhook(webhookURL, hook)
|
||||
result := "ok"
|
||||
if err != nil {
|
||||
result = "error"
|
||||
}
|
||||
metrics.DiscordWebhooks.Inc(kind, result)
|
||||
return err
|
||||
}
|
||||
|
||||
func SendCustomWebhook(webhookURL string, hook models.CustomHook) error {
|
||||
|
||||
Reference in new issue
Block a user