From 1dd8b1105e83d835427339d38a6fcef0b8a093b0 Mon Sep 17 00:00:00 2001 From: Sotirios Pupakis Date: Sat, 6 Sep 2025 02:24:49 +0200 Subject: [PATCH] indexer support --- config.yaml | 1 + controllers/indexerController.go | 164 +++++++++++++++++++++++++++++++ controllers/scraperController.go | 7 +- go.mod | 1 + go.sum | 11 +++ initializers/initialize.go | 3 + main.go | 2 + models/addon.go | 2 +- models/addonFiles.go | 4 +- 9 files changed, 191 insertions(+), 4 deletions(-) create mode 100644 controllers/indexerController.go diff --git a/config.yaml b/config.yaml index 1eb0207..fcce8eb 100644 --- a/config.yaml +++ b/config.yaml @@ -2,3 +2,4 @@ port: 8083 ip: "localhost" secret: "secret" db: "crawler.db" +discordWebhook: "https://discord.com/api/webhooks/1413673169792925787/bfbCJ8wiFeKsYkLGCp1U8sr5jOOWDcF0NP9DsHuGXy8NjrfKOncYWJbPnROophTXr-kH" diff --git a/controllers/indexerController.go b/controllers/indexerController.go new file mode 100644 index 0000000..8607555 --- /dev/null +++ b/controllers/indexerController.go @@ -0,0 +1,164 @@ +package controllers + +import ( + "encoding/json" + "fmt" + "strconv" + "time" + + "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" + "github.com/gin-gonic/gin" + "gorm.io/gorm" + + discordwebhook "github.com/bensch777/discord-webhook-golang" +) + +func GetNextToBeIndexed(c *gin.Context) { + + checkIndexingTimeout() + + var addon models.Addon + ret := initializers.DB.Where("to_be_indexed = ?", true).Where("priority_indexing = ?", true).Order("updated_at asc").First(&addon) + + if ret.Error == nil { + addon.IsBeingIndexed = true + addon.IndexStartTime = time.Now() + initializers.DB.Save(&addon) + + c.JSON(200, gin.H{"guid": addon.ID, "currentVersionId": addon.CurrentVersionID}) + return + } + + if ret.Error != nil && ret.Error != gorm.ErrRecordNotFound { + c.JSON(500, gin.H{"error": ret.Error.Error()}) + return + } + + ret = initializers.DB.Where("to_be_indexed = ?", true).Order("updated_at asc").First(&addon) + if ret.Error != nil && ret.Error != gorm.ErrRecordNotFound { + c.JSON(500, gin.H{"error": ret.Error.Error()}) + return + } + + if ret.Error == nil { + addon.IsBeingIndexed = true + addon.IndexStartTime = time.Now() + initializers.DB.Save(&addon) + + c.JSON(200, gin.H{"guid": addon.ID, "currentVersion": addon.CurrentVersionNumber}) + return + } + + // nothing to index + c.JSON(200, gin.H{"guid": "", "currentVersion": ""}) +} + +func SaveIndexingResult(c *gin.Context) { + var result struct { + GUID string `json:"guid"` + CurrentVersion string `json:"currentVersion"` + Success bool `json:"success"` + Files []models.AddonFile `json:"files"` + } + err := c.ShouldBindJSON(&result) + + if err != nil { + c.JSON(400, gin.H{"error": "Invalid JSON"}) + return + } + + var addon models.Addon + ret := initializers.DB.Where("id = ?", result.GUID).First(&addon) + if ret.Error != nil { + c.JSON(404, gin.H{"error": "Addon not found"}) + return + } + + addon.CurrentVersionNumber = result.CurrentVersion + addon.IsBeingIndexed = false + addon.ToBeIndexed = false + initializers.DB.Save(&addon) + + // Add new files with versionID + for _, file := range result.Files { + initializers.DB.Create(&file) + } + + if initializers.DiscordWebhook != "" { + text := "Indexing of addon " + addon.ID + " (" + addon.Name + ") was successful." + colour := 2228479 + nbr := strconv.Itoa(len(result.Files)) + + addonsToBeIndexed := int64(0) + addonsCount := int64(0) + initializers.DB.Model(&models.Addon{}).Where("to_be_indexed = ?", true).Count(&addonsToBeIndexed) + initializers.DB.Model(&models.Addon{}).Count(&addonsCount) + + percentage := 0.0 + if addonsCount > 0 { + percentage = (float64(addonsCount - addonsToBeIndexed) / float64(addonsCount)) * 100 + } + + txt := fmt.Sprintf("%d/%d (%.2f%%).", addonsCount - addonsToBeIndexed, addonsCount, percentage) + + embed := discordwebhook.Embed{ + Title: text, + Thumbnail: discordwebhook.Thumbnail{ + Url: addon.Preview, + }, + Description: txt, + Color: colour, + Timestamp: time.Now(), + Fields: []discordwebhook.Field{ + discordwebhook.Field{ + Name: "Name", + Value: addon.Name, + Inline: true, + }, + discordwebhook.Field{ + Name: "GUID", + Value: addon.ID, + Inline: true, + }, + discordwebhook.Field{ + Name: "Version", + Value: addon.CurrentVersionNumber, + Inline: false, + }, + discordwebhook.Field{ + Name: "Files Indexed", + Value: nbr, + Inline: false, + }, + }, + + } + + sendEmbed(initializers.DiscordWebhook, embed) + } + + c.JSON(200, gin.H{"status": "success"}) +} + +func checkIndexingTimeout() { + var addons []models.Addon + initializers.DB.Where("is_being_indexed = ?", true).Find(&addons) + + for _, addon := range addons { + if time.Since(addon.IndexStartTime) > 30*time.Minute { + addon.IsBeingIndexed = false + initializers.DB.Save(&addon) + } + } +} + +func sendEmbed(webhookUrl string, embed discordwebhook.Embed) { + hook := discordwebhook.Hook{ + Username: "Reforger Crawler", + Embeds: []discordwebhook.Embed{embed}, + } + + payload, _ := json.Marshal(hook) + discordwebhook.ExecuteWebhook(webhookUrl, payload) +} diff --git a/controllers/scraperController.go b/controllers/scraperController.go index 3974e11..f2c6518 100644 --- a/controllers/scraperController.go +++ b/controllers/scraperController.go @@ -1,14 +1,19 @@ package controllers import ( + "fmt" + "strconv" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" "github.com/gin-gonic/gin" - "strconv" ) func CreateAddon(c *gin.Context) { var addonR models.Addon + body := make([]byte, 0) + c.Request.Body.Read(body) + fmt.Println(body) err := c.ShouldBindJSON(&addonR) if err != nil { c.JSON(400, gin.H{"error": "Invalid JSON"}) diff --git a/go.mod b/go.mod index 0c7d9fd..10105c1 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module gitea.tbdevent.eu/TBD/reforger_crawler_main go 1.25.0 require ( + github.com/bensch777/discord-webhook-golang v0.0.6 gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/sqlite v1.6.0 gorm.io/gorm v1.30.3 diff --git a/go.sum b/go.sum index 2eb6f0d..7a564d7 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/bensch777/discord-webhook-golang v0.0.6 h1:91BMU6vKgymAMfRwtXPMUrKX+SUoPPHTDJHTFA/1Kgk= +github.com/bensch777/discord-webhook-golang v0.0.6/go.mod h1:GcIorMZAZaHZyQJkjNoYKvZ6VpZo8XLib/eD51xN7Is= github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= @@ -7,6 +9,7 @@ github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJ github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= @@ -14,6 +17,8 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= @@ -22,6 +27,8 @@ github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBEx github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= @@ -46,6 +53,7 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -57,6 +65,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= @@ -75,6 +84,8 @@ golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= diff --git a/initializers/initialize.go b/initializers/initialize.go index e74ae52..6cb691a 100644 --- a/initializers/initialize.go +++ b/initializers/initialize.go @@ -15,6 +15,7 @@ var PORT string var IP string var SECRET string var DB_NAME string +var DiscordWebhook string func ConnectToDB() { db, err := gorm.Open(sqlite.Open(DB_NAME), &gorm.Config{}) @@ -32,6 +33,7 @@ type Configuration struct { IP string `yaml:"ip"` Secret string `yaml:"secret"` DB string `yaml:"db"` + DiscordWebhook string `yaml:"discordWebhook"` } func Load() { @@ -52,4 +54,5 @@ file, err := os.ReadFile("config.yaml") IP = configuration.IP SECRET = configuration.Secret DB_NAME = configuration.DB + DiscordWebhook = configuration.DiscordWebhook } diff --git a/main.go b/main.go index 78ef2a3..ec0d221 100644 --- a/main.go +++ b/main.go @@ -22,6 +22,8 @@ func main() { { back.GET("/addon", controllers.CheckAllowed, controllers.GetCurrentState) back.POST("/addon", controllers.CheckAllowed, controllers.CreateAddon) + back.GET("/nextToBeIndexed", controllers.CheckAllowed, controllers.GetNextToBeIndexed) + back.POST("/submitAddon", controllers.CheckAllowed, controllers.SaveIndexingResult) } r.Run(initializers.IP + ":" + initializers.PORT) diff --git a/models/addon.go b/models/addon.go index c481f97..eb90239 100644 --- a/models/addon.go +++ b/models/addon.go @@ -24,6 +24,6 @@ type Addon struct { PriorityIndexing bool `json:"priorityIndexing" gorm:"default:false"` AddonFiles []AddonFile `json:"addonFiles"` - Previews string `json:"preview"` + Preview string `json:"preview"` Author string `json:"author"` } diff --git a/models/addonFiles.go b/models/addonFiles.go index fa37504..da248b3 100644 --- a/models/addonFiles.go +++ b/models/addonFiles.go @@ -5,7 +5,7 @@ import "gorm.io/gorm" type AddonFile struct { gorm.Model Path string `json:"path"` - Hash string `json:"hash"` + Hash string `json:"hash" gorm:"index"` AddonID string `json:"-"` - VersionID int `json:"versionId"` + Version string `json:"version"` }