indexer support

config.yaml

@@ -2,3 +2,4 @@ port: 8083
 ip: "localhost"
 secret: "secret"
 db: "crawler.db"
+discordWebhook: "https://discord.com/api/webhooks/1413673169792925787/bfbCJ8wiFeKsYkLGCp1U8sr5jOOWDcF0NP9DsHuGXy8NjrfKOncYWJbPnROophTXr-kH"

controllers/indexerController.go

@@ -0,0 +1,164 @@
+package controllers
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
+	"github.com/gin-gonic/gin"
+	"gorm.io/gorm"
+
+	discordwebhook "github.com/bensch777/discord-webhook-golang"
+)
+
+func GetNextToBeIndexed(c *gin.Context) {
+
+	checkIndexingTimeout()
+
+	var addon models.Addon
+	ret := initializers.DB.Where("to_be_indexed = ?", true).Where("priority_indexing = ?", true).Order("updated_at asc").First(&addon)
+
+	if ret.Error == nil {
+		addon.IsBeingIndexed = true
+		addon.IndexStartTime = time.Now()
+		initializers.DB.Save(&addon)
+
+		c.JSON(200, gin.H{"guid": addon.ID, "currentVersionId": addon.CurrentVersionID})
+		return
+	}
+
+	if ret.Error != nil && ret.Error != gorm.ErrRecordNotFound {
+		c.JSON(500, gin.H{"error": ret.Error.Error()})
+		return
+	}
+
+	ret = initializers.DB.Where("to_be_indexed = ?", true).Order("updated_at asc").First(&addon)
+	if ret.Error != nil && ret.Error != gorm.ErrRecordNotFound {
+		c.JSON(500, gin.H{"error": ret.Error.Error()})
+		return
+	}
+
+	if ret.Error == nil {
+		addon.IsBeingIndexed = true
+		addon.IndexStartTime = time.Now()
+		initializers.DB.Save(&addon)
+
+		c.JSON(200, gin.H{"guid": addon.ID, "currentVersion": addon.CurrentVersionNumber})
+		return
+	}
+
+	// nothing to index
+	c.JSON(200, gin.H{"guid": "", "currentVersion": ""})
+}
+
+func SaveIndexingResult(c *gin.Context) {
+	var result struct {
+		GUID           string             `json:"guid"`
+		CurrentVersion string             `json:"currentVersion"`
+		Success        bool               `json:"success"`
+		Files          []models.AddonFile `json:"files"`
+	}
+	err := c.ShouldBindJSON(&result)
+
+	if err != nil {
+		c.JSON(400, gin.H{"error": "Invalid JSON"})
+		return
+	}
+
+	var addon models.Addon
+	ret := initializers.DB.Where("id = ?", result.GUID).First(&addon)
+	if ret.Error != nil {
+		c.JSON(404, gin.H{"error": "Addon not found"})
+		return
+	}
+
+	addon.CurrentVersionNumber = result.CurrentVersion
+	addon.IsBeingIndexed = false
+	addon.ToBeIndexed = false
+	initializers.DB.Save(&addon)
+
+	// Add new files with versionID
+	for _, file := range result.Files {
+		initializers.DB.Create(&file)
+	}
+
+	if initializers.DiscordWebhook != "" {
+		text := "Indexing of addon " + addon.ID + " (" + addon.Name + ") was successful."
+		colour := 2228479
+		nbr := strconv.Itoa(len(result.Files))
+
+		addonsToBeIndexed := int64(0)
+		addonsCount := int64(0)
+		initializers.DB.Model(&models.Addon{}).Where("to_be_indexed = ?", true).Count(&addonsToBeIndexed)
+		initializers.DB.Model(&models.Addon{}).Count(&addonsCount)
+
+		percentage := 0.0
+		if addonsCount > 0 {
+			percentage = (float64(addonsCount - addonsToBeIndexed) / float64(addonsCount)) * 100
+		}
+
+		txt := fmt.Sprintf("%d/%d (%.2f%%).", addonsCount - addonsToBeIndexed, addonsCount, percentage)
+
+		embed := discordwebhook.Embed{
+			Title:     text,
+			Thumbnail: discordwebhook.Thumbnail{
+				Url: addon.Preview,
+			},
+			Description: txt,
+			Color:     colour,
+			Timestamp: time.Now(),
+			Fields: []discordwebhook.Field{
+				discordwebhook.Field{
+					Name:   "Name",
+					Value:  addon.Name,
+					Inline: true,
+				},
+				discordwebhook.Field{
+					Name:   "GUID",
+					Value:  addon.ID,
+					Inline: true,
+				},
+				discordwebhook.Field{
+					Name:   "Version",
+					Value:  addon.CurrentVersionNumber,
+					Inline: false,
+				},
+				discordwebhook.Field{
+					Name:   "Files Indexed",
+					Value:  nbr,
+					Inline: false,
+				},
+			},
+			
+		}
+
+		sendEmbed(initializers.DiscordWebhook, embed)
+	}
+
+	c.JSON(200, gin.H{"status": "success"})
+}
+
+func checkIndexingTimeout() {
+	var addons []models.Addon
+	initializers.DB.Where("is_being_indexed = ?", true).Find(&addons)
+
+	for _, addon := range addons {
+		if time.Since(addon.IndexStartTime) > 30*time.Minute {
+			addon.IsBeingIndexed = false
+			initializers.DB.Save(&addon)
+		}
+	}
+}
+
+func sendEmbed(webhookUrl string, embed discordwebhook.Embed) {
+	hook := discordwebhook.Hook{
+		Username: "Reforger Crawler",
+		Embeds:   []discordwebhook.Embed{embed},
+	}
+
+	payload, _ := json.Marshal(hook)
+	discordwebhook.ExecuteWebhook(webhookUrl, payload)
+}

controllers/scraperController.go

@@ -1,14 +1,19 @@
 package controllers
 
 import (
+	"fmt"
+	"strconv"
+
 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
 	"github.com/gin-gonic/gin"
-	"strconv"
 )
 
 func CreateAddon(c *gin.Context) {
 	var addonR models.Addon
+	body := make([]byte, 0)
+	c.Request.Body.Read(body)
+	fmt.Println(body)
 	err := c.ShouldBindJSON(&addonR)
 	if err != nil {
 		c.JSON(400, gin.H{"error": "Invalid JSON"})

go.mod

@@ -3,6 +3,7 @@ module gitea.tbdevent.eu/TBD/reforger_crawler_main
 go 1.25.0
 
 require (
+	github.com/bensch777/discord-webhook-golang v0.0.6
 	gopkg.in/yaml.v3 v3.0.1
 	gorm.io/driver/sqlite v1.6.0
 	gorm.io/gorm v1.30.3

go.sum

@@ -1,3 +1,5 @@
+github.com/bensch777/discord-webhook-golang v0.0.6 h1:91BMU6vKgymAMfRwtXPMUrKX+SUoPPHTDJHTFA/1Kgk=
+github.com/bensch777/discord-webhook-golang v0.0.6/go.mod h1:GcIorMZAZaHZyQJkjNoYKvZ6VpZo8XLib/eD51xN7Is=
 github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0=
 github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
 github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
@@ -7,6 +9,7 @@ github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJ
 github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
 github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
 github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
@@ -14,6 +17,8 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ=
 github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=
+github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
+github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
 github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
 github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
@@ -22,6 +27,8 @@ github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBEx
 github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
@@ -46,6 +53,7 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
 github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
@@ -57,6 +65,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
@@ -75,6 +84,8 @@ golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
 golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
 google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

initializers/initialize.go

@@ -15,6 +15,7 @@ var PORT string
 var IP string
 var SECRET string
 var DB_NAME string
+var DiscordWebhook string
 
 func ConnectToDB() {
 	db, err := gorm.Open(sqlite.Open(DB_NAME), &gorm.Config{})
@@ -32,6 +33,7 @@ type Configuration struct {
 	IP     string `yaml:"ip"`
 	Secret string `yaml:"secret"`
 	DB     string `yaml:"db"`
+	DiscordWebhook string `yaml:"discordWebhook"`
 }
 
 func Load() {
@@ -52,4 +54,5 @@ file, err := os.ReadFile("config.yaml")
 	IP = configuration.IP
 	SECRET = configuration.Secret
 	DB_NAME = configuration.DB
+	DiscordWebhook = configuration.DiscordWebhook
 }

main.go

@@ -22,6 +22,8 @@ func main() {
 	{
 		back.GET("/addon", controllers.CheckAllowed, controllers.GetCurrentState)
 		back.POST("/addon", controllers.CheckAllowed, controllers.CreateAddon)
+		back.GET("/nextToBeIndexed", controllers.CheckAllowed, controllers.GetNextToBeIndexed)
+		back.POST("/submitAddon", controllers.CheckAllowed, controllers.SaveIndexingResult)
 	}
 
 	r.Run(initializers.IP + ":" + initializers.PORT)

models/addon.go

@@ -24,6 +24,6 @@ type Addon struct {
 	PriorityIndexing     bool        `json:"priorityIndexing" gorm:"default:false"`
 	AddonFiles           []AddonFile `json:"addonFiles"`
 
-	Previews string `json:"preview"`
+	Preview string `json:"preview"`
 	Author   string `json:"author"`
 }

models/addonFiles.go

@@ -5,7 +5,7 @@ import "gorm.io/gorm"
 type AddonFile struct {
 	gorm.Model
 	Path		 string `json:"path"`
-	Hash		 string `json:"hash"`
+	Hash		 string `json:"hash" gorm:"index"`
 	AddonID      string `json:"-"`
-	VersionID    int    `json:"versionId"`
+	Version      string `json:"version"`
 }