TBD/reforger_crawler_main
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

indexer support

Browse Source
This commit is contained in:
Sotirios Pupakis
2025-09-06 02:24:49 +02:00
parent 092c1a4080
commit 1dd8b1105e
9 changed files with 191 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
config.yaml
View File

@@ -2,3 +2,4 @@ port: 8083
ip: "localhost"
secret: "secret"
db: "crawler.db"
discordWebhook: "https://discord.com/api/webhooks/1413673169792925787/bfbCJ8wiFeKsYkLGCp1U8sr5jOOWDcF0NP9DsHuGXy8NjrfKOncYWJbPnROophTXr-kH"

164
controllers/indexerController.go Normal file
View File

@@ -0,0 +1,164 @@
package controllers
import (
"encoding/json"
"fmt"
"strconv"
"time"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
"github.com/gin-gonic/gin"
"gorm.io/gorm"
discordwebhook "github.com/bensch777/discord-webhook-golang"
)
func GetNextToBeIndexed(c *gin.Context) {
checkIndexingTimeout()
var addon models.Addon
ret := initializers.DB.Where("to_be_indexed = ?", true).Where("priority_indexing = ?", true).Order("updated_at asc").First(&addon)
if ret.Error == nil {
addon.IsBeingIndexed = true
addon.IndexStartTime = time.Now()
initializers.DB.Save(&addon)
c.JSON(200, gin.H{"guid": addon.ID, "currentVersionId": addon.CurrentVersionID})
return
}
if ret.Error != nil && ret.Error != gorm.ErrRecordNotFound {
c.JSON(500, gin.H{"error": ret.Error.Error()})
return
}
ret = initializers.DB.Where("to_be_indexed = ?", true).Order("updated_at asc").First(&addon)
if ret.Error != nil && ret.Error != gorm.ErrRecordNotFound {
c.JSON(500, gin.H{"error": ret.Error.Error()})
return
}
if ret.Error == nil {
addon.IsBeingIndexed = true
addon.IndexStartTime = time.Now()
initializers.DB.Save(&addon)
c.JSON(200, gin.H{"guid": addon.ID, "currentVersion": addon.CurrentVersionNumber})
return
}
// nothing to index
c.JSON(200, gin.H{"guid": "", "currentVersion": ""})
}
func SaveIndexingResult(c *gin.Context) {
var result struct {
GUID string `json:"guid"`
CurrentVersion string `json:"currentVersion"`
Success bool `json:"success"`
Files []models.AddonFile `json:"files"`
}
err := c.ShouldBindJSON(&result)
if err != nil {
c.JSON(400, gin.H{"error": "Invalid JSON"})
return
}
var addon models.Addon
ret := initializers.DB.Where("id = ?", result.GUID).First(&addon)
if ret.Error != nil {
c.JSON(404, gin.H{"error": "Addon not found"})
return
}
addon.CurrentVersionNumber = result.CurrentVersion
addon.IsBeingIndexed = false
addon.ToBeIndexed = false
initializers.DB.Save(&addon)
// Add new files with versionID
for _, file := range result.Files {
initializers.DB.Create(&file)
}
if initializers.DiscordWebhook != "" {
text := "Indexing of addon " + addon.ID + " (" + addon.Name + ") was successful."
colour := 2228479
nbr := strconv.Itoa(len(result.Files))
addonsToBeIndexed := int64(0)
addonsCount := int64(0)
initializers.DB.Model(&models.Addon{}).Where("to_be_indexed = ?", true).Count(&addonsToBeIndexed)
initializers.DB.Model(&models.Addon{}).Count(&addonsCount)
percentage := 0.0
if addonsCount > 0 {
percentage = (float64(addonsCount - addonsToBeIndexed) / float64(addonsCount)) * 100
}
txt := fmt.Sprintf("%d/%d (%.2f%%).", addonsCount - addonsToBeIndexed, addonsCount, percentage)
embed := discordwebhook.Embed{
Title: text,
Thumbnail: discordwebhook.Thumbnail{
Url: addon.Preview,
},
Description: txt,
Color: colour,
Timestamp: time.Now(),
Fields: []discordwebhook.Field{
discordwebhook.Field{
Name: "Name",
Value: addon.Name,
Inline: true,
},
discordwebhook.Field{
Name: "GUID",
Value: addon.ID,
Inline: true,
},
discordwebhook.Field{
Name: "Version",
Value: addon.CurrentVersionNumber,
Inline: false,
},
discordwebhook.Field{
Name: "Files Indexed",
Value: nbr,
Inline: false,
},
},
}
sendEmbed(initializers.DiscordWebhook, embed)
}
c.JSON(200, gin.H{"status": "success"})
}
func checkIndexingTimeout() {
var addons []models.Addon
initializers.DB.Where("is_being_indexed = ?", true).Find(&addons)
for _, addon := range addons {
if time.Since(addon.IndexStartTime) > 30*time.Minute {
addon.IsBeingIndexed = false
initializers.DB.Save(&addon)
}
}
}
func sendEmbed(webhookUrl string, embed discordwebhook.Embed) {
hook := discordwebhook.Hook{
Username: "Reforger Crawler",
Embeds: []discordwebhook.Embed{embed},
}
payload, _ := json.Marshal(hook)
discordwebhook.ExecuteWebhook(webhookUrl, payload)
}

7
controllers/scraperController.go
View File

@@ -1,14 +1,19 @@
package controllers
import (
"fmt"
"strconv"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
"github.com/gin-gonic/gin"
"strconv"
)
func CreateAddon(c *gin.Context) {
var addonR models.Addon
body := make([]byte, 0)
c.Request.Body.Read(body)
fmt.Println(body)
err := c.ShouldBindJSON(&addonR)
if err != nil {
c.JSON(400, gin.H{"error": "Invalid JSON"})

1
go.mod
View File

@@ -3,6 +3,7 @@ module gitea.tbdevent.eu/TBD/reforger_crawler_main
go 1.25.0
require (
github.com/bensch777/discord-webhook-golang v0.0.6
gopkg.in/yaml.v3 v3.0.1
gorm.io/driver/sqlite v1.6.0
gorm.io/gorm v1.30.3

11
go.sum
View File

@@ -1,3 +1,5 @@
github.com/bensch777/discord-webhook-golang v0.0.6 h1:91BMU6vKgymAMfRwtXPMUrKX+SUoPPHTDJHTFA/1Kgk=
github.com/bensch777/discord-webhook-golang v0.0.6/go.mod h1:GcIorMZAZaHZyQJkjNoYKvZ6VpZo8XLib/eD51xN7Is=
github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0=
github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
@@ -7,6 +9,7 @@ github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJ
github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
@@ -14,6 +17,8 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ=
github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
@@ -22,6 +27,8 @@ github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBEx
github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
@@ -46,6 +53,7 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
@@ -57,6 +65,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
@@ -75,6 +84,8 @@ golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

3
initializers/initialize.go
View File

@@ -15,6 +15,7 @@ var PORT string
var IP string
var SECRET string
var DB_NAME string
var DiscordWebhook string
func ConnectToDB() {
db, err := gorm.Open(sqlite.Open(DB_NAME), &gorm.Config{})
@@ -32,6 +33,7 @@ type Configuration struct {
IP string `yaml:"ip"`
Secret string `yaml:"secret"`
DB string `yaml:"db"`
DiscordWebhook string `yaml:"discordWebhook"`
}
func Load() {
@@ -52,4 +54,5 @@ file, err := os.ReadFile("config.yaml")
IP = configuration.IP
SECRET = configuration.Secret
DB_NAME = configuration.DB
DiscordWebhook = configuration.DiscordWebhook
}

2
main.go
View File

@@ -22,6 +22,8 @@ func main() {
{
back.GET("/addon", controllers.CheckAllowed, controllers.GetCurrentState)
back.POST("/addon", controllers.CheckAllowed, controllers.CreateAddon)
back.GET("/nextToBeIndexed", controllers.CheckAllowed, controllers.GetNextToBeIndexed)
back.POST("/submitAddon", controllers.CheckAllowed, controllers.SaveIndexingResult)
}
r.Run(initializers.IP + ":" + initializers.PORT)

2
models/addon.go
View File

@@ -24,6 +24,6 @@ type Addon struct {
PriorityIndexing bool `json:"priorityIndexing" gorm:"default:false"`
AddonFiles []AddonFile `json:"addonFiles"`
Previews string `json:"preview"`
Preview string `json:"preview"`
Author string `json:"author"`
}

4
models/addonFiles.go
View File

@@ -5,7 +5,7 @@ import "gorm.io/gorm"
type AddonFile struct {
gorm.Model
Path string `json:"path"`
Hash string `json:"hash"`
Hash string `json:"hash" gorm:"index"`
AddonID string `json:"-"`
VersionID int `json:"versionId"`
Version string `json:"version"`
}