indexer support

controllers/indexerController.go

@@ -0,0 +1,164 @@
+package controllers
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
+	"github.com/gin-gonic/gin"
+	"gorm.io/gorm"
+
+	discordwebhook "github.com/bensch777/discord-webhook-golang"
+)
+
+func GetNextToBeIndexed(c *gin.Context) {
+
+	checkIndexingTimeout()
+
+	var addon models.Addon
+	ret := initializers.DB.Where("to_be_indexed = ?", true).Where("priority_indexing = ?", true).Order("updated_at asc").First(&addon)
+
+	if ret.Error == nil {
+		addon.IsBeingIndexed = true
+		addon.IndexStartTime = time.Now()
+		initializers.DB.Save(&addon)
+
+		c.JSON(200, gin.H{"guid": addon.ID, "currentVersionId": addon.CurrentVersionID})
+		return
+	}
+
+	if ret.Error != nil && ret.Error != gorm.ErrRecordNotFound {
+		c.JSON(500, gin.H{"error": ret.Error.Error()})
+		return
+	}
+
+	ret = initializers.DB.Where("to_be_indexed = ?", true).Order("updated_at asc").First(&addon)
+	if ret.Error != nil && ret.Error != gorm.ErrRecordNotFound {
+		c.JSON(500, gin.H{"error": ret.Error.Error()})
+		return
+	}
+
+	if ret.Error == nil {
+		addon.IsBeingIndexed = true
+		addon.IndexStartTime = time.Now()
+		initializers.DB.Save(&addon)
+
+		c.JSON(200, gin.H{"guid": addon.ID, "currentVersion": addon.CurrentVersionNumber})
+		return
+	}
+
+	// nothing to index
+	c.JSON(200, gin.H{"guid": "", "currentVersion": ""})
+}
+
+func SaveIndexingResult(c *gin.Context) {
+	var result struct {
+		GUID           string             `json:"guid"`
+		CurrentVersion string             `json:"currentVersion"`
+		Success        bool               `json:"success"`
+		Files          []models.AddonFile `json:"files"`
+	}
+	err := c.ShouldBindJSON(&result)
+
+	if err != nil {
+		c.JSON(400, gin.H{"error": "Invalid JSON"})
+		return
+	}
+
+	var addon models.Addon
+	ret := initializers.DB.Where("id = ?", result.GUID).First(&addon)
+	if ret.Error != nil {
+		c.JSON(404, gin.H{"error": "Addon not found"})
+		return
+	}
+
+	addon.CurrentVersionNumber = result.CurrentVersion
+	addon.IsBeingIndexed = false
+	addon.ToBeIndexed = false
+	initializers.DB.Save(&addon)
+
+	// Add new files with versionID
+	for _, file := range result.Files {
+		initializers.DB.Create(&file)
+	}
+
+	if initializers.DiscordWebhook != "" {
+		text := "Indexing of addon " + addon.ID + " (" + addon.Name + ") was successful."
+		colour := 2228479
+		nbr := strconv.Itoa(len(result.Files))
+
+		addonsToBeIndexed := int64(0)
+		addonsCount := int64(0)
+		initializers.DB.Model(&models.Addon{}).Where("to_be_indexed = ?", true).Count(&addonsToBeIndexed)
+		initializers.DB.Model(&models.Addon{}).Count(&addonsCount)
+
+		percentage := 0.0
+		if addonsCount > 0 {
+			percentage = (float64(addonsCount - addonsToBeIndexed) / float64(addonsCount)) * 100
+		}
+
+		txt := fmt.Sprintf("%d/%d (%.2f%%).", addonsCount - addonsToBeIndexed, addonsCount, percentage)
+
+		embed := discordwebhook.Embed{
+			Title:     text,
+			Thumbnail: discordwebhook.Thumbnail{
+				Url: addon.Preview,
+			},
+			Description: txt,
+			Color:     colour,
+			Timestamp: time.Now(),
+			Fields: []discordwebhook.Field{
+				discordwebhook.Field{
+					Name:   "Name",
+					Value:  addon.Name,
+					Inline: true,
+				},
+				discordwebhook.Field{
+					Name:   "GUID",
+					Value:  addon.ID,
+					Inline: true,
+				},
+				discordwebhook.Field{
+					Name:   "Version",
+					Value:  addon.CurrentVersionNumber,
+					Inline: false,
+				},
+				discordwebhook.Field{
+					Name:   "Files Indexed",
+					Value:  nbr,
+					Inline: false,
+				},
+			},
+			
+		}
+
+		sendEmbed(initializers.DiscordWebhook, embed)
+	}
+
+	c.JSON(200, gin.H{"status": "success"})
+}
+
+func checkIndexingTimeout() {
+	var addons []models.Addon
+	initializers.DB.Where("is_being_indexed = ?", true).Find(&addons)
+
+	for _, addon := range addons {
+		if time.Since(addon.IndexStartTime) > 30*time.Minute {
+			addon.IsBeingIndexed = false
+			initializers.DB.Save(&addon)
+		}
+	}
+}
+
+func sendEmbed(webhookUrl string, embed discordwebhook.Embed) {
+	hook := discordwebhook.Hook{
+		Username: "Reforger Crawler",
+		Embeds:   []discordwebhook.Embed{embed},
+	}
+
+	payload, _ := json.Marshal(hook)
+	discordwebhook.ExecuteWebhook(webhookUrl, payload)
+}