reforger_crawler_main/controllers/indexerController.go

package controllers

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
	"time"

	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
	"github.com/gin-gonic/gin"
	"gorm.io/gorm"

	discordwebhook "github.com/bensch777/discord-webhook-golang"
)

func GetNextToBeIndexed(c *gin.Context) {

	checkIndexingTimeout()

	maxFileSizeMB := c.Query("maxFileSizeMB")
	maxSize := 10_000_000
	if maxFileSizeMB != "" {
		if val, err := strconv.Atoi(maxFileSizeMB); err == nil {
			maxSize = val
		}
	}

	var addon models.Addon
	ret := initializers.DB.Where("to_be_indexed = ?", true).Where("priority_indexing = ?", true).Where("current_version_size <= ?", maxSize).Order("updated_at asc").First(&addon)

	if ret.Error == nil {
		addon.IsBeingIndexed = true
		addon.IndexStartTime = time.Now()
		initializers.DB.Save(&addon)

		c.JSON(200, gin.H{"guid": addon.ID, "currentVersionId": addon.CurrentVersionID})
		return
	}

	if ret.Error != nil && ret.Error != gorm.ErrRecordNotFound {
		c.JSON(500, gin.H{"error": ret.Error.Error()})
		return
	}

	ret = initializers.DB.Where("to_be_indexed = ?", true).Where("priority_indexing = ?", true).Order("updated_at asc").First(&addon)
	if ret.Error != nil && ret.Error != gorm.ErrRecordNotFound {
		c.JSON(500, gin.H{"error": ret.Error.Error()})
		return
	}

	if ret.Error == nil {
		addon.IsBeingIndexed = true
		addon.IndexStartTime = time.Now()
		initializers.DB.Save(&addon)

		c.JSON(200, gin.H{"guid": addon.ID, "currentVersion": addon.CurrentVersionNumber})
		return
	}

	// nothing to index
	c.JSON(200, gin.H{"guid": "", "currentVersion": ""})
}

func SaveIndexingResult(c *gin.Context) {
	var result struct {
		GUID           string             `json:"guid"`
		CurrentVersion string             `json:"currentVersion"`
		Success        bool               `json:"success"`
		Files          []models.AddonFile `json:"files"`
	}
	err := c.ShouldBindJSON(&result)

	if err != nil {
		c.JSON(400, gin.H{"error": "Invalid JSON"})
		return
	}

	var addon models.Addon
	ret := initializers.DB.Where("id = ?", result.GUID).First(&addon)
	if ret.Error != nil {
		c.JSON(404, gin.H{"error": "Addon not found"})
		return
	}

	addon.CurrentVersionNumber = result.CurrentVersion
	addon.IsBeingIndexed = false
	addon.ToBeIndexed = false
	initializers.DB.Save(&addon)

	// Add new files with versionID
	for _, file := range result.Files {
		initializers.DB.Create(&file)
	}

	if initializers.DiscordWebhook != "" {
		text := "Indexing of addon " + addon.ID + " (" + addon.Name + ") was successful."
		colour := 2228479
		nbr := strconv.Itoa(len(result.Files))

		addonsToBeIndexed := int64(0)
		addonsCount := int64(0)
		initializers.DB.Model(&models.Addon{}).Where("to_be_indexed = ?", true).Count(&addonsToBeIndexed)
		initializers.DB.Model(&models.Addon{}).Count(&addonsCount)

		percentage := 0.0
		if addonsCount > 0 {
			percentage = (float64(addonsCount - addonsToBeIndexed) / float64(addonsCount)) * 100
		}

		txt := fmt.Sprintf("%d/%d (%.2f%%).", addonsCount - addonsToBeIndexed, addonsCount, percentage)

		myEmbed := models.CustomEmbed{
			Title: text,
			Description: fmt.Sprintf("Files Indexed: %s\nOverall Progress: %s", nbr, txt),
			Color: colour,
			Timestamp: time.Now(),
			Image: models.CustomImage{
				URL: addon.Preview,
			},
		}

		myHook := models.CustomHook{
			Username: "Reforger Crawler",
			Embeds: []models.CustomEmbed{myEmbed},
		}

		err := SendCustomWebhook(initializers.DiscordWebhook, myHook)
		if err != nil {
			fmt.Println("Error sending webhook:", err)
		}

		/*embed := discordwebhook.Embed{
			Title:     text,
			Thumbnail: discordwebhook.Thumbnail{
				Url: addon.Preview,
			},
			Description: txt,
			Color:     colour,
			Timestamp: time.Now(),
			Fields: []discordwebhook.Field{
				discordwebhook.Field{
					Name:   "Name",
					Value:  addon.Name,
					Inline: true,
				},
				discordwebhook.Field{
					Name:   "GUID",
					Value:  addon.ID,
					Inline: true,
				},
				discordwebhook.Field{
					Name:   "Version",
					Value:  addon.CurrentVersionNumber,
					Inline: false,
				},
				discordwebhook.Field{
					Name:   "Files Indexed",
					Value:  nbr,
					Inline: false,
				},
			},
			
		}

		sendEmbed(initializers.DiscordWebhook, embed)*/
	}

	c.JSON(200, gin.H{"status": "success"})
}

func checkIndexingTimeout() {
	var addons []models.Addon
	initializers.DB.Where("is_being_indexed = ?", true).Find(&addons)

	for _, addon := range addons {
		if time.Since(addon.IndexStartTime) > 30*time.Minute {
			addon.IsBeingIndexed = false
			initializers.DB.Save(&addon)
		}
	}
}

func sendEmbed(webhookUrl string, embed discordwebhook.Embed) {
	hook := discordwebhook.Hook{
		Username: "Reforger Crawler",
		Embeds:   []discordwebhook.Embed{embed},
	}

	payload, _ := json.Marshal(hook)
	discordwebhook.ExecuteWebhook(webhookUrl, payload)
}

func SendCustomWebhook(webhookURL string, hook models.CustomHook) error {
    payload, err := json.Marshal(hook)
    if err != nil {
        return err
    }

    req, err := http.NewRequest("POST", webhookURL, bytes.NewBuffer(payload))
    if err != nil {
        return err
    }
    req.Header.Set("Content-Type", "application/json")

    client := &http.Client{}
    resp, err := client.Do(req)
    if err != nil {
        return err
    }
    defer resp.Body.Close()

    if resp.StatusCode < 200 || resp.StatusCode > 299 {
        return fmt.Errorf("webhook failed with status code %d", resp.StatusCode)
    }

    return nil
}