reforger_crawler_main/controllers/indexerController.go

package controllers

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
	"time"

	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
	"github.com/gin-gonic/gin"
	"gorm.io/gorm"
)

func GetNextToBeIndexed(c *gin.Context) {

	checkIndexingTimeout()

	maxFileSize := c.Query("maxFileSize")
	maxSize := 10_000_000_000
	if maxFileSize != "" {
		if val, err := strconv.Atoi(maxFileSize); err == nil {
			maxSize = val
		}
	}

	var addon models.Addon
	ret := initializers.DB.Where("to_be_indexed = ?", true).Where("is_being_indexed = ?", false).Where("priority_indexing = ?", true).Where("current_version_size <= ?", maxSize).Order("updated_at asc").First(&addon)

	if ret.Error == nil {
		addon.IsBeingIndexed = true
		addon.IndexStartTime = time.Now()
		initializers.DB.Save(&addon)

		c.JSON(200, gin.H{"guid": addon.ID, "currentVersionId": addon.CurrentVersionID})
		return
	}

	if ret.Error != nil && ret.Error != gorm.ErrRecordNotFound {
		c.JSON(500, gin.H{"error": ret.Error.Error()})
		return
	}

	ret = initializers.DB.Where("to_be_indexed = ?", true).Where("is_being_indexed = ?", false).Where("current_version_size <= ?", maxSize).Order("updated_at asc").First(&addon)
	if ret.Error != nil && ret.Error != gorm.ErrRecordNotFound {
		c.JSON(500, gin.H{"error": ret.Error.Error()})
		return
	}

	if ret.Error == nil {
		addon.IsBeingIndexed = true
		addon.IndexStartTime = time.Now()
		initializers.DB.Save(&addon)

		c.JSON(200, gin.H{"guid": addon.ID, "currentVersion": addon.CurrentVersionNumber})
		return
	}

	// nothing to index
	c.JSON(200, gin.H{"guid": "", "currentVersion": ""})
}

func SaveIndexingResult(c *gin.Context) {
	var result struct {
		GUID           string             `json:"guid"`
		CurrentVersion string             `json:"currentVersion"`
		Success        bool               `json:"success"`
		Files          []models.AddonFile `json:"files"`
	}
	err := c.ShouldBindJSON(&result)

	if err != nil {
		c.JSON(400, gin.H{"error": "Invalid JSON"})
		return
	}

	var addon models.Addon
	ret := initializers.DB.Where("id = ?", result.GUID).First(&addon)
	if ret.Error != nil {
		c.JSON(404, gin.H{"error": "Addon not found"})
		return
	}

	// get all files for this addon and delete them - old version
	ret = initializers.DB.Where("addon_id = ?", addon.ID).Delete(&models.AddonFile{})
	if ret.Error != nil {
		c.JSON(500, gin.H{"error": ret.Error.Error()})
		return
	}

	ret = initializers.DB.CreateInBatches(&result.Files, 100)
	if ret.Error != nil {
		c.JSON(500, gin.H{"error": ret.Error.Error()})
		return
	}

	addon.CurrentVersionNumber = result.CurrentVersion
	addon.IsBeingIndexed = false
	addon.ToBeIndexed = false
	ret = initializers.DB.Save(&addon)
	if ret.Error != nil {
		c.JSON(500, gin.H{"error": ret.Error.Error()})
		return
	}

	if initializers.DiscordWebhook != "" {
		text := "Indexing of addon " + addon.Name + " (" + addon.ID + ") was successful."
		colour := 2228479
		nbr := strconv.Itoa(len(result.Files))
		size := fmt.Sprintf("%.2f MB", float64(addon.CurrentVersionSize)/1_000_000)

		addonsToBeIndexed := int64(0)
		addonsCount := int64(0)
		initializers.DB.Model(&models.Addon{}).Where("to_be_indexed = ?", true).Count(&addonsToBeIndexed)
		initializers.DB.Model(&models.Addon{}).Count(&addonsCount)

		percentage := 0.0
		if addonsCount > 0 {
			percentage = (float64(addonsCount - addonsToBeIndexed) / float64(addonsCount)) * 100
		}

		txt := fmt.Sprintf("%d/%d (%.2f%%).", addonsCount - addonsToBeIndexed, addonsCount, percentage)

		myEmbed := models.CustomEmbed{
			Title: text,
			Description: fmt.Sprintf("Files Indexed: %s\nOverall Progress: %s", nbr, txt),
			Color: colour,
			Timestamp: time.Now(),
			Image: models.CustomImage{
				URL: addon.Preview,
			},
			Fields: []models.CustomEmbedField{
				{
					Name:   "Size",
					Value:  size,
					Inline: true,
				},
				{
					Name:   "Current Version",
					Value:  addon.CurrentVersionNumber,
					Inline: true,
				},
				// {
				// 	Name:   "Indexed by",
				// 	Value:  c.ClientIP(),
				// 	Inline: false,
				// },
			},
		}

		myHook := models.CustomHook{
			Username: "Reforger Crawler",
			Embeds: []models.CustomEmbed{myEmbed},
		}

		err := SendCustomWebhook(initializers.DiscordWebhook, myHook)
		if err != nil {
			fmt.Println("Error sending webhook:", err)
		}
	}

	c.JSON(200, gin.H{"status": "success"})
}

func checkIndexingTimeout() {
	var addons []models.Addon
	initializers.DB.Where("is_being_indexed = ?", true).Find(&addons)

	for _, addon := range addons {
		if time.Since(addon.IndexStartTime) > 30*time.Minute {
			addon.IsBeingIndexed = false
			initializers.DB.Save(&addon)
		}
	}
}

func SendCustomWebhook(webhookURL string, hook models.CustomHook) error {
    payload, err := json.Marshal(hook)
    if err != nil {
        return err
    }

    req, err := http.NewRequest("POST", webhookURL, bytes.NewBuffer(payload))
    if err != nil {
        return err
    }
    req.Header.Set("Content-Type", "application/json")

    client := &http.Client{}
    resp, err := client.Do(req)
    if err != nil {
        return err
    }
    defer resp.Body.Close()

    if resp.StatusCode < 200 || resp.StatusCode > 299 {
        return fmt.Errorf("webhook failed with status code %d", resp.StatusCode)
    }

    return nil
}