reforger_crawler_main/controllers/indexerController.go

package controllers

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
	"time"

	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
	"github.com/gin-gonic/gin"
	"gorm.io/gorm"
)

func GetNextToBeIndexed(c *gin.Context) {

	metrics.IndexerTimeoutsReset.Add(float64(ResetTimedOutIndexing(initializers.IndexTimeout)))

	maxFileSize := c.Query("maxFileSize")
	maxSize := 10_000_000_000
	if maxFileSize != "" {
		if val, err := strconv.Atoi(maxFileSize); err == nil {
			maxSize = val
		}
	}

	queue := "priority"
	addon, err := claimNextAddon(maxSize, true)
	if err != nil {
		metrics.IndexerClaims.Inc("error")
		c.JSON(500, gin.H{"error": err.Error()})
		return
	}
	if addon == nil {
		queue = "normal"
		addon, err = claimNextAddon(maxSize, false)
		if err != nil {
			metrics.IndexerClaims.Inc("error")
			c.JSON(500, gin.H{"error": err.Error()})
			return
		}
	}

	if addon != nil {
		metrics.IndexerClaims.Inc(queue)
		c.JSON(200, gin.H{"guid": addon.ID, "currentVersion": addon.CurrentVersionNumber})
		return
	}

	// nothing to index
	metrics.IndexerClaims.Inc("empty")
	c.JSON(200, gin.H{"guid": "", "currentVersion": ""})
}

// claimNextAddon picks the oldest indexable addon and claims it with an
// optimistic UPDATE, so concurrent indexers can never claim the same job.
// Returns nil when no candidate is available.
func claimNextAddon(maxSize int, priorityOnly bool) (*models.Addon, error) {
	for range 5 {
		query := initializers.DB.Where("to_be_indexed = ?", true).Where("is_being_indexed = ?", false).Where("blocked = ?", false).Where("current_version_size <= ?", maxSize)
		if priorityOnly {
			query = query.Where("priority_indexing = ?", true)
		}

		var addon models.Addon
		ret := query.Order("updated_at asc").First(&addon)
		if ret.Error == gorm.ErrRecordNotFound {
			return nil, nil
		}
		if ret.Error != nil {
			return nil, ret.Error
		}

		claim := initializers.DB.Model(&models.Addon{}).
			Where("id = ?", addon.ID).
			Where("is_being_indexed = ?", false).
			Where("to_be_indexed = ?", true).
			Updates(map[string]any{
				"is_being_indexed": true,
				"index_start_time": time.Now(),
			})
		if claim.Error != nil {
			return nil, claim.Error
		}
		if claim.RowsAffected == 1 {
			return &addon, nil
		}
		// lost the race to another indexer - retry with the next candidate
	}
	return nil, nil
}

func SaveIndexingResult(c *gin.Context) {
	var result struct {
		GUID           string             `json:"guid"`
		CurrentVersion string             `json:"currentVersion"`
		Success        bool               `json:"success"`
		Files          []models.AddonFile `json:"files"`
	}
	err := c.ShouldBindJSON(&result)

	if err != nil {
		metrics.IndexerSubmissions.Inc("invalid")
		c.JSON(400, gin.H{"error": "Invalid JSON"})
		return
	}

	var addon models.Addon
	ret := initializers.DB.Where("id = ?", result.GUID).First(&addon)
	if ret.Error != nil {
		metrics.IndexerSubmissions.Inc("not_found")
		c.JSON(404, gin.H{"error": "Addon not found"})
		return
	}

	// get all files for this addon and delete them - old version
	ret = initializers.DB.Where("addon_id = ?", addon.ID).Delete(&models.AddonFile{})
	if ret.Error != nil {
		metrics.IndexerSubmissions.Inc("error")
		c.JSON(500, gin.H{"error": ret.Error.Error()})
		return
	}

	ret = initializers.DB.CreateInBatches(&result.Files, 100)
	if ret.Error != nil {
		metrics.IndexerSubmissions.Inc("error")
		c.JSON(500, gin.H{"error": ret.Error.Error()})
		return
	}

	now := time.Now()
	wasClaimed := addon.IsBeingIndexed && !addon.IndexStartTime.IsZero()
	addon.CurrentVersionNumber = result.CurrentVersion
	addon.IsBeingIndexed = false
	addon.ToBeIndexed = false
	addon.LastIndexedAt = &now
	ret = initializers.DB.Save(&addon)
	if ret.Error != nil {
		metrics.IndexerSubmissions.Inc("error")
		c.JSON(500, gin.H{"error": ret.Error.Error()})
		return
	}

	outcome := "ok"
	if !result.Success {
		outcome = "unsuccessful"
	}
	metrics.IndexerSubmissions.Inc(outcome)
	if wasClaimed {
		metrics.IndexerJobDuration.Observe(now.Sub(addon.IndexStartTime).Seconds())
	}
	metrics.IndexerFilesPerSubmission.Observe(float64(len(result.Files)))
	metrics.IndexerBytesIndexed.Add(float64(addon.CurrentVersionSize))

	if initializers.DiscordWebhookURL != "" {
		text := "Indexing of addon " + addon.Name + " (" + addon.ID + ") was successful."
		colour := 2228479
		nbr := strconv.Itoa(len(result.Files))
		size := fmt.Sprintf("%.2f MB", float64(addon.CurrentVersionSize)/1_000_000)

		addonsToBeIndexed := int64(0)
		addonsCount := int64(0)
		initializers.DB.Model(&models.Addon{}).Where("to_be_indexed = ?", true).Count(&addonsToBeIndexed)
		initializers.DB.Model(&models.Addon{}).Count(&addonsCount)

		percentage := 0.0
		if addonsCount > 0 {
			percentage = (float64(addonsCount-addonsToBeIndexed) / float64(addonsCount)) * 100
		}

		txt := fmt.Sprintf("%d/%d (%.2f%%).", addonsCount-addonsToBeIndexed, addonsCount, percentage)

		myEmbed := models.CustomEmbed{
			Title:       text,
			Description: fmt.Sprintf("Files Indexed: %s\nOverall Progress: %s", nbr, txt),
			Color:       colour,
			Timestamp:   time.Now(),
			Image: models.CustomImage{
				URL: addon.Preview,
			},
			Fields: []models.CustomEmbedField{
				{
					Name:   "Size",
					Value:  size,
					Inline: true,
				},
				{
					Name:   "Current Version",
					Value:  addon.CurrentVersionNumber,
					Inline: true,
				},
				// {
				// 	Name:   "Indexed by",
				// 	Value:  c.ClientIP(),
				// 	Inline: false,
				// },
			},
		}

		myHook := models.CustomHook{
			Username: "Reforger Crawler",
			Embeds:   []models.CustomEmbed{myEmbed},
		}

		err := sendWebhook("index", initializers.DiscordWebhookURL, myHook)
		if err != nil {
			fmt.Println("Error sending webhook:", err)
		}
	}

	c.JSON(200, gin.H{"status": "success"})
}

func DeleteAddon(c *gin.Context) {
	guid := c.Query("guid")
	if guid == "" {
		c.JSON(400, gin.H{"error": "GUID is required"})
		return
	}

	var addon models.Addon
	ret := initializers.DB.Where("id = ?", guid).First(&addon)
	if ret.Error != nil {
		metrics.ScraperDeletes.Inc("not_found")
		c.JSON(404, gin.H{"error": "Addon not found"})
		return
	}

	err := initializers.DB.Transaction(func(tx *gorm.DB) error {
		return DeleteAddonCascade(tx, &addon)
	})
	if err != nil {
		metrics.ScraperDeletes.Inc("error")
		c.JSON(500, gin.H{"error": err.Error()})
		return
	}

	metrics.ScraperDeletes.Inc("deleted")
	c.JSON(200, gin.H{"status": "addon deleted"})
}

// DeleteAddonCascade soft-deletes an addon and its files and removes its
// outgoing dependency edges.
func DeleteAddonCascade(tx *gorm.DB, addon *models.Addon) error {
	// Delete associated files
	if err := tx.Where("addon_id = ?", addon.ID).Delete(&models.AddonFile{}).Error; err != nil {
		return err
	}

	// Delete outgoing dependency edges; incoming edges stay - other addons
	// still declare this dependency even if the addon row is gone
	if err := tx.Where("addon_id = ?", addon.ID).Delete(&models.AddonDependency{}).Error; err != nil {
		return err
	}

	// Delete the addon
	return tx.Delete(addon).Error
}

// ResetTimedOutIndexing releases claims older than olderThan so another
// indexer can pick the addon up. Returns the number of released claims.
func ResetTimedOutIndexing(olderThan time.Duration) int64 {
	return initializers.DB.Model(&models.Addon{}).
		Where("is_being_indexed = ?", true).
		Where("index_start_time < ?", time.Now().Add(-olderThan)).
		Update("is_being_indexed", false).RowsAffected
}

// sendWebhook sends a Discord webhook and records the outcome.
func sendWebhook(kind, webhookURL string, hook models.CustomHook) error {
	err := SendCustomWebhook(webhookURL, hook)
	result := "ok"
	if err != nil {
		result = "error"
	}
	metrics.DiscordWebhooks.Inc(kind, result)
	return err
}

func SendCustomWebhook(webhookURL string, hook models.CustomHook) error {
	payload, err := json.Marshal(hook)
	if err != nil {
		return err
	}

	req, err := http.NewRequest("POST", webhookURL, bytes.NewBuffer(payload))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		return fmt.Errorf("webhook failed with status code %d", resp.StatusCode)
	}

	return nil
}