reforger_crawler_main/controllers/scraperController.go

package controllers

import (
	"fmt"
	"strconv"
	"sync"
	"time"

	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
	"github.com/gin-gonic/gin"
)

var webhookMu sync.Mutex
var lastWebhookTime time.Time
var nbrOfUnsentWebhooks int

func CreateAddon(c *gin.Context) {
	var addonR models.Addon
	err := c.ShouldBindJSON(&addonR)
	if err != nil {
		metrics.ScraperAddons.Inc("invalid")
		c.JSON(400, gin.H{"error": "Invalid JSON"})
		return
	}

	addon := models.Addon{
		ID: addonR.ID,
	}

	result := initializers.DB.Where(&addon).First(&addon)
	if result.Error != nil {
		// addon does not exist, create it
		if err := initializers.DB.Create(&addonR).Error; err != nil {
			// usually a soft-deleted addon with the same ID; it can be restored from the admin dashboard
			metrics.ScraperAddons.Inc("create_failed")
			fmt.Println("Failed to create addon", addonR.ID, err)
		} else {
			metrics.ScraperAddons.Inc("created")
		}

		// send webhook about creation
		webhookMu.Lock()
		if time.Since(lastWebhookTime) < time.Second*10 {
			nbrOfUnsentWebhooks++
			webhookMu.Unlock()
			return
		}

		lastWebhookTime = time.Now()
		oldNbr := nbrOfUnsentWebhooks
		nbrOfUnsentWebhooks = 0
		webhookMu.Unlock()

		go func() {
			webhookURL := initializers.ScraperWebhookURL
			if webhookURL != "" {
				text := fmt.Sprintf("New %d addons found\nLast: %s (%s).", oldNbr, addon.ID, addon.Name)
				colour := 2228479
				size := fmt.Sprintf("%.2f MB", float64(addon.CurrentVersionSize)/1_000_000)

				myEmbed := models.CustomEmbed{
					Title:     text,
					Color:     colour,
					Timestamp: time.Now(),
					Image: models.CustomImage{
						URL: addon.Preview,
					},
					Fields: []models.CustomEmbedField{
						{
							Name:   "Size",
							Value:  size,
							Inline: true,
						},
						{
							Name:   "Current Version",
							Value:  addon.CurrentVersionNumber,
							Inline: true,
						},
					},
				}

				myHook := models.CustomHook{
					Username: "Reforger Crawler",
					Embeds:   []models.CustomEmbed{myEmbed},
				}

				err := sendWebhook("scraper", webhookURL, myHook)
				if err != nil {
					fmt.Println("Error sending webhook:", err)
				}
			}
		}()
	} else {
		// addon exists, check version, update it and queue for reindexing if needed
		if addon.CurrentVersionNumber != addonR.CurrentVersionNumber {
			addon.Name = addonR.Name
			addon.Type = addonR.Type
			addon.Summary = addonR.Summary
			addon.Unlisted = addonR.Unlisted
			addon.Private = addonR.Private
			addon.SubscriberCount = addonR.SubscriberCount
			addon.Author = addonR.Author
			addon.Preview = addonR.Preview
			addon.CurrentVersionNumber = addonR.CurrentVersionNumber
			addon.CurrentVersionID = addonR.CurrentVersionID
			addon.CurrentVersionSize = addonR.CurrentVersionSize
			addon.Blocked = addonR.Blocked
			addon.ToBeIndexed = true
			initializers.DB.Save(&addon)
			metrics.ScraperAddons.Inc("updated")

			// send webhook about update
			go func() {
				webhookURL := initializers.ScraperWebhookURL
				if webhookURL != "" {
					text := fmt.Sprintf("Addon: %s (%s) updated", addon.ID, addon.Name)
					colour := 2228479
					size := fmt.Sprintf("%.2f MB", float64(addon.CurrentVersionSize)/1_000_000)

					myEmbed := models.CustomEmbed{
						Title:     text,
						Color:     colour,
						Timestamp: time.Now(),
						Image: models.CustomImage{
							URL: addon.Preview,
						},
						Fields: []models.CustomEmbedField{
							{
								Name:   "Size",
								Value:  size,
								Inline: true,
							},
							{
								Name:   "Current Version",
								Value:  addon.CurrentVersionNumber,
								Inline: true,
							},
						},
					}

					myHook := models.CustomHook{
						Username: "Reforger Crawler",
						Embeds:   []models.CustomEmbed{myEmbed},
					}

					err := sendWebhook("scraper", webhookURL, myHook)
					if err != nil {
						fmt.Println("Error sending webhook:", err)
					}
				}
			}()
		} else {
			metrics.ScraperAddons.Inc("unchanged")
		}
	}
	c.JSON(200, gin.H{"status": "success"})
}
func GetCurrentState(c *gin.Context) {
	offsetStr := c.Query("offset")
	limitStr := c.Query("limit")

	offset := 0
	limit := 0
	if offsetStr != "" {
		if val, err := strconv.Atoi(offsetStr); err == nil {
			offset = val
		}
	}
	if limitStr != "" {
		if val, err := strconv.Atoi(limitStr); err == nil {
			limit = val
		}
	}

	var addons []models.Addon
	initializers.DB.Limit(limit).Offset(offset).Find(&addons)

	c.JSON(200, addons)
}