package controllers import ( "bytes" "encoding/json" "fmt" "net/http" "strconv" "time" "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" "github.com/gin-gonic/gin" "gorm.io/gorm" ) func GetNextToBeIndexed(c *gin.Context) { checkIndexingTimeout() maxFileSize := c.Query("maxFileSize") maxSize := 10_000_000_000 if maxFileSize != "" { if val, err := strconv.Atoi(maxFileSize); err == nil { maxSize = val } } var addon models.Addon ret := initializers.DB.Where("to_be_indexed = ?", true).Where("is_being_indexed = ?", false).Where("priority_indexing = ?", true).Where("blocked = ?", false).Where("current_version_size <= ?", maxSize).Order("updated_at asc").First(&addon) if ret.Error == nil { addon.IsBeingIndexed = true addon.IndexStartTime = time.Now() initializers.DB.Save(&addon) c.JSON(200, gin.H{"guid": addon.ID, "currentVersion": addon.CurrentVersionNumber}) return } if ret.Error != nil && ret.Error != gorm.ErrRecordNotFound { c.JSON(500, gin.H{"error": ret.Error.Error()}) return } ret = initializers.DB.Where("to_be_indexed = ?", true).Where("is_being_indexed = ?", false).Where("blocked = ?", false).Where("current_version_size <= ?", maxSize).Order("updated_at asc").First(&addon) if ret.Error != nil && ret.Error != gorm.ErrRecordNotFound { c.JSON(500, gin.H{"error": ret.Error.Error()}) return } if ret.Error == nil { addon.IsBeingIndexed = true addon.IndexStartTime = time.Now() initializers.DB.Save(&addon) c.JSON(200, gin.H{"guid": addon.ID, "currentVersion": addon.CurrentVersionNumber}) return } // nothing to index c.JSON(200, gin.H{"guid": "", "currentVersion": ""}) } func SaveIndexingResult(c *gin.Context) { var result struct { GUID string `json:"guid"` CurrentVersion string `json:"currentVersion"` Success bool `json:"success"` Files []models.AddonFile `json:"files"` } err := c.ShouldBindJSON(&result) if err != nil { c.JSON(400, gin.H{"error": "Invalid JSON"}) return } var addon models.Addon ret := initializers.DB.Where("id = ?", result.GUID).First(&addon) if ret.Error != nil { c.JSON(404, gin.H{"error": "Addon not found"}) return } // get all files for this addon and delete them - old version ret = initializers.DB.Where("addon_id = ?", addon.ID).Delete(&models.AddonFile{}) if ret.Error != nil { c.JSON(500, gin.H{"error": ret.Error.Error()}) return } ret = initializers.DB.CreateInBatches(&result.Files, 100) if ret.Error != nil { c.JSON(500, gin.H{"error": ret.Error.Error()}) return } addon.CurrentVersionNumber = result.CurrentVersion addon.IsBeingIndexed = false addon.ToBeIndexed = false ret = initializers.DB.Save(&addon) if ret.Error != nil { c.JSON(500, gin.H{"error": ret.Error.Error()}) return } if initializers.DiscordWebhookURL != "" { text := "Indexing of addon " + addon.Name + " (" + addon.ID + ") was successful." colour := 2228479 nbr := strconv.Itoa(len(result.Files)) size := fmt.Sprintf("%.2f MB", float64(addon.CurrentVersionSize)/1_000_000) addonsToBeIndexed := int64(0) addonsCount := int64(0) initializers.DB.Model(&models.Addon{}).Where("to_be_indexed = ?", true).Count(&addonsToBeIndexed) initializers.DB.Model(&models.Addon{}).Count(&addonsCount) percentage := 0.0 if addonsCount > 0 { percentage = (float64(addonsCount-addonsToBeIndexed) / float64(addonsCount)) * 100 } txt := fmt.Sprintf("%d/%d (%.2f%%).", addonsCount-addonsToBeIndexed, addonsCount, percentage) myEmbed := models.CustomEmbed{ Title: text, Description: fmt.Sprintf("Files Indexed: %s\nOverall Progress: %s", nbr, txt), Color: colour, Timestamp: time.Now(), Image: models.CustomImage{ URL: addon.Preview, }, Fields: []models.CustomEmbedField{ { Name: "Size", Value: size, Inline: true, }, { Name: "Current Version", Value: addon.CurrentVersionNumber, Inline: true, }, // { // Name: "Indexed by", // Value: c.ClientIP(), // Inline: false, // }, }, } myHook := models.CustomHook{ Username: "Reforger Crawler", Embeds: []models.CustomEmbed{myEmbed}, } err := SendCustomWebhook(initializers.DiscordWebhookURL, myHook) if err != nil { fmt.Println("Error sending webhook:", err) } } c.JSON(200, gin.H{"status": "success"}) } func DeleteAddon(c *gin.Context) { guid := c.Query("guid") if guid == "" { c.JSON(400, gin.H{"error": "GUID is required"}) return } var addon models.Addon ret := initializers.DB.Where("id = ?", guid).First(&addon) if ret.Error != nil { c.JSON(404, gin.H{"error": "Addon not found"}) return } // Delete associated files ret = initializers.DB.Where("addon_id = ?", addon.ID).Delete(&models.AddonFile{}) if ret.Error != nil { c.JSON(500, gin.H{"error": ret.Error.Error()}) return } // Delete the addon ret = initializers.DB.Delete(&addon) if ret.Error != nil { c.JSON(500, gin.H{"error": ret.Error.Error()}) return } c.JSON(200, gin.H{"status": "addon deleted"}) } func checkIndexingTimeout() { var addons []models.Addon initializers.DB.Where("is_being_indexed = ?", true).Find(&addons) for _, addon := range addons { if time.Since(addon.IndexStartTime) > 30*time.Minute { addon.IsBeingIndexed = false initializers.DB.Save(&addon) } } } func SendCustomWebhook(webhookURL string, hook models.CustomHook) error { payload, err := json.Marshal(hook) if err != nil { return err } req, err := http.NewRequest("POST", webhookURL, bytes.NewBuffer(payload)) if err != nil { return err } req.Header.Set("Content-Type", "application/json") client := &http.Client{} resp, err := client.Do(req) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode > 299 { return fmt.Errorf("webhook failed with status code %d", resp.StatusCode) } return nil }