Basic global hash whitelist

controllers/adminController.go

@@ -1,2 +1,30 @@
 package controllers
 
+import (
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
+	"github.com/gin-gonic/gin"
+)
+
+func AddWhitelistedHash(c *gin.Context) {
+	hash := c.PostForm("hash")
+	if hash == "" {
+		c.JSON(400, gin.H{"error": "Hash is required"})
+		return
+	}
+
+	hashAddon := models.WhitelistedHash{Hash: hash}
+
+	ret := initializers.DB.Find(&models.WhitelistedHash{}, "hash = ?", hash)
+	if ret.RowsAffected > 0 {
+		c.JSON(400, gin.H{"error": "Hash already whitelisted"})
+		return
+	}
+
+	if err := initializers.DB.Create(&hashAddon).Error; err != nil {
+		c.JSON(500, gin.H{"error": "Failed to add whitelisted hash"})
+		return
+	}
+
+	c.JSON(200, gin.H{"message": "Whitelisted hash added successfully"})
+}

controllers/feController.go

@@ -73,10 +73,23 @@ func GetDuplicates(c *gin.Context) {
 		return
 	}
 
-	// Extract all hashes from source addon files
+	// Get all whitelisted hashes
+	var whitelistedHashes []string
+	if err := initializers.DB.Model(&models.WhitelistedHash{}).Pluck("hash", &whitelistedHashes).Error; err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to fetch whitelisted hashes"})
+		return
+	}
+
+	// Create a map for faster lookup
+	whitelistMap := make(map[string]bool)
+	for _, hash := range whitelistedHashes {
+		whitelistMap[hash] = true
+	}
+
+	// Extract all hashes from source addon files, excluding whitelisted ones
 	var sourceHashes []string
 	for _, file := range sourceAddon.AddonFiles {
-		if file.Hash != "" {
+		if file.Hash != "" && !whitelistMap[file.Hash] {
 			sourceHashes = append(sourceHashes, file.Hash)
 		}
 	}

initializers/initialize.go

@@ -27,7 +27,7 @@ func ConnectToDB() {
 
 	DB = db
 
-	DB.AutoMigrate(&models.Addon{}, &models.AddonFile{})
+	DB.AutoMigrate(&models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{})
 }
 
 type Configuration struct {

main.go

@@ -28,6 +28,7 @@ func main() {
 
 	admin := r.Group("/admin")
 	{
+		admin.POST("/addWhitelistedHash",controllers.CheckAdmin, controllers.AddWhitelistedHash)
 		admin.GET("/duplicates", controllers.CheckAdmin)
 		admin.POST("/setPriority", controllers.CheckAdmin)
 		admin.POST("/setToBeIndexed", controllers.CheckAdmin)

models/whitelisted.go

@@ -0,0 +1,8 @@
+package models
+
+import "gorm.io/gorm"
+
+type WhitelistedHash struct {
+	gorm.Model
+	Hash    string `json:"hash"`
+}