diff --git a/controllers/feController.go b/controllers/feController.go index 256b1f8..15bed31 100644 --- a/controllers/feController.go +++ b/controllers/feController.go @@ -52,6 +52,113 @@ func convertToSimplifiedAddonWithoutFiles(addon models.Addon) models.SimplifiedA func GetDuplicates(c *gin.Context) { id := c.Param("id") + + // First check if addon exists (lightweight query) + var exists bool + if err := initializers.DB.Model(&models.Addon{}).Select("1").Where("id = ?", id).Limit(1).Find(&exists).Error; err != nil { + c.JSON(http.StatusNotFound, gin.H{"error": "Addon not found"}) + return + } + + // Get only the hashes from source addon files (don't load full addon yet) + var sourceHashes []string + if err := initializers.DB.Model(&models.AddonFile{}). + Where("addon_id = ? AND hash != ''", id). + Pluck("hash", &sourceHashes).Error; err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to fetch source addon files"}) + return + } + + // Now load the full source addon for the response + var sourceAddon models.Addon + if err := initializers.DB.Preload("AddonFiles").First(&sourceAddon, "id = ?", id).Error; err != nil { + c.JSON(http.StatusNotFound, gin.H{"error": "Addon not found"}) + return + } + + simplifiedSourceAddon := convertToSimplifiedAddon(sourceAddon) + + if len(sourceHashes) == 0 { + c.JSON(http.StatusOK, models.DuplicatesResponse{ + SourceAddon: simplifiedSourceAddon, + DuplicateAddons: make(map[string]models.AddonWithDuplicates), + }) + return + } + + // Single query with JOIN to get all data + type DuplicateRow struct { + Hash string + Path string + Version string + AddonID string + AddonName string + AddonType string + Summary string + Preview string + Author string + SubscriberCount int + CurrentVersionNumber string + } + + var duplicates []DuplicateRow + err := initializers.DB.Table("addon_files af"). + Select(`af.hash, af.path, af.version, af.addon_id, + a.name as addon_name, a.type as addon_type, a.summary, + a.preview, a.author, a.subscriber_count, a.current_version_number`). + Joins("INNER JOIN addons a ON a.id = af.addon_id"). + Joins("LEFT JOIN whitelisted_hashes wh ON wh.hash = af.hash"). + Where(`af.hash IN ? + AND af.addon_id != ? + AND af.deleted_at IS NULL + AND wh.hash IS NULL`, + sourceHashes, id). + Scan(&duplicates).Error + + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to fetch duplicates"}) + return + } + + // Build response from single result set + response := models.DuplicatesResponse{ + SourceAddon: simplifiedSourceAddon, + DuplicateAddons: make(map[string]models.AddonWithDuplicates), + } + + for _, dup := range duplicates { + if _, exists := response.DuplicateAddons[dup.AddonID]; !exists { + response.DuplicateAddons[dup.AddonID] = models.AddonWithDuplicates{ + Addon: models.SimplifiedAddon{ + ID: dup.AddonID, + Name: dup.AddonName, + Type: dup.AddonType, + Summary: dup.Summary, + Preview: dup.Preview, + Author: dup.Author, + SubscriberCount: dup.SubscriberCount, + CurrentVersionNumber: dup.CurrentVersionNumber, + AddonFiles: []models.SimplifiedAddonFile{}, + }, + Duplicates: []models.DuplicateFileInfo{}, + } + } + + entry := response.DuplicateAddons[dup.AddonID] + entry.Duplicates = append(entry.Duplicates, models.DuplicateFileInfo{ + Path: dup.Path, + Hash: dup.Hash, + Version: dup.Version, + AddonID: dup.AddonID, + }) + response.DuplicateAddons[dup.AddonID] = entry + } + + c.JSON(http.StatusOK, response) +} + +/*func GetDuplicates(c *gin.Context) { + id := c.Param("id") fmt.Println("Fetching addon with ID:", id) // Fetch the source addon with its files @@ -162,7 +269,7 @@ func GetDuplicates(c *gin.Context) { } c.JSON(http.StatusOK, response) -} +}*/ func GetPossibleAddons(c *gin.Context) { query := c.Query("q") diff --git a/initializers/initialize.go b/initializers/initialize.go index 4bf3388..bb5f6a8 100644 --- a/initializers/initialize.go +++ b/initializers/initialize.go @@ -20,28 +20,55 @@ var ADMIN_SECRET string var ScraperWebhook string func ConnectToDB() { - db, err := gorm.Open(sqlite.Open(DB_NAME), &gorm.Config{}) + db, err := gorm.Open(sqlite.Open(DB_NAME), &gorm.Config{ + PrepareStmt: true, // Cache prepared statements + SkipDefaultTransaction: true, // Disable default transactions for better performance + }) if err != nil { log.Fatal("Failed to connect to database") } DB = db + // Optimize SQLite for performance + DB.Exec("PRAGMA journal_mode=WAL") // Write-Ahead Logging for better concurrency + DB.Exec("PRAGMA synchronous=NORMAL") // Balance between safety and speed + DB.Exec("PRAGMA cache_size=-64000") // 64MB cache (negative = KB) + DB.Exec("PRAGMA temp_store=MEMORY") // Use memory for temp tables + DB.Exec("PRAGMA mmap_size=268435456") // 256MB memory-mapped I/O + DB.AutoMigrate(&models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{}) + + // Create optimized indexes for duplicate detection queries + // Composite index for efficient hash lookups excluding specific addon_id + DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_hash_addon_optimized + ON addon_files(hash, addon_id) WHERE deleted_at IS NULL`) + + // Index for whitelisted hashes lookup + DB.Exec(`CREATE INDEX IF NOT EXISTS idx_whitelisted_hash + ON whitelisted_hashes(hash)`) + + // Additional index on addon_id for the JOIN operation + DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addons_id + ON addons(id)`) + + // Covering index for addon_files to avoid table lookups + DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_covering + ON addon_files(addon_id, hash, path, version) WHERE deleted_at IS NULL`) } type Configuration struct { - Port string `yaml:"port"` - IP string `yaml:"ip"` - Secret string `yaml:"secret"` - DB string `yaml:"db"` + Port string `yaml:"port"` + IP string `yaml:"ip"` + Secret string `yaml:"secret"` + DB string `yaml:"db"` DiscordWebhook string `yaml:"discordWebhook"` - ADMIN_SECRET string `yaml:"adminSecret"` - ScraperWebhook string `yaml:"scraperWebhook"` + ADMIN_SECRET string `yaml:"adminSecret"` + ScraperWebhook string `yaml:"scraperWebhook"` } func Load() { -file, err := os.ReadFile("config.yaml") + file, err := os.ReadFile("config.yaml") if err != nil { log.Fatal("Failed to open config file") }