speedup test

controllers/feController.go

@@ -52,6 +52,113 @@ func convertToSimplifiedAddonWithoutFiles(addon models.Addon) models.SimplifiedA
 
 func GetDuplicates(c *gin.Context) {
 	id := c.Param("id")
+
+	// First check if addon exists (lightweight query)
+	var exists bool
+	if err := initializers.DB.Model(&models.Addon{}).Select("1").Where("id = ?", id).Limit(1).Find(&exists).Error; err != nil {
+		c.JSON(http.StatusNotFound, gin.H{"error": "Addon not found"})
+		return
+	}
+
+	// Get only the hashes from source addon files (don't load full addon yet)
+	var sourceHashes []string
+	if err := initializers.DB.Model(&models.AddonFile{}).
+		Where("addon_id = ? AND hash != ''", id).
+		Pluck("hash", &sourceHashes).Error; err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to fetch source addon files"})
+		return
+	}
+
+	// Now load the full source addon for the response
+	var sourceAddon models.Addon
+	if err := initializers.DB.Preload("AddonFiles").First(&sourceAddon, "id = ?", id).Error; err != nil {
+		c.JSON(http.StatusNotFound, gin.H{"error": "Addon not found"})
+		return
+	}
+
+	simplifiedSourceAddon := convertToSimplifiedAddon(sourceAddon)
+
+	if len(sourceHashes) == 0 {
+		c.JSON(http.StatusOK, models.DuplicatesResponse{
+			SourceAddon:     simplifiedSourceAddon,
+			DuplicateAddons: make(map[string]models.AddonWithDuplicates),
+		})
+		return
+	}
+
+	// Single query with JOIN to get all data
+	type DuplicateRow struct {
+		Hash                 string
+		Path                 string
+		Version              string
+		AddonID              string
+		AddonName            string
+		AddonType            string
+		Summary              string
+		Preview              string
+		Author               string
+		SubscriberCount      int
+		CurrentVersionNumber string
+	}
+
+	var duplicates []DuplicateRow
+	err := initializers.DB.Table("addon_files af").
+		Select(`af.hash, af.path, af.version, af.addon_id,
+                a.name as addon_name, a.type as addon_type, a.summary,
+                a.preview, a.author, a.subscriber_count, a.current_version_number`).
+		Joins("INNER JOIN addons a ON a.id = af.addon_id").
+		Joins("LEFT JOIN whitelisted_hashes wh ON wh.hash = af.hash").
+		Where(`af.hash IN ? 
+               AND af.addon_id != ? 
+               AND af.deleted_at IS NULL
+               AND wh.hash IS NULL`,
+			sourceHashes, id).
+		Scan(&duplicates).Error
+
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to fetch duplicates"})
+		return
+	}
+
+	// Build response from single result set
+	response := models.DuplicatesResponse{
+		SourceAddon:     simplifiedSourceAddon,
+		DuplicateAddons: make(map[string]models.AddonWithDuplicates),
+	}
+
+	for _, dup := range duplicates {
+		if _, exists := response.DuplicateAddons[dup.AddonID]; !exists {
+			response.DuplicateAddons[dup.AddonID] = models.AddonWithDuplicates{
+				Addon: models.SimplifiedAddon{
+					ID:                   dup.AddonID,
+					Name:                 dup.AddonName,
+					Type:                 dup.AddonType,
+					Summary:              dup.Summary,
+					Preview:              dup.Preview,
+					Author:               dup.Author,
+					SubscriberCount:      dup.SubscriberCount,
+					CurrentVersionNumber: dup.CurrentVersionNumber,
+					AddonFiles:           []models.SimplifiedAddonFile{},
+				},
+				Duplicates: []models.DuplicateFileInfo{},
+			}
+		}
+
+		entry := response.DuplicateAddons[dup.AddonID]
+		entry.Duplicates = append(entry.Duplicates, models.DuplicateFileInfo{
+			Path:    dup.Path,
+			Hash:    dup.Hash,
+			Version: dup.Version,
+			AddonID: dup.AddonID,
+		})
+		response.DuplicateAddons[dup.AddonID] = entry
+	}
+
+	c.JSON(http.StatusOK, response)
+}
+
+/*func GetDuplicates(c *gin.Context) {
+	id := c.Param("id")
 	fmt.Println("Fetching addon with ID:", id)
 
 	// Fetch the source addon with its files
@@ -162,7 +269,7 @@ func GetDuplicates(c *gin.Context) {
 	}
 
 	c.JSON(http.StatusOK, response)
-}
+}*/
 
 func GetPossibleAddons(c *gin.Context) {
 	query := c.Query("q")

initializers/initialize.go

@@ -20,14 +20,41 @@ var ADMIN_SECRET string
 var ScraperWebhook string
 
 func ConnectToDB() {
-	db, err := gorm.Open(sqlite.Open(DB_NAME), &gorm.Config{})
+	db, err := gorm.Open(sqlite.Open(DB_NAME), &gorm.Config{
+		PrepareStmt:            true, // Cache prepared statements
+		SkipDefaultTransaction: true, // Disable default transactions for better performance
+	})
 	if err != nil {
 		log.Fatal("Failed to connect to database")
 	}
 
 	DB = db
 
+	// Optimize SQLite for performance
+	DB.Exec("PRAGMA journal_mode=WAL")    // Write-Ahead Logging for better concurrency
+	DB.Exec("PRAGMA synchronous=NORMAL")  // Balance between safety and speed
+	DB.Exec("PRAGMA cache_size=-64000")   // 64MB cache (negative = KB)
+	DB.Exec("PRAGMA temp_store=MEMORY")   // Use memory for temp tables
+	DB.Exec("PRAGMA mmap_size=268435456") // 256MB memory-mapped I/O
+
 	DB.AutoMigrate(&models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{})
+
+	// Create optimized indexes for duplicate detection queries
+	// Composite index for efficient hash lookups excluding specific addon_id
+	DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_hash_addon_optimized 
+		ON addon_files(hash, addon_id) WHERE deleted_at IS NULL`)
+
+	// Index for whitelisted hashes lookup
+	DB.Exec(`CREATE INDEX IF NOT EXISTS idx_whitelisted_hash 
+		ON whitelisted_hashes(hash)`)
+
+	// Additional index on addon_id for the JOIN operation
+	DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addons_id 
+		ON addons(id)`)
+
+	// Covering index for addon_files to avoid table lookups
+	DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_covering 
+		ON addon_files(addon_id, hash, path, version) WHERE deleted_at IS NULL`)
 }
 
 type Configuration struct {