dependencies added

config.yaml

@@ -6,12 +6,12 @@ admin_secret: "123456789"
 discord_webhook_url: ""
 scraper_webhook_url: ""
 
-db_driver: "sqlite"
+db_driver: "postgres"
 sqlite_path: "crawler.db"
 
 # Set db_driver to "postgres" and provide either postgres_dsn
 # or individual postgres_* fields.
-postgres_dsn: ""
+postgres_dsn: "postgresql://postgres:YaRvF8cQLKaXxDtakkvx@159.195.6.13:5432/dev"
 postgres_host: ""
 postgres_port: "5432"
 postgres_user: "postgres"

controllers/dependencyController.go

@@ -0,0 +1,134 @@
+package controllers
+
+import (
+	"strconv"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
+	"github.com/gin-gonic/gin"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+)
+
+// SetAddonDependencies replaces all outgoing dependency edges for an addon.
+// An empty dependency list is valid and clears the edges.
+func SetAddonDependencies(c *gin.Context) {
+	var report models.DependencyReport
+	if err := c.ShouldBindJSON(&report); err != nil {
+		c.JSON(400, gin.H{"error": "Invalid JSON"})
+		return
+	}
+
+	if report.AddonID == "" {
+		c.JSON(400, gin.H{"error": "addonId is required"})
+		return
+	}
+
+	seen := make(map[string]bool)
+	edges := make([]models.AddonDependency, 0, len(report.Dependencies))
+	for _, dep := range report.Dependencies {
+		if dep.ID == "" || dep.ID == report.AddonID || seen[dep.ID] {
+			continue
+		}
+		seen[dep.ID] = true
+		edges = append(edges, models.AddonDependency{
+			AddonID:        report.AddonID,
+			DependencyID:   dep.ID,
+			Version:        dep.Version,
+			DependencyName: dep.Name,
+		})
+	}
+
+	err := initializers.DB.Transaction(func(tx *gorm.DB) error {
+		if err := tx.Where("addon_id = ?", report.AddonID).Delete(&models.AddonDependency{}).Error; err != nil {
+			return err
+		}
+		if len(edges) == 0 {
+			return nil
+		}
+		return tx.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&edges, 100).Error
+	})
+	if err != nil {
+		c.JSON(500, gin.H{"error": err.Error()})
+		return
+	}
+
+	c.JSON(200, gin.H{"status": "success", "stored": len(edges)})
+}
+
+// GetDependents returns the addons that depend on the given addon.
+func GetDependents(c *gin.Context) {
+	addonID := c.Param("id")
+
+	var addon models.Addon
+	if err := initializers.DB.Where("id = ?", addonID).First(&addon).Error; err != nil {
+		c.JSON(404, gin.H{"error": "Addon not found"})
+		return
+	}
+
+	limit := parseLimit(c.Query("limit"))
+
+	dependents := []models.AddonSearchResult{}
+	err := initializers.DB.Model(&models.Addon{}).
+		Select("addons.id, addons.name, addons.type, addons.summary, addons.preview, addons.subscriber_count, addons.current_version_number, addons.author").
+		Joins("JOIN addon_dependencies ad ON ad.addon_id = addons.id").
+		Where("ad.dependency_id = ?", addonID).
+		Order("addons.subscriber_count DESC").
+		Limit(limit).
+		Find(&dependents).Error
+	if err != nil {
+		c.JSON(500, gin.H{"error": err.Error()})
+		return
+	}
+
+	c.JSON(200, models.DependentsResponse{
+		AddonID:    addonID,
+		Dependents: dependents,
+		Total:      len(dependents),
+	})
+}
+
+// GetDependencies returns the addons that the given addon depends on.
+func GetDependencies(c *gin.Context) {
+	addonID := c.Param("id")
+
+	var addon models.Addon
+	if err := initializers.DB.Where("id = ?", addonID).First(&addon).Error; err != nil {
+		c.JSON(404, gin.H{"error": "Addon not found"})
+		return
+	}
+
+	limit := parseLimit(c.Query("limit"))
+
+	dependencies := []models.AddonSearchResult{}
+	err := initializers.DB.Model(&models.Addon{}).
+		Select("addons.id, addons.name, addons.type, addons.summary, addons.preview, addons.subscriber_count, addons.current_version_number, addons.author").
+		Joins("JOIN addon_dependencies ad ON ad.dependency_id = addons.id").
+		Where("ad.addon_id = ?", addonID).
+		Order("addons.subscriber_count DESC").
+		Limit(limit).
+		Find(&dependencies).Error
+	if err != nil {
+		c.JSON(500, gin.H{"error": err.Error()})
+		return
+	}
+
+	c.JSON(200, models.DependenciesResponse{
+		AddonID:      addonID,
+		Dependencies: dependencies,
+		Total:        len(dependencies),
+	})
+}
+
+func parseLimit(raw string) int {
+	limit := 100
+	if raw != "" {
+		if val, err := strconv.Atoi(raw); err == nil && val > 0 {
+			limit = val
+		}
+	}
+	if limit > 500 {
+		limit = 500
+	}
+	return limit
+}

controllers/indexerController.go

@@ -26,34 +26,20 @@ func GetNextToBeIndexed(c *gin.Context) {
 		}
 	}
 
-	var addon models.Addon
-	ret := initializers.DB.Where("to_be_indexed = ?", true).Where("is_being_indexed = ?", false).Where("priority_indexing = ?", true).Where("blocked = ?", false).Where("current_version_size <= ?", maxSize).Order("updated_at asc").First(&addon)
-
-	if ret.Error == nil {
-		addon.IsBeingIndexed = true
-		addon.IndexStartTime = time.Now()
-		initializers.DB.Save(&addon)
-
-		c.JSON(200, gin.H{"guid": addon.ID, "currentVersion": addon.CurrentVersionNumber})
+	addon, err := claimNextAddon(maxSize, true)
+	if err != nil {
+		c.JSON(500, gin.H{"error": err.Error()})
 		return
 	}
-
-	if ret.Error != nil && ret.Error != gorm.ErrRecordNotFound {
-		c.JSON(500, gin.H{"error": ret.Error.Error()})
+	if addon == nil {
+		addon, err = claimNextAddon(maxSize, false)
+		if err != nil {
+			c.JSON(500, gin.H{"error": err.Error()})
 			return
 		}
-
-	ret = initializers.DB.Where("to_be_indexed = ?", true).Where("is_being_indexed = ?", false).Where("blocked = ?", false).Where("current_version_size <= ?", maxSize).Order("updated_at asc").First(&addon)
-	if ret.Error != nil && ret.Error != gorm.ErrRecordNotFound {
-		c.JSON(500, gin.H{"error": ret.Error.Error()})
-		return
 	}
 
-	if ret.Error == nil {
-		addon.IsBeingIndexed = true
-		addon.IndexStartTime = time.Now()
-		initializers.DB.Save(&addon)
-
+	if addon != nil {
 		c.JSON(200, gin.H{"guid": addon.ID, "currentVersion": addon.CurrentVersionNumber})
 		return
 	}
@@ -62,6 +48,44 @@ func GetNextToBeIndexed(c *gin.Context) {
 	c.JSON(200, gin.H{"guid": "", "currentVersion": ""})
 }
 
+// claimNextAddon picks the oldest indexable addon and claims it with an
+// optimistic UPDATE, so concurrent indexers can never claim the same job.
+// Returns nil when no candidate is available.
+func claimNextAddon(maxSize int, priorityOnly bool) (*models.Addon, error) {
+	for range 5 {
+		query := initializers.DB.Where("to_be_indexed = ?", true).Where("is_being_indexed = ?", false).Where("blocked = ?", false).Where("current_version_size <= ?", maxSize)
+		if priorityOnly {
+			query = query.Where("priority_indexing = ?", true)
+		}
+
+		var addon models.Addon
+		ret := query.Order("updated_at asc").First(&addon)
+		if ret.Error == gorm.ErrRecordNotFound {
+			return nil, nil
+		}
+		if ret.Error != nil {
+			return nil, ret.Error
+		}
+
+		claim := initializers.DB.Model(&models.Addon{}).
+			Where("id = ?", addon.ID).
+			Where("is_being_indexed = ?", false).
+			Where("to_be_indexed = ?", true).
+			Updates(map[string]any{
+				"is_being_indexed": true,
+				"index_start_time": time.Now(),
+			})
+		if claim.Error != nil {
+			return nil, claim.Error
+		}
+		if claim.RowsAffected == 1 {
+			return &addon, nil
+		}
+		// lost the race to another indexer - retry with the next candidate
+	}
+	return nil, nil
+}
+
 func SaveIndexingResult(c *gin.Context) {
 	var result struct {
 		GUID           string             `json:"guid"`
@@ -185,6 +209,14 @@ func DeleteAddon(c *gin.Context) {
 		return
 	}
 
+	// Delete outgoing dependency edges; incoming edges stay - other addons
+	// still declare this dependency even if the addon row is gone
+	ret = initializers.DB.Where("addon_id = ?", addon.ID).Delete(&models.AddonDependency{})
+	if ret.Error != nil {
+		c.JSON(500, gin.H{"error": ret.Error.Error()})
+		return
+	}
+
 	// Delete the addon
 	ret = initializers.DB.Delete(&addon)
 	if ret.Error != nil {
@@ -196,15 +228,10 @@ func DeleteAddon(c *gin.Context) {
 }
 
 func checkIndexingTimeout() {
-	var addons []models.Addon
-	initializers.DB.Where("is_being_indexed = ?", true).Find(&addons)
-
-	for _, addon := range addons {
-		if time.Since(addon.IndexStartTime) > 30*time.Minute {
-			addon.IsBeingIndexed = false
-			initializers.DB.Save(&addon)
-		}
-	}
+	initializers.DB.Model(&models.Addon{}).
+		Where("is_being_indexed = ?", true).
+		Where("index_start_time < ?", time.Now().Add(-30*time.Minute)).
+		Update("is_being_indexed", false)
 }
 
 func SendCustomWebhook(webhookURL string, hook models.CustomHook) error {

initializers/initialize.go

@@ -75,7 +75,7 @@ func ConnectToDB() {
 
 	DB = db
 
-	if err := DB.AutoMigrate(&models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{}); err != nil {
+	if err := DB.AutoMigrate(&models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{}, &models.AddonDependency{}); err != nil {
 		log.Fatalf("Failed to run migrations: %v", err)
 	}
 

main.go

@@ -28,6 +28,7 @@ func main() {
 		back.GET("/nextToBeIndexed", controllers.CheckAllowed, controllers.GetNextToBeIndexed)
 		back.POST("/submitAddon", controllers.CheckAllowed, controllers.SaveIndexingResult)
 		back.DELETE("/deleteAddon", controllers.CheckAllowed, controllers.DeleteAddon)
+		back.POST("/addonDependencies", controllers.CheckAllowed, controllers.SetAddonDependencies)
 	}
 
 	admin := r.Group("/admin")
@@ -41,6 +42,8 @@ func main() {
 	v1 := r.Group("/v1")
 	{
 		v1.GET("/addon/:id", controllers.GetDuplicates)
+		v1.GET("/addon/:id/dependents", controllers.GetDependents)
+		v1.GET("/addon/:id/dependencies", controllers.GetDependencies)
 		v1.GET("/getPossible", controllers.GetPossibleAddons)
 	}
 

models/addonDependency.go

@@ -0,0 +1,14 @@
+package models
+
+import "time"
+
+// AddonDependency is a directed edge: AddonID depends on DependencyID.
+// No soft delete - edges are replaced wholesale on every report.
+type AddonDependency struct {
+	AddonID        string    `gorm:"primaryKey;size:64" json:"addonId"`
+	DependencyID   string    `gorm:"primaryKey;size:64;index:idx_addon_dependencies_dependency_id" json:"dependencyId"`
+	Version        string    `json:"version"`
+	DependencyName string    `json:"dependencyName"`
+	CreatedAt      time.Time `json:"createdAt"`
+	UpdatedAt      time.Time `json:"updatedAt"`
+}

models/apiModels.go

@@ -53,6 +53,33 @@ type AddonSearchResult struct {
 	Author               string `json:"author"`
 }
 
+// DependencyReportItem is one dependency entry reported by the scraper
+type DependencyReportItem struct {
+	ID      string `json:"id"`
+	Name    string `json:"name"`
+	Version string `json:"version"`
+}
+
+// DependencyReport is the scraper payload that replaces all edges for an addon
+type DependencyReport struct {
+	AddonID      string                 `json:"addonId"`
+	Dependencies []DependencyReportItem `json:"dependencies"`
+}
+
+// DependentsResponse lists the addons that depend on a given addon
+type DependentsResponse struct {
+	AddonID    string              `json:"addonId"`
+	Dependents []AddonSearchResult `json:"dependents"`
+	Total      int                 `json:"total"`
+}
+
+// DependenciesResponse lists the addons a given addon depends on
+type DependenciesResponse struct {
+	AddonID      string              `json:"addonId"`
+	Dependencies []AddonSearchResult `json:"dependencies"`
+	Total        int                 `json:"total"`
+}
+
 // SearchResponse contains the search results
 type SearchResponse struct {
 	Query   string              `json:"query"`