diff --git a/config.yaml b/config.yaml index 93e00c1..187a22c 100644 --- a/config.yaml +++ b/config.yaml @@ -6,12 +6,12 @@ admin_secret: "123456789" discord_webhook_url: "" scraper_webhook_url: "" -db_driver: "sqlite" +db_driver: "postgres" sqlite_path: "crawler.db" # Set db_driver to "postgres" and provide either postgres_dsn # or individual postgres_* fields. -postgres_dsn: "" +postgres_dsn: "postgresql://postgres:YaRvF8cQLKaXxDtakkvx@159.195.6.13:5432/dev" postgres_host: "" postgres_port: "5432" postgres_user: "postgres" diff --git a/controllers/dependencyController.go b/controllers/dependencyController.go new file mode 100644 index 0000000..46762e5 --- /dev/null +++ b/controllers/dependencyController.go @@ -0,0 +1,134 @@ +package controllers + +import ( + "strconv" + + "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" + "github.com/gin-gonic/gin" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +// SetAddonDependencies replaces all outgoing dependency edges for an addon. +// An empty dependency list is valid and clears the edges. +func SetAddonDependencies(c *gin.Context) { + var report models.DependencyReport + if err := c.ShouldBindJSON(&report); err != nil { + c.JSON(400, gin.H{"error": "Invalid JSON"}) + return + } + + if report.AddonID == "" { + c.JSON(400, gin.H{"error": "addonId is required"}) + return + } + + seen := make(map[string]bool) + edges := make([]models.AddonDependency, 0, len(report.Dependencies)) + for _, dep := range report.Dependencies { + if dep.ID == "" || dep.ID == report.AddonID || seen[dep.ID] { + continue + } + seen[dep.ID] = true + edges = append(edges, models.AddonDependency{ + AddonID: report.AddonID, + DependencyID: dep.ID, + Version: dep.Version, + DependencyName: dep.Name, + }) + } + + err := initializers.DB.Transaction(func(tx *gorm.DB) error { + if err := tx.Where("addon_id = ?", report.AddonID).Delete(&models.AddonDependency{}).Error; err != nil { + return err + } + if len(edges) == 0 { + return nil + } + return tx.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&edges, 100).Error + }) + if err != nil { + c.JSON(500, gin.H{"error": err.Error()}) + return + } + + c.JSON(200, gin.H{"status": "success", "stored": len(edges)}) +} + +// GetDependents returns the addons that depend on the given addon. +func GetDependents(c *gin.Context) { + addonID := c.Param("id") + + var addon models.Addon + if err := initializers.DB.Where("id = ?", addonID).First(&addon).Error; err != nil { + c.JSON(404, gin.H{"error": "Addon not found"}) + return + } + + limit := parseLimit(c.Query("limit")) + + dependents := []models.AddonSearchResult{} + err := initializers.DB.Model(&models.Addon{}). + Select("addons.id, addons.name, addons.type, addons.summary, addons.preview, addons.subscriber_count, addons.current_version_number, addons.author"). + Joins("JOIN addon_dependencies ad ON ad.addon_id = addons.id"). + Where("ad.dependency_id = ?", addonID). + Order("addons.subscriber_count DESC"). + Limit(limit). + Find(&dependents).Error + if err != nil { + c.JSON(500, gin.H{"error": err.Error()}) + return + } + + c.JSON(200, models.DependentsResponse{ + AddonID: addonID, + Dependents: dependents, + Total: len(dependents), + }) +} + +// GetDependencies returns the addons that the given addon depends on. +func GetDependencies(c *gin.Context) { + addonID := c.Param("id") + + var addon models.Addon + if err := initializers.DB.Where("id = ?", addonID).First(&addon).Error; err != nil { + c.JSON(404, gin.H{"error": "Addon not found"}) + return + } + + limit := parseLimit(c.Query("limit")) + + dependencies := []models.AddonSearchResult{} + err := initializers.DB.Model(&models.Addon{}). + Select("addons.id, addons.name, addons.type, addons.summary, addons.preview, addons.subscriber_count, addons.current_version_number, addons.author"). + Joins("JOIN addon_dependencies ad ON ad.dependency_id = addons.id"). + Where("ad.addon_id = ?", addonID). + Order("addons.subscriber_count DESC"). + Limit(limit). + Find(&dependencies).Error + if err != nil { + c.JSON(500, gin.H{"error": err.Error()}) + return + } + + c.JSON(200, models.DependenciesResponse{ + AddonID: addonID, + Dependencies: dependencies, + Total: len(dependencies), + }) +} + +func parseLimit(raw string) int { + limit := 100 + if raw != "" { + if val, err := strconv.Atoi(raw); err == nil && val > 0 { + limit = val + } + } + if limit > 500 { + limit = 500 + } + return limit +} diff --git a/controllers/indexerController.go b/controllers/indexerController.go index 4f078cc..b7cbe81 100644 --- a/controllers/indexerController.go +++ b/controllers/indexerController.go @@ -26,34 +26,20 @@ func GetNextToBeIndexed(c *gin.Context) { } } - var addon models.Addon - ret := initializers.DB.Where("to_be_indexed = ?", true).Where("is_being_indexed = ?", false).Where("priority_indexing = ?", true).Where("blocked = ?", false).Where("current_version_size <= ?", maxSize).Order("updated_at asc").First(&addon) - - if ret.Error == nil { - addon.IsBeingIndexed = true - addon.IndexStartTime = time.Now() - initializers.DB.Save(&addon) - - c.JSON(200, gin.H{"guid": addon.ID, "currentVersion": addon.CurrentVersionNumber}) + addon, err := claimNextAddon(maxSize, true) + if err != nil { + c.JSON(500, gin.H{"error": err.Error()}) return } - - if ret.Error != nil && ret.Error != gorm.ErrRecordNotFound { - c.JSON(500, gin.H{"error": ret.Error.Error()}) - return + if addon == nil { + addon, err = claimNextAddon(maxSize, false) + if err != nil { + c.JSON(500, gin.H{"error": err.Error()}) + return + } } - ret = initializers.DB.Where("to_be_indexed = ?", true).Where("is_being_indexed = ?", false).Where("blocked = ?", false).Where("current_version_size <= ?", maxSize).Order("updated_at asc").First(&addon) - if ret.Error != nil && ret.Error != gorm.ErrRecordNotFound { - c.JSON(500, gin.H{"error": ret.Error.Error()}) - return - } - - if ret.Error == nil { - addon.IsBeingIndexed = true - addon.IndexStartTime = time.Now() - initializers.DB.Save(&addon) - + if addon != nil { c.JSON(200, gin.H{"guid": addon.ID, "currentVersion": addon.CurrentVersionNumber}) return } @@ -62,6 +48,44 @@ func GetNextToBeIndexed(c *gin.Context) { c.JSON(200, gin.H{"guid": "", "currentVersion": ""}) } +// claimNextAddon picks the oldest indexable addon and claims it with an +// optimistic UPDATE, so concurrent indexers can never claim the same job. +// Returns nil when no candidate is available. +func claimNextAddon(maxSize int, priorityOnly bool) (*models.Addon, error) { + for range 5 { + query := initializers.DB.Where("to_be_indexed = ?", true).Where("is_being_indexed = ?", false).Where("blocked = ?", false).Where("current_version_size <= ?", maxSize) + if priorityOnly { + query = query.Where("priority_indexing = ?", true) + } + + var addon models.Addon + ret := query.Order("updated_at asc").First(&addon) + if ret.Error == gorm.ErrRecordNotFound { + return nil, nil + } + if ret.Error != nil { + return nil, ret.Error + } + + claim := initializers.DB.Model(&models.Addon{}). + Where("id = ?", addon.ID). + Where("is_being_indexed = ?", false). + Where("to_be_indexed = ?", true). + Updates(map[string]any{ + "is_being_indexed": true, + "index_start_time": time.Now(), + }) + if claim.Error != nil { + return nil, claim.Error + } + if claim.RowsAffected == 1 { + return &addon, nil + } + // lost the race to another indexer - retry with the next candidate + } + return nil, nil +} + func SaveIndexingResult(c *gin.Context) { var result struct { GUID string `json:"guid"` @@ -185,6 +209,14 @@ func DeleteAddon(c *gin.Context) { return } + // Delete outgoing dependency edges; incoming edges stay - other addons + // still declare this dependency even if the addon row is gone + ret = initializers.DB.Where("addon_id = ?", addon.ID).Delete(&models.AddonDependency{}) + if ret.Error != nil { + c.JSON(500, gin.H{"error": ret.Error.Error()}) + return + } + // Delete the addon ret = initializers.DB.Delete(&addon) if ret.Error != nil { @@ -196,15 +228,10 @@ func DeleteAddon(c *gin.Context) { } func checkIndexingTimeout() { - var addons []models.Addon - initializers.DB.Where("is_being_indexed = ?", true).Find(&addons) - - for _, addon := range addons { - if time.Since(addon.IndexStartTime) > 30*time.Minute { - addon.IsBeingIndexed = false - initializers.DB.Save(&addon) - } - } + initializers.DB.Model(&models.Addon{}). + Where("is_being_indexed = ?", true). + Where("index_start_time < ?", time.Now().Add(-30*time.Minute)). + Update("is_being_indexed", false) } func SendCustomWebhook(webhookURL string, hook models.CustomHook) error { diff --git a/initializers/initialize.go b/initializers/initialize.go index 6e0024a..b3bb6df 100644 --- a/initializers/initialize.go +++ b/initializers/initialize.go @@ -75,7 +75,7 @@ func ConnectToDB() { DB = db - if err := DB.AutoMigrate(&models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{}); err != nil { + if err := DB.AutoMigrate(&models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{}, &models.AddonDependency{}); err != nil { log.Fatalf("Failed to run migrations: %v", err) } diff --git a/main.go b/main.go index 1dcc697..1cb88c4 100644 --- a/main.go +++ b/main.go @@ -28,6 +28,7 @@ func main() { back.GET("/nextToBeIndexed", controllers.CheckAllowed, controllers.GetNextToBeIndexed) back.POST("/submitAddon", controllers.CheckAllowed, controllers.SaveIndexingResult) back.DELETE("/deleteAddon", controllers.CheckAllowed, controllers.DeleteAddon) + back.POST("/addonDependencies", controllers.CheckAllowed, controllers.SetAddonDependencies) } admin := r.Group("/admin") @@ -41,6 +42,8 @@ func main() { v1 := r.Group("/v1") { v1.GET("/addon/:id", controllers.GetDuplicates) + v1.GET("/addon/:id/dependents", controllers.GetDependents) + v1.GET("/addon/:id/dependencies", controllers.GetDependencies) v1.GET("/getPossible", controllers.GetPossibleAddons) } diff --git a/models/addonDependency.go b/models/addonDependency.go new file mode 100644 index 0000000..43c021c --- /dev/null +++ b/models/addonDependency.go @@ -0,0 +1,14 @@ +package models + +import "time" + +// AddonDependency is a directed edge: AddonID depends on DependencyID. +// No soft delete - edges are replaced wholesale on every report. +type AddonDependency struct { + AddonID string `gorm:"primaryKey;size:64" json:"addonId"` + DependencyID string `gorm:"primaryKey;size:64;index:idx_addon_dependencies_dependency_id" json:"dependencyId"` + Version string `json:"version"` + DependencyName string `json:"dependencyName"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} diff --git a/models/apiModels.go b/models/apiModels.go index 4d0367c..ab6ad91 100644 --- a/models/apiModels.go +++ b/models/apiModels.go @@ -53,6 +53,33 @@ type AddonSearchResult struct { Author string `json:"author"` } +// DependencyReportItem is one dependency entry reported by the scraper +type DependencyReportItem struct { + ID string `json:"id"` + Name string `json:"name"` + Version string `json:"version"` +} + +// DependencyReport is the scraper payload that replaces all edges for an addon +type DependencyReport struct { + AddonID string `json:"addonId"` + Dependencies []DependencyReportItem `json:"dependencies"` +} + +// DependentsResponse lists the addons that depend on a given addon +type DependentsResponse struct { + AddonID string `json:"addonId"` + Dependents []AddonSearchResult `json:"dependents"` + Total int `json:"total"` +} + +// DependenciesResponse lists the addons a given addon depends on +type DependenciesResponse struct { + AddonID string `json:"addonId"` + Dependencies []AddonSearchResult `json:"dependencies"` + Total int `json:"total"` +} + // SearchResponse contains the search results type SearchResponse struct { Query string `json:"query"`