diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9841f16 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ + +reforger_crawler_main + +crawler.db diff --git a/api/reforger_crawler/GetDuplicates.bru b/api/reforger_crawler/GetDuplicates.bru new file mode 100644 index 0000000..265adb4 --- /dev/null +++ b/api/reforger_crawler/GetDuplicates.bru @@ -0,0 +1,20 @@ +meta { + name: GetDuplicates + type: http + seq: 1 +} + +get { + url: {{host}}/v1/addon/{{ACE}} + body: none + auth: inherit +} + +vars:pre-request { + RHS: 1337C0DE5DABBEEF + ACE: 60C4E0B49618CC62 +} + +settings { + encodeUrl: true +} diff --git a/api/reforger_crawler/GetPoosibleAddons.bru b/api/reforger_crawler/GetPoosibleAddons.bru new file mode 100644 index 0000000..f07167e --- /dev/null +++ b/api/reforger_crawler/GetPoosibleAddons.bru @@ -0,0 +1,19 @@ +meta { + name: GetPoosibleAddons + type: http + seq: 2 +} + +get { + url: {{host}}/v1/getPossible?q=RHS + body: none + auth: inherit +} + +params:query { + q: RHS +} + +settings { + encodeUrl: true +} diff --git a/api/reforger_crawler/bruno.json b/api/reforger_crawler/bruno.json new file mode 100644 index 0000000..fceabca --- /dev/null +++ b/api/reforger_crawler/bruno.json @@ -0,0 +1,9 @@ +{ + "version": "1", + "name": "reforger_crawler", + "type": "collection", + "ignore": [ + "node_modules", + ".git" + ] +} \ No newline at end of file diff --git a/api/reforger_crawler/collection.bru b/api/reforger_crawler/collection.bru new file mode 100644 index 0000000..c55bc26 --- /dev/null +++ b/api/reforger_crawler/collection.bru @@ -0,0 +1,3 @@ +vars:pre-request { + host: http://localhost:8083 +} diff --git a/config.yaml b/config.yaml index 9c14f16..e8c8121 100644 --- a/config.yaml +++ b/config.yaml @@ -1,6 +1,6 @@ port: 8083 -ip: "localhost" +ip: "0.0.0.0" secret: "secret" db: "crawler.db" discordWebhook: "https://discord.com/api/webhooks/1413673169792925787/bfbCJ8wiFeKsYkLGCp1U8sr5jOOWDcF0NP9DsHuGXy8NjrfKOncYWJbPnROophTXr-kH" -adminSecret: "123456789" \ No newline at end of file +adminSecret: "123456789" diff --git a/controllers/feController.go b/controllers/feController.go new file mode 100644 index 0000000..3f6f986 --- /dev/null +++ b/controllers/feController.go @@ -0,0 +1,206 @@ +package controllers + +import ( + "fmt" + "net/http" + "strconv" + + "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" + "github.com/gin-gonic/gin" +) + +// convertToSimplifiedAddon converts a full Addon to SimplifiedAddon +func convertToSimplifiedAddon(addon models.Addon) models.SimplifiedAddon { + var simplifiedFiles []models.SimplifiedAddonFile + for _, file := range addon.AddonFiles { + simplifiedFiles = append(simplifiedFiles, models.SimplifiedAddonFile{ + Path: file.Path, + Hash: file.Hash, + Version: file.Version, + AddonID: file.AddonID, + }) + } + + return models.SimplifiedAddon{ + ID: addon.ID, + Name: addon.Name, + Type: addon.Type, + Summary: addon.Summary, + SubscriberCount: addon.SubscriberCount, + CurrentVersionNumber: addon.CurrentVersionNumber, + Preview: addon.Preview, + Author: addon.Author, + AddonFiles: simplifiedFiles, + } +} + +// convertToSimplifiedAddonWithoutFiles converts a full Addon to SimplifiedAddon without files +func convertToSimplifiedAddonWithoutFiles(addon models.Addon) models.SimplifiedAddon { + return models.SimplifiedAddon{ + ID: addon.ID, + Name: addon.Name, + Type: addon.Type, + Summary: addon.Summary, + SubscriberCount: addon.SubscriberCount, + CurrentVersionNumber: addon.CurrentVersionNumber, + Preview: addon.Preview, + Author: addon.Author, + AddonFiles: []models.SimplifiedAddonFile{}, // Empty slice + } +} + +func GetDuplicates(c *gin.Context) { + id := c.Param("id") + fmt.Println("Fetching addon with ID:", id) + + // Fetch the source addon with its files + var sourceAddon models.Addon + if err := initializers.DB.Preload("AddonFiles").First(&sourceAddon, "id = ?", id).Error; err != nil { + c.JSON(http.StatusNotFound, gin.H{"error": "Addon not found"}) + return + } + + // Convert to simplified addon + simplifiedSourceAddon := convertToSimplifiedAddon(sourceAddon) + + // If the addon has no files, return early + if len(sourceAddon.AddonFiles) == 0 { + c.JSON(http.StatusOK, models.DuplicatesResponse{ + SourceAddon: simplifiedSourceAddon, + DuplicateAddons: make(map[string]models.AddonWithDuplicates), + }) + return + } + + // Extract all hashes from source addon files + var sourceHashes []string + for _, file := range sourceAddon.AddonFiles { + if file.Hash != "" { + sourceHashes = append(sourceHashes, file.Hash) + } + } + + if len(sourceHashes) == 0 { + c.JSON(http.StatusOK, models.DuplicatesResponse{ + SourceAddon: simplifiedSourceAddon, + DuplicateAddons: make(map[string]models.AddonWithDuplicates), + }) + return + } + + // Find all addon files that have matching hashes but belong to different addons + var duplicateFiles []models.AddonFile + if err := initializers.DB.Where("hash IN ? AND addon_id != ?", sourceHashes, id).Find(&duplicateFiles).Error; err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to fetch duplicate files"}) + return + } + + // Group duplicate files by addon ID + duplicatesByAddonID := make(map[string][]models.AddonFile) + for _, file := range duplicateFiles { + duplicatesByAddonID[file.AddonID] = append(duplicatesByAddonID[file.AddonID], file) + } + + // Fetch addon information for each addon that has duplicates + var addonIDs []string + for addonID := range duplicatesByAddonID { + addonIDs = append(addonIDs, addonID) + } + + var duplicateAddons []models.Addon + if len(addonIDs) > 0 { + if err := initializers.DB.Where("id IN ?", addonIDs).Find(&duplicateAddons).Error; err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to fetch duplicate addons"}) + return + } + } + + // Build the response + response := models.DuplicatesResponse{ + SourceAddon: simplifiedSourceAddon, + DuplicateAddons: make(map[string]models.AddonWithDuplicates), + } + + // Create a map for easy addon lookup + addonMap := make(map[string]models.Addon) + for _, addon := range duplicateAddons { + addonMap[addon.ID] = addon + } + + // Build the duplicate addons response + for addonID, files := range duplicatesByAddonID { + var duplicateFileInfos []models.DuplicateFileInfo + for _, file := range files { + duplicateFileInfos = append(duplicateFileInfos, models.DuplicateFileInfo{ + Path: file.Path, + Hash: file.Hash, + Version: file.Version, + AddonID: file.AddonID, + }) + } + + if addon, exists := addonMap[addonID]; exists { + response.DuplicateAddons[addonID] = models.AddonWithDuplicates{ + Addon: convertToSimplifiedAddonWithoutFiles(addon), + Duplicates: duplicateFileInfos, + } + } + } + + c.JSON(http.StatusOK, response) +} + +func GetPossibleAddons(c *gin.Context) { + query := c.Query("q") + if query == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "Query parameter 'q' is required"}) + return + } + + // Set default limit + limit := 20 + if limitParam := c.Query("limit"); limitParam != "" { + if parsedLimit, err := strconv.Atoi(limitParam); err == nil && parsedLimit > 0 && parsedLimit <= 100 { + limit = parsedLimit + } + } + + fmt.Printf("Searching for addons with query: %s (limit: %d)\n", query, limit) + + var addons []models.Addon + + // Search by ID (exact match) or by name (case-insensitive partial match) + // Using OR condition to search both ID and name fields + dbQuery := initializers.DB.Where("id = ? OR LOWER(name) LIKE LOWER(?)", query, "%"+query+"%"). + Order("subscriber_count DESC"). // Order by popularity + Limit(limit) + + if err := dbQuery.Find(&addons).Error; err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to search addons"}) + return + } + + // Convert to search results + var results []models.AddonSearchResult + for _, addon := range addons { + results = append(results, models.AddonSearchResult{ + ID: addon.ID, + Name: addon.Name, + Type: addon.Type, + Summary: addon.Summary, + Preview: addon.Preview, + SubscriberCount: addon.SubscriberCount, + CurrentVersionNumber: addon.CurrentVersionNumber, + Author: addon.Author, + }) + } + + response := models.SearchResponse{ + Query: query, + Results: results, + Total: len(results), + } + + c.JSON(http.StatusOK, response) +} diff --git a/crawler - kopie.db b/crawler - kopie.db deleted file mode 100644 index 9967c3a..0000000 Binary files a/crawler - kopie.db and /dev/null differ diff --git a/crawler.db b/crawler.db deleted file mode 100644 index ed8de1d..0000000 Binary files a/crawler.db and /dev/null differ diff --git a/main.go b/main.go index 0da4beb..00d551e 100644 --- a/main.go +++ b/main.go @@ -33,6 +33,12 @@ func main() { admin.POST("/setToBeIndexed", controllers.CheckAdmin) } + v1 := r.Group("/v1") + { + v1.GET("/addon/:id", controllers.GetDuplicates) + v1.GET("/getPossible", controllers.GetPossibleAddons) + } + r.Run(initializers.IP + ":" + initializers.PORT) } diff --git a/models/addonFiles.go b/models/addonFiles.go index cde5d6e..0a63d63 100644 --- a/models/addonFiles.go +++ b/models/addonFiles.go @@ -4,8 +4,8 @@ import "gorm.io/gorm" type AddonFile struct { gorm.Model - Path string `json:"path"` - Hash string `json:"hash" gorm:"index"` - AddonID string `json:"addonid" gorm:"index"` - Version string `json:"version"` + Path string `json:"path"` + Hash string `json:"hash" gorm:"index:idx_hash_addon,priority:1;index"` + AddonID string `json:"addonid" gorm:"index:idx_hash_addon,priority:2;index"` + Version string `json:"version"` } diff --git a/models/apiModels.go b/models/apiModels.go new file mode 100644 index 0000000..4d0367c --- /dev/null +++ b/models/apiModels.go @@ -0,0 +1,61 @@ +package models + +// SimplifiedAddon contains only essential addon information for API responses +type SimplifiedAddon struct { + ID string `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + Summary string `json:"summary"` + SubscriberCount int `json:"subscriberCount"` + CurrentVersionNumber string `json:"currentVersionNumber"` + Preview string `json:"preview"` + Author string `json:"author"` + AddonFiles []SimplifiedAddonFile `json:"addonFiles"` +} + +// SimplifiedAddonFile contains only essential addon file information for API responses +type SimplifiedAddonFile struct { + Path string `json:"path"` + Hash string `json:"hash"` + Version string `json:"version"` + AddonID string `json:"addonId"` +} + +// DuplicateFileInfo holds information about a duplicate file +type DuplicateFileInfo struct { + Path string `json:"path"` + Hash string `json:"hash"` + Version string `json:"version"` + AddonID string `json:"addonId"` +} + +// AddonWithDuplicates holds addon information and its duplicate files +type AddonWithDuplicates struct { + Addon SimplifiedAddon `json:"addon"` + Duplicates []DuplicateFileInfo `json:"duplicates"` +} + +// DuplicatesResponse is the main response structure +type DuplicatesResponse struct { + SourceAddon SimplifiedAddon `json:"sourceAddon"` + DuplicateAddons map[string]AddonWithDuplicates `json:"duplicateAddons"` +} + +// AddonSearchResult contains minimal addon information for search results +type AddonSearchResult struct { + ID string `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + Summary string `json:"summary"` + Preview string `json:"preview"` + SubscriberCount int `json:"subscriberCount"` + CurrentVersionNumber string `json:"currentVersionNumber"` + Author string `json:"author"` +} + +// SearchResponse contains the search results +type SearchResponse struct { + Query string `json:"query"` + Results []AddonSearchResult `json:"results"` + Total int `json:"total"` +}