Merge pull request 'support for FE' (#1) from wibecoded_fe into main

Reviewed-on: #1

.gitignore

@@ -0,0 +1,4 @@
+
+reforger_crawler_main
+
+crawler.db

api/reforger_crawler/GetDuplicates.bru

@@ -0,0 +1,20 @@
+meta {
+  name: GetDuplicates
+  type: http
+  seq: 1
+}
+
+get {
+  url: {{host}}/v1/addon/{{ACE}}
+  body: none
+  auth: inherit
+}
+
+vars:pre-request {
+  RHS: 1337C0DE5DABBEEF
+  ACE: 60C4E0B49618CC62
+}
+
+settings {
+  encodeUrl: true
+}

api/reforger_crawler/GetPoosibleAddons.bru

@@ -0,0 +1,19 @@
+meta {
+  name: GetPoosibleAddons
+  type: http
+  seq: 2
+}
+
+get {
+  url: {{host}}/v1/getPossible?q=RHS
+  body: none
+  auth: inherit
+}
+
+params:query {
+  q: RHS
+}
+
+settings {
+  encodeUrl: true
+}

api/reforger_crawler/bruno.json

@@ -0,0 +1,9 @@
+{
+  "version": "1",
+  "name": "reforger_crawler",
+  "type": "collection",
+  "ignore": [
+    "node_modules",
+    ".git"
+  ]
+}

api/reforger_crawler/collection.bru

@@ -0,0 +1,3 @@
+vars:pre-request {
+  host: http://localhost:8083
+}

config.yaml

@@ -1,6 +1,6 @@
 port: 8083
-ip: "localhost"
+ip: "0.0.0.0"
 secret: "secret"
 db: "crawler.db"
 discordWebhook: "https://discord.com/api/webhooks/1413673169792925787/bfbCJ8wiFeKsYkLGCp1U8sr5jOOWDcF0NP9DsHuGXy8NjrfKOncYWJbPnROophTXr-kH"
-adminSecret: "123456789"
+adminSecret: "123456789"

controllers/feController.go

@@ -0,0 +1,206 @@
+package controllers
+
+import (
+	"fmt"
+	"net/http"
+	"strconv"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
+	"github.com/gin-gonic/gin"
+)
+
+// convertToSimplifiedAddon converts a full Addon to SimplifiedAddon
+func convertToSimplifiedAddon(addon models.Addon) models.SimplifiedAddon {
+	var simplifiedFiles []models.SimplifiedAddonFile
+	for _, file := range addon.AddonFiles {
+		simplifiedFiles = append(simplifiedFiles, models.SimplifiedAddonFile{
+			Path:    file.Path,
+			Hash:    file.Hash,
+			Version: file.Version,
+			AddonID: file.AddonID,
+		})
+	}
+
+	return models.SimplifiedAddon{
+		ID:                   addon.ID,
+		Name:                 addon.Name,
+		Type:                 addon.Type,
+		Summary:              addon.Summary,
+		SubscriberCount:      addon.SubscriberCount,
+		CurrentVersionNumber: addon.CurrentVersionNumber,
+		Preview:              addon.Preview,
+		Author:               addon.Author,
+		AddonFiles:           simplifiedFiles,
+	}
+}
+
+// convertToSimplifiedAddonWithoutFiles converts a full Addon to SimplifiedAddon without files
+func convertToSimplifiedAddonWithoutFiles(addon models.Addon) models.SimplifiedAddon {
+	return models.SimplifiedAddon{
+		ID:                   addon.ID,
+		Name:                 addon.Name,
+		Type:                 addon.Type,
+		Summary:              addon.Summary,
+		SubscriberCount:      addon.SubscriberCount,
+		CurrentVersionNumber: addon.CurrentVersionNumber,
+		Preview:              addon.Preview,
+		Author:               addon.Author,
+		AddonFiles:           []models.SimplifiedAddonFile{}, // Empty slice
+	}
+}
+
+func GetDuplicates(c *gin.Context) {
+	id := c.Param("id")
+	fmt.Println("Fetching addon with ID:", id)
+
+	// Fetch the source addon with its files
+	var sourceAddon models.Addon
+	if err := initializers.DB.Preload("AddonFiles").First(&sourceAddon, "id = ?", id).Error; err != nil {
+		c.JSON(http.StatusNotFound, gin.H{"error": "Addon not found"})
+		return
+	}
+
+	// Convert to simplified addon
+	simplifiedSourceAddon := convertToSimplifiedAddon(sourceAddon)
+
+	// If the addon has no files, return early
+	if len(sourceAddon.AddonFiles) == 0 {
+		c.JSON(http.StatusOK, models.DuplicatesResponse{
+			SourceAddon:     simplifiedSourceAddon,
+			DuplicateAddons: make(map[string]models.AddonWithDuplicates),
+		})
+		return
+	}
+
+	// Extract all hashes from source addon files
+	var sourceHashes []string
+	for _, file := range sourceAddon.AddonFiles {
+		if file.Hash != "" {
+			sourceHashes = append(sourceHashes, file.Hash)
+		}
+	}
+
+	if len(sourceHashes) == 0 {
+		c.JSON(http.StatusOK, models.DuplicatesResponse{
+			SourceAddon:     simplifiedSourceAddon,
+			DuplicateAddons: make(map[string]models.AddonWithDuplicates),
+		})
+		return
+	}
+
+	// Find all addon files that have matching hashes but belong to different addons
+	var duplicateFiles []models.AddonFile
+	if err := initializers.DB.Where("hash IN ? AND addon_id != ?", sourceHashes, id).Find(&duplicateFiles).Error; err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to fetch duplicate files"})
+		return
+	}
+
+	// Group duplicate files by addon ID
+	duplicatesByAddonID := make(map[string][]models.AddonFile)
+	for _, file := range duplicateFiles {
+		duplicatesByAddonID[file.AddonID] = append(duplicatesByAddonID[file.AddonID], file)
+	}
+
+	// Fetch addon information for each addon that has duplicates
+	var addonIDs []string
+	for addonID := range duplicatesByAddonID {
+		addonIDs = append(addonIDs, addonID)
+	}
+
+	var duplicateAddons []models.Addon
+	if len(addonIDs) > 0 {
+		if err := initializers.DB.Where("id IN ?", addonIDs).Find(&duplicateAddons).Error; err != nil {
+			c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to fetch duplicate addons"})
+			return
+		}
+	}
+
+	// Build the response
+	response := models.DuplicatesResponse{
+		SourceAddon:     simplifiedSourceAddon,
+		DuplicateAddons: make(map[string]models.AddonWithDuplicates),
+	}
+
+	// Create a map for easy addon lookup
+	addonMap := make(map[string]models.Addon)
+	for _, addon := range duplicateAddons {
+		addonMap[addon.ID] = addon
+	}
+
+	// Build the duplicate addons response
+	for addonID, files := range duplicatesByAddonID {
+		var duplicateFileInfos []models.DuplicateFileInfo
+		for _, file := range files {
+			duplicateFileInfos = append(duplicateFileInfos, models.DuplicateFileInfo{
+				Path:    file.Path,
+				Hash:    file.Hash,
+				Version: file.Version,
+				AddonID: file.AddonID,
+			})
+		}
+
+		if addon, exists := addonMap[addonID]; exists {
+			response.DuplicateAddons[addonID] = models.AddonWithDuplicates{
+				Addon:      convertToSimplifiedAddonWithoutFiles(addon),
+				Duplicates: duplicateFileInfos,
+			}
+		}
+	}
+
+	c.JSON(http.StatusOK, response)
+}
+
+func GetPossibleAddons(c *gin.Context) {
+	query := c.Query("q")
+	if query == "" {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Query parameter 'q' is required"})
+		return
+	}
+
+	// Set default limit
+	limit := 20
+	if limitParam := c.Query("limit"); limitParam != "" {
+		if parsedLimit, err := strconv.Atoi(limitParam); err == nil && parsedLimit > 0 && parsedLimit <= 100 {
+			limit = parsedLimit
+		}
+	}
+
+	fmt.Printf("Searching for addons with query: %s (limit: %d)\n", query, limit)
+
+	var addons []models.Addon
+
+	// Search by ID (exact match) or by name (case-insensitive partial match)
+	// Using OR condition to search both ID and name fields
+	dbQuery := initializers.DB.Where("id = ? OR LOWER(name) LIKE LOWER(?)", query, "%"+query+"%").
+		Order("subscriber_count DESC"). // Order by popularity
+		Limit(limit)
+
+	if err := dbQuery.Find(&addons).Error; err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to search addons"})
+		return
+	}
+
+	// Convert to search results
+	var results []models.AddonSearchResult
+	for _, addon := range addons {
+		results = append(results, models.AddonSearchResult{
+			ID:                   addon.ID,
+			Name:                 addon.Name,
+			Type:                 addon.Type,
+			Summary:              addon.Summary,
+			Preview:              addon.Preview,
+			SubscriberCount:      addon.SubscriberCount,
+			CurrentVersionNumber: addon.CurrentVersionNumber,
+			Author:               addon.Author,
+		})
+	}
+
+	response := models.SearchResponse{
+		Query:   query,
+		Results: results,
+		Total:   len(results),
+	}
+
+	c.JSON(http.StatusOK, response)
+}

main.go

@@ -33,6 +33,12 @@ func main() {
 		admin.POST("/setToBeIndexed", controllers.CheckAdmin)
 	}
 
+	v1 := r.Group("/v1")
+	{
+		v1.GET("/addon/:id", controllers.GetDuplicates)
+		v1.GET("/getPossible", controllers.GetPossibleAddons)
+	}
+
 	r.Run(initializers.IP + ":" + initializers.PORT)
 }
 

models/addonFiles.go

@@ -4,8 +4,8 @@ import "gorm.io/gorm"
 
 type AddonFile struct {
 	gorm.Model
-	Path		 string `json:"path"`
-	Hash		 string `json:"hash" gorm:"index"`
-	AddonID      string `json:"addonid" gorm:"index"`
-	Version      string `json:"version"`
+	Path    string `json:"path"`
+	Hash    string `json:"hash" gorm:"index:idx_hash_addon,priority:1;index"`
+	AddonID string `json:"addonid" gorm:"index:idx_hash_addon,priority:2;index"`
+	Version string `json:"version"`
 }

models/apiModels.go

@@ -0,0 +1,61 @@
+package models
+
+// SimplifiedAddon contains only essential addon information for API responses
+type SimplifiedAddon struct {
+	ID                   string                `json:"id"`
+	Name                 string                `json:"name"`
+	Type                 string                `json:"type"`
+	Summary              string                `json:"summary"`
+	SubscriberCount      int                   `json:"subscriberCount"`
+	CurrentVersionNumber string                `json:"currentVersionNumber"`
+	Preview              string                `json:"preview"`
+	Author               string                `json:"author"`
+	AddonFiles           []SimplifiedAddonFile `json:"addonFiles"`
+}
+
+// SimplifiedAddonFile contains only essential addon file information for API responses
+type SimplifiedAddonFile struct {
+	Path    string `json:"path"`
+	Hash    string `json:"hash"`
+	Version string `json:"version"`
+	AddonID string `json:"addonId"`
+}
+
+// DuplicateFileInfo holds information about a duplicate file
+type DuplicateFileInfo struct {
+	Path    string `json:"path"`
+	Hash    string `json:"hash"`
+	Version string `json:"version"`
+	AddonID string `json:"addonId"`
+}
+
+// AddonWithDuplicates holds addon information and its duplicate files
+type AddonWithDuplicates struct {
+	Addon      SimplifiedAddon     `json:"addon"`
+	Duplicates []DuplicateFileInfo `json:"duplicates"`
+}
+
+// DuplicatesResponse is the main response structure
+type DuplicatesResponse struct {
+	SourceAddon     SimplifiedAddon                `json:"sourceAddon"`
+	DuplicateAddons map[string]AddonWithDuplicates `json:"duplicateAddons"`
+}
+
+// AddonSearchResult contains minimal addon information for search results
+type AddonSearchResult struct {
+	ID                   string `json:"id"`
+	Name                 string `json:"name"`
+	Type                 string `json:"type"`
+	Summary              string `json:"summary"`
+	Preview              string                `json:"preview"`
+	SubscriberCount      int    `json:"subscriberCount"`
+	CurrentVersionNumber string `json:"currentVersionNumber"`
+	Author               string `json:"author"`
+}
+
+// SearchResponse contains the search results
+type SearchResponse struct {
+	Query   string              `json:"query"`
+	Results []AddonSearchResult `json:"results"`
+	Total   int                 `json:"total"`
+}