diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..31efec1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +.git +.gitea +.vscode +api +*.db +*.db-journal +*.db-shm +*.db-wal +.env +.env.* +README.md diff --git a/.env.dokploy.example b/.env.dokploy.example new file mode 100644 index 0000000..7bdea0c --- /dev/null +++ b/.env.dokploy.example @@ -0,0 +1,28 @@ +# Dokploy environment example for Reforger Crawler +# Copy these values into Dokploy Environment Variables. + +SERVER_HOST=0.0.0.0 +SERVER_PORT=8083 + +SECRET_KEY=change-me-secret +ADMIN_SECRET=change-me-admin-secret + +DISCORD_WEBHOOK_URL= +SCRAPER_WEBHOOK_URL= + +# sqlite or postgres +DB_DRIVER=postgres + +# Used only when DB_DRIVER=sqlite +SQLITE_PATH=crawler.db + +# Option A: full DSN +POSTGRES_DSN=postgresql://postgres:change-me-password@postgres-host:5432/dev + +# Option B: split fields (used when POSTGRES_DSN is empty) +POSTGRES_HOST=postgres-host +POSTGRES_PORT=5432 +POSTGRES_USER=postgres +POSTGRES_PASSWORD=change-me-password +POSTGRES_DB_NAME=dev +POSTGRES_SSLMODE=disable diff --git a/.gitignore b/.gitignore index 9841f16..669d57f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ reforger_crawler_main crawler.db +.env +.env.* +!.env.dokploy.example diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..680e0b2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM golang:1.25-bookworm AS builder + +WORKDIR /app + +# Required for CGO build (sqlite driver). +RUN apt-get update \ + && apt-get install -y --no-install-recommends build-essential ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . +RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o /out/reforger_crawler_main . + +FROM debian:bookworm-slim AS runtime + +WORKDIR /app + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates tzdata \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /out/reforger_crawler_main /app/reforger_crawler_main +COPY config.yaml /app/config.yaml + +EXPOSE 8083 + +ENTRYPOINT ["/app/reforger_crawler_main"] diff --git a/config.yaml b/config.yaml index a4356d5..93e00c1 100644 --- a/config.yaml +++ b/config.yaml @@ -1,7 +1,20 @@ -port: 8083 -ip: "localhost" -secret: "secret" -db: "crawler.db" -discordWebhook: "" -scraperWebhook: "" -adminSecret: "123456789" +server_port: "8083" +server_host: "localhost" +secret_key: "secret" +admin_secret: "123456789" + +discord_webhook_url: "" +scraper_webhook_url: "" + +db_driver: "sqlite" +sqlite_path: "crawler.db" + +# Set db_driver to "postgres" and provide either postgres_dsn +# or individual postgres_* fields. +postgres_dsn: "" +postgres_host: "" +postgres_port: "5432" +postgres_user: "postgres" +postgres_password: "" +postgres_db_name: "dev" +postgres_sslmode: "disable" diff --git a/controllers/authController.go b/controllers/authController.go index 7a20636..460e51e 100644 --- a/controllers/authController.go +++ b/controllers/authController.go @@ -9,7 +9,7 @@ import ( func CheckAllowed(c *gin.Context) { secret := c.GetHeader("X-SECRET-KEY") - if secret != initializers.SECRET { + if secret != initializers.SecretKey { c.JSON(http.StatusForbidden, gin.H{"error": "Forbidden"}) c.Abort() return @@ -18,7 +18,7 @@ func CheckAllowed(c *gin.Context) { func CheckAdmin(c *gin.Context) { adminSecret := c.GetHeader("X-ADMIN-KEY") - if adminSecret != initializers.ADMIN_SECRET { + if adminSecret != initializers.AdminSecret { c.JSON(http.StatusForbidden, gin.H{"error": "Forbidden"}) c.Abort() return diff --git a/controllers/feController.go b/controllers/feController.go index 15bed31..80ede00 100644 --- a/controllers/feController.go +++ b/controllers/feController.go @@ -52,113 +52,6 @@ func convertToSimplifiedAddonWithoutFiles(addon models.Addon) models.SimplifiedA func GetDuplicates(c *gin.Context) { id := c.Param("id") - - // First check if addon exists (lightweight query) - var exists bool - if err := initializers.DB.Model(&models.Addon{}).Select("1").Where("id = ?", id).Limit(1).Find(&exists).Error; err != nil { - c.JSON(http.StatusNotFound, gin.H{"error": "Addon not found"}) - return - } - - // Get only the hashes from source addon files (don't load full addon yet) - var sourceHashes []string - if err := initializers.DB.Model(&models.AddonFile{}). - Where("addon_id = ? AND hash != ''", id). - Pluck("hash", &sourceHashes).Error; err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to fetch source addon files"}) - return - } - - // Now load the full source addon for the response - var sourceAddon models.Addon - if err := initializers.DB.Preload("AddonFiles").First(&sourceAddon, "id = ?", id).Error; err != nil { - c.JSON(http.StatusNotFound, gin.H{"error": "Addon not found"}) - return - } - - simplifiedSourceAddon := convertToSimplifiedAddon(sourceAddon) - - if len(sourceHashes) == 0 { - c.JSON(http.StatusOK, models.DuplicatesResponse{ - SourceAddon: simplifiedSourceAddon, - DuplicateAddons: make(map[string]models.AddonWithDuplicates), - }) - return - } - - // Single query with JOIN to get all data - type DuplicateRow struct { - Hash string - Path string - Version string - AddonID string - AddonName string - AddonType string - Summary string - Preview string - Author string - SubscriberCount int - CurrentVersionNumber string - } - - var duplicates []DuplicateRow - err := initializers.DB.Table("addon_files af"). - Select(`af.hash, af.path, af.version, af.addon_id, - a.name as addon_name, a.type as addon_type, a.summary, - a.preview, a.author, a.subscriber_count, a.current_version_number`). - Joins("INNER JOIN addons a ON a.id = af.addon_id"). - Joins("LEFT JOIN whitelisted_hashes wh ON wh.hash = af.hash"). - Where(`af.hash IN ? - AND af.addon_id != ? - AND af.deleted_at IS NULL - AND wh.hash IS NULL`, - sourceHashes, id). - Scan(&duplicates).Error - - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to fetch duplicates"}) - return - } - - // Build response from single result set - response := models.DuplicatesResponse{ - SourceAddon: simplifiedSourceAddon, - DuplicateAddons: make(map[string]models.AddonWithDuplicates), - } - - for _, dup := range duplicates { - if _, exists := response.DuplicateAddons[dup.AddonID]; !exists { - response.DuplicateAddons[dup.AddonID] = models.AddonWithDuplicates{ - Addon: models.SimplifiedAddon{ - ID: dup.AddonID, - Name: dup.AddonName, - Type: dup.AddonType, - Summary: dup.Summary, - Preview: dup.Preview, - Author: dup.Author, - SubscriberCount: dup.SubscriberCount, - CurrentVersionNumber: dup.CurrentVersionNumber, - AddonFiles: []models.SimplifiedAddonFile{}, - }, - Duplicates: []models.DuplicateFileInfo{}, - } - } - - entry := response.DuplicateAddons[dup.AddonID] - entry.Duplicates = append(entry.Duplicates, models.DuplicateFileInfo{ - Path: dup.Path, - Hash: dup.Hash, - Version: dup.Version, - AddonID: dup.AddonID, - }) - response.DuplicateAddons[dup.AddonID] = entry - } - - c.JSON(http.StatusOK, response) -} - -/*func GetDuplicates(c *gin.Context) { - id := c.Param("id") fmt.Println("Fetching addon with ID:", id) // Fetch the source addon with its files @@ -236,6 +129,19 @@ func GetDuplicates(c *gin.Context) { } } + // Set Recheck flag for source addon and all duplicate addons + if err := initializers.DB.Model(&models.Addon{}).Where("id = ?", id).Update("recheck", true).Error; err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update source addon"}) + return + } + + for _, addon := range duplicateAddons { + if err := initializers.DB.Model(&models.Addon{}).Where("id = ?", addon.ID).Update("recheck", true).Error; err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update duplicate addon"}) + return + } + } + // Build the response response := models.DuplicatesResponse{ SourceAddon: simplifiedSourceAddon, @@ -269,7 +175,7 @@ func GetDuplicates(c *gin.Context) { } c.JSON(http.StatusOK, response) -}*/ +} func GetPossibleAddons(c *gin.Context) { query := c.Query("q") diff --git a/controllers/indexerController.go b/controllers/indexerController.go index 85a3bf8..ecee92b 100644 --- a/controllers/indexerController.go +++ b/controllers/indexerController.go @@ -105,7 +105,7 @@ func SaveIndexingResult(c *gin.Context) { return } - if initializers.DiscordWebhook != "" { + if initializers.DiscordWebhookURL != "" { text := "Indexing of addon " + addon.Name + " (" + addon.ID + ") was successful." colour := 2228479 nbr := strconv.Itoa(len(result.Files)) @@ -118,16 +118,16 @@ func SaveIndexingResult(c *gin.Context) { percentage := 0.0 if addonsCount > 0 { - percentage = (float64(addonsCount - addonsToBeIndexed) / float64(addonsCount)) * 100 + percentage = (float64(addonsCount-addonsToBeIndexed) / float64(addonsCount)) * 100 } - txt := fmt.Sprintf("%d/%d (%.2f%%).", addonsCount - addonsToBeIndexed, addonsCount, percentage) + txt := fmt.Sprintf("%d/%d (%.2f%%).", addonsCount-addonsToBeIndexed, addonsCount, percentage) myEmbed := models.CustomEmbed{ - Title: text, + Title: text, Description: fmt.Sprintf("Files Indexed: %s\nOverall Progress: %s", nbr, txt), - Color: colour, - Timestamp: time.Now(), + Color: colour, + Timestamp: time.Now(), Image: models.CustomImage{ URL: addon.Preview, }, @@ -152,10 +152,10 @@ func SaveIndexingResult(c *gin.Context) { myHook := models.CustomHook{ Username: "Reforger Crawler", - Embeds: []models.CustomEmbed{myEmbed}, + Embeds: []models.CustomEmbed{myEmbed}, } - err := SendCustomWebhook(initializers.DiscordWebhook, myHook) + err := SendCustomWebhook(initializers.DiscordWebhookURL, myHook) if err != nil { fmt.Println("Error sending webhook:", err) } @@ -208,27 +208,27 @@ func checkIndexingTimeout() { } func SendCustomWebhook(webhookURL string, hook models.CustomHook) error { - payload, err := json.Marshal(hook) - if err != nil { - return err - } + payload, err := json.Marshal(hook) + if err != nil { + return err + } - req, err := http.NewRequest("POST", webhookURL, bytes.NewBuffer(payload)) - if err != nil { - return err - } - req.Header.Set("Content-Type", "application/json") + req, err := http.NewRequest("POST", webhookURL, bytes.NewBuffer(payload)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() - if resp.StatusCode < 200 || resp.StatusCode > 299 { - return fmt.Errorf("webhook failed with status code %d", resp.StatusCode) - } + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return fmt.Errorf("webhook failed with status code %d", resp.StatusCode) + } - return nil -} \ No newline at end of file + return nil +} diff --git a/controllers/scraperController.go b/controllers/scraperController.go index bc0869b..b9efbed 100644 --- a/controllers/scraperController.go +++ b/controllers/scraperController.go @@ -1,7 +1,9 @@ package controllers import ( + "bytes" "fmt" + "io" "strconv" "time" @@ -15,10 +17,14 @@ var nbrOfUnsentWebhooks int func CreateAddon(c *gin.Context) { var addonR models.Addon - body := make([]byte, 0) - c.Request.Body.Read(body) - fmt.Println(body) - err := c.ShouldBindJSON(&addonR) + body, err := io.ReadAll(c.Request.Body) + if err != nil { + c.JSON(400, gin.H{"error": "Failed to read request body"}) + return + } + fmt.Println(string(body)) + c.Request.Body = io.NopCloser(bytes.NewBuffer(body)) + err = c.ShouldBindJSON(&addonR) if err != nil { c.JSON(400, gin.H{"error": "Invalid JSON"}) return @@ -44,7 +50,7 @@ func CreateAddon(c *gin.Context) { nbrOfUnsentWebhooks = 0 go func() { - webhookURL := initializers.ScraperWebhook + webhookURL := initializers.ScraperWebhookURL if webhookURL != "" { text := fmt.Sprintf("New %d addons found\nLast: %s (%s).", oldNbr, addon.ID, addon.Name) colour := 2228479 @@ -102,44 +108,44 @@ func CreateAddon(c *gin.Context) { // send webhook about update go func() { - webhookURL := initializers.ScraperWebhook - if webhookURL != "" { - text := fmt.Sprintf("Addon: %s (%s) updated", addon.ID, addon.Name) - colour := 2228479 - size := fmt.Sprintf("%.2f MB", float64(addon.CurrentVersionSize)/1_000_000) + webhookURL := initializers.ScraperWebhookURL + if webhookURL != "" { + text := fmt.Sprintf("Addon: %s (%s) updated", addon.ID, addon.Name) + colour := 2228479 + size := fmt.Sprintf("%.2f MB", float64(addon.CurrentVersionSize)/1_000_000) - myEmbed := models.CustomEmbed{ - Title: text, - Color: colour, - Timestamp: time.Now(), - Image: models.CustomImage{ - URL: addon.Preview, - }, - Fields: []models.CustomEmbedField{ - { - Name: "Size", - Value: size, - Inline: true, + myEmbed := models.CustomEmbed{ + Title: text, + Color: colour, + Timestamp: time.Now(), + Image: models.CustomImage{ + URL: addon.Preview, }, - { - Name: "Current Version", - Value: addon.CurrentVersionNumber, - Inline: true, + Fields: []models.CustomEmbedField{ + { + Name: "Size", + Value: size, + Inline: true, + }, + { + Name: "Current Version", + Value: addon.CurrentVersionNumber, + Inline: true, + }, }, - }, - } + } - myHook := models.CustomHook{ - Username: "Reforger Crawler", - Embeds: []models.CustomEmbed{myEmbed}, - } + myHook := models.CustomHook{ + Username: "Reforger Crawler", + Embeds: []models.CustomEmbed{myEmbed}, + } - err := SendCustomWebhook(webhookURL, myHook) - if err != nil { - fmt.Println("Error sending webhook:", err) + err := SendCustomWebhook(webhookURL, myHook) + if err != nil { + fmt.Println("Error sending webhook:", err) + } } - } - }() + }() } } c.JSON(200, gin.H{"status": "success"}) diff --git a/go.mod b/go.mod index 10105c1..9451edc 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,10 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.20.0 // indirect github.com/goccy/go-json v0.10.2 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.6.0 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/leodido/go-urn v1.4.0 // indirect @@ -30,10 +34,12 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect golang.org/x/arch v0.8.0 // indirect - golang.org/x/crypto v0.23.0 // indirect + golang.org/x/crypto v0.31.0 // indirect golang.org/x/net v0.25.0 // indirect - golang.org/x/sys v0.20.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/sys v0.28.0 // indirect google.golang.org/protobuf v1.34.1 // indirect + gorm.io/driver/postgres v1.6.0 // indirect ) require ( diff --git a/go.sum b/go.sum index 7a564d7..f2644ec 100644 --- a/go.sum +++ b/go.sum @@ -30,6 +30,14 @@ github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MG github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= +github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= @@ -76,12 +84,18 @@ golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -93,6 +107,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4= +gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo= gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8= gorm.io/gorm v1.30.3 h1:QiG8upl0Sg9ba2Zatfjy0fy4It2iNBL2/eMdvEkdXNs= diff --git a/initializers/initialize.go b/initializers/initialize.go index bb5f6a8..6e0024a 100644 --- a/initializers/initialize.go +++ b/initializers/initialize.go @@ -1,91 +1,240 @@ package initializers import ( + "errors" + "fmt" "log" "os" + "strings" "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" "gopkg.in/yaml.v3" + "gorm.io/driver/postgres" "gorm.io/driver/sqlite" "gorm.io/gorm" ) var DB *gorm.DB -var PORT string -var IP string -var SECRET string -var DB_NAME string -var DiscordWebhook string -var ADMIN_SECRET string -var ScraperWebhook string +var ServerPort string +var ServerHost string +var SecretKey string +var DiscordWebhookURL string +var AdminSecret string +var ScraperWebhookURL string +var DatabaseDriver string +var SQLitePath string +var PostgresDSN string func ConnectToDB() { - db, err := gorm.Open(sqlite.Open(DB_NAME), &gorm.Config{ - PrepareStmt: true, // Cache prepared statements - SkipDefaultTransaction: true, // Disable default transactions for better performance - }) - if err != nil { - log.Fatal("Failed to connect to database") + var ( + db *gorm.DB + err error + ) + + switch strings.ToLower(DatabaseDriver) { + case "", "sqlite", "sqlite3": + sqlitePath := SQLitePath + if sqlitePath == "" { + sqlitePath = "crawler.db" + } + + dsn := fmt.Sprintf("%s?_busy_timeout=5000&_journal_mode=WAL", sqlitePath) + db, err = gorm.Open(sqlite.Open(dsn), &gorm.Config{ + SkipDefaultTransaction: true, + }) + if err != nil { + log.Fatalf("Failed to connect to sqlite database: %v", err) + } + + if err := db.Exec("PRAGMA synchronous=NORMAL").Error; err != nil { + log.Fatalf("Failed to set PRAGMA synchronous: %v", err) + } + if err := db.Exec("PRAGMA cache_size=-64000").Error; err != nil { + log.Fatalf("Failed to set PRAGMA cache_size: %v", err) + } + if err := db.Exec("PRAGMA temp_store=MEMORY").Error; err != nil { + log.Fatalf("Failed to set PRAGMA temp_store: %v", err) + } + if err := db.Exec("PRAGMA mmap_size=268435456").Error; err != nil { + log.Fatalf("Failed to set PRAGMA mmap_size: %v", err) + } + case "postgres", "postgresql": + if PostgresDSN == "" { + log.Fatal("PostgreSQL selected but no DSN is configured") + } + + db, err = gorm.Open(postgres.Open(PostgresDSN), &gorm.Config{ + SkipDefaultTransaction: true, + }) + if err != nil { + log.Fatalf("Failed to connect to postgres database: %v", err) + } + default: + log.Fatalf("Unsupported database driver: %s", DatabaseDriver) } DB = db - // Optimize SQLite for performance - DB.Exec("PRAGMA journal_mode=WAL") // Write-Ahead Logging for better concurrency - DB.Exec("PRAGMA synchronous=NORMAL") // Balance between safety and speed - DB.Exec("PRAGMA cache_size=-64000") // 64MB cache (negative = KB) - DB.Exec("PRAGMA temp_store=MEMORY") // Use memory for temp tables - DB.Exec("PRAGMA mmap_size=268435456") // 256MB memory-mapped I/O + if err := DB.AutoMigrate(&models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{}); err != nil { + log.Fatalf("Failed to run migrations: %v", err) + } - DB.AutoMigrate(&models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{}) + if err := DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_hash_addon_optimized + ON addon_files(hash, addon_id) WHERE deleted_at IS NULL`).Error; err != nil { + log.Fatalf("Failed to create idx_addon_files_hash_addon_optimized: %v", err) + } - // Create optimized indexes for duplicate detection queries - // Composite index for efficient hash lookups excluding specific addon_id - DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_hash_addon_optimized - ON addon_files(hash, addon_id) WHERE deleted_at IS NULL`) + if err := DB.Exec(`CREATE INDEX IF NOT EXISTS idx_whitelisted_hash + ON whitelisted_hashes(hash)`).Error; err != nil { + log.Fatalf("Failed to create idx_whitelisted_hash: %v", err) + } - // Index for whitelisted hashes lookup - DB.Exec(`CREATE INDEX IF NOT EXISTS idx_whitelisted_hash - ON whitelisted_hashes(hash)`) + if err := DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addons_id + ON addons(id)`).Error; err != nil { + log.Fatalf("Failed to create idx_addons_id: %v", err) + } - // Additional index on addon_id for the JOIN operation - DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addons_id - ON addons(id)`) - - // Covering index for addon_files to avoid table lookups - DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_covering - ON addon_files(addon_id, hash, path, version) WHERE deleted_at IS NULL`) + if err := DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_covering + ON addon_files(addon_id, hash, path, version) WHERE deleted_at IS NULL`).Error; err != nil { + log.Fatalf("Failed to create idx_addon_files_covering: %v", err) + } } type Configuration struct { - Port string `yaml:"port"` - IP string `yaml:"ip"` - Secret string `yaml:"secret"` - DB string `yaml:"db"` - DiscordWebhook string `yaml:"discordWebhook"` - ADMIN_SECRET string `yaml:"adminSecret"` - ScraperWebhook string `yaml:"scraperWebhook"` + ServerPort string `yaml:"server_port"` + ServerHost string `yaml:"server_host"` + SecretKey string `yaml:"secret_key"` + AdminSecret string `yaml:"admin_secret"` + DiscordWebhookURL string `yaml:"discord_webhook_url"` + ScraperWebhookURL string `yaml:"scraper_webhook_url"` + DatabaseDriver string `yaml:"db_driver"` + SQLitePath string `yaml:"sqlite_path"` + PostgresDSN string `yaml:"postgres_dsn"` + PostgresHost string `yaml:"postgres_host"` + PostgresPort string `yaml:"postgres_port"` + PostgresUser string `yaml:"postgres_user"` + PostgresPassword string `yaml:"postgres_password"` + PostgresDBName string `yaml:"postgres_db_name"` + PostgresSSLMode string `yaml:"postgres_sslmode"` + LegacyPort string `yaml:"port"` + LegacyIP string `yaml:"ip"` + LegacySecret string `yaml:"secret"` + LegacyDB string `yaml:"db"` + LegacyDiscordHook string `yaml:"discordWebhook"` + LegacyAdminSecret string `yaml:"adminSecret"` + LegacyScraperHook string `yaml:"scraperWebhook"` } func Load() { - file, err := os.ReadFile("config.yaml") - if err != nil { - log.Fatal("Failed to open config file") - } - configuration := Configuration{ - DB: "register.db", - } - err = yaml.Unmarshal(file, &configuration) - if err != nil { - log.Fatal("Failed to read yaml file") + ServerPort: "8083", + ServerHost: "0.0.0.0", + DatabaseDriver: "sqlite", + SQLitePath: "crawler.db", + PostgresSSLMode: "disable", } - PORT = configuration.Port - IP = configuration.IP - SECRET = configuration.Secret - DB_NAME = configuration.DB - DiscordWebhook = configuration.DiscordWebhook - ADMIN_SECRET = configuration.ADMIN_SECRET - ScraperWebhook = configuration.ScraperWebhook + file, err := os.ReadFile("config.yaml") + if err == nil { + if err := yaml.Unmarshal(file, &configuration); err != nil { + log.Fatalf("Failed to read yaml file: %v", err) + } + } else if !errors.Is(err, os.ErrNotExist) { + log.Fatalf("Failed to open config file: %v", err) + } + + applyLegacyConfigFallbacks(&configuration) + applyEnvOverrides(&configuration) + + ServerPort = configuration.ServerPort + ServerHost = configuration.ServerHost + SecretKey = configuration.SecretKey + AdminSecret = configuration.AdminSecret + DiscordWebhookURL = configuration.DiscordWebhookURL + ScraperWebhookURL = configuration.ScraperWebhookURL + DatabaseDriver = strings.ToLower(configuration.DatabaseDriver) + SQLitePath = configuration.SQLitePath + + if strings.TrimSpace(configuration.PostgresDSN) != "" { + PostgresDSN = configuration.PostgresDSN + } else { + PostgresDSN = buildPostgresDSN(configuration) + } +} + +func applyLegacyConfigFallbacks(c *Configuration) { + if c.ServerPort == "" { + c.ServerPort = c.LegacyPort + } + if c.ServerHost == "" { + c.ServerHost = c.LegacyIP + } + if c.SecretKey == "" { + c.SecretKey = c.LegacySecret + } + if c.AdminSecret == "" { + c.AdminSecret = c.LegacyAdminSecret + } + if c.DiscordWebhookURL == "" { + c.DiscordWebhookURL = c.LegacyDiscordHook + } + if c.ScraperWebhookURL == "" { + c.ScraperWebhookURL = c.LegacyScraperHook + } + if c.SQLitePath == "" { + c.SQLitePath = c.LegacyDB + } +} + +func applyEnvOverrides(c *Configuration) { + c.ServerPort = firstEnv([]string{"SERVER_PORT", "PORT"}, c.ServerPort) + c.ServerHost = firstEnv([]string{"SERVER_HOST", "IP"}, c.ServerHost) + c.SecretKey = firstEnv([]string{"SECRET_KEY", "SECRET"}, c.SecretKey) + c.AdminSecret = firstEnv([]string{"ADMIN_SECRET"}, c.AdminSecret) + c.DiscordWebhookURL = firstEnv([]string{"DISCORD_WEBHOOK_URL", "DISCORD_WEBHOOK"}, c.DiscordWebhookURL) + c.ScraperWebhookURL = firstEnv([]string{"SCRAPER_WEBHOOK_URL", "SCRAPER_WEBHOOK"}, c.ScraperWebhookURL) + c.DatabaseDriver = firstEnv([]string{"DB_DRIVER"}, c.DatabaseDriver) + c.SQLitePath = firstEnv([]string{"SQLITE_PATH", "DB"}, c.SQLitePath) + c.PostgresDSN = firstEnv([]string{"POSTGRES_DSN"}, c.PostgresDSN) + c.PostgresHost = firstEnv([]string{"POSTGRES_HOST"}, c.PostgresHost) + c.PostgresPort = firstEnv([]string{"POSTGRES_PORT"}, c.PostgresPort) + c.PostgresUser = firstEnv([]string{"POSTGRES_USER"}, c.PostgresUser) + c.PostgresPassword = firstEnv([]string{"POSTGRES_PASSWORD"}, c.PostgresPassword) + c.PostgresDBName = firstEnv([]string{"POSTGRES_DB_NAME"}, c.PostgresDBName) + c.PostgresSSLMode = firstEnv([]string{"POSTGRES_SSLMODE"}, c.PostgresSSLMode) +} + +func firstEnv(keys []string, fallback string) string { + for _, k := range keys { + if value, ok := os.LookupEnv(k); ok && strings.TrimSpace(value) != "" { + return value + } + } + return fallback +} + +func buildPostgresDSN(c Configuration) string { + if c.PostgresHost == "" || c.PostgresUser == "" || c.PostgresDBName == "" { + return "" + } + + port := c.PostgresPort + if port == "" { + port = "5432" + } + + sslMode := c.PostgresSSLMode + if sslMode == "" { + sslMode = "disable" + } + + return fmt.Sprintf( + "host=%s user=%s password=%s dbname=%s port=%s sslmode=%s", + c.PostgresHost, + c.PostgresUser, + c.PostgresPassword, + c.PostgresDBName, + port, + sslMode, + ) } diff --git a/main.go b/main.go index 0b9b4ac..1dcc697 100644 --- a/main.go +++ b/main.go @@ -17,6 +17,9 @@ func init() { func main() { r := gin.Default() r.Use(CORSMiddleware()) + r.GET("/health", func(c *gin.Context) { + c.JSON(http.StatusOK, gin.H{"status": "ok"}) + }) back := r.Group("/back") { @@ -29,7 +32,7 @@ func main() { admin := r.Group("/admin") { - admin.POST("/addWhitelistedHash",controllers.CheckAdmin, controllers.AddWhitelistedHash) + admin.POST("/addWhitelistedHash", controllers.CheckAdmin, controllers.AddWhitelistedHash) admin.GET("/duplicates", controllers.CheckAdmin) admin.POST("/setPriority", controllers.CheckAdmin) admin.POST("/setToBeIndexed", controllers.CheckAdmin) @@ -41,32 +44,32 @@ func main() { v1.GET("/getPossible", controllers.GetPossibleAddons) } - r.Run(initializers.IP + ":" + initializers.PORT) + r.Run(initializers.ServerHost + ":" + initializers.ServerPort) } func CORSMiddleware() gin.HandlerFunc { - return func(c *gin.Context) { + return func(c *gin.Context) { orig := c.Request.Header.Clone().Get("Origin") - c.Writer.Header().Set("Access-Control-Allow-Origin", orig) - c.Writer.Header().Set("Access-Control-Allow-Credentials", "true") - c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With") - c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT, DELETE") + c.Writer.Header().Set("Access-Control-Allow-Origin", orig) + c.Writer.Header().Set("Access-Control-Allow-Credentials", "true") + c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With") + c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT, DELETE") - if c.Request.Method == "OPTIONS" { - c.AbortWithStatus(http.StatusNoContent) - return - } + if c.Request.Method == "OPTIONS" { + c.AbortWithStatus(http.StatusNoContent) + return + } - c.Next() - } + c.Next() + } } func OptionsMiddleware() gin.HandlerFunc { - return func(c *gin.Context) { + return func(c *gin.Context) { if c.Request.Method != "OPTIONS" { c.Next() } else { c.AbortWithStatus(http.StatusOK) } - } + } } diff --git a/models/addon.go b/models/addon.go index eb90239..dbb092b 100644 --- a/models/addon.go +++ b/models/addon.go @@ -2,28 +2,32 @@ package models import ( "time" + + "gorm.io/gorm" ) type Addon struct { - ID string `gorm:"primaryKey" json:"id"` - Name string `json:"name"` - Type string `json:"type"` - Summary string `json:"summary"` - Unlisted bool `json:"unlisted"` - Private bool `json:"private"` - Blocked bool `json:"blocked"` - SubscriberCount int `json:"subscriberCount"` - CurrentVersionNumber string `json:"currentVersionNumber"` - CurrentVersionSize int `json:"currentVersionSize"` - CurrentVersionID int `json:"currentVersionId"` - CreatedAt time.Time `json:"createdAt"` - UpdatedAt time.Time `json:"updatedAt"` - ToBeIndexed bool `json:"toBeIndexed" gorm:"default:true"` - IsBeingIndexed bool `json:"isBeingIndexed" gorm:"default:false"` - IndexStartTime time.Time `json:"indexStartTime"` - PriorityIndexing bool `json:"priorityIndexing" gorm:"default:false"` - AddonFiles []AddonFile `json:"addonFiles"` + ID string `gorm:"primaryKey" json:"id"` + Name string `json:"name"` + Type string `json:"type"` + Summary string `json:"summary"` + Unlisted bool `json:"unlisted"` + Private bool `json:"private"` + Blocked bool `json:"blocked"` + SubscriberCount int `json:"subscriberCount"` + CurrentVersionNumber string `json:"currentVersionNumber"` + CurrentVersionSize int `json:"currentVersionSize"` + CurrentVersionID int `json:"currentVersionId"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` + DeletedAt gorm.DeletedAt `gorm:"index"` + ToBeIndexed bool `json:"toBeIndexed" gorm:"default:true"` + IsBeingIndexed bool `json:"isBeingIndexed" gorm:"default:false"` + IndexStartTime time.Time `json:"indexStartTime"` + PriorityIndexing bool `json:"priorityIndexing" gorm:"default:false"` + AddonFiles []AddonFile `json:"addonFiles"` + Recheck bool `json:"recheck" gorm:"default:false"` Preview string `json:"preview"` - Author string `json:"author"` + Author string `json:"author"` }