TBD/reforger_crawler_main
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updates, dokploy
All checks were successful
Cross Compile Go / build (push) Successful in 12m11s
Details

Browse Source
This commit is contained in:
ilbinek
2026-04-09 01:39:37 +02:00
parent fcebe00175
commit e39a92c6d3
14 changed files with 450 additions and 276 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
.dockerignore Normal file
View File

@@ -0,0 +1,11 @@
.git
.gitea
.vscode
api
*.db
*.db-journal
*.db-shm
*.db-wal
.env
.env.*
README.md

28
.env.dokploy.example Normal file
View File

@@ -0,0 +1,28 @@
# Dokploy environment example for Reforger Crawler
# Copy these values into Dokploy Environment Variables.
SERVER_HOST=0.0.0.0
SERVER_PORT=8083
SECRET_KEY=change-me-secret
ADMIN_SECRET=change-me-admin-secret
DISCORD_WEBHOOK_URL=
SCRAPER_WEBHOOK_URL=
# sqlite or postgres
DB_DRIVER=postgres
# Used only when DB_DRIVER=sqlite
SQLITE_PATH=crawler.db
# Option A: full DSN
POSTGRES_DSN=postgresql://postgres:change-me-password@postgres-host:5432/dev
# Option B: split fields (used when POSTGRES_DSN is empty)
POSTGRES_HOST=postgres-host
POSTGRES_PORT=5432
POSTGRES_USER=postgres
POSTGRES_PASSWORD=change-me-password
POSTGRES_DB_NAME=dev
POSTGRES_SSLMODE=disable

3
.gitignore vendored
View File

@@ -2,3 +2,6 @@
reforger_crawler_main reforger_crawler_main
crawler.db crawler.db
.env
.env.*
!.env.dokploy.example

29
Dockerfile Normal file
View File

@@ -0,0 +1,29 @@
FROM golang:1.25-bookworm AS builder
WORKDIR /app
# Required for CGO build (sqlite driver).
RUN apt-get update \
&& apt-get install -y --no-install-recommends build-essential ca-certificates \
&& rm -rf /var/lib/apt/lists/*
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o /out/reforger_crawler_main .
FROM debian:bookworm-slim AS runtime
WORKDIR /app
RUN apt-get update \
&& apt-get install -y --no-install-recommends ca-certificates tzdata \
&& rm -rf /var/lib/apt/lists/*
COPY --from=builder /out/reforger_crawler_main /app/reforger_crawler_main
COPY config.yaml /app/config.yaml
EXPOSE 8083
ENTRYPOINT ["/app/reforger_crawler_main"]

27
config.yaml
View File

@@ -1,7 +1,20 @@
port: 8083 server_port: "8083"
ip: "localhost" server_host: "localhost"
secret: "secret" secret_key: "secret"
db: "crawler.db" admin_secret: "123456789"
discordWebhook: ""
scraperWebhook: "" discord_webhook_url: ""
adminSecret: "123456789" scraper_webhook_url: ""
db_driver: "sqlite"
sqlite_path: "crawler.db"
# Set db_driver to "postgres" and provide either postgres_dsn
# or individual postgres_* fields.
postgres_dsn: ""
postgres_host: ""
postgres_port: "5432"
postgres_user: "postgres"
postgres_password: ""
postgres_db_name: "dev"
postgres_sslmode: "disable"

4
controllers/authController.go
View File

@@ -9,7 +9,7 @@ import (
func CheckAllowed(c *gin.Context) { func CheckAllowed(c *gin.Context) {
secret := c.GetHeader("X-SECRET-KEY") secret := c.GetHeader("X-SECRET-KEY")
if secret != initializers.SECRET { if secret != initializers.SecretKey {
c.JSON(http.StatusForbidden, gin.H{"error": "Forbidden"}) c.JSON(http.StatusForbidden, gin.H{"error": "Forbidden"})
c.Abort() c.Abort()
return return
@@ -18,7 +18,7 @@ func CheckAllowed(c *gin.Context) {
func CheckAdmin(c *gin.Context) { func CheckAdmin(c *gin.Context) {
adminSecret := c.GetHeader("X-ADMIN-KEY") adminSecret := c.GetHeader("X-ADMIN-KEY")
if adminSecret != initializers.ADMIN_SECRET { if adminSecret != initializers.AdminSecret {
c.JSON(http.StatusForbidden, gin.H{"error": "Forbidden"}) c.JSON(http.StatusForbidden, gin.H{"error": "Forbidden"})
c.Abort() c.Abort()
return return

122
controllers/feController.go
View File

@@ -52,113 +52,6 @@ func convertToSimplifiedAddonWithoutFiles(addon models.Addon) models.SimplifiedA
func GetDuplicates(c *gin.Context) { func GetDuplicates(c *gin.Context) {
id := c.Param("id") id := c.Param("id")
// First check if addon exists (lightweight query)
var exists bool
if err := initializers.DB.Model(&models.Addon{}).Select("1").Where("id = ?", id).Limit(1).Find(&exists).Error; err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "Addon not found"})
return
}
// Get only the hashes from source addon files (don't load full addon yet)
var sourceHashes []string
if err := initializers.DB.Model(&models.AddonFile{}).
Where("addon_id = ? AND hash != ''", id).
Pluck("hash", &sourceHashes).Error; err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to fetch source addon files"})
return
}
// Now load the full source addon for the response
var sourceAddon models.Addon
if err := initializers.DB.Preload("AddonFiles").First(&sourceAddon, "id = ?", id).Error; err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "Addon not found"})
return
}
simplifiedSourceAddon := convertToSimplifiedAddon(sourceAddon)
if len(sourceHashes) == 0 {
c.JSON(http.StatusOK, models.DuplicatesResponse{
SourceAddon: simplifiedSourceAddon,
DuplicateAddons: make(map[string]models.AddonWithDuplicates),
})
return
}
// Single query with JOIN to get all data
type DuplicateRow struct {
Hash string
Path string
Version string
AddonID string
AddonName string
AddonType string
Summary string
Preview string
Author string
SubscriberCount int
CurrentVersionNumber string
}
var duplicates []DuplicateRow
err := initializers.DB.Table("addon_files af").
Select(`af.hash, af.path, af.version, af.addon_id,
a.name as addon_name, a.type as addon_type, a.summary,
a.preview, a.author, a.subscriber_count, a.current_version_number`).
Joins("INNER JOIN addons a ON a.id = af.addon_id").
Joins("LEFT JOIN whitelisted_hashes wh ON wh.hash = af.hash").
Where(`af.hash IN ?
AND af.addon_id != ?
AND af.deleted_at IS NULL
AND wh.hash IS NULL`,
sourceHashes, id).
Scan(&duplicates).Error
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to fetch duplicates"})
return
}
// Build response from single result set
response := models.DuplicatesResponse{
SourceAddon: simplifiedSourceAddon,
DuplicateAddons: make(map[string]models.AddonWithDuplicates),
}
for _, dup := range duplicates {
if _, exists := response.DuplicateAddons[dup.AddonID]; !exists {
response.DuplicateAddons[dup.AddonID] = models.AddonWithDuplicates{
Addon: models.SimplifiedAddon{
ID: dup.AddonID,
Name: dup.AddonName,
Type: dup.AddonType,
Summary: dup.Summary,
Preview: dup.Preview,
Author: dup.Author,
SubscriberCount: dup.SubscriberCount,
CurrentVersionNumber: dup.CurrentVersionNumber,
AddonFiles: []models.SimplifiedAddonFile{},
},
Duplicates: []models.DuplicateFileInfo{},
}
}
entry := response.DuplicateAddons[dup.AddonID]
entry.Duplicates = append(entry.Duplicates, models.DuplicateFileInfo{
Path: dup.Path,
Hash: dup.Hash,
Version: dup.Version,
AddonID: dup.AddonID,
})
response.DuplicateAddons[dup.AddonID] = entry
}
c.JSON(http.StatusOK, response)
}
/*func GetDuplicates(c *gin.Context) {
id := c.Param("id")
fmt.Println("Fetching addon with ID:", id) fmt.Println("Fetching addon with ID:", id)
// Fetch the source addon with its files // Fetch the source addon with its files
@@ -236,6 +129,19 @@ func GetDuplicates(c *gin.Context) {
} }
} }
// Set Recheck flag for source addon and all duplicate addons
if err := initializers.DB.Model(&models.Addon{}).Where("id = ?", id).Update("recheck", true).Error; err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update source addon"})
return
}
for _, addon := range duplicateAddons {
if err := initializers.DB.Model(&models.Addon{}).Where("id = ?", addon.ID).Update("recheck", true).Error; err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update duplicate addon"})
return
}
}
// Build the response // Build the response
response := models.DuplicatesResponse{ response := models.DuplicatesResponse{
SourceAddon: simplifiedSourceAddon, SourceAddon: simplifiedSourceAddon,
@@ -269,7 +175,7 @@ func GetDuplicates(c *gin.Context) {
} }
c.JSON(http.StatusOK, response) c.JSON(http.StatusOK, response)
}*/ }
func GetPossibleAddons(c *gin.Context) { func GetPossibleAddons(c *gin.Context) {
query := c.Query("q") query := c.Query("q")

8
controllers/indexerController.go
View File

@@ -105,7 +105,7 @@ func SaveIndexingResult(c *gin.Context) {
return return
} }
if initializers.DiscordWebhook != "" { if initializers.DiscordWebhookURL != "" {
text := "Indexing of addon " + addon.Name + " (" + addon.ID + ") was successful." text := "Indexing of addon " + addon.Name + " (" + addon.ID + ") was successful."
colour := 2228479 colour := 2228479
nbr := strconv.Itoa(len(result.Files)) nbr := strconv.Itoa(len(result.Files))
@@ -118,10 +118,10 @@ func SaveIndexingResult(c *gin.Context) {
percentage := 0.0 percentage := 0.0
if addonsCount > 0 { if addonsCount > 0 {
percentage = (float64(addonsCount - addonsToBeIndexed) / float64(addonsCount)) * 100 percentage = (float64(addonsCount-addonsToBeIndexed) / float64(addonsCount)) * 100
} }
txt := fmt.Sprintf("%d/%d (%.2f%%).", addonsCount - addonsToBeIndexed, addonsCount, percentage) txt := fmt.Sprintf("%d/%d (%.2f%%).", addonsCount-addonsToBeIndexed, addonsCount, percentage)
myEmbed := models.CustomEmbed{ myEmbed := models.CustomEmbed{
Title: text, Title: text,
@@ -155,7 +155,7 @@ func SaveIndexingResult(c *gin.Context) {
Embeds: []models.CustomEmbed{myEmbed}, Embeds: []models.CustomEmbed{myEmbed},
} }
err := SendCustomWebhook(initializers.DiscordWebhook, myHook) err := SendCustomWebhook(initializers.DiscordWebhookURL, myHook)
if err != nil { if err != nil {
fmt.Println("Error sending webhook:", err) fmt.Println("Error sending webhook:", err)
} }

18
controllers/scraperController.go
View File

@@ -1,7 +1,9 @@
package controllers package controllers
import ( import (
"bytes"
"fmt" "fmt"
"io"
"strconv" "strconv"
"time" "time"
@@ -15,10 +17,14 @@ var nbrOfUnsentWebhooks int
func CreateAddon(c *gin.Context) { func CreateAddon(c *gin.Context) {
var addonR models.Addon var addonR models.Addon
body := make([]byte, 0) body, err := io.ReadAll(c.Request.Body)
c.Request.Body.Read(body) if err != nil {
fmt.Println(body) c.JSON(400, gin.H{"error": "Failed to read request body"})
err := c.ShouldBindJSON(&addonR) return
}
fmt.Println(string(body))
c.Request.Body = io.NopCloser(bytes.NewBuffer(body))
err = c.ShouldBindJSON(&addonR)
if err != nil { if err != nil {
c.JSON(400, gin.H{"error": "Invalid JSON"}) c.JSON(400, gin.H{"error": "Invalid JSON"})
return return
@@ -44,7 +50,7 @@ func CreateAddon(c *gin.Context) {
nbrOfUnsentWebhooks = 0 nbrOfUnsentWebhooks = 0
go func() { go func() {
webhookURL := initializers.ScraperWebhook webhookURL := initializers.ScraperWebhookURL
if webhookURL != "" { if webhookURL != "" {
text := fmt.Sprintf("New %d addons found\nLast: %s (%s).", oldNbr, addon.ID, addon.Name) text := fmt.Sprintf("New %d addons found\nLast: %s (%s).", oldNbr, addon.ID, addon.Name)
colour := 2228479 colour := 2228479
@@ -102,7 +108,7 @@ func CreateAddon(c *gin.Context) {
// send webhook about update // send webhook about update
go func() { go func() {
webhookURL := initializers.ScraperWebhook webhookURL := initializers.ScraperWebhookURL
if webhookURL != "" { if webhookURL != "" {
text := fmt.Sprintf("Addon: %s (%s) updated", addon.ID, addon.Name) text := fmt.Sprintf("Addon: %s (%s) updated", addon.ID, addon.Name)
colour := 2228479 colour := 2228479

10
go.mod
View File

@@ -20,6 +20,10 @@ require (
github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.20.0 // indirect github.com/go-playground/validator/v10 v10.20.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect github.com/goccy/go-json v0.10.2 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/pgx/v5 v5.6.0 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/json-iterator/go v1.1.12 // indirect github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect
github.com/leodido/go-urn v1.4.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect
@@ -30,10 +34,12 @@ require (
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.12 // indirect github.com/ugorji/go/codec v1.2.12 // indirect
golang.org/x/arch v0.8.0 // indirect golang.org/x/arch v0.8.0 // indirect
golang.org/x/crypto v0.23.0 // indirect golang.org/x/crypto v0.31.0 // indirect
golang.org/x/net v0.25.0 // indirect golang.org/x/net v0.25.0 // indirect
golang.org/x/sys v0.20.0 // indirect golang.org/x/sync v0.16.0 // indirect
golang.org/x/sys v0.28.0 // indirect
google.golang.org/protobuf v1.34.1 // indirect google.golang.org/protobuf v1.34.1 // indirect
gorm.io/driver/postgres v1.6.0 // indirect
) )
require ( require (

16
go.sum
View File

@@ -30,6 +30,14 @@ github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MG
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY=
github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
@@ -76,12 +84,18 @@ golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc=
golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
@@ -93,6 +107,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4=
gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo=
gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ=
gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8= gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8=
gorm.io/gorm v1.30.3 h1:QiG8upl0Sg9ba2Zatfjy0fy4It2iNBL2/eMdvEkdXNs= gorm.io/gorm v1.30.3 h1:QiG8upl0Sg9ba2Zatfjy0fy4It2iNBL2/eMdvEkdXNs=

261
initializers/initialize.go
View File

@@ -1,91 +1,240 @@
package initializers package initializers
import ( import (
"errors"
"fmt"
"log" "log"
"os" "os"
"strings"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models" "gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
"gorm.io/driver/postgres"
"gorm.io/driver/sqlite" "gorm.io/driver/sqlite"
"gorm.io/gorm" "gorm.io/gorm"
) )
var DB *gorm.DB var DB *gorm.DB
var PORT string var ServerPort string
var IP string var ServerHost string
var SECRET string var SecretKey string
var DB_NAME string var DiscordWebhookURL string
var DiscordWebhook string var AdminSecret string
var ADMIN_SECRET string var ScraperWebhookURL string
var ScraperWebhook string var DatabaseDriver string
var SQLitePath string
var PostgresDSN string
func ConnectToDB() { func ConnectToDB() {
db, err := gorm.Open(sqlite.Open(DB_NAME), &gorm.Config{ var (
PrepareStmt: true, // Cache prepared statements db *gorm.DB
SkipDefaultTransaction: true, // Disable default transactions for better performance err error
)
switch strings.ToLower(DatabaseDriver) {
case "", "sqlite", "sqlite3":
sqlitePath := SQLitePath
if sqlitePath == "" {
sqlitePath = "crawler.db"
}
dsn := fmt.Sprintf("%s?_busy_timeout=5000&_journal_mode=WAL", sqlitePath)
db, err = gorm.Open(sqlite.Open(dsn), &gorm.Config{
SkipDefaultTransaction: true,
}) })
if err != nil { if err != nil {
log.Fatal("Failed to connect to database") log.Fatalf("Failed to connect to sqlite database: %v", err)
}
if err := db.Exec("PRAGMA synchronous=NORMAL").Error; err != nil {
log.Fatalf("Failed to set PRAGMA synchronous: %v", err)
}
if err := db.Exec("PRAGMA cache_size=-64000").Error; err != nil {
log.Fatalf("Failed to set PRAGMA cache_size: %v", err)
}
if err := db.Exec("PRAGMA temp_store=MEMORY").Error; err != nil {
log.Fatalf("Failed to set PRAGMA temp_store: %v", err)
}
if err := db.Exec("PRAGMA mmap_size=268435456").Error; err != nil {
log.Fatalf("Failed to set PRAGMA mmap_size: %v", err)
}
case "postgres", "postgresql":
if PostgresDSN == "" {
log.Fatal("PostgreSQL selected but no DSN is configured")
}
db, err = gorm.Open(postgres.Open(PostgresDSN), &gorm.Config{
SkipDefaultTransaction: true,
})
if err != nil {
log.Fatalf("Failed to connect to postgres database: %v", err)
}
default:
log.Fatalf("Unsupported database driver: %s", DatabaseDriver)
} }
DB = db DB = db
// Optimize SQLite for performance if err := DB.AutoMigrate(&models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{}); err != nil {
DB.Exec("PRAGMA journal_mode=WAL") // Write-Ahead Logging for better concurrency log.Fatalf("Failed to run migrations: %v", err)
DB.Exec("PRAGMA synchronous=NORMAL") // Balance between safety and speed }
DB.Exec("PRAGMA cache_size=-64000") // 64MB cache (negative = KB)
DB.Exec("PRAGMA temp_store=MEMORY") // Use memory for temp tables
DB.Exec("PRAGMA mmap_size=268435456") // 256MB memory-mapped I/O
DB.AutoMigrate(&models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{}) if err := DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_hash_addon_optimized
ON addon_files(hash, addon_id) WHERE deleted_at IS NULL`).Error; err != nil {
log.Fatalf("Failed to create idx_addon_files_hash_addon_optimized: %v", err)
}
// Create optimized indexes for duplicate detection queries if err := DB.Exec(`CREATE INDEX IF NOT EXISTS idx_whitelisted_hash
// Composite index for efficient hash lookups excluding specific addon_id ON whitelisted_hashes(hash)`).Error; err != nil {
DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_hash_addon_optimized log.Fatalf("Failed to create idx_whitelisted_hash: %v", err)
ON addon_files(hash, addon_id) WHERE deleted_at IS NULL`) }
// Index for whitelisted hashes lookup if err := DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addons_id
DB.Exec(`CREATE INDEX IF NOT EXISTS idx_whitelisted_hash ON addons(id)`).Error; err != nil {
ON whitelisted_hashes(hash)`) log.Fatalf("Failed to create idx_addons_id: %v", err)
}
// Additional index on addon_id for the JOIN operation if err := DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_covering
DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addons_id ON addon_files(addon_id, hash, path, version) WHERE deleted_at IS NULL`).Error; err != nil {
ON addons(id)`) log.Fatalf("Failed to create idx_addon_files_covering: %v", err)
}
// Covering index for addon_files to avoid table lookups
DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_covering
ON addon_files(addon_id, hash, path, version) WHERE deleted_at IS NULL`)
} }
type Configuration struct { type Configuration struct {
Port string `yaml:"port"` ServerPort string `yaml:"server_port"`
IP string `yaml:"ip"` ServerHost string `yaml:"server_host"`
Secret string `yaml:"secret"` SecretKey string `yaml:"secret_key"`
DB string `yaml:"db"` AdminSecret string `yaml:"admin_secret"`
DiscordWebhook string `yaml:"discordWebhook"` DiscordWebhookURL string `yaml:"discord_webhook_url"`
ADMIN_SECRET string `yaml:"adminSecret"` ScraperWebhookURL string `yaml:"scraper_webhook_url"`
ScraperWebhook string `yaml:"scraperWebhook"` DatabaseDriver string `yaml:"db_driver"`
SQLitePath string `yaml:"sqlite_path"`
PostgresDSN string `yaml:"postgres_dsn"`
PostgresHost string `yaml:"postgres_host"`
PostgresPort string `yaml:"postgres_port"`
PostgresUser string `yaml:"postgres_user"`
PostgresPassword string `yaml:"postgres_password"`
PostgresDBName string `yaml:"postgres_db_name"`
PostgresSSLMode string `yaml:"postgres_sslmode"`
LegacyPort string `yaml:"port"`
LegacyIP string `yaml:"ip"`
LegacySecret string `yaml:"secret"`
LegacyDB string `yaml:"db"`
LegacyDiscordHook string `yaml:"discordWebhook"`
LegacyAdminSecret string `yaml:"adminSecret"`
LegacyScraperHook string `yaml:"scraperWebhook"`
} }
func Load() { func Load() {
file, err := os.ReadFile("config.yaml")
if err != nil {
log.Fatal("Failed to open config file")
}
configuration := Configuration{ configuration := Configuration{
DB: "register.db", ServerPort: "8083",
} ServerHost: "0.0.0.0",
err = yaml.Unmarshal(file, &configuration) DatabaseDriver: "sqlite",
if err != nil { SQLitePath: "crawler.db",
log.Fatal("Failed to read yaml file") PostgresSSLMode: "disable",
} }
PORT = configuration.Port file, err := os.ReadFile("config.yaml")
IP = configuration.IP if err == nil {
SECRET = configuration.Secret if err := yaml.Unmarshal(file, &configuration); err != nil {
DB_NAME = configuration.DB log.Fatalf("Failed to read yaml file: %v", err)
DiscordWebhook = configuration.DiscordWebhook }
ADMIN_SECRET = configuration.ADMIN_SECRET } else if !errors.Is(err, os.ErrNotExist) {
ScraperWebhook = configuration.ScraperWebhook log.Fatalf("Failed to open config file: %v", err)
}
applyLegacyConfigFallbacks(&configuration)
applyEnvOverrides(&configuration)
ServerPort = configuration.ServerPort
ServerHost = configuration.ServerHost
SecretKey = configuration.SecretKey
AdminSecret = configuration.AdminSecret
DiscordWebhookURL = configuration.DiscordWebhookURL
ScraperWebhookURL = configuration.ScraperWebhookURL
DatabaseDriver = strings.ToLower(configuration.DatabaseDriver)
SQLitePath = configuration.SQLitePath
if strings.TrimSpace(configuration.PostgresDSN) != "" {
PostgresDSN = configuration.PostgresDSN
} else {
PostgresDSN = buildPostgresDSN(configuration)
}
}
func applyLegacyConfigFallbacks(c *Configuration) {
if c.ServerPort == "" {
c.ServerPort = c.LegacyPort
}
if c.ServerHost == "" {
c.ServerHost = c.LegacyIP
}
if c.SecretKey == "" {
c.SecretKey = c.LegacySecret
}
if c.AdminSecret == "" {
c.AdminSecret = c.LegacyAdminSecret
}
if c.DiscordWebhookURL == "" {
c.DiscordWebhookURL = c.LegacyDiscordHook
}
if c.ScraperWebhookURL == "" {
c.ScraperWebhookURL = c.LegacyScraperHook
}
if c.SQLitePath == "" {
c.SQLitePath = c.LegacyDB
}
}
func applyEnvOverrides(c *Configuration) {
c.ServerPort = firstEnv([]string{"SERVER_PORT", "PORT"}, c.ServerPort)
c.ServerHost = firstEnv([]string{"SERVER_HOST", "IP"}, c.ServerHost)
c.SecretKey = firstEnv([]string{"SECRET_KEY", "SECRET"}, c.SecretKey)
c.AdminSecret = firstEnv([]string{"ADMIN_SECRET"}, c.AdminSecret)
c.DiscordWebhookURL = firstEnv([]string{"DISCORD_WEBHOOK_URL", "DISCORD_WEBHOOK"}, c.DiscordWebhookURL)
c.ScraperWebhookURL = firstEnv([]string{"SCRAPER_WEBHOOK_URL", "SCRAPER_WEBHOOK"}, c.ScraperWebhookURL)
c.DatabaseDriver = firstEnv([]string{"DB_DRIVER"}, c.DatabaseDriver)
c.SQLitePath = firstEnv([]string{"SQLITE_PATH", "DB"}, c.SQLitePath)
c.PostgresDSN = firstEnv([]string{"POSTGRES_DSN"}, c.PostgresDSN)
c.PostgresHost = firstEnv([]string{"POSTGRES_HOST"}, c.PostgresHost)
c.PostgresPort = firstEnv([]string{"POSTGRES_PORT"}, c.PostgresPort)
c.PostgresUser = firstEnv([]string{"POSTGRES_USER"}, c.PostgresUser)
c.PostgresPassword = firstEnv([]string{"POSTGRES_PASSWORD"}, c.PostgresPassword)
c.PostgresDBName = firstEnv([]string{"POSTGRES_DB_NAME"}, c.PostgresDBName)
c.PostgresSSLMode = firstEnv([]string{"POSTGRES_SSLMODE"}, c.PostgresSSLMode)
}
func firstEnv(keys []string, fallback string) string {
for _, k := range keys {
if value, ok := os.LookupEnv(k); ok && strings.TrimSpace(value) != "" {
return value
}
}
return fallback
}
func buildPostgresDSN(c Configuration) string {
if c.PostgresHost == "" || c.PostgresUser == "" || c.PostgresDBName == "" {
return ""
}
port := c.PostgresPort
if port == "" {
port = "5432"
}
sslMode := c.PostgresSSLMode
if sslMode == "" {
sslMode = "disable"
}
return fmt.Sprintf(
"host=%s user=%s password=%s dbname=%s port=%s sslmode=%s",
c.PostgresHost,
c.PostgresUser,
c.PostgresPassword,
c.PostgresDBName,
port,
sslMode,
)
} }

7
main.go
View File

@@ -17,6 +17,9 @@ func init() {
func main() { func main() {
r := gin.Default() r := gin.Default()
r.Use(CORSMiddleware()) r.Use(CORSMiddleware())
r.GET("/health", func(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{"status": "ok"})
})
back := r.Group("/back") back := r.Group("/back")
{ {
@@ -29,7 +32,7 @@ func main() {
admin := r.Group("/admin") admin := r.Group("/admin")
{ {
admin.POST("/addWhitelistedHash",controllers.CheckAdmin, controllers.AddWhitelistedHash) admin.POST("/addWhitelistedHash", controllers.CheckAdmin, controllers.AddWhitelistedHash)
admin.GET("/duplicates", controllers.CheckAdmin) admin.GET("/duplicates", controllers.CheckAdmin)
admin.POST("/setPriority", controllers.CheckAdmin) admin.POST("/setPriority", controllers.CheckAdmin)
admin.POST("/setToBeIndexed", controllers.CheckAdmin) admin.POST("/setToBeIndexed", controllers.CheckAdmin)
@@ -41,7 +44,7 @@ func main() {
v1.GET("/getPossible", controllers.GetPossibleAddons) v1.GET("/getPossible", controllers.GetPossibleAddons)
} }
r.Run(initializers.IP + ":" + initializers.PORT) r.Run(initializers.ServerHost + ":" + initializers.ServerPort)
} }
func CORSMiddleware() gin.HandlerFunc { func CORSMiddleware() gin.HandlerFunc {

4
models/addon.go
View File

@@ -2,6 +2,8 @@ package models
import ( import (
"time" "time"
"gorm.io/gorm"
) )
type Addon struct { type Addon struct {
@@ -18,11 +20,13 @@ type Addon struct {
CurrentVersionID int `json:"currentVersionId"` CurrentVersionID int `json:"currentVersionId"`
CreatedAt time.Time `json:"createdAt"` CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"` UpdatedAt time.Time `json:"updatedAt"`
DeletedAt gorm.DeletedAt `gorm:"index"`
ToBeIndexed bool `json:"toBeIndexed" gorm:"default:true"` ToBeIndexed bool `json:"toBeIndexed" gorm:"default:true"`
IsBeingIndexed bool `json:"isBeingIndexed" gorm:"default:false"` IsBeingIndexed bool `json:"isBeingIndexed" gorm:"default:false"`
IndexStartTime time.Time `json:"indexStartTime"` IndexStartTime time.Time `json:"indexStartTime"`
PriorityIndexing bool `json:"priorityIndexing" gorm:"default:false"` PriorityIndexing bool `json:"priorityIndexing" gorm:"default:false"`
AddonFiles []AddonFile `json:"addonFiles"` AddonFiles []AddonFile `json:"addonFiles"`
Recheck bool `json:"recheck" gorm:"default:false"`
Preview string `json:"preview"` Preview string `json:"preview"`
Author string `json:"author"` Author string `json:"author"`