TBD/reforger_crawler_main
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updates, dokploy
All checks were successful
Cross Compile Go / build (push) Successful in 12m11s
Details

Browse Source
This commit is contained in:
ilbinek
2026-04-09 01:39:37 +02:00
parent fcebe00175
commit e39a92c6d3
14 changed files with 450 additions and 276 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
.dockerignore Normal file
View File

@@ -0,0 +1,11 @@
.git
.gitea
.vscode
api
*.db
*.db-journal
*.db-shm
*.db-wal
.env
.env.*
README.md

28
.env.dokploy.example Normal file
View File

@@ -0,0 +1,28 @@
# Dokploy environment example for Reforger Crawler
# Copy these values into Dokploy Environment Variables.
SERVER_HOST=0.0.0.0
SERVER_PORT=8083
SECRET_KEY=change-me-secret
ADMIN_SECRET=change-me-admin-secret
DISCORD_WEBHOOK_URL=
SCRAPER_WEBHOOK_URL=
# sqlite or postgres
DB_DRIVER=postgres
# Used only when DB_DRIVER=sqlite
SQLITE_PATH=crawler.db
# Option A: full DSN
POSTGRES_DSN=postgresql://postgres:change-me-password@postgres-host:5432/dev
# Option B: split fields (used when POSTGRES_DSN is empty)
POSTGRES_HOST=postgres-host
POSTGRES_PORT=5432
POSTGRES_USER=postgres
POSTGRES_PASSWORD=change-me-password
POSTGRES_DB_NAME=dev
POSTGRES_SSLMODE=disable

3
.gitignore vendored
View File

@@ -2,3 +2,6 @@
reforger_crawler_main
crawler.db
.env
.env.*
!.env.dokploy.example

29
Dockerfile Normal file
View File

@@ -0,0 +1,29 @@
FROM golang:1.25-bookworm AS builder
WORKDIR /app
# Required for CGO build (sqlite driver).
RUN apt-get update \
&& apt-get install -y --no-install-recommends build-essential ca-certificates \
&& rm -rf /var/lib/apt/lists/*
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o /out/reforger_crawler_main .
FROM debian:bookworm-slim AS runtime
WORKDIR /app
RUN apt-get update \
&& apt-get install -y --no-install-recommends ca-certificates tzdata \
&& rm -rf /var/lib/apt/lists/*
COPY --from=builder /out/reforger_crawler_main /app/reforger_crawler_main
COPY config.yaml /app/config.yaml
EXPOSE 8083
ENTRYPOINT ["/app/reforger_crawler_main"]

27
config.yaml
View File

@@ -1,7 +1,20 @@
port: 8083
ip: "localhost"
secret: "secret"
db: "crawler.db"
discordWebhook: ""
scraperWebhook: ""
adminSecret: "123456789"
server_port: "8083"
server_host: "localhost"
secret_key: "secret"
admin_secret: "123456789"
discord_webhook_url: ""
scraper_webhook_url: ""
db_driver: "sqlite"
sqlite_path: "crawler.db"
# Set db_driver to "postgres" and provide either postgres_dsn
# or individual postgres_* fields.
postgres_dsn: ""
postgres_host: ""
postgres_port: "5432"
postgres_user: "postgres"
postgres_password: ""
postgres_db_name: "dev"
postgres_sslmode: "disable"

4
controllers/authController.go
View File

@@ -9,7 +9,7 @@ import (
func CheckAllowed(c *gin.Context) {
secret := c.GetHeader("X-SECRET-KEY")
if secret != initializers.SECRET {
if secret != initializers.SecretKey {
c.JSON(http.StatusForbidden, gin.H{"error": "Forbidden"})
c.Abort()
return
@@ -18,7 +18,7 @@ func CheckAllowed(c *gin.Context) {
func CheckAdmin(c *gin.Context) {
adminSecret := c.GetHeader("X-ADMIN-KEY")
if adminSecret != initializers.ADMIN_SECRET {
if adminSecret != initializers.AdminSecret {
c.JSON(http.StatusForbidden, gin.H{"error": "Forbidden"})
c.Abort()
return

122
controllers/feController.go
View File

@@ -52,113 +52,6 @@ func convertToSimplifiedAddonWithoutFiles(addon models.Addon) models.SimplifiedA
func GetDuplicates(c *gin.Context) {
id := c.Param("id")
// First check if addon exists (lightweight query)
var exists bool
if err := initializers.DB.Model(&models.Addon{}).Select("1").Where("id = ?", id).Limit(1).Find(&exists).Error; err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "Addon not found"})
return
}
// Get only the hashes from source addon files (don't load full addon yet)
var sourceHashes []string
if err := initializers.DB.Model(&models.AddonFile{}).
Where("addon_id = ? AND hash != ''", id).
Pluck("hash", &sourceHashes).Error; err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to fetch source addon files"})
return
}
// Now load the full source addon for the response
var sourceAddon models.Addon
if err := initializers.DB.Preload("AddonFiles").First(&sourceAddon, "id = ?", id).Error; err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "Addon not found"})
return
}
simplifiedSourceAddon := convertToSimplifiedAddon(sourceAddon)
if len(sourceHashes) == 0 {
c.JSON(http.StatusOK, models.DuplicatesResponse{
SourceAddon: simplifiedSourceAddon,
DuplicateAddons: make(map[string]models.AddonWithDuplicates),
})
return
}
// Single query with JOIN to get all data
type DuplicateRow struct {
Hash string
Path string
Version string
AddonID string
AddonName string
AddonType string
Summary string
Preview string
Author string
SubscriberCount int
CurrentVersionNumber string
}
var duplicates []DuplicateRow
err := initializers.DB.Table("addon_files af").
Select(`af.hash, af.path, af.version, af.addon_id,
a.name as addon_name, a.type as addon_type, a.summary,
a.preview, a.author, a.subscriber_count, a.current_version_number`).
Joins("INNER JOIN addons a ON a.id = af.addon_id").
Joins("LEFT JOIN whitelisted_hashes wh ON wh.hash = af.hash").
Where(`af.hash IN ?
AND af.addon_id != ?
AND af.deleted_at IS NULL
AND wh.hash IS NULL`,
sourceHashes, id).
Scan(&duplicates).Error
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to fetch duplicates"})
return
}
// Build response from single result set
response := models.DuplicatesResponse{
SourceAddon: simplifiedSourceAddon,
DuplicateAddons: make(map[string]models.AddonWithDuplicates),
}
for _, dup := range duplicates {
if _, exists := response.DuplicateAddons[dup.AddonID]; !exists {
response.DuplicateAddons[dup.AddonID] = models.AddonWithDuplicates{
Addon: models.SimplifiedAddon{
ID: dup.AddonID,
Name: dup.AddonName,
Type: dup.AddonType,
Summary: dup.Summary,
Preview: dup.Preview,
Author: dup.Author,
SubscriberCount: dup.SubscriberCount,
CurrentVersionNumber: dup.CurrentVersionNumber,
AddonFiles: []models.SimplifiedAddonFile{},
},
Duplicates: []models.DuplicateFileInfo{},
}
}
entry := response.DuplicateAddons[dup.AddonID]
entry.Duplicates = append(entry.Duplicates, models.DuplicateFileInfo{
Path: dup.Path,
Hash: dup.Hash,
Version: dup.Version,
AddonID: dup.AddonID,
})
response.DuplicateAddons[dup.AddonID] = entry
}
c.JSON(http.StatusOK, response)
}
/*func GetDuplicates(c *gin.Context) {
id := c.Param("id")
fmt.Println("Fetching addon with ID:", id)
// Fetch the source addon with its files
@@ -236,6 +129,19 @@ func GetDuplicates(c *gin.Context) {
}
}
// Set Recheck flag for source addon and all duplicate addons
if err := initializers.DB.Model(&models.Addon{}).Where("id = ?", id).Update("recheck", true).Error; err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update source addon"})
return
}
for _, addon := range duplicateAddons {
if err := initializers.DB.Model(&models.Addon{}).Where("id = ?", addon.ID).Update("recheck", true).Error; err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update duplicate addon"})
return
}
}
// Build the response
response := models.DuplicatesResponse{
SourceAddon: simplifiedSourceAddon,
@@ -269,7 +175,7 @@ func GetDuplicates(c *gin.Context) {
}
c.JSON(http.StatusOK, response)
}*/
}
func GetPossibleAddons(c *gin.Context) {
query := c.Query("q")

54
controllers/indexerController.go
View File

@@ -105,7 +105,7 @@ func SaveIndexingResult(c *gin.Context) {
return
}
if initializers.DiscordWebhook != "" {
if initializers.DiscordWebhookURL != "" {
text := "Indexing of addon " + addon.Name + " (" + addon.ID + ") was successful."
colour := 2228479
nbr := strconv.Itoa(len(result.Files))
@@ -118,16 +118,16 @@ func SaveIndexingResult(c *gin.Context) {
percentage := 0.0
if addonsCount > 0 {
percentage = (float64(addonsCount - addonsToBeIndexed) / float64(addonsCount)) * 100
percentage = (float64(addonsCount-addonsToBeIndexed) / float64(addonsCount)) * 100
}
txt := fmt.Sprintf("%d/%d (%.2f%%).", addonsCount - addonsToBeIndexed, addonsCount, percentage)
txt := fmt.Sprintf("%d/%d (%.2f%%).", addonsCount-addonsToBeIndexed, addonsCount, percentage)
myEmbed := models.CustomEmbed{
Title: text,
Title: text,
Description: fmt.Sprintf("Files Indexed: %s\nOverall Progress: %s", nbr, txt),
Color: colour,
Timestamp: time.Now(),
Color: colour,
Timestamp: time.Now(),
Image: models.CustomImage{
URL: addon.Preview,
},
@@ -152,10 +152,10 @@ func SaveIndexingResult(c *gin.Context) {
myHook := models.CustomHook{
Username: "Reforger Crawler",
Embeds: []models.CustomEmbed{myEmbed},
Embeds: []models.CustomEmbed{myEmbed},
}
err := SendCustomWebhook(initializers.DiscordWebhook, myHook)
err := SendCustomWebhook(initializers.DiscordWebhookURL, myHook)
if err != nil {
fmt.Println("Error sending webhook:", err)
}
@@ -208,27 +208,27 @@ func checkIndexingTimeout() {
}
func SendCustomWebhook(webhookURL string, hook models.CustomHook) error {
payload, err := json.Marshal(hook)
if err != nil {
return err
}
payload, err := json.Marshal(hook)
if err != nil {
return err
}
req, err := http.NewRequest("POST", webhookURL, bytes.NewBuffer(payload))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
req, err := http.NewRequest("POST", webhookURL, bytes.NewBuffer(payload))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode > 299 {
return fmt.Errorf("webhook failed with status code %d", resp.StatusCode)
}
if resp.StatusCode < 200 || resp.StatusCode > 299 {
return fmt.Errorf("webhook failed with status code %d", resp.StatusCode)
}
return nil
return nil
}

80
controllers/scraperController.go
View File

@@ -1,7 +1,9 @@
package controllers
import (
"bytes"
"fmt"
"io"
"strconv"
"time"
@@ -15,10 +17,14 @@ var nbrOfUnsentWebhooks int
func CreateAddon(c *gin.Context) {
var addonR models.Addon
body := make([]byte, 0)
c.Request.Body.Read(body)
fmt.Println(body)
err := c.ShouldBindJSON(&addonR)
body, err := io.ReadAll(c.Request.Body)
if err != nil {
c.JSON(400, gin.H{"error": "Failed to read request body"})
return
}
fmt.Println(string(body))
c.Request.Body = io.NopCloser(bytes.NewBuffer(body))
err = c.ShouldBindJSON(&addonR)
if err != nil {
c.JSON(400, gin.H{"error": "Invalid JSON"})
return
@@ -44,7 +50,7 @@ func CreateAddon(c *gin.Context) {
nbrOfUnsentWebhooks = 0
go func() {
webhookURL := initializers.ScraperWebhook
webhookURL := initializers.ScraperWebhookURL
if webhookURL != "" {
text := fmt.Sprintf("New %d addons found\nLast: %s (%s).", oldNbr, addon.ID, addon.Name)
colour := 2228479
@@ -102,44 +108,44 @@ func CreateAddon(c *gin.Context) {
// send webhook about update
go func() {
webhookURL := initializers.ScraperWebhook
if webhookURL != "" {
text := fmt.Sprintf("Addon: %s (%s) updated", addon.ID, addon.Name)
colour := 2228479
size := fmt.Sprintf("%.2f MB", float64(addon.CurrentVersionSize)/1_000_000)
webhookURL := initializers.ScraperWebhookURL
if webhookURL != "" {
text := fmt.Sprintf("Addon: %s (%s) updated", addon.ID, addon.Name)
colour := 2228479
size := fmt.Sprintf("%.2f MB", float64(addon.CurrentVersionSize)/1_000_000)
myEmbed := models.CustomEmbed{
Title: text,
Color: colour,
Timestamp: time.Now(),
Image: models.CustomImage{
URL: addon.Preview,
},
Fields: []models.CustomEmbedField{
{
Name: "Size",
Value: size,
Inline: true,
myEmbed := models.CustomEmbed{
Title: text,
Color: colour,
Timestamp: time.Now(),
Image: models.CustomImage{
URL: addon.Preview,
},
{
Name: "Current Version",
Value: addon.CurrentVersionNumber,
Inline: true,
Fields: []models.CustomEmbedField{
{
Name: "Size",
Value: size,
Inline: true,
},
{
Name: "Current Version",
Value: addon.CurrentVersionNumber,
Inline: true,
},
},
},
}
}
myHook := models.CustomHook{
Username: "Reforger Crawler",
Embeds: []models.CustomEmbed{myEmbed},
}
myHook := models.CustomHook{
Username: "Reforger Crawler",
Embeds: []models.CustomEmbed{myEmbed},
}
err := SendCustomWebhook(webhookURL, myHook)
if err != nil {
fmt.Println("Error sending webhook:", err)
err := SendCustomWebhook(webhookURL, myHook)
if err != nil {
fmt.Println("Error sending webhook:", err)
}
}
}
}()
}()
}
}
c.JSON(200, gin.H{"status": "success"})

10
go.mod
View File

@@ -20,6 +20,10 @@ require (
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.20.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/pgx/v5 v5.6.0 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
@@ -30,10 +34,12 @@ require (
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.12 // indirect
golang.org/x/arch v0.8.0 // indirect
golang.org/x/crypto v0.23.0 // indirect
golang.org/x/crypto v0.31.0 // indirect
golang.org/x/net v0.25.0 // indirect
golang.org/x/sys v0.20.0 // indirect
golang.org/x/sync v0.16.0 // indirect
golang.org/x/sys v0.28.0 // indirect
google.golang.org/protobuf v1.34.1 // indirect
gorm.io/driver/postgres v1.6.0 // indirect
)
require (

16
go.sum
View File

@@ -30,6 +30,14 @@ github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MG
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY=
github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
@@ -76,12 +84,18 @@ golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc=
golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
@@ -93,6 +107,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4=
gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo=
gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ=
gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8=
gorm.io/gorm v1.30.3 h1:QiG8upl0Sg9ba2Zatfjy0fy4It2iNBL2/eMdvEkdXNs=

265
initializers/initialize.go
View File

@@ -1,91 +1,240 @@
package initializers
import (
"errors"
"fmt"
"log"
"os"
"strings"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
"gopkg.in/yaml.v3"
"gorm.io/driver/postgres"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
var DB *gorm.DB
var PORT string
var IP string
var SECRET string
var DB_NAME string
var DiscordWebhook string
var ADMIN_SECRET string
var ScraperWebhook string
var ServerPort string
var ServerHost string
var SecretKey string
var DiscordWebhookURL string
var AdminSecret string
var ScraperWebhookURL string
var DatabaseDriver string
var SQLitePath string
var PostgresDSN string
func ConnectToDB() {
db, err := gorm.Open(sqlite.Open(DB_NAME), &gorm.Config{
PrepareStmt: true, // Cache prepared statements
SkipDefaultTransaction: true, // Disable default transactions for better performance
})
if err != nil {
log.Fatal("Failed to connect to database")
var (
db *gorm.DB
err error
)
switch strings.ToLower(DatabaseDriver) {
case "", "sqlite", "sqlite3":
sqlitePath := SQLitePath
if sqlitePath == "" {
sqlitePath = "crawler.db"
}
dsn := fmt.Sprintf("%s?_busy_timeout=5000&_journal_mode=WAL", sqlitePath)
db, err = gorm.Open(sqlite.Open(dsn), &gorm.Config{
SkipDefaultTransaction: true,
})
if err != nil {
log.Fatalf("Failed to connect to sqlite database: %v", err)
}
if err := db.Exec("PRAGMA synchronous=NORMAL").Error; err != nil {
log.Fatalf("Failed to set PRAGMA synchronous: %v", err)
}
if err := db.Exec("PRAGMA cache_size=-64000").Error; err != nil {
log.Fatalf("Failed to set PRAGMA cache_size: %v", err)
}
if err := db.Exec("PRAGMA temp_store=MEMORY").Error; err != nil {
log.Fatalf("Failed to set PRAGMA temp_store: %v", err)
}
if err := db.Exec("PRAGMA mmap_size=268435456").Error; err != nil {
log.Fatalf("Failed to set PRAGMA mmap_size: %v", err)
}
case "postgres", "postgresql":
if PostgresDSN == "" {
log.Fatal("PostgreSQL selected but no DSN is configured")
}
db, err = gorm.Open(postgres.Open(PostgresDSN), &gorm.Config{
SkipDefaultTransaction: true,
})
if err != nil {
log.Fatalf("Failed to connect to postgres database: %v", err)
}
default:
log.Fatalf("Unsupported database driver: %s", DatabaseDriver)
}
DB = db
// Optimize SQLite for performance
DB.Exec("PRAGMA journal_mode=WAL") // Write-Ahead Logging for better concurrency
DB.Exec("PRAGMA synchronous=NORMAL") // Balance between safety and speed
DB.Exec("PRAGMA cache_size=-64000") // 64MB cache (negative = KB)
DB.Exec("PRAGMA temp_store=MEMORY") // Use memory for temp tables
DB.Exec("PRAGMA mmap_size=268435456") // 256MB memory-mapped I/O
if err := DB.AutoMigrate(&models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{}); err != nil {
log.Fatalf("Failed to run migrations: %v", err)
}
DB.AutoMigrate(&models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{})
if err := DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_hash_addon_optimized
ON addon_files(hash, addon_id) WHERE deleted_at IS NULL`).Error; err != nil {
log.Fatalf("Failed to create idx_addon_files_hash_addon_optimized: %v", err)
}
// Create optimized indexes for duplicate detection queries
// Composite index for efficient hash lookups excluding specific addon_id
DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_hash_addon_optimized
ON addon_files(hash, addon_id) WHERE deleted_at IS NULL`)
if err := DB.Exec(`CREATE INDEX IF NOT EXISTS idx_whitelisted_hash
ON whitelisted_hashes(hash)`).Error; err != nil {
log.Fatalf("Failed to create idx_whitelisted_hash: %v", err)
}
// Index for whitelisted hashes lookup
DB.Exec(`CREATE INDEX IF NOT EXISTS idx_whitelisted_hash
ON whitelisted_hashes(hash)`)
if err := DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addons_id
ON addons(id)`).Error; err != nil {
log.Fatalf("Failed to create idx_addons_id: %v", err)
}
// Additional index on addon_id for the JOIN operation
DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addons_id
ON addons(id)`)
// Covering index for addon_files to avoid table lookups
DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_covering
ON addon_files(addon_id, hash, path, version) WHERE deleted_at IS NULL`)
if err := DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_covering
ON addon_files(addon_id, hash, path, version) WHERE deleted_at IS NULL`).Error; err != nil {
log.Fatalf("Failed to create idx_addon_files_covering: %v", err)
}
}
type Configuration struct {
Port string `yaml:"port"`
IP string `yaml:"ip"`
Secret string `yaml:"secret"`
DB string `yaml:"db"`
DiscordWebhook string `yaml:"discordWebhook"`
ADMIN_SECRET string `yaml:"adminSecret"`
ScraperWebhook string `yaml:"scraperWebhook"`
ServerPort string `yaml:"server_port"`
ServerHost string `yaml:"server_host"`
SecretKey string `yaml:"secret_key"`
AdminSecret string `yaml:"admin_secret"`
DiscordWebhookURL string `yaml:"discord_webhook_url"`
ScraperWebhookURL string `yaml:"scraper_webhook_url"`
DatabaseDriver string `yaml:"db_driver"`
SQLitePath string `yaml:"sqlite_path"`
PostgresDSN string `yaml:"postgres_dsn"`
PostgresHost string `yaml:"postgres_host"`
PostgresPort string `yaml:"postgres_port"`
PostgresUser string `yaml:"postgres_user"`
PostgresPassword string `yaml:"postgres_password"`
PostgresDBName string `yaml:"postgres_db_name"`
PostgresSSLMode string `yaml:"postgres_sslmode"`
LegacyPort string `yaml:"port"`
LegacyIP string `yaml:"ip"`
LegacySecret string `yaml:"secret"`
LegacyDB string `yaml:"db"`
LegacyDiscordHook string `yaml:"discordWebhook"`
LegacyAdminSecret string `yaml:"adminSecret"`
LegacyScraperHook string `yaml:"scraperWebhook"`
}
func Load() {
file, err := os.ReadFile("config.yaml")
if err != nil {
log.Fatal("Failed to open config file")
}
configuration := Configuration{
DB: "register.db",
}
err = yaml.Unmarshal(file, &configuration)
if err != nil {
log.Fatal("Failed to read yaml file")
ServerPort: "8083",
ServerHost: "0.0.0.0",
DatabaseDriver: "sqlite",
SQLitePath: "crawler.db",
PostgresSSLMode: "disable",
}
PORT = configuration.Port
IP = configuration.IP
SECRET = configuration.Secret
DB_NAME = configuration.DB
DiscordWebhook = configuration.DiscordWebhook
ADMIN_SECRET = configuration.ADMIN_SECRET
ScraperWebhook = configuration.ScraperWebhook
file, err := os.ReadFile("config.yaml")
if err == nil {
if err := yaml.Unmarshal(file, &configuration); err != nil {
log.Fatalf("Failed to read yaml file: %v", err)
}
} else if !errors.Is(err, os.ErrNotExist) {
log.Fatalf("Failed to open config file: %v", err)
}
applyLegacyConfigFallbacks(&configuration)
applyEnvOverrides(&configuration)
ServerPort = configuration.ServerPort
ServerHost = configuration.ServerHost
SecretKey = configuration.SecretKey
AdminSecret = configuration.AdminSecret
DiscordWebhookURL = configuration.DiscordWebhookURL
ScraperWebhookURL = configuration.ScraperWebhookURL
DatabaseDriver = strings.ToLower(configuration.DatabaseDriver)
SQLitePath = configuration.SQLitePath
if strings.TrimSpace(configuration.PostgresDSN) != "" {
PostgresDSN = configuration.PostgresDSN
} else {
PostgresDSN = buildPostgresDSN(configuration)
}
}
func applyLegacyConfigFallbacks(c *Configuration) {
if c.ServerPort == "" {
c.ServerPort = c.LegacyPort
}
if c.ServerHost == "" {
c.ServerHost = c.LegacyIP
}
if c.SecretKey == "" {
c.SecretKey = c.LegacySecret
}
if c.AdminSecret == "" {
c.AdminSecret = c.LegacyAdminSecret
}
if c.DiscordWebhookURL == "" {
c.DiscordWebhookURL = c.LegacyDiscordHook
}
if c.ScraperWebhookURL == "" {
c.ScraperWebhookURL = c.LegacyScraperHook
}
if c.SQLitePath == "" {
c.SQLitePath = c.LegacyDB
}
}
func applyEnvOverrides(c *Configuration) {
c.ServerPort = firstEnv([]string{"SERVER_PORT", "PORT"}, c.ServerPort)
c.ServerHost = firstEnv([]string{"SERVER_HOST", "IP"}, c.ServerHost)
c.SecretKey = firstEnv([]string{"SECRET_KEY", "SECRET"}, c.SecretKey)
c.AdminSecret = firstEnv([]string{"ADMIN_SECRET"}, c.AdminSecret)
c.DiscordWebhookURL = firstEnv([]string{"DISCORD_WEBHOOK_URL", "DISCORD_WEBHOOK"}, c.DiscordWebhookURL)
c.ScraperWebhookURL = firstEnv([]string{"SCRAPER_WEBHOOK_URL", "SCRAPER_WEBHOOK"}, c.ScraperWebhookURL)
c.DatabaseDriver = firstEnv([]string{"DB_DRIVER"}, c.DatabaseDriver)
c.SQLitePath = firstEnv([]string{"SQLITE_PATH", "DB"}, c.SQLitePath)
c.PostgresDSN = firstEnv([]string{"POSTGRES_DSN"}, c.PostgresDSN)
c.PostgresHost = firstEnv([]string{"POSTGRES_HOST"}, c.PostgresHost)
c.PostgresPort = firstEnv([]string{"POSTGRES_PORT"}, c.PostgresPort)
c.PostgresUser = firstEnv([]string{"POSTGRES_USER"}, c.PostgresUser)
c.PostgresPassword = firstEnv([]string{"POSTGRES_PASSWORD"}, c.PostgresPassword)
c.PostgresDBName = firstEnv([]string{"POSTGRES_DB_NAME"}, c.PostgresDBName)
c.PostgresSSLMode = firstEnv([]string{"POSTGRES_SSLMODE"}, c.PostgresSSLMode)
}
func firstEnv(keys []string, fallback string) string {
for _, k := range keys {
if value, ok := os.LookupEnv(k); ok && strings.TrimSpace(value) != "" {
return value
}
}
return fallback
}
func buildPostgresDSN(c Configuration) string {
if c.PostgresHost == "" || c.PostgresUser == "" || c.PostgresDBName == "" {
return ""
}
port := c.PostgresPort
if port == "" {
port = "5432"
}
sslMode := c.PostgresSSLMode
if sslMode == "" {
sslMode = "disable"
}
return fmt.Sprintf(
"host=%s user=%s password=%s dbname=%s port=%s sslmode=%s",
c.PostgresHost,
c.PostgresUser,
c.PostgresPassword,
c.PostgresDBName,
port,
sslMode,
)
}

33
main.go
View File

@@ -17,6 +17,9 @@ func init() {
func main() {
r := gin.Default()
r.Use(CORSMiddleware())
r.GET("/health", func(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{"status": "ok"})
})
back := r.Group("/back")
{
@@ -29,7 +32,7 @@ func main() {
admin := r.Group("/admin")
{
admin.POST("/addWhitelistedHash",controllers.CheckAdmin, controllers.AddWhitelistedHash)
admin.POST("/addWhitelistedHash", controllers.CheckAdmin, controllers.AddWhitelistedHash)
admin.GET("/duplicates", controllers.CheckAdmin)
admin.POST("/setPriority", controllers.CheckAdmin)
admin.POST("/setToBeIndexed", controllers.CheckAdmin)
@@ -41,32 +44,32 @@ func main() {
v1.GET("/getPossible", controllers.GetPossibleAddons)
}
r.Run(initializers.IP + ":" + initializers.PORT)
r.Run(initializers.ServerHost + ":" + initializers.ServerPort)
}
func CORSMiddleware() gin.HandlerFunc {
return func(c *gin.Context) {
return func(c *gin.Context) {
orig := c.Request.Header.Clone().Get("Origin")
c.Writer.Header().Set("Access-Control-Allow-Origin", orig)
c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With")
c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT, DELETE")
c.Writer.Header().Set("Access-Control-Allow-Origin", orig)
c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With")
c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT, DELETE")
if c.Request.Method == "OPTIONS" {
c.AbortWithStatus(http.StatusNoContent)
return
}
if c.Request.Method == "OPTIONS" {
c.AbortWithStatus(http.StatusNoContent)
return
}
c.Next()
}
c.Next()
}
}
func OptionsMiddleware() gin.HandlerFunc {
return func(c *gin.Context) {
return func(c *gin.Context) {
if c.Request.Method != "OPTIONS" {
c.Next()
} else {
c.AbortWithStatus(http.StatusOK)
}
}
}
}

42
models/addon.go
View File

@@ -2,28 +2,32 @@ package models
import (
"time"
"gorm.io/gorm"
)
type Addon struct {
ID string `gorm:"primaryKey" json:"id"`
Name string `json:"name"`
Type string `json:"type"`
Summary string `json:"summary"`
Unlisted bool `json:"unlisted"`
Private bool `json:"private"`
Blocked bool `json:"blocked"`
SubscriberCount int `json:"subscriberCount"`
CurrentVersionNumber string `json:"currentVersionNumber"`
CurrentVersionSize int `json:"currentVersionSize"`
CurrentVersionID int `json:"currentVersionId"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
ToBeIndexed bool `json:"toBeIndexed" gorm:"default:true"`
IsBeingIndexed bool `json:"isBeingIndexed" gorm:"default:false"`
IndexStartTime time.Time `json:"indexStartTime"`
PriorityIndexing bool `json:"priorityIndexing" gorm:"default:false"`
AddonFiles []AddonFile `json:"addonFiles"`
ID string `gorm:"primaryKey" json:"id"`
Name string `json:"name"`
Type string `json:"type"`
Summary string `json:"summary"`
Unlisted bool `json:"unlisted"`
Private bool `json:"private"`
Blocked bool `json:"blocked"`
SubscriberCount int `json:"subscriberCount"`
CurrentVersionNumber string `json:"currentVersionNumber"`
CurrentVersionSize int `json:"currentVersionSize"`
CurrentVersionID int `json:"currentVersionId"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
DeletedAt gorm.DeletedAt `gorm:"index"`
ToBeIndexed bool `json:"toBeIndexed" gorm:"default:true"`
IsBeingIndexed bool `json:"isBeingIndexed" gorm:"default:false"`
IndexStartTime time.Time `json:"indexStartTime"`
PriorityIndexing bool `json:"priorityIndexing" gorm:"default:false"`
AddonFiles []AddonFile `json:"addonFiles"`
Recheck bool `json:"recheck" gorm:"default:false"`
Preview string `json:"preview"`
Author string `json:"author"`
Author string `json:"author"`
}