updates, dokploy

2026-04-09 01:39:37 +02:00
parent fcebe00175
commit e39a92c6d3
14 changed files with 450 additions and 276 deletions
--- a/.dockerignore
+++ b/.dockerignore
@@ -0,0 +1,11 @@
+.git
+.gitea
+.vscode
+api
+*.db
+*.db-journal
+*.db-shm
+*.db-wal
+.env
+.env.*
+README.md
--- a/.env.dokploy.example
+++ b/.env.dokploy.example
@@ -0,0 +1,28 @@
+# Dokploy environment example for Reforger Crawler
+# Copy these values into Dokploy Environment Variables.
+
+SERVER_HOST=0.0.0.0
+SERVER_PORT=8083
+
+SECRET_KEY=change-me-secret
+ADMIN_SECRET=change-me-admin-secret
+
+DISCORD_WEBHOOK_URL=
+SCRAPER_WEBHOOK_URL=
+
+# sqlite or postgres
+DB_DRIVER=postgres
+
+# Used only when DB_DRIVER=sqlite
+SQLITE_PATH=crawler.db
+
+# Option A: full DSN
+POSTGRES_DSN=postgresql://postgres:change-me-password@postgres-host:5432/dev
+
+# Option B: split fields (used when POSTGRES_DSN is empty)
+POSTGRES_HOST=postgres-host
+POSTGRES_PORT=5432
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=change-me-password
+POSTGRES_DB_NAME=dev
+POSTGRES_SSLMODE=disable
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,6 @@
 reforger_crawler_main

 crawler.db
+.env
+.env.*
+!.env.dokploy.example
--- a/29
+++ b/29
@@ -0,0 +1,29 @@
+FROM golang:1.25-bookworm AS builder
+
+WORKDIR /app
+
+# Required for CGO build (sqlite driver).
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends build-essential ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . .
+RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o /out/reforger_crawler_main .
+
+FROM debian:bookworm-slim AS runtime
+
+WORKDIR /app
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends ca-certificates tzdata \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /out/reforger_crawler_main /app/reforger_crawler_main
+COPY config.yaml /app/config.yaml
+
+EXPOSE 8083
+
+ENTRYPOINT ["/app/reforger_crawler_main"]
--- a/config.yaml
+++ b/config.yaml
@@ -1,7 +1,20 @@
-port: 8083
-ip: "localhost"
-secret: "secret"
-db: "crawler.db"
-discordWebhook: ""
-scraperWebhook: ""
-adminSecret: "123456789"
+server_port: "8083"
+server_host: "localhost"
+secret_key: "secret"
+admin_secret: "123456789"
+
+discord_webhook_url: ""
+scraper_webhook_url: ""
+
+db_driver: "sqlite"
+sqlite_path: "crawler.db"
+
+# Set db_driver to "postgres" and provide either postgres_dsn
+# or individual postgres_* fields.
+postgres_dsn: ""
+postgres_host: ""
+postgres_port: "5432"
+postgres_user: "postgres"
+postgres_password: ""
+postgres_db_name: "dev"
+postgres_sslmode: "disable"
--- a/controllers/authController.go
+++ b/controllers/authController.go
@@ -9,7 +9,7 @@ import (

 func CheckAllowed(c *gin.Context) {
 	secret := c.GetHeader("X-SECRET-KEY")
-	if secret != initializers.SECRET {
+	if secret != initializers.SecretKey {
 		c.JSON(http.StatusForbidden, gin.H{"error": "Forbidden"})
 		c.Abort()
 		return
@@ -18,7 +18,7 @@ func CheckAllowed(c *gin.Context) {

 func CheckAdmin(c *gin.Context) {
 	adminSecret := c.GetHeader("X-ADMIN-KEY")
-	if adminSecret != initializers.ADMIN_SECRET {
+	if adminSecret != initializers.AdminSecret {
 		c.JSON(http.StatusForbidden, gin.H{"error": "Forbidden"})
 		c.Abort()
 		return
--- a/controllers/feController.go
+++ b/controllers/feController.go
@@ -52,113 +52,6 @@ func convertToSimplifiedAddonWithoutFiles(addon models.Addon) models.SimplifiedA

 func GetDuplicates(c *gin.Context) {
 	id := c.Param("id")
-
-	// First check if addon exists (lightweight query)
-	var exists bool
-	if err := initializers.DB.Model(&models.Addon{}).Select("1").Where("id = ?", id).Limit(1).Find(&exists).Error; err != nil {
-		c.JSON(http.StatusNotFound, gin.H{"error": "Addon not found"})
-		return
-	}
-
-	// Get only the hashes from source addon files (don't load full addon yet)
-	var sourceHashes []string
-	if err := initializers.DB.Model(&models.AddonFile{}).
-		Where("addon_id = ? AND hash != ''", id).
-		Pluck("hash", &sourceHashes).Error; err != nil {
-		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to fetch source addon files"})
-		return
-	}
-
-	// Now load the full source addon for the response
-	var sourceAddon models.Addon
-	if err := initializers.DB.Preload("AddonFiles").First(&sourceAddon, "id = ?", id).Error; err != nil {
-		c.JSON(http.StatusNotFound, gin.H{"error": "Addon not found"})
-		return
-	}
-
-	simplifiedSourceAddon := convertToSimplifiedAddon(sourceAddon)
-
-	if len(sourceHashes) == 0 {
-		c.JSON(http.StatusOK, models.DuplicatesResponse{
-			SourceAddon:     simplifiedSourceAddon,
-			DuplicateAddons: make(map[string]models.AddonWithDuplicates),
-		})
-		return
-	}
-
-	// Single query with JOIN to get all data
-	type DuplicateRow struct {
-		Hash                 string
-		Path                 string
-		Version              string
-		AddonID              string
-		AddonName            string
-		AddonType            string
-		Summary              string
-		Preview              string
-		Author               string
-		SubscriberCount      int
-		CurrentVersionNumber string
-	}
-
-	var duplicates []DuplicateRow
-	err := initializers.DB.Table("addon_files af").
-		Select(`af.hash, af.path, af.version, af.addon_id,
-                a.name as addon_name, a.type as addon_type, a.summary,
-                a.preview, a.author, a.subscriber_count, a.current_version_number`).
-		Joins("INNER JOIN addons a ON a.id = af.addon_id").
-		Joins("LEFT JOIN whitelisted_hashes wh ON wh.hash = af.hash").
-		Where(`af.hash IN ? 
-               AND af.addon_id != ? 
-               AND af.deleted_at IS NULL
-               AND wh.hash IS NULL`,
-			sourceHashes, id).
-		Scan(&duplicates).Error
-
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to fetch duplicates"})
-		return
-	}
-
-	// Build response from single result set
-	response := models.DuplicatesResponse{
-		SourceAddon:     simplifiedSourceAddon,
-		DuplicateAddons: make(map[string]models.AddonWithDuplicates),
-	}
-
-	for _, dup := range duplicates {
-		if _, exists := response.DuplicateAddons[dup.AddonID]; !exists {
-			response.DuplicateAddons[dup.AddonID] = models.AddonWithDuplicates{
-				Addon: models.SimplifiedAddon{
-					ID:                   dup.AddonID,
-					Name:                 dup.AddonName,
-					Type:                 dup.AddonType,
-					Summary:              dup.Summary,
-					Preview:              dup.Preview,
-					Author:               dup.Author,
-					SubscriberCount:      dup.SubscriberCount,
-					CurrentVersionNumber: dup.CurrentVersionNumber,
-					AddonFiles:           []models.SimplifiedAddonFile{},
-				},
-				Duplicates: []models.DuplicateFileInfo{},
-			}
-		}
-
-		entry := response.DuplicateAddons[dup.AddonID]
-		entry.Duplicates = append(entry.Duplicates, models.DuplicateFileInfo{
-			Path:    dup.Path,
-			Hash:    dup.Hash,
-			Version: dup.Version,
-			AddonID: dup.AddonID,
-		})
-		response.DuplicateAddons[dup.AddonID] = entry
-	}
-
-	c.JSON(http.StatusOK, response)
-}
-
-/*func GetDuplicates(c *gin.Context) {
-	id := c.Param("id")
 	fmt.Println("Fetching addon with ID:", id)

 	// Fetch the source addon with its files
@@ -236,6 +129,19 @@ func GetDuplicates(c *gin.Context) {
 		}
 	}

+	// Set Recheck flag for source addon and all duplicate addons
+	if err := initializers.DB.Model(&models.Addon{}).Where("id = ?", id).Update("recheck", true).Error; err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update source addon"})
+		return
+	}
+
+	for _, addon := range duplicateAddons {
+		if err := initializers.DB.Model(&models.Addon{}).Where("id = ?", addon.ID).Update("recheck", true).Error; err != nil {
+			c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update duplicate addon"})
+			return
+		}
+	}
+
 	// Build the response
 	response := models.DuplicatesResponse{
 		SourceAddon:     simplifiedSourceAddon,
@@ -269,7 +175,7 @@ func GetDuplicates(c *gin.Context) {
 	}

 	c.JSON(http.StatusOK, response)
-}*/
+}

 func GetPossibleAddons(c *gin.Context) {
 	query := c.Query("q")
--- a/controllers/indexerController.go
+++ b/controllers/indexerController.go
@@ -105,7 +105,7 @@ func SaveIndexingResult(c *gin.Context) {
 		return
 	}

-	if initializers.DiscordWebhook != "" {
+	if initializers.DiscordWebhookURL != "" {
 		text := "Indexing of addon " + addon.Name + " (" + addon.ID + ") was successful."
 		colour := 2228479
 		nbr := strconv.Itoa(len(result.Files))
@@ -118,10 +118,10 @@ func SaveIndexingResult(c *gin.Context) {

 		percentage := 0.0
 		if addonsCount > 0 {
-			percentage = (float64(addonsCount - addonsToBeIndexed) / float64(addonsCount)) * 100
+			percentage = (float64(addonsCount-addonsToBeIndexed) / float64(addonsCount)) * 100
 		}

-		txt := fmt.Sprintf("%d/%d (%.2f%%).", addonsCount - addonsToBeIndexed, addonsCount, percentage)
+		txt := fmt.Sprintf("%d/%d (%.2f%%).", addonsCount-addonsToBeIndexed, addonsCount, percentage)

 		myEmbed := models.CustomEmbed{
 			Title:       text,
@@ -155,7 +155,7 @@ func SaveIndexingResult(c *gin.Context) {
 			Embeds:   []models.CustomEmbed{myEmbed},
 		}

-		err := SendCustomWebhook(initializers.DiscordWebhook, myHook)
+		err := SendCustomWebhook(initializers.DiscordWebhookURL, myHook)
 		if err != nil {
 			fmt.Println("Error sending webhook:", err)
 		}
--- a/controllers/scraperController.go
+++ b/controllers/scraperController.go
@@ -1,7 +1,9 @@
 package controllers

 import (
+	"bytes"
 	"fmt"
+	"io"
 	"strconv"
 	"time"

@@ -15,10 +17,14 @@ var nbrOfUnsentWebhooks int

 func CreateAddon(c *gin.Context) {
 	var addonR models.Addon
-	body := make([]byte, 0)
-	c.Request.Body.Read(body)
-	fmt.Println(body)
-	err := c.ShouldBindJSON(&addonR)
+	body, err := io.ReadAll(c.Request.Body)
+	if err != nil {
+		c.JSON(400, gin.H{"error": "Failed to read request body"})
+		return
+	}
+	fmt.Println(string(body))
+	c.Request.Body = io.NopCloser(bytes.NewBuffer(body))
+	err = c.ShouldBindJSON(&addonR)
 	if err != nil {
 		c.JSON(400, gin.H{"error": "Invalid JSON"})
 		return
@@ -44,7 +50,7 @@ func CreateAddon(c *gin.Context) {
 		nbrOfUnsentWebhooks = 0

 		go func() {
-			webhookURL := initializers.ScraperWebhook
+			webhookURL := initializers.ScraperWebhookURL
 			if webhookURL != "" {
 				text := fmt.Sprintf("New %d addons found\nLast: %s (%s).", oldNbr, addon.ID, addon.Name)
 				colour := 2228479
@@ -102,7 +108,7 @@ func CreateAddon(c *gin.Context) {

 			// send webhook about update
 			go func() {
-			webhookURL := initializers.ScraperWebhook
+				webhookURL := initializers.ScraperWebhookURL
 				if webhookURL != "" {
 					text := fmt.Sprintf("Addon: %s (%s) updated", addon.ID, addon.Name)
 					colour := 2228479
--- a/go.mod
+++ b/go.mod
@@ -20,6 +20,10 @@ require (
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-playground/validator/v10 v10.20.0 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
+	github.com/jackc/pgx/v5 v5.6.0 // indirect
+	github.com/jackc/puddle/v2 v2.2.2 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
@@ -30,10 +34,12 @@ require (
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
 	golang.org/x/arch v0.8.0 // indirect
-	golang.org/x/crypto v0.23.0 // indirect
+	golang.org/x/crypto v0.31.0 // indirect
 	golang.org/x/net v0.25.0 // indirect
-	golang.org/x/sys v0.20.0 // indirect
+	golang.org/x/sync v0.16.0 // indirect
+	golang.org/x/sys v0.28.0 // indirect
 	google.golang.org/protobuf v1.34.1 // indirect
+	gorm.io/driver/postgres v1.6.0 // indirect
 )

 require (
--- a/go.sum
+++ b/go.sum
@@ -30,6 +30,14 @@ github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MG
 github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY=
+github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw=
+github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
+github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
@@ -76,12 +84,18 @@ golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc=
 golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
 golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
 golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
+golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
 golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
@@ -93,6 +107,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4=
+gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo=
 gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ=
 gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8=
 gorm.io/gorm v1.30.3 h1:QiG8upl0Sg9ba2Zatfjy0fy4It2iNBL2/eMdvEkdXNs=
--- a/initializers/initialize.go
+++ b/initializers/initialize.go
@@ -1,91 +1,240 @@
 package initializers

 import (
+	"errors"
+	"fmt"
 	"log"
 	"os"
+	"strings"

 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
 	"gopkg.in/yaml.v3"
+	"gorm.io/driver/postgres"
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
 )

 var DB *gorm.DB
-var PORT string
-var IP string
-var SECRET string
-var DB_NAME string
-var DiscordWebhook string
-var ADMIN_SECRET string
-var ScraperWebhook string
+var ServerPort string
+var ServerHost string
+var SecretKey string
+var DiscordWebhookURL string
+var AdminSecret string
+var ScraperWebhookURL string
+var DatabaseDriver string
+var SQLitePath string
+var PostgresDSN string

 func ConnectToDB() {
-	db, err := gorm.Open(sqlite.Open(DB_NAME), &gorm.Config{
-		PrepareStmt:            true, // Cache prepared statements
-		SkipDefaultTransaction: true, // Disable default transactions for better performance
+	var (
+		db  *gorm.DB
+		err error
+	)
+
+	switch strings.ToLower(DatabaseDriver) {
+	case "", "sqlite", "sqlite3":
+		sqlitePath := SQLitePath
+		if sqlitePath == "" {
+			sqlitePath = "crawler.db"
+		}
+
+		dsn := fmt.Sprintf("%s?_busy_timeout=5000&_journal_mode=WAL", sqlitePath)
+		db, err = gorm.Open(sqlite.Open(dsn), &gorm.Config{
+			SkipDefaultTransaction: true,
 		})
 		if err != nil {
-		log.Fatal("Failed to connect to database")
+			log.Fatalf("Failed to connect to sqlite database: %v", err)
+		}
+
+		if err := db.Exec("PRAGMA synchronous=NORMAL").Error; err != nil {
+			log.Fatalf("Failed to set PRAGMA synchronous: %v", err)
+		}
+		if err := db.Exec("PRAGMA cache_size=-64000").Error; err != nil {
+			log.Fatalf("Failed to set PRAGMA cache_size: %v", err)
+		}
+		if err := db.Exec("PRAGMA temp_store=MEMORY").Error; err != nil {
+			log.Fatalf("Failed to set PRAGMA temp_store: %v", err)
+		}
+		if err := db.Exec("PRAGMA mmap_size=268435456").Error; err != nil {
+			log.Fatalf("Failed to set PRAGMA mmap_size: %v", err)
+		}
+	case "postgres", "postgresql":
+		if PostgresDSN == "" {
+			log.Fatal("PostgreSQL selected but no DSN is configured")
+		}
+
+		db, err = gorm.Open(postgres.Open(PostgresDSN), &gorm.Config{
+			SkipDefaultTransaction: true,
+		})
+		if err != nil {
+			log.Fatalf("Failed to connect to postgres database: %v", err)
+		}
+	default:
+		log.Fatalf("Unsupported database driver: %s", DatabaseDriver)
 	}

 	DB = db

-	// Optimize SQLite for performance
-	DB.Exec("PRAGMA journal_mode=WAL")    // Write-Ahead Logging for better concurrency
-	DB.Exec("PRAGMA synchronous=NORMAL")  // Balance between safety and speed
-	DB.Exec("PRAGMA cache_size=-64000")   // 64MB cache (negative = KB)
-	DB.Exec("PRAGMA temp_store=MEMORY")   // Use memory for temp tables
-	DB.Exec("PRAGMA mmap_size=268435456") // 256MB memory-mapped I/O
+	if err := DB.AutoMigrate(&models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{}); err != nil {
+		log.Fatalf("Failed to run migrations: %v", err)
+	}

-	DB.AutoMigrate(&models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{})
+	if err := DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_hash_addon_optimized 
+		ON addon_files(hash, addon_id) WHERE deleted_at IS NULL`).Error; err != nil {
+		log.Fatalf("Failed to create idx_addon_files_hash_addon_optimized: %v", err)
+	}

-	// Create optimized indexes for duplicate detection queries
-	// Composite index for efficient hash lookups excluding specific addon_id
-	DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_hash_addon_optimized 
-		ON addon_files(hash, addon_id) WHERE deleted_at IS NULL`)
+	if err := DB.Exec(`CREATE INDEX IF NOT EXISTS idx_whitelisted_hash 
+		ON whitelisted_hashes(hash)`).Error; err != nil {
+		log.Fatalf("Failed to create idx_whitelisted_hash: %v", err)
+	}

-	// Index for whitelisted hashes lookup
-	DB.Exec(`CREATE INDEX IF NOT EXISTS idx_whitelisted_hash 
-		ON whitelisted_hashes(hash)`)
+	if err := DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addons_id 
+		ON addons(id)`).Error; err != nil {
+		log.Fatalf("Failed to create idx_addons_id: %v", err)
+	}

-	// Additional index on addon_id for the JOIN operation
-	DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addons_id 
-		ON addons(id)`)
-
-	// Covering index for addon_files to avoid table lookups
-	DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_covering 
-		ON addon_files(addon_id, hash, path, version) WHERE deleted_at IS NULL`)
+	if err := DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_covering 
+		ON addon_files(addon_id, hash, path, version) WHERE deleted_at IS NULL`).Error; err != nil {
+		log.Fatalf("Failed to create idx_addon_files_covering: %v", err)
+	}
 }

 type Configuration struct {
-	Port           string `yaml:"port"`
-	IP             string `yaml:"ip"`
-	Secret         string `yaml:"secret"`
-	DB             string `yaml:"db"`
-	DiscordWebhook string `yaml:"discordWebhook"`
-	ADMIN_SECRET   string `yaml:"adminSecret"`
-	ScraperWebhook string `yaml:"scraperWebhook"`
+	ServerPort        string `yaml:"server_port"`
+	ServerHost        string `yaml:"server_host"`
+	SecretKey         string `yaml:"secret_key"`
+	AdminSecret       string `yaml:"admin_secret"`
+	DiscordWebhookURL string `yaml:"discord_webhook_url"`
+	ScraperWebhookURL string `yaml:"scraper_webhook_url"`
+	DatabaseDriver    string `yaml:"db_driver"`
+	SQLitePath        string `yaml:"sqlite_path"`
+	PostgresDSN       string `yaml:"postgres_dsn"`
+	PostgresHost      string `yaml:"postgres_host"`
+	PostgresPort      string `yaml:"postgres_port"`
+	PostgresUser      string `yaml:"postgres_user"`
+	PostgresPassword  string `yaml:"postgres_password"`
+	PostgresDBName    string `yaml:"postgres_db_name"`
+	PostgresSSLMode   string `yaml:"postgres_sslmode"`
+	LegacyPort        string `yaml:"port"`
+	LegacyIP          string `yaml:"ip"`
+	LegacySecret      string `yaml:"secret"`
+	LegacyDB          string `yaml:"db"`
+	LegacyDiscordHook string `yaml:"discordWebhook"`
+	LegacyAdminSecret string `yaml:"adminSecret"`
+	LegacyScraperHook string `yaml:"scraperWebhook"`
 }

 func Load() {
-	file, err := os.ReadFile("config.yaml")
-	if err != nil {
-		log.Fatal("Failed to open config file")
-	}
-
 	configuration := Configuration{
-		DB: "register.db",
-	}
-	err = yaml.Unmarshal(file, &configuration)
-	if err != nil {
-		log.Fatal("Failed to read yaml file")
+		ServerPort:      "8083",
+		ServerHost:      "0.0.0.0",
+		DatabaseDriver:  "sqlite",
+		SQLitePath:      "crawler.db",
+		PostgresSSLMode: "disable",
 	}

-	PORT = configuration.Port
-	IP = configuration.IP
-	SECRET = configuration.Secret
-	DB_NAME = configuration.DB
-	DiscordWebhook = configuration.DiscordWebhook
-	ADMIN_SECRET = configuration.ADMIN_SECRET
-	ScraperWebhook = configuration.ScraperWebhook
+	file, err := os.ReadFile("config.yaml")
+	if err == nil {
+		if err := yaml.Unmarshal(file, &configuration); err != nil {
+			log.Fatalf("Failed to read yaml file: %v", err)
+		}
+	} else if !errors.Is(err, os.ErrNotExist) {
+		log.Fatalf("Failed to open config file: %v", err)
+	}
+
+	applyLegacyConfigFallbacks(&configuration)
+	applyEnvOverrides(&configuration)
+
+	ServerPort = configuration.ServerPort
+	ServerHost = configuration.ServerHost
+	SecretKey = configuration.SecretKey
+	AdminSecret = configuration.AdminSecret
+	DiscordWebhookURL = configuration.DiscordWebhookURL
+	ScraperWebhookURL = configuration.ScraperWebhookURL
+	DatabaseDriver = strings.ToLower(configuration.DatabaseDriver)
+	SQLitePath = configuration.SQLitePath
+
+	if strings.TrimSpace(configuration.PostgresDSN) != "" {
+		PostgresDSN = configuration.PostgresDSN
+	} else {
+		PostgresDSN = buildPostgresDSN(configuration)
+	}
+}
+
+func applyLegacyConfigFallbacks(c *Configuration) {
+	if c.ServerPort == "" {
+		c.ServerPort = c.LegacyPort
+	}
+	if c.ServerHost == "" {
+		c.ServerHost = c.LegacyIP
+	}
+	if c.SecretKey == "" {
+		c.SecretKey = c.LegacySecret
+	}
+	if c.AdminSecret == "" {
+		c.AdminSecret = c.LegacyAdminSecret
+	}
+	if c.DiscordWebhookURL == "" {
+		c.DiscordWebhookURL = c.LegacyDiscordHook
+	}
+	if c.ScraperWebhookURL == "" {
+		c.ScraperWebhookURL = c.LegacyScraperHook
+	}
+	if c.SQLitePath == "" {
+		c.SQLitePath = c.LegacyDB
+	}
+}
+
+func applyEnvOverrides(c *Configuration) {
+	c.ServerPort = firstEnv([]string{"SERVER_PORT", "PORT"}, c.ServerPort)
+	c.ServerHost = firstEnv([]string{"SERVER_HOST", "IP"}, c.ServerHost)
+	c.SecretKey = firstEnv([]string{"SECRET_KEY", "SECRET"}, c.SecretKey)
+	c.AdminSecret = firstEnv([]string{"ADMIN_SECRET"}, c.AdminSecret)
+	c.DiscordWebhookURL = firstEnv([]string{"DISCORD_WEBHOOK_URL", "DISCORD_WEBHOOK"}, c.DiscordWebhookURL)
+	c.ScraperWebhookURL = firstEnv([]string{"SCRAPER_WEBHOOK_URL", "SCRAPER_WEBHOOK"}, c.ScraperWebhookURL)
+	c.DatabaseDriver = firstEnv([]string{"DB_DRIVER"}, c.DatabaseDriver)
+	c.SQLitePath = firstEnv([]string{"SQLITE_PATH", "DB"}, c.SQLitePath)
+	c.PostgresDSN = firstEnv([]string{"POSTGRES_DSN"}, c.PostgresDSN)
+	c.PostgresHost = firstEnv([]string{"POSTGRES_HOST"}, c.PostgresHost)
+	c.PostgresPort = firstEnv([]string{"POSTGRES_PORT"}, c.PostgresPort)
+	c.PostgresUser = firstEnv([]string{"POSTGRES_USER"}, c.PostgresUser)
+	c.PostgresPassword = firstEnv([]string{"POSTGRES_PASSWORD"}, c.PostgresPassword)
+	c.PostgresDBName = firstEnv([]string{"POSTGRES_DB_NAME"}, c.PostgresDBName)
+	c.PostgresSSLMode = firstEnv([]string{"POSTGRES_SSLMODE"}, c.PostgresSSLMode)
+}
+
+func firstEnv(keys []string, fallback string) string {
+	for _, k := range keys {
+		if value, ok := os.LookupEnv(k); ok && strings.TrimSpace(value) != "" {
+			return value
+		}
+	}
+	return fallback
+}
+
+func buildPostgresDSN(c Configuration) string {
+	if c.PostgresHost == "" || c.PostgresUser == "" || c.PostgresDBName == "" {
+		return ""
+	}
+
+	port := c.PostgresPort
+	if port == "" {
+		port = "5432"
+	}
+
+	sslMode := c.PostgresSSLMode
+	if sslMode == "" {
+		sslMode = "disable"
+	}
+
+	return fmt.Sprintf(
+		"host=%s user=%s password=%s dbname=%s port=%s sslmode=%s",
+		c.PostgresHost,
+		c.PostgresUser,
+		c.PostgresPassword,
+		c.PostgresDBName,
+		port,
+		sslMode,
+	)
 }
--- a/main.go
+++ b/main.go
@@ -17,6 +17,9 @@ func init() {
 func main() {
 	r := gin.Default()
 	r.Use(CORSMiddleware())
+	r.GET("/health", func(c *gin.Context) {
+		c.JSON(http.StatusOK, gin.H{"status": "ok"})
+	})

 	back := r.Group("/back")
 	{
@@ -29,7 +32,7 @@ func main() {

 	admin := r.Group("/admin")
 	{
-		admin.POST("/addWhitelistedHash",controllers.CheckAdmin, controllers.AddWhitelistedHash)
+		admin.POST("/addWhitelistedHash", controllers.CheckAdmin, controllers.AddWhitelistedHash)
 		admin.GET("/duplicates", controllers.CheckAdmin)
 		admin.POST("/setPriority", controllers.CheckAdmin)
 		admin.POST("/setToBeIndexed", controllers.CheckAdmin)
@@ -41,7 +44,7 @@ func main() {
 		v1.GET("/getPossible", controllers.GetPossibleAddons)
 	}

-	r.Run(initializers.IP + ":" + initializers.PORT)
+	r.Run(initializers.ServerHost + ":" + initializers.ServerPort)
 }

 func CORSMiddleware() gin.HandlerFunc {
--- a/models/addon.go
+++ b/models/addon.go
@@ -2,6 +2,8 @@ package models

 import (
 	"time"
+
+	"gorm.io/gorm"
 )

 type Addon struct {
@@ -18,11 +20,13 @@ type Addon struct {
 	CurrentVersionID     int            `json:"currentVersionId"`
 	CreatedAt            time.Time      `json:"createdAt"`
 	UpdatedAt            time.Time      `json:"updatedAt"`
+	DeletedAt            gorm.DeletedAt `gorm:"index"`
 	ToBeIndexed          bool           `json:"toBeIndexed" gorm:"default:true"`
 	IsBeingIndexed       bool           `json:"isBeingIndexed" gorm:"default:false"`
 	IndexStartTime       time.Time      `json:"indexStartTime"`
 	PriorityIndexing     bool           `json:"priorityIndexing" gorm:"default:false"`
 	AddonFiles           []AddonFile    `json:"addonFiles"`
+	Recheck              bool           `json:"recheck" gorm:"default:false"`

 	Preview string `json:"preview"`
 	Author  string `json:"author"`