updates, dokploy

2026-04-09 01:39:37 +02:00
parent fcebe00175
commit e39a92c6d3
14 changed files with 450 additions and 276 deletions
--- a/initializers/initialize.go
+++ b/initializers/initialize.go
@@ -1,91 +1,240 @@
 package initializers

 import (
+	"errors"
+	"fmt"
 	"log"
 	"os"
+	"strings"

 	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
 	"gopkg.in/yaml.v3"
+	"gorm.io/driver/postgres"
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
 )

 var DB *gorm.DB
-var PORT string
-var IP string
-var SECRET string
-var DB_NAME string
-var DiscordWebhook string
-var ADMIN_SECRET string
-var ScraperWebhook string
+var ServerPort string
+var ServerHost string
+var SecretKey string
+var DiscordWebhookURL string
+var AdminSecret string
+var ScraperWebhookURL string
+var DatabaseDriver string
+var SQLitePath string
+var PostgresDSN string

 func ConnectToDB() {
-	db, err := gorm.Open(sqlite.Open(DB_NAME), &gorm.Config{
-		PrepareStmt:            true, // Cache prepared statements
-		SkipDefaultTransaction: true, // Disable default transactions for better performance
-	})
-	if err != nil {
-		log.Fatal("Failed to connect to database")
+	var (
+		db  *gorm.DB
+		err error
+	)
+
+	switch strings.ToLower(DatabaseDriver) {
+	case "", "sqlite", "sqlite3":
+		sqlitePath := SQLitePath
+		if sqlitePath == "" {
+			sqlitePath = "crawler.db"
+		}
+
+		dsn := fmt.Sprintf("%s?_busy_timeout=5000&_journal_mode=WAL", sqlitePath)
+		db, err = gorm.Open(sqlite.Open(dsn), &gorm.Config{
+			SkipDefaultTransaction: true,
+		})
+		if err != nil {
+			log.Fatalf("Failed to connect to sqlite database: %v", err)
+		}
+
+		if err := db.Exec("PRAGMA synchronous=NORMAL").Error; err != nil {
+			log.Fatalf("Failed to set PRAGMA synchronous: %v", err)
+		}
+		if err := db.Exec("PRAGMA cache_size=-64000").Error; err != nil {
+			log.Fatalf("Failed to set PRAGMA cache_size: %v", err)
+		}
+		if err := db.Exec("PRAGMA temp_store=MEMORY").Error; err != nil {
+			log.Fatalf("Failed to set PRAGMA temp_store: %v", err)
+		}
+		if err := db.Exec("PRAGMA mmap_size=268435456").Error; err != nil {
+			log.Fatalf("Failed to set PRAGMA mmap_size: %v", err)
+		}
+	case "postgres", "postgresql":
+		if PostgresDSN == "" {
+			log.Fatal("PostgreSQL selected but no DSN is configured")
+		}
+
+		db, err = gorm.Open(postgres.Open(PostgresDSN), &gorm.Config{
+			SkipDefaultTransaction: true,
+		})
+		if err != nil {
+			log.Fatalf("Failed to connect to postgres database: %v", err)
+		}
+	default:
+		log.Fatalf("Unsupported database driver: %s", DatabaseDriver)
 	}

 	DB = db

-	// Optimize SQLite for performance
-	DB.Exec("PRAGMA journal_mode=WAL")    // Write-Ahead Logging for better concurrency
-	DB.Exec("PRAGMA synchronous=NORMAL")  // Balance between safety and speed
-	DB.Exec("PRAGMA cache_size=-64000")   // 64MB cache (negative = KB)
-	DB.Exec("PRAGMA temp_store=MEMORY")   // Use memory for temp tables
-	DB.Exec("PRAGMA mmap_size=268435456") // 256MB memory-mapped I/O
+	if err := DB.AutoMigrate(&models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{}); err != nil {
+		log.Fatalf("Failed to run migrations: %v", err)
+	}

-	DB.AutoMigrate(&models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{})
+	if err := DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_hash_addon_optimized 
+		ON addon_files(hash, addon_id) WHERE deleted_at IS NULL`).Error; err != nil {
+		log.Fatalf("Failed to create idx_addon_files_hash_addon_optimized: %v", err)
+	}

-	// Create optimized indexes for duplicate detection queries
-	// Composite index for efficient hash lookups excluding specific addon_id
-	DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_hash_addon_optimized 
-		ON addon_files(hash, addon_id) WHERE deleted_at IS NULL`)
+	if err := DB.Exec(`CREATE INDEX IF NOT EXISTS idx_whitelisted_hash 
+		ON whitelisted_hashes(hash)`).Error; err != nil {
+		log.Fatalf("Failed to create idx_whitelisted_hash: %v", err)
+	}

-	// Index for whitelisted hashes lookup
-	DB.Exec(`CREATE INDEX IF NOT EXISTS idx_whitelisted_hash 
-		ON whitelisted_hashes(hash)`)
+	if err := DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addons_id 
+		ON addons(id)`).Error; err != nil {
+		log.Fatalf("Failed to create idx_addons_id: %v", err)
+	}

-	// Additional index on addon_id for the JOIN operation
-	DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addons_id 
-		ON addons(id)`)
-
-	// Covering index for addon_files to avoid table lookups
-	DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_covering 
-		ON addon_files(addon_id, hash, path, version) WHERE deleted_at IS NULL`)
+	if err := DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_covering 
+		ON addon_files(addon_id, hash, path, version) WHERE deleted_at IS NULL`).Error; err != nil {
+		log.Fatalf("Failed to create idx_addon_files_covering: %v", err)
+	}
 }

 type Configuration struct {
-	Port           string `yaml:"port"`
-	IP             string `yaml:"ip"`
-	Secret         string `yaml:"secret"`
-	DB             string `yaml:"db"`
-	DiscordWebhook string `yaml:"discordWebhook"`
-	ADMIN_SECRET   string `yaml:"adminSecret"`
-	ScraperWebhook string `yaml:"scraperWebhook"`
+	ServerPort        string `yaml:"server_port"`
+	ServerHost        string `yaml:"server_host"`
+	SecretKey         string `yaml:"secret_key"`
+	AdminSecret       string `yaml:"admin_secret"`
+	DiscordWebhookURL string `yaml:"discord_webhook_url"`
+	ScraperWebhookURL string `yaml:"scraper_webhook_url"`
+	DatabaseDriver    string `yaml:"db_driver"`
+	SQLitePath        string `yaml:"sqlite_path"`
+	PostgresDSN       string `yaml:"postgres_dsn"`
+	PostgresHost      string `yaml:"postgres_host"`
+	PostgresPort      string `yaml:"postgres_port"`
+	PostgresUser      string `yaml:"postgres_user"`
+	PostgresPassword  string `yaml:"postgres_password"`
+	PostgresDBName    string `yaml:"postgres_db_name"`
+	PostgresSSLMode   string `yaml:"postgres_sslmode"`
+	LegacyPort        string `yaml:"port"`
+	LegacyIP          string `yaml:"ip"`
+	LegacySecret      string `yaml:"secret"`
+	LegacyDB          string `yaml:"db"`
+	LegacyDiscordHook string `yaml:"discordWebhook"`
+	LegacyAdminSecret string `yaml:"adminSecret"`
+	LegacyScraperHook string `yaml:"scraperWebhook"`
 }

 func Load() {
-	file, err := os.ReadFile("config.yaml")
-	if err != nil {
-		log.Fatal("Failed to open config file")
-	}
-
 	configuration := Configuration{
-		DB: "register.db",
-	}
-	err = yaml.Unmarshal(file, &configuration)
-	if err != nil {
-		log.Fatal("Failed to read yaml file")
+		ServerPort:      "8083",
+		ServerHost:      "0.0.0.0",
+		DatabaseDriver:  "sqlite",
+		SQLitePath:      "crawler.db",
+		PostgresSSLMode: "disable",
 	}

-	PORT = configuration.Port
-	IP = configuration.IP
-	SECRET = configuration.Secret
-	DB_NAME = configuration.DB
-	DiscordWebhook = configuration.DiscordWebhook
-	ADMIN_SECRET = configuration.ADMIN_SECRET
-	ScraperWebhook = configuration.ScraperWebhook
+	file, err := os.ReadFile("config.yaml")
+	if err == nil {
+		if err := yaml.Unmarshal(file, &configuration); err != nil {
+			log.Fatalf("Failed to read yaml file: %v", err)
+		}
+	} else if !errors.Is(err, os.ErrNotExist) {
+		log.Fatalf("Failed to open config file: %v", err)
+	}
+
+	applyLegacyConfigFallbacks(&configuration)
+	applyEnvOverrides(&configuration)
+
+	ServerPort = configuration.ServerPort
+	ServerHost = configuration.ServerHost
+	SecretKey = configuration.SecretKey
+	AdminSecret = configuration.AdminSecret
+	DiscordWebhookURL = configuration.DiscordWebhookURL
+	ScraperWebhookURL = configuration.ScraperWebhookURL
+	DatabaseDriver = strings.ToLower(configuration.DatabaseDriver)
+	SQLitePath = configuration.SQLitePath
+
+	if strings.TrimSpace(configuration.PostgresDSN) != "" {
+		PostgresDSN = configuration.PostgresDSN
+	} else {
+		PostgresDSN = buildPostgresDSN(configuration)
+	}
+}
+
+func applyLegacyConfigFallbacks(c *Configuration) {
+	if c.ServerPort == "" {
+		c.ServerPort = c.LegacyPort
+	}
+	if c.ServerHost == "" {
+		c.ServerHost = c.LegacyIP
+	}
+	if c.SecretKey == "" {
+		c.SecretKey = c.LegacySecret
+	}
+	if c.AdminSecret == "" {
+		c.AdminSecret = c.LegacyAdminSecret
+	}
+	if c.DiscordWebhookURL == "" {
+		c.DiscordWebhookURL = c.LegacyDiscordHook
+	}
+	if c.ScraperWebhookURL == "" {
+		c.ScraperWebhookURL = c.LegacyScraperHook
+	}
+	if c.SQLitePath == "" {
+		c.SQLitePath = c.LegacyDB
+	}
+}
+
+func applyEnvOverrides(c *Configuration) {
+	c.ServerPort = firstEnv([]string{"SERVER_PORT", "PORT"}, c.ServerPort)
+	c.ServerHost = firstEnv([]string{"SERVER_HOST", "IP"}, c.ServerHost)
+	c.SecretKey = firstEnv([]string{"SECRET_KEY", "SECRET"}, c.SecretKey)
+	c.AdminSecret = firstEnv([]string{"ADMIN_SECRET"}, c.AdminSecret)
+	c.DiscordWebhookURL = firstEnv([]string{"DISCORD_WEBHOOK_URL", "DISCORD_WEBHOOK"}, c.DiscordWebhookURL)
+	c.ScraperWebhookURL = firstEnv([]string{"SCRAPER_WEBHOOK_URL", "SCRAPER_WEBHOOK"}, c.ScraperWebhookURL)
+	c.DatabaseDriver = firstEnv([]string{"DB_DRIVER"}, c.DatabaseDriver)
+	c.SQLitePath = firstEnv([]string{"SQLITE_PATH", "DB"}, c.SQLitePath)
+	c.PostgresDSN = firstEnv([]string{"POSTGRES_DSN"}, c.PostgresDSN)
+	c.PostgresHost = firstEnv([]string{"POSTGRES_HOST"}, c.PostgresHost)
+	c.PostgresPort = firstEnv([]string{"POSTGRES_PORT"}, c.PostgresPort)
+	c.PostgresUser = firstEnv([]string{"POSTGRES_USER"}, c.PostgresUser)
+	c.PostgresPassword = firstEnv([]string{"POSTGRES_PASSWORD"}, c.PostgresPassword)
+	c.PostgresDBName = firstEnv([]string{"POSTGRES_DB_NAME"}, c.PostgresDBName)
+	c.PostgresSSLMode = firstEnv([]string{"POSTGRES_SSLMODE"}, c.PostgresSSLMode)
+}
+
+func firstEnv(keys []string, fallback string) string {
+	for _, k := range keys {
+		if value, ok := os.LookupEnv(k); ok && strings.TrimSpace(value) != "" {
+			return value
+		}
+	}
+	return fallback
+}
+
+func buildPostgresDSN(c Configuration) string {
+	if c.PostgresHost == "" || c.PostgresUser == "" || c.PostgresDBName == "" {
+		return ""
+	}
+
+	port := c.PostgresPort
+	if port == "" {
+		port = "5432"
+	}
+
+	sslMode := c.PostgresSSLMode
+	if sslMode == "" {
+		sslMode = "disable"
+	}
+
+	return fmt.Sprintf(
+		"host=%s user=%s password=%s dbname=%s port=%s sslmode=%s",
+		c.PostgresHost,
+		c.PostgresUser,
+		c.PostgresPassword,
+		c.PostgresDBName,
+		port,
+		sslMode,
+	)
 }