241 lines
7.2 KiB
Go
241 lines
7.2 KiB
Go
package initializers
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"strings"
|
|
|
|
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
|
|
"gopkg.in/yaml.v3"
|
|
"gorm.io/driver/postgres"
|
|
"gorm.io/driver/sqlite"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
var DB *gorm.DB
|
|
var ServerPort string
|
|
var ServerHost string
|
|
var SecretKey string
|
|
var DiscordWebhookURL string
|
|
var AdminSecret string
|
|
var ScraperWebhookURL string
|
|
var DatabaseDriver string
|
|
var SQLitePath string
|
|
var PostgresDSN string
|
|
|
|
func ConnectToDB() {
|
|
var (
|
|
db *gorm.DB
|
|
err error
|
|
)
|
|
|
|
switch strings.ToLower(DatabaseDriver) {
|
|
case "", "sqlite", "sqlite3":
|
|
sqlitePath := SQLitePath
|
|
if sqlitePath == "" {
|
|
sqlitePath = "crawler.db"
|
|
}
|
|
|
|
dsn := fmt.Sprintf("%s?_busy_timeout=5000&_journal_mode=WAL", sqlitePath)
|
|
db, err = gorm.Open(sqlite.Open(dsn), &gorm.Config{
|
|
SkipDefaultTransaction: true,
|
|
})
|
|
if err != nil {
|
|
log.Fatalf("Failed to connect to sqlite database: %v", err)
|
|
}
|
|
|
|
if err := db.Exec("PRAGMA synchronous=NORMAL").Error; err != nil {
|
|
log.Fatalf("Failed to set PRAGMA synchronous: %v", err)
|
|
}
|
|
if err := db.Exec("PRAGMA cache_size=-64000").Error; err != nil {
|
|
log.Fatalf("Failed to set PRAGMA cache_size: %v", err)
|
|
}
|
|
if err := db.Exec("PRAGMA temp_store=MEMORY").Error; err != nil {
|
|
log.Fatalf("Failed to set PRAGMA temp_store: %v", err)
|
|
}
|
|
if err := db.Exec("PRAGMA mmap_size=268435456").Error; err != nil {
|
|
log.Fatalf("Failed to set PRAGMA mmap_size: %v", err)
|
|
}
|
|
case "postgres", "postgresql":
|
|
if PostgresDSN == "" {
|
|
log.Fatal("PostgreSQL selected but no DSN is configured")
|
|
}
|
|
|
|
db, err = gorm.Open(postgres.Open(PostgresDSN), &gorm.Config{
|
|
SkipDefaultTransaction: true,
|
|
})
|
|
if err != nil {
|
|
log.Fatalf("Failed to connect to postgres database: %v", err)
|
|
}
|
|
default:
|
|
log.Fatalf("Unsupported database driver: %s", DatabaseDriver)
|
|
}
|
|
|
|
DB = db
|
|
|
|
if err := DB.AutoMigrate(&models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{}); err != nil {
|
|
log.Fatalf("Failed to run migrations: %v", err)
|
|
}
|
|
|
|
if err := DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_hash_addon_optimized
|
|
ON addon_files(hash, addon_id) WHERE deleted_at IS NULL`).Error; err != nil {
|
|
log.Fatalf("Failed to create idx_addon_files_hash_addon_optimized: %v", err)
|
|
}
|
|
|
|
if err := DB.Exec(`CREATE INDEX IF NOT EXISTS idx_whitelisted_hash
|
|
ON whitelisted_hashes(hash)`).Error; err != nil {
|
|
log.Fatalf("Failed to create idx_whitelisted_hash: %v", err)
|
|
}
|
|
|
|
if err := DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addons_id
|
|
ON addons(id)`).Error; err != nil {
|
|
log.Fatalf("Failed to create idx_addons_id: %v", err)
|
|
}
|
|
|
|
if err := DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_covering
|
|
ON addon_files(addon_id, hash, path, version) WHERE deleted_at IS NULL`).Error; err != nil {
|
|
log.Fatalf("Failed to create idx_addon_files_covering: %v", err)
|
|
}
|
|
}
|
|
|
|
type Configuration struct {
|
|
ServerPort string `yaml:"server_port"`
|
|
ServerHost string `yaml:"server_host"`
|
|
SecretKey string `yaml:"secret_key"`
|
|
AdminSecret string `yaml:"admin_secret"`
|
|
DiscordWebhookURL string `yaml:"discord_webhook_url"`
|
|
ScraperWebhookURL string `yaml:"scraper_webhook_url"`
|
|
DatabaseDriver string `yaml:"db_driver"`
|
|
SQLitePath string `yaml:"sqlite_path"`
|
|
PostgresDSN string `yaml:"postgres_dsn"`
|
|
PostgresHost string `yaml:"postgres_host"`
|
|
PostgresPort string `yaml:"postgres_port"`
|
|
PostgresUser string `yaml:"postgres_user"`
|
|
PostgresPassword string `yaml:"postgres_password"`
|
|
PostgresDBName string `yaml:"postgres_db_name"`
|
|
PostgresSSLMode string `yaml:"postgres_sslmode"`
|
|
LegacyPort string `yaml:"port"`
|
|
LegacyIP string `yaml:"ip"`
|
|
LegacySecret string `yaml:"secret"`
|
|
LegacyDB string `yaml:"db"`
|
|
LegacyDiscordHook string `yaml:"discordWebhook"`
|
|
LegacyAdminSecret string `yaml:"adminSecret"`
|
|
LegacyScraperHook string `yaml:"scraperWebhook"`
|
|
}
|
|
|
|
func Load() {
|
|
configuration := Configuration{
|
|
ServerPort: "8083",
|
|
ServerHost: "0.0.0.0",
|
|
DatabaseDriver: "sqlite",
|
|
SQLitePath: "crawler.db",
|
|
PostgresSSLMode: "disable",
|
|
}
|
|
|
|
file, err := os.ReadFile("config.yaml")
|
|
if err == nil {
|
|
if err := yaml.Unmarshal(file, &configuration); err != nil {
|
|
log.Fatalf("Failed to read yaml file: %v", err)
|
|
}
|
|
} else if !errors.Is(err, os.ErrNotExist) {
|
|
log.Fatalf("Failed to open config file: %v", err)
|
|
}
|
|
|
|
applyLegacyConfigFallbacks(&configuration)
|
|
applyEnvOverrides(&configuration)
|
|
|
|
ServerPort = configuration.ServerPort
|
|
ServerHost = configuration.ServerHost
|
|
SecretKey = configuration.SecretKey
|
|
AdminSecret = configuration.AdminSecret
|
|
DiscordWebhookURL = configuration.DiscordWebhookURL
|
|
ScraperWebhookURL = configuration.ScraperWebhookURL
|
|
DatabaseDriver = strings.ToLower(configuration.DatabaseDriver)
|
|
SQLitePath = configuration.SQLitePath
|
|
|
|
if strings.TrimSpace(configuration.PostgresDSN) != "" {
|
|
PostgresDSN = configuration.PostgresDSN
|
|
} else {
|
|
PostgresDSN = buildPostgresDSN(configuration)
|
|
}
|
|
}
|
|
|
|
func applyLegacyConfigFallbacks(c *Configuration) {
|
|
if c.ServerPort == "" {
|
|
c.ServerPort = c.LegacyPort
|
|
}
|
|
if c.ServerHost == "" {
|
|
c.ServerHost = c.LegacyIP
|
|
}
|
|
if c.SecretKey == "" {
|
|
c.SecretKey = c.LegacySecret
|
|
}
|
|
if c.AdminSecret == "" {
|
|
c.AdminSecret = c.LegacyAdminSecret
|
|
}
|
|
if c.DiscordWebhookURL == "" {
|
|
c.DiscordWebhookURL = c.LegacyDiscordHook
|
|
}
|
|
if c.ScraperWebhookURL == "" {
|
|
c.ScraperWebhookURL = c.LegacyScraperHook
|
|
}
|
|
if c.SQLitePath == "" {
|
|
c.SQLitePath = c.LegacyDB
|
|
}
|
|
}
|
|
|
|
func applyEnvOverrides(c *Configuration) {
|
|
c.ServerPort = firstEnv([]string{"SERVER_PORT", "PORT"}, c.ServerPort)
|
|
c.ServerHost = firstEnv([]string{"SERVER_HOST", "IP"}, c.ServerHost)
|
|
c.SecretKey = firstEnv([]string{"SECRET_KEY", "SECRET"}, c.SecretKey)
|
|
c.AdminSecret = firstEnv([]string{"ADMIN_SECRET"}, c.AdminSecret)
|
|
c.DiscordWebhookURL = firstEnv([]string{"DISCORD_WEBHOOK_URL", "DISCORD_WEBHOOK"}, c.DiscordWebhookURL)
|
|
c.ScraperWebhookURL = firstEnv([]string{"SCRAPER_WEBHOOK_URL", "SCRAPER_WEBHOOK"}, c.ScraperWebhookURL)
|
|
c.DatabaseDriver = firstEnv([]string{"DB_DRIVER"}, c.DatabaseDriver)
|
|
c.SQLitePath = firstEnv([]string{"SQLITE_PATH", "DB"}, c.SQLitePath)
|
|
c.PostgresDSN = firstEnv([]string{"POSTGRES_DSN"}, c.PostgresDSN)
|
|
c.PostgresHost = firstEnv([]string{"POSTGRES_HOST"}, c.PostgresHost)
|
|
c.PostgresPort = firstEnv([]string{"POSTGRES_PORT"}, c.PostgresPort)
|
|
c.PostgresUser = firstEnv([]string{"POSTGRES_USER"}, c.PostgresUser)
|
|
c.PostgresPassword = firstEnv([]string{"POSTGRES_PASSWORD"}, c.PostgresPassword)
|
|
c.PostgresDBName = firstEnv([]string{"POSTGRES_DB_NAME"}, c.PostgresDBName)
|
|
c.PostgresSSLMode = firstEnv([]string{"POSTGRES_SSLMODE"}, c.PostgresSSLMode)
|
|
}
|
|
|
|
func firstEnv(keys []string, fallback string) string {
|
|
for _, k := range keys {
|
|
if value, ok := os.LookupEnv(k); ok && strings.TrimSpace(value) != "" {
|
|
return value
|
|
}
|
|
}
|
|
return fallback
|
|
}
|
|
|
|
func buildPostgresDSN(c Configuration) string {
|
|
if c.PostgresHost == "" || c.PostgresUser == "" || c.PostgresDBName == "" {
|
|
return ""
|
|
}
|
|
|
|
port := c.PostgresPort
|
|
if port == "" {
|
|
port = "5432"
|
|
}
|
|
|
|
sslMode := c.PostgresSSLMode
|
|
if sslMode == "" {
|
|
sslMode = "disable"
|
|
}
|
|
|
|
return fmt.Sprintf(
|
|
"host=%s user=%s password=%s dbname=%s port=%s sslmode=%s",
|
|
c.PostgresHost,
|
|
c.PostgresUser,
|
|
c.PostgresPassword,
|
|
c.PostgresDBName,
|
|
port,
|
|
sslMode,
|
|
)
|
|
}
|