package initializers import ( "log" "os" "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" "gopkg.in/yaml.v3" "gorm.io/driver/sqlite" "gorm.io/gorm" ) var DB *gorm.DB var PORT string var IP string var SECRET string var DB_NAME string var DiscordWebhook string var ADMIN_SECRET string var ScraperWebhook string func ConnectToDB() { db, err := gorm.Open(sqlite.Open(DB_NAME), &gorm.Config{ PrepareStmt: true, // Cache prepared statements SkipDefaultTransaction: true, // Disable default transactions for better performance }) if err != nil { log.Fatal("Failed to connect to database") } DB = db // Optimize SQLite for performance DB.Exec("PRAGMA journal_mode=WAL") // Write-Ahead Logging for better concurrency DB.Exec("PRAGMA synchronous=NORMAL") // Balance between safety and speed DB.Exec("PRAGMA cache_size=-64000") // 64MB cache (negative = KB) DB.Exec("PRAGMA temp_store=MEMORY") // Use memory for temp tables DB.Exec("PRAGMA mmap_size=268435456") // 256MB memory-mapped I/O DB.AutoMigrate(&models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{}) // Create optimized indexes for duplicate detection queries // Composite index for efficient hash lookups excluding specific addon_id DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_hash_addon_optimized ON addon_files(hash, addon_id) WHERE deleted_at IS NULL`) // Index for whitelisted hashes lookup DB.Exec(`CREATE INDEX IF NOT EXISTS idx_whitelisted_hash ON whitelisted_hashes(hash)`) // Additional index on addon_id for the JOIN operation DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addons_id ON addons(id)`) // Covering index for addon_files to avoid table lookups DB.Exec(`CREATE INDEX IF NOT EXISTS idx_addon_files_covering ON addon_files(addon_id, hash, path, version) WHERE deleted_at IS NULL`) } type Configuration struct { Port string `yaml:"port"` IP string `yaml:"ip"` Secret string `yaml:"secret"` DB string `yaml:"db"` DiscordWebhook string `yaml:"discordWebhook"` ADMIN_SECRET string `yaml:"adminSecret"` ScraperWebhook string `yaml:"scraperWebhook"` } func Load() { file, err := os.ReadFile("config.yaml") if err != nil { log.Fatal("Failed to open config file") } configuration := Configuration{ DB: "register.db", } err = yaml.Unmarshal(file, &configuration) if err != nil { log.Fatal("Failed to read yaml file") } PORT = configuration.Port IP = configuration.IP SECRET = configuration.Secret DB_NAME = configuration.DB DiscordWebhook = configuration.DiscordWebhook ADMIN_SECRET = configuration.ADMIN_SECRET ScraperWebhook = configuration.ScraperWebhook }