- embedded React admin UI at /admin/ui (go:embed), login with ADMIN_SECRET via signed HttpOnly session cookie, X-ADMIN-KEY still works for scripts - admin API: overview stats, indexing queue, addon management (priority, reindex, delete/restore/purge, bulk), whitelist CRUD, duplicate hashes, traffic, metrics explorer, services, events/audit log, system info - built-in metrics: request latency per route, visitors, top addons and searches, indexer/scraper counters; minute buckets in the DB with hourly rollups and retention; optional Prometheus /metrics (METRICS_TOKEN) - POST /back/telemetry for indexers/scrapers (metrics, events, heartbeat) and a stdlib-only Go client in telemetry/ - auth hardening: empty secrets no longer allow access, constant-time compare, CORS not applied to /admin and /back, TRUSTED_PROXIES setting - Dockerfile builds the UI and no longer bakes config.yaml into the image; CI builds the UI, runs tests and builds arm64 with CGO
This commit is contained in:
ilbinek committed
1 parent
0ef27b0072
commit
56d9e10621
79 files changed
+13146
-118
No files matched your search
+105
-1
@@ -5,7 +5,9 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
|
||||
"gopkg.in/yaml.v3"
|
||||
@@ -25,6 +27,18 @@ var DatabaseDriver string
|
||||
var SQLitePath string
|
||||
var PostgresDSN string
|
||||
|
||||
// Admin dashboard, metrics and telemetry settings
|
||||
var TrustedProxies []string
|
||||
var AdminSessionTTL time.Duration
|
||||
var AdminCookieSecure string // auto|true|false
|
||||
var MetricsEnabled bool
|
||||
var MetricsToken string
|
||||
var MetricsMinuteRetentionDays int
|
||||
var MetricsHourRetentionDays int
|
||||
var EventsRetentionDays int
|
||||
var TopItemsRetentionDays int
|
||||
var IndexTimeout time.Duration
|
||||
|
||||
func ConnectToDB() {
|
||||
var (
|
||||
db *gorm.DB
|
||||
@@ -75,7 +89,10 @@ func ConnectToDB() {
|
||||
|
||||
DB = db
|
||||
|
||||
if err := DB.AutoMigrate(&models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{}, &models.AddonDependency{}); err != nil {
|
||||
if err := DB.AutoMigrate(
|
||||
&models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{}, &models.AddonDependency{},
|
||||
&models.MetricSeries{}, &models.MetricPoint{}, &models.ServiceInstance{}, &models.ServiceEvent{}, &models.DailyTopItem{},
|
||||
); err != nil {
|
||||
log.Fatalf("Failed to run migrations: %v", err)
|
||||
}
|
||||
|
||||
@@ -98,6 +115,21 @@ func ConnectToDB() {
|
||||
ON addon_files(addon_id, hash, path, version) WHERE deleted_at IS NULL`).Error; err != nil {
|
||||
log.Fatalf("Failed to create idx_addon_files_covering: %v", err)
|
||||
}
|
||||
|
||||
// Indexes used by the indexing queue and the admin dashboard; failures are
|
||||
// not fatal - they only cost performance
|
||||
optionalIndexes := []string{
|
||||
`CREATE INDEX IF NOT EXISTS idx_addons_queue ON addons(priority_indexing, updated_at)
|
||||
WHERE to_be_indexed = true AND is_being_indexed = false AND deleted_at IS NULL`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_addons_created_at ON addons(created_at)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_addons_subscriber_count ON addons(subscriber_count)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_addons_current_version_size ON addons(current_version_size)`,
|
||||
}
|
||||
for _, stmt := range optionalIndexes {
|
||||
if err := DB.Exec(stmt).Error; err != nil {
|
||||
log.Printf("Warning: failed to create index: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type Configuration struct {
|
||||
@@ -116,6 +148,16 @@ type Configuration struct {
|
||||
PostgresPassword string `yaml:"postgres_password"`
|
||||
PostgresDBName string `yaml:"postgres_db_name"`
|
||||
PostgresSSLMode string `yaml:"postgres_sslmode"`
|
||||
TrustedProxies string `yaml:"trusted_proxies"`
|
||||
AdminSessionTTLH int `yaml:"admin_session_ttl_hours"`
|
||||
AdminCookieSecure string `yaml:"admin_cookie_secure"`
|
||||
MetricsEnabled *bool `yaml:"metrics_enabled"`
|
||||
MetricsToken string `yaml:"metrics_token"`
|
||||
MetricsMinuteDays int `yaml:"metrics_minute_retention_days"`
|
||||
MetricsHourDays int `yaml:"metrics_hour_retention_days"`
|
||||
EventsDays int `yaml:"events_retention_days"`
|
||||
TopItemsDays int `yaml:"top_items_retention_days"`
|
||||
IndexTimeoutMin int `yaml:"index_timeout_minutes"`
|
||||
LegacyPort string `yaml:"port"`
|
||||
LegacyIP string `yaml:"ip"`
|
||||
LegacySecret string `yaml:"secret"`
|
||||
@@ -132,6 +174,14 @@ func Load() {
|
||||
DatabaseDriver: "sqlite",
|
||||
SQLitePath: "crawler.db",
|
||||
PostgresSSLMode: "disable",
|
||||
|
||||
AdminSessionTTLH: 168,
|
||||
AdminCookieSecure: "auto",
|
||||
MetricsMinuteDays: 7,
|
||||
MetricsHourDays: 180,
|
||||
EventsDays: 30,
|
||||
TopItemsDays: 90,
|
||||
IndexTimeoutMin: 30,
|
||||
}
|
||||
|
||||
file, err := os.ReadFile("config.yaml")
|
||||
@@ -155,6 +205,17 @@ func Load() {
|
||||
DatabaseDriver = strings.ToLower(configuration.DatabaseDriver)
|
||||
SQLitePath = configuration.SQLitePath
|
||||
|
||||
TrustedProxies = splitList(configuration.TrustedProxies)
|
||||
AdminSessionTTL = time.Duration(positiveOr(configuration.AdminSessionTTLH, 168)) * time.Hour
|
||||
AdminCookieSecure = strings.ToLower(configuration.AdminCookieSecure)
|
||||
MetricsEnabled = configuration.MetricsEnabled == nil || *configuration.MetricsEnabled
|
||||
MetricsToken = configuration.MetricsToken
|
||||
MetricsMinuteRetentionDays = positiveOr(configuration.MetricsMinuteDays, 7)
|
||||
MetricsHourRetentionDays = positiveOr(configuration.MetricsHourDays, 180)
|
||||
EventsRetentionDays = positiveOr(configuration.EventsDays, 30)
|
||||
TopItemsRetentionDays = positiveOr(configuration.TopItemsDays, 90)
|
||||
IndexTimeout = time.Duration(positiveOr(configuration.IndexTimeoutMin, 30)) * time.Minute
|
||||
|
||||
if strings.TrimSpace(configuration.PostgresDSN) != "" {
|
||||
PostgresDSN = configuration.PostgresDSN
|
||||
} else {
|
||||
@@ -202,6 +263,49 @@ func applyEnvOverrides(c *Configuration) {
|
||||
c.PostgresPassword = firstEnv([]string{"POSTGRES_PASSWORD"}, c.PostgresPassword)
|
||||
c.PostgresDBName = firstEnv([]string{"POSTGRES_DB_NAME"}, c.PostgresDBName)
|
||||
c.PostgresSSLMode = firstEnv([]string{"POSTGRES_SSLMODE"}, c.PostgresSSLMode)
|
||||
c.TrustedProxies = firstEnv([]string{"TRUSTED_PROXIES"}, c.TrustedProxies)
|
||||
c.AdminSessionTTLH = firstEnvInt([]string{"ADMIN_SESSION_TTL_HOURS"}, c.AdminSessionTTLH)
|
||||
c.AdminCookieSecure = firstEnv([]string{"ADMIN_COOKIE_SECURE"}, c.AdminCookieSecure)
|
||||
if v := firstEnv([]string{"METRICS_ENABLED"}, ""); v != "" {
|
||||
enabled := v == "1" || strings.EqualFold(v, "true") || strings.EqualFold(v, "yes")
|
||||
c.MetricsEnabled = &enabled
|
||||
}
|
||||
c.MetricsToken = firstEnv([]string{"METRICS_TOKEN"}, c.MetricsToken)
|
||||
c.MetricsMinuteDays = firstEnvInt([]string{"METRICS_MINUTE_RETENTION_DAYS"}, c.MetricsMinuteDays)
|
||||
c.MetricsHourDays = firstEnvInt([]string{"METRICS_HOUR_RETENTION_DAYS"}, c.MetricsHourDays)
|
||||
c.EventsDays = firstEnvInt([]string{"EVENTS_RETENTION_DAYS"}, c.EventsDays)
|
||||
c.TopItemsDays = firstEnvInt([]string{"TOP_ITEMS_RETENTION_DAYS"}, c.TopItemsDays)
|
||||
c.IndexTimeoutMin = firstEnvInt([]string{"INDEX_TIMEOUT_MINUTES"}, c.IndexTimeoutMin)
|
||||
}
|
||||
|
||||
func firstEnvInt(keys []string, fallback int) int {
|
||||
raw := firstEnv(keys, "")
|
||||
if raw == "" {
|
||||
return fallback
|
||||
}
|
||||
val, err := strconv.Atoi(strings.TrimSpace(raw))
|
||||
if err != nil {
|
||||
log.Printf("Warning: ignoring invalid integer %q for %s", raw, strings.Join(keys, "/"))
|
||||
return fallback
|
||||
}
|
||||
return val
|
||||
}
|
||||
|
||||
func positiveOr(val, fallback int) int {
|
||||
if val > 0 {
|
||||
return val
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func splitList(raw string) []string {
|
||||
var out []string
|
||||
for _, part := range strings.Split(raw, ",") {
|
||||
if part = strings.TrimSpace(part); part != "" {
|
||||
out = append(out, part)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func firstEnv(keys []string, fallback string) string {
|
||||
|
||||
Reference in new issue
Block a user