reforger_crawler_main/admin/overview.go

package admin

import (
	"context"
	"log"
	"net/http"
	"os"
	"strconv"
	"sync"
	"time"

	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
	"github.com/gin-gonic/gin"
	"gorm.io/gorm"
)

type addonCounts struct {
	Total      int64 `json:"total"`
	Indexed    int64 `json:"indexed"`
	Queued     int64 `json:"queued"`
	Indexing   int64 `json:"indexing"`
	Priority   int64 `json:"priority"`
	Recheck    int64 `json:"recheck"`
	Blocked    int64 `json:"blocked"`
	Unlisted   int64 `json:"unlisted"`
	Private    int64 `json:"private"`
	TotalSize  int64 `json:"totalSizeBytes"`
	Added24h   int64 `json:"added24h"`
	Added7d    int64 `json:"added7d"`
	Indexed24h int64 `json:"indexed24h"`

	SoftDeleted int64 `json:"softDeleted"`
}

type typeCount struct {
	Type  string `json:"type"`
	Count int64  `json:"count"`
}

type miniAddon struct {
	ID                 string     `json:"id"`
	Name               string     `json:"name"`
	Author             string     `json:"author"`
	SubscriberCount    int64      `json:"subscriberCount"`
	CurrentVersionSize int64      `json:"currentVersionSize"`
	LastIndexedAt      *time.Time `json:"lastIndexedAt,omitempty"`
	CreatedAt          time.Time  `json:"createdAt"`
}

type overviewData struct {
	Addons          addonCounts    `json:"addons"`
	Types           []typeCount    `json:"types"`
	Whitelist       int64          `json:"whitelistCount"`
	Dependencies    map[string]any `json:"dependencies"`
	Traffic         map[string]any `json:"traffic"`
	Activity        map[string]any `json:"activity"`
	Services        map[string]any `json:"services"`
	TopSubscribed   []miniAddon    `json:"topSubscribed"`
	Largest         []miniAddon    `json:"largest"`
	RecentlyAdded   []miniAddon    `json:"recentlyAdded"`
	RecentlyIndexed []miniAddon    `json:"recentlyIndexed"`
	Heavy           *heavyStats    `json:"heavy"`
}

func loadAddonCounts() (addonCounts, error) {
	var ac addonCounts
	now := time.Now()
	err := initializers.DB.Raw(`SELECT COUNT(*) AS total,
		COALESCE(SUM(CASE WHEN to_be_indexed = ? THEN 1 ELSE 0 END), 0) AS indexed,
		COALESCE(SUM(CASE WHEN to_be_indexed = ? AND is_being_indexed = ? THEN 1 ELSE 0 END), 0) AS queued,
		COALESCE(SUM(CASE WHEN is_being_indexed = ? THEN 1 ELSE 0 END), 0) AS indexing,
		COALESCE(SUM(CASE WHEN priority_indexing = ? AND to_be_indexed = ? THEN 1 ELSE 0 END), 0) AS priority,
		COALESCE(SUM(CASE WHEN recheck = ? THEN 1 ELSE 0 END), 0) AS recheck,
		COALESCE(SUM(CASE WHEN blocked = ? THEN 1 ELSE 0 END), 0) AS blocked,
		COALESCE(SUM(CASE WHEN unlisted = ? THEN 1 ELSE 0 END), 0) AS unlisted,
		COALESCE(SUM(CASE WHEN private = ? THEN 1 ELSE 0 END), 0) AS private,
		CAST(COALESCE(SUM(current_version_size), 0) AS BIGINT) AS total_size,
		COALESCE(SUM(CASE WHEN created_at >= ? THEN 1 ELSE 0 END), 0) AS added24h,
		COALESCE(SUM(CASE WHEN created_at >= ? THEN 1 ELSE 0 END), 0) AS added7d,
		COALESCE(SUM(CASE WHEN last_indexed_at >= ? THEN 1 ELSE 0 END), 0) AS indexed24h
		FROM addons WHERE deleted_at IS NULL`,
		false, true, false, true, true, true, true, true, true, true,
		now.Add(-24*time.Hour), now.Add(-7*24*time.Hour), now.Add(-24*time.Hour),
	).Scan(&ac).Error
	if err != nil {
		return ac, err
	}
	err = initializers.DB.Unscoped().Model(&models.Addon{}).Where("deleted_at IS NOT NULL").Count(&ac.SoftDeleted).Error
	return ac, err
}

func loadMiniAddons(order string, where string, args ...any) []miniAddon {
	out := []miniAddon{}
	q := initializers.DB.Model(&models.Addon{}).
		Select("id, name, author, subscriber_count, current_version_size, last_indexed_at, created_at")
	if where != "" {
		q = q.Where(where, args...)
	}
	q.Order(order).Limit(8).Scan(&out)
	return out
}

func buildOverview() (*overviewData, error) {
	ac, err := loadAddonCounts()
	if err != nil {
		return nil, err
	}
	d := &overviewData{Addons: ac}

	d.Types = []typeCount{}
	initializers.DB.Model(&models.Addon{}).Select("type, COUNT(*) AS count").Group("type").Order("count DESC").Limit(12).Scan(&d.Types)

	initializers.DB.Model(&models.WhitelistedHash{}).Count(&d.Whitelist)

	var deps struct {
		Edges      int64
		WithDeps   int64
		Dependents int64
	}
	initializers.DB.Raw(`SELECT COUNT(*) AS edges, COUNT(DISTINCT addon_id) AS with_deps,
		COUNT(DISTINCT dependency_id) AS dependents FROM addon_dependencies`).Scan(&deps)
	d.Dependencies = map[string]any{"edges": deps.Edges, "addonsWithDependencies": deps.WithDeps, "addonsDependedOn": deps.Dependents}

	now := time.Now().Unix()
	dayAgo := now - 86400
	httpSel := metrics.Selector{Service: metrics.MainService, Name: "http_request_duration_ms"}
	traffic := map[string]any{}
	if t, err := metrics.Totals(metrics.Selector{Service: httpSel.Service, Name: httpSel.Name, GroupBy: []string{"group"}}, dayAgo, now); err == nil {
		var total int64
		for _, g := range t {
			total += g.Count
			if g.Labels["group"] == "v1" {
				traffic["websiteRequests24h"] = g.Count
				traffic["websiteP95Ms"] = g.P95
				traffic["websiteP50Ms"] = g.P50
			}
			if g.Labels["group"] == "back" {
				traffic["workerRequests24h"] = g.Count
			}
		}
		traffic["requests24h"] = total
	}
	traffic["errors5xx24h"] = metrics.Sum(metrics.Selector{Service: httpSel.Service, Name: httpSel.Name, Match: map[string]string{"status": "5xx"}}, dayAgo, now)
	humans, bots := metrics.Visitors()
	traffic["visitorsToday"] = humans
	traffic["botsToday"] = bots
	d.Traffic = traffic

	sum := func(name string, match map[string]string) float64 {
		return metrics.Sum(metrics.Selector{Service: metrics.MainService, Name: name, Match: match}, dayAgo, now)
	}
	d.Activity = map[string]any{
		"scraperCreated24h":      sum("scraper_addons_total", map[string]string{"action": "created"}),
		"scraperUpdated24h":      sum("scraper_addons_total", map[string]string{"action": "updated"}),
		"scraperCreateFailed24h": sum("scraper_addons_total", map[string]string{"action": "create_failed"}),
		"deleted24h":             sum("scraper_addons_deleted_total", map[string]string{"result": "deleted"}),
		"indexed24h":             sum("indexer_submissions_total", map[string]string{"result": "ok"}),
		"claims24h":              sum("indexer_claims_total", nil),
		"timeoutsReset24h":       sum("indexer_timeouts_reset_total", nil),
		"dependencyReports24h":   sum("dependency_reports_total", nil),
	}

	var instances []models.ServiceInstance
	initializers.DB.Find(&instances)
	online := 0
	for i := range instances {
		if metrics.Online(&instances[i], time.Now()) {
			online++
		}
	}
	d.Services = map[string]any{"online": online, "total": len(instances)}

	d.TopSubscribed = loadMiniAddons("subscriber_count DESC", "")
	d.Largest = loadMiniAddons("current_version_size DESC", "")
	d.RecentlyAdded = loadMiniAddons("created_at DESC", "")
	d.RecentlyIndexed = loadMiniAddons("last_indexed_at DESC", "last_indexed_at IS NOT NULL")

	d.Heavy = currentHeavy()
	return d, nil
}

func overview(c *gin.Context) {
	data, at, err := cached("overview", 20*time.Second, buildOverview)
	if err != nil {
		fail(c, http.StatusInternalServerError, err.Error())
		return
	}
	c.JSON(http.StatusOK, gin.H{"asOf": at.Unix(), "data": data})
}

// ---- heavy statistics (background) -----------------------------------------

type tableInfo struct {
	Name      string `json:"name"`
	Rows      int64  `json:"rows"`
	DeadRows  int64  `json:"deadRows"`
	SizeBytes int64  `json:"sizeBytes"`
}

type dupHash struct {
	Hash        string `json:"hash"`
	AddonCount  int64  `json:"addonCount"`
	FileCount   int64  `json:"fileCount"`
	SamplePath  string `json:"samplePath"`
	Whitelisted bool   `json:"whitelisted"`
}

type heavyStats struct {
	AsOf             int64       `json:"asOf"`
	FilesLive        int64       `json:"filesLive"`
	FilesSoftDeleted int64       `json:"filesSoftDeleted"`
	FilesApprox      bool        `json:"filesApprox"`
	DistinctHashes   int64       `json:"distinctHashes"`
	DistinctAsOf     int64       `json:"distinctAsOf"`
	DBDriver         string      `json:"dbDriver"`
	DBSizeBytes      int64       `json:"dbSizeBytes"`
	Tables           []tableInfo `json:"tables"`
	Duplicates       []dupHash   `json:"-"`
	DuplicatesAsOf   int64       `json:"duplicatesAsOf"`
	Error            string      `json:"error,omitempty"`
}

var (
	heavyMu   sync.Mutex
	heavy     *heavyStats
	refreshCh = make(chan bool, 1)
)

func currentHeavy() *heavyStats {
	heavyMu.Lock()
	defer heavyMu.Unlock()
	return heavy
}

// requestHeavyRefresh triggers a refresh; full=true also recomputes the
// expensive duplicate statistics.
func requestHeavyRefresh(full bool) {
	select {
	case refreshCh <- full:
	default:
	}
}

func heavyStatsLoop(ctx context.Context) {
	lastFull := time.Time{}
	run := func(full bool) {
		defer func() {
			if r := recover(); r != nil {
				log.Printf("admin: heavy stats panicked: %v", r)
			}
		}()
		full = full || time.Since(lastFull) > time.Hour
		computeHeavy(full)
		if full {
			lastFull = time.Now()
		}
	}
	// give the server a moment to start before the first scan
	select {
	case <-ctx.Done():
		return
	case <-time.After(5 * time.Second):
	}
	run(true)
	ticker := time.NewTicker(10 * time.Minute)
	defer ticker.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			run(false)
		case full := <-refreshCh:
			run(full)
		}
	}
}

// withTimeout runs fn in a transaction with a statement timeout on Postgres.
func withTimeout(seconds int, fn func(tx *gorm.DB) error) error {
	return initializers.DB.Transaction(func(tx *gorm.DB) error {
		if isPostgres() {
			if err := tx.Exec("SET LOCAL statement_timeout = " + strconv.Itoa(seconds*1000)).Error; err != nil {
				return err
			}
		}
		return fn(tx)
	})
}

var ourTables = []string{"addons", "addon_files", "addon_dependencies", "whitelisted_hashes",
	"metric_series", "metric_points", "service_instances", "service_events", "daily_top_items"}

func computeHeavy(full bool) {
	started := time.Now()
	h := &heavyStats{AsOf: time.Now().Unix(), DBDriver: initializers.DB.Dialector.Name(), Tables: []tableInfo{}}
	prev := currentHeavy()
	if prev != nil {
		h.DistinctHashes, h.DistinctAsOf = prev.DistinctHashes, prev.DistinctAsOf
		h.Duplicates, h.DuplicatesAsOf = prev.Duplicates, prev.DuplicatesAsOf
	}

	var errs []string
	note := func(what string, err error) {
		if err != nil {
			errs = append(errs, what+": "+err.Error())
		}
	}

	if isPostgres() {
		note("tables", initializers.DB.Raw(`SELECT relname AS name, n_live_tup AS rows, n_dead_tup AS dead_rows,
			pg_total_relation_size(relid) AS size_bytes FROM pg_stat_user_tables ORDER BY size_bytes DESC`).Scan(&h.Tables).Error)
		note("db size", initializers.DB.Raw(`SELECT pg_database_size(current_database())`).Scan(&h.DBSizeBytes).Error)
		note("soft-deleted files", withTimeout(60, func(tx *gorm.DB) error {
			return tx.Raw(`SELECT COUNT(*) FROM addon_files WHERE deleted_at IS NOT NULL`).Scan(&h.FilesSoftDeleted).Error
		}))
		for _, t := range h.Tables {
			if t.Name == "addon_files" {
				h.FilesLive = max(0, t.Rows-h.FilesSoftDeleted)
				h.FilesApprox = true
			}
		}
	} else {
		var pageCount, pageSize int64
		initializers.DB.Raw("PRAGMA page_count").Scan(&pageCount)
		initializers.DB.Raw("PRAGMA page_size").Scan(&pageSize)
		h.DBSizeBytes = pageCount * pageSize
		if st, err := os.Stat(sqlitePath() + "-wal"); err == nil {
			h.DBSizeBytes += st.Size()
		}
		for _, name := range ourTables {
			var n int64
			if err := initializers.DB.Table(name).Count(&n).Error; err == nil {
				h.Tables = append(h.Tables, tableInfo{Name: name, Rows: n})
			}
		}
		note("files", initializers.DB.Raw(`SELECT
			COALESCE(SUM(CASE WHEN deleted_at IS NULL THEN 1 ELSE 0 END), 0),
			COALESCE(SUM(CASE WHEN deleted_at IS NOT NULL THEN 1 ELSE 0 END), 0) FROM addon_files`).Row().
			Scan(&h.FilesLive, &h.FilesSoftDeleted))
	}

	if full {
		var distinct int64
		err := withTimeout(120, func(tx *gorm.DB) error {
			return tx.Raw(`SELECT COUNT(DISTINCT hash) FROM addon_files WHERE deleted_at IS NULL`).Scan(&distinct).Error
		})
		note("distinct hashes", err)
		if err == nil {
			h.DistinctHashes, h.DistinctAsOf = distinct, time.Now().Unix()
		}

		var dups []dupHash
		err = withTimeout(120, func(tx *gorm.DB) error {
			return tx.Raw(`SELECT hash, COUNT(DISTINCT addon_id) AS addon_count, COUNT(*) AS file_count
				FROM addon_files WHERE deleted_at IS NULL
				GROUP BY hash HAVING COUNT(DISTINCT addon_id) > 1
				ORDER BY addon_count DESC, file_count DESC LIMIT 200`).Scan(&dups).Error
		})
		note("duplicates", err)
		if err == nil {
			h.Duplicates, h.DuplicatesAsOf = dups, time.Now().Unix()
			if len(dups) > 0 {
				hashes := make([]string, len(dups))
				for i, d := range dups {
					hashes[i] = d.Hash
				}
				var samples []struct {
					Hash string
					Path string
				}
				initializers.DB.Raw(`SELECT hash, MIN(path) AS path FROM addon_files
					WHERE deleted_at IS NULL AND hash IN ? GROUP BY hash`, hashes).Scan(&samples)
				byHash := map[string]string{}
				for _, s := range samples {
					byHash[s.Hash] = s.Path
				}
				for i := range h.Duplicates {
					h.Duplicates[i].SamplePath = byHash[h.Duplicates[i].Hash]
				}
			}
		}
	}

	if len(errs) > 0 {
		h.Error = errs[0]
		for _, e := range errs {
			log.Printf("admin: heavy stats: %s", e)
		}
	}
	heavyMu.Lock()
	heavy = h
	heavyMu.Unlock()
	invalidate("overview")
	if time.Since(started) > 10*time.Second {
		log.Printf("admin: heavy stats took %s", time.Since(started).Round(time.Millisecond))
	}
}

func sqlitePath() string {
	if initializers.SQLitePath != "" {
		return initializers.SQLitePath
	}
	return "crawler.db"
}

// duplicateHashes lists the hashes shared by the most addons.
func duplicateHashes(c *gin.Context) {
	h := currentHeavy()
	if h == nil {
		c.JSON(http.StatusOK, gin.H{"asOf": 0, "items": []dupHash{}, "pending": true})
		return
	}
	items := make([]dupHash, len(h.Duplicates))
	copy(items, h.Duplicates)
	if len(items) > 0 {
		hashes := make([]string, len(items))
		for i, d := range items {
			hashes[i] = d.Hash
		}
		var wl []string
		initializers.DB.Model(&models.WhitelistedHash{}).Where("hash IN ?", hashes).Pluck("hash", &wl)
		set := map[string]bool{}
		for _, w := range wl {
			set[w] = true
		}
		for i := range items {
			items[i].Whitelisted = set[items[i].Hash]
		}
	}
	if c.Query("hideWhitelisted") == "1" {
		kept := items[:0]
		for _, it := range items {
			if !it.Whitelisted {
				kept = append(kept, it)
			}
		}
		items = kept
	}
	c.JSON(http.StatusOK, gin.H{"asOf": h.DuplicatesAsOf, "items": items})
}