ilbinek
748db94d55
Cross Compile Go / build (push) Canceled after 0s
- compute duplicate-hash statistics in their own job (startup, hourly, on demand) so they no longer block the table stats - single-pass query over DISTINCT (hash, addon_id), 15 min timeout - record errors and panics; never stay stuck in a computing state - whitelist page polls while computing, shows errors and a Recompute button; tab is linkable via ?tab=shared
358 lines
12 KiB
Go
358 lines
12 KiB
Go
package admin
|
|
|
|
import (
|
|
"context"
|
|
"log"
|
|
"net/http"
|
|
"os"
|
|
"strconv"
|
|
"sync"
|
|
"time"
|
|
|
|
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
|
|
"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
|
|
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
|
|
"github.com/gin-gonic/gin"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
type addonCounts struct {
|
|
Total int64 `json:"total"`
|
|
Indexed int64 `json:"indexed"`
|
|
Queued int64 `json:"queued"`
|
|
Indexing int64 `json:"indexing"`
|
|
Priority int64 `json:"priority"`
|
|
Recheck int64 `json:"recheck"`
|
|
Blocked int64 `json:"blocked"`
|
|
Unlisted int64 `json:"unlisted"`
|
|
Private int64 `json:"private"`
|
|
TotalSize int64 `json:"totalSizeBytes"`
|
|
Added24h int64 `json:"added24h"`
|
|
Added7d int64 `json:"added7d"`
|
|
Indexed24h int64 `json:"indexed24h"`
|
|
|
|
SoftDeleted int64 `json:"softDeleted"`
|
|
}
|
|
|
|
type typeCount struct {
|
|
Type string `json:"type"`
|
|
Count int64 `json:"count"`
|
|
}
|
|
|
|
type miniAddon struct {
|
|
ID string `json:"id"`
|
|
Name string `json:"name"`
|
|
Author string `json:"author"`
|
|
SubscriberCount int64 `json:"subscriberCount"`
|
|
CurrentVersionSize int64 `json:"currentVersionSize"`
|
|
LastIndexedAt *time.Time `json:"lastIndexedAt,omitempty"`
|
|
CreatedAt time.Time `json:"createdAt"`
|
|
}
|
|
|
|
type overviewData struct {
|
|
Addons addonCounts `json:"addons"`
|
|
Types []typeCount `json:"types"`
|
|
Whitelist int64 `json:"whitelistCount"`
|
|
Dependencies map[string]any `json:"dependencies"`
|
|
Traffic map[string]any `json:"traffic"`
|
|
Activity map[string]any `json:"activity"`
|
|
Services map[string]any `json:"services"`
|
|
TopSubscribed []miniAddon `json:"topSubscribed"`
|
|
Largest []miniAddon `json:"largest"`
|
|
RecentlyAdded []miniAddon `json:"recentlyAdded"`
|
|
RecentlyIndexed []miniAddon `json:"recentlyIndexed"`
|
|
Heavy *heavyStats `json:"heavy"`
|
|
}
|
|
|
|
func loadAddonCounts() (addonCounts, error) {
|
|
var ac addonCounts
|
|
now := time.Now()
|
|
err := initializers.DB.Raw(`SELECT COUNT(*) AS total,
|
|
COALESCE(SUM(CASE WHEN to_be_indexed = ? THEN 1 ELSE 0 END), 0) AS indexed,
|
|
COALESCE(SUM(CASE WHEN to_be_indexed = ? AND is_being_indexed = ? THEN 1 ELSE 0 END), 0) AS queued,
|
|
COALESCE(SUM(CASE WHEN is_being_indexed = ? THEN 1 ELSE 0 END), 0) AS indexing,
|
|
COALESCE(SUM(CASE WHEN priority_indexing = ? AND to_be_indexed = ? THEN 1 ELSE 0 END), 0) AS priority,
|
|
COALESCE(SUM(CASE WHEN recheck = ? THEN 1 ELSE 0 END), 0) AS recheck,
|
|
COALESCE(SUM(CASE WHEN blocked = ? THEN 1 ELSE 0 END), 0) AS blocked,
|
|
COALESCE(SUM(CASE WHEN unlisted = ? THEN 1 ELSE 0 END), 0) AS unlisted,
|
|
COALESCE(SUM(CASE WHEN private = ? THEN 1 ELSE 0 END), 0) AS private,
|
|
CAST(COALESCE(SUM(current_version_size), 0) AS BIGINT) AS total_size,
|
|
COALESCE(SUM(CASE WHEN created_at >= ? THEN 1 ELSE 0 END), 0) AS added24h,
|
|
COALESCE(SUM(CASE WHEN created_at >= ? THEN 1 ELSE 0 END), 0) AS added7d,
|
|
COALESCE(SUM(CASE WHEN last_indexed_at >= ? THEN 1 ELSE 0 END), 0) AS indexed24h
|
|
FROM addons WHERE deleted_at IS NULL`,
|
|
false, true, false, true, true, true, true, true, true, true,
|
|
now.Add(-24*time.Hour), now.Add(-7*24*time.Hour), now.Add(-24*time.Hour),
|
|
).Scan(&ac).Error
|
|
if err != nil {
|
|
return ac, err
|
|
}
|
|
err = initializers.DB.Unscoped().Model(&models.Addon{}).Where("deleted_at IS NOT NULL").Count(&ac.SoftDeleted).Error
|
|
return ac, err
|
|
}
|
|
|
|
func loadMiniAddons(order string, where string, args ...any) []miniAddon {
|
|
out := []miniAddon{}
|
|
q := initializers.DB.Model(&models.Addon{}).
|
|
Select("id, name, author, subscriber_count, current_version_size, last_indexed_at, created_at")
|
|
if where != "" {
|
|
q = q.Where(where, args...)
|
|
}
|
|
q.Order(order).Limit(8).Scan(&out)
|
|
return out
|
|
}
|
|
|
|
func buildOverview() (*overviewData, error) {
|
|
ac, err := loadAddonCounts()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
d := &overviewData{Addons: ac}
|
|
|
|
d.Types = []typeCount{}
|
|
initializers.DB.Model(&models.Addon{}).Select("type, COUNT(*) AS count").Group("type").Order("count DESC").Limit(12).Scan(&d.Types)
|
|
|
|
initializers.DB.Model(&models.WhitelistedHash{}).Count(&d.Whitelist)
|
|
|
|
var deps struct {
|
|
Edges int64
|
|
WithDeps int64
|
|
Dependents int64
|
|
}
|
|
initializers.DB.Raw(`SELECT COUNT(*) AS edges, COUNT(DISTINCT addon_id) AS with_deps,
|
|
COUNT(DISTINCT dependency_id) AS dependents FROM addon_dependencies`).Scan(&deps)
|
|
d.Dependencies = map[string]any{"edges": deps.Edges, "addonsWithDependencies": deps.WithDeps, "addonsDependedOn": deps.Dependents}
|
|
|
|
now := time.Now().Unix()
|
|
dayAgo := now - 86400
|
|
httpSel := metrics.Selector{Service: metrics.MainService, Name: "http_request_duration_ms"}
|
|
traffic := map[string]any{}
|
|
if t, err := metrics.Totals(metrics.Selector{Service: httpSel.Service, Name: httpSel.Name, GroupBy: []string{"group"}}, dayAgo, now); err == nil {
|
|
var total int64
|
|
for _, g := range t {
|
|
total += g.Count
|
|
if g.Labels["group"] == "v1" {
|
|
traffic["websiteRequests24h"] = g.Count
|
|
traffic["websiteP95Ms"] = g.P95
|
|
traffic["websiteP50Ms"] = g.P50
|
|
}
|
|
if g.Labels["group"] == "back" {
|
|
traffic["workerRequests24h"] = g.Count
|
|
}
|
|
}
|
|
traffic["requests24h"] = total
|
|
}
|
|
traffic["errors5xx24h"] = metrics.Sum(metrics.Selector{Service: httpSel.Service, Name: httpSel.Name, Match: map[string]string{"status": "5xx"}}, dayAgo, now)
|
|
humans, bots := metrics.Visitors()
|
|
traffic["visitorsToday"] = humans
|
|
traffic["botsToday"] = bots
|
|
d.Traffic = traffic
|
|
|
|
sum := func(name string, match map[string]string) float64 {
|
|
return metrics.Sum(metrics.Selector{Service: metrics.MainService, Name: name, Match: match}, dayAgo, now)
|
|
}
|
|
d.Activity = map[string]any{
|
|
"scraperCreated24h": sum("scraper_addons_total", map[string]string{"action": "created"}),
|
|
"scraperUpdated24h": sum("scraper_addons_total", map[string]string{"action": "updated"}),
|
|
"scraperCreateFailed24h": sum("scraper_addons_total", map[string]string{"action": "create_failed"}),
|
|
"deleted24h": sum("scraper_addons_deleted_total", map[string]string{"result": "deleted"}),
|
|
"indexed24h": sum("indexer_submissions_total", map[string]string{"result": "ok"}),
|
|
"claims24h": sum("indexer_claims_total", nil),
|
|
"timeoutsReset24h": sum("indexer_timeouts_reset_total", nil),
|
|
"dependencyReports24h": sum("dependency_reports_total", nil),
|
|
}
|
|
|
|
var instances []models.ServiceInstance
|
|
initializers.DB.Find(&instances)
|
|
online := 0
|
|
for i := range instances {
|
|
if metrics.Online(&instances[i], time.Now()) {
|
|
online++
|
|
}
|
|
}
|
|
d.Services = map[string]any{"online": online, "total": len(instances)}
|
|
|
|
d.TopSubscribed = loadMiniAddons("subscriber_count DESC", "")
|
|
d.Largest = loadMiniAddons("current_version_size DESC", "")
|
|
d.RecentlyAdded = loadMiniAddons("created_at DESC", "")
|
|
d.RecentlyIndexed = loadMiniAddons("last_indexed_at DESC", "last_indexed_at IS NOT NULL")
|
|
|
|
d.Heavy = currentHeavy()
|
|
return d, nil
|
|
}
|
|
|
|
func overview(c *gin.Context) {
|
|
data, at, err := cached("overview", 20*time.Second, buildOverview)
|
|
if err != nil {
|
|
fail(c, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
c.JSON(http.StatusOK, gin.H{"asOf": at.Unix(), "data": data})
|
|
}
|
|
|
|
// ---- heavy statistics (background) -----------------------------------------
|
|
|
|
type tableInfo struct {
|
|
Name string `json:"name"`
|
|
Rows int64 `json:"rows"`
|
|
DeadRows int64 `json:"deadRows"`
|
|
SizeBytes int64 `json:"sizeBytes"`
|
|
}
|
|
|
|
type heavyStats struct {
|
|
AsOf int64 `json:"asOf"`
|
|
FilesLive int64 `json:"filesLive"`
|
|
FilesSoftDeleted int64 `json:"filesSoftDeleted"`
|
|
FilesApprox bool `json:"filesApprox"`
|
|
DistinctHashes int64 `json:"distinctHashes"`
|
|
DistinctAsOf int64 `json:"distinctAsOf"`
|
|
DBDriver string `json:"dbDriver"`
|
|
DBSizeBytes int64 `json:"dbSizeBytes"`
|
|
Tables []tableInfo `json:"tables"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
var (
|
|
heavyMu sync.Mutex
|
|
heavy *heavyStats
|
|
refreshCh = make(chan struct{}, 1)
|
|
)
|
|
|
|
// currentHeavy returns a copy of the table statistics with the distinct-hash
|
|
// count from the duplicate job, or nil before the first run.
|
|
func currentHeavy() *heavyStats {
|
|
heavyMu.Lock()
|
|
defer heavyMu.Unlock()
|
|
if heavy == nil {
|
|
return nil
|
|
}
|
|
h := *heavy
|
|
d := currentDups()
|
|
h.DistinctHashes, h.DistinctAsOf = d.DistinctHashes, d.AsOf
|
|
return &h
|
|
}
|
|
|
|
// requestHeavyRefresh recomputes the table statistics soon.
|
|
func requestHeavyRefresh() {
|
|
select {
|
|
case refreshCh <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
|
|
func heavyStatsLoop(ctx context.Context) {
|
|
run := func() {
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
log.Printf("admin: table stats panicked: %v", r)
|
|
}
|
|
}()
|
|
computeHeavy()
|
|
}
|
|
// give the server a moment to start before the first scan
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-time.After(5 * time.Second):
|
|
}
|
|
// the slow duplicate scan runs in its own loop so it never delays these
|
|
go dupLoop(ctx)
|
|
requestDupRefresh()
|
|
run()
|
|
ticker := time.NewTicker(10 * time.Minute)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
run()
|
|
case <-refreshCh:
|
|
run()
|
|
}
|
|
}
|
|
}
|
|
|
|
// withTimeout runs fn in a transaction with a statement timeout on Postgres.
|
|
func withTimeout(seconds int, fn func(tx *gorm.DB) error) error {
|
|
return initializers.DB.Transaction(func(tx *gorm.DB) error {
|
|
if isPostgres() {
|
|
if err := tx.Exec("SET LOCAL statement_timeout = " + strconv.Itoa(seconds*1000)).Error; err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return fn(tx)
|
|
})
|
|
}
|
|
|
|
var ourTables = []string{"addons", "addon_files", "addon_dependencies", "whitelisted_hashes",
|
|
"metric_series", "metric_points", "service_instances", "service_events", "daily_top_items"}
|
|
|
|
// computeHeavy refreshes table sizes and file row counts. The duplicate-hash
|
|
// statistics are computed separately by runDupJob.
|
|
func computeHeavy() {
|
|
started := time.Now()
|
|
h := &heavyStats{AsOf: time.Now().Unix(), DBDriver: initializers.DB.Dialector.Name(), Tables: []tableInfo{}}
|
|
|
|
var errs []string
|
|
note := func(what string, err error) {
|
|
if err != nil {
|
|
errs = append(errs, what+": "+err.Error())
|
|
}
|
|
}
|
|
|
|
if isPostgres() {
|
|
note("tables", initializers.DB.Raw(`SELECT relname AS name, n_live_tup AS rows, n_dead_tup AS dead_rows,
|
|
pg_total_relation_size(relid) AS size_bytes FROM pg_stat_user_tables ORDER BY size_bytes DESC`).Scan(&h.Tables).Error)
|
|
note("db size", initializers.DB.Raw(`SELECT pg_database_size(current_database())`).Scan(&h.DBSizeBytes).Error)
|
|
note("soft-deleted files", withTimeout(60, func(tx *gorm.DB) error {
|
|
return tx.Raw(`SELECT COUNT(*) FROM addon_files WHERE deleted_at IS NOT NULL`).Scan(&h.FilesSoftDeleted).Error
|
|
}))
|
|
for _, t := range h.Tables {
|
|
if t.Name == "addon_files" {
|
|
h.FilesLive = max(0, t.Rows-h.FilesSoftDeleted)
|
|
h.FilesApprox = true
|
|
}
|
|
}
|
|
} else {
|
|
var pageCount, pageSize int64
|
|
initializers.DB.Raw("PRAGMA page_count").Scan(&pageCount)
|
|
initializers.DB.Raw("PRAGMA page_size").Scan(&pageSize)
|
|
h.DBSizeBytes = pageCount * pageSize
|
|
if st, err := os.Stat(sqlitePath() + "-wal"); err == nil {
|
|
h.DBSizeBytes += st.Size()
|
|
}
|
|
for _, name := range ourTables {
|
|
var n int64
|
|
if err := initializers.DB.Table(name).Count(&n).Error; err == nil {
|
|
h.Tables = append(h.Tables, tableInfo{Name: name, Rows: n})
|
|
}
|
|
}
|
|
note("files", initializers.DB.Raw(`SELECT
|
|
COALESCE(SUM(CASE WHEN deleted_at IS NULL THEN 1 ELSE 0 END), 0),
|
|
COALESCE(SUM(CASE WHEN deleted_at IS NOT NULL THEN 1 ELSE 0 END), 0) FROM addon_files`).Row().
|
|
Scan(&h.FilesLive, &h.FilesSoftDeleted))
|
|
}
|
|
|
|
if len(errs) > 0 {
|
|
h.Error = errs[0]
|
|
for _, e := range errs {
|
|
log.Printf("admin: table stats: %s", e)
|
|
}
|
|
}
|
|
heavyMu.Lock()
|
|
heavy = h
|
|
heavyMu.Unlock()
|
|
invalidate("overview")
|
|
if time.Since(started) > 10*time.Second {
|
|
log.Printf("admin: table stats took %s", time.Since(started).Round(time.Millisecond))
|
|
}
|
|
}
|
|
|
|
func sqlitePath() string {
|
|
if initializers.SQLitePath != "" {
|
|
return initializers.SQLitePath
|
|
}
|
|
return "crawler.db"
|
|
}
|