package admin import ( "context" "fmt" "log" "net/http" "strings" "sync" "time" "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" "github.com/gin-gonic/gin" "gorm.io/gorm" ) type dupHash struct { Hash string `json:"hash"` AddonCount int64 `json:"addonCount"` FileCount int64 `json:"fileCount"` SamplePath string `json:"samplePath"` Whitelisted bool `json:"whitelisted"` } // dupState is the state of the background job that finds the hashes shared by // the most addons. It runs separately from the cheap table statistics because // it scans every live file row. type dupState struct { Running bool StartedAt int64 AsOf int64 // last successful run TookMs int64 Error string ErrorAt int64 DistinctHashes int64 SharedHashes int64 Items []dupHash } const ( dupTopN = 500 // keep more than shown so hiding whitelisted ones still leaves a full list dupShown = 200 dupTimeoutSeconds = 15 * 60 ) var ( dupMu sync.Mutex dups dupState dupReqCh = make(chan struct{}, 1) ) func currentDups() dupState { dupMu.Lock() defer dupMu.Unlock() return dups } // requestDupRefresh asks the duplicate job to run as soon as possible. func requestDupRefresh() { select { case dupReqCh <- struct{}{}: default: } } // dupLoop runs the duplicate job on request and hourly. func dupLoop(ctx context.Context) { ticker := time.NewTicker(time.Hour) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: runDupJob() case <-dupReqCh: runDupJob() } } } // runDupJob computes the duplicate statistics. Errors and panics are recorded // for the dashboard, so the state never stays "running". func runDupJob() { dupMu.Lock() if dups.Running { dupMu.Unlock() return } dups.Running = true dups.StartedAt = time.Now().Unix() dupMu.Unlock() started := time.Now() log.Printf("admin: duplicate hashes: computing") var ( items []dupHash distinct, shared int64 err error ) func() { defer func() { if r := recover(); r != nil { err = fmt.Errorf("panic: %v", r) } }() items, distinct, shared, err = computeDuplicates() }() took := time.Since(started) dupMu.Lock() dups.Running = false dups.TookMs = took.Milliseconds() if err != nil { dups.Error, dups.ErrorAt = err.Error(), time.Now().Unix() } else { dups.Error, dups.ErrorAt = "", 0 dups.AsOf = time.Now().Unix() dups.Items, dups.DistinctHashes, dups.SharedHashes = items, distinct, shared } dupMu.Unlock() invalidate("overview") if err != nil { log.Printf("admin: duplicate hashes: failed after %s: %v", took.Round(time.Millisecond), err) } else { log.Printf("admin: duplicate hashes: computed in %s (%d shared of %d distinct)", took.Round(time.Millisecond), shared, distinct) } } // computeDuplicates scans the live file rows once. The inner DISTINCT over // (hash, addon_id) can be answered from idx_addon_files_hash_addon_optimized, // and the window functions return the totals alongside the top hashes. func computeDuplicates() ([]dupHash, int64, int64, error) { var rows []struct { Hash string AddonCount int64 DistinctHashes int64 SharedHashes int64 } err := withTimeout(dupTimeoutSeconds, func(tx *gorm.DB) error { return tx.Raw(`SELECT hash, addon_count, COUNT(*) OVER () AS distinct_hashes, SUM(CASE WHEN addon_count > 1 THEN 1 ELSE 0 END) OVER () AS shared_hashes FROM (SELECT hash, COUNT(*) AS addon_count FROM (SELECT DISTINCT hash, addon_id FROM addon_files WHERE deleted_at IS NULL) d GROUP BY hash) g ORDER BY addon_count DESC, hash ASC LIMIT ?`, dupTopN).Scan(&rows).Error }) if err != nil { if strings.Contains(err.Error(), "statement timeout") { err = fmt.Errorf("query timed out after %d minutes: %w", dupTimeoutSeconds/60, err) } return nil, 0, 0, err } items := []dupHash{} var distinct, shared int64 for _, r := range rows { distinct, shared = r.DistinctHashes, r.SharedHashes if r.AddonCount > 1 { items = append(items, dupHash{Hash: r.Hash, AddonCount: r.AddonCount}) } } if len(items) == 0 { return items, distinct, shared, nil } hashes := make([]string, len(items)) for i, d := range items { hashes[i] = d.Hash } var details []struct { Hash string Files int64 Path string } if err := initializers.DB.Raw(`SELECT hash, COUNT(*) AS files, MIN(path) AS path FROM addon_files WHERE deleted_at IS NULL AND hash IN ? GROUP BY hash`, hashes).Scan(&details).Error; err != nil { return nil, 0, 0, fmt.Errorf("file details: %w", err) } byHash := make(map[string]int, len(details)) for i, d := range details { byHash[d.Hash] = i } for i := range items { if j, ok := byHash[items[i].Hash]; ok { items[i].FileCount, items[i].SamplePath = details[j].Files, details[j].Path } } return items, distinct, shared, nil } // duplicateHashes lists the hashes shared by the most addons together with // the state of the job that computes them. func duplicateHashes(c *gin.Context) { d := currentDups() items := make([]dupHash, len(d.Items)) copy(items, d.Items) if len(items) > 0 { hashes := make([]string, len(items)) for i, it := range items { hashes[i] = it.Hash } var wl []string initializers.DB.Model(&models.WhitelistedHash{}).Where("hash IN ?", hashes).Pluck("hash", &wl) set := make(map[string]bool, len(wl)) for _, w := range wl { set[w] = true } for i := range items { items[i].Whitelisted = set[items[i].Hash] } } if c.Query("hideWhitelisted") == "1" { kept := items[:0] for _, it := range items { if !it.Whitelisted { kept = append(kept, it) } } items = kept } if len(items) > dupShown { items = items[:dupShown] } c.JSON(http.StatusOK, gin.H{ "running": d.Running, "startedAt": d.StartedAt, "asOf": d.AsOf, "tookMs": d.TookMs, "error": d.Error, "errorAt": d.ErrorAt, "distinctHashes": d.DistinctHashes, "sharedHashes": d.SharedHashes, "items": items, }) } func refreshDuplicates(c *gin.Context) { requestDupRefresh() audit(c, "duplicates_refresh", nil) c.JSON(http.StatusAccepted, gin.H{"started": true}) }