reforger_crawler_main/admin/duplicates.go

package admin

import (
	"context"
	"fmt"
	"log"
	"net/http"
	"strings"
	"sync"
	"time"

	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
	"github.com/gin-gonic/gin"
	"gorm.io/gorm"
)

type dupHash struct {
	Hash        string `json:"hash"`
	AddonCount  int64  `json:"addonCount"`
	FileCount   int64  `json:"fileCount"`
	SamplePath  string `json:"samplePath"`
	Whitelisted bool   `json:"whitelisted"`
}

// dupState is the state of the background job that finds the hashes shared by
// the most addons. It runs separately from the cheap table statistics because
// it scans every live file row.
type dupState struct {
	Running        bool
	StartedAt      int64
	AsOf           int64 // last successful run
	TookMs         int64
	Error          string
	ErrorAt        int64
	DistinctHashes int64
	SharedHashes   int64
	Items          []dupHash
}

const (
	dupTopN           = 500 // keep more than shown so hiding whitelisted ones still leaves a full list
	dupShown          = 200
	dupTimeoutSeconds = 15 * 60
)

var (
	dupMu    sync.Mutex
	dups     dupState
	dupReqCh = make(chan struct{}, 1)
)

func currentDups() dupState {
	dupMu.Lock()
	defer dupMu.Unlock()
	return dups
}

// requestDupRefresh asks the duplicate job to run as soon as possible.
func requestDupRefresh() {
	select {
	case dupReqCh <- struct{}{}:
	default:
	}
}

// dupLoop runs the duplicate job on request and hourly.
func dupLoop(ctx context.Context) {
	ticker := time.NewTicker(time.Hour)
	defer ticker.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			runDupJob()
		case <-dupReqCh:
			runDupJob()
		}
	}
}

// runDupJob computes the duplicate statistics. Errors and panics are recorded
// for the dashboard, so the state never stays "running".
func runDupJob() {
	dupMu.Lock()
	if dups.Running {
		dupMu.Unlock()
		return
	}
	dups.Running = true
	dups.StartedAt = time.Now().Unix()
	dupMu.Unlock()

	started := time.Now()
	log.Printf("admin: duplicate hashes: computing")
	var (
		items            []dupHash
		distinct, shared int64
		err              error
	)
	func() {
		defer func() {
			if r := recover(); r != nil {
				err = fmt.Errorf("panic: %v", r)
			}
		}()
		items, distinct, shared, err = computeDuplicates()
	}()
	took := time.Since(started)

	dupMu.Lock()
	dups.Running = false
	dups.TookMs = took.Milliseconds()
	if err != nil {
		dups.Error, dups.ErrorAt = err.Error(), time.Now().Unix()
	} else {
		dups.Error, dups.ErrorAt = "", 0
		dups.AsOf = time.Now().Unix()
		dups.Items, dups.DistinctHashes, dups.SharedHashes = items, distinct, shared
	}
	dupMu.Unlock()
	invalidate("overview")

	if err != nil {
		log.Printf("admin: duplicate hashes: failed after %s: %v", took.Round(time.Millisecond), err)
	} else {
		log.Printf("admin: duplicate hashes: computed in %s (%d shared of %d distinct)", took.Round(time.Millisecond), shared, distinct)
	}
}

// computeDuplicates scans the live file rows once. The inner DISTINCT over
// (hash, addon_id) can be answered from idx_addon_files_hash_addon_optimized,
// and the window functions return the totals alongside the top hashes.
func computeDuplicates() ([]dupHash, int64, int64, error) {
	var rows []struct {
		Hash           string
		AddonCount     int64
		DistinctHashes int64
		SharedHashes   int64
	}
	err := withTimeout(dupTimeoutSeconds, func(tx *gorm.DB) error {
		return tx.Raw(`SELECT hash, addon_count,
				COUNT(*) OVER () AS distinct_hashes,
				SUM(CASE WHEN addon_count > 1 THEN 1 ELSE 0 END) OVER () AS shared_hashes
			FROM (SELECT hash, COUNT(*) AS addon_count
				FROM (SELECT DISTINCT hash, addon_id FROM addon_files WHERE deleted_at IS NULL) d
				GROUP BY hash) g
			ORDER BY addon_count DESC, hash ASC
			LIMIT ?`, dupTopN).Scan(&rows).Error
	})
	if err != nil {
		if strings.Contains(err.Error(), "statement timeout") {
			err = fmt.Errorf("query timed out after %d minutes: %w", dupTimeoutSeconds/60, err)
		}
		return nil, 0, 0, err
	}

	items := []dupHash{}
	var distinct, shared int64
	for _, r := range rows {
		distinct, shared = r.DistinctHashes, r.SharedHashes
		if r.AddonCount > 1 {
			items = append(items, dupHash{Hash: r.Hash, AddonCount: r.AddonCount})
		}
	}
	if len(items) == 0 {
		return items, distinct, shared, nil
	}

	hashes := make([]string, len(items))
	for i, d := range items {
		hashes[i] = d.Hash
	}
	var details []struct {
		Hash  string
		Files int64
		Path  string
	}
	if err := initializers.DB.Raw(`SELECT hash, COUNT(*) AS files, MIN(path) AS path FROM addon_files
		WHERE deleted_at IS NULL AND hash IN ? GROUP BY hash`, hashes).Scan(&details).Error; err != nil {
		return nil, 0, 0, fmt.Errorf("file details: %w", err)
	}
	byHash := make(map[string]int, len(details))
	for i, d := range details {
		byHash[d.Hash] = i
	}
	for i := range items {
		if j, ok := byHash[items[i].Hash]; ok {
			items[i].FileCount, items[i].SamplePath = details[j].Files, details[j].Path
		}
	}
	return items, distinct, shared, nil
}

// duplicateHashes lists the hashes shared by the most addons together with
// the state of the job that computes them.
func duplicateHashes(c *gin.Context) {
	d := currentDups()
	items := make([]dupHash, len(d.Items))
	copy(items, d.Items)
	if len(items) > 0 {
		hashes := make([]string, len(items))
		for i, it := range items {
			hashes[i] = it.Hash
		}
		var wl []string
		initializers.DB.Model(&models.WhitelistedHash{}).Where("hash IN ?", hashes).Pluck("hash", &wl)
		set := make(map[string]bool, len(wl))
		for _, w := range wl {
			set[w] = true
		}
		for i := range items {
			items[i].Whitelisted = set[items[i].Hash]
		}
	}
	if c.Query("hideWhitelisted") == "1" {
		kept := items[:0]
		for _, it := range items {
			if !it.Whitelisted {
				kept = append(kept, it)
			}
		}
		items = kept
	}
	if len(items) > dupShown {
		items = items[:dupShown]
	}
	c.JSON(http.StatusOK, gin.H{
		"running":        d.Running,
		"startedAt":      d.StartedAt,
		"asOf":           d.AsOf,
		"tookMs":         d.TookMs,
		"error":          d.Error,
		"errorAt":        d.ErrorAt,
		"distinctHashes": d.DistinctHashes,
		"sharedHashes":   d.SharedHashes,
		"items":          items,
	})
}

func refreshDuplicates(c *gin.Context) {
	requestDupRefresh()
	audit(c, "duplicates_refresh", nil)
	c.JSON(http.StatusAccepted, gin.H{"started": true})
}