shared hashes: separate background job with visible state
Cross Compile Go / build (push) Canceled after 0s
Cross Compile Go / build (push) Canceled after 0s
- compute duplicate-hash statistics in their own job (startup, hourly, on demand) so they no longer block the table stats - single-pass query over DISTINCT (hash, addon_id), 15 min timeout - record errors and panics; never stay stuck in a computing state - whitelist page polls while computing, shows errors and a Recompute button; tab is linkable via ?tab=shared
This commit is contained in:
ilbinek committed
1 parent
56d9e10621
commit
748db94d55
8 files changed
+436
-127
No files matched your search
@@ -0,0 +1,246 @@
|
||||
package admin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
|
||||
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
|
||||
"github.com/gin-gonic/gin"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type dupHash struct {
|
||||
Hash string `json:"hash"`
|
||||
AddonCount int64 `json:"addonCount"`
|
||||
FileCount int64 `json:"fileCount"`
|
||||
SamplePath string `json:"samplePath"`
|
||||
Whitelisted bool `json:"whitelisted"`
|
||||
}
|
||||
|
||||
// dupState is the state of the background job that finds the hashes shared by
|
||||
// the most addons. It runs separately from the cheap table statistics because
|
||||
// it scans every live file row.
|
||||
type dupState struct {
|
||||
Running bool
|
||||
StartedAt int64
|
||||
AsOf int64 // last successful run
|
||||
TookMs int64
|
||||
Error string
|
||||
ErrorAt int64
|
||||
DistinctHashes int64
|
||||
SharedHashes int64
|
||||
Items []dupHash
|
||||
}
|
||||
|
||||
const (
|
||||
dupTopN = 500 // keep more than shown so hiding whitelisted ones still leaves a full list
|
||||
dupShown = 200
|
||||
dupTimeoutSeconds = 15 * 60
|
||||
)
|
||||
|
||||
var (
|
||||
dupMu sync.Mutex
|
||||
dups dupState
|
||||
dupReqCh = make(chan struct{}, 1)
|
||||
)
|
||||
|
||||
func currentDups() dupState {
|
||||
dupMu.Lock()
|
||||
defer dupMu.Unlock()
|
||||
return dups
|
||||
}
|
||||
|
||||
// requestDupRefresh asks the duplicate job to run as soon as possible.
|
||||
func requestDupRefresh() {
|
||||
select {
|
||||
case dupReqCh <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// dupLoop runs the duplicate job on request and hourly.
|
||||
func dupLoop(ctx context.Context) {
|
||||
ticker := time.NewTicker(time.Hour)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
runDupJob()
|
||||
case <-dupReqCh:
|
||||
runDupJob()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// runDupJob computes the duplicate statistics. Errors and panics are recorded
|
||||
// for the dashboard, so the state never stays "running".
|
||||
func runDupJob() {
|
||||
dupMu.Lock()
|
||||
if dups.Running {
|
||||
dupMu.Unlock()
|
||||
return
|
||||
}
|
||||
dups.Running = true
|
||||
dups.StartedAt = time.Now().Unix()
|
||||
dupMu.Unlock()
|
||||
|
||||
started := time.Now()
|
||||
log.Printf("admin: duplicate hashes: computing")
|
||||
var (
|
||||
items []dupHash
|
||||
distinct, shared int64
|
||||
err error
|
||||
)
|
||||
func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
err = fmt.Errorf("panic: %v", r)
|
||||
}
|
||||
}()
|
||||
items, distinct, shared, err = computeDuplicates()
|
||||
}()
|
||||
took := time.Since(started)
|
||||
|
||||
dupMu.Lock()
|
||||
dups.Running = false
|
||||
dups.TookMs = took.Milliseconds()
|
||||
if err != nil {
|
||||
dups.Error, dups.ErrorAt = err.Error(), time.Now().Unix()
|
||||
} else {
|
||||
dups.Error, dups.ErrorAt = "", 0
|
||||
dups.AsOf = time.Now().Unix()
|
||||
dups.Items, dups.DistinctHashes, dups.SharedHashes = items, distinct, shared
|
||||
}
|
||||
dupMu.Unlock()
|
||||
invalidate("overview")
|
||||
|
||||
if err != nil {
|
||||
log.Printf("admin: duplicate hashes: failed after %s: %v", took.Round(time.Millisecond), err)
|
||||
} else {
|
||||
log.Printf("admin: duplicate hashes: computed in %s (%d shared of %d distinct)", took.Round(time.Millisecond), shared, distinct)
|
||||
}
|
||||
}
|
||||
|
||||
// computeDuplicates scans the live file rows once. The inner DISTINCT over
|
||||
// (hash, addon_id) can be answered from idx_addon_files_hash_addon_optimized,
|
||||
// and the window functions return the totals alongside the top hashes.
|
||||
func computeDuplicates() ([]dupHash, int64, int64, error) {
|
||||
var rows []struct {
|
||||
Hash string
|
||||
AddonCount int64
|
||||
DistinctHashes int64
|
||||
SharedHashes int64
|
||||
}
|
||||
err := withTimeout(dupTimeoutSeconds, func(tx *gorm.DB) error {
|
||||
return tx.Raw(`SELECT hash, addon_count,
|
||||
COUNT(*) OVER () AS distinct_hashes,
|
||||
SUM(CASE WHEN addon_count > 1 THEN 1 ELSE 0 END) OVER () AS shared_hashes
|
||||
FROM (SELECT hash, COUNT(*) AS addon_count
|
||||
FROM (SELECT DISTINCT hash, addon_id FROM addon_files WHERE deleted_at IS NULL) d
|
||||
GROUP BY hash) g
|
||||
ORDER BY addon_count DESC, hash ASC
|
||||
LIMIT ?`, dupTopN).Scan(&rows).Error
|
||||
})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "statement timeout") {
|
||||
err = fmt.Errorf("query timed out after %d minutes: %w", dupTimeoutSeconds/60, err)
|
||||
}
|
||||
return nil, 0, 0, err
|
||||
}
|
||||
|
||||
items := []dupHash{}
|
||||
var distinct, shared int64
|
||||
for _, r := range rows {
|
||||
distinct, shared = r.DistinctHashes, r.SharedHashes
|
||||
if r.AddonCount > 1 {
|
||||
items = append(items, dupHash{Hash: r.Hash, AddonCount: r.AddonCount})
|
||||
}
|
||||
}
|
||||
if len(items) == 0 {
|
||||
return items, distinct, shared, nil
|
||||
}
|
||||
|
||||
hashes := make([]string, len(items))
|
||||
for i, d := range items {
|
||||
hashes[i] = d.Hash
|
||||
}
|
||||
var details []struct {
|
||||
Hash string
|
||||
Files int64
|
||||
Path string
|
||||
}
|
||||
if err := initializers.DB.Raw(`SELECT hash, COUNT(*) AS files, MIN(path) AS path FROM addon_files
|
||||
WHERE deleted_at IS NULL AND hash IN ? GROUP BY hash`, hashes).Scan(&details).Error; err != nil {
|
||||
return nil, 0, 0, fmt.Errorf("file details: %w", err)
|
||||
}
|
||||
byHash := make(map[string]int, len(details))
|
||||
for i, d := range details {
|
||||
byHash[d.Hash] = i
|
||||
}
|
||||
for i := range items {
|
||||
if j, ok := byHash[items[i].Hash]; ok {
|
||||
items[i].FileCount, items[i].SamplePath = details[j].Files, details[j].Path
|
||||
}
|
||||
}
|
||||
return items, distinct, shared, nil
|
||||
}
|
||||
|
||||
// duplicateHashes lists the hashes shared by the most addons together with
|
||||
// the state of the job that computes them.
|
||||
func duplicateHashes(c *gin.Context) {
|
||||
d := currentDups()
|
||||
items := make([]dupHash, len(d.Items))
|
||||
copy(items, d.Items)
|
||||
if len(items) > 0 {
|
||||
hashes := make([]string, len(items))
|
||||
for i, it := range items {
|
||||
hashes[i] = it.Hash
|
||||
}
|
||||
var wl []string
|
||||
initializers.DB.Model(&models.WhitelistedHash{}).Where("hash IN ?", hashes).Pluck("hash", &wl)
|
||||
set := make(map[string]bool, len(wl))
|
||||
for _, w := range wl {
|
||||
set[w] = true
|
||||
}
|
||||
for i := range items {
|
||||
items[i].Whitelisted = set[items[i].Hash]
|
||||
}
|
||||
}
|
||||
if c.Query("hideWhitelisted") == "1" {
|
||||
kept := items[:0]
|
||||
for _, it := range items {
|
||||
if !it.Whitelisted {
|
||||
kept = append(kept, it)
|
||||
}
|
||||
}
|
||||
items = kept
|
||||
}
|
||||
if len(items) > dupShown {
|
||||
items = items[:dupShown]
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"running": d.Running,
|
||||
"startedAt": d.StartedAt,
|
||||
"asOf": d.AsOf,
|
||||
"tookMs": d.TookMs,
|
||||
"error": d.Error,
|
||||
"errorAt": d.ErrorAt,
|
||||
"distinctHashes": d.DistinctHashes,
|
||||
"sharedHashes": d.SharedHashes,
|
||||
"items": items,
|
||||
})
|
||||
}
|
||||
|
||||
func refreshDuplicates(c *gin.Context) {
|
||||
requestDupRefresh()
|
||||
audit(c, "duplicates_refresh", nil)
|
||||
c.JSON(http.StatusAccepted, gin.H{"started": true})
|
||||
}
|
||||
Reference in new issue
Block a user