shared hashes: separate background job with visible state

- compute duplicate-hash statistics in their own job (startup, hourly,
  on demand) so they no longer block the table stats
- single-pass query over DISTINCT (hash, addon_id), 15 min timeout
- record errors and panics; never stay stuck in a computing state
- whitelist page polls while computing, shows errors and a Recompute
  button; tab is linkable via ?tab=shared

admin/duplicates_test.go

@@ -0,0 +1,90 @@
+package admin
+
+import (
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
+	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+)
+
+func setupDupDB(t *testing.T) {
+	t.Helper()
+	db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "d.db")), &gorm.Config{Logger: logger.Discard})
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		if sqlDB, err := db.DB(); err == nil {
+			sqlDB.Close()
+		}
+	})
+	if err := db.AutoMigrate(&models.AddonFile{}, &models.WhitelistedHash{}); err != nil {
+		t.Fatal(err)
+	}
+	initializers.DB = db
+	dupMu.Lock()
+	dups = dupState{}
+	dupMu.Unlock()
+}
+
+func hash(c string) string { return strings.Repeat(c, 128) }
+
+func TestDuplicateJob(t *testing.T) {
+	setupDupDB(t)
+	files := []models.AddonFile{
+		// hash a: in three addons (twice in addon 1)
+		{AddonID: "1", Hash: hash("a"), Path: "x/a.et"},
+		{AddonID: "1", Hash: hash("a"), Path: "y/a.et"},
+		{AddonID: "2", Hash: hash("a"), Path: "a.et"},
+		{AddonID: "3", Hash: hash("a"), Path: "a.et"},
+		// hash b: only in one addon
+		{AddonID: "1", Hash: hash("b"), Path: "b.et"},
+		// hash c: in two addons, but one row is soft-deleted
+		{AddonID: "1", Hash: hash("c"), Path: "c.et"},
+		{AddonID: "2", Hash: hash("c"), Path: "c.et"},
+	}
+	if err := initializers.DB.Create(&files).Error; err != nil {
+		t.Fatal(err)
+	}
+	initializers.DB.Where("addon_id = ? AND hash = ?", "2", hash("c")).Delete(&models.AddonFile{})
+
+	runDupJob()
+	d := currentDups()
+	if d.Running || d.Error != "" || d.AsOf == 0 {
+		t.Fatalf("unexpected state %+v", d)
+	}
+	if d.DistinctHashes != 3 || d.SharedHashes != 1 {
+		t.Fatalf("distinct=%d shared=%d, want 3 and 1", d.DistinctHashes, d.SharedHashes)
+	}
+	if len(d.Items) != 1 {
+		t.Fatalf("items = %+v, want only hash a", d.Items)
+	}
+	it := d.Items[0]
+	if it.Hash != hash("a") || it.AddonCount != 3 || it.FileCount != 4 || it.SamplePath != "a.et" {
+		t.Fatalf("unexpected item %+v", it)
+	}
+}
+
+func TestDuplicateJobRecordsErrors(t *testing.T) {
+	setupDupDB(t)
+	if err := initializers.DB.Migrator().DropTable(&models.AddonFile{}); err != nil {
+		t.Fatal(err)
+	}
+	runDupJob()
+	d := currentDups()
+	if d.Running {
+		t.Fatal("job must not stay running after a failure")
+	}
+	if d.Error == "" || d.ErrorAt == 0 || d.AsOf != 0 {
+		t.Fatalf("error not recorded: %+v", d)
+	}
+	if time.Now().Unix()-d.ErrorAt > 5 {
+		t.Fatalf("errorAt not set to now: %d", d.ErrorAt)
+	}
+}