ilbinek
748db94d55
Cross Compile Go / build (push) Canceled after 0s
- compute duplicate-hash statistics in their own job (startup, hourly, on demand) so they no longer block the table stats - single-pass query over DISTINCT (hash, addon_id), 15 min timeout - record errors and panics; never stay stuck in a computing state - whitelist page polls while computing, shows errors and a Recompute button; tab is linkable via ?tab=shared
91 lines
2.4 KiB
Go
91 lines
2.4 KiB
Go
package admin
|
|
|
|
import (
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
|
|
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
|
|
"gorm.io/driver/sqlite"
|
|
"gorm.io/gorm"
|
|
"gorm.io/gorm/logger"
|
|
)
|
|
|
|
func setupDupDB(t *testing.T) {
|
|
t.Helper()
|
|
db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "d.db")), &gorm.Config{Logger: logger.Discard})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() {
|
|
if sqlDB, err := db.DB(); err == nil {
|
|
sqlDB.Close()
|
|
}
|
|
})
|
|
if err := db.AutoMigrate(&models.AddonFile{}, &models.WhitelistedHash{}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
initializers.DB = db
|
|
dupMu.Lock()
|
|
dups = dupState{}
|
|
dupMu.Unlock()
|
|
}
|
|
|
|
func hash(c string) string { return strings.Repeat(c, 128) }
|
|
|
|
func TestDuplicateJob(t *testing.T) {
|
|
setupDupDB(t)
|
|
files := []models.AddonFile{
|
|
// hash a: in three addons (twice in addon 1)
|
|
{AddonID: "1", Hash: hash("a"), Path: "x/a.et"},
|
|
{AddonID: "1", Hash: hash("a"), Path: "y/a.et"},
|
|
{AddonID: "2", Hash: hash("a"), Path: "a.et"},
|
|
{AddonID: "3", Hash: hash("a"), Path: "a.et"},
|
|
// hash b: only in one addon
|
|
{AddonID: "1", Hash: hash("b"), Path: "b.et"},
|
|
// hash c: in two addons, but one row is soft-deleted
|
|
{AddonID: "1", Hash: hash("c"), Path: "c.et"},
|
|
{AddonID: "2", Hash: hash("c"), Path: "c.et"},
|
|
}
|
|
if err := initializers.DB.Create(&files).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
initializers.DB.Where("addon_id = ? AND hash = ?", "2", hash("c")).Delete(&models.AddonFile{})
|
|
|
|
runDupJob()
|
|
d := currentDups()
|
|
if d.Running || d.Error != "" || d.AsOf == 0 {
|
|
t.Fatalf("unexpected state %+v", d)
|
|
}
|
|
if d.DistinctHashes != 3 || d.SharedHashes != 1 {
|
|
t.Fatalf("distinct=%d shared=%d, want 3 and 1", d.DistinctHashes, d.SharedHashes)
|
|
}
|
|
if len(d.Items) != 1 {
|
|
t.Fatalf("items = %+v, want only hash a", d.Items)
|
|
}
|
|
it := d.Items[0]
|
|
if it.Hash != hash("a") || it.AddonCount != 3 || it.FileCount != 4 || it.SamplePath != "a.et" {
|
|
t.Fatalf("unexpected item %+v", it)
|
|
}
|
|
}
|
|
|
|
func TestDuplicateJobRecordsErrors(t *testing.T) {
|
|
setupDupDB(t)
|
|
if err := initializers.DB.Migrator().DropTable(&models.AddonFile{}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runDupJob()
|
|
d := currentDups()
|
|
if d.Running {
|
|
t.Fatal("job must not stay running after a failure")
|
|
}
|
|
if d.Error == "" || d.ErrorAt == 0 || d.AsOf != 0 {
|
|
t.Fatalf("error not recorded: %+v", d)
|
|
}
|
|
if time.Now().Unix()-d.ErrorAt > 5 {
|
|
t.Fatalf("errorAt not set to now: %d", d.ErrorAt)
|
|
}
|
|
}
|