shared hashes: separate background job with visible state
Cross Compile Go / build (push) Canceled after 0s
Cross Compile Go / build (push) Canceled after 0s
- compute duplicate-hash statistics in their own job (startup, hourly, on demand) so they no longer block the table stats - single-pass query over DISTINCT (hash, addon_id), 15 min timeout - record errors and panics; never stay stuck in a computing state - whitelist page polls while computing, shows errors and a Recompute button; tab is linkable via ?tab=shared
This commit is contained in:
ilbinek committed
1 parent
56d9e10621
commit
748db94d55
8 files changed
+436
-127
No files matched your search
+28
-114
@@ -199,14 +199,6 @@ type tableInfo struct {
|
||||
SizeBytes int64 `json:"sizeBytes"`
|
||||
}
|
||||
|
||||
type dupHash struct {
|
||||
Hash string `json:"hash"`
|
||||
AddonCount int64 `json:"addonCount"`
|
||||
FileCount int64 `json:"fileCount"`
|
||||
SamplePath string `json:"samplePath"`
|
||||
Whitelisted bool `json:"whitelisted"`
|
||||
}
|
||||
|
||||
type heavyStats struct {
|
||||
AsOf int64 `json:"asOf"`
|
||||
FilesLive int64 `json:"filesLive"`
|
||||
@@ -217,45 +209,45 @@ type heavyStats struct {
|
||||
DBDriver string `json:"dbDriver"`
|
||||
DBSizeBytes int64 `json:"dbSizeBytes"`
|
||||
Tables []tableInfo `json:"tables"`
|
||||
Duplicates []dupHash `json:"-"`
|
||||
DuplicatesAsOf int64 `json:"duplicatesAsOf"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
var (
|
||||
heavyMu sync.Mutex
|
||||
heavy *heavyStats
|
||||
refreshCh = make(chan bool, 1)
|
||||
refreshCh = make(chan struct{}, 1)
|
||||
)
|
||||
|
||||
// currentHeavy returns a copy of the table statistics with the distinct-hash
|
||||
// count from the duplicate job, or nil before the first run.
|
||||
func currentHeavy() *heavyStats {
|
||||
heavyMu.Lock()
|
||||
defer heavyMu.Unlock()
|
||||
return heavy
|
||||
if heavy == nil {
|
||||
return nil
|
||||
}
|
||||
h := *heavy
|
||||
d := currentDups()
|
||||
h.DistinctHashes, h.DistinctAsOf = d.DistinctHashes, d.AsOf
|
||||
return &h
|
||||
}
|
||||
|
||||
// requestHeavyRefresh triggers a refresh; full=true also recomputes the
|
||||
// expensive duplicate statistics.
|
||||
func requestHeavyRefresh(full bool) {
|
||||
// requestHeavyRefresh recomputes the table statistics soon.
|
||||
func requestHeavyRefresh() {
|
||||
select {
|
||||
case refreshCh <- full:
|
||||
case refreshCh <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func heavyStatsLoop(ctx context.Context) {
|
||||
lastFull := time.Time{}
|
||||
run := func(full bool) {
|
||||
run := func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("admin: heavy stats panicked: %v", r)
|
||||
log.Printf("admin: table stats panicked: %v", r)
|
||||
}
|
||||
}()
|
||||
full = full || time.Since(lastFull) > time.Hour
|
||||
computeHeavy(full)
|
||||
if full {
|
||||
lastFull = time.Now()
|
||||
}
|
||||
computeHeavy()
|
||||
}
|
||||
// give the server a moment to start before the first scan
|
||||
select {
|
||||
@@ -263,7 +255,10 @@ func heavyStatsLoop(ctx context.Context) {
|
||||
return
|
||||
case <-time.After(5 * time.Second):
|
||||
}
|
||||
run(true)
|
||||
// the slow duplicate scan runs in its own loop so it never delays these
|
||||
go dupLoop(ctx)
|
||||
requestDupRefresh()
|
||||
run()
|
||||
ticker := time.NewTicker(10 * time.Minute)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
@@ -271,9 +266,9 @@ func heavyStatsLoop(ctx context.Context) {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
run(false)
|
||||
case full := <-refreshCh:
|
||||
run(full)
|
||||
run()
|
||||
case <-refreshCh:
|
||||
run()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -293,14 +288,11 @@ func withTimeout(seconds int, fn func(tx *gorm.DB) error) error {
|
||||
var ourTables = []string{"addons", "addon_files", "addon_dependencies", "whitelisted_hashes",
|
||||
"metric_series", "metric_points", "service_instances", "service_events", "daily_top_items"}
|
||||
|
||||
func computeHeavy(full bool) {
|
||||
// computeHeavy refreshes table sizes and file row counts. The duplicate-hash
|
||||
// statistics are computed separately by runDupJob.
|
||||
func computeHeavy() {
|
||||
started := time.Now()
|
||||
h := &heavyStats{AsOf: time.Now().Unix(), DBDriver: initializers.DB.Dialector.Name(), Tables: []tableInfo{}}
|
||||
prev := currentHeavy()
|
||||
if prev != nil {
|
||||
h.DistinctHashes, h.DistinctAsOf = prev.DistinctHashes, prev.DistinctAsOf
|
||||
h.Duplicates, h.DuplicatesAsOf = prev.Duplicates, prev.DuplicatesAsOf
|
||||
}
|
||||
|
||||
var errs []string
|
||||
note := func(what string, err error) {
|
||||
@@ -342,52 +334,10 @@ func computeHeavy(full bool) {
|
||||
Scan(&h.FilesLive, &h.FilesSoftDeleted))
|
||||
}
|
||||
|
||||
if full {
|
||||
var distinct int64
|
||||
err := withTimeout(120, func(tx *gorm.DB) error {
|
||||
return tx.Raw(`SELECT COUNT(DISTINCT hash) FROM addon_files WHERE deleted_at IS NULL`).Scan(&distinct).Error
|
||||
})
|
||||
note("distinct hashes", err)
|
||||
if err == nil {
|
||||
h.DistinctHashes, h.DistinctAsOf = distinct, time.Now().Unix()
|
||||
}
|
||||
|
||||
var dups []dupHash
|
||||
err = withTimeout(120, func(tx *gorm.DB) error {
|
||||
return tx.Raw(`SELECT hash, COUNT(DISTINCT addon_id) AS addon_count, COUNT(*) AS file_count
|
||||
FROM addon_files WHERE deleted_at IS NULL
|
||||
GROUP BY hash HAVING COUNT(DISTINCT addon_id) > 1
|
||||
ORDER BY addon_count DESC, file_count DESC LIMIT 200`).Scan(&dups).Error
|
||||
})
|
||||
note("duplicates", err)
|
||||
if err == nil {
|
||||
h.Duplicates, h.DuplicatesAsOf = dups, time.Now().Unix()
|
||||
if len(dups) > 0 {
|
||||
hashes := make([]string, len(dups))
|
||||
for i, d := range dups {
|
||||
hashes[i] = d.Hash
|
||||
}
|
||||
var samples []struct {
|
||||
Hash string
|
||||
Path string
|
||||
}
|
||||
initializers.DB.Raw(`SELECT hash, MIN(path) AS path FROM addon_files
|
||||
WHERE deleted_at IS NULL AND hash IN ? GROUP BY hash`, hashes).Scan(&samples)
|
||||
byHash := map[string]string{}
|
||||
for _, s := range samples {
|
||||
byHash[s.Hash] = s.Path
|
||||
}
|
||||
for i := range h.Duplicates {
|
||||
h.Duplicates[i].SamplePath = byHash[h.Duplicates[i].Hash]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(errs) > 0 {
|
||||
h.Error = errs[0]
|
||||
for _, e := range errs {
|
||||
log.Printf("admin: heavy stats: %s", e)
|
||||
log.Printf("admin: table stats: %s", e)
|
||||
}
|
||||
}
|
||||
heavyMu.Lock()
|
||||
@@ -395,7 +345,7 @@ func computeHeavy(full bool) {
|
||||
heavyMu.Unlock()
|
||||
invalidate("overview")
|
||||
if time.Since(started) > 10*time.Second {
|
||||
log.Printf("admin: heavy stats took %s", time.Since(started).Round(time.Millisecond))
|
||||
log.Printf("admin: table stats took %s", time.Since(started).Round(time.Millisecond))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -405,39 +355,3 @@ func sqlitePath() string {
|
||||
}
|
||||
return "crawler.db"
|
||||
}
|
||||
|
||||
// duplicateHashes lists the hashes shared by the most addons.
|
||||
func duplicateHashes(c *gin.Context) {
|
||||
h := currentHeavy()
|
||||
if h == nil {
|
||||
c.JSON(http.StatusOK, gin.H{"asOf": 0, "items": []dupHash{}, "pending": true})
|
||||
return
|
||||
}
|
||||
items := make([]dupHash, len(h.Duplicates))
|
||||
copy(items, h.Duplicates)
|
||||
if len(items) > 0 {
|
||||
hashes := make([]string, len(items))
|
||||
for i, d := range items {
|
||||
hashes[i] = d.Hash
|
||||
}
|
||||
var wl []string
|
||||
initializers.DB.Model(&models.WhitelistedHash{}).Where("hash IN ?", hashes).Pluck("hash", &wl)
|
||||
set := map[string]bool{}
|
||||
for _, w := range wl {
|
||||
set[w] = true
|
||||
}
|
||||
for i := range items {
|
||||
items[i].Whitelisted = set[items[i].Hash]
|
||||
}
|
||||
}
|
||||
if c.Query("hideWhitelisted") == "1" {
|
||||
kept := items[:0]
|
||||
for _, it := range items {
|
||||
if !it.Whitelisted {
|
||||
kept = append(kept, it)
|
||||
}
|
||||
}
|
||||
items = kept
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"asOf": h.DuplicatesAsOf, "items": items})
|
||||
}
|
||||
Reference in new issue
Block a user