TBD/reforger_crawler_main
2
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

admin dashboard, metrics and telemetry
Cross Compile Go / build (push) Canceled after 0s
Details

Browse Source 
- embedded React admin UI at /admin/ui (go:embed), login with ADMIN_SECRET
  via signed HttpOnly session cookie, X-ADMIN-KEY still works for scripts
- admin API: overview stats, indexing queue, addon management (priority,
  reindex, delete/restore/purge, bulk), whitelist CRUD, duplicate hashes,
  traffic, metrics explorer, services, events/audit log, system info
- built-in metrics: request latency per route, visitors, top addons and
  searches, indexer/scraper counters; minute buckets in the DB with hourly
  rollups and retention; optional Prometheus /metrics (METRICS_TOKEN)
- POST /back/telemetry for indexers/scrapers (metrics, events, heartbeat)
  and a stdlib-only Go client in telemetry/
- auth hardening: empty secrets no longer allow access, constant-time
  compare, CORS not applied to /admin and /back, TRUSTED_PROXIES setting
- Dockerfile builds the UI and no longer bakes config.yaml into the image;
  CI builds the UI, runs tests and builds arm64 with CGO
This commit is contained in:
ilbinek committed 2026-10-03 18:10:57 +02:00
1 parent 0ef27b0072
commit 56d9e10621
79 files changed
+13144 -116
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.dockerignore
+2
View File
@@ -9,3 +9,5 @@ api
.env
.env.*
README.md
web/node_modules
web/dist
.env.dokploy.example
+23
View File
@@ -26,3 +26,26 @@ POSTGRES_USER=postgres
POSTGRES_PASSWORD=change-me-password
POSTGRES_DB_NAME=dev
POSTGRES_SSLMODE=disable
# --- Admin dashboard (/admin/ui/) ---------------------------------------------
# Log in with ADMIN_SECRET (use 16+ random characters). Rotating it signs
# everyone out. The X-ADMIN-KEY header keeps working for scripts.
ADMIN_SESSION_TTL_HOURS=168
# auto = Secure cookie when the request arrived over HTTPS (X-Forwarded-Proto)
ADMIN_COOKIE_SECURE=auto
# Proxies whose X-Forwarded-For is trusted (comma separated IPs/CIDRs), e.g. the
# Traefik network. Without it, client IPs (visitor counts, login throttling)
# can be spoofed.
TRUSTED_PROXIES=10.0.0.0/8,172.16.0.0/12
# --- Metrics & telemetry ---------------------------------------------------------
METRICS_ENABLED=true
# Enables GET /metrics (Prometheus) with "Authorization: Bearer <token>".
# Leave empty to disable the endpoint.
METRICS_TOKEN=
METRICS_MINUTE_RETENTION_DAYS=7
METRICS_HOUR_RETENTION_DAYS=180
EVENTS_RETENTION_DAYS=30
TOP_ITEMS_RETENTION_DAYS=90
# Indexing claims older than this go back to the queue
INDEX_TIMEOUT_MINUTES=30
.gitea/workflows/build.yaml
+25 -3
View File
@@ -13,18 +13,40 @@ jobs:
- name: Checkout Code
uses: actions/checkout@v4
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: '22'
cache: npm
cache-dependency-path: web/package-lock.json
# The admin dashboard is embedded into the binary, so build it first
- name: Build admin UI
run: |
npm ci --prefix web
npm run build --prefix web
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: '1.23'
go-version: '1.25'
- name: Vet and test
run: |
go vet ./...
go test ./...
# SQLite needs CGO, so cross-compile ARM64 with the aarch64 C toolchain
- name: Install ARM64 C toolchain
run: sudo apt-get update && sudo apt-get install -y gcc-aarch64-linux-gnu
# Build ARM64
- name: Build ARM64
run: GOOS=linux GOARCH=arm64 go build -v -o crawler_main-linux-arm64 .
run: CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc GOOS=linux GOARCH=arm64 go build -v -ldflags "-s -w -X main.buildCommit=${{ github.sha }}" -o crawler_main-linux-arm64 .
# Build AMD64
- name: Build AMD64
run: GOOS=linux GOARCH=amd64 go build -v -o crawler_main-linux-amd64 .
run: CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -v -ldflags "-s -w -X main.buildCommit=${{ github.sha }}" -o crawler_main-linux-amd64 .
# Create organized archives for download
- name: Bundle Artifacts
.gitignore
+5
View File
@@ -5,3 +5,8 @@ crawler.db
.env
.env.*
!.env.dokploy.example
# admin UI
web/node_modules/
web/dist/*
!web/dist/.gitkeep
Dockerfile
+18 -2
View File
@@ -1,3 +1,14 @@
# Admin dashboard (React) - built once, embedded into the Go binary via go:embed.
FROM --platform=$BUILDPLATFORM node:22-bookworm-slim AS ui
WORKDIR /web
COPY web/package.json web/package-lock.json ./
RUN npm ci
COPY web/ ./
RUN npm run build
FROM golang:1.25-bookworm AS builder
WORKDIR /app
@@ -11,7 +22,10 @@ COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o /out/reforger_crawler_main .
COPY --from=ui /web/dist ./web/dist
ARG GIT_SHA=""
RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -ldflags "-s -w -X main.buildCommit=${GIT_SHA}" -o /out/reforger_crawler_main .
FROM debian:bookworm-slim AS runtime
@@ -22,7 +36,9 @@ RUN apt-get update \
&& rm -rf /var/lib/apt/lists/*
COPY --from=builder /out/reforger_crawler_main /app/reforger_crawler_main
COPY config.yaml /app/config.yaml
# Configuration comes from environment variables (see .env.dokploy.example).
# Mount a config.yaml into /app if you prefer a file.
EXPOSE 8083
admin/addons.go
+510
View File
@@ -0,0 +1,510 @@
package admin
import (
"errors"
"net/http"
"strings"
"time"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/controllers"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
"github.com/gin-gonic/gin"
"gorm.io/gorm"
)
// addonRow is the admin view of an addon (no files).
type addonRow struct {
ID string `json:"id"`
Name string `json:"name"`
Type string `json:"type"`
Author string `json:"author"`
Preview string `json:"preview"`
SubscriberCount int64 `json:"subscriberCount"`
CurrentVersionNumber string `json:"currentVersionNumber"`
CurrentVersionSize int64 `json:"currentVersionSize"`
Unlisted bool `json:"unlisted"`
Private bool `json:"private"`
Blocked bool `json:"blocked"`
ToBeIndexed bool `json:"toBeIndexed"`
IsBeingIndexed bool `json:"isBeingIndexed"`
PriorityIndexing bool `json:"priorityIndexing"`
Recheck bool `json:"recheck"`
IndexStartTime *time.Time `json:"indexStartTime"`
LastIndexedAt *time.Time `json:"lastIndexedAt"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
DeletedAt *time.Time `json:"deletedAt"`
}
const addonColumns = `id, name, type, author, preview, subscriber_count, current_version_number,
current_version_size, unlisted, private, blocked, to_be_indexed, is_being_indexed, priority_indexing,
recheck, index_start_time, last_indexed_at, created_at, updated_at, deleted_at`
var addonSorts = map[string]string{
"name": "name",
"subscribers": "subscriber_count",
"size": "current_version_size",
"created": "created_at",
"updated": "updated_at",
"indexed": "last_indexed_at",
"deleted": "deleted_at",
}
func listAddons(c *gin.Context) {
limit := queryInt(c, "limit", 50, 1, 200)
offset := queryInt(c, "offset", 0, 0, 1<<30)
status := c.DefaultQuery("status", "all")
q := initializers.DB.Model(&models.Addon{})
switch status {
case "all":
case "indexed":
q = q.Where("to_be_indexed = ?", false)
case "queued":
q = q.Where("to_be_indexed = ? AND is_being_indexed = ?", true, false)
case "indexing":
q = q.Where("is_being_indexed = ?", true)
case "priority":
q = q.Where("priority_indexing = ?", true)
case "recheck":
q = q.Where("recheck = ?", true)
case "blocked":
q = q.Where("blocked = ?", true)
case "unlisted":
q = q.Where("unlisted = ?", true)
case "private":
q = q.Where("private = ?", true)
case "deleted":
q = q.Unscoped().Where("deleted_at IS NOT NULL")
default:
fail(c, http.StatusBadRequest, "unknown status")
return
}
if t := c.Query("type"); t != "" {
q = q.Where("type = ?", t)
}
if s := strings.TrimSpace(c.Query("q")); s != "" {
like := "%" + likeEscape(strings.ToLower(s)) + "%"
q = q.Where(`id = ? OR LOWER(name) LIKE ? ESCAPE '\' OR LOWER(author) LIKE ? ESCAPE '\'`, s, like, like)
}
var total int64
if err := q.Session(&gorm.Session{}).Count(&total).Error; err != nil {
fail(c, http.StatusInternalServerError, err.Error())
return
}
sortCol, ok := addonSorts[c.DefaultQuery("sort", "subscribers")]
if !ok {
sortCol = "subscriber_count"
}
dir := "DESC"
if c.Query("order") == "asc" {
dir = "ASC"
}
nulls := ""
if isPostgres() && dir == "DESC" {
nulls = " NULLS LAST"
}
rows := []addonRow{}
err := q.Select(addonColumns).Order(sortCol + " " + dir + nulls + ", id ASC").Limit(limit).Offset(offset).Scan(&rows).Error
if err != nil {
fail(c, http.StatusInternalServerError, err.Error())
return
}
c.JSON(http.StatusOK, gin.H{"items": rows, "total": total, "limit": limit, "offset": offset})
}
type sharedAddon struct {
AddonID string `json:"addonId"`
Name string `json:"name"`
SharedFiles int64 `json:"sharedFiles"`
}
func addonDetail(c *gin.Context) {
id := c.Param("id")
var row addonRow
res := initializers.DB.Unscoped().Model(&models.Addon{}).Select(addonColumns).Where("id = ?", id).Limit(1).Scan(&row)
if res.Error != nil {
fail(c, http.StatusInternalServerError, res.Error.Error())
return
}
if res.RowsAffected == 0 {
fail(c, http.StatusNotFound, "addon not found")
return
}
var extra struct {
Summary string
CurrentVersionID int64
}
initializers.DB.Unscoped().Model(&models.Addon{}).Select("summary, current_version_id").Where("id = ?", id).Limit(1).Scan(&extra)
var fileCount, deletedFiles int64
initializers.DB.Model(&models.AddonFile{}).Where("addon_id = ?", id).Count(&fileCount)
initializers.DB.Unscoped().Model(&models.AddonFile{}).Where("addon_id = ? AND deleted_at IS NOT NULL", id).Count(&deletedFiles)
deps, depsTotal, _ := controllers.QueryRelatedAddons(id, "ad.dependency_id", "ad.addon_id", 50, 0)
dependents, dependentsTotal, _ := controllers.QueryRelatedAddons(id, "ad.addon_id", "ad.dependency_id", 50, 0)
// declared dependencies that are not (or no longer) in the addons table
var missingDeps []struct {
DependencyID string `json:"id"`
DependencyName string `json:"name"`
Version string `json:"version"`
}
initializers.DB.Raw(`SELECT ad.dependency_id, ad.dependency_name, ad.version FROM addon_dependencies ad
LEFT JOIN addons a ON a.id = ad.dependency_id AND a.deleted_at IS NULL
WHERE ad.addon_id = ? AND a.id IS NULL LIMIT 50`, id).Scan(&missingDeps)
shared := []sharedAddon{}
err := initializers.DB.Raw(`SELECT f.addon_id AS addon_id, COALESCE(MAX(a.name), '') AS name, COUNT(*) AS shared_files
FROM addon_files f
LEFT JOIN addons a ON a.id = f.addon_id
WHERE f.deleted_at IS NULL AND f.addon_id <> ?
AND f.hash IN (SELECT s.hash FROM addon_files s WHERE s.addon_id = ? AND s.deleted_at IS NULL)
AND NOT EXISTS (SELECT 1 FROM whitelisted_hashes w WHERE w.hash = f.hash AND w.deleted_at IS NULL)
GROUP BY f.addon_id ORDER BY shared_files DESC LIMIT 25`, id, id).Scan(&shared).Error
if err != nil {
fail(c, http.StatusInternalServerError, err.Error())
return
}
c.JSON(http.StatusOK, gin.H{
"addon": row,
"summary": extra.Summary,
"currentVersionId": extra.CurrentVersionID,
"fileCount": fileCount,
"deletedFileRows": deletedFiles,
"dependencies": gin.H{"total": depsTotal, "items": deps, "missing": missingDeps},
"dependents": gin.H{"total": dependentsTotal, "items": dependents},
"duplicates": shared,
})
}
type fileRow struct {
ID uint `json:"id"`
Path string `json:"path"`
Hash string `json:"hash"`
Version string `json:"version"`
Whitelisted bool `json:"whitelisted"`
OtherAddons int64 `json:"otherAddons"`
}
func addonFiles(c *gin.Context) {
id := c.Param("id")
limit := queryInt(c, "limit", 100, 1, 500)
offset := queryInt(c, "offset", 0, 0, 1<<30)
q := initializers.DB.Model(&models.AddonFile{}).Where("addon_id = ?", id)
if s := strings.TrimSpace(c.Query("q")); s != "" {
like := "%" + likeEscape(strings.ToLower(s)) + "%"
q = q.Where(`LOWER(path) LIKE ? ESCAPE '\' OR hash = ?`, like, strings.ToLower(s))
}
var total int64
q.Session(&gorm.Session{}).Count(&total)
rows := []fileRow{}
if err := q.Select("id, path, hash, version").Order("path ASC").Limit(limit).Offset(offset).Scan(&rows).Error; err != nil {
fail(c, http.StatusInternalServerError, err.Error())
return
}
if len(rows) > 0 && c.Query("shared") != "0" {
hashes := make([]string, 0, len(rows))
for _, r := range rows {
hashes = append(hashes, r.Hash)
}
var wl []string
initializers.DB.Model(&models.WhitelistedHash{}).Where("hash IN ?", hashes).Pluck("hash", &wl)
wlSet := map[string]bool{}
for _, h := range wl {
wlSet[h] = true
}
var counts []struct {
Hash string
Addons int64
}
initializers.DB.Raw(`SELECT hash, COUNT(DISTINCT addon_id) AS addons FROM addon_files
WHERE deleted_at IS NULL AND hash IN ? GROUP BY hash`, hashes).Scan(&counts)
cnt := map[string]int64{}
for _, x := range counts {
cnt[x.Hash] = x.Addons
}
for i := range rows {
rows[i].Whitelisted = wlSet[rows[i].Hash]
rows[i].OtherAddons = max(0, cnt[rows[i].Hash]-1)
}
}
c.JSON(http.StatusOK, gin.H{"items": rows, "total": total, "limit": limit, "offset": offset})
}
var errNotFound = errors.New("addon not found")
// applyAction changes queue flags of the given addons. UpdateColumns is used
// so updated_at (the queue order) is left alone.
func applyAction(tx *gorm.DB, ids []string, action string, value bool) (int64, error) {
base := tx.Model(&models.Addon{}).Where("id IN ?", ids)
var res *gorm.DB
switch action {
case "priority":
cols := map[string]any{"priority_indexing": value}
if value {
cols["to_be_indexed"] = true
}
res = base.UpdateColumns(cols)
case "to-be-indexed":
res = base.UpdateColumns(map[string]any{"to_be_indexed": value})
case "reindex":
cols := map[string]any{"to_be_indexed": true, "is_being_indexed": false}
if value {
cols["priority_indexing"] = true
}
res = base.UpdateColumns(cols)
case "reset-indexing":
res = base.UpdateColumns(map[string]any{"is_being_indexed": false})
case "recheck":
res = base.UpdateColumns(map[string]any{"recheck": value})
case "restore":
res = tx.Unscoped().Model(&models.Addon{}).Where("id IN ? AND deleted_at IS NOT NULL", ids).
UpdateColumns(map[string]any{"deleted_at": nil, "to_be_indexed": true, "is_being_indexed": false})
case "delete":
var addons []models.Addon
if err := tx.Where("id IN ?", ids).Find(&addons).Error; err != nil {
return 0, err
}
for i := range addons {
if err := controllers.DeleteAddonCascade(tx, &addons[i]); err != nil {
return 0, err
}
}
return int64(len(addons)), nil
default:
return 0, errors.New("unknown action")
}
return res.RowsAffected, res.Error
}
func addonAction(c *gin.Context) {
id, action := c.Param("id"), c.Param("action")
var body struct {
Value *bool `json:"value"`
}
_ = c.ShouldBindJSON(&body)
value := body.Value == nil || *body.Value
if action == "delete" {
fail(c, http.StatusBadRequest, "use DELETE /addons/:id")
return
}
n, err := applyAction(initializers.DB, []string{id}, action, value)
if err != nil {
fail(c, http.StatusBadRequest, err.Error())
return
}
if n == 0 {
fail(c, http.StatusNotFound, "addon not found or nothing to change")
return
}
audit(c, "addon_"+strings.ReplaceAll(action, "-", "_"), map[string]any{"addon": id, "value": value})
invalidate("overview", "queue")
c.JSON(http.StatusOK, gin.H{"updated": n})
}
func deleteAddon(c *gin.Context) {
id := c.Param("id")
err := initializers.DB.Transaction(func(tx *gorm.DB) error {
var addon models.Addon
if err := tx.Where("id = ?", id).First(&addon).Error; err != nil {
return errNotFound
}
return controllers.DeleteAddonCascade(tx, &addon)
})
if errors.Is(err, errNotFound) {
fail(c, http.StatusNotFound, err.Error())
return
}
if err != nil {
fail(c, http.StatusInternalServerError, err.Error())
return
}
audit(c, "addon_delete", map[string]any{"addon": id})
invalidate("overview", "queue")
c.JSON(http.StatusOK, gin.H{"deleted": id})
}
// purgeAddon permanently removes an already soft-deleted addon and its rows.
func purgeAddon(c *gin.Context) {
id := c.Param("id")
if c.Query("confirm") != id {
fail(c, http.StatusBadRequest, "pass ?confirm=<addon id> to purge")
return
}
var files int64
err := initializers.DB.Transaction(func(tx *gorm.DB) error {
var addon models.Addon
if err := tx.Unscoped().Where("id = ? AND deleted_at IS NOT NULL", id).First(&addon).Error; err != nil {
return errNotFound
}
res := tx.Unscoped().Where("addon_id = ?", id).Delete(&models.AddonFile{})
if res.Error != nil {
return res.Error
}
files = res.RowsAffected
if err := tx.Where("addon_id = ?", id).Delete(&models.AddonDependency{}).Error; err != nil {
return err
}
return tx.Unscoped().Delete(&addon).Error
})
if errors.Is(err, errNotFound) {
fail(c, http.StatusNotFound, "addon not found or not deleted")
return
}
if err != nil {
fail(c, http.StatusInternalServerError, err.Error())
return
}
audit(c, "addon_purge", map[string]any{"addon": id, "files": files})
invalidate("overview")
c.JSON(http.StatusOK, gin.H{"purged": id, "files": files})
}
func bulkAddons(c *gin.Context) {
var body struct {
IDs []string `json:"ids"`
Action string `json:"action"`
Value *bool `json:"value"`
}
if err := c.ShouldBindJSON(&body); err != nil || len(body.IDs) == 0 {
fail(c, http.StatusBadRequest, "ids and action are required")
return
}
if len(body.IDs) > 500 {
fail(c, http.StatusBadRequest, "at most 500 ids per request")
return
}
value := body.Value == nil || *body.Value
var n int64
err := initializers.DB.Transaction(func(tx *gorm.DB) error {
var err error
n, err = applyAction(tx, body.IDs, body.Action, value)
return err
})
if err != nil {
fail(c, http.StatusBadRequest, err.Error())
return
}
audit(c, "bulk_"+strings.ReplaceAll(body.Action, "-", "_"), map[string]any{"count": len(body.IDs), "updated": n, "value": value})
invalidate("overview", "queue")
c.JSON(http.StatusOK, gin.H{"updated": n})
}
// ---- queue ---------------------------------------------------------------
type queueRow struct {
ID string `json:"id"`
Name string `json:"name"`
CurrentVersionSize int64 `json:"currentVersionSize"`
SubscriberCount int64 `json:"subscriberCount"`
PriorityIndexing bool `json:"priorityIndexing"`
Blocked bool `json:"blocked"`
IndexStartTime *time.Time `json:"indexStartTime"`
UpdatedAt time.Time `json:"updatedAt"`
ElapsedSec int64 `json:"elapsedSec"`
Stuck bool `json:"stuck"`
}
const queueColumns = "id, name, current_version_size, subscriber_count, priority_indexing, blocked, index_start_time, updated_at"
func buildQueue() (gin.H, error) {
counts, err := metrics.LoadQueueCounts()
if err != nil {
return nil, err
}
var blockedQueued int64
initializers.DB.Model(&models.Addon{}).Where("to_be_indexed = ? AND blocked = ?", true, true).Count(&blockedQueued)
now := time.Now()
indexing := []queueRow{}
initializers.DB.Model(&models.Addon{}).Select(queueColumns).Where("is_being_indexed = ?", true).
Order("index_start_time ASC").Limit(100).Scan(&indexing)
for i := range indexing {
if t := indexing[i].IndexStartTime; t != nil && !t.IsZero() {
indexing[i].ElapsedSec = int64(now.Sub(*t).Seconds())
indexing[i].Stuck = now.Sub(*t) > initializers.IndexTimeout
}
}
priority := []queueRow{}
initializers.DB.Model(&models.Addon{}).Select(queueColumns).
Where("to_be_indexed = ? AND is_being_indexed = ? AND priority_indexing = ?", true, false, true).
Order("updated_at ASC").Limit(50).Scan(&priority)
// same order the indexers claim jobs in
next := []queueRow{}
initializers.DB.Model(&models.Addon{}).Select(queueColumns).
Where("to_be_indexed = ? AND is_being_indexed = ? AND blocked = ?", true, false, false).
Order("priority_indexing DESC, updated_at ASC").Limit(25).Scan(&next)
ts := now.Unix()
ok := map[string]string{"result": "ok"}
sel := metrics.Selector{Service: metrics.MainService, Name: "indexer_submissions_total", Match: ok}
perHour1h := metrics.Sum(sel, ts-3600, ts)
perHour6h := metrics.Sum(sel, ts-6*3600, ts) / 6
indexed24h := metrics.Sum(sel, ts-86400, ts)
var etaHours *float64
if perHour6h > 0 {
v := float64(counts.Queued) / perHour6h
etaHours = &v
}
duration, _ := metrics.Totals(metrics.Selector{Service: metrics.MainService, Name: "indexer_job_duration_seconds"}, ts-86400, ts)
var durationStats any
if len(duration) > 0 {
durationStats = duration[0]
}
return gin.H{
"counts": counts,
"blockedQueued": blockedQueued,
"timeoutMinutes": int(initializers.IndexTimeout.Minutes()),
"indexing": indexing,
"priority": priority,
"next": next,
"throughput": gin.H{
"perHour1h": perHour1h,
"perHour6h": perHour6h,
"indexed24h": indexed24h,
"etaHours": etaHours,
"duration24h": durationStats,
},
}, nil
}
func queue(c *gin.Context) {
data, at, err := cached("queue", 5*time.Second, buildQueue)
if err != nil {
fail(c, http.StatusInternalServerError, err.Error())
return
}
out := gin.H{"asOf": at.Unix()}
for k, v := range data {
out[k] = v
}
c.JSON(http.StatusOK, out)
}
func resetStuck(c *gin.Context) {
var body struct {
OlderThanMinutes int `json:"olderThanMinutes"`
}
_ = c.ShouldBindJSON(&body)
older := initializers.IndexTimeout
if body.OlderThanMinutes > 0 {
older = time.Duration(body.OlderThanMinutes) * time.Minute
}
n := controllers.ResetTimedOutIndexing(older)
metrics.IndexerTimeoutsReset.Add(float64(n))
audit(c, "queue_reset_stuck", map[string]any{"olderThanMinutes": older.Minutes(), "reset": n})
invalidate("queue", "overview")
c.JSON(http.StatusOK, gin.H{"reset": n})
}
admin/authapi.go
+71
View File
@@ -0,0 +1,71 @@
package admin
import (
"net/http"
"time"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/auth"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
"github.com/gin-gonic/gin"
)
var limiter = auth.NewLoginLimiter(5, 10*time.Minute)
func login(c *gin.Context) {
if initializers.AdminSecret == "" {
fail(c, http.StatusServiceUnavailable, "admin disabled: ADMIN_SECRET not set")
return
}
if c.GetHeader(auth.CSRFHeader) != "1" {
fail(c, http.StatusForbidden, "csrf check failed")
return
}
ip := c.ClientIP()
if limiter.Blocked(ip) {
fail(c, http.StatusTooManyRequests, "too many failed attempts, try again later")
return
}
var body struct {
Secret string `json:"secret"`
}
if err := c.ShouldBindJSON(&body); err != nil {
fail(c, http.StatusBadRequest, "invalid JSON")
return
}
if !auth.SecretEqual(body.Secret, initializers.AdminSecret) {
limiter.Fail(ip)
time.Sleep(300 * time.Millisecond)
metrics.AdminActions.Inc("login_failed")
metrics.Event("warn", "admin: login failed", map[string]any{"kind": "audit", "ip": ip})
fail(c, http.StatusUnauthorized, "invalid secret")
return
}
limiter.Reset(ip)
token, exp := auth.IssueToken(time.Now())
auth.SetSessionCookie(c, token, exp)
c.Set("adminAuth", "login")
audit(c, "login", nil)
c.JSON(http.StatusOK, gin.H{"expiresAt": exp.Unix()})
}
func logout(c *gin.Context) {
auth.ClearSessionCookie(c)
c.Status(http.StatusNoContent)
}
func me(c *gin.Context) {
method := c.GetString("adminAuth")
resp := gin.H{"method": method}
if v, ok := c.Get("adminSessionExp"); ok {
exp := v.(time.Time)
// sliding renewal: extend sessions that are about to expire
if time.Until(exp) < 24*time.Hour {
token, newExp := auth.IssueToken(time.Now())
auth.SetSessionCookie(c, token, newExp)
exp = newExp
}
resp["expiresAt"] = exp.Unix()
}
c.JSON(http.StatusOK, resp)
}
admin/metricsapi.go
+259
View File
@@ -0,0 +1,259 @@
package admin
import (
"encoding/json"
"net/http"
"sort"
"strconv"
"strings"
"time"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
"github.com/gin-gonic/gin"
)
func metricsCatalog(c *gin.Context) {
service := c.Query("service")
data, _, err := cached("catalog:"+service, 15*time.Second, func() ([]metrics.CatalogEntry, error) {
return metrics.Catalog(service)
})
if err != nil {
fail(c, http.StatusInternalServerError, err.Error())
return
}
c.JSON(http.StatusOK, gin.H{"items": data, "aggregations": metrics.Aggregations})
}
func selectorFrom(c *gin.Context) (metrics.Selector, bool) {
sel := metrics.Selector{Service: c.Query("service"), Name: c.Query("name")}
if m := c.Query("match"); m != "" {
if err := json.Unmarshal([]byte(m), &sel.Match); err != nil {
fail(c, http.StatusBadRequest, "match must be a JSON object of label values")
return sel, false
}
}
if g := c.Query("groupBy"); g != "" {
for _, part := range strings.Split(g, ",") {
if part = strings.TrimSpace(part); part != "" {
sel.GroupBy = append(sel.GroupBy, part)
}
}
}
if sel.Name == "" {
fail(c, http.StatusBadRequest, "name is required")
return sel, false
}
return sel, true
}
func metricsQuery(c *gin.Context) {
sel, ok := selectorFrom(c)
if !ok {
return
}
from, to := timeRange(c, time.Hour)
res, err := metrics.Query(sel, c.Query("agg"), from, to, queryInt64(c, "step", 0))
if err != nil {
fail(c, http.StatusBadRequest, err.Error())
return
}
c.JSON(http.StatusOK, res)
}
func autoStep(from, to int64) int64 {
span := to - from
switch {
case span <= 3*3600:
return 60
case span <= 12*3600:
return 300
case span <= 48*3600:
return 900
case span <= 8*86400:
return 3600
default:
return 6 * 3600
}
}
type routeStat struct {
Method string `json:"method"`
Route string `json:"route"`
Group string `json:"group"`
Count int64 `json:"count"`
Errors int64 `json:"errors"`
Client4 int64 `json:"clientErrors"`
P50 *float64 `json:"p50"`
P95 *float64 `json:"p95"`
P99 *float64 `json:"p99"`
Max *float64 `json:"max"`
Avg *float64 `json:"avg"`
}
// traffic returns pre-composed request, latency and error data for a range.
func traffic(c *gin.Context) {
from, to := timeRange(c, 24*time.Hour)
step := queryInt64(c, "step", 0)
if step <= 0 {
step = autoStep(from, to)
}
key := "traffic:" + fmtInt64(from/30) + ":" + fmtInt64(to/30) + ":" + fmtInt64(step)
data, at, err := cached(key, 30*time.Second, func() (gin.H, error) {
return buildTraffic(from, to, step)
})
if err != nil {
fail(c, http.StatusInternalServerError, err.Error())
return
}
out := gin.H{"asOf": at.Unix()}
for k, v := range data {
out[k] = v
}
c.JSON(http.StatusOK, out)
}
func fmtInt64(v int64) string { return strconv.FormatInt(v, 10) }
func buildTraffic(from, to, step int64) (gin.H, error) {
const name = "http_request_duration_ms"
sel := func(match map[string]string, groupBy ...string) metrics.Selector {
return metrics.Selector{Service: metrics.MainService, Name: name, Match: match, GroupBy: groupBy}
}
v1 := map[string]string{"group": "v1"}
byGroup, err := metrics.Query(sel(nil, "group"), "rate", from, to, step)
if err != nil {
return nil, err
}
latency := gin.H{}
for _, q := range []string{"p50", "p95", "p99"} {
r, err := metrics.Query(sel(v1), q, from, to, step)
if err != nil {
return nil, err
}
latency[q] = r
}
errors5xx, _ := metrics.Query(sel(map[string]string{"status": "5xx"}, "group"), "rate", from, to, step)
errors4xx, _ := metrics.Query(sel(map[string]string{"status": "4xx", "group": "v1"}), "rate", from, to, step)
statusTotals, _ := metrics.Totals(sel(nil, "status"), from, to)
clientTotals, _ := metrics.Totals(sel(v1, "client"), from, to)
groupTotals, _ := metrics.Totals(sel(nil, "group"), from, to)
routeTotals, err := metrics.Totals(sel(nil, "method", "route", "group"), from, to)
if err != nil {
return nil, err
}
statusByRoute, _ := metrics.Totals(sel(nil, "method", "route", "status"), from, to)
errs := map[string][2]int64{}
for _, t := range statusByRoute {
k := t.Labels["method"] + " " + t.Labels["route"]
e := errs[k]
switch t.Labels["status"] {
case "5xx":
e[0] += t.Count
case "4xx":
e[1] += t.Count
}
errs[k] = e
}
routes := make([]routeStat, 0, len(routeTotals))
for _, t := range routeTotals {
k := t.Labels["method"] + " " + t.Labels["route"]
routes = append(routes, routeStat{
Method: t.Labels["method"], Route: t.Labels["route"], Group: t.Labels["group"],
Count: t.Count, Errors: errs[k][0], Client4: errs[k][1],
P50: t.P50, P95: t.P95, P99: t.P99, Max: t.Max, Avg: t.Avg,
})
}
sort.Slice(routes, func(i, j int) bool { return routes[i].Count > routes[j].Count })
now := time.Now().Unix()
visitorsDaily, _ := metrics.Query(metrics.Selector{Service: metrics.MainService, Name: "visitors_unique", GroupBy: []string{"kind"}},
"max", now-30*86400, now, 86400)
humans, bots := metrics.Visitors()
var v1Total *metrics.Total
if all, _ := metrics.Totals(sel(v1), from, to); len(all) > 0 {
v1Total = &all[0]
}
return gin.H{
"from": from, "to": to, "step": step,
"requestsByGroup": byGroup,
"latency": latency,
"errors5xx": errors5xx,
"errors4xx": errors4xx,
"statusTotals": statusTotals,
"clientTotals": clientTotals,
"groupTotals": groupTotals,
"website": v1Total,
"routes": routes,
"visitorsDaily": visitorsDaily,
"visitorsToday": gin.H{"human": humans, "bot": bots},
}, nil
}
type topEntry struct {
Item string `json:"item"`
Count int64 `json:"count"`
Name string `json:"name,omitempty"`
}
// topItems returns the most looked-up addons or search queries.
func topItems(c *gin.Context) {
kind := c.DefaultQuery("kind", "addon")
if kind != "addon" && kind != "search" {
fail(c, http.StatusBadRequest, "kind must be addon or search")
return
}
days := queryInt(c, "days", 1, 1, 90)
limit := queryInt(c, "limit", 20, 1, 200)
today, counts := metrics.Top.Today(kind)
if today == "" {
today = time.Now().UTC().Format("2006-01-02")
}
if days > 1 {
since := time.Now().UTC().AddDate(0, 0, -(days - 1)).Format("2006-01-02")
var rows []models.DailyTopItem
initializers.DB.Where("kind = ? AND day >= ? AND day < ?", kind, since, today).Find(&rows)
for _, r := range rows {
counts[r.Item] += r.Count
}
}
list := make([]topEntry, 0, len(counts))
for k, v := range counts {
list = append(list, topEntry{Item: k, Count: v})
}
sort.Slice(list, func(i, j int) bool {
if list[i].Count != list[j].Count {
return list[i].Count > list[j].Count
}
return list[i].Item < list[j].Item
})
if len(list) > limit {
list = list[:limit]
}
if kind == "addon" && len(list) > 0 {
ids := make([]string, len(list))
for i, e := range list {
ids[i] = e.Item
}
var names []struct {
ID string
Name string
}
initializers.DB.Unscoped().Model(&models.Addon{}).Select("id, name").Where("id IN ?", ids).Scan(&names)
byID := map[string]string{}
for _, n := range names {
byID[n.ID] = n.Name
}
for i := range list {
list[i].Name = byID[list[i].Item]
}
}
c.JSON(http.StatusOK, gin.H{"kind": kind, "days": days, "items": list})
}
admin/overview.go
+443
View File
@@ -0,0 +1,443 @@
package admin
import (
"context"
"log"
"net/http"
"os"
"strconv"
"sync"
"time"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
"github.com/gin-gonic/gin"
"gorm.io/gorm"
)
type addonCounts struct {
Total int64 `json:"total"`
Indexed int64 `json:"indexed"`
Queued int64 `json:"queued"`
Indexing int64 `json:"indexing"`
Priority int64 `json:"priority"`
Recheck int64 `json:"recheck"`
Blocked int64 `json:"blocked"`
Unlisted int64 `json:"unlisted"`
Private int64 `json:"private"`
TotalSize int64 `json:"totalSizeBytes"`
Added24h int64 `json:"added24h"`
Added7d int64 `json:"added7d"`
Indexed24h int64 `json:"indexed24h"`
SoftDeleted int64 `json:"softDeleted"`
}
type typeCount struct {
Type string `json:"type"`
Count int64 `json:"count"`
}
type miniAddon struct {
ID string `json:"id"`
Name string `json:"name"`
Author string `json:"author"`
SubscriberCount int64 `json:"subscriberCount"`
CurrentVersionSize int64 `json:"currentVersionSize"`
LastIndexedAt *time.Time `json:"lastIndexedAt,omitempty"`
CreatedAt time.Time `json:"createdAt"`
}
type overviewData struct {
Addons addonCounts `json:"addons"`
Types []typeCount `json:"types"`
Whitelist int64 `json:"whitelistCount"`
Dependencies map[string]any `json:"dependencies"`
Traffic map[string]any `json:"traffic"`
Activity map[string]any `json:"activity"`
Services map[string]any `json:"services"`
TopSubscribed []miniAddon `json:"topSubscribed"`
Largest []miniAddon `json:"largest"`
RecentlyAdded []miniAddon `json:"recentlyAdded"`
RecentlyIndexed []miniAddon `json:"recentlyIndexed"`
Heavy *heavyStats `json:"heavy"`
}
func loadAddonCounts() (addonCounts, error) {
var ac addonCounts
now := time.Now()
err := initializers.DB.Raw(`SELECT COUNT(*) AS total,
COALESCE(SUM(CASE WHEN to_be_indexed = ? THEN 1 ELSE 0 END), 0) AS indexed,
COALESCE(SUM(CASE WHEN to_be_indexed = ? AND is_being_indexed = ? THEN 1 ELSE 0 END), 0) AS queued,
COALESCE(SUM(CASE WHEN is_being_indexed = ? THEN 1 ELSE 0 END), 0) AS indexing,
COALESCE(SUM(CASE WHEN priority_indexing = ? AND to_be_indexed = ? THEN 1 ELSE 0 END), 0) AS priority,
COALESCE(SUM(CASE WHEN recheck = ? THEN 1 ELSE 0 END), 0) AS recheck,
COALESCE(SUM(CASE WHEN blocked = ? THEN 1 ELSE 0 END), 0) AS blocked,
COALESCE(SUM(CASE WHEN unlisted = ? THEN 1 ELSE 0 END), 0) AS unlisted,
COALESCE(SUM(CASE WHEN private = ? THEN 1 ELSE 0 END), 0) AS private,
CAST(COALESCE(SUM(current_version_size), 0) AS BIGINT) AS total_size,
COALESCE(SUM(CASE WHEN created_at >= ? THEN 1 ELSE 0 END), 0) AS added24h,
COALESCE(SUM(CASE WHEN created_at >= ? THEN 1 ELSE 0 END), 0) AS added7d,
COALESCE(SUM(CASE WHEN last_indexed_at >= ? THEN 1 ELSE 0 END), 0) AS indexed24h
FROM addons WHERE deleted_at IS NULL`,
false, true, false, true, true, true, true, true, true, true,
now.Add(-24*time.Hour), now.Add(-7*24*time.Hour), now.Add(-24*time.Hour),
).Scan(&ac).Error
if err != nil {
return ac, err
}
err = initializers.DB.Unscoped().Model(&models.Addon{}).Where("deleted_at IS NOT NULL").Count(&ac.SoftDeleted).Error
return ac, err
}
func loadMiniAddons(order string, where string, args ...any) []miniAddon {
out := []miniAddon{}
q := initializers.DB.Model(&models.Addon{}).
Select("id, name, author, subscriber_count, current_version_size, last_indexed_at, created_at")
if where != "" {
q = q.Where(where, args...)
}
q.Order(order).Limit(8).Scan(&out)
return out
}
func buildOverview() (*overviewData, error) {
ac, err := loadAddonCounts()
if err != nil {
return nil, err
}
d := &overviewData{Addons: ac}
d.Types = []typeCount{}
initializers.DB.Model(&models.Addon{}).Select("type, COUNT(*) AS count").Group("type").Order("count DESC").Limit(12).Scan(&d.Types)
initializers.DB.Model(&models.WhitelistedHash{}).Count(&d.Whitelist)
var deps struct {
Edges int64
WithDeps int64
Dependents int64
}
initializers.DB.Raw(`SELECT COUNT(*) AS edges, COUNT(DISTINCT addon_id) AS with_deps,
COUNT(DISTINCT dependency_id) AS dependents FROM addon_dependencies`).Scan(&deps)
d.Dependencies = map[string]any{"edges": deps.Edges, "addonsWithDependencies": deps.WithDeps, "addonsDependedOn": deps.Dependents}
now := time.Now().Unix()
dayAgo := now - 86400
httpSel := metrics.Selector{Service: metrics.MainService, Name: "http_request_duration_ms"}
traffic := map[string]any{}
if t, err := metrics.Totals(metrics.Selector{Service: httpSel.Service, Name: httpSel.Name, GroupBy: []string{"group"}}, dayAgo, now); err == nil {
var total int64
for _, g := range t {
total += g.Count
if g.Labels["group"] == "v1" {
traffic["websiteRequests24h"] = g.Count
traffic["websiteP95Ms"] = g.P95
traffic["websiteP50Ms"] = g.P50
}
if g.Labels["group"] == "back" {
traffic["workerRequests24h"] = g.Count
}
}
traffic["requests24h"] = total
}
traffic["errors5xx24h"] = metrics.Sum(metrics.Selector{Service: httpSel.Service, Name: httpSel.Name, Match: map[string]string{"status": "5xx"}}, dayAgo, now)
humans, bots := metrics.Visitors()
traffic["visitorsToday"] = humans
traffic["botsToday"] = bots
d.Traffic = traffic
sum := func(name string, match map[string]string) float64 {
return metrics.Sum(metrics.Selector{Service: metrics.MainService, Name: name, Match: match}, dayAgo, now)
}
d.Activity = map[string]any{
"scraperCreated24h": sum("scraper_addons_total", map[string]string{"action": "created"}),
"scraperUpdated24h": sum("scraper_addons_total", map[string]string{"action": "updated"}),
"scraperCreateFailed24h": sum("scraper_addons_total", map[string]string{"action": "create_failed"}),
"deleted24h": sum("scraper_addons_deleted_total", map[string]string{"result": "deleted"}),
"indexed24h": sum("indexer_submissions_total", map[string]string{"result": "ok"}),
"claims24h": sum("indexer_claims_total", nil),
"timeoutsReset24h": sum("indexer_timeouts_reset_total", nil),
"dependencyReports24h": sum("dependency_reports_total", nil),
}
var instances []models.ServiceInstance
initializers.DB.Find(&instances)
online := 0
for i := range instances {
if metrics.Online(&instances[i], time.Now()) {
online++
}
}
d.Services = map[string]any{"online": online, "total": len(instances)}
d.TopSubscribed = loadMiniAddons("subscriber_count DESC", "")
d.Largest = loadMiniAddons("current_version_size DESC", "")
d.RecentlyAdded = loadMiniAddons("created_at DESC", "")
d.RecentlyIndexed = loadMiniAddons("last_indexed_at DESC", "last_indexed_at IS NOT NULL")
d.Heavy = currentHeavy()
return d, nil
}
func overview(c *gin.Context) {
data, at, err := cached("overview", 20*time.Second, buildOverview)
if err != nil {
fail(c, http.StatusInternalServerError, err.Error())
return
}
c.JSON(http.StatusOK, gin.H{"asOf": at.Unix(), "data": data})
}
// ---- heavy statistics (background) -----------------------------------------
type tableInfo struct {
Name string `json:"name"`
Rows int64 `json:"rows"`
DeadRows int64 `json:"deadRows"`
SizeBytes int64 `json:"sizeBytes"`
}
type dupHash struct {
Hash string `json:"hash"`
AddonCount int64 `json:"addonCount"`
FileCount int64 `json:"fileCount"`
SamplePath string `json:"samplePath"`
Whitelisted bool `json:"whitelisted"`
}
type heavyStats struct {
AsOf int64 `json:"asOf"`
FilesLive int64 `json:"filesLive"`
FilesSoftDeleted int64 `json:"filesSoftDeleted"`
FilesApprox bool `json:"filesApprox"`
DistinctHashes int64 `json:"distinctHashes"`
DistinctAsOf int64 `json:"distinctAsOf"`
DBDriver string `json:"dbDriver"`
DBSizeBytes int64 `json:"dbSizeBytes"`
Tables []tableInfo `json:"tables"`
Duplicates []dupHash `json:"-"`
DuplicatesAsOf int64 `json:"duplicatesAsOf"`
Error string `json:"error,omitempty"`
}
var (
heavyMu sync.Mutex
heavy *heavyStats
refreshCh = make(chan bool, 1)
)
func currentHeavy() *heavyStats {
heavyMu.Lock()
defer heavyMu.Unlock()
return heavy
}
// requestHeavyRefresh triggers a refresh; full=true also recomputes the
// expensive duplicate statistics.
func requestHeavyRefresh(full bool) {
select {
case refreshCh <- full:
default:
}
}
func heavyStatsLoop(ctx context.Context) {
lastFull := time.Time{}
run := func(full bool) {
defer func() {
if r := recover(); r != nil {
log.Printf("admin: heavy stats panicked: %v", r)
}
}()
full = full || time.Since(lastFull) > time.Hour
computeHeavy(full)
if full {
lastFull = time.Now()
}
}
// give the server a moment to start before the first scan
select {
case <-ctx.Done():
return
case <-time.After(5 * time.Second):
}
run(true)
ticker := time.NewTicker(10 * time.Minute)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
run(false)
case full := <-refreshCh:
run(full)
}
}
}
// withTimeout runs fn in a transaction with a statement timeout on Postgres.
func withTimeout(seconds int, fn func(tx *gorm.DB) error) error {
return initializers.DB.Transaction(func(tx *gorm.DB) error {
if isPostgres() {
if err := tx.Exec("SET LOCAL statement_timeout = " + strconv.Itoa(seconds*1000)).Error; err != nil {
return err
}
}
return fn(tx)
})
}
var ourTables = []string{"addons", "addon_files", "addon_dependencies", "whitelisted_hashes",
"metric_series", "metric_points", "service_instances", "service_events", "daily_top_items"}
func computeHeavy(full bool) {
started := time.Now()
h := &heavyStats{AsOf: time.Now().Unix(), DBDriver: initializers.DB.Dialector.Name(), Tables: []tableInfo{}}
prev := currentHeavy()
if prev != nil {
h.DistinctHashes, h.DistinctAsOf = prev.DistinctHashes, prev.DistinctAsOf
h.Duplicates, h.DuplicatesAsOf = prev.Duplicates, prev.DuplicatesAsOf
}
var errs []string
note := func(what string, err error) {
if err != nil {
errs = append(errs, what+": "+err.Error())
}
}
if isPostgres() {
note("tables", initializers.DB.Raw(`SELECT relname AS name, n_live_tup AS rows, n_dead_tup AS dead_rows,
pg_total_relation_size(relid) AS size_bytes FROM pg_stat_user_tables ORDER BY size_bytes DESC`).Scan(&h.Tables).Error)
note("db size", initializers.DB.Raw(`SELECT pg_database_size(current_database())`).Scan(&h.DBSizeBytes).Error)
note("soft-deleted files", withTimeout(60, func(tx *gorm.DB) error {
return tx.Raw(`SELECT COUNT(*) FROM addon_files WHERE deleted_at IS NOT NULL`).Scan(&h.FilesSoftDeleted).Error
}))
for _, t := range h.Tables {
if t.Name == "addon_files" {
h.FilesLive = max(0, t.Rows-h.FilesSoftDeleted)
h.FilesApprox = true
}
}
} else {
var pageCount, pageSize int64
initializers.DB.Raw("PRAGMA page_count").Scan(&pageCount)
initializers.DB.Raw("PRAGMA page_size").Scan(&pageSize)
h.DBSizeBytes = pageCount * pageSize
if st, err := os.Stat(sqlitePath() + "-wal"); err == nil {
h.DBSizeBytes += st.Size()
}
for _, name := range ourTables {
var n int64
if err := initializers.DB.Table(name).Count(&n).Error; err == nil {
h.Tables = append(h.Tables, tableInfo{Name: name, Rows: n})
}
}
note("files", initializers.DB.Raw(`SELECT
COALESCE(SUM(CASE WHEN deleted_at IS NULL THEN 1 ELSE 0 END), 0),
COALESCE(SUM(CASE WHEN deleted_at IS NOT NULL THEN 1 ELSE 0 END), 0) FROM addon_files`).Row().
Scan(&h.FilesLive, &h.FilesSoftDeleted))
}
if full {
var distinct int64
err := withTimeout(120, func(tx *gorm.DB) error {
return tx.Raw(`SELECT COUNT(DISTINCT hash) FROM addon_files WHERE deleted_at IS NULL`).Scan(&distinct).Error
})
note("distinct hashes", err)
if err == nil {
h.DistinctHashes, h.DistinctAsOf = distinct, time.Now().Unix()
}
var dups []dupHash
err = withTimeout(120, func(tx *gorm.DB) error {
return tx.Raw(`SELECT hash, COUNT(DISTINCT addon_id) AS addon_count, COUNT(*) AS file_count
FROM addon_files WHERE deleted_at IS NULL
GROUP BY hash HAVING COUNT(DISTINCT addon_id) > 1
ORDER BY addon_count DESC, file_count DESC LIMIT 200`).Scan(&dups).Error
})
note("duplicates", err)
if err == nil {
h.Duplicates, h.DuplicatesAsOf = dups, time.Now().Unix()
if len(dups) > 0 {
hashes := make([]string, len(dups))
for i, d := range dups {
hashes[i] = d.Hash
}
var samples []struct {
Hash string
Path string
}
initializers.DB.Raw(`SELECT hash, MIN(path) AS path FROM addon_files
WHERE deleted_at IS NULL AND hash IN ? GROUP BY hash`, hashes).Scan(&samples)
byHash := map[string]string{}
for _, s := range samples {
byHash[s.Hash] = s.Path
}
for i := range h.Duplicates {
h.Duplicates[i].SamplePath = byHash[h.Duplicates[i].Hash]
}
}
}
}
if len(errs) > 0 {
h.Error = errs[0]
for _, e := range errs {
log.Printf("admin: heavy stats: %s", e)
}
}
heavyMu.Lock()
heavy = h
heavyMu.Unlock()
invalidate("overview")
if time.Since(started) > 10*time.Second {
log.Printf("admin: heavy stats took %s", time.Since(started).Round(time.Millisecond))
}
}
func sqlitePath() string {
if initializers.SQLitePath != "" {
return initializers.SQLitePath
}
return "crawler.db"
}
// duplicateHashes lists the hashes shared by the most addons.
func duplicateHashes(c *gin.Context) {
h := currentHeavy()
if h == nil {
c.JSON(http.StatusOK, gin.H{"asOf": 0, "items": []dupHash{}, "pending": true})
return
}
items := make([]dupHash, len(h.Duplicates))
copy(items, h.Duplicates)
if len(items) > 0 {
hashes := make([]string, len(items))
for i, d := range items {
hashes[i] = d.Hash
}
var wl []string
initializers.DB.Model(&models.WhitelistedHash{}).Where("hash IN ?", hashes).Pluck("hash", &wl)
set := map[string]bool{}
for _, w := range wl {
set[w] = true
}
for i := range items {
items[i].Whitelisted = set[items[i].Hash]
}
}
if c.Query("hideWhitelisted") == "1" {
kept := items[:0]
for _, it := range items {
if !it.Whitelisted {
kept = append(kept, it)
}
}
items = kept
}
c.JSON(http.StatusOK, gin.H{"asOf": h.DuplicatesAsOf, "items": items})
}
admin/routes.go
+57
View File
@@ -0,0 +1,57 @@
package admin
import (
"context"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/auth"
"github.com/gin-gonic/gin"
)
// Register mounts the admin JSON API under /admin/api and the legacy
// header-authenticated /admin/addWhitelistedHash endpoint.
func Register(r *gin.Engine) {
r.POST("/admin/addWhitelistedHash", auth.RequireAdminHeader, legacyAddWhitelistedHash)
api := r.Group("/admin/api")
api.POST("/auth/login", login)
api.POST("/auth/logout", logout)
p := api.Group("", auth.RequireAdmin)
p.GET("/auth/me", me)
p.GET("/overview", overview)
p.GET("/overview/duplicates", duplicateHashes)
p.GET("/queue", queue)
p.POST("/queue/reset-stuck", resetStuck)
p.GET("/addons", listAddons)
p.GET("/addons/:id", addonDetail)
p.GET("/addons/:id/files", addonFiles)
p.POST("/addons/:id/:action", addonAction)
p.DELETE("/addons/:id", deleteAddon)
p.DELETE("/addons/:id/purge", purgeAddon)
p.POST("/bulk/addons", bulkAddons)
p.GET("/whitelist", listWhitelist)
p.POST("/whitelist", addWhitelist)
p.DELETE("/whitelist/:id", deleteWhitelist)
p.GET("/hashes/:hash", hashUsage)
p.GET("/metrics/catalog", metricsCatalog)
p.GET("/metrics/query", metricsQuery)
p.GET("/traffic", traffic)
p.GET("/top", topItems)
p.GET("/services", listServices)
p.GET("/services/events", listEvents)
p.DELETE("/services/:id", deleteService)
p.GET("/system", system)
p.POST("/system/maintenance", maintenance)
}
// StartBackground runs the periodic heavy statistics job.
func StartBackground(ctx context.Context) {
go heavyStatsLoop(ctx)
}
admin/system.go
+244
View File
@@ -0,0 +1,244 @@
package admin
import (
"encoding/json"
"net/http"
"os"
"runtime"
"runtime/debug"
"strconv"
"strings"
"time"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
"github.com/gin-gonic/gin"
)
// BuildCommit is set by main from -ldflags.
var BuildCommit = ""
type serviceRow struct {
models.ServiceInstance
Online bool `json:"online"`
UptimeSec int64 `json:"uptimeSec"`
MetaMap map[string]any `json:"metaData,omitempty"`
}
func listServices(c *gin.Context) {
var rows []models.ServiceInstance
if err := initializers.DB.Order("service ASC, instance ASC").Find(&rows).Error; err != nil {
fail(c, http.StatusInternalServerError, err.Error())
return
}
now := time.Now()
out := make([]serviceRow, 0, len(rows))
for _, r := range rows {
s := serviceRow{ServiceInstance: r, Online: metrics.Online(&r, now)}
if r.StartedAt > 0 && s.Online {
s.UptimeSec = now.Unix() - r.StartedAt
}
if r.Meta != "" {
_ = json.Unmarshal([]byte(r.Meta), &s.MetaMap)
}
out = append(out, s)
}
c.JSON(http.StatusOK, gin.H{"items": out})
}
func deleteService(c *gin.Context) {
id, err := strconv.ParseUint(c.Param("id"), 10, 64)
if err != nil {
fail(c, http.StatusBadRequest, "invalid id")
return
}
var row models.ServiceInstance
if err := initializers.DB.First(&row, id).Error; err != nil {
fail(c, http.StatusNotFound, "not found")
return
}
initializers.DB.Delete(&row)
audit(c, "service_forget", map[string]any{"service": row.Service, "instance": row.Instance})
c.JSON(http.StatusOK, gin.H{"deleted": id})
}
type eventRow struct {
models.ServiceEvent
FieldsMap map[string]any `json:"fieldsData,omitempty"`
}
func listEvents(c *gin.Context) {
limit := queryInt(c, "limit", 100, 1, 500)
q := initializers.DB.Model(&models.ServiceEvent{})
if v := c.Query("service"); v != "" {
q = q.Where("service = ?", v)
}
if v := c.Query("instance"); v != "" {
q = q.Where("instance = ?", v)
}
if v := c.Query("level"); v != "" {
levels := strings.Split(v, ",")
q = q.Where("level IN ?", levels)
}
if c.Query("audit") == "1" {
q = q.Where("fields LIKE ?", `%"kind":"audit"%`)
}
if v := strings.TrimSpace(c.Query("q")); v != "" {
q = q.Where(`LOWER(message) LIKE ? ESCAPE '\'`, "%"+likeEscape(strings.ToLower(v))+"%")
}
if before := queryInt64(c, "before", 0); before > 0 {
q = q.Where("id < ?", before)
}
var rows []models.ServiceEvent
if err := q.Order("id DESC").Limit(limit).Find(&rows).Error; err != nil {
fail(c, http.StatusInternalServerError, err.Error())
return
}
out := make([]eventRow, 0, len(rows))
for _, r := range rows {
e := eventRow{ServiceEvent: r}
if r.Fields != "" {
_ = json.Unmarshal([]byte(r.Fields), &e.FieldsMap)
}
out = append(out, e)
}
var next uint
if len(rows) == limit {
next = rows[len(rows)-1].ID
}
c.JSON(http.StatusOK, gin.H{"items": out, "next": next})
}
func buildInfo() gin.H {
info := gin.H{"commit": BuildCommit}
if bi, ok := debug.ReadBuildInfo(); ok {
info["module"] = bi.Main.Path
for _, s := range bi.Settings {
switch s.Key {
case "vcs.revision":
if BuildCommit == "" {
info["commit"] = s.Value
}
case "vcs.time":
info["time"] = s.Value
case "vcs.modified":
info["modified"] = s.Value == "true"
}
}
}
return info
}
func system(c *gin.Context) {
var ms runtime.MemStats
runtime.ReadMemStats(&ms)
host, _ := os.Hostname()
started := metrics.StartedAt()
resp := gin.H{
"go": gin.H{
"version": runtime.Version(), "os": runtime.GOOS, "arch": runtime.GOARCH,
"cpus": runtime.NumCPU(), "maxProcs": runtime.GOMAXPROCS(0), "goroutines": runtime.NumGoroutine(),
},
"process": gin.H{
"pid": os.Getpid(), "hostname": host, "startedAt": started.Unix(),
"uptimeSec": int64(time.Since(started).Seconds()),
},
"memory": gin.H{
"heapAlloc": ms.HeapAlloc, "heapInuse": ms.HeapInuse, "sys": ms.Sys,
"numGC": ms.NumGC, "pauseTotalNs": ms.PauseTotalNs, "lastGC": int64(ms.LastGC / 1e9),
},
"build": buildInfo(),
"metrics": metrics.Status(),
"config": gin.H{
"dbDriver": initializers.DB.Dialector.Name(),
"trustedProxies": initializers.TrustedProxies,
"cookieSecure": initializers.AdminCookieSecure,
"sessionTtlHours": initializers.AdminSessionTTL.Hours(),
"metricsEnabled": initializers.MetricsEnabled,
"prometheusEndpoint": initializers.MetricsToken != "",
"metricsMinuteRetentionDays": initializers.MetricsMinuteRetentionDays,
"metricsHourRetentionDays": initializers.MetricsHourRetentionDays,
"eventsRetentionDays": initializers.EventsRetentionDays,
"topItemsRetentionDays": initializers.TopItemsRetentionDays,
"indexTimeoutMinutes": initializers.IndexTimeout.Minutes(),
"adminSecretStrong": len(initializers.AdminSecret) >= 16,
"secretKeySet": initializers.SecretKey != "",
"discordWebhook": initializers.DiscordWebhookURL != "",
"scraperWebhook": initializers.ScraperWebhookURL != "",
},
}
if sqlDB, err := initializers.DB.DB(); err == nil {
st := sqlDB.Stats()
resp["dbPool"] = gin.H{
"open": st.OpenConnections, "inUse": st.InUse, "idle": st.Idle,
"maxOpen": st.MaxOpenConnections, "waitCount": st.WaitCount, "waitMs": st.WaitDuration.Milliseconds(),
}
start := time.Now()
if err := sqlDB.Ping(); err == nil {
resp["dbPingMs"] = float64(time.Since(start).Microseconds()) / 1000
} else {
resp["dbPingError"] = err.Error()
}
}
if h := currentHeavy(); h != nil {
resp["db"] = h
}
c.JSON(http.StatusOK, resp)
}
func maintenance(c *gin.Context) {
var body struct {
Action string `json:"action"`
OlderDays int `json:"olderThanDays"`
}
if err := c.ShouldBindJSON(&body); err != nil {
fail(c, http.StatusBadRequest, "invalid JSON")
return
}
switch body.Action {
case "refresh-stats":
requestHeavyRefresh(true)
case "flush-metrics":
metrics.FlushAll()
case "rollup":
go metrics.RunMaintenance(time.Now())
case "purge-deleted-files":
days := body.OlderDays
if days <= 0 {
days = 7
}
go purgeDeletedFiles(days)
default:
fail(c, http.StatusBadRequest, "unknown action")
return
}
audit(c, "maintenance_"+strings.ReplaceAll(body.Action, "-", "_"), map[string]any{"olderThanDays": body.OlderDays})
c.JSON(http.StatusAccepted, gin.H{"started": body.Action})
}
// purgeDeletedFiles hard-deletes file rows that were soft-deleted (replaced by
// a newer index) more than `days` ago, in batches.
func purgeDeletedFiles(days int) {
cutoff := time.Now().AddDate(0, 0, -days)
started := time.Now()
var total int64
for {
res := initializers.DB.Exec(`DELETE FROM addon_files WHERE id IN (
SELECT id FROM addon_files WHERE deleted_at IS NOT NULL AND deleted_at < ? LIMIT 5000)`, cutoff)
if res.Error != nil {
metrics.Event("error", "purge of deleted file rows failed", map[string]any{"kind": "maintenance", "error": res.Error.Error(), "deleted": total})
return
}
total += res.RowsAffected
if res.RowsAffected == 0 {
break
}
time.Sleep(200 * time.Millisecond)
}
metrics.Event("info", "purged deleted file rows", map[string]any{
"kind": "maintenance", "deleted": total, "olderThanDays": days, "tookSec": int64(time.Since(started).Seconds()),
})
requestHeavyRefresh(false)
}
admin/util.go
+136
View File
@@ -0,0 +1,136 @@
package admin
import (
"strconv"
"strings"
"sync"
"time"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
"github.com/gin-gonic/gin"
)
// ---- TTL cache with request coalescing -------------------------------------
type cacheEntry struct {
val any
at time.Time
}
type inflight struct {
wg sync.WaitGroup
val any
err error
}
var (
cacheMu sync.Mutex
cacheData = map[string]cacheEntry{}
cacheRun = map[string]*inflight{}
)
// cached returns a cached value younger than ttl, or computes it once even if
// many requests ask at the same time. It also returns when the value was made.
func cached[T any](key string, ttl time.Duration, fn func() (T, error)) (T, time.Time, error) {
cacheMu.Lock()
if e, ok := cacheData[key]; ok && time.Since(e.at) < ttl {
cacheMu.Unlock()
return e.val.(T), e.at, nil
}
if call, ok := cacheRun[key]; ok {
cacheMu.Unlock()
call.wg.Wait()
var zero T
if call.err != nil {
return zero, time.Time{}, call.err
}
return call.val.(T), time.Now(), nil
}
call := &inflight{}
call.wg.Add(1)
cacheRun[key] = call
cacheMu.Unlock()
val, err := fn()
call.val, call.err = val, err
now := time.Now()
cacheMu.Lock()
delete(cacheRun, key)
if err == nil {
cacheData[key] = cacheEntry{val: val, at: now}
}
cacheMu.Unlock()
call.wg.Done()
return val, now, err
}
// invalidate drops cached values whose key starts with one of the prefixes.
func invalidate(prefixes ...string) {
cacheMu.Lock()
defer cacheMu.Unlock()
for k := range cacheData {
for _, p := range prefixes {
if strings.HasPrefix(k, p) {
delete(cacheData, k)
}
}
}
}
// ---- request helpers -----------------------------------------------------
func fail(c *gin.Context, code int, msg string) {
c.AbortWithStatusJSON(code, gin.H{"error": msg})
}
func queryInt(c *gin.Context, name string, def, lo, hi int) int {
v, err := strconv.Atoi(c.Query(name))
if err != nil {
return def
}
return max(lo, min(hi, v))
}
func queryInt64(c *gin.Context, name string, def int64) int64 {
v, err := strconv.ParseInt(c.Query(name), 10, 64)
if err != nil {
return def
}
return v
}
// timeRange reads from/to (unix seconds) with a default window ending now.
func timeRange(c *gin.Context, def time.Duration) (int64, int64) {
now := time.Now().Unix()
to := queryInt64(c, "to", now)
if to > now {
to = now
}
from := queryInt64(c, "from", to-int64(def.Seconds()))
if from >= to {
from = to - int64(def.Seconds())
}
return from, to
}
func isPostgres() bool {
return initializers.DB.Dialector.Name() == "postgres"
}
// likeEscape escapes LIKE wildcards in user input.
func likeEscape(s string) string {
return strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(s)
}
// audit records an admin action as a metric and an event.
func audit(c *gin.Context, action string, fields map[string]any) {
if fields == nil {
fields = map[string]any{}
}
fields["kind"] = "audit"
fields["ip"] = c.ClientIP()
fields["auth"] = c.GetString("adminAuth")
metrics.AdminActions.Inc(action)
metrics.Event("info", "admin: "+action, fields)
}
admin/whitelist.go
+227
View File
@@ -0,0 +1,227 @@
package admin
import (
"net/http"
"regexp"
"strconv"
"strings"
"time"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
"github.com/gin-gonic/gin"
"gorm.io/gorm"
)
var sha512Re = regexp.MustCompile(`^[0-9a-f]{128}$`)
// legacyAddWhitelistedHash keeps the original form-based endpoint working.
func legacyAddWhitelistedHash(c *gin.Context) {
hash := c.PostForm("hash")
if hash == "" {
c.JSON(400, gin.H{"error": "Hash is required"})
return
}
ret := initializers.DB.Find(&models.WhitelistedHash{}, "hash = ?", hash)
if ret.RowsAffected > 0 {
c.JSON(400, gin.H{"error": "Hash already whitelisted"})
return
}
if err := initializers.DB.Create(&models.WhitelistedHash{Hash: hash}).Error; err != nil {
c.JSON(500, gin.H{"error": "Failed to add whitelisted hash"})
return
}
audit(c, "whitelist_add", map[string]any{"added": 1, "legacy": true})
invalidate("overview")
c.JSON(200, gin.H{"message": "Whitelisted hash added successfully"})
}
type whitelistRow struct {
ID uint `json:"id"`
Hash string `json:"hash"`
Note string `json:"note"`
CreatedAt time.Time `json:"createdAt"`
Files int64 `json:"files"`
Addons int64 `json:"addons"`
SamplePath string `json:"samplePath"`
}
func listWhitelist(c *gin.Context) {
limit := queryInt(c, "limit", 50, 1, 500)
offset := queryInt(c, "offset", 0, 0, 1<<30)
q := initializers.DB.Model(&models.WhitelistedHash{})
if s := strings.TrimSpace(strings.ToLower(c.Query("q"))); s != "" {
like := "%" + likeEscape(s) + "%"
q = q.Where(`hash LIKE ? ESCAPE '\' OR LOWER(note) LIKE ? ESCAPE '\'`, likeEscape(s)+"%", like)
}
var total int64
q.Session(&gorm.Session{}).Count(&total)
rows := []whitelistRow{}
if err := q.Select("id, hash, note, created_at").Order("id DESC").Limit(limit).Offset(offset).Scan(&rows).Error; err != nil {
fail(c, http.StatusInternalServerError, err.Error())
return
}
if len(rows) > 0 {
hashes := make([]string, len(rows))
for i, r := range rows {
hashes[i] = r.Hash
}
var usage []struct {
Hash string
Files int64
Addons int64
Sample string
}
initializers.DB.Raw(`SELECT hash, COUNT(*) AS files, COUNT(DISTINCT addon_id) AS addons, MIN(path) AS sample
FROM addon_files WHERE deleted_at IS NULL AND hash IN ? GROUP BY hash`, hashes).Scan(&usage)
byHash := map[string]int{}
for i, u := range usage {
byHash[u.Hash] = i
}
for i := range rows {
if j, ok := byHash[rows[i].Hash]; ok {
rows[i].Files, rows[i].Addons, rows[i].SamplePath = usage[j].Files, usage[j].Addons, usage[j].Sample
}
}
}
c.JSON(http.StatusOK, gin.H{"items": rows, "total": total, "limit": limit, "offset": offset})
}
type skipped struct {
Hash string `json:"hash"`
Reason string `json:"reason"`
}
func addWhitelist(c *gin.Context) {
var body struct {
Hashes []string `json:"hashes"`
Text string `json:"text"`
Note string `json:"note"`
}
if err := c.ShouldBindJSON(&body); err != nil {
fail(c, http.StatusBadRequest, "invalid JSON")
return
}
candidates := append([]string{}, body.Hashes...)
candidates = append(candidates, strings.FieldsFunc(body.Text, func(r rune) bool {
return r == ',' || r == ';' || r == ' ' || r == '\n' || r == '\r' || r == '\t'
})...)
if len(candidates) == 0 {
fail(c, http.StatusBadRequest, "no hashes given")
return
}
if len(candidates) > 5000 {
fail(c, http.StatusBadRequest, "at most 5000 hashes per request")
return
}
note := body.Note
if len(note) > 256 {
note = note[:256]
}
var skips []skipped
seen := map[string]bool{}
valid := []string{}
for _, raw := range candidates {
h := strings.ToLower(strings.TrimSpace(raw))
switch {
case h == "":
continue
case !sha512Re.MatchString(h):
skips = append(skips, skipped{raw, "not a SHA-512 hex hash"})
case seen[h]:
skips = append(skips, skipped{h, "duplicate in request"})
default:
seen[h] = true
valid = append(valid, h)
}
}
added := 0
err := initializers.DB.Transaction(func(tx *gorm.DB) error {
existing := map[string]bool{}
for start := 0; start < len(valid); start += 500 {
end := min(start+500, len(valid))
var found []string
if err := tx.Model(&models.WhitelistedHash{}).Where("hash IN ?", valid[start:end]).Pluck("hash", &found).Error; err != nil {
return err
}
for _, h := range found {
existing[h] = true
}
}
rows := []models.WhitelistedHash{}
for _, h := range valid {
if existing[h] {
skips = append(skips, skipped{h, "already whitelisted"})
continue
}
rows = append(rows, models.WhitelistedHash{Hash: h, Note: note})
}
if len(rows) == 0 {
return nil
}
added = len(rows)
return tx.CreateInBatches(&rows, 200).Error
})
if err != nil {
fail(c, http.StatusInternalServerError, err.Error())
return
}
if added > 0 {
audit(c, "whitelist_add", map[string]any{"added": added, "note": note})
invalidate("overview")
}
if skips == nil {
skips = []skipped{}
}
c.JSON(http.StatusOK, gin.H{"added": added, "skipped": skips})
}
func deleteWhitelist(c *gin.Context) {
id, err := strconv.ParseUint(c.Param("id"), 10, 64)
if err != nil {
fail(c, http.StatusBadRequest, "invalid id")
return
}
var row models.WhitelistedHash
if err := initializers.DB.Unscoped().First(&row, id).Error; err != nil {
fail(c, http.StatusNotFound, "not found")
return
}
// remove every row of this hash, the table has no unique constraint
res := initializers.DB.Unscoped().Where("hash = ?", row.Hash).Delete(&models.WhitelistedHash{})
if res.Error != nil {
fail(c, http.StatusInternalServerError, res.Error.Error())
return
}
audit(c, "whitelist_remove", map[string]any{"hash": row.Hash})
invalidate("overview")
c.JSON(http.StatusOK, gin.H{"deleted": res.RowsAffected})
}
// hashUsage lists the addons that contain a file with the given hash.
func hashUsage(c *gin.Context) {
hash := strings.ToLower(c.Param("hash"))
var total int64
initializers.DB.Model(&models.AddonFile{}).Where("hash = ?", hash).Count(&total)
items := []struct {
AddonID string `json:"addonId"`
Name string `json:"name"`
Path string `json:"path"`
Version string `json:"version"`
}{}
err := initializers.DB.Raw(`SELECT f.addon_id, COALESCE(a.name, '') AS name, f.path, f.version
FROM addon_files f LEFT JOIN addons a ON a.id = f.addon_id
WHERE f.hash = ? AND f.deleted_at IS NULL ORDER BY a.subscriber_count DESC LIMIT 200`, hash).Scan(&items).Error
if err != nil {
fail(c, http.StatusInternalServerError, err.Error())
return
}
var wl int64
initializers.DB.Model(&models.WhitelistedHash{}).Where("hash = ?", hash).Count(&wl)
c.JSON(http.StatusOK, gin.H{"hash": hash, "whitelisted": wl > 0, "total": total, "items": items})
}
auth/auth_test.go
+80
View File
@@ -0,0 +1,80 @@
package auth
import (
"strings"
"testing"
"time"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
)
func TestSecretEqual(t *testing.T) {
if SecretEqual("", "") {
t.Fatal("empty configured secret must never match")
}
if SecretEqual("anything", "") {
t.Fatal("empty configured secret must never match")
}
if !SecretEqual("abc", "abc") {
t.Fatal("equal secrets must match")
}
if SecretEqual("abd", "abc") || SecretEqual("ab", "abc") {
t.Fatal("different secrets must not match")
}
}
func TestSessionToken(t *testing.T) {
initializers.AdminSecret = "first-secret-value"
initializers.AdminSessionTTL = time.Hour
now := time.Now()
tok, exp := IssueToken(now)
got, ok := VerifyToken(tok, now)
if !ok || got.Unix() != exp.Unix() {
t.Fatalf("fresh token should verify, ok=%v", ok)
}
if _, ok := VerifyToken(tok, now.Add(2*time.Hour)); ok {
t.Fatal("expired token must not verify")
}
parts := strings.Split(tok, ".")
parts[1] = "9999999999"
if _, ok := VerifyToken(strings.Join(parts, "."), now); ok {
t.Fatal("tampered token must not verify")
}
if _, ok := VerifyToken("garbage", now); ok {
t.Fatal("garbage must not verify")
}
initializers.AdminSecret = "rotated-secret-value"
if _, ok := VerifyToken(tok, now); ok {
t.Fatal("rotating the secret must invalidate sessions")
}
initializers.AdminSecret = ""
if _, ok := VerifyToken(tok, now); ok {
t.Fatal("no secret configured must reject all sessions")
}
}
func TestLoginLimiter(t *testing.T) {
l := NewLoginLimiter(3, time.Minute)
for range 3 {
if l.Blocked("1.2.3.4") {
t.Fatal("blocked too early")
}
l.Fail("1.2.3.4")
}
if !l.Blocked("1.2.3.4") {
t.Fatal("should be blocked after max failures")
}
if l.Blocked("5.6.7.8") {
t.Fatal("other IPs must not be affected")
}
l.Reset("1.2.3.4")
if l.Blocked("1.2.3.4") {
t.Fatal("reset should unblock")
}
}
auth/middleware.go
+142
View File
@@ -0,0 +1,142 @@
package auth
import (
"net/http"
"net/url"
"sync"
"time"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
"github.com/gin-gonic/gin"
)
// CSRFHeader must be sent with every state-changing cookie-authenticated
// request. Cross-site forms cannot set custom headers.
const CSRFHeader = "X-Admin-CSRF"
// RequireSecret guards the /back endpoints used by scrapers and indexers.
func RequireSecret(c *gin.Context) {
if initializers.SecretKey == "" {
c.AbortWithStatusJSON(http.StatusServiceUnavailable, gin.H{"error": "backend auth not configured"})
return
}
if !SecretEqual(c.GetHeader("X-SECRET-KEY"), initializers.SecretKey) {
c.AbortWithStatusJSON(http.StatusForbidden, gin.H{"error": "Forbidden"})
return
}
}
// RequireAdminHeader is the legacy admin check: X-ADMIN-KEY only, 403 on failure.
func RequireAdminHeader(c *gin.Context) {
if !SecretEqual(c.GetHeader("X-ADMIN-KEY"), initializers.AdminSecret) {
c.AbortWithStatusJSON(http.StatusForbidden, gin.H{"error": "Forbidden"})
return
}
c.Set("adminAuth", "header")
}
// RequireAdmin accepts either the X-ADMIN-KEY header (scripts) or a valid
// session cookie (dashboard). Cookie requests that change state must also carry
// the CSRF header and, when present, a same-host Origin.
func RequireAdmin(c *gin.Context) {
if initializers.AdminSecret == "" {
c.AbortWithStatusJSON(http.StatusServiceUnavailable, gin.H{"error": "admin disabled: ADMIN_SECRET not set"})
return
}
if key := c.GetHeader("X-ADMIN-KEY"); key != "" {
if !SecretEqual(key, initializers.AdminSecret) {
c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "invalid admin key"})
return
}
c.Set("adminAuth", "header")
return
}
cookie, err := c.Cookie(CookieName)
if err != nil || cookie == "" {
c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "not authenticated"})
return
}
exp, ok := VerifyToken(cookie, time.Now())
if !ok {
ClearSessionCookie(c)
c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "session expired"})
return
}
if c.Request.Method != http.MethodGet && c.Request.Method != http.MethodHead {
if c.GetHeader(CSRFHeader) != "1" || !sameOrigin(c) {
c.AbortWithStatusJSON(http.StatusForbidden, gin.H{"error": "csrf check failed"})
return
}
}
c.Set("adminAuth", "cookie")
c.Set("adminSessionExp", exp)
}
func sameOrigin(c *gin.Context) bool {
origin := c.GetHeader("Origin")
if origin == "" {
return true
}
u, err := url.Parse(origin)
if err != nil {
return false
}
return u.Host == c.Request.Host
}
// LoginLimiter throttles failed logins per client IP.
type LoginLimiter struct {
mu sync.Mutex
failures map[string][]time.Time
max int
window time.Duration
}
func NewLoginLimiter(max int, window time.Duration) *LoginLimiter {
return &LoginLimiter{failures: make(map[string][]time.Time), max: max, window: window}
}
func (l *LoginLimiter) prune(ip string, now time.Time) []time.Time {
kept := l.failures[ip][:0]
for _, t := range l.failures[ip] {
if now.Sub(t) < l.window {
kept = append(kept, t)
}
}
if len(kept) == 0 {
delete(l.failures, ip)
return nil
}
l.failures[ip] = kept
return kept
}
// Blocked reports whether the IP has used up its failed attempts.
func (l *LoginLimiter) Blocked(ip string) bool {
l.mu.Lock()
defer l.mu.Unlock()
return len(l.prune(ip, time.Now())) >= l.max
}
func (l *LoginLimiter) Fail(ip string) {
l.mu.Lock()
defer l.mu.Unlock()
now := time.Now()
l.failures[ip] = append(l.prune(ip, now), now)
// keep the map bounded if someone sprays from many addresses
if len(l.failures) > 10000 {
for k := range l.failures {
l.prune(k, now)
}
}
}
func (l *LoginLimiter) Reset(ip string) {
l.mu.Lock()
defer l.mu.Unlock()
delete(l.failures, ip)
}
auth/secret.go
+19
View File
@@ -0,0 +1,19 @@
package auth
import (
"crypto/sha256"
"crypto/subtle"
)
// SecretEqual compares a client-supplied secret with the configured one in
// constant time. An empty configured secret never matches, so an unset secret
// disables access instead of letting requests without the header through.
func SecretEqual(given, want string) bool {
if want == "" {
return false
}
// hashing first keeps the comparison length-independent
g := sha256.Sum256([]byte(given))
w := sha256.Sum256([]byte(want))
return subtle.ConstantTimeCompare(g[:], w[:]) == 1
}
auth/session.go
+108
View File
@@ -0,0 +1,108 @@
package auth
import (
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"fmt"
"net/http"
"strconv"
"strings"
"time"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
"github.com/gin-gonic/gin"
)
// CookieName is the admin session cookie. It is scoped to /admin so it is never
// sent to the public API.
const CookieName = "rc_admin"
var b64 = base64.RawURLEncoding
// The signing key is derived from the admin secret, so rotating ADMIN_SECRET
// invalidates every issued session.
func sessionKey() []byte {
m := hmac.New(sha256.New, []byte(initializers.AdminSecret))
m.Write([]byte("rc-admin-session/v1"))
return m.Sum(nil)
}
func sign(payload string) string {
m := hmac.New(sha256.New, sessionKey())
m.Write([]byte(payload))
return b64.EncodeToString(m.Sum(nil))
}
// IssueToken creates a stateless session token: v1.<expiry>.<nonce>.<mac>
func IssueToken(now time.Time) (string, time.Time) {
exp := now.Add(initializers.AdminSessionTTL)
nonce := make([]byte, 16)
_, _ = rand.Read(nonce)
payload := fmt.Sprintf("v1.%d.%s", exp.Unix(), b64.EncodeToString(nonce))
return payload + "." + sign(payload), exp
}
// VerifyToken checks the signature and expiry and returns the expiry time.
func VerifyToken(token string, now time.Time) (time.Time, bool) {
if initializers.AdminSecret == "" {
return time.Time{}, false
}
idx := strings.LastIndexByte(token, '.')
if idx <= 0 {
return time.Time{}, false
}
payload, mac := token[:idx], token[idx+1:]
if !hmac.Equal([]byte(mac), []byte(sign(payload))) {
return time.Time{}, false
}
parts := strings.Split(payload, ".")
if len(parts) != 3 || parts[0] != "v1" {
return time.Time{}, false
}
expUnix, err := strconv.ParseInt(parts[1], 10, 64)
if err != nil {
return time.Time{}, false
}
exp := time.Unix(expUnix, 0)
if !now.Before(exp) {
return time.Time{}, false
}
return exp, true
}
func cookieSecure(c *gin.Context) bool {
switch initializers.AdminCookieSecure {
case "true", "1", "yes":
return true
case "false", "0", "no":
return false
}
return c.Request.TLS != nil || strings.EqualFold(c.GetHeader("X-Forwarded-Proto"), "https")
}
func SetSessionCookie(c *gin.Context, token string, exp time.Time) {
http.SetCookie(c.Writer, &http.Cookie{
Name: CookieName,
Value: token,
Path: "/admin",
Expires: exp,
MaxAge: int(time.Until(exp).Seconds()),
HttpOnly: true,
Secure: cookieSecure(c),
SameSite: http.SameSiteStrictMode,
})
}
func ClearSessionCookie(c *gin.Context) {
http.SetCookie(c.Writer, &http.Cookie{
Name: CookieName,
Value: "",
Path: "/admin",
MaxAge: -1,
HttpOnly: true,
Secure: cookieSecure(c),
SameSite: http.SameSiteStrictMode,
})
}
controllers/adminController.go
-30
View File
@@ -1,30 +0,0 @@
package controllers
import (
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
"github.com/gin-gonic/gin"
)
func AddWhitelistedHash(c *gin.Context) {
hash := c.PostForm("hash")
if hash == "" {
c.JSON(400, gin.H{"error": "Hash is required"})
return
}
hashAddon := models.WhitelistedHash{Hash: hash}
ret := initializers.DB.Find(&models.WhitelistedHash{}, "hash = ?", hash)
if ret.RowsAffected > 0 {
c.JSON(400, gin.H{"error": "Hash already whitelisted"})
return
}
if err := initializers.DB.Create(&hashAddon).Error; err != nil {
c.JSON(500, gin.H{"error": "Failed to add whitelisted hash"})
return
}
c.JSON(200, gin.H{"message": "Whitelisted hash added successfully"})
}
controllers/authController.go
+5 -15
View File
@@ -1,26 +1,16 @@
package controllers
import (
"net/http"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/auth"
"github.com/gin-gonic/gin"
)
// CheckAllowed guards the scraper/indexer endpoints (X-SECRET-KEY).
func CheckAllowed(c *gin.Context) {
secret := c.GetHeader("X-SECRET-KEY")
if secret != initializers.SecretKey {
c.JSON(http.StatusForbidden, gin.H{"error": "Forbidden"})
c.Abort()
return
}
auth.RequireSecret(c)
}
// CheckAdmin is the legacy header-only admin check (X-ADMIN-KEY).
func CheckAdmin(c *gin.Context) {
adminSecret := c.GetHeader("X-ADMIN-KEY")
if adminSecret != initializers.AdminSecret {
c.JSON(http.StatusForbidden, gin.H{"error": "Forbidden"})
c.Abort()
return
}
auth.RequireAdminHeader(c)
}
controllers/dependencyController.go
+7 -4
View File
@@ -4,6 +4,7 @@ import (
"strconv"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
"github.com/gin-gonic/gin"
"gorm.io/gorm"
@@ -53,6 +54,8 @@ func SetAddonDependencies(c *gin.Context) {
return
}
metrics.DependencyReports.Inc()
metrics.DependencyEdges.Add(float64(len(edges)))
c.JSON(200, gin.H{"status": "success", "stored": len(edges)})
}
@@ -69,7 +72,7 @@ func GetDependents(c *gin.Context) {
limit := parseLimit(c.Query("limit"))
offset := parseOffset(c.Query("offset"))
dependents, total, err := queryRelatedAddons(addonID, "ad.addon_id", "ad.dependency_id", limit, offset)
dependents, total, err := QueryRelatedAddons(addonID, "ad.addon_id", "ad.dependency_id", limit, offset)
if err != nil {
c.JSON(500, gin.H{"error": err.Error()})
return
@@ -97,7 +100,7 @@ func GetDependencies(c *gin.Context) {
limit := parseLimit(c.Query("limit"))
offset := parseOffset(c.Query("offset"))
dependencies, total, err := queryRelatedAddons(addonID, "ad.dependency_id", "ad.addon_id", limit, offset)
dependencies, total, err := QueryRelatedAddons(addonID, "ad.dependency_id", "ad.addon_id", limit, offset)
if err != nil {
c.JSON(500, gin.H{"error": err.Error()})
return
@@ -112,7 +115,7 @@ func GetDependencies(c *gin.Context) {
})
}
// queryRelatedAddons returns one page of addons linked to addonID through the
// QueryRelatedAddons returns one page of addons linked to addonID through the
// addon_dependencies edge table, plus the total number of matches.
//
// joinCol is the edge column joined to addons.id; filterCol is the edge column
@@ -120,7 +123,7 @@ func GetDependencies(c *gin.Context) {
//
// The addons.id tiebreaker in the ORDER BY is required: subscriber_count alone
// is not a unique ordering, so without it paging could repeat or skip rows.
func queryRelatedAddons(addonID, joinCol, filterCol string, limit, offset int) ([]models.AddonSearchResult, int, error) {
func QueryRelatedAddons(addonID, joinCol, filterCol string, limit, offset int) ([]models.AddonSearchResult, int, error) {
base := func() *gorm.DB {
return initializers.DB.Model(&models.Addon{}).
Joins("JOIN addon_dependencies ad ON "+joinCol+" = addons.id").
controllers/feController.go
+2
View File
@@ -6,6 +6,7 @@ import (
"strconv"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
"github.com/gin-gonic/gin"
)
@@ -114,6 +115,7 @@ func GetDuplicates(c *gin.Context) {
for _, file := range duplicateFiles {
duplicatesByAddonID[file.AddonID] = append(duplicatesByAddonID[file.AddonID], file)
}
metrics.DuplicatesFound.Observe(float64(len(duplicatesByAddonID)))
// Fetch addon information for each addon that has duplicates
var addonIDs []string
controllers/indexerController.go
+67 -21
View File
@@ -9,6 +9,7 @@ import (
"time"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
"github.com/gin-gonic/gin"
"gorm.io/gorm"
@@ -16,7 +17,7 @@ import (
func GetNextToBeIndexed(c *gin.Context) {
checkIndexingTimeout()
metrics.IndexerTimeoutsReset.Add(float64(ResetTimedOutIndexing(initializers.IndexTimeout)))
maxFileSize := c.Query("maxFileSize")
maxSize := 10_000_000_000
@@ -26,25 +27,31 @@ func GetNextToBeIndexed(c *gin.Context) {
}
}
queue := "priority"
addon, err := claimNextAddon(maxSize, true)
if err != nil {
metrics.IndexerClaims.Inc("error")
c.JSON(500, gin.H{"error": err.Error()})
return
}
if addon == nil {
queue = "normal"
addon, err = claimNextAddon(maxSize, false)
if err != nil {
metrics.IndexerClaims.Inc("error")
c.JSON(500, gin.H{"error": err.Error()})
return
}
}
if addon != nil {
metrics.IndexerClaims.Inc(queue)
c.JSON(200, gin.H{"guid": addon.ID, "currentVersion": addon.CurrentVersionNumber})
return
}
// nothing to index
metrics.IndexerClaims.Inc("empty")
c.JSON(200, gin.H{"guid": "", "currentVersion": ""})
}
@@ -96,6 +103,7 @@ func SaveIndexingResult(c *gin.Context) {
err := c.ShouldBindJSON(&result)
if err != nil {
metrics.IndexerSubmissions.Inc("invalid")
c.JSON(400, gin.H{"error": "Invalid JSON"})
return
}
@@ -103,6 +111,7 @@ func SaveIndexingResult(c *gin.Context) {
var addon models.Addon
ret := initializers.DB.Where("id = ?", result.GUID).First(&addon)
if ret.Error != nil {
metrics.IndexerSubmissions.Inc("not_found")
c.JSON(404, gin.H{"error": "Addon not found"})
return
}
@@ -110,25 +119,42 @@ func SaveIndexingResult(c *gin.Context) {
// get all files for this addon and delete them - old version
ret = initializers.DB.Where("addon_id = ?", addon.ID).Delete(&models.AddonFile{})
if ret.Error != nil {
metrics.IndexerSubmissions.Inc("error")
c.JSON(500, gin.H{"error": ret.Error.Error()})
return
}
ret = initializers.DB.CreateInBatches(&result.Files, 100)
if ret.Error != nil {
metrics.IndexerSubmissions.Inc("error")
c.JSON(500, gin.H{"error": ret.Error.Error()})
return
}
now := time.Now()
wasClaimed := addon.IsBeingIndexed && !addon.IndexStartTime.IsZero()
addon.CurrentVersionNumber = result.CurrentVersion
addon.IsBeingIndexed = false
addon.ToBeIndexed = false
addon.LastIndexedAt = &now
ret = initializers.DB.Save(&addon)
if ret.Error != nil {
metrics.IndexerSubmissions.Inc("error")
c.JSON(500, gin.H{"error": ret.Error.Error()})
return
}
outcome := "ok"
if !result.Success {
outcome = "unsuccessful"
}
metrics.IndexerSubmissions.Inc(outcome)
if wasClaimed {
metrics.IndexerJobDuration.Observe(now.Sub(addon.IndexStartTime).Seconds())
}
metrics.IndexerFilesPerSubmission.Observe(float64(len(result.Files)))
metrics.IndexerBytesIndexed.Add(float64(addon.CurrentVersionSize))
if initializers.DiscordWebhookURL != "" {
text := "Indexing of addon " + addon.Name + " (" + addon.ID + ") was successful."
colour := 2228479
@@ -179,7 +205,7 @@ func SaveIndexingResult(c *gin.Context) {
Embeds: []models.CustomEmbed{myEmbed},
}
err := SendCustomWebhook(initializers.DiscordWebhookURL, myHook)
err := sendWebhook("index", initializers.DiscordWebhookURL, myHook)
if err != nil {
fmt.Println("Error sending webhook:", err)
}
@@ -198,40 +224,60 @@ func DeleteAddon(c *gin.Context) {
var addon models.Addon
ret := initializers.DB.Where("id = ?", guid).First(&addon)
if ret.Error != nil {
metrics.ScraperDeletes.Inc("not_found")
c.JSON(404, gin.H{"error": "Addon not found"})
return
}
// Delete associated files
ret = initializers.DB.Where("addon_id = ?", addon.ID).Delete(&models.AddonFile{})
if ret.Error != nil {
c.JSON(500, gin.H{"error": ret.Error.Error()})
err := initializers.DB.Transaction(func(tx *gorm.DB) error {
return DeleteAddonCascade(tx, &addon)
})
if err != nil {
metrics.ScraperDeletes.Inc("error")
c.JSON(500, gin.H{"error": err.Error()})
return
}
metrics.ScraperDeletes.Inc("deleted")
c.JSON(200, gin.H{"status": "addon deleted"})
}
// DeleteAddonCascade soft-deletes an addon and its files and removes its
// outgoing dependency edges.
func DeleteAddonCascade(tx *gorm.DB, addon *models.Addon) error {
// Delete associated files
if err := tx.Where("addon_id = ?", addon.ID).Delete(&models.AddonFile{}).Error; err != nil {
return err
}
// Delete outgoing dependency edges; incoming edges stay - other addons
// still declare this dependency even if the addon row is gone
ret = initializers.DB.Where("addon_id = ?", addon.ID).Delete(&models.AddonDependency{})
if ret.Error != nil {
c.JSON(500, gin.H{"error": ret.Error.Error()})
return
if err := tx.Where("addon_id = ?", addon.ID).Delete(&models.AddonDependency{}).Error; err != nil {
return err
}
// Delete the addon
ret = initializers.DB.Delete(&addon)
if ret.Error != nil {
c.JSON(500, gin.H{"error": ret.Error.Error()})
return
}
c.JSON(200, gin.H{"status": "addon deleted"})
return tx.Delete(addon).Error
}
func checkIndexingTimeout() {
initializers.DB.Model(&models.Addon{}).
// ResetTimedOutIndexing releases claims older than olderThan so another
// indexer can pick the addon up. Returns the number of released claims.
func ResetTimedOutIndexing(olderThan time.Duration) int64 {
return initializers.DB.Model(&models.Addon{}).
Where("is_being_indexed = ?", true).
Where("index_start_time < ?", time.Now().Add(-30*time.Minute)).
Update("is_being_indexed", false)
Where("index_start_time < ?", time.Now().Add(-olderThan)).
Update("is_being_indexed", false).RowsAffected
}
// sendWebhook sends a Discord webhook and records the outcome.
func sendWebhook(kind, webhookURL string, hook models.CustomHook) error {
err := SendCustomWebhook(webhookURL, hook)
result := "ok"
if err != nil {
result = "error"
}
metrics.DiscordWebhooks.Inc(kind, result)
return err
}
func SendCustomWebhook(webhookURL string, hook models.CustomHook) error {
controllers/scraperController.go
+20 -13
View File
@@ -1,31 +1,26 @@
package controllers
import (
"bytes"
"fmt"
"io"
"strconv"
"sync"
"time"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
"github.com/gin-gonic/gin"
)
var webhookMu sync.Mutex
var lastWebhookTime time.Time
var nbrOfUnsentWebhooks int
func CreateAddon(c *gin.Context) {
var addonR models.Addon
body, err := io.ReadAll(c.Request.Body)
if err != nil {
c.JSON(400, gin.H{"error": "Failed to read request body"})
return
}
fmt.Println(string(body))
c.Request.Body = io.NopCloser(bytes.NewBuffer(body))
err = c.ShouldBindJSON(&addonR)
err := c.ShouldBindJSON(&addonR)
if err != nil {
metrics.ScraperAddons.Inc("invalid")
c.JSON(400, gin.H{"error": "Invalid JSON"})
return
}
@@ -37,17 +32,26 @@ func CreateAddon(c *gin.Context) {
result := initializers.DB.Where(&addon).First(&addon)
if result.Error != nil {
// addon does not exist, create it
initializers.DB.Create(&addonR)
if err := initializers.DB.Create(&addonR).Error; err != nil {
// usually a soft-deleted addon with the same ID; it can be restored from the admin dashboard
metrics.ScraperAddons.Inc("create_failed")
fmt.Println("Failed to create addon", addonR.ID, err)
} else {
metrics.ScraperAddons.Inc("created")
}
// send webhook about creation
webhookMu.Lock()
if time.Since(lastWebhookTime) < time.Second*10 {
nbrOfUnsentWebhooks++
webhookMu.Unlock()
return
}
lastWebhookTime = time.Now()
oldNbr := nbrOfUnsentWebhooks
nbrOfUnsentWebhooks = 0
webhookMu.Unlock()
go func() {
webhookURL := initializers.ScraperWebhookURL
@@ -82,7 +86,7 @@ func CreateAddon(c *gin.Context) {
Embeds: []models.CustomEmbed{myEmbed},
}
err := SendCustomWebhook(webhookURL, myHook)
err := sendWebhook("scraper", webhookURL, myHook)
if err != nil {
fmt.Println("Error sending webhook:", err)
}
@@ -105,6 +109,7 @@ func CreateAddon(c *gin.Context) {
addon.Blocked = addonR.Blocked
addon.ToBeIndexed = true
initializers.DB.Save(&addon)
metrics.ScraperAddons.Inc("updated")
// send webhook about update
go func() {
@@ -140,12 +145,14 @@ func CreateAddon(c *gin.Context) {
Embeds: []models.CustomEmbed{myEmbed},
}
err := SendCustomWebhook(webhookURL, myHook)
err := sendWebhook("scraper", webhookURL, myHook)
if err != nil {
fmt.Println("Error sending webhook:", err)
}
}
}()
} else {
metrics.ScraperAddons.Inc("unchanged")
}
}
c.JSON(200, gin.H{"status": "success"})
controllers/telemetryController.go
+53
View File
@@ -0,0 +1,53 @@
package controllers
import (
"net/http"
"time"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
"github.com/gin-gonic/gin"
)
const (
maxTelemetryBody = 1 << 20
maxTelemetryMetrics = 1000
maxTelemetryEvents = 200
)
// SubmitTelemetry receives metrics, events and a heartbeat from a service
// (indexer, scraper, ...). See models.TelemetryBatch for the payload.
func SubmitTelemetry(c *gin.Context) {
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxTelemetryBody)
var batch models.TelemetryBatch
if err := c.ShouldBindJSON(&batch); err != nil {
c.JSON(400, gin.H{"error": "Invalid JSON"})
return
}
if !metrics.ValidName(batch.Service) || batch.Service == metrics.MainService {
c.JSON(400, gin.H{"error": "service must match ^[a-z_][a-z0-9_]{0,63}$ and not be \"main\""})
return
}
if batch.Instance == "" || len(batch.Instance) > 128 {
c.JSON(400, gin.H{"error": "instance is required (max 128 characters)"})
return
}
if len(batch.Metrics) > maxTelemetryMetrics || len(batch.Events) > maxTelemetryEvents {
c.JSON(413, gin.H{"error": "too many metrics or events in one batch"})
return
}
now := time.Now()
if err := metrics.Heartbeat(&batch, c.ClientIP(), now); err != nil {
c.JSON(500, gin.H{"error": "failed to store heartbeat"})
return
}
result := metrics.Ingest(batch.Service, batch.Instance, batch.Metrics, now)
if err := metrics.RecordEvents(batch.Service, batch.Instance, batch.Events, now); err != nil {
c.JSON(500, gin.H{"error": "failed to store events"})
return
}
c.JSON(200, result)
}
docs/admin-dashboard.md
+124
View File
@@ -0,0 +1,124 @@
# Admin dashboard, metrics and telemetry
The API binary serves an admin dashboard at **`/admin/ui/`** (`/admin` redirects there).
It is a React app in `web/`, built into `web/dist` and embedded with `go:embed`.
## Access
- Sign in with `ADMIN_SECRET`. The server sets an HttpOnly, `SameSite=Strict`
session cookie (`ADMIN_SESSION_TTL_HOURS`, default 7 days).
- Rotating `ADMIN_SECRET` signs every session out.
- If `ADMIN_SECRET` is empty, the dashboard and admin API are disabled.
- Set `TRUSTED_PROXIES` behind Traefik/Dokploy. Without it, client IPs can be
spoofed via `X-Forwarded-For`, which affects visitor counts and login throttling.
## Admin API
The JSON API lives under `/admin/api`. Scripts can call it with the
`X-ADMIN-KEY: <ADMIN_SECRET>` header instead of a cookie. Cookie requests that
change data must also send `X-Admin-CSRF: 1`.
| Area | Endpoints |
|---|---|
| Stats | `GET overview`, `GET overview/duplicates`, `GET traffic?from&to`, `GET top?kind=addon\|search&days` |
| Queue | `GET queue`, `POST queue/reset-stuck` |
| Addons | `GET addons?q&status&type&sort&order&limit&offset`, `GET addons/:id`, `GET addons/:id/files` |
| Addon actions | `POST addons/:id/{priority\|to-be-indexed\|reindex\|reset-indexing\|recheck\|restore}` with body `{"value": bool}`, `DELETE addons/:id`, `DELETE addons/:id/purge?confirm=<id>`, `POST bulk/addons` |
| Whitelist | `GET whitelist`, `POST whitelist` with body `{"hashes":[], "text":"", "note":""}`, `DELETE whitelist/:id`, `GET hashes/:hash` |
| Metrics | `GET metrics/catalog`, `GET metrics/query?service&name&agg&groupBy&match&from&to&step` |
| Services | `GET services`, `GET services/events`, `DELETE services/:id` |
| System | `GET system`, `POST system/maintenance` with body `{"action": "refresh-stats\|flush-metrics\|rollup\|purge-deleted-files"}` |
The legacy endpoint `POST /admin/addWhitelistedHash` (form field `hash`, `X-ADMIN-KEY`) still works.
## Metrics
- Every request is timed. Latency is recorded per route as a histogram, plus
status class, caller group and web vs direct client.
- The handlers also emit metrics for indexer claims and submissions, job
duration, scraper creates/updates/deletes, Discord webhooks and admin actions.
- Every minute, gauges sample the queue depth and total addon count.
Storage:
- Data is aggregated in memory per minute and flushed to the database every
minute (`metric_series` and `metric_points` tables).
- Minute data is rolled up hourly.
- Retention is configurable: `METRICS_MINUTE_RETENTION_DAYS` (default 7) and
`METRICS_HOUR_RETENTION_DAYS` (default 180).
Website visitors are counted as a salted hash of IP and user agent. The salt
rotates daily and only lives in memory, so the count is approximate. The most
looked-up addons and search queries are stored per day in `daily_top_items`.
Prometheus:
- Set `METRICS_TOKEN` to expose `GET /metrics` in the Prometheus text format.
- Authenticate with `Authorization: Bearer <token>`.
- This lets you add an external Prometheus/Grafana later.
## Telemetry from indexers and scrapers
Services report to `POST /back/telemetry` with `X-SECRET-KEY`. Every push is
also a heartbeat. An instance is shown as offline after three missed
`heartbeatIntervalSec`.
```json
{
"service": "indexer", "instance": "indexer-1", "version": "1.4.0",
"startedAt": 1727950000, "heartbeatIntervalSec": 15, "status": "ok",
"metrics": [
{"name": "jobs_total", "type": "counter", "value": 3, "labels": {"result": "ok"}},
{"name": "data_window_bytes", "type": "gauge", "value": 7.1e9},
{"name": "download_seconds", "type": "summary", "count": 3, "sum": 41.2, "min": 8.1, "max": 20.3}
],
"events": [{"level": "warn", "message": "data limit reached", "fields": {"limitGB": 50}}]
}
```
Metric types:
- `counter`: the increase since the last push.
- `gauge`: the current value.
- `summary`: pre-aggregated values from the client.
Limits:
- Names must match `^[a-z_][a-z0-9_]{0,63}$`.
- At most 8 labels per metric.
- At most 1000 metrics and 200 events per push.
- At most 200 series per metric and 5000 external series in total. Anything
over these limits is dropped and counted.
A stdlib-only Go client is in `telemetry/`, as its own module:
```go
import "gitea.tbdevent.eu/TBD/reforger_crawler_main/telemetry"
c := telemetry.New(telemetry.Config{BaseURL: apiURL, SecretKey: secret, Service: "indexer", Version: version})
c.Start(ctx)
defer c.Close(context.Background())
c.Counter("addons_indexed_total", "result", "ok").Inc()
c.Summary("download_seconds").Observe(took.Seconds())
c.Gauge("data_window_bytes").Set(float64(used))
c.Event(telemetry.Warn, "data limit reached", nil)
```
Reported metrics show up in the dashboard under **Services** and in the **Metrics explorer**.
## Development
```sh
# API on SQLite (overrides config.yaml)
DB_DRIVER=sqlite SQLITE_PATH=dev.db SECRET_KEY=dev ADMIN_SECRET=devadminsecret1234 go run .
# UI with hot reload, proxied to :8083
npm --prefix web install
npm --prefix web run dev # http://localhost:5173/admin/ui/
# production build (needed before `go build` to embed the UI)
npm --prefix web run build
```
If the UI was not built, the binary still compiles and serves a placeholder page.
go.mod
+6 -2
View File
@@ -3,8 +3,8 @@ module gitea.tbdevent.eu/TBD/reforger_crawler_main
go 1.25.0
require (
github.com/bensch777/discord-webhook-golang v0.0.6
gopkg.in/yaml.v3 v3.0.1
gorm.io/driver/postgres v1.6.0
gorm.io/driver/sqlite v1.6.0
gorm.io/gorm v1.30.3
)
@@ -26,11 +26,13 @@ require (
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
github.com/rogpeppe/go-internal v1.16.0 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.12 // indirect
golang.org/x/arch v0.8.0 // indirect
@@ -39,7 +41,6 @@ require (
golang.org/x/sync v0.16.0 // indirect
golang.org/x/sys v0.28.0 // indirect
google.golang.org/protobuf v1.34.1 // indirect
gorm.io/driver/postgres v1.6.0 // indirect
)
require (
@@ -49,3 +50,6 @@ require (
github.com/mattn/go-sqlite3 v1.14.32 // indirect
golang.org/x/text v0.28.0 // indirect
)
// keep the admin UI dependencies out of ./... package patterns
ignore ./web/node_modules
go.sum
+9 -7
View File
@@ -1,5 +1,3 @@
github.com/bensch777/discord-webhook-golang v0.0.6 h1:91BMU6vKgymAMfRwtXPMUrKX+SUoPPHTDJHTFA/1Kgk=
github.com/bensch777/discord-webhook-golang v0.0.6/go.mod h1:GcIorMZAZaHZyQJkjNoYKvZ6VpZo8XLib/eD51xN7Is=
github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0=
github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
@@ -8,6 +6,7 @@ github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/
github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -48,6 +47,10 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02
github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
@@ -63,6 +66,8 @@ github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6
github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g=
github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -82,8 +87,6 @@ github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZ
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc=
golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
@@ -92,8 +95,6 @@ golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
@@ -102,8 +103,9 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IV
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
initializers/initialize.go
+105 -1
View File
@@ -5,7 +5,9 @@ import (
"fmt"
"log"
"os"
"strconv"
"strings"
"time"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
"gopkg.in/yaml.v3"
@@ -25,6 +27,18 @@ var DatabaseDriver string
var SQLitePath string
var PostgresDSN string
// Admin dashboard, metrics and telemetry settings
var TrustedProxies []string
var AdminSessionTTL time.Duration
var AdminCookieSecure string // auto|true|false
var MetricsEnabled bool
var MetricsToken string
var MetricsMinuteRetentionDays int
var MetricsHourRetentionDays int
var EventsRetentionDays int
var TopItemsRetentionDays int
var IndexTimeout time.Duration
func ConnectToDB() {
var (
db *gorm.DB
@@ -75,7 +89,10 @@ func ConnectToDB() {
DB = db
if err := DB.AutoMigrate(&models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{}, &models.AddonDependency{}); err != nil {
if err := DB.AutoMigrate(
&models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{}, &models.AddonDependency{},
&models.MetricSeries{}, &models.MetricPoint{}, &models.ServiceInstance{}, &models.ServiceEvent{}, &models.DailyTopItem{},
); err != nil {
log.Fatalf("Failed to run migrations: %v", err)
}
@@ -98,6 +115,21 @@ func ConnectToDB() {
ON addon_files(addon_id, hash, path, version) WHERE deleted_at IS NULL`).Error; err != nil {
log.Fatalf("Failed to create idx_addon_files_covering: %v", err)
}
// Indexes used by the indexing queue and the admin dashboard; failures are
// not fatal - they only cost performance
optionalIndexes := []string{
`CREATE INDEX IF NOT EXISTS idx_addons_queue ON addons(priority_indexing, updated_at)
WHERE to_be_indexed = true AND is_being_indexed = false AND deleted_at IS NULL`,
`CREATE INDEX IF NOT EXISTS idx_addons_created_at ON addons(created_at)`,
`CREATE INDEX IF NOT EXISTS idx_addons_subscriber_count ON addons(subscriber_count)`,
`CREATE INDEX IF NOT EXISTS idx_addons_current_version_size ON addons(current_version_size)`,
}
for _, stmt := range optionalIndexes {
if err := DB.Exec(stmt).Error; err != nil {
log.Printf("Warning: failed to create index: %v", err)
}
}
}
type Configuration struct {
@@ -116,6 +148,16 @@ type Configuration struct {
PostgresPassword string `yaml:"postgres_password"`
PostgresDBName string `yaml:"postgres_db_name"`
PostgresSSLMode string `yaml:"postgres_sslmode"`
TrustedProxies string `yaml:"trusted_proxies"`
AdminSessionTTLH int `yaml:"admin_session_ttl_hours"`
AdminCookieSecure string `yaml:"admin_cookie_secure"`
MetricsEnabled *bool `yaml:"metrics_enabled"`
MetricsToken string `yaml:"metrics_token"`
MetricsMinuteDays int `yaml:"metrics_minute_retention_days"`
MetricsHourDays int `yaml:"metrics_hour_retention_days"`
EventsDays int `yaml:"events_retention_days"`
TopItemsDays int `yaml:"top_items_retention_days"`
IndexTimeoutMin int `yaml:"index_timeout_minutes"`
LegacyPort string `yaml:"port"`
LegacyIP string `yaml:"ip"`
LegacySecret string `yaml:"secret"`
@@ -132,6 +174,14 @@ func Load() {
DatabaseDriver: "sqlite",
SQLitePath: "crawler.db",
PostgresSSLMode: "disable",
AdminSessionTTLH: 168,
AdminCookieSecure: "auto",
MetricsMinuteDays: 7,
MetricsHourDays: 180,
EventsDays: 30,
TopItemsDays: 90,
IndexTimeoutMin: 30,
}
file, err := os.ReadFile("config.yaml")
@@ -155,6 +205,17 @@ func Load() {
DatabaseDriver = strings.ToLower(configuration.DatabaseDriver)
SQLitePath = configuration.SQLitePath
TrustedProxies = splitList(configuration.TrustedProxies)
AdminSessionTTL = time.Duration(positiveOr(configuration.AdminSessionTTLH, 168)) * time.Hour
AdminCookieSecure = strings.ToLower(configuration.AdminCookieSecure)
MetricsEnabled = configuration.MetricsEnabled == nil || *configuration.MetricsEnabled
MetricsToken = configuration.MetricsToken
MetricsMinuteRetentionDays = positiveOr(configuration.MetricsMinuteDays, 7)
MetricsHourRetentionDays = positiveOr(configuration.MetricsHourDays, 180)
EventsRetentionDays = positiveOr(configuration.EventsDays, 30)
TopItemsRetentionDays = positiveOr(configuration.TopItemsDays, 90)
IndexTimeout = time.Duration(positiveOr(configuration.IndexTimeoutMin, 30)) * time.Minute
if strings.TrimSpace(configuration.PostgresDSN) != "" {
PostgresDSN = configuration.PostgresDSN
} else {
@@ -202,6 +263,49 @@ func applyEnvOverrides(c *Configuration) {
c.PostgresPassword = firstEnv([]string{"POSTGRES_PASSWORD"}, c.PostgresPassword)
c.PostgresDBName = firstEnv([]string{"POSTGRES_DB_NAME"}, c.PostgresDBName)
c.PostgresSSLMode = firstEnv([]string{"POSTGRES_SSLMODE"}, c.PostgresSSLMode)
c.TrustedProxies = firstEnv([]string{"TRUSTED_PROXIES"}, c.TrustedProxies)
c.AdminSessionTTLH = firstEnvInt([]string{"ADMIN_SESSION_TTL_HOURS"}, c.AdminSessionTTLH)
c.AdminCookieSecure = firstEnv([]string{"ADMIN_COOKIE_SECURE"}, c.AdminCookieSecure)
if v := firstEnv([]string{"METRICS_ENABLED"}, ""); v != "" {
enabled := v == "1" || strings.EqualFold(v, "true") || strings.EqualFold(v, "yes")
c.MetricsEnabled = &enabled
}
c.MetricsToken = firstEnv([]string{"METRICS_TOKEN"}, c.MetricsToken)
c.MetricsMinuteDays = firstEnvInt([]string{"METRICS_MINUTE_RETENTION_DAYS"}, c.MetricsMinuteDays)
c.MetricsHourDays = firstEnvInt([]string{"METRICS_HOUR_RETENTION_DAYS"}, c.MetricsHourDays)
c.EventsDays = firstEnvInt([]string{"EVENTS_RETENTION_DAYS"}, c.EventsDays)
c.TopItemsDays = firstEnvInt([]string{"TOP_ITEMS_RETENTION_DAYS"}, c.TopItemsDays)
c.IndexTimeoutMin = firstEnvInt([]string{"INDEX_TIMEOUT_MINUTES"}, c.IndexTimeoutMin)
}
func firstEnvInt(keys []string, fallback int) int {
raw := firstEnv(keys, "")
if raw == "" {
return fallback
}
val, err := strconv.Atoi(strings.TrimSpace(raw))
if err != nil {
log.Printf("Warning: ignoring invalid integer %q for %s", raw, strings.Join(keys, "/"))
return fallback
}
return val
}
func positiveOr(val, fallback int) int {
if val > 0 {
return val
}
return fallback
}
func splitList(raw string) []string {
var out []string
for _, part := range strings.Split(raw, ",") {
if part = strings.TrimSpace(part); part != "" {
out = append(out, part)
}
}
return out
}
func firstEnv(keys []string, fallback string) string {
main.go
+123 -18
View File
@@ -1,22 +1,49 @@
package main
import (
"context"
"errors"
"log"
"net/http"
"os/signal"
"strings"
"syscall"
"time"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/admin"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/auth"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/controllers"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/web"
"github.com/gin-gonic/gin"
)
// buildCommit is set at build time: -ldflags "-X main.buildCommit=<sha>"
var buildCommit = ""
func init() {
initializers.Load()
initializers.ConnectToDB()
}
func main() {
r := gin.Default()
warnAboutConfig()
r := gin.New()
r.Use(gin.LoggerWithConfig(gin.LoggerConfig{Skip: skipLog}), gin.Recovery())
if len(initializers.TrustedProxies) > 0 {
if err := r.SetTrustedProxies(initializers.TrustedProxies); err != nil {
log.Fatalf("Invalid TRUSTED_PROXIES: %v", err)
}
}
metrics.SetEnabled(initializers.MetricsEnabled)
if initializers.MetricsEnabled {
r.Use(metrics.Middleware())
}
r.Use(CORSMiddleware())
r.GET("/health", func(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{"status": "ok"})
})
@@ -29,14 +56,18 @@ func main() {
back.POST("/submitAddon", controllers.CheckAllowed, controllers.SaveIndexingResult)
back.DELETE("/deleteAddon", controllers.CheckAllowed, controllers.DeleteAddon)
back.POST("/addonDependencies", controllers.CheckAllowed, controllers.SetAddonDependencies)
back.POST("/telemetry", controllers.CheckAllowed, controllers.SubmitTelemetry)
}
admin := r.Group("/admin")
{
admin.POST("/addWhitelistedHash", controllers.CheckAdmin, controllers.AddWhitelistedHash)
admin.GET("/duplicates", controllers.CheckAdmin)
admin.POST("/setPriority", controllers.CheckAdmin)
admin.POST("/setToBeIndexed", controllers.CheckAdmin)
// admin JSON API, legacy admin endpoint and the embedded dashboard
admin.Register(r)
web.Register(r)
if initializers.MetricsToken != "" {
r.GET("/metrics", requireMetricsToken, func(c *gin.Context) {
c.Header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
metrics.WritePrometheus(c.Writer)
})
}
v1 := r.Group("/v1")
@@ -47,11 +78,95 @@ func main() {
v1.GET("/getPossible", controllers.GetPossibleAddons)
}
r.Run(initializers.ServerHost + ":" + initializers.ServerPort)
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
version := buildCommit
if version == "" {
version = "dev"
}
admin.BuildCommit = buildCommit
metrics.SetVersion(version)
if initializers.MetricsEnabled {
metrics.Start(ctx)
}
admin.StartBackground(ctx)
srv := &http.Server{
Addr: initializers.ServerHost + ":" + initializers.ServerPort,
Handler: r,
ReadHeaderTimeout: 10 * time.Second,
}
go func() {
log.Printf("Listening on %s (admin dashboard at /admin/ui/)", srv.Addr)
if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
log.Fatalf("Server error: %v", err)
}
}()
<-ctx.Done()
log.Println("Shutting down...")
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := srv.Shutdown(shutdownCtx); err != nil {
log.Printf("Shutdown error: %v", err)
}
if initializers.MetricsEnabled {
metrics.FlushAll()
}
}
// skipLog keeps health checks, dashboard assets and dashboard polling out of
// the access log.
func skipLog(c *gin.Context) bool {
p := c.Request.URL.Path
if p == "/health" || strings.HasPrefix(p, "/admin/ui") {
return true
}
return strings.HasPrefix(p, "/admin/api") && c.Request.Method == http.MethodGet && c.Writer.Status() < 400
}
func requireMetricsToken(c *gin.Context) {
token := strings.TrimPrefix(c.GetHeader("Authorization"), "Bearer ")
if auth.SecretEqual(token, initializers.MetricsToken) {
return
}
// also allow a logged-in admin to look at the raw exposition
if auth.SecretEqual(c.GetHeader("X-ADMIN-KEY"), initializers.AdminSecret) {
return
}
if cookie, err := c.Cookie(auth.CookieName); err == nil {
if _, ok := auth.VerifyToken(cookie, time.Now()); ok {
return
}
}
c.Header("WWW-Authenticate", `Bearer realm="metrics"`)
c.AbortWithStatus(http.StatusUnauthorized)
}
func warnAboutConfig() {
if initializers.SecretKey == "" {
log.Println("Warning: SECRET_KEY is not set - /back endpoints are disabled")
}
if initializers.AdminSecret == "" {
log.Println("Warning: ADMIN_SECRET is not set - the admin dashboard is disabled")
} else if len(initializers.AdminSecret) < 16 {
log.Println("Warning: ADMIN_SECRET is shorter than 16 characters")
}
if len(initializers.TrustedProxies) == 0 {
log.Println("Warning: TRUSTED_PROXIES is not set - client IPs from X-Forwarded-For are trusted from any source")
}
}
func CORSMiddleware() gin.HandlerFunc {
return func(c *gin.Context) {
// the admin API and worker endpoints are never called cross-origin
p := c.Request.URL.Path
if strings.HasPrefix(p, "/admin") || strings.HasPrefix(p, "/back") || p == "/metrics" {
c.Next()
return
}
orig := c.Request.Header.Clone().Get("Origin")
c.Writer.Header().Set("Access-Control-Allow-Origin", orig)
c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
@@ -66,13 +181,3 @@ func CORSMiddleware() gin.HandlerFunc {
c.Next()
}
}
func OptionsMiddleware() gin.HandlerFunc {
return func(c *gin.Context) {
if c.Request.Method != "OPTIONS" {
c.Next()
} else {
c.AbortWithStatus(http.StatusOK)
}
}
}
metrics/aggregator.go
+286
View File
@@ -0,0 +1,286 @@
package metrics
import (
"encoding/json"
"math"
"sort"
"sync"
"sync/atomic"
)
// disabled turns all recording into no-ops (METRICS_ENABLED=false).
var disabled atomic.Bool
// SetEnabled switches metric recording on or off.
func SetEnabled(on bool) { disabled.Store(!on) }
// Agg is the aggregate of all samples of one series inside one bucket.
type Agg struct {
Count int64
Sum float64
Min float64
Max float64
Last float64
LastTs int64
Hist []uint64 // len(bounds)+1, last element is +Inf
}
func (a *Agg) add(v float64, ts int64, bounds []float64) {
if a.Count == 0 {
a.Min, a.Max = v, v
} else {
a.Min = math.Min(a.Min, v)
a.Max = math.Max(a.Max, v)
}
a.Count++
a.Sum += v
if ts >= a.LastTs {
a.Last, a.LastTs = v, ts
}
if len(bounds) > 0 {
if a.Hist == nil {
a.Hist = make([]uint64, len(bounds)+1)
}
a.Hist[bucketIndex(bounds, v)]++
}
}
func (a *Agg) addSummary(count int64, sum, min, max float64, ts int64) {
if count <= 0 {
return
}
if a.Count == 0 {
a.Min, a.Max = min, max
} else {
a.Min = math.Min(a.Min, min)
a.Max = math.Max(a.Max, max)
}
a.Count += count
a.Sum += sum
if ts >= a.LastTs {
a.Last, a.LastTs = sum/float64(count), ts
}
}
// merge folds b into a.
func (a *Agg) merge(b *Agg) {
if b == nil || b.Count == 0 {
return
}
if a.Count == 0 {
a.Min, a.Max = b.Min, b.Max
} else {
a.Min = math.Min(a.Min, b.Min)
a.Max = math.Max(a.Max, b.Max)
}
a.Count += b.Count
a.Sum += b.Sum
if b.LastTs >= a.LastTs {
a.Last, a.LastTs = b.Last, b.LastTs
}
if len(b.Hist) > 0 {
if len(a.Hist) == 0 {
a.Hist = make([]uint64, len(b.Hist))
}
if len(a.Hist) == len(b.Hist) {
for i, v := range b.Hist {
a.Hist[i] += v
}
}
}
}
func (a *Agg) clone() *Agg {
c := *a
if a.Hist != nil {
c.Hist = append([]uint64(nil), a.Hist...)
}
return &c
}
func (a *Agg) histJSON() string {
if len(a.Hist) == 0 {
return ""
}
b, _ := json.Marshal(a.Hist)
return string(b)
}
func bucketIndex(bounds []float64, v float64) int {
return sort.SearchFloat64s(bounds, v) // first bound >= v; len(bounds) = +Inf
}
// quantile estimates the q-quantile from histogram buckets with linear
// interpolation inside the bucket (like Prometheus), clamped to [min, max].
func quantile(a *Agg, bounds []float64, q float64) (float64, bool) {
if a == nil || a.Count == 0 || len(a.Hist) != len(bounds)+1 {
return 0, false
}
var total uint64
for _, c := range a.Hist {
total += c
}
if total == 0 {
return 0, false
}
rank := q * float64(total)
var cum uint64
for i, c := range a.Hist {
if c == 0 {
continue
}
if float64(cum+c) >= rank {
lower := a.Min
if i > 0 {
lower = math.Max(bounds[i-1], a.Min)
}
upper := a.Max
if i < len(bounds) {
upper = math.Min(bounds[i], a.Max)
}
if upper < lower {
upper = lower
}
frac := (rank - float64(cum)) / float64(c)
return lower + (upper-lower)*frac, true
}
cum += c
}
return a.Max, true
}
type pointKey struct {
s *Series
bucket int64
}
// cumulative is the since-start state of a series, used for /metrics.
type cumulative struct {
count int64
sum float64
last float64
hist []uint64
}
// aggregator holds the not yet persisted minute buckets and cumulative state.
type aggregator struct {
mu sync.Mutex
open map[pointKey]*Agg
cum map[*Series]*cumulative
}
var defaultAgg = &aggregator{
open: make(map[pointKey]*Agg),
cum: make(map[*Series]*cumulative),
}
func minuteOf(ts int64) int64 { return ts - ts%60 }
func (g *aggregator) entry(s *Series, ts int64) (*Agg, *cumulative) {
k := pointKey{s, minuteOf(ts)}
a := g.open[k]
if a == nil {
a = &Agg{}
g.open[k] = a
}
c := g.cum[s]
if c == nil {
c = &cumulative{}
g.cum[s] = c
}
return a, c
}
func (g *aggregator) observe(s *Series, ts int64, v float64) {
if disabled.Load() {
return
}
g.mu.Lock()
a, c := g.entry(s, ts)
a.add(v, ts, s.Bounds)
c.count++
c.sum += v
c.last = v
if len(s.Bounds) > 0 {
if c.hist == nil {
c.hist = make([]uint64, len(s.Bounds)+1)
}
c.hist[bucketIndex(s.Bounds, v)]++
}
g.mu.Unlock()
reg.touch(s, ts)
}
func (g *aggregator) observeSummary(s *Series, ts int64, count int64, sum, min, max float64) {
if disabled.Load() {
return
}
g.mu.Lock()
a, c := g.entry(s, ts)
a.addSummary(count, sum, min, max, ts)
c.count += count
c.sum += sum
c.last = sum / float64(count)
g.mu.Unlock()
reg.touch(s, ts)
}
// takeClosed removes and returns all buckets that started before `before`.
func (g *aggregator) takeClosed(before int64) map[pointKey]*Agg {
g.mu.Lock()
defer g.mu.Unlock()
out := make(map[pointKey]*Agg)
for k, a := range g.open {
if k.bucket < before {
out[k] = a
delete(g.open, k)
}
}
return out
}
// putBack re-queues buckets after a failed flush.
func (g *aggregator) putBack(points map[pointKey]*Agg) {
g.mu.Lock()
defer g.mu.Unlock()
for k, a := range points {
if cur := g.open[k]; cur != nil {
cur.merge(a)
} else {
g.open[k] = a
}
}
}
// snapshotOpen copies the open buckets of the given series.
func (g *aggregator) snapshotOpen(want map[*Series]bool) map[pointKey]*Agg {
g.mu.Lock()
defer g.mu.Unlock()
out := make(map[pointKey]*Agg)
for k, a := range g.open {
if want[k.s] {
out[k] = a.clone()
}
}
return out
}
func (g *aggregator) openCount() int {
g.mu.Lock()
defer g.mu.Unlock()
return len(g.open)
}
func (g *aggregator) snapshotCum() map[*Series]cumulative {
g.mu.Lock()
defer g.mu.Unlock()
out := make(map[*Series]cumulative, len(g.cum))
for s, c := range g.cum {
cc := *c
if c.hist != nil {
cc.hist = append([]uint64(nil), c.hist...)
}
out[s] = cc
}
return out
}
metrics/internal.go
+38
View File
@@ -0,0 +1,38 @@
package metrics
// Metrics emitted by the API itself (service "main").
// HTTPBoundsMs are the latency histogram buckets in milliseconds.
var HTTPBoundsMs = []float64{1, 2, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000}
var (
HTTPDuration = NewHistogram("http_request_duration_ms", "HTTP request latency in milliseconds", HTTPBoundsMs,
"method", "route", "status", "group", "client")
HTTPResponseBytes = NewCounter("http_response_bytes_total", "HTTP response body bytes", "group")
IndexerClaims = NewCounter("indexer_claims_total", "Indexing jobs requested by indexers, by outcome", "queue")
IndexerTimeoutsReset = NewCounter("indexer_timeouts_reset_total", "Indexing claims released after the timeout")
IndexerSubmissions = NewCounter("indexer_submissions_total", "Indexing results submitted", "result")
IndexerJobDuration = NewHistogram("indexer_job_duration_seconds", "Time from claim to submitted result",
[]float64{1, 5, 15, 30, 60, 120, 300, 600, 1200, 1800, 3600})
IndexerFilesPerSubmission = NewHistogram("indexer_files_per_submission", "Files per indexing result",
[]float64{1, 10, 50, 100, 500, 1000, 5000, 10000, 50000})
IndexerBytesIndexed = NewCounter("indexer_bytes_indexed_total", "Addon bytes indexed")
ScraperAddons = NewCounter("scraper_addons_total", "Addon reports from scrapers", "action")
ScraperDeletes = NewCounter("scraper_addons_deleted_total", "Addon deletions requested by scrapers", "result")
DependencyReports = NewCounter("dependency_reports_total", "Dependency reports received")
DependencyEdges = NewCounter("dependency_edges_reported_total", "Dependency edges stored")
DuplicatesFound = NewHistogram("duplicates_found_addons", "Addons sharing files with a looked-up addon",
[]float64{0, 1, 2, 5, 10, 25, 50, 100})
DiscordWebhooks = NewCounter("discord_webhook_total", "Discord webhook sends", "kind", "result")
AdminActions = NewCounter("admin_actions_total", "Admin dashboard actions", "action")
AddonsTotal = NewGauge("addons_total", "Addons in the database (not deleted)")
QueueDepth = NewGauge("queue_depth", "Addons waiting to be indexed", "kind")
BeingIndexed = NewGauge("being_indexed", "Addons currently claimed by an indexer")
VisitorsUnique = NewGauge("visitors_unique", "Unique website visitors so far today (approximate)", "kind")
TelemetrySamples = NewCounter("telemetry_samples_total", "Telemetry samples received", "result")
PointsDropped = NewCounter("metrics_points_dropped_total", "Metric points dropped after failed writes")
)
metrics/metrics_test.go
+216
View File
@@ -0,0 +1,216 @@
package metrics
import (
"math"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
)
func setupDB(t *testing.T) {
t.Helper()
db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "m.db")), &gorm.Config{Logger: logger.Discard})
if err != nil {
t.Fatal(err)
}
if err := db.AutoMigrate(&models.Addon{}, &models.MetricSeries{}, &models.MetricPoint{},
&models.ServiceInstance{}, &models.ServiceEvent{}, &models.DailyTopItem{}); err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if sqlDB, err := db.DB(); err == nil {
sqlDB.Close()
}
})
initializers.DB = db
initializers.MetricsMinuteRetentionDays = 7
initializers.MetricsHourRetentionDays = 180
// fresh in-memory state per test
reg = &registry{series: map[seriesKey]*Series{}, perName: map[string]int{}, lastSeen: map[*Series]int64{}}
defaultAgg = &aggregator{open: map[pointKey]*Agg{}, cum: map[*Series]*cumulative{}}
seriesID = map[*Series]uint{}
rolledUntil.Store(0)
for _, f := range families {
f.cache = map[string]*Series{}
}
}
func TestQuantileInterpolation(t *testing.T) {
bounds := []float64{10, 20, 30}
a := &Agg{}
for i := 1; i <= 100; i++ {
a.add(float64(i%30)+0.5, 0, bounds)
}
p50, ok := quantile(a, bounds, 0.5)
if !ok || p50 < 10 || p50 > 20 {
t.Fatalf("p50 = %v, want within (10, 20]", p50)
}
p99, _ := quantile(a, bounds, 0.99)
if p99 > a.Max || p99 < 20 {
t.Fatalf("p99 = %v, want in [20, max=%v]", p99, a.Max)
}
if _, ok := quantile(&Agg{}, bounds, 0.5); ok {
t.Fatal("empty histogram must not produce a quantile")
}
}
func TestAggMerge(t *testing.T) {
a, b := &Agg{}, &Agg{}
a.add(5, 100, nil)
a.add(1, 101, nil)
b.add(9, 99, nil)
a.merge(b)
if a.Count != 3 || a.Sum != 15 || a.Min != 1 || a.Max != 9 || a.Last != 1 {
t.Fatalf("unexpected merge result %+v", a)
}
}
func TestConcurrentObserve(t *testing.T) {
setupDB(t)
f := NewCounter("test_concurrent_total", "test", "worker")
var wg sync.WaitGroup
for w := range 8 {
wg.Add(1)
go func(w int) {
defer wg.Done()
for range 1000 {
f.Inc(string(rune('a' + w%2)))
}
}(w)
}
wg.Wait()
var total float64
for s, c := range defaultAgg.snapshotCum() {
if s.Name == "test_concurrent_total" {
total += c.sum
}
}
if total != 8000 {
t.Fatalf("total = %v, want 8000", total)
}
}
func TestFlushQueryRollup(t *testing.T) {
setupDB(t)
h := NewHistogram("test_latency_ms", "test", []float64{1, 10, 100}, "route")
s := h.series([]string{"/a"})
// two hours ago, spread over three minutes, plus one point in the current minute
base := hourOf(time.Now().Unix()) - 2*3600
for i := range 30 {
defaultAgg.observe(s, base+int64(i%3)*60, float64(i))
}
now := time.Now().Unix()
defaultAgg.observe(s, now, 50)
if err := flush(minuteOf(now)); err != nil {
t.Fatal(err)
}
var rows int64
initializers.DB.Model(&models.MetricPoint{}).Where("resolution = ?", resMinute).Count(&rows)
if rows != 3 {
t.Fatalf("minute rows = %d, want 3 (current minute stays in memory)", rows)
}
// flushing the same bucket again must merge, not overwrite
defaultAgg.observe(s, base, 1000)
if err := flush(minuteOf(now)); err != nil {
t.Fatal(err)
}
var p models.MetricPoint
initializers.DB.Where("resolution = ? AND bucket = ?", resMinute, base).First(&p)
if p.Count != 11 || p.Max != 1000 {
t.Fatalf("merged point = %+v, want count 11 and max 1000", p)
}
// query sees DB rows plus the unflushed current minute
sel := Selector{Service: MainService, Name: "test_latency_ms"}
res, err := Query(sel, "count", now-3*3600, now, 60)
if err != nil {
t.Fatal(err)
}
var count float64
for _, pt := range res.Series[0].Points {
if pt.V != nil {
count += *pt.V
}
}
if count != 32 {
t.Fatalf("queried count = %v, want 32", count)
}
if err := rollup(now); err != nil {
t.Fatal(err)
}
var hourly models.MetricPoint
if err := initializers.DB.Where("resolution = ? AND bucket = ?", resHour, base).First(&hourly).Error; err != nil {
t.Fatalf("missing hourly rollup: %v", err)
}
if hourly.Count != 31 || hourly.Max != 1000 {
t.Fatalf("hourly = %+v, want count 31 max 1000", hourly)
}
// rolling up again is idempotent
rolledUntil.Store(0)
if err := rollup(now); err != nil {
t.Fatal(err)
}
initializers.DB.Where("resolution = ? AND bucket = ?", resHour, base).First(&hourly)
if hourly.Count != 31 {
t.Fatalf("re-rollup changed count to %d", hourly.Count)
}
totals, err := Totals(sel, now-3*3600, now)
if err != nil || len(totals) != 1 || totals[0].P95 == nil || math.IsNaN(*totals[0].P95) {
t.Fatalf("totals = %+v, err %v", totals, err)
}
}
func TestIngestValidation(t *testing.T) {
setupDB(t)
now := time.Now()
res := Ingest("indexer", "i-1", []models.TelemetryMetric{
{Name: "ok_total", Type: "counter", Value: 2},
{Name: "Bad-Name", Type: "gauge", Value: 1},
{Name: "neg_total", Type: "counter", Value: -1},
{Name: "lat", Type: "summary", Count: 2, Sum: 3, Min: 1, Max: 2},
{Name: "ok_total", Type: "gauge", Value: 1},
{Name: "weird", Type: "nope"},
}, now)
if res.Accepted != 2 || res.Dropped != 4 {
t.Fatalf("accepted %d dropped %d (%v)", res.Accepted, res.Dropped, res.Reasons)
}
if res.Reasons["type_mismatch"] != 1 || res.Reasons["invalid_name"] != 1 {
t.Fatalf("unexpected reasons %v", res.Reasons)
}
}
func TestPrometheusOutput(t *testing.T) {
setupDB(t)
c := NewCounter("test_prom_total", "a \"quoted\" help", "kind")
c.Add(3, `va"l`)
h := NewHistogram("test_prom_ms", "hist", []float64{1, 10})
h.Observe(5)
var b strings.Builder
WritePrometheus(&b)
out := b.String()
for _, want := range []string{
`test_prom_total{kind="va\"l"} 3`,
`test_prom_ms_bucket{le="1"} 0`,
`test_prom_ms_bucket{le="10"} 1`,
`test_prom_ms_bucket{le="+Inf"} 1`,
`# TYPE test_prom_ms histogram`,
} {
if !strings.Contains(out, want) {
t.Errorf("missing %q in output", want)
}
}
}
metrics/pipeline.go
+156
View File
@@ -0,0 +1,156 @@
package metrics
import (
"context"
"log"
"math"
"sync"
"time"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
)
var startOnce sync.Once
// Start launches the background jobs: minute flush and samplers, top-N
// persistence, hourly rollup and retention, and service heartbeat checks.
func Start(ctx context.Context) {
startOnce.Do(func() {
Top.seed()
go every(ctx, "flush", time.Minute, 5*time.Second, func(now time.Time) {
sample()
if err := flush(minuteOf(now.Unix())); err != nil {
log.Printf("metrics: flush failed: %v", err)
}
})
go every(ctx, "top", 5*time.Minute, 0, func(time.Time) {
if err := Top.flush(); err != nil {
log.Printf("metrics: top items flush failed: %v", err)
}
})
go every(ctx, "rollup", time.Hour, 2*time.Minute, func(now time.Time) {
RunMaintenance(now)
})
go every(ctx, "services", 30*time.Second, 0, func(now time.Time) {
selfHeartbeat(now)
checkServices(now)
})
// initialize the rollup watermark and service state right away
go func() {
selfHeartbeat(time.Now())
checkServices(time.Now())
if err := rollup(time.Now().Unix()); err != nil {
log.Printf("metrics: rollup failed: %v", err)
}
}()
})
}
// RunMaintenance runs the rollup and retention jobs.
func RunMaintenance(now time.Time) {
if err := rollup(now.Unix()); err != nil {
log.Printf("metrics: rollup failed: %v", err)
}
if err := retention(now); err != nil {
log.Printf("metrics: retention failed: %v", err)
}
}
// every runs fn on a fixed period aligned to wall clock (+offset).
func every(ctx context.Context, name string, period, offset time.Duration, fn func(time.Time)) {
for {
now := time.Now()
next := now.Truncate(period).Add(offset)
if !next.After(now) {
next = next.Add(period)
}
select {
case <-ctx.Done():
return
case t := <-time.After(time.Until(next)):
func() {
defer func() {
if r := recover(); r != nil {
log.Printf("metrics: %s job panicked: %v", name, r)
}
}()
fn(t)
}()
}
}
}
// FlushAll writes everything still in memory; used on shutdown.
func FlushAll() {
sample()
if err := flush(math.MaxInt64); err != nil {
log.Printf("metrics: final flush failed: %v", err)
}
if err := Top.flush(); err != nil {
log.Printf("metrics: final top items flush failed: %v", err)
}
}
// QueueCounts is a cheap snapshot of the indexing queue.
type QueueCounts struct {
Total int64 `json:"total"`
Queued int64 `json:"queued"`
Priority int64 `json:"priority"`
Indexing int64 `json:"indexing"`
}
// LoadQueueCounts counts addons by queue state with one query.
func LoadQueueCounts() (QueueCounts, error) {
var qc QueueCounts
err := initializers.DB.Raw(`SELECT COUNT(*) AS total,
COALESCE(SUM(CASE WHEN to_be_indexed = ? AND is_being_indexed = ? THEN 1 ELSE 0 END), 0) AS queued,
COALESCE(SUM(CASE WHEN to_be_indexed = ? AND is_being_indexed = ? AND priority_indexing = ? THEN 1 ELSE 0 END), 0) AS priority,
COALESCE(SUM(CASE WHEN is_being_indexed = ? THEN 1 ELSE 0 END), 0) AS indexing
FROM addons WHERE deleted_at IS NULL`,
true, false, true, false, true, true).Scan(&qc).Error
return qc, err
}
// sample records gauges that are read from the database or memory.
func sample() {
if qc, err := LoadQueueCounts(); err == nil {
AddonsTotal.Set(float64(qc.Total))
QueueDepth.Set(float64(qc.Queued), "total")
QueueDepth.Set(float64(qc.Priority), "priority")
BeingIndexed.Set(float64(qc.Indexing))
}
humans, bots := liveVisitors()
VisitorsUnique.Set(float64(humans), "human")
VisitorsUnique.Set(float64(bots), "bot")
}
// PipelineStatus describes the health of the metrics pipeline.
type PipelineStatus struct {
Enabled bool `json:"enabled"`
LastFlush int64 `json:"lastFlush"`
LastFlushMs int64 `json:"lastFlushMs"`
LastFlushErr string `json:"lastFlushError"`
LastRollup int64 `json:"lastRollup"`
LastRetention int64 `json:"lastRetention"`
RolledUntil int64 `json:"rolledUntil"`
OpenPoints int `json:"openPoints"`
LiveSeries int `json:"liveSeries"`
DroppedPoints int64 `json:"droppedPoints"`
}
func Status() PipelineStatus {
status.mu.Lock()
defer status.mu.Unlock()
return PipelineStatus{
Enabled: initializers.MetricsEnabled,
LastFlush: status.LastFlush,
LastFlushMs: status.LastFlushMs,
LastFlushErr: status.LastFlushErr,
LastRollup: status.LastRollup,
LastRetention: status.LastRetention,
RolledUntil: rolledUntil.Load(),
OpenPoints: defaultAgg.openCount(),
LiveSeries: len(reg.all()),
DroppedPoints: status.Dropped,
}
}
metrics/prometheus.go
+181
View File
@@ -0,0 +1,181 @@
package metrics
import (
"bufio"
"io"
"runtime"
"sort"
"strconv"
"strings"
"time"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
)
// WritePrometheus writes all cumulative series in the Prometheus text format
// (version 0.0.4). External metrics are exposed as <service>_<name> with an
// "instance" label; counters are totals since this process started.
func WritePrometheus(w io.Writer) {
bw := bufio.NewWriter(w)
defer bw.Flush()
cum := defaultAgg.snapshotCum()
type entry struct {
s *Series
c cumulative
}
byName := map[string][]entry{}
meta := map[string]*Series{}
for s, c := range cum {
name := s.Name
if s.Service != MainService {
name = s.Service + "_" + s.Name
}
byName[name] = append(byName[name], entry{s, c})
if meta[name] == nil {
meta[name] = s
}
}
names := make([]string, 0, len(byName))
for n := range byName {
names = append(names, n)
}
sort.Strings(names)
for _, name := range names {
s := meta[name]
writeHeader(bw, name, s.Help, promType(s.Kind))
entries := byName[name]
sort.Slice(entries, func(i, j int) bool {
if entries[i].s.Instance != entries[j].s.Instance {
return entries[i].s.Instance < entries[j].s.Instance
}
return entries[i].s.LabelsJSON < entries[j].s.LabelsJSON
})
for _, e := range entries {
labels := e.s.Labels
if e.s.Service != MainService {
labels = withLabel(labels, "instance", e.s.Instance)
}
switch e.s.Kind {
case Counter:
writeSample(bw, name, labels, e.c.sum)
case Gauge:
writeSample(bw, name, labels, e.c.last)
case Summary:
writeSample(bw, name+"_sum", labels, e.c.sum)
writeSample(bw, name+"_count", labels, float64(e.c.count))
case Histogram:
var acc uint64
for i, b := range e.s.Bounds {
if i < len(e.c.hist) {
acc += e.c.hist[i]
}
writeSample(bw, name+"_bucket", withLabel(labels, "le", formatFloat(b)), float64(acc))
}
writeSample(bw, name+"_bucket", withLabel(labels, "le", "+Inf"), float64(e.c.count))
writeSample(bw, name+"_sum", labels, e.c.sum)
writeSample(bw, name+"_count", labels, float64(e.c.count))
}
}
}
up := ServiceUp()
if len(up) > 0 {
writeHeader(bw, "reforger_service_up", "Whether a reporting service instance is sending heartbeats", "gauge")
keys := make([][2]string, 0, len(up))
for k := range up {
keys = append(keys, k)
}
sort.Slice(keys, func(i, j int) bool { return keys[i][0]+keys[i][1] < keys[j][0]+keys[j][1] })
for _, k := range keys {
v := 0.0
if up[k] {
v = 1
}
writeSample(bw, "reforger_service_up", map[string]string{"service": k[0], "instance": k[1]}, v)
}
}
var ms runtime.MemStats
runtime.ReadMemStats(&ms)
writeHeader(bw, "go_goroutines", "Number of goroutines", "gauge")
writeSample(bw, "go_goroutines", nil, float64(runtime.NumGoroutine()))
writeHeader(bw, "go_memstats_heap_alloc_bytes", "Heap bytes allocated and in use", "gauge")
writeSample(bw, "go_memstats_heap_alloc_bytes", nil, float64(ms.HeapAlloc))
writeHeader(bw, "go_memstats_sys_bytes", "Bytes obtained from the OS", "gauge")
writeSample(bw, "go_memstats_sys_bytes", nil, float64(ms.Sys))
writeHeader(bw, "go_gc_cycles_total", "Completed GC cycles", "counter")
writeSample(bw, "go_gc_cycles_total", nil, float64(ms.NumGC))
writeHeader(bw, "process_start_time_seconds", "Process start time", "gauge")
writeSample(bw, "process_start_time_seconds", nil, float64(startedAt.Unix()))
writeHeader(bw, "process_uptime_seconds", "Process uptime", "gauge")
writeSample(bw, "process_uptime_seconds", nil, time.Since(startedAt).Seconds())
if sqlDB, err := initializers.DB.DB(); err == nil {
st := sqlDB.Stats()
writeHeader(bw, "db_open_connections", "Open database connections", "gauge")
writeSample(bw, "db_open_connections", nil, float64(st.OpenConnections))
writeHeader(bw, "db_in_use_connections", "Database connections in use", "gauge")
writeSample(bw, "db_in_use_connections", nil, float64(st.InUse))
writeHeader(bw, "db_wait_count_total", "Waits for a database connection", "counter")
writeSample(bw, "db_wait_count_total", nil, float64(st.WaitCount))
writeHeader(bw, "db_wait_seconds_total", "Time spent waiting for a database connection", "counter")
writeSample(bw, "db_wait_seconds_total", nil, st.WaitDuration.Seconds())
}
}
func promType(k Kind) string {
switch k {
case Counter:
return "counter"
case Histogram:
return "histogram"
case Summary:
return "summary"
}
return "gauge"
}
func withLabel(labels map[string]string, k, v string) map[string]string {
out := make(map[string]string, len(labels)+1)
for lk, lv := range labels {
out[lk] = lv
}
out[k] = v
return out
}
func formatFloat(v float64) string { return strconv.FormatFloat(v, 'g', -1, 64) }
var helpEscaper = strings.NewReplacer(`\`, `\\`, "\n", `\n`)
var valueEscaper = strings.NewReplacer(`\`, `\\`, "\n", `\n`, `"`, `\"`)
func writeHeader(w *bufio.Writer, name, help, typ string) {
if help != "" {
w.WriteString("# HELP " + name + " " + helpEscaper.Replace(help) + "\n")
}
w.WriteString("# TYPE " + name + " " + typ + "\n")
}
func writeSample(w *bufio.Writer, name string, labels map[string]string, v float64) {
w.WriteString(name)
if len(labels) > 0 {
keys := make([]string, 0, len(labels))
for k := range labels {
keys = append(keys, k)
}
sort.Strings(keys)
w.WriteByte('{')
for i, k := range keys {
if i > 0 {
w.WriteByte(',')
}
w.WriteString(k + `="` + valueEscaper.Replace(labels[k]) + `"`)
}
w.WriteByte('}')
}
w.WriteByte(' ')
w.WriteString(formatFloat(v))
w.WriteByte('\n')
}
metrics/query.go
+526
View File
@@ -0,0 +1,526 @@
package metrics
import (
"encoding/json"
"fmt"
"math"
"sort"
"strconv"
"strings"
"time"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
)
const (
maxQuerySeries = 50
maxQueryPoints = 1500
)
// Selector picks series by service, name and exact label matches, and groups
// them by the GroupBy labels ("service" and "instance" work as labels too).
type Selector struct {
Service string
Name string
Match map[string]string
GroupBy []string
}
// Aggregations supported by Query and Totals.
var Aggregations = []string{"sum", "count", "rate", "avg", "min", "max", "last", "p50", "p90", "p95", "p99"}
func validAgg(agg string) bool {
for _, a := range Aggregations {
if a == agg {
return true
}
}
return false
}
// DefaultAgg picks a sensible aggregation for a metric kind.
func DefaultAgg(kind Kind) string {
switch kind {
case Counter:
return "sum"
case Gauge:
return "last"
case Histogram:
return "p95"
default:
return "avg"
}
}
// Point is [unix seconds, value or null] in JSON.
type Point struct {
T int64
V *float64
}
func (p Point) MarshalJSON() ([]byte, error) {
if p.V == nil {
return []byte("[" + strconv.FormatInt(p.T, 10) + ",null]"), nil
}
return []byte("[" + strconv.FormatInt(p.T, 10) + "," + strconv.FormatFloat(*p.V, 'g', 10, 64) + "]"), nil
}
type SeriesResult struct {
Labels map[string]string `json:"labels"`
Points []Point `json:"points"`
}
type QueryResult struct {
Name string `json:"name"`
Kind Kind `json:"kind"`
Agg string `json:"agg"`
Resolution int64 `json:"resolution"`
Step int64 `json:"step"`
From int64 `json:"from"`
To int64 `json:"to"`
Series []SeriesResult `json:"series"`
Truncated bool `json:"truncated"`
}
// matched is a series selected for a query, possibly only known from the DB.
type matched struct {
id uint
live *Series // nil when this process never saw the series
labels map[string]string
svc string
inst string
kind Kind
bounds []float64
}
func (m *matched) label(name string) string {
switch name {
case "service":
if v, ok := m.labels["service"]; ok {
return v
}
return m.svc
case "instance":
if v, ok := m.labels["instance"]; ok {
return v
}
return m.inst
}
return m.labels[name]
}
func (sel Selector) matches(m *matched) bool {
if sel.Service != "" && m.svc != sel.Service {
return false
}
for k, v := range sel.Match {
if m.label(k) != v {
return false
}
}
return true
}
func findSeries(sel Selector) ([]*matched, error) {
byKey := map[seriesKey]*matched{}
q := initializers.DB.Model(&models.MetricSeries{}).Where("name = ?", sel.Name)
if sel.Service != "" {
q = q.Where("service = ?", sel.Service)
}
var rows []models.MetricSeries
if err := q.Find(&rows).Error; err != nil {
return nil, err
}
for _, r := range rows {
m := &matched{id: r.ID, svc: r.Service, inst: r.Instance, kind: Kind(r.Kind), labels: map[string]string{}}
_ = json.Unmarshal([]byte(r.Labels), &m.labels)
if r.Bounds != "" {
_ = json.Unmarshal([]byte(r.Bounds), &m.bounds)
}
byKey[seriesKey{r.Service, r.Instance, r.Name, r.Labels}] = m
}
for _, s := range reg.all() {
if s.Name != sel.Name || (sel.Service != "" && s.Service != sel.Service) {
continue
}
k := s.key()
if m := byKey[k]; m != nil {
m.live = s
continue
}
byKey[k] = &matched{live: s, svc: s.Service, inst: s.Instance, kind: s.Kind, bounds: s.Bounds, labels: s.Labels}
}
out := make([]*matched, 0, len(byKey))
for _, m := range byKey {
if sel.matches(m) {
out = append(out, m)
}
}
return out, nil
}
type groupAcc struct {
labels map[string]string
steps map[int64]*Agg
lastSum map[int64]float64
}
// collect loads all matching data in [from, to] and folds it into groups and
// step buckets chosen by bucketFn. Gauge "last" values are summed across the
// series of a group (e.g. queue depth over all instances).
func collect(sel Selector, from, to, res int64, bucketFn func(int64) int64) (map[string]*groupAcc, Kind, []float64, error) {
series, err := findSeries(sel)
if err != nil {
return nil, "", nil, err
}
if len(series) == 0 {
return map[string]*groupAcc{}, "", nil, nil
}
kind, bounds := series[0].kind, series[0].bounds
perSeries := map[*matched]map[int64]*Agg{}
add := func(m *matched, ts int64, a *Agg) {
b := bucketFn(ts)
steps := perSeries[m]
if steps == nil {
steps = map[int64]*Agg{}
perSeries[m] = steps
}
if cur := steps[b]; cur != nil {
cur.merge(a)
} else {
steps[b] = a.clone()
}
}
byID := map[uint]*matched{}
var ids []uint
for _, m := range series {
if m.id != 0 {
byID[m.id] = m
ids = append(ids, m.id)
}
}
load := func(resolution int, lo, hi int64) error {
if lo > hi || len(ids) == 0 {
return nil
}
for start := 0; start < len(ids); start += 300 {
end := min(start+300, len(ids))
var rows []models.MetricPoint
err := initializers.DB.
Where("resolution = ? AND series_id IN ? AND bucket >= ? AND bucket <= ?", resolution, ids[start:end], lo, hi).
Find(&rows).Error
if err != nil {
return err
}
for i := range rows {
add(byID[rows[i].SeriesID], rows[i].Bucket, pointToAgg(&rows[i]))
}
}
return nil
}
if res == resHour {
cut := rolledUntil.Load()
if err := load(resHour, hourOf(from), cut-1); err != nil {
return nil, "", nil, err
}
if err := load(resMinute, max(cut, from), to); err != nil {
return nil, "", nil, err
}
} else if err := load(resMinute, minuteOf(from), to); err != nil {
return nil, "", nil, err
}
// not yet flushed buckets
live := map[*Series]*matched{}
want := map[*Series]bool{}
for _, m := range series {
if m.live != nil {
live[m.live] = m
want[m.live] = true
}
}
for k, a := range defaultAgg.snapshotOpen(want) {
if k.bucket >= minuteOf(from) && k.bucket <= to {
add(live[k.s], k.bucket, a)
}
}
groups := map[string]*groupAcc{}
for m, steps := range perSeries {
labels := make(map[string]string, len(sel.GroupBy))
parts := make([]string, len(sel.GroupBy))
for i, g := range sel.GroupBy {
labels[g] = m.label(g)
parts[i] = labels[g]
}
gk := strings.Join(parts, "\x00")
g := groups[gk]
if g == nil {
g = &groupAcc{labels: labels, steps: map[int64]*Agg{}, lastSum: map[int64]float64{}}
groups[gk] = g
}
for b, a := range steps {
if cur := g.steps[b]; cur != nil {
cur.merge(a)
} else {
g.steps[b] = a.clone()
}
g.lastSum[b] += a.Last
}
}
return groups, kind, bounds, nil
}
func evaluate(a *Agg, lastSum float64, agg string, kind Kind, bounds []float64, span int64) (float64, bool) {
if a == nil || a.Count == 0 {
return 0, false
}
switch agg {
case "count":
return float64(a.Count), true
case "sum":
return a.Sum, true
case "rate":
if span <= 0 {
return 0, false
}
if kind == Counter {
return a.Sum / float64(span), true
}
return float64(a.Count) / float64(span), true
case "avg":
return a.Sum / float64(a.Count), true
case "min":
return a.Min, true
case "max":
return a.Max, true
case "last":
return lastSum, true
case "p50", "p90", "p95", "p99":
q, _ := strconv.ParseFloat(agg[1:], 64)
return quantile(a, bounds, q/100)
}
return 0, false
}
func pickResolution(from, to, step int64) int64 {
minuteHorizon := time.Now().Unix() - int64(initializers.MetricsMinuteRetentionDays)*86400
if to-from > 48*3600 || step >= resHour || from < minuteHorizon {
return resHour
}
return resMinute
}
// Query returns one time series per group with values at every step.
func Query(sel Selector, agg string, from, to, step int64) (*QueryResult, error) {
if sel.Name == "" {
return nil, fmt.Errorf("name is required")
}
if to <= from {
return nil, fmt.Errorf("to must be after from")
}
res := pickResolution(from, to, step)
if step < res {
step = res
}
if (to-from)/step > maxQueryPoints {
step = (to - from) / maxQueryPoints
}
step = int64(math.Ceil(float64(step)/float64(res))) * res
align := func(ts int64) int64 { return ts - ((ts%step)+step)%step }
groups, kind, bounds, err := collect(sel, from, to, res, align)
if err != nil {
return nil, err
}
if agg == "" {
agg = DefaultAgg(kind)
}
if !validAgg(agg) {
return nil, fmt.Errorf("unknown aggregation %q", agg)
}
result := &QueryResult{Name: sel.Name, Kind: kind, Agg: agg, Resolution: res, Step: step, From: from, To: to, Series: []SeriesResult{}}
keys := make([]string, 0, len(groups))
for k := range groups {
keys = append(keys, k)
}
sort.Strings(keys)
if len(keys) > maxQuerySeries {
keys = keys[:maxQuerySeries]
result.Truncated = true
}
for _, k := range keys {
g := groups[k]
sr := SeriesResult{Labels: g.labels}
for b := align(from); b <= to; b += step {
p := Point{T: b}
if v, ok := evaluate(g.steps[b], g.lastSum[b], agg, kind, bounds, step); ok && !math.IsNaN(v) && !math.IsInf(v, 0) {
p.V = &v
}
sr.Points = append(sr.Points, p)
}
result.Series = append(result.Series, sr)
}
return result, nil
}
// Total is the whole-range aggregate of one group.
type Total struct {
Labels map[string]string `json:"labels"`
Count int64 `json:"count"`
Sum float64 `json:"sum"`
Min *float64 `json:"min"`
Max *float64 `json:"max"`
Avg *float64 `json:"avg"`
Last *float64 `json:"last"`
Rate *float64 `json:"rate"`
P50 *float64 `json:"p50"`
P95 *float64 `json:"p95"`
P99 *float64 `json:"p99"`
}
// Totals aggregates [from, to] into one value set per group.
func Totals(sel Selector, from, to int64) ([]Total, error) {
res := pickResolution(from, to, 0)
groups, kind, bounds, err := collect(sel, from, to, res, func(int64) int64 { return 0 })
if err != nil {
return nil, err
}
span := to - from
out := make([]Total, 0, len(groups))
for _, g := range groups {
a := g.steps[0]
if a == nil {
continue
}
t := Total{Labels: g.labels, Count: a.Count, Sum: a.Sum}
val := func(agg string) *float64 {
if v, ok := evaluate(a, g.lastSum[0], agg, kind, bounds, span); ok && !math.IsNaN(v) && !math.IsInf(v, 0) {
return &v
}
return nil
}
t.Min, t.Max, t.Avg, t.Last, t.Rate = val("min"), val("max"), val("avg"), val("last"), val("rate")
t.P50, t.P95, t.P99 = val("p50"), val("p95"), val("p99")
out = append(out, t)
}
sort.Slice(out, func(i, j int) bool { return out[i].Count > out[j].Count })
return out, nil
}
// Sum returns the summed value of a counter (or count of observations for
// other kinds) over the range, across all matching series.
func Sum(sel Selector, from, to int64) float64 {
sel.GroupBy = nil
totals, err := Totals(sel, from, to)
if err != nil || len(totals) == 0 {
return 0
}
if f := familyByName(sel.Name); f != nil && f.Kind != Counter {
return float64(totals[0].Count)
}
return totals[0].Sum
}
// CatalogEntry describes one metric name of one service.
type CatalogEntry struct {
Service string `json:"service"`
Name string `json:"name"`
Kind Kind `json:"kind"`
Help string `json:"help"`
Labels map[string][]string `json:"labels"`
Instances []string `json:"instances"`
Series int `json:"series"`
LastSeen int64 `json:"lastSeen"`
}
// Catalog lists every metric known in the DB or in memory.
func Catalog(service string) ([]CatalogEntry, error) {
type acc struct {
e CatalogEntry
labels map[string]map[string]bool
inst map[string]bool
keys map[seriesKey]bool
}
entries := map[string]*acc{}
addSeries := func(svc, inst, name string, kind Kind, help, labelsJSON string, labels map[string]string, lastSeen int64) {
k := svc + "/" + name
a := entries[k]
if a == nil {
a = &acc{e: CatalogEntry{Service: svc, Name: name, Kind: kind, Help: help}, labels: map[string]map[string]bool{}, inst: map[string]bool{}, keys: map[seriesKey]bool{}}
entries[k] = a
}
if a.e.Help == "" {
a.e.Help = help
}
sk := seriesKey{svc, inst, name, labelsJSON}
if !a.keys[sk] {
a.keys[sk] = true
a.e.Series++
}
a.inst[inst] = true
for lk, lv := range labels {
if a.labels[lk] == nil {
a.labels[lk] = map[string]bool{}
}
a.labels[lk][lv] = true
}
if lastSeen > a.e.LastSeen {
a.e.LastSeen = lastSeen
}
}
q := initializers.DB.Model(&models.MetricSeries{})
if service != "" {
q = q.Where("service = ?", service)
}
var rows []models.MetricSeries
if err := q.Find(&rows).Error; err != nil {
return nil, err
}
for _, r := range rows {
labels := map[string]string{}
_ = json.Unmarshal([]byte(r.Labels), &labels)
addSeries(r.Service, r.Instance, r.Name, Kind(r.Kind), r.Help, r.Labels, labels, r.LastSeen)
}
for _, s := range reg.all() {
if service != "" && s.Service != service {
continue
}
addSeries(s.Service, s.Instance, s.Name, s.Kind, s.Help, s.LabelsJSON, s.Labels, reg.seen(s))
}
out := make([]CatalogEntry, 0, len(entries))
for _, a := range entries {
a.e.Labels = map[string][]string{}
for lk, vals := range a.labels {
list := make([]string, 0, len(vals))
for v := range vals {
list = append(list, v)
}
sort.Strings(list)
a.e.Labels[lk] = list
}
for inst := range a.inst {
a.e.Instances = append(a.e.Instances, inst)
}
sort.Strings(a.e.Instances)
out = append(out, a.e)
}
sort.Slice(out, func(i, j int) bool {
if out[i].Service != out[j].Service {
return out[i].Service < out[j].Service
}
return out[i].Name < out[j].Name
})
return out, nil
}
metrics/registry.go
+284
View File
@@ -0,0 +1,284 @@
package metrics
import (
"encoding/json"
"errors"
"os"
"sort"
"strings"
"sync"
"time"
)
type Kind string
const (
Counter Kind = "counter" // value is an increment; Sum holds the total
Gauge Kind = "gauge" // value is a sample; Last holds the latest
Summary Kind = "summary" // pre-aggregated count/sum/min/max from a client
Histogram Kind = "histogram" // observations bucketed by Bounds
)
// MainService is the service name the API uses for its own metrics.
const MainService = "main"
var (
errSeriesLimit = errors.New("series limit reached")
errKindMismatch = errors.New("metric reported with a different type")
)
// Limits for series created from external telemetry.
const (
maxSeriesPerName = 200
maxExternal = 5000
)
var hostname = func() string {
h, err := os.Hostname()
if err != nil || h == "" {
return "api"
}
return h
}()
// Instance returns the instance name used for this process' own metrics.
func Instance() string { return hostname }
// Series is one metric name + label set from one service instance. Series are
// interned, so pointer equality means same series.
type Series struct {
Service string
Instance string
Name string
Labels map[string]string
LabelsJSON string
Kind Kind
Bounds []float64
Help string
}
type seriesKey struct{ service, instance, name, labels string }
func (s *Series) key() seriesKey {
return seriesKey{s.Service, s.Instance, s.Name, s.LabelsJSON}
}
// Label returns a label value, including the service/instance pseudo labels.
func (s *Series) Label(name string) string {
switch name {
case "service":
if v, ok := s.Labels["service"]; ok {
return v
}
return s.Service
case "instance":
if v, ok := s.Labels["instance"]; ok {
return v
}
return s.Instance
}
return s.Labels[name]
}
// canonicalLabels serializes labels deterministically (encoding/json sorts map keys).
func canonicalLabels(labels map[string]string) string {
if len(labels) == 0 {
return "{}"
}
b, _ := json.Marshal(labels)
return string(b)
}
type registry struct {
mu sync.RWMutex
series map[seriesKey]*Series
perName map[string]int // external series per service/name
external int
lastSeen map[*Series]int64
}
var reg = &registry{
series: make(map[seriesKey]*Series),
perName: make(map[string]int),
lastSeen: make(map[*Series]int64),
}
func (r *registry) get(service, instance, name string, labels map[string]string, kind Kind, bounds []float64, help string, external bool) (*Series, error) {
lj := canonicalLabels(labels)
k := seriesKey{service, instance, name, lj}
r.mu.RLock()
s := r.series[k]
r.mu.RUnlock()
if s != nil {
if s.Kind != kind {
return nil, errKindMismatch
}
return s, nil
}
r.mu.Lock()
defer r.mu.Unlock()
if s = r.series[k]; s != nil {
return s, nil
}
nameKey := service + "/" + name
if external {
if r.external >= maxExternal || r.perName[nameKey] >= maxSeriesPerName {
return nil, errSeriesLimit
}
r.external++
r.perName[nameKey]++
}
copied := make(map[string]string, len(labels))
for k, v := range labels {
copied[k] = v
}
s = &Series{
Service: service,
Instance: instance,
Name: name,
Labels: copied,
LabelsJSON: lj,
Kind: kind,
Bounds: bounds,
Help: help,
}
r.series[k] = s
return s, nil
}
func (r *registry) lookup(k seriesKey) *Series {
r.mu.RLock()
defer r.mu.RUnlock()
return r.series[k]
}
// all returns every known series, sorted by name then labels.
func (r *registry) all() []*Series {
r.mu.RLock()
out := make([]*Series, 0, len(r.series))
for _, s := range r.series {
out = append(out, s)
}
r.mu.RUnlock()
sort.Slice(out, func(i, j int) bool {
if out[i].Name != out[j].Name {
return out[i].Name < out[j].Name
}
if out[i].Service != out[j].Service {
return out[i].Service < out[j].Service
}
if out[i].Instance != out[j].Instance {
return out[i].Instance < out[j].Instance
}
return out[i].LabelsJSON < out[j].LabelsJSON
})
return out
}
func (r *registry) touch(s *Series, ts int64) {
r.mu.Lock()
if ts > r.lastSeen[s] {
r.lastSeen[s] = ts
}
r.mu.Unlock()
}
func (r *registry) seen(s *Series) int64 {
r.mu.RLock()
defer r.mu.RUnlock()
return r.lastSeen[s]
}
// Family is a metric emitted by this process, with a fixed set of label names.
type Family struct {
Name string
Help string
Kind Kind
Bounds []float64
LabelNames []string
mu sync.Mutex
cache map[string]*Series
}
var (
familiesMu sync.Mutex
families = map[string]*Family{}
)
func newFamily(name, help string, kind Kind, bounds []float64, labels []string) *Family {
f := &Family{Name: name, Help: help, Kind: kind, Bounds: bounds, LabelNames: labels, cache: map[string]*Series{}}
familiesMu.Lock()
families[name] = f
familiesMu.Unlock()
return f
}
func familyByName(name string) *Family {
familiesMu.Lock()
defer familiesMu.Unlock()
return families[name]
}
func NewCounter(name, help string, labels ...string) *Family {
return newFamily(name, help, Counter, nil, labels)
}
func NewGauge(name, help string, labels ...string) *Family {
return newFamily(name, help, Gauge, nil, labels)
}
func NewHistogram(name, help string, bounds []float64, labels ...string) *Family {
return newFamily(name, help, Histogram, bounds, labels)
}
func (f *Family) series(values []string) *Series {
key := strings.Join(values, "\x00")
f.mu.Lock()
defer f.mu.Unlock()
if s := f.cache[key]; s != nil {
return s
}
labels := make(map[string]string, len(f.LabelNames))
for i, n := range f.LabelNames {
if i < len(values) {
labels[n] = values[i]
} else {
labels[n] = ""
}
}
s, err := reg.get(MainService, Instance(), f.Name, labels, f.Kind, f.Bounds, f.Help, false)
if err != nil {
return nil
}
f.cache[key] = s
return s
}
// Add increments a counter. Zero increments are ignored.
func (f *Family) Add(v float64, labelValues ...string) {
if v == 0 {
return
}
if s := f.series(labelValues); s != nil {
defaultAgg.observe(s, time.Now().Unix(), v)
}
}
func (f *Family) Inc(labelValues ...string) { f.Add(1, labelValues...) }
// Set records a gauge sample.
func (f *Family) Set(v float64, labelValues ...string) {
if s := f.series(labelValues); s != nil {
defaultAgg.observe(s, time.Now().Unix(), v)
}
}
// Observe records a histogram observation.
func (f *Family) Observe(v float64, labelValues ...string) {
if s := f.series(labelValues); s != nil {
defaultAgg.observe(s, time.Now().Unix(), v)
}
}
metrics/services.go
+325
View File
@@ -0,0 +1,325 @@
package metrics
import (
"encoding/json"
"errors"
"fmt"
"math"
"regexp"
"sync"
"time"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
"gorm.io/gorm/clause"
)
var nameRe = regexp.MustCompile(`^[a-z_][a-z0-9_]{0,63}$`)
// ValidName reports whether a service, metric or label name is acceptable.
func ValidName(s string) bool { return nameRe.MatchString(s) }
const (
defaultHeartbeat = 30
maxLabels = 8
maxLabelValue = 128
)
// normalizeTs converts unix seconds or milliseconds to seconds and clamps it
// to [now-1h, now+1m]; anything outside becomes now.
func normalizeTs(ts int64, now int64) int64 {
if ts <= 0 {
return now
}
if ts > 1e12 {
ts /= 1000
}
if ts < now-3600 || ts > now+60 {
return now
}
return ts
}
// IngestResult reports what happened to a telemetry batch.
type IngestResult struct {
Accepted int `json:"accepted"`
Dropped int `json:"dropped"`
Reasons map[string]int `json:"reasons,omitempty"`
}
// Ingest records external metric samples for a service instance.
func Ingest(service, instance string, samples []models.TelemetryMetric, now time.Time) IngestResult {
res := IngestResult{Reasons: map[string]int{}}
drop := func(reason string) {
res.Dropped++
res.Reasons[reason]++
}
nowTs := now.Unix()
for _, m := range samples {
if !ValidName(m.Name) {
drop("invalid_name")
continue
}
if len(m.Labels) > maxLabels {
drop("too_many_labels")
continue
}
badLabel := false
for k, v := range m.Labels {
if !ValidName(k) || len(v) > maxLabelValue {
badLabel = true
break
}
}
if badLabel {
drop("invalid_label")
continue
}
var kind Kind
switch m.Type {
case "counter":
kind = Counter
case "gauge", "":
kind = Gauge
case "summary":
kind = Summary
default:
drop("invalid_type")
continue
}
if math.IsNaN(m.Value) || math.IsInf(m.Value, 0) || math.IsNaN(m.Sum) || math.IsInf(m.Sum, 0) {
drop("invalid_value")
continue
}
if kind == Counter && m.Value < 0 {
drop("negative_counter")
continue
}
if kind == Summary && m.Count <= 0 {
drop("empty_summary")
continue
}
help := m.Help
if len(help) > 256 {
help = help[:256]
}
s, err := reg.get(service, instance, m.Name, m.Labels, kind, nil, help, true)
if err != nil {
if errors.Is(err, errKindMismatch) {
drop("type_mismatch")
} else {
drop("series_limit")
}
continue
}
ts := normalizeTs(m.Ts, nowTs)
if kind == Summary {
lo, hi := m.Min, m.Max
if lo == 0 && hi == 0 {
avg := m.Sum / float64(m.Count)
lo, hi = avg, avg
}
defaultAgg.observeSummary(s, ts, m.Count, m.Sum, lo, hi)
} else {
defaultAgg.observe(s, ts, m.Value)
}
res.Accepted++
}
TelemetrySamples.Add(float64(res.Accepted), "accepted")
TelemetrySamples.Add(float64(res.Dropped), "dropped")
return res
}
func jsonString(v map[string]any) string {
if len(v) == 0 {
return ""
}
b, err := json.Marshal(v)
if err != nil {
return ""
}
return string(b)
}
func truncate(s string, n int) string {
if len(s) > n {
return s[:n]
}
return s
}
// Heartbeat upserts the service instance row.
func Heartbeat(b *models.TelemetryBatch, remoteAddr string, now time.Time) error {
interval := b.HeartbeatIntervalSec
if interval <= 0 {
interval = defaultHeartbeat
}
status := b.Status
if status == "" {
status = "ok"
}
row := models.ServiceInstance{
Service: b.Service,
Instance: b.Instance,
Version: truncate(b.Version, 64),
Status: truncate(status, 16),
StatusMessage: truncate(b.StatusMessage, 512),
StartedAt: normalizeStarted(b.StartedAt),
LastSeen: now.Unix(),
HeartbeatIntervalSec: interval,
RemoteAddr: truncate(remoteAddr, 64),
Meta: jsonString(b.Meta),
}
return initializers.DB.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "service"}, {Name: "instance"}},
DoUpdates: clause.AssignmentColumns([]string{
"version", "status", "status_message", "started_at", "last_seen",
"heartbeat_interval_sec", "remote_addr", "meta", "updated_at",
}),
}).Create(&row).Error
}
func normalizeStarted(ts int64) int64 {
if ts > 1e12 {
return ts / 1000
}
return ts
}
// RecordEvents stores service events.
func RecordEvents(service, instance string, events []models.TelemetryEvent, now time.Time) error {
if len(events) == 0 {
return nil
}
rows := make([]models.ServiceEvent, 0, len(events))
for _, e := range events {
level := e.Level
switch level {
case "debug", "info", "warn", "error":
case "warning":
level = "warn"
default:
level = "info"
}
ts := e.Ts
if ts <= 0 {
ts = now.UnixMilli()
} else if ts < 1e12 {
ts *= 1000
}
rows = append(rows, models.ServiceEvent{
Service: service, Instance: instance, Level: level,
Message: truncate(e.Message, 1024), Fields: jsonString(e.Fields), TS: ts,
})
}
return initializers.DB.CreateInBatches(&rows, 100).Error
}
// Event records a single event from the API itself.
func Event(level, message string, fields map[string]any) {
err := RecordEvents(MainService, Instance(), []models.TelemetryEvent{{Level: level, Message: message, Fields: fields}}, time.Now())
if err != nil {
fmt.Printf("metrics: failed to record event: %v\n", err)
}
}
// Online reports whether an instance has sent a heartbeat recently enough.
func Online(inst *models.ServiceInstance, now time.Time) bool {
interval := inst.HeartbeatIntervalSec
if interval <= 0 {
interval = defaultHeartbeat
}
return now.Unix()-inst.LastSeen < int64(3*interval)
}
var (
upMu sync.Mutex
upState = map[string]bool{} // "service/instance" -> online
)
// ServiceUp returns the last known online state of every instance.
func ServiceUp() map[[2]string]bool {
upMu.Lock()
defer upMu.Unlock()
out := make(map[[2]string]bool, len(upState))
for k, v := range upState {
var svc, inst string
for i := 0; i < len(k); i++ {
if k[i] == '/' {
svc, inst = k[:i], k[i+1:]
break
}
}
out[[2]string{svc, inst}] = v
}
return out
}
// checkServices records online/offline transitions as events.
func checkServices(now time.Time) {
var rows []models.ServiceInstance
if err := initializers.DB.Find(&rows).Error; err != nil {
return
}
upMu.Lock()
known := make(map[string]bool, len(rows))
type change struct {
inst models.ServiceInstance
online bool
}
var changes []change
for _, r := range rows {
k := r.Service + "/" + r.Instance
known[k] = true
online := Online(&r, now)
prev, seen := upState[k]
upState[k] = online
if seen && prev != online {
changes = append(changes, change{r, online})
}
}
for k := range upState {
if !known[k] {
delete(upState, k)
}
}
upMu.Unlock()
for _, ch := range changes {
level, msg := "info", "back online"
if !ch.online {
level, msg = "warn", "heartbeat lost"
}
_ = RecordEvents(ch.inst.Service, ch.inst.Instance, []models.TelemetryEvent{{
Level: level, Message: msg,
Fields: map[string]any{"kind": "status", "lastSeen": ch.inst.LastSeen},
}}, now)
}
}
var (
buildMu sync.Mutex
buildVersion = "dev"
startedAt = time.Now()
)
// SetVersion sets the version reported in the API's own heartbeat.
func SetVersion(v string) {
buildMu.Lock()
buildVersion = v
buildMu.Unlock()
}
// StartedAt is when this process started.
func StartedAt() time.Time { return startedAt }
func selfHeartbeat(now time.Time) {
buildMu.Lock()
v := buildVersion
buildMu.Unlock()
_ = Heartbeat(&models.TelemetryBatch{
Service: MainService, Instance: Instance(), Version: v,
StartedAt: startedAt.Unix(), HeartbeatIntervalSec: 30, Status: "ok",
}, "", now)
}
metrics/store.go
+371
View File
@@ -0,0 +1,371 @@
package metrics
import (
"encoding/json"
"fmt"
"log"
"sort"
"sync"
"sync/atomic"
"time"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
const (
resMinute = 60
resHour = 3600
// points that cannot be written for this long are dropped
maxRetryAge = 15 * 60
)
var (
idMu sync.Mutex
seriesID = map[*Series]uint{}
// rolledUntil is the start of the first hour without an hourly rollup.
rolledUntil atomic.Int64
// flushMu serializes writers of metric_points (flush, rollup, retention).
flushMu sync.Mutex
)
func pointToAgg(p *models.MetricPoint) *Agg {
a := &Agg{Count: p.Count, Sum: p.Sum, Min: p.Min, Max: p.Max, Last: p.Last, LastTs: p.Bucket}
if p.Hist != "" {
_ = json.Unmarshal([]byte(p.Hist), &a.Hist)
}
return a
}
func aggToPoint(id uint, res int, bucket int64, a *Agg) models.MetricPoint {
return models.MetricPoint{
SeriesID: id, Resolution: res, Bucket: bucket,
Count: a.Count, Sum: a.Sum, Min: a.Min, Max: a.Max, Last: a.Last, Hist: a.histJSON(),
}
}
// resolveIDs makes sure every series has a metric_series row.
func resolveIDs(list []*Series) (map[*Series]uint, error) {
out := make(map[*Series]uint, len(list))
var missing []*Series
idMu.Lock()
for _, s := range list {
if id, ok := seriesID[s]; ok {
out[s] = id
} else {
missing = append(missing, s)
}
}
idMu.Unlock()
for _, s := range missing {
bounds := ""
if len(s.Bounds) > 0 {
b, _ := json.Marshal(s.Bounds)
bounds = string(b)
}
row := models.MetricSeries{
Service: s.Service, Instance: s.Instance, Name: s.Name, Labels: s.LabelsJSON,
Kind: string(s.Kind), Bounds: bounds, Help: s.Help, LastSeen: time.Now().Unix(),
}
if err := initializers.DB.Clauses(clause.OnConflict{DoNothing: true}).Create(&row).Error; err != nil {
return nil, err
}
if row.ID == 0 {
if err := initializers.DB.
Where("service = ? AND instance = ? AND name = ? AND labels = ?", s.Service, s.Instance, s.Name, s.LabelsJSON).
First(&row).Error; err != nil {
return nil, err
}
}
idMu.Lock()
seriesID[s] = row.ID
idMu.Unlock()
out[s] = row.ID
}
return out, nil
}
type pointID struct {
id uint
bucket int64
}
// persist merges the given buckets into existing rows and upserts them. The
// read-merge-write runs in one transaction; this process is the only writer.
func persist(tx *gorm.DB, res int, points map[pointID]*Agg) error {
if len(points) == 0 {
return nil
}
keys := make([]pointID, 0, len(points))
for k := range points {
keys = append(keys, k)
}
sort.Slice(keys, func(i, j int) bool {
if keys[i].id != keys[j].id {
return keys[i].id < keys[j].id
}
return keys[i].bucket < keys[j].bucket
})
for start := 0; start < len(keys); start += 400 {
end := min(start+400, len(keys))
chunk := keys[start:end]
idSet, bucketSet := map[uint]bool{}, map[int64]bool{}
for _, k := range chunk {
idSet[k.id] = true
bucketSet[k.bucket] = true
}
ids := make([]uint, 0, len(idSet))
for id := range idSet {
ids = append(ids, id)
}
buckets := make([]int64, 0, len(bucketSet))
for b := range bucketSet {
buckets = append(buckets, b)
}
var existing []models.MetricPoint
if err := tx.Where("resolution = ? AND series_id IN ? AND bucket IN ?", res, ids, buckets).Find(&existing).Error; err != nil {
return err
}
prev := make(map[pointID]*Agg, len(existing))
for i := range existing {
prev[pointID{existing[i].SeriesID, existing[i].Bucket}] = pointToAgg(&existing[i])
}
rows := make([]models.MetricPoint, 0, len(chunk))
for _, k := range chunk {
a := points[k]
if old := prev[k]; old != nil {
old.merge(a)
a = old
}
rows = append(rows, aggToPoint(k.id, res, k.bucket, a))
}
err := tx.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "series_id"}, {Name: "resolution"}, {Name: "bucket"}},
DoUpdates: clause.AssignmentColumns([]string{"count", "sum", "min", "max", "last", "hist"}),
}).CreateInBatches(&rows, 200).Error
if err != nil {
return err
}
}
return nil
}
type pipelineStatus struct {
mu sync.Mutex
LastFlush int64 `json:"lastFlush"`
LastFlushMs int64 `json:"lastFlushMs"`
LastFlushErr string `json:"lastFlushError"`
LastRollup int64 `json:"lastRollup"`
LastRetention int64 `json:"lastRetention"`
Dropped int64 `json:"droppedPoints"`
}
var status pipelineStatus
// flush writes every bucket that started before `before`.
func flush(before int64) error {
flushMu.Lock()
defer flushMu.Unlock()
started := time.Now()
points := defaultAgg.takeClosed(before)
if len(points) == 0 {
return nil
}
fail := func(err error) error {
now := time.Now().Unix()
keep := make(map[pointKey]*Agg)
var dropped int64
for k, a := range points {
if now-k.bucket <= maxRetryAge {
keep[k] = a
} else {
dropped++
}
}
defaultAgg.putBack(keep)
status.mu.Lock()
status.LastFlushErr = err.Error()
status.Dropped += dropped
status.mu.Unlock()
if dropped > 0 {
PointsDropped.Add(float64(dropped))
}
return err
}
set := map[*Series]bool{}
for k := range points {
set[k.s] = true
}
list := make([]*Series, 0, len(set))
for s := range set {
list = append(list, s)
}
ids, err := resolveIDs(list)
if err != nil {
return fail(fmt.Errorf("resolve series: %w", err))
}
rows := make(map[pointID]*Agg, len(points))
for k, a := range points {
rows[pointID{ids[k.s], k.bucket}] = a
}
err = initializers.DB.Transaction(func(tx *gorm.DB) error {
return persist(tx, resMinute, rows)
})
if err != nil {
return fail(fmt.Errorf("write points: %w", err))
}
// last_seen lets the catalog hide series that stopped reporting
idList := make([]uint, 0, len(ids))
for _, id := range ids {
idList = append(idList, id)
}
for start := 0; start < len(idList); start += 500 {
end := min(start+500, len(idList))
initializers.DB.Model(&models.MetricSeries{}).Where("id IN ?", idList[start:end]).
Update("last_seen", time.Now().Unix())
}
status.mu.Lock()
status.LastFlush = time.Now().Unix()
status.LastFlushMs = time.Since(started).Milliseconds()
status.LastFlushErr = ""
status.mu.Unlock()
return nil
}
func hourOf(ts int64) int64 { return ts - ts%resHour }
// rollup aggregates complete hours of minute data into hourly points. The last
// rolled hour is redone each time to pick up late telemetry; it is idempotent.
func rollup(now int64) error {
flushMu.Lock()
defer flushMu.Unlock()
// the minute data of an hour is complete ~1 flush after the hour ends
lastFull := hourOf(now - 2*resMinute)
start := rolledUntil.Load()
if start == 0 {
var maxHour, minMinute *int64
initializers.DB.Model(&models.MetricPoint{}).Where("resolution = ?", resHour).Select("MAX(bucket)").Scan(&maxHour)
if maxHour != nil {
start = *maxHour + resHour
} else {
initializers.DB.Model(&models.MetricPoint{}).Where("resolution = ?", resMinute).Select("MIN(bucket)").Scan(&minMinute)
if minMinute == nil {
status.mu.Lock()
status.LastRollup = now
status.mu.Unlock()
return nil
}
start = hourOf(*minMinute)
}
}
from := start - resHour
if earliest := hourOf(now - int64(initializers.MetricsMinuteRetentionDays)*86400); from < earliest {
from = earliest
}
for h := from; h < lastFull; h += resHour {
var rows []models.MetricPoint
err := initializers.DB.Where("resolution = ? AND bucket >= ? AND bucket < ?", resMinute, h, h+resHour).
Order("bucket asc").Find(&rows).Error
if err != nil {
return err
}
if len(rows) == 0 {
continue
}
merged := map[pointID]*Agg{}
for i := range rows {
k := pointID{rows[i].SeriesID, h}
a := merged[k]
if a == nil {
a = &Agg{}
merged[k] = a
}
a.merge(pointToAgg(&rows[i]))
}
// replace (not merge) so re-rolling an hour is idempotent
out := make([]models.MetricPoint, 0, len(merged))
for k, a := range merged {
out = append(out, aggToPoint(k.id, resHour, k.bucket, a))
}
err = initializers.DB.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "series_id"}, {Name: "resolution"}, {Name: "bucket"}},
DoUpdates: clause.AssignmentColumns([]string{"count", "sum", "min", "max", "last", "hist"}),
}).CreateInBatches(&out, 200).Error
if err != nil {
return err
}
}
if lastFull > rolledUntil.Load() {
rolledUntil.Store(lastFull)
}
status.mu.Lock()
status.LastRollup = now
status.mu.Unlock()
return nil
}
// retention deletes data older than the configured windows.
func retention(now time.Time) error {
flushMu.Lock()
defer flushMu.Unlock()
day := int64(86400)
ts := now.Unix()
steps := []struct {
what string
run func() error
}{
{"minute points", func() error {
return initializers.DB.Where("resolution = ? AND bucket < ?", resMinute, ts-int64(initializers.MetricsMinuteRetentionDays)*day).
Delete(&models.MetricPoint{}).Error
}},
{"hour points", func() error {
return initializers.DB.Where("resolution = ? AND bucket < ?", resHour, ts-int64(initializers.MetricsHourRetentionDays)*day).
Delete(&models.MetricPoint{}).Error
}},
{"events", func() error {
return initializers.DB.Where("ts < ?", (ts-int64(initializers.EventsRetentionDays)*day)*1000).
Delete(&models.ServiceEvent{}).Error
}},
{"top items", func() error {
cutoff := now.UTC().AddDate(0, 0, -initializers.TopItemsRetentionDays).Format("2006-01-02")
return initializers.DB.Where("day < ?", cutoff).Delete(&models.DailyTopItem{}).Error
}},
{"stale series", func() error {
stale := ts - 30*day
res := initializers.DB.Where("last_seen < ? AND id NOT IN (SELECT DISTINCT series_id FROM metric_points)", stale).
Delete(&models.MetricSeries{})
if res.Error == nil && res.RowsAffected > 0 {
// forget cached ids so a revived series gets a new row
idMu.Lock()
seriesID = map[*Series]uint{}
idMu.Unlock()
}
return res.Error
}},
}
for _, step := range steps {
if err := step.run(); err != nil {
log.Printf("metrics retention (%s): %v", step.what, err)
}
}
status.mu.Lock()
status.LastRetention = now.Unix()
status.mu.Unlock()
return nil
}
metrics/traffic.go
+328
View File
@@ -0,0 +1,328 @@
package metrics
import (
"crypto/rand"
"crypto/sha256"
"encoding/binary"
"log"
"sort"
"strconv"
"strings"
"sync"
"time"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
"github.com/gin-gonic/gin"
"gorm.io/gorm/clause"
)
// Middleware records latency and traffic for every request except static admin
// UI files and health checks.
func Middleware() gin.HandlerFunc {
return func(c *gin.Context) {
path := c.Request.URL.Path
if c.Request.Method == "OPTIONS" || path == "/health" || strings.HasPrefix(path, "/admin/ui") {
c.Next()
return
}
start := time.Now()
c.Next()
ms := float64(time.Since(start).Microseconds()) / 1000
route := c.FullPath()
if route == "" {
route = "(unmatched)"
}
status := c.Writer.Status()
group := routeGroup(path)
client := "direct"
if c.GetHeader("Origin") != "" || c.GetHeader("Referer") != "" {
client = "web"
}
HTTPDuration.Observe(ms, c.Request.Method, route, strconv.Itoa(status/100)+"xx", group, client)
if size := c.Writer.Size(); size > 0 {
HTTPResponseBytes.Add(float64(size), group)
}
if group == "v1" && status < 500 {
visitors.add(c.ClientIP(), c.Request.UserAgent())
}
if status == 200 {
switch route {
case "/v1/addon/:id":
Top.Add("addon", c.Param("id"))
case "/v1/getPossible":
Top.Add("search", normalizeSearch(c.Query("q")))
}
}
}
}
func routeGroup(path string) string {
for _, g := range []string{"v1", "back", "admin", "metrics"} {
if path == "/"+g || strings.HasPrefix(path, "/"+g+"/") {
return g
}
}
return "other"
}
func normalizeSearch(q string) string {
q = strings.Join(strings.Fields(strings.ToLower(q)), " ")
if len(q) > 64 {
q = q[:64]
}
return q
}
// visitorSet counts distinct (IP, user agent) pairs per UTC day. Only salted
// 64-bit hashes are kept and the salt never leaves memory.
type visitorSet struct {
mu sync.Mutex
day string
salt []byte
humans map[uint64]struct{}
bots map[uint64]struct{}
}
var visitors = &visitorSet{}
const maxVisitors = 2_000_000
func (v *visitorSet) rotate(day string) {
v.day = day
v.salt = make([]byte, 32)
_, _ = rand.Read(v.salt)
v.humans = make(map[uint64]struct{})
v.bots = make(map[uint64]struct{})
}
func isBot(ua string) bool {
ua = strings.ToLower(ua)
if ua == "" {
return true
}
for _, s := range []string{"bot", "spider", "crawl", "curl", "wget", "python", "go-http", "headless", "preview", "monitor"} {
if strings.Contains(ua, s) {
return true
}
}
return false
}
func (v *visitorSet) add(ip, ua string) {
day := time.Now().UTC().Format("2006-01-02")
v.mu.Lock()
defer v.mu.Unlock()
if v.day != day {
v.rotate(day)
}
h := sha256.New()
h.Write(v.salt)
h.Write([]byte(ip))
h.Write([]byte{0})
h.Write([]byte(ua))
key := binary.BigEndian.Uint64(h.Sum(nil)[:8])
set := v.humans
if isBot(ua) {
set = v.bots
}
if len(set) < maxVisitors {
set[key] = struct{}{}
}
}
// Visitors returns today's approximate unique human and bot visitors. The set
// lives in memory, so after a restart the day's highest recorded value is used
// until the new set catches up.
func Visitors() (humans, bots int) {
now := time.Now().UTC()
day := now.Format("2006-01-02")
visitors.mu.Lock()
if visitors.day == day {
humans, bots = len(visitors.humans), len(visitors.bots)
}
visitors.mu.Unlock()
dayStart := now.Truncate(24 * time.Hour).Unix()
if dayStart < startedAt.Unix() {
if t, err := Totals(Selector{Service: MainService, Name: "visitors_unique", GroupBy: []string{"kind"}}, dayStart, now.Unix()); err == nil {
for _, g := range t {
if g.Max == nil {
continue
}
switch g.Labels["kind"] {
case "human":
humans = max(humans, int(*g.Max))
case "bot":
bots = max(bots, int(*g.Max))
}
}
}
}
return humans, bots
}
// liveVisitors is the in-memory count only, used when recording the gauge.
func liveVisitors() (humans, bots int) {
day := time.Now().UTC().Format("2006-01-02")
visitors.mu.Lock()
defer visitors.mu.Unlock()
if visitors.day != day {
return 0, 0
}
return len(visitors.humans), len(visitors.bots)
}
// topCounter counts requested items per UTC day and persists the most popular.
type topCounter struct {
mu sync.Mutex
day string
counts map[string]map[string]int64
pending []topSnapshot // previous days not yet written
}
type topSnapshot struct {
day string
counts map[string]map[string]int64
}
// Top tracks the most looked-up addons and search queries.
var Top = &topCounter{counts: map[string]map[string]int64{}}
const (
maxTopKeys = 50_000
topPersistMax = 500
)
func (t *topCounter) Add(kind, item string) {
if item == "" {
return
}
if len(item) > 128 {
item = item[:128]
}
day := time.Now().UTC().Format("2006-01-02")
t.mu.Lock()
defer t.mu.Unlock()
if t.day != day {
if t.day != "" {
t.pending = append(t.pending, topSnapshot{t.day, t.counts})
}
t.day = day
t.counts = map[string]map[string]int64{}
}
m := t.counts[kind]
if m == nil {
m = map[string]int64{}
t.counts[kind] = m
}
if _, ok := m[item]; !ok && len(m) >= maxTopKeys {
for k, v := range m {
if v <= 1 {
delete(m, k)
}
}
if len(m) >= maxTopKeys {
return
}
}
m[item]++
}
// Today returns today's in-memory counts for a kind.
func (t *topCounter) Today(kind string) (string, map[string]int64) {
t.mu.Lock()
defer t.mu.Unlock()
out := make(map[string]int64, len(t.counts[kind]))
for k, v := range t.counts[kind] {
out[k] = v
}
return t.day, out
}
// seed loads today's persisted counts so a restart does not reset them.
func (t *topCounter) seed() {
day := time.Now().UTC().Format("2006-01-02")
var rows []models.DailyTopItem
if err := initializers.DB.Where("day = ?", day).Find(&rows).Error; err != nil {
log.Printf("metrics: failed to load top items: %v", err)
return
}
t.mu.Lock()
defer t.mu.Unlock()
if t.day != day {
t.day = day
t.counts = map[string]map[string]int64{}
}
for _, r := range rows {
if t.counts[r.Kind] == nil {
t.counts[r.Kind] = map[string]int64{}
}
t.counts[r.Kind][r.Item] += r.Count
}
}
func topRows(snap topSnapshot) []models.DailyTopItem {
var rows []models.DailyTopItem
for kind, m := range snap.counts {
type kv struct {
k string
v int64
}
list := make([]kv, 0, len(m))
for k, v := range m {
list = append(list, kv{k, v})
}
sort.Slice(list, func(i, j int) bool { return list[i].v > list[j].v })
if len(list) > topPersistMax {
list = list[:topPersistMax]
}
for _, e := range list {
rows = append(rows, models.DailyTopItem{Day: snap.day, Kind: kind, Item: e.k, Count: e.v})
}
}
return rows
}
func (t *topCounter) flush() error {
t.mu.Lock()
snaps := append([]topSnapshot{}, t.pending...)
t.pending = nil
cur := topSnapshot{day: t.day, counts: map[string]map[string]int64{}}
for kind, m := range t.counts {
cp := make(map[string]int64, len(m))
for k, v := range m {
cp[k] = v
}
cur.counts[kind] = cp
}
t.mu.Unlock()
if cur.day != "" {
snaps = append(snaps, cur)
}
for i, snap := range snaps {
rows := topRows(snap)
if len(rows) == 0 {
continue
}
err := initializers.DB.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "day"}, {Name: "kind"}, {Name: "item"}},
DoUpdates: clause.AssignmentColumns([]string{"count"}),
}).CreateInBatches(&rows, 200).Error
if err != nil {
// keep finished days for the next attempt; today is re-read from memory
t.mu.Lock()
for _, old := range snaps[i:] {
if old.day != cur.day {
t.pending = append(t.pending, old)
}
}
t.mu.Unlock()
return err
}
}
return nil
}
models/addon.go
+1
View File
@@ -27,6 +27,7 @@ type Addon struct {
PriorityIndexing bool `json:"priorityIndexing" gorm:"default:false"`
AddonFiles []AddonFile `json:"addonFiles"`
Recheck bool `json:"recheck" gorm:"default:false"`
LastIndexedAt *time.Time `json:"lastIndexedAt"`
Preview string `json:"preview"`
Author string `json:"author"`
models/metrics.go
+74
View File
@@ -0,0 +1,74 @@
package models
import "time"
// All timestamps in the metrics tables are int64 unix seconds (events: unix
// milliseconds) so range queries behave the same on SQLite and Postgres.
// MetricSeries identifies one time series: a metric name plus a fixed label set
// reported by one service instance.
type MetricSeries struct {
ID uint `gorm:"primaryKey" json:"id"`
Service string `gorm:"size:64;not null;uniqueIndex:ux_metric_series,priority:1" json:"service"`
Instance string `gorm:"size:128;not null;uniqueIndex:ux_metric_series,priority:2" json:"instance"`
Name string `gorm:"size:96;not null;uniqueIndex:ux_metric_series,priority:3;index" json:"name"`
Labels string `gorm:"size:512;not null;uniqueIndex:ux_metric_series,priority:4" json:"labels"` // canonical JSON object
Kind string `gorm:"size:16;not null" json:"kind"` // counter|gauge|summary|histogram
Bounds string `gorm:"type:text" json:"bounds"` // JSON []float64, histograms only
Help string `gorm:"size:256" json:"help"`
CreatedAt time.Time
LastSeen int64 `gorm:"index" json:"lastSeen"`
}
// MetricPoint is one aggregated bucket of a series. Resolution is the bucket
// width in seconds (60 = raw minute data, 3600 = hourly rollup).
type MetricPoint struct {
SeriesID uint `gorm:"primaryKey;autoIncrement:false"`
Resolution int `gorm:"primaryKey;autoIncrement:false;index:idx_metric_points_res_bucket,priority:1"`
Bucket int64 `gorm:"primaryKey;autoIncrement:false;index:idx_metric_points_res_bucket,priority:2"`
Count int64 `gorm:"not null;default:0"`
Sum float64 `gorm:"not null;default:0"`
Min float64 `gorm:"not null;default:0"`
Max float64 `gorm:"not null;default:0"`
Last float64 `gorm:"not null;default:0"`
Hist string `gorm:"type:text"` // JSON []uint64, histograms only
}
// ServiceInstance is one running process (indexer, scraper, the API itself)
// that reports telemetry. Every telemetry push counts as a heartbeat.
type ServiceInstance struct {
ID uint `gorm:"primaryKey" json:"id"`
Service string `gorm:"size:64;not null;uniqueIndex:ux_service_instance,priority:1" json:"service"`
Instance string `gorm:"size:128;not null;uniqueIndex:ux_service_instance,priority:2" json:"instance"`
Version string `gorm:"size:64" json:"version"`
Status string `gorm:"size:16" json:"status"` // self-reported: ok|degraded|error
StatusMessage string `gorm:"size:512" json:"statusMessage"`
StartedAt int64 `json:"startedAt"`
LastSeen int64 `gorm:"index" json:"lastSeen"`
HeartbeatIntervalSec int `json:"heartbeatIntervalSec"`
RemoteAddr string `gorm:"size:64" json:"remoteAddr"`
Meta string `gorm:"type:text" json:"meta"` // JSON object
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
// ServiceEvent is a log line reported by a service. Admin actions are recorded
// here too (service "main", fields.kind "audit").
type ServiceEvent struct {
ID uint `gorm:"primaryKey" json:"id"`
Service string `gorm:"size:64;index:idx_service_events_svc_ts,priority:1" json:"service"`
Instance string `gorm:"size:128" json:"instance"`
Level string `gorm:"size:8;index" json:"level"` // debug|info|warn|error
Message string `gorm:"size:1024" json:"message"`
Fields string `gorm:"type:text" json:"fields"` // JSON object
TS int64 `gorm:"index:idx_service_events_svc_ts,priority:2;index" json:"ts"` // unix ms
}
// DailyTopItem counts how often something was requested on one UTC day
// (Kind "addon": addon lookups, Kind "search": search queries).
type DailyTopItem struct {
Day string `gorm:"primaryKey;size:10"` // YYYY-MM-DD
Kind string `gorm:"primaryKey;size:16"`
Item string `gorm:"primaryKey;size:128"`
Count int64
}
models/telemetry.go
+39
View File
@@ -0,0 +1,39 @@
package models
// TelemetryBatch is the payload of POST /back/telemetry. The client lives in
// the telemetry/ module; keep both definitions in sync.
type TelemetryBatch struct {
Service string `json:"service"`
Instance string `json:"instance"`
Version string `json:"version"`
StartedAt int64 `json:"startedAt"`
HeartbeatIntervalSec int `json:"heartbeatIntervalSec"`
Status string `json:"status"`
StatusMessage string `json:"statusMessage"`
Meta map[string]any `json:"meta"`
Metrics []TelemetryMetric `json:"metrics"`
Events []TelemetryEvent `json:"events"`
}
// TelemetryMetric is one sample. Type "counter" carries a delta since the last
// push, "gauge" the current value, "summary" client-side pre-aggregated
// Count/Sum/Min/Max. Ts is unix seconds or milliseconds, optional.
type TelemetryMetric struct {
Name string `json:"name"`
Type string `json:"type"`
Help string `json:"help"`
Value float64 `json:"value"`
Count int64 `json:"count"`
Sum float64 `json:"sum"`
Min float64 `json:"min"`
Max float64 `json:"max"`
Labels map[string]string `json:"labels"`
Ts int64 `json:"ts"`
}
type TelemetryEvent struct {
Level string `json:"level"`
Message string `json:"message"`
Ts int64 `json:"ts"`
Fields map[string]any `json:"fields"`
}
models/whitelisted.go
+1
View File
@@ -5,4 +5,5 @@ import "gorm.io/gorm"
type WhitelistedHash struct {
gorm.Model
Hash string `json:"hash"`
Note string `json:"note" gorm:"size:256"`
}
telemetry/client.go
+430
View File
@@ -0,0 +1,430 @@
// Package telemetry reports metrics, events and heartbeats from a crawler
// service (indexer, scraper, ...) to the main API's POST /back/telemetry.
//
// It has no dependencies outside the standard library, so it can be imported
// by any service, or the file can simply be copied.
//
// c := telemetry.New(telemetry.Config{
// BaseURL: "https://crawler.example.com",
// SecretKey: os.Getenv("SECRET_KEY"),
// Service: "indexer",
// Version: "1.4.0",
// })
// c.Start(ctx)
// defer c.Close(context.Background())
//
// c.Counter("jobs_total", "result", "ok").Add(1)
// c.Gauge("data_window_bytes").Set(float64(used))
// c.Summary("download_seconds").Observe(took.Seconds())
// c.Event(telemetry.Warn, "data limit reached", map[string]any{"limitGB": 50})
// c.SetStatus(telemetry.Degraded, "rate limited by workshop")
//
// Recording never blocks on the network. A push that fails keeps the counter
// increments for the next attempt; a few failed batches of events and samples
// are buffered and the oldest are dropped first.
package telemetry
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"math"
"net/http"
"os"
"sort"
"strings"
"sync"
"time"
)
// Level of an event.
type Level string
const (
Debug Level = "debug"
Info Level = "info"
Warn Level = "warn"
Error Level = "error"
)
// Status is the self-reported health shown on the dashboard.
type Status string
const (
OK Status = "ok"
Degraded Status = "degraded"
Failing Status = "error"
)
// Config configures a Client. BaseURL, SecretKey and Service are required.
type Config struct {
BaseURL string // e.g. https://crawler.example.com (no trailing /back)
SecretKey string // the API's SECRET_KEY
Service string // [a-z0-9_], e.g. "indexer"
Instance string // defaults to the hostname
Version string // shown on the dashboard
Interval time.Duration // push interval, default 15s (also the heartbeat)
Meta map[string]any
// HTTPClient defaults to a client with a 10s timeout.
HTTPClient *http.Client
// OnError is called when a push fails (optional).
OnError func(error)
}
// Client buffers metrics and events and pushes them periodically.
type Client struct {
cfg Config
started time.Time
mu sync.Mutex
counters map[string]*Counter
gauges map[string]*Gauge
summaries map[string]*Summary
events []event
status Status
statusMsg string
pending [][]byte // failed batches waiting for a retry
stop chan struct{}
done chan struct{}
}
type metric struct {
Name string `json:"name"`
Type string `json:"type"`
Value float64 `json:"value,omitempty"`
Count int64 `json:"count,omitempty"`
Sum float64 `json:"sum,omitempty"`
Min float64 `json:"min,omitempty"`
Max float64 `json:"max,omitempty"`
Labels map[string]string `json:"labels,omitempty"`
Ts int64 `json:"ts"`
}
type event struct {
Level Level `json:"level"`
Message string `json:"message"`
Ts int64 `json:"ts"`
Fields map[string]any `json:"fields,omitempty"`
}
type batch struct {
Service string `json:"service"`
Instance string `json:"instance"`
Version string `json:"version"`
StartedAt int64 `json:"startedAt"`
HeartbeatIntervalSec int `json:"heartbeatIntervalSec"`
Status Status `json:"status"`
StatusMessage string `json:"statusMessage"`
Meta map[string]any `json:"meta,omitempty"`
Metrics []metric `json:"metrics"`
Events []event `json:"events"`
}
const (
maxPending = 20
maxEvents = 200
maxMetricsPerReq = 1000
)
// New creates a client. Call Start to begin pushing.
func New(cfg Config) *Client {
if cfg.Instance == "" {
cfg.Instance, _ = os.Hostname()
}
if cfg.Interval <= 0 {
cfg.Interval = 15 * time.Second
}
if cfg.HTTPClient == nil {
cfg.HTTPClient = &http.Client{Timeout: 10 * time.Second}
}
cfg.BaseURL = strings.TrimRight(cfg.BaseURL, "/")
return &Client{
cfg: cfg,
started: time.Now(),
counters: map[string]*Counter{},
gauges: map[string]*Gauge{},
summaries: map[string]*Summary{},
status: OK,
}
}
func key(name string, labels []string) string {
return name + "\x00" + strings.Join(labels, "\x00")
}
func labelMap(kv []string) map[string]string {
if len(kv) < 2 {
return nil
}
m := make(map[string]string, len(kv)/2)
for i := 0; i+1 < len(kv); i += 2 {
m[kv[i]] = kv[i+1]
}
return m
}
// Counter returns a counter; labels are key/value pairs.
func (c *Client) Counter(name string, labels ...string) *Counter {
k := key(name, labels)
c.mu.Lock()
defer c.mu.Unlock()
ctr := c.counters[k]
if ctr == nil {
ctr = &Counter{name: name, labels: labelMap(labels)}
c.counters[k] = ctr
}
return ctr
}
// Gauge returns a gauge; labels are key/value pairs.
func (c *Client) Gauge(name string, labels ...string) *Gauge {
k := key(name, labels)
c.mu.Lock()
defer c.mu.Unlock()
g := c.gauges[k]
if g == nil {
g = &Gauge{name: name, labels: labelMap(labels)}
c.gauges[k] = g
}
return g
}
// Summary returns a summary (count/sum/min/max per push); labels are key/value pairs.
func (c *Client) Summary(name string, labels ...string) *Summary {
k := key(name, labels)
c.mu.Lock()
defer c.mu.Unlock()
s := c.summaries[k]
if s == nil {
s = &Summary{name: name, labels: labelMap(labels)}
c.summaries[k] = s
}
return s
}
// Event queues a log event for the dashboard.
func (c *Client) Event(level Level, message string, fields map[string]any) {
c.mu.Lock()
defer c.mu.Unlock()
if len(c.events) >= maxEvents {
c.events = c.events[1:]
}
c.events = append(c.events, event{Level: level, Message: message, Ts: time.Now().UnixMilli(), Fields: fields})
}
// SetStatus sets the self-reported health shown on the dashboard.
func (c *Client) SetStatus(s Status, message string) {
c.mu.Lock()
c.status, c.statusMsg = s, message
c.mu.Unlock()
}
// Counter accumulates increments between pushes.
type Counter struct {
mu sync.Mutex
name string
labels map[string]string
delta float64
}
func (m *Counter) Add(v float64) {
if v <= 0 || math.IsNaN(v) || math.IsInf(v, 0) {
return
}
m.mu.Lock()
m.delta += v
m.mu.Unlock()
}
func (m *Counter) Inc() { m.Add(1) }
// Gauge holds the latest value.
type Gauge struct {
mu sync.Mutex
name string
labels map[string]string
value float64
set bool
}
func (m *Gauge) Set(v float64) {
if math.IsNaN(v) || math.IsInf(v, 0) {
return
}
m.mu.Lock()
m.value, m.set = v, true
m.mu.Unlock()
}
// Summary aggregates observations between pushes.
type Summary struct {
mu sync.Mutex
name string
labels map[string]string
count int64
sum float64
min, max float64
}
func (m *Summary) Observe(v float64) {
if math.IsNaN(v) || math.IsInf(v, 0) {
return
}
m.mu.Lock()
if m.count == 0 || v < m.min {
m.min = v
}
if m.count == 0 || v > m.max {
m.max = v
}
m.count++
m.sum += v
m.mu.Unlock()
}
// collect drains the current values into a batch payload.
func (c *Client) collect() []byte {
now := time.Now()
c.mu.Lock()
b := batch{
Service: c.cfg.Service,
Instance: c.cfg.Instance,
Version: c.cfg.Version,
StartedAt: c.started.Unix(),
HeartbeatIntervalSec: int(c.cfg.Interval.Seconds()),
Status: c.status,
StatusMessage: c.statusMsg,
Meta: c.cfg.Meta,
Events: c.events,
}
c.events = nil
counters := make([]*Counter, 0, len(c.counters))
for _, m := range c.counters {
counters = append(counters, m)
}
gauges := make([]*Gauge, 0, len(c.gauges))
for _, m := range c.gauges {
gauges = append(gauges, m)
}
summaries := make([]*Summary, 0, len(c.summaries))
for _, m := range c.summaries {
summaries = append(summaries, m)
}
c.mu.Unlock()
ts := now.UnixMilli()
for _, m := range counters {
m.mu.Lock()
if m.delta > 0 {
b.Metrics = append(b.Metrics, metric{Name: m.name, Type: "counter", Value: m.delta, Labels: m.labels, Ts: ts})
m.delta = 0
}
m.mu.Unlock()
}
for _, m := range gauges {
m.mu.Lock()
if m.set {
b.Metrics = append(b.Metrics, metric{Name: m.name, Type: "gauge", Value: m.value, Labels: m.labels, Ts: ts})
}
m.mu.Unlock()
}
for _, m := range summaries {
m.mu.Lock()
if m.count > 0 {
b.Metrics = append(b.Metrics, metric{Name: m.name, Type: "summary", Count: m.count, Sum: m.sum, Min: m.min, Max: m.max, Labels: m.labels, Ts: ts})
m.count, m.sum, m.min, m.max = 0, 0, 0, 0
}
m.mu.Unlock()
}
sort.Slice(b.Metrics, func(i, j int) bool { return b.Metrics[i].Name < b.Metrics[j].Name })
if len(b.Metrics) > maxMetricsPerReq {
b.Metrics = b.Metrics[:maxMetricsPerReq]
}
if b.Metrics == nil {
b.Metrics = []metric{}
}
if b.Events == nil {
b.Events = []event{}
}
payload, _ := json.Marshal(b)
return payload
}
func (c *Client) post(ctx context.Context, payload []byte) error {
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.cfg.BaseURL+"/back/telemetry", bytes.NewReader(payload))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-SECRET-KEY", c.cfg.SecretKey)
resp, err := c.cfg.HTTPClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
if resp.StatusCode/100 != 2 {
return fmt.Errorf("telemetry push: %s: %s", resp.Status, strings.TrimSpace(string(body)))
}
return nil
}
// Flush pushes buffered batches and the current values now.
func (c *Client) Flush(ctx context.Context) error {
payload := c.collect()
c.mu.Lock()
queue := append(c.pending, payload)
c.pending = nil
c.mu.Unlock()
for i, p := range queue {
if err := c.post(ctx, p); err != nil {
c.mu.Lock()
c.pending = append(queue[i:], c.pending...)
if len(c.pending) > maxPending {
c.pending = c.pending[len(c.pending)-maxPending:]
}
c.mu.Unlock()
if c.cfg.OnError != nil {
c.cfg.OnError(err)
}
return err
}
}
return nil
}
// Start pushes in the background every Interval until ctx ends or Close is called.
func (c *Client) Start(ctx context.Context) {
c.stop = make(chan struct{})
c.done = make(chan struct{})
go func() {
defer close(c.done)
t := time.NewTicker(c.cfg.Interval)
defer t.Stop()
_ = c.Flush(ctx)
for {
select {
case <-ctx.Done():
return
case <-c.stop:
return
case <-t.C:
_ = c.Flush(ctx)
}
}
}()
}
// Close stops the background loop and makes a final push.
func (c *Client) Close(ctx context.Context) error {
if c.stop != nil {
close(c.stop)
<-c.done
c.stop = nil
}
return c.Flush(ctx)
}
telemetry/client_test.go
+73
View File
@@ -0,0 +1,73 @@
package telemetry
import (
"context"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
)
func TestFlushAndRetry(t *testing.T) {
var fail atomic.Bool
var got []batch
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/back/telemetry" || r.Header.Get("X-SECRET-KEY") != "k" {
w.WriteHeader(http.StatusForbidden)
return
}
if fail.Load() {
w.WriteHeader(http.StatusServiceUnavailable)
return
}
body, _ := io.ReadAll(r.Body)
var b batch
if err := json.Unmarshal(body, &b); err != nil {
t.Error(err)
}
got = append(got, b)
w.Write([]byte(`{"accepted":1}`))
}))
defer srv.Close()
c := New(Config{BaseURL: srv.URL + "/", SecretKey: "k", Service: "indexer", Instance: "i1"})
c.Counter("jobs_total", "result", "ok").Add(2)
c.Gauge("queue").Set(7)
c.Summary("took_seconds").Observe(1)
c.Summary("took_seconds").Observe(3)
c.Event(Warn, "hello", nil)
fail.Store(true)
if err := c.Flush(context.Background()); err == nil {
t.Fatal("expected an error while the server fails")
}
c.Counter("jobs_total", "result", "ok").Add(1)
fail.Store(false)
if err := c.Flush(context.Background()); err != nil {
t.Fatal(err)
}
if len(got) != 2 {
t.Fatalf("got %d batches, want the retried one plus the new one", len(got))
}
first := got[0]
if first.Service != "indexer" || first.Instance != "i1" || len(first.Events) != 1 {
t.Fatalf("unexpected first batch %+v", first)
}
var sum float64
for _, b := range got {
for _, m := range b.Metrics {
if m.Name == "jobs_total" {
sum += m.Value
}
if m.Name == "took_seconds" && (m.Count != 2 || m.Min != 1 || m.Max != 3) {
t.Fatalf("bad summary %+v", m)
}
}
}
if sum != 3 {
t.Fatalf("counter total = %v, want 3", sum)
}
}
telemetry/go.mod
+3
View File
@@ -0,0 +1,3 @@
module gitea.tbdevent.eu/TBD/reforger_crawler_main/telemetry
go 1.22
web/dist/.gitkeep
Vendored
View File
Whitespace-only changes.
web/embed.go
+13
View File
@@ -0,0 +1,13 @@
// Package web serves the admin dashboard (a React app built into web/dist).
//
// Build the UI before building the Go binary:
//
// npm --prefix web ci && npm --prefix web run build
//
// Without a build the binary still compiles and serves a placeholder page.
package web
import "embed"
//go:embed all:dist
var dist embed.FS
web/index.html
+15
View File
@@ -0,0 +1,15 @@
<!doctype html>
<html lang="en" data-theme="dark">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="robots" content="noindex" />
<link rel="icon" type="image/svg+xml" href="/admin/ui/favicon.svg" />
<title>Crawler Admin</title>
<script src="/admin/ui/theme-init.js"></script>
</head>
<body>
<div id="root"></div>
<script type="module" src="/src/main.tsx"></script>
</body>
</html>
web/package-lock.json
Generated
+2534
View File
File diff suppressed because it is too large. Load diff
web/package.json
+29
View File
@@ -0,0 +1,29 @@
{
"name": "reforger-crawler-admin",
"private": true,
"version": "1.0.0",
"type": "module",
"scripts": {
"dev": "vite",
"clean": "node -e \"const fs=require('fs');fs.rmSync('dist/assets',{recursive:true,force:true});fs.rmSync('dist/index.html',{force:true})\"",
"build": "npm run clean && tsc -b && vite build",
"typecheck": "tsc -b"
},
"dependencies": {
"@tanstack/react-query": "^5.80.0",
"lucide-react": "^0.511.0",
"react": "^19.1.0",
"react-dom": "^19.1.0",
"react-router": "^7.6.0",
"uplot": "^1.6.32"
},
"devDependencies": {
"@tailwindcss/vite": "^4.1.8",
"@types/react": "^19.1.6",
"@types/react-dom": "^19.1.5",
"@vitejs/plugin-react": "^4.5.0",
"tailwindcss": "^4.1.8",
"typescript": "^5.8.3",
"vite": "^6.3.5"
}
}
web/public/favicon.svg
+1
View File
@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32"><rect width="32" height="32" rx="8" fill="#1a1a19"/><path d="M8 21l5-6 4 4 7-9" fill="none" stroke="#3987e5" stroke-width="3" stroke-linecap="round" stroke-linejoin="round"/></svg>
Side by Side

After

Width:  |  Height:  |  Size: 241 B

web/public/theme-init.js
+5
View File
@@ -0,0 +1,5 @@
// Applies the saved theme before first paint (external file: the CSP blocks inline scripts).
try {
var t = localStorage.getItem("rc-theme");
document.documentElement.setAttribute("data-theme", t === "light" ? "light" : "dark");
} catch (e) {}
web/spa.go
+75
View File
@@ -0,0 +1,75 @@
package web
import (
"io/fs"
"net/http"
"strings"
"github.com/gin-gonic/gin"
)
const notBuiltHTML = `<!doctype html><html><head><meta charset="utf-8"><title>Admin UI not built</title>
<style>body{font-family:system-ui,sans-serif;background:#0b0d10;color:#c9d1d9;display:grid;place-items:center;height:100vh;margin:0}
code{background:#161b22;padding:2px 6px;border-radius:4px}</style></head>
<body><div><h1>Admin UI not built</h1><p>Run <code>npm --prefix web ci &amp;&amp; npm --prefix web run build</code> and rebuild the binary.</p>
<p>The JSON API is available under <code>/admin/api</code>.</p></div></body></html>`
const csp = "default-src 'self'; img-src 'self' https: data:; style-src 'self' 'unsafe-inline'; " +
"script-src 'self'; connect-src 'self'; font-src 'self' data:; frame-ancestors 'none'; base-uri 'self'; form-action 'self'"
func securityHeaders(c *gin.Context) {
c.Header("Content-Security-Policy", csp)
c.Header("X-Content-Type-Options", "nosniff")
c.Header("Referrer-Policy", "no-referrer")
c.Header("X-Frame-Options", "DENY")
}
// Handler returns the SPA handler: real files are served from the embedded
// build, every other path gets index.html so client-side routes deep-link.
func Handler() gin.HandlerFunc {
sub, err := fs.Sub(dist, "dist")
if err != nil {
panic(err)
}
index, err := fs.ReadFile(sub, "index.html")
if err != nil {
index = []byte(notBuiltHTML)
}
files := http.FileServer(http.FS(sub))
return func(c *gin.Context) {
securityHeaders(c)
p := strings.TrimPrefix(c.Param("filepath"), "/")
if p != "" && p != "index.html" {
if st, err := fs.Stat(sub, p); err == nil && !st.IsDir() {
if strings.HasPrefix(p, "assets/") {
c.Header("Cache-Control", "public, max-age=31536000, immutable")
} else {
c.Header("Cache-Control", "no-cache")
}
req := c.Request.Clone(c.Request.Context())
req.URL.Path = "/" + p
files.ServeHTTP(c.Writer, req)
return
}
// a missing hashed asset must not turn into an HTML page
if strings.HasPrefix(p, "assets/") {
c.Status(http.StatusNotFound)
return
}
}
// serve the bytes directly; FileServer would redirect /index.html to ./
c.Header("Cache-Control", "no-cache")
c.Data(http.StatusOK, "text/html; charset=utf-8", index)
}
}
// Register mounts the dashboard at /admin/ui and redirects /admin there.
func Register(r *gin.Engine) {
h := Handler()
r.GET("/admin/ui/*filepath", h)
r.HEAD("/admin/ui/*filepath", h)
redirect := func(c *gin.Context) { c.Redirect(http.StatusFound, "/admin/ui/") }
r.GET("/admin", redirect)
r.GET("/admin/ui", redirect)
}
web/spa_test.go
+51
View File
@@ -0,0 +1,51 @@
package web
import (
"io/fs"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/gin-gonic/gin"
)
func TestSPA(t *testing.T) {
gin.SetMode(gin.TestMode)
r := gin.New()
Register(r)
get := func(path string) *httptest.ResponseRecorder {
w := httptest.NewRecorder()
r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil))
return w
}
for _, p := range []string{"/admin/ui/", "/admin/ui/addons/123", "/admin/ui/index.html"} {
w := get(p)
if w.Code != http.StatusOK || !strings.Contains(w.Header().Get("Content-Type"), "text/html") {
t.Errorf("%s: got %d %s", p, w.Code, w.Header().Get("Content-Type"))
}
if w.Header().Get("Content-Security-Policy") == "" {
t.Errorf("%s: missing CSP", p)
}
}
if w := get("/admin/ui/assets/missing.js"); w.Code != http.StatusNotFound {
t.Errorf("missing asset: got %d, want 404", w.Code)
}
if w := get("/admin"); w.Code != http.StatusFound || w.Header().Get("Location") != "/admin/ui/" {
t.Errorf("/admin: got %d -> %s", w.Code, w.Header().Get("Location"))
}
// when the UI is built, hashed assets are served with long caching
sub, _ := fs.Sub(dist, "dist")
entries, _ := fs.ReadDir(sub, "assets")
for _, e := range entries {
w := get("/admin/ui/assets/" + e.Name())
if w.Code != http.StatusOK || !strings.Contains(w.Header().Get("Cache-Control"), "immutable") {
t.Errorf("asset %s: got %d cache %q", e.Name(), w.Code, w.Header().Get("Cache-Control"))
}
break
}
}
web/src/components/Shell.tsx
+160
View File
@@ -0,0 +1,160 @@
import { useEffect, type ReactNode } from "react";
import { NavLink, Navigate, Outlet, useLocation, useNavigate } from "react-router";
import { useIsFetching, useQuery, useQueryClient } from "@tanstack/react-query";
import {
Activity,
Cpu,
LayoutDashboard,
LineChart,
ListOrdered,
LogOut,
Moon,
Package,
RefreshCw,
ScrollText,
Server,
ShieldCheck,
Sun,
} from "lucide-react";
import { api, ApiError, UNAUTHORIZED_EVENT } from "../lib/api";
import { RANGES, REFRESH, useSettings } from "../lib/settings";
import { Loading, Segmented, Select, cn } from "./ui";
const NAV: { to: string; label: string; icon: ReactNode }[] = [
{ to: "/", label: "Overview", icon: <LayoutDashboard className="size-4" /> },
{ to: "/traffic", label: "Traffic", icon: <Activity className="size-4" /> },
{ to: "/queue", label: "Indexing queue", icon: <ListOrdered className="size-4" /> },
{ to: "/addons", label: "Addons", icon: <Package className="size-4" /> },
{ to: "/whitelist", label: "Whitelist", icon: <ShieldCheck className="size-4" /> },
{ to: "/services", label: "Services", icon: <Server className="size-4" /> },
{ to: "/explorer", label: "Metrics explorer", icon: <LineChart className="size-4" /> },
{ to: "/events", label: "Events", icon: <ScrollText className="size-4" /> },
{ to: "/system", label: "System", icon: <Cpu className="size-4" /> },
];
/** Redirects to /login when the session is missing or expired. */
export function RequireAuth() {
const navigate = useNavigate();
const qc = useQueryClient();
const location = useLocation();
const me = useQuery({ queryKey: ["me"], queryFn: () => api<{ method: string }>("/auth/me"), retry: false, staleTime: 60_000 });
useEffect(() => {
const onUnauthorized = () => {
qc.clear();
navigate("/login", { replace: true, state: { from: location.pathname } });
};
window.addEventListener(UNAUTHORIZED_EVENT, onUnauthorized);
return () => window.removeEventListener(UNAUTHORIZED_EVENT, onUnauthorized);
}, [navigate, qc, location.pathname]);
if (me.isLoading) return <div className="p-8"><Loading /></div>;
if (me.error instanceof ApiError && (me.error.status === 401 || me.error.status === 503)) {
return <Navigate to="/login" replace state={{ from: location.pathname }} />;
}
return <Shell />;
}
function Shell() {
const { theme, setTheme, range, setRange, refresh, setRefresh } = useSettings();
const fetching = useIsFetching();
const navigate = useNavigate();
const qc = useQueryClient();
const logout = async () => {
await api("/auth/logout", { method: "POST" }).catch(() => undefined);
qc.clear();
navigate("/login", { replace: true });
};
return (
<div className="flex min-h-screen">
<aside className="sticky top-0 hidden h-screen w-56 shrink-0 flex-col border-r border-line bg-surface md:flex">
<div className="flex items-center gap-2.5 px-4 py-4">
<div className="flex size-7 items-center justify-center rounded-lg bg-accent/15">
<svg viewBox="0 0 32 32" className="size-4"><path d="M6 22l6-7 5 5 9-11" fill="none" stroke="var(--accent)" strokeWidth="3.5" strokeLinecap="round" strokeLinejoin="round" /></svg>
</div>
<div className="leading-tight">
<div className="text-[13px] font-semibold text-ink">Reforger Crawler</div>
<div className="text-[11px] text-muted">Admin</div>
</div>
</div>
<nav className="flex-1 space-y-0.5 overflow-y-auto px-2 py-2">
{NAV.map((n) => (
<NavLink
key={n.to}
to={n.to}
end={n.to === "/"}
className={({ isActive }) =>
cn(
"flex items-center gap-2.5 rounded-lg px-2.5 py-1.5 text-[13px] transition-colors",
isActive ? "bg-surface-2 font-medium text-ink" : "text-ink-2 hover:bg-surface-2/60 hover:text-ink",
)
}
>
{n.icon}
{n.label}
</NavLink>
))}
</nav>
<div className="space-y-0.5 border-t border-line p-2">
<button
onClick={() => setTheme(theme === "dark" ? "light" : "dark")}
className="flex w-full items-center gap-2.5 rounded-lg px-2.5 py-1.5 text-[13px] text-ink-2 hover:bg-surface-2/60 hover:text-ink"
>
{theme === "dark" ? <Sun className="size-4" /> : <Moon className="size-4" />}
{theme === "dark" ? "Light mode" : "Dark mode"}
</button>
<button
onClick={logout}
className="flex w-full items-center gap-2.5 rounded-lg px-2.5 py-1.5 text-[13px] text-ink-2 hover:bg-surface-2/60 hover:text-ink"
>
<LogOut className="size-4" />
Sign out
</button>
</div>
</aside>
<div className="flex min-w-0 flex-1 flex-col">
<div className="sticky top-0 z-20 border-b border-line bg-page/85 backdrop-blur">
<div className="flex items-center gap-2 overflow-x-auto px-4 py-2 md:hidden">
{NAV.map((n) => (
<NavLink
key={n.to}
to={n.to}
end={n.to === "/"}
className={({ isActive }) =>
cn("flex shrink-0 items-center gap-1.5 rounded-lg px-2 py-1 text-xs", isActive ? "bg-surface-2 text-ink" : "text-ink-2")
}
>
{n.icon}
{n.label}
</NavLink>
))}
</div>
<div className="flex flex-wrap items-center justify-end gap-2 px-4 py-2 md:px-6">
<RefreshCw className={cn("size-3.5 text-muted", fetching > 0 && "animate-spin text-accent")} aria-label={fetching ? "Refreshing" : "Idle"} />
<Select value={refresh} onChange={(e) => setRefresh(Number(e.target.value))} className="h-7.5 text-xs" aria-label="Auto refresh">
{REFRESH.map((r) => (
<option key={r.key} value={r.key}>
Refresh: {r.label}
</option>
))}
</Select>
<Segmented size="sm" options={RANGES.map((r) => ({ key: r.key, label: r.label }))} value={range} onChange={setRange} />
<button
onClick={() => setTheme(theme === "dark" ? "light" : "dark")}
className="inline-flex size-7 items-center justify-center rounded-lg text-ink-2 hover:bg-surface-2 md:hidden"
aria-label="Toggle theme"
>
{theme === "dark" ? <Sun className="size-4" /> : <Moon className="size-4" />}
</button>
</div>
</div>
<main className="mx-auto w-full max-w-[1400px] flex-1 px-4 py-6 md:px-6">
<Outlet />
</main>
</div>
</div>
);
}
web/src/components/charts/BarList.tsx
+37
View File
@@ -0,0 +1,37 @@
import type { ReactNode } from "react";
import { compact } from "../../lib/format";
import { cn } from "../ui";
export interface BarItem {
key: string;
label: ReactNode;
value: number;
hint?: string;
onClick?: () => void;
}
/** Ranked list with a thin magnitude bar under each label. */
export function BarList({ items, format = compact, empty = "Nothing yet" }: { items: BarItem[]; format?: (v: number) => string; empty?: string }) {
if (items.length === 0) return <div className="py-6 text-center text-sm text-muted">{empty}</div>;
const max = Math.max(...items.map((i) => i.value), 1);
return (
<ul className="space-y-2.5">
{items.map((it) => (
<li
key={it.key}
title={it.hint}
onClick={it.onClick}
className={cn("group", it.onClick && "cursor-pointer")}
>
<div className="flex items-baseline justify-between gap-3 text-[13px]">
<span className={cn("min-w-0 truncate text-ink-2", it.onClick && "group-hover:text-ink")}>{it.label}</span>
<span className="tnum shrink-0 text-ink">{format(it.value)}</span>
</div>
<div className="mt-1 h-1 w-full rounded-full bg-surface-2">
<div className="h-1 rounded-full bg-accent" style={{ width: `${Math.max(2, (it.value / max) * 100)}%` }} />
</div>
</li>
))}
</ul>
);
}
web/src/components/charts/Sparkline.tsx
+36
View File
@@ -0,0 +1,36 @@
/** Tiny trend line: de-emphasised line, latest value marked in the accent. */
export function Sparkline({ data, className }: { data: (number | null)[]; className?: string }) {
const vals = data.map((v) => (v === null || Number.isNaN(v) ? null : v));
const nums = vals.filter((v): v is number => v !== null);
if (nums.length < 2) return null;
const min = Math.min(...nums, 0);
const max = Math.max(...nums);
const W = 96;
const H = 32;
const pad = 4;
const x = (i: number) => (i / (vals.length - 1)) * (W - pad * 2) + pad;
const y = (v: number) => H - pad - (max === min ? 0.5 : (v - min) / (max - min)) * (H - pad * 2);
const segments: string[] = [];
let cur = "";
vals.forEach((v, i) => {
if (v === null) {
if (cur) segments.push(cur);
cur = "";
return;
}
cur += `${cur ? "L" : "M"}${x(i).toFixed(1)},${y(v).toFixed(1)}`;
});
if (cur) segments.push(cur);
let lastIdx = vals.length - 1;
while (lastIdx >= 0 && vals[lastIdx] === null) lastIdx--;
return (
<svg viewBox={`0 0 ${W} ${H}`} className={className} preserveAspectRatio="none" aria-hidden>
{segments.map((d, i) => (
<path key={i} d={d} fill="none" stroke="var(--muted)" strokeWidth={1.5} strokeLinejoin="round" strokeLinecap="round" vectorEffect="non-scaling-stroke" />
))}
{lastIdx >= 0 && <circle cx={x(lastIdx)} cy={y(vals[lastIdx] as number)} r={2.5} fill="var(--accent)" stroke="var(--surface)" strokeWidth={1.5} />}
</svg>
);
}
web/src/components/charts/TimeChart.tsx
+233
View File
@@ -0,0 +1,233 @@
import { useEffect, useMemo, useRef, useState } from "react";
import uPlot from "uplot";
import { useSettings } from "../../lib/settings";
import type { QueryResult } from "../../lib/types";
import { compact } from "../../lib/format";
import { cn } from "../ui";
export interface ChartSeries {
label: string;
data: (number | null)[];
/** fixed categorical slot (1-8) so an entity keeps its colour across filters */
slot?: number;
}
export interface ChartData {
x: number[];
series: ChartSeries[];
}
/** Converts a metrics query result into chart data. */
export function fromQuery(
q: QueryResult | null | undefined,
label: (labels: Record<string, string>, i: number) => string = (l) => Object.values(l).join(" · ") || q?.name || "value",
scale = 1,
slotOf?: (labels: Record<string, string>) => number | undefined,
): ChartData {
if (!q || q.series.length === 0) return { x: [], series: [] };
const x = q.series[0].points.map((p) => p[0]);
return {
x,
series: q.series.map((s, i) => ({
label: label(s.labels, i),
slot: slotOf?.(s.labels),
data: s.points.map((p) => (p[1] === null ? null : p[1] * scale)),
})),
};
}
function cssVar(name: string): string {
return getComputedStyle(document.documentElement).getPropertyValue(name).trim();
}
function seriesColor(i: number, slot?: number): string {
const n = slot ?? i + 1;
return cssVar(`--series-${((n - 1) % 8) + 1}`);
}
function fmtTick(ts: number, span: number): string {
const d = new Date(ts * 1000);
if (span >= 2 * 86400) return d.toLocaleDateString("en-GB", { day: "2-digit", month: "short" });
return d.toLocaleTimeString("en-GB", { hour: "2-digit", minute: "2-digit" });
}
function fmtTime(ts: number, span: number): string {
const d = new Date(ts * 1000);
if (span > 2 * 86400) return d.toLocaleString("en-GB", { month: "short", day: "2-digit", hour: "2-digit", minute: "2-digit" });
return d.toLocaleTimeString("en-GB", { hour: "2-digit", minute: "2-digit" });
}
function tooltipPlugin(format: (v: number) => string, labels: () => ChartSeries[]): uPlot.Plugin {
let tip: HTMLDivElement;
return {
hooks: {
init: (u) => {
tip = document.createElement("div");
tip.className =
"pointer-events-none absolute z-10 hidden min-w-36 rounded-lg border border-line bg-surface px-2.5 py-2 text-xs text-ink shadow-lg";
u.over.appendChild(tip);
u.over.addEventListener("mouseleave", () => tip.classList.add("hidden"));
},
setCursor: (u) => {
const idx = u.cursor.idx;
if (idx === null || idx === undefined || u.cursor.left === undefined || u.cursor.left < 0) {
tip.classList.add("hidden");
return;
}
const ts = u.data[0][idx];
const span = (u.data[0][u.data[0].length - 1] ?? 0) - (u.data[0][0] ?? 0);
const rows = labels()
.map((s, i) => {
const v = u.data[i + 1][idx];
const color = (u.series[i + 1].stroke as () => string)?.() ?? "";
return `<div class="flex items-center justify-between gap-4 py-0.5"><span class="flex items-center gap-1.5 text-ink-2"><span style="background:${color}" class="inline-block h-0.5 w-3 rounded"></span>${escapeHtml(s.label)}</span><span class="tnum text-ink">${v === null || v === undefined ? "—" : format(v)}</span></div>`;
})
.join("");
tip.innerHTML = `<div class="mb-1 text-muted">${fmtTime(ts, span)}${span > 2 * 86400 ? "" : " · " + new Date(ts * 1000).toLocaleDateString("en-GB", { day: "2-digit", month: "short" })}</div>${rows}`;
tip.classList.remove("hidden");
const left = u.cursor.left;
const w = tip.offsetWidth;
const overW = u.over.clientWidth;
tip.style.left = `${left + 12 + w > overW ? left - w - 12 : left + 12}px`;
tip.style.top = `4px`;
},
},
};
}
function escapeHtml(s: string): string {
return s.replace(/[&<>"]/g, (c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" })[c] as string);
}
export function TimeChart({
data,
height = 220,
format = compact,
area,
bars,
yMin = 0,
className,
empty = "No data for this range yet",
}: {
data: ChartData;
height?: number;
format?: (v: number) => string;
area?: boolean;
bars?: boolean;
yMin?: number | null;
className?: string;
empty?: string;
}) {
const { theme } = useSettings();
const wrapRef = useRef<HTMLDivElement>(null);
const plotRef = useRef<uPlot | null>(null);
const seriesRef = useRef<ChartSeries[]>(data.series);
seriesRef.current = data.series;
const [width, setWidth] = useState(0);
const hasData = data.x.length > 0 && data.series.some((s) => s.data.some((v) => v !== null));
const shape = `${data.series.map((s) => `${s.label}#${s.slot ?? ""}`).join("|")}`;
const aligned = useMemo<uPlot.AlignedData>(() => [data.x, ...data.series.map((s) => s.data)] as uPlot.AlignedData, [data]);
useEffect(() => {
const el = wrapRef.current;
if (!el) return;
const ro = new ResizeObserver((entries) => setWidth(Math.floor(entries[0].contentRect.width)));
ro.observe(el);
return () => ro.disconnect();
}, []);
// (re)create when the theme or the set of series changes
useEffect(() => {
const el = wrapRef.current;
if (!el || !hasData || width === 0) return;
const muted = cssVar("--muted");
const grid = cssVar("--grid");
const font = "11px system-ui, -apple-system, Segoe UI, sans-serif";
const single = data.series.length === 1;
const opts: uPlot.Options = {
width,
height,
padding: [8, 14, 0, 0],
legend: { show: false },
cursor: { points: { size: 8, width: 2 }, drag: { x: false, y: false } },
scales: { x: { time: true }, y: { range: (_u, min, max) => [yMin === null ? min : Math.min(yMin, min), max <= 0 ? 1 : max * 1.08] } },
axes: [
{
stroke: muted,
font,
grid: { show: false },
ticks: { stroke: grid, width: 1, size: 4 },
space: 70,
values: (_u, splits) => {
const span = (splits[splits.length - 1] ?? 0) - (splits[0] ?? 0);
return splits.map((ts) => fmtTick(ts, span));
},
},
{
stroke: muted,
font,
size: 48,
space: 36,
grid: { stroke: grid, width: 1 },
ticks: { show: false },
values: (_u, vals) => vals.map((v) => format(v)),
},
],
series: [
{},
...data.series.map((s, i) => {
const color = seriesColor(i, s.slot);
return {
label: s.label,
stroke: color,
width: 2,
spanGaps: false,
// uPlot shows markers only when samples are far apart, so sparse data stays visible
points: { size: 7, width: 2, stroke: cssVar("--surface"), fill: color },
fill: bars ? color : area || single ? `color-mix(in srgb, ${color} 12%, transparent)` : undefined,
paths: bars ? uPlot.paths.bars!({ size: [0.6, 24], radius: 0.15 }) : undefined,
} as uPlot.Series;
}),
],
plugins: [tooltipPlugin(format, () => seriesRef.current)],
};
plotRef.current?.destroy();
plotRef.current = new uPlot(opts, aligned, el);
return () => {
plotRef.current?.destroy();
plotRef.current = null;
};
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [theme, shape, hasData, height, bars, area, width === 0]);
useEffect(() => {
if (plotRef.current && width > 0) plotRef.current.setSize({ width, height });
}, [width, height]);
useEffect(() => {
plotRef.current?.setData(aligned);
}, [aligned]);
return (
<div className={cn("min-w-0", className)}>
<div ref={wrapRef} style={{ height }} className="relative w-full">
{!hasData && <div className="flex h-full items-center justify-center text-sm text-muted">{empty}</div>}
</div>
{hasData && data.series.length > 1 && <Legend series={data.series} />}
</div>
);
}
export function Legend({ series }: { series: ChartSeries[] }) {
return (
<div className="mt-2 flex flex-wrap gap-x-4 gap-y-1 text-xs text-ink-2">
{series.map((s, i) => (
<span key={s.label + i} className="inline-flex items-center gap-1.5">
<span className="inline-block h-0.5 w-3 rounded" style={{ background: `var(--series-${(((s.slot ?? i + 1) - 1) % 8) + 1})` }} />
{s.label}
</span>
))}
</div>
);
}
web/src/components/ui.tsx
+484
View File
@@ -0,0 +1,484 @@
import {
createContext,
useCallback,
useContext,
useEffect,
useRef,
useState,
type ButtonHTMLAttributes,
type InputHTMLAttributes,
type ReactNode,
type SelectHTMLAttributes,
type TextareaHTMLAttributes,
} from "react";
import { AlertTriangle, Check, CheckCircle2, ChevronLeft, ChevronRight, Copy, Info, Loader2, X, XCircle } from "lucide-react";
import { Sparkline } from "./charts/Sparkline";
export function cn(...parts: (string | false | null | undefined)[]): string {
return parts.filter(Boolean).join(" ");
}
// ---- containers -------------------------------------------------------------
export function Card({
title,
subtitle,
actions,
children,
className,
bodyClass,
}: {
title?: ReactNode;
subtitle?: ReactNode;
actions?: ReactNode;
children: ReactNode;
className?: string;
bodyClass?: string;
}) {
return (
<section className={cn("rounded-xl border border-line bg-surface", className)}>
{(title || actions) && (
<header className="flex items-start justify-between gap-3 px-4 pt-3.5">
<div className="min-w-0">
{title && <h2 className="text-[13px] font-medium text-ink">{title}</h2>}
{subtitle && <p className="mt-0.5 text-xs text-muted">{subtitle}</p>}
</div>
{actions && <div className="flex shrink-0 items-center gap-2">{actions}</div>}
</header>
)}
<div className={cn("p-4", bodyClass)}>{children}</div>
</section>
);
}
export function PageHeader({ title, description, actions }: { title: string; description?: ReactNode; actions?: ReactNode }) {
return (
<div className="mb-5 flex flex-wrap items-end justify-between gap-3">
<div>
<h1 className="text-xl font-semibold tracking-tight text-ink">{title}</h1>
{description && <p className="mt-1 text-sm text-ink-2">{description}</p>}
</div>
{actions && <div className="flex flex-wrap items-center gap-2">{actions}</div>}
</div>
);
}
// ---- controls ---------------------------------------------------------------
type ButtonVariant = "primary" | "secondary" | "ghost" | "danger";
export function Button({
variant = "secondary",
size = "md",
icon,
loading,
className,
children,
...rest
}: ButtonHTMLAttributes<HTMLButtonElement> & {
variant?: ButtonVariant;
size?: "sm" | "md";
icon?: ReactNode;
loading?: boolean;
}) {
const styles: Record<ButtonVariant, string> = {
primary: "bg-accent text-accent-ink hover:brightness-110 border-transparent",
secondary: "bg-surface-2 text-ink border-line hover:border-ink/25",
ghost: "bg-transparent text-ink-2 border-transparent hover:bg-surface-2 hover:text-ink",
danger: "bg-transparent text-critical border-critical/40 hover:bg-critical/10",
};
return (
<button
{...rest}
disabled={rest.disabled || loading}
className={cn(
"inline-flex items-center justify-center gap-1.5 whitespace-nowrap rounded-lg border font-medium transition-colors",
"disabled:cursor-not-allowed disabled:opacity-50 focus-visible:outline-2 focus-visible:outline-accent",
size === "sm" ? "h-7 px-2.5 text-xs" : "h-8.5 px-3 text-[13px]",
styles[variant],
className,
)}
>
{loading ? <Loader2 className="size-3.5 animate-spin" /> : icon}
{children}
</button>
);
}
const fieldClass =
"h-8.5 rounded-lg border border-line bg-surface px-2.5 text-[13px] text-ink placeholder:text-muted " +
"focus:border-accent focus:outline-none";
export function Input({ className, ...rest }: InputHTMLAttributes<HTMLInputElement>) {
return <input {...rest} className={cn(fieldClass, className)} />;
}
export function Select({ className, children, ...rest }: SelectHTMLAttributes<HTMLSelectElement>) {
return (
<select {...rest} className={cn(fieldClass, "pr-7", className)}>
{children}
</select>
);
}
export function Textarea({ className, ...rest }: TextareaHTMLAttributes<HTMLTextAreaElement>) {
return <textarea {...rest} className={cn(fieldClass, "h-auto py-2 font-mono text-xs", className)} />;
}
export function Segmented<T extends string | number>({
options,
value,
onChange,
size = "md",
}: {
options: readonly { key: T; label: ReactNode }[];
value: T;
onChange: (v: T) => void;
size?: "sm" | "md";
}) {
return (
<div className="inline-flex rounded-lg border border-line bg-surface p-0.5">
{options.map((o) => (
<button
key={String(o.key)}
onClick={() => onChange(o.key)}
className={cn(
"rounded-md px-2.5 font-medium transition-colors",
size === "sm" ? "h-6 text-xs" : "h-7 text-[13px]",
o.key === value ? "bg-surface-2 text-ink shadow-sm" : "text-muted hover:text-ink",
)}
>
{o.label}
</button>
))}
</div>
);
}
export function Checkbox({ checked, onChange, label }: { checked: boolean; onChange: (v: boolean) => void; label?: string }) {
return (
<input
type="checkbox"
aria-label={label}
checked={checked}
onChange={(e) => onChange(e.target.checked)}
onClick={(e) => e.stopPropagation()}
className="size-3.5 cursor-pointer accent-[var(--accent)]"
/>
);
}
// ---- status -----------------------------------------------------------------
export type Tone = "neutral" | "accent" | "good" | "warning" | "serious" | "critical";
const toneDot: Record<Tone, string> = {
neutral: "bg-muted",
accent: "bg-accent",
good: "bg-good",
warning: "bg-warning",
serious: "bg-serious",
critical: "bg-critical",
};
/** A status label: always a dot plus text, never colour alone. */
export function Badge({ tone = "neutral", children, title }: { tone?: Tone; children: ReactNode; title?: string }) {
return (
<span
title={title}
className="inline-flex items-center gap-1.5 whitespace-nowrap rounded-full border border-line bg-surface-2 px-2 py-0.5 text-[11px] font-medium text-ink-2"
>
<span className={cn("size-1.5 rounded-full", toneDot[tone])} />
{children}
</span>
);
}
export function StatusDot({ online }: { online: boolean }) {
return (
<span className="relative inline-flex size-2">
{online && <span className="absolute inline-flex size-full animate-ping rounded-full bg-good opacity-50" />}
<span className={cn("relative inline-flex size-2 rounded-full", online ? "bg-good" : "bg-critical")} />
</span>
);
}
// ---- figures ----------------------------------------------------------------
export function StatTile({
label,
value,
sub,
trend,
hero,
onClick,
}: {
label: string;
value: ReactNode;
sub?: ReactNode;
trend?: (number | null)[];
hero?: boolean;
onClick?: () => void;
}) {
return (
<div
onClick={onClick}
className={cn(
"flex min-w-0 flex-col justify-between rounded-xl border border-line bg-surface p-4",
onClick && "cursor-pointer transition-colors hover:border-ink/25",
)}
>
<div className="text-xs text-muted">{label}</div>
<div className="mt-2 flex items-end justify-between gap-3">
<div className="min-w-0">
<div className={cn("truncate font-semibold tracking-tight text-ink", hero ? "text-[40px] leading-none" : "text-2xl")}>{value}</div>
{sub && <div className="mt-1.5 truncate text-xs text-ink-2">{sub}</div>}
</div>
{trend && trend.length > 1 && <Sparkline data={trend} className="h-8 w-24 shrink-0" />}
</div>
</div>
);
}
/** Progress meter: unfilled track is a lighter step of the fill. */
export function Meter({ value, max, tone = "accent" }: { value: number; max: number; tone?: "accent" | "good" | "warning" | "critical" }) {
const p = max > 0 ? Math.min(100, (value / max) * 100) : 0;
const color = { accent: "var(--accent)", good: "var(--good)", warning: "var(--warning)", critical: "var(--critical)" }[tone];
return (
<div className="h-1.5 w-full overflow-hidden rounded-full" style={{ background: `color-mix(in srgb, ${color} 18%, transparent)` }}>
<div className="h-full rounded-full" style={{ width: `${p}%`, background: color }} />
</div>
);
}
// ---- feedback -----------------------------------------------------------------
export function Spinner({ className }: { className?: string }) {
return <Loader2 className={cn("size-4 animate-spin text-muted", className)} />;
}
export function Loading({ label = "Loading…" }: { label?: string }) {
return (
<div className="flex items-center gap-2 py-8 text-sm text-muted">
<Spinner /> {label}
</div>
);
}
export function Empty({ children }: { children: ReactNode }) {
return <div className="py-8 text-center text-sm text-muted">{children}</div>;
}
export function ErrorBox({ error }: { error: unknown }) {
const msg = error instanceof Error ? error.message : String(error);
return (
<div className="flex items-center gap-2 rounded-lg border border-critical/30 bg-critical/10 px-3 py-2 text-sm text-ink">
<XCircle className="size-4 shrink-0 text-critical" />
{msg}
</div>
);
}
export function CopyButton({ value, className }: { value: string; className?: string }) {
const [done, setDone] = useState(false);
return (
<button
title="Copy"
onClick={(e) => {
e.stopPropagation();
navigator.clipboard?.writeText(value).then(() => {
setDone(true);
setTimeout(() => setDone(false), 1200);
});
}}
className={cn("inline-flex size-6 items-center justify-center rounded-md text-muted hover:bg-surface-2 hover:text-ink", className)}
>
{done ? <Check className="size-3.5 text-good" /> : <Copy className="size-3.5" />}
</button>
);
}
export function Pager({ total, limit, offset, onChange }: { total: number; limit: number; offset: number; onChange: (offset: number) => void }) {
const page = Math.floor(offset / limit) + 1;
const pages = Math.max(1, Math.ceil(total / limit));
return (
<div className="flex items-center justify-between gap-3 text-xs text-muted">
<span className="tnum">
{total === 0 ? "No results" : `${(offset + 1).toLocaleString()}–${Math.min(offset + limit, total).toLocaleString()} of ${total.toLocaleString()}`}
</span>
<div className="flex items-center gap-1">
<Button size="sm" variant="ghost" disabled={page <= 1} onClick={() => onChange(Math.max(0, offset - limit))} icon={<ChevronLeft className="size-3.5" />} />
<span className="tnum px-1">
{page} / {pages}
</span>
<Button size="sm" variant="ghost" disabled={page >= pages} onClick={() => onChange(offset + limit)} icon={<ChevronRight className="size-3.5" />} />
</div>
</div>
);
}
// ---- table --------------------------------------------------------------------
export function Table({ children, className }: { children: ReactNode; className?: string }) {
return (
<div className={cn("-mx-4 overflow-x-auto", className)}>
<table className="w-full border-collapse text-[13px]">{children}</table>
</div>
);
}
export function Th({ children, className, onClick }: { children?: ReactNode; className?: string; onClick?: () => void }) {
return (
<th
onClick={onClick}
className={cn(
"border-b border-line px-4 py-2 text-left text-[11px] font-medium uppercase tracking-wide text-muted first:pl-4 whitespace-nowrap",
onClick && "cursor-pointer select-none hover:text-ink",
className,
)}
>
{children}
</th>
);
}
export function Td({ children, className, title }: { children?: ReactNode; className?: string; title?: string }) {
return (
<td title={title} className={cn("border-b border-line/60 px-4 py-2 align-middle text-ink-2", className)}>
{children}
</td>
);
}
// ---- toasts & confirm ---------------------------------------------------------
interface ToastItem {
id: number;
tone: "good" | "critical" | "neutral";
text: string;
}
interface Feedback {
toast: (text: string, tone?: ToastItem["tone"]) => void;
confirm: (opts: { title: string; body?: ReactNode; confirmLabel?: string; danger?: boolean; requireText?: string }) => Promise<boolean>;
}
const FeedbackCtx = createContext<Feedback | null>(null);
export function useFeedback(): Feedback {
const f = useContext(FeedbackCtx);
if (!f) throw new Error("FeedbackProvider missing");
return f;
}
interface ConfirmState {
title: string;
body?: ReactNode;
confirmLabel?: string;
danger?: boolean;
requireText?: string;
resolve: (ok: boolean) => void;
}
export function FeedbackProvider({ children }: { children: ReactNode }) {
const [toasts, setToasts] = useState<ToastItem[]>([]);
const [confirmState, setConfirmState] = useState<ConfirmState | null>(null);
const [typed, setTyped] = useState("");
const dialogRef = useRef<HTMLDialogElement>(null);
const nextId = useRef(1);
const toast = useCallback((text: string, tone: ToastItem["tone"] = "good") => {
const id = nextId.current++;
setToasts((t) => [...t, { id, tone, text }]);
setTimeout(() => setToasts((t) => t.filter((x) => x.id !== id)), tone === "critical" ? 6000 : 3500);
}, []);
const confirm = useCallback<Feedback["confirm"]>(
(opts) =>
new Promise<boolean>((resolve) => {
setTyped("");
setConfirmState({ ...opts, resolve });
}),
[],
);
useEffect(() => {
const d = dialogRef.current;
if (!d) return;
if (confirmState && !d.open) d.showModal();
if (!confirmState && d.open) d.close();
}, [confirmState]);
const close = (ok: boolean) => {
confirmState?.resolve(ok);
setConfirmState(null);
};
const blocked = !!confirmState?.requireText && typed !== confirmState.requireText;
return (
<FeedbackCtx.Provider value={{ toast, confirm }}>
{children}
<dialog
ref={dialogRef}
onCancel={(e) => {
e.preventDefault();
close(false);
}}
className="m-auto w-[min(440px,calc(100vw-32px))] rounded-xl border border-line bg-surface p-0 text-ink shadow-2xl"
>
{confirmState && (
<div className="p-5">
<div className="flex items-start gap-3">
{confirmState.danger ? (
<AlertTriangle className="mt-0.5 size-5 shrink-0 text-critical" />
) : (
<Info className="mt-0.5 size-5 shrink-0 text-accent" />
)}
<div className="min-w-0">
<h3 className="font-medium">{confirmState.title}</h3>
{confirmState.body && <div className="mt-1.5 text-sm text-ink-2">{confirmState.body}</div>}
</div>
</div>
{confirmState.requireText && (
<div className="mt-4">
<label className="text-xs text-muted">
Type <span className="font-mono text-ink">{confirmState.requireText}</span> to confirm
</label>
<Input autoFocus className="mt-1 w-full font-mono" value={typed} onChange={(e) => setTyped(e.target.value)} />
</div>
)}
<div className="mt-5 flex justify-end gap-2">
<Button variant="ghost" onClick={() => close(false)}>
Cancel
</Button>
<Button variant={confirmState.danger ? "danger" : "primary"} disabled={blocked} onClick={() => close(true)} autoFocus={!confirmState.requireText}>
{confirmState.confirmLabel ?? "Confirm"}
</Button>
</div>
</div>
)}
</dialog>
<div className="pointer-events-none fixed right-4 bottom-4 z-50 flex flex-col gap-2">
{toasts.map((t) => (
<div
key={t.id}
className="pointer-events-auto flex max-w-sm items-center gap-2 rounded-lg border border-line bg-surface px-3 py-2 text-sm text-ink shadow-lg"
>
{t.tone === "good" && <CheckCircle2 className="size-4 shrink-0 text-good" />}
{t.tone === "critical" && <XCircle className="size-4 shrink-0 text-critical" />}
{t.tone === "neutral" && <Info className="size-4 shrink-0 text-accent" />}
<span className="min-w-0 break-words">{t.text}</span>
<button className="ml-1 text-muted hover:text-ink" onClick={() => setToasts((x) => x.filter((y) => y.id !== t.id))}>
<X className="size-3.5" />
</button>
</div>
))}
</div>
</FeedbackCtx.Provider>
);
}
export function LevelBadge({ level }: { level: string }) {
const tone: Tone = level === "error" ? "critical" : level === "warn" ? "warning" : level === "debug" ? "neutral" : "accent";
return <Badge tone={tone}>{level}</Badge>;
}
web/src/index.css
+176
View File
@@ -0,0 +1,176 @@
@import "tailwindcss";
@custom-variant dark (&:where([data-theme="dark"], [data-theme="dark"] *));
/* Colour tokens. Chart colours follow the validated reference palette
(categorical slots in fixed order, separate light/dark steps). */
:root {
color-scheme: light;
--page: #f9f9f7;
--surface: #fcfcfb;
--surface-2: #f2f2ef;
--ink: #0b0b0b;
--ink-2: #52514e;
--muted: #898781;
--line: rgba(11, 11, 11, 0.1);
--grid: #e1e0d9;
--axis: #c3c2b7;
--accent: #2a78d6;
--accent-ink: #ffffff;
--good: #0ca30c;
--good-ink: #006300;
--warning: #fab219;
--serious: #ec835a;
--critical: #d03b3b;
--series-1: #2a78d6;
--series-2: #eb6834;
--series-3: #1baf7a;
--series-4: #eda100;
--series-5: #e87ba4;
--series-6: #008300;
--series-7: #4a3aa7;
--series-8: #e34948;
}
:root[data-theme="dark"] {
color-scheme: dark;
--page: #0d0d0d;
--surface: #1a1a19;
--surface-2: #232321;
--ink: #ffffff;
--ink-2: #c3c2b7;
--muted: #898781;
--line: rgba(255, 255, 255, 0.1);
--grid: #2c2c2a;
--axis: #383835;
--accent: #3987e5;
--accent-ink: #ffffff;
--good-ink: #0ca30c;
--series-1: #3987e5;
--series-2: #d95926;
--series-3: #199e70;
--series-4: #c98500;
--series-5: #d55181;
--series-6: #008300;
--series-7: #9085e9;
--series-8: #e66767;
}
@theme inline {
--color-page: var(--page);
--color-surface: var(--surface);
--color-surface-2: var(--surface-2);
--color-ink: var(--ink);
--color-ink-2: var(--ink-2);
--color-muted: var(--muted);
--color-line: var(--line);
--color-grid: var(--grid);
--color-accent: var(--accent);
--color-accent-ink: var(--accent-ink);
--color-good: var(--good);
--color-good-ink: var(--good-ink);
--color-warning: var(--warning);
--color-serious: var(--serious);
--color-critical: var(--critical);
--font-sans: system-ui, -apple-system, "Segoe UI", Roboto, sans-serif;
--font-mono: ui-monospace, "SF Mono", "Cascadia Code", Consolas, monospace;
}
html,
body {
background: var(--page);
color: var(--ink);
}
body {
font-family: var(--font-sans);
font-size: 14px;
-webkit-font-smoothing: antialiased;
}
.tnum {
font-variant-numeric: tabular-nums;
}
/* uPlot base styles (trimmed from uplot.min.css) */
.uplot,
.uplot *,
.uplot *::before,
.uplot *::after {
box-sizing: border-box;
}
.uplot {
font-family: var(--font-sans);
line-height: 1.5;
width: min-content;
}
.u-wrap {
position: relative;
user-select: none;
}
.u-over,
.u-under {
position: absolute;
}
.u-under {
overflow: hidden;
}
.uplot canvas {
display: block;
position: relative;
width: 100%;
height: 100%;
}
.u-axis {
position: absolute;
}
.u-select {
background: color-mix(in srgb, var(--accent) 12%, transparent);
position: absolute;
pointer-events: none;
}
.u-cursor-x,
.u-cursor-y {
position: absolute;
left: 0;
top: 0;
pointer-events: none;
will-change: transform;
}
.u-cursor-x {
height: 100%;
border-right: 1px solid var(--axis);
}
.u-cursor-y {
width: 100%;
border-bottom: 0;
}
.u-cursor-pt {
position: absolute;
top: 0;
left: 0;
border-radius: 50%;
border: 2px solid var(--surface);
pointer-events: none;
will-change: transform;
background-clip: padding-box !important;
}
.u-hz .u-cursor-y {
display: none;
}
dialog::backdrop {
background: rgba(0, 0, 0, 0.5);
}
::-webkit-scrollbar {
width: 10px;
height: 10px;
}
::-webkit-scrollbar-thumb {
background: var(--grid);
border-radius: 10px;
border: 2px solid var(--page);
}
web/src/lib/api.ts
+53
View File
@@ -0,0 +1,53 @@
// Thin fetch wrapper for the admin JSON API. Auth is the HttpOnly session
// cookie; every request carries the CSRF header the server requires.
export class ApiError extends Error {
status: number;
constructor(status: number, message: string) {
super(message);
this.status = status;
}
}
export const UNAUTHORIZED_EVENT = "rc:unauthorized";
type Method = "GET" | "POST" | "DELETE";
export async function api<T>(path: string, opts: { method?: Method; body?: unknown } = {}): Promise<T> {
const headers: Record<string, string> = { "X-Admin-CSRF": "1", Accept: "application/json" };
if (opts.body !== undefined) headers["Content-Type"] = "application/json";
const res = await fetch(`/admin/api${path}`, {
method: opts.method ?? "GET",
credentials: "same-origin",
headers,
body: opts.body !== undefined ? JSON.stringify(opts.body) : undefined,
});
if (res.status === 204) return undefined as T;
let data: unknown = null;
const text = await res.text();
if (text) {
try {
data = JSON.parse(text);
} catch {
data = { error: text };
}
}
if (!res.ok) {
if (res.status === 401 && !path.startsWith("/auth/")) {
window.dispatchEvent(new Event(UNAUTHORIZED_EVENT));
}
const msg = (data as { error?: string } | null)?.error ?? `${res.status} ${res.statusText}`;
throw new ApiError(res.status, msg);
}
return data as T;
}
export function qs(params: Record<string, string | number | boolean | undefined | null>): string {
const p = new URLSearchParams();
for (const [k, v] of Object.entries(params)) {
if (v === undefined || v === null || v === "") continue;
p.set(k, String(v));
}
const s = p.toString();
return s ? `?${s}` : "";
}
web/src/lib/format.ts
+94
View File
@@ -0,0 +1,94 @@
const compactFmt = new Intl.NumberFormat("en", { notation: "compact", maximumFractionDigits: 1 });
const intFmt = new Intl.NumberFormat("en");
export function compact(n: number | null | undefined): string {
if (n === null || n === undefined || Number.isNaN(n)) return "—";
if (Math.abs(n) < 1000) return intFmt.format(Math.round(n * 10) / 10);
return compactFmt.format(n);
}
export function int(n: number | null | undefined): string {
if (n === null || n === undefined || Number.isNaN(n)) return "—";
return intFmt.format(Math.round(n));
}
export function num(n: number | null | undefined, digits = 1): string {
if (n === null || n === undefined || Number.isNaN(n)) return "—";
return n.toLocaleString("en", { maximumFractionDigits: digits });
}
export function pct(part: number, total: number, digits = 1): string {
if (!total) return "—";
return `${((part / total) * 100).toFixed(digits)}%`;
}
export function bytes(n: number | null | undefined): string {
if (n === null || n === undefined || Number.isNaN(n)) return "—";
const units = ["B", "KB", "MB", "GB", "TB", "PB"];
let v = n;
let i = 0;
while (Math.abs(v) >= 1000 && i < units.length - 1) {
v /= 1000;
i++;
}
return `${v.toFixed(v < 10 && i > 0 ? 1 : 0)} ${units[i]}`;
}
export function ms(n: number | null | undefined): string {
if (n === null || n === undefined || Number.isNaN(n)) return "—";
if (n < 1) return `${(n * 1000).toFixed(0)} µs`;
if (n < 1000) return `${n.toFixed(n < 10 ? 1 : 0)} ms`;
return `${(n / 1000).toFixed(2)} s`;
}
export function duration(sec: number | null | undefined): string {
if (sec === null || sec === undefined || Number.isNaN(sec)) return "—";
sec = Math.max(0, Math.round(sec));
const d = Math.floor(sec / 86400);
const h = Math.floor((sec % 86400) / 3600);
const m = Math.floor((sec % 3600) / 60);
const s = sec % 60;
if (d) return `${d}d ${h}h`;
if (h) return `${h}h ${m}m`;
if (m) return `${m}m ${s}s`;
return `${s}s`;
}
/** Parses an ISO string or unix seconds/ms; returns null for zero/empty times. */
export function toDate(v: string | number | null | undefined): Date | null {
if (v === null || v === undefined || v === "" || v === 0) return null;
const d = typeof v === "number" ? new Date(v > 1e12 ? v : v * 1000) : new Date(v);
if (Number.isNaN(d.getTime()) || d.getFullYear() < 2000) return null;
return d;
}
export function relative(v: string | number | null | undefined): string {
const d = toDate(v);
if (!d) return "—";
const diff = (Date.now() - d.getTime()) / 1000;
if (Math.abs(diff) < 45) return "just now";
const future = diff < 0;
const a = Math.abs(diff);
let out: string;
if (a < 3600) out = `${Math.round(a / 60)}m`;
else if (a < 86400) out = `${Math.round(a / 3600)}h`;
else if (a < 86400 * 60) out = `${Math.round(a / 86400)}d`;
else out = `${Math.round(a / (86400 * 30))}mo`;
return future ? `in ${out}` : `${out} ago`;
}
export function dateTime(v: string | number | null | undefined): string {
const d = toDate(v);
if (!d) return "—";
return d.toLocaleString("en-GB", { year: "numeric", month: "short", day: "2-digit", hour: "2-digit", minute: "2-digit" });
}
export function shortHash(h: string, n = 10): string {
return h.length > n * 2 ? `${h.slice(0, n)}…${h.slice(-4)}` : h;
}
/** Seconds for chart axes: keeps one decimal below a minute so ticks stay distinct. */
export function seconds(v: number): string {
if (v < 60) return `${num(v, v < 10 ? 1 : 0)}s`;
return duration(v);
}
web/src/lib/queries.ts
+59
View File
@@ -0,0 +1,59 @@
import { useQuery, keepPreviousData } from "@tanstack/react-query";
import { api, qs } from "./api";
import { rangeSeconds, useRefetch, useSettings } from "./settings";
import type { QueryResult } from "./types";
export interface MetricParams {
service?: string;
name: string;
agg?: string;
groupBy?: string[];
match?: Record<string, string>;
step?: number;
/** override the global range (seconds) */
rangeSec?: number;
}
/** Fetches a metric time series for the global time range. */
export function useMetric(p: MetricParams, enabled = true) {
const { range } = useSettings();
const refetchInterval = useRefetch();
const span = p.rangeSec ?? rangeSeconds(range);
return useQuery({
queryKey: ["metric", p, span],
enabled,
refetchInterval,
placeholderData: keepPreviousData,
queryFn: () => {
const to = Math.floor(Date.now() / 1000);
return api<QueryResult>(
"/metrics/query" +
qs({
service: p.service ?? "main",
name: p.name,
agg: p.agg,
groupBy: p.groupBy?.join(","),
match: p.match ? JSON.stringify(p.match) : undefined,
step: p.step,
from: to - span,
to,
}),
);
},
});
}
/** Default step for a range so charts stay readable. */
export function stepFor(seconds: number): number {
if (seconds <= 3 * 3600) return 60;
if (seconds <= 12 * 3600) return 300;
if (seconds <= 48 * 3600) return 900;
if (seconds <= 8 * 86400) return 3600;
return 6 * 3600;
}
export function useRangeStep() {
const { range } = useSettings();
const seconds = rangeSeconds(range);
return { seconds, step: stepFor(seconds) };
}
web/src/lib/settings.tsx
+102
View File
@@ -0,0 +1,102 @@
import { createContext, useCallback, useContext, useEffect, useMemo, useState, type ReactNode } from "react";
export type Theme = "dark" | "light";
export const RANGES = [
{ key: "1h", label: "1h", seconds: 3600 },
{ key: "6h", label: "6h", seconds: 6 * 3600 },
{ key: "24h", label: "24h", seconds: 86400 },
{ key: "7d", label: "7d", seconds: 7 * 86400 },
{ key: "30d", label: "30d", seconds: 30 * 86400 },
] as const;
export type RangeKey = (typeof RANGES)[number]["key"];
export const REFRESH = [
{ key: 0, label: "Off" },
{ key: 10, label: "10s" },
{ key: 30, label: "30s" },
{ key: 60, label: "1m" },
] as const;
interface Settings {
theme: Theme;
setTheme: (t: Theme) => void;
range: RangeKey;
setRange: (r: RangeKey) => void;
refresh: number;
setRefresh: (s: number) => void;
}
const Ctx = createContext<Settings | null>(null);
function load<T>(key: string, fallback: T, valid: (v: unknown) => boolean): T {
try {
const raw = localStorage.getItem(key);
if (raw === null) return fallback;
const v = JSON.parse(raw);
return valid(v) ? (v as T) : fallback;
} catch {
return fallback;
}
}
function save(key: string, v: unknown) {
try {
localStorage.setItem(key, JSON.stringify(v));
} catch {
/* storage unavailable */
}
}
export function SettingsProvider({ children }: { children: ReactNode }) {
const [theme, setThemeState] = useState<Theme>(() =>
document.documentElement.getAttribute("data-theme") === "light" ? "light" : "dark",
);
const [range, setRangeState] = useState<RangeKey>(() =>
load("rc-range", "24h", (v) => RANGES.some((r) => r.key === v)),
);
const [refresh, setRefreshState] = useState<number>(() =>
load("rc-refresh", 30, (v) => REFRESH.some((r) => r.key === v)),
);
useEffect(() => {
document.documentElement.setAttribute("data-theme", theme);
try {
localStorage.setItem("rc-theme", theme);
} catch {
/* ignore */
}
}, [theme]);
const setTheme = useCallback((t: Theme) => setThemeState(t), []);
const setRange = useCallback((r: RangeKey) => {
setRangeState(r);
save("rc-range", r);
}, []);
const setRefresh = useCallback((s: number) => {
setRefreshState(s);
save("rc-refresh", s);
}, []);
const value = useMemo(
() => ({ theme, setTheme, range, setRange, refresh, setRefresh }),
[theme, setTheme, range, setRange, refresh, setRefresh],
);
return <Ctx.Provider value={value}>{children}</Ctx.Provider>;
}
export function useSettings(): Settings {
const s = useContext(Ctx);
if (!s) throw new Error("SettingsProvider missing");
return s;
}
export function rangeSeconds(key: RangeKey): number {
return RANGES.find((r) => r.key === key)?.seconds ?? 86400;
}
/** Refetch interval in ms for TanStack Query (false when auto-refresh is off). */
export function useRefetch(): number | false {
const { refresh } = useSettings();
return refresh > 0 ? refresh * 1000 : false;
}
web/src/lib/types.ts
+289
View File
@@ -0,0 +1,289 @@
// Response shapes of the admin API (see admin/*.go).
export type Pt = [number, number | null];
export interface SeriesResult {
labels: Record<string, string>;
points: Pt[];
}
export interface QueryResult {
name: string;
kind: string;
agg: string;
resolution: number;
step: number;
from: number;
to: number;
series: SeriesResult[];
truncated: boolean;
}
export interface Total {
labels: Record<string, string>;
count: number;
sum: number;
min: number | null;
max: number | null;
avg: number | null;
last: number | null;
rate: number | null;
p50: number | null;
p95: number | null;
p99: number | null;
}
export interface MiniAddon {
id: string;
name: string;
author: string;
subscriberCount: number;
currentVersionSize: number;
lastIndexedAt?: string | null;
createdAt: string;
}
export interface TableInfo {
name: string;
rows: number;
deadRows: number;
sizeBytes: number;
}
export interface HeavyStats {
asOf: number;
filesLive: number;
filesSoftDeleted: number;
filesApprox: boolean;
distinctHashes: number;
distinctAsOf: number;
dbDriver: string;
dbSizeBytes: number;
tables: TableInfo[];
duplicatesAsOf: number;
error?: string;
}
export interface Overview {
asOf: number;
data: {
addons: {
total: number;
indexed: number;
queued: number;
indexing: number;
priority: number;
recheck: number;
blocked: number;
unlisted: number;
private: number;
totalSizeBytes: number;
added24h: number;
added7d: number;
indexed24h: number;
softDeleted: number;
};
types: { type: string; count: number }[];
whitelistCount: number;
dependencies: { edges: number; addonsWithDependencies: number; addonsDependedOn: number };
traffic: {
requests24h?: number;
websiteRequests24h?: number;
workerRequests24h?: number;
websiteP95Ms?: number | null;
websiteP50Ms?: number | null;
errors5xx24h: number;
visitorsToday: number;
botsToday: number;
};
activity: Record<string, number>;
services: { online: number; total: number };
topSubscribed: MiniAddon[];
largest: MiniAddon[];
recentlyAdded: MiniAddon[];
recentlyIndexed: MiniAddon[];
heavy: HeavyStats | null;
};
}
export interface AddonRow {
id: string;
name: string;
type: string;
author: string;
preview: string;
subscriberCount: number;
currentVersionNumber: string;
currentVersionSize: number;
unlisted: boolean;
private: boolean;
blocked: boolean;
toBeIndexed: boolean;
isBeingIndexed: boolean;
priorityIndexing: boolean;
recheck: boolean;
indexStartTime: string | null;
lastIndexedAt: string | null;
createdAt: string;
updatedAt: string;
deletedAt: string | null;
}
export interface Paged<T> {
items: T[];
total: number;
limit: number;
offset: number;
}
export interface RelatedAddon {
id: string;
name: string;
type: string;
author: string;
subscriberCount: number;
currentVersionNumber: string;
}
export interface AddonDetail {
addon: AddonRow;
summary: string;
currentVersionId: number;
fileCount: number;
deletedFileRows: number;
dependencies: { total: number; items: RelatedAddon[] | null; missing: { id: string; name: string; version: string }[] | null };
dependents: { total: number; items: RelatedAddon[] | null };
duplicates: { addonId: string; name: string; sharedFiles: number }[];
}
export interface FileRow {
id: number;
path: string;
hash: string;
version: string;
whitelisted: boolean;
otherAddons: number;
}
export interface QueueRow {
id: string;
name: string;
currentVersionSize: number;
subscriberCount: number;
priorityIndexing: boolean;
blocked: boolean;
indexStartTime: string | null;
updatedAt: string;
elapsedSec: number;
stuck: boolean;
}
export interface QueueData {
asOf: number;
counts: { total: number; queued: number; priority: number; indexing: number };
blockedQueued: number;
timeoutMinutes: number;
indexing: QueueRow[];
priority: QueueRow[];
next: QueueRow[];
throughput: {
perHour1h: number;
perHour6h: number;
indexed24h: number;
etaHours: number | null;
duration24h: Total | null;
};
}
export interface WhitelistRow {
id: number;
hash: string;
note: string;
createdAt: string;
files: number;
addons: number;
samplePath: string;
}
export interface DupHash {
hash: string;
addonCount: number;
fileCount: number;
samplePath: string;
whitelisted: boolean;
}
export interface ServiceInstance {
id: number;
service: string;
instance: string;
version: string;
status: string;
statusMessage: string;
startedAt: number;
lastSeen: number;
heartbeatIntervalSec: number;
remoteAddr: string;
online: boolean;
uptimeSec: number;
metaData?: Record<string, unknown>;
}
export interface ServiceEvent {
id: number;
service: string;
instance: string;
level: "debug" | "info" | "warn" | "error";
message: string;
ts: number;
fieldsData?: Record<string, unknown>;
}
export interface CatalogEntry {
service: string;
name: string;
kind: string;
help: string;
labels: Record<string, string[]>;
instances: string[];
series: number;
lastSeen: number;
}
export interface RouteStat {
method: string;
route: string;
group: string;
count: number;
errors: number;
clientErrors: number;
p50: number | null;
p95: number | null;
p99: number | null;
max: number | null;
avg: number | null;
}
export interface TrafficData {
asOf: number;
from: number;
to: number;
step: number;
requestsByGroup: QueryResult;
latency: { p50: QueryResult; p95: QueryResult; p99: QueryResult };
errors5xx: QueryResult | null;
errors4xx: QueryResult | null;
statusTotals: Total[] | null;
clientTotals: Total[] | null;
groupTotals: Total[] | null;
website: Total | null;
routes: RouteStat[];
visitorsDaily: QueryResult | null;
visitorsToday: { human: number; bot: number };
}
export interface TopEntry {
item: string;
count: number;
name?: string;
}
web/src/main.tsx
+67
View File
@@ -0,0 +1,67 @@
import { StrictMode, lazy, Suspense } from "react";
import { createRoot } from "react-dom/client";
import { createBrowserRouter, RouterProvider } from "react-router";
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
import "./index.css";
import { SettingsProvider } from "./lib/settings";
import { FeedbackProvider, Loading } from "./components/ui";
import { RequireAuth } from "./components/Shell";
import { ApiError } from "./lib/api";
import Login from "./pages/Login";
import Overview from "./pages/Overview";
const Traffic = lazy(() => import("./pages/Traffic"));
const Queue = lazy(() => import("./pages/Queue"));
const Addons = lazy(() => import("./pages/Addons"));
const Whitelist = lazy(() => import("./pages/Whitelist"));
const Services = lazy(() => import("./pages/Services"));
const Explorer = lazy(() => import("./pages/Explorer"));
const Events = lazy(() => import("./pages/Events"));
const System = lazy(() => import("./pages/System"));
const queryClient = new QueryClient({
defaultOptions: {
queries: {
staleTime: 10_000,
refetchOnWindowFocus: false,
retry: (count, err) => !(err instanceof ApiError && err.status < 500) && count < 2,
},
},
});
const page = (el: React.ReactNode) => <Suspense fallback={<Loading />}>{el}</Suspense>;
const router = createBrowserRouter(
[
{ path: "/login", element: <Login /> },
{
element: <RequireAuth />,
children: [
{ index: true, element: <Overview /> },
{ path: "traffic", element: page(<Traffic />) },
{ path: "queue", element: page(<Queue />) },
{ path: "addons", element: page(<Addons />) },
{ path: "addons/:id", element: page(<Addons />) },
{ path: "whitelist", element: page(<Whitelist />) },
{ path: "services", element: page(<Services />) },
{ path: "explorer", element: page(<Explorer />) },
{ path: "events", element: page(<Events />) },
{ path: "system", element: page(<System />) },
{ path: "*", element: <Overview /> },
],
},
],
{ basename: "/admin/ui" },
);
createRoot(document.getElementById("root")!).render(
<StrictMode>
<SettingsProvider>
<QueryClientProvider client={queryClient}>
<FeedbackProvider>
<RouterProvider router={router} />
</FeedbackProvider>
</QueryClientProvider>
</SettingsProvider>
</StrictMode>,
);
web/src/pages/Addons.tsx
+606
View File
@@ -0,0 +1,606 @@
import { useEffect, useState } from "react";
import { useMutation, useQuery, useQueryClient, keepPreviousData } from "@tanstack/react-query";
import { useNavigate, useParams, useSearchParams } from "react-router";
import { ExternalLink, RefreshCw, RotateCcw, Search, ShieldCheck, Star, StarOff, Trash2, Undo2, X } from "lucide-react";
import { api, qs } from "../lib/api";
import { useRefetch } from "../lib/settings";
import type { AddonDetail, AddonRow, FileRow, Overview, Paged } from "../lib/types";
import { bytes, compact, dateTime, int, relative, shortHash } from "../lib/format";
import {
Badge,
Button,
Card,
Checkbox,
CopyButton,
Empty,
ErrorBox,
Input,
Loading,
PageHeader,
Pager,
Segmented,
Select,
Table,
Td,
Th,
useFeedback,
type Tone,
} from "../components/ui";
const STATUSES = [
["all", "All"],
["indexed", "Indexed"],
["queued", "Queued"],
["indexing", "Indexing"],
["priority", "Priority"],
["recheck", "Recheck"],
["blocked", "Blocked"],
["unlisted", "Unlisted"],
["private", "Private"],
["deleted", "Deleted"],
] as const;
const SORTS = [
["subscribers", "Subscribers"],
["size", "Size"],
["name", "Name"],
["created", "Created"],
["updated", "Updated"],
["indexed", "Last indexed"],
["deleted", "Deleted at"],
] as const;
export function addonBadges(a: AddonRow): { tone: Tone; label: string }[] {
const out: { tone: Tone; label: string }[] = [];
if (a.deletedAt) out.push({ tone: "serious", label: "deleted" });
if (a.blocked) out.push({ tone: "critical", label: "blocked" });
if (a.isBeingIndexed) out.push({ tone: "good", label: "indexing" });
else if (a.toBeIndexed) out.push({ tone: "warning", label: "queued" });
if (a.priorityIndexing) out.push({ tone: "accent", label: "priority" });
if (a.unlisted) out.push({ tone: "neutral", label: "unlisted" });
if (a.private) out.push({ tone: "neutral", label: "private" });
return out;
}
const workshopUrl = (id: string) => `https://reforger.armaplatform.com/workshop/${id}`;
export default function Addons() {
const { id } = useParams();
const navigate = useNavigate();
const [sp, setSp] = useSearchParams();
const refetchInterval = useRefetch();
const qc = useQueryClient();
const { toast, confirm } = useFeedback();
const [selected, setSelected] = useState<Set<string>>(new Set());
const status = sp.get("status") ?? "all";
const search = sp.get("q") ?? "";
const type = sp.get("type") ?? "";
const sort = sp.get("sort") ?? (status === "deleted" ? "deleted" : "subscribers");
const order = sp.get("order") ?? "desc";
const offset = Number(sp.get("offset") ?? 0);
const limit = 50;
const [draft, setDraft] = useState(search);
useEffect(() => setDraft(search), [search]);
const set = (patch: Record<string, string | null>) => {
const next = new URLSearchParams(sp);
for (const [k, v] of Object.entries(patch)) {
if (v === null || v === "") next.delete(k);
else next.set(k, v);
}
if (!("offset" in patch)) next.delete("offset");
setSp(next, { replace: true });
setSelected(new Set());
};
const list = useQuery({
queryKey: ["addons", status, search, type, sort, order, offset],
queryFn: () => api<Paged<AddonRow>>("/addons" + qs({ status, q: search, type, sort, order, limit, offset })),
placeholderData: keepPreviousData,
refetchInterval,
});
const overview = useQuery({ queryKey: ["overview"], queryFn: () => api<Overview>("/overview"), staleTime: 60_000 });
const bulk = useMutation({
mutationFn: (body: { action: string; value?: boolean }) =>
api<{ updated: number }>("/bulk/addons", { method: "POST", body: { ids: [...selected], ...body } }),
onSuccess: (r, v) => {
toast(`${v.action}: ${r.updated} addon${r.updated === 1 ? "" : "s"} updated`);
setSelected(new Set());
qc.invalidateQueries({ queryKey: ["addons"] });
qc.invalidateQueries({ queryKey: ["overview"] });
},
onError: (e) => toast((e as Error).message, "critical"),
});
const items = list.data?.items ?? [];
const allSelected = items.length > 0 && items.every((a) => selected.has(a.id));
const query = sp.toString() ? `?${sp.toString()}` : "";
return (
<div className="space-y-4">
<PageHeader title="Addons" description="Search, inspect and manage every addon the crawler knows about" />
<Card bodyClass="p-0">
<div className="flex flex-wrap items-center gap-2 border-b border-line p-3">
<form
className="relative min-w-56 flex-1"
onSubmit={(e) => {
e.preventDefault();
set({ q: draft.trim() });
}}
>
<Search className="pointer-events-none absolute top-1/2 left-2.5 size-3.5 -translate-y-1/2 text-muted" />
<Input value={draft} onChange={(e) => setDraft(e.target.value)} placeholder="Search name, author or exact ID…" className="w-full pl-8" />
</form>
<Select value={status} onChange={(e) => set({ status: e.target.value, sort: null })}>
{STATUSES.map(([k, l]) => (
<option key={k} value={k}>
{l}
</option>
))}
</Select>
<Select value={type} onChange={(e) => set({ type: e.target.value })}>
<option value="">All types</option>
{(overview.data?.data.types ?? []).map((t) => (
<option key={t.type} value={t.type}>
{t.type || "(none)"}
</option>
))}
</Select>
<Select value={sort} onChange={(e) => set({ sort: e.target.value })}>
{SORTS.map(([k, l]) => (
<option key={k} value={k}>
Sort: {l}
</option>
))}
</Select>
<Segmented size="sm" options={[{ key: "desc", label: "Desc" }, { key: "asc", label: "Asc" }]} value={order} onChange={(v) => set({ order: v })} />
</div>
{selected.size > 0 && (
<div className="flex flex-wrap items-center gap-2 border-b border-line bg-surface-2/50 px-3 py-2 text-xs">
<span className="text-ink">{selected.size} selected</span>
<Button size="sm" icon={<Star className="size-3.5" />} onClick={() => bulk.mutate({ action: "priority", value: true })}>Prioritize</Button>
<Button size="sm" icon={<StarOff className="size-3.5" />} onClick={() => bulk.mutate({ action: "priority", value: false })}>Unprioritize</Button>
<Button size="sm" icon={<RefreshCw className="size-3.5" />} onClick={() => bulk.mutate({ action: "reindex", value: false })}>Re-index</Button>
{status === "deleted" ? (
<Button size="sm" icon={<Undo2 className="size-3.5" />} onClick={() => bulk.mutate({ action: "restore" })}>Restore</Button>
) : (
<Button
size="sm"
variant="danger"
icon={<Trash2 className="size-3.5" />}
onClick={async () => {
if (await confirm({ title: `Delete ${selected.size} addons?`, body: "They are soft-deleted with their files and can be restored later.", danger: true, confirmLabel: "Delete" }))
bulk.mutate({ action: "delete" });
}}
>
Delete
</Button>
)}
<Button size="sm" variant="ghost" onClick={() => setSelected(new Set())}>Clear</Button>
</div>
)}
{list.error ? (
<div className="p-4"><ErrorBox error={list.error} /></div>
) : list.isLoading ? (
<div className="px-4"><Loading /></div>
) : items.length === 0 ? (
<Empty>No addons match these filters</Empty>
) : (
<div className="px-4">
<Table>
<thead>
<tr>
<Th className="w-8">
<Checkbox
label="Select all"
checked={allSelected}
onChange={(v) => setSelected(v ? new Set(items.map((a) => a.id)) : new Set())}
/>
</Th>
<Th>Addon</Th>
<Th>State</Th>
<Th className="text-right">Subscribers</Th>
<Th className="text-right">Size</Th>
<Th>Version</Th>
<Th>{status === "deleted" ? "Deleted" : "Last indexed"}</Th>
</tr>
</thead>
<tbody>
{items.map((a) => (
<tr key={a.id} onClick={() => navigate(`/addons/${a.id}${query}`)} className="cursor-pointer hover:bg-surface-2/40">
<Td>
<Checkbox
label={`Select ${a.name}`}
checked={selected.has(a.id)}
onChange={(v) => {
const next = new Set(selected);
if (v) next.add(a.id);
else next.delete(a.id);
setSelected(next);
}}
/>
</Td>
<Td className="max-w-[340px]">
<div className="truncate text-ink" title={a.name}>{a.name || "(unnamed)"}</div>
<div className="truncate text-xs text-muted">
<span className="font-mono">{a.id}</span> · {a.author || "unknown"} · {a.type}
</div>
</Td>
<Td>
<div className="flex flex-wrap gap-1">
{addonBadges(a).map((b) => (
<Badge key={b.label} tone={b.tone}>{b.label}</Badge>
))}
</div>
</Td>
<Td className="tnum text-right">{compact(a.subscriberCount)}</Td>
<Td className="tnum text-right">{bytes(a.currentVersionSize)}</Td>
<Td className="font-mono text-xs">{a.currentVersionNumber}</Td>
<Td className="whitespace-nowrap">{relative(status === "deleted" ? a.deletedAt : a.lastIndexedAt)}</Td>
</tr>
))}
</tbody>
</Table>
</div>
)}
{list.data && (
<div className="border-t border-line px-4 py-2.5">
<Pager total={list.data.total} limit={limit} offset={offset} onChange={(o) => set({ offset: String(o) })} />
</div>
)}
</Card>
{id && <AddonDrawer id={id} onClose={() => navigate(`/addons${query}`)} />}
</div>
);
}
function AddonDrawer({ id, onClose }: { id: string; onClose: () => void }) {
const qc = useQueryClient();
const navigate = useNavigate();
const { toast, confirm } = useFeedback();
const [tab, setTab] = useState<"overview" | "files">("overview");
const detail = useQuery({ queryKey: ["addon", id], queryFn: () => api<AddonDetail>(`/addons/${encodeURIComponent(id)}`) });
useEffect(() => {
const onKey = (e: KeyboardEvent) => e.key === "Escape" && onClose();
window.addEventListener("keydown", onKey);
return () => window.removeEventListener("keydown", onKey);
}, [onClose]);
useEffect(() => setTab("overview"), [id]);
const refresh = () => {
qc.invalidateQueries({ queryKey: ["addon", id] });
qc.invalidateQueries({ queryKey: ["addons"] });
qc.invalidateQueries({ queryKey: ["overview"] });
qc.invalidateQueries({ queryKey: ["queue"] });
};
const act = useMutation({
mutationFn: ({ action, value, method }: { action: string; value?: boolean; method?: "POST" | "DELETE" }) =>
method === "DELETE"
? api(`/addons/${encodeURIComponent(id)}${action ? `/${action}` : ""}${action === "purge" ? `?confirm=${encodeURIComponent(id)}` : ""}`, { method: "DELETE" })
: api(`/addons/${encodeURIComponent(id)}/${action}`, { method: "POST", body: { value: value ?? true } }),
onSuccess: (_r, v) => {
toast(v.action === "purge" ? "Addon purged" : "Saved");
if (v.action === "purge") onClose();
refresh();
},
onError: (e) => toast((e as Error).message, "critical"),
});
const a = detail.data?.addon;
return (
<div className="fixed inset-0 z-40 flex justify-end">
<div className="absolute inset-0 bg-black/40" onClick={onClose} />
<aside className="relative flex h-full w-[min(760px,100vw)] flex-col border-l border-line bg-page shadow-2xl">
<header className="flex items-start justify-between gap-3 border-b border-line bg-surface px-5 py-4">
<div className="min-w-0">
<div className="flex items-center gap-2">
<h2 className="truncate text-lg font-semibold text-ink">{a?.name || id}</h2>
<a href={workshopUrl(id)} target="_blank" rel="noreferrer" className="text-muted hover:text-accent" title="Open on the workshop">
<ExternalLink className="size-4" />
</a>
</div>
<div className="mt-0.5 flex items-center gap-1 font-mono text-xs text-muted">
{id} <CopyButton value={id} />
</div>
{a && (
<div className="mt-2 flex flex-wrap gap-1">
{addonBadges(a).map((b) => <Badge key={b.label} tone={b.tone}>{b.label}</Badge>)}
{a.recheck && <Badge tone="warning">recheck</Badge>}
</div>
)}
</div>
<Button variant="ghost" size="sm" onClick={onClose} icon={<X className="size-4" />} aria-label="Close" />
</header>
{detail.isLoading ? (
<div className="px-5"><Loading /></div>
) : detail.error ? (
<div className="p-5"><ErrorBox error={detail.error} /></div>
) : (
a && (
<>
<div className="flex flex-wrap gap-2 border-b border-line bg-surface px-5 py-3">
{a.deletedAt ? (
<>
<Button size="sm" variant="primary" icon={<Undo2 className="size-3.5" />} onClick={() => act.mutate({ action: "restore" })}>
Restore
</Button>
<Button
size="sm"
variant="danger"
icon={<Trash2 className="size-3.5" />}
onClick={async () => {
if (
await confirm({
title: "Permanently purge this addon?",
body: "Removes the addon row, all its file rows and dependency edges. This cannot be undone.",
danger: true,
confirmLabel: "Purge",
requireText: id,
})
)
act.mutate({ action: "purge", method: "DELETE" });
}}
>
Purge permanently
</Button>
</>
) : (
<>
<Button
size="sm"
icon={a.priorityIndexing ? <StarOff className="size-3.5" /> : <Star className="size-3.5" />}
onClick={() => act.mutate({ action: "priority", value: !a.priorityIndexing })}
>
{a.priorityIndexing ? "Remove priority" : "Prioritize"}
</Button>
<Button size="sm" icon={<RefreshCw className="size-3.5" />} onClick={() => act.mutate({ action: "reindex", value: false })}>
Re-index
</Button>
{a.toBeIndexed && !a.isBeingIndexed && (
<Button size="sm" onClick={() => act.mutate({ action: "to-be-indexed", value: false })}>
Remove from queue
</Button>
)}
{a.isBeingIndexed && (
<Button size="sm" icon={<RotateCcw className="size-3.5" />} onClick={() => act.mutate({ action: "reset-indexing" })}>
Release claim
</Button>
)}
{a.recheck && (
<Button size="sm" onClick={() => act.mutate({ action: "recheck", value: false })}>
Clear recheck
</Button>
)}
<Button
size="sm"
variant="danger"
icon={<Trash2 className="size-3.5" />}
onClick={async () => {
if (await confirm({ title: "Delete this addon?", body: "Soft-deletes the addon and its files; it can be restored later.", danger: true, confirmLabel: "Delete" }))
act.mutate({ action: "", method: "DELETE" });
}}
>
Delete
</Button>
</>
)}
</div>
<div className="border-b border-line bg-surface px-5 py-2">
<Segmented
size="sm"
options={[
{ key: "overview", label: "Overview" },
{ key: "files", label: `Files (${int(detail.data!.fileCount)})` },
]}
value={tab}
onChange={setTab}
/>
</div>
<div className="flex-1 space-y-4 overflow-y-auto p-5">
{tab === "overview" ? (
<>
<Card>
<dl className="grid grid-cols-2 gap-x-6 gap-y-3 text-[13px] sm:grid-cols-3">
{[
["Author", a.author || "—"],
["Type", a.type || "—"],
["Subscribers", int(a.subscriberCount)],
["Version", a.currentVersionNumber || "—"],
["Size", bytes(a.currentVersionSize)],
["Files", `${int(detail.data!.fileCount)}${detail.data!.deletedFileRows ? ` (+${int(detail.data!.deletedFileRows)} old rows)` : ""}`],
["Added", dateTime(a.createdAt)],
["Updated", dateTime(a.updatedAt)],
["Last indexed", dateTime(a.lastIndexedAt)],
...(a.isBeingIndexed ? [["Claimed", relative(a.indexStartTime)]] : []),
...(a.deletedAt ? [["Deleted", dateTime(a.deletedAt)]] : []),
].map(([k, v]) => (
<div key={k}>
<dt className="text-xs text-muted">{k}</dt>
<dd className="mt-0.5 truncate text-ink">{v}</dd>
</div>
))}
</dl>
{detail.data!.summary && <p className="mt-4 border-t border-line pt-3 text-[13px] text-ink-2">{detail.data!.summary}</p>}
</Card>
<Card title="Shares files with" subtitle="Other addons containing identical files (whitelisted hashes excluded)">
{detail.data!.duplicates.length === 0 ? (
<div className="text-sm text-muted">No shared files</div>
) : (
<Table>
<tbody>
{detail.data!.duplicates.map((d) => (
<tr key={d.addonId} className="cursor-pointer hover:bg-surface-2/40" onClick={() => navigate(`/addons/${d.addonId}`)}>
<Td className="text-ink">
{d.name || d.addonId}
<div className="font-mono text-[11px] text-muted">{d.addonId}</div>
</Td>
<Td className="tnum text-right">{int(d.sharedFiles)} {d.sharedFiles === 1 ? "file" : "files"}</Td>
</tr>
))}
</tbody>
</Table>
)}
</Card>
<div className="grid grid-cols-1 gap-4 sm:grid-cols-2">
<Card title={`Depends on (${detail.data!.dependencies.total})`}>
<Related items={detail.data!.dependencies.items ?? []} onOpen={(x) => navigate(`/addons/${x}`)} />
{(detail.data!.dependencies.missing ?? []).length > 0 && (
<div className="mt-3 border-t border-line pt-3">
<div className="mb-1.5 text-xs text-muted">Not in the database</div>
{detail.data!.dependencies.missing!.map((m) => (
<div key={m.id} className="flex items-center justify-between gap-2 py-0.5 text-[13px]">
<span className="truncate text-ink-2">{m.name || m.id}</span>
<span className="font-mono text-[11px] text-muted">{m.id}</span>
</div>
))}
</div>
)}
</Card>
<Card title={`Used by (${detail.data!.dependents.total})`}>
<Related items={detail.data!.dependents.items ?? []} onOpen={(x) => navigate(`/addons/${x}`)} />
</Card>
</div>
</>
) : (
<Files id={id} />
)}
</div>
</>
)
)}
</aside>
</div>
);
}
function Related({ items, onOpen }: { items: { id: string; name: string; subscriberCount: number }[]; onOpen: (id: string) => void }) {
if (items.length === 0) return <div className="text-sm text-muted">None</div>;
return (
<ul className="space-y-1">
{items.map((x) => (
<li key={x.id}>
<button onClick={() => onOpen(x.id)} className="flex w-full items-center justify-between gap-2 rounded px-1 py-0.5 text-left text-[13px] hover:bg-surface-2">
<span className="truncate text-ink-2">{x.name || x.id}</span>
<span className="tnum shrink-0 text-xs text-muted">{compact(x.subscriberCount)}</span>
</button>
</li>
))}
</ul>
);
}
function Files({ id }: { id: string }) {
const [q, setQ] = useState("");
const [draft, setDraft] = useState("");
const [offset, setOffset] = useState(0);
const qc = useQueryClient();
const { toast, confirm } = useFeedback();
const limit = 100;
const files = useQuery({
queryKey: ["addon-files", id, q, offset],
queryFn: () => api<Paged<FileRow>>(`/addons/${encodeURIComponent(id)}/files` + qs({ q, offset, limit })),
placeholderData: keepPreviousData,
});
const whitelist = useMutation({
mutationFn: (f: FileRow) => api<{ added: number }>("/whitelist", { method: "POST", body: { hashes: [f.hash], note: f.path } }),
onSuccess: (r) => {
toast(r.added ? "Hash whitelisted" : "Already whitelisted");
qc.invalidateQueries({ queryKey: ["addon-files", id] });
qc.invalidateQueries({ queryKey: ["addon", id] });
},
onError: (e) => toast((e as Error).message, "critical"),
});
return (
<Card bodyClass="p-0">
<form
className="border-b border-line p-3"
onSubmit={(e) => {
e.preventDefault();
setOffset(0);
setQ(draft.trim());
}}
>
<Input value={draft} onChange={(e) => setDraft(e.target.value)} placeholder="Filter by path or exact hash…" className="w-full" />
</form>
{files.isLoading ? (
<div className="px-4"><Loading /></div>
) : files.error ? (
<div className="p-4"><ErrorBox error={files.error} /></div>
) : (files.data?.items.length ?? 0) === 0 ? (
<Empty>No files</Empty>
) : (
<div className="px-4">
<Table>
<thead>
<tr>
<Th>Path</Th>
<Th>Hash</Th>
<Th className="text-right">Also in</Th>
<Th />
</tr>
</thead>
<tbody>
{files.data!.items.map((f) => (
<tr key={f.id}>
<Td className="max-w-[300px] truncate font-mono text-xs text-ink" title={f.path}>{f.path}</Td>
<Td className="whitespace-nowrap font-mono text-xs">
{shortHash(f.hash, 8)} <CopyButton value={f.hash} />
</Td>
<Td className="tnum text-right">{f.otherAddons ? `${int(f.otherAddons)} addons` : "—"}</Td>
<Td className="text-right">
{f.whitelisted ? (
<Badge tone="good">whitelisted</Badge>
) : (
<Button
size="sm"
variant="ghost"
icon={<ShieldCheck className="size-3.5" />}
onClick={async () => {
if (
await confirm({
title: "Whitelist this hash?",
body: (
<>
Files with this content will be ignored by duplicate detection for <b>every</b> addon.
<div className="mt-2 font-mono text-xs break-all text-muted">{f.path}</div>
</>
),
confirmLabel: "Whitelist",
})
)
whitelist.mutate(f);
}}
>
Whitelist
</Button>
)}
</Td>
</tr>
))}
</tbody>
</Table>
</div>
)}
{files.data && files.data.total > limit && (
<div className="border-t border-line px-4 py-2.5">
<Pager total={files.data.total} limit={limit} offset={offset} onChange={setOffset} />
</div>
)}
</Card>
);
}
web/src/pages/Events.tsx
+117
View File
@@ -0,0 +1,117 @@
import { useState } from "react";
import { useInfiniteQuery, useQuery } from "@tanstack/react-query";
import { api, qs } from "../lib/api";
import { useRefetch } from "../lib/settings";
import type { ServiceEvent, ServiceInstance } from "../lib/types";
import { dateTime, relative } from "../lib/format";
import { Button, Card, Checkbox, Empty, ErrorBox, Input, LevelBadge, Loading, PageHeader, Select, Table, Td, Th } from "../components/ui";
export default function Events() {
const refetchInterval = useRefetch();
const [service, setService] = useState("");
const [level, setLevel] = useState("");
const [audit, setAudit] = useState(false);
const [search, setSearch] = useState("");
const [draft, setDraft] = useState("");
const services = useQuery({ queryKey: ["services"], queryFn: () => api<{ items: ServiceInstance[] }>("/services") });
const events = useInfiniteQuery({
queryKey: ["events", "log", service, level, audit, search],
initialPageParam: 0,
refetchInterval,
queryFn: ({ pageParam }) =>
api<{ items: ServiceEvent[]; next: number }>(
"/services/events" + qs({ service, level, audit: audit ? "1" : undefined, q: search, before: pageParam || undefined, limit: 100 }),
),
getNextPageParam: (last) => last.next || undefined,
});
const rows = events.data?.pages.flatMap((p) => p.items) ?? [];
const names = [...new Set((services.data?.items ?? []).map((s) => s.service))];
return (
<div className="space-y-4">
<PageHeader title="Events" description="Logs reported by services, status changes and the admin audit trail" />
<Card bodyClass="p-0">
<div className="flex flex-wrap items-center gap-2 border-b border-line p-3">
<form
className="min-w-56 flex-1"
onSubmit={(e) => {
e.preventDefault();
setSearch(draft.trim());
}}
>
<Input value={draft} onChange={(e) => setDraft(e.target.value)} placeholder="Search messages…" className="w-full" />
</form>
<Select value={service} onChange={(e) => setService(e.target.value)}>
<option value="">All services</option>
{names.map((n) => <option key={n} value={n}>{n}</option>)}
</Select>
<Select value={level} onChange={(e) => setLevel(e.target.value)}>
<option value="">All levels</option>
<option value="error">Errors</option>
<option value="warn,error">Warnings & errors</option>
<option value="info">Info</option>
<option value="debug">Debug</option>
</Select>
<label className="flex items-center gap-1.5 text-xs text-ink-2">
<Checkbox checked={audit} onChange={setAudit} label="Admin audit only" /> Admin audit only
</label>
</div>
{events.isLoading ? (
<div className="px-4"><Loading /></div>
) : events.error ? (
<div className="p-4"><ErrorBox error={events.error} /></div>
) : rows.length === 0 ? (
<Empty>No events</Empty>
) : (
<div className="px-4">
<Table>
<thead>
<tr>
<Th>Time</Th>
<Th>Level</Th>
<Th>Service</Th>
<Th>Message</Th>
</tr>
</thead>
<tbody>
{rows.map((e) => (
<tr key={e.id} className="align-top">
<Td className="whitespace-nowrap text-xs" title={relative(e.ts)}>{dateTime(e.ts)}</Td>
<Td><LevelBadge level={e.level} /></Td>
<Td className="whitespace-nowrap text-xs">
{e.service}
<div className="text-muted">{e.instance}</div>
</Td>
<Td className="text-ink">
{e.message}
{e.fieldsData && Object.keys(e.fieldsData).length > 0 && (
<div className="mt-0.5 flex flex-wrap gap-x-3 font-mono text-[11px] text-muted">
{Object.entries(e.fieldsData)
.filter(([k]) => k !== "kind")
.map(([k, v]) => (
<span key={k}>
{k}={typeof v === "object" ? JSON.stringify(v) : String(v)}
</span>
))}
</div>
)}
</Td>
</tr>
))}
</tbody>
</Table>
</div>
)}
{events.hasNextPage && (
<div className="border-t border-line p-3 text-center">
<Button size="sm" loading={events.isFetchingNextPage} onClick={() => events.fetchNextPage()}>
Load older events
</Button>
</div>
)}
</Card>
</div>
);
}
web/src/pages/Explorer.tsx
+232
View File
@@ -0,0 +1,232 @@
import { useMemo } from "react";
import { useQuery } from "@tanstack/react-query";
import { useSearchParams } from "react-router";
import { X } from "lucide-react";
import { api } from "../lib/api";
import { stepFor, useMetric } from "../lib/queries";
import { rangeSeconds, useSettings } from "../lib/settings";
import type { CatalogEntry } from "../lib/types";
import { compact, num, relative } from "../lib/format";
import { Card, Empty, ErrorBox, Loading, PageHeader, Select, Table, Td, Th } from "../components/ui";
import { fromQuery, TimeChart } from "../components/charts/TimeChart";
const AGG_HELP: Record<string, string> = {
sum: "Total of values (counters: increase per interval)",
count: "Number of samples / observations",
rate: "Per second (counters: increase, others: observations)",
avg: "Average value",
min: "Minimum",
max: "Maximum",
last: "Latest value (summed over series)",
p50: "Median (histograms)",
p90: "90th percentile (histograms)",
p95: "95th percentile (histograms)",
p99: "99th percentile (histograms)",
};
const STEPS = [
{ key: 0, label: "Auto" },
{ key: 60, label: "1 min" },
{ key: 300, label: "5 min" },
{ key: 900, label: "15 min" },
{ key: 3600, label: "1 hour" },
{ key: 86400, label: "1 day" },
];
export default function Explorer() {
const [sp, setSp] = useSearchParams();
const { range } = useSettings();
const catalog = useQuery({ queryKey: ["catalog", "all"], queryFn: () => api<{ items: CatalogEntry[]; aggregations: string[] }>("/metrics/catalog") });
const service = sp.get("service") ?? "main";
const name = sp.get("name") ?? "";
const groupBy = (sp.get("groupBy") ?? "").split(",").filter(Boolean);
const match: Record<string, string> = useMemo(() => {
try {
return JSON.parse(sp.get("match") ?? "{}");
} catch {
return {};
}
}, [sp]);
const stepParam = Number(sp.get("step") ?? 0);
const update = (patch: Record<string, string | null>) => {
const next = new URLSearchParams(sp);
for (const [k, v] of Object.entries(patch)) {
if (v === null || v === "") next.delete(k);
else next.set(k, v);
}
setSp(next, { replace: true });
};
const services = useMemo(() => [...new Set((catalog.data?.items ?? []).map((c) => c.service))], [catalog.data]);
const metrics = (catalog.data?.items ?? []).filter((c) => c.service === service);
const entry = metrics.find((m) => m.name === name) ?? null;
const defaultAgg = entry ? (entry.kind === "counter" ? "sum" : entry.kind === "gauge" ? "last" : entry.kind === "histogram" ? "p95" : "avg") : "";
const agg = sp.get("agg") ?? defaultAgg;
const labelKeys = entry ? [...Object.keys(entry.labels), ...(entry.instances.length > 1 ? ["instance"] : [])] : [];
const step = stepParam || stepFor(rangeSeconds(range));
const q = useMetric({ service, name, agg, groupBy, match, step }, !!entry);
// colour only the 8 largest series (fixed slots, never cycled); the table lists all of them
const MAX_CHART_SERIES = 8;
const full = fromQuery(q.data, (l) => (Object.keys(l).length ? Object.entries(l).map(([k, v]) => `${k}=${v}`).join(", ") : name));
const ranked = full.series
.map((s) => ({ s, w: s.data.reduce<number>((a, v) => a + (v ?? 0), 0) }))
.sort((a, b) => b.w - a.w)
.slice(0, MAX_CHART_SERIES)
.map((x, i) => ({ ...x.s, slot: i + 1 }));
const data = { x: full.x, series: ranked };
const hidden = full.series.length - ranked.length;
const fmt = agg === "rate" ? (v: number) => num(v, 3) : compact;
const summary = (q.data?.series ?? []).map((s) => {
const vals = s.points.map((p) => p[1]).filter((v): v is number => v !== null);
return {
labels: s.labels,
last: vals.at(-1) ?? null,
min: vals.length ? Math.min(...vals) : null,
max: vals.length ? Math.max(...vals) : null,
avg: vals.length ? vals.reduce((a, b) => a + b, 0) / vals.length : null,
};
});
return (
<div className="space-y-4">
<PageHeader title="Metrics explorer" description="Query any metric from the API or a reporting service. The URL keeps the query, so views can be bookmarked." />
<Card>
{catalog.isLoading ? (
<Loading />
) : catalog.error ? (
<ErrorBox error={catalog.error} />
) : (
<div className="flex flex-wrap items-end gap-3">
<Field label="Service">
<Select value={service} onChange={(e) => update({ service: e.target.value, name: null, agg: null, groupBy: null, match: null })}>
{services.map((s) => <option key={s} value={s}>{s}</option>)}
</Select>
</Field>
<Field label="Metric">
<Select value={name} onChange={(e) => update({ name: e.target.value, agg: null, groupBy: null, match: null })} className="min-w-64">
<option value="">Choose a metric…</option>
{metrics.map((m) => <option key={m.name} value={m.name}>{m.name} ({m.kind})</option>)}
</Select>
</Field>
{entry && (
<>
<Field label="Aggregation">
<Select value={agg} onChange={(e) => update({ agg: e.target.value })} title={AGG_HELP[agg]}>
{(catalog.data?.aggregations ?? []).map((a) => <option key={a} value={a}>{a}</option>)}
</Select>
</Field>
<Field label="Interval">
<Select value={stepParam} onChange={(e) => update({ step: e.target.value === "0" ? null : e.target.value })}>
{STEPS.map((s) => <option key={s.key} value={s.key}>{s.label}</option>)}
</Select>
</Field>
{labelKeys.length > 0 && (
<Field label="Split by">
<div className="flex flex-wrap gap-1">
{labelKeys.map((k) => {
const on = groupBy.includes(k);
return (
<button
key={k}
onClick={() => update({ groupBy: (on ? groupBy.filter((g) => g !== k) : [...groupBy, k]).join(",") })}
className={`h-8.5 rounded-lg border px-2.5 text-xs ${on ? "border-accent bg-accent/15 text-ink" : "border-line text-ink-2 hover:text-ink"}`}
>
{k}
</button>
);
})}
</div>
</Field>
)}
</>
)}
</div>
)}
{entry && Object.keys(entry.labels).length > 0 && (
<div className="mt-3 flex flex-wrap items-center gap-2 border-t border-line pt-3">
<span className="text-xs text-muted">Filter</span>
{Object.entries(entry.labels).map(([k, vals]) => (
<Select
key={k}
value={match[k] ?? ""}
onChange={(e) => {
const next = { ...match };
if (e.target.value) next[k] = e.target.value;
else delete next[k];
update({ match: Object.keys(next).length ? JSON.stringify(next) : null });
}}
className="h-7.5 text-xs"
>
<option value="">{k}: any</option>
{vals.map((v) => <option key={v} value={v}>{k}={v || "(empty)"}</option>)}
</Select>
))}
{Object.keys(match).length > 0 && (
<button className="inline-flex items-center gap-1 text-xs text-muted hover:text-ink" onClick={() => update({ match: null })}>
<X className="size-3" /> clear
</button>
)}
</div>
)}
</Card>
{!entry ? (
<Card><Empty>Pick a service and a metric to plot it</Empty></Card>
) : (
<>
<Card
title={<span className="font-mono">{name}</span>}
subtitle={`${entry.help || entry.kind} · ${agg}: ${AGG_HELP[agg] ?? ""}${q.data ? ` · ${q.data.resolution === 3600 ? "hourly" : "minute"} data, ${q.data.step / 60} min interval` : ""}`}
>
{q.error ? <ErrorBox error={q.error} /> : <TimeChart data={data} height={320} format={fmt} />}
{hidden > 0 && (
<p className="mt-2 text-xs text-muted">
Charting the {MAX_CHART_SERIES} largest of {full.series.length} series — use a filter or fewer split labels to see the rest.
</p>
)}
{q.data?.truncated && <p className="mt-2 text-xs text-muted">The query returned the first 50 series only.</p>}
</Card>
<Card title="Series" subtitle={`${entry.series} series · ${entry.instances.length} instance(s) · last sample ${relative(entry.lastSeen)}`}>
<Table>
<thead>
<tr>
<Th>Labels</Th>
<Th className="text-right">Last</Th>
<Th className="text-right">Min</Th>
<Th className="text-right">Avg</Th>
<Th className="text-right">Max</Th>
</tr>
</thead>
<tbody>
{summary.map((s, i) => (
<tr key={i}>
<Td className="font-mono text-xs text-ink">{Object.entries(s.labels).map(([k, v]) => `${k}=${v}`).join(", ") || name}</Td>
<Td className="tnum text-right">{s.last === null ? "—" : fmt(s.last)}</Td>
<Td className="tnum text-right">{s.min === null ? "—" : fmt(s.min)}</Td>
<Td className="tnum text-right">{s.avg === null ? "—" : fmt(s.avg)}</Td>
<Td className="tnum text-right">{s.max === null ? "—" : fmt(s.max)}</Td>
</tr>
))}
</tbody>
</Table>
</Card>
</>
)}
</div>
);
}
function Field({ label, children }: { label: string; children: React.ReactNode }) {
return (
<label className="flex flex-col gap-1">
<span className="text-xs text-muted">{label}</span>
{children}
</label>
);
}
web/src/pages/Login.tsx
+64
View File
@@ -0,0 +1,64 @@
import { useState, type FormEvent } from "react";
import { useLocation, useNavigate } from "react-router";
import { useQueryClient } from "@tanstack/react-query";
import { KeyRound } from "lucide-react";
import { api } from "../lib/api";
import { Button, ErrorBox, Input } from "../components/ui";
export default function Login() {
const [secret, setSecret] = useState("");
const [error, setError] = useState<unknown>(null);
const [busy, setBusy] = useState(false);
const navigate = useNavigate();
const location = useLocation();
const qc = useQueryClient();
const submit = async (e: FormEvent) => {
e.preventDefault();
setBusy(true);
setError(null);
try {
await api("/auth/login", { method: "POST", body: { secret } });
qc.clear();
const from = (location.state as { from?: string } | null)?.from;
navigate(from && from !== "/login" ? from : "/", { replace: true });
} catch (err) {
setError(err);
} finally {
setBusy(false);
}
};
return (
<div className="flex min-h-screen items-center justify-center px-4">
<form onSubmit={submit} className="w-full max-w-sm rounded-2xl border border-line bg-surface p-6 shadow-sm">
<div className="mb-6 flex items-center gap-3">
<div className="flex size-10 items-center justify-center rounded-xl bg-accent/15">
<KeyRound className="size-5 text-accent" />
</div>
<div>
<h1 className="font-semibold text-ink">Reforger Crawler admin</h1>
<p className="text-xs text-muted">Sign in with the admin secret</p>
</div>
</div>
<label className="text-xs text-ink-2" htmlFor="secret">
Admin secret
</label>
<Input
id="secret"
type="password"
autoFocus
autoComplete="current-password"
value={secret}
onChange={(e) => setSecret(e.target.value)}
className="mt-1 w-full"
/>
{error !== null && <div className="mt-3"><ErrorBox error={error} /></div>}
<Button type="submit" variant="primary" className="mt-4 w-full" loading={busy} disabled={!secret}>
Sign in
</Button>
<p className="mt-4 text-center text-[11px] text-muted">The session is kept in an HttpOnly cookie for 7 days.</p>
</form>
</div>
);
}
web/src/pages/Overview.tsx
+209
View File
@@ -0,0 +1,209 @@
import { useQuery, keepPreviousData } from "@tanstack/react-query";
import { useNavigate } from "react-router";
import { api } from "../lib/api";
import { useMetric, useRangeStep } from "../lib/queries";
import { useRefetch } from "../lib/settings";
import type { MiniAddon, Overview as OverviewT, ServiceEvent, TopEntry } from "../lib/types";
import { bytes, compact, int, ms, num, pct, relative } from "../lib/format";
import { Badge, Card, ErrorBox, LevelBadge, Loading, Meter, PageHeader, StatTile, Table, Td } from "../components/ui";
import { fromQuery, TimeChart } from "../components/charts/TimeChart";
import { BarList } from "../components/charts/BarList";
const GROUP_SLOT: Record<string, number> = { v1: 1, back: 2, admin: 3, metrics: 4, other: 5 };
const GROUP_LABEL: Record<string, string> = { v1: "Website API", back: "Workers", admin: "Admin", metrics: "Prometheus", other: "Other" };
function values(q: { series: { points: [number, number | null][] }[] } | undefined): (number | null)[] {
return q?.series[0]?.points.map((p) => p[1]) ?? [];
}
export default function Overview() {
const navigate = useNavigate();
const refetchInterval = useRefetch();
const { step } = useRangeStep();
const ov = useQuery({ queryKey: ["overview"], queryFn: () => api<OverviewT>("/overview"), refetchInterval, placeholderData: keepPreviousData });
const topAddons = useQuery({ queryKey: ["top", "addon"], queryFn: () => api<{ items: TopEntry[] }>("/top?kind=addon&days=1&limit=8"), refetchInterval });
const topSearch = useQuery({ queryKey: ["top", "search"], queryFn: () => api<{ items: TopEntry[] }>("/top?kind=search&days=1&limit=8"), refetchInterval });
const events = useQuery({ queryKey: ["events", "recent"], queryFn: () => api<{ items: ServiceEvent[] }>("/services/events?limit=6"), refetchInterval });
const requests = useMetric({ name: "http_request_duration_ms", agg: "rate", groupBy: ["group"], step });
const p95 = useMetric({ name: "http_request_duration_ms", agg: "p95", match: { group: "v1" }, step });
const queueDepth = useMetric({ name: "queue_depth", agg: "last", groupBy: ["kind"], step });
const indexed = useMetric({ name: "indexer_submissions_total", agg: "sum", groupBy: ["result"], step });
const addonsTotal = useMetric({ name: "addons_total", agg: "last", step });
const visitors = useMetric({ name: "visitors_unique", agg: "max", match: { kind: "human" }, step: 86400, rangeSec: 14 * 86400 });
if (ov.isLoading) return <Loading />;
if (ov.error) return <ErrorBox error={ov.error} />;
const d = ov.data!.data;
const a = d.addons;
const heavy = d.heavy;
const openAddon = (id: string) => navigate(`/addons/${id}`);
return (
<div className="space-y-4">
<PageHeader title="Overview" description={<>Snapshot of the crawler, its workers and the public website · updated {relative(ov.data!.asOf)}</>} />
<div className="grid grid-cols-1 gap-3 sm:grid-cols-2 xl:grid-cols-4">
<StatTile hero label="Addons tracked" value={int(a.total)} sub={`+${int(a.added24h)} today · +${int(a.added7d)} this week`} trend={values(addonsTotal.data)} onClick={() => navigate("/addons")} />
<div className="rounded-xl border border-line bg-surface p-4">
<div className="text-xs text-muted">Indexed</div>
<div className="mt-2 text-2xl font-semibold tracking-tight text-ink">{pct(a.indexed, a.total)}</div>
<div className="mt-1.5 text-xs text-ink-2">
{int(a.indexed)} of {int(a.total)} · {int(a.indexed24h)} in last 24h
</div>
<div className="mt-3">
<Meter value={a.indexed} max={a.total} />
</div>
</div>
<StatTile
label="Website visitors today"
value={int(d.traffic.visitorsToday)}
sub={`${int(d.traffic.botsToday)} bots / scripts · approx.`}
trend={values(visitors.data)}
onClick={() => navigate("/traffic")}
/>
<StatTile
label="Website latency p95 (24h)"
value={ms(d.traffic.websiteP95Ms)}
sub={`p50 ${ms(d.traffic.websiteP50Ms)} · ${compact(d.traffic.websiteRequests24h ?? 0)} requests`}
trend={values(p95.data)}
onClick={() => navigate("/traffic")}
/>
<StatTile label="Indexing queue" value={int(a.queued)} sub={`${int(a.priority)} priority · ${int(a.indexing)} in progress`} onClick={() => navigate("/queue")} />
<StatTile
label="Indexed files"
value={compact(heavy?.filesLive)}
sub={heavy ? `${compact(heavy.distinctHashes)} distinct hashes${heavy.filesApprox ? " · approx." : ""}` : "computing…"}
/>
<StatTile label="Server errors (24h)" value={int(d.traffic.errors5xx24h)} sub={`${compact(d.traffic.requests24h ?? 0)} requests in total`} onClick={() => navigate("/traffic")} />
<StatTile label="Services online" value={`${d.services.online} / ${d.services.total}`} sub="indexers, scrapers, API" onClick={() => navigate("/services")} />
</div>
<div className="grid grid-cols-1 gap-4 xl:grid-cols-3">
<Card title="Requests per minute" subtitle="All HTTP traffic by caller" className="xl:col-span-2">
<TimeChart
data={fromQuery(requests.data, (l) => GROUP_LABEL[l.group] ?? l.group, 60, (l) => GROUP_SLOT[l.group])}
format={(v) => num(v, 1)}
/>
</Card>
<Card title="Queue depth" subtitle="Addons waiting to be indexed">
<TimeChart
data={fromQuery(queueDepth.data, (l) => (l.kind === "priority" ? "Priority" : "All queued"), 1, (l) => (l.kind === "priority" ? 2 : 1))}
format={compact}
/>
</Card>
</div>
<div className="grid grid-cols-1 gap-4 xl:grid-cols-3">
<Card title="Indexing results" subtitle="Submissions from indexers per interval" className="xl:col-span-2">
<TimeChart data={fromQuery(indexed.data, (l) => l.result)} format={compact} />
</Card>
<Card title="Last 24 hours">
<dl className="space-y-2 text-[13px]">
{[
["New addons from scrapers", d.activity.scraperCreated24h],
["Updated addons", d.activity.scraperUpdated24h],
["Failed creates (soft-deleted IDs)", d.activity.scraperCreateFailed24h],
["Deleted by scrapers", d.activity.deleted24h],
["Indexed", d.activity.indexed24h],
["Indexing claims", d.activity.claims24h],
["Timed-out claims reset", d.activity.timeoutsReset24h],
["Dependency reports", d.activity.dependencyReports24h],
].map(([k, v]) => (
<div key={k as string} className="flex justify-between gap-3">
<dt className="text-ink-2">{k}</dt>
<dd className="tnum text-ink">{int(v as number)}</dd>
</div>
))}
</dl>
</Card>
</div>
<div className="grid grid-cols-1 gap-4 md:grid-cols-2 xl:grid-cols-3">
<Card title="Most viewed addons today">
<BarList
items={(topAddons.data?.items ?? []).map((t) => ({ key: t.item, label: t.name || t.item, value: t.count, onClick: () => openAddon(t.item) }))}
empty="No lookups yet today"
/>
</Card>
<Card title="Top searches today">
<BarList items={(topSearch.data?.items ?? []).map((t) => ({ key: t.item, label: t.item, value: t.count }))} empty="No searches yet today" />
</Card>
<Card title="Addon states">
<div className="space-y-2.5 text-[13px]">
{[
{ label: "Blocked on workshop", value: a.blocked, tone: "critical" as const, status: "blocked" },
{ label: "Soft-deleted", value: a.softDeleted, tone: "serious" as const, status: "deleted" },
{ label: "Unlisted", value: a.unlisted, tone: "neutral" as const, status: "unlisted" },
{ label: "Private", value: a.private, tone: "neutral" as const, status: "private" },
{ label: "Priority indexing", value: a.priority, tone: "accent" as const, status: "priority" },
{ label: "Marked for recheck", value: a.recheck, tone: "warning" as const, status: "recheck" },
].map((s) => (
<button key={s.label} onClick={() => navigate(`/addons?status=${s.status}`)} className="flex w-full items-center justify-between gap-3 text-left hover:text-ink">
<Badge tone={s.tone}>{s.label}</Badge>
<span className="tnum text-ink">{int(s.value)}</span>
</button>
))}
<div className="border-t border-line pt-2.5 text-xs text-ink-2">
{int(d.whitelistCount)} whitelisted hashes · {int(d.dependencies.edges)} dependency edges · {bytes(a.totalSizeBytes)} total size
</div>
</div>
</Card>
</div>
<div className="grid grid-cols-1 gap-4 md:grid-cols-2 xl:grid-cols-3">
<AddonList title="Most subscribed" items={d.topSubscribed} metric={(x) => compact(x.subscriberCount)} onOpen={openAddon} />
<AddonList title="Largest" items={d.largest} metric={(x) => bytes(x.currentVersionSize)} onOpen={openAddon} />
<AddonList title="Recently indexed" items={d.recentlyIndexed} metric={(x) => relative(x.lastIndexedAt)} onOpen={openAddon} />
</div>
<div className="grid grid-cols-1 gap-4 xl:grid-cols-3">
<Card title="Addon types">
<BarList items={d.types.map((t) => ({ key: t.type || "(none)", label: t.type || "(none)", value: t.count }))} />
</Card>
<Card title="Recent events" className="xl:col-span-2" actions={<button onClick={() => navigate("/events")} className="text-xs text-accent hover:underline">All events</button>}>
{(events.data?.items ?? []).length === 0 ? (
<div className="py-6 text-center text-sm text-muted">No events yet</div>
) : (
<Table>
<tbody>
{events.data!.items.map((e) => (
<tr key={e.id}>
<Td className="w-24"><LevelBadge level={e.level} /></Td>
<Td className="text-ink">{e.message}</Td>
<Td className="whitespace-nowrap">{e.service}</Td>
<Td className="whitespace-nowrap text-right text-muted">{relative(e.ts)}</Td>
</tr>
))}
</tbody>
</Table>
)}
</Card>
</div>
</div>
);
}
function AddonList({ title, items, metric, onOpen }: { title: string; items: MiniAddon[]; metric: (a: MiniAddon) => string; onOpen: (id: string) => void }) {
return (
<Card title={title}>
{items.length === 0 ? (
<div className="py-6 text-center text-sm text-muted">Nothing yet</div>
) : (
<ul className="-mx-2">
{items.map((x) => (
<li key={x.id}>
<button onClick={() => onOpen(x.id)} className="flex w-full items-center justify-between gap-3 rounded-lg px-2 py-1.5 text-left hover:bg-surface-2/60">
<span className="min-w-0">
<span className="block truncate text-[13px] text-ink" title={x.name}>{x.name || x.id}</span>
<span className="block truncate text-xs text-muted">{x.author || "unknown author"}</span>
</span>
<span className="tnum shrink-0 text-[13px] text-ink-2">{metric(x)}</span>
</button>
</li>
))}
</ul>
)}
</Card>
);
}
web/src/pages/Queue.tsx
+208
View File
@@ -0,0 +1,208 @@
import { useMutation, useQuery, useQueryClient, keepPreviousData } from "@tanstack/react-query";
import { useNavigate } from "react-router";
import { RotateCcw, Star, StarOff } from "lucide-react";
import { api } from "../lib/api";
import { useMetric, useRangeStep } from "../lib/queries";
import { useRefetch } from "../lib/settings";
import type { QueueData, QueueRow } from "../lib/types";
import { bytes, compact, duration, int, num, relative, seconds } from "../lib/format";
import { Badge, Button, Card, ErrorBox, Loading, PageHeader, StatTile, Table, Td, Th, useFeedback } from "../components/ui";
import { fromQuery, TimeChart } from "../components/charts/TimeChart";
export default function Queue() {
const refetchInterval = useRefetch();
const { step } = useRangeStep();
const qc = useQueryClient();
const { toast, confirm } = useFeedback();
const navigate = useNavigate();
const q = useQuery({ queryKey: ["queue"], queryFn: () => api<QueueData>("/queue"), refetchInterval: refetchInterval || 15000, placeholderData: keepPreviousData });
const depth = useMetric({ name: "queue_depth", agg: "last", groupBy: ["kind"], step });
const claims = useMetric({ name: "indexer_claims_total", agg: "sum", groupBy: ["queue"], step });
const jobDuration = useMetric({ name: "indexer_job_duration_seconds", agg: "p50", step });
const resetStuck = useMutation({
mutationFn: () => api<{ reset: number }>("/queue/reset-stuck", { method: "POST", body: {} }),
onSuccess: (r) => {
toast(`Released ${r.reset} timed-out claim${r.reset === 1 ? "" : "s"}`);
qc.invalidateQueries({ queryKey: ["queue"] });
},
onError: (e) => toast(String((e as Error).message), "critical"),
});
const action = useMutation({
mutationFn: ({ id, act, value }: { id: string; act: string; value?: boolean }) =>
api(`/addons/${encodeURIComponent(id)}/${act}`, { method: "POST", body: { value: value ?? true } }),
onSuccess: () => {
toast("Updated");
qc.invalidateQueries({ queryKey: ["queue"] });
},
onError: (e) => toast(String((e as Error).message), "critical"),
});
if (q.isLoading) return <Loading />;
if (q.error) return <ErrorBox error={q.error} />;
const d = q.data!;
const stuck = d.indexing.filter((r) => r.stuck).length;
return (
<div className="space-y-4">
<PageHeader
title="Indexing queue"
description={<>Jobs are claimed priority-first, then by oldest update · claims time out after {d.timeoutMinutes} min · updated {relative(d.asOf)}</>}
actions={
<Button
icon={<RotateCcw className="size-3.5" />}
loading={resetStuck.isPending}
onClick={async () => {
if (await confirm({ title: "Release timed-out claims?", body: `Addons claimed more than ${d.timeoutMinutes} minutes ago go back to the queue.` }))
resetStuck.mutate();
}}
>
Release stuck claims
</Button>
}
/>
<div className="grid grid-cols-1 gap-3 sm:grid-cols-2 xl:grid-cols-4">
<StatTile label="Waiting" value={int(d.counts.queued)} sub={`${int(d.counts.priority)} priority · ${int(d.blockedQueued)} blocked (never claimed)`} />
<StatTile label="In progress" value={int(d.counts.indexing)} sub={stuck ? `${stuck} past the timeout` : "none stuck"} />
<StatTile
label="Throughput"
value={`${num(d.throughput.perHour6h, 1)}/h`}
sub={`${num(d.throughput.perHour1h, 0)} last hour · ${int(d.throughput.indexed24h)} in 24h`}
/>
<StatTile
label="Queue drains in"
value={d.throughput.etaHours === null ? "—" : duration(d.throughput.etaHours * 3600)}
sub={d.throughput.duration24h ? `job p50 ${duration(d.throughput.duration24h.p50)} · p95 ${duration(d.throughput.duration24h.p95)}` : "at the 6h average rate"}
/>
</div>
<div className="grid grid-cols-1 gap-4 xl:grid-cols-3">
<Card title="Queue depth">
<TimeChart data={fromQuery(depth.data, (l) => (l.kind === "priority" ? "Priority" : "All queued"), 1, (l) => (l.kind === "priority" ? 2 : 1))} />
</Card>
<Card title="Claims per interval" subtitle="What indexers got when asking for work">
<TimeChart data={fromQuery(claims.data, (l) => l.queue)} />
</Card>
<Card title="Job duration p50" subtitle="Claim to submitted result">
<TimeChart data={fromQuery(jobDuration.data, () => "p50")} format={seconds} />
</Card>
</div>
<QueueTable
title={`In progress (${d.indexing.length})`}
rows={d.indexing}
empty="No addon is being indexed right now"
extra={(r) => (
<>
<Td className="tnum whitespace-nowrap">{duration(r.elapsedSec)}</Td>
<Td>{r.stuck ? <Badge tone="critical">past timeout</Badge> : <Badge tone="good">running</Badge>}</Td>
</>
)}
extraHead={
<>
<Th>Elapsed</Th>
<Th>State</Th>
</>
}
actions={(r) => (
<Button size="sm" variant="ghost" onClick={() => action.mutate({ id: r.id, act: "reset-indexing" })}>
Release
</Button>
)}
onOpen={(id) => navigate(`/addons/${id}`)}
/>
<QueueTable
title={`Priority (${d.counts.priority})`}
rows={d.priority}
empty="No priority jobs. Mark addons as priority from the addon list."
actions={(r) => (
<Button size="sm" variant="ghost" icon={<StarOff className="size-3.5" />} onClick={() => action.mutate({ id: r.id, act: "priority", value: false })}>
Unprioritize
</Button>
)}
onOpen={(id) => navigate(`/addons/${id}`)}
/>
<QueueTable
title="Next up"
subtitle="The next jobs in claim order (an indexer may skip addons above its size limit)"
rows={d.next}
empty="The queue is empty"
actions={(r) =>
r.priorityIndexing ? null : (
<Button size="sm" variant="ghost" icon={<Star className="size-3.5" />} onClick={() => action.mutate({ id: r.id, act: "priority", value: true })}>
Prioritize
</Button>
)
}
onOpen={(id) => navigate(`/addons/${id}`)}
/>
</div>
);
}
function QueueTable({
title,
subtitle,
rows,
empty,
extra,
extraHead,
actions,
onOpen,
}: {
title: string;
subtitle?: string;
rows: QueueRow[];
empty: string;
extra?: (r: QueueRow) => React.ReactNode;
extraHead?: React.ReactNode;
actions?: (r: QueueRow) => React.ReactNode;
onOpen: (id: string) => void;
}) {
return (
<Card title={title} subtitle={subtitle}>
{rows.length === 0 ? (
<div className="py-4 text-sm text-muted">{empty}</div>
) : (
<Table>
<thead>
<tr>
<Th>Addon</Th>
<Th className="text-right">Size</Th>
<Th className="text-right">Subscribers</Th>
<Th>Waiting since</Th>
{extraHead}
<Th />
</tr>
</thead>
<tbody>
{rows.map((r) => (
<tr key={r.id} className="cursor-pointer hover:bg-surface-2/40" onClick={() => onOpen(r.id)}>
<Td className="text-ink">
<div className="flex items-center gap-2">
{r.name || r.id}
{r.priorityIndexing && <Badge tone="accent">priority</Badge>}
{r.blocked && <Badge tone="critical">blocked</Badge>}
</div>
<div className="font-mono text-[11px] text-muted">{r.id}</div>
</Td>
<Td className="tnum text-right">{bytes(r.currentVersionSize)}</Td>
<Td className="tnum text-right">{compact(r.subscriberCount)}</Td>
<Td className="whitespace-nowrap">{relative(r.updatedAt)}</Td>
{extra?.(r)}
<Td className="text-right" >
<span onClick={(e) => e.stopPropagation()}>{actions?.(r)}</span>
</Td>
</tr>
))}
</tbody>
</Table>
)}
</Card>
);
}
web/src/pages/Services.tsx
+212
View File
@@ -0,0 +1,212 @@
import { useState } from "react";
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { useNavigate } from "react-router";
import { LineChart, Trash2 } from "lucide-react";
import { api, qs } from "../lib/api";
import { useMetric, useRangeStep } from "../lib/queries";
import { useRefetch } from "../lib/settings";
import type { CatalogEntry, ServiceEvent, ServiceInstance } from "../lib/types";
import { compact, dateTime, duration, relative } from "../lib/format";
import { Badge, Button, Card, Empty, ErrorBox, LevelBadge, Loading, PageHeader, StatusDot, Table, Td, cn, useFeedback } from "../components/ui";
import { fromQuery, TimeChart } from "../components/charts/TimeChart";
const EXAMPLE = `POST /back/telemetry
X-SECRET-KEY: <SECRET_KEY>
Content-Type: application/json
{
"service": "indexer", // [a-z0-9_], not "main"
"instance": "indexer-1",
"version": "1.4.0",
"startedAt": 1727950000,
"heartbeatIntervalSec": 15, // offline after 3 missed beats
"status": "ok", // ok | degraded | error
"metrics": [
{"name": "jobs_total", "type": "counter", "value": 3, "labels": {"result": "ok"}},
{"name": "data_window_bytes", "type": "gauge", "value": 7.1e9},
{"name": "download_seconds", "type": "summary", "count": 3, "sum": 41.2, "min": 8.1, "max": 20.3}
],
"events": [{"level": "warn", "message": "data limit reached", "fields": {"limitGB": 50}}]
}`;
export default function Services() {
const refetchInterval = useRefetch();
const qc = useQueryClient();
const { toast, confirm } = useFeedback();
const [selected, setSelected] = useState<string | null>(null);
const services = useQuery({ queryKey: ["services"], queryFn: () => api<{ items: ServiceInstance[] }>("/services"), refetchInterval: refetchInterval || 30000 });
const forget = useMutation({
mutationFn: (id: number) => api(`/services/${id}`, { method: "DELETE" }),
onSuccess: () => {
toast("Instance removed");
qc.invalidateQueries({ queryKey: ["services"] });
},
});
if (services.isLoading) return <Loading />;
if (services.error) return <ErrorBox error={services.error} />;
const items = services.data!.items;
const names = [...new Set(items.map((s) => s.service))];
const current = selected ?? names.find((n) => n !== "main") ?? names[0] ?? null;
return (
<div className="space-y-4">
<PageHeader title="Services" description="Indexers, scrapers and the API report heartbeats, metrics and events here" />
{items.length === 0 ? (
<Card><Empty>No service has reported yet</Empty></Card>
) : (
<div className="grid grid-cols-1 gap-3 md:grid-cols-2 xl:grid-cols-3">
{items.map((s) => (
<div
key={s.id}
onClick={() => setSelected(s.service)}
className={cn(
"cursor-pointer rounded-xl border bg-surface p-4 transition-colors",
current === s.service ? "border-accent/60" : "border-line hover:border-ink/25",
)}
>
<div className="flex items-start justify-between gap-3">
<div className="min-w-0">
<div className="flex items-center gap-2">
<StatusDot online={s.online} />
<span className="font-medium text-ink">{s.service}</span>
<span className="truncate text-xs text-muted">{s.instance}</span>
</div>
<div className="mt-2 flex flex-wrap gap-1">
<Badge tone={s.online ? "good" : "critical"}>{s.online ? "online" : "offline"}</Badge>
{s.status && s.status !== "ok" && <Badge tone={s.status === "error" ? "critical" : "warning"}>{s.status}</Badge>}
{s.version && <Badge>v{s.version}</Badge>}
</div>
</div>
{!s.online && s.service !== "main" && (
<Button
size="sm"
variant="ghost"
icon={<Trash2 className="size-3.5" />}
aria-label="Forget instance"
onClick={async (e) => {
e.stopPropagation();
if (await confirm({ title: `Forget ${s.service}/${s.instance}?`, body: "Its metric history stays; the instance will reappear if it reports again." }))
forget.mutate(s.id);
}}
/>
)}
</div>
{s.statusMessage && <div className="mt-2 text-xs text-ink-2">{s.statusMessage}</div>}
<dl className="mt-3 grid grid-cols-2 gap-2 text-xs">
<div>
<dt className="text-muted">Last seen</dt>
<dd className="text-ink-2">{relative(s.lastSeen)}</dd>
</div>
<div>
<dt className="text-muted">Uptime</dt>
<dd className="text-ink-2">{s.online && s.uptimeSec ? duration(s.uptimeSec) : "—"}</dd>
</div>
<div>
<dt className="text-muted">Heartbeat</dt>
<dd className="text-ink-2">every {s.heartbeatIntervalSec}s</dd>
</div>
<div>
<dt className="text-muted">Address</dt>
<dd className="truncate text-ink-2">{s.remoteAddr || "local"}</dd>
</div>
{s.metaData &&
Object.entries(s.metaData).slice(0, 4).map(([k, v]) => (
<div key={k}>
<dt className="text-muted">{k}</dt>
<dd className="truncate text-ink-2">{String(v)}</dd>
</div>
))}
</dl>
</div>
))}
</div>
)}
{current && <ServiceDetail service={current} />}
<Card title="Sending telemetry" subtitle="Any process can report; a stdlib-only Go client lives in the telemetry/ module of this repo">
<pre className="overflow-x-auto rounded-lg bg-surface-2 p-3 font-mono text-xs leading-relaxed text-ink-2">{EXAMPLE}</pre>
<p className="mt-3 text-xs text-ink-2">
Counters carry the increase since the previous push, gauges the current value, summaries a pre-aggregated count/sum/min/max. Every push counts as a heartbeat.
</p>
</Card>
</div>
);
}
function ServiceDetail({ service }: { service: string }) {
const navigate = useNavigate();
const refetchInterval = useRefetch();
const catalog = useQuery({ queryKey: ["catalog", service], queryFn: () => api<{ items: CatalogEntry[] }>("/metrics/catalog" + qs({ service })) });
const events = useQuery({
queryKey: ["events", service],
queryFn: () => api<{ items: ServiceEvent[] }>("/services/events" + qs({ service, limit: 15 })),
refetchInterval,
});
const metrics = (catalog.data?.items ?? []).slice(0, service === "main" ? 12 : 24);
return (
<div className="space-y-4">
<div className="flex items-center justify-between">
<h2 className="text-base font-semibold text-ink">{service} metrics</h2>
<Button size="sm" icon={<LineChart className="size-3.5" />} onClick={() => navigate(`/explorer?service=${service}`)}>
Open in explorer
</Button>
</div>
{catalog.isLoading ? (
<Loading />
) : metrics.length === 0 ? (
<Card><Empty>This service has not sent any metrics yet</Empty></Card>
) : (
<div className="grid grid-cols-1 gap-4 md:grid-cols-2 xl:grid-cols-3">
{metrics.map((m) => (
<MiniMetric key={m.name} entry={m} onOpen={() => navigate(`/explorer?service=${service}&name=${m.name}`)} />
))}
</div>
)}
<Card title="Recent events">
{(events.data?.items ?? []).length === 0 ? (
<div className="text-sm text-muted">No events</div>
) : (
<Table>
<tbody>
{events.data!.items.map((e) => (
<tr key={e.id}>
<Td className="w-20"><LevelBadge level={e.level} /></Td>
<Td className="text-ink">
{e.message}
{e.fieldsData && <div className="truncate font-mono text-[11px] text-muted">{JSON.stringify(e.fieldsData)}</div>}
</Td>
<Td className="whitespace-nowrap text-xs">{e.instance}</Td>
<Td className="whitespace-nowrap text-right text-xs text-muted" title={dateTime(e.ts)}>{relative(e.ts)}</Td>
</tr>
))}
</tbody>
</Table>
)}
</Card>
</div>
);
}
function aggFor(kind: string): string {
return kind === "counter" ? "sum" : kind === "gauge" ? "last" : kind === "histogram" ? "p95" : "avg";
}
function MiniMetric({ entry, onOpen }: { entry: CatalogEntry; onOpen: () => void }) {
const { step } = useRangeStep();
const groupBy = entry.instances.length > 1 ? ["instance"] : undefined;
const q = useMetric({ service: entry.service, name: entry.name, agg: aggFor(entry.kind), groupBy, step });
return (
<Card
title={<span className="font-mono text-xs">{entry.name}</span>}
subtitle={`${entry.kind} · ${aggFor(entry.kind)}${entry.help ? ` · ${entry.help}` : ""}`}
actions={<button className="text-xs text-accent hover:underline" onClick={onOpen}>Explore</button>}
>
<TimeChart data={fromQuery(q.data, (l) => l.instance ?? entry.name)} height={140} format={compact} />
</Card>
);
}
web/src/pages/System.tsx
+248
View File
@@ -0,0 +1,248 @@
import { useState } from "react";
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { Database, Eraser, Gauge, RefreshCw } from "lucide-react";
import { api } from "../lib/api";
import { useRefetch } from "../lib/settings";
import type { HeavyStats } from "../lib/types";
import { bytes, dateTime, duration, int, num, relative } from "../lib/format";
import { Badge, Button, Card, ErrorBox, Input, Loading, PageHeader, StatTile, Table, Td, Th, useFeedback } from "../components/ui";
interface SystemData {
go: { version: string; os: string; arch: string; cpus: number; maxProcs: number; goroutines: number };
process: { pid: number; hostname: string; startedAt: number; uptimeSec: number };
memory: { heapAlloc: number; heapInuse: number; sys: number; numGC: number; pauseTotalNs: number; lastGC: number };
build: { commit?: string; time?: string; modified?: boolean; module?: string };
metrics: {
enabled: boolean;
lastFlush: number;
lastFlushMs: number;
lastFlushError: string;
lastRollup: number;
lastRetention: number;
rolledUntil: number;
openPoints: number;
liveSeries: number;
droppedPoints: number;
};
config: Record<string, unknown>;
dbPool?: { open: number; inUse: number; idle: number; maxOpen: number; waitCount: number; waitMs: number };
dbPingMs?: number;
dbPingError?: string;
db?: HeavyStats;
}
export default function System() {
const refetchInterval = useRefetch();
const qc = useQueryClient();
const { toast, confirm } = useFeedback();
const [days, setDays] = useState("7");
const sys = useQuery({ queryKey: ["system"], queryFn: () => api<SystemData>("/system"), refetchInterval });
const run = useMutation({
mutationFn: (body: { action: string; olderThanDays?: number }) => api("/system/maintenance", { method: "POST", body }),
onSuccess: (_r, v) => {
toast(`Started: ${v.action}`);
setTimeout(() => qc.invalidateQueries({ queryKey: ["system"] }), 1500);
},
onError: (e) => toast((e as Error).message, "critical"),
});
if (sys.isLoading) return <Loading />;
if (sys.error) return <ErrorBox error={sys.error} />;
const s = sys.data!;
const db = s.db;
return (
<div className="space-y-4">
<PageHeader title="System" description={<>{s.process.hostname} · pid {s.process.pid} · build {s.build.commit ? s.build.commit.slice(0, 10) : "dev"}{s.build.modified ? " (modified)" : ""}</>} />
<div className="grid grid-cols-1 gap-3 sm:grid-cols-2 xl:grid-cols-4">
<StatTile label="Uptime" value={duration(s.process.uptimeSec)} sub={`since ${dateTime(s.process.startedAt)}`} />
<StatTile label="Heap in use" value={bytes(s.memory.heapAlloc)} sub={`${bytes(s.memory.sys)} from OS · ${int(s.memory.numGC)} GCs`} />
<StatTile label="Goroutines" value={int(s.go.goroutines)} sub={`${s.go.version} · ${s.go.os}/${s.go.arch} · ${s.go.cpus} CPUs`} />
<StatTile
label="Database"
value={db ? bytes(db.dbSizeBytes) : "—"}
sub={`${db?.dbDriver ?? ""} · ping ${s.dbPingMs !== undefined ? `${num(s.dbPingMs, 1)} ms` : s.dbPingError ?? "—"}`}
/>
</div>
<div className="grid grid-cols-1 gap-4 xl:grid-cols-3">
<Card
title="Tables"
subtitle={db ? `Updated ${relative(db.asOf)}${db.dbDriver === "postgres" ? " · row counts are estimates" : ""}` : "Computing…"}
className="xl:col-span-2"
actions={
<Button size="sm" icon={<RefreshCw className="size-3.5" />} onClick={() => run.mutate({ action: "refresh-stats" })}>
Refresh
</Button>
}
>
{db?.error && <div className="mb-3"><ErrorBox error={db.error} /></div>}
<Table>
<thead>
<tr>
<Th>Table</Th>
<Th className="text-right">Rows</Th>
<Th className="text-right">Dead rows</Th>
<Th className="text-right">Size</Th>
</tr>
</thead>
<tbody>
{(db?.tables ?? []).map((t) => (
<tr key={t.name}>
<Td className="font-mono text-xs text-ink">{t.name}</Td>
<Td className="tnum text-right">{int(t.rows)}</Td>
<Td className="tnum text-right">{t.deadRows ? int(t.deadRows) : "—"}</Td>
<Td className="tnum text-right">{t.sizeBytes ? bytes(t.sizeBytes) : "—"}</Td>
</tr>
))}
</tbody>
</Table>
{db && (
<div className="mt-3 grid grid-cols-2 gap-3 text-xs text-ink-2 sm:grid-cols-3">
<div>
<div className="text-muted">Live file rows</div>
<div className="text-ink">{int(db.filesLive)}{db.filesApprox ? " (est.)" : ""}</div>
</div>
<div>
<div className="text-muted">Soft-deleted file rows</div>
<div className="text-ink">{int(db.filesSoftDeleted)}</div>
</div>
<div>
<div className="text-muted">Distinct hashes</div>
<div className="text-ink">{int(db.distinctHashes)} <span className="text-muted">· {relative(db.distinctAsOf)}</span></div>
</div>
</div>
)}
</Card>
<Card title="Connection pool">
{s.dbPool ? (
<dl className="space-y-2 text-[13px]">
{[
["Open", s.dbPool.open],
["In use", s.dbPool.inUse],
["Idle", s.dbPool.idle],
["Max open", s.dbPool.maxOpen || "unlimited"],
["Waits", s.dbPool.waitCount],
["Wait time", `${int(s.dbPool.waitMs)} ms`],
].map(([k, v]) => (
<div key={k as string} className="flex justify-between">
<dt className="text-ink-2">{k}</dt>
<dd className="tnum text-ink">{typeof v === "number" ? int(v) : v}</dd>
</div>
))}
</dl>
) : (
<div className="text-sm text-muted">Unavailable</div>
)}
</Card>
</div>
<div className="grid grid-cols-1 gap-4 xl:grid-cols-3">
<Card title="Metrics pipeline" subtitle="Minute buckets are flushed to the database every minute and rolled up hourly">
<dl className="space-y-2 text-[13px]">
<Row k="Status" v={s.metrics.enabled ? <Badge tone="good">enabled</Badge> : <Badge tone="warning">disabled</Badge>} />
<Row k="Last flush" v={`${relative(s.metrics.lastFlush)} · ${s.metrics.lastFlushMs} ms`} />
{s.metrics.lastFlushError && <Row k="Flush error" v={<span className="text-critical">{s.metrics.lastFlushError}</span>} />}
<Row k="Last rollup" v={relative(s.metrics.lastRollup)} />
<Row k="Rolled up until" v={dateTime(s.metrics.rolledUntil)} />
<Row k="Last retention run" v={relative(s.metrics.lastRetention)} />
<Row k="Unflushed buckets" v={int(s.metrics.openPoints)} />
<Row k="Series in memory" v={int(s.metrics.liveSeries)} />
<Row k="Dropped points" v={int(s.metrics.droppedPoints)} />
</dl>
</Card>
<Card title="Configuration" subtitle="Effective settings (secrets are never shown)">
<dl className="space-y-2 text-[13px]">
{Object.entries(s.config).map(([k, v]) => (
<Row
key={k}
k={k}
v={
typeof v === "boolean" ? (
<Badge tone={v ? "good" : k === "adminSecretStrong" || k === "secretKeySet" ? "critical" : "neutral"}>{v ? "yes" : "no"}</Badge>
) : Array.isArray(v) ? (
v.join(", ") || "—"
) : v === null || v === "" ? (
"—"
) : (
String(v)
)
}
/>
))}
</dl>
</Card>
<Card title="Maintenance">
<div className="space-y-4 text-[13px]">
<Action
icon={<Gauge className="size-4" />}
title="Flush metrics now"
text="Writes all buffered metric buckets to the database."
button={<Button size="sm" onClick={() => run.mutate({ action: "flush-metrics" })}>Flush</Button>}
/>
<Action
icon={<Database className="size-4" />}
title="Roll up & apply retention"
text="Builds hourly rollups and deletes data past the retention windows."
button={<Button size="sm" onClick={() => run.mutate({ action: "rollup" })}>Run</Button>}
/>
<Action
icon={<Eraser className="size-4" />}
title="Purge replaced file rows"
text={`Every re-index soft-deletes the previous file rows (${int(db?.filesSoftDeleted ?? 0)} now). Permanently remove rows replaced more than N days ago.`}
button={
<div className="flex items-center gap-1.5">
<Input value={days} onChange={(e) => setDays(e.target.value.replace(/\D/g, ""))} className="h-7 w-14 text-xs" aria-label="Days" />
<Button
size="sm"
variant="danger"
onClick={async () => {
if (
await confirm({
title: "Purge old file rows?",
body: `Permanently deletes soft-deleted addon_files rows older than ${days} days. Runs in the background in batches.`,
danger: true,
confirmLabel: "Purge",
})
)
run.mutate({ action: "purge-deleted-files", olderThanDays: Number(days) || 7 });
}}
>
Purge
</Button>
</div>
}
/>
</div>
</Card>
</div>
</div>
);
}
function Row({ k, v }: { k: string; v: React.ReactNode }) {
return (
<div className="flex items-center justify-between gap-3">
<dt className="truncate text-ink-2">{k}</dt>
<dd className="tnum shrink-0 text-right text-ink">{v}</dd>
</div>
);
}
function Action({ icon, title, text, button }: { icon: React.ReactNode; title: string; text: string; button: React.ReactNode }) {
return (
<div className="flex items-start gap-3">
<div className="mt-0.5 text-muted">{icon}</div>
<div className="min-w-0 flex-1">
<div className="text-ink">{title}</div>
<div className="mt-0.5 text-xs text-ink-2">{text}</div>
</div>
<div className="shrink-0">{button}</div>
</div>
);
}
web/src/pages/Traffic.tsx
+156
View File
@@ -0,0 +1,156 @@
import { useMemo, useState } from "react";
import { useQuery, keepPreviousData } from "@tanstack/react-query";
import { api, qs } from "../lib/api";
import { rangeSeconds, useRefetch, useSettings } from "../lib/settings";
import type { RouteStat, TrafficData } from "../lib/types";
import { compact, int, ms, num, pct, relative } from "../lib/format";
import { Badge, Card, ErrorBox, Loading, PageHeader, Segmented, StatTile, Table, Td, Th } from "../components/ui";
import { fromQuery, TimeChart, type ChartData } from "../components/charts/TimeChart";
import { BarList } from "../components/charts/BarList";
const GROUP_SLOT: Record<string, number> = { v1: 1, back: 2, admin: 3, metrics: 4, other: 5 };
const GROUP_LABEL: Record<string, string> = { v1: "Website API", back: "Workers", admin: "Admin", metrics: "Prometheus", other: "Other" };
type SortKey = "count" | "errors" | "p50" | "p95" | "p99" | "max";
export default function Traffic() {
const { range } = useSettings();
const refetchInterval = useRefetch();
const [groupFilter, setGroupFilter] = useState<string>("all");
const [sort, setSort] = useState<SortKey>("count");
const q = useQuery({
queryKey: ["traffic", range],
refetchInterval,
placeholderData: keepPreviousData,
queryFn: () => {
const to = Math.floor(Date.now() / 1000);
return api<TrafficData>("/traffic" + qs({ from: to - rangeSeconds(range), to }));
},
});
const latency = useMemo<ChartData>(() => {
if (!q.data) return { x: [], series: [] };
const parts = (["p50", "p95", "p99"] as const).map((k) => fromQuery(q.data!.latency[k], () => k));
return { x: parts[0].x, series: parts.flatMap((p, i) => p.series.map((s) => ({ ...s, slot: i + 1 }))) };
}, [q.data]);
const errors = useMemo<ChartData>(() => {
if (!q.data) return { x: [], series: [] };
const e5 = fromQuery(q.data.errors5xx, (l) => `5xx · ${GROUP_LABEL[l.group] ?? l.group}`, 60);
const e4 = fromQuery(q.data.errors4xx, () => "4xx · Website API", 60);
const x = e5.x.length ? e5.x : e4.x;
return { x, series: [...e5.series, ...e4.series].map((s, i) => ({ ...s, slot: i + 1 })) };
}, [q.data]);
if (q.isLoading) return <Loading />;
if (q.error) return <ErrorBox error={q.error} />;
const t = q.data!;
const totalReq = (t.groupTotals ?? []).reduce((s, g) => s + g.count, 0);
const status = Object.fromEntries((t.statusTotals ?? []).map((s) => [s.labels.status, s.count]));
const web = t.website;
const visitors = fromQuery(t.visitorsDaily, (l) => (l.kind === "human" ? "Visitors" : "Bots & scripts"), 1, (l) => (l.kind === "human" ? 1 : 2));
const routes = t.routes
.filter((r) => groupFilter === "all" || r.group === groupFilter)
.sort((a, b) => ((b[sort] as number | null) ?? -1) - ((a[sort] as number | null) ?? -1));
return (
<div className="space-y-4">
<PageHeader title="Traffic & performance" description={<>Request rates, latency and errors for the API · updated {relative(t.asOf)}</>} />
<div className="grid grid-cols-1 gap-3 sm:grid-cols-2 xl:grid-cols-4">
<StatTile label="Requests in range" value={compact(totalReq)} sub={`${compact(web?.count ?? 0)} from the website API`} />
<StatTile label="Website latency p95" value={ms(web?.p95)} sub={`p50 ${ms(web?.p50)} · p99 ${ms(web?.p99)}`} />
<StatTile
label="Error rate"
value={pct(status["5xx"] ?? 0, totalReq, 2)}
sub={`${int(status["5xx"] ?? 0)} server · ${int(status["4xx"] ?? 0)} client errors`}
/>
<StatTile label="Visitors today" value={int(t.visitorsToday.human)} sub={`${int(t.visitorsToday.bot)} bots & scripts · approx.`} />
</div>
<div className="grid grid-cols-1 gap-4 xl:grid-cols-2">
<Card title="Requests per minute" subtitle="By caller">
<TimeChart data={fromQuery(t.requestsByGroup, (l) => GROUP_LABEL[l.group] ?? l.group, 60, (l) => GROUP_SLOT[l.group])} format={(v) => num(v, 1)} />
</Card>
<Card title="Website API latency" subtitle="Percentiles per interval, milliseconds">
<TimeChart data={latency} format={(v) => ms(v)} />
</Card>
<Card title="Errors per minute" subtitle="Server errors by caller, client errors on the website API">
<TimeChart data={errors} format={(v) => num(v, 2)} empty="No errors in this range" />
</Card>
<Card title="Unique visitors per day" subtitle="Last 30 days · salted hash of IP + user agent, approximate">
<TimeChart data={visitors} bars format={compact} />
</Card>
</div>
<div className="grid grid-cols-1 gap-4 md:grid-cols-3">
<Card title="Status codes">
<BarList items={(t.statusTotals ?? []).map((s) => ({ key: s.labels.status, label: s.labels.status, value: s.count }))} />
</Card>
<Card title="Callers">
<BarList items={(t.groupTotals ?? []).map((g) => ({ key: g.labels.group, label: GROUP_LABEL[g.labels.group] ?? g.labels.group, value: g.count }))} />
</Card>
<Card title="Website API clients" subtitle="Browser (Origin/Referer set) vs direct calls">
<BarList items={(t.clientTotals ?? []).map((g) => ({ key: g.labels.client, label: g.labels.client === "web" ? "Browser" : "Direct / scripts", value: g.count }))} />
</Card>
</div>
<Card
title="Routes"
subtitle="Latency in milliseconds over the selected range"
actions={
<Segmented
size="sm"
options={[
{ key: "all", label: "All" },
{ key: "v1", label: "Website" },
{ key: "back", label: "Workers" },
{ key: "admin", label: "Admin" },
]}
value={groupFilter}
onChange={setGroupFilter}
/>
}
>
<Table>
<thead>
<tr>
<Th>Route</Th>
<Th>Caller</Th>
{(["count", "errors", "p50", "p95", "p99", "max"] as SortKey[]).map((k) => (
<Th key={k} className="text-right" onClick={() => setSort(k)}>
{k === "count" ? "Requests" : k === "errors" ? "5xx" : k}
{sort === k ? " ↓" : ""}
</Th>
))}
</tr>
</thead>
<tbody>
{routes.map((r: RouteStat) => (
<tr key={`${r.method} ${r.route} ${r.group}`} className="hover:bg-surface-2/40">
<Td className="font-mono text-xs text-ink">
<span className="mr-2 inline-block w-12 text-muted">{r.method}</span>
{r.route}
</Td>
<Td><Badge tone={r.group === "v1" ? "accent" : "neutral"}>{GROUP_LABEL[r.group] ?? r.group}</Badge></Td>
<Td className="tnum text-right text-ink">{int(r.count)}</Td>
<Td className="tnum text-right">{r.errors ? <span className="text-critical">{int(r.errors)}</span> : "0"}</Td>
<Td className="tnum text-right">{ms(r.p50)}</Td>
<Td className="tnum text-right">{ms(r.p95)}</Td>
<Td className="tnum text-right">{ms(r.p99)}</Td>
<Td className="tnum text-right">{ms(r.max)}</Td>
</tr>
))}
{routes.length === 0 && (
<tr>
<Td className="py-8 text-center text-muted">No requests in this range</Td>
</tr>
)}
</tbody>
</Table>
</Card>
</div>
);
}
web/src/pages/Whitelist.tsx
+315
View File
@@ -0,0 +1,315 @@
import { useState } from "react";
import { useMutation, useQuery, useQueryClient, keepPreviousData } from "@tanstack/react-query";
import { useNavigate } from "react-router";
import { Plus, Search, ShieldCheck, Trash2 } from "lucide-react";
import { api, qs } from "../lib/api";
import type { DupHash, Paged, WhitelistRow } from "../lib/types";
import { int, relative, shortHash } from "../lib/format";
import {
Badge,
Button,
Card,
Checkbox,
CopyButton,
Empty,
ErrorBox,
Input,
Loading,
PageHeader,
Pager,
Segmented,
Table,
Td,
Textarea,
Th,
useFeedback,
} from "../components/ui";
interface HashUsage {
hash: string;
whitelisted: boolean;
total: number;
items: { addonId: string; name: string; path: string; version: string }[];
}
export default function Whitelist() {
const [tab, setTab] = useState<"list" | "candidates">("list");
const [inspect, setInspect] = useState<string | null>(null);
return (
<div className="space-y-4">
<PageHeader
title="Whitelisted hashes"
description="Files whose hash is whitelisted are ignored by duplicate detection for every addon — use it for shared engine or template files."
actions={
<Segmented
options={[
{ key: "list", label: "Whitelist" },
{ key: "candidates", label: "Most shared hashes" },
]}
value={tab}
onChange={setTab}
/>
}
/>
{tab === "list" ? <List onInspect={setInspect} /> : <Candidates onInspect={setInspect} />}
{inspect && <UsageDialog hash={inspect} onClose={() => setInspect(null)} />}
</div>
);
}
function List({ onInspect }: { onInspect: (h: string) => void }) {
const qc = useQueryClient();
const { toast, confirm } = useFeedback();
const [q, setQ] = useState("");
const [draft, setDraft] = useState("");
const [offset, setOffset] = useState(0);
const [text, setText] = useState("");
const [note, setNote] = useState("");
const limit = 50;
const list = useQuery({
queryKey: ["whitelist", q, offset],
queryFn: () => api<Paged<WhitelistRow>>("/whitelist" + qs({ q, offset, limit })),
placeholderData: keepPreviousData,
});
const add = useMutation({
mutationFn: () => api<{ added: number; skipped: { hash: string; reason: string }[] }>("/whitelist", { method: "POST", body: { text, note } }),
onSuccess: (r) => {
const reasons = r.skipped.length ? ` · ${r.skipped.length} skipped (${[...new Set(r.skipped.map((s) => s.reason))].join(", ")})` : "";
toast(`${r.added} hash${r.added === 1 ? "" : "es"} added${reasons}`, r.added ? "good" : "neutral");
if (r.added) {
setText("");
setNote("");
}
qc.invalidateQueries({ queryKey: ["whitelist"] });
qc.invalidateQueries({ queryKey: ["dups"] });
},
onError: (e) => toast((e as Error).message, "critical"),
});
const remove = useMutation({
mutationFn: (id: number) => api(`/whitelist/${id}`, { method: "DELETE" }),
onSuccess: () => {
toast("Removed from whitelist");
qc.invalidateQueries({ queryKey: ["whitelist"] });
},
onError: (e) => toast((e as Error).message, "critical"),
});
const count = text.split(/[\s,;]+/).filter(Boolean).length;
return (
<div className="grid grid-cols-1 gap-4 xl:grid-cols-3">
<Card title="Add hashes" subtitle="SHA-512 hex, one per line (or separated by commas/spaces)" className="xl:order-2">
<Textarea rows={8} value={text} onChange={(e) => setText(e.target.value)} placeholder="9f86d081884c7d659a2feaa0c55ad015…" className="w-full" />
<Input value={note} onChange={(e) => setNote(e.target.value)} placeholder="Note (optional) — e.g. vanilla UI layout" className="mt-2 w-full" />
<Button variant="primary" className="mt-3 w-full" icon={<Plus className="size-4" />} disabled={!count} loading={add.isPending} onClick={() => add.mutate()}>
Add {count ? int(count) : ""} hash{count === 1 ? "" : "es"}
</Button>
</Card>
<Card bodyClass="p-0" className="xl:col-span-2">
<form
className="relative border-b border-line p-3"
onSubmit={(e) => {
e.preventDefault();
setOffset(0);
setQ(draft.trim());
}}
>
<Search className="pointer-events-none absolute top-1/2 left-5.5 size-3.5 -translate-y-1/2 text-muted" />
<Input value={draft} onChange={(e) => setDraft(e.target.value)} placeholder="Search by hash prefix or note…" className="w-full pl-8" />
</form>
{list.isLoading ? (
<div className="px-4"><Loading /></div>
) : list.error ? (
<div className="p-4"><ErrorBox error={list.error} /></div>
) : list.data!.items.length === 0 ? (
<Empty>{q ? "No matching hashes" : "No whitelisted hashes yet"}</Empty>
) : (
<div className="px-4">
<Table>
<thead>
<tr>
<Th>Hash</Th>
<Th>Example file</Th>
<Th className="text-right">Used by</Th>
<Th>Added</Th>
<Th />
</tr>
</thead>
<tbody>
{list.data!.items.map((w) => (
<tr key={w.id} className="hover:bg-surface-2/40">
<Td className="whitespace-nowrap">
<button className="font-mono text-xs text-ink hover:text-accent" onClick={() => onInspect(w.hash)}>
{shortHash(w.hash, 10)}
</button>
<CopyButton value={w.hash} />
{w.note && <div className="max-w-[240px] truncate text-xs text-muted" title={w.note}>{w.note}</div>}
</Td>
<Td className="max-w-[260px] truncate font-mono text-xs" title={w.samplePath}>{w.samplePath || "—"}</Td>
<Td className="tnum whitespace-nowrap text-right">
{int(w.addons)} addons
<div className="text-xs text-muted">{int(w.files)} files</div>
</Td>
<Td className="whitespace-nowrap">{relative(w.createdAt)}</Td>
<Td className="text-right">
<Button
size="sm"
variant="ghost"
icon={<Trash2 className="size-3.5" />}
aria-label="Remove"
onClick={async () => {
if (await confirm({ title: "Remove from whitelist?", body: "Files with this hash will count as duplicates again.", danger: true, confirmLabel: "Remove" }))
remove.mutate(w.id);
}}
/>
</Td>
</tr>
))}
</tbody>
</Table>
</div>
)}
{list.data && (
<div className="border-t border-line px-4 py-2.5">
<Pager total={list.data.total} limit={limit} offset={offset} onChange={setOffset} />
</div>
)}
</Card>
</div>
);
}
function Candidates({ onInspect }: { onInspect: (h: string) => void }) {
const qc = useQueryClient();
const { toast } = useFeedback();
const [hide, setHide] = useState(true);
const [selected, setSelected] = useState<Set<string>>(new Set());
const dups = useQuery({
queryKey: ["dups", hide],
queryFn: () => api<{ asOf: number; items: DupHash[]; pending?: boolean }>(`/overview/duplicates${hide ? "?hideWhitelisted=1" : ""}`),
});
const add = useMutation({
mutationFn: (hashes: string[]) => api<{ added: number }>("/whitelist", { method: "POST", body: { hashes, note: "from most shared hashes" } }),
onSuccess: (r) => {
toast(`${r.added} hash${r.added === 1 ? "" : "es"} whitelisted`);
setSelected(new Set());
qc.invalidateQueries({ queryKey: ["dups"] });
qc.invalidateQueries({ queryKey: ["whitelist"] });
},
onError: (e) => toast((e as Error).message, "critical"),
});
if (dups.isLoading) return <Loading />;
if (dups.error) return <ErrorBox error={dups.error} />;
const items = dups.data!.items;
return (
<Card
title="Hashes shared by the most addons"
subtitle={dups.data!.pending ? "Statistics are being computed…" : `Computed ${relative(dups.data!.asOf)} · refreshed hourly`}
actions={
<>
<label className="flex items-center gap-1.5 text-xs text-ink-2">
<Checkbox checked={hide} onChange={setHide} label="Hide whitelisted" /> Hide whitelisted
</label>
<Button size="sm" variant="primary" icon={<ShieldCheck className="size-3.5" />} disabled={selected.size === 0} loading={add.isPending} onClick={() => add.mutate([...selected])}>
Whitelist {selected.size || ""}
</Button>
</>
}
>
{items.length === 0 ? (
<Empty>No shared hashes</Empty>
) : (
<Table>
<thead>
<tr>
<Th className="w-8" />
<Th>Hash</Th>
<Th>Example file</Th>
<Th className="text-right">Addons</Th>
<Th className="text-right">Files</Th>
<Th />
</tr>
</thead>
<tbody>
{items.map((d) => (
<tr key={d.hash} className="hover:bg-surface-2/40">
<Td>
{!d.whitelisted && (
<Checkbox
label="Select"
checked={selected.has(d.hash)}
onChange={(v) => {
const next = new Set(selected);
if (v) next.add(d.hash);
else next.delete(d.hash);
setSelected(next);
}}
/>
)}
</Td>
<Td className="whitespace-nowrap">
<button className="font-mono text-xs text-ink hover:text-accent" onClick={() => onInspect(d.hash)}>
{shortHash(d.hash, 10)}
</button>
<CopyButton value={d.hash} />
</Td>
<Td className="max-w-[340px] truncate font-mono text-xs" title={d.samplePath}>{d.samplePath}</Td>
<Td className="tnum text-right text-ink">{int(d.addonCount)}</Td>
<Td className="tnum text-right">{int(d.fileCount)}</Td>
<Td className="text-right">{d.whitelisted && <Badge tone="good">whitelisted</Badge>}</Td>
</tr>
))}
</tbody>
</Table>
)}
</Card>
);
}
function UsageDialog({ hash, onClose }: { hash: string; onClose: () => void }) {
const navigate = useNavigate();
const usage = useQuery({ queryKey: ["hash", hash], queryFn: () => api<HashUsage>(`/hashes/${hash}`) });
return (
<div className="fixed inset-0 z-40 flex items-center justify-center p-4">
<div className="absolute inset-0 bg-black/50" onClick={onClose} />
<div className="relative max-h-[80vh] w-[min(720px,100%)] overflow-hidden rounded-xl border border-line bg-surface shadow-2xl">
<div className="flex items-start justify-between gap-3 border-b border-line px-5 py-4">
<div className="min-w-0">
<h3 className="font-medium text-ink">Files with this hash</h3>
<div className="mt-1 font-mono text-[11px] break-all text-muted">{hash}</div>
</div>
<Button size="sm" variant="ghost" onClick={onClose}>Close</Button>
</div>
<div className="max-h-[60vh] overflow-y-auto px-5 py-3">
{usage.isLoading ? (
<Loading />
) : usage.error ? (
<ErrorBox error={usage.error} />
) : (
<>
<div className="mb-2 flex items-center gap-2 text-xs text-ink-2">
{int(usage.data!.total)} files {usage.data!.whitelisted ? <Badge tone="good">whitelisted</Badge> : <Badge>not whitelisted</Badge>}
</div>
<Table>
<tbody>
{usage.data!.items.map((u, i) => (
<tr key={u.addonId + i} className="cursor-pointer hover:bg-surface-2/40" onClick={() => navigate(`/addons/${u.addonId}`)}>
<Td className="text-ink">{u.name || u.addonId}</Td>
<Td className="max-w-[320px] truncate font-mono text-xs" title={u.path}>{u.path}</Td>
</tr>
))}
</tbody>
</Table>
</>
)}
</div>
</div>
</div>
);
}
web/tsconfig.json
+20
View File
@@ -0,0 +1,20 @@
{
"compilerOptions": {
"target": "ES2022",
"lib": ["ES2022", "DOM", "DOM.Iterable"],
"module": "ESNext",
"moduleResolution": "bundler",
"jsx": "react-jsx",
"strict": true,
"noUnusedLocals": true,
"noUnusedParameters": true,
"noFallthroughCasesInSwitch": true,
"skipLibCheck": true,
"isolatedModules": true,
"resolveJsonModule": true,
"noEmit": true,
"types": ["vite/client"],
"tsBuildInfoFile": "node_modules/.tmp/tsconfig.tsbuildinfo"
},
"include": ["src"]
}
web/vite.config.ts
+22
View File
@@ -0,0 +1,22 @@
import { defineConfig } from "vite";
import react from "@vitejs/plugin-react";
import tailwindcss from "@tailwindcss/vite";
// The build is embedded into the Go binary (web/embed.go) and served at /admin/ui/.
// In dev, run the Go server on :8083 and `npm run dev`, then open
// http://localhost:5173/admin/ui/
export default defineConfig({
base: "/admin/ui/",
plugins: [react(), tailwindcss()],
build: {
outDir: "dist",
// keep dist/.gitkeep so go:embed always has a directory
emptyOutDir: false,
chunkSizeWarningLimit: 600,
},
server: {
proxy: {
"/admin/api": "http://localhost:8083",
},
},
});