From 56d9e106215c040b1162d02b5afba5cbc5b571a2 Mon Sep 17 00:00:00 2001 From: ilbinek Date: Sat, 3 Oct 2026 18:10:57 +0200 Subject: [PATCH] admin dashboard, metrics and telemetry - embedded React admin UI at /admin/ui (go:embed), login with ADMIN_SECRET via signed HttpOnly session cookie, X-ADMIN-KEY still works for scripts - admin API: overview stats, indexing queue, addon management (priority, reindex, delete/restore/purge, bulk), whitelist CRUD, duplicate hashes, traffic, metrics explorer, services, events/audit log, system info - built-in metrics: request latency per route, visitors, top addons and searches, indexer/scraper counters; minute buckets in the DB with hourly rollups and retention; optional Prometheus /metrics (METRICS_TOKEN) - POST /back/telemetry for indexers/scrapers (metrics, events, heartbeat) and a stdlib-only Go client in telemetry/ - auth hardening: empty secrets no longer allow access, constant-time compare, CORS not applied to /admin and /back, TRUSTED_PROXIES setting - Dockerfile builds the UI and no longer bakes config.yaml into the image; CI builds the UI, runs tests and builds arm64 with CGO --- .dockerignore | 2 + .env.dokploy.example | 23 + .gitea/workflows/build.yaml | 30 +- .gitignore | 5 + Dockerfile | 20 +- admin/addons.go | 510 +++++ admin/authapi.go | 71 + admin/metricsapi.go | 259 +++ admin/overview.go | 443 ++++ admin/routes.go | 57 + admin/system.go | 244 +++ admin/util.go | 136 ++ admin/whitelist.go | 227 ++ auth/auth_test.go | 80 + auth/middleware.go | 142 ++ auth/secret.go | 19 + auth/session.go | 108 + controllers/adminController.go | 30 - controllers/authController.go | 20 +- controllers/dependencyController.go | 11 +- controllers/feController.go | 2 + controllers/indexerController.go | 88 +- controllers/scraperController.go | 33 +- controllers/telemetryController.go | 53 + docs/admin-dashboard.md | 124 ++ go.mod | 8 +- go.sum | 16 +- initializers/initialize.go | 106 +- main.go | 141 +- metrics/aggregator.go | 286 +++ metrics/internal.go | 38 + metrics/metrics_test.go | 216 ++ metrics/pipeline.go | 156 ++ metrics/prometheus.go | 181 ++ metrics/query.go | 526 +++++ metrics/registry.go | 284 +++ metrics/services.go | 325 +++ metrics/store.go | 371 ++++ metrics/traffic.go | 328 +++ models/addon.go | 1 + models/metrics.go | 74 + models/telemetry.go | 39 + models/whitelisted.go | 3 +- telemetry/client.go | 430 ++++ telemetry/client_test.go | 73 + telemetry/go.mod | 3 + web/dist/.gitkeep | 0 web/embed.go | 13 + web/index.html | 15 + web/package-lock.json | 2534 +++++++++++++++++++++++ web/package.json | 29 + web/public/favicon.svg | 1 + web/public/theme-init.js | 5 + web/spa.go | 75 + web/spa_test.go | 51 + web/src/components/Shell.tsx | 160 ++ web/src/components/charts/BarList.tsx | 37 + web/src/components/charts/Sparkline.tsx | 36 + web/src/components/charts/TimeChart.tsx | 233 +++ web/src/components/ui.tsx | 484 +++++ web/src/index.css | 176 ++ web/src/lib/api.ts | 53 + web/src/lib/format.ts | 94 + web/src/lib/queries.ts | 59 + web/src/lib/settings.tsx | 102 + web/src/lib/types.ts | 289 +++ web/src/main.tsx | 67 + web/src/pages/Addons.tsx | 606 ++++++ web/src/pages/Events.tsx | 117 ++ web/src/pages/Explorer.tsx | 232 +++ web/src/pages/Login.tsx | 64 + web/src/pages/Overview.tsx | 209 ++ web/src/pages/Queue.tsx | 208 ++ web/src/pages/Services.tsx | 212 ++ web/src/pages/System.tsx | 248 +++ web/src/pages/Traffic.tsx | 156 ++ web/src/pages/Whitelist.tsx | 315 +++ web/tsconfig.json | 20 + web/vite.config.ts | 22 + 79 files changed, 13146 insertions(+), 118 deletions(-) create mode 100644 admin/addons.go create mode 100644 admin/authapi.go create mode 100644 admin/metricsapi.go create mode 100644 admin/overview.go create mode 100644 admin/routes.go create mode 100644 admin/system.go create mode 100644 admin/util.go create mode 100644 admin/whitelist.go create mode 100644 auth/auth_test.go create mode 100644 auth/middleware.go create mode 100644 auth/secret.go create mode 100644 auth/session.go delete mode 100644 controllers/adminController.go create mode 100644 controllers/telemetryController.go create mode 100644 docs/admin-dashboard.md create mode 100644 metrics/aggregator.go create mode 100644 metrics/internal.go create mode 100644 metrics/metrics_test.go create mode 100644 metrics/pipeline.go create mode 100644 metrics/prometheus.go create mode 100644 metrics/query.go create mode 100644 metrics/registry.go create mode 100644 metrics/services.go create mode 100644 metrics/store.go create mode 100644 metrics/traffic.go create mode 100644 models/metrics.go create mode 100644 models/telemetry.go create mode 100644 telemetry/client.go create mode 100644 telemetry/client_test.go create mode 100644 telemetry/go.mod create mode 100644 web/dist/.gitkeep create mode 100644 web/embed.go create mode 100644 web/index.html create mode 100644 web/package-lock.json create mode 100644 web/package.json create mode 100644 web/public/favicon.svg create mode 100644 web/public/theme-init.js create mode 100644 web/spa.go create mode 100644 web/spa_test.go create mode 100644 web/src/components/Shell.tsx create mode 100644 web/src/components/charts/BarList.tsx create mode 100644 web/src/components/charts/Sparkline.tsx create mode 100644 web/src/components/charts/TimeChart.tsx create mode 100644 web/src/components/ui.tsx create mode 100644 web/src/index.css create mode 100644 web/src/lib/api.ts create mode 100644 web/src/lib/format.ts create mode 100644 web/src/lib/queries.ts create mode 100644 web/src/lib/settings.tsx create mode 100644 web/src/lib/types.ts create mode 100644 web/src/main.tsx create mode 100644 web/src/pages/Addons.tsx create mode 100644 web/src/pages/Events.tsx create mode 100644 web/src/pages/Explorer.tsx create mode 100644 web/src/pages/Login.tsx create mode 100644 web/src/pages/Overview.tsx create mode 100644 web/src/pages/Queue.tsx create mode 100644 web/src/pages/Services.tsx create mode 100644 web/src/pages/System.tsx create mode 100644 web/src/pages/Traffic.tsx create mode 100644 web/src/pages/Whitelist.tsx create mode 100644 web/tsconfig.json create mode 100644 web/vite.config.ts diff --git a/.dockerignore b/.dockerignore index 31efec1..ee9dd50 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,3 +9,5 @@ api .env .env.* README.md +web/node_modules +web/dist diff --git a/.env.dokploy.example b/.env.dokploy.example index 7bdea0c..3c96d1e 100644 --- a/.env.dokploy.example +++ b/.env.dokploy.example @@ -26,3 +26,26 @@ POSTGRES_USER=postgres POSTGRES_PASSWORD=change-me-password POSTGRES_DB_NAME=dev POSTGRES_SSLMODE=disable + +# --- Admin dashboard (/admin/ui/) --------------------------------------------- +# Log in with ADMIN_SECRET (use 16+ random characters). Rotating it signs +# everyone out. The X-ADMIN-KEY header keeps working for scripts. +ADMIN_SESSION_TTL_HOURS=168 +# auto = Secure cookie when the request arrived over HTTPS (X-Forwarded-Proto) +ADMIN_COOKIE_SECURE=auto +# Proxies whose X-Forwarded-For is trusted (comma separated IPs/CIDRs), e.g. the +# Traefik network. Without it, client IPs (visitor counts, login throttling) +# can be spoofed. +TRUSTED_PROXIES=10.0.0.0/8,172.16.0.0/12 + +# --- Metrics & telemetry --------------------------------------------------------- +METRICS_ENABLED=true +# Enables GET /metrics (Prometheus) with "Authorization: Bearer ". +# Leave empty to disable the endpoint. +METRICS_TOKEN= +METRICS_MINUTE_RETENTION_DAYS=7 +METRICS_HOUR_RETENTION_DAYS=180 +EVENTS_RETENTION_DAYS=30 +TOP_ITEMS_RETENTION_DAYS=90 +# Indexing claims older than this go back to the queue +INDEX_TIMEOUT_MINUTES=30 diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml index 0128563..88bf0c0 100644 --- a/.gitea/workflows/build.yaml +++ b/.gitea/workflows/build.yaml @@ -13,18 +13,40 @@ jobs: - name: Checkout Code uses: actions/checkout@v4 + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + cache-dependency-path: web/package-lock.json + + # The admin dashboard is embedded into the binary, so build it first + - name: Build admin UI + run: | + npm ci --prefix web + npm run build --prefix web + - name: Setup Go uses: actions/setup-go@v5 with: - go-version: '1.23' + go-version: '1.25' + + - name: Vet and test + run: | + go vet ./... + go test ./... + + # SQLite needs CGO, so cross-compile ARM64 with the aarch64 C toolchain + - name: Install ARM64 C toolchain + run: sudo apt-get update && sudo apt-get install -y gcc-aarch64-linux-gnu # Build ARM64 - name: Build ARM64 - run: GOOS=linux GOARCH=arm64 go build -v -o crawler_main-linux-arm64 . + run: CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc GOOS=linux GOARCH=arm64 go build -v -ldflags "-s -w -X main.buildCommit=${{ github.sha }}" -o crawler_main-linux-arm64 . # Build AMD64 - name: Build AMD64 - run: GOOS=linux GOARCH=amd64 go build -v -o crawler_main-linux-amd64 . + run: CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -v -ldflags "-s -w -X main.buildCommit=${{ github.sha }}" -o crawler_main-linux-amd64 . # Create organized archives for download - name: Bundle Artifacts @@ -52,4 +74,4 @@ jobs: with: name: linux-amd64 path: release-amd64.tar.gz - if-no-files-found: error \ No newline at end of file + if-no-files-found: error diff --git a/.gitignore b/.gitignore index 669d57f..99134d0 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,8 @@ crawler.db .env .env.* !.env.dokploy.example + +# admin UI +web/node_modules/ +web/dist/* +!web/dist/.gitkeep diff --git a/Dockerfile b/Dockerfile index 680e0b2..40d8df8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,14 @@ +# Admin dashboard (React) - built once, embedded into the Go binary via go:embed. +FROM --platform=$BUILDPLATFORM node:22-bookworm-slim AS ui + +WORKDIR /web + +COPY web/package.json web/package-lock.json ./ +RUN npm ci + +COPY web/ ./ +RUN npm run build + FROM golang:1.25-bookworm AS builder WORKDIR /app @@ -11,7 +22,10 @@ COPY go.mod go.sum ./ RUN go mod download COPY . . -RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o /out/reforger_crawler_main . +COPY --from=ui /web/dist ./web/dist + +ARG GIT_SHA="" +RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -ldflags "-s -w -X main.buildCommit=${GIT_SHA}" -o /out/reforger_crawler_main . FROM debian:bookworm-slim AS runtime @@ -22,7 +36,9 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* COPY --from=builder /out/reforger_crawler_main /app/reforger_crawler_main -COPY config.yaml /app/config.yaml + +# Configuration comes from environment variables (see .env.dokploy.example). +# Mount a config.yaml into /app if you prefer a file. EXPOSE 8083 diff --git a/admin/addons.go b/admin/addons.go new file mode 100644 index 0000000..16adf0d --- /dev/null +++ b/admin/addons.go @@ -0,0 +1,510 @@ +package admin + +import ( + "errors" + "net/http" + "strings" + "time" + + "gitea.tbdevent.eu/TBD/reforger_crawler_main/controllers" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" + "github.com/gin-gonic/gin" + "gorm.io/gorm" +) + +// addonRow is the admin view of an addon (no files). +type addonRow struct { + ID string `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + Author string `json:"author"` + Preview string `json:"preview"` + SubscriberCount int64 `json:"subscriberCount"` + CurrentVersionNumber string `json:"currentVersionNumber"` + CurrentVersionSize int64 `json:"currentVersionSize"` + Unlisted bool `json:"unlisted"` + Private bool `json:"private"` + Blocked bool `json:"blocked"` + ToBeIndexed bool `json:"toBeIndexed"` + IsBeingIndexed bool `json:"isBeingIndexed"` + PriorityIndexing bool `json:"priorityIndexing"` + Recheck bool `json:"recheck"` + IndexStartTime *time.Time `json:"indexStartTime"` + LastIndexedAt *time.Time `json:"lastIndexedAt"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` + DeletedAt *time.Time `json:"deletedAt"` +} + +const addonColumns = `id, name, type, author, preview, subscriber_count, current_version_number, + current_version_size, unlisted, private, blocked, to_be_indexed, is_being_indexed, priority_indexing, + recheck, index_start_time, last_indexed_at, created_at, updated_at, deleted_at` + +var addonSorts = map[string]string{ + "name": "name", + "subscribers": "subscriber_count", + "size": "current_version_size", + "created": "created_at", + "updated": "updated_at", + "indexed": "last_indexed_at", + "deleted": "deleted_at", +} + +func listAddons(c *gin.Context) { + limit := queryInt(c, "limit", 50, 1, 200) + offset := queryInt(c, "offset", 0, 0, 1<<30) + status := c.DefaultQuery("status", "all") + + q := initializers.DB.Model(&models.Addon{}) + switch status { + case "all": + case "indexed": + q = q.Where("to_be_indexed = ?", false) + case "queued": + q = q.Where("to_be_indexed = ? AND is_being_indexed = ?", true, false) + case "indexing": + q = q.Where("is_being_indexed = ?", true) + case "priority": + q = q.Where("priority_indexing = ?", true) + case "recheck": + q = q.Where("recheck = ?", true) + case "blocked": + q = q.Where("blocked = ?", true) + case "unlisted": + q = q.Where("unlisted = ?", true) + case "private": + q = q.Where("private = ?", true) + case "deleted": + q = q.Unscoped().Where("deleted_at IS NOT NULL") + default: + fail(c, http.StatusBadRequest, "unknown status") + return + } + if t := c.Query("type"); t != "" { + q = q.Where("type = ?", t) + } + if s := strings.TrimSpace(c.Query("q")); s != "" { + like := "%" + likeEscape(strings.ToLower(s)) + "%" + q = q.Where(`id = ? OR LOWER(name) LIKE ? ESCAPE '\' OR LOWER(author) LIKE ? ESCAPE '\'`, s, like, like) + } + + var total int64 + if err := q.Session(&gorm.Session{}).Count(&total).Error; err != nil { + fail(c, http.StatusInternalServerError, err.Error()) + return + } + + sortCol, ok := addonSorts[c.DefaultQuery("sort", "subscribers")] + if !ok { + sortCol = "subscriber_count" + } + dir := "DESC" + if c.Query("order") == "asc" { + dir = "ASC" + } + nulls := "" + if isPostgres() && dir == "DESC" { + nulls = " NULLS LAST" + } + + rows := []addonRow{} + err := q.Select(addonColumns).Order(sortCol + " " + dir + nulls + ", id ASC").Limit(limit).Offset(offset).Scan(&rows).Error + if err != nil { + fail(c, http.StatusInternalServerError, err.Error()) + return + } + c.JSON(http.StatusOK, gin.H{"items": rows, "total": total, "limit": limit, "offset": offset}) +} + +type sharedAddon struct { + AddonID string `json:"addonId"` + Name string `json:"name"` + SharedFiles int64 `json:"sharedFiles"` +} + +func addonDetail(c *gin.Context) { + id := c.Param("id") + var row addonRow + res := initializers.DB.Unscoped().Model(&models.Addon{}).Select(addonColumns).Where("id = ?", id).Limit(1).Scan(&row) + if res.Error != nil { + fail(c, http.StatusInternalServerError, res.Error.Error()) + return + } + if res.RowsAffected == 0 { + fail(c, http.StatusNotFound, "addon not found") + return + } + var extra struct { + Summary string + CurrentVersionID int64 + } + initializers.DB.Unscoped().Model(&models.Addon{}).Select("summary, current_version_id").Where("id = ?", id).Limit(1).Scan(&extra) + + var fileCount, deletedFiles int64 + initializers.DB.Model(&models.AddonFile{}).Where("addon_id = ?", id).Count(&fileCount) + initializers.DB.Unscoped().Model(&models.AddonFile{}).Where("addon_id = ? AND deleted_at IS NOT NULL", id).Count(&deletedFiles) + + deps, depsTotal, _ := controllers.QueryRelatedAddons(id, "ad.dependency_id", "ad.addon_id", 50, 0) + dependents, dependentsTotal, _ := controllers.QueryRelatedAddons(id, "ad.addon_id", "ad.dependency_id", 50, 0) + + // declared dependencies that are not (or no longer) in the addons table + var missingDeps []struct { + DependencyID string `json:"id"` + DependencyName string `json:"name"` + Version string `json:"version"` + } + initializers.DB.Raw(`SELECT ad.dependency_id, ad.dependency_name, ad.version FROM addon_dependencies ad + LEFT JOIN addons a ON a.id = ad.dependency_id AND a.deleted_at IS NULL + WHERE ad.addon_id = ? AND a.id IS NULL LIMIT 50`, id).Scan(&missingDeps) + + shared := []sharedAddon{} + err := initializers.DB.Raw(`SELECT f.addon_id AS addon_id, COALESCE(MAX(a.name), '') AS name, COUNT(*) AS shared_files + FROM addon_files f + LEFT JOIN addons a ON a.id = f.addon_id + WHERE f.deleted_at IS NULL AND f.addon_id <> ? + AND f.hash IN (SELECT s.hash FROM addon_files s WHERE s.addon_id = ? AND s.deleted_at IS NULL) + AND NOT EXISTS (SELECT 1 FROM whitelisted_hashes w WHERE w.hash = f.hash AND w.deleted_at IS NULL) + GROUP BY f.addon_id ORDER BY shared_files DESC LIMIT 25`, id, id).Scan(&shared).Error + if err != nil { + fail(c, http.StatusInternalServerError, err.Error()) + return + } + + c.JSON(http.StatusOK, gin.H{ + "addon": row, + "summary": extra.Summary, + "currentVersionId": extra.CurrentVersionID, + "fileCount": fileCount, + "deletedFileRows": deletedFiles, + "dependencies": gin.H{"total": depsTotal, "items": deps, "missing": missingDeps}, + "dependents": gin.H{"total": dependentsTotal, "items": dependents}, + "duplicates": shared, + }) +} + +type fileRow struct { + ID uint `json:"id"` + Path string `json:"path"` + Hash string `json:"hash"` + Version string `json:"version"` + Whitelisted bool `json:"whitelisted"` + OtherAddons int64 `json:"otherAddons"` +} + +func addonFiles(c *gin.Context) { + id := c.Param("id") + limit := queryInt(c, "limit", 100, 1, 500) + offset := queryInt(c, "offset", 0, 0, 1<<30) + q := initializers.DB.Model(&models.AddonFile{}).Where("addon_id = ?", id) + if s := strings.TrimSpace(c.Query("q")); s != "" { + like := "%" + likeEscape(strings.ToLower(s)) + "%" + q = q.Where(`LOWER(path) LIKE ? ESCAPE '\' OR hash = ?`, like, strings.ToLower(s)) + } + var total int64 + q.Session(&gorm.Session{}).Count(&total) + + rows := []fileRow{} + if err := q.Select("id, path, hash, version").Order("path ASC").Limit(limit).Offset(offset).Scan(&rows).Error; err != nil { + fail(c, http.StatusInternalServerError, err.Error()) + return + } + + if len(rows) > 0 && c.Query("shared") != "0" { + hashes := make([]string, 0, len(rows)) + for _, r := range rows { + hashes = append(hashes, r.Hash) + } + var wl []string + initializers.DB.Model(&models.WhitelistedHash{}).Where("hash IN ?", hashes).Pluck("hash", &wl) + wlSet := map[string]bool{} + for _, h := range wl { + wlSet[h] = true + } + var counts []struct { + Hash string + Addons int64 + } + initializers.DB.Raw(`SELECT hash, COUNT(DISTINCT addon_id) AS addons FROM addon_files + WHERE deleted_at IS NULL AND hash IN ? GROUP BY hash`, hashes).Scan(&counts) + cnt := map[string]int64{} + for _, x := range counts { + cnt[x.Hash] = x.Addons + } + for i := range rows { + rows[i].Whitelisted = wlSet[rows[i].Hash] + rows[i].OtherAddons = max(0, cnt[rows[i].Hash]-1) + } + } + c.JSON(http.StatusOK, gin.H{"items": rows, "total": total, "limit": limit, "offset": offset}) +} + +var errNotFound = errors.New("addon not found") + +// applyAction changes queue flags of the given addons. UpdateColumns is used +// so updated_at (the queue order) is left alone. +func applyAction(tx *gorm.DB, ids []string, action string, value bool) (int64, error) { + base := tx.Model(&models.Addon{}).Where("id IN ?", ids) + var res *gorm.DB + switch action { + case "priority": + cols := map[string]any{"priority_indexing": value} + if value { + cols["to_be_indexed"] = true + } + res = base.UpdateColumns(cols) + case "to-be-indexed": + res = base.UpdateColumns(map[string]any{"to_be_indexed": value}) + case "reindex": + cols := map[string]any{"to_be_indexed": true, "is_being_indexed": false} + if value { + cols["priority_indexing"] = true + } + res = base.UpdateColumns(cols) + case "reset-indexing": + res = base.UpdateColumns(map[string]any{"is_being_indexed": false}) + case "recheck": + res = base.UpdateColumns(map[string]any{"recheck": value}) + case "restore": + res = tx.Unscoped().Model(&models.Addon{}).Where("id IN ? AND deleted_at IS NOT NULL", ids). + UpdateColumns(map[string]any{"deleted_at": nil, "to_be_indexed": true, "is_being_indexed": false}) + case "delete": + var addons []models.Addon + if err := tx.Where("id IN ?", ids).Find(&addons).Error; err != nil { + return 0, err + } + for i := range addons { + if err := controllers.DeleteAddonCascade(tx, &addons[i]); err != nil { + return 0, err + } + } + return int64(len(addons)), nil + default: + return 0, errors.New("unknown action") + } + return res.RowsAffected, res.Error +} + +func addonAction(c *gin.Context) { + id, action := c.Param("id"), c.Param("action") + var body struct { + Value *bool `json:"value"` + } + _ = c.ShouldBindJSON(&body) + value := body.Value == nil || *body.Value + if action == "delete" { + fail(c, http.StatusBadRequest, "use DELETE /addons/:id") + return + } + n, err := applyAction(initializers.DB, []string{id}, action, value) + if err != nil { + fail(c, http.StatusBadRequest, err.Error()) + return + } + if n == 0 { + fail(c, http.StatusNotFound, "addon not found or nothing to change") + return + } + audit(c, "addon_"+strings.ReplaceAll(action, "-", "_"), map[string]any{"addon": id, "value": value}) + invalidate("overview", "queue") + c.JSON(http.StatusOK, gin.H{"updated": n}) +} + +func deleteAddon(c *gin.Context) { + id := c.Param("id") + err := initializers.DB.Transaction(func(tx *gorm.DB) error { + var addon models.Addon + if err := tx.Where("id = ?", id).First(&addon).Error; err != nil { + return errNotFound + } + return controllers.DeleteAddonCascade(tx, &addon) + }) + if errors.Is(err, errNotFound) { + fail(c, http.StatusNotFound, err.Error()) + return + } + if err != nil { + fail(c, http.StatusInternalServerError, err.Error()) + return + } + audit(c, "addon_delete", map[string]any{"addon": id}) + invalidate("overview", "queue") + c.JSON(http.StatusOK, gin.H{"deleted": id}) +} + +// purgeAddon permanently removes an already soft-deleted addon and its rows. +func purgeAddon(c *gin.Context) { + id := c.Param("id") + if c.Query("confirm") != id { + fail(c, http.StatusBadRequest, "pass ?confirm= to purge") + return + } + var files int64 + err := initializers.DB.Transaction(func(tx *gorm.DB) error { + var addon models.Addon + if err := tx.Unscoped().Where("id = ? AND deleted_at IS NOT NULL", id).First(&addon).Error; err != nil { + return errNotFound + } + res := tx.Unscoped().Where("addon_id = ?", id).Delete(&models.AddonFile{}) + if res.Error != nil { + return res.Error + } + files = res.RowsAffected + if err := tx.Where("addon_id = ?", id).Delete(&models.AddonDependency{}).Error; err != nil { + return err + } + return tx.Unscoped().Delete(&addon).Error + }) + if errors.Is(err, errNotFound) { + fail(c, http.StatusNotFound, "addon not found or not deleted") + return + } + if err != nil { + fail(c, http.StatusInternalServerError, err.Error()) + return + } + audit(c, "addon_purge", map[string]any{"addon": id, "files": files}) + invalidate("overview") + c.JSON(http.StatusOK, gin.H{"purged": id, "files": files}) +} + +func bulkAddons(c *gin.Context) { + var body struct { + IDs []string `json:"ids"` + Action string `json:"action"` + Value *bool `json:"value"` + } + if err := c.ShouldBindJSON(&body); err != nil || len(body.IDs) == 0 { + fail(c, http.StatusBadRequest, "ids and action are required") + return + } + if len(body.IDs) > 500 { + fail(c, http.StatusBadRequest, "at most 500 ids per request") + return + } + value := body.Value == nil || *body.Value + var n int64 + err := initializers.DB.Transaction(func(tx *gorm.DB) error { + var err error + n, err = applyAction(tx, body.IDs, body.Action, value) + return err + }) + if err != nil { + fail(c, http.StatusBadRequest, err.Error()) + return + } + audit(c, "bulk_"+strings.ReplaceAll(body.Action, "-", "_"), map[string]any{"count": len(body.IDs), "updated": n, "value": value}) + invalidate("overview", "queue") + c.JSON(http.StatusOK, gin.H{"updated": n}) +} + +// ---- queue --------------------------------------------------------------- + +type queueRow struct { + ID string `json:"id"` + Name string `json:"name"` + CurrentVersionSize int64 `json:"currentVersionSize"` + SubscriberCount int64 `json:"subscriberCount"` + PriorityIndexing bool `json:"priorityIndexing"` + Blocked bool `json:"blocked"` + IndexStartTime *time.Time `json:"indexStartTime"` + UpdatedAt time.Time `json:"updatedAt"` + ElapsedSec int64 `json:"elapsedSec"` + Stuck bool `json:"stuck"` +} + +const queueColumns = "id, name, current_version_size, subscriber_count, priority_indexing, blocked, index_start_time, updated_at" + +func buildQueue() (gin.H, error) { + counts, err := metrics.LoadQueueCounts() + if err != nil { + return nil, err + } + var blockedQueued int64 + initializers.DB.Model(&models.Addon{}).Where("to_be_indexed = ? AND blocked = ?", true, true).Count(&blockedQueued) + + now := time.Now() + indexing := []queueRow{} + initializers.DB.Model(&models.Addon{}).Select(queueColumns).Where("is_being_indexed = ?", true). + Order("index_start_time ASC").Limit(100).Scan(&indexing) + for i := range indexing { + if t := indexing[i].IndexStartTime; t != nil && !t.IsZero() { + indexing[i].ElapsedSec = int64(now.Sub(*t).Seconds()) + indexing[i].Stuck = now.Sub(*t) > initializers.IndexTimeout + } + } + + priority := []queueRow{} + initializers.DB.Model(&models.Addon{}).Select(queueColumns). + Where("to_be_indexed = ? AND is_being_indexed = ? AND priority_indexing = ?", true, false, true). + Order("updated_at ASC").Limit(50).Scan(&priority) + + // same order the indexers claim jobs in + next := []queueRow{} + initializers.DB.Model(&models.Addon{}).Select(queueColumns). + Where("to_be_indexed = ? AND is_being_indexed = ? AND blocked = ?", true, false, false). + Order("priority_indexing DESC, updated_at ASC").Limit(25).Scan(&next) + + ts := now.Unix() + ok := map[string]string{"result": "ok"} + sel := metrics.Selector{Service: metrics.MainService, Name: "indexer_submissions_total", Match: ok} + perHour1h := metrics.Sum(sel, ts-3600, ts) + perHour6h := metrics.Sum(sel, ts-6*3600, ts) / 6 + indexed24h := metrics.Sum(sel, ts-86400, ts) + var etaHours *float64 + if perHour6h > 0 { + v := float64(counts.Queued) / perHour6h + etaHours = &v + } + duration, _ := metrics.Totals(metrics.Selector{Service: metrics.MainService, Name: "indexer_job_duration_seconds"}, ts-86400, ts) + var durationStats any + if len(duration) > 0 { + durationStats = duration[0] + } + + return gin.H{ + "counts": counts, + "blockedQueued": blockedQueued, + "timeoutMinutes": int(initializers.IndexTimeout.Minutes()), + "indexing": indexing, + "priority": priority, + "next": next, + "throughput": gin.H{ + "perHour1h": perHour1h, + "perHour6h": perHour6h, + "indexed24h": indexed24h, + "etaHours": etaHours, + "duration24h": durationStats, + }, + }, nil +} + +func queue(c *gin.Context) { + data, at, err := cached("queue", 5*time.Second, buildQueue) + if err != nil { + fail(c, http.StatusInternalServerError, err.Error()) + return + } + out := gin.H{"asOf": at.Unix()} + for k, v := range data { + out[k] = v + } + c.JSON(http.StatusOK, out) +} + +func resetStuck(c *gin.Context) { + var body struct { + OlderThanMinutes int `json:"olderThanMinutes"` + } + _ = c.ShouldBindJSON(&body) + older := initializers.IndexTimeout + if body.OlderThanMinutes > 0 { + older = time.Duration(body.OlderThanMinutes) * time.Minute + } + n := controllers.ResetTimedOutIndexing(older) + metrics.IndexerTimeoutsReset.Add(float64(n)) + audit(c, "queue_reset_stuck", map[string]any{"olderThanMinutes": older.Minutes(), "reset": n}) + invalidate("queue", "overview") + c.JSON(http.StatusOK, gin.H{"reset": n}) +} diff --git a/admin/authapi.go b/admin/authapi.go new file mode 100644 index 0000000..19d9627 --- /dev/null +++ b/admin/authapi.go @@ -0,0 +1,71 @@ +package admin + +import ( + "net/http" + "time" + + "gitea.tbdevent.eu/TBD/reforger_crawler_main/auth" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics" + "github.com/gin-gonic/gin" +) + +var limiter = auth.NewLoginLimiter(5, 10*time.Minute) + +func login(c *gin.Context) { + if initializers.AdminSecret == "" { + fail(c, http.StatusServiceUnavailable, "admin disabled: ADMIN_SECRET not set") + return + } + if c.GetHeader(auth.CSRFHeader) != "1" { + fail(c, http.StatusForbidden, "csrf check failed") + return + } + ip := c.ClientIP() + if limiter.Blocked(ip) { + fail(c, http.StatusTooManyRequests, "too many failed attempts, try again later") + return + } + var body struct { + Secret string `json:"secret"` + } + if err := c.ShouldBindJSON(&body); err != nil { + fail(c, http.StatusBadRequest, "invalid JSON") + return + } + if !auth.SecretEqual(body.Secret, initializers.AdminSecret) { + limiter.Fail(ip) + time.Sleep(300 * time.Millisecond) + metrics.AdminActions.Inc("login_failed") + metrics.Event("warn", "admin: login failed", map[string]any{"kind": "audit", "ip": ip}) + fail(c, http.StatusUnauthorized, "invalid secret") + return + } + limiter.Reset(ip) + token, exp := auth.IssueToken(time.Now()) + auth.SetSessionCookie(c, token, exp) + c.Set("adminAuth", "login") + audit(c, "login", nil) + c.JSON(http.StatusOK, gin.H{"expiresAt": exp.Unix()}) +} + +func logout(c *gin.Context) { + auth.ClearSessionCookie(c) + c.Status(http.StatusNoContent) +} + +func me(c *gin.Context) { + method := c.GetString("adminAuth") + resp := gin.H{"method": method} + if v, ok := c.Get("adminSessionExp"); ok { + exp := v.(time.Time) + // sliding renewal: extend sessions that are about to expire + if time.Until(exp) < 24*time.Hour { + token, newExp := auth.IssueToken(time.Now()) + auth.SetSessionCookie(c, token, newExp) + exp = newExp + } + resp["expiresAt"] = exp.Unix() + } + c.JSON(http.StatusOK, resp) +} diff --git a/admin/metricsapi.go b/admin/metricsapi.go new file mode 100644 index 0000000..ed6b214 --- /dev/null +++ b/admin/metricsapi.go @@ -0,0 +1,259 @@ +package admin + +import ( + "encoding/json" + "net/http" + "sort" + "strconv" + "strings" + "time" + + "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" + "github.com/gin-gonic/gin" +) + +func metricsCatalog(c *gin.Context) { + service := c.Query("service") + data, _, err := cached("catalog:"+service, 15*time.Second, func() ([]metrics.CatalogEntry, error) { + return metrics.Catalog(service) + }) + if err != nil { + fail(c, http.StatusInternalServerError, err.Error()) + return + } + c.JSON(http.StatusOK, gin.H{"items": data, "aggregations": metrics.Aggregations}) +} + +func selectorFrom(c *gin.Context) (metrics.Selector, bool) { + sel := metrics.Selector{Service: c.Query("service"), Name: c.Query("name")} + if m := c.Query("match"); m != "" { + if err := json.Unmarshal([]byte(m), &sel.Match); err != nil { + fail(c, http.StatusBadRequest, "match must be a JSON object of label values") + return sel, false + } + } + if g := c.Query("groupBy"); g != "" { + for _, part := range strings.Split(g, ",") { + if part = strings.TrimSpace(part); part != "" { + sel.GroupBy = append(sel.GroupBy, part) + } + } + } + if sel.Name == "" { + fail(c, http.StatusBadRequest, "name is required") + return sel, false + } + return sel, true +} + +func metricsQuery(c *gin.Context) { + sel, ok := selectorFrom(c) + if !ok { + return + } + from, to := timeRange(c, time.Hour) + res, err := metrics.Query(sel, c.Query("agg"), from, to, queryInt64(c, "step", 0)) + if err != nil { + fail(c, http.StatusBadRequest, err.Error()) + return + } + c.JSON(http.StatusOK, res) +} + +func autoStep(from, to int64) int64 { + span := to - from + switch { + case span <= 3*3600: + return 60 + case span <= 12*3600: + return 300 + case span <= 48*3600: + return 900 + case span <= 8*86400: + return 3600 + default: + return 6 * 3600 + } +} + +type routeStat struct { + Method string `json:"method"` + Route string `json:"route"` + Group string `json:"group"` + Count int64 `json:"count"` + Errors int64 `json:"errors"` + Client4 int64 `json:"clientErrors"` + P50 *float64 `json:"p50"` + P95 *float64 `json:"p95"` + P99 *float64 `json:"p99"` + Max *float64 `json:"max"` + Avg *float64 `json:"avg"` +} + +// traffic returns pre-composed request, latency and error data for a range. +func traffic(c *gin.Context) { + from, to := timeRange(c, 24*time.Hour) + step := queryInt64(c, "step", 0) + if step <= 0 { + step = autoStep(from, to) + } + key := "traffic:" + fmtInt64(from/30) + ":" + fmtInt64(to/30) + ":" + fmtInt64(step) + data, at, err := cached(key, 30*time.Second, func() (gin.H, error) { + return buildTraffic(from, to, step) + }) + if err != nil { + fail(c, http.StatusInternalServerError, err.Error()) + return + } + out := gin.H{"asOf": at.Unix()} + for k, v := range data { + out[k] = v + } + c.JSON(http.StatusOK, out) +} + +func fmtInt64(v int64) string { return strconv.FormatInt(v, 10) } + +func buildTraffic(from, to, step int64) (gin.H, error) { + const name = "http_request_duration_ms" + sel := func(match map[string]string, groupBy ...string) metrics.Selector { + return metrics.Selector{Service: metrics.MainService, Name: name, Match: match, GroupBy: groupBy} + } + v1 := map[string]string{"group": "v1"} + + byGroup, err := metrics.Query(sel(nil, "group"), "rate", from, to, step) + if err != nil { + return nil, err + } + latency := gin.H{} + for _, q := range []string{"p50", "p95", "p99"} { + r, err := metrics.Query(sel(v1), q, from, to, step) + if err != nil { + return nil, err + } + latency[q] = r + } + errors5xx, _ := metrics.Query(sel(map[string]string{"status": "5xx"}, "group"), "rate", from, to, step) + errors4xx, _ := metrics.Query(sel(map[string]string{"status": "4xx", "group": "v1"}), "rate", from, to, step) + + statusTotals, _ := metrics.Totals(sel(nil, "status"), from, to) + clientTotals, _ := metrics.Totals(sel(v1, "client"), from, to) + groupTotals, _ := metrics.Totals(sel(nil, "group"), from, to) + + routeTotals, err := metrics.Totals(sel(nil, "method", "route", "group"), from, to) + if err != nil { + return nil, err + } + statusByRoute, _ := metrics.Totals(sel(nil, "method", "route", "status"), from, to) + errs := map[string][2]int64{} + for _, t := range statusByRoute { + k := t.Labels["method"] + " " + t.Labels["route"] + e := errs[k] + switch t.Labels["status"] { + case "5xx": + e[0] += t.Count + case "4xx": + e[1] += t.Count + } + errs[k] = e + } + routes := make([]routeStat, 0, len(routeTotals)) + for _, t := range routeTotals { + k := t.Labels["method"] + " " + t.Labels["route"] + routes = append(routes, routeStat{ + Method: t.Labels["method"], Route: t.Labels["route"], Group: t.Labels["group"], + Count: t.Count, Errors: errs[k][0], Client4: errs[k][1], + P50: t.P50, P95: t.P95, P99: t.P99, Max: t.Max, Avg: t.Avg, + }) + } + sort.Slice(routes, func(i, j int) bool { return routes[i].Count > routes[j].Count }) + + now := time.Now().Unix() + visitorsDaily, _ := metrics.Query(metrics.Selector{Service: metrics.MainService, Name: "visitors_unique", GroupBy: []string{"kind"}}, + "max", now-30*86400, now, 86400) + humans, bots := metrics.Visitors() + + var v1Total *metrics.Total + if all, _ := metrics.Totals(sel(v1), from, to); len(all) > 0 { + v1Total = &all[0] + } + + return gin.H{ + "from": from, "to": to, "step": step, + "requestsByGroup": byGroup, + "latency": latency, + "errors5xx": errors5xx, + "errors4xx": errors4xx, + "statusTotals": statusTotals, + "clientTotals": clientTotals, + "groupTotals": groupTotals, + "website": v1Total, + "routes": routes, + "visitorsDaily": visitorsDaily, + "visitorsToday": gin.H{"human": humans, "bot": bots}, + }, nil +} + +type topEntry struct { + Item string `json:"item"` + Count int64 `json:"count"` + Name string `json:"name,omitempty"` +} + +// topItems returns the most looked-up addons or search queries. +func topItems(c *gin.Context) { + kind := c.DefaultQuery("kind", "addon") + if kind != "addon" && kind != "search" { + fail(c, http.StatusBadRequest, "kind must be addon or search") + return + } + days := queryInt(c, "days", 1, 1, 90) + limit := queryInt(c, "limit", 20, 1, 200) + + today, counts := metrics.Top.Today(kind) + if today == "" { + today = time.Now().UTC().Format("2006-01-02") + } + if days > 1 { + since := time.Now().UTC().AddDate(0, 0, -(days - 1)).Format("2006-01-02") + var rows []models.DailyTopItem + initializers.DB.Where("kind = ? AND day >= ? AND day < ?", kind, since, today).Find(&rows) + for _, r := range rows { + counts[r.Item] += r.Count + } + } + list := make([]topEntry, 0, len(counts)) + for k, v := range counts { + list = append(list, topEntry{Item: k, Count: v}) + } + sort.Slice(list, func(i, j int) bool { + if list[i].Count != list[j].Count { + return list[i].Count > list[j].Count + } + return list[i].Item < list[j].Item + }) + if len(list) > limit { + list = list[:limit] + } + if kind == "addon" && len(list) > 0 { + ids := make([]string, len(list)) + for i, e := range list { + ids[i] = e.Item + } + var names []struct { + ID string + Name string + } + initializers.DB.Unscoped().Model(&models.Addon{}).Select("id, name").Where("id IN ?", ids).Scan(&names) + byID := map[string]string{} + for _, n := range names { + byID[n.ID] = n.Name + } + for i := range list { + list[i].Name = byID[list[i].Item] + } + } + c.JSON(http.StatusOK, gin.H{"kind": kind, "days": days, "items": list}) +} diff --git a/admin/overview.go b/admin/overview.go new file mode 100644 index 0000000..31f0509 --- /dev/null +++ b/admin/overview.go @@ -0,0 +1,443 @@ +package admin + +import ( + "context" + "log" + "net/http" + "os" + "strconv" + "sync" + "time" + + "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" + "github.com/gin-gonic/gin" + "gorm.io/gorm" +) + +type addonCounts struct { + Total int64 `json:"total"` + Indexed int64 `json:"indexed"` + Queued int64 `json:"queued"` + Indexing int64 `json:"indexing"` + Priority int64 `json:"priority"` + Recheck int64 `json:"recheck"` + Blocked int64 `json:"blocked"` + Unlisted int64 `json:"unlisted"` + Private int64 `json:"private"` + TotalSize int64 `json:"totalSizeBytes"` + Added24h int64 `json:"added24h"` + Added7d int64 `json:"added7d"` + Indexed24h int64 `json:"indexed24h"` + + SoftDeleted int64 `json:"softDeleted"` +} + +type typeCount struct { + Type string `json:"type"` + Count int64 `json:"count"` +} + +type miniAddon struct { + ID string `json:"id"` + Name string `json:"name"` + Author string `json:"author"` + SubscriberCount int64 `json:"subscriberCount"` + CurrentVersionSize int64 `json:"currentVersionSize"` + LastIndexedAt *time.Time `json:"lastIndexedAt,omitempty"` + CreatedAt time.Time `json:"createdAt"` +} + +type overviewData struct { + Addons addonCounts `json:"addons"` + Types []typeCount `json:"types"` + Whitelist int64 `json:"whitelistCount"` + Dependencies map[string]any `json:"dependencies"` + Traffic map[string]any `json:"traffic"` + Activity map[string]any `json:"activity"` + Services map[string]any `json:"services"` + TopSubscribed []miniAddon `json:"topSubscribed"` + Largest []miniAddon `json:"largest"` + RecentlyAdded []miniAddon `json:"recentlyAdded"` + RecentlyIndexed []miniAddon `json:"recentlyIndexed"` + Heavy *heavyStats `json:"heavy"` +} + +func loadAddonCounts() (addonCounts, error) { + var ac addonCounts + now := time.Now() + err := initializers.DB.Raw(`SELECT COUNT(*) AS total, + COALESCE(SUM(CASE WHEN to_be_indexed = ? THEN 1 ELSE 0 END), 0) AS indexed, + COALESCE(SUM(CASE WHEN to_be_indexed = ? AND is_being_indexed = ? THEN 1 ELSE 0 END), 0) AS queued, + COALESCE(SUM(CASE WHEN is_being_indexed = ? THEN 1 ELSE 0 END), 0) AS indexing, + COALESCE(SUM(CASE WHEN priority_indexing = ? AND to_be_indexed = ? THEN 1 ELSE 0 END), 0) AS priority, + COALESCE(SUM(CASE WHEN recheck = ? THEN 1 ELSE 0 END), 0) AS recheck, + COALESCE(SUM(CASE WHEN blocked = ? THEN 1 ELSE 0 END), 0) AS blocked, + COALESCE(SUM(CASE WHEN unlisted = ? THEN 1 ELSE 0 END), 0) AS unlisted, + COALESCE(SUM(CASE WHEN private = ? THEN 1 ELSE 0 END), 0) AS private, + CAST(COALESCE(SUM(current_version_size), 0) AS BIGINT) AS total_size, + COALESCE(SUM(CASE WHEN created_at >= ? THEN 1 ELSE 0 END), 0) AS added24h, + COALESCE(SUM(CASE WHEN created_at >= ? THEN 1 ELSE 0 END), 0) AS added7d, + COALESCE(SUM(CASE WHEN last_indexed_at >= ? THEN 1 ELSE 0 END), 0) AS indexed24h + FROM addons WHERE deleted_at IS NULL`, + false, true, false, true, true, true, true, true, true, true, + now.Add(-24*time.Hour), now.Add(-7*24*time.Hour), now.Add(-24*time.Hour), + ).Scan(&ac).Error + if err != nil { + return ac, err + } + err = initializers.DB.Unscoped().Model(&models.Addon{}).Where("deleted_at IS NOT NULL").Count(&ac.SoftDeleted).Error + return ac, err +} + +func loadMiniAddons(order string, where string, args ...any) []miniAddon { + out := []miniAddon{} + q := initializers.DB.Model(&models.Addon{}). + Select("id, name, author, subscriber_count, current_version_size, last_indexed_at, created_at") + if where != "" { + q = q.Where(where, args...) + } + q.Order(order).Limit(8).Scan(&out) + return out +} + +func buildOverview() (*overviewData, error) { + ac, err := loadAddonCounts() + if err != nil { + return nil, err + } + d := &overviewData{Addons: ac} + + d.Types = []typeCount{} + initializers.DB.Model(&models.Addon{}).Select("type, COUNT(*) AS count").Group("type").Order("count DESC").Limit(12).Scan(&d.Types) + + initializers.DB.Model(&models.WhitelistedHash{}).Count(&d.Whitelist) + + var deps struct { + Edges int64 + WithDeps int64 + Dependents int64 + } + initializers.DB.Raw(`SELECT COUNT(*) AS edges, COUNT(DISTINCT addon_id) AS with_deps, + COUNT(DISTINCT dependency_id) AS dependents FROM addon_dependencies`).Scan(&deps) + d.Dependencies = map[string]any{"edges": deps.Edges, "addonsWithDependencies": deps.WithDeps, "addonsDependedOn": deps.Dependents} + + now := time.Now().Unix() + dayAgo := now - 86400 + httpSel := metrics.Selector{Service: metrics.MainService, Name: "http_request_duration_ms"} + traffic := map[string]any{} + if t, err := metrics.Totals(metrics.Selector{Service: httpSel.Service, Name: httpSel.Name, GroupBy: []string{"group"}}, dayAgo, now); err == nil { + var total int64 + for _, g := range t { + total += g.Count + if g.Labels["group"] == "v1" { + traffic["websiteRequests24h"] = g.Count + traffic["websiteP95Ms"] = g.P95 + traffic["websiteP50Ms"] = g.P50 + } + if g.Labels["group"] == "back" { + traffic["workerRequests24h"] = g.Count + } + } + traffic["requests24h"] = total + } + traffic["errors5xx24h"] = metrics.Sum(metrics.Selector{Service: httpSel.Service, Name: httpSel.Name, Match: map[string]string{"status": "5xx"}}, dayAgo, now) + humans, bots := metrics.Visitors() + traffic["visitorsToday"] = humans + traffic["botsToday"] = bots + d.Traffic = traffic + + sum := func(name string, match map[string]string) float64 { + return metrics.Sum(metrics.Selector{Service: metrics.MainService, Name: name, Match: match}, dayAgo, now) + } + d.Activity = map[string]any{ + "scraperCreated24h": sum("scraper_addons_total", map[string]string{"action": "created"}), + "scraperUpdated24h": sum("scraper_addons_total", map[string]string{"action": "updated"}), + "scraperCreateFailed24h": sum("scraper_addons_total", map[string]string{"action": "create_failed"}), + "deleted24h": sum("scraper_addons_deleted_total", map[string]string{"result": "deleted"}), + "indexed24h": sum("indexer_submissions_total", map[string]string{"result": "ok"}), + "claims24h": sum("indexer_claims_total", nil), + "timeoutsReset24h": sum("indexer_timeouts_reset_total", nil), + "dependencyReports24h": sum("dependency_reports_total", nil), + } + + var instances []models.ServiceInstance + initializers.DB.Find(&instances) + online := 0 + for i := range instances { + if metrics.Online(&instances[i], time.Now()) { + online++ + } + } + d.Services = map[string]any{"online": online, "total": len(instances)} + + d.TopSubscribed = loadMiniAddons("subscriber_count DESC", "") + d.Largest = loadMiniAddons("current_version_size DESC", "") + d.RecentlyAdded = loadMiniAddons("created_at DESC", "") + d.RecentlyIndexed = loadMiniAddons("last_indexed_at DESC", "last_indexed_at IS NOT NULL") + + d.Heavy = currentHeavy() + return d, nil +} + +func overview(c *gin.Context) { + data, at, err := cached("overview", 20*time.Second, buildOverview) + if err != nil { + fail(c, http.StatusInternalServerError, err.Error()) + return + } + c.JSON(http.StatusOK, gin.H{"asOf": at.Unix(), "data": data}) +} + +// ---- heavy statistics (background) ----------------------------------------- + +type tableInfo struct { + Name string `json:"name"` + Rows int64 `json:"rows"` + DeadRows int64 `json:"deadRows"` + SizeBytes int64 `json:"sizeBytes"` +} + +type dupHash struct { + Hash string `json:"hash"` + AddonCount int64 `json:"addonCount"` + FileCount int64 `json:"fileCount"` + SamplePath string `json:"samplePath"` + Whitelisted bool `json:"whitelisted"` +} + +type heavyStats struct { + AsOf int64 `json:"asOf"` + FilesLive int64 `json:"filesLive"` + FilesSoftDeleted int64 `json:"filesSoftDeleted"` + FilesApprox bool `json:"filesApprox"` + DistinctHashes int64 `json:"distinctHashes"` + DistinctAsOf int64 `json:"distinctAsOf"` + DBDriver string `json:"dbDriver"` + DBSizeBytes int64 `json:"dbSizeBytes"` + Tables []tableInfo `json:"tables"` + Duplicates []dupHash `json:"-"` + DuplicatesAsOf int64 `json:"duplicatesAsOf"` + Error string `json:"error,omitempty"` +} + +var ( + heavyMu sync.Mutex + heavy *heavyStats + refreshCh = make(chan bool, 1) +) + +func currentHeavy() *heavyStats { + heavyMu.Lock() + defer heavyMu.Unlock() + return heavy +} + +// requestHeavyRefresh triggers a refresh; full=true also recomputes the +// expensive duplicate statistics. +func requestHeavyRefresh(full bool) { + select { + case refreshCh <- full: + default: + } +} + +func heavyStatsLoop(ctx context.Context) { + lastFull := time.Time{} + run := func(full bool) { + defer func() { + if r := recover(); r != nil { + log.Printf("admin: heavy stats panicked: %v", r) + } + }() + full = full || time.Since(lastFull) > time.Hour + computeHeavy(full) + if full { + lastFull = time.Now() + } + } + // give the server a moment to start before the first scan + select { + case <-ctx.Done(): + return + case <-time.After(5 * time.Second): + } + run(true) + ticker := time.NewTicker(10 * time.Minute) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + run(false) + case full := <-refreshCh: + run(full) + } + } +} + +// withTimeout runs fn in a transaction with a statement timeout on Postgres. +func withTimeout(seconds int, fn func(tx *gorm.DB) error) error { + return initializers.DB.Transaction(func(tx *gorm.DB) error { + if isPostgres() { + if err := tx.Exec("SET LOCAL statement_timeout = " + strconv.Itoa(seconds*1000)).Error; err != nil { + return err + } + } + return fn(tx) + }) +} + +var ourTables = []string{"addons", "addon_files", "addon_dependencies", "whitelisted_hashes", + "metric_series", "metric_points", "service_instances", "service_events", "daily_top_items"} + +func computeHeavy(full bool) { + started := time.Now() + h := &heavyStats{AsOf: time.Now().Unix(), DBDriver: initializers.DB.Dialector.Name(), Tables: []tableInfo{}} + prev := currentHeavy() + if prev != nil { + h.DistinctHashes, h.DistinctAsOf = prev.DistinctHashes, prev.DistinctAsOf + h.Duplicates, h.DuplicatesAsOf = prev.Duplicates, prev.DuplicatesAsOf + } + + var errs []string + note := func(what string, err error) { + if err != nil { + errs = append(errs, what+": "+err.Error()) + } + } + + if isPostgres() { + note("tables", initializers.DB.Raw(`SELECT relname AS name, n_live_tup AS rows, n_dead_tup AS dead_rows, + pg_total_relation_size(relid) AS size_bytes FROM pg_stat_user_tables ORDER BY size_bytes DESC`).Scan(&h.Tables).Error) + note("db size", initializers.DB.Raw(`SELECT pg_database_size(current_database())`).Scan(&h.DBSizeBytes).Error) + note("soft-deleted files", withTimeout(60, func(tx *gorm.DB) error { + return tx.Raw(`SELECT COUNT(*) FROM addon_files WHERE deleted_at IS NOT NULL`).Scan(&h.FilesSoftDeleted).Error + })) + for _, t := range h.Tables { + if t.Name == "addon_files" { + h.FilesLive = max(0, t.Rows-h.FilesSoftDeleted) + h.FilesApprox = true + } + } + } else { + var pageCount, pageSize int64 + initializers.DB.Raw("PRAGMA page_count").Scan(&pageCount) + initializers.DB.Raw("PRAGMA page_size").Scan(&pageSize) + h.DBSizeBytes = pageCount * pageSize + if st, err := os.Stat(sqlitePath() + "-wal"); err == nil { + h.DBSizeBytes += st.Size() + } + for _, name := range ourTables { + var n int64 + if err := initializers.DB.Table(name).Count(&n).Error; err == nil { + h.Tables = append(h.Tables, tableInfo{Name: name, Rows: n}) + } + } + note("files", initializers.DB.Raw(`SELECT + COALESCE(SUM(CASE WHEN deleted_at IS NULL THEN 1 ELSE 0 END), 0), + COALESCE(SUM(CASE WHEN deleted_at IS NOT NULL THEN 1 ELSE 0 END), 0) FROM addon_files`).Row(). + Scan(&h.FilesLive, &h.FilesSoftDeleted)) + } + + if full { + var distinct int64 + err := withTimeout(120, func(tx *gorm.DB) error { + return tx.Raw(`SELECT COUNT(DISTINCT hash) FROM addon_files WHERE deleted_at IS NULL`).Scan(&distinct).Error + }) + note("distinct hashes", err) + if err == nil { + h.DistinctHashes, h.DistinctAsOf = distinct, time.Now().Unix() + } + + var dups []dupHash + err = withTimeout(120, func(tx *gorm.DB) error { + return tx.Raw(`SELECT hash, COUNT(DISTINCT addon_id) AS addon_count, COUNT(*) AS file_count + FROM addon_files WHERE deleted_at IS NULL + GROUP BY hash HAVING COUNT(DISTINCT addon_id) > 1 + ORDER BY addon_count DESC, file_count DESC LIMIT 200`).Scan(&dups).Error + }) + note("duplicates", err) + if err == nil { + h.Duplicates, h.DuplicatesAsOf = dups, time.Now().Unix() + if len(dups) > 0 { + hashes := make([]string, len(dups)) + for i, d := range dups { + hashes[i] = d.Hash + } + var samples []struct { + Hash string + Path string + } + initializers.DB.Raw(`SELECT hash, MIN(path) AS path FROM addon_files + WHERE deleted_at IS NULL AND hash IN ? GROUP BY hash`, hashes).Scan(&samples) + byHash := map[string]string{} + for _, s := range samples { + byHash[s.Hash] = s.Path + } + for i := range h.Duplicates { + h.Duplicates[i].SamplePath = byHash[h.Duplicates[i].Hash] + } + } + } + } + + if len(errs) > 0 { + h.Error = errs[0] + for _, e := range errs { + log.Printf("admin: heavy stats: %s", e) + } + } + heavyMu.Lock() + heavy = h + heavyMu.Unlock() + invalidate("overview") + if time.Since(started) > 10*time.Second { + log.Printf("admin: heavy stats took %s", time.Since(started).Round(time.Millisecond)) + } +} + +func sqlitePath() string { + if initializers.SQLitePath != "" { + return initializers.SQLitePath + } + return "crawler.db" +} + +// duplicateHashes lists the hashes shared by the most addons. +func duplicateHashes(c *gin.Context) { + h := currentHeavy() + if h == nil { + c.JSON(http.StatusOK, gin.H{"asOf": 0, "items": []dupHash{}, "pending": true}) + return + } + items := make([]dupHash, len(h.Duplicates)) + copy(items, h.Duplicates) + if len(items) > 0 { + hashes := make([]string, len(items)) + for i, d := range items { + hashes[i] = d.Hash + } + var wl []string + initializers.DB.Model(&models.WhitelistedHash{}).Where("hash IN ?", hashes).Pluck("hash", &wl) + set := map[string]bool{} + for _, w := range wl { + set[w] = true + } + for i := range items { + items[i].Whitelisted = set[items[i].Hash] + } + } + if c.Query("hideWhitelisted") == "1" { + kept := items[:0] + for _, it := range items { + if !it.Whitelisted { + kept = append(kept, it) + } + } + items = kept + } + c.JSON(http.StatusOK, gin.H{"asOf": h.DuplicatesAsOf, "items": items}) +} diff --git a/admin/routes.go b/admin/routes.go new file mode 100644 index 0000000..22b763c --- /dev/null +++ b/admin/routes.go @@ -0,0 +1,57 @@ +package admin + +import ( + "context" + + "gitea.tbdevent.eu/TBD/reforger_crawler_main/auth" + "github.com/gin-gonic/gin" +) + +// Register mounts the admin JSON API under /admin/api and the legacy +// header-authenticated /admin/addWhitelistedHash endpoint. +func Register(r *gin.Engine) { + r.POST("/admin/addWhitelistedHash", auth.RequireAdminHeader, legacyAddWhitelistedHash) + + api := r.Group("/admin/api") + api.POST("/auth/login", login) + api.POST("/auth/logout", logout) + + p := api.Group("", auth.RequireAdmin) + p.GET("/auth/me", me) + + p.GET("/overview", overview) + p.GET("/overview/duplicates", duplicateHashes) + + p.GET("/queue", queue) + p.POST("/queue/reset-stuck", resetStuck) + + p.GET("/addons", listAddons) + p.GET("/addons/:id", addonDetail) + p.GET("/addons/:id/files", addonFiles) + p.POST("/addons/:id/:action", addonAction) + p.DELETE("/addons/:id", deleteAddon) + p.DELETE("/addons/:id/purge", purgeAddon) + p.POST("/bulk/addons", bulkAddons) + + p.GET("/whitelist", listWhitelist) + p.POST("/whitelist", addWhitelist) + p.DELETE("/whitelist/:id", deleteWhitelist) + p.GET("/hashes/:hash", hashUsage) + + p.GET("/metrics/catalog", metricsCatalog) + p.GET("/metrics/query", metricsQuery) + p.GET("/traffic", traffic) + p.GET("/top", topItems) + + p.GET("/services", listServices) + p.GET("/services/events", listEvents) + p.DELETE("/services/:id", deleteService) + + p.GET("/system", system) + p.POST("/system/maintenance", maintenance) +} + +// StartBackground runs the periodic heavy statistics job. +func StartBackground(ctx context.Context) { + go heavyStatsLoop(ctx) +} diff --git a/admin/system.go b/admin/system.go new file mode 100644 index 0000000..e373125 --- /dev/null +++ b/admin/system.go @@ -0,0 +1,244 @@ +package admin + +import ( + "encoding/json" + "net/http" + "os" + "runtime" + "runtime/debug" + "strconv" + "strings" + "time" + + "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" + "github.com/gin-gonic/gin" +) + +// BuildCommit is set by main from -ldflags. +var BuildCommit = "" + +type serviceRow struct { + models.ServiceInstance + Online bool `json:"online"` + UptimeSec int64 `json:"uptimeSec"` + MetaMap map[string]any `json:"metaData,omitempty"` +} + +func listServices(c *gin.Context) { + var rows []models.ServiceInstance + if err := initializers.DB.Order("service ASC, instance ASC").Find(&rows).Error; err != nil { + fail(c, http.StatusInternalServerError, err.Error()) + return + } + now := time.Now() + out := make([]serviceRow, 0, len(rows)) + for _, r := range rows { + s := serviceRow{ServiceInstance: r, Online: metrics.Online(&r, now)} + if r.StartedAt > 0 && s.Online { + s.UptimeSec = now.Unix() - r.StartedAt + } + if r.Meta != "" { + _ = json.Unmarshal([]byte(r.Meta), &s.MetaMap) + } + out = append(out, s) + } + c.JSON(http.StatusOK, gin.H{"items": out}) +} + +func deleteService(c *gin.Context) { + id, err := strconv.ParseUint(c.Param("id"), 10, 64) + if err != nil { + fail(c, http.StatusBadRequest, "invalid id") + return + } + var row models.ServiceInstance + if err := initializers.DB.First(&row, id).Error; err != nil { + fail(c, http.StatusNotFound, "not found") + return + } + initializers.DB.Delete(&row) + audit(c, "service_forget", map[string]any{"service": row.Service, "instance": row.Instance}) + c.JSON(http.StatusOK, gin.H{"deleted": id}) +} + +type eventRow struct { + models.ServiceEvent + FieldsMap map[string]any `json:"fieldsData,omitempty"` +} + +func listEvents(c *gin.Context) { + limit := queryInt(c, "limit", 100, 1, 500) + q := initializers.DB.Model(&models.ServiceEvent{}) + if v := c.Query("service"); v != "" { + q = q.Where("service = ?", v) + } + if v := c.Query("instance"); v != "" { + q = q.Where("instance = ?", v) + } + if v := c.Query("level"); v != "" { + levels := strings.Split(v, ",") + q = q.Where("level IN ?", levels) + } + if c.Query("audit") == "1" { + q = q.Where("fields LIKE ?", `%"kind":"audit"%`) + } + if v := strings.TrimSpace(c.Query("q")); v != "" { + q = q.Where(`LOWER(message) LIKE ? ESCAPE '\'`, "%"+likeEscape(strings.ToLower(v))+"%") + } + if before := queryInt64(c, "before", 0); before > 0 { + q = q.Where("id < ?", before) + } + var rows []models.ServiceEvent + if err := q.Order("id DESC").Limit(limit).Find(&rows).Error; err != nil { + fail(c, http.StatusInternalServerError, err.Error()) + return + } + out := make([]eventRow, 0, len(rows)) + for _, r := range rows { + e := eventRow{ServiceEvent: r} + if r.Fields != "" { + _ = json.Unmarshal([]byte(r.Fields), &e.FieldsMap) + } + out = append(out, e) + } + var next uint + if len(rows) == limit { + next = rows[len(rows)-1].ID + } + c.JSON(http.StatusOK, gin.H{"items": out, "next": next}) +} + +func buildInfo() gin.H { + info := gin.H{"commit": BuildCommit} + if bi, ok := debug.ReadBuildInfo(); ok { + info["module"] = bi.Main.Path + for _, s := range bi.Settings { + switch s.Key { + case "vcs.revision": + if BuildCommit == "" { + info["commit"] = s.Value + } + case "vcs.time": + info["time"] = s.Value + case "vcs.modified": + info["modified"] = s.Value == "true" + } + } + } + return info +} + +func system(c *gin.Context) { + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + host, _ := os.Hostname() + started := metrics.StartedAt() + + resp := gin.H{ + "go": gin.H{ + "version": runtime.Version(), "os": runtime.GOOS, "arch": runtime.GOARCH, + "cpus": runtime.NumCPU(), "maxProcs": runtime.GOMAXPROCS(0), "goroutines": runtime.NumGoroutine(), + }, + "process": gin.H{ + "pid": os.Getpid(), "hostname": host, "startedAt": started.Unix(), + "uptimeSec": int64(time.Since(started).Seconds()), + }, + "memory": gin.H{ + "heapAlloc": ms.HeapAlloc, "heapInuse": ms.HeapInuse, "sys": ms.Sys, + "numGC": ms.NumGC, "pauseTotalNs": ms.PauseTotalNs, "lastGC": int64(ms.LastGC / 1e9), + }, + "build": buildInfo(), + "metrics": metrics.Status(), + "config": gin.H{ + "dbDriver": initializers.DB.Dialector.Name(), + "trustedProxies": initializers.TrustedProxies, + "cookieSecure": initializers.AdminCookieSecure, + "sessionTtlHours": initializers.AdminSessionTTL.Hours(), + "metricsEnabled": initializers.MetricsEnabled, + "prometheusEndpoint": initializers.MetricsToken != "", + "metricsMinuteRetentionDays": initializers.MetricsMinuteRetentionDays, + "metricsHourRetentionDays": initializers.MetricsHourRetentionDays, + "eventsRetentionDays": initializers.EventsRetentionDays, + "topItemsRetentionDays": initializers.TopItemsRetentionDays, + "indexTimeoutMinutes": initializers.IndexTimeout.Minutes(), + "adminSecretStrong": len(initializers.AdminSecret) >= 16, + "secretKeySet": initializers.SecretKey != "", + "discordWebhook": initializers.DiscordWebhookURL != "", + "scraperWebhook": initializers.ScraperWebhookURL != "", + }, + } + if sqlDB, err := initializers.DB.DB(); err == nil { + st := sqlDB.Stats() + resp["dbPool"] = gin.H{ + "open": st.OpenConnections, "inUse": st.InUse, "idle": st.Idle, + "maxOpen": st.MaxOpenConnections, "waitCount": st.WaitCount, "waitMs": st.WaitDuration.Milliseconds(), + } + start := time.Now() + if err := sqlDB.Ping(); err == nil { + resp["dbPingMs"] = float64(time.Since(start).Microseconds()) / 1000 + } else { + resp["dbPingError"] = err.Error() + } + } + if h := currentHeavy(); h != nil { + resp["db"] = h + } + c.JSON(http.StatusOK, resp) +} + +func maintenance(c *gin.Context) { + var body struct { + Action string `json:"action"` + OlderDays int `json:"olderThanDays"` + } + if err := c.ShouldBindJSON(&body); err != nil { + fail(c, http.StatusBadRequest, "invalid JSON") + return + } + switch body.Action { + case "refresh-stats": + requestHeavyRefresh(true) + case "flush-metrics": + metrics.FlushAll() + case "rollup": + go metrics.RunMaintenance(time.Now()) + case "purge-deleted-files": + days := body.OlderDays + if days <= 0 { + days = 7 + } + go purgeDeletedFiles(days) + default: + fail(c, http.StatusBadRequest, "unknown action") + return + } + audit(c, "maintenance_"+strings.ReplaceAll(body.Action, "-", "_"), map[string]any{"olderThanDays": body.OlderDays}) + c.JSON(http.StatusAccepted, gin.H{"started": body.Action}) +} + +// purgeDeletedFiles hard-deletes file rows that were soft-deleted (replaced by +// a newer index) more than `days` ago, in batches. +func purgeDeletedFiles(days int) { + cutoff := time.Now().AddDate(0, 0, -days) + started := time.Now() + var total int64 + for { + res := initializers.DB.Exec(`DELETE FROM addon_files WHERE id IN ( + SELECT id FROM addon_files WHERE deleted_at IS NOT NULL AND deleted_at < ? LIMIT 5000)`, cutoff) + if res.Error != nil { + metrics.Event("error", "purge of deleted file rows failed", map[string]any{"kind": "maintenance", "error": res.Error.Error(), "deleted": total}) + return + } + total += res.RowsAffected + if res.RowsAffected == 0 { + break + } + time.Sleep(200 * time.Millisecond) + } + metrics.Event("info", "purged deleted file rows", map[string]any{ + "kind": "maintenance", "deleted": total, "olderThanDays": days, "tookSec": int64(time.Since(started).Seconds()), + }) + requestHeavyRefresh(false) +} diff --git a/admin/util.go b/admin/util.go new file mode 100644 index 0000000..277fcaf --- /dev/null +++ b/admin/util.go @@ -0,0 +1,136 @@ +package admin + +import ( + "strconv" + "strings" + "sync" + "time" + + "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics" + "github.com/gin-gonic/gin" +) + +// ---- TTL cache with request coalescing ------------------------------------- + +type cacheEntry struct { + val any + at time.Time +} + +type inflight struct { + wg sync.WaitGroup + val any + err error +} + +var ( + cacheMu sync.Mutex + cacheData = map[string]cacheEntry{} + cacheRun = map[string]*inflight{} +) + +// cached returns a cached value younger than ttl, or computes it once even if +// many requests ask at the same time. It also returns when the value was made. +func cached[T any](key string, ttl time.Duration, fn func() (T, error)) (T, time.Time, error) { + cacheMu.Lock() + if e, ok := cacheData[key]; ok && time.Since(e.at) < ttl { + cacheMu.Unlock() + return e.val.(T), e.at, nil + } + if call, ok := cacheRun[key]; ok { + cacheMu.Unlock() + call.wg.Wait() + var zero T + if call.err != nil { + return zero, time.Time{}, call.err + } + return call.val.(T), time.Now(), nil + } + call := &inflight{} + call.wg.Add(1) + cacheRun[key] = call + cacheMu.Unlock() + + val, err := fn() + call.val, call.err = val, err + now := time.Now() + cacheMu.Lock() + delete(cacheRun, key) + if err == nil { + cacheData[key] = cacheEntry{val: val, at: now} + } + cacheMu.Unlock() + call.wg.Done() + return val, now, err +} + +// invalidate drops cached values whose key starts with one of the prefixes. +func invalidate(prefixes ...string) { + cacheMu.Lock() + defer cacheMu.Unlock() + for k := range cacheData { + for _, p := range prefixes { + if strings.HasPrefix(k, p) { + delete(cacheData, k) + } + } + } +} + +// ---- request helpers ----------------------------------------------------- + +func fail(c *gin.Context, code int, msg string) { + c.AbortWithStatusJSON(code, gin.H{"error": msg}) +} + +func queryInt(c *gin.Context, name string, def, lo, hi int) int { + v, err := strconv.Atoi(c.Query(name)) + if err != nil { + return def + } + return max(lo, min(hi, v)) +} + +func queryInt64(c *gin.Context, name string, def int64) int64 { + v, err := strconv.ParseInt(c.Query(name), 10, 64) + if err != nil { + return def + } + return v +} + +// timeRange reads from/to (unix seconds) with a default window ending now. +func timeRange(c *gin.Context, def time.Duration) (int64, int64) { + now := time.Now().Unix() + to := queryInt64(c, "to", now) + if to > now { + to = now + } + from := queryInt64(c, "from", to-int64(def.Seconds())) + if from >= to { + from = to - int64(def.Seconds()) + } + return from, to +} + +func isPostgres() bool { + return initializers.DB.Dialector.Name() == "postgres" +} + +// likeEscape escapes LIKE wildcards in user input. +func likeEscape(s string) string { + return strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(s) +} + +// audit records an admin action as a metric and an event. +func audit(c *gin.Context, action string, fields map[string]any) { + if fields == nil { + fields = map[string]any{} + } + fields["kind"] = "audit" + fields["ip"] = c.ClientIP() + fields["auth"] = c.GetString("adminAuth") + metrics.AdminActions.Inc(action) + metrics.Event("info", "admin: "+action, fields) +} diff --git a/admin/whitelist.go b/admin/whitelist.go new file mode 100644 index 0000000..670cd76 --- /dev/null +++ b/admin/whitelist.go @@ -0,0 +1,227 @@ +package admin + +import ( + "net/http" + "regexp" + "strconv" + "strings" + "time" + + "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" + "github.com/gin-gonic/gin" + "gorm.io/gorm" +) + +var sha512Re = regexp.MustCompile(`^[0-9a-f]{128}$`) + +// legacyAddWhitelistedHash keeps the original form-based endpoint working. +func legacyAddWhitelistedHash(c *gin.Context) { + hash := c.PostForm("hash") + if hash == "" { + c.JSON(400, gin.H{"error": "Hash is required"}) + return + } + + ret := initializers.DB.Find(&models.WhitelistedHash{}, "hash = ?", hash) + if ret.RowsAffected > 0 { + c.JSON(400, gin.H{"error": "Hash already whitelisted"}) + return + } + + if err := initializers.DB.Create(&models.WhitelistedHash{Hash: hash}).Error; err != nil { + c.JSON(500, gin.H{"error": "Failed to add whitelisted hash"}) + return + } + audit(c, "whitelist_add", map[string]any{"added": 1, "legacy": true}) + invalidate("overview") + c.JSON(200, gin.H{"message": "Whitelisted hash added successfully"}) +} + +type whitelistRow struct { + ID uint `json:"id"` + Hash string `json:"hash"` + Note string `json:"note"` + CreatedAt time.Time `json:"createdAt"` + Files int64 `json:"files"` + Addons int64 `json:"addons"` + SamplePath string `json:"samplePath"` +} + +func listWhitelist(c *gin.Context) { + limit := queryInt(c, "limit", 50, 1, 500) + offset := queryInt(c, "offset", 0, 0, 1<<30) + q := initializers.DB.Model(&models.WhitelistedHash{}) + if s := strings.TrimSpace(strings.ToLower(c.Query("q"))); s != "" { + like := "%" + likeEscape(s) + "%" + q = q.Where(`hash LIKE ? ESCAPE '\' OR LOWER(note) LIKE ? ESCAPE '\'`, likeEscape(s)+"%", like) + } + var total int64 + q.Session(&gorm.Session{}).Count(&total) + + rows := []whitelistRow{} + if err := q.Select("id, hash, note, created_at").Order("id DESC").Limit(limit).Offset(offset).Scan(&rows).Error; err != nil { + fail(c, http.StatusInternalServerError, err.Error()) + return + } + if len(rows) > 0 { + hashes := make([]string, len(rows)) + for i, r := range rows { + hashes[i] = r.Hash + } + var usage []struct { + Hash string + Files int64 + Addons int64 + Sample string + } + initializers.DB.Raw(`SELECT hash, COUNT(*) AS files, COUNT(DISTINCT addon_id) AS addons, MIN(path) AS sample + FROM addon_files WHERE deleted_at IS NULL AND hash IN ? GROUP BY hash`, hashes).Scan(&usage) + byHash := map[string]int{} + for i, u := range usage { + byHash[u.Hash] = i + } + for i := range rows { + if j, ok := byHash[rows[i].Hash]; ok { + rows[i].Files, rows[i].Addons, rows[i].SamplePath = usage[j].Files, usage[j].Addons, usage[j].Sample + } + } + } + c.JSON(http.StatusOK, gin.H{"items": rows, "total": total, "limit": limit, "offset": offset}) +} + +type skipped struct { + Hash string `json:"hash"` + Reason string `json:"reason"` +} + +func addWhitelist(c *gin.Context) { + var body struct { + Hashes []string `json:"hashes"` + Text string `json:"text"` + Note string `json:"note"` + } + if err := c.ShouldBindJSON(&body); err != nil { + fail(c, http.StatusBadRequest, "invalid JSON") + return + } + candidates := append([]string{}, body.Hashes...) + candidates = append(candidates, strings.FieldsFunc(body.Text, func(r rune) bool { + return r == ',' || r == ';' || r == ' ' || r == '\n' || r == '\r' || r == '\t' + })...) + if len(candidates) == 0 { + fail(c, http.StatusBadRequest, "no hashes given") + return + } + if len(candidates) > 5000 { + fail(c, http.StatusBadRequest, "at most 5000 hashes per request") + return + } + note := body.Note + if len(note) > 256 { + note = note[:256] + } + + var skips []skipped + seen := map[string]bool{} + valid := []string{} + for _, raw := range candidates { + h := strings.ToLower(strings.TrimSpace(raw)) + switch { + case h == "": + continue + case !sha512Re.MatchString(h): + skips = append(skips, skipped{raw, "not a SHA-512 hex hash"}) + case seen[h]: + skips = append(skips, skipped{h, "duplicate in request"}) + default: + seen[h] = true + valid = append(valid, h) + } + } + + added := 0 + err := initializers.DB.Transaction(func(tx *gorm.DB) error { + existing := map[string]bool{} + for start := 0; start < len(valid); start += 500 { + end := min(start+500, len(valid)) + var found []string + if err := tx.Model(&models.WhitelistedHash{}).Where("hash IN ?", valid[start:end]).Pluck("hash", &found).Error; err != nil { + return err + } + for _, h := range found { + existing[h] = true + } + } + rows := []models.WhitelistedHash{} + for _, h := range valid { + if existing[h] { + skips = append(skips, skipped{h, "already whitelisted"}) + continue + } + rows = append(rows, models.WhitelistedHash{Hash: h, Note: note}) + } + if len(rows) == 0 { + return nil + } + added = len(rows) + return tx.CreateInBatches(&rows, 200).Error + }) + if err != nil { + fail(c, http.StatusInternalServerError, err.Error()) + return + } + if added > 0 { + audit(c, "whitelist_add", map[string]any{"added": added, "note": note}) + invalidate("overview") + } + if skips == nil { + skips = []skipped{} + } + c.JSON(http.StatusOK, gin.H{"added": added, "skipped": skips}) +} + +func deleteWhitelist(c *gin.Context) { + id, err := strconv.ParseUint(c.Param("id"), 10, 64) + if err != nil { + fail(c, http.StatusBadRequest, "invalid id") + return + } + var row models.WhitelistedHash + if err := initializers.DB.Unscoped().First(&row, id).Error; err != nil { + fail(c, http.StatusNotFound, "not found") + return + } + // remove every row of this hash, the table has no unique constraint + res := initializers.DB.Unscoped().Where("hash = ?", row.Hash).Delete(&models.WhitelistedHash{}) + if res.Error != nil { + fail(c, http.StatusInternalServerError, res.Error.Error()) + return + } + audit(c, "whitelist_remove", map[string]any{"hash": row.Hash}) + invalidate("overview") + c.JSON(http.StatusOK, gin.H{"deleted": res.RowsAffected}) +} + +// hashUsage lists the addons that contain a file with the given hash. +func hashUsage(c *gin.Context) { + hash := strings.ToLower(c.Param("hash")) + var total int64 + initializers.DB.Model(&models.AddonFile{}).Where("hash = ?", hash).Count(&total) + items := []struct { + AddonID string `json:"addonId"` + Name string `json:"name"` + Path string `json:"path"` + Version string `json:"version"` + }{} + err := initializers.DB.Raw(`SELECT f.addon_id, COALESCE(a.name, '') AS name, f.path, f.version + FROM addon_files f LEFT JOIN addons a ON a.id = f.addon_id + WHERE f.hash = ? AND f.deleted_at IS NULL ORDER BY a.subscriber_count DESC LIMIT 200`, hash).Scan(&items).Error + if err != nil { + fail(c, http.StatusInternalServerError, err.Error()) + return + } + var wl int64 + initializers.DB.Model(&models.WhitelistedHash{}).Where("hash = ?", hash).Count(&wl) + c.JSON(http.StatusOK, gin.H{"hash": hash, "whitelisted": wl > 0, "total": total, "items": items}) +} diff --git a/auth/auth_test.go b/auth/auth_test.go new file mode 100644 index 0000000..86554a2 --- /dev/null +++ b/auth/auth_test.go @@ -0,0 +1,80 @@ +package auth + +import ( + "strings" + "testing" + "time" + + "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" +) + +func TestSecretEqual(t *testing.T) { + if SecretEqual("", "") { + t.Fatal("empty configured secret must never match") + } + if SecretEqual("anything", "") { + t.Fatal("empty configured secret must never match") + } + if !SecretEqual("abc", "abc") { + t.Fatal("equal secrets must match") + } + if SecretEqual("abd", "abc") || SecretEqual("ab", "abc") { + t.Fatal("different secrets must not match") + } +} + +func TestSessionToken(t *testing.T) { + initializers.AdminSecret = "first-secret-value" + initializers.AdminSessionTTL = time.Hour + now := time.Now() + + tok, exp := IssueToken(now) + got, ok := VerifyToken(tok, now) + if !ok || got.Unix() != exp.Unix() { + t.Fatalf("fresh token should verify, ok=%v", ok) + } + + if _, ok := VerifyToken(tok, now.Add(2*time.Hour)); ok { + t.Fatal("expired token must not verify") + } + + parts := strings.Split(tok, ".") + parts[1] = "9999999999" + if _, ok := VerifyToken(strings.Join(parts, "."), now); ok { + t.Fatal("tampered token must not verify") + } + + if _, ok := VerifyToken("garbage", now); ok { + t.Fatal("garbage must not verify") + } + + initializers.AdminSecret = "rotated-secret-value" + if _, ok := VerifyToken(tok, now); ok { + t.Fatal("rotating the secret must invalidate sessions") + } + + initializers.AdminSecret = "" + if _, ok := VerifyToken(tok, now); ok { + t.Fatal("no secret configured must reject all sessions") + } +} + +func TestLoginLimiter(t *testing.T) { + l := NewLoginLimiter(3, time.Minute) + for range 3 { + if l.Blocked("1.2.3.4") { + t.Fatal("blocked too early") + } + l.Fail("1.2.3.4") + } + if !l.Blocked("1.2.3.4") { + t.Fatal("should be blocked after max failures") + } + if l.Blocked("5.6.7.8") { + t.Fatal("other IPs must not be affected") + } + l.Reset("1.2.3.4") + if l.Blocked("1.2.3.4") { + t.Fatal("reset should unblock") + } +} diff --git a/auth/middleware.go b/auth/middleware.go new file mode 100644 index 0000000..2529c5c --- /dev/null +++ b/auth/middleware.go @@ -0,0 +1,142 @@ +package auth + +import ( + "net/http" + "net/url" + "sync" + "time" + + "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" + "github.com/gin-gonic/gin" +) + +// CSRFHeader must be sent with every state-changing cookie-authenticated +// request. Cross-site forms cannot set custom headers. +const CSRFHeader = "X-Admin-CSRF" + +// RequireSecret guards the /back endpoints used by scrapers and indexers. +func RequireSecret(c *gin.Context) { + if initializers.SecretKey == "" { + c.AbortWithStatusJSON(http.StatusServiceUnavailable, gin.H{"error": "backend auth not configured"}) + return + } + if !SecretEqual(c.GetHeader("X-SECRET-KEY"), initializers.SecretKey) { + c.AbortWithStatusJSON(http.StatusForbidden, gin.H{"error": "Forbidden"}) + return + } +} + +// RequireAdminHeader is the legacy admin check: X-ADMIN-KEY only, 403 on failure. +func RequireAdminHeader(c *gin.Context) { + if !SecretEqual(c.GetHeader("X-ADMIN-KEY"), initializers.AdminSecret) { + c.AbortWithStatusJSON(http.StatusForbidden, gin.H{"error": "Forbidden"}) + return + } + c.Set("adminAuth", "header") +} + +// RequireAdmin accepts either the X-ADMIN-KEY header (scripts) or a valid +// session cookie (dashboard). Cookie requests that change state must also carry +// the CSRF header and, when present, a same-host Origin. +func RequireAdmin(c *gin.Context) { + if initializers.AdminSecret == "" { + c.AbortWithStatusJSON(http.StatusServiceUnavailable, gin.H{"error": "admin disabled: ADMIN_SECRET not set"}) + return + } + + if key := c.GetHeader("X-ADMIN-KEY"); key != "" { + if !SecretEqual(key, initializers.AdminSecret) { + c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "invalid admin key"}) + return + } + c.Set("adminAuth", "header") + return + } + + cookie, err := c.Cookie(CookieName) + if err != nil || cookie == "" { + c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "not authenticated"}) + return + } + exp, ok := VerifyToken(cookie, time.Now()) + if !ok { + ClearSessionCookie(c) + c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "session expired"}) + return + } + + if c.Request.Method != http.MethodGet && c.Request.Method != http.MethodHead { + if c.GetHeader(CSRFHeader) != "1" || !sameOrigin(c) { + c.AbortWithStatusJSON(http.StatusForbidden, gin.H{"error": "csrf check failed"}) + return + } + } + + c.Set("adminAuth", "cookie") + c.Set("adminSessionExp", exp) +} + +func sameOrigin(c *gin.Context) bool { + origin := c.GetHeader("Origin") + if origin == "" { + return true + } + u, err := url.Parse(origin) + if err != nil { + return false + } + return u.Host == c.Request.Host +} + +// LoginLimiter throttles failed logins per client IP. +type LoginLimiter struct { + mu sync.Mutex + failures map[string][]time.Time + max int + window time.Duration +} + +func NewLoginLimiter(max int, window time.Duration) *LoginLimiter { + return &LoginLimiter{failures: make(map[string][]time.Time), max: max, window: window} +} + +func (l *LoginLimiter) prune(ip string, now time.Time) []time.Time { + kept := l.failures[ip][:0] + for _, t := range l.failures[ip] { + if now.Sub(t) < l.window { + kept = append(kept, t) + } + } + if len(kept) == 0 { + delete(l.failures, ip) + return nil + } + l.failures[ip] = kept + return kept +} + +// Blocked reports whether the IP has used up its failed attempts. +func (l *LoginLimiter) Blocked(ip string) bool { + l.mu.Lock() + defer l.mu.Unlock() + return len(l.prune(ip, time.Now())) >= l.max +} + +func (l *LoginLimiter) Fail(ip string) { + l.mu.Lock() + defer l.mu.Unlock() + now := time.Now() + l.failures[ip] = append(l.prune(ip, now), now) + // keep the map bounded if someone sprays from many addresses + if len(l.failures) > 10000 { + for k := range l.failures { + l.prune(k, now) + } + } +} + +func (l *LoginLimiter) Reset(ip string) { + l.mu.Lock() + defer l.mu.Unlock() + delete(l.failures, ip) +} diff --git a/auth/secret.go b/auth/secret.go new file mode 100644 index 0000000..28a79eb --- /dev/null +++ b/auth/secret.go @@ -0,0 +1,19 @@ +package auth + +import ( + "crypto/sha256" + "crypto/subtle" +) + +// SecretEqual compares a client-supplied secret with the configured one in +// constant time. An empty configured secret never matches, so an unset secret +// disables access instead of letting requests without the header through. +func SecretEqual(given, want string) bool { + if want == "" { + return false + } + // hashing first keeps the comparison length-independent + g := sha256.Sum256([]byte(given)) + w := sha256.Sum256([]byte(want)) + return subtle.ConstantTimeCompare(g[:], w[:]) == 1 +} diff --git a/auth/session.go b/auth/session.go new file mode 100644 index 0000000..d5d70c0 --- /dev/null +++ b/auth/session.go @@ -0,0 +1,108 @@ +package auth + +import ( + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" + "github.com/gin-gonic/gin" +) + +// CookieName is the admin session cookie. It is scoped to /admin so it is never +// sent to the public API. +const CookieName = "rc_admin" + +var b64 = base64.RawURLEncoding + +// The signing key is derived from the admin secret, so rotating ADMIN_SECRET +// invalidates every issued session. +func sessionKey() []byte { + m := hmac.New(sha256.New, []byte(initializers.AdminSecret)) + m.Write([]byte("rc-admin-session/v1")) + return m.Sum(nil) +} + +func sign(payload string) string { + m := hmac.New(sha256.New, sessionKey()) + m.Write([]byte(payload)) + return b64.EncodeToString(m.Sum(nil)) +} + +// IssueToken creates a stateless session token: v1... +func IssueToken(now time.Time) (string, time.Time) { + exp := now.Add(initializers.AdminSessionTTL) + nonce := make([]byte, 16) + _, _ = rand.Read(nonce) + payload := fmt.Sprintf("v1.%d.%s", exp.Unix(), b64.EncodeToString(nonce)) + return payload + "." + sign(payload), exp +} + +// VerifyToken checks the signature and expiry and returns the expiry time. +func VerifyToken(token string, now time.Time) (time.Time, bool) { + if initializers.AdminSecret == "" { + return time.Time{}, false + } + idx := strings.LastIndexByte(token, '.') + if idx <= 0 { + return time.Time{}, false + } + payload, mac := token[:idx], token[idx+1:] + if !hmac.Equal([]byte(mac), []byte(sign(payload))) { + return time.Time{}, false + } + parts := strings.Split(payload, ".") + if len(parts) != 3 || parts[0] != "v1" { + return time.Time{}, false + } + expUnix, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return time.Time{}, false + } + exp := time.Unix(expUnix, 0) + if !now.Before(exp) { + return time.Time{}, false + } + return exp, true +} + +func cookieSecure(c *gin.Context) bool { + switch initializers.AdminCookieSecure { + case "true", "1", "yes": + return true + case "false", "0", "no": + return false + } + return c.Request.TLS != nil || strings.EqualFold(c.GetHeader("X-Forwarded-Proto"), "https") +} + +func SetSessionCookie(c *gin.Context, token string, exp time.Time) { + http.SetCookie(c.Writer, &http.Cookie{ + Name: CookieName, + Value: token, + Path: "/admin", + Expires: exp, + MaxAge: int(time.Until(exp).Seconds()), + HttpOnly: true, + Secure: cookieSecure(c), + SameSite: http.SameSiteStrictMode, + }) +} + +func ClearSessionCookie(c *gin.Context) { + http.SetCookie(c.Writer, &http.Cookie{ + Name: CookieName, + Value: "", + Path: "/admin", + MaxAge: -1, + HttpOnly: true, + Secure: cookieSecure(c), + SameSite: http.SameSiteStrictMode, + }) +} diff --git a/controllers/adminController.go b/controllers/adminController.go deleted file mode 100644 index 7721855..0000000 --- a/controllers/adminController.go +++ /dev/null @@ -1,30 +0,0 @@ -package controllers - -import ( - "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" - "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" - "github.com/gin-gonic/gin" -) - -func AddWhitelistedHash(c *gin.Context) { - hash := c.PostForm("hash") - if hash == "" { - c.JSON(400, gin.H{"error": "Hash is required"}) - return - } - - hashAddon := models.WhitelistedHash{Hash: hash} - - ret := initializers.DB.Find(&models.WhitelistedHash{}, "hash = ?", hash) - if ret.RowsAffected > 0 { - c.JSON(400, gin.H{"error": "Hash already whitelisted"}) - return - } - - if err := initializers.DB.Create(&hashAddon).Error; err != nil { - c.JSON(500, gin.H{"error": "Failed to add whitelisted hash"}) - return - } - - c.JSON(200, gin.H{"message": "Whitelisted hash added successfully"}) -} diff --git a/controllers/authController.go b/controllers/authController.go index 460e51e..27c1a77 100644 --- a/controllers/authController.go +++ b/controllers/authController.go @@ -1,26 +1,16 @@ package controllers import ( - "net/http" - - "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/auth" "github.com/gin-gonic/gin" ) +// CheckAllowed guards the scraper/indexer endpoints (X-SECRET-KEY). func CheckAllowed(c *gin.Context) { - secret := c.GetHeader("X-SECRET-KEY") - if secret != initializers.SecretKey { - c.JSON(http.StatusForbidden, gin.H{"error": "Forbidden"}) - c.Abort() - return - } + auth.RequireSecret(c) } +// CheckAdmin is the legacy header-only admin check (X-ADMIN-KEY). func CheckAdmin(c *gin.Context) { - adminSecret := c.GetHeader("X-ADMIN-KEY") - if adminSecret != initializers.AdminSecret { - c.JSON(http.StatusForbidden, gin.H{"error": "Forbidden"}) - c.Abort() - return - } + auth.RequireAdminHeader(c) } diff --git a/controllers/dependencyController.go b/controllers/dependencyController.go index 6fc9223..4e24085 100644 --- a/controllers/dependencyController.go +++ b/controllers/dependencyController.go @@ -4,6 +4,7 @@ import ( "strconv" "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics" "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" "github.com/gin-gonic/gin" "gorm.io/gorm" @@ -53,6 +54,8 @@ func SetAddonDependencies(c *gin.Context) { return } + metrics.DependencyReports.Inc() + metrics.DependencyEdges.Add(float64(len(edges))) c.JSON(200, gin.H{"status": "success", "stored": len(edges)}) } @@ -69,7 +72,7 @@ func GetDependents(c *gin.Context) { limit := parseLimit(c.Query("limit")) offset := parseOffset(c.Query("offset")) - dependents, total, err := queryRelatedAddons(addonID, "ad.addon_id", "ad.dependency_id", limit, offset) + dependents, total, err := QueryRelatedAddons(addonID, "ad.addon_id", "ad.dependency_id", limit, offset) if err != nil { c.JSON(500, gin.H{"error": err.Error()}) return @@ -97,7 +100,7 @@ func GetDependencies(c *gin.Context) { limit := parseLimit(c.Query("limit")) offset := parseOffset(c.Query("offset")) - dependencies, total, err := queryRelatedAddons(addonID, "ad.dependency_id", "ad.addon_id", limit, offset) + dependencies, total, err := QueryRelatedAddons(addonID, "ad.dependency_id", "ad.addon_id", limit, offset) if err != nil { c.JSON(500, gin.H{"error": err.Error()}) return @@ -112,7 +115,7 @@ func GetDependencies(c *gin.Context) { }) } -// queryRelatedAddons returns one page of addons linked to addonID through the +// QueryRelatedAddons returns one page of addons linked to addonID through the // addon_dependencies edge table, plus the total number of matches. // // joinCol is the edge column joined to addons.id; filterCol is the edge column @@ -120,7 +123,7 @@ func GetDependencies(c *gin.Context) { // // The addons.id tiebreaker in the ORDER BY is required: subscriber_count alone // is not a unique ordering, so without it paging could repeat or skip rows. -func queryRelatedAddons(addonID, joinCol, filterCol string, limit, offset int) ([]models.AddonSearchResult, int, error) { +func QueryRelatedAddons(addonID, joinCol, filterCol string, limit, offset int) ([]models.AddonSearchResult, int, error) { base := func() *gorm.DB { return initializers.DB.Model(&models.Addon{}). Joins("JOIN addon_dependencies ad ON "+joinCol+" = addons.id"). diff --git a/controllers/feController.go b/controllers/feController.go index 80ede00..b26fd5d 100644 --- a/controllers/feController.go +++ b/controllers/feController.go @@ -6,6 +6,7 @@ import ( "strconv" "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics" "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" "github.com/gin-gonic/gin" ) @@ -114,6 +115,7 @@ func GetDuplicates(c *gin.Context) { for _, file := range duplicateFiles { duplicatesByAddonID[file.AddonID] = append(duplicatesByAddonID[file.AddonID], file) } + metrics.DuplicatesFound.Observe(float64(len(duplicatesByAddonID))) // Fetch addon information for each addon that has duplicates var addonIDs []string diff --git a/controllers/indexerController.go b/controllers/indexerController.go index b7cbe81..02b8243 100644 --- a/controllers/indexerController.go +++ b/controllers/indexerController.go @@ -9,6 +9,7 @@ import ( "time" "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics" "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" "github.com/gin-gonic/gin" "gorm.io/gorm" @@ -16,7 +17,7 @@ import ( func GetNextToBeIndexed(c *gin.Context) { - checkIndexingTimeout() + metrics.IndexerTimeoutsReset.Add(float64(ResetTimedOutIndexing(initializers.IndexTimeout))) maxFileSize := c.Query("maxFileSize") maxSize := 10_000_000_000 @@ -26,25 +27,31 @@ func GetNextToBeIndexed(c *gin.Context) { } } + queue := "priority" addon, err := claimNextAddon(maxSize, true) if err != nil { + metrics.IndexerClaims.Inc("error") c.JSON(500, gin.H{"error": err.Error()}) return } if addon == nil { + queue = "normal" addon, err = claimNextAddon(maxSize, false) if err != nil { + metrics.IndexerClaims.Inc("error") c.JSON(500, gin.H{"error": err.Error()}) return } } if addon != nil { + metrics.IndexerClaims.Inc(queue) c.JSON(200, gin.H{"guid": addon.ID, "currentVersion": addon.CurrentVersionNumber}) return } // nothing to index + metrics.IndexerClaims.Inc("empty") c.JSON(200, gin.H{"guid": "", "currentVersion": ""}) } @@ -96,6 +103,7 @@ func SaveIndexingResult(c *gin.Context) { err := c.ShouldBindJSON(&result) if err != nil { + metrics.IndexerSubmissions.Inc("invalid") c.JSON(400, gin.H{"error": "Invalid JSON"}) return } @@ -103,6 +111,7 @@ func SaveIndexingResult(c *gin.Context) { var addon models.Addon ret := initializers.DB.Where("id = ?", result.GUID).First(&addon) if ret.Error != nil { + metrics.IndexerSubmissions.Inc("not_found") c.JSON(404, gin.H{"error": "Addon not found"}) return } @@ -110,25 +119,42 @@ func SaveIndexingResult(c *gin.Context) { // get all files for this addon and delete them - old version ret = initializers.DB.Where("addon_id = ?", addon.ID).Delete(&models.AddonFile{}) if ret.Error != nil { + metrics.IndexerSubmissions.Inc("error") c.JSON(500, gin.H{"error": ret.Error.Error()}) return } ret = initializers.DB.CreateInBatches(&result.Files, 100) if ret.Error != nil { + metrics.IndexerSubmissions.Inc("error") c.JSON(500, gin.H{"error": ret.Error.Error()}) return } + now := time.Now() + wasClaimed := addon.IsBeingIndexed && !addon.IndexStartTime.IsZero() addon.CurrentVersionNumber = result.CurrentVersion addon.IsBeingIndexed = false addon.ToBeIndexed = false + addon.LastIndexedAt = &now ret = initializers.DB.Save(&addon) if ret.Error != nil { + metrics.IndexerSubmissions.Inc("error") c.JSON(500, gin.H{"error": ret.Error.Error()}) return } + outcome := "ok" + if !result.Success { + outcome = "unsuccessful" + } + metrics.IndexerSubmissions.Inc(outcome) + if wasClaimed { + metrics.IndexerJobDuration.Observe(now.Sub(addon.IndexStartTime).Seconds()) + } + metrics.IndexerFilesPerSubmission.Observe(float64(len(result.Files))) + metrics.IndexerBytesIndexed.Add(float64(addon.CurrentVersionSize)) + if initializers.DiscordWebhookURL != "" { text := "Indexing of addon " + addon.Name + " (" + addon.ID + ") was successful." colour := 2228479 @@ -179,7 +205,7 @@ func SaveIndexingResult(c *gin.Context) { Embeds: []models.CustomEmbed{myEmbed}, } - err := SendCustomWebhook(initializers.DiscordWebhookURL, myHook) + err := sendWebhook("index", initializers.DiscordWebhookURL, myHook) if err != nil { fmt.Println("Error sending webhook:", err) } @@ -198,40 +224,60 @@ func DeleteAddon(c *gin.Context) { var addon models.Addon ret := initializers.DB.Where("id = ?", guid).First(&addon) if ret.Error != nil { + metrics.ScraperDeletes.Inc("not_found") c.JSON(404, gin.H{"error": "Addon not found"}) return } - // Delete associated files - ret = initializers.DB.Where("addon_id = ?", addon.ID).Delete(&models.AddonFile{}) - if ret.Error != nil { - c.JSON(500, gin.H{"error": ret.Error.Error()}) + err := initializers.DB.Transaction(func(tx *gorm.DB) error { + return DeleteAddonCascade(tx, &addon) + }) + if err != nil { + metrics.ScraperDeletes.Inc("error") + c.JSON(500, gin.H{"error": err.Error()}) return } + metrics.ScraperDeletes.Inc("deleted") + c.JSON(200, gin.H{"status": "addon deleted"}) +} + +// DeleteAddonCascade soft-deletes an addon and its files and removes its +// outgoing dependency edges. +func DeleteAddonCascade(tx *gorm.DB, addon *models.Addon) error { + // Delete associated files + if err := tx.Where("addon_id = ?", addon.ID).Delete(&models.AddonFile{}).Error; err != nil { + return err + } + // Delete outgoing dependency edges; incoming edges stay - other addons // still declare this dependency even if the addon row is gone - ret = initializers.DB.Where("addon_id = ?", addon.ID).Delete(&models.AddonDependency{}) - if ret.Error != nil { - c.JSON(500, gin.H{"error": ret.Error.Error()}) - return + if err := tx.Where("addon_id = ?", addon.ID).Delete(&models.AddonDependency{}).Error; err != nil { + return err } // Delete the addon - ret = initializers.DB.Delete(&addon) - if ret.Error != nil { - c.JSON(500, gin.H{"error": ret.Error.Error()}) - return - } - - c.JSON(200, gin.H{"status": "addon deleted"}) + return tx.Delete(addon).Error } -func checkIndexingTimeout() { - initializers.DB.Model(&models.Addon{}). +// ResetTimedOutIndexing releases claims older than olderThan so another +// indexer can pick the addon up. Returns the number of released claims. +func ResetTimedOutIndexing(olderThan time.Duration) int64 { + return initializers.DB.Model(&models.Addon{}). Where("is_being_indexed = ?", true). - Where("index_start_time < ?", time.Now().Add(-30*time.Minute)). - Update("is_being_indexed", false) + Where("index_start_time < ?", time.Now().Add(-olderThan)). + Update("is_being_indexed", false).RowsAffected +} + +// sendWebhook sends a Discord webhook and records the outcome. +func sendWebhook(kind, webhookURL string, hook models.CustomHook) error { + err := SendCustomWebhook(webhookURL, hook) + result := "ok" + if err != nil { + result = "error" + } + metrics.DiscordWebhooks.Inc(kind, result) + return err } func SendCustomWebhook(webhookURL string, hook models.CustomHook) error { diff --git a/controllers/scraperController.go b/controllers/scraperController.go index b9efbed..2e946d1 100644 --- a/controllers/scraperController.go +++ b/controllers/scraperController.go @@ -1,31 +1,26 @@ package controllers import ( - "bytes" "fmt" - "io" "strconv" + "sync" "time" "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics" "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" "github.com/gin-gonic/gin" ) +var webhookMu sync.Mutex var lastWebhookTime time.Time var nbrOfUnsentWebhooks int func CreateAddon(c *gin.Context) { var addonR models.Addon - body, err := io.ReadAll(c.Request.Body) - if err != nil { - c.JSON(400, gin.H{"error": "Failed to read request body"}) - return - } - fmt.Println(string(body)) - c.Request.Body = io.NopCloser(bytes.NewBuffer(body)) - err = c.ShouldBindJSON(&addonR) + err := c.ShouldBindJSON(&addonR) if err != nil { + metrics.ScraperAddons.Inc("invalid") c.JSON(400, gin.H{"error": "Invalid JSON"}) return } @@ -37,17 +32,26 @@ func CreateAddon(c *gin.Context) { result := initializers.DB.Where(&addon).First(&addon) if result.Error != nil { // addon does not exist, create it - initializers.DB.Create(&addonR) + if err := initializers.DB.Create(&addonR).Error; err != nil { + // usually a soft-deleted addon with the same ID; it can be restored from the admin dashboard + metrics.ScraperAddons.Inc("create_failed") + fmt.Println("Failed to create addon", addonR.ID, err) + } else { + metrics.ScraperAddons.Inc("created") + } // send webhook about creation + webhookMu.Lock() if time.Since(lastWebhookTime) < time.Second*10 { nbrOfUnsentWebhooks++ + webhookMu.Unlock() return } lastWebhookTime = time.Now() oldNbr := nbrOfUnsentWebhooks nbrOfUnsentWebhooks = 0 + webhookMu.Unlock() go func() { webhookURL := initializers.ScraperWebhookURL @@ -82,7 +86,7 @@ func CreateAddon(c *gin.Context) { Embeds: []models.CustomEmbed{myEmbed}, } - err := SendCustomWebhook(webhookURL, myHook) + err := sendWebhook("scraper", webhookURL, myHook) if err != nil { fmt.Println("Error sending webhook:", err) } @@ -105,6 +109,7 @@ func CreateAddon(c *gin.Context) { addon.Blocked = addonR.Blocked addon.ToBeIndexed = true initializers.DB.Save(&addon) + metrics.ScraperAddons.Inc("updated") // send webhook about update go func() { @@ -140,12 +145,14 @@ func CreateAddon(c *gin.Context) { Embeds: []models.CustomEmbed{myEmbed}, } - err := SendCustomWebhook(webhookURL, myHook) + err := sendWebhook("scraper", webhookURL, myHook) if err != nil { fmt.Println("Error sending webhook:", err) } } }() + } else { + metrics.ScraperAddons.Inc("unchanged") } } c.JSON(200, gin.H{"status": "success"}) diff --git a/controllers/telemetryController.go b/controllers/telemetryController.go new file mode 100644 index 0000000..58d14f8 --- /dev/null +++ b/controllers/telemetryController.go @@ -0,0 +1,53 @@ +package controllers + +import ( + "net/http" + "time" + + "gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" + "github.com/gin-gonic/gin" +) + +const ( + maxTelemetryBody = 1 << 20 + maxTelemetryMetrics = 1000 + maxTelemetryEvents = 200 +) + +// SubmitTelemetry receives metrics, events and a heartbeat from a service +// (indexer, scraper, ...). See models.TelemetryBatch for the payload. +func SubmitTelemetry(c *gin.Context) { + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxTelemetryBody) + + var batch models.TelemetryBatch + if err := c.ShouldBindJSON(&batch); err != nil { + c.JSON(400, gin.H{"error": "Invalid JSON"}) + return + } + if !metrics.ValidName(batch.Service) || batch.Service == metrics.MainService { + c.JSON(400, gin.H{"error": "service must match ^[a-z_][a-z0-9_]{0,63}$ and not be \"main\""}) + return + } + if batch.Instance == "" || len(batch.Instance) > 128 { + c.JSON(400, gin.H{"error": "instance is required (max 128 characters)"}) + return + } + if len(batch.Metrics) > maxTelemetryMetrics || len(batch.Events) > maxTelemetryEvents { + c.JSON(413, gin.H{"error": "too many metrics or events in one batch"}) + return + } + + now := time.Now() + if err := metrics.Heartbeat(&batch, c.ClientIP(), now); err != nil { + c.JSON(500, gin.H{"error": "failed to store heartbeat"}) + return + } + result := metrics.Ingest(batch.Service, batch.Instance, batch.Metrics, now) + if err := metrics.RecordEvents(batch.Service, batch.Instance, batch.Events, now); err != nil { + c.JSON(500, gin.H{"error": "failed to store events"}) + return + } + + c.JSON(200, result) +} diff --git a/docs/admin-dashboard.md b/docs/admin-dashboard.md new file mode 100644 index 0000000..10ab27d --- /dev/null +++ b/docs/admin-dashboard.md @@ -0,0 +1,124 @@ +# Admin dashboard, metrics and telemetry + +The API binary serves an admin dashboard at **`/admin/ui/`** (`/admin` redirects there). +It is a React app in `web/`, built into `web/dist` and embedded with `go:embed`. + +## Access + +- Sign in with `ADMIN_SECRET`. The server sets an HttpOnly, `SameSite=Strict` + session cookie (`ADMIN_SESSION_TTL_HOURS`, default 7 days). +- Rotating `ADMIN_SECRET` signs every session out. +- If `ADMIN_SECRET` is empty, the dashboard and admin API are disabled. +- Set `TRUSTED_PROXIES` behind Traefik/Dokploy. Without it, client IPs can be + spoofed via `X-Forwarded-For`, which affects visitor counts and login throttling. + +## Admin API + +The JSON API lives under `/admin/api`. Scripts can call it with the +`X-ADMIN-KEY: ` header instead of a cookie. Cookie requests that +change data must also send `X-Admin-CSRF: 1`. + +| Area | Endpoints | +|---|---| +| Stats | `GET overview`, `GET overview/duplicates`, `GET traffic?from&to`, `GET top?kind=addon\|search&days` | +| Queue | `GET queue`, `POST queue/reset-stuck` | +| Addons | `GET addons?q&status&type&sort&order&limit&offset`, `GET addons/:id`, `GET addons/:id/files` | +| Addon actions | `POST addons/:id/{priority\|to-be-indexed\|reindex\|reset-indexing\|recheck\|restore}` with body `{"value": bool}`, `DELETE addons/:id`, `DELETE addons/:id/purge?confirm=`, `POST bulk/addons` | +| Whitelist | `GET whitelist`, `POST whitelist` with body `{"hashes":[], "text":"", "note":""}`, `DELETE whitelist/:id`, `GET hashes/:hash` | +| Metrics | `GET metrics/catalog`, `GET metrics/query?service&name&agg&groupBy&match&from&to&step` | +| Services | `GET services`, `GET services/events`, `DELETE services/:id` | +| System | `GET system`, `POST system/maintenance` with body `{"action": "refresh-stats\|flush-metrics\|rollup\|purge-deleted-files"}` | + +The legacy endpoint `POST /admin/addWhitelistedHash` (form field `hash`, `X-ADMIN-KEY`) still works. + +## Metrics + +- Every request is timed. Latency is recorded per route as a histogram, plus + status class, caller group and web vs direct client. +- The handlers also emit metrics for indexer claims and submissions, job + duration, scraper creates/updates/deletes, Discord webhooks and admin actions. +- Every minute, gauges sample the queue depth and total addon count. + +Storage: + +- Data is aggregated in memory per minute and flushed to the database every + minute (`metric_series` and `metric_points` tables). +- Minute data is rolled up hourly. +- Retention is configurable: `METRICS_MINUTE_RETENTION_DAYS` (default 7) and + `METRICS_HOUR_RETENTION_DAYS` (default 180). + +Website visitors are counted as a salted hash of IP and user agent. The salt +rotates daily and only lives in memory, so the count is approximate. The most +looked-up addons and search queries are stored per day in `daily_top_items`. + +Prometheus: + +- Set `METRICS_TOKEN` to expose `GET /metrics` in the Prometheus text format. +- Authenticate with `Authorization: Bearer `. +- This lets you add an external Prometheus/Grafana later. + +## Telemetry from indexers and scrapers + +Services report to `POST /back/telemetry` with `X-SECRET-KEY`. Every push is +also a heartbeat. An instance is shown as offline after three missed +`heartbeatIntervalSec`. + +```json +{ + "service": "indexer", "instance": "indexer-1", "version": "1.4.0", + "startedAt": 1727950000, "heartbeatIntervalSec": 15, "status": "ok", + "metrics": [ + {"name": "jobs_total", "type": "counter", "value": 3, "labels": {"result": "ok"}}, + {"name": "data_window_bytes", "type": "gauge", "value": 7.1e9}, + {"name": "download_seconds", "type": "summary", "count": 3, "sum": 41.2, "min": 8.1, "max": 20.3} + ], + "events": [{"level": "warn", "message": "data limit reached", "fields": {"limitGB": 50}}] +} +``` + +Metric types: + +- `counter`: the increase since the last push. +- `gauge`: the current value. +- `summary`: pre-aggregated values from the client. + +Limits: + +- Names must match `^[a-z_][a-z0-9_]{0,63}$`. +- At most 8 labels per metric. +- At most 1000 metrics and 200 events per push. +- At most 200 series per metric and 5000 external series in total. Anything + over these limits is dropped and counted. + +A stdlib-only Go client is in `telemetry/`, as its own module: + +```go +import "gitea.tbdevent.eu/TBD/reforger_crawler_main/telemetry" + +c := telemetry.New(telemetry.Config{BaseURL: apiURL, SecretKey: secret, Service: "indexer", Version: version}) +c.Start(ctx) +defer c.Close(context.Background()) + +c.Counter("addons_indexed_total", "result", "ok").Inc() +c.Summary("download_seconds").Observe(took.Seconds()) +c.Gauge("data_window_bytes").Set(float64(used)) +c.Event(telemetry.Warn, "data limit reached", nil) +``` + +Reported metrics show up in the dashboard under **Services** and in the **Metrics explorer**. + +## Development + +```sh +# API on SQLite (overrides config.yaml) +DB_DRIVER=sqlite SQLITE_PATH=dev.db SECRET_KEY=dev ADMIN_SECRET=devadminsecret1234 go run . + +# UI with hot reload, proxied to :8083 +npm --prefix web install +npm --prefix web run dev # http://localhost:5173/admin/ui/ + +# production build (needed before `go build` to embed the UI) +npm --prefix web run build +``` + +If the UI was not built, the binary still compiles and serves a placeholder page. diff --git a/go.mod b/go.mod index 9451edc..fd3d516 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module gitea.tbdevent.eu/TBD/reforger_crawler_main go 1.25.0 require ( - github.com/bensch777/discord-webhook-golang v0.0.6 gopkg.in/yaml.v3 v3.0.1 + gorm.io/driver/postgres v1.6.0 gorm.io/driver/sqlite v1.6.0 gorm.io/gorm v1.30.3 ) @@ -26,11 +26,13 @@ require ( github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/kr/text v0.2.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/rogpeppe/go-internal v1.16.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect golang.org/x/arch v0.8.0 // indirect @@ -39,7 +41,6 @@ require ( golang.org/x/sync v0.16.0 // indirect golang.org/x/sys v0.28.0 // indirect google.golang.org/protobuf v1.34.1 // indirect - gorm.io/driver/postgres v1.6.0 // indirect ) require ( @@ -49,3 +50,6 @@ require ( github.com/mattn/go-sqlite3 v1.14.32 // indirect golang.org/x/text v0.28.0 // indirect ) + +// keep the admin UI dependencies out of ./... package patterns +ignore ./web/node_modules diff --git a/go.sum b/go.sum index f2644ec..a2ced9b 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -github.com/bensch777/discord-webhook-golang v0.0.6 h1:91BMU6vKgymAMfRwtXPMUrKX+SUoPPHTDJHTFA/1Kgk= -github.com/bensch777/discord-webhook-golang v0.0.6/go.mod h1:GcIorMZAZaHZyQJkjNoYKvZ6VpZo8XLib/eD51xN7Is= github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= @@ -8,6 +6,7 @@ github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/ github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -48,6 +47,10 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -63,6 +66,8 @@ github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6 github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g= +github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -82,8 +87,6 @@ github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZ golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= @@ -92,8 +95,6 @@ golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= @@ -102,8 +103,9 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IV golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/initializers/initialize.go b/initializers/initialize.go index b3bb6df..038504b 100644 --- a/initializers/initialize.go +++ b/initializers/initialize.go @@ -5,7 +5,9 @@ import ( "fmt" "log" "os" + "strconv" "strings" + "time" "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" "gopkg.in/yaml.v3" @@ -25,6 +27,18 @@ var DatabaseDriver string var SQLitePath string var PostgresDSN string +// Admin dashboard, metrics and telemetry settings +var TrustedProxies []string +var AdminSessionTTL time.Duration +var AdminCookieSecure string // auto|true|false +var MetricsEnabled bool +var MetricsToken string +var MetricsMinuteRetentionDays int +var MetricsHourRetentionDays int +var EventsRetentionDays int +var TopItemsRetentionDays int +var IndexTimeout time.Duration + func ConnectToDB() { var ( db *gorm.DB @@ -75,7 +89,10 @@ func ConnectToDB() { DB = db - if err := DB.AutoMigrate(&models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{}, &models.AddonDependency{}); err != nil { + if err := DB.AutoMigrate( + &models.Addon{}, &models.AddonFile{}, &models.WhitelistedHash{}, &models.AddonDependency{}, + &models.MetricSeries{}, &models.MetricPoint{}, &models.ServiceInstance{}, &models.ServiceEvent{}, &models.DailyTopItem{}, + ); err != nil { log.Fatalf("Failed to run migrations: %v", err) } @@ -98,6 +115,21 @@ func ConnectToDB() { ON addon_files(addon_id, hash, path, version) WHERE deleted_at IS NULL`).Error; err != nil { log.Fatalf("Failed to create idx_addon_files_covering: %v", err) } + + // Indexes used by the indexing queue and the admin dashboard; failures are + // not fatal - they only cost performance + optionalIndexes := []string{ + `CREATE INDEX IF NOT EXISTS idx_addons_queue ON addons(priority_indexing, updated_at) + WHERE to_be_indexed = true AND is_being_indexed = false AND deleted_at IS NULL`, + `CREATE INDEX IF NOT EXISTS idx_addons_created_at ON addons(created_at)`, + `CREATE INDEX IF NOT EXISTS idx_addons_subscriber_count ON addons(subscriber_count)`, + `CREATE INDEX IF NOT EXISTS idx_addons_current_version_size ON addons(current_version_size)`, + } + for _, stmt := range optionalIndexes { + if err := DB.Exec(stmt).Error; err != nil { + log.Printf("Warning: failed to create index: %v", err) + } + } } type Configuration struct { @@ -116,6 +148,16 @@ type Configuration struct { PostgresPassword string `yaml:"postgres_password"` PostgresDBName string `yaml:"postgres_db_name"` PostgresSSLMode string `yaml:"postgres_sslmode"` + TrustedProxies string `yaml:"trusted_proxies"` + AdminSessionTTLH int `yaml:"admin_session_ttl_hours"` + AdminCookieSecure string `yaml:"admin_cookie_secure"` + MetricsEnabled *bool `yaml:"metrics_enabled"` + MetricsToken string `yaml:"metrics_token"` + MetricsMinuteDays int `yaml:"metrics_minute_retention_days"` + MetricsHourDays int `yaml:"metrics_hour_retention_days"` + EventsDays int `yaml:"events_retention_days"` + TopItemsDays int `yaml:"top_items_retention_days"` + IndexTimeoutMin int `yaml:"index_timeout_minutes"` LegacyPort string `yaml:"port"` LegacyIP string `yaml:"ip"` LegacySecret string `yaml:"secret"` @@ -132,6 +174,14 @@ func Load() { DatabaseDriver: "sqlite", SQLitePath: "crawler.db", PostgresSSLMode: "disable", + + AdminSessionTTLH: 168, + AdminCookieSecure: "auto", + MetricsMinuteDays: 7, + MetricsHourDays: 180, + EventsDays: 30, + TopItemsDays: 90, + IndexTimeoutMin: 30, } file, err := os.ReadFile("config.yaml") @@ -155,6 +205,17 @@ func Load() { DatabaseDriver = strings.ToLower(configuration.DatabaseDriver) SQLitePath = configuration.SQLitePath + TrustedProxies = splitList(configuration.TrustedProxies) + AdminSessionTTL = time.Duration(positiveOr(configuration.AdminSessionTTLH, 168)) * time.Hour + AdminCookieSecure = strings.ToLower(configuration.AdminCookieSecure) + MetricsEnabled = configuration.MetricsEnabled == nil || *configuration.MetricsEnabled + MetricsToken = configuration.MetricsToken + MetricsMinuteRetentionDays = positiveOr(configuration.MetricsMinuteDays, 7) + MetricsHourRetentionDays = positiveOr(configuration.MetricsHourDays, 180) + EventsRetentionDays = positiveOr(configuration.EventsDays, 30) + TopItemsRetentionDays = positiveOr(configuration.TopItemsDays, 90) + IndexTimeout = time.Duration(positiveOr(configuration.IndexTimeoutMin, 30)) * time.Minute + if strings.TrimSpace(configuration.PostgresDSN) != "" { PostgresDSN = configuration.PostgresDSN } else { @@ -202,6 +263,49 @@ func applyEnvOverrides(c *Configuration) { c.PostgresPassword = firstEnv([]string{"POSTGRES_PASSWORD"}, c.PostgresPassword) c.PostgresDBName = firstEnv([]string{"POSTGRES_DB_NAME"}, c.PostgresDBName) c.PostgresSSLMode = firstEnv([]string{"POSTGRES_SSLMODE"}, c.PostgresSSLMode) + c.TrustedProxies = firstEnv([]string{"TRUSTED_PROXIES"}, c.TrustedProxies) + c.AdminSessionTTLH = firstEnvInt([]string{"ADMIN_SESSION_TTL_HOURS"}, c.AdminSessionTTLH) + c.AdminCookieSecure = firstEnv([]string{"ADMIN_COOKIE_SECURE"}, c.AdminCookieSecure) + if v := firstEnv([]string{"METRICS_ENABLED"}, ""); v != "" { + enabled := v == "1" || strings.EqualFold(v, "true") || strings.EqualFold(v, "yes") + c.MetricsEnabled = &enabled + } + c.MetricsToken = firstEnv([]string{"METRICS_TOKEN"}, c.MetricsToken) + c.MetricsMinuteDays = firstEnvInt([]string{"METRICS_MINUTE_RETENTION_DAYS"}, c.MetricsMinuteDays) + c.MetricsHourDays = firstEnvInt([]string{"METRICS_HOUR_RETENTION_DAYS"}, c.MetricsHourDays) + c.EventsDays = firstEnvInt([]string{"EVENTS_RETENTION_DAYS"}, c.EventsDays) + c.TopItemsDays = firstEnvInt([]string{"TOP_ITEMS_RETENTION_DAYS"}, c.TopItemsDays) + c.IndexTimeoutMin = firstEnvInt([]string{"INDEX_TIMEOUT_MINUTES"}, c.IndexTimeoutMin) +} + +func firstEnvInt(keys []string, fallback int) int { + raw := firstEnv(keys, "") + if raw == "" { + return fallback + } + val, err := strconv.Atoi(strings.TrimSpace(raw)) + if err != nil { + log.Printf("Warning: ignoring invalid integer %q for %s", raw, strings.Join(keys, "/")) + return fallback + } + return val +} + +func positiveOr(val, fallback int) int { + if val > 0 { + return val + } + return fallback +} + +func splitList(raw string) []string { + var out []string + for _, part := range strings.Split(raw, ",") { + if part = strings.TrimSpace(part); part != "" { + out = append(out, part) + } + } + return out } func firstEnv(keys []string, fallback string) string { diff --git a/main.go b/main.go index 1cb88c4..6d03c03 100644 --- a/main.go +++ b/main.go @@ -1,22 +1,49 @@ package main import ( + "context" + "errors" + "log" "net/http" + "os/signal" + "strings" + "syscall" + "time" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/admin" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/auth" "gitea.tbdevent.eu/TBD/reforger_crawler_main/controllers" "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/web" "github.com/gin-gonic/gin" ) +// buildCommit is set at build time: -ldflags "-X main.buildCommit=" +var buildCommit = "" + func init() { initializers.Load() initializers.ConnectToDB() } func main() { - r := gin.Default() + warnAboutConfig() + + r := gin.New() + r.Use(gin.LoggerWithConfig(gin.LoggerConfig{Skip: skipLog}), gin.Recovery()) + if len(initializers.TrustedProxies) > 0 { + if err := r.SetTrustedProxies(initializers.TrustedProxies); err != nil { + log.Fatalf("Invalid TRUSTED_PROXIES: %v", err) + } + } + metrics.SetEnabled(initializers.MetricsEnabled) + if initializers.MetricsEnabled { + r.Use(metrics.Middleware()) + } r.Use(CORSMiddleware()) + r.GET("/health", func(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"status": "ok"}) }) @@ -29,14 +56,18 @@ func main() { back.POST("/submitAddon", controllers.CheckAllowed, controllers.SaveIndexingResult) back.DELETE("/deleteAddon", controllers.CheckAllowed, controllers.DeleteAddon) back.POST("/addonDependencies", controllers.CheckAllowed, controllers.SetAddonDependencies) + back.POST("/telemetry", controllers.CheckAllowed, controllers.SubmitTelemetry) } - admin := r.Group("/admin") - { - admin.POST("/addWhitelistedHash", controllers.CheckAdmin, controllers.AddWhitelistedHash) - admin.GET("/duplicates", controllers.CheckAdmin) - admin.POST("/setPriority", controllers.CheckAdmin) - admin.POST("/setToBeIndexed", controllers.CheckAdmin) + // admin JSON API, legacy admin endpoint and the embedded dashboard + admin.Register(r) + web.Register(r) + + if initializers.MetricsToken != "" { + r.GET("/metrics", requireMetricsToken, func(c *gin.Context) { + c.Header("Content-Type", "text/plain; version=0.0.4; charset=utf-8") + metrics.WritePrometheus(c.Writer) + }) } v1 := r.Group("/v1") @@ -47,11 +78,95 @@ func main() { v1.GET("/getPossible", controllers.GetPossibleAddons) } - r.Run(initializers.ServerHost + ":" + initializers.ServerPort) + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + version := buildCommit + if version == "" { + version = "dev" + } + admin.BuildCommit = buildCommit + metrics.SetVersion(version) + if initializers.MetricsEnabled { + metrics.Start(ctx) + } + admin.StartBackground(ctx) + + srv := &http.Server{ + Addr: initializers.ServerHost + ":" + initializers.ServerPort, + Handler: r, + ReadHeaderTimeout: 10 * time.Second, + } + go func() { + log.Printf("Listening on %s (admin dashboard at /admin/ui/)", srv.Addr) + if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Fatalf("Server error: %v", err) + } + }() + + <-ctx.Done() + log.Println("Shutting down...") + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := srv.Shutdown(shutdownCtx); err != nil { + log.Printf("Shutdown error: %v", err) + } + if initializers.MetricsEnabled { + metrics.FlushAll() + } +} + +// skipLog keeps health checks, dashboard assets and dashboard polling out of +// the access log. +func skipLog(c *gin.Context) bool { + p := c.Request.URL.Path + if p == "/health" || strings.HasPrefix(p, "/admin/ui") { + return true + } + return strings.HasPrefix(p, "/admin/api") && c.Request.Method == http.MethodGet && c.Writer.Status() < 400 +} + +func requireMetricsToken(c *gin.Context) { + token := strings.TrimPrefix(c.GetHeader("Authorization"), "Bearer ") + if auth.SecretEqual(token, initializers.MetricsToken) { + return + } + // also allow a logged-in admin to look at the raw exposition + if auth.SecretEqual(c.GetHeader("X-ADMIN-KEY"), initializers.AdminSecret) { + return + } + if cookie, err := c.Cookie(auth.CookieName); err == nil { + if _, ok := auth.VerifyToken(cookie, time.Now()); ok { + return + } + } + c.Header("WWW-Authenticate", `Bearer realm="metrics"`) + c.AbortWithStatus(http.StatusUnauthorized) +} + +func warnAboutConfig() { + if initializers.SecretKey == "" { + log.Println("Warning: SECRET_KEY is not set - /back endpoints are disabled") + } + if initializers.AdminSecret == "" { + log.Println("Warning: ADMIN_SECRET is not set - the admin dashboard is disabled") + } else if len(initializers.AdminSecret) < 16 { + log.Println("Warning: ADMIN_SECRET is shorter than 16 characters") + } + if len(initializers.TrustedProxies) == 0 { + log.Println("Warning: TRUSTED_PROXIES is not set - client IPs from X-Forwarded-For are trusted from any source") + } } func CORSMiddleware() gin.HandlerFunc { return func(c *gin.Context) { + // the admin API and worker endpoints are never called cross-origin + p := c.Request.URL.Path + if strings.HasPrefix(p, "/admin") || strings.HasPrefix(p, "/back") || p == "/metrics" { + c.Next() + return + } + orig := c.Request.Header.Clone().Get("Origin") c.Writer.Header().Set("Access-Control-Allow-Origin", orig) c.Writer.Header().Set("Access-Control-Allow-Credentials", "true") @@ -66,13 +181,3 @@ func CORSMiddleware() gin.HandlerFunc { c.Next() } } - -func OptionsMiddleware() gin.HandlerFunc { - return func(c *gin.Context) { - if c.Request.Method != "OPTIONS" { - c.Next() - } else { - c.AbortWithStatus(http.StatusOK) - } - } -} diff --git a/metrics/aggregator.go b/metrics/aggregator.go new file mode 100644 index 0000000..3a6a06a --- /dev/null +++ b/metrics/aggregator.go @@ -0,0 +1,286 @@ +package metrics + +import ( + "encoding/json" + "math" + "sort" + "sync" + "sync/atomic" +) + +// disabled turns all recording into no-ops (METRICS_ENABLED=false). +var disabled atomic.Bool + +// SetEnabled switches metric recording on or off. +func SetEnabled(on bool) { disabled.Store(!on) } + +// Agg is the aggregate of all samples of one series inside one bucket. +type Agg struct { + Count int64 + Sum float64 + Min float64 + Max float64 + Last float64 + LastTs int64 + Hist []uint64 // len(bounds)+1, last element is +Inf +} + +func (a *Agg) add(v float64, ts int64, bounds []float64) { + if a.Count == 0 { + a.Min, a.Max = v, v + } else { + a.Min = math.Min(a.Min, v) + a.Max = math.Max(a.Max, v) + } + a.Count++ + a.Sum += v + if ts >= a.LastTs { + a.Last, a.LastTs = v, ts + } + if len(bounds) > 0 { + if a.Hist == nil { + a.Hist = make([]uint64, len(bounds)+1) + } + a.Hist[bucketIndex(bounds, v)]++ + } +} + +func (a *Agg) addSummary(count int64, sum, min, max float64, ts int64) { + if count <= 0 { + return + } + if a.Count == 0 { + a.Min, a.Max = min, max + } else { + a.Min = math.Min(a.Min, min) + a.Max = math.Max(a.Max, max) + } + a.Count += count + a.Sum += sum + if ts >= a.LastTs { + a.Last, a.LastTs = sum/float64(count), ts + } +} + +// merge folds b into a. +func (a *Agg) merge(b *Agg) { + if b == nil || b.Count == 0 { + return + } + if a.Count == 0 { + a.Min, a.Max = b.Min, b.Max + } else { + a.Min = math.Min(a.Min, b.Min) + a.Max = math.Max(a.Max, b.Max) + } + a.Count += b.Count + a.Sum += b.Sum + if b.LastTs >= a.LastTs { + a.Last, a.LastTs = b.Last, b.LastTs + } + if len(b.Hist) > 0 { + if len(a.Hist) == 0 { + a.Hist = make([]uint64, len(b.Hist)) + } + if len(a.Hist) == len(b.Hist) { + for i, v := range b.Hist { + a.Hist[i] += v + } + } + } +} + +func (a *Agg) clone() *Agg { + c := *a + if a.Hist != nil { + c.Hist = append([]uint64(nil), a.Hist...) + } + return &c +} + +func (a *Agg) histJSON() string { + if len(a.Hist) == 0 { + return "" + } + b, _ := json.Marshal(a.Hist) + return string(b) +} + +func bucketIndex(bounds []float64, v float64) int { + return sort.SearchFloat64s(bounds, v) // first bound >= v; len(bounds) = +Inf +} + +// quantile estimates the q-quantile from histogram buckets with linear +// interpolation inside the bucket (like Prometheus), clamped to [min, max]. +func quantile(a *Agg, bounds []float64, q float64) (float64, bool) { + if a == nil || a.Count == 0 || len(a.Hist) != len(bounds)+1 { + return 0, false + } + var total uint64 + for _, c := range a.Hist { + total += c + } + if total == 0 { + return 0, false + } + rank := q * float64(total) + var cum uint64 + for i, c := range a.Hist { + if c == 0 { + continue + } + if float64(cum+c) >= rank { + lower := a.Min + if i > 0 { + lower = math.Max(bounds[i-1], a.Min) + } + upper := a.Max + if i < len(bounds) { + upper = math.Min(bounds[i], a.Max) + } + if upper < lower { + upper = lower + } + frac := (rank - float64(cum)) / float64(c) + return lower + (upper-lower)*frac, true + } + cum += c + } + return a.Max, true +} + +type pointKey struct { + s *Series + bucket int64 +} + +// cumulative is the since-start state of a series, used for /metrics. +type cumulative struct { + count int64 + sum float64 + last float64 + hist []uint64 +} + +// aggregator holds the not yet persisted minute buckets and cumulative state. +type aggregator struct { + mu sync.Mutex + open map[pointKey]*Agg + cum map[*Series]*cumulative +} + +var defaultAgg = &aggregator{ + open: make(map[pointKey]*Agg), + cum: make(map[*Series]*cumulative), +} + +func minuteOf(ts int64) int64 { return ts - ts%60 } + +func (g *aggregator) entry(s *Series, ts int64) (*Agg, *cumulative) { + k := pointKey{s, minuteOf(ts)} + a := g.open[k] + if a == nil { + a = &Agg{} + g.open[k] = a + } + c := g.cum[s] + if c == nil { + c = &cumulative{} + g.cum[s] = c + } + return a, c +} + +func (g *aggregator) observe(s *Series, ts int64, v float64) { + if disabled.Load() { + return + } + g.mu.Lock() + a, c := g.entry(s, ts) + a.add(v, ts, s.Bounds) + c.count++ + c.sum += v + c.last = v + if len(s.Bounds) > 0 { + if c.hist == nil { + c.hist = make([]uint64, len(s.Bounds)+1) + } + c.hist[bucketIndex(s.Bounds, v)]++ + } + g.mu.Unlock() + reg.touch(s, ts) +} + +func (g *aggregator) observeSummary(s *Series, ts int64, count int64, sum, min, max float64) { + if disabled.Load() { + return + } + g.mu.Lock() + a, c := g.entry(s, ts) + a.addSummary(count, sum, min, max, ts) + c.count += count + c.sum += sum + c.last = sum / float64(count) + g.mu.Unlock() + reg.touch(s, ts) +} + +// takeClosed removes and returns all buckets that started before `before`. +func (g *aggregator) takeClosed(before int64) map[pointKey]*Agg { + g.mu.Lock() + defer g.mu.Unlock() + out := make(map[pointKey]*Agg) + for k, a := range g.open { + if k.bucket < before { + out[k] = a + delete(g.open, k) + } + } + return out +} + +// putBack re-queues buckets after a failed flush. +func (g *aggregator) putBack(points map[pointKey]*Agg) { + g.mu.Lock() + defer g.mu.Unlock() + for k, a := range points { + if cur := g.open[k]; cur != nil { + cur.merge(a) + } else { + g.open[k] = a + } + } +} + +// snapshotOpen copies the open buckets of the given series. +func (g *aggregator) snapshotOpen(want map[*Series]bool) map[pointKey]*Agg { + g.mu.Lock() + defer g.mu.Unlock() + out := make(map[pointKey]*Agg) + for k, a := range g.open { + if want[k.s] { + out[k] = a.clone() + } + } + return out +} + +func (g *aggregator) openCount() int { + g.mu.Lock() + defer g.mu.Unlock() + return len(g.open) +} + +func (g *aggregator) snapshotCum() map[*Series]cumulative { + g.mu.Lock() + defer g.mu.Unlock() + out := make(map[*Series]cumulative, len(g.cum)) + for s, c := range g.cum { + cc := *c + if c.hist != nil { + cc.hist = append([]uint64(nil), c.hist...) + } + out[s] = cc + } + return out +} diff --git a/metrics/internal.go b/metrics/internal.go new file mode 100644 index 0000000..b5693cf --- /dev/null +++ b/metrics/internal.go @@ -0,0 +1,38 @@ +package metrics + +// Metrics emitted by the API itself (service "main"). + +// HTTPBoundsMs are the latency histogram buckets in milliseconds. +var HTTPBoundsMs = []float64{1, 2, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000} + +var ( + HTTPDuration = NewHistogram("http_request_duration_ms", "HTTP request latency in milliseconds", HTTPBoundsMs, + "method", "route", "status", "group", "client") + HTTPResponseBytes = NewCounter("http_response_bytes_total", "HTTP response body bytes", "group") + + IndexerClaims = NewCounter("indexer_claims_total", "Indexing jobs requested by indexers, by outcome", "queue") + IndexerTimeoutsReset = NewCounter("indexer_timeouts_reset_total", "Indexing claims released after the timeout") + IndexerSubmissions = NewCounter("indexer_submissions_total", "Indexing results submitted", "result") + IndexerJobDuration = NewHistogram("indexer_job_duration_seconds", "Time from claim to submitted result", + []float64{1, 5, 15, 30, 60, 120, 300, 600, 1200, 1800, 3600}) + IndexerFilesPerSubmission = NewHistogram("indexer_files_per_submission", "Files per indexing result", + []float64{1, 10, 50, 100, 500, 1000, 5000, 10000, 50000}) + IndexerBytesIndexed = NewCounter("indexer_bytes_indexed_total", "Addon bytes indexed") + + ScraperAddons = NewCounter("scraper_addons_total", "Addon reports from scrapers", "action") + ScraperDeletes = NewCounter("scraper_addons_deleted_total", "Addon deletions requested by scrapers", "result") + DependencyReports = NewCounter("dependency_reports_total", "Dependency reports received") + DependencyEdges = NewCounter("dependency_edges_reported_total", "Dependency edges stored") + DuplicatesFound = NewHistogram("duplicates_found_addons", "Addons sharing files with a looked-up addon", + []float64{0, 1, 2, 5, 10, 25, 50, 100}) + DiscordWebhooks = NewCounter("discord_webhook_total", "Discord webhook sends", "kind", "result") + AdminActions = NewCounter("admin_actions_total", "Admin dashboard actions", "action") + + AddonsTotal = NewGauge("addons_total", "Addons in the database (not deleted)") + QueueDepth = NewGauge("queue_depth", "Addons waiting to be indexed", "kind") + BeingIndexed = NewGauge("being_indexed", "Addons currently claimed by an indexer") + VisitorsUnique = NewGauge("visitors_unique", "Unique website visitors so far today (approximate)", "kind") + + TelemetrySamples = NewCounter("telemetry_samples_total", "Telemetry samples received", "result") + PointsDropped = NewCounter("metrics_points_dropped_total", "Metric points dropped after failed writes") +) diff --git a/metrics/metrics_test.go b/metrics/metrics_test.go new file mode 100644 index 0000000..b5a1393 --- /dev/null +++ b/metrics/metrics_test.go @@ -0,0 +1,216 @@ +package metrics + +import ( + "math" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +func setupDB(t *testing.T) { + t.Helper() + db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "m.db")), &gorm.Config{Logger: logger.Discard}) + if err != nil { + t.Fatal(err) + } + if err := db.AutoMigrate(&models.Addon{}, &models.MetricSeries{}, &models.MetricPoint{}, + &models.ServiceInstance{}, &models.ServiceEvent{}, &models.DailyTopItem{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if sqlDB, err := db.DB(); err == nil { + sqlDB.Close() + } + }) + initializers.DB = db + initializers.MetricsMinuteRetentionDays = 7 + initializers.MetricsHourRetentionDays = 180 + + // fresh in-memory state per test + reg = ®istry{series: map[seriesKey]*Series{}, perName: map[string]int{}, lastSeen: map[*Series]int64{}} + defaultAgg = &aggregator{open: map[pointKey]*Agg{}, cum: map[*Series]*cumulative{}} + seriesID = map[*Series]uint{} + rolledUntil.Store(0) + for _, f := range families { + f.cache = map[string]*Series{} + } +} + +func TestQuantileInterpolation(t *testing.T) { + bounds := []float64{10, 20, 30} + a := &Agg{} + for i := 1; i <= 100; i++ { + a.add(float64(i%30)+0.5, 0, bounds) + } + p50, ok := quantile(a, bounds, 0.5) + if !ok || p50 < 10 || p50 > 20 { + t.Fatalf("p50 = %v, want within (10, 20]", p50) + } + p99, _ := quantile(a, bounds, 0.99) + if p99 > a.Max || p99 < 20 { + t.Fatalf("p99 = %v, want in [20, max=%v]", p99, a.Max) + } + if _, ok := quantile(&Agg{}, bounds, 0.5); ok { + t.Fatal("empty histogram must not produce a quantile") + } +} + +func TestAggMerge(t *testing.T) { + a, b := &Agg{}, &Agg{} + a.add(5, 100, nil) + a.add(1, 101, nil) + b.add(9, 99, nil) + a.merge(b) + if a.Count != 3 || a.Sum != 15 || a.Min != 1 || a.Max != 9 || a.Last != 1 { + t.Fatalf("unexpected merge result %+v", a) + } +} + +func TestConcurrentObserve(t *testing.T) { + setupDB(t) + f := NewCounter("test_concurrent_total", "test", "worker") + var wg sync.WaitGroup + for w := range 8 { + wg.Add(1) + go func(w int) { + defer wg.Done() + for range 1000 { + f.Inc(string(rune('a' + w%2))) + } + }(w) + } + wg.Wait() + var total float64 + for s, c := range defaultAgg.snapshotCum() { + if s.Name == "test_concurrent_total" { + total += c.sum + } + } + if total != 8000 { + t.Fatalf("total = %v, want 8000", total) + } +} + +func TestFlushQueryRollup(t *testing.T) { + setupDB(t) + h := NewHistogram("test_latency_ms", "test", []float64{1, 10, 100}, "route") + s := h.series([]string{"/a"}) + + // two hours ago, spread over three minutes, plus one point in the current minute + base := hourOf(time.Now().Unix()) - 2*3600 + for i := range 30 { + defaultAgg.observe(s, base+int64(i%3)*60, float64(i)) + } + now := time.Now().Unix() + defaultAgg.observe(s, now, 50) + + if err := flush(minuteOf(now)); err != nil { + t.Fatal(err) + } + var rows int64 + initializers.DB.Model(&models.MetricPoint{}).Where("resolution = ?", resMinute).Count(&rows) + if rows != 3 { + t.Fatalf("minute rows = %d, want 3 (current minute stays in memory)", rows) + } + + // flushing the same bucket again must merge, not overwrite + defaultAgg.observe(s, base, 1000) + if err := flush(minuteOf(now)); err != nil { + t.Fatal(err) + } + var p models.MetricPoint + initializers.DB.Where("resolution = ? AND bucket = ?", resMinute, base).First(&p) + if p.Count != 11 || p.Max != 1000 { + t.Fatalf("merged point = %+v, want count 11 and max 1000", p) + } + + // query sees DB rows plus the unflushed current minute + sel := Selector{Service: MainService, Name: "test_latency_ms"} + res, err := Query(sel, "count", now-3*3600, now, 60) + if err != nil { + t.Fatal(err) + } + var count float64 + for _, pt := range res.Series[0].Points { + if pt.V != nil { + count += *pt.V + } + } + if count != 32 { + t.Fatalf("queried count = %v, want 32", count) + } + + if err := rollup(now); err != nil { + t.Fatal(err) + } + var hourly models.MetricPoint + if err := initializers.DB.Where("resolution = ? AND bucket = ?", resHour, base).First(&hourly).Error; err != nil { + t.Fatalf("missing hourly rollup: %v", err) + } + if hourly.Count != 31 || hourly.Max != 1000 { + t.Fatalf("hourly = %+v, want count 31 max 1000", hourly) + } + // rolling up again is idempotent + rolledUntil.Store(0) + if err := rollup(now); err != nil { + t.Fatal(err) + } + initializers.DB.Where("resolution = ? AND bucket = ?", resHour, base).First(&hourly) + if hourly.Count != 31 { + t.Fatalf("re-rollup changed count to %d", hourly.Count) + } + + totals, err := Totals(sel, now-3*3600, now) + if err != nil || len(totals) != 1 || totals[0].P95 == nil || math.IsNaN(*totals[0].P95) { + t.Fatalf("totals = %+v, err %v", totals, err) + } +} + +func TestIngestValidation(t *testing.T) { + setupDB(t) + now := time.Now() + res := Ingest("indexer", "i-1", []models.TelemetryMetric{ + {Name: "ok_total", Type: "counter", Value: 2}, + {Name: "Bad-Name", Type: "gauge", Value: 1}, + {Name: "neg_total", Type: "counter", Value: -1}, + {Name: "lat", Type: "summary", Count: 2, Sum: 3, Min: 1, Max: 2}, + {Name: "ok_total", Type: "gauge", Value: 1}, + {Name: "weird", Type: "nope"}, + }, now) + if res.Accepted != 2 || res.Dropped != 4 { + t.Fatalf("accepted %d dropped %d (%v)", res.Accepted, res.Dropped, res.Reasons) + } + if res.Reasons["type_mismatch"] != 1 || res.Reasons["invalid_name"] != 1 { + t.Fatalf("unexpected reasons %v", res.Reasons) + } +} + +func TestPrometheusOutput(t *testing.T) { + setupDB(t) + c := NewCounter("test_prom_total", "a \"quoted\" help", "kind") + c.Add(3, `va"l`) + h := NewHistogram("test_prom_ms", "hist", []float64{1, 10}) + h.Observe(5) + var b strings.Builder + WritePrometheus(&b) + out := b.String() + for _, want := range []string{ + `test_prom_total{kind="va\"l"} 3`, + `test_prom_ms_bucket{le="1"} 0`, + `test_prom_ms_bucket{le="10"} 1`, + `test_prom_ms_bucket{le="+Inf"} 1`, + `# TYPE test_prom_ms histogram`, + } { + if !strings.Contains(out, want) { + t.Errorf("missing %q in output", want) + } + } +} diff --git a/metrics/pipeline.go b/metrics/pipeline.go new file mode 100644 index 0000000..a5816c7 --- /dev/null +++ b/metrics/pipeline.go @@ -0,0 +1,156 @@ +package metrics + +import ( + "context" + "log" + "math" + "sync" + "time" + + "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" +) + +var startOnce sync.Once + +// Start launches the background jobs: minute flush and samplers, top-N +// persistence, hourly rollup and retention, and service heartbeat checks. +func Start(ctx context.Context) { + startOnce.Do(func() { + Top.seed() + go every(ctx, "flush", time.Minute, 5*time.Second, func(now time.Time) { + sample() + if err := flush(minuteOf(now.Unix())); err != nil { + log.Printf("metrics: flush failed: %v", err) + } + }) + go every(ctx, "top", 5*time.Minute, 0, func(time.Time) { + if err := Top.flush(); err != nil { + log.Printf("metrics: top items flush failed: %v", err) + } + }) + go every(ctx, "rollup", time.Hour, 2*time.Minute, func(now time.Time) { + RunMaintenance(now) + }) + go every(ctx, "services", 30*time.Second, 0, func(now time.Time) { + selfHeartbeat(now) + checkServices(now) + }) + // initialize the rollup watermark and service state right away + go func() { + selfHeartbeat(time.Now()) + checkServices(time.Now()) + if err := rollup(time.Now().Unix()); err != nil { + log.Printf("metrics: rollup failed: %v", err) + } + }() + }) +} + +// RunMaintenance runs the rollup and retention jobs. +func RunMaintenance(now time.Time) { + if err := rollup(now.Unix()); err != nil { + log.Printf("metrics: rollup failed: %v", err) + } + if err := retention(now); err != nil { + log.Printf("metrics: retention failed: %v", err) + } +} + +// every runs fn on a fixed period aligned to wall clock (+offset). +func every(ctx context.Context, name string, period, offset time.Duration, fn func(time.Time)) { + for { + now := time.Now() + next := now.Truncate(period).Add(offset) + if !next.After(now) { + next = next.Add(period) + } + select { + case <-ctx.Done(): + return + case t := <-time.After(time.Until(next)): + func() { + defer func() { + if r := recover(); r != nil { + log.Printf("metrics: %s job panicked: %v", name, r) + } + }() + fn(t) + }() + } + } +} + +// FlushAll writes everything still in memory; used on shutdown. +func FlushAll() { + sample() + if err := flush(math.MaxInt64); err != nil { + log.Printf("metrics: final flush failed: %v", err) + } + if err := Top.flush(); err != nil { + log.Printf("metrics: final top items flush failed: %v", err) + } +} + +// QueueCounts is a cheap snapshot of the indexing queue. +type QueueCounts struct { + Total int64 `json:"total"` + Queued int64 `json:"queued"` + Priority int64 `json:"priority"` + Indexing int64 `json:"indexing"` +} + +// LoadQueueCounts counts addons by queue state with one query. +func LoadQueueCounts() (QueueCounts, error) { + var qc QueueCounts + err := initializers.DB.Raw(`SELECT COUNT(*) AS total, + COALESCE(SUM(CASE WHEN to_be_indexed = ? AND is_being_indexed = ? THEN 1 ELSE 0 END), 0) AS queued, + COALESCE(SUM(CASE WHEN to_be_indexed = ? AND is_being_indexed = ? AND priority_indexing = ? THEN 1 ELSE 0 END), 0) AS priority, + COALESCE(SUM(CASE WHEN is_being_indexed = ? THEN 1 ELSE 0 END), 0) AS indexing + FROM addons WHERE deleted_at IS NULL`, + true, false, true, false, true, true).Scan(&qc).Error + return qc, err +} + +// sample records gauges that are read from the database or memory. +func sample() { + if qc, err := LoadQueueCounts(); err == nil { + AddonsTotal.Set(float64(qc.Total)) + QueueDepth.Set(float64(qc.Queued), "total") + QueueDepth.Set(float64(qc.Priority), "priority") + BeingIndexed.Set(float64(qc.Indexing)) + } + humans, bots := liveVisitors() + VisitorsUnique.Set(float64(humans), "human") + VisitorsUnique.Set(float64(bots), "bot") +} + +// PipelineStatus describes the health of the metrics pipeline. +type PipelineStatus struct { + Enabled bool `json:"enabled"` + LastFlush int64 `json:"lastFlush"` + LastFlushMs int64 `json:"lastFlushMs"` + LastFlushErr string `json:"lastFlushError"` + LastRollup int64 `json:"lastRollup"` + LastRetention int64 `json:"lastRetention"` + RolledUntil int64 `json:"rolledUntil"` + OpenPoints int `json:"openPoints"` + LiveSeries int `json:"liveSeries"` + DroppedPoints int64 `json:"droppedPoints"` +} + +func Status() PipelineStatus { + status.mu.Lock() + defer status.mu.Unlock() + return PipelineStatus{ + Enabled: initializers.MetricsEnabled, + LastFlush: status.LastFlush, + LastFlushMs: status.LastFlushMs, + LastFlushErr: status.LastFlushErr, + LastRollup: status.LastRollup, + LastRetention: status.LastRetention, + RolledUntil: rolledUntil.Load(), + OpenPoints: defaultAgg.openCount(), + LiveSeries: len(reg.all()), + DroppedPoints: status.Dropped, + } +} diff --git a/metrics/prometheus.go b/metrics/prometheus.go new file mode 100644 index 0000000..3923bab --- /dev/null +++ b/metrics/prometheus.go @@ -0,0 +1,181 @@ +package metrics + +import ( + "bufio" + "io" + "runtime" + "sort" + "strconv" + "strings" + "time" + + "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" +) + +// WritePrometheus writes all cumulative series in the Prometheus text format +// (version 0.0.4). External metrics are exposed as _ with an +// "instance" label; counters are totals since this process started. +func WritePrometheus(w io.Writer) { + bw := bufio.NewWriter(w) + defer bw.Flush() + + cum := defaultAgg.snapshotCum() + type entry struct { + s *Series + c cumulative + } + byName := map[string][]entry{} + meta := map[string]*Series{} + for s, c := range cum { + name := s.Name + if s.Service != MainService { + name = s.Service + "_" + s.Name + } + byName[name] = append(byName[name], entry{s, c}) + if meta[name] == nil { + meta[name] = s + } + } + names := make([]string, 0, len(byName)) + for n := range byName { + names = append(names, n) + } + sort.Strings(names) + + for _, name := range names { + s := meta[name] + writeHeader(bw, name, s.Help, promType(s.Kind)) + entries := byName[name] + sort.Slice(entries, func(i, j int) bool { + if entries[i].s.Instance != entries[j].s.Instance { + return entries[i].s.Instance < entries[j].s.Instance + } + return entries[i].s.LabelsJSON < entries[j].s.LabelsJSON + }) + for _, e := range entries { + labels := e.s.Labels + if e.s.Service != MainService { + labels = withLabel(labels, "instance", e.s.Instance) + } + switch e.s.Kind { + case Counter: + writeSample(bw, name, labels, e.c.sum) + case Gauge: + writeSample(bw, name, labels, e.c.last) + case Summary: + writeSample(bw, name+"_sum", labels, e.c.sum) + writeSample(bw, name+"_count", labels, float64(e.c.count)) + case Histogram: + var acc uint64 + for i, b := range e.s.Bounds { + if i < len(e.c.hist) { + acc += e.c.hist[i] + } + writeSample(bw, name+"_bucket", withLabel(labels, "le", formatFloat(b)), float64(acc)) + } + writeSample(bw, name+"_bucket", withLabel(labels, "le", "+Inf"), float64(e.c.count)) + writeSample(bw, name+"_sum", labels, e.c.sum) + writeSample(bw, name+"_count", labels, float64(e.c.count)) + } + } + } + + up := ServiceUp() + if len(up) > 0 { + writeHeader(bw, "reforger_service_up", "Whether a reporting service instance is sending heartbeats", "gauge") + keys := make([][2]string, 0, len(up)) + for k := range up { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { return keys[i][0]+keys[i][1] < keys[j][0]+keys[j][1] }) + for _, k := range keys { + v := 0.0 + if up[k] { + v = 1 + } + writeSample(bw, "reforger_service_up", map[string]string{"service": k[0], "instance": k[1]}, v) + } + } + + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + writeHeader(bw, "go_goroutines", "Number of goroutines", "gauge") + writeSample(bw, "go_goroutines", nil, float64(runtime.NumGoroutine())) + writeHeader(bw, "go_memstats_heap_alloc_bytes", "Heap bytes allocated and in use", "gauge") + writeSample(bw, "go_memstats_heap_alloc_bytes", nil, float64(ms.HeapAlloc)) + writeHeader(bw, "go_memstats_sys_bytes", "Bytes obtained from the OS", "gauge") + writeSample(bw, "go_memstats_sys_bytes", nil, float64(ms.Sys)) + writeHeader(bw, "go_gc_cycles_total", "Completed GC cycles", "counter") + writeSample(bw, "go_gc_cycles_total", nil, float64(ms.NumGC)) + writeHeader(bw, "process_start_time_seconds", "Process start time", "gauge") + writeSample(bw, "process_start_time_seconds", nil, float64(startedAt.Unix())) + writeHeader(bw, "process_uptime_seconds", "Process uptime", "gauge") + writeSample(bw, "process_uptime_seconds", nil, time.Since(startedAt).Seconds()) + + if sqlDB, err := initializers.DB.DB(); err == nil { + st := sqlDB.Stats() + writeHeader(bw, "db_open_connections", "Open database connections", "gauge") + writeSample(bw, "db_open_connections", nil, float64(st.OpenConnections)) + writeHeader(bw, "db_in_use_connections", "Database connections in use", "gauge") + writeSample(bw, "db_in_use_connections", nil, float64(st.InUse)) + writeHeader(bw, "db_wait_count_total", "Waits for a database connection", "counter") + writeSample(bw, "db_wait_count_total", nil, float64(st.WaitCount)) + writeHeader(bw, "db_wait_seconds_total", "Time spent waiting for a database connection", "counter") + writeSample(bw, "db_wait_seconds_total", nil, st.WaitDuration.Seconds()) + } +} + +func promType(k Kind) string { + switch k { + case Counter: + return "counter" + case Histogram: + return "histogram" + case Summary: + return "summary" + } + return "gauge" +} + +func withLabel(labels map[string]string, k, v string) map[string]string { + out := make(map[string]string, len(labels)+1) + for lk, lv := range labels { + out[lk] = lv + } + out[k] = v + return out +} + +func formatFloat(v float64) string { return strconv.FormatFloat(v, 'g', -1, 64) } + +var helpEscaper = strings.NewReplacer(`\`, `\\`, "\n", `\n`) +var valueEscaper = strings.NewReplacer(`\`, `\\`, "\n", `\n`, `"`, `\"`) + +func writeHeader(w *bufio.Writer, name, help, typ string) { + if help != "" { + w.WriteString("# HELP " + name + " " + helpEscaper.Replace(help) + "\n") + } + w.WriteString("# TYPE " + name + " " + typ + "\n") +} + +func writeSample(w *bufio.Writer, name string, labels map[string]string, v float64) { + w.WriteString(name) + if len(labels) > 0 { + keys := make([]string, 0, len(labels)) + for k := range labels { + keys = append(keys, k) + } + sort.Strings(keys) + w.WriteByte('{') + for i, k := range keys { + if i > 0 { + w.WriteByte(',') + } + w.WriteString(k + `="` + valueEscaper.Replace(labels[k]) + `"`) + } + w.WriteByte('}') + } + w.WriteByte(' ') + w.WriteString(formatFloat(v)) + w.WriteByte('\n') +} diff --git a/metrics/query.go b/metrics/query.go new file mode 100644 index 0000000..fc9cb09 --- /dev/null +++ b/metrics/query.go @@ -0,0 +1,526 @@ +package metrics + +import ( + "encoding/json" + "fmt" + "math" + "sort" + "strconv" + "strings" + "time" + + "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" +) + +const ( + maxQuerySeries = 50 + maxQueryPoints = 1500 +) + +// Selector picks series by service, name and exact label matches, and groups +// them by the GroupBy labels ("service" and "instance" work as labels too). +type Selector struct { + Service string + Name string + Match map[string]string + GroupBy []string +} + +// Aggregations supported by Query and Totals. +var Aggregations = []string{"sum", "count", "rate", "avg", "min", "max", "last", "p50", "p90", "p95", "p99"} + +func validAgg(agg string) bool { + for _, a := range Aggregations { + if a == agg { + return true + } + } + return false +} + +// DefaultAgg picks a sensible aggregation for a metric kind. +func DefaultAgg(kind Kind) string { + switch kind { + case Counter: + return "sum" + case Gauge: + return "last" + case Histogram: + return "p95" + default: + return "avg" + } +} + +// Point is [unix seconds, value or null] in JSON. +type Point struct { + T int64 + V *float64 +} + +func (p Point) MarshalJSON() ([]byte, error) { + if p.V == nil { + return []byte("[" + strconv.FormatInt(p.T, 10) + ",null]"), nil + } + return []byte("[" + strconv.FormatInt(p.T, 10) + "," + strconv.FormatFloat(*p.V, 'g', 10, 64) + "]"), nil +} + +type SeriesResult struct { + Labels map[string]string `json:"labels"` + Points []Point `json:"points"` +} + +type QueryResult struct { + Name string `json:"name"` + Kind Kind `json:"kind"` + Agg string `json:"agg"` + Resolution int64 `json:"resolution"` + Step int64 `json:"step"` + From int64 `json:"from"` + To int64 `json:"to"` + Series []SeriesResult `json:"series"` + Truncated bool `json:"truncated"` +} + +// matched is a series selected for a query, possibly only known from the DB. +type matched struct { + id uint + live *Series // nil when this process never saw the series + labels map[string]string + svc string + inst string + kind Kind + bounds []float64 +} + +func (m *matched) label(name string) string { + switch name { + case "service": + if v, ok := m.labels["service"]; ok { + return v + } + return m.svc + case "instance": + if v, ok := m.labels["instance"]; ok { + return v + } + return m.inst + } + return m.labels[name] +} + +func (sel Selector) matches(m *matched) bool { + if sel.Service != "" && m.svc != sel.Service { + return false + } + for k, v := range sel.Match { + if m.label(k) != v { + return false + } + } + return true +} + +func findSeries(sel Selector) ([]*matched, error) { + byKey := map[seriesKey]*matched{} + + q := initializers.DB.Model(&models.MetricSeries{}).Where("name = ?", sel.Name) + if sel.Service != "" { + q = q.Where("service = ?", sel.Service) + } + var rows []models.MetricSeries + if err := q.Find(&rows).Error; err != nil { + return nil, err + } + for _, r := range rows { + m := &matched{id: r.ID, svc: r.Service, inst: r.Instance, kind: Kind(r.Kind), labels: map[string]string{}} + _ = json.Unmarshal([]byte(r.Labels), &m.labels) + if r.Bounds != "" { + _ = json.Unmarshal([]byte(r.Bounds), &m.bounds) + } + byKey[seriesKey{r.Service, r.Instance, r.Name, r.Labels}] = m + } + + for _, s := range reg.all() { + if s.Name != sel.Name || (sel.Service != "" && s.Service != sel.Service) { + continue + } + k := s.key() + if m := byKey[k]; m != nil { + m.live = s + continue + } + byKey[k] = &matched{live: s, svc: s.Service, inst: s.Instance, kind: s.Kind, bounds: s.Bounds, labels: s.Labels} + } + + out := make([]*matched, 0, len(byKey)) + for _, m := range byKey { + if sel.matches(m) { + out = append(out, m) + } + } + return out, nil +} + +type groupAcc struct { + labels map[string]string + steps map[int64]*Agg + lastSum map[int64]float64 +} + +// collect loads all matching data in [from, to] and folds it into groups and +// step buckets chosen by bucketFn. Gauge "last" values are summed across the +// series of a group (e.g. queue depth over all instances). +func collect(sel Selector, from, to, res int64, bucketFn func(int64) int64) (map[string]*groupAcc, Kind, []float64, error) { + series, err := findSeries(sel) + if err != nil { + return nil, "", nil, err + } + if len(series) == 0 { + return map[string]*groupAcc{}, "", nil, nil + } + kind, bounds := series[0].kind, series[0].bounds + + perSeries := map[*matched]map[int64]*Agg{} + add := func(m *matched, ts int64, a *Agg) { + b := bucketFn(ts) + steps := perSeries[m] + if steps == nil { + steps = map[int64]*Agg{} + perSeries[m] = steps + } + if cur := steps[b]; cur != nil { + cur.merge(a) + } else { + steps[b] = a.clone() + } + } + + byID := map[uint]*matched{} + var ids []uint + for _, m := range series { + if m.id != 0 { + byID[m.id] = m + ids = append(ids, m.id) + } + } + + load := func(resolution int, lo, hi int64) error { + if lo > hi || len(ids) == 0 { + return nil + } + for start := 0; start < len(ids); start += 300 { + end := min(start+300, len(ids)) + var rows []models.MetricPoint + err := initializers.DB. + Where("resolution = ? AND series_id IN ? AND bucket >= ? AND bucket <= ?", resolution, ids[start:end], lo, hi). + Find(&rows).Error + if err != nil { + return err + } + for i := range rows { + add(byID[rows[i].SeriesID], rows[i].Bucket, pointToAgg(&rows[i])) + } + } + return nil + } + + if res == resHour { + cut := rolledUntil.Load() + if err := load(resHour, hourOf(from), cut-1); err != nil { + return nil, "", nil, err + } + if err := load(resMinute, max(cut, from), to); err != nil { + return nil, "", nil, err + } + } else if err := load(resMinute, minuteOf(from), to); err != nil { + return nil, "", nil, err + } + + // not yet flushed buckets + live := map[*Series]*matched{} + want := map[*Series]bool{} + for _, m := range series { + if m.live != nil { + live[m.live] = m + want[m.live] = true + } + } + for k, a := range defaultAgg.snapshotOpen(want) { + if k.bucket >= minuteOf(from) && k.bucket <= to { + add(live[k.s], k.bucket, a) + } + } + + groups := map[string]*groupAcc{} + for m, steps := range perSeries { + labels := make(map[string]string, len(sel.GroupBy)) + parts := make([]string, len(sel.GroupBy)) + for i, g := range sel.GroupBy { + labels[g] = m.label(g) + parts[i] = labels[g] + } + gk := strings.Join(parts, "\x00") + g := groups[gk] + if g == nil { + g = &groupAcc{labels: labels, steps: map[int64]*Agg{}, lastSum: map[int64]float64{}} + groups[gk] = g + } + for b, a := range steps { + if cur := g.steps[b]; cur != nil { + cur.merge(a) + } else { + g.steps[b] = a.clone() + } + g.lastSum[b] += a.Last + } + } + return groups, kind, bounds, nil +} + +func evaluate(a *Agg, lastSum float64, agg string, kind Kind, bounds []float64, span int64) (float64, bool) { + if a == nil || a.Count == 0 { + return 0, false + } + switch agg { + case "count": + return float64(a.Count), true + case "sum": + return a.Sum, true + case "rate": + if span <= 0 { + return 0, false + } + if kind == Counter { + return a.Sum / float64(span), true + } + return float64(a.Count) / float64(span), true + case "avg": + return a.Sum / float64(a.Count), true + case "min": + return a.Min, true + case "max": + return a.Max, true + case "last": + return lastSum, true + case "p50", "p90", "p95", "p99": + q, _ := strconv.ParseFloat(agg[1:], 64) + return quantile(a, bounds, q/100) + } + return 0, false +} + +func pickResolution(from, to, step int64) int64 { + minuteHorizon := time.Now().Unix() - int64(initializers.MetricsMinuteRetentionDays)*86400 + if to-from > 48*3600 || step >= resHour || from < minuteHorizon { + return resHour + } + return resMinute +} + +// Query returns one time series per group with values at every step. +func Query(sel Selector, agg string, from, to, step int64) (*QueryResult, error) { + if sel.Name == "" { + return nil, fmt.Errorf("name is required") + } + if to <= from { + return nil, fmt.Errorf("to must be after from") + } + res := pickResolution(from, to, step) + if step < res { + step = res + } + if (to-from)/step > maxQueryPoints { + step = (to - from) / maxQueryPoints + } + step = int64(math.Ceil(float64(step)/float64(res))) * res + align := func(ts int64) int64 { return ts - ((ts%step)+step)%step } + + groups, kind, bounds, err := collect(sel, from, to, res, align) + if err != nil { + return nil, err + } + if agg == "" { + agg = DefaultAgg(kind) + } + if !validAgg(agg) { + return nil, fmt.Errorf("unknown aggregation %q", agg) + } + + result := &QueryResult{Name: sel.Name, Kind: kind, Agg: agg, Resolution: res, Step: step, From: from, To: to, Series: []SeriesResult{}} + keys := make([]string, 0, len(groups)) + for k := range groups { + keys = append(keys, k) + } + sort.Strings(keys) + if len(keys) > maxQuerySeries { + keys = keys[:maxQuerySeries] + result.Truncated = true + } + for _, k := range keys { + g := groups[k] + sr := SeriesResult{Labels: g.labels} + for b := align(from); b <= to; b += step { + p := Point{T: b} + if v, ok := evaluate(g.steps[b], g.lastSum[b], agg, kind, bounds, step); ok && !math.IsNaN(v) && !math.IsInf(v, 0) { + p.V = &v + } + sr.Points = append(sr.Points, p) + } + result.Series = append(result.Series, sr) + } + return result, nil +} + +// Total is the whole-range aggregate of one group. +type Total struct { + Labels map[string]string `json:"labels"` + Count int64 `json:"count"` + Sum float64 `json:"sum"` + Min *float64 `json:"min"` + Max *float64 `json:"max"` + Avg *float64 `json:"avg"` + Last *float64 `json:"last"` + Rate *float64 `json:"rate"` + P50 *float64 `json:"p50"` + P95 *float64 `json:"p95"` + P99 *float64 `json:"p99"` +} + +// Totals aggregates [from, to] into one value set per group. +func Totals(sel Selector, from, to int64) ([]Total, error) { + res := pickResolution(from, to, 0) + groups, kind, bounds, err := collect(sel, from, to, res, func(int64) int64 { return 0 }) + if err != nil { + return nil, err + } + span := to - from + out := make([]Total, 0, len(groups)) + for _, g := range groups { + a := g.steps[0] + if a == nil { + continue + } + t := Total{Labels: g.labels, Count: a.Count, Sum: a.Sum} + val := func(agg string) *float64 { + if v, ok := evaluate(a, g.lastSum[0], agg, kind, bounds, span); ok && !math.IsNaN(v) && !math.IsInf(v, 0) { + return &v + } + return nil + } + t.Min, t.Max, t.Avg, t.Last, t.Rate = val("min"), val("max"), val("avg"), val("last"), val("rate") + t.P50, t.P95, t.P99 = val("p50"), val("p95"), val("p99") + out = append(out, t) + } + sort.Slice(out, func(i, j int) bool { return out[i].Count > out[j].Count }) + return out, nil +} + +// Sum returns the summed value of a counter (or count of observations for +// other kinds) over the range, across all matching series. +func Sum(sel Selector, from, to int64) float64 { + sel.GroupBy = nil + totals, err := Totals(sel, from, to) + if err != nil || len(totals) == 0 { + return 0 + } + if f := familyByName(sel.Name); f != nil && f.Kind != Counter { + return float64(totals[0].Count) + } + return totals[0].Sum +} + +// CatalogEntry describes one metric name of one service. +type CatalogEntry struct { + Service string `json:"service"` + Name string `json:"name"` + Kind Kind `json:"kind"` + Help string `json:"help"` + Labels map[string][]string `json:"labels"` + Instances []string `json:"instances"` + Series int `json:"series"` + LastSeen int64 `json:"lastSeen"` +} + +// Catalog lists every metric known in the DB or in memory. +func Catalog(service string) ([]CatalogEntry, error) { + type acc struct { + e CatalogEntry + labels map[string]map[string]bool + inst map[string]bool + keys map[seriesKey]bool + } + entries := map[string]*acc{} + addSeries := func(svc, inst, name string, kind Kind, help, labelsJSON string, labels map[string]string, lastSeen int64) { + k := svc + "/" + name + a := entries[k] + if a == nil { + a = &acc{e: CatalogEntry{Service: svc, Name: name, Kind: kind, Help: help}, labels: map[string]map[string]bool{}, inst: map[string]bool{}, keys: map[seriesKey]bool{}} + entries[k] = a + } + if a.e.Help == "" { + a.e.Help = help + } + sk := seriesKey{svc, inst, name, labelsJSON} + if !a.keys[sk] { + a.keys[sk] = true + a.e.Series++ + } + a.inst[inst] = true + for lk, lv := range labels { + if a.labels[lk] == nil { + a.labels[lk] = map[string]bool{} + } + a.labels[lk][lv] = true + } + if lastSeen > a.e.LastSeen { + a.e.LastSeen = lastSeen + } + } + + q := initializers.DB.Model(&models.MetricSeries{}) + if service != "" { + q = q.Where("service = ?", service) + } + var rows []models.MetricSeries + if err := q.Find(&rows).Error; err != nil { + return nil, err + } + for _, r := range rows { + labels := map[string]string{} + _ = json.Unmarshal([]byte(r.Labels), &labels) + addSeries(r.Service, r.Instance, r.Name, Kind(r.Kind), r.Help, r.Labels, labels, r.LastSeen) + } + for _, s := range reg.all() { + if service != "" && s.Service != service { + continue + } + addSeries(s.Service, s.Instance, s.Name, s.Kind, s.Help, s.LabelsJSON, s.Labels, reg.seen(s)) + } + + out := make([]CatalogEntry, 0, len(entries)) + for _, a := range entries { + a.e.Labels = map[string][]string{} + for lk, vals := range a.labels { + list := make([]string, 0, len(vals)) + for v := range vals { + list = append(list, v) + } + sort.Strings(list) + a.e.Labels[lk] = list + } + for inst := range a.inst { + a.e.Instances = append(a.e.Instances, inst) + } + sort.Strings(a.e.Instances) + out = append(out, a.e) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Service != out[j].Service { + return out[i].Service < out[j].Service + } + return out[i].Name < out[j].Name + }) + return out, nil +} diff --git a/metrics/registry.go b/metrics/registry.go new file mode 100644 index 0000000..360377d --- /dev/null +++ b/metrics/registry.go @@ -0,0 +1,284 @@ +package metrics + +import ( + "encoding/json" + "errors" + "os" + "sort" + "strings" + "sync" + "time" +) + +type Kind string + +const ( + Counter Kind = "counter" // value is an increment; Sum holds the total + Gauge Kind = "gauge" // value is a sample; Last holds the latest + Summary Kind = "summary" // pre-aggregated count/sum/min/max from a client + Histogram Kind = "histogram" // observations bucketed by Bounds +) + +// MainService is the service name the API uses for its own metrics. +const MainService = "main" + +var ( + errSeriesLimit = errors.New("series limit reached") + errKindMismatch = errors.New("metric reported with a different type") +) + +// Limits for series created from external telemetry. +const ( + maxSeriesPerName = 200 + maxExternal = 5000 +) + +var hostname = func() string { + h, err := os.Hostname() + if err != nil || h == "" { + return "api" + } + return h +}() + +// Instance returns the instance name used for this process' own metrics. +func Instance() string { return hostname } + +// Series is one metric name + label set from one service instance. Series are +// interned, so pointer equality means same series. +type Series struct { + Service string + Instance string + Name string + Labels map[string]string + LabelsJSON string + Kind Kind + Bounds []float64 + Help string +} + +type seriesKey struct{ service, instance, name, labels string } + +func (s *Series) key() seriesKey { + return seriesKey{s.Service, s.Instance, s.Name, s.LabelsJSON} +} + +// Label returns a label value, including the service/instance pseudo labels. +func (s *Series) Label(name string) string { + switch name { + case "service": + if v, ok := s.Labels["service"]; ok { + return v + } + return s.Service + case "instance": + if v, ok := s.Labels["instance"]; ok { + return v + } + return s.Instance + } + return s.Labels[name] +} + +// canonicalLabels serializes labels deterministically (encoding/json sorts map keys). +func canonicalLabels(labels map[string]string) string { + if len(labels) == 0 { + return "{}" + } + b, _ := json.Marshal(labels) + return string(b) +} + +type registry struct { + mu sync.RWMutex + series map[seriesKey]*Series + perName map[string]int // external series per service/name + external int + lastSeen map[*Series]int64 +} + +var reg = ®istry{ + series: make(map[seriesKey]*Series), + perName: make(map[string]int), + lastSeen: make(map[*Series]int64), +} + +func (r *registry) get(service, instance, name string, labels map[string]string, kind Kind, bounds []float64, help string, external bool) (*Series, error) { + lj := canonicalLabels(labels) + k := seriesKey{service, instance, name, lj} + + r.mu.RLock() + s := r.series[k] + r.mu.RUnlock() + if s != nil { + if s.Kind != kind { + return nil, errKindMismatch + } + return s, nil + } + + r.mu.Lock() + defer r.mu.Unlock() + if s = r.series[k]; s != nil { + return s, nil + } + nameKey := service + "/" + name + if external { + if r.external >= maxExternal || r.perName[nameKey] >= maxSeriesPerName { + return nil, errSeriesLimit + } + r.external++ + r.perName[nameKey]++ + } + copied := make(map[string]string, len(labels)) + for k, v := range labels { + copied[k] = v + } + s = &Series{ + Service: service, + Instance: instance, + Name: name, + Labels: copied, + LabelsJSON: lj, + Kind: kind, + Bounds: bounds, + Help: help, + } + r.series[k] = s + return s, nil +} + +func (r *registry) lookup(k seriesKey) *Series { + r.mu.RLock() + defer r.mu.RUnlock() + return r.series[k] +} + +// all returns every known series, sorted by name then labels. +func (r *registry) all() []*Series { + r.mu.RLock() + out := make([]*Series, 0, len(r.series)) + for _, s := range r.series { + out = append(out, s) + } + r.mu.RUnlock() + sort.Slice(out, func(i, j int) bool { + if out[i].Name != out[j].Name { + return out[i].Name < out[j].Name + } + if out[i].Service != out[j].Service { + return out[i].Service < out[j].Service + } + if out[i].Instance != out[j].Instance { + return out[i].Instance < out[j].Instance + } + return out[i].LabelsJSON < out[j].LabelsJSON + }) + return out +} + +func (r *registry) touch(s *Series, ts int64) { + r.mu.Lock() + if ts > r.lastSeen[s] { + r.lastSeen[s] = ts + } + r.mu.Unlock() +} + +func (r *registry) seen(s *Series) int64 { + r.mu.RLock() + defer r.mu.RUnlock() + return r.lastSeen[s] +} + +// Family is a metric emitted by this process, with a fixed set of label names. +type Family struct { + Name string + Help string + Kind Kind + Bounds []float64 + LabelNames []string + + mu sync.Mutex + cache map[string]*Series +} + +var ( + familiesMu sync.Mutex + families = map[string]*Family{} +) + +func newFamily(name, help string, kind Kind, bounds []float64, labels []string) *Family { + f := &Family{Name: name, Help: help, Kind: kind, Bounds: bounds, LabelNames: labels, cache: map[string]*Series{}} + familiesMu.Lock() + families[name] = f + familiesMu.Unlock() + return f +} + +func familyByName(name string) *Family { + familiesMu.Lock() + defer familiesMu.Unlock() + return families[name] +} + +func NewCounter(name, help string, labels ...string) *Family { + return newFamily(name, help, Counter, nil, labels) +} + +func NewGauge(name, help string, labels ...string) *Family { + return newFamily(name, help, Gauge, nil, labels) +} + +func NewHistogram(name, help string, bounds []float64, labels ...string) *Family { + return newFamily(name, help, Histogram, bounds, labels) +} + +func (f *Family) series(values []string) *Series { + key := strings.Join(values, "\x00") + f.mu.Lock() + defer f.mu.Unlock() + if s := f.cache[key]; s != nil { + return s + } + labels := make(map[string]string, len(f.LabelNames)) + for i, n := range f.LabelNames { + if i < len(values) { + labels[n] = values[i] + } else { + labels[n] = "" + } + } + s, err := reg.get(MainService, Instance(), f.Name, labels, f.Kind, f.Bounds, f.Help, false) + if err != nil { + return nil + } + f.cache[key] = s + return s +} + +// Add increments a counter. Zero increments are ignored. +func (f *Family) Add(v float64, labelValues ...string) { + if v == 0 { + return + } + if s := f.series(labelValues); s != nil { + defaultAgg.observe(s, time.Now().Unix(), v) + } +} + +func (f *Family) Inc(labelValues ...string) { f.Add(1, labelValues...) } + +// Set records a gauge sample. +func (f *Family) Set(v float64, labelValues ...string) { + if s := f.series(labelValues); s != nil { + defaultAgg.observe(s, time.Now().Unix(), v) + } +} + +// Observe records a histogram observation. +func (f *Family) Observe(v float64, labelValues ...string) { + if s := f.series(labelValues); s != nil { + defaultAgg.observe(s, time.Now().Unix(), v) + } +} diff --git a/metrics/services.go b/metrics/services.go new file mode 100644 index 0000000..4e90d18 --- /dev/null +++ b/metrics/services.go @@ -0,0 +1,325 @@ +package metrics + +import ( + "encoding/json" + "errors" + "fmt" + "math" + "regexp" + "sync" + "time" + + "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" + "gorm.io/gorm/clause" +) + +var nameRe = regexp.MustCompile(`^[a-z_][a-z0-9_]{0,63}$`) + +// ValidName reports whether a service, metric or label name is acceptable. +func ValidName(s string) bool { return nameRe.MatchString(s) } + +const ( + defaultHeartbeat = 30 + maxLabels = 8 + maxLabelValue = 128 +) + +// normalizeTs converts unix seconds or milliseconds to seconds and clamps it +// to [now-1h, now+1m]; anything outside becomes now. +func normalizeTs(ts int64, now int64) int64 { + if ts <= 0 { + return now + } + if ts > 1e12 { + ts /= 1000 + } + if ts < now-3600 || ts > now+60 { + return now + } + return ts +} + +// IngestResult reports what happened to a telemetry batch. +type IngestResult struct { + Accepted int `json:"accepted"` + Dropped int `json:"dropped"` + Reasons map[string]int `json:"reasons,omitempty"` +} + +// Ingest records external metric samples for a service instance. +func Ingest(service, instance string, samples []models.TelemetryMetric, now time.Time) IngestResult { + res := IngestResult{Reasons: map[string]int{}} + drop := func(reason string) { + res.Dropped++ + res.Reasons[reason]++ + } + nowTs := now.Unix() + + for _, m := range samples { + if !ValidName(m.Name) { + drop("invalid_name") + continue + } + if len(m.Labels) > maxLabels { + drop("too_many_labels") + continue + } + badLabel := false + for k, v := range m.Labels { + if !ValidName(k) || len(v) > maxLabelValue { + badLabel = true + break + } + } + if badLabel { + drop("invalid_label") + continue + } + var kind Kind + switch m.Type { + case "counter": + kind = Counter + case "gauge", "": + kind = Gauge + case "summary": + kind = Summary + default: + drop("invalid_type") + continue + } + if math.IsNaN(m.Value) || math.IsInf(m.Value, 0) || math.IsNaN(m.Sum) || math.IsInf(m.Sum, 0) { + drop("invalid_value") + continue + } + if kind == Counter && m.Value < 0 { + drop("negative_counter") + continue + } + if kind == Summary && m.Count <= 0 { + drop("empty_summary") + continue + } + help := m.Help + if len(help) > 256 { + help = help[:256] + } + s, err := reg.get(service, instance, m.Name, m.Labels, kind, nil, help, true) + if err != nil { + if errors.Is(err, errKindMismatch) { + drop("type_mismatch") + } else { + drop("series_limit") + } + continue + } + ts := normalizeTs(m.Ts, nowTs) + if kind == Summary { + lo, hi := m.Min, m.Max + if lo == 0 && hi == 0 { + avg := m.Sum / float64(m.Count) + lo, hi = avg, avg + } + defaultAgg.observeSummary(s, ts, m.Count, m.Sum, lo, hi) + } else { + defaultAgg.observe(s, ts, m.Value) + } + res.Accepted++ + } + TelemetrySamples.Add(float64(res.Accepted), "accepted") + TelemetrySamples.Add(float64(res.Dropped), "dropped") + return res +} + +func jsonString(v map[string]any) string { + if len(v) == 0 { + return "" + } + b, err := json.Marshal(v) + if err != nil { + return "" + } + return string(b) +} + +func truncate(s string, n int) string { + if len(s) > n { + return s[:n] + } + return s +} + +// Heartbeat upserts the service instance row. +func Heartbeat(b *models.TelemetryBatch, remoteAddr string, now time.Time) error { + interval := b.HeartbeatIntervalSec + if interval <= 0 { + interval = defaultHeartbeat + } + status := b.Status + if status == "" { + status = "ok" + } + row := models.ServiceInstance{ + Service: b.Service, + Instance: b.Instance, + Version: truncate(b.Version, 64), + Status: truncate(status, 16), + StatusMessage: truncate(b.StatusMessage, 512), + StartedAt: normalizeStarted(b.StartedAt), + LastSeen: now.Unix(), + HeartbeatIntervalSec: interval, + RemoteAddr: truncate(remoteAddr, 64), + Meta: jsonString(b.Meta), + } + return initializers.DB.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "service"}, {Name: "instance"}}, + DoUpdates: clause.AssignmentColumns([]string{ + "version", "status", "status_message", "started_at", "last_seen", + "heartbeat_interval_sec", "remote_addr", "meta", "updated_at", + }), + }).Create(&row).Error +} + +func normalizeStarted(ts int64) int64 { + if ts > 1e12 { + return ts / 1000 + } + return ts +} + +// RecordEvents stores service events. +func RecordEvents(service, instance string, events []models.TelemetryEvent, now time.Time) error { + if len(events) == 0 { + return nil + } + rows := make([]models.ServiceEvent, 0, len(events)) + for _, e := range events { + level := e.Level + switch level { + case "debug", "info", "warn", "error": + case "warning": + level = "warn" + default: + level = "info" + } + ts := e.Ts + if ts <= 0 { + ts = now.UnixMilli() + } else if ts < 1e12 { + ts *= 1000 + } + rows = append(rows, models.ServiceEvent{ + Service: service, Instance: instance, Level: level, + Message: truncate(e.Message, 1024), Fields: jsonString(e.Fields), TS: ts, + }) + } + return initializers.DB.CreateInBatches(&rows, 100).Error +} + +// Event records a single event from the API itself. +func Event(level, message string, fields map[string]any) { + err := RecordEvents(MainService, Instance(), []models.TelemetryEvent{{Level: level, Message: message, Fields: fields}}, time.Now()) + if err != nil { + fmt.Printf("metrics: failed to record event: %v\n", err) + } +} + +// Online reports whether an instance has sent a heartbeat recently enough. +func Online(inst *models.ServiceInstance, now time.Time) bool { + interval := inst.HeartbeatIntervalSec + if interval <= 0 { + interval = defaultHeartbeat + } + return now.Unix()-inst.LastSeen < int64(3*interval) +} + +var ( + upMu sync.Mutex + upState = map[string]bool{} // "service/instance" -> online +) + +// ServiceUp returns the last known online state of every instance. +func ServiceUp() map[[2]string]bool { + upMu.Lock() + defer upMu.Unlock() + out := make(map[[2]string]bool, len(upState)) + for k, v := range upState { + var svc, inst string + for i := 0; i < len(k); i++ { + if k[i] == '/' { + svc, inst = k[:i], k[i+1:] + break + } + } + out[[2]string{svc, inst}] = v + } + return out +} + +// checkServices records online/offline transitions as events. +func checkServices(now time.Time) { + var rows []models.ServiceInstance + if err := initializers.DB.Find(&rows).Error; err != nil { + return + } + upMu.Lock() + known := make(map[string]bool, len(rows)) + type change struct { + inst models.ServiceInstance + online bool + } + var changes []change + for _, r := range rows { + k := r.Service + "/" + r.Instance + known[k] = true + online := Online(&r, now) + prev, seen := upState[k] + upState[k] = online + if seen && prev != online { + changes = append(changes, change{r, online}) + } + } + for k := range upState { + if !known[k] { + delete(upState, k) + } + } + upMu.Unlock() + + for _, ch := range changes { + level, msg := "info", "back online" + if !ch.online { + level, msg = "warn", "heartbeat lost" + } + _ = RecordEvents(ch.inst.Service, ch.inst.Instance, []models.TelemetryEvent{{ + Level: level, Message: msg, + Fields: map[string]any{"kind": "status", "lastSeen": ch.inst.LastSeen}, + }}, now) + } +} + +var ( + buildMu sync.Mutex + buildVersion = "dev" + startedAt = time.Now() +) + +// SetVersion sets the version reported in the API's own heartbeat. +func SetVersion(v string) { + buildMu.Lock() + buildVersion = v + buildMu.Unlock() +} + +// StartedAt is when this process started. +func StartedAt() time.Time { return startedAt } + +func selfHeartbeat(now time.Time) { + buildMu.Lock() + v := buildVersion + buildMu.Unlock() + _ = Heartbeat(&models.TelemetryBatch{ + Service: MainService, Instance: Instance(), Version: v, + StartedAt: startedAt.Unix(), HeartbeatIntervalSec: 30, Status: "ok", + }, "", now) +} diff --git a/metrics/store.go b/metrics/store.go new file mode 100644 index 0000000..d669800 --- /dev/null +++ b/metrics/store.go @@ -0,0 +1,371 @@ +package metrics + +import ( + "encoding/json" + "fmt" + "log" + "sort" + "sync" + "sync/atomic" + "time" + + "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +const ( + resMinute = 60 + resHour = 3600 + // points that cannot be written for this long are dropped + maxRetryAge = 15 * 60 +) + +var ( + idMu sync.Mutex + seriesID = map[*Series]uint{} + + // rolledUntil is the start of the first hour without an hourly rollup. + rolledUntil atomic.Int64 + // flushMu serializes writers of metric_points (flush, rollup, retention). + flushMu sync.Mutex +) + +func pointToAgg(p *models.MetricPoint) *Agg { + a := &Agg{Count: p.Count, Sum: p.Sum, Min: p.Min, Max: p.Max, Last: p.Last, LastTs: p.Bucket} + if p.Hist != "" { + _ = json.Unmarshal([]byte(p.Hist), &a.Hist) + } + return a +} + +func aggToPoint(id uint, res int, bucket int64, a *Agg) models.MetricPoint { + return models.MetricPoint{ + SeriesID: id, Resolution: res, Bucket: bucket, + Count: a.Count, Sum: a.Sum, Min: a.Min, Max: a.Max, Last: a.Last, Hist: a.histJSON(), + } +} + +// resolveIDs makes sure every series has a metric_series row. +func resolveIDs(list []*Series) (map[*Series]uint, error) { + out := make(map[*Series]uint, len(list)) + var missing []*Series + idMu.Lock() + for _, s := range list { + if id, ok := seriesID[s]; ok { + out[s] = id + } else { + missing = append(missing, s) + } + } + idMu.Unlock() + + for _, s := range missing { + bounds := "" + if len(s.Bounds) > 0 { + b, _ := json.Marshal(s.Bounds) + bounds = string(b) + } + row := models.MetricSeries{ + Service: s.Service, Instance: s.Instance, Name: s.Name, Labels: s.LabelsJSON, + Kind: string(s.Kind), Bounds: bounds, Help: s.Help, LastSeen: time.Now().Unix(), + } + if err := initializers.DB.Clauses(clause.OnConflict{DoNothing: true}).Create(&row).Error; err != nil { + return nil, err + } + if row.ID == 0 { + if err := initializers.DB. + Where("service = ? AND instance = ? AND name = ? AND labels = ?", s.Service, s.Instance, s.Name, s.LabelsJSON). + First(&row).Error; err != nil { + return nil, err + } + } + idMu.Lock() + seriesID[s] = row.ID + idMu.Unlock() + out[s] = row.ID + } + return out, nil +} + +type pointID struct { + id uint + bucket int64 +} + +// persist merges the given buckets into existing rows and upserts them. The +// read-merge-write runs in one transaction; this process is the only writer. +func persist(tx *gorm.DB, res int, points map[pointID]*Agg) error { + if len(points) == 0 { + return nil + } + keys := make([]pointID, 0, len(points)) + for k := range points { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { + if keys[i].id != keys[j].id { + return keys[i].id < keys[j].id + } + return keys[i].bucket < keys[j].bucket + }) + + for start := 0; start < len(keys); start += 400 { + end := min(start+400, len(keys)) + chunk := keys[start:end] + idSet, bucketSet := map[uint]bool{}, map[int64]bool{} + for _, k := range chunk { + idSet[k.id] = true + bucketSet[k.bucket] = true + } + ids := make([]uint, 0, len(idSet)) + for id := range idSet { + ids = append(ids, id) + } + buckets := make([]int64, 0, len(bucketSet)) + for b := range bucketSet { + buckets = append(buckets, b) + } + + var existing []models.MetricPoint + if err := tx.Where("resolution = ? AND series_id IN ? AND bucket IN ?", res, ids, buckets).Find(&existing).Error; err != nil { + return err + } + prev := make(map[pointID]*Agg, len(existing)) + for i := range existing { + prev[pointID{existing[i].SeriesID, existing[i].Bucket}] = pointToAgg(&existing[i]) + } + + rows := make([]models.MetricPoint, 0, len(chunk)) + for _, k := range chunk { + a := points[k] + if old := prev[k]; old != nil { + old.merge(a) + a = old + } + rows = append(rows, aggToPoint(k.id, res, k.bucket, a)) + } + err := tx.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "series_id"}, {Name: "resolution"}, {Name: "bucket"}}, + DoUpdates: clause.AssignmentColumns([]string{"count", "sum", "min", "max", "last", "hist"}), + }).CreateInBatches(&rows, 200).Error + if err != nil { + return err + } + } + return nil +} + +type pipelineStatus struct { + mu sync.Mutex + LastFlush int64 `json:"lastFlush"` + LastFlushMs int64 `json:"lastFlushMs"` + LastFlushErr string `json:"lastFlushError"` + LastRollup int64 `json:"lastRollup"` + LastRetention int64 `json:"lastRetention"` + Dropped int64 `json:"droppedPoints"` +} + +var status pipelineStatus + +// flush writes every bucket that started before `before`. +func flush(before int64) error { + flushMu.Lock() + defer flushMu.Unlock() + + started := time.Now() + points := defaultAgg.takeClosed(before) + if len(points) == 0 { + return nil + } + + fail := func(err error) error { + now := time.Now().Unix() + keep := make(map[pointKey]*Agg) + var dropped int64 + for k, a := range points { + if now-k.bucket <= maxRetryAge { + keep[k] = a + } else { + dropped++ + } + } + defaultAgg.putBack(keep) + status.mu.Lock() + status.LastFlushErr = err.Error() + status.Dropped += dropped + status.mu.Unlock() + if dropped > 0 { + PointsDropped.Add(float64(dropped)) + } + return err + } + + set := map[*Series]bool{} + for k := range points { + set[k.s] = true + } + list := make([]*Series, 0, len(set)) + for s := range set { + list = append(list, s) + } + ids, err := resolveIDs(list) + if err != nil { + return fail(fmt.Errorf("resolve series: %w", err)) + } + + rows := make(map[pointID]*Agg, len(points)) + for k, a := range points { + rows[pointID{ids[k.s], k.bucket}] = a + } + err = initializers.DB.Transaction(func(tx *gorm.DB) error { + return persist(tx, resMinute, rows) + }) + if err != nil { + return fail(fmt.Errorf("write points: %w", err)) + } + + // last_seen lets the catalog hide series that stopped reporting + idList := make([]uint, 0, len(ids)) + for _, id := range ids { + idList = append(idList, id) + } + for start := 0; start < len(idList); start += 500 { + end := min(start+500, len(idList)) + initializers.DB.Model(&models.MetricSeries{}).Where("id IN ?", idList[start:end]). + Update("last_seen", time.Now().Unix()) + } + + status.mu.Lock() + status.LastFlush = time.Now().Unix() + status.LastFlushMs = time.Since(started).Milliseconds() + status.LastFlushErr = "" + status.mu.Unlock() + return nil +} + +func hourOf(ts int64) int64 { return ts - ts%resHour } + +// rollup aggregates complete hours of minute data into hourly points. The last +// rolled hour is redone each time to pick up late telemetry; it is idempotent. +func rollup(now int64) error { + flushMu.Lock() + defer flushMu.Unlock() + + // the minute data of an hour is complete ~1 flush after the hour ends + lastFull := hourOf(now - 2*resMinute) + start := rolledUntil.Load() + if start == 0 { + var maxHour, minMinute *int64 + initializers.DB.Model(&models.MetricPoint{}).Where("resolution = ?", resHour).Select("MAX(bucket)").Scan(&maxHour) + if maxHour != nil { + start = *maxHour + resHour + } else { + initializers.DB.Model(&models.MetricPoint{}).Where("resolution = ?", resMinute).Select("MIN(bucket)").Scan(&minMinute) + if minMinute == nil { + status.mu.Lock() + status.LastRollup = now + status.mu.Unlock() + return nil + } + start = hourOf(*minMinute) + } + } + from := start - resHour + if earliest := hourOf(now - int64(initializers.MetricsMinuteRetentionDays)*86400); from < earliest { + from = earliest + } + + for h := from; h < lastFull; h += resHour { + var rows []models.MetricPoint + err := initializers.DB.Where("resolution = ? AND bucket >= ? AND bucket < ?", resMinute, h, h+resHour). + Order("bucket asc").Find(&rows).Error + if err != nil { + return err + } + if len(rows) == 0 { + continue + } + merged := map[pointID]*Agg{} + for i := range rows { + k := pointID{rows[i].SeriesID, h} + a := merged[k] + if a == nil { + a = &Agg{} + merged[k] = a + } + a.merge(pointToAgg(&rows[i])) + } + // replace (not merge) so re-rolling an hour is idempotent + out := make([]models.MetricPoint, 0, len(merged)) + for k, a := range merged { + out = append(out, aggToPoint(k.id, resHour, k.bucket, a)) + } + err = initializers.DB.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "series_id"}, {Name: "resolution"}, {Name: "bucket"}}, + DoUpdates: clause.AssignmentColumns([]string{"count", "sum", "min", "max", "last", "hist"}), + }).CreateInBatches(&out, 200).Error + if err != nil { + return err + } + } + if lastFull > rolledUntil.Load() { + rolledUntil.Store(lastFull) + } + status.mu.Lock() + status.LastRollup = now + status.mu.Unlock() + return nil +} + +// retention deletes data older than the configured windows. +func retention(now time.Time) error { + flushMu.Lock() + defer flushMu.Unlock() + + day := int64(86400) + ts := now.Unix() + steps := []struct { + what string + run func() error + }{ + {"minute points", func() error { + return initializers.DB.Where("resolution = ? AND bucket < ?", resMinute, ts-int64(initializers.MetricsMinuteRetentionDays)*day). + Delete(&models.MetricPoint{}).Error + }}, + {"hour points", func() error { + return initializers.DB.Where("resolution = ? AND bucket < ?", resHour, ts-int64(initializers.MetricsHourRetentionDays)*day). + Delete(&models.MetricPoint{}).Error + }}, + {"events", func() error { + return initializers.DB.Where("ts < ?", (ts-int64(initializers.EventsRetentionDays)*day)*1000). + Delete(&models.ServiceEvent{}).Error + }}, + {"top items", func() error { + cutoff := now.UTC().AddDate(0, 0, -initializers.TopItemsRetentionDays).Format("2006-01-02") + return initializers.DB.Where("day < ?", cutoff).Delete(&models.DailyTopItem{}).Error + }}, + {"stale series", func() error { + stale := ts - 30*day + res := initializers.DB.Where("last_seen < ? AND id NOT IN (SELECT DISTINCT series_id FROM metric_points)", stale). + Delete(&models.MetricSeries{}) + if res.Error == nil && res.RowsAffected > 0 { + // forget cached ids so a revived series gets a new row + idMu.Lock() + seriesID = map[*Series]uint{} + idMu.Unlock() + } + return res.Error + }}, + } + for _, step := range steps { + if err := step.run(); err != nil { + log.Printf("metrics retention (%s): %v", step.what, err) + } + } + status.mu.Lock() + status.LastRetention = now.Unix() + status.mu.Unlock() + return nil +} diff --git a/metrics/traffic.go b/metrics/traffic.go new file mode 100644 index 0000000..b8b7012 --- /dev/null +++ b/metrics/traffic.go @@ -0,0 +1,328 @@ +package metrics + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "log" + "sort" + "strconv" + "strings" + "sync" + "time" + + "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" + "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" + "github.com/gin-gonic/gin" + "gorm.io/gorm/clause" +) + +// Middleware records latency and traffic for every request except static admin +// UI files and health checks. +func Middleware() gin.HandlerFunc { + return func(c *gin.Context) { + path := c.Request.URL.Path + if c.Request.Method == "OPTIONS" || path == "/health" || strings.HasPrefix(path, "/admin/ui") { + c.Next() + return + } + start := time.Now() + c.Next() + ms := float64(time.Since(start).Microseconds()) / 1000 + + route := c.FullPath() + if route == "" { + route = "(unmatched)" + } + status := c.Writer.Status() + group := routeGroup(path) + client := "direct" + if c.GetHeader("Origin") != "" || c.GetHeader("Referer") != "" { + client = "web" + } + HTTPDuration.Observe(ms, c.Request.Method, route, strconv.Itoa(status/100)+"xx", group, client) + if size := c.Writer.Size(); size > 0 { + HTTPResponseBytes.Add(float64(size), group) + } + + if group == "v1" && status < 500 { + visitors.add(c.ClientIP(), c.Request.UserAgent()) + } + if status == 200 { + switch route { + case "/v1/addon/:id": + Top.Add("addon", c.Param("id")) + case "/v1/getPossible": + Top.Add("search", normalizeSearch(c.Query("q"))) + } + } + } +} + +func routeGroup(path string) string { + for _, g := range []string{"v1", "back", "admin", "metrics"} { + if path == "/"+g || strings.HasPrefix(path, "/"+g+"/") { + return g + } + } + return "other" +} + +func normalizeSearch(q string) string { + q = strings.Join(strings.Fields(strings.ToLower(q)), " ") + if len(q) > 64 { + q = q[:64] + } + return q +} + +// visitorSet counts distinct (IP, user agent) pairs per UTC day. Only salted +// 64-bit hashes are kept and the salt never leaves memory. +type visitorSet struct { + mu sync.Mutex + day string + salt []byte + humans map[uint64]struct{} + bots map[uint64]struct{} +} + +var visitors = &visitorSet{} + +const maxVisitors = 2_000_000 + +func (v *visitorSet) rotate(day string) { + v.day = day + v.salt = make([]byte, 32) + _, _ = rand.Read(v.salt) + v.humans = make(map[uint64]struct{}) + v.bots = make(map[uint64]struct{}) +} + +func isBot(ua string) bool { + ua = strings.ToLower(ua) + if ua == "" { + return true + } + for _, s := range []string{"bot", "spider", "crawl", "curl", "wget", "python", "go-http", "headless", "preview", "monitor"} { + if strings.Contains(ua, s) { + return true + } + } + return false +} + +func (v *visitorSet) add(ip, ua string) { + day := time.Now().UTC().Format("2006-01-02") + v.mu.Lock() + defer v.mu.Unlock() + if v.day != day { + v.rotate(day) + } + h := sha256.New() + h.Write(v.salt) + h.Write([]byte(ip)) + h.Write([]byte{0}) + h.Write([]byte(ua)) + key := binary.BigEndian.Uint64(h.Sum(nil)[:8]) + set := v.humans + if isBot(ua) { + set = v.bots + } + if len(set) < maxVisitors { + set[key] = struct{}{} + } +} + +// Visitors returns today's approximate unique human and bot visitors. The set +// lives in memory, so after a restart the day's highest recorded value is used +// until the new set catches up. +func Visitors() (humans, bots int) { + now := time.Now().UTC() + day := now.Format("2006-01-02") + visitors.mu.Lock() + if visitors.day == day { + humans, bots = len(visitors.humans), len(visitors.bots) + } + visitors.mu.Unlock() + + dayStart := now.Truncate(24 * time.Hour).Unix() + if dayStart < startedAt.Unix() { + if t, err := Totals(Selector{Service: MainService, Name: "visitors_unique", GroupBy: []string{"kind"}}, dayStart, now.Unix()); err == nil { + for _, g := range t { + if g.Max == nil { + continue + } + switch g.Labels["kind"] { + case "human": + humans = max(humans, int(*g.Max)) + case "bot": + bots = max(bots, int(*g.Max)) + } + } + } + } + return humans, bots +} + +// liveVisitors is the in-memory count only, used when recording the gauge. +func liveVisitors() (humans, bots int) { + day := time.Now().UTC().Format("2006-01-02") + visitors.mu.Lock() + defer visitors.mu.Unlock() + if visitors.day != day { + return 0, 0 + } + return len(visitors.humans), len(visitors.bots) +} + +// topCounter counts requested items per UTC day and persists the most popular. +type topCounter struct { + mu sync.Mutex + day string + counts map[string]map[string]int64 + pending []topSnapshot // previous days not yet written +} + +type topSnapshot struct { + day string + counts map[string]map[string]int64 +} + +// Top tracks the most looked-up addons and search queries. +var Top = &topCounter{counts: map[string]map[string]int64{}} + +const ( + maxTopKeys = 50_000 + topPersistMax = 500 +) + +func (t *topCounter) Add(kind, item string) { + if item == "" { + return + } + if len(item) > 128 { + item = item[:128] + } + day := time.Now().UTC().Format("2006-01-02") + t.mu.Lock() + defer t.mu.Unlock() + if t.day != day { + if t.day != "" { + t.pending = append(t.pending, topSnapshot{t.day, t.counts}) + } + t.day = day + t.counts = map[string]map[string]int64{} + } + m := t.counts[kind] + if m == nil { + m = map[string]int64{} + t.counts[kind] = m + } + if _, ok := m[item]; !ok && len(m) >= maxTopKeys { + for k, v := range m { + if v <= 1 { + delete(m, k) + } + } + if len(m) >= maxTopKeys { + return + } + } + m[item]++ +} + +// Today returns today's in-memory counts for a kind. +func (t *topCounter) Today(kind string) (string, map[string]int64) { + t.mu.Lock() + defer t.mu.Unlock() + out := make(map[string]int64, len(t.counts[kind])) + for k, v := range t.counts[kind] { + out[k] = v + } + return t.day, out +} + +// seed loads today's persisted counts so a restart does not reset them. +func (t *topCounter) seed() { + day := time.Now().UTC().Format("2006-01-02") + var rows []models.DailyTopItem + if err := initializers.DB.Where("day = ?", day).Find(&rows).Error; err != nil { + log.Printf("metrics: failed to load top items: %v", err) + return + } + t.mu.Lock() + defer t.mu.Unlock() + if t.day != day { + t.day = day + t.counts = map[string]map[string]int64{} + } + for _, r := range rows { + if t.counts[r.Kind] == nil { + t.counts[r.Kind] = map[string]int64{} + } + t.counts[r.Kind][r.Item] += r.Count + } +} + +func topRows(snap topSnapshot) []models.DailyTopItem { + var rows []models.DailyTopItem + for kind, m := range snap.counts { + type kv struct { + k string + v int64 + } + list := make([]kv, 0, len(m)) + for k, v := range m { + list = append(list, kv{k, v}) + } + sort.Slice(list, func(i, j int) bool { return list[i].v > list[j].v }) + if len(list) > topPersistMax { + list = list[:topPersistMax] + } + for _, e := range list { + rows = append(rows, models.DailyTopItem{Day: snap.day, Kind: kind, Item: e.k, Count: e.v}) + } + } + return rows +} + +func (t *topCounter) flush() error { + t.mu.Lock() + snaps := append([]topSnapshot{}, t.pending...) + t.pending = nil + cur := topSnapshot{day: t.day, counts: map[string]map[string]int64{}} + for kind, m := range t.counts { + cp := make(map[string]int64, len(m)) + for k, v := range m { + cp[k] = v + } + cur.counts[kind] = cp + } + t.mu.Unlock() + if cur.day != "" { + snaps = append(snaps, cur) + } + + for i, snap := range snaps { + rows := topRows(snap) + if len(rows) == 0 { + continue + } + err := initializers.DB.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "day"}, {Name: "kind"}, {Name: "item"}}, + DoUpdates: clause.AssignmentColumns([]string{"count"}), + }).CreateInBatches(&rows, 200).Error + if err != nil { + // keep finished days for the next attempt; today is re-read from memory + t.mu.Lock() + for _, old := range snaps[i:] { + if old.day != cur.day { + t.pending = append(t.pending, old) + } + } + t.mu.Unlock() + return err + } + } + return nil +} diff --git a/models/addon.go b/models/addon.go index dbb092b..2a2b4f1 100644 --- a/models/addon.go +++ b/models/addon.go @@ -27,6 +27,7 @@ type Addon struct { PriorityIndexing bool `json:"priorityIndexing" gorm:"default:false"` AddonFiles []AddonFile `json:"addonFiles"` Recheck bool `json:"recheck" gorm:"default:false"` + LastIndexedAt *time.Time `json:"lastIndexedAt"` Preview string `json:"preview"` Author string `json:"author"` diff --git a/models/metrics.go b/models/metrics.go new file mode 100644 index 0000000..c05d8e6 --- /dev/null +++ b/models/metrics.go @@ -0,0 +1,74 @@ +package models + +import "time" + +// All timestamps in the metrics tables are int64 unix seconds (events: unix +// milliseconds) so range queries behave the same on SQLite and Postgres. + +// MetricSeries identifies one time series: a metric name plus a fixed label set +// reported by one service instance. +type MetricSeries struct { + ID uint `gorm:"primaryKey" json:"id"` + Service string `gorm:"size:64;not null;uniqueIndex:ux_metric_series,priority:1" json:"service"` + Instance string `gorm:"size:128;not null;uniqueIndex:ux_metric_series,priority:2" json:"instance"` + Name string `gorm:"size:96;not null;uniqueIndex:ux_metric_series,priority:3;index" json:"name"` + Labels string `gorm:"size:512;not null;uniqueIndex:ux_metric_series,priority:4" json:"labels"` // canonical JSON object + Kind string `gorm:"size:16;not null" json:"kind"` // counter|gauge|summary|histogram + Bounds string `gorm:"type:text" json:"bounds"` // JSON []float64, histograms only + Help string `gorm:"size:256" json:"help"` + CreatedAt time.Time + LastSeen int64 `gorm:"index" json:"lastSeen"` +} + +// MetricPoint is one aggregated bucket of a series. Resolution is the bucket +// width in seconds (60 = raw minute data, 3600 = hourly rollup). +type MetricPoint struct { + SeriesID uint `gorm:"primaryKey;autoIncrement:false"` + Resolution int `gorm:"primaryKey;autoIncrement:false;index:idx_metric_points_res_bucket,priority:1"` + Bucket int64 `gorm:"primaryKey;autoIncrement:false;index:idx_metric_points_res_bucket,priority:2"` + Count int64 `gorm:"not null;default:0"` + Sum float64 `gorm:"not null;default:0"` + Min float64 `gorm:"not null;default:0"` + Max float64 `gorm:"not null;default:0"` + Last float64 `gorm:"not null;default:0"` + Hist string `gorm:"type:text"` // JSON []uint64, histograms only +} + +// ServiceInstance is one running process (indexer, scraper, the API itself) +// that reports telemetry. Every telemetry push counts as a heartbeat. +type ServiceInstance struct { + ID uint `gorm:"primaryKey" json:"id"` + Service string `gorm:"size:64;not null;uniqueIndex:ux_service_instance,priority:1" json:"service"` + Instance string `gorm:"size:128;not null;uniqueIndex:ux_service_instance,priority:2" json:"instance"` + Version string `gorm:"size:64" json:"version"` + Status string `gorm:"size:16" json:"status"` // self-reported: ok|degraded|error + StatusMessage string `gorm:"size:512" json:"statusMessage"` + StartedAt int64 `json:"startedAt"` + LastSeen int64 `gorm:"index" json:"lastSeen"` + HeartbeatIntervalSec int `json:"heartbeatIntervalSec"` + RemoteAddr string `gorm:"size:64" json:"remoteAddr"` + Meta string `gorm:"type:text" json:"meta"` // JSON object + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +// ServiceEvent is a log line reported by a service. Admin actions are recorded +// here too (service "main", fields.kind "audit"). +type ServiceEvent struct { + ID uint `gorm:"primaryKey" json:"id"` + Service string `gorm:"size:64;index:idx_service_events_svc_ts,priority:1" json:"service"` + Instance string `gorm:"size:128" json:"instance"` + Level string `gorm:"size:8;index" json:"level"` // debug|info|warn|error + Message string `gorm:"size:1024" json:"message"` + Fields string `gorm:"type:text" json:"fields"` // JSON object + TS int64 `gorm:"index:idx_service_events_svc_ts,priority:2;index" json:"ts"` // unix ms +} + +// DailyTopItem counts how often something was requested on one UTC day +// (Kind "addon": addon lookups, Kind "search": search queries). +type DailyTopItem struct { + Day string `gorm:"primaryKey;size:10"` // YYYY-MM-DD + Kind string `gorm:"primaryKey;size:16"` + Item string `gorm:"primaryKey;size:128"` + Count int64 +} diff --git a/models/telemetry.go b/models/telemetry.go new file mode 100644 index 0000000..d43d738 --- /dev/null +++ b/models/telemetry.go @@ -0,0 +1,39 @@ +package models + +// TelemetryBatch is the payload of POST /back/telemetry. The client lives in +// the telemetry/ module; keep both definitions in sync. +type TelemetryBatch struct { + Service string `json:"service"` + Instance string `json:"instance"` + Version string `json:"version"` + StartedAt int64 `json:"startedAt"` + HeartbeatIntervalSec int `json:"heartbeatIntervalSec"` + Status string `json:"status"` + StatusMessage string `json:"statusMessage"` + Meta map[string]any `json:"meta"` + Metrics []TelemetryMetric `json:"metrics"` + Events []TelemetryEvent `json:"events"` +} + +// TelemetryMetric is one sample. Type "counter" carries a delta since the last +// push, "gauge" the current value, "summary" client-side pre-aggregated +// Count/Sum/Min/Max. Ts is unix seconds or milliseconds, optional. +type TelemetryMetric struct { + Name string `json:"name"` + Type string `json:"type"` + Help string `json:"help"` + Value float64 `json:"value"` + Count int64 `json:"count"` + Sum float64 `json:"sum"` + Min float64 `json:"min"` + Max float64 `json:"max"` + Labels map[string]string `json:"labels"` + Ts int64 `json:"ts"` +} + +type TelemetryEvent struct { + Level string `json:"level"` + Message string `json:"message"` + Ts int64 `json:"ts"` + Fields map[string]any `json:"fields"` +} diff --git a/models/whitelisted.go b/models/whitelisted.go index 414a35e..1cbc77c 100644 --- a/models/whitelisted.go +++ b/models/whitelisted.go @@ -4,5 +4,6 @@ import "gorm.io/gorm" type WhitelistedHash struct { gorm.Model - Hash string `json:"hash"` + Hash string `json:"hash"` + Note string `json:"note" gorm:"size:256"` } diff --git a/telemetry/client.go b/telemetry/client.go new file mode 100644 index 0000000..3523d6f --- /dev/null +++ b/telemetry/client.go @@ -0,0 +1,430 @@ +// Package telemetry reports metrics, events and heartbeats from a crawler +// service (indexer, scraper, ...) to the main API's POST /back/telemetry. +// +// It has no dependencies outside the standard library, so it can be imported +// by any service, or the file can simply be copied. +// +// c := telemetry.New(telemetry.Config{ +// BaseURL: "https://crawler.example.com", +// SecretKey: os.Getenv("SECRET_KEY"), +// Service: "indexer", +// Version: "1.4.0", +// }) +// c.Start(ctx) +// defer c.Close(context.Background()) +// +// c.Counter("jobs_total", "result", "ok").Add(1) +// c.Gauge("data_window_bytes").Set(float64(used)) +// c.Summary("download_seconds").Observe(took.Seconds()) +// c.Event(telemetry.Warn, "data limit reached", map[string]any{"limitGB": 50}) +// c.SetStatus(telemetry.Degraded, "rate limited by workshop") +// +// Recording never blocks on the network. A push that fails keeps the counter +// increments for the next attempt; a few failed batches of events and samples +// are buffered and the oldest are dropped first. +package telemetry + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "math" + "net/http" + "os" + "sort" + "strings" + "sync" + "time" +) + +// Level of an event. +type Level string + +const ( + Debug Level = "debug" + Info Level = "info" + Warn Level = "warn" + Error Level = "error" +) + +// Status is the self-reported health shown on the dashboard. +type Status string + +const ( + OK Status = "ok" + Degraded Status = "degraded" + Failing Status = "error" +) + +// Config configures a Client. BaseURL, SecretKey and Service are required. +type Config struct { + BaseURL string // e.g. https://crawler.example.com (no trailing /back) + SecretKey string // the API's SECRET_KEY + Service string // [a-z0-9_], e.g. "indexer" + Instance string // defaults to the hostname + Version string // shown on the dashboard + Interval time.Duration // push interval, default 15s (also the heartbeat) + Meta map[string]any + // HTTPClient defaults to a client with a 10s timeout. + HTTPClient *http.Client + // OnError is called when a push fails (optional). + OnError func(error) +} + +// Client buffers metrics and events and pushes them periodically. +type Client struct { + cfg Config + started time.Time + + mu sync.Mutex + counters map[string]*Counter + gauges map[string]*Gauge + summaries map[string]*Summary + events []event + status Status + statusMsg string + pending [][]byte // failed batches waiting for a retry + + stop chan struct{} + done chan struct{} +} + +type metric struct { + Name string `json:"name"` + Type string `json:"type"` + Value float64 `json:"value,omitempty"` + Count int64 `json:"count,omitempty"` + Sum float64 `json:"sum,omitempty"` + Min float64 `json:"min,omitempty"` + Max float64 `json:"max,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Ts int64 `json:"ts"` +} + +type event struct { + Level Level `json:"level"` + Message string `json:"message"` + Ts int64 `json:"ts"` + Fields map[string]any `json:"fields,omitempty"` +} + +type batch struct { + Service string `json:"service"` + Instance string `json:"instance"` + Version string `json:"version"` + StartedAt int64 `json:"startedAt"` + HeartbeatIntervalSec int `json:"heartbeatIntervalSec"` + Status Status `json:"status"` + StatusMessage string `json:"statusMessage"` + Meta map[string]any `json:"meta,omitempty"` + Metrics []metric `json:"metrics"` + Events []event `json:"events"` +} + +const ( + maxPending = 20 + maxEvents = 200 + maxMetricsPerReq = 1000 +) + +// New creates a client. Call Start to begin pushing. +func New(cfg Config) *Client { + if cfg.Instance == "" { + cfg.Instance, _ = os.Hostname() + } + if cfg.Interval <= 0 { + cfg.Interval = 15 * time.Second + } + if cfg.HTTPClient == nil { + cfg.HTTPClient = &http.Client{Timeout: 10 * time.Second} + } + cfg.BaseURL = strings.TrimRight(cfg.BaseURL, "/") + return &Client{ + cfg: cfg, + started: time.Now(), + counters: map[string]*Counter{}, + gauges: map[string]*Gauge{}, + summaries: map[string]*Summary{}, + status: OK, + } +} + +func key(name string, labels []string) string { + return name + "\x00" + strings.Join(labels, "\x00") +} + +func labelMap(kv []string) map[string]string { + if len(kv) < 2 { + return nil + } + m := make(map[string]string, len(kv)/2) + for i := 0; i+1 < len(kv); i += 2 { + m[kv[i]] = kv[i+1] + } + return m +} + +// Counter returns a counter; labels are key/value pairs. +func (c *Client) Counter(name string, labels ...string) *Counter { + k := key(name, labels) + c.mu.Lock() + defer c.mu.Unlock() + ctr := c.counters[k] + if ctr == nil { + ctr = &Counter{name: name, labels: labelMap(labels)} + c.counters[k] = ctr + } + return ctr +} + +// Gauge returns a gauge; labels are key/value pairs. +func (c *Client) Gauge(name string, labels ...string) *Gauge { + k := key(name, labels) + c.mu.Lock() + defer c.mu.Unlock() + g := c.gauges[k] + if g == nil { + g = &Gauge{name: name, labels: labelMap(labels)} + c.gauges[k] = g + } + return g +} + +// Summary returns a summary (count/sum/min/max per push); labels are key/value pairs. +func (c *Client) Summary(name string, labels ...string) *Summary { + k := key(name, labels) + c.mu.Lock() + defer c.mu.Unlock() + s := c.summaries[k] + if s == nil { + s = &Summary{name: name, labels: labelMap(labels)} + c.summaries[k] = s + } + return s +} + +// Event queues a log event for the dashboard. +func (c *Client) Event(level Level, message string, fields map[string]any) { + c.mu.Lock() + defer c.mu.Unlock() + if len(c.events) >= maxEvents { + c.events = c.events[1:] + } + c.events = append(c.events, event{Level: level, Message: message, Ts: time.Now().UnixMilli(), Fields: fields}) +} + +// SetStatus sets the self-reported health shown on the dashboard. +func (c *Client) SetStatus(s Status, message string) { + c.mu.Lock() + c.status, c.statusMsg = s, message + c.mu.Unlock() +} + +// Counter accumulates increments between pushes. +type Counter struct { + mu sync.Mutex + name string + labels map[string]string + delta float64 +} + +func (m *Counter) Add(v float64) { + if v <= 0 || math.IsNaN(v) || math.IsInf(v, 0) { + return + } + m.mu.Lock() + m.delta += v + m.mu.Unlock() +} + +func (m *Counter) Inc() { m.Add(1) } + +// Gauge holds the latest value. +type Gauge struct { + mu sync.Mutex + name string + labels map[string]string + value float64 + set bool +} + +func (m *Gauge) Set(v float64) { + if math.IsNaN(v) || math.IsInf(v, 0) { + return + } + m.mu.Lock() + m.value, m.set = v, true + m.mu.Unlock() +} + +// Summary aggregates observations between pushes. +type Summary struct { + mu sync.Mutex + name string + labels map[string]string + count int64 + sum float64 + min, max float64 +} + +func (m *Summary) Observe(v float64) { + if math.IsNaN(v) || math.IsInf(v, 0) { + return + } + m.mu.Lock() + if m.count == 0 || v < m.min { + m.min = v + } + if m.count == 0 || v > m.max { + m.max = v + } + m.count++ + m.sum += v + m.mu.Unlock() +} + +// collect drains the current values into a batch payload. +func (c *Client) collect() []byte { + now := time.Now() + c.mu.Lock() + b := batch{ + Service: c.cfg.Service, + Instance: c.cfg.Instance, + Version: c.cfg.Version, + StartedAt: c.started.Unix(), + HeartbeatIntervalSec: int(c.cfg.Interval.Seconds()), + Status: c.status, + StatusMessage: c.statusMsg, + Meta: c.cfg.Meta, + Events: c.events, + } + c.events = nil + counters := make([]*Counter, 0, len(c.counters)) + for _, m := range c.counters { + counters = append(counters, m) + } + gauges := make([]*Gauge, 0, len(c.gauges)) + for _, m := range c.gauges { + gauges = append(gauges, m) + } + summaries := make([]*Summary, 0, len(c.summaries)) + for _, m := range c.summaries { + summaries = append(summaries, m) + } + c.mu.Unlock() + + ts := now.UnixMilli() + for _, m := range counters { + m.mu.Lock() + if m.delta > 0 { + b.Metrics = append(b.Metrics, metric{Name: m.name, Type: "counter", Value: m.delta, Labels: m.labels, Ts: ts}) + m.delta = 0 + } + m.mu.Unlock() + } + for _, m := range gauges { + m.mu.Lock() + if m.set { + b.Metrics = append(b.Metrics, metric{Name: m.name, Type: "gauge", Value: m.value, Labels: m.labels, Ts: ts}) + } + m.mu.Unlock() + } + for _, m := range summaries { + m.mu.Lock() + if m.count > 0 { + b.Metrics = append(b.Metrics, metric{Name: m.name, Type: "summary", Count: m.count, Sum: m.sum, Min: m.min, Max: m.max, Labels: m.labels, Ts: ts}) + m.count, m.sum, m.min, m.max = 0, 0, 0, 0 + } + m.mu.Unlock() + } + sort.Slice(b.Metrics, func(i, j int) bool { return b.Metrics[i].Name < b.Metrics[j].Name }) + if len(b.Metrics) > maxMetricsPerReq { + b.Metrics = b.Metrics[:maxMetricsPerReq] + } + if b.Metrics == nil { + b.Metrics = []metric{} + } + if b.Events == nil { + b.Events = []event{} + } + payload, _ := json.Marshal(b) + return payload +} + +func (c *Client) post(ctx context.Context, payload []byte) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.cfg.BaseURL+"/back/telemetry", bytes.NewReader(payload)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-SECRET-KEY", c.cfg.SecretKey) + resp, err := c.cfg.HTTPClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + if resp.StatusCode/100 != 2 { + return fmt.Errorf("telemetry push: %s: %s", resp.Status, strings.TrimSpace(string(body))) + } + return nil +} + +// Flush pushes buffered batches and the current values now. +func (c *Client) Flush(ctx context.Context) error { + payload := c.collect() + c.mu.Lock() + queue := append(c.pending, payload) + c.pending = nil + c.mu.Unlock() + + for i, p := range queue { + if err := c.post(ctx, p); err != nil { + c.mu.Lock() + c.pending = append(queue[i:], c.pending...) + if len(c.pending) > maxPending { + c.pending = c.pending[len(c.pending)-maxPending:] + } + c.mu.Unlock() + if c.cfg.OnError != nil { + c.cfg.OnError(err) + } + return err + } + } + return nil +} + +// Start pushes in the background every Interval until ctx ends or Close is called. +func (c *Client) Start(ctx context.Context) { + c.stop = make(chan struct{}) + c.done = make(chan struct{}) + go func() { + defer close(c.done) + t := time.NewTicker(c.cfg.Interval) + defer t.Stop() + _ = c.Flush(ctx) + for { + select { + case <-ctx.Done(): + return + case <-c.stop: + return + case <-t.C: + _ = c.Flush(ctx) + } + } + }() +} + +// Close stops the background loop and makes a final push. +func (c *Client) Close(ctx context.Context) error { + if c.stop != nil { + close(c.stop) + <-c.done + c.stop = nil + } + return c.Flush(ctx) +} diff --git a/telemetry/client_test.go b/telemetry/client_test.go new file mode 100644 index 0000000..d7937cc --- /dev/null +++ b/telemetry/client_test.go @@ -0,0 +1,73 @@ +package telemetry + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" +) + +func TestFlushAndRetry(t *testing.T) { + var fail atomic.Bool + var got []batch + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/back/telemetry" || r.Header.Get("X-SECRET-KEY") != "k" { + w.WriteHeader(http.StatusForbidden) + return + } + if fail.Load() { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + body, _ := io.ReadAll(r.Body) + var b batch + if err := json.Unmarshal(body, &b); err != nil { + t.Error(err) + } + got = append(got, b) + w.Write([]byte(`{"accepted":1}`)) + })) + defer srv.Close() + + c := New(Config{BaseURL: srv.URL + "/", SecretKey: "k", Service: "indexer", Instance: "i1"}) + c.Counter("jobs_total", "result", "ok").Add(2) + c.Gauge("queue").Set(7) + c.Summary("took_seconds").Observe(1) + c.Summary("took_seconds").Observe(3) + c.Event(Warn, "hello", nil) + + fail.Store(true) + if err := c.Flush(context.Background()); err == nil { + t.Fatal("expected an error while the server fails") + } + c.Counter("jobs_total", "result", "ok").Add(1) + + fail.Store(false) + if err := c.Flush(context.Background()); err != nil { + t.Fatal(err) + } + if len(got) != 2 { + t.Fatalf("got %d batches, want the retried one plus the new one", len(got)) + } + first := got[0] + if first.Service != "indexer" || first.Instance != "i1" || len(first.Events) != 1 { + t.Fatalf("unexpected first batch %+v", first) + } + var sum float64 + for _, b := range got { + for _, m := range b.Metrics { + if m.Name == "jobs_total" { + sum += m.Value + } + if m.Name == "took_seconds" && (m.Count != 2 || m.Min != 1 || m.Max != 3) { + t.Fatalf("bad summary %+v", m) + } + } + } + if sum != 3 { + t.Fatalf("counter total = %v, want 3", sum) + } +} diff --git a/telemetry/go.mod b/telemetry/go.mod new file mode 100644 index 0000000..a7eb002 --- /dev/null +++ b/telemetry/go.mod @@ -0,0 +1,3 @@ +module gitea.tbdevent.eu/TBD/reforger_crawler_main/telemetry + +go 1.22 diff --git a/web/dist/.gitkeep b/web/dist/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/web/embed.go b/web/embed.go new file mode 100644 index 0000000..9ee00a0 --- /dev/null +++ b/web/embed.go @@ -0,0 +1,13 @@ +// Package web serves the admin dashboard (a React app built into web/dist). +// +// Build the UI before building the Go binary: +// +// npm --prefix web ci && npm --prefix web run build +// +// Without a build the binary still compiles and serves a placeholder page. +package web + +import "embed" + +//go:embed all:dist +var dist embed.FS diff --git a/web/index.html b/web/index.html new file mode 100644 index 0000000..ff99c87 --- /dev/null +++ b/web/index.html @@ -0,0 +1,15 @@ + + + + + + + + Crawler Admin + + + +
+ + + diff --git a/web/package-lock.json b/web/package-lock.json new file mode 100644 index 0000000..d51a8c8 --- /dev/null +++ b/web/package-lock.json @@ -0,0 +1,2534 @@ +{ + "name": "reforger-crawler-admin", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "reforger-crawler-admin", + "version": "1.0.0", + "dependencies": { + "@tanstack/react-query": "^5.80.0", + "lucide-react": "^0.511.0", + "react": "^19.1.0", + "react-dom": "^19.1.0", + "react-router": "^7.6.0", + "uplot": "^1.6.32" + }, + "devDependencies": { + "@tailwindcss/vite": "^4.1.8", + "@types/react": "^19.1.6", + "@types/react-dom": "^19.1.5", + "@vitejs/plugin-react": "^4.5.0", + "tailwindcss": "^4.1.8", + "typescript": "^5.8.3", + "vite": "^6.3.5" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", + "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.29.7", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.7.tgz", + "integrity": "sha512-locTkQyKvwIEgBzVrn8693ebc97F2U8ZHjbXwDXJ5Fn2TCpNwTlKcaKLkdHop5c/icOFE7qt7Q9JC5hnKNa6Gg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz", + "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.7", + "@babel/helper-compilation-targets": "^7.29.7", + "@babel/helper-module-transforms": "^7.29.7", + "@babel/helpers": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7", + "@jridgewell/remapping": "^2.3.5", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/generator": { + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.8.tgz", + "integrity": "sha512-gZbepsdh3WDtgZKWL+vTPh71LSBrm/Y4/QDZBVCcYfmeTEEuoOYwlSy+G1StfJg+/Zy550u/3TATbm7qDbbMtg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.8", + "@babel/types": "^7.29.8", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.29.7.tgz", + "integrity": "sha512-wem6WaBj4NaVYVdNhLPPVacES6ZJ+KBBfSkTMD3YZxbP3rm3Di85tJU5ljaUNhaOynt+Aj0xruhYuzQBt8n71g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.29.7", + "@babel/helper-validator-option": "^7.29.7", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-globals": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.29.7.tgz", + "integrity": "sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.29.7.tgz", + "integrity": "sha512-ejHwrQQYcm9xnTivShn2IDOlIzInN34AXskvq9QicvCtEzq1Vzclu/tKF8Jq1Cg8JG2GL6/EmjgsCT7lXepE3g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.29.7.tgz", + "integrity": "sha512-UPUVSyXbOh627KiCIGQSgwWzGeBKLkaJ9PJEdrngIwMSzxLR4jS4+f1f1jb7VzBbg8nFLaYotvVPFCTqdrmTAg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7", + "@babel/traverse": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.29.7.tgz", + "integrity": "sha512-G7sHYigPY17oO5SYWnfD/0MTBwVR781S/JI643e/JhUYgVgWE/61SoW3NH9KWUKyKq5LVh3npif99Wkt6j86Jw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.29.7.tgz", + "integrity": "sha512-N9ZErrD+yW5geCDtBqnOoxmR8+tNKiGuxKlDpuJxfsqpa2dFcexaziGAE/qoHLiDDreVNMupxGmSoNlyvsA3gw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.7.tgz", + "integrity": "sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.9", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.9.tgz", + "integrity": "sha512-CjXrNHTnvqBVqHgdBysY3vk2T8tpJHb5/RMeHJBTyVa9xgugCB0CJTx/3oO8RV2QRQP391RWpB7D6hLjm8V9uA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.8" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-self": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.29.7.tgz", + "integrity": "sha512-TL0hMc9xzy86VD31nUiwzd5otRAcyEPcsegCxolO0PvcXuH1v0kECe/UIznYFihpkvU5wg/jk4v0TTEFfm53fw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-source": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.29.7.tgz", + "integrity": "sha512-06IyK09H3wi4cGbhDBwp5gUGo0IKtnYa8tyTiephirPCK6fbobVGiXMMI5zLQ4aKEYP3wZ3ArU44o+8KMrSG/Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/template": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.29.7.tgz", + "integrity": "sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.8.tgz", + "integrity": "sha512-I5z7H3bf/41ktsNVLtpN0wAa336HkqIHQ5BuPLEhTkt1jVSyZpeNKIzTgEWmlxjdg81R0IgUCcaE+Ok3NvrfZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.8", + "@babel/helper-globals": "^7.29.7", + "@babel/parser": "^7.29.8", + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.8", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.8.tgz", + "integrity": "sha512-Vj1jF3cPfxg7OAfoI7QnVKLoILlm2JF9pnVHrX8qx7AHMiYWT+NDAA7jChlNgRS4WTLc/fD1lXLmPixluj+3Gg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", + "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", + "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", + "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", + "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", + "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", + "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", + "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", + "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", + "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", + "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", + "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", + "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", + "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", + "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", + "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", + "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", + "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", + "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", + "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", + "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", + "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", + "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", + "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", + "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", + "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", + "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@napi-rs/lzma-linux-x64-gnu": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-linux-x64-gnu/-/lzma-linux-x64-gnu-1.5.1.tgz", + "integrity": "sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^22.20 || ^24.12 || >=25" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.0-beta.27", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.27.tgz", + "integrity": "sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.64.0.tgz", + "integrity": "sha512-FLLpX941CD/7aBvX3GqLGBM9OvY5PGXTryPJkG5r5/OuBpZpV5k45Uo3RxAvqd35tsyc3hH1bhtz921IKzoIzg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.64.0.tgz", + "integrity": "sha512-iI32XM/jnDYJsvx5ai/za/7gWO9XoSeFFtU06/ZCaRgyilfJOjaopD3nOpk2WrTCGOyeDt4hleUiGOAnd5tHNg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.64.0.tgz", + "integrity": "sha512-dRBZlbt2sEtF8Mn4MODThze9fIPdDoOhVSlXMAKy5dQbOYK9pOOoPY4Fat3OTtr0+8zbJo5TTZwlsfxY/kDNJg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.64.0.tgz", + "integrity": "sha512-Y3whrJOMucfkhgNkvcM93M6j8C53iEAMmPMYD99C6G1kUy4ELigLWk9ar2/4aCBSRgsql0zdlOemLgW2m5GZgQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.64.0.tgz", + "integrity": "sha512-S6S1GEzPjFF6IONYJX3AKt/799+8RgBgCcwRnUo+Ep1IvBxUbw5M51iU0TDdK2NWWVnSvCQ8yi6Em0fGASMOng==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.64.0.tgz", + "integrity": "sha512-Ed1DPoFENkXj4IGzSB12SesV40iP6LxCCrbTIkSGIQBpOOxpBM+0kj62lmaeIHfMWTDoddqTOtqc7s8LYfn+Pw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.64.0.tgz", + "integrity": "sha512-J/6xhutJPtFyIkmHdf4olOdHHKH5kOJNumojdWn+Elv5ICEYTVOa362Vw27TxsK0BLzb1le3EDlO3UbvFK1lnQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.64.0.tgz", + "integrity": "sha512-G4L/Nnx59d2LvDoV/P4S5lZ7oVLT6hodOHMacqjiq2LL2gts86KbkEBTYL53+fBt0odhf7XPIbIATGoYiJctxg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.64.0.tgz", + "integrity": "sha512-zF16pvJDEUksGYRbfnXne7pdklAyj2XDmJeYKgP93oq0cUyX9Y8O7lUo1R+IY1GagP/YBvCEDCe7eEcAKpw//g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.64.0.tgz", + "integrity": "sha512-nIggWn8n+w12+R5lsV+PYYoUY8vuObxPAQqJI7WES+9v0qspYuAHoZMfYM0yLxlipasT5BY9ht5W8t5c/rSCkg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.64.0.tgz", + "integrity": "sha512-WLOTiHPzszP0dxIMsHpFeDSOSB5I49Mj8K4+yKE3H1X/D0JRG5XZ5jrctJLMjnUc/LUzaKiwq6EHWNJ8pup8PA==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.64.0.tgz", + "integrity": "sha512-35RK9aRhh7g7Pmeu5gQ01aCvrhXzj9LoBE6xtUcEH1ns+nyTaSzxSpADmBQINu4QiTDGvmQYwo1j5hlg1rRXFw==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.64.0.tgz", + "integrity": "sha512-RqTfnpoTjuWfPAmqK5p2edt2AMCDJWm6oTn7sKfmyx+6tRga0ip05SJ977o6udTcIiDeF/CJMUfkaN4gtWt6Aw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.64.0.tgz", + "integrity": "sha512-RQjZPigjxFvFZDijG6rjlSXxhRaTf9zLYVpGK+lVcga64oMqP1GbgVqCb46eHTTceSIgp89Ja0y5aPe3GFBYHg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.64.0.tgz", + "integrity": "sha512-tjL58mgQcTRnDRpeky+Sth1MKQeYi2Ct6r3Luq9E5aTSjnM/i/JbozqZkKqp9thM5+oDSZxo+05f9kdQedfTGA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.64.0.tgz", + "integrity": "sha512-ictoRyVaoZ8C67myHef+8hhCa6IplXJJ/xUCkVCxziV4zxMZh+jvhkhGsPBLqj1S72fnqD0QH5zL4lcaa4X7DQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.64.0.tgz", + "integrity": "sha512-8aVvZ2hfr9RLFzBQkQNmYeHLDja8fLeQAmLt757zdKN/B+8L809KlLJKctQjS22wZotptV8G6o7fVPuAs8S1tQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.64.0.tgz", + "integrity": "sha512-2dEF8GAcDKwshUfydD+GhosppNyBzhVUHkdFF3CXd3FpOkzj/RZzF5aoEtzklujklT5qFVRU2GK/cFeLaTNKVg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.64.0.tgz", + "integrity": "sha512-3XHTvoo2hKkh0MiOfS7SkhHjMaaWF7P4F3drocZZaho/Z8w7h/jpnuBHm91BOQLfvrKHs1yR+zir4E8RI3XzTQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.64.0.tgz", + "integrity": "sha512-HKtkzodL4a3KFWDXDoO61cRynAWTR3LnAE0oGtfG9lcNFpNuVYUFOByxDgNqEvic9rvaQbAj+SFwTEWDptRaIw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.64.0.tgz", + "integrity": "sha512-JBsxqiY16uyyCujtoC3s7zL+SrYyNElptD7K4V+KUoJ2V8/XG3hPWpheK+SuPwNh1cikJvIZmIdsrYXvD6PLJw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.64.0.tgz", + "integrity": "sha512-S5Qrijh37qhnUYQ0ay8WN3LuWeKjow7wTUyCOPMZtnXiBXYXdsZETI6ZLfYnQAKovltckPLBFdNeD61P2rJc7g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.64.0.tgz", + "integrity": "sha512-zbZ6iMmLFEsCtTdc2/y1F/cDFVNNCPF7Ae8PCT/wdPE//hGwYp6r+MyCLMFX+kOeIe09Pz6ASfnnfpSTN1FJQQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.64.0.tgz", + "integrity": "sha512-L6Hw69oNpuYaOr1zCG30v0/DqKYCSRwodPGwIcVZ/6f8SOzq9rxtL/xNVI5MO4wjli+Ko0ErK9uHJL3mNPt8wQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.64.0.tgz", + "integrity": "sha512-nl+CTWIgOGszQmUz/kanfgrfFZ4qiwxKIfT/ryl3N3Ewk5tqizmYtNXiqb7Bu/fDtNgu1kyxdFiAUJNur3iysg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@tailwindcss/node": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.3.3.tgz", + "integrity": "sha512-/T8IKEsf9VTU6tLjgC7+sv2mOPtQxzE2jMw7u4Tt40Tx+QSZxpzh95/H6cMKoja9XuW7iMdLJYBB0o9G1CaAgg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/remapping": "^2.3.5", + "enhanced-resolve": "^5.24.1", + "jiti": "^2.7.0", + "lightningcss": "1.32.0", + "magic-string": "^0.30.21", + "source-map-js": "^1.2.1", + "tailwindcss": "4.3.3" + } + }, + "node_modules/@tailwindcss/oxide": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide/-/oxide-4.3.3.tgz", + "integrity": "sha512-krXjAikiaFSPaK/FkAQT5UTx3VormQaiZ5hBFlJZ9UFQGB/rwg1MZIhHAG9smMQRTdyJxP6Qt5MwMtdyU5FWrA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 20" + }, + "optionalDependencies": { + "@tailwindcss/oxide-android-arm64": "4.3.3", + "@tailwindcss/oxide-darwin-arm64": "4.3.3", + "@tailwindcss/oxide-darwin-x64": "4.3.3", + "@tailwindcss/oxide-freebsd-x64": "4.3.3", + "@tailwindcss/oxide-linux-arm-gnueabihf": "4.3.3", + "@tailwindcss/oxide-linux-arm64-gnu": "4.3.3", + "@tailwindcss/oxide-linux-arm64-musl": "4.3.3", + "@tailwindcss/oxide-linux-x64-gnu": "4.3.3", + "@tailwindcss/oxide-linux-x64-musl": "4.3.3", + "@tailwindcss/oxide-wasm32-wasi": "4.3.3", + "@tailwindcss/oxide-win32-arm64-msvc": "4.3.3", + "@tailwindcss/oxide-win32-x64-msvc": "4.3.3" + } + }, + "node_modules/@tailwindcss/oxide-android-arm64": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-android-arm64/-/oxide-android-arm64-4.3.3.tgz", + "integrity": "sha512-Y85A2gmPSkl5Ve5qR86GL4HT509cFqQh1aes9p3sSkyTPwt0Pppf3GkwGe4JPACcRYjgJIEhQgM6dBClnr0NYw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-darwin-arm64": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-arm64/-/oxide-darwin-arm64-4.3.3.tgz", + "integrity": "sha512-BiaWatpBcERQFDlOjRDpIVXuFK5PJez5SA4JMg6VYZdBYU+qKfV/vqjcIs+IYmtitf1xYQZTwXvU/8y4lfZUGw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-darwin-x64": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-x64/-/oxide-darwin-x64-4.3.3.tgz", + "integrity": "sha512-fAeUqfV5ndhxRwai8cXGzdLvul9utWOmeTkv69unv4ZXixjn61Z+p9lCWdwOwA3TYboG3BwdVuN/RDjhBRl0mw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-freebsd-x64": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-freebsd-x64/-/oxide-freebsd-x64-4.3.3.tgz", + "integrity": "sha512-iyf5bV6+wnAlflVeEy7R25dupxTNECZN5QMI0qNT6eT+EgaGdZcKhGkr5SdoaWiLJ3spLqIY9VCeSGrwmtg4kw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm-gnueabihf": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm-gnueabihf/-/oxide-linux-arm-gnueabihf-4.3.3.tgz", + "integrity": "sha512-aAYUprJAJQWWbRrPvtjdroZ56Md+JM8pMiopS6xGEwDfLhqj+2ver2p4nU4Mb3CRqcMmNBjo8KkUgcxhkzVQGQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm64-gnu": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-gnu/-/oxide-linux-arm64-gnu-4.3.3.tgz", + "integrity": "sha512-nDxldcEENOxZRzC2uu9jrutZdAAQtb+8WWDCSnWL1zvBk1+FN+x6MtDViPB5AJMfttVCUhehGWus3XBPgatM/w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm64-musl": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-musl/-/oxide-linux-arm64-musl-4.3.3.tgz", + "integrity": "sha512-Md44bD6veX/PC5iyF8cDVnw4HBIANZepRZZ7a8DQOvkfo5WUBwcp6iAuCUz23u+4SUkhJlD3eL7hNdW8ezd/kA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-x64-gnu": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-gnu/-/oxide-linux-x64-gnu-4.3.3.tgz", + "integrity": "sha512-tx7us1muwOKAKWao2v/GaafFeQboE6aj88vC6ziN2NCGcRm8gWUhwjzg+YdVB1e4boAtdtma4L43onunI6NS4w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-x64-musl": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-musl/-/oxide-linux-x64-musl-4.3.3.tgz", + "integrity": "sha512-SJxX60smvHgasZoBy11dX6YRjXJFovwWBoedhbQPOBzgFWBHGB+TVPWB9BxzR7TTxU8FQZAI2AyiNCMzFm8Img==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-wasm32-wasi/-/oxide-wasm32-wasi-4.3.3.tgz", + "integrity": "sha512-jx1+rPhY/5Ympkktd656HBWEBLxP7dH06losBLjjf5vgCODXvi9KhtftWcMIwTFIDqBr7cRnQkdLnAG+IOlGvQ==", + "bundleDependencies": [ + "@napi-rs/wasm-runtime", + "@emnapi/core", + "@emnapi/runtime", + "@tybys/wasm-util", + "@emnapi/wasi-threads", + "tslib" + ], + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "^1.11.1", + "@emnapi/runtime": "^1.11.1", + "@emnapi/wasi-threads": "^1.2.2", + "@napi-rs/wasm-runtime": "^1.1.4", + "@tybys/wasm-util": "^0.10.2", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@tailwindcss/oxide-win32-arm64-msvc": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.3.3.tgz", + "integrity": "sha512-3rc292Ca2ceK6Ulcc/bAVnTs/3nDtoPhyEKlgPv+yQJQi/JS/AMJlqzxvlDacL1nekbrcf6bTqp/jV4qgnPxNQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-win32-x64-msvc": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-x64-msvc/-/oxide-win32-x64-msvc-4.3.3.tgz", + "integrity": "sha512-yJ0pwIVc/nYeGoV02WtsN8KYyLQv7kyI2wDnkezyJlGGjkd4QLwDGAwl47YpPJeuI0M0ObaXGSPjvWDPeTPggw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/vite": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/vite/-/vite-4.3.3.tgz", + "integrity": "sha512-yYU8cogLeSh/ms2jh8Fj7jaba/EWa7Ja6GoUqYZaraEuCI5YS6ms6ObZgjjedm+jm6XZjdNRWBpPP6Z86oOxcw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@tailwindcss/node": "4.3.3", + "@tailwindcss/oxide": "4.3.3", + "tailwindcss": "4.3.3" + }, + "peerDependencies": { + "vite": "^5.2.0 || ^6 || ^7 || ^8" + } + }, + "node_modules/@tanstack/query-core": { + "version": "5.104.1", + "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.104.1.tgz", + "integrity": "sha512-TiRghcrGkUTE+M4JBxvpPSpBhxvIzQairw+cksrHYGr+2uwC77yW1SY9nYxnYRYOppvucf8rEg6b5byoz4rTpw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + } + }, + "node_modules/@tanstack/react-query": { + "version": "5.104.1", + "resolved": "https://registry.npmjs.org/@tanstack/react-query/-/react-query-5.104.1.tgz", + "integrity": "sha512-Zz70EgjahNI7aCz8BZxM3vUGU44ycxEAzMuCUjLeA7SVhhOkg3ZgPjPJPvHjs6Rmky/dtmnu1WzjGHNHmhCoZQ==", + "license": "MIT", + "dependencies": { + "@tanstack/query-core": "5.104.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + }, + "peerDependencies": { + "react": "^18 || ^19" + } + }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.27.0", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.27.0.tgz", + "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", + "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.28.2" + } + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/react": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.3.0.tgz", + "integrity": "sha512-N0rFCuH9YoxG9/m61l9MfpJKfmLOVU0em7ipIz6TRgSSkvReLB9vL85GB+yr8Bs5leqpvg96JSwF4ZS1s4viQg==", + "dev": true, + "license": "MIT", + "dependencies": { + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.3.0.tgz", + "integrity": "sha512-ZI7bU42mZXXKHn/qNLEw2IrbiINU7X5+vfgdixBHkCNpYWXjKgfQ/P+uyGb5CjOLB9UcnTeg3rylQtV2hym44Q==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^19.3.0" + } + }, + "node_modules/@vitejs/plugin-react": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.7.0.tgz", + "integrity": "sha512-gUu9hwfWvvEDBBmgtAowQCojwZmJ5mcLn3aufeCsitijs3+f2NsrPtlAWIR6OPiqljl96GVCUbLe0HyqIpVaoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.28.0", + "@babel/plugin-transform-react-jsx-self": "^7.27.1", + "@babel/plugin-transform-react-jsx-source": "^7.27.1", + "@rolldown/pluginutils": "1.0.0-beta.27", + "@types/babel__core": "^7.20.5", + "react-refresh": "^0.17.0" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "peerDependencies": { + "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" + } + }, + "node_modules/baseline-browser-mapping": { + "version": "2.11.27", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.27.tgz", + "integrity": "sha512-ElY12DaROGuan+lMmZ8Cvo/ZUbXPe7Enc/9VU/b1T3Kp4dwytRcNdR8DoSJN5SNJT/CuvcCA0DHDVmMOCePdRQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.cjs" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/browserslist": { + "version": "4.29.3", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.29.3.tgz", + "integrity": "sha512-1R4kiYKXGViqEN0CnoDrXc1StD9niAwu+j2dukWzrD4bJgsD4lDmEp0CRbc6E/vYJIfTHwPmwyaKtVSudICdPA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "baseline-browser-mapping": "^2.11.26", + "caniuse-lite": "^1.0.30001813", + "electron-to-chromium": "^1.5.439", + "node-releases": "^2.0.57", + "update-browserslist-db": "^1.3.3" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001814", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001814.tgz", + "integrity": "sha512-/Uaf1lAzr59XcMpW0o96WoEfr+VXK2OX4U9AgFoiSHsVJ4HppnIFUjtYzsyDH2+tgANaQb2/oxYGwCPapN1FpA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/cookie": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", + "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/electron-to-chromium": { + "version": "1.5.444", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.444.tgz", + "integrity": "sha512-5ss/uJfoDYDHT0lfJzT6FbcskIzROIOPf0BbbFkGcvDzoJU7i//9GDrwwIHQVmIsrAGiF3ihpADBRIsrEFt1rQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/enhanced-resolve": { + "version": "5.26.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.26.0.tgz", + "integrity": "sha512-9vhedylFonb2YGogzUKX6+Ja72gOJbN1QHAqdrvqLwhdl/QWbKopzoUC9EbQNVsAns/bx/4uyqalrQAoy1IByw==", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "tapable": "^2.3.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/esbuild": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", + "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.12", + "@esbuild/android-arm": "0.25.12", + "@esbuild/android-arm64": "0.25.12", + "@esbuild/android-x64": "0.25.12", + "@esbuild/darwin-arm64": "0.25.12", + "@esbuild/darwin-x64": "0.25.12", + "@esbuild/freebsd-arm64": "0.25.12", + "@esbuild/freebsd-x64": "0.25.12", + "@esbuild/linux-arm": "0.25.12", + "@esbuild/linux-arm64": "0.25.12", + "@esbuild/linux-ia32": "0.25.12", + "@esbuild/linux-loong64": "0.25.12", + "@esbuild/linux-mips64el": "0.25.12", + "@esbuild/linux-ppc64": "0.25.12", + "@esbuild/linux-riscv64": "0.25.12", + "@esbuild/linux-s390x": "0.25.12", + "@esbuild/linux-x64": "0.25.12", + "@esbuild/netbsd-arm64": "0.25.12", + "@esbuild/netbsd-x64": "0.25.12", + "@esbuild/openbsd-arm64": "0.25.12", + "@esbuild/openbsd-x64": "0.25.12", + "@esbuild/openharmony-arm64": "0.25.12", + "@esbuild/sunos-x64": "0.25.12", + "@esbuild/win32-arm64": "0.25.12", + "@esbuild/win32-ia32": "0.25.12", + "@esbuild/win32-x64": "0.25.12" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/jiti": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.7.0.tgz", + "integrity": "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==", + "dev": true, + "license": "MIT", + "bin": { + "jiti": "lib/jiti-cli.mjs" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/lightningcss": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", + "integrity": "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.32.0", + "lightningcss-darwin-arm64": "1.32.0", + "lightningcss-darwin-x64": "1.32.0", + "lightningcss-freebsd-x64": "1.32.0", + "lightningcss-linux-arm-gnueabihf": "1.32.0", + "lightningcss-linux-arm64-gnu": "1.32.0", + "lightningcss-linux-arm64-musl": "1.32.0", + "lightningcss-linux-x64-gnu": "1.32.0", + "lightningcss-linux-x64-musl": "1.32.0", + "lightningcss-win32-arm64-msvc": "1.32.0", + "lightningcss-win32-x64-msvc": "1.32.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz", + "integrity": "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.32.0.tgz", + "integrity": "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.32.0.tgz", + "integrity": "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.32.0.tgz", + "integrity": "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.32.0.tgz", + "integrity": "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.32.0.tgz", + "integrity": "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.32.0.tgz", + "integrity": "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.32.0.tgz", + "integrity": "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.32.0.tgz", + "integrity": "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.32.0.tgz", + "integrity": "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.32.0.tgz", + "integrity": "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/lucide-react": { + "version": "0.511.0", + "resolved": "https://registry.npmjs.org/lucide-react/-/lucide-react-0.511.0.tgz", + "integrity": "sha512-VK5a2ydJ7xm8GvBeKLS9mu1pVK6ucef9780JVUjw6bAjJL/QXnd4Y0p7SPeOUMC27YhzNCZvm5d/QX0Tp3rc0w==", + "license": "ISC", + "peerDependencies": { + "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.19", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.19.tgz", + "integrity": "sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/node-releases": { + "version": "2.0.57", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.57.tgz", + "integrity": "sha512-kQK9LGGFiHtrWiNhZtA7Qbw17AQz+dmsEKODRIVTXA9+e5MS/2gZEBhYJt13GrAz5/IOZKddH/0Z3TP/Zgo+yw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.28", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz", + "integrity": "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.18", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/react": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/react/-/react-19.3.0.tgz", + "integrity": "sha512-E8LUcbtBWt20bbl2YoHfx4ZDBdxVTfOKtCZn9cDSJ4l6/nuoApcpIBcj47t2wZoVX8g2ZHuMHbiShgCR1T5Sog==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.3.0.tgz", + "integrity": "sha512-JDk8dgif51OjFoDE70+OT9ICyYr+69HlmihNwp1+Nsfbna3t5sIiCa9ZJktDmQ4/1b/rn26hIAR2uYXDMr5r0Q==", + "license": "MIT", + "dependencies": { + "scheduler": "^0.28.0" + }, + "peerDependencies": { + "react": "^19.3.0" + } + }, + "node_modules/react-refresh": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.17.0.tgz", + "integrity": "sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-router": { + "version": "7.18.4", + "resolved": "https://registry.npmjs.org/react-router/-/react-router-7.18.4.tgz", + "integrity": "sha512-PUPQcMhMGRAslLcvtlPz/kmzBEWPhLdgLFrL7pLNepBL6dX0lWj4WD2cUYVgYCuT3jxvghYFg81cDTj44DhetQ==", + "license": "MIT", + "dependencies": { + "cookie": "^1.0.1", + "set-cookie-parser": "^2.6.0" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "react": ">=18", + "react-dom": ">=18" + }, + "peerDependenciesMeta": { + "react-dom": { + "optional": true + } + } + }, + "node_modules/rollup": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.64.0.tgz", + "integrity": "sha512-gjA/kFeDfffII8COoPsvLPvE16hWDTEc1+0QKQNKUkuV8iJjqsC4xXs8/FzuL0v9EV39kuxTbAgcFoTRyy2TGQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.9" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@napi-rs/lzma-linux-x64-gnu": "1.5.1", + "@rollup/rollup-android-arm-eabi": "4.64.0", + "@rollup/rollup-android-arm64": "4.64.0", + "@rollup/rollup-darwin-arm64": "4.64.0", + "@rollup/rollup-darwin-x64": "4.64.0", + "@rollup/rollup-freebsd-arm64": "4.64.0", + "@rollup/rollup-freebsd-x64": "4.64.0", + "@rollup/rollup-linux-arm-gnueabihf": "4.64.0", + "@rollup/rollup-linux-arm-musleabihf": "4.64.0", + "@rollup/rollup-linux-arm64-gnu": "4.64.0", + "@rollup/rollup-linux-arm64-musl": "4.64.0", + "@rollup/rollup-linux-loong64-gnu": "4.64.0", + "@rollup/rollup-linux-loong64-musl": "4.64.0", + "@rollup/rollup-linux-ppc64-gnu": "4.64.0", + "@rollup/rollup-linux-ppc64-musl": "4.64.0", + "@rollup/rollup-linux-riscv64-gnu": "4.64.0", + "@rollup/rollup-linux-riscv64-musl": "4.64.0", + "@rollup/rollup-linux-s390x-gnu": "4.64.0", + "@rollup/rollup-linux-x64-gnu": "4.64.0", + "@rollup/rollup-linux-x64-musl": "4.64.0", + "@rollup/rollup-openbsd-x64": "4.64.0", + "@rollup/rollup-openharmony-arm64": "4.64.0", + "@rollup/rollup-win32-arm64-msvc": "4.64.0", + "@rollup/rollup-win32-ia32-msvc": "4.64.0", + "@rollup/rollup-win32-x64-gnu": "4.64.0", + "@rollup/rollup-win32-x64-msvc": "4.64.0", + "fsevents": "~2.3.2" + } + }, + "node_modules/scheduler": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.28.0.tgz", + "integrity": "sha512-juorfCmIkIw8tT+p5BXSm6PJjQF/ycEYmKyzURCIt/RaZIhL+PulbQ9Yu2z1HdOJDdqDTlxA1+xKBmHXJsczAw==", + "license": "MIT" + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/set-cookie-parser": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.2.tgz", + "integrity": "sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==", + "license": "MIT" + }, + "node_modules/source-map-js": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz", + "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tailwindcss": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.3.3.tgz", + "integrity": "sha512-gOhV3P7ufE62QDGg1zVaTgCR+EtPv92k2nIhVcVKcLmxT1sUBsQGhnZj175j+MqRt4zLF7ic+sCYjfhxMxj7YQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/tapable": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.3.tgz", + "integrity": "sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.3.3.tgz", + "integrity": "sha512-pJ2sYawQS0R/WI928Gj5GlPhTGzbMelq0+4INtSYNDV9ErKJcX6xjGWkoG/VnB3dpUm00zALaqkrUD77pO5TDQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/uplot": { + "version": "1.6.32", + "resolved": "https://registry.npmjs.org/uplot/-/uplot-1.6.32.tgz", + "integrity": "sha512-KIMVnG68zvu5XXUbC4LQEPnhwOxBuLyW1AHtpm6IKTXImkbLgkMy+jabjLgSLMasNuGGzQm/ep3tOkyTxpiQIw==", + "license": "MIT" + }, + "node_modules/vite": { + "version": "6.4.3", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.3.tgz", + "integrity": "sha512-NTKlcQjlAK7MlQoyb6LgaqHc8sso/pVyUJYWMws3jg21uTJw/LddqIFPcPqP6PzpgbIcZyKI85sFE4HBrQDA8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.25.0", + "fdir": "^6.4.4", + "picomatch": "^4.0.2", + "postcss": "^8.5.3", + "rollup": "^4.34.9", + "tinyglobby": "^0.2.13" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "jiti": ">=1.21.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true, + "license": "ISC" + } + } +} diff --git a/web/package.json b/web/package.json new file mode 100644 index 0000000..14227c4 --- /dev/null +++ b/web/package.json @@ -0,0 +1,29 @@ +{ + "name": "reforger-crawler-admin", + "private": true, + "version": "1.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "clean": "node -e \"const fs=require('fs');fs.rmSync('dist/assets',{recursive:true,force:true});fs.rmSync('dist/index.html',{force:true})\"", + "build": "npm run clean && tsc -b && vite build", + "typecheck": "tsc -b" + }, + "dependencies": { + "@tanstack/react-query": "^5.80.0", + "lucide-react": "^0.511.0", + "react": "^19.1.0", + "react-dom": "^19.1.0", + "react-router": "^7.6.0", + "uplot": "^1.6.32" + }, + "devDependencies": { + "@tailwindcss/vite": "^4.1.8", + "@types/react": "^19.1.6", + "@types/react-dom": "^19.1.5", + "@vitejs/plugin-react": "^4.5.0", + "tailwindcss": "^4.1.8", + "typescript": "^5.8.3", + "vite": "^6.3.5" + } +} diff --git a/web/public/favicon.svg b/web/public/favicon.svg new file mode 100644 index 0000000..5fbe536 --- /dev/null +++ b/web/public/favicon.svg @@ -0,0 +1 @@ + diff --git a/web/public/theme-init.js b/web/public/theme-init.js new file mode 100644 index 0000000..de0d16f --- /dev/null +++ b/web/public/theme-init.js @@ -0,0 +1,5 @@ +// Applies the saved theme before first paint (external file: the CSP blocks inline scripts). +try { + var t = localStorage.getItem("rc-theme"); + document.documentElement.setAttribute("data-theme", t === "light" ? "light" : "dark"); +} catch (e) {} diff --git a/web/spa.go b/web/spa.go new file mode 100644 index 0000000..f99d8e6 --- /dev/null +++ b/web/spa.go @@ -0,0 +1,75 @@ +package web + +import ( + "io/fs" + "net/http" + "strings" + + "github.com/gin-gonic/gin" +) + +const notBuiltHTML = `Admin UI not built + +

Admin UI not built

Run npm --prefix web ci && npm --prefix web run build and rebuild the binary.

+

The JSON API is available under /admin/api.

` + +const csp = "default-src 'self'; img-src 'self' https: data:; style-src 'self' 'unsafe-inline'; " + + "script-src 'self'; connect-src 'self'; font-src 'self' data:; frame-ancestors 'none'; base-uri 'self'; form-action 'self'" + +func securityHeaders(c *gin.Context) { + c.Header("Content-Security-Policy", csp) + c.Header("X-Content-Type-Options", "nosniff") + c.Header("Referrer-Policy", "no-referrer") + c.Header("X-Frame-Options", "DENY") +} + +// Handler returns the SPA handler: real files are served from the embedded +// build, every other path gets index.html so client-side routes deep-link. +func Handler() gin.HandlerFunc { + sub, err := fs.Sub(dist, "dist") + if err != nil { + panic(err) + } + index, err := fs.ReadFile(sub, "index.html") + if err != nil { + index = []byte(notBuiltHTML) + } + files := http.FileServer(http.FS(sub)) + + return func(c *gin.Context) { + securityHeaders(c) + p := strings.TrimPrefix(c.Param("filepath"), "/") + if p != "" && p != "index.html" { + if st, err := fs.Stat(sub, p); err == nil && !st.IsDir() { + if strings.HasPrefix(p, "assets/") { + c.Header("Cache-Control", "public, max-age=31536000, immutable") + } else { + c.Header("Cache-Control", "no-cache") + } + req := c.Request.Clone(c.Request.Context()) + req.URL.Path = "/" + p + files.ServeHTTP(c.Writer, req) + return + } + // a missing hashed asset must not turn into an HTML page + if strings.HasPrefix(p, "assets/") { + c.Status(http.StatusNotFound) + return + } + } + // serve the bytes directly; FileServer would redirect /index.html to ./ + c.Header("Cache-Control", "no-cache") + c.Data(http.StatusOK, "text/html; charset=utf-8", index) + } +} + +// Register mounts the dashboard at /admin/ui and redirects /admin there. +func Register(r *gin.Engine) { + h := Handler() + r.GET("/admin/ui/*filepath", h) + r.HEAD("/admin/ui/*filepath", h) + redirect := func(c *gin.Context) { c.Redirect(http.StatusFound, "/admin/ui/") } + r.GET("/admin", redirect) + r.GET("/admin/ui", redirect) +} diff --git a/web/spa_test.go b/web/spa_test.go new file mode 100644 index 0000000..6e3b341 --- /dev/null +++ b/web/spa_test.go @@ -0,0 +1,51 @@ +package web + +import ( + "io/fs" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gin-gonic/gin" +) + +func TestSPA(t *testing.T) { + gin.SetMode(gin.TestMode) + r := gin.New() + Register(r) + + get := func(path string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + return w + } + + for _, p := range []string{"/admin/ui/", "/admin/ui/addons/123", "/admin/ui/index.html"} { + w := get(p) + if w.Code != http.StatusOK || !strings.Contains(w.Header().Get("Content-Type"), "text/html") { + t.Errorf("%s: got %d %s", p, w.Code, w.Header().Get("Content-Type")) + } + if w.Header().Get("Content-Security-Policy") == "" { + t.Errorf("%s: missing CSP", p) + } + } + + if w := get("/admin/ui/assets/missing.js"); w.Code != http.StatusNotFound { + t.Errorf("missing asset: got %d, want 404", w.Code) + } + if w := get("/admin"); w.Code != http.StatusFound || w.Header().Get("Location") != "/admin/ui/" { + t.Errorf("/admin: got %d -> %s", w.Code, w.Header().Get("Location")) + } + + // when the UI is built, hashed assets are served with long caching + sub, _ := fs.Sub(dist, "dist") + entries, _ := fs.ReadDir(sub, "assets") + for _, e := range entries { + w := get("/admin/ui/assets/" + e.Name()) + if w.Code != http.StatusOK || !strings.Contains(w.Header().Get("Cache-Control"), "immutable") { + t.Errorf("asset %s: got %d cache %q", e.Name(), w.Code, w.Header().Get("Cache-Control")) + } + break + } +} diff --git a/web/src/components/Shell.tsx b/web/src/components/Shell.tsx new file mode 100644 index 0000000..753e5e8 --- /dev/null +++ b/web/src/components/Shell.tsx @@ -0,0 +1,160 @@ +import { useEffect, type ReactNode } from "react"; +import { NavLink, Navigate, Outlet, useLocation, useNavigate } from "react-router"; +import { useIsFetching, useQuery, useQueryClient } from "@tanstack/react-query"; +import { + Activity, + Cpu, + LayoutDashboard, + LineChart, + ListOrdered, + LogOut, + Moon, + Package, + RefreshCw, + ScrollText, + Server, + ShieldCheck, + Sun, +} from "lucide-react"; +import { api, ApiError, UNAUTHORIZED_EVENT } from "../lib/api"; +import { RANGES, REFRESH, useSettings } from "../lib/settings"; +import { Loading, Segmented, Select, cn } from "./ui"; + +const NAV: { to: string; label: string; icon: ReactNode }[] = [ + { to: "/", label: "Overview", icon: }, + { to: "/traffic", label: "Traffic", icon: }, + { to: "/queue", label: "Indexing queue", icon: }, + { to: "/addons", label: "Addons", icon: }, + { to: "/whitelist", label: "Whitelist", icon: }, + { to: "/services", label: "Services", icon: }, + { to: "/explorer", label: "Metrics explorer", icon: }, + { to: "/events", label: "Events", icon: }, + { to: "/system", label: "System", icon: }, +]; + +/** Redirects to /login when the session is missing or expired. */ +export function RequireAuth() { + const navigate = useNavigate(); + const qc = useQueryClient(); + const location = useLocation(); + const me = useQuery({ queryKey: ["me"], queryFn: () => api<{ method: string }>("/auth/me"), retry: false, staleTime: 60_000 }); + + useEffect(() => { + const onUnauthorized = () => { + qc.clear(); + navigate("/login", { replace: true, state: { from: location.pathname } }); + }; + window.addEventListener(UNAUTHORIZED_EVENT, onUnauthorized); + return () => window.removeEventListener(UNAUTHORIZED_EVENT, onUnauthorized); + }, [navigate, qc, location.pathname]); + + if (me.isLoading) return
; + if (me.error instanceof ApiError && (me.error.status === 401 || me.error.status === 503)) { + return ; + } + return ; +} + +function Shell() { + const { theme, setTheme, range, setRange, refresh, setRefresh } = useSettings(); + const fetching = useIsFetching(); + const navigate = useNavigate(); + const qc = useQueryClient(); + + const logout = async () => { + await api("/auth/logout", { method: "POST" }).catch(() => undefined); + qc.clear(); + navigate("/login", { replace: true }); + }; + + return ( +
+ + +
+
+
+ {NAV.map((n) => ( + + cn("flex shrink-0 items-center gap-1.5 rounded-lg px-2 py-1 text-xs", isActive ? "bg-surface-2 text-ink" : "text-ink-2") + } + > + {n.icon} + {n.label} + + ))} +
+
+ 0 && "animate-spin text-accent")} aria-label={fetching ? "Refreshing" : "Idle"} /> + + ({ key: r.key, label: r.label }))} value={range} onChange={setRange} /> + +
+
+
+ +
+
+
+ ); +} diff --git a/web/src/components/charts/BarList.tsx b/web/src/components/charts/BarList.tsx new file mode 100644 index 0000000..1debfcb --- /dev/null +++ b/web/src/components/charts/BarList.tsx @@ -0,0 +1,37 @@ +import type { ReactNode } from "react"; +import { compact } from "../../lib/format"; +import { cn } from "../ui"; + +export interface BarItem { + key: string; + label: ReactNode; + value: number; + hint?: string; + onClick?: () => void; +} + +/** Ranked list with a thin magnitude bar under each label. */ +export function BarList({ items, format = compact, empty = "Nothing yet" }: { items: BarItem[]; format?: (v: number) => string; empty?: string }) { + if (items.length === 0) return
{empty}
; + const max = Math.max(...items.map((i) => i.value), 1); + return ( +
    + {items.map((it) => ( +
  • +
    + {it.label} + {format(it.value)} +
    +
    +
    +
    +
    • + ))} +
+ ); +} diff --git a/web/src/components/charts/Sparkline.tsx b/web/src/components/charts/Sparkline.tsx new file mode 100644 index 0000000..e32300e --- /dev/null +++ b/web/src/components/charts/Sparkline.tsx @@ -0,0 +1,36 @@ +/** Tiny trend line: de-emphasised line, latest value marked in the accent. */ +export function Sparkline({ data, className }: { data: (number | null)[]; className?: string }) { + const vals = data.map((v) => (v === null || Number.isNaN(v) ? null : v)); + const nums = vals.filter((v): v is number => v !== null); + if (nums.length < 2) return null; + const min = Math.min(...nums, 0); + const max = Math.max(...nums); + const W = 96; + const H = 32; + const pad = 4; + const x = (i: number) => (i / (vals.length - 1)) * (W - pad * 2) + pad; + const y = (v: number) => H - pad - (max === min ? 0.5 : (v - min) / (max - min)) * (H - pad * 2); + + const segments: string[] = []; + let cur = ""; + vals.forEach((v, i) => { + if (v === null) { + if (cur) segments.push(cur); + cur = ""; + return; + } + cur += `${cur ? "L" : "M"}${x(i).toFixed(1)},${y(v).toFixed(1)}`; + }); + if (cur) segments.push(cur); + let lastIdx = vals.length - 1; + while (lastIdx >= 0 && vals[lastIdx] === null) lastIdx--; + + return ( + + {segments.map((d, i) => ( + + ))} + {lastIdx >= 0 && } + + ); +} diff --git a/web/src/components/charts/TimeChart.tsx b/web/src/components/charts/TimeChart.tsx new file mode 100644 index 0000000..6c6cb1e --- /dev/null +++ b/web/src/components/charts/TimeChart.tsx @@ -0,0 +1,233 @@ +import { useEffect, useMemo, useRef, useState } from "react"; +import uPlot from "uplot"; +import { useSettings } from "../../lib/settings"; +import type { QueryResult } from "../../lib/types"; +import { compact } from "../../lib/format"; +import { cn } from "../ui"; + +export interface ChartSeries { + label: string; + data: (number | null)[]; + /** fixed categorical slot (1-8) so an entity keeps its colour across filters */ + slot?: number; +} + +export interface ChartData { + x: number[]; + series: ChartSeries[]; +} + +/** Converts a metrics query result into chart data. */ +export function fromQuery( + q: QueryResult | null | undefined, + label: (labels: Record, i: number) => string = (l) => Object.values(l).join(" · ") || q?.name || "value", + scale = 1, + slotOf?: (labels: Record) => number | undefined, +): ChartData { + if (!q || q.series.length === 0) return { x: [], series: [] }; + const x = q.series[0].points.map((p) => p[0]); + return { + x, + series: q.series.map((s, i) => ({ + label: label(s.labels, i), + slot: slotOf?.(s.labels), + data: s.points.map((p) => (p[1] === null ? null : p[1] * scale)), + })), + }; +} + +function cssVar(name: string): string { + return getComputedStyle(document.documentElement).getPropertyValue(name).trim(); +} + +function seriesColor(i: number, slot?: number): string { + const n = slot ?? i + 1; + return cssVar(`--series-${((n - 1) % 8) + 1}`); +} + +function fmtTick(ts: number, span: number): string { + const d = new Date(ts * 1000); + if (span >= 2 * 86400) return d.toLocaleDateString("en-GB", { day: "2-digit", month: "short" }); + return d.toLocaleTimeString("en-GB", { hour: "2-digit", minute: "2-digit" }); +} + +function fmtTime(ts: number, span: number): string { + const d = new Date(ts * 1000); + if (span > 2 * 86400) return d.toLocaleString("en-GB", { month: "short", day: "2-digit", hour: "2-digit", minute: "2-digit" }); + return d.toLocaleTimeString("en-GB", { hour: "2-digit", minute: "2-digit" }); +} + +function tooltipPlugin(format: (v: number) => string, labels: () => ChartSeries[]): uPlot.Plugin { + let tip: HTMLDivElement; + return { + hooks: { + init: (u) => { + tip = document.createElement("div"); + tip.className = + "pointer-events-none absolute z-10 hidden min-w-36 rounded-lg border border-line bg-surface px-2.5 py-2 text-xs text-ink shadow-lg"; + u.over.appendChild(tip); + u.over.addEventListener("mouseleave", () => tip.classList.add("hidden")); + }, + setCursor: (u) => { + const idx = u.cursor.idx; + if (idx === null || idx === undefined || u.cursor.left === undefined || u.cursor.left < 0) { + tip.classList.add("hidden"); + return; + } + const ts = u.data[0][idx]; + const span = (u.data[0][u.data[0].length - 1] ?? 0) - (u.data[0][0] ?? 0); + const rows = labels() + .map((s, i) => { + const v = u.data[i + 1][idx]; + const color = (u.series[i + 1].stroke as () => string)?.() ?? ""; + return `
${escapeHtml(s.label)}${v === null || v === undefined ? "—" : format(v)}
`; + }) + .join(""); + tip.innerHTML = `
${fmtTime(ts, span)}${span > 2 * 86400 ? "" : " · " + new Date(ts * 1000).toLocaleDateString("en-GB", { day: "2-digit", month: "short" })}
${rows}`; + tip.classList.remove("hidden"); + const left = u.cursor.left; + const w = tip.offsetWidth; + const overW = u.over.clientWidth; + tip.style.left = `${left + 12 + w > overW ? left - w - 12 : left + 12}px`; + tip.style.top = `4px`; + }, + }, + }; +} + +function escapeHtml(s: string): string { + return s.replace(/[&<>"]/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """ })[c] as string); +} + +export function TimeChart({ + data, + height = 220, + format = compact, + area, + bars, + yMin = 0, + className, + empty = "No data for this range yet", +}: { + data: ChartData; + height?: number; + format?: (v: number) => string; + area?: boolean; + bars?: boolean; + yMin?: number | null; + className?: string; + empty?: string; +}) { + const { theme } = useSettings(); + const wrapRef = useRef(null); + const plotRef = useRef(null); + const seriesRef = useRef(data.series); + seriesRef.current = data.series; + const [width, setWidth] = useState(0); + + const hasData = data.x.length > 0 && data.series.some((s) => s.data.some((v) => v !== null)); + const shape = `${data.series.map((s) => `${s.label}#${s.slot ?? ""}`).join("|")}`; + const aligned = useMemo(() => [data.x, ...data.series.map((s) => s.data)] as uPlot.AlignedData, [data]); + + useEffect(() => { + const el = wrapRef.current; + if (!el) return; + const ro = new ResizeObserver((entries) => setWidth(Math.floor(entries[0].contentRect.width))); + ro.observe(el); + return () => ro.disconnect(); + }, []); + + // (re)create when the theme or the set of series changes + useEffect(() => { + const el = wrapRef.current; + if (!el || !hasData || width === 0) return; + const muted = cssVar("--muted"); + const grid = cssVar("--grid"); + const font = "11px system-ui, -apple-system, Segoe UI, sans-serif"; + const single = data.series.length === 1; + const opts: uPlot.Options = { + width, + height, + padding: [8, 14, 0, 0], + legend: { show: false }, + cursor: { points: { size: 8, width: 2 }, drag: { x: false, y: false } }, + scales: { x: { time: true }, y: { range: (_u, min, max) => [yMin === null ? min : Math.min(yMin, min), max <= 0 ? 1 : max * 1.08] } }, + axes: [ + { + stroke: muted, + font, + grid: { show: false }, + ticks: { stroke: grid, width: 1, size: 4 }, + space: 70, + values: (_u, splits) => { + const span = (splits[splits.length - 1] ?? 0) - (splits[0] ?? 0); + return splits.map((ts) => fmtTick(ts, span)); + }, + }, + { + stroke: muted, + font, + size: 48, + space: 36, + grid: { stroke: grid, width: 1 }, + ticks: { show: false }, + values: (_u, vals) => vals.map((v) => format(v)), + }, + ], + series: [ + {}, + ...data.series.map((s, i) => { + const color = seriesColor(i, s.slot); + return { + label: s.label, + stroke: color, + width: 2, + spanGaps: false, + // uPlot shows markers only when samples are far apart, so sparse data stays visible + points: { size: 7, width: 2, stroke: cssVar("--surface"), fill: color }, + fill: bars ? color : area || single ? `color-mix(in srgb, ${color} 12%, transparent)` : undefined, + paths: bars ? uPlot.paths.bars!({ size: [0.6, 24], radius: 0.15 }) : undefined, + } as uPlot.Series; + }), + ], + plugins: [tooltipPlugin(format, () => seriesRef.current)], + }; + plotRef.current?.destroy(); + plotRef.current = new uPlot(opts, aligned, el); + return () => { + plotRef.current?.destroy(); + plotRef.current = null; + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [theme, shape, hasData, height, bars, area, width === 0]); + + useEffect(() => { + if (plotRef.current && width > 0) plotRef.current.setSize({ width, height }); + }, [width, height]); + + useEffect(() => { + plotRef.current?.setData(aligned); + }, [aligned]); + + return ( +
+
+ {!hasData &&
{empty}
} +
+ {hasData && data.series.length > 1 && } +
+ ); +} + +export function Legend({ series }: { series: ChartSeries[] }) { + return ( +
+ {series.map((s, i) => ( + + + {s.label} + + ))} +
+ ); +} diff --git a/web/src/components/ui.tsx b/web/src/components/ui.tsx new file mode 100644 index 0000000..b32cf60 --- /dev/null +++ b/web/src/components/ui.tsx @@ -0,0 +1,484 @@ +import { + createContext, + useCallback, + useContext, + useEffect, + useRef, + useState, + type ButtonHTMLAttributes, + type InputHTMLAttributes, + type ReactNode, + type SelectHTMLAttributes, + type TextareaHTMLAttributes, +} from "react"; +import { AlertTriangle, Check, CheckCircle2, ChevronLeft, ChevronRight, Copy, Info, Loader2, X, XCircle } from "lucide-react"; +import { Sparkline } from "./charts/Sparkline"; + +export function cn(...parts: (string | false | null | undefined)[]): string { + return parts.filter(Boolean).join(" "); +} + +// ---- containers ------------------------------------------------------------- + +export function Card({ + title, + subtitle, + actions, + children, + className, + bodyClass, +}: { + title?: ReactNode; + subtitle?: ReactNode; + actions?: ReactNode; + children: ReactNode; + className?: string; + bodyClass?: string; +}) { + return ( +
+ {(title || actions) && ( +
+
+ {title &&

{title}

} + {subtitle &&

{subtitle}

} +
+ {actions &&
{actions}
} +
+ )} +
{children}
+
+ ); +} + +export function PageHeader({ title, description, actions }: { title: string; description?: ReactNode; actions?: ReactNode }) { + return ( +
+
+

{title}

+ {description &&

{description}

} +
+ {actions &&
{actions}
} +
+ ); +} + +// ---- controls --------------------------------------------------------------- + +type ButtonVariant = "primary" | "secondary" | "ghost" | "danger"; + +export function Button({ + variant = "secondary", + size = "md", + icon, + loading, + className, + children, + ...rest +}: ButtonHTMLAttributes & { + variant?: ButtonVariant; + size?: "sm" | "md"; + icon?: ReactNode; + loading?: boolean; +}) { + const styles: Record = { + primary: "bg-accent text-accent-ink hover:brightness-110 border-transparent", + secondary: "bg-surface-2 text-ink border-line hover:border-ink/25", + ghost: "bg-transparent text-ink-2 border-transparent hover:bg-surface-2 hover:text-ink", + danger: "bg-transparent text-critical border-critical/40 hover:bg-critical/10", + }; + return ( + + ); +} + +const fieldClass = + "h-8.5 rounded-lg border border-line bg-surface px-2.5 text-[13px] text-ink placeholder:text-muted " + + "focus:border-accent focus:outline-none"; + +export function Input({ className, ...rest }: InputHTMLAttributes) { + return ; +} + +export function Select({ className, children, ...rest }: SelectHTMLAttributes) { + return ( + + ); +} + +export function Textarea({ className, ...rest }: TextareaHTMLAttributes) { + return