- embedded React admin UI at /admin/ui (go:embed), login with ADMIN_SECRET via signed HttpOnly session cookie, X-ADMIN-KEY still works for scripts - admin API: overview stats, indexing queue, addon management (priority, reindex, delete/restore/purge, bulk), whitelist CRUD, duplicate hashes, traffic, metrics explorer, services, events/audit log, system info - built-in metrics: request latency per route, visitors, top addons and searches, indexer/scraper counters; minute buckets in the DB with hourly rollups and retention; optional Prometheus /metrics (METRICS_TOKEN) - POST /back/telemetry for indexers/scrapers (metrics, events, heartbeat) and a stdlib-only Go client in telemetry/ - auth hardening: empty secrets no longer allow access, constant-time compare, CORS not applied to /admin and /back, TRUSTED_PROXIES setting - Dockerfile builds the UI and no longer bakes config.yaml into the image; CI builds the UI, runs tests and builds arm64 with CGO
This commit is contained in:
ilbinek committed
1 parent
0ef27b0072
commit
56d9e10621
79 files changed
+13146
-118
No files matched your search
@@ -0,0 +1,325 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"regexp"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
|
||||
"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
var nameRe = regexp.MustCompile(`^[a-z_][a-z0-9_]{0,63}$`)
|
||||
|
||||
// ValidName reports whether a service, metric or label name is acceptable.
|
||||
func ValidName(s string) bool { return nameRe.MatchString(s) }
|
||||
|
||||
const (
|
||||
defaultHeartbeat = 30
|
||||
maxLabels = 8
|
||||
maxLabelValue = 128
|
||||
)
|
||||
|
||||
// normalizeTs converts unix seconds or milliseconds to seconds and clamps it
|
||||
// to [now-1h, now+1m]; anything outside becomes now.
|
||||
func normalizeTs(ts int64, now int64) int64 {
|
||||
if ts <= 0 {
|
||||
return now
|
||||
}
|
||||
if ts > 1e12 {
|
||||
ts /= 1000
|
||||
}
|
||||
if ts < now-3600 || ts > now+60 {
|
||||
return now
|
||||
}
|
||||
return ts
|
||||
}
|
||||
|
||||
// IngestResult reports what happened to a telemetry batch.
|
||||
type IngestResult struct {
|
||||
Accepted int `json:"accepted"`
|
||||
Dropped int `json:"dropped"`
|
||||
Reasons map[string]int `json:"reasons,omitempty"`
|
||||
}
|
||||
|
||||
// Ingest records external metric samples for a service instance.
|
||||
func Ingest(service, instance string, samples []models.TelemetryMetric, now time.Time) IngestResult {
|
||||
res := IngestResult{Reasons: map[string]int{}}
|
||||
drop := func(reason string) {
|
||||
res.Dropped++
|
||||
res.Reasons[reason]++
|
||||
}
|
||||
nowTs := now.Unix()
|
||||
|
||||
for _, m := range samples {
|
||||
if !ValidName(m.Name) {
|
||||
drop("invalid_name")
|
||||
continue
|
||||
}
|
||||
if len(m.Labels) > maxLabels {
|
||||
drop("too_many_labels")
|
||||
continue
|
||||
}
|
||||
badLabel := false
|
||||
for k, v := range m.Labels {
|
||||
if !ValidName(k) || len(v) > maxLabelValue {
|
||||
badLabel = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if badLabel {
|
||||
drop("invalid_label")
|
||||
continue
|
||||
}
|
||||
var kind Kind
|
||||
switch m.Type {
|
||||
case "counter":
|
||||
kind = Counter
|
||||
case "gauge", "":
|
||||
kind = Gauge
|
||||
case "summary":
|
||||
kind = Summary
|
||||
default:
|
||||
drop("invalid_type")
|
||||
continue
|
||||
}
|
||||
if math.IsNaN(m.Value) || math.IsInf(m.Value, 0) || math.IsNaN(m.Sum) || math.IsInf(m.Sum, 0) {
|
||||
drop("invalid_value")
|
||||
continue
|
||||
}
|
||||
if kind == Counter && m.Value < 0 {
|
||||
drop("negative_counter")
|
||||
continue
|
||||
}
|
||||
if kind == Summary && m.Count <= 0 {
|
||||
drop("empty_summary")
|
||||
continue
|
||||
}
|
||||
help := m.Help
|
||||
if len(help) > 256 {
|
||||
help = help[:256]
|
||||
}
|
||||
s, err := reg.get(service, instance, m.Name, m.Labels, kind, nil, help, true)
|
||||
if err != nil {
|
||||
if errors.Is(err, errKindMismatch) {
|
||||
drop("type_mismatch")
|
||||
} else {
|
||||
drop("series_limit")
|
||||
}
|
||||
continue
|
||||
}
|
||||
ts := normalizeTs(m.Ts, nowTs)
|
||||
if kind == Summary {
|
||||
lo, hi := m.Min, m.Max
|
||||
if lo == 0 && hi == 0 {
|
||||
avg := m.Sum / float64(m.Count)
|
||||
lo, hi = avg, avg
|
||||
}
|
||||
defaultAgg.observeSummary(s, ts, m.Count, m.Sum, lo, hi)
|
||||
} else {
|
||||
defaultAgg.observe(s, ts, m.Value)
|
||||
}
|
||||
res.Accepted++
|
||||
}
|
||||
TelemetrySamples.Add(float64(res.Accepted), "accepted")
|
||||
TelemetrySamples.Add(float64(res.Dropped), "dropped")
|
||||
return res
|
||||
}
|
||||
|
||||
func jsonString(v map[string]any) string {
|
||||
if len(v) == 0 {
|
||||
return ""
|
||||
}
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
if len(s) > n {
|
||||
return s[:n]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Heartbeat upserts the service instance row.
|
||||
func Heartbeat(b *models.TelemetryBatch, remoteAddr string, now time.Time) error {
|
||||
interval := b.HeartbeatIntervalSec
|
||||
if interval <= 0 {
|
||||
interval = defaultHeartbeat
|
||||
}
|
||||
status := b.Status
|
||||
if status == "" {
|
||||
status = "ok"
|
||||
}
|
||||
row := models.ServiceInstance{
|
||||
Service: b.Service,
|
||||
Instance: b.Instance,
|
||||
Version: truncate(b.Version, 64),
|
||||
Status: truncate(status, 16),
|
||||
StatusMessage: truncate(b.StatusMessage, 512),
|
||||
StartedAt: normalizeStarted(b.StartedAt),
|
||||
LastSeen: now.Unix(),
|
||||
HeartbeatIntervalSec: interval,
|
||||
RemoteAddr: truncate(remoteAddr, 64),
|
||||
Meta: jsonString(b.Meta),
|
||||
}
|
||||
return initializers.DB.Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "service"}, {Name: "instance"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{
|
||||
"version", "status", "status_message", "started_at", "last_seen",
|
||||
"heartbeat_interval_sec", "remote_addr", "meta", "updated_at",
|
||||
}),
|
||||
}).Create(&row).Error
|
||||
}
|
||||
|
||||
func normalizeStarted(ts int64) int64 {
|
||||
if ts > 1e12 {
|
||||
return ts / 1000
|
||||
}
|
||||
return ts
|
||||
}
|
||||
|
||||
// RecordEvents stores service events.
|
||||
func RecordEvents(service, instance string, events []models.TelemetryEvent, now time.Time) error {
|
||||
if len(events) == 0 {
|
||||
return nil
|
||||
}
|
||||
rows := make([]models.ServiceEvent, 0, len(events))
|
||||
for _, e := range events {
|
||||
level := e.Level
|
||||
switch level {
|
||||
case "debug", "info", "warn", "error":
|
||||
case "warning":
|
||||
level = "warn"
|
||||
default:
|
||||
level = "info"
|
||||
}
|
||||
ts := e.Ts
|
||||
if ts <= 0 {
|
||||
ts = now.UnixMilli()
|
||||
} else if ts < 1e12 {
|
||||
ts *= 1000
|
||||
}
|
||||
rows = append(rows, models.ServiceEvent{
|
||||
Service: service, Instance: instance, Level: level,
|
||||
Message: truncate(e.Message, 1024), Fields: jsonString(e.Fields), TS: ts,
|
||||
})
|
||||
}
|
||||
return initializers.DB.CreateInBatches(&rows, 100).Error
|
||||
}
|
||||
|
||||
// Event records a single event from the API itself.
|
||||
func Event(level, message string, fields map[string]any) {
|
||||
err := RecordEvents(MainService, Instance(), []models.TelemetryEvent{{Level: level, Message: message, Fields: fields}}, time.Now())
|
||||
if err != nil {
|
||||
fmt.Printf("metrics: failed to record event: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Online reports whether an instance has sent a heartbeat recently enough.
|
||||
func Online(inst *models.ServiceInstance, now time.Time) bool {
|
||||
interval := inst.HeartbeatIntervalSec
|
||||
if interval <= 0 {
|
||||
interval = defaultHeartbeat
|
||||
}
|
||||
return now.Unix()-inst.LastSeen < int64(3*interval)
|
||||
}
|
||||
|
||||
var (
|
||||
upMu sync.Mutex
|
||||
upState = map[string]bool{} // "service/instance" -> online
|
||||
)
|
||||
|
||||
// ServiceUp returns the last known online state of every instance.
|
||||
func ServiceUp() map[[2]string]bool {
|
||||
upMu.Lock()
|
||||
defer upMu.Unlock()
|
||||
out := make(map[[2]string]bool, len(upState))
|
||||
for k, v := range upState {
|
||||
var svc, inst string
|
||||
for i := 0; i < len(k); i++ {
|
||||
if k[i] == '/' {
|
||||
svc, inst = k[:i], k[i+1:]
|
||||
break
|
||||
}
|
||||
}
|
||||
out[[2]string{svc, inst}] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// checkServices records online/offline transitions as events.
|
||||
func checkServices(now time.Time) {
|
||||
var rows []models.ServiceInstance
|
||||
if err := initializers.DB.Find(&rows).Error; err != nil {
|
||||
return
|
||||
}
|
||||
upMu.Lock()
|
||||
known := make(map[string]bool, len(rows))
|
||||
type change struct {
|
||||
inst models.ServiceInstance
|
||||
online bool
|
||||
}
|
||||
var changes []change
|
||||
for _, r := range rows {
|
||||
k := r.Service + "/" + r.Instance
|
||||
known[k] = true
|
||||
online := Online(&r, now)
|
||||
prev, seen := upState[k]
|
||||
upState[k] = online
|
||||
if seen && prev != online {
|
||||
changes = append(changes, change{r, online})
|
||||
}
|
||||
}
|
||||
for k := range upState {
|
||||
if !known[k] {
|
||||
delete(upState, k)
|
||||
}
|
||||
}
|
||||
upMu.Unlock()
|
||||
|
||||
for _, ch := range changes {
|
||||
level, msg := "info", "back online"
|
||||
if !ch.online {
|
||||
level, msg = "warn", "heartbeat lost"
|
||||
}
|
||||
_ = RecordEvents(ch.inst.Service, ch.inst.Instance, []models.TelemetryEvent{{
|
||||
Level: level, Message: msg,
|
||||
Fields: map[string]any{"kind": "status", "lastSeen": ch.inst.LastSeen},
|
||||
}}, now)
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
buildMu sync.Mutex
|
||||
buildVersion = "dev"
|
||||
startedAt = time.Now()
|
||||
)
|
||||
|
||||
// SetVersion sets the version reported in the API's own heartbeat.
|
||||
func SetVersion(v string) {
|
||||
buildMu.Lock()
|
||||
buildVersion = v
|
||||
buildMu.Unlock()
|
||||
}
|
||||
|
||||
// StartedAt is when this process started.
|
||||
func StartedAt() time.Time { return startedAt }
|
||||
|
||||
func selfHeartbeat(now time.Time) {
|
||||
buildMu.Lock()
|
||||
v := buildVersion
|
||||
buildMu.Unlock()
|
||||
_ = Heartbeat(&models.TelemetryBatch{
|
||||
Service: MainService, Instance: Instance(), Version: v,
|
||||
StartedAt: startedAt.Unix(), HeartbeatIntervalSec: 30, Status: "ok",
|
||||
}, "", now)
|
||||
}
|
||||
Reference in new issue
Block a user