package metrics import ( "encoding/json" "errors" "fmt" "math" "regexp" "sync" "time" "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" "gorm.io/gorm/clause" ) var nameRe = regexp.MustCompile(`^[a-z_][a-z0-9_]{0,63}$`) // ValidName reports whether a service, metric or label name is acceptable. func ValidName(s string) bool { return nameRe.MatchString(s) } const ( defaultHeartbeat = 30 maxLabels = 8 maxLabelValue = 128 ) // normalizeTs converts unix seconds or milliseconds to seconds and clamps it // to [now-1h, now+1m]; anything outside becomes now. func normalizeTs(ts int64, now int64) int64 { if ts <= 0 { return now } if ts > 1e12 { ts /= 1000 } if ts < now-3600 || ts > now+60 { return now } return ts } // IngestResult reports what happened to a telemetry batch. type IngestResult struct { Accepted int `json:"accepted"` Dropped int `json:"dropped"` Reasons map[string]int `json:"reasons,omitempty"` } // Ingest records external metric samples for a service instance. func Ingest(service, instance string, samples []models.TelemetryMetric, now time.Time) IngestResult { res := IngestResult{Reasons: map[string]int{}} drop := func(reason string) { res.Dropped++ res.Reasons[reason]++ } nowTs := now.Unix() for _, m := range samples { if !ValidName(m.Name) { drop("invalid_name") continue } if len(m.Labels) > maxLabels { drop("too_many_labels") continue } badLabel := false for k, v := range m.Labels { if !ValidName(k) || len(v) > maxLabelValue { badLabel = true break } } if badLabel { drop("invalid_label") continue } var kind Kind switch m.Type { case "counter": kind = Counter case "gauge", "": kind = Gauge case "summary": kind = Summary default: drop("invalid_type") continue } if math.IsNaN(m.Value) || math.IsInf(m.Value, 0) || math.IsNaN(m.Sum) || math.IsInf(m.Sum, 0) { drop("invalid_value") continue } if kind == Counter && m.Value < 0 { drop("negative_counter") continue } if kind == Summary && m.Count <= 0 { drop("empty_summary") continue } help := m.Help if len(help) > 256 { help = help[:256] } s, err := reg.get(service, instance, m.Name, m.Labels, kind, nil, help, true) if err != nil { if errors.Is(err, errKindMismatch) { drop("type_mismatch") } else { drop("series_limit") } continue } ts := normalizeTs(m.Ts, nowTs) if kind == Summary { lo, hi := m.Min, m.Max if lo == 0 && hi == 0 { avg := m.Sum / float64(m.Count) lo, hi = avg, avg } defaultAgg.observeSummary(s, ts, m.Count, m.Sum, lo, hi) } else { defaultAgg.observe(s, ts, m.Value) } res.Accepted++ } TelemetrySamples.Add(float64(res.Accepted), "accepted") TelemetrySamples.Add(float64(res.Dropped), "dropped") return res } func jsonString(v map[string]any) string { if len(v) == 0 { return "" } b, err := json.Marshal(v) if err != nil { return "" } return string(b) } func truncate(s string, n int) string { if len(s) > n { return s[:n] } return s } // Heartbeat upserts the service instance row. func Heartbeat(b *models.TelemetryBatch, remoteAddr string, now time.Time) error { interval := b.HeartbeatIntervalSec if interval <= 0 { interval = defaultHeartbeat } status := b.Status if status == "" { status = "ok" } row := models.ServiceInstance{ Service: b.Service, Instance: b.Instance, Version: truncate(b.Version, 64), Status: truncate(status, 16), StatusMessage: truncate(b.StatusMessage, 512), StartedAt: normalizeStarted(b.StartedAt), LastSeen: now.Unix(), HeartbeatIntervalSec: interval, RemoteAddr: truncate(remoteAddr, 64), Meta: jsonString(b.Meta), } return initializers.DB.Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "service"}, {Name: "instance"}}, DoUpdates: clause.AssignmentColumns([]string{ "version", "status", "status_message", "started_at", "last_seen", "heartbeat_interval_sec", "remote_addr", "meta", "updated_at", }), }).Create(&row).Error } func normalizeStarted(ts int64) int64 { if ts > 1e12 { return ts / 1000 } return ts } // RecordEvents stores service events. func RecordEvents(service, instance string, events []models.TelemetryEvent, now time.Time) error { if len(events) == 0 { return nil } rows := make([]models.ServiceEvent, 0, len(events)) for _, e := range events { level := e.Level switch level { case "debug", "info", "warn", "error": case "warning": level = "warn" default: level = "info" } ts := e.Ts if ts <= 0 { ts = now.UnixMilli() } else if ts < 1e12 { ts *= 1000 } rows = append(rows, models.ServiceEvent{ Service: service, Instance: instance, Level: level, Message: truncate(e.Message, 1024), Fields: jsonString(e.Fields), TS: ts, }) } return initializers.DB.CreateInBatches(&rows, 100).Error } // Event records a single event from the API itself. func Event(level, message string, fields map[string]any) { err := RecordEvents(MainService, Instance(), []models.TelemetryEvent{{Level: level, Message: message, Fields: fields}}, time.Now()) if err != nil { fmt.Printf("metrics: failed to record event: %v\n", err) } } // Online reports whether an instance has sent a heartbeat recently enough. func Online(inst *models.ServiceInstance, now time.Time) bool { interval := inst.HeartbeatIntervalSec if interval <= 0 { interval = defaultHeartbeat } return now.Unix()-inst.LastSeen < int64(3*interval) } var ( upMu sync.Mutex upState = map[string]bool{} // "service/instance" -> online ) // ServiceUp returns the last known online state of every instance. func ServiceUp() map[[2]string]bool { upMu.Lock() defer upMu.Unlock() out := make(map[[2]string]bool, len(upState)) for k, v := range upState { var svc, inst string for i := 0; i < len(k); i++ { if k[i] == '/' { svc, inst = k[:i], k[i+1:] break } } out[[2]string{svc, inst}] = v } return out } // checkServices records online/offline transitions as events. func checkServices(now time.Time) { var rows []models.ServiceInstance if err := initializers.DB.Find(&rows).Error; err != nil { return } upMu.Lock() known := make(map[string]bool, len(rows)) type change struct { inst models.ServiceInstance online bool } var changes []change for _, r := range rows { k := r.Service + "/" + r.Instance known[k] = true online := Online(&r, now) prev, seen := upState[k] upState[k] = online if seen && prev != online { changes = append(changes, change{r, online}) } } for k := range upState { if !known[k] { delete(upState, k) } } upMu.Unlock() for _, ch := range changes { level, msg := "info", "back online" if !ch.online { level, msg = "warn", "heartbeat lost" } _ = RecordEvents(ch.inst.Service, ch.inst.Instance, []models.TelemetryEvent{{ Level: level, Message: msg, Fields: map[string]any{"kind": "status", "lastSeen": ch.inst.LastSeen}, }}, now) } } var ( buildMu sync.Mutex buildVersion = "dev" startedAt = time.Now() ) // SetVersion sets the version reported in the API's own heartbeat. func SetVersion(v string) { buildMu.Lock() buildVersion = v buildMu.Unlock() } // StartedAt is when this process started. func StartedAt() time.Time { return startedAt } func selfHeartbeat(now time.Time) { buildMu.Lock() v := buildVersion buildMu.Unlock() _ = Heartbeat(&models.TelemetryBatch{ Service: MainService, Instance: Instance(), Version: v, StartedAt: startedAt.Unix(), HeartbeatIntervalSec: 30, Status: "ok", }, "", now) }