reforger_crawler_main/metrics/services.go

package metrics

import (
	"encoding/json"
	"errors"
	"fmt"
	"math"
	"regexp"
	"sync"
	"time"

	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
	"gorm.io/gorm/clause"
)

var nameRe = regexp.MustCompile(`^[a-z_][a-z0-9_]{0,63}$`)

// ValidName reports whether a service, metric or label name is acceptable.
func ValidName(s string) bool { return nameRe.MatchString(s) }

const (
	defaultHeartbeat = 30
	maxLabels        = 8
	maxLabelValue    = 128
)

// normalizeTs converts unix seconds or milliseconds to seconds and clamps it
// to [now-1h, now+1m]; anything outside becomes now.
func normalizeTs(ts int64, now int64) int64 {
	if ts <= 0 {
		return now
	}
	if ts > 1e12 {
		ts /= 1000
	}
	if ts < now-3600 || ts > now+60 {
		return now
	}
	return ts
}

// IngestResult reports what happened to a telemetry batch.
type IngestResult struct {
	Accepted int            `json:"accepted"`
	Dropped  int            `json:"dropped"`
	Reasons  map[string]int `json:"reasons,omitempty"`
}

// Ingest records external metric samples for a service instance.
func Ingest(service, instance string, samples []models.TelemetryMetric, now time.Time) IngestResult {
	res := IngestResult{Reasons: map[string]int{}}
	drop := func(reason string) {
		res.Dropped++
		res.Reasons[reason]++
	}
	nowTs := now.Unix()

	for _, m := range samples {
		if !ValidName(m.Name) {
			drop("invalid_name")
			continue
		}
		if len(m.Labels) > maxLabels {
			drop("too_many_labels")
			continue
		}
		badLabel := false
		for k, v := range m.Labels {
			if !ValidName(k) || len(v) > maxLabelValue {
				badLabel = true
				break
			}
		}
		if badLabel {
			drop("invalid_label")
			continue
		}
		var kind Kind
		switch m.Type {
		case "counter":
			kind = Counter
		case "gauge", "":
			kind = Gauge
		case "summary":
			kind = Summary
		default:
			drop("invalid_type")
			continue
		}
		if math.IsNaN(m.Value) || math.IsInf(m.Value, 0) || math.IsNaN(m.Sum) || math.IsInf(m.Sum, 0) {
			drop("invalid_value")
			continue
		}
		if kind == Counter && m.Value < 0 {
			drop("negative_counter")
			continue
		}
		if kind == Summary && m.Count <= 0 {
			drop("empty_summary")
			continue
		}
		help := m.Help
		if len(help) > 256 {
			help = help[:256]
		}
		s, err := reg.get(service, instance, m.Name, m.Labels, kind, nil, help, true)
		if err != nil {
			if errors.Is(err, errKindMismatch) {
				drop("type_mismatch")
			} else {
				drop("series_limit")
			}
			continue
		}
		ts := normalizeTs(m.Ts, nowTs)
		if kind == Summary {
			lo, hi := m.Min, m.Max
			if lo == 0 && hi == 0 {
				avg := m.Sum / float64(m.Count)
				lo, hi = avg, avg
			}
			defaultAgg.observeSummary(s, ts, m.Count, m.Sum, lo, hi)
		} else {
			defaultAgg.observe(s, ts, m.Value)
		}
		res.Accepted++
	}
	TelemetrySamples.Add(float64(res.Accepted), "accepted")
	TelemetrySamples.Add(float64(res.Dropped), "dropped")
	return res
}

func jsonString(v map[string]any) string {
	if len(v) == 0 {
		return ""
	}
	b, err := json.Marshal(v)
	if err != nil {
		return ""
	}
	return string(b)
}

func truncate(s string, n int) string {
	if len(s) > n {
		return s[:n]
	}
	return s
}

// Heartbeat upserts the service instance row.
func Heartbeat(b *models.TelemetryBatch, remoteAddr string, now time.Time) error {
	interval := b.HeartbeatIntervalSec
	if interval <= 0 {
		interval = defaultHeartbeat
	}
	status := b.Status
	if status == "" {
		status = "ok"
	}
	row := models.ServiceInstance{
		Service:              b.Service,
		Instance:             b.Instance,
		Version:              truncate(b.Version, 64),
		Status:               truncate(status, 16),
		StatusMessage:        truncate(b.StatusMessage, 512),
		StartedAt:            normalizeStarted(b.StartedAt),
		LastSeen:             now.Unix(),
		HeartbeatIntervalSec: interval,
		RemoteAddr:           truncate(remoteAddr, 64),
		Meta:                 jsonString(b.Meta),
	}
	return initializers.DB.Clauses(clause.OnConflict{
		Columns: []clause.Column{{Name: "service"}, {Name: "instance"}},
		DoUpdates: clause.AssignmentColumns([]string{
			"version", "status", "status_message", "started_at", "last_seen",
			"heartbeat_interval_sec", "remote_addr", "meta", "updated_at",
		}),
	}).Create(&row).Error
}

func normalizeStarted(ts int64) int64 {
	if ts > 1e12 {
		return ts / 1000
	}
	return ts
}

// RecordEvents stores service events.
func RecordEvents(service, instance string, events []models.TelemetryEvent, now time.Time) error {
	if len(events) == 0 {
		return nil
	}
	rows := make([]models.ServiceEvent, 0, len(events))
	for _, e := range events {
		level := e.Level
		switch level {
		case "debug", "info", "warn", "error":
		case "warning":
			level = "warn"
		default:
			level = "info"
		}
		ts := e.Ts
		if ts <= 0 {
			ts = now.UnixMilli()
		} else if ts < 1e12 {
			ts *= 1000
		}
		rows = append(rows, models.ServiceEvent{
			Service: service, Instance: instance, Level: level,
			Message: truncate(e.Message, 1024), Fields: jsonString(e.Fields), TS: ts,
		})
	}
	return initializers.DB.CreateInBatches(&rows, 100).Error
}

// Event records a single event from the API itself.
func Event(level, message string, fields map[string]any) {
	err := RecordEvents(MainService, Instance(), []models.TelemetryEvent{{Level: level, Message: message, Fields: fields}}, time.Now())
	if err != nil {
		fmt.Printf("metrics: failed to record event: %v\n", err)
	}
}

// Online reports whether an instance has sent a heartbeat recently enough.
func Online(inst *models.ServiceInstance, now time.Time) bool {
	interval := inst.HeartbeatIntervalSec
	if interval <= 0 {
		interval = defaultHeartbeat
	}
	return now.Unix()-inst.LastSeen < int64(3*interval)
}

var (
	upMu    sync.Mutex
	upState = map[string]bool{} // "service/instance" -> online
)

// ServiceUp returns the last known online state of every instance.
func ServiceUp() map[[2]string]bool {
	upMu.Lock()
	defer upMu.Unlock()
	out := make(map[[2]string]bool, len(upState))
	for k, v := range upState {
		var svc, inst string
		for i := 0; i < len(k); i++ {
			if k[i] == '/' {
				svc, inst = k[:i], k[i+1:]
				break
			}
		}
		out[[2]string{svc, inst}] = v
	}
	return out
}

// checkServices records online/offline transitions as events.
func checkServices(now time.Time) {
	var rows []models.ServiceInstance
	if err := initializers.DB.Find(&rows).Error; err != nil {
		return
	}
	upMu.Lock()
	known := make(map[string]bool, len(rows))
	type change struct {
		inst   models.ServiceInstance
		online bool
	}
	var changes []change
	for _, r := range rows {
		k := r.Service + "/" + r.Instance
		known[k] = true
		online := Online(&r, now)
		prev, seen := upState[k]
		upState[k] = online
		if seen && prev != online {
			changes = append(changes, change{r, online})
		}
	}
	for k := range upState {
		if !known[k] {
			delete(upState, k)
		}
	}
	upMu.Unlock()

	for _, ch := range changes {
		level, msg := "info", "back online"
		if !ch.online {
			level, msg = "warn", "heartbeat lost"
		}
		_ = RecordEvents(ch.inst.Service, ch.inst.Instance, []models.TelemetryEvent{{
			Level: level, Message: msg,
			Fields: map[string]any{"kind": "status", "lastSeen": ch.inst.LastSeen},
		}}, now)
	}
}

var (
	buildMu      sync.Mutex
	buildVersion = "dev"
	startedAt    = time.Now()
)

// SetVersion sets the version reported in the API's own heartbeat.
func SetVersion(v string) {
	buildMu.Lock()
	buildVersion = v
	buildMu.Unlock()
}

// StartedAt is when this process started.
func StartedAt() time.Time { return startedAt }

func selfHeartbeat(now time.Time) {
	buildMu.Lock()
	v := buildVersion
	buildMu.Unlock()
	_ = Heartbeat(&models.TelemetryBatch{
		Service: MainService, Instance: Instance(), Version: v,
		StartedAt: startedAt.Unix(), HeartbeatIntervalSec: 30, Status: "ok",
	}, "", now)
}