- embedded React admin UI at /admin/ui (go:embed), login with ADMIN_SECRET via signed HttpOnly session cookie, X-ADMIN-KEY still works for scripts - admin API: overview stats, indexing queue, addon management (priority, reindex, delete/restore/purge, bulk), whitelist CRUD, duplicate hashes, traffic, metrics explorer, services, events/audit log, system info - built-in metrics: request latency per route, visitors, top addons and searches, indexer/scraper counters; minute buckets in the DB with hourly rollups and retention; optional Prometheus /metrics (METRICS_TOKEN) - POST /back/telemetry for indexers/scrapers (metrics, events, heartbeat) and a stdlib-only Go client in telemetry/ - auth hardening: empty secrets no longer allow access, constant-time compare, CORS not applied to /admin and /back, TRUSTED_PROXIES setting - Dockerfile builds the UI and no longer bakes config.yaml into the image; CI builds the UI, runs tests and builds arm64 with CGO
This commit is contained in:
ilbinek committed
1 parent
0ef27b0072
commit
56d9e10621
79 files changed
+13146
-118
No files matched your search
@@ -0,0 +1,430 @@
|
||||
// Package telemetry reports metrics, events and heartbeats from a crawler
|
||||
// service (indexer, scraper, ...) to the main API's POST /back/telemetry.
|
||||
//
|
||||
// It has no dependencies outside the standard library, so it can be imported
|
||||
// by any service, or the file can simply be copied.
|
||||
//
|
||||
// c := telemetry.New(telemetry.Config{
|
||||
// BaseURL: "https://crawler.example.com",
|
||||
// SecretKey: os.Getenv("SECRET_KEY"),
|
||||
// Service: "indexer",
|
||||
// Version: "1.4.0",
|
||||
// })
|
||||
// c.Start(ctx)
|
||||
// defer c.Close(context.Background())
|
||||
//
|
||||
// c.Counter("jobs_total", "result", "ok").Add(1)
|
||||
// c.Gauge("data_window_bytes").Set(float64(used))
|
||||
// c.Summary("download_seconds").Observe(took.Seconds())
|
||||
// c.Event(telemetry.Warn, "data limit reached", map[string]any{"limitGB": 50})
|
||||
// c.SetStatus(telemetry.Degraded, "rate limited by workshop")
|
||||
//
|
||||
// Recording never blocks on the network. A push that fails keeps the counter
|
||||
// increments for the next attempt; a few failed batches of events and samples
|
||||
// are buffered and the oldest are dropped first.
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Level of an event.
|
||||
type Level string
|
||||
|
||||
const (
|
||||
Debug Level = "debug"
|
||||
Info Level = "info"
|
||||
Warn Level = "warn"
|
||||
Error Level = "error"
|
||||
)
|
||||
|
||||
// Status is the self-reported health shown on the dashboard.
|
||||
type Status string
|
||||
|
||||
const (
|
||||
OK Status = "ok"
|
||||
Degraded Status = "degraded"
|
||||
Failing Status = "error"
|
||||
)
|
||||
|
||||
// Config configures a Client. BaseURL, SecretKey and Service are required.
|
||||
type Config struct {
|
||||
BaseURL string // e.g. https://crawler.example.com (no trailing /back)
|
||||
SecretKey string // the API's SECRET_KEY
|
||||
Service string // [a-z0-9_], e.g. "indexer"
|
||||
Instance string // defaults to the hostname
|
||||
Version string // shown on the dashboard
|
||||
Interval time.Duration // push interval, default 15s (also the heartbeat)
|
||||
Meta map[string]any
|
||||
// HTTPClient defaults to a client with a 10s timeout.
|
||||
HTTPClient *http.Client
|
||||
// OnError is called when a push fails (optional).
|
||||
OnError func(error)
|
||||
}
|
||||
|
||||
// Client buffers metrics and events and pushes them periodically.
|
||||
type Client struct {
|
||||
cfg Config
|
||||
started time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
counters map[string]*Counter
|
||||
gauges map[string]*Gauge
|
||||
summaries map[string]*Summary
|
||||
events []event
|
||||
status Status
|
||||
statusMsg string
|
||||
pending [][]byte // failed batches waiting for a retry
|
||||
|
||||
stop chan struct{}
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
type metric struct {
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
Value float64 `json:"value,omitempty"`
|
||||
Count int64 `json:"count,omitempty"`
|
||||
Sum float64 `json:"sum,omitempty"`
|
||||
Min float64 `json:"min,omitempty"`
|
||||
Max float64 `json:"max,omitempty"`
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
Ts int64 `json:"ts"`
|
||||
}
|
||||
|
||||
type event struct {
|
||||
Level Level `json:"level"`
|
||||
Message string `json:"message"`
|
||||
Ts int64 `json:"ts"`
|
||||
Fields map[string]any `json:"fields,omitempty"`
|
||||
}
|
||||
|
||||
type batch struct {
|
||||
Service string `json:"service"`
|
||||
Instance string `json:"instance"`
|
||||
Version string `json:"version"`
|
||||
StartedAt int64 `json:"startedAt"`
|
||||
HeartbeatIntervalSec int `json:"heartbeatIntervalSec"`
|
||||
Status Status `json:"status"`
|
||||
StatusMessage string `json:"statusMessage"`
|
||||
Meta map[string]any `json:"meta,omitempty"`
|
||||
Metrics []metric `json:"metrics"`
|
||||
Events []event `json:"events"`
|
||||
}
|
||||
|
||||
const (
|
||||
maxPending = 20
|
||||
maxEvents = 200
|
||||
maxMetricsPerReq = 1000
|
||||
)
|
||||
|
||||
// New creates a client. Call Start to begin pushing.
|
||||
func New(cfg Config) *Client {
|
||||
if cfg.Instance == "" {
|
||||
cfg.Instance, _ = os.Hostname()
|
||||
}
|
||||
if cfg.Interval <= 0 {
|
||||
cfg.Interval = 15 * time.Second
|
||||
}
|
||||
if cfg.HTTPClient == nil {
|
||||
cfg.HTTPClient = &http.Client{Timeout: 10 * time.Second}
|
||||
}
|
||||
cfg.BaseURL = strings.TrimRight(cfg.BaseURL, "/")
|
||||
return &Client{
|
||||
cfg: cfg,
|
||||
started: time.Now(),
|
||||
counters: map[string]*Counter{},
|
||||
gauges: map[string]*Gauge{},
|
||||
summaries: map[string]*Summary{},
|
||||
status: OK,
|
||||
}
|
||||
}
|
||||
|
||||
func key(name string, labels []string) string {
|
||||
return name + "\x00" + strings.Join(labels, "\x00")
|
||||
}
|
||||
|
||||
func labelMap(kv []string) map[string]string {
|
||||
if len(kv) < 2 {
|
||||
return nil
|
||||
}
|
||||
m := make(map[string]string, len(kv)/2)
|
||||
for i := 0; i+1 < len(kv); i += 2 {
|
||||
m[kv[i]] = kv[i+1]
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// Counter returns a counter; labels are key/value pairs.
|
||||
func (c *Client) Counter(name string, labels ...string) *Counter {
|
||||
k := key(name, labels)
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
ctr := c.counters[k]
|
||||
if ctr == nil {
|
||||
ctr = &Counter{name: name, labels: labelMap(labels)}
|
||||
c.counters[k] = ctr
|
||||
}
|
||||
return ctr
|
||||
}
|
||||
|
||||
// Gauge returns a gauge; labels are key/value pairs.
|
||||
func (c *Client) Gauge(name string, labels ...string) *Gauge {
|
||||
k := key(name, labels)
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
g := c.gauges[k]
|
||||
if g == nil {
|
||||
g = &Gauge{name: name, labels: labelMap(labels)}
|
||||
c.gauges[k] = g
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
// Summary returns a summary (count/sum/min/max per push); labels are key/value pairs.
|
||||
func (c *Client) Summary(name string, labels ...string) *Summary {
|
||||
k := key(name, labels)
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
s := c.summaries[k]
|
||||
if s == nil {
|
||||
s = &Summary{name: name, labels: labelMap(labels)}
|
||||
c.summaries[k] = s
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Event queues a log event for the dashboard.
|
||||
func (c *Client) Event(level Level, message string, fields map[string]any) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if len(c.events) >= maxEvents {
|
||||
c.events = c.events[1:]
|
||||
}
|
||||
c.events = append(c.events, event{Level: level, Message: message, Ts: time.Now().UnixMilli(), Fields: fields})
|
||||
}
|
||||
|
||||
// SetStatus sets the self-reported health shown on the dashboard.
|
||||
func (c *Client) SetStatus(s Status, message string) {
|
||||
c.mu.Lock()
|
||||
c.status, c.statusMsg = s, message
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
// Counter accumulates increments between pushes.
|
||||
type Counter struct {
|
||||
mu sync.Mutex
|
||||
name string
|
||||
labels map[string]string
|
||||
delta float64
|
||||
}
|
||||
|
||||
func (m *Counter) Add(v float64) {
|
||||
if v <= 0 || math.IsNaN(v) || math.IsInf(v, 0) {
|
||||
return
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.delta += v
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
func (m *Counter) Inc() { m.Add(1) }
|
||||
|
||||
// Gauge holds the latest value.
|
||||
type Gauge struct {
|
||||
mu sync.Mutex
|
||||
name string
|
||||
labels map[string]string
|
||||
value float64
|
||||
set bool
|
||||
}
|
||||
|
||||
func (m *Gauge) Set(v float64) {
|
||||
if math.IsNaN(v) || math.IsInf(v, 0) {
|
||||
return
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.value, m.set = v, true
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
// Summary aggregates observations between pushes.
|
||||
type Summary struct {
|
||||
mu sync.Mutex
|
||||
name string
|
||||
labels map[string]string
|
||||
count int64
|
||||
sum float64
|
||||
min, max float64
|
||||
}
|
||||
|
||||
func (m *Summary) Observe(v float64) {
|
||||
if math.IsNaN(v) || math.IsInf(v, 0) {
|
||||
return
|
||||
}
|
||||
m.mu.Lock()
|
||||
if m.count == 0 || v < m.min {
|
||||
m.min = v
|
||||
}
|
||||
if m.count == 0 || v > m.max {
|
||||
m.max = v
|
||||
}
|
||||
m.count++
|
||||
m.sum += v
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
// collect drains the current values into a batch payload.
|
||||
func (c *Client) collect() []byte {
|
||||
now := time.Now()
|
||||
c.mu.Lock()
|
||||
b := batch{
|
||||
Service: c.cfg.Service,
|
||||
Instance: c.cfg.Instance,
|
||||
Version: c.cfg.Version,
|
||||
StartedAt: c.started.Unix(),
|
||||
HeartbeatIntervalSec: int(c.cfg.Interval.Seconds()),
|
||||
Status: c.status,
|
||||
StatusMessage: c.statusMsg,
|
||||
Meta: c.cfg.Meta,
|
||||
Events: c.events,
|
||||
}
|
||||
c.events = nil
|
||||
counters := make([]*Counter, 0, len(c.counters))
|
||||
for _, m := range c.counters {
|
||||
counters = append(counters, m)
|
||||
}
|
||||
gauges := make([]*Gauge, 0, len(c.gauges))
|
||||
for _, m := range c.gauges {
|
||||
gauges = append(gauges, m)
|
||||
}
|
||||
summaries := make([]*Summary, 0, len(c.summaries))
|
||||
for _, m := range c.summaries {
|
||||
summaries = append(summaries, m)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
|
||||
ts := now.UnixMilli()
|
||||
for _, m := range counters {
|
||||
m.mu.Lock()
|
||||
if m.delta > 0 {
|
||||
b.Metrics = append(b.Metrics, metric{Name: m.name, Type: "counter", Value: m.delta, Labels: m.labels, Ts: ts})
|
||||
m.delta = 0
|
||||
}
|
||||
m.mu.Unlock()
|
||||
}
|
||||
for _, m := range gauges {
|
||||
m.mu.Lock()
|
||||
if m.set {
|
||||
b.Metrics = append(b.Metrics, metric{Name: m.name, Type: "gauge", Value: m.value, Labels: m.labels, Ts: ts})
|
||||
}
|
||||
m.mu.Unlock()
|
||||
}
|
||||
for _, m := range summaries {
|
||||
m.mu.Lock()
|
||||
if m.count > 0 {
|
||||
b.Metrics = append(b.Metrics, metric{Name: m.name, Type: "summary", Count: m.count, Sum: m.sum, Min: m.min, Max: m.max, Labels: m.labels, Ts: ts})
|
||||
m.count, m.sum, m.min, m.max = 0, 0, 0, 0
|
||||
}
|
||||
m.mu.Unlock()
|
||||
}
|
||||
sort.Slice(b.Metrics, func(i, j int) bool { return b.Metrics[i].Name < b.Metrics[j].Name })
|
||||
if len(b.Metrics) > maxMetricsPerReq {
|
||||
b.Metrics = b.Metrics[:maxMetricsPerReq]
|
||||
}
|
||||
if b.Metrics == nil {
|
||||
b.Metrics = []metric{}
|
||||
}
|
||||
if b.Events == nil {
|
||||
b.Events = []event{}
|
||||
}
|
||||
payload, _ := json.Marshal(b)
|
||||
return payload
|
||||
}
|
||||
|
||||
func (c *Client) post(ctx context.Context, payload []byte) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.cfg.BaseURL+"/back/telemetry", bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("X-SECRET-KEY", c.cfg.SecretKey)
|
||||
resp, err := c.cfg.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||||
if resp.StatusCode/100 != 2 {
|
||||
return fmt.Errorf("telemetry push: %s: %s", resp.Status, strings.TrimSpace(string(body)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Flush pushes buffered batches and the current values now.
|
||||
func (c *Client) Flush(ctx context.Context) error {
|
||||
payload := c.collect()
|
||||
c.mu.Lock()
|
||||
queue := append(c.pending, payload)
|
||||
c.pending = nil
|
||||
c.mu.Unlock()
|
||||
|
||||
for i, p := range queue {
|
||||
if err := c.post(ctx, p); err != nil {
|
||||
c.mu.Lock()
|
||||
c.pending = append(queue[i:], c.pending...)
|
||||
if len(c.pending) > maxPending {
|
||||
c.pending = c.pending[len(c.pending)-maxPending:]
|
||||
}
|
||||
c.mu.Unlock()
|
||||
if c.cfg.OnError != nil {
|
||||
c.cfg.OnError(err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Start pushes in the background every Interval until ctx ends or Close is called.
|
||||
func (c *Client) Start(ctx context.Context) {
|
||||
c.stop = make(chan struct{})
|
||||
c.done = make(chan struct{})
|
||||
go func() {
|
||||
defer close(c.done)
|
||||
t := time.NewTicker(c.cfg.Interval)
|
||||
defer t.Stop()
|
||||
_ = c.Flush(ctx)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-c.stop:
|
||||
return
|
||||
case <-t.C:
|
||||
_ = c.Flush(ctx)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Close stops the background loop and makes a final push.
|
||||
func (c *Client) Close(ctx context.Context) error {
|
||||
if c.stop != nil {
|
||||
close(c.stop)
|
||||
<-c.done
|
||||
c.stop = nil
|
||||
}
|
||||
return c.Flush(ctx)
|
||||
}
|
||||
Reference in new issue
Block a user