- embedded React admin UI at /admin/ui (go:embed), login with ADMIN_SECRET via signed HttpOnly session cookie, X-ADMIN-KEY still works for scripts - admin API: overview stats, indexing queue, addon management (priority, reindex, delete/restore/purge, bulk), whitelist CRUD, duplicate hashes, traffic, metrics explorer, services, events/audit log, system info - built-in metrics: request latency per route, visitors, top addons and searches, indexer/scraper counters; minute buckets in the DB with hourly rollups and retention; optional Prometheus /metrics (METRICS_TOKEN) - POST /back/telemetry for indexers/scrapers (metrics, events, heartbeat) and a stdlib-only Go client in telemetry/ - auth hardening: empty secrets no longer allow access, constant-time compare, CORS not applied to /admin and /back, TRUSTED_PROXIES setting - Dockerfile builds the UI and no longer bakes config.yaml into the image; CI builds the UI, runs tests and builds arm64 with CGO
This commit is contained in:
ilbinek committed
1 parent
0ef27b0072
commit
56d9e10621
79 files changed
+13146
-118
No files matched your search
@@ -0,0 +1,284 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Kind string
|
||||
|
||||
const (
|
||||
Counter Kind = "counter" // value is an increment; Sum holds the total
|
||||
Gauge Kind = "gauge" // value is a sample; Last holds the latest
|
||||
Summary Kind = "summary" // pre-aggregated count/sum/min/max from a client
|
||||
Histogram Kind = "histogram" // observations bucketed by Bounds
|
||||
)
|
||||
|
||||
// MainService is the service name the API uses for its own metrics.
|
||||
const MainService = "main"
|
||||
|
||||
var (
|
||||
errSeriesLimit = errors.New("series limit reached")
|
||||
errKindMismatch = errors.New("metric reported with a different type")
|
||||
)
|
||||
|
||||
// Limits for series created from external telemetry.
|
||||
const (
|
||||
maxSeriesPerName = 200
|
||||
maxExternal = 5000
|
||||
)
|
||||
|
||||
var hostname = func() string {
|
||||
h, err := os.Hostname()
|
||||
if err != nil || h == "" {
|
||||
return "api"
|
||||
}
|
||||
return h
|
||||
}()
|
||||
|
||||
// Instance returns the instance name used for this process' own metrics.
|
||||
func Instance() string { return hostname }
|
||||
|
||||
// Series is one metric name + label set from one service instance. Series are
|
||||
// interned, so pointer equality means same series.
|
||||
type Series struct {
|
||||
Service string
|
||||
Instance string
|
||||
Name string
|
||||
Labels map[string]string
|
||||
LabelsJSON string
|
||||
Kind Kind
|
||||
Bounds []float64
|
||||
Help string
|
||||
}
|
||||
|
||||
type seriesKey struct{ service, instance, name, labels string }
|
||||
|
||||
func (s *Series) key() seriesKey {
|
||||
return seriesKey{s.Service, s.Instance, s.Name, s.LabelsJSON}
|
||||
}
|
||||
|
||||
// Label returns a label value, including the service/instance pseudo labels.
|
||||
func (s *Series) Label(name string) string {
|
||||
switch name {
|
||||
case "service":
|
||||
if v, ok := s.Labels["service"]; ok {
|
||||
return v
|
||||
}
|
||||
return s.Service
|
||||
case "instance":
|
||||
if v, ok := s.Labels["instance"]; ok {
|
||||
return v
|
||||
}
|
||||
return s.Instance
|
||||
}
|
||||
return s.Labels[name]
|
||||
}
|
||||
|
||||
// canonicalLabels serializes labels deterministically (encoding/json sorts map keys).
|
||||
func canonicalLabels(labels map[string]string) string {
|
||||
if len(labels) == 0 {
|
||||
return "{}"
|
||||
}
|
||||
b, _ := json.Marshal(labels)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
type registry struct {
|
||||
mu sync.RWMutex
|
||||
series map[seriesKey]*Series
|
||||
perName map[string]int // external series per service/name
|
||||
external int
|
||||
lastSeen map[*Series]int64
|
||||
}
|
||||
|
||||
var reg = ®istry{
|
||||
series: make(map[seriesKey]*Series),
|
||||
perName: make(map[string]int),
|
||||
lastSeen: make(map[*Series]int64),
|
||||
}
|
||||
|
||||
func (r *registry) get(service, instance, name string, labels map[string]string, kind Kind, bounds []float64, help string, external bool) (*Series, error) {
|
||||
lj := canonicalLabels(labels)
|
||||
k := seriesKey{service, instance, name, lj}
|
||||
|
||||
r.mu.RLock()
|
||||
s := r.series[k]
|
||||
r.mu.RUnlock()
|
||||
if s != nil {
|
||||
if s.Kind != kind {
|
||||
return nil, errKindMismatch
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if s = r.series[k]; s != nil {
|
||||
return s, nil
|
||||
}
|
||||
nameKey := service + "/" + name
|
||||
if external {
|
||||
if r.external >= maxExternal || r.perName[nameKey] >= maxSeriesPerName {
|
||||
return nil, errSeriesLimit
|
||||
}
|
||||
r.external++
|
||||
r.perName[nameKey]++
|
||||
}
|
||||
copied := make(map[string]string, len(labels))
|
||||
for k, v := range labels {
|
||||
copied[k] = v
|
||||
}
|
||||
s = &Series{
|
||||
Service: service,
|
||||
Instance: instance,
|
||||
Name: name,
|
||||
Labels: copied,
|
||||
LabelsJSON: lj,
|
||||
Kind: kind,
|
||||
Bounds: bounds,
|
||||
Help: help,
|
||||
}
|
||||
r.series[k] = s
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (r *registry) lookup(k seriesKey) *Series {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
return r.series[k]
|
||||
}
|
||||
|
||||
// all returns every known series, sorted by name then labels.
|
||||
func (r *registry) all() []*Series {
|
||||
r.mu.RLock()
|
||||
out := make([]*Series, 0, len(r.series))
|
||||
for _, s := range r.series {
|
||||
out = append(out, s)
|
||||
}
|
||||
r.mu.RUnlock()
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Name != out[j].Name {
|
||||
return out[i].Name < out[j].Name
|
||||
}
|
||||
if out[i].Service != out[j].Service {
|
||||
return out[i].Service < out[j].Service
|
||||
}
|
||||
if out[i].Instance != out[j].Instance {
|
||||
return out[i].Instance < out[j].Instance
|
||||
}
|
||||
return out[i].LabelsJSON < out[j].LabelsJSON
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func (r *registry) touch(s *Series, ts int64) {
|
||||
r.mu.Lock()
|
||||
if ts > r.lastSeen[s] {
|
||||
r.lastSeen[s] = ts
|
||||
}
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
func (r *registry) seen(s *Series) int64 {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
return r.lastSeen[s]
|
||||
}
|
||||
|
||||
// Family is a metric emitted by this process, with a fixed set of label names.
|
||||
type Family struct {
|
||||
Name string
|
||||
Help string
|
||||
Kind Kind
|
||||
Bounds []float64
|
||||
LabelNames []string
|
||||
|
||||
mu sync.Mutex
|
||||
cache map[string]*Series
|
||||
}
|
||||
|
||||
var (
|
||||
familiesMu sync.Mutex
|
||||
families = map[string]*Family{}
|
||||
)
|
||||
|
||||
func newFamily(name, help string, kind Kind, bounds []float64, labels []string) *Family {
|
||||
f := &Family{Name: name, Help: help, Kind: kind, Bounds: bounds, LabelNames: labels, cache: map[string]*Series{}}
|
||||
familiesMu.Lock()
|
||||
families[name] = f
|
||||
familiesMu.Unlock()
|
||||
return f
|
||||
}
|
||||
|
||||
func familyByName(name string) *Family {
|
||||
familiesMu.Lock()
|
||||
defer familiesMu.Unlock()
|
||||
return families[name]
|
||||
}
|
||||
|
||||
func NewCounter(name, help string, labels ...string) *Family {
|
||||
return newFamily(name, help, Counter, nil, labels)
|
||||
}
|
||||
|
||||
func NewGauge(name, help string, labels ...string) *Family {
|
||||
return newFamily(name, help, Gauge, nil, labels)
|
||||
}
|
||||
|
||||
func NewHistogram(name, help string, bounds []float64, labels ...string) *Family {
|
||||
return newFamily(name, help, Histogram, bounds, labels)
|
||||
}
|
||||
|
||||
func (f *Family) series(values []string) *Series {
|
||||
key := strings.Join(values, "\x00")
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if s := f.cache[key]; s != nil {
|
||||
return s
|
||||
}
|
||||
labels := make(map[string]string, len(f.LabelNames))
|
||||
for i, n := range f.LabelNames {
|
||||
if i < len(values) {
|
||||
labels[n] = values[i]
|
||||
} else {
|
||||
labels[n] = ""
|
||||
}
|
||||
}
|
||||
s, err := reg.get(MainService, Instance(), f.Name, labels, f.Kind, f.Bounds, f.Help, false)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
f.cache[key] = s
|
||||
return s
|
||||
}
|
||||
|
||||
// Add increments a counter. Zero increments are ignored.
|
||||
func (f *Family) Add(v float64, labelValues ...string) {
|
||||
if v == 0 {
|
||||
return
|
||||
}
|
||||
if s := f.series(labelValues); s != nil {
|
||||
defaultAgg.observe(s, time.Now().Unix(), v)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Family) Inc(labelValues ...string) { f.Add(1, labelValues...) }
|
||||
|
||||
// Set records a gauge sample.
|
||||
func (f *Family) Set(v float64, labelValues ...string) {
|
||||
if s := f.series(labelValues); s != nil {
|
||||
defaultAgg.observe(s, time.Now().Unix(), v)
|
||||
}
|
||||
}
|
||||
|
||||
// Observe records a histogram observation.
|
||||
func (f *Family) Observe(v float64, labelValues ...string) {
|
||||
if s := f.series(labelValues); s != nil {
|
||||
defaultAgg.observe(s, time.Now().Unix(), v)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user