reforger_crawler_main/admin/metricsapi.go

package admin

import (
	"encoding/json"
	"net/http"
	"sort"
	"strconv"
	"strings"
	"time"

	"gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers"
	"gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics"
	"gitea.tbdevent.eu/TBD/reforger_crawler_main/models"
	"github.com/gin-gonic/gin"
)

func metricsCatalog(c *gin.Context) {
	service := c.Query("service")
	data, _, err := cached("catalog:"+service, 15*time.Second, func() ([]metrics.CatalogEntry, error) {
		return metrics.Catalog(service)
	})
	if err != nil {
		fail(c, http.StatusInternalServerError, err.Error())
		return
	}
	c.JSON(http.StatusOK, gin.H{"items": data, "aggregations": metrics.Aggregations})
}

func selectorFrom(c *gin.Context) (metrics.Selector, bool) {
	sel := metrics.Selector{Service: c.Query("service"), Name: c.Query("name")}
	if m := c.Query("match"); m != "" {
		if err := json.Unmarshal([]byte(m), &sel.Match); err != nil {
			fail(c, http.StatusBadRequest, "match must be a JSON object of label values")
			return sel, false
		}
	}
	if g := c.Query("groupBy"); g != "" {
		for _, part := range strings.Split(g, ",") {
			if part = strings.TrimSpace(part); part != "" {
				sel.GroupBy = append(sel.GroupBy, part)
			}
		}
	}
	if sel.Name == "" {
		fail(c, http.StatusBadRequest, "name is required")
		return sel, false
	}
	return sel, true
}

func metricsQuery(c *gin.Context) {
	sel, ok := selectorFrom(c)
	if !ok {
		return
	}
	from, to := timeRange(c, time.Hour)
	res, err := metrics.Query(sel, c.Query("agg"), from, to, queryInt64(c, "step", 0))
	if err != nil {
		fail(c, http.StatusBadRequest, err.Error())
		return
	}
	c.JSON(http.StatusOK, res)
}

func autoStep(from, to int64) int64 {
	span := to - from
	switch {
	case span <= 3*3600:
		return 60
	case span <= 12*3600:
		return 300
	case span <= 48*3600:
		return 900
	case span <= 8*86400:
		return 3600
	default:
		return 6 * 3600
	}
}

type routeStat struct {
	Method  string   `json:"method"`
	Route   string   `json:"route"`
	Group   string   `json:"group"`
	Count   int64    `json:"count"`
	Errors  int64    `json:"errors"`
	Client4 int64    `json:"clientErrors"`
	P50     *float64 `json:"p50"`
	P95     *float64 `json:"p95"`
	P99     *float64 `json:"p99"`
	Max     *float64 `json:"max"`
	Avg     *float64 `json:"avg"`
}

// traffic returns pre-composed request, latency and error data for a range.
func traffic(c *gin.Context) {
	from, to := timeRange(c, 24*time.Hour)
	step := queryInt64(c, "step", 0)
	if step <= 0 {
		step = autoStep(from, to)
	}
	key := "traffic:" + fmtInt64(from/30) + ":" + fmtInt64(to/30) + ":" + fmtInt64(step)
	data, at, err := cached(key, 30*time.Second, func() (gin.H, error) {
		return buildTraffic(from, to, step)
	})
	if err != nil {
		fail(c, http.StatusInternalServerError, err.Error())
		return
	}
	out := gin.H{"asOf": at.Unix()}
	for k, v := range data {
		out[k] = v
	}
	c.JSON(http.StatusOK, out)
}

func fmtInt64(v int64) string { return strconv.FormatInt(v, 10) }

func buildTraffic(from, to, step int64) (gin.H, error) {
	const name = "http_request_duration_ms"
	sel := func(match map[string]string, groupBy ...string) metrics.Selector {
		return metrics.Selector{Service: metrics.MainService, Name: name, Match: match, GroupBy: groupBy}
	}
	v1 := map[string]string{"group": "v1"}

	byGroup, err := metrics.Query(sel(nil, "group"), "rate", from, to, step)
	if err != nil {
		return nil, err
	}
	latency := gin.H{}
	for _, q := range []string{"p50", "p95", "p99"} {
		r, err := metrics.Query(sel(v1), q, from, to, step)
		if err != nil {
			return nil, err
		}
		latency[q] = r
	}
	errors5xx, _ := metrics.Query(sel(map[string]string{"status": "5xx"}, "group"), "rate", from, to, step)
	errors4xx, _ := metrics.Query(sel(map[string]string{"status": "4xx", "group": "v1"}), "rate", from, to, step)

	statusTotals, _ := metrics.Totals(sel(nil, "status"), from, to)
	clientTotals, _ := metrics.Totals(sel(v1, "client"), from, to)
	groupTotals, _ := metrics.Totals(sel(nil, "group"), from, to)

	routeTotals, err := metrics.Totals(sel(nil, "method", "route", "group"), from, to)
	if err != nil {
		return nil, err
	}
	statusByRoute, _ := metrics.Totals(sel(nil, "method", "route", "status"), from, to)
	errs := map[string][2]int64{}
	for _, t := range statusByRoute {
		k := t.Labels["method"] + " " + t.Labels["route"]
		e := errs[k]
		switch t.Labels["status"] {
		case "5xx":
			e[0] += t.Count
		case "4xx":
			e[1] += t.Count
		}
		errs[k] = e
	}
	routes := make([]routeStat, 0, len(routeTotals))
	for _, t := range routeTotals {
		k := t.Labels["method"] + " " + t.Labels["route"]
		routes = append(routes, routeStat{
			Method: t.Labels["method"], Route: t.Labels["route"], Group: t.Labels["group"],
			Count: t.Count, Errors: errs[k][0], Client4: errs[k][1],
			P50: t.P50, P95: t.P95, P99: t.P99, Max: t.Max, Avg: t.Avg,
		})
	}
	sort.Slice(routes, func(i, j int) bool { return routes[i].Count > routes[j].Count })

	now := time.Now().Unix()
	visitorsDaily, _ := metrics.Query(metrics.Selector{Service: metrics.MainService, Name: "visitors_unique", GroupBy: []string{"kind"}},
		"max", now-30*86400, now, 86400)
	humans, bots := metrics.Visitors()

	var v1Total *metrics.Total
	if all, _ := metrics.Totals(sel(v1), from, to); len(all) > 0 {
		v1Total = &all[0]
	}

	return gin.H{
		"from": from, "to": to, "step": step,
		"requestsByGroup": byGroup,
		"latency":         latency,
		"errors5xx":       errors5xx,
		"errors4xx":       errors4xx,
		"statusTotals":    statusTotals,
		"clientTotals":    clientTotals,
		"groupTotals":     groupTotals,
		"website":         v1Total,
		"routes":          routes,
		"visitorsDaily":   visitorsDaily,
		"visitorsToday":   gin.H{"human": humans, "bot": bots},
	}, nil
}

type topEntry struct {
	Item  string `json:"item"`
	Count int64  `json:"count"`
	Name  string `json:"name,omitempty"`
}

// topItems returns the most looked-up addons or search queries.
func topItems(c *gin.Context) {
	kind := c.DefaultQuery("kind", "addon")
	if kind != "addon" && kind != "search" {
		fail(c, http.StatusBadRequest, "kind must be addon or search")
		return
	}
	days := queryInt(c, "days", 1, 1, 90)
	limit := queryInt(c, "limit", 20, 1, 200)

	today, counts := metrics.Top.Today(kind)
	if today == "" {
		today = time.Now().UTC().Format("2006-01-02")
	}
	if days > 1 {
		since := time.Now().UTC().AddDate(0, 0, -(days - 1)).Format("2006-01-02")
		var rows []models.DailyTopItem
		initializers.DB.Where("kind = ? AND day >= ? AND day < ?", kind, since, today).Find(&rows)
		for _, r := range rows {
			counts[r.Item] += r.Count
		}
	}
	list := make([]topEntry, 0, len(counts))
	for k, v := range counts {
		list = append(list, topEntry{Item: k, Count: v})
	}
	sort.Slice(list, func(i, j int) bool {
		if list[i].Count != list[j].Count {
			return list[i].Count > list[j].Count
		}
		return list[i].Item < list[j].Item
	})
	if len(list) > limit {
		list = list[:limit]
	}
	if kind == "addon" && len(list) > 0 {
		ids := make([]string, len(list))
		for i, e := range list {
			ids[i] = e.Item
		}
		var names []struct {
			ID   string
			Name string
		}
		initializers.DB.Unscoped().Model(&models.Addon{}).Select("id, name").Where("id IN ?", ids).Scan(&names)
		byID := map[string]string{}
		for _, n := range names {
			byID[n.ID] = n.Name
		}
		for i := range list {
			list[i].Name = byID[list[i].Item]
		}
	}
	c.JSON(http.StatusOK, gin.H{"kind": kind, "days": days, "items": list})
}