package admin import ( "encoding/json" "net/http" "sort" "strconv" "strings" "time" "gitea.tbdevent.eu/TBD/reforger_crawler_main/initializers" "gitea.tbdevent.eu/TBD/reforger_crawler_main/metrics" "gitea.tbdevent.eu/TBD/reforger_crawler_main/models" "github.com/gin-gonic/gin" ) func metricsCatalog(c *gin.Context) { service := c.Query("service") data, _, err := cached("catalog:"+service, 15*time.Second, func() ([]metrics.CatalogEntry, error) { return metrics.Catalog(service) }) if err != nil { fail(c, http.StatusInternalServerError, err.Error()) return } c.JSON(http.StatusOK, gin.H{"items": data, "aggregations": metrics.Aggregations}) } func selectorFrom(c *gin.Context) (metrics.Selector, bool) { sel := metrics.Selector{Service: c.Query("service"), Name: c.Query("name")} if m := c.Query("match"); m != "" { if err := json.Unmarshal([]byte(m), &sel.Match); err != nil { fail(c, http.StatusBadRequest, "match must be a JSON object of label values") return sel, false } } if g := c.Query("groupBy"); g != "" { for _, part := range strings.Split(g, ",") { if part = strings.TrimSpace(part); part != "" { sel.GroupBy = append(sel.GroupBy, part) } } } if sel.Name == "" { fail(c, http.StatusBadRequest, "name is required") return sel, false } return sel, true } func metricsQuery(c *gin.Context) { sel, ok := selectorFrom(c) if !ok { return } from, to := timeRange(c, time.Hour) res, err := metrics.Query(sel, c.Query("agg"), from, to, queryInt64(c, "step", 0)) if err != nil { fail(c, http.StatusBadRequest, err.Error()) return } c.JSON(http.StatusOK, res) } func autoStep(from, to int64) int64 { span := to - from switch { case span <= 3*3600: return 60 case span <= 12*3600: return 300 case span <= 48*3600: return 900 case span <= 8*86400: return 3600 default: return 6 * 3600 } } type routeStat struct { Method string `json:"method"` Route string `json:"route"` Group string `json:"group"` Count int64 `json:"count"` Errors int64 `json:"errors"` Client4 int64 `json:"clientErrors"` P50 *float64 `json:"p50"` P95 *float64 `json:"p95"` P99 *float64 `json:"p99"` Max *float64 `json:"max"` Avg *float64 `json:"avg"` } // traffic returns pre-composed request, latency and error data for a range. func traffic(c *gin.Context) { from, to := timeRange(c, 24*time.Hour) step := queryInt64(c, "step", 0) if step <= 0 { step = autoStep(from, to) } key := "traffic:" + fmtInt64(from/30) + ":" + fmtInt64(to/30) + ":" + fmtInt64(step) data, at, err := cached(key, 30*time.Second, func() (gin.H, error) { return buildTraffic(from, to, step) }) if err != nil { fail(c, http.StatusInternalServerError, err.Error()) return } out := gin.H{"asOf": at.Unix()} for k, v := range data { out[k] = v } c.JSON(http.StatusOK, out) } func fmtInt64(v int64) string { return strconv.FormatInt(v, 10) } func buildTraffic(from, to, step int64) (gin.H, error) { const name = "http_request_duration_ms" sel := func(match map[string]string, groupBy ...string) metrics.Selector { return metrics.Selector{Service: metrics.MainService, Name: name, Match: match, GroupBy: groupBy} } v1 := map[string]string{"group": "v1"} byGroup, err := metrics.Query(sel(nil, "group"), "rate", from, to, step) if err != nil { return nil, err } latency := gin.H{} for _, q := range []string{"p50", "p95", "p99"} { r, err := metrics.Query(sel(v1), q, from, to, step) if err != nil { return nil, err } latency[q] = r } errors5xx, _ := metrics.Query(sel(map[string]string{"status": "5xx"}, "group"), "rate", from, to, step) errors4xx, _ := metrics.Query(sel(map[string]string{"status": "4xx", "group": "v1"}), "rate", from, to, step) statusTotals, _ := metrics.Totals(sel(nil, "status"), from, to) clientTotals, _ := metrics.Totals(sel(v1, "client"), from, to) groupTotals, _ := metrics.Totals(sel(nil, "group"), from, to) routeTotals, err := metrics.Totals(sel(nil, "method", "route", "group"), from, to) if err != nil { return nil, err } statusByRoute, _ := metrics.Totals(sel(nil, "method", "route", "status"), from, to) errs := map[string][2]int64{} for _, t := range statusByRoute { k := t.Labels["method"] + " " + t.Labels["route"] e := errs[k] switch t.Labels["status"] { case "5xx": e[0] += t.Count case "4xx": e[1] += t.Count } errs[k] = e } routes := make([]routeStat, 0, len(routeTotals)) for _, t := range routeTotals { k := t.Labels["method"] + " " + t.Labels["route"] routes = append(routes, routeStat{ Method: t.Labels["method"], Route: t.Labels["route"], Group: t.Labels["group"], Count: t.Count, Errors: errs[k][0], Client4: errs[k][1], P50: t.P50, P95: t.P95, P99: t.P99, Max: t.Max, Avg: t.Avg, }) } sort.Slice(routes, func(i, j int) bool { return routes[i].Count > routes[j].Count }) now := time.Now().Unix() visitorsDaily, _ := metrics.Query(metrics.Selector{Service: metrics.MainService, Name: "visitors_unique", GroupBy: []string{"kind"}}, "max", now-30*86400, now, 86400) humans, bots := metrics.Visitors() var v1Total *metrics.Total if all, _ := metrics.Totals(sel(v1), from, to); len(all) > 0 { v1Total = &all[0] } return gin.H{ "from": from, "to": to, "step": step, "requestsByGroup": byGroup, "latency": latency, "errors5xx": errors5xx, "errors4xx": errors4xx, "statusTotals": statusTotals, "clientTotals": clientTotals, "groupTotals": groupTotals, "website": v1Total, "routes": routes, "visitorsDaily": visitorsDaily, "visitorsToday": gin.H{"human": humans, "bot": bots}, }, nil } type topEntry struct { Item string `json:"item"` Count int64 `json:"count"` Name string `json:"name,omitempty"` } // topItems returns the most looked-up addons or search queries. func topItems(c *gin.Context) { kind := c.DefaultQuery("kind", "addon") if kind != "addon" && kind != "search" { fail(c, http.StatusBadRequest, "kind must be addon or search") return } days := queryInt(c, "days", 1, 1, 90) limit := queryInt(c, "limit", 20, 1, 200) today, counts := metrics.Top.Today(kind) if today == "" { today = time.Now().UTC().Format("2006-01-02") } if days > 1 { since := time.Now().UTC().AddDate(0, 0, -(days - 1)).Format("2006-01-02") var rows []models.DailyTopItem initializers.DB.Where("kind = ? AND day >= ? AND day < ?", kind, since, today).Find(&rows) for _, r := range rows { counts[r.Item] += r.Count } } list := make([]topEntry, 0, len(counts)) for k, v := range counts { list = append(list, topEntry{Item: k, Count: v}) } sort.Slice(list, func(i, j int) bool { if list[i].Count != list[j].Count { return list[i].Count > list[j].Count } return list[i].Item < list[j].Item }) if len(list) > limit { list = list[:limit] } if kind == "addon" && len(list) > 0 { ids := make([]string, len(list)) for i, e := range list { ids[i] = e.Item } var names []struct { ID string Name string } initializers.DB.Unscoped().Model(&models.Addon{}).Select("id, name").Where("id IN ?", ids).Scan(&names) byID := map[string]string{} for _, n := range names { byID[n.ID] = n.Name } for i := range list { list[i].Name = byID[list[i].Item] } } c.JSON(http.StatusOK, gin.H{"kind": kind, "days": days, "items": list}) }