feat: add path & method labels to prometheus metrics for current requests (coder#17362)

Closes: coder#17212

Author: ibetitsmike

coderd/httpmw/prometheus.go

@@ -3,6 +3,7 @@ package httpmw
 import (
 	"net/http"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/go-chi/chi/v5"
@@ -22,18 +23,18 @@ func Prometheus(register prometheus.Registerer) func(http.Handler) http.Handler
 		Name:      "requests_processed_total",
 		Help:      "The total number of processed API requests",
 	}, []string{"code", "method", "path"})
-	requestsConcurrent := factory.NewGauge(prometheus.GaugeOpts{
+	requestsConcurrent := factory.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: "coderd",
 		Subsystem: "api",
 		Name:      "concurrent_requests",
 		Help:      "The number of concurrent API requests.",
-	})
-	websocketsConcurrent := factory.NewGauge(prometheus.GaugeOpts{
+	}, []string{"method", "path"})
+	websocketsConcurrent := factory.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: "coderd",
 		Subsystem: "api",
 		Name:      "concurrent_websockets",
 		Help:      "The total number of concurrent API websockets.",
-	})
+	}, []string{"path"})
 	websocketsDist := factory.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: "coderd",
 		Subsystem: "api",
@@ -61,7 +62,6 @@ func Prometheus(register prometheus.Registerer) func(http.Handler) http.Handler
 			var (
 				start  = time.Now()
 				method = r.Method
-				rctx   = chi.RouteContext(r.Context())
 			)
 
 			sw, ok := w.(*tracing.StatusWriter)
@@ -72,24 +72,25 @@ func Prometheus(register prometheus.Registerer) func(http.Handler) http.Handler
 			var (
 				dist     *prometheus.HistogramVec
 				distOpts []string
+				path     = getRoutePattern(r)
 			)
+
 			// We want to count WebSockets separately.
 			if httpapi.IsWebsocketUpgrade(r) {
-				websocketsConcurrent.Inc()
-				defer websocketsConcurrent.Dec()
+				websocketsConcurrent.WithLabelValues(path).Inc()
+				defer websocketsConcurrent.WithLabelValues(path).Dec()
 
 				dist = websocketsDist
 			} else {
-				requestsConcurrent.Inc()
-				defer requestsConcurrent.Dec()
+				requestsConcurrent.WithLabelValues(method, path).Inc()
+				defer requestsConcurrent.WithLabelValues(method, path).Dec()
 
 				dist = requestsDist
 				distOpts = []string{method}
 			}
 
 			next.ServeHTTP(w, r)
 
-			path := rctx.RoutePattern()
 			distOpts = append(distOpts, path)
 			statusStr := strconv.Itoa(sw.Status)
 
@@ -98,3 +99,34 @@ func Prometheus(register prometheus.Registerer) func(http.Handler) http.Handler
 		})
 	}
 }
+
+func getRoutePattern(r *http.Request) string {
+	rctx := chi.RouteContext(r.Context())
+	if rctx == nil {
+		return ""
+	}
+
+	if pattern := rctx.RoutePattern(); pattern != "" {
+		// Pattern is already available
+		return pattern
+	}
+
+	routePath := r.URL.Path
+	if r.URL.RawPath != "" {
+		routePath = r.URL.RawPath
+	}
+
+	tctx := chi.NewRouteContext()
+	routes := rctx.Routes
+	if routes != nil && !routes.Match(tctx, r.Method, routePath) {
+		// No matching pattern. /api/* requests will be matched as "UNKNOWN"
+		// All other ones will be matched as "STATIC".
+		if strings.HasPrefix(routePath, "/api/") {
+			return "UNKNOWN"
+		}
+		return "STATIC"
+	}
+
+	// tctx has the updated pattern, since Match mutates it
+	return tctx.RoutePattern()
+}

coderd/httpmw/prometheus_test.go

@@ -8,14 +8,19 @@ import (
 
 	"github.com/go-chi/chi/v5"
 	"github.com/prometheus/client_golang/prometheus"
+	cm "github.com/prometheus/client_model/go"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/coder/coder/v2/coderd/httpmw"
 	"github.com/coder/coder/v2/coderd/tracing"
+	"github.com/coder/coder/v2/testutil"
+	"github.com/coder/websocket"
 )
 
 func TestPrometheus(t *testing.T) {
 	t.Parallel()
+
 	t.Run("All", func(t *testing.T) {
 		t.Parallel()
 		req := httptest.NewRequest("GET", "/", nil)
@@ -29,4 +34,90 @@ func TestPrometheus(t *testing.T) {
 		require.NoError(t, err)
 		require.Greater(t, len(metrics), 0)
 	})
+
+	t.Run("Concurrent", func(t *testing.T) {
+		t.Parallel()
+		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
+		defer cancel()
+
+		reg := prometheus.NewRegistry()
+		promMW := httpmw.Prometheus(reg)
+
+		// Create a test handler to simulate a WebSocket connection
+		testHandler := http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+			conn, err := websocket.Accept(rw, r, nil)
+			if !assert.NoError(t, err, "failed to accept websocket") {
+				return
+			}
+			defer conn.Close(websocket.StatusGoingAway, "")
+		})
+
+		wrappedHandler := promMW(testHandler)
+
+		r := chi.NewRouter()
+		r.Use(tracing.StatusWriterMiddleware, promMW)
+		r.Get("/api/v2/build/{build}/logs", func(rw http.ResponseWriter, r *http.Request) {
+			wrappedHandler.ServeHTTP(rw, r)
+		})
+
+		srv := httptest.NewServer(r)
+		defer srv.Close()
+		// nolint: bodyclose
+		conn, _, err := websocket.Dial(ctx, srv.URL+"/api/v2/build/1/logs", nil)
+		require.NoError(t, err, "failed to dial WebSocket")
+		defer conn.Close(websocket.StatusNormalClosure, "")
+
+		metrics, err := reg.Gather()
+		require.NoError(t, err)
+		require.Greater(t, len(metrics), 0)
+		metricLabels := getMetricLabels(metrics)
+
+		concurrentWebsockets, ok := metricLabels["coderd_api_concurrent_websockets"]
+		require.True(t, ok, "coderd_api_concurrent_websockets metric not found")
+		require.Equal(t, "/api/v2/build/{build}/logs", concurrentWebsockets["path"])
+	})
+
+	t.Run("UserRoute", func(t *testing.T) {
+		t.Parallel()
+		reg := prometheus.NewRegistry()
+		promMW := httpmw.Prometheus(reg)
+
+		r := chi.NewRouter()
+		r.With(promMW).Get("/api/v2/users/{user}", func(w http.ResponseWriter, r *http.Request) {})
+
+		req := httptest.NewRequest("GET", "/api/v2/users/john", nil)
+
+		sw := &tracing.StatusWriter{ResponseWriter: httptest.NewRecorder()}
+
+		r.ServeHTTP(sw, req)
+
+		metrics, err := reg.Gather()
+		require.NoError(t, err)
+		require.Greater(t, len(metrics), 0)
+		metricLabels := getMetricLabels(metrics)
+
+		reqProcessed, ok := metricLabels["coderd_api_requests_processed_total"]
+		require.True(t, ok, "coderd_api_requests_processed_total metric not found")
+		require.Equal(t, "/api/v2/users/{user}", reqProcessed["path"])
+		require.Equal(t, "GET", reqProcessed["method"])
+
+		concurrentRequests, ok := metricLabels["coderd_api_concurrent_requests"]
+		require.True(t, ok, "coderd_api_concurrent_requests metric not found")
+		require.Equal(t, "/api/v2/users/{user}", concurrentRequests["path"])
+		require.Equal(t, "GET", concurrentRequests["method"])
+	})
+}
+
+func getMetricLabels(metrics []*cm.MetricFamily) map[string]map[string]string {
+	metricLabels := map[string]map[string]string{}
+	for _, metricFamily := range metrics {
+		metricName := metricFamily.GetName()
+		metricLabels[metricName] = map[string]string{}
+		for _, metric := range metricFamily.GetMetric() {
+			for _, labelPair := range metric.GetLabel() {
+				metricLabels[metricName][labelPair.GetName()] = labelPair.GetValue()
+			}
+		}
+	}
+	return metricLabels
 }