coderd/httpmw/prometheus.go

Rama main · réplica de lectura del repositorio original.

package httpmw

import (
	"context"
	"net/http"
	"strconv"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"

	"github.com/coder/coder/v2/coderd/httpapi"
	"github.com/coder/coder/v2/coderd/tracing"
)

// WSMetrics groups all WebSocket-related Prometheus metrics so they
// can be created once and shared between the HTTP middleware and the
// WSWatcher probe recorder.
type WSMetrics struct {
	Concurrent *prometheus.GaugeVec
	Durations  *prometheus.HistogramVec
	Probes     *prometheus.CounterVec
}

// NewWSMetrics registers and returns WebSocket metrics. The returned
// struct is safe to pass to both Prometheus() and
// WSMetrics.RecordProbe.
func NewWSMetrics(reg prometheus.Registerer) *WSMetrics {
	factory := promauto.With(reg)
	return &WSMetrics{
		Concurrent: factory.NewGaugeVec(prometheus.GaugeOpts{
			Namespace: "coderd",
			Subsystem: "api",
			Name:      "concurrent_websockets",
			Help:      "The total number of concurrent API websockets.",
		}, []string{"path"}),
		Durations: factory.NewHistogramVec(prometheus.HistogramOpts{
			Namespace: "coderd",
			Subsystem: "api",
			Name:      "websocket_durations_seconds",
			Help:      "Websocket duration distribution of requests in seconds.",
			Buckets: []float64{
				0.001, // 1ms
				1,
				60,           // 1 minute
				60 * 60,      // 1 hour
				60 * 60 * 15, // 15 hours
				60 * 60 * 30, // 30 hours
			},
		}, []string{"path"}),
		Probes: factory.NewCounterVec(prometheus.CounterOpts{
			Namespace: "coderd",
			Subsystem: "api",
			Name:      "websocket_probes_total",
			Help: "WebSocket liveness probe outcomes by route. " +
				"Compare rate(...{result=\"ok\"}[1m]) against " +
				"coderd_api_concurrent_websockets to detect " +
				"unresponsive WebSocket connections.",
		}, []string{"path", "result"}),
	}
}

// RecordProbe records a single liveness probe outcome. It extracts
// the HTTP route from ctx via ExtractHTTPRoute.
func (m *WSMetrics) RecordProbe(ctx context.Context, r httpapi.ProbeResult) {
	m.Probes.WithLabelValues(ExtractHTTPRoute(ctx), string(r)).Inc()
}

func Prometheus(register prometheus.Registerer, ws *WSMetrics) func(http.Handler) http.Handler {
	if ws == nil {
		panic("developer error: WSMetrics is nil")
	}
	factory := promauto.With(register)
	requestsProcessed := factory.NewCounterVec(prometheus.CounterOpts{
		Namespace: "coderd",
		Subsystem: "api",
		Name:      "requests_processed_total",
		Help:      "The total number of processed API requests",
	}, []string{"code", "method", "path"})
	requestsConcurrent := factory.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: "coderd",
		Subsystem: "api",
		Name:      "concurrent_requests",
		Help:      "The number of concurrent API requests.",
	}, []string{"method", "path"})
	requestsDist := factory.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: "coderd",
		Subsystem: "api",
		Name:      "request_latencies_seconds",
		Help:      "Latency distribution of requests in seconds.",
		Buckets:   []float64{0.001, 0.005, 0.010, 0.025, 0.050, 0.100, 0.500, 1, 5, 10, 30},
	}, []string{"method", "path"})
	// Series here exist only for routes that have actually rejected a body,
	// which is what makes this readable at a glance where filtering
	// requests_processed_total by code is not.
	requestsTooLarge := factory.NewCounterVec(prometheus.CounterOpts{
		Namespace: "coderd",
		Subsystem: "api",
		Name:      "requests_too_large_total",
		Help: "The total number of API requests answered 413, by the reason " +
			"they were rejected. A sustained rate of reason=\"request_body\" " +
			"on one route is more often a limit set too tight for a " +
			"legitimate payload than an attempt to exhaust memory. " +
			"reason=\"other\" counts 413s raised for reasons unrelated to " +
			"the size of the request body, such as agent log storage " +
			"overflow or an archive that is too large once expanded.",
	}, []string{"method", "path", "reason"})

	return func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			var (
				start  = time.Now()
				method = r.Method
			)

			sw, ok := w.(*tracing.StatusWriter)
			if !ok {
				panic("dev error: http.ResponseWriter is not *tracing.StatusWriter")
			}

			var (
				dist     *prometheus.HistogramVec
				distOpts []string
				path     = ExtractHTTPRoute(r.Context())
			)

			// We want to count WebSockets separately.
			if httpapi.IsWebsocketUpgrade(r) {
				ws.Concurrent.WithLabelValues(path).Inc()
				defer ws.Concurrent.WithLabelValues(path).Dec()

				dist = ws.Durations
			} else {
				requestsConcurrent.WithLabelValues(method, path).Inc()
				defer requestsConcurrent.WithLabelValues(method, path).Dec()

				dist = requestsDist
				distOpts = []string{method}
			}

			// Counting by status rather than at the point of rejection reaches
			// handlers that bound their bodies themselves to keep their own
			// error shapes, such as aibridge. It also reaches the 413s that have
			// nothing to do with body size, such as agent log storage overflow,
			// so the tracker separates the two.
			bodyLimit := &httpapi.RequestBodyLimitTracker{}
			r = r.WithContext(httpapi.WithRequestBodyLimitTracker(r.Context(), bodyLimit))

			next.ServeHTTP(w, r)

			distOpts = append(distOpts, path)
			statusStr := strconv.Itoa(sw.Status)

			requestsProcessed.WithLabelValues(statusStr, method, path).Inc()
			dist.WithLabelValues(distOpts...).Observe(time.Since(start).Seconds())

			if sw.Status == http.StatusRequestEntityTooLarge {
				reason := "other"
				if bodyLimit.Exceeded() {
					reason = "request_body"
				}
				requestsTooLarge.WithLabelValues(method, path, reason).Inc()
			}
		})
	}
}