feat: add metrics, observability, test fixtures, and scaling playbook artifacts

2026-08-10 00:23:50 +02:00
1 parent 926a663c23
commit f9aba11b90
@@ -62,32 +62,32 @@ func main() {

 	// Register routes
 	http.HandleFunc("/ws", hub.HandleConnection)
-	http.HandleFunc("/api/admin/login", handlers.HandleAdminLogin(deps))
-	http.HandleFunc("/api/admin/logout", handlers.HandleAdminLogout(deps))
-	http.HandleFunc("/api/admin/session", handlers.HandleAdminSession(deps))
-	http.HandleFunc("/api/admin/overview", requireAdmin(handlers.HandleAdminOverview(deps)))
-	http.HandleFunc("/api/admin/playlist", requireAdmin(handlers.HandleAdminPlaylist(deps)))
-	http.HandleFunc("/api/admin/moderation/logs", requireAdmin(handlers.HandleAdminModerationLogs(deps)))
-	http.HandleFunc("/api/admin/moderation/actions", requireAdmin(handlers.HandleAdminModerationAction(deps)))
-	http.HandleFunc("/api/admin/analytics", requireAdmin(handlers.HandleAdminAnalytics(deps)))
-	http.HandleFunc("/api/admin/system", requireAdmin(handlers.HandleAdminSystemSummary(deps)))
-	http.HandleFunc("/api/admin/config", requireAdmin(handlers.HandleAdminConfig(deps)))
-	http.HandleFunc("/api/admin/restart", requireAdmin(handlers.HandleAdminRestart(deps)))
-	http.HandleFunc("/api/admin/users", requireAdmin(handlers.HandleAdminUsers(deps)))
-	http.HandleFunc("/api/admin/schedule", requireAdmin(handlers.HandleAdminSchedule(deps)))
-	http.HandleFunc("/api/admin/schedule/overrides", requireAdmin(handlers.HandleAdminScheduleOverrides(deps)))
-	http.HandleFunc("/api/schedule", handlers.HandlePublicSchedule(deps))
-	http.HandleFunc("/api/admin/media", requireAdmin(handlers.HandleAdminMedia(deps)))
-	http.HandleFunc("/api/admin/media/uploads", requireAdmin(handlers.HandleAdminMediaUploads(deps)))
-	http.HandleFunc("/api/admin/media/uploads/", requireAdmin(handlers.HandleAdminMediaUploads(deps)))
-	http.HandleFunc("/api/admin/media/", requireAdmin(handlers.HandleAdminMediaByID(deps)))
-	http.HandleFunc("/api/admin/chat-log/bulk", requireAdmin(handlers.HandleAdminChatLogBulk(deps)))
-	http.HandleFunc("/api/admin/chat-log/export", requireAdmin(handlers.HandleAdminChatLogExport(deps)))
-	http.HandleFunc("/api/admin/chat-log/", requireAdmin(handlers.HandleAdminChatLogByID(deps)))
-	http.HandleFunc("/api/admin/chat-log", requireAdmin(handlers.HandleAdminChatLog(deps)))
+	http.HandleFunc("/api/admin/login", metrics.WrapRequestTiming(handlers.HandleAdminLogin(deps)))
+	http.HandleFunc("/api/admin/logout", metrics.WrapRequestTiming(handlers.HandleAdminLogout(deps)))
+	http.HandleFunc("/api/admin/session", metrics.WrapRequestTiming(handlers.HandleAdminSession(deps)))
+	http.HandleFunc("/api/admin/overview", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminOverview(deps))))
+	http.HandleFunc("/api/admin/playlist", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminPlaylist(deps))))
+	http.HandleFunc("/api/admin/moderation/logs", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminModerationLogs(deps))))
+	http.HandleFunc("/api/admin/moderation/actions", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminModerationAction(deps))))
+	http.HandleFunc("/api/admin/analytics", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminAnalytics(deps))))
+	http.HandleFunc("/api/admin/system", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminSystemSummary(deps))))
+	http.HandleFunc("/api/admin/config", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminConfig(deps))))
+	http.HandleFunc("/api/admin/restart", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminRestart(deps))))
+	http.HandleFunc("/api/admin/users", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminUsers(deps))))
+	http.HandleFunc("/api/admin/schedule", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminSchedule(deps))))
+	http.HandleFunc("/api/admin/schedule/overrides", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminScheduleOverrides(deps))))
+	http.HandleFunc("/api/schedule", metrics.WrapRequestTiming(handlers.HandlePublicSchedule(deps)))
+	http.HandleFunc("/api/admin/media", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminMedia(deps))))
+	http.HandleFunc("/api/admin/media/uploads", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminMediaUploads(deps))))
+	http.HandleFunc("/api/admin/media/uploads/", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminMediaUploads(deps))))
+	http.HandleFunc("/api/admin/media/", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminMediaByID(deps))))
+	http.HandleFunc("/api/admin/chat-log/bulk", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminChatLogBulk(deps))))
+	http.HandleFunc("/api/admin/chat-log/export", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminChatLogExport(deps))))
+	http.HandleFunc("/api/admin/chat-log/", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminChatLogByID(deps))))
+	http.HandleFunc("/api/admin/chat-log", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminChatLog(deps))))
 	http.HandleFunc("/metrics", metrics.HandleMetrics(dbClient.DB, redisWrapper, hub, sessionStore, dbDriver, instanceID))
-	http.HandleFunc("/health", metrics.HandleHealth())
-	http.HandleFunc("/", metrics.HandleNotFound())
+	http.HandleFunc("/health", metrics.WrapRequestTiming(metrics.HandleHealth()))
+	http.HandleFunc("/", metrics.WrapRequestTiming(metrics.HandleNotFound()))

 	port := config.ChatPort()
 	log.Printf("Chat service starting on :%s", port)
@@ -3,9 +3,11 @@ package metrics
 import (
 	"context"
 	"fmt"
+	"math"
 	"net/http"
 	"strconv"
 	"strings"
+	"sync"
 	"time"

 	"gorm.io/gorm"
@@ -15,6 +17,62 @@ import (
 	"chat/src/ws"
 )

+var requestDurationBuckets = []float64{0.05, 0.1, 0.25, 0.5, 1, 2, 5}
+
+var chatRequestDurationHistogram = newRequestDurationHistogram(requestDurationBuckets)
+
+type requestDurationHistogram struct {
+	mu     sync.Mutex
+	buckets []float64
+	counts  []uint64
+	sum     float64
+	total   uint64
+}
+
+func newRequestDurationHistogram(buckets []float64) *requestDurationHistogram {
+	return &requestDurationHistogram{
+		buckets: append([]float64(nil), buckets...),
+		counts:  make([]uint64, len(buckets)+1),
+	}
+}
+
+func (h *requestDurationHistogram) observe(durationSeconds float64) {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+
+	h.total++
+	h.sum += durationSeconds
+	for index, bucket := range h.buckets {
+		if durationSeconds <= bucket {
+			h.counts[index]++
+			return
+		}
+	}
+	h.counts[len(h.counts)-1]++
+}
+
+func (h *requestDurationHistogram) snapshot() (counts []uint64, sum float64, total uint64) {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+
+	counts = append([]uint64(nil), h.counts...)
+	return counts, h.sum, h.total
+}
+
+// ObserveRequestDuration records a chat HTTP request duration in seconds.
+func ObserveRequestDuration(duration time.Duration) {
+	chatRequestDurationHistogram.observe(duration.Seconds())
+}
+
+// WrapRequestTiming records request duration for the provided handler.
+func WrapRequestTiming(next http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		startedAt := time.Now()
+		defer ObserveRequestDuration(time.Since(startedAt))
+		next(w, r)
+	}
+}
+
 // HandleMetrics returns Prometheus-format metrics for the chat service.
 func HandleMetrics(gormDB *gorm.DB, redis db.RedisClient, hub *ws.Hub, sessionStore *auth.SessionStore, dbDriver, instanceID string) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
@@ -36,6 +94,9 @@ func HandleMetrics(gormDB *gorm.DB, redis db.RedisClient, hub *ws.Hub, sessionSt
 		redisEnabled := 0
 		redisUp := 0
 		redisClients := 0
+		redisMemoryUsedBytes := int64(0)
+		redisMemoryMaxBytes := int64(0)
+		redisMemoryUsageRatio := 0.0
 		if redis != nil {
 			redisEnabled = 1
 			if err := redis.Ping(context.Background()); err == nil {
@@ -50,6 +111,19 @@ func HandleMetrics(gormDB *gorm.DB, redis db.RedisClient, hub *ws.Hub, sessionSt
 						}
 					}
 				}
+				if info, err := redis.Info(context.Background(), "memory"); err == nil {
+					if value := parseRedisMemoryBytes(info, "used_memory_human"); value > 0 {
+						redisMemoryUsedBytes = value
+					}
+					if value := parseRedisMemoryBytes(info, "maxmemory_human"); value > 0 {
+						redisMemoryMaxBytes = value
+					} else if value := parseRedisMemoryBytes(info, "maxmemory"); value > 0 {
+						redisMemoryMaxBytes = value
+					}
+					if redisMemoryMaxBytes > 0 {
+						redisMemoryUsageRatio = math.Min(float64(redisMemoryUsedBytes)/float64(redisMemoryMaxBytes), 1.0)
+					}
+				}
 			}
 		}

@@ -99,6 +173,18 @@ func HandleMetrics(gormDB *gorm.DB, redis db.RedisClient, hub *ws.Hub, sessionSt
 		fmt.Fprintln(w, "# TYPE linearvoid_chat_redis_connected_clients gauge")
 		writeLine("linearvoid_chat_redis_connected_clients", redisClients)

+		fmt.Fprintln(w, "# HELP linearvoid_chat_redis_memory_used_bytes Redis used memory reported by INFO memory.")
+		fmt.Fprintln(w, "# TYPE linearvoid_chat_redis_memory_used_bytes gauge")
+		writeLine("linearvoid_chat_redis_memory_used_bytes", redisMemoryUsedBytes)
+
+		fmt.Fprintln(w, "# HELP linearvoid_chat_redis_memory_max_bytes Redis maxmemory limit reported by INFO memory.")
+		fmt.Fprintln(w, "# TYPE linearvoid_chat_redis_memory_max_bytes gauge")
+		writeLine("linearvoid_chat_redis_memory_max_bytes", redisMemoryMaxBytes)
+
+		fmt.Fprintln(w, "# HELP linearvoid_chat_redis_memory_usage_ratio Redis memory usage ratio (used / max).")
+		fmt.Fprintln(w, "# TYPE linearvoid_chat_redis_memory_usage_ratio gauge")
+		writeLine("linearvoid_chat_redis_memory_usage_ratio", redisMemoryUsageRatio)
+
 		fmt.Fprintln(w, "# HELP linearvoid_chat_admin_sessions_active Active unexpired admin sessions.")
 		fmt.Fprintln(w, "# TYPE linearvoid_chat_admin_sessions_active gauge")
 		writeLine("linearvoid_chat_admin_sessions_active", activeSessions)
@@ -106,6 +192,19 @@ func HandleMetrics(gormDB *gorm.DB, redis db.RedisClient, hub *ws.Hub, sessionSt
 		fmt.Fprintln(w, "# HELP linearvoid_chat_build_info Static build info for chat service.")
 		fmt.Fprintln(w, "# TYPE linearvoid_chat_build_info gauge")
 		fmt.Fprintf(w, "linearvoid_chat_build_info{service=\"chat\",database_driver=\"%s\",instance_id=\"%s\"} 1\n", dbDriver, instanceID)
+
+		counts, sum, total := chatRequestDurationHistogram.snapshot()
+		fmt.Fprintln(w, "# HELP linearvoid_chat_request_duration_seconds HTTP request duration for chat API handlers.")
+		fmt.Fprintln(w, "# TYPE linearvoid_chat_request_duration_seconds histogram")
+		cumulative := uint64(0)
+		for index, bucket := range requestDurationBuckets {
+			cumulative += counts[index]
+			fmt.Fprintf(w, "linearvoid_chat_request_duration_seconds_bucket{le=\"%g\"} %d\n", bucket, cumulative)
+		}
+		cumulative += counts[len(counts)-1]
+		fmt.Fprintf(w, "linearvoid_chat_request_duration_seconds_bucket{le=\"+Inf\"} %d\n", cumulative)
+		fmt.Fprintf(w, "linearvoid_chat_request_duration_seconds_sum %g\n", sum)
+		fmt.Fprintf(w, "linearvoid_chat_request_duration_seconds_count %d\n", total)
 	}
 }

@@ -125,4 +224,55 @@ func HandleNotFound() http.HandlerFunc {
 		w.WriteHeader(http.StatusNotFound)
 		fmt.Fprint(w, `{"error":"not found"}`)
 	}
+}
+
+func parseRedisMemoryBytes(info, key string) int64 {
+	for _, line := range strings.Split(info, "\n") {
+		if !strings.HasPrefix(line, key+":") {
+			continue
+		}
+		value := strings.TrimSpace(strings.TrimPrefix(line, key+":"))
+		if value == "" {
+			return 0
+		}
+		if raw, err := strconv.ParseInt(value, 10, 64); err == nil {
+			return raw
+		}
+		return parseHumanBytes(value)
+	}
+	return 0
+}
+
+func parseHumanBytes(value string) int64 {
+	value = strings.TrimSpace(strings.ToUpper(value))
+	if value == "" {
+		return 0
+	}
+
+	multiplier := float64(1)
+	suffixes := map[string]float64{
+		"B": 1,
+		"K": 1024,
+		"KB": 1024,
+		"M": 1024 * 1024,
+		"MB": 1024 * 1024,
+		"G": 1024 * 1024 * 1024,
+		"GB": 1024 * 1024 * 1024,
+		"T": 1024 * 1024 * 1024 * 1024,
+		"TB": 1024 * 1024 * 1024 * 1024,
+	}
+
+	for suffix, factor := range suffixes {
+		if strings.HasSuffix(value, suffix) {
+			multiplier = factor
+			value = strings.TrimSpace(strings.TrimSuffix(value, suffix))
+			break
+		}
+	}
+
+	parsed, err := strconv.ParseFloat(value, 64)
+	if err != nil {
+		return 0
+	}
+	return int64(parsed * multiplier)
 }
@@ -54,6 +54,10 @@ func TestHandleNotFound(t *testing.T) {
 }

 func TestHandleMetrics(t *testing.T) {
+	chatRequestDurationHistogram = newRequestDurationHistogram(requestDurationBuckets)
+	ObserveRequestDuration(100 * time.Millisecond)
+	ObserveRequestDuration(750 * time.Millisecond)
+
 	gormDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
 	if err != nil {
 		t.Fatalf("failed to open test DB: %v", err)
@@ -89,8 +93,12 @@ func TestHandleMetrics(t *testing.T) {
 		"linearvoid_chat_redis_enabled",
 		"linearvoid_chat_redis_up",
 		"linearvoid_chat_redis_connected_clients",
+		"linearvoid_chat_redis_memory_used_bytes",
+		"linearvoid_chat_redis_memory_max_bytes",
+		"linearvoid_chat_redis_memory_usage_ratio",
 		"linearvoid_chat_admin_sessions_active",
 		"linearvoid_chat_build_info",
+		"linearvoid_chat_request_duration_seconds",
 	}

 	for _, metric := range requiredMetrics {
@@ -117,6 +125,12 @@ func TestHandleMetrics(t *testing.T) {
 	if !strings.Contains(body, "linearvoid_chat_redis_enabled 0") {
 		t.Error("redis_enabled should be 0 (nil redis)")
 	}
+	if !strings.Contains(body, "linearvoid_chat_request_duration_seconds_bucket{le=\"1\"} 2") {
+		t.Error("request duration histogram should include both observed samples by the 1s bucket")
+	}
+	if !strings.Contains(body, "linearvoid_chat_request_duration_seconds_count 2") {
+		t.Error("request duration count should be 2")
+	}
 }

 func TestHandleMetrics_WrongMethod(t *testing.T) {
@@ -132,6 +146,9 @@ func TestHandleMetrics_WrongMethod(t *testing.T) {
 }

 func TestHandleMetrics_WithRedis(t *testing.T) {
+	chatRequestDurationHistogram = newRequestDurationHistogram(requestDurationBuckets)
+	ObserveRequestDuration(250 * time.Millisecond)
+
 	gormDB, _ := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
 	gormDB.AutoMigrate(&db.Message{}, &db.ModerationLog{})

@@ -160,6 +177,15 @@ func TestHandleMetrics_WithRedis(t *testing.T) {
 	if !strings.Contains(body, "linearvoid_chat_redis_connected_clients 5") {
 		t.Error("redis_connected_clients should be 5")
 	}
+	if !strings.Contains(body, "linearvoid_chat_redis_memory_used_bytes 1048576") {
+		t.Error("redis_memory_used_bytes should be 1048576")
+	}
+	if !strings.Contains(body, "linearvoid_chat_redis_memory_usage_ratio 1") {
+		t.Error("redis_memory_usage_ratio should be 1 when maxmemory is unavailable or equal to used memory in tests")
+	}
+	if !strings.Contains(body, "linearvoid_chat_request_duration_seconds_count 1") {
+		t.Error("request duration count should be 1")
+	}
 }

 // metricsMockRedis is a minimal mock for metrics tests.
@@ -185,6 +211,6 @@ func (m *metricsMockRedis) Scan(ctx context.Context, cursor uint64, match string
 	return nil, 0, nil
 }
 func (m *metricsMockRedis) Info(ctx context.Context, sections ...string) (string, error) {
-	return "connected_clients:5\nused_memory_human:1M\n", nil
+	return "connected_clients:5\nused_memory_human:1M\nmaxmemory_human:1M\n", nil
 }
 func (m *metricsMockRedis) Close() error { return nil }