feat: add metrics, observability, test fixtures, and scaling playbook artifacts
This commit is contained in:
zwitschi committed
1 parent
926a663c23
commit
f9aba11b90
32 files changed
+1788
-124
No files matched your search
@@ -3,9 +3,11 @@ package metrics
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
@@ -15,6 +17,62 @@ import (
|
||||
"chat/src/ws"
|
||||
)
|
||||
|
||||
var requestDurationBuckets = []float64{0.05, 0.1, 0.25, 0.5, 1, 2, 5}
|
||||
|
||||
var chatRequestDurationHistogram = newRequestDurationHistogram(requestDurationBuckets)
|
||||
|
||||
type requestDurationHistogram struct {
|
||||
mu sync.Mutex
|
||||
buckets []float64
|
||||
counts []uint64
|
||||
sum float64
|
||||
total uint64
|
||||
}
|
||||
|
||||
func newRequestDurationHistogram(buckets []float64) *requestDurationHistogram {
|
||||
return &requestDurationHistogram{
|
||||
buckets: append([]float64(nil), buckets...),
|
||||
counts: make([]uint64, len(buckets)+1),
|
||||
}
|
||||
}
|
||||
|
||||
func (h *requestDurationHistogram) observe(durationSeconds float64) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
|
||||
h.total++
|
||||
h.sum += durationSeconds
|
||||
for index, bucket := range h.buckets {
|
||||
if durationSeconds <= bucket {
|
||||
h.counts[index]++
|
||||
return
|
||||
}
|
||||
}
|
||||
h.counts[len(h.counts)-1]++
|
||||
}
|
||||
|
||||
func (h *requestDurationHistogram) snapshot() (counts []uint64, sum float64, total uint64) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
|
||||
counts = append([]uint64(nil), h.counts...)
|
||||
return counts, h.sum, h.total
|
||||
}
|
||||
|
||||
// ObserveRequestDuration records a chat HTTP request duration in seconds.
|
||||
func ObserveRequestDuration(duration time.Duration) {
|
||||
chatRequestDurationHistogram.observe(duration.Seconds())
|
||||
}
|
||||
|
||||
// WrapRequestTiming records request duration for the provided handler.
|
||||
func WrapRequestTiming(next http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
startedAt := time.Now()
|
||||
defer ObserveRequestDuration(time.Since(startedAt))
|
||||
next(w, r)
|
||||
}
|
||||
}
|
||||
|
||||
// HandleMetrics returns Prometheus-format metrics for the chat service.
|
||||
func HandleMetrics(gormDB *gorm.DB, redis db.RedisClient, hub *ws.Hub, sessionStore *auth.SessionStore, dbDriver, instanceID string) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -36,6 +94,9 @@ func HandleMetrics(gormDB *gorm.DB, redis db.RedisClient, hub *ws.Hub, sessionSt
|
||||
redisEnabled := 0
|
||||
redisUp := 0
|
||||
redisClients := 0
|
||||
redisMemoryUsedBytes := int64(0)
|
||||
redisMemoryMaxBytes := int64(0)
|
||||
redisMemoryUsageRatio := 0.0
|
||||
if redis != nil {
|
||||
redisEnabled = 1
|
||||
if err := redis.Ping(context.Background()); err == nil {
|
||||
@@ -50,6 +111,19 @@ func HandleMetrics(gormDB *gorm.DB, redis db.RedisClient, hub *ws.Hub, sessionSt
|
||||
}
|
||||
}
|
||||
}
|
||||
if info, err := redis.Info(context.Background(), "memory"); err == nil {
|
||||
if value := parseRedisMemoryBytes(info, "used_memory_human"); value > 0 {
|
||||
redisMemoryUsedBytes = value
|
||||
}
|
||||
if value := parseRedisMemoryBytes(info, "maxmemory_human"); value > 0 {
|
||||
redisMemoryMaxBytes = value
|
||||
} else if value := parseRedisMemoryBytes(info, "maxmemory"); value > 0 {
|
||||
redisMemoryMaxBytes = value
|
||||
}
|
||||
if redisMemoryMaxBytes > 0 {
|
||||
redisMemoryUsageRatio = math.Min(float64(redisMemoryUsedBytes)/float64(redisMemoryMaxBytes), 1.0)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -99,6 +173,18 @@ func HandleMetrics(gormDB *gorm.DB, redis db.RedisClient, hub *ws.Hub, sessionSt
|
||||
fmt.Fprintln(w, "# TYPE linearvoid_chat_redis_connected_clients gauge")
|
||||
writeLine("linearvoid_chat_redis_connected_clients", redisClients)
|
||||
|
||||
fmt.Fprintln(w, "# HELP linearvoid_chat_redis_memory_used_bytes Redis used memory reported by INFO memory.")
|
||||
fmt.Fprintln(w, "# TYPE linearvoid_chat_redis_memory_used_bytes gauge")
|
||||
writeLine("linearvoid_chat_redis_memory_used_bytes", redisMemoryUsedBytes)
|
||||
|
||||
fmt.Fprintln(w, "# HELP linearvoid_chat_redis_memory_max_bytes Redis maxmemory limit reported by INFO memory.")
|
||||
fmt.Fprintln(w, "# TYPE linearvoid_chat_redis_memory_max_bytes gauge")
|
||||
writeLine("linearvoid_chat_redis_memory_max_bytes", redisMemoryMaxBytes)
|
||||
|
||||
fmt.Fprintln(w, "# HELP linearvoid_chat_redis_memory_usage_ratio Redis memory usage ratio (used / max).")
|
||||
fmt.Fprintln(w, "# TYPE linearvoid_chat_redis_memory_usage_ratio gauge")
|
||||
writeLine("linearvoid_chat_redis_memory_usage_ratio", redisMemoryUsageRatio)
|
||||
|
||||
fmt.Fprintln(w, "# HELP linearvoid_chat_admin_sessions_active Active unexpired admin sessions.")
|
||||
fmt.Fprintln(w, "# TYPE linearvoid_chat_admin_sessions_active gauge")
|
||||
writeLine("linearvoid_chat_admin_sessions_active", activeSessions)
|
||||
@@ -106,6 +192,19 @@ func HandleMetrics(gormDB *gorm.DB, redis db.RedisClient, hub *ws.Hub, sessionSt
|
||||
fmt.Fprintln(w, "# HELP linearvoid_chat_build_info Static build info for chat service.")
|
||||
fmt.Fprintln(w, "# TYPE linearvoid_chat_build_info gauge")
|
||||
fmt.Fprintf(w, "linearvoid_chat_build_info{service=\"chat\",database_driver=\"%s\",instance_id=\"%s\"} 1\n", dbDriver, instanceID)
|
||||
|
||||
counts, sum, total := chatRequestDurationHistogram.snapshot()
|
||||
fmt.Fprintln(w, "# HELP linearvoid_chat_request_duration_seconds HTTP request duration for chat API handlers.")
|
||||
fmt.Fprintln(w, "# TYPE linearvoid_chat_request_duration_seconds histogram")
|
||||
cumulative := uint64(0)
|
||||
for index, bucket := range requestDurationBuckets {
|
||||
cumulative += counts[index]
|
||||
fmt.Fprintf(w, "linearvoid_chat_request_duration_seconds_bucket{le=\"%g\"} %d\n", bucket, cumulative)
|
||||
}
|
||||
cumulative += counts[len(counts)-1]
|
||||
fmt.Fprintf(w, "linearvoid_chat_request_duration_seconds_bucket{le=\"+Inf\"} %d\n", cumulative)
|
||||
fmt.Fprintf(w, "linearvoid_chat_request_duration_seconds_sum %g\n", sum)
|
||||
fmt.Fprintf(w, "linearvoid_chat_request_duration_seconds_count %d\n", total)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -125,4 +224,55 @@ func HandleNotFound() http.HandlerFunc {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
fmt.Fprint(w, `{"error":"not found"}`)
|
||||
}
|
||||
}
|
||||
|
||||
func parseRedisMemoryBytes(info, key string) int64 {
|
||||
for _, line := range strings.Split(info, "\n") {
|
||||
if !strings.HasPrefix(line, key+":") {
|
||||
continue
|
||||
}
|
||||
value := strings.TrimSpace(strings.TrimPrefix(line, key+":"))
|
||||
if value == "" {
|
||||
return 0
|
||||
}
|
||||
if raw, err := strconv.ParseInt(value, 10, 64); err == nil {
|
||||
return raw
|
||||
}
|
||||
return parseHumanBytes(value)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func parseHumanBytes(value string) int64 {
|
||||
value = strings.TrimSpace(strings.ToUpper(value))
|
||||
if value == "" {
|
||||
return 0
|
||||
}
|
||||
|
||||
multiplier := float64(1)
|
||||
suffixes := map[string]float64{
|
||||
"B": 1,
|
||||
"K": 1024,
|
||||
"KB": 1024,
|
||||
"M": 1024 * 1024,
|
||||
"MB": 1024 * 1024,
|
||||
"G": 1024 * 1024 * 1024,
|
||||
"GB": 1024 * 1024 * 1024,
|
||||
"T": 1024 * 1024 * 1024 * 1024,
|
||||
"TB": 1024 * 1024 * 1024 * 1024,
|
||||
}
|
||||
|
||||
for suffix, factor := range suffixes {
|
||||
if strings.HasSuffix(value, suffix) {
|
||||
multiplier = factor
|
||||
value = strings.TrimSpace(strings.TrimSuffix(value, suffix))
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
parsed, err := strconv.ParseFloat(value, 64)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return int64(parsed * multiplier)
|
||||
}
|
||||
Reference in new issue
Block a user