zwitschi/linearvoid
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 9 Wiki Activity

feat: monitoring exporters, alerting, rate limiting, backups, incident runbook
CI / test (push) Successful in 58s
Details
CI / docker (push) Successful in 2m22s
Details
CI / release (push) Skipped
Details

Browse Source 
- Add 4 Prometheus exporters: nginx (9113), redis (9121), postgres (9187), node (9100)
- Create 8 alerting rules in monitoring/prometheus/alerts.yml
- Update prometheus.yml with all scrape targets + rule_files
- WS connection rate limiting: max 5/IP/60s (Redis INCR)
- Admin login rate limiting: max 5/IP/5min (Redis INCR)
- 3 backup scripts: daily to R2 (14d), weekly full (8w), verify with restore test
- docs/incident-runbook.md with 5-tier triage + escalation
- Update CD workflow: full prometheus config + alerts seeding
This commit is contained in:
zwitschi committed 2026-08-06 19:31:40 +02:00
1 parent b5710548b1
commit 0f81a81029
11 files changed
+683 -73
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

backend/chat/src/main.go
+97 -61
View File
@@ -32,9 +32,9 @@ var (
upgrader = websocket.Upgrader{
CheckOrigin: func(r *http.Request) bool { return true },
}
clients = make(map[*websocket.Conn]bool)
clientMeta = make(map[*websocket.Conn]clientInfo)
mutex = sync.Mutex{}
clients = make(map[*websocket.Conn]bool)
clientMeta = make(map[*websocket.Conn]clientInfo)
mutex = sync.Mutex{}
authSessions = map[string]time.Time{}
authSessionsMu = sync.Mutex{}
@@ -101,17 +101,17 @@ type Presence struct {
}
type AdminOverview struct {
Viewers int `json:"viewers"`
StoredMessages int64 `json:"storedMessages"`
ModerationActions int64 `json:"moderationActions"`
MediaCount int64 `json:"mediaCount"`
Viewers int `json:"viewers"`
StoredMessages int64 `json:"storedMessages"`
ModerationActions int64 `json:"moderationActions"`
MediaCount int64 `json:"mediaCount"`
MediaTotalDurationSeconds int64 `json:"mediaTotalDurationSeconds"`
MediaTotalRuntime string `json:"mediaTotalRuntime"`
RedisEnabled bool `json:"redisEnabled"`
RedisClients int `json:"redisClients"`
RedisMemoryUsed string `json:"redisMemoryUsed"`
DatabaseDriver string `json:"databaseDriver"`
GeneratedAt string `json:"generatedAt"`
MediaTotalRuntime string `json:"mediaTotalRuntime"`
RedisEnabled bool `json:"redisEnabled"`
RedisClients int `json:"redisClients"`
RedisMemoryUsed string `json:"redisMemoryUsed"`
DatabaseDriver string `json:"databaseDriver"`
GeneratedAt string `json:"generatedAt"`
}
type PlaylistPayload struct {
@@ -136,15 +136,15 @@ type RoomAnalytics struct {
}
type AdminAnalytics struct {
TotalMessages int64 `json:"totalMessages"`
MessagesLast24Hours int64 `json:"messagesLast24Hours"`
DistinctRooms int64 `json:"distinctRooms"`
ModerationActions int64 `json:"moderationActions"`
MediaCount int64 `json:"mediaCount"`
MediaTotalDurationSeconds int64 `json:"mediaTotalDurationSeconds"`
MediaTotalRuntime string `json:"mediaTotalRuntime"`
TopRooms []RoomAnalytics `json:"topRooms"`
GeneratedAt string `json:"generatedAt"`
TotalMessages int64 `json:"totalMessages"`
MessagesLast24Hours int64 `json:"messagesLast24Hours"`
DistinctRooms int64 `json:"distinctRooms"`
ModerationActions int64 `json:"moderationActions"`
MediaCount int64 `json:"mediaCount"`
MediaTotalDurationSeconds int64 `json:"mediaTotalDurationSeconds"`
MediaTotalRuntime string `json:"mediaTotalRuntime"`
TopRooms []RoomAnalytics `json:"topRooms"`
GeneratedAt string `json:"generatedAt"`
}
type ServiceHealth struct {
@@ -171,9 +171,9 @@ type ActiveUser struct {
}
type AdminUsersResponse struct {
Users []ActiveUser `json:"users"`
RedisEnabled bool `json:"redisEnabled"`
GeneratedAt string `json:"generatedAt"`
Users []ActiveUser `json:"users"`
RedisEnabled bool `json:"redisEnabled"`
GeneratedAt string `json:"generatedAt"`
}
type ChatLogEntry struct {
@@ -205,12 +205,12 @@ type ScheduleEntry struct {
}
type AdminScheduleResponse struct {
Entries []ScheduleEntry `json:"entries"`
TotalEntries int `json:"totalEntries"`
MatchedEntries int `json:"matchedEntries"`
TotalDurationSeconds int `json:"totalDurationSeconds"`
TotalRuntime string `json:"totalRuntime"`
GeneratedAt string `json:"generatedAt"`
Entries []ScheduleEntry `json:"entries"`
TotalEntries int `json:"totalEntries"`
MatchedEntries int `json:"matchedEntries"`
TotalDurationSeconds int `json:"totalDurationSeconds"`
TotalRuntime string `json:"totalRuntime"`
GeneratedAt string `json:"generatedAt"`
}
type MediaFileEntry struct {
@@ -234,12 +234,12 @@ type AdminMediaResponse struct {
}
type AdminMediaUploadInitRequest struct {
UploadID string `json:"uploadId"`
FileName string `json:"fileName"`
FileSize int64 `json:"fileSize"`
MimeType string `json:"mimeType"`
ChunkSize int64 `json:"chunkSize"`
TotalChunks int `json:"totalChunks"`
UploadID string `json:"uploadId"`
FileName string `json:"fileName"`
FileSize int64 `json:"fileSize"`
MimeType string `json:"mimeType"`
ChunkSize int64 `json:"chunkSize"`
TotalChunks int `json:"totalChunks"`
}
type AdminMediaUploadSession struct {
@@ -254,13 +254,13 @@ type AdminMediaUploadSession struct {
}
type AdminMediaUploadStatusResponse struct {
UploadID string `json:"uploadId"`
FileName string `json:"fileName"`
FileSize int64 `json:"fileSize"`
ChunkSize int64 `json:"chunkSize"`
TotalChunks int `json:"totalChunks"`
UploadedChunks int `json:"uploadedChunks"`
Progress float64 `json:"progress"`
UploadID string `json:"uploadId"`
FileName string `json:"fileName"`
FileSize int64 `json:"fileSize"`
ChunkSize int64 `json:"chunkSize"`
TotalChunks int `json:"totalChunks"`
UploadedChunks int `json:"uploadedChunks"`
Progress float64 `json:"progress"`
}
type AdminLoginPayload struct {
@@ -278,9 +278,9 @@ var databaseDriverName = "sqlite"
const adminSessionCookieName = "linearvoid_admin_session"
const (
maxAdminUploadBytes int64 = 2 * 1024 * 1024 * 1024
defaultAdminChunkSize int64 = 8 * 1024 * 1024
maxAdminChunkSize int64 = 32 * 1024 * 1024
maxAdminUploadBytes int64 = 2 * 1024 * 1024 * 1024
defaultAdminChunkSize int64 = 8 * 1024 * 1024
maxAdminChunkSize int64 = 32 * 1024 * 1024
)
// initRedis initializes Redis for presence, rate limiting, and pub/sub (ADR-007)
@@ -380,11 +380,11 @@ func parseRedisInfoValue(info, key string) string {
func getAdminOverview() AdminOverview {
overview := AdminOverview{
RedisEnabled: redisClient != nil,
RedisMemoryUsed: "unavailable",
DatabaseDriver: databaseDriverName,
RedisEnabled: redisClient != nil,
RedisMemoryUsed: "unavailable",
DatabaseDriver: databaseDriverName,
MediaTotalRuntime: "00:00:00",
GeneratedAt: time.Now().UTC().Format(time.RFC3339),
GeneratedAt: time.Now().UTC().Format(time.RFC3339),
}
mutex.Lock()
@@ -422,6 +422,23 @@ func handleAdminLogin(w http.ResponseWriter, r *http.Request) {
return
}
// Rate limit: max 5 attempts per IP per 5 minutes
clientIP := r.Header.Get("X-Forwarded-For")
if clientIP == "" {
clientIP = r.RemoteAddr
}
if redisClient != nil {
loginKey := fmt.Sprintf("login_attempts:%s", clientIP)
count, _ := redisClient.Incr(context.Background(), loginKey).Result()
if count == 1 {
redisClient.Expire(context.Background(), loginKey, 5*time.Minute)
}
if count > 5 {
http.Error(w, "too many login attempts, try again later", http.StatusTooManyRequests)
return
}
}
var payload AdminLoginPayload
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
http.Error(w, fmt.Sprintf("invalid request body: %v", err), http.StatusBadRequest)
@@ -907,9 +924,9 @@ func handleAdminModerationAction(w http.ResponseWriter, r *http.Request) {
func getAdminAnalytics() AdminAnalytics {
analytics := AdminAnalytics{
TopRooms: []RoomAnalytics{},
TopRooms: []RoomAnalytics{},
MediaTotalRuntime: "00:00:00",
GeneratedAt: time.Now().UTC().Format(time.RFC3339),
GeneratedAt: time.Now().UTC().Format(time.RFC3339),
}
if db == nil {
@@ -1114,7 +1131,7 @@ func handleAdminUsers(w http.ResponseWriter, r *http.Request) {
case http.MethodPost:
var payload struct {
Action string `json:"action"` // mute, unmute, ban, unban
Action string `json:"action"` // mute, unmute, ban, unban
Username string `json:"username"`
Duration int `json:"duration"` // minutes, for mute
}
@@ -1309,13 +1326,13 @@ func buildUploadStatus(session AdminMediaUploadSession) AdminMediaUploadStatusRe
}
return AdminMediaUploadStatusResponse{
UploadID: session.UploadID,
FileName: session.FileName,
FileSize: session.FileSize,
ChunkSize: session.ChunkSize,
TotalChunks: session.TotalChunks,
UploadID: session.UploadID,
FileName: session.FileName,
FileSize: session.FileSize,
ChunkSize: session.ChunkSize,
TotalChunks: session.TotalChunks,
UploadedChunks: uploadedChunks,
Progress: progress,
Progress: progress,
}
}
@@ -2165,6 +2182,25 @@ func broadcast(msg Message) {
// handleConnection handles WebSocket connections for chat (public rooms, private DMs, moderation)
func handleConnection(w http.ResponseWriter, r *http.Request) {
// IP-based connection rate limiting: max 5 concurrent per IP
clientIP := r.Header.Get("X-Forwarded-For")
if clientIP == "" {
clientIP = r.RemoteAddr
}
if redisClient != nil {
ipKey := fmt.Sprintf("ws_conn:%s", clientIP)
count, _ := redisClient.Incr(context.Background(), ipKey).Result()
if count == 1 {
redisClient.Expire(context.Background(), ipKey, 60*time.Second)
}
if count > 5 {
http.Error(w, "too many connections from this IP", http.StatusTooManyRequests)
return
}
// Decrement on disconnect
defer redisClient.Decr(context.Background(), ipKey)
}
conn, err := upgrader.Upgrade(w, r, nil)
if err != nil {
log.Println("Upgrade error:", err)
@@ -2649,4 +2685,4 @@ func main() {
log.Printf("Chat Backend starting on :%s (WebSocket at /ws, health at /health, database persistence enabled)", port)
log.Fatal(http.ListenAndServe(":"+port, nil))
}
}