diff --git a/API.md b/API.md new file mode 100644 index 0000000..0010c58 --- /dev/null +++ b/API.md @@ -0,0 +1,282 @@ +# API Reference + +This document describes the currently implemented API surfaces in Linear Void. + +## Base URLs + +- Public app and admin: +- WebSocket endpoint: +- Local fallback: and + +## Authentication Model + +Admin REST endpoints use a cookie session. + +- Cookie name: linearvoid_admin_session +- Cookie attributes: HttpOnly, SameSite=Strict, Secure when request is HTTPS (or X-Forwarded-Proto=https) +- Session TTL: controlled by ADMIN_SESSION_TTL_HOURS (default 12h) + +## Chat WebSocket API + +### Connect + +Endpoint: + +- GET /ws?username={name}&room={room} + +Client sends JSON messages over the socket. + +### Message Envelope (client -> server) + +```json +{ + "room": "main", + "username": "viewer123", + "text": "hello world", + "type": "public" +} +``` + +Notes: + +- type is usually public for room chat and private for DM room routing. +- Private room names are formatted like dm:userA-userB (sorted usernames). +- Moderation/system events are sent as type=system messages. + +### Message Envelope (server -> client) + +```json +{ + "room": "main", + "username": "viewer123", + "text": "hello world", + "type": "public", + "timestamp": "2026-08-04T12:34:56Z" +} +``` + +## Admin Session API + +### Login + +- POST /api/admin/login +- Auth: none + +Request: + +```json +{ + "username": "admin", + "password": "your-password" +} +``` + +Success response: + +```json +{ + "authenticated": true, + "username": "admin" +} +``` + +### Logout + +- POST /api/admin/logout +- Auth: optional (clears cookie either way) + +Response: + +```json +{ + "authenticated": false +} +``` + +### Session Check + +- GET /api/admin/session +- Auth: cookie required + +Success response: + +```json +{ + "authenticated": true, + "username": "admin" +} +``` + +Failure: + +- 401 unauthorized + +## Admin Data API + +All endpoints below require admin authentication. + +### Overview + +- GET /api/admin/overview + +Returns aggregate counters and runtime/system snapshot fields. + +### Playlist + +- GET /api/admin/playlist +- PUT /api/admin/playlist + +PUT request: + +```json +{ + "content": "file '/app/media/video1.mp4'\nfile '/app/media/video2.mp4'\n" +} +``` + +### Schedule + +- GET /api/admin/schedule + +Returns parsed playlist entries plus matched media metadata/runtime. + +### Moderation + +- GET /api/admin/moderation/logs?limit=50 +- POST /api/admin/moderation/actions + +Action request: + +```json +{ + "action": "mute", + "target": "username-or-id", + "moderator": "admin", + "reason": "spam", + "room": "main" +} +``` + +### Analytics + +- GET /api/admin/analytics + +Returns totals (messages, moderation actions, rooms) and top room summary. + +### System + +- GET /api/admin/system + +Returns service health snapshot for chat/playout/redis/database and config metadata. + +### Active Users + +- GET /api/admin/users + +Returns currently discovered presence users (room + username) when Redis is available. + +## Media Library API + +### Media Collection + +- GET /api/admin/media +- POST /api/admin/media (multipart upload, legacy path) + +Multipart form field name: file + +### Media Item + +- GET /api/admin/media/{id} +- PUT /api/admin/media/{id} +- DELETE /api/admin/media/{id} + +PUT request: + +```json +{ + "status": "ready", + "title": "My Clip", + "tags": "music,live" +} +``` + +## Resumable Upload API + +Maximum file size: 2GB + +Default chunk size: 8MB + +### Initialize or Resume Session + +- POST /api/admin/media/uploads + +Request: + +```json +{ + "uploadId": "optional-existing-id", + "fileName": "clip.mp4", + "fileSize": 123456789, + "mimeType": "video/mp4", + "chunkSize": 8388608, + "totalChunks": 15 +} +``` + +Response: + +```json +{ + "uploadId": "hex-token", + "fileName": "clip.mp4", + "fileSize": 123456789, + "chunkSize": 8388608, + "totalChunks": 15, + "uploadedChunks": 3, + "progress": 20 +} +``` + +### Check Upload Status + +- GET /api/admin/media/uploads/{uploadId} + +Returns same status envelope as initialize. + +### Upload Chunk + +- PUT /api/admin/media/uploads/{uploadId}/chunk?index={chunkIndex} +- Content-Type: application/octet-stream + +Response returns updated upload status envelope. + +### Complete Upload + +- POST /api/admin/media/uploads/{uploadId}/complete + +On success returns admin media listing payload (files + generatedAt). + +## Metrics and Health + +### Chat metrics + +- GET /metrics + +Prometheus exposition includes connected clients, stored messages, moderation actions, Redis indicators, active admin sessions, and build info. + +### Health + +- GET /health on chat service +- GET /health on playout service +- GET /health on nginx reverse proxy + +## HTTP Status Conventions + +- 200/201: successful request +- 400: malformed payload, invalid input, missing required fields +- 401: unauthorized admin request +- 404: not found +- 405: method not allowed +- 409: logical conflict (for example, mismatched resumed upload) +- 413: payload too large +- 500: internal server error diff --git a/CHANGELOG.md b/CHANGELOG.md index 9438796..05eeb5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - Added a dedicated plain TypeScript admin module for tab navigation, media CRUD actions, schedule/system analytics refresh, and moderation workflows without Alpine dependencies. - Added resumable chunked admin media upload flow with upload-session endpoints and client progress indicators. - Expanded architecture documentation with scaling strategy, API surface reference, detailed video normalization spec, and Cloudflare configuration reference. +- Added `API.md`, `DEPLOYMENT.md`, `MONITORING.md`, and `SECURITY.md` as dedicated operational documentation guides. ### Changed @@ -24,6 +25,7 @@ - Switched the frontend theme to dark-mode-first token defaults with a light-scheme override path for users who prefer light appearance. - Refactored the viewer page to a single header plus unified content layout with responsive video/chat split, mobile stacking, and chat panel toggle behavior. - Code-split `hls.js` via lazy loading in the viewer bootstrap to shrink the main frontend bundle and isolate stream playback logic into a dedicated chunk. +- Added navigation links in root README and architecture docs to the new API, deployment, monitoring, and security guide files. ### Fixed diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md new file mode 100644 index 0000000..b8089ae --- /dev/null +++ b/DEPLOYMENT.md @@ -0,0 +1,169 @@ +# Production Deployment Guide + +This guide describes a production-style deployment for Linear Void using Docker Compose, Gitea registry images, and Cloudflare fronting. + +## Target Topology + +- Host: Linux server with Docker Engine +- Reverse proxy: nginx container (linearvoid-nginx) +- Core services: playout, chat, redis, postgres +- Observability: prometheus, grafana +- CDN/edge: Cloudflare + +## Prerequisites + +- Docker Engine + Docker Compose plugin +- Network access to git.allucanget.biz +- A valid .env file with production values +- DNS configured for linearvoid.tv (and media.linearvoid.tv if split media domain is used) + +## Environment File + +Use .env with at least: + +- CI_REGISTRY_URL +- CI_REGISTRY_NAMESPACE +- CHAT_PORT +- NGINX_PORT +- DATABASE_URL (if overridden) +- REDIS_URL (if overridden) +- ADMIN_USERNAME +- ADMIN_PASSWORD +- VITE_PUBLIC_APP_URL +- VITE_PUBLIC_WS_URL +- VITE_PUBLIC_HLS_URL +- Cloudflare/R2 variables when media sync is enabled + +See .env.example and SETUP.md for full variable reference. + +## Pull-and-Run Deployment + +From repository root: + +```bash +docker login git.allucanget.biz + +docker compose -f docker-compose.prod.yml pull +docker compose -f docker-compose.prod.yml up -d --remove-orphans +``` + +## First-Time or Recovery Provisioning + +1. Ensure host paths exist: + +```bash +mkdir -p data/chat data/playlist data/media data/hls +``` + +1. Confirm volumes and networks: + +```bash +docker volume ls +docker network ls +``` + +1. Start stack: + +```bash +docker compose -f docker-compose.prod.yml up -d +``` + +## Health Verification + +After startup, check service status: + +```bash +docker compose -f docker-compose.prod.yml ps +``` + +Endpoint checks: + +- App: +- Nginx health: +- Chat health: +- Playout health: +- Prometheus: +- Grafana: + +Domain checks: + +- +- +- + +## Zero/Low-Downtime Update Pattern + +1. Pull newest images. +1. Recreate changed services only: + +```bash +docker compose -f docker-compose.prod.yml pull + +docker compose -f docker-compose.prod.yml up -d nginx chat playout +``` + +1. Re-run health checks. + +## Rollback + +If new images are unhealthy: + +1. Pin previous tags in docker-compose.prod.yml. +2. Pull pinned tags. +3. Restart impacted services. + +```bash +docker compose -f docker-compose.prod.yml pull + +docker compose -f docker-compose.prod.yml up -d nginx chat playout +``` + +## Operational Tasks + +### View logs + +```bash +docker compose -f docker-compose.prod.yml logs -f nginx chat playout +``` + +### Restart one service + +```bash +docker compose -f docker-compose.prod.yml restart chat +``` + +### Stop stack + +```bash +docker compose -f docker-compose.prod.yml down +``` + +### Keep volumes (recommended) vs remove volumes (destructive) + +```bash +# keep data +docker compose -f docker-compose.prod.yml down + +# destructive (remove data) +docker compose -f docker-compose.prod.yml down -v +``` + +## Nginx and Proxy Considerations + +- If an external proxy (for example Nginx Proxy Manager) fronts the stack, align request body limits with internal nginx. +- For resumable admin uploads, ensure client_max_body_size supports chunk requests in both proxy layers. + +## Security Baseline + +- Use strong ADMIN_PASSWORD and rotate regularly. +- Keep Docker host access restricted. +- Prefer HTTPS/WSS through Cloudflare. +- Restrict exposed ports if only proxy access is needed. + +## Common Deployment Pitfalls + +- Missing frontend build artifacts in local-only compose workflows. +- Wrong registry credentials or missing login. +- Misaligned domain env vars (app/ws/hls) causing mixed-origin errors. +- Proxy-layer body size limits returning 413 on upload. +- Health checks failing before service warmup is complete. diff --git a/MONITORING.md b/MONITORING.md new file mode 100644 index 0000000..b999879 --- /dev/null +++ b/MONITORING.md @@ -0,0 +1,137 @@ +# Monitoring Guide + +This guide covers Prometheus and Grafana setup for Linear Void. + +## Current Monitoring Components + +- Prometheus container: linearvoid-prometheus +- Grafana container: linearvoid-grafana +- Metrics target: chat service /metrics +- Base scrape config: monitoring/prometheus/prometheus.yml + +## Prometheus Configuration + +Current scrape config: + +- Job chat -> chat:3000 +- Job prometheus -> prometheus:9090 +- Global scrape interval: 15s + +File: + +- monitoring/prometheus/prometheus.yml + +## Start Monitoring Stack + +With compose stack up: + +- Prometheus UI: +- Grafana UI: + +Compose services include prometheus and grafana in both docker-compose.yml and docker-compose.prod.yml. + +## Grafana Provisioning + +Current mount paths: + +- Dashboards directory: monitoring/grafana/dashboards +- Datasources directory: monitoring/grafana/datasources + +Notes: + +- The dashboards directory includes a Prometheus overview dashboard file. +- Datasource provisioning can be done with a YAML file in monitoring/grafana/datasources, or by configuring a datasource in Grafana UI. + +Example datasource provisioning file (create if needed): + +```yaml +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true +``` + +## Dashboard References + +- Provisioned dashboard: [monitoring/grafana/dashboards/dashboard.json](monitoring/grafana/dashboards/dashboard.json) +- Root dashboard snapshot: [monitoring/grafana/dashboard.json](monitoring/grafana/dashboard.json) + +Grafana screenshot: + +![Grafana dashboard screenshot](docs/img/grafana-screenshot.png) + +## Key Metrics to Track + +### Chat/Application + +- linearvoid_chat_connected_clients +- linearvoid_chat_stored_messages_total +- linearvoid_chat_moderation_actions_total +- linearvoid_chat_redis_enabled +- linearvoid_chat_redis_up +- linearvoid_chat_active_admin_sessions + +### Prometheus Runtime + +- up +- process_cpu_seconds_total +- process_resident_memory_bytes +- prometheus_tsdb_head_series +- prometheus_tsdb_wal_writes_failed_total +- prometheus_tsdb_compactions_failed_total + +## Suggested Alerts + +Create a Prometheus rules file (for example monitoring/prometheus/alerts.yml) and wire it into Prometheus. + +Recommended baseline alerts: + +- Chat target down for 2m +- Prometheus target down for 2m +- Redis indicator down for 2m +- TSDB WAL/compaction failure rate above zero +- Elevated handler/query latency p99 + +## Validate Monitoring End-to-End + +1. Verify /metrics through nginx: + +- + +1. Confirm targets in Prometheus: + +- Status -> Targets page in Prometheus UI + +1. Confirm dashboard data in Grafana: + +- Import or open provisioned dashboards + +## Troubleshooting + +### No chat metrics in Prometheus + +- Check chat container health and logs. +- Confirm /metrics is reachable from Prometheus container. +- Verify scrape target is chat:3000 in prometheus.yml. + +### Grafana shows no data + +- Ensure datasource points to . +- Confirm Prometheus is healthy. +- Check dashboard query labels match active jobs. + +### High cardinality or query slowness + +- Reduce unneeded labels. +- Limit expensive dashboard panels. +- Tune retention and query windows. + +## Operational Recommendations + +- Keep alert thresholds tied to actual user impact. +- Review dashboards after each release that changes endpoints or metrics names. +- Periodically test alert routing and incident response flow. diff --git a/README.md b/README.md index 7a9995a..4f9b4e7 100644 --- a/README.md +++ b/README.md @@ -204,6 +204,10 @@ Contributions, issues, and further evolution (LL-HLS, ABR, multi-channel, E2EE) - [CHANGELOG.md](CHANGELOG.md) - Version history and completed features. - [CONTRIBUTING.md](CONTRIBUTING.md) - Guidelines for contributing to the project. - [SETUP.md](SETUP.md) - Complete setup guide with prerequisites, environment configuration, running the application, and troubleshooting. +- [API.md](API.md) - Implemented WebSocket and REST API reference. +- [DEPLOYMENT.md](DEPLOYMENT.md) - Production deployment and operations guide. +- [MONITORING.md](MONITORING.md) - Prometheus and Grafana setup and troubleshooting. +- [SECURITY.md](SECURITY.md) - Security controls, hardening checklist, and incident baseline. - [Architecture Documentation](docs/architecture/README.md) - Full arc42 template with diagrams and decisions. - [ADRs](docs/architecture/adr/) - Detailed decision records (ADR-001 to ADR-007). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..577008c --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,127 @@ +# Security Guide + +This document summarizes security controls and best practices for Linear Void. + +## Security Model Overview + +Security is layered across: + +- Cloudflare edge +- nginx reverse proxy +- chat/admin backend +- container and host operations + +## Authentication and Sessions + +Admin access is session-cookie based. + +- Login endpoint: /api/admin/login +- Session endpoint: /api/admin/session +- Logout endpoint: /api/admin/logout +- Cookie: linearvoid_admin_session +- Cookie flags: HttpOnly, SameSite=Strict, Secure over HTTPS +- Session expiry enforced server-side + +Best practices: + +- Set strong ADMIN_USERNAME and ADMIN_PASSWORD values. +- Rotate admin credentials periodically. +- Keep ADMIN_SESSION_TTL_HOURS aligned with operational policy. + +## Transport Security + +- Enforce HTTPS for app/admin endpoints. +- Use WSS for chat WebSocket. +- Keep Cloudflare TLS mode strict and valid at origin. + +## Reverse Proxy Controls + +nginx controls include: + +- auth_request guard for /admin and /api/admin paths +- dedicated unauthorized behavior for admin API responses +- explicit websocket upgrade headers for /ws +- upload body-size limits for resumable media chunk uploads + +Best practices: + +- Keep proxy body-size and timeout settings aligned across all proxy layers. +- Avoid exposing backend service ports publicly unless required. + +## Input Validation and Upload Safety + +Server-side protections include: + +- extension allow-list for media uploads +- MIME checks +- file-size ceiling (2GB) +- resumable chunk bounds and chunk-index validation +- media path safety checks on delete + +Best practices: + +- Keep upload chunk size bounded. +- Reject unsupported content types consistently. +- Monitor 4xx/5xx rates on upload routes. + +## Rate Limiting and Abuse Controls + +- Chat rate limiting uses Redis INCR + EXPIRE windows. +- Current baseline: more than 10 messages in 10 seconds is throttled. +- Moderation actions are logged for auditability. + +Best practices: + +- Tune thresholds by observed abuse patterns. +- Keep moderation logs reviewed and retained per policy. + +## Infrastructure and Secrets + +- Store secrets in .env (not in committed source). +- Use Gitea Actions secrets for CI/CD credentials and admin credentials. +- Restrict Docker host control plane exposure. +- Limit who can read deployment environment files. + +## Cloudflare and Edge Security + +Recommended baseline: + +- WAF enabled +- Rate limits on sensitive paths +- Bot protection where appropriate +- Cache rules scoped carefully to avoid caching private/control-plane content + +## Data Protection and Retention + +- Persistent data: postgres, redis append-only, chat state, playlist/media metadata +- Backup and restore drills should be periodic +- Restore procedures should be tested in isolated environments + +## Logging and Monitoring for Security + +Track and alert on: + +- repeated failed admin login attempts +- spikes in 401 and 403 responses +- sudden upload errors (413/400/500 patterns) +- unusual websocket disconnect or message-rate anomalies + +## Hardening Checklist + +- [ ] HTTPS/WSS enforced end-to-end +- [ ] Strong admin credentials configured +- [ ] Secrets only in env/secrets stores +- [ ] Proxy auth and websocket headers validated +- [ ] Upload limits and validation active +- [ ] Rate limiting active and tested +- [ ] Cloudflare WAF/rate-limit rules active +- [ ] Backup/restore drills completed +- [ ] Security-related monitoring alerts routed + +## Incident Response Baseline + +1. Detect issue (alerts/logs/user reports). +2. Triage scope (edge, proxy, app, data plane). +3. Contain impact (rate limit, credential rotate, route disable). +4. Recover services and verify health. +5. Record timeline and corrective actions. diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 99d4785..848456a 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -5,6 +5,7 @@ ## Table of Contents - [Introduction and Goals](#introduction-and-goals) +- [Companion Documentation](#companion-documentation) - [Architecture Constraints](#architecture-constraints) - [Context and Scope](#context-and-scope) - [Solution Strategy](#solution-strategy) @@ -17,6 +18,13 @@ - [Risks and Technical Debts](#risks-and-technical-debts) - [Glossary](#glossary) +## Companion Documentation + +- [API.md](../../API.md) - Implemented WebSocket and REST API reference. +- [DEPLOYMENT.md](../../DEPLOYMENT.md) - Production deployment and operational runbook. +- [MONITORING.md](../../MONITORING.md) - Metrics, dashboards, and monitoring troubleshooting. +- [SECURITY.md](../../SECURITY.md) - Security model, controls, and hardening baseline. + ## Introduction and Goals ### Requirements Overview diff --git a/docs/img/grafana-screenshot.png b/docs/img/grafana-screenshot.png new file mode 100644 index 0000000..cc10f84 Binary files /dev/null and b/docs/img/grafana-screenshot.png differ diff --git a/monitoring/grafana/dashboards/dashboard.json b/monitoring/grafana/dashboards/dashboard.json new file mode 100644 index 0000000..06b0707 --- /dev/null +++ b/monitoring/grafana/dashboards/dashboard.json @@ -0,0 +1,1629 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Prometheus server health, TSDB, query, HTTP and Go runtime metrics.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "text": "DOWN" + }, + "1": { + "color": "green", + "text": "UP" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "expr": "min(up{job=~\"$job\",instance=~\"$instance\"})", + "instant": true, + "legendFormat": "Prometheus", + "refId": "A" + } + ], + "title": "Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "expr": "max(time() - process_start_time_seconds{job=~\"$job\",instance=~\"$instance\"})", + "instant": true, + "legendFormat": "Uptime", + "refId": "A" + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "name" + }, + "targets": [ + { + "expr": "max by (version) (prometheus_build_info{job=~\"$job\",instance=~\"$instance\"})", + "instant": true, + "legendFormat": "Prometheus {{version}}", + "refId": "A" + } + ], + "title": "Version", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "text": "FAILED" + }, + "1": { + "color": "green", + "text": "SUCCESS" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "expr": "min(prometheus_config_last_reload_successful{job=~\"$job\",instance=~\"$instance\"})", + "instant": true, + "legendFormat": "Config reload", + "refId": "A" + } + ], + "title": "Configuration", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "expr": "sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\"}[5m]))", + "instant": true, + "legendFormat": "CPU", + "refId": "A" + } + ], + "title": "Process CPU", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1073741824 + }, + { + "color": "red", + "value": 2147483648 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "expr": "sum(process_resident_memory_bytes{job=~\"$job\",instance=~\"$instance\"})", + "instant": true, + "legendFormat": "RSS", + "refId": "A" + } + ], + "title": "Resident Memory", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 500000 + }, + { + "color": "red", + "value": 1000000 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "expr": "sum(prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"})", + "instant": true, + "legendFormat": "Series", + "refId": "A" + } + ], + "title": "Head Series", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "expr": "sum(rate(prometheus_tsdb_head_samples_appended_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "instant": true, + "legendFormat": "Samples/sec", + "refId": "A" + } + ], + "title": "Ingestion Rate", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [], + "title": "Process and Runtime", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 6 + }, + "id": 11, + "options": { + "legend": { + "calcs": ["lastNotNull", "max", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Process CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 6 + }, + "id": 12, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "RSS {{instance}}", + "refId": "A" + }, + { + "expr": "go_memstats_heap_alloc_bytes{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Heap allocated {{instance}}", + "refId": "B" + }, + { + "expr": "go_memstats_heap_sys_bytes{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Heap system {{instance}}", + "refId": "C" + }, + { + "expr": "go_gc_heap_goal_bytes{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "GC heap goal {{instance}}", + "refId": "D" + } + ], + "title": "Process and Heap Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 13, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "go_goroutines{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Goroutines {{instance}}", + "refId": "A" + }, + { + "expr": "go_threads{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "OS threads {{instance}}", + "refId": "B" + }, + { + "expr": "process_open_fds{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Open FDs {{instance}}", + "refId": "C" + }, + { + "expr": "go_sched_goroutines_waiting_goroutines{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Waiting goroutines {{instance}}", + "refId": "D" + } + ], + "title": "Goroutines, Threads and File Descriptors", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 14, + "panels": [], + "title": "TSDB and Ingestion", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 15, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 15, + "options": { + "legend": { + "calcs": ["lastNotNull", "max", "mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (instance,type) (rate(prometheus_tsdb_head_samples_appended_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{instance}} {{type}}", + "refId": "A" + } + ], + "title": "Samples Appended per Second", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 15 + }, + "id": 16, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Series {{instance}}", + "refId": "A" + }, + { + "expr": "prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Chunks {{instance}}", + "refId": "B" + }, + { + "expr": "prometheus_tsdb_blocks_loaded{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Blocks {{instance}}", + "refId": "C" + }, + { + "expr": "prometheus_tsdb_head_stale_series{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Stale series {{instance}}", + "refId": "D" + } + ], + "title": "TSDB Head", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 15, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 15 + }, + "id": 17, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "prometheus_tsdb_storage_blocks_bytes{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Blocks {{instance}}", + "refId": "A" + }, + { + "expr": "prometheus_tsdb_wal_storage_size_bytes{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "WAL {{instance}}", + "refId": "B" + }, + { + "expr": "prometheus_tsdb_head_chunks_storage_size_bytes{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Head chunks {{instance}}", + "refId": "C" + } + ], + "title": "TSDB Storage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 18, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (instance) (rate(prometheus_tsdb_head_series_created_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "Created {{instance}}", + "refId": "A" + }, + { + "expr": "sum by (instance) (rate(prometheus_tsdb_head_series_removed_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "Removed {{instance}}", + "refId": "B" + }, + { + "expr": "sum by (instance) (rate(prometheus_tsdb_head_chunks_created_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "Chunks created {{instance}}", + "refId": "C" + } + ], + "title": "Series and Chunk Churn", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 20, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 19, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (reason) (rate(prometheus_tsdb_wal_writes_failed_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "WAL write failures", + "refId": "A" + }, + { + "expr": "sum(rate(prometheus_tsdb_wal_corruptions_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "WAL corruptions", + "refId": "B" + }, + { + "expr": "sum(rate(prometheus_tsdb_compactions_failed_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "Compaction failures", + "refId": "C" + }, + { + "expr": "sum(rate(prometheus_tsdb_head_truncations_failed_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "Truncation failures", + "refId": "D" + }, + { + "expr": "sum(rate(prometheus_tsdb_out_of_order_samples_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "Out-of-order samples", + "refId": "E" + } + ], + "title": "TSDB and WAL Errors", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 20, + "panels": [], + "title": "HTTP and Queries", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 20, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 21, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (code) (rate(prometheus_http_requests_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "HTTP {{code}}", + "refId": "A" + } + ], + "title": "HTTP Requests by Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 22, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "topk(10, histogram_quantile(0.99, sum by (le,handler) (rate(prometheus_http_request_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))))", + "legendFormat": "p99 {{handler}}", + "refId": "A" + } + ], + "title": "HTTP Handler p99 Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 23, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (le,slice) (rate(prometheus_engine_query_duration_histogram_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "p99 {{slice}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.50, sum by (le,slice) (rate(prometheus_engine_query_duration_histogram_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "p50 {{slice}}", + "refId": "B" + } + ], + "title": "PromQL Query Execution Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 24, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "prometheus_engine_queries{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Active queries {{instance}}", + "refId": "A" + }, + { + "expr": "prometheus_engine_queries_concurrent_max{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "Concurrency limit {{instance}}", + "refId": "B" + }, + { + "expr": "rate(prometheus_engine_query_samples_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "Samples loaded/sec {{instance}}", + "refId": "C" + }, + { + "expr": "rate(prometheus_engine_query_samples_read_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "Samples read/sec {{instance}}", + "refId": "D" + } + ], + "title": "Query Load", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 25, + "panels": [], + "title": "Garbage Collection and Scheduler", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 26, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (le,instance) (rate(go_sched_pauses_total_gc_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "GC pause p99 {{instance}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.50, sum by (le,instance) (rate(go_sched_pauses_total_gc_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "GC pause p50 {{instance}}", + "refId": "B" + }, + { + "expr": "rate(go_gc_duration_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]) / clamp_min(rate(go_gc_duration_seconds_count{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]), 0.000001)", + "legendFormat": "Average GC pause {{instance}}", + "refId": "C" + } + ], + "title": "GC Pause Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 27, + "options": { + "legend": { + "calcs": ["lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (le,instance) (rate(go_sched_latencies_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "Scheduler p99 {{instance}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.90, sum by (le,instance) (rate(go_sched_latencies_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "Scheduler p90 {{instance}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.50, sum by (le,instance) (rate(go_sched_latencies_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "Scheduler p50 {{instance}}", + "refId": "C" + } + ], + "title": "Go Scheduler Latency", + "type": "timeseries" + } + ], + "refresh": "15s", + "schemaVersion": 41, + "tags": ["prometheus", "linearvoid", "go", "tsdb"], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Prometheus datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": "linearvoid-metrics", + "value": "linearvoid-metrics" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(prometheus_build_info, job)", + "hide": 0, + "includeAll": false, + "label": "Job", + "multi": true, + "name": "job", + "options": [], + "query": { + "query": "label_values(prometheus_build_info, job)", + "refId": "PrometheusVariableQueryEditor-Job" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(up{job=~\"$job\"}, instance)", + "hide": 0, + "includeAll": true, + "allValue": ".*", + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "query": "label_values(up{job=~\"$job\"}, instance)", + "refId": "PrometheusVariableQueryEditor-Instance" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "15s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h" + ] + }, + "timezone": "browser", + "title": "Linearvoid Prometheus Overview", + "uid": "linearvoid-prometheus", + "version": 1, + "weekStart": "" +}