feat: add metrics, observability, test fixtures, and scaling playbook artifacts

scripts/loadtest/README.md

@@ -0,0 +1,90 @@
+# Load Test Scaffolding
+
+These files back the scaling/performance playbook.
+
+## WebSocket chat
+
+Run with an explicit WebSocket URL, for example:
+
+```bash
+WS_URL=wss://linearvoid.tv/ws?username=loadtest&room=main npx artillery run scripts/loadtest/ws-chat.yml
+```
+
+Capture output to an artifact file for the worksheet:
+
+```bash
+WS_URL=wss://linearvoid.tv/ws?username=loadtest&room=main npx artillery run scripts/loadtest/ws-chat.yml --output artifacts/ws-chat-report.json
+```
+
+PowerShell equivalent:
+
+```powershell
+$env:WS_URL = 'wss://linearvoid.tv/ws?username=loadtest&room=main'
+New-Item -ItemType Directory -Force artifacts | Out-Null
+npx artillery run scripts/loadtest/ws-chat.yml --output artifacts/ws-chat-report.json
+```
+
+Summarize key counters, rates, and available percentile stats from a saved Artillery JSON report:
+
+```powershell
+pwsh -File scripts/loadtest/summarize-artillery.ps1 -ReportFile artifacts/ws-chat-report.json
+```
+
+## Admin API
+
+Run with a cookie or session header that can access admin endpoints:
+
+```bash
+ADMIN_BASE_URL=https://linearvoid.tv ADMIN_SESSION_COOKIE="session=..." k6 run scripts/loadtest/admin-api.js
+```
+
+Capture machine-readable metrics and a text summary:
+
+```bash
+ADMIN_BASE_URL=https://linearvoid.tv ADMIN_SESSION_COOKIE="session=..." k6 run --summary-export artifacts/admin-api-summary.json scripts/loadtest/admin-api.js > artifacts/admin-api-output.txt
+```
+
+PowerShell equivalent:
+
+```powershell
+$env:ADMIN_BASE_URL = 'https://linearvoid.tv'
+$env:ADMIN_SESSION_COOKIE = 'session=...'
+New-Item -ItemType Directory -Force artifacts | Out-Null
+k6 run --summary-export artifacts/admin-api-summary.json scripts/loadtest/admin-api.js | Tee-Object -FilePath artifacts/admin-api-output.txt
+```
+
+The script emits global and per-endpoint thresholds (overview, analytics, chat-log, users) so p95 and error-rate regressions are visible per route.
+
+Summarize endpoint p95 and failure rates from a saved k6 JSON summary:
+
+```powershell
+pwsh -File scripts/loadtest/summarize-k6.ps1 -SummaryFile artifacts/admin-api-summary.json
+```
+
+## PowerShell helper
+
+Use the helper to run one scenario or both and automatically write outputs into a dated artifact folder:
+
+- The helper resolves scenario paths from its own script location, so it can be run from any current working directory.
+- WebSocket runs also emit `ws-chat-summary.txt` with connection/error counters and available summary percentiles.
+- Admin runs also emit `admin-api-endpoint-summary.txt` with per-endpoint p95/error values.
+
+```powershell
+pwsh -File scripts/loadtest/run-loadtests.ps1 -Scenario ws -WsUrl 'wss://linearvoid.tv/ws?username=loadtest&room=main'
+```
+
+```powershell
+pwsh -File scripts/loadtest/run-loadtests.ps1 -Scenario admin -AdminBaseUrl 'https://linearvoid.tv' -AdminSessionCookie 'session=...'
+```
+
+```powershell
+pwsh -File scripts/loadtest/run-loadtests.ps1 -Scenario all -WsUrl 'wss://linearvoid.tv/ws?username=loadtest&room=main' -AdminBaseUrl 'https://linearvoid.tv' -AdminSessionCookie 'session=...'
+```
+
+## Notes
+
+- Adjust target URLs for test lab or production.
+- Keep the same commit SHA or image tag when comparing runs.
+- Record results using the worksheet in `docs/operations/scaling-performance-playbook.md`.
+- Store raw outputs under an `artifacts/` folder per run date (for example `artifacts/2026-08-09/`).
+- The repository `.gitignore` excludes benchmark artifacts by default.

scripts/loadtest/admin-api.js

@@ -0,0 +1,58 @@
+import http from "k6/http";
+import { check, sleep } from "k6";
+
+const baseUrl = __ENV.ADMIN_BASE_URL || "https://linearvoid.tv";
+const adminCookie = __ENV.ADMIN_SESSION_COOKIE || "";
+const requestParams = adminCookie ? { headers: { Cookie: adminCookie } } : {};
+
+const endpoints = [
+  { name: "overview", path: "/api/admin/overview" },
+  { name: "analytics", path: "/api/admin/analytics" },
+  { name: "chat_log", path: "/api/admin/chat-log" },
+  { name: "users", path: "/api/admin/users" },
+];
+
+export const options = {
+  stages: [
+    { duration: "30s", target: 10 },
+    { duration: "60s", target: 50 },
+    { duration: "60s", target: 100 },
+    { duration: "30s", target: 0 },
+  ],
+  thresholds: {
+    http_req_duration: ["p(95)<1000"],
+    http_req_failed: ["rate<0.01"],
+    "http_req_duration{endpoint:overview}": ["p(95)<1000"],
+    "http_req_duration{endpoint:analytics}": ["p(95)<1000"],
+    "http_req_duration{endpoint:chat_log}": ["p(95)<1000"],
+    "http_req_duration{endpoint:users}": ["p(95)<1000"],
+    "http_req_failed{endpoint:overview}": ["rate<0.01"],
+    "http_req_failed{endpoint:analytics}": ["rate<0.01"],
+    "http_req_failed{endpoint:chat_log}": ["rate<0.01"],
+    "http_req_failed{endpoint:users}": ["rate<0.01"],
+  },
+};
+
+export default function () {
+  const requests = endpoints.map((endpoint) => [
+    "GET",
+    `${baseUrl}${endpoint.path}`,
+    null,
+    {
+      ...requestParams,
+      tags: {
+        endpoint: endpoint.name,
+      },
+    },
+  ]);
+  const responses = http.batch(requests);
+
+  responses.forEach((response, index) => {
+    const endpointPath = endpoints[index].path;
+    check(response, {
+      [`status is 200 for ${endpointPath}`]: (r) => r.status === 200,
+    });
+  });
+
+  sleep(1);
+}

scripts/loadtest/run-loadtests.ps1

@@ -0,0 +1,99 @@
+#!/usr/bin/env pwsh
+param(
+  [ValidateSet('ws', 'admin', 'all')]
+  [string]$Scenario = 'all',
+  [string]$ArtifactRoot = 'artifacts',
+  [string]$WsUrl = $env:WS_URL,
+  [string]$AdminBaseUrl = $env:ADMIN_BASE_URL,
+  [string]$AdminSessionCookie = $env:ADMIN_SESSION_COOKIE
+)
+
+$ErrorActionPreference = 'Stop'
+
+$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
+$repoRoot = Resolve-Path (Join-Path $scriptDir '..\..')
+$wsScenarioPath = Join-Path $scriptDir 'ws-chat.yml'
+$wsSummaryScriptPath = Join-Path $scriptDir 'summarize-artillery.ps1'
+$adminScriptPath = Join-Path $scriptDir 'admin-api.js'
+$adminSummaryScriptPath = Join-Path $scriptDir 'summarize-k6.ps1'
+
+if ([System.IO.Path]::IsPathRooted($ArtifactRoot)) {
+  $artifactBase = $ArtifactRoot
+} else {
+  $artifactBase = Join-Path $repoRoot $ArtifactRoot
+}
+
+if ($Scenario -in @('ws', 'all')) {
+  if (-not (Get-Command npx -ErrorAction SilentlyContinue)) {
+    throw 'npx is required for ws scenario but was not found in PATH.'
+  }
+}
+
+if ($Scenario -in @('admin', 'all')) {
+  if (-not (Get-Command k6 -ErrorAction SilentlyContinue)) {
+    throw 'k6 is required for admin scenario but was not found in PATH.'
+  }
+}
+
+$timestamp = Get-Date -Format 'yyyy-MM-dd_HHmmss'
+$artifactDir = Join-Path $artifactBase $timestamp
+New-Item -ItemType Directory -Force -Path $artifactDir | Out-Null
+
+function Invoke-WebSocketLoadTest {
+  param(
+    [string]$TargetWsUrl,
+    [string]$OutputPath
+  )
+
+  if ([string]::IsNullOrWhiteSpace($TargetWsUrl)) {
+    throw 'WS_URL is required for ws scenario. Pass -WsUrl or set $env:WS_URL.'
+  }
+
+  $env:WS_URL = $TargetWsUrl
+  & npx artillery run $wsScenarioPath --output $OutputPath
+}
+
+function Invoke-AdminApiLoadTest {
+  param(
+    [string]$BaseUrl,
+    [string]$SessionCookie,
+    [string]$SummaryPath,
+    [string]$OutputPath
+  )
+
+  if ([string]::IsNullOrWhiteSpace($BaseUrl)) {
+    throw 'ADMIN_BASE_URL is required for admin scenario. Pass -AdminBaseUrl or set $env:ADMIN_BASE_URL.'
+  }
+  if ([string]::IsNullOrWhiteSpace($SessionCookie)) {
+    throw 'ADMIN_SESSION_COOKIE is required for admin scenario. Pass -AdminSessionCookie or set $env:ADMIN_SESSION_COOKIE.'
+  }
+
+  $env:ADMIN_BASE_URL = $BaseUrl
+  $env:ADMIN_SESSION_COOKIE = $SessionCookie
+  & k6 run --summary-export $SummaryPath $adminScriptPath | Tee-Object -FilePath $OutputPath
+}
+
+if ($Scenario -in @('ws', 'all')) {
+  if (-not (Test-Path $wsSummaryScriptPath)) {
+    throw "Missing summary helper: $wsSummaryScriptPath"
+  }
+
+  $wsOutput = Join-Path $artifactDir 'ws-chat-report.json'
+  $wsSummary = Join-Path $artifactDir 'ws-chat-summary.txt'
+  Invoke-WebSocketLoadTest -TargetWsUrl $WsUrl -OutputPath $wsOutput
+  & pwsh -File $wsSummaryScriptPath -ReportFile $wsOutput | Tee-Object -FilePath $wsSummary
+}
+
+if ($Scenario -in @('admin', 'all')) {
+  if (-not (Test-Path $adminSummaryScriptPath)) {
+    throw "Missing summary helper: $adminSummaryScriptPath"
+  }
+
+  $adminSummary = Join-Path $artifactDir 'admin-api-summary.json'
+  $adminOutput = Join-Path $artifactDir 'admin-api-output.txt'
+  $adminEndpointSummary = Join-Path $artifactDir 'admin-api-endpoint-summary.txt'
+  Invoke-AdminApiLoadTest -BaseUrl $AdminBaseUrl -SessionCookie $AdminSessionCookie -SummaryPath $adminSummary -OutputPath $adminOutput
+  & pwsh -File $adminSummaryScriptPath -SummaryFile $adminSummary | Tee-Object -FilePath $adminEndpointSummary
+}
+
+Write-Host "Artifacts written to $artifactDir"

scripts/loadtest/summarize-artillery.ps1

@@ -0,0 +1,115 @@
+#!/usr/bin/env pwsh
+param(
+  [Parameter(Mandatory = $true)]
+  [string]$ReportFile
+)
+
+$ErrorActionPreference = 'Stop'
+
+if (-not (Test-Path $ReportFile)) {
+  throw "Report file not found: $ReportFile"
+}
+
+$report = Get-Content -Raw -Path $ReportFile | ConvertFrom-Json -Depth 100
+$aggregate = $report.aggregate
+
+if (-not $aggregate) {
+  throw 'Artillery report is missing aggregate data.'
+}
+
+function Get-PropertyValue {
+  param(
+    [Parameter(Mandatory = $true)]
+    [object]$Object,
+    [Parameter(Mandatory = $true)]
+    [string]$Name
+  )
+
+  if (-not $Object) {
+    return $null
+  }
+
+  $property = $Object.PSObject.Properties[$Name]
+  if ($property) {
+    return $property.Value
+  }
+
+  return $null
+}
+
+function Format-StatLine {
+  param(
+    [string]$Prefix,
+    [object]$Stats
+  )
+
+  if (-not $Stats) {
+    return $null
+  }
+
+  $parts = @()
+  foreach ($name in @('count', 'min', 'max', 'mean', 'median', 'p95', 'p99')) {
+    $value = Get-PropertyValue -Object $Stats -Name $name
+    if ($null -ne $value) {
+      $parts += "$name=$value"
+    }
+  }
+
+  if ($parts.Count -eq 0) {
+    return $null
+  }
+
+  return "$Prefix: $($parts -join ', ')"
+}
+
+$counters = Get-PropertyValue -Object $aggregate -Name 'counters'
+$rates = Get-PropertyValue -Object $aggregate -Name 'rates'
+$summaries = Get-PropertyValue -Object $aggregate -Name 'summaries'
+
+$created = [double](Get-PropertyValue -Object $counters -Name 'vusers.created' ?? 0)
+$failed = [double](Get-PropertyValue -Object $counters -Name 'vusers.failed' ?? 0)
+$completed = [double](Get-PropertyValue -Object $counters -Name 'vusers.completed' ?? 0)
+$errorRate = if ($created -gt 0) { [math]::Round(($failed / $created) * 100, 2) } else { 0 }
+
+Write-Output '=== Artillery WebSocket Summary ==='
+Write-Output "source: $ReportFile"
+
+$timestamp = Get-PropertyValue -Object $report -Name 'timestamp'
+if ($timestamp) {
+  Write-Output "timestamp: $timestamp"
+}
+
+if ($null -ne $created) {
+  Write-Output "vusers.created: $created"
+}
+if ($null -ne $completed) {
+  Write-Output "vusers.completed: $completed"
+}
+if ($null -ne $failed) {
+  Write-Output "vusers.failed: $failed"
+}
+Write-Output "vusers.failure_rate_percent: $errorRate"
+
+if ($counters) {
+  foreach ($metric in @('ws.messages_sent', 'ws.messages_received', 'ws.ping', 'plugins.metrics-by-endpoint.response_time')) {
+    $value = Get-PropertyValue -Object $counters -Name $metric
+    if ($null -ne $value) {
+      Write-Output "$metric: $value"
+    }
+  }
+}
+
+if ($rates) {
+  foreach ($property in $rates.PSObject.Properties) {
+    Write-Output "rate.$($property.Name): $($property.Value)"
+  }
+}
+
+if ($summaries) {
+  foreach ($property in $summaries.PSObject.Properties) {
+    $line = Format-StatLine -Prefix "summary.$($property.Name)" -Stats $property.Value
+    if ($line) {
+      Write-Output $line
+    }
+  }
+}

scripts/loadtest/summarize-k6.ps1

@@ -0,0 +1,68 @@
+#!/usr/bin/env pwsh
+param(
+  [Parameter(Mandatory = $true)]
+  [string]$SummaryFile
+)
+
+$ErrorActionPreference = 'Stop'
+
+if (-not (Test-Path $SummaryFile)) {
+  throw "Summary file not found: $SummaryFile"
+}
+
+$summary = Get-Content -Raw -Path $SummaryFile | ConvertFrom-Json
+$metrics = $summary.metrics
+
+if (-not $metrics) {
+  throw 'Invalid k6 summary file: missing metrics object.'
+}
+
+$endpoints = @('overview', 'analytics', 'chat_log', 'users')
+
+function Get-MetricValue {
+  param(
+    [object]$Metric,
+    [string]$Key
+  )
+
+  if (-not $Metric) {
+    return $null
+  }
+  if (-not $Metric.values) {
+    return $null
+  }
+
+  $value = $Metric.values.$Key
+  if ($null -ne $value) {
+    return [double]$value
+  }
+  return $null
+}
+
+$globalDuration = Get-MetricValue -Metric $metrics.http_req_duration -Key 'p(95)'
+$globalFailRate = Get-MetricValue -Metric $metrics.http_req_failed -Key 'rate'
+
+Write-Output "k6 Summary: $SummaryFile"
+if ($null -ne $globalDuration) {
+  Write-Output ("Global p95 duration (ms): {0:N2}" -f $globalDuration)
+}
+if ($null -ne $globalFailRate) {
+  Write-Output ("Global failure rate: {0:P2}" -f $globalFailRate)
+}
+Write-Output ''
+
+$rows = foreach ($endpoint in $endpoints) {
+  $durationMetricName = "http_req_duration{endpoint:$endpoint}"
+  $failMetricName = "http_req_failed{endpoint:$endpoint}"
+
+  $durationP95 = Get-MetricValue -Metric $metrics.$durationMetricName -Key 'p(95)'
+  $failRate = Get-MetricValue -Metric $metrics.$failMetricName -Key 'rate'
+
+  [PSCustomObject]@{
+    Endpoint   = $endpoint
+    P95Ms      = if ($null -ne $durationP95) { [math]::Round($durationP95, 2) } else { $null }
+    FailurePct = if ($null -ne $failRate) { [math]::Round($failRate * 100, 4) } else { $null }
+  }
+}
+
+$rows | Format-Table -AutoSize | Out-String | Write-Output

scripts/loadtest/ws-chat.yml

@@ -0,0 +1,28 @@
+config:
+  target: "{{ $processEnvironment.WS_URL }}"
+  phases:
+    - duration: 60
+      arrivalRate: 10
+      name: warmup-10
+    - duration: 120
+      arrivalRate: 100
+      name: sustain-100
+    - duration: 180
+      arrivalRate: 500
+      name: sustain-500
+    - duration: 240
+      arrivalRate: 1000
+      name: sustain-1000
+  engines:
+    ws: {}
+scenarios:
+  - engine: ws
+    name: chat-presence-and-send
+    flow:
+      - think: 1
+      - send:
+          text: '{"type":"ping","text":"loadtest"}'
+      - think: 1
+      - send:
+          text: '{"type":"ping","text":"loadtest-2"}'
+      - think: 1