feat: enhance media management with CRUD operations and runtime tracking in admin UI
CI / test (push) Successful in 1m39s
CI / docker (push) Successful in 3m9s
CI / release (push) Skipped

This commit is contained in:
zwitschi committed 2026-08-04 19:28:23 +02:00
1 parent 47740e3f4a
commit 7cfddfc7ef
6 files changed
+525 -93

No files matched your search

+21
View File
@@ -2,6 +2,27 @@
<!-- markdownlint-disable MD024 --> <!-- markdownlint-disable MD024 -->
## Unreleased
### Added
- Documentation for Admin UI Media Library (arc42 updates, refined TODO, media DB model decisions per ADR-006).
- Started implementation of full Media CRUD + video duration extraction using ffprobe (Media Library - Full CRUD section).
- Added `/api/admin/media/{id}` GET/PUT/DELETE handlers for media metadata retrieval, updates, and file+record deletion.
- Extended admin schedule and dashboard data endpoints with media duration totals, runtime strings, and media counts.
- Added admin UI runtime display for schedule/dashboard plus media library edit/delete actions.
### Changed
- Revised ADR-006 to reflect current implemented storage: PostgreSQL is the compose default and deployed primary backend, with SQLite retained as optional fallback.
- Updated chat runtime image to include ffmpeg/ffprobe for media duration probing on upload.
### Fixed
- Started Admin UI Media Library backend implementation by adding the `Media` GORM model and AutoMigrate integration in chat startup.
- Implemented media upload duration probing with ffprobe and persisted media metadata to the database alongside filesystem storage.
- Updated media listing to read from the media table and return duration metadata with HH:MM:SS runtime formatting.
## [1.0.3] - 2026-08-03 ## [1.0.3] - 2026-08-03
### Added ### Added
+1 -1
View File
@@ -9,7 +9,7 @@ RUN go mod tidy
RUN CGO_ENABLED=1 GOOS=linux go build -o /chat-server ./src/main.go RUN CGO_ENABLED=1 GOOS=linux go build -o /chat-server ./src/main.go
FROM alpine:3.20 FROM alpine:3.20
RUN apk --no-cache add ca-certificates curl curl RUN apk --no-cache add ca-certificates curl ffmpeg
WORKDIR /app WORKDIR /app
COPY --from=builder /chat-server /usr/local/bin/chat-server COPY --from=builder /chat-server /usr/local/bin/chat-server
+329 -26
View File
@@ -10,6 +10,7 @@ import (
"log" "log"
"net/http" "net/http"
"os" "os"
"os/exec"
"path/filepath" "path/filepath"
"sort" "sort"
"strconv" "strconv"
@@ -37,7 +38,7 @@ var (
authSessionsMu = sync.Mutex{} authSessionsMu = sync.Mutex{}
) )
// Message represents a chat message (persisted in SQLite per ADR-006) // Message represents a chat message (persisted in SQL DB per ADR-006)
type Message struct { type Message struct {
ID uint `gorm:"primarykey" json:"id"` ID uint `gorm:"primarykey" json:"id"`
Room string `json:"room"` Room string `json:"room"`
@@ -57,6 +58,21 @@ type ModerationLog struct {
Timestamp time.Time `json:"timestamp"` Timestamp time.Time `json:"timestamp"`
} }
// Media stores admin media library metadata (filesystem + DB model per ADR-006)
type Media struct {
ID uint `gorm:"primarykey" json:"id"`
Filename string `gorm:"index" json:"filename"`
OriginalName string `json:"originalName"`
Path string `json:"path"`
SizeBytes int64 `json:"sizeBytes"`
DurationSeconds int `json:"durationSeconds"`
Status string `gorm:"index" json:"status"`
Title string `json:"title"`
Tags string `json:"tags"`
UploadedAt time.Time `gorm:"autoCreateTime" json:"uploadedAt"`
UpdatedAt time.Time `gorm:"autoUpdateTime" json:"updatedAt"`
}
// Presence tracks online users per room (using Redis per ADR-007) // Presence tracks online users per room (using Redis per ADR-007)
type Presence struct { type Presence struct {
Username string `json:"username"` Username string `json:"username"`
@@ -68,6 +84,9 @@ type AdminOverview struct {
Viewers int `json:"viewers"` Viewers int `json:"viewers"`
StoredMessages int64 `json:"storedMessages"` StoredMessages int64 `json:"storedMessages"`
ModerationActions int64 `json:"moderationActions"` ModerationActions int64 `json:"moderationActions"`
MediaCount int64 `json:"mediaCount"`
MediaTotalDurationSeconds int64 `json:"mediaTotalDurationSeconds"`
MediaTotalRuntime string `json:"mediaTotalRuntime"`
RedisEnabled bool `json:"redisEnabled"` RedisEnabled bool `json:"redisEnabled"`
RedisClients int `json:"redisClients"` RedisClients int `json:"redisClients"`
RedisMemoryUsed string `json:"redisMemoryUsed"` RedisMemoryUsed string `json:"redisMemoryUsed"`
@@ -101,6 +120,9 @@ type AdminAnalytics struct {
MessagesLast24Hours int64 `json:"messagesLast24Hours"` MessagesLast24Hours int64 `json:"messagesLast24Hours"`
DistinctRooms int64 `json:"distinctRooms"` DistinctRooms int64 `json:"distinctRooms"`
ModerationActions int64 `json:"moderationActions"` ModerationActions int64 `json:"moderationActions"`
MediaCount int64 `json:"mediaCount"`
MediaTotalDurationSeconds int64 `json:"mediaTotalDurationSeconds"`
MediaTotalRuntime string `json:"mediaTotalRuntime"`
TopRooms []RoomAnalytics `json:"topRooms"` TopRooms []RoomAnalytics `json:"topRooms"`
GeneratedAt string `json:"generatedAt"` GeneratedAt string `json:"generatedAt"`
} }
@@ -138,16 +160,33 @@ type ScheduleEntry struct {
Index int `json:"index"` Index int `json:"index"`
Path string `json:"path"` Path string `json:"path"`
Label string `json:"label"` Label string `json:"label"`
DurationSeconds int `json:"durationSeconds"`
Runtime string `json:"runtime"`
Status string `json:"status"`
Title string `json:"title"`
} }
type AdminScheduleResponse struct { type AdminScheduleResponse struct {
Entries []ScheduleEntry `json:"entries"` Entries []ScheduleEntry `json:"entries"`
TotalEntries int `json:"totalEntries"`
MatchedEntries int `json:"matchedEntries"`
TotalDurationSeconds int `json:"totalDurationSeconds"`
TotalRuntime string `json:"totalRuntime"`
GeneratedAt string `json:"generatedAt"` GeneratedAt string `json:"generatedAt"`
} }
type MediaFileEntry struct { type MediaFileEntry struct {
ID uint `json:"id"`
Name string `json:"name"` Name string `json:"name"`
OriginalName string `json:"originalName"`
Path string `json:"path"`
Size int64 `json:"size"` Size int64 `json:"size"`
DurationSeconds int `json:"durationSeconds"`
Runtime string `json:"runtime"`
Status string `json:"status"`
Title string `json:"title"`
Tags string `json:"tags"`
UploadedAt string `json:"uploadedAt"`
ModifiedAt string `json:"modifiedAt"` ModifiedAt string `json:"modifiedAt"`
} }
@@ -199,7 +238,7 @@ func initRedis() {
} }
} }
// initDB initializes SQLite for message history and moderation audit log (ADR-006) // initDB initializes configured SQL storage for chat, moderation, and admin metadata (ADR-006)
func initDB() { func initDB() {
var err error var err error
databaseURL := strings.TrimSpace(os.Getenv("DATABASE_URL")) databaseURL := strings.TrimSpace(os.Getenv("DATABASE_URL"))
@@ -226,8 +265,8 @@ func initDB() {
} }
databaseDriverName = driverName databaseDriverName = driverName
db.AutoMigrate(&Message{}, &ModerationLog{}) db.AutoMigrate(&Message{}, &ModerationLog{}, &Media{})
log.Printf("%s database initialized (chat history and moderation log)", strings.Title(driverName)) log.Printf("%s database initialized (chat history, moderation log, media metadata)", strings.Title(driverName))
} }
func parseRedisInfoValue(info, key string) string { func parseRedisInfoValue(info, key string) string {
@@ -246,6 +285,7 @@ func getAdminOverview() AdminOverview {
RedisEnabled: redisClient != nil, RedisEnabled: redisClient != nil,
RedisMemoryUsed: "unavailable", RedisMemoryUsed: "unavailable",
DatabaseDriver: databaseDriverName, DatabaseDriver: databaseDriverName,
MediaTotalRuntime: "00:00:00",
GeneratedAt: time.Now().UTC().Format(time.RFC3339), GeneratedAt: time.Now().UTC().Format(time.RFC3339),
} }
@@ -256,6 +296,9 @@ func getAdminOverview() AdminOverview {
if db != nil { if db != nil {
db.Model(&Message{}).Count(&overview.StoredMessages) db.Model(&Message{}).Count(&overview.StoredMessages)
db.Model(&ModerationLog{}).Count(&overview.ModerationActions) db.Model(&ModerationLog{}).Count(&overview.ModerationActions)
db.Model(&Media{}).Count(&overview.MediaCount)
db.Model(&Media{}).Select("COALESCE(SUM(duration_seconds), 0)").Scan(&overview.MediaTotalDurationSeconds)
overview.MediaTotalRuntime = formatDuration(int(overview.MediaTotalDurationSeconds))
} }
if redisClient != nil { if redisClient != nil {
@@ -541,9 +584,54 @@ func handleAdminSchedule(w http.ResponseWriter, r *http.Request) {
return return
} }
entries := parsePlaylistEntries(string(content))
var mediaRecords []Media
if err := db.Find(&mediaRecords).Error; err != nil {
http.Error(w, fmt.Sprintf("failed to load media metadata: %v", err), http.StatusInternalServerError)
return
}
mediaByFilename := make(map[string]Media, len(mediaRecords))
mediaByPath := make(map[string]Media, len(mediaRecords))
for _, record := range mediaRecords {
mediaByFilename[strings.ToLower(record.Filename)] = record
mediaByPath[strings.ToLower(filepath.Clean(record.Path))] = record
}
totalDurationSeconds := 0
matchedEntries := 0
for i := range entries {
entry := &entries[i]
entry.Status = "missing"
entry.Runtime = "00:00:00"
cleanPath := strings.ToLower(filepath.Clean(entry.Path))
baseName := strings.ToLower(filepath.Base(entry.Path))
record, ok := mediaByPath[cleanPath]
if !ok {
record, ok = mediaByFilename[baseName]
}
if !ok {
continue
}
entry.DurationSeconds = record.DurationSeconds
entry.Runtime = formatDuration(record.DurationSeconds)
entry.Status = record.Status
entry.Title = record.Title
totalDurationSeconds += record.DurationSeconds
matchedEntries++
}
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(AdminScheduleResponse{ json.NewEncoder(w).Encode(AdminScheduleResponse{
Entries: parsePlaylistEntries(string(content)), Entries: entries,
TotalEntries: len(entries),
MatchedEntries: matchedEntries,
TotalDurationSeconds: totalDurationSeconds,
TotalRuntime: formatDuration(totalDurationSeconds),
GeneratedAt: time.Now().UTC().Format(time.RFC3339), GeneratedAt: time.Now().UTC().Format(time.RFC3339),
}) })
} }
@@ -624,6 +712,7 @@ func handleAdminModerationAction(w http.ResponseWriter, r *http.Request) {
func getAdminAnalytics() AdminAnalytics { func getAdminAnalytics() AdminAnalytics {
analytics := AdminAnalytics{ analytics := AdminAnalytics{
TopRooms: []RoomAnalytics{}, TopRooms: []RoomAnalytics{},
MediaTotalRuntime: "00:00:00",
GeneratedAt: time.Now().UTC().Format(time.RFC3339), GeneratedAt: time.Now().UTC().Format(time.RFC3339),
} }
@@ -637,6 +726,9 @@ func getAdminAnalytics() AdminAnalytics {
Count(&analytics.MessagesLast24Hours) Count(&analytics.MessagesLast24Hours)
db.Model(&Message{}).Distinct("room").Count(&analytics.DistinctRooms) db.Model(&Message{}).Distinct("room").Count(&analytics.DistinctRooms)
db.Model(&ModerationLog{}).Count(&analytics.ModerationActions) db.Model(&ModerationLog{}).Count(&analytics.ModerationActions)
db.Model(&Media{}).Count(&analytics.MediaCount)
db.Model(&Media{}).Select("COALESCE(SUM(duration_seconds), 0)").Scan(&analytics.MediaTotalDurationSeconds)
analytics.MediaTotalRuntime = formatDuration(int(analytics.MediaTotalDurationSeconds))
var topRooms []RoomAnalytics var topRooms []RoomAnalytics
db.Model(&Message{}). db.Model(&Message{}).
@@ -784,37 +876,126 @@ func handleAdminUsers(w http.ResponseWriter, r *http.Request) {
} }
func listMediaFiles() ([]MediaFileEntry, error) { func listMediaFiles() ([]MediaFileEntry, error) {
mediaDir := getMediaDir() var records []Media
entries, err := os.ReadDir(mediaDir) if err := db.Order("uploaded_at desc").Find(&records).Error; err != nil {
if err != nil {
return nil, err return nil, err
} }
files := make([]MediaFileEntry, 0, len(entries)) files := make([]MediaFileEntry, 0, len(records))
for _, entry := range entries { for _, record := range records {
if entry.IsDir() {
continue
}
info, err := entry.Info()
if err != nil {
continue
}
files = append(files, MediaFileEntry{ files = append(files, MediaFileEntry{
Name: entry.Name(), ID: record.ID,
Size: info.Size(), Name: record.Filename,
ModifiedAt: info.ModTime().UTC().Format(time.RFC3339), OriginalName: record.OriginalName,
Path: record.Path,
Size: record.SizeBytes,
DurationSeconds: record.DurationSeconds,
Runtime: formatDuration(record.DurationSeconds),
Status: record.Status,
Title: record.Title,
Tags: record.Tags,
UploadedAt: record.UploadedAt.UTC().Format(time.RFC3339),
ModifiedAt: record.UpdatedAt.UTC().Format(time.RFC3339),
}) })
} }
sort.Slice(files, func(i, j int) bool { sort.Slice(files, func(i, j int) bool {
if files[i].UploadedAt == files[j].UploadedAt {
return files[i].Name < files[j].Name return files[i].Name < files[j].Name
}
return files[i].UploadedAt > files[j].UploadedAt
}) })
return files, nil return files, nil
} }
func formatDuration(seconds int) string {
if seconds < 0 {
seconds = 0
}
hours := seconds / 3600
minutes := (seconds % 3600) / 60
remainingSeconds := seconds % 60
return fmt.Sprintf("%02d:%02d:%02d", hours, minutes, remainingSeconds)
}
func getMediaIDFromPath(path string) (uint, error) {
prefix := "/api/admin/media/"
if !strings.HasPrefix(path, prefix) {
return 0, fmt.Errorf("invalid media path")
}
idPart := strings.TrimPrefix(path, prefix)
idPart = strings.TrimSpace(idPart)
if idPart == "" || strings.Contains(idPart, "/") {
return 0, fmt.Errorf("invalid media id")
}
id, err := strconv.ParseUint(idPart, 10, 64)
if err != nil || id == 0 {
return 0, fmt.Errorf("invalid media id")
}
return uint(id), nil
}
func mediaEntryFromRecord(record Media) MediaFileEntry {
return MediaFileEntry{
ID: record.ID,
Name: record.Filename,
OriginalName: record.OriginalName,
Path: record.Path,
Size: record.SizeBytes,
DurationSeconds: record.DurationSeconds,
Runtime: formatDuration(record.DurationSeconds),
Status: record.Status,
Title: record.Title,
Tags: record.Tags,
UploadedAt: record.UploadedAt.UTC().Format(time.RFC3339),
ModifiedAt: record.UpdatedAt.UTC().Format(time.RFC3339),
}
}
func probeMediaDurationSeconds(mediaPath string) (int, error) {
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
out, err := exec.CommandContext(
ctx,
"ffprobe",
"-v", "error",
"-show_entries", "format=duration",
"-of", "default=noprint_wrappers=1:nokey=1",
mediaPath,
).CombinedOutput()
if ctx.Err() == context.DeadlineExceeded {
return 0, fmt.Errorf("ffprobe timed out")
}
if err != nil {
return 0, fmt.Errorf("ffprobe failed: %w (%s)", err, strings.TrimSpace(string(out)))
}
rawDuration := strings.TrimSpace(string(out))
if rawDuration == "" {
return 0, fmt.Errorf("ffprobe returned empty duration")
}
secondsFloat, err := strconv.ParseFloat(rawDuration, 64)
if err != nil {
return 0, fmt.Errorf("failed to parse duration %q: %w", rawDuration, err)
}
if secondsFloat < 0 {
secondsFloat = 0
}
seconds := int(secondsFloat + 0.5)
if secondsFloat > 0 && seconds == 0 {
seconds = 1
}
return seconds, nil
}
func handleAdminMedia(w http.ResponseWriter, r *http.Request) { func handleAdminMedia(w http.ResponseWriter, r *http.Request) {
switch r.Method { switch r.Method {
case http.MethodGet: case http.MethodGet:
@@ -851,12 +1032,49 @@ func handleAdminMedia(w http.ResponseWriter, r *http.Request) {
http.Error(w, fmt.Sprintf("failed to create media file: %v", err), http.StatusInternalServerError) http.Error(w, fmt.Sprintf("failed to create media file: %v", err), http.StatusInternalServerError)
return return
} }
defer targetFile.Close()
if _, err := io.Copy(targetFile, file); err != nil { if _, err := io.Copy(targetFile, file); err != nil {
targetFile.Close()
_ = os.Remove(targetPath)
http.Error(w, fmt.Sprintf("failed to store media file: %v", err), http.StatusInternalServerError) http.Error(w, fmt.Sprintf("failed to store media file: %v", err), http.StatusInternalServerError)
return return
} }
if err := targetFile.Close(); err != nil {
_ = os.Remove(targetPath)
http.Error(w, fmt.Sprintf("failed to finalize media file: %v", err), http.StatusInternalServerError)
return
}
fileInfo, err := os.Stat(targetPath)
if err != nil {
_ = os.Remove(targetPath)
http.Error(w, fmt.Sprintf("failed to read media file info: %v", err), http.StatusInternalServerError)
return
}
durationSeconds, err := probeMediaDurationSeconds(targetPath)
if err != nil {
_ = os.Remove(targetPath)
http.Error(w, fmt.Sprintf("failed to probe media duration: %v", err), http.StatusBadRequest)
return
}
filename := filepath.Base(header.Filename)
mediaRecord := Media{
Filename: filename,
OriginalName: header.Filename,
Path: targetPath,
SizeBytes: fileInfo.Size(),
DurationSeconds: durationSeconds,
Status: "ready",
Title: strings.TrimSuffix(filename, filepath.Ext(filename)),
Tags: "",
}
if err := db.Create(&mediaRecord).Error; err != nil {
_ = os.Remove(targetPath)
http.Error(w, fmt.Sprintf("failed to persist media metadata: %v", err), http.StatusInternalServerError)
return
}
files, err := listMediaFiles() files, err := listMediaFiles()
if err != nil { if err != nil {
@@ -871,6 +1089,90 @@ func handleAdminMedia(w http.ResponseWriter, r *http.Request) {
} }
} }
func handleAdminMediaByID(w http.ResponseWriter, r *http.Request) {
mediaID, err := getMediaIDFromPath(r.URL.Path)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
var record Media
if err := db.First(&record, mediaID).Error; err != nil {
http.Error(w, "media item not found", http.StatusNotFound)
return
}
switch r.Method {
case http.MethodGet:
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(mediaEntryFromRecord(record))
case http.MethodPut:
var payload struct {
Status string `json:"status"`
Title string `json:"title"`
Tags string `json:"tags"`
}
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
http.Error(w, fmt.Sprintf("invalid request body: %v", err), http.StatusBadRequest)
return
}
updates := map[string]any{}
if payload.Status != "" {
updates["status"] = strings.TrimSpace(payload.Status)
}
if payload.Title != "" {
updates["title"] = strings.TrimSpace(payload.Title)
}
updates["tags"] = strings.TrimSpace(payload.Tags)
if err := db.Model(&record).Updates(updates).Error; err != nil {
http.Error(w, fmt.Sprintf("failed to update media metadata: %v", err), http.StatusInternalServerError)
return
}
if err := db.First(&record, mediaID).Error; err != nil {
http.Error(w, "media item not found after update", http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(mediaEntryFromRecord(record))
case http.MethodDelete:
absMediaDir, err := filepath.Abs(getMediaDir())
if err != nil {
http.Error(w, fmt.Sprintf("failed to resolve media directory: %v", err), http.StatusInternalServerError)
return
}
absTargetPath, err := filepath.Abs(record.Path)
if err != nil {
http.Error(w, fmt.Sprintf("failed to resolve media path: %v", err), http.StatusInternalServerError)
return
}
prefix := absMediaDir + string(os.PathSeparator)
if absTargetPath != absMediaDir && !strings.HasPrefix(absTargetPath, prefix) {
http.Error(w, "media path consistency check failed", http.StatusConflict)
return
}
if err := os.Remove(absTargetPath); err != nil && !os.IsNotExist(err) {
http.Error(w, fmt.Sprintf("failed to delete media file: %v", err), http.StatusInternalServerError)
return
}
if err := db.Delete(&record).Error; err != nil {
http.Error(w, fmt.Sprintf("failed to delete media record: %v", err), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]any{"deleted": true, "id": mediaID})
default:
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
}
}
func countConnectedClients() int { func countConnectedClients() int {
mutex.Lock() mutex.Lock()
defer mutex.Unlock() defer mutex.Unlock()
@@ -976,7 +1278,7 @@ func broadcast(msg Message) {
mutex.Lock() mutex.Lock()
defer mutex.Unlock() defer mutex.Unlock()
// Save to SQLite // Persist to configured SQL backend.
db.Create(&msg) db.Create(&msg)
for client := range clients { for client := range clients {
@@ -1054,7 +1356,7 @@ func handleConnection(w http.ResponseWriter, r *http.Request) {
log.Printf("Chat message in room %s from %s: %s", msg.Room, msg.Username, msg.Text) log.Printf("Chat message in room %s from %s: %s", msg.Room, msg.Username, msg.Text)
broadcast(msg) broadcast(msg)
// Save to SQLite (already done in broadcast, but ensure for private DMs) // Save private messages explicitly as well.
if msg.Type == "private" { if msg.Type == "private" {
db.Create(&msg) db.Create(&msg)
} }
@@ -1063,7 +1365,7 @@ func handleConnection(w http.ResponseWriter, r *http.Request) {
// handlePrivateDM routes private messages to the specific user pair // handlePrivateDM routes private messages to the specific user pair
func handlePrivateDM(msg Message) { func handlePrivateDM(msg Message) {
// Save to SQLite for history // Persist private messages for history.
db.Create(&msg) db.Create(&msg)
// Broadcast only to participants (simplified - in production use a map of user connections) // Broadcast only to participants (simplified - in production use a map of user connections)
@@ -1122,6 +1424,7 @@ func main() {
http.HandleFunc("/api/admin/users", requireAdminAuth(handleAdminUsers)) http.HandleFunc("/api/admin/users", requireAdminAuth(handleAdminUsers))
http.HandleFunc("/api/admin/schedule", requireAdminAuth(handleAdminSchedule)) http.HandleFunc("/api/admin/schedule", requireAdminAuth(handleAdminSchedule))
http.HandleFunc("/api/admin/media", requireAdminAuth(handleAdminMedia)) http.HandleFunc("/api/admin/media", requireAdminAuth(handleAdminMedia))
http.HandleFunc("/api/admin/media/", requireAdminAuth(handleAdminMediaByID))
http.HandleFunc("/metrics", handleMetrics) http.HandleFunc("/metrics", handleMetrics)
http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
+45 -56
View File
@@ -1,7 +1,7 @@
# ADR-006: Data Storage Strategy (SQLite vs PostgreSQL) # ADR-006: Data Storage Strategy (SQLite vs PostgreSQL)
**Status:** Proposed **Status:** Accepted
**Date:** 2026-08-02 **Date:** 2026-08-04
**Author:** AI Assistant (following AGENTS.md, ADR-002, ADR-003, ADR-004, ADR-005) **Author:** AI Assistant (following AGENTS.md, ADR-002, ADR-003, ADR-004, ADR-005)
**Related ADRs:** [ADR-001](./ADR-001.md), [ADR-003](./ADR-003-tech-stack.md), [ADR-005](./ADR-005-admin-moderation-analytics.md), [ADR-007](./ADR-007-redis-usage.md) **Related ADRs:** [ADR-001](./ADR-001.md), [ADR-003](./ADR-003-tech-stack.md), [ADR-005](./ADR-005-admin-moderation-analytics.md), [ADR-007](./ADR-007-redis-usage.md)
@@ -12,89 +12,78 @@ The system needs persistent storage for:
- Chat messages and history (public rooms and private DMs). - Chat messages and history (public rooms and private DMs).
- User accounts, roles, and ban lists. - User accounts, roles, and ban lists.
- Moderation audit log (immutable record of admin and moderator actions). - Moderation audit log (immutable record of admin and moderator actions).
- Playlist metadata and schedule overrides. - Playlist and media metadata used by admin flows.
- System configuration (including Redis settings adjustable via admin UI per ADR-005/ADR-007). - System configuration visibility in admin APIs.
From ADR-003 we prefer operational simplicity. ADR-005 requires a robust audit trail. We must decide between **SQLite** (embedded, zero-config) and **PostgreSQL** (full-featured, scalable) and document how the admin interface can view and adjust storage-related configuration. The codebase now ships with a PostgreSQL service in Docker Compose (`db`) and chat is configured with a PostgreSQL `DATABASE_URL` by default. The backend still supports SQLite for optional local fallback, but the deployed and compose-default path is PostgreSQL.
## Decision Drivers ## Decision Drivers
1. **Operational Simplicity** – Minimize infrastructure (no separate DB server for SQLite). 1. **Implemented Reality** - Docker Compose already provisions PostgreSQL and chat depends on it.
2. **Performance & Concurrency** – Chat and moderation may have high write volume. 2. **Concurrency & Throughput** - Chat, moderation, and media metadata writes require stronger concurrent write behavior.
3. **Backup and Recovery** – Easy point-in-time recovery and replication. 3. **Operational Traceability** - Structured backups, service health checks, and persistent volume management are already in place for PostgreSQL.
4. **Admin Capabilities** – Admin UI must be able to view storage stats, run vacuum/analyze, and switch between SQLite and PostgreSQL (with migration path). 4. **Admin Growth** - Upcoming admin features (media CRUD, analytics, moderation tooling) benefit from relational querying and indexing.
5. **Auditability** – Moderation log must be immutable and queryable. 5. **Fallback Simplicity** - Keep SQLite support for lightweight local or recovery scenarios without blocking developer workflows.
6. **Cost** – Avoid managed database costs during early stages.
## Decision ## Decision
**Primary storage for v1: SQLite** (embedded in the chat and admin services). **Primary storage for current implementation: PostgreSQL (Docker Compose `db` service).**
**Migration Path**: Provide a clear upgrade path to PostgreSQL when concurrency or replication needs exceed SQLite capabilities (target v2 or when concurrent users > 500). **Secondary/Fallback mode:** SQLite remains supported via `DATABASE_URL=sqlite://...` for local-only scenarios.
**Admin Interface Capabilities** (extension of ADR-005): **Current Backend Behavior**:
- "System Configuration" tab showing current storage engine, database size, connection stats, and vacuum status. - Chat backend (`backend/chat/src/main.go`) resolves `DATABASE_URL` at startup.
- Buttons to run `VACUUM`, `ANALYZE`, or trigger a backup. - `postgres://` or `postgresql://` selects PostgreSQL driver.
- Configuration form to switch connection string (SQLite file path vs PostgreSQL URL) and restart services gracefully. - `sqlite://` selects SQLite driver.
- Audit log of all configuration changes. - Empty `DATABASE_URL` falls back to local `chat.db` for developer convenience.
- GORM AutoMigrate is used for chat/admin data models.
**Technical Details**: **Media Model Decision (Admin UI Feature Track)**:
- Use SQLite with WAL mode for better concurrency. - Media metadata is persisted in the same database as messages and moderation logs.
- Use GORM or sqlx (Go) or Prisma (if Node.js fallback) for ORM/migrations. - Physical media files remain on the shared filesystem (`/app/media`, mapped from `data/media`).
- Store large chat history in a separate table with retention policy (e.g., 30 days). - The database is the source of truth for media library metadata and future runtime analytics.
- Moderation log as an append-only table with triggers for immutability.
**Folder Placement** (per ADR-002):
- Database models and migrations in `backend/shared/db/` or `backend/chat/internal/db/`.
- Admin UI pages for storage management in `frontend/admin/system/`.
## Rationale ## Rationale
- SQLite aligns with ADR-003's emphasis on operational simplicity and low resource usage for the initial deployment on a single Hetzner VPS. - The repository already operationalizes PostgreSQL in both `docker-compose.yml` and `docker-compose.prod.yml` with persistent storage.
- It eliminates the need for a separate database container in `docker-compose.yml` during early development. - Default environment documentation (`.env.example`, setup docs) already points to PostgreSQL.
- The admin interface can provide visibility and basic maintenance operations, fulfilling the requirement from the previous TODO plan. - This aligns architecture documentation with implemented behavior and avoids drift.
- PostgreSQL can be introduced later with a migration script when scale demands it (replication, advanced querying, full-text search on chat). - Keeping SQLite support preserves local flexibility while avoiding a hard breaking change.
## Consequences ## Consequences
**Positive**: **Positive**:
- Zero-config database for quick local development and small-scale production. - Better write concurrency and query flexibility for chat + admin workloads.
- Simple backups (just copy the `.db` file). - Clear operational model for containerized deployments (health checks, named volume, service dependency).
- Admin UI can directly manage and monitor storage without external tools. - Easier extension for admin analytics and media metadata features.
**Negative / Risks**: **Negative / Risks**:
- SQLite has limited concurrency compared to PostgreSQL (WAL mode helps but is not a full replacement). - Additional service to operate versus embedded-only SQLite.
- Large chat history or analytics queries may become slow – plan retention policy and archiving. - Misconfigured `DATABASE_URL` can still route to SQLite fallback unintentionally if left empty outside compose.
- Migration to PostgreSQL will require downtime or dual-write strategy (documented in this ADR). - Requires backup/restore procedures for PostgreSQL volume in production runbooks.
**Alternatives Considered**: ## Alternatives Considered
- PostgreSQL from day one – rejected for added operational complexity (separate service, connection pooling, managed service cost). - SQLite-first in production: rejected for current implementation because compose and deployment already run PostgreSQL.
- Pure key-value store (e.g., BoltDB) – rejected because SQL is better for audit logs and complex moderation queries. - PostgreSQL-only with no fallback: rejected to preserve local dev and troubleshooting flexibility.
- Managed database (Neon, Supabase) – rejected to maintain full self-hosted independence per ADR-001. - Managed external PostgreSQL: deferred; self-hosted compose remains the preferred operational model.
## Implementation Plan ## Implementation Status
1. Create this ADR-006 (done). 1. ADR updated to match implemented state. Done.
2. Add SQLite support in `backend/shared/db/` with GORM migrations. 2. Compose includes PostgreSQL service and persistent volume. Done.
3. Implement admin UI pages for storage monitoring and configuration (linked to ADR-005). 3. Chat backend supports PostgreSQL and SQLite based on `DATABASE_URL`. Done.
4. Add configuration reload capability without full restart where possible. 4. Media metadata persistence model documented here for Admin UI track. In progress (implementation ongoing).
5. Write migration script from SQLite to PostgreSQL. 5. Admin storage observability and operations UI (vacuum/analyze/backup flows). Planned.
6. Update `docker-compose.yml` to optionally include PostgreSQL.
7. Add tests for data access patterns and admin configuration changes.
**Next ADR**: ADR-007 (Redis usage and admin configuration capabilities).
## References ## References
- ADR-001: Chat persistence, moderation log, monitoring requirements. - ADR-001: Chat persistence, moderation log, monitoring requirements.
- ADR-003: Preference for operational simplicity and Go backend. - ADR-003: Preference for operational simplicity and Go backend.
- ADR-005: Admin interface for user management, moderation, and analytics – this ADR extends it with storage and Redis config capabilities. - ADR-005: Admin interface for user management, moderation, and analytics.
- arc42 documentation: Cross-cutting Concepts (data management, monitoring) and Risks & Technical Debts. - ADR-007: Redis usage for presence, rate limiting, pub/sub.
- `docker-compose.yml` and `docker-compose.prod.yml` database service definitions.
This ADR ensures all data storage decisions are documented before code is written, maintaining full traceability.
+128 -9
View File
@@ -38,7 +38,7 @@
<template x-if="currentTab === 'dashboard'"> <template x-if="currentTab === 'dashboard'">
<div> <div>
<h1 class="text-3xl font-bold mb-8">Dashboard</h1> <h1 class="text-3xl font-bold mb-8">Dashboard</h1>
<div class="grid grid-cols-3 gap-6"> <div class="grid grid-cols-1 md:grid-cols-2 xl:grid-cols-4 gap-6">
<div class="bg-slate-800 p-6 rounded-3xl"> <div class="bg-slate-800 p-6 rounded-3xl">
<div class="text-emerald-400 text-sm">Concurrent Viewers</div> <div class="text-emerald-400 text-sm">Concurrent Viewers</div>
<div class="text-5xl font-bold mt-2" x-text="stats.viewers"></div> <div class="text-5xl font-bold mt-2" x-text="stats.viewers"></div>
@@ -51,6 +51,13 @@
<div class="text-amber-400 text-sm">Moderation Actions</div> <div class="text-amber-400 text-sm">Moderation Actions</div>
<div class="text-5xl font-bold mt-2" x-text="stats.moderationActions"></div> <div class="text-5xl font-bold mt-2" x-text="stats.moderationActions"></div>
</div> </div>
<div class="bg-slate-800 p-6 rounded-3xl">
<div class="text-violet-400 text-sm">Media Runtime</div>
<div class="text-3xl font-bold mt-2" x-text="stats.mediaTotalRuntime"></div>
<div class="text-xs text-slate-400 mt-2">
<span x-text="stats.mediaCount"></span> files
</div>
</div>
</div> </div>
<div class="mt-6 text-sm text-slate-400" x-text="statusMessage"></div> <div class="mt-6 text-sm text-slate-400" x-text="statusMessage"></div>
</div> </div>
@@ -82,21 +89,34 @@
<thead class="text-slate-400 border-b border-slate-700"> <thead class="text-slate-400 border-b border-slate-700">
<tr> <tr>
<th class="text-left py-3 pr-4 font-medium">File</th> <th class="text-left py-3 pr-4 font-medium">File</th>
<th class="text-left py-3 pr-4 font-medium">Title</th>
<th class="text-left py-3 pr-4 font-medium">Runtime</th>
<th class="text-left py-3 pr-4 font-medium">Status</th>
<th class="text-left py-3 pr-4 font-medium">Size</th> <th class="text-left py-3 pr-4 font-medium">Size</th>
<th class="text-left py-3 font-medium">Modified</th> <th class="text-left py-3 pr-4 font-medium">Modified</th>
<th class="text-left py-3 font-medium">Actions</th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
<template x-if="media.files.length === 0"> <template x-if="media.files.length === 0">
<tr> <tr>
<td colspan="3" class="py-6 text-slate-500">No media files available.</td> <td colspan="7" class="py-6 text-slate-500">No media files available.</td>
</tr> </tr>
</template> </template>
<template x-for="file in media.files" :key="file.name"> <template x-for="file in media.files" :key="file.id || file.name">
<tr class="border-b border-slate-800"> <tr class="border-b border-slate-800">
<td class="py-3 pr-4" x-text="file.name"></td> <td class="py-3 pr-4" x-text="file.name"></td>
<td class="py-3 pr-4" x-text="file.title || '-' "></td>
<td class="py-3 pr-4 font-mono text-xs" x-text="file.runtime || '00:00:00'"></td>
<td class="py-3 pr-4" x-text="file.status || 'unknown'"></td>
<td class="py-3 pr-4" x-text="file.size"></td> <td class="py-3 pr-4" x-text="file.size"></td>
<td class="py-3 font-mono text-xs" x-text="file.modifiedAt"></td> <td class="py-3 pr-4 font-mono text-xs" x-text="file.modifiedAt"></td>
<td class="py-3">
<div class="flex gap-2">
<button @click="editMedia(file)" class="bg-slate-700 hover:bg-slate-600 px-3 py-1 rounded-lg text-xs">Edit</button>
<button @click="deleteMedia(file)" class="bg-rose-700 hover:bg-rose-600 px-3 py-1 rounded-lg text-xs">Delete</button>
</div>
</td>
</tr> </tr>
</template> </template>
</tbody> </tbody>
@@ -116,25 +136,33 @@
<button @click="refreshSchedule()" class="bg-slate-700 hover:bg-slate-600 px-4 py-2 rounded-xl text-sm">Reload schedule</button> <button @click="refreshSchedule()" class="bg-slate-700 hover:bg-slate-600 px-4 py-2 rounded-xl text-sm">Reload schedule</button>
</div> </div>
<div class="bg-slate-800 rounded-3xl p-6"> <div class="bg-slate-800 rounded-3xl p-6">
<div class="mb-4 text-sm text-slate-400">
<span x-text="schedule.matchedEntries"></span>/<span x-text="schedule.totalEntries"></span> matched
• total runtime <span class="font-mono" x-text="schedule.totalRuntime"></span>
</div>
<div class="overflow-x-auto"> <div class="overflow-x-auto">
<table class="w-full text-sm"> <table class="w-full text-sm">
<thead class="text-slate-400 border-b border-slate-700"> <thead class="text-slate-400 border-b border-slate-700">
<tr> <tr>
<th class="text-left py-3 pr-4 font-medium">#</th> <th class="text-left py-3 pr-4 font-medium">#</th>
<th class="text-left py-3 pr-4 font-medium">Label</th> <th class="text-left py-3 pr-4 font-medium">Label</th>
<th class="text-left py-3 pr-4 font-medium">Runtime</th>
<th class="text-left py-3 pr-4 font-medium">Status</th>
<th class="text-left py-3 font-medium">Path</th> <th class="text-left py-3 font-medium">Path</th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
<template x-if="schedule.entries.length === 0"> <template x-if="schedule.entries.length === 0">
<tr> <tr>
<td colspan="3" class="py-6 text-slate-500">No schedule entries found.</td> <td colspan="5" class="py-6 text-slate-500">No schedule entries found.</td>
</tr> </tr>
</template> </template>
<template x-for="entry in schedule.entries" :key="entry.index + ':' + entry.path"> <template x-for="entry in schedule.entries" :key="entry.index + ':' + entry.path">
<tr class="border-b border-slate-800"> <tr class="border-b border-slate-800">
<td class="py-3 pr-4" x-text="entry.index"></td> <td class="py-3 pr-4" x-text="entry.index"></td>
<td class="py-3 pr-4" x-text="entry.label"></td> <td class="py-3 pr-4" x-text="entry.label"></td>
<td class="py-3 pr-4 font-mono text-xs" x-text="entry.runtime || '00:00:00'"></td>
<td class="py-3 pr-4" x-text="entry.status || 'missing'"></td>
<td class="py-3 font-mono text-xs" x-text="entry.path"></td> <td class="py-3 font-mono text-xs" x-text="entry.path"></td>
</tr> </tr>
</template> </template>
@@ -448,7 +476,9 @@
stats: { stats: {
viewers: 0, viewers: 0,
storedMessages: 0, storedMessages: 0,
moderationActions: 0 moderationActions: 0,
mediaCount: 0,
mediaTotalRuntime: '00:00:00'
}, },
system: { system: {
redisEnabled: false, redisEnabled: false,
@@ -467,6 +497,8 @@
messagesLast24Hours: 0, messagesLast24Hours: 0,
distinctRooms: 0, distinctRooms: 0,
moderationActions: 0, moderationActions: 0,
mediaCount: 0,
mediaTotalRuntime: '00:00:00',
topRooms: [], topRooms: [],
generatedAt: 'loading...', generatedAt: 'loading...',
status: 'Loading analytics...', status: 'Loading analytics...',
@@ -474,6 +506,9 @@
}, },
schedule: { schedule: {
entries: [], entries: [],
totalEntries: 0,
matchedEntries: 0,
totalRuntime: '00:00:00',
status: 'Loading schedule...', status: 'Loading schedule...',
error: false error: false
}, },
@@ -520,6 +555,8 @@
this.stats.viewers = overview.viewers; this.stats.viewers = overview.viewers;
this.stats.storedMessages = overview.storedMessages; this.stats.storedMessages = overview.storedMessages;
this.stats.moderationActions = overview.moderationActions; this.stats.moderationActions = overview.moderationActions;
this.stats.mediaCount = overview.mediaCount || 0;
this.stats.mediaTotalRuntime = overview.mediaTotalRuntime || '00:00:00';
this.system.redisEnabled = overview.redisEnabled; this.system.redisEnabled = overview.redisEnabled;
this.system.redisClients = overview.redisClients; this.system.redisClients = overview.redisClients;
this.system.redisMemoryUsed = overview.redisMemoryUsed; this.system.redisMemoryUsed = overview.redisMemoryUsed;
@@ -549,6 +586,8 @@
this.analytics.messagesLast24Hours = payload.messagesLast24Hours; this.analytics.messagesLast24Hours = payload.messagesLast24Hours;
this.analytics.distinctRooms = payload.distinctRooms; this.analytics.distinctRooms = payload.distinctRooms;
this.analytics.moderationActions = payload.moderationActions; this.analytics.moderationActions = payload.moderationActions;
this.analytics.mediaCount = payload.mediaCount || 0;
this.analytics.mediaTotalRuntime = payload.mediaTotalRuntime || '00:00:00';
this.analytics.topRooms = payload.topRooms || []; this.analytics.topRooms = payload.topRooms || [];
this.analytics.generatedAt = payload.generatedAt; this.analytics.generatedAt = payload.generatedAt;
this.analytics.status = 'Analytics loaded.'; this.analytics.status = 'Analytics loaded.';
@@ -589,7 +628,10 @@
const payload = await response.json(); const payload = await response.json();
this.schedule.entries = payload.entries || []; this.schedule.entries = payload.entries || [];
this.schedule.status = 'Schedule loaded.'; this.schedule.totalEntries = payload.totalEntries || 0;
this.schedule.matchedEntries = payload.matchedEntries || 0;
this.schedule.totalRuntime = payload.totalRuntime || '00:00:00';
this.schedule.status = `Schedule loaded. Total runtime ${this.schedule.totalRuntime}.`;
} catch (error) { } catch (error) {
console.error('Failed to load schedule', error); console.error('Failed to load schedule', error);
this.schedule.status = 'Failed to load schedule.'; this.schedule.status = 'Failed to load schedule.';
@@ -641,6 +683,83 @@
this.media.error = true; this.media.error = true;
} }
}, },
async editMedia(file) {
if (!file.id) {
this.media.status = 'Cannot edit media item without id.';
this.media.error = true;
return;
}
const title = window.prompt('Title', file.title || file.name || '');
if (title === null) {
return;
}
const status = window.prompt('Status', file.status || 'ready');
if (status === null) {
return;
}
const tags = window.prompt('Tags (comma-separated)', file.tags || '');
if (tags === null) {
return;
}
this.media.error = false;
this.media.status = `Saving metadata for ${file.name}...`;
try {
const response = await fetch(`/api/admin/media/${file.id}`, {
method: 'PUT',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({ title, status, tags })
});
if (!response.ok) {
throw new Error(`HTTP ${response.status}`);
}
this.media.status = 'Media metadata updated.';
await this.refreshMediaLibrary();
await this.refreshOverview();
await this.refreshSchedule();
} catch (error) {
console.error('Failed to update media metadata', error);
this.media.status = 'Failed to update media metadata.';
this.media.error = true;
}
},
async deleteMedia(file) {
if (!file.id) {
this.media.status = 'Cannot delete media item without id.';
this.media.error = true;
return;
}
if (!window.confirm(`Delete ${file.name}? This removes file and metadata.`)) {
return;
}
this.media.error = false;
this.media.status = `Deleting ${file.name}...`;
try {
const response = await fetch(`/api/admin/media/${file.id}`, {
method: 'DELETE'
});
if (!response.ok) {
throw new Error(`HTTP ${response.status}`);
}
this.media.status = `${file.name} deleted.`;
await this.refreshMediaLibrary();
await this.refreshOverview();
await this.refreshSchedule();
} catch (error) {
console.error('Failed to delete media', error);
this.media.status = 'Failed to delete media file.';
this.media.error = true;
}
},
async savePlaylist() { async savePlaylist() {
this.playlist.saving = true; this.playlist.saving = true;
this.playlist.status = 'Saving playlist...'; this.playlist.status = 'Saving playlist...';
@@ -799,7 +918,7 @@
} }
function runVacuum() { function runVacuum() {
alert('VACUUM executed on SQLite database (per ADR-006). Audit log entry created.'); alert('Database maintenance action placeholder. Use storage backend-specific operations from server-side APIs.');
} }
</script> </script>
</body> </body>
+1 -1
View File
@@ -39,7 +39,7 @@
</div> </div>
<div class="p-4 bg-slate-900 border-t border-slate-800 text-xs text-slate-400 flex justify-between"> <div class="p-4 bg-slate-900 border-t border-slate-800 text-xs text-slate-400 flex justify-between">
<div>Latency: ~25s • Synced for all viewers (per ADR-001)</div> <div>Latency: ~25s • Synced</div>
<div class="flex gap-6"> <div class="flex gap-6">
<a href="#" onclick="switchRoom('main')" class="hover:text-white">Main Channel</a> <a href="#" onclick="switchRoom('main')" class="hover:text-white">Main Channel</a>
<a href="#" onclick="switchRoom('chill')" class="hover:text-white">Chill Room</a> <a href="#" onclick="switchRoom('chill')" class="hover:text-white">Chill Room</a>