feat(playout): replace time-based R2 retention with run-count and size caps
This commit is contained in:
zwitschi committed
1 parent
06b277e9c1
commit
6950bba85d
8 files changed
+329
-90
No files matched your search
@@ -593,54 +593,95 @@ func TestSyncToR2_NoCacheHeadersForUnknownExt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestR2SegmentRetentionHours(t *testing.T) {
|
||||
os.Setenv("R2_SEGMENT_RETENTION_HOURS", "")
|
||||
if got := r2SegmentRetentionHours(); got != 24*time.Hour {
|
||||
t.Errorf("default = %v, want 24h", got)
|
||||
func TestR2KeepRuns(t *testing.T) {
|
||||
os.Setenv("R2_KEEP_RUNS", "")
|
||||
if got := r2KeepRuns(); got != 2 {
|
||||
t.Errorf("default = %d, want 2", got)
|
||||
}
|
||||
|
||||
os.Setenv("R2_SEGMENT_RETENTION_HOURS", "48")
|
||||
if got := r2SegmentRetentionHours(); got != 48*time.Hour {
|
||||
t.Errorf("override = %v, want 48h", got)
|
||||
os.Setenv("R2_KEEP_RUNS", "5")
|
||||
if got := r2KeepRuns(); got != 5 {
|
||||
t.Errorf("override = %d, want 5", got)
|
||||
}
|
||||
|
||||
os.Setenv("R2_SEGMENT_RETENTION_HOURS", "invalid")
|
||||
if got := r2SegmentRetentionHours(); got != 24*time.Hour {
|
||||
t.Errorf("invalid = %v, want 24h fallback", got)
|
||||
os.Setenv("R2_KEEP_RUNS", "0")
|
||||
if got := r2KeepRuns(); got != 2 {
|
||||
t.Errorf("invalid = %d, want 2 fallback", got)
|
||||
}
|
||||
|
||||
os.Unsetenv("R2_SEGMENT_RETENTION_HOURS")
|
||||
os.Unsetenv("R2_KEEP_RUNS")
|
||||
}
|
||||
|
||||
func TestCleanupOldSegments(t *testing.T) {
|
||||
func TestR2MaxRetentionBytes(t *testing.T) {
|
||||
os.Setenv("R2_MAX_RETENTION_BYTES", "")
|
||||
if got := r2MaxRetentionBytes(); got != 10*1024*1024*1024 {
|
||||
t.Errorf("default = %d, want 10GiB", got)
|
||||
}
|
||||
|
||||
os.Setenv("R2_MAX_RETENTION_BYTES", "100")
|
||||
if got := r2MaxRetentionBytes(); got != 100 {
|
||||
t.Errorf("override = %d, want 100", got)
|
||||
}
|
||||
|
||||
os.Setenv("R2_MAX_RETENTION_BYTES", "invalid")
|
||||
if got := r2MaxRetentionBytes(); got != 10*1024*1024*1024 {
|
||||
t.Errorf("invalid = %d, want 10GiB fallback", got)
|
||||
}
|
||||
|
||||
os.Unsetenv("R2_MAX_RETENTION_BYTES")
|
||||
}
|
||||
|
||||
func TestR2RunFromKey(t *testing.T) {
|
||||
cases := []struct {
|
||||
key string
|
||||
run string
|
||||
ok bool
|
||||
}{
|
||||
{"hls/r-20260815T120000Z/seg-1.m4s", "r-20260815T120000Z", true},
|
||||
{"hls/r-20260815T120000Z/init.mp4", "r-20260815T120000Z", true},
|
||||
{"hls/playlist.m3u8", "", false},
|
||||
{"hls/playlist.m3u8.tmp", "", false},
|
||||
{"other/r-x/seg.m4s", "", false},
|
||||
{"hls/r-noslash", "", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
run, ok := r2RunFromKey(c.key)
|
||||
if run != c.run || ok != c.ok {
|
||||
t.Errorf("r2RunFromKey(%q) = (%q, %v), want (%q, %v)", c.key, run, ok, c.run, c.ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupOldRuns(t *testing.T) {
|
||||
prevClient := r2Client
|
||||
prevBucket := bucket
|
||||
prevList := listObjects
|
||||
prevDelete := deleteObjects
|
||||
prevTotal := r2CleanupObjectsTotal
|
||||
prevRuns := r2RunsRetained
|
||||
prevBytes := r2BytesRetained
|
||||
defer func() {
|
||||
r2Client = prevClient
|
||||
bucket = prevBucket
|
||||
listObjects = prevList
|
||||
deleteObjects = prevDelete
|
||||
r2CleanupObjectsTotal = prevTotal
|
||||
r2RunsRetained = prevRuns
|
||||
r2BytesRetained = prevBytes
|
||||
}()
|
||||
|
||||
bucket = "linearvoid-media"
|
||||
r2Client = &s3.Client{}
|
||||
|
||||
now := time.Now().UTC()
|
||||
old := now.Add(-25 * time.Hour)
|
||||
recent := now.Add(-1 * time.Hour)
|
||||
|
||||
size := int64(1024)
|
||||
listObjects = func(_ *s3.Client, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
|
||||
return &s3.ListObjectsV2Output{
|
||||
Contents: []types.Object{
|
||||
{Key: aws.String("hls/r-old/seg-1.m4s"), LastModified: &old},
|
||||
{Key: aws.String("hls/r-old/init.mp4"), LastModified: &old},
|
||||
{Key: aws.String("hls/r-new/seg-2.m4s"), LastModified: &recent},
|
||||
{Key: aws.String("hls/playlist.m3u8"), LastModified: &old},
|
||||
{Key: aws.String("hls/playlist.m3u8.tmp"), LastModified: &old},
|
||||
{Key: aws.String("hls/r-20260813T000000Z/seg-1.m4s"), Size: &size},
|
||||
{Key: aws.String("hls/r-20260814T000000Z/seg-2.m4s"), Size: &size},
|
||||
{Key: aws.String("hls/r-20260815T000000Z/seg-3.m4s"), Size: &size},
|
||||
{Key: aws.String("hls/playlist.m3u8"), Size: &size},
|
||||
{Key: aws.String("hls/playlist.m3u8.tmp"), Size: &size},
|
||||
},
|
||||
IsTruncated: aws.Bool(false),
|
||||
}, nil
|
||||
@@ -654,23 +695,101 @@ func TestCleanupOldSegments(t *testing.T) {
|
||||
return &s3.DeleteObjectsOutput{}, nil
|
||||
}
|
||||
|
||||
n, err := cleanupOldSegments(context.Background())
|
||||
n, err := cleanupOldRuns(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("cleanupOldSegments error: %v", err)
|
||||
t.Fatalf("cleanupOldRuns error: %v", err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("deleted = %d, want 1 (oldest run)", n)
|
||||
}
|
||||
|
||||
joined := strings.Join(deletedKeys, ",")
|
||||
if !strings.Contains(joined, "hls/r-20260813T000000Z/seg-1.m4s") {
|
||||
t.Fatalf("expected oldest run deleted, got: %v", deletedKeys)
|
||||
}
|
||||
if strings.Contains(joined, "playlist.m3u8") {
|
||||
t.Fatalf("manifest must not be deleted: %v", deletedKeys)
|
||||
}
|
||||
if r2RunsRetained != 2 {
|
||||
t.Fatalf("r2RunsRetained = %d, want 2", r2RunsRetained)
|
||||
}
|
||||
if r2BytesRetained != 2*size {
|
||||
t.Fatalf("r2BytesRetained = %d, want %d", r2BytesRetained, 2*size)
|
||||
}
|
||||
if r2CleanupObjectsTotal != 1 {
|
||||
t.Fatalf("r2CleanupObjectsTotal = %d, want 1", r2CleanupObjectsTotal)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupOldRuns_SizeCap(t *testing.T) {
|
||||
prevClient := r2Client
|
||||
prevBucket := bucket
|
||||
prevList := listObjects
|
||||
prevDelete := deleteObjects
|
||||
prevTotal := r2CleanupObjectsTotal
|
||||
prevRuns := r2RunsRetained
|
||||
prevBytes := r2BytesRetained
|
||||
defer func() {
|
||||
r2Client = prevClient
|
||||
bucket = prevBucket
|
||||
listObjects = prevList
|
||||
deleteObjects = prevDelete
|
||||
r2CleanupObjectsTotal = prevTotal
|
||||
r2RunsRetained = prevRuns
|
||||
r2BytesRetained = prevBytes
|
||||
}()
|
||||
|
||||
bucket = "linearvoid-media"
|
||||
r2Client = &s3.Client{}
|
||||
|
||||
// Force a small cap so the size policy trims even the last 2 runs.
|
||||
os.Setenv("R2_KEEP_RUNS", "2")
|
||||
os.Setenv("R2_MAX_RETENTION_BYTES", "1024")
|
||||
defer os.Unsetenv("R2_KEEP_RUNS")
|
||||
defer os.Unsetenv("R2_MAX_RETENTION_BYTES")
|
||||
|
||||
size := int64(1024)
|
||||
listObjects = func(_ *s3.Client, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
|
||||
return &s3.ListObjectsV2Output{
|
||||
Contents: []types.Object{
|
||||
{Key: aws.String("hls/r-20260813T000000Z/seg-1.m4s"), Size: &size},
|
||||
{Key: aws.String("hls/r-20260814T000000Z/seg-2.m4s"), Size: &size},
|
||||
{Key: aws.String("hls/r-20260815T000000Z/seg-3.m4s"), Size: &size},
|
||||
},
|
||||
IsTruncated: aws.Bool(false),
|
||||
}, nil
|
||||
}
|
||||
|
||||
var deletedKeys []string
|
||||
deleteObjects = func(_ *s3.Client, input *s3.DeleteObjectsInput) (*s3.DeleteObjectsOutput, error) {
|
||||
for _, obj := range input.Delete.Objects {
|
||||
deletedKeys = append(deletedKeys, aws.ToString(obj.Key))
|
||||
}
|
||||
return &s3.DeleteObjectsOutput{}, nil
|
||||
}
|
||||
|
||||
n, err := cleanupOldRuns(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("cleanupOldRuns error: %v", err)
|
||||
}
|
||||
// 3 runs -> keep 2 -> oldest deleted by count; remaining 2 bytes (2048) exceed
|
||||
// the 1024 cap -> the older of the kept two is deleted, leaving only the newest.
|
||||
if n != 2 {
|
||||
t.Fatalf("deleted = %d, want 2", n)
|
||||
}
|
||||
|
||||
joined := strings.Join(deletedKeys, ",")
|
||||
if !strings.Contains(joined, "hls/r-old/seg-1.m4s") || !strings.Contains(joined, "hls/r-old/init.mp4") {
|
||||
t.Fatalf("unexpected deleted keys: %v", deletedKeys)
|
||||
if !strings.Contains(joined, "hls/r-20260813T000000Z/seg-1.m4s") {
|
||||
t.Fatalf("expected oldest run deleted, got: %v", deletedKeys)
|
||||
}
|
||||
if strings.Contains(joined, "playlist.m3u8") {
|
||||
t.Fatalf("manifest must not be deleted: %v", deletedKeys)
|
||||
if !strings.Contains(joined, "hls/r-20260814T000000Z/seg-2.m4s") {
|
||||
t.Fatalf("expected second-oldest run deleted by size cap, got: %v", deletedKeys)
|
||||
}
|
||||
if r2CleanupObjectsTotal != 2 {
|
||||
t.Fatalf("r2CleanupObjectsTotal = %d, want 2", r2CleanupObjectsTotal)
|
||||
if strings.Contains(joined, "hls/r-20260815T000000Z/seg-3.m4s") {
|
||||
t.Fatalf("newest run must be kept, got: %v", deletedKeys)
|
||||
}
|
||||
if r2RunsRetained != 1 {
|
||||
t.Fatalf("r2RunsRetained = %d, want 1", r2RunsRetained)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user