From 21f2a75ed3bd4258a5020b9a60e7f4d312a058fd Mon Sep 17 00:00:00 2001 From: Joe Lothan Date: Mon, 18 May 2026 00:49:50 -0400 Subject: [PATCH] delete old tab bundles before making new ones --- pipeline/05_bundle_gen/main.go | 10 +++++++++- pipeline/05_bundle_gen/s3.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/pipeline/05_bundle_gen/main.go b/pipeline/05_bundle_gen/main.go index f7525ca..06c531a 100644 --- a/pipeline/05_bundle_gen/main.go +++ b/pipeline/05_bundle_gen/main.go @@ -131,8 +131,16 @@ func main() { } wg.Wait() + // Clean old bundles before writing new ones (avoids orphans if count changed) + if !cfg.DryRun { + fmt.Println("\nCleaning old bundles from S3...") + if err := s3DeletePrefix(ctx, cfg.SiteBucket, "tabs/"); err != nil { + log.Fatalf("Failed to clean old bundles: %v", err) + } + } + // Chunk into bundles and write - fmt.Println("\nWriting bundles...") + fmt.Println("Writing bundles...") bundleCount := 0 var totalBytes int64 diff --git a/pipeline/05_bundle_gen/s3.go b/pipeline/05_bundle_gen/s3.go index 734f7ef..1e02253 100644 --- a/pipeline/05_bundle_gen/s3.go +++ b/pipeline/05_bundle_gen/s3.go @@ -3,11 +3,13 @@ package main import ( "bytes" "context" + "fmt" "io" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" ) var s3Client *s3.Client @@ -44,3 +46,35 @@ func s3UploadBundle(ctx context.Context, bucket, key string, data []byte) error }) return err } + +// s3DeletePrefix deletes all objects under a prefix in S3. +func s3DeletePrefix(ctx context.Context, bucket, prefix string) error { + paginator := s3.NewListObjectsV2Paginator(s3Client, &s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + Prefix: aws.String(prefix), + }) + + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + return err + } + if len(page.Contents) == 0 { + continue + } + + var objects []types.ObjectIdentifier + for _, obj := range page.Contents { + objects = append(objects, types.ObjectIdentifier{Key: obj.Key}) + } + + _, err = s3Client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: aws.String(bucket), + Delete: &types.Delete{Objects: objects}, + }) + if err != nil { + return fmt.Errorf("delete batch: %w", err) + } + } + return nil +}