added icon downloader

pipeline/03_icon_download/s3.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+)
+
+var (
+	s3Client *s3.Client
+	bucket   string
+)
+
+func initS3(bucketName string) error {
+	cfg, err := config.LoadDefaultConfig(context.Background(), config.WithRegion("us-east-1"))
+	if err != nil {
+		return err
+	}
+	s3Client = s3.NewFromConfig(cfg)
+	bucket = bucketName
+	return nil
+}
+
+// s3Exists checks if an object already exists in S3 (for dedup).
+func s3Exists(ctx context.Context, key string) (bool, error) {
+	_, err := s3Client.HeadObject(ctx, &s3.HeadObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+	})
+	if err != nil {
+		var notFound *types.NotFound
+		if errors.As(err, &notFound) {
+			return false, nil
+		}
+		// NoSuchKey error type
+		var nsk *types.NoSuchKey
+		if errors.As(err, &nsk) {
+			return false, nil
+		}
+		// Some S3 errors return 404 as a generic error
+		if ctx.Err() != nil {
+			return false, ctx.Err()
+		}
+		// Treat other errors as "not found" to avoid blocking uploads
+		return false, nil
+	}
+	return true, nil
+}
+
+// s3Upload uploads icon data to S3 with the given key.
+func s3Upload(ctx context.Context, key string, data []byte, contentType string) error {
+	_, err := s3Client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket:      aws.String(bucket),
+		Key:         aws.String(key),
+		Body:        bytes.NewReader(data),
+		ContentType: aws.String(contentType),
+	})
+	return err
+}