joe/everytab
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

switched from s3 to disk for saving icons

Browse source
This commit is contained in:
Joe Lothan 2026-05-18 12:43:50 -04:00
parent 113a261dae
commit 5b3f6a6870
9 changed files with 84 additions and 112 deletions
Download patch file Download diff file Expand all files Collapse all files

30
pipeline/05_bundle_gen/s3.go
View file

@ -4,7 +4,8 @@ import (
"bytes"
"context"
"fmt"
"io"
"os"
"path/filepath"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
@ -23,22 +24,22 @@ func initS3() error {
return nil
}
// s3Download fetches an object from S3.
func s3Download(ctx context.Context, bucket, key string) ([]byte, error) {
resp, err := s3Client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
})
if err != nil {
return nil, err
// iconHashToPath converts a SHA-256 hash to a sharded file path.
func iconHashToPath(iconsDir, hash string) string {
if len(hash) < 6 {
return filepath.Join(iconsDir, hash)
}
defer resp.Body.Close()
return io.ReadAll(resp.Body)
return filepath.Join(iconsDir, hash[:2], hash[2:4], hash[4:6], hash)
}
// readIconFromDisk reads an icon file from the local sharded directory.
func readIconFromDisk(iconsDir, hash string) ([]byte, error) {
return os.ReadFile(iconHashToPath(iconsDir, hash))
}
// s3UploadBundle uploads a bundle JSON to S3.
func s3UploadBundle(ctx context.Context, bucket, key string, data []byte) error {
_, err := s3Client.PutObject(ctx, &s3.PutObjectInput{
func s3UploadBundle(bucket, key string, data []byte) error {
_, err := s3Client.PutObject(context.Background(), &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
Body: bytes.NewReader(data),
@ -48,7 +49,8 @@ func s3UploadBundle(ctx context.Context, bucket, key string, data []byte) error
}
// s3DeletePrefix deletes all objects under a prefix in S3.
func s3DeletePrefix(ctx context.Context, bucket, prefix string) error {
func s3DeletePrefix(bucket, prefix string) error {
ctx := context.Background()
paginator := s3.NewListObjectsV2Paginator(s3Client, &s3.ListObjectsV2Input{
Bucket: aws.String(bucket),
Prefix: aws.String(prefix),