joe/everytab
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

switched from s3 to disk for saving icons

Browse source
This commit is contained in:
Joe Lothan 2026-05-18 12:43:50 -04:00
parent 113a261dae
commit 5b3f6a6870
9 changed files with 84 additions and 112 deletions
Download patch file Download diff file Expand all files Collapse all files

14
pipeline/03_icon_download/download.go
View file

@ -1,7 +1,6 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
@ -12,8 +11,8 @@ import (
"time"
)
// processIcon downloads, validates, and uploads a single icon.
func processIcon(ctx context.Context, icon IconRow, cfg Config) DownloadResult {
// processIcon downloads, validates, and stores a single icon.
func processIcon(icon IconRow, cfg Config) DownloadResult {
// Download
data, contentType, err := downloadIcon(icon.URL, cfg.Timeout, cfg.MaxSize)
if err != nil {
@ -39,15 +38,14 @@ func processIcon(ctx context.Context, icon IconRow, cfg Config) DownloadResult {
hash := sha256.Sum256(data)
s3Key := hex.EncodeToString(hash[:])
// Upload to S3 (skip if already exists — dedup)
// Write to disk (skip if already exists — dedup)
dedup := false
if !cfg.DryRun {
exists, err := s3Exists(ctx, s3Key)
if err == nil && exists {
if iconExists(s3Key) {
dedup = true
} else {
if err := s3Upload(ctx, s3Key, data, contentType); err != nil {
return DownloadResult{Err: fmt.Sprintf("s3 upload: %v", err), ErrType: "other"}
if err := iconWrite(s3Key, data); err != nil {
return DownloadResult{Err: fmt.Sprintf("disk write: %v", err), ErrType: "other"}
}
}
}