joe/everytab
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

switched from s3 to disk for saving icons

Browse source
This commit is contained in:
Joe Lothan 2026-05-18 12:43:50 -04:00
parent 113a261dae
commit 5b3f6a6870
9 changed files with 84 additions and 112 deletions
Download patch file Download diff file Expand all files Collapse all files

16
pipeline/03_icon_download/main.go
View file

@ -15,7 +15,7 @@ import (
type Config struct {
DBUrl string
S3Bucket string
IconsDir string
BatchSize int
Concurrency int
Limit int
@ -45,13 +45,13 @@ type Stats struct {
func main() {
cfg := Config{}
flag.StringVar(&cfg.DBUrl, "db", "", "Postgres connection string (required)")
flag.StringVar(&cfg.S3Bucket, "s3-bucket", "everytab-icons", "S3 bucket for icons")
flag.StringVar(&cfg.IconsDir, "icons-dir", "icons", "Directory to store downloaded icons")
flag.IntVar(&cfg.BatchSize, "batch-size", 200, "Rows to claim per batch")
flag.IntVar(&cfg.Concurrency, "concurrency", 200, "Number of concurrent goroutines")
flag.IntVar(&cfg.Limit, "limit", 0, "Max icons to process (0 = all)")
flag.DurationVar(&cfg.Timeout, "timeout", 10*time.Second, "HTTP request timeout")
flag.Int64Var(&cfg.MaxSize, "max-size", 512*1024, "Max icon download size in bytes")
flag.BoolVar(&cfg.DryRun, "dry-run", false, "Download but don't upload to S3 or update DB")
flag.BoolVar(&cfg.DryRun, "dry-run", false, "Download but don't write to disk or update DB")
flag.StringVar(&cfg.LogFile, "log-file", "", "Mirror log lines to this file")
flag.BoolVar(&cfg.LogErrors, "log-errors-only", false, "Only write errors to log file")
flag.Parse()
@ -64,9 +64,9 @@ func main() {
ctx := context.Background()
// Init S3
if err := initS3(cfg.S3Bucket); err != nil {
log.Fatalf("Failed to init S3: %v", err)
// Init storage
if err := initStorage(cfg.IconsDir); err != nil {
log.Fatalf("Failed to init storage: %v", err)
}
// Init DB pool
@ -103,7 +103,7 @@ func main() {
fmt.Printf("Concurrency: %d\n", cfg.Concurrency)
fmt.Printf("Timeout: %s\n", cfg.Timeout)
fmt.Printf("Max size: %dKB\n", cfg.MaxSize/1024)
fmt.Printf("S3 bucket: %s\n", cfg.S3Bucket)
fmt.Printf("Icons dir: %s\n", cfg.IconsDir)
fmt.Printf("Dry run: %v\n\n", cfg.DryRun)
// Setup log file
@ -168,7 +168,7 @@ func main() {
}
}()
result := processIcon(ctx, icon, cfg)
result := processIcon(icon, cfg)
// Log line
logLine := formatLogLine(icon, result)