From 426abe1c905e521e73b5e4561eaf13dac35140ce Mon Sep 17 00:00:00 2001 From: Joe Lothan Date: Wed, 20 May 2026 09:47:18 -0400 Subject: [PATCH] upped concurrency of icon downloading --- pipeline/03_icon_download/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/03_icon_download/main.go b/pipeline/03_icon_download/main.go index 262f0b6..18b9d77 100644 --- a/pipeline/03_icon_download/main.go +++ b/pipeline/03_icon_download/main.go @@ -48,7 +48,7 @@ func main() { flag.StringVar(&cfg.DBUrl, "db", "", "Postgres connection string (required)") flag.StringVar(&cfg.IconsDir, "icons-dir", "icons", "Directory to store downloaded icons") flag.IntVar(&cfg.BatchSize, "batch-size", 5000, "Rows to claim per batch") - flag.IntVar(&cfg.Concurrency, "concurrency", 200, "Number of concurrent goroutines") + flag.IntVar(&cfg.Concurrency, "concurrency", 1000, "Number of concurrent goroutines") flag.IntVar(&cfg.Limit, "limit", 0, "Max icons to process (0 = all)") flag.DurationVar(&cfg.Timeout, "timeout", 10*time.Second, "HTTP request timeout") flag.Int64Var(&cfg.MaxSize, "max-size", 512*1024, "Max icon download size in bytes")