joe/everytab
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

upped concurrency of icon downloading

Browse source
This commit is contained in:
Joe Lothan 2026-05-20 09:47:18 -04:00
parent 3bc355e503
commit 426abe1c90
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
pipeline/03_icon_download/main.go
View file

@ -48,7 +48,7 @@ func main() {
flag.StringVar(&cfg.DBUrl, "db", "", "Postgres connection string (required)") flag.StringVar(&cfg.DBUrl, "db", "", "Postgres connection string (required)")
flag.StringVar(&cfg.IconsDir, "icons-dir", "icons", "Directory to store downloaded icons") flag.StringVar(&cfg.IconsDir, "icons-dir", "icons", "Directory to store downloaded icons")
flag.IntVar(&cfg.BatchSize, "batch-size", 5000, "Rows to claim per batch") flag.IntVar(&cfg.BatchSize, "batch-size", 5000, "Rows to claim per batch")
flag.IntVar(&cfg.Concurrency, "concurrency", 200, "Number of concurrent goroutines") flag.IntVar(&cfg.Concurrency, "concurrency", 1000, "Number of concurrent goroutines")
flag.IntVar(&cfg.Limit, "limit", 0, "Max icons to process (0 = all)") flag.IntVar(&cfg.Limit, "limit", 0, "Max icons to process (0 = all)")
flag.DurationVar(&cfg.Timeout, "timeout", 10*time.Second, "HTTP request timeout") flag.DurationVar(&cfg.Timeout, "timeout", 10*time.Second, "HTTP request timeout")
flag.Int64Var(&cfg.MaxSize, "max-size", 512*1024, "Max icon download size in bytes") flag.Int64Var(&cfg.MaxSize, "max-size", 512*1024, "Max icon download size in bytes")