joe/everytab
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fixed icon downloading performance issues

Browse source
This commit is contained in:
Joe Lothan 2026-05-19 10:32:34 -04:00
parent 2745e75408
commit cf17fc42b1
2 changed files with 3 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

10
pipeline/03_icon_download/main.go
View file

@ -46,7 +46,7 @@ func main() {
cfg := Config{}
flag.StringVar(&cfg.DBUrl, "db", "", "Postgres connection string (required)")
flag.StringVar(&cfg.IconsDir, "icons-dir", "icons", "Directory to store downloaded icons")
flag.IntVar(&cfg.BatchSize, "batch-size", 200, "Rows to claim per batch")
flag.IntVar(&cfg.BatchSize, "batch-size", 5000, "Rows to claim per batch")
flag.IntVar(&cfg.Concurrency, "concurrency", 200, "Number of concurrent goroutines")
flag.IntVar(&cfg.Limit, "limit", 0, "Max icons to process (0 = all)")
flag.DurationVar(&cfg.Timeout, "timeout", 10*time.Second, "HTTP request timeout")
@ -79,11 +79,7 @@ func main() {
// Count eligible icons
var total int64
err = pool.QueryRow(ctx, `
SELECT COUNT(*) FROM icons
WHERE scan_state = 'unscanned'
AND (source = 'favicon_ico'
OR rel_sizes IS NULL
OR rel_sizes IN ('16x16','32x32','48x48','64x64'))
SELECT COUNT(*) FROM icons WHERE scan_state = 'unscanned'
`).Scan(&total)
if err != nil {
log.Fatalf("Failed to count icons: %v", err)
@ -119,7 +115,7 @@ func main() {
stats := &Stats{StartedAt: time.Now()}
// Feed icons into a channel so workers never starve waiting for batch claims
iconCh := make(chan IconRow, cfg.Concurrency*2)
iconCh := make(chan IconRow, cfg.BatchSize)
go func() {
defer close(iconCh)
claimed := 0