joe/everytab
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fixed icon downloading performance issues

Browse source
This commit is contained in:
Joe Lothan 2026-05-19 10:32:34 -04:00
parent 2745e75408
commit cf17fc42b1
2 changed files with 3 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

4
pipeline/03_icon_download/db.go
View file

@ -20,10 +20,6 @@ func claimBatch(ctx context.Context, pool *pgxpool.Pool, limit int) ([]IconRow,
WHERE id IN ( WHERE id IN (
SELECT id FROM icons SELECT id FROM icons
WHERE scan_state = 'unscanned' WHERE scan_state = 'unscanned'
AND (source = 'favicon_ico'
OR rel_sizes IS NULL
OR rel_sizes IN ('16x16','32x32','48x48','64x64'))
ORDER BY md5(id::text)
LIMIT $1 LIMIT $1
FOR UPDATE SKIP LOCKED FOR UPDATE SKIP LOCKED
) )

10
pipeline/03_icon_download/main.go
View file

@ -46,7 +46,7 @@ func main() {
cfg := Config{} cfg := Config{}
flag.StringVar(&cfg.DBUrl, "db", "", "Postgres connection string (required)") flag.StringVar(&cfg.DBUrl, "db", "", "Postgres connection string (required)")
flag.StringVar(&cfg.IconsDir, "icons-dir", "icons", "Directory to store downloaded icons") flag.StringVar(&cfg.IconsDir, "icons-dir", "icons", "Directory to store downloaded icons")
flag.IntVar(&cfg.BatchSize, "batch-size", 200, "Rows to claim per batch") flag.IntVar(&cfg.BatchSize, "batch-size", 5000, "Rows to claim per batch")
flag.IntVar(&cfg.Concurrency, "concurrency", 200, "Number of concurrent goroutines") flag.IntVar(&cfg.Concurrency, "concurrency", 200, "Number of concurrent goroutines")
flag.IntVar(&cfg.Limit, "limit", 0, "Max icons to process (0 = all)") flag.IntVar(&cfg.Limit, "limit", 0, "Max icons to process (0 = all)")
flag.DurationVar(&cfg.Timeout, "timeout", 10*time.Second, "HTTP request timeout") flag.DurationVar(&cfg.Timeout, "timeout", 10*time.Second, "HTTP request timeout")
@ -79,11 +79,7 @@ func main() {
// Count eligible icons // Count eligible icons
var total int64 var total int64
err = pool.QueryRow(ctx, ` err = pool.QueryRow(ctx, `
SELECT COUNT(*) FROM icons SELECT COUNT(*) FROM icons WHERE scan_state = 'unscanned'
WHERE scan_state = 'unscanned'
AND (source = 'favicon_ico'
OR rel_sizes IS NULL
OR rel_sizes IN ('16x16','32x32','48x48','64x64'))
`).Scan(&total) `).Scan(&total)
if err != nil { if err != nil {
log.Fatalf("Failed to count icons: %v", err) log.Fatalf("Failed to count icons: %v", err)
@ -119,7 +115,7 @@ func main() {
stats := &Stats{StartedAt: time.Now()} stats := &Stats{StartedAt: time.Now()}
// Feed icons into a channel so workers never starve waiting for batch claims // Feed icons into a channel so workers never starve waiting for batch claims
iconCh := make(chan IconRow, cfg.Concurrency*2) iconCh := make(chan IconRow, cfg.BatchSize)
go func() { go func() {
defer close(iconCh) defer close(iconCh)
claimed := 0 claimed := 0