joe/everytab
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

optimize db for bulk insert by turning off indexes and vacuum

Browse source
This commit is contained in:
Joe Lothan 2026-05-25 14:16:40 -04:00
parent eec486880a
commit ca90b7071e
1 changed files with 16 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

24
pipeline/02_warc_parse/main.go
View file

@ -87,12 +87,17 @@ func main() {
defer logWriter.Close() defer logWriter.Close()
} }
// Disable autovacuum during heavy writes — it competes for disk I/O // Optimize DB for bulk loading:
// and causes writer stalls. Re-enabled + manual VACUUM at end. // - Disable autovacuum (competes for disk I/O, causes writer stalls)
// - Drop icons indexes (index maintenance during 80M inserts is the main IOPS cost)
// Both are re-enabled/recreated at the end.
if !cfg.DryRun { if !cfg.DryRun {
fmt.Println("Disabling autovacuum on hosts and icons tables...") fmt.Println("Preparing DB for bulk load...")
pool.Exec(ctx, "ALTER TABLE hosts SET (autovacuum_enabled = false)") pool.Exec(ctx, "ALTER TABLE hosts SET (autovacuum_enabled = false)")
pool.Exec(ctx, "ALTER TABLE icons SET (autovacuum_enabled = false)") pool.Exec(ctx, "ALTER TABLE icons SET (autovacuum_enabled = false)")
pool.Exec(ctx, "DROP INDEX IF EXISTS idx_icons_unscanned")
pool.Exec(ctx, "DROP INDEX IF EXISTS idx_icons_host_id")
fmt.Println("Autovacuum disabled, icons indexes dropped")
} }
stats := &Stats{StartedAt: time.Now()} stats := &Stats{StartedAt: time.Now()}
@ -237,14 +242,17 @@ func main() {
} }
writerWg.Wait() writerWg.Wait()
// Re-enable autovacuum and run manual vacuum // Restore DB: recreate indexes, re-enable autovacuum, update planner stats
if !cfg.DryRun { if !cfg.DryRun {
fmt.Println("Re-enabling autovacuum and running VACUUM ANALYZE...") fmt.Println("Recreating icons indexes...")
pool.Exec(ctx, "CREATE INDEX IF NOT EXISTS idx_icons_unscanned ON icons(id) WHERE scan_state = 'unscanned'")
pool.Exec(ctx, "CREATE INDEX IF NOT EXISTS idx_icons_host_id ON icons(host_id)")
fmt.Println("Indexes created. Running ANALYZE...")
pool.Exec(ctx, "ALTER TABLE hosts SET (autovacuum_enabled = true)") pool.Exec(ctx, "ALTER TABLE hosts SET (autovacuum_enabled = true)")
pool.Exec(ctx, "ALTER TABLE icons SET (autovacuum_enabled = true)") pool.Exec(ctx, "ALTER TABLE icons SET (autovacuum_enabled = true)")
pool.Exec(ctx, "VACUUM ANALYZE hosts") pool.Exec(ctx, "ANALYZE hosts")
pool.Exec(ctx, "VACUUM ANALYZE icons") pool.Exec(ctx, "ANALYZE icons")
fmt.Println("VACUUM complete") fmt.Println("ANALYZE complete")
} }
// Print summary // Print summary