joe/everytab
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

optimize db for bulk insert by turning off indexes and vacuum

Browse source
This commit is contained in:
Joe Lothan 2026-05-25 14:16:40 -04:00
parent eec486880a
commit ca90b7071e
1 changed files with 16 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

24
pipeline/02_warc_parse/main.go
View file

@ -87,12 +87,17 @@ func main() {
defer logWriter.Close()
}
// Disable autovacuum during heavy writes — it competes for disk I/O
// and causes writer stalls. Re-enabled + manual VACUUM at end.
// Optimize DB for bulk loading:
// - Disable autovacuum (competes for disk I/O, causes writer stalls)
// - Drop icons indexes (index maintenance during 80M inserts is the main IOPS cost)
// Both are re-enabled/recreated at the end.
if !cfg.DryRun {
fmt.Println("Disabling autovacuum on hosts and icons tables...")
fmt.Println("Preparing DB for bulk load...")
pool.Exec(ctx, "ALTER TABLE hosts SET (autovacuum_enabled = false)")
pool.Exec(ctx, "ALTER TABLE icons SET (autovacuum_enabled = false)")
pool.Exec(ctx, "DROP INDEX IF EXISTS idx_icons_unscanned")
pool.Exec(ctx, "DROP INDEX IF EXISTS idx_icons_host_id")
fmt.Println("Autovacuum disabled, icons indexes dropped")
}
stats := &Stats{StartedAt: time.Now()}
@ -237,14 +242,17 @@ func main() {
}
writerWg.Wait()
// Re-enable autovacuum and run manual vacuum
// Restore DB: recreate indexes, re-enable autovacuum, update planner stats
if !cfg.DryRun {
fmt.Println("Re-enabling autovacuum and running VACUUM ANALYZE...")
fmt.Println("Recreating icons indexes...")
pool.Exec(ctx, "CREATE INDEX IF NOT EXISTS idx_icons_unscanned ON icons(id) WHERE scan_state = 'unscanned'")
pool.Exec(ctx, "CREATE INDEX IF NOT EXISTS idx_icons_host_id ON icons(host_id)")
fmt.Println("Indexes created. Running ANALYZE...")
pool.Exec(ctx, "ALTER TABLE hosts SET (autovacuum_enabled = true)")
pool.Exec(ctx, "ALTER TABLE icons SET (autovacuum_enabled = true)")
pool.Exec(ctx, "VACUUM ANALYZE hosts")
pool.Exec(ctx, "VACUUM ANALYZE icons")
fmt.Println("VACUUM complete")
pool.Exec(ctx, "ANALYZE hosts")
pool.Exec(ctx, "ANALYZE icons")
fmt.Println("ANALYZE complete")
}
// Print summary