From a819dabb57ab09e1be40241fae98d6bc4cdb342d Mon Sep 17 00:00:00 2001
From: Joe Lothan <joe@lothan.net>
Date: Mon, 25 May 2026 19:41:25 -0400
Subject: [PATCH] updated number of async writers to warc_parse to accomidate
 faster db nvme write speeds

---
 pipeline/02_warc_parse/main.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pipeline/02_warc_parse/main.go b/pipeline/02_warc_parse/main.go
index 3895b15..16766b4 100644
--- a/pipeline/02_warc_parse/main.go
+++ b/pipeline/02_warc_parse/main.go
@@ -204,7 +204,7 @@ func main() {
 	}()
 
 	// Stage 3: DB writers — multiple goroutines batch writes for efficiency
-	const numWriters = 3
+	const numWriters = 8
 	var writerWg sync.WaitGroup
 	if !cfg.DryRun {
 		for w := 0; w < numWriters; w++ {