joe/everytab
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

updated number of async writers to warc_parse to accomidate faster db nvme write speeds

Browse source
This commit is contained in:
Joe Lothan 2026-05-25 19:41:25 -04:00
parent bfb7d8f883
commit a819dabb57
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
pipeline/02_warc_parse/main.go
View file

@ -204,7 +204,7 @@ func main() {
}() }()
// Stage 3: DB writers — multiple goroutines batch writes for efficiency // Stage 3: DB writers — multiple goroutines batch writes for efficiency
const numWriters = 3 const numWriters = 8
var writerWg sync.WaitGroup var writerWg sync.WaitGroup
if !cfg.DryRun { if !cfg.DryRun {
for w := 0; w < numWriters; w++ { for w := 0; w < numWriters; w++ {