joe/everytab
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

upped buffer sizes and switched to 2xlarge to increase speed

Browse source
This commit is contained in:
Joe Lothan 2026-05-20 12:59:12 -04:00
parent 1df9a234cf
commit 8dce702e8d
5 changed files with 10 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

4
pipeline/02_warc_parse/main.go
View file

@ -106,8 +106,8 @@ func main() {
// [DB fetcher] → hostCh → [N workers] → resultCh → [DB writer]
// Workers do S3 fetch + parsing (I/O-bound). Writer batches DB writes.
hostCh := make(chan Host, cfg.BatchSize)
resultCh := make(chan WorkResult, cfg.Concurrency)
hostCh := make(chan Host, 20000)
resultCh := make(chan WorkResult, 1000)
// Stage 1: DB fetcher — continuously fetches pages into hostCh
go func() {