From 6c64ffcf949170ba1714fd389c018c6eaac88d15 Mon Sep 17 00:00:00 2001
From: Joe Lothan <joe@lothan.net>
Date: Mon, 25 May 2026 21:53:57 -0400
Subject: [PATCH] upped concurrent s3 requests to speed up cc-index download

---
 pipeline/01_cc_index/query.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pipeline/01_cc_index/query.sh b/pipeline/01_cc_index/query.sh
index 05634bf..d480486 100755
--- a/pipeline/01_cc_index/query.sh
+++ b/pipeline/01_cc_index/query.sh
@@ -118,6 +118,7 @@ if [ -d "$LOCAL_INDEX" ] && [ "$(ls -A "$LOCAL_INDEX"/*.parquet 2>/dev/null)" ];
 else
     echo "Downloading CC-Index parquet files..."
     mkdir -p "$LOCAL_INDEX"
+    aws configure set default.s3.max_concurrent_requests 50
     aws s3 sync "$S3_PATH" "$LOCAL_INDEX/" --quiet
     echo "Downloaded $(ls "$LOCAL_INDEX"/*.parquet | wc -l) parquet files"
 fi