From 7c4572aafb9c2634b08bd10c55636176d9e70a76 Mon Sep 17 00:00:00 2001 From: Joe Lothan Date: Mon, 25 May 2026 22:00:53 -0400 Subject: [PATCH] EVEN MORE CONCURRENCYYYY --- pipeline/01_cc_index/query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/01_cc_index/query.sh b/pipeline/01_cc_index/query.sh index d480486..71d0e85 100755 --- a/pipeline/01_cc_index/query.sh +++ b/pipeline/01_cc_index/query.sh @@ -118,7 +118,7 @@ if [ -d "$LOCAL_INDEX" ] && [ "$(ls -A "$LOCAL_INDEX"/*.parquet 2>/dev/null)" ]; else echo "Downloading CC-Index parquet files..." mkdir -p "$LOCAL_INDEX" - aws configure set default.s3.max_concurrent_requests 50 + aws configure set default.s3.max_concurrent_requests 100 aws s3 sync "$S3_PATH" "$LOCAL_INDEX/" --quiet echo "Downloaded $(ls "$LOCAL_INDEX"/*.parquet | wc -l) parquet files" fi