joe/everytab
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

upped concurrent s3 requests to speed up cc-index download

Browse source
This commit is contained in:
Joe Lothan 2026-05-25 21:53:57 -04:00
parent 4bfe165fac
commit 6c64ffcf94
1 changed files with 1 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
pipeline/01_cc_index/query.sh
View file

@ -118,6 +118,7 @@ if [ -d "$LOCAL_INDEX" ] && [ "$(ls -A "$LOCAL_INDEX"/*.parquet 2>/dev/null)" ];
else else
echo "Downloading CC-Index parquet files..." echo "Downloading CC-Index parquet files..."
mkdir -p "$LOCAL_INDEX" mkdir -p "$LOCAL_INDEX"
aws configure set default.s3.max_concurrent_requests 50
aws s3 sync "$S3_PATH" "$LOCAL_INDEX/" --quiet aws s3 sync "$S3_PATH" "$LOCAL_INDEX/" --quiet
echo "Downloaded $(ls "$LOCAL_INDEX"/*.parquet | wc -l) parquet files" echo "Downloaded $(ls "$LOCAL_INDEX"/*.parquet | wc -l) parquet files"
fi fi