diff --git a/infra/db-setup.sh b/infra/db-setup.sh index 5341024..954c6cf 100755 --- a/infra/db-setup.sh +++ b/infra/db-setup.sh @@ -15,14 +15,13 @@ NVME_DEV="/dev/nvme1n1" NVME_MOUNT="/data" if [ ! -d "$NVME_MOUNT" ]; then - # Find the NVMe instance store (not the root EBS) - # i3.large has one 475GB NVMe at /dev/nvme1n1 or similar + # Find the NVMe instance store — look for unmounted nvme devices if [ ! -b "$NVME_DEV" ]; then - # Try finding it - NVME_DEV=$(lsblk -dpno NAME,SIZE | grep -v "$(lsblk -dpno NAME /)" | head -1 | awk '{print $1}') + NVME_DEV=$(lsblk -dpno NAME | grep nvme | head -1) if [ -z "$NVME_DEV" ]; then echo "ERROR: Could not find NVMe instance store device" - echo "Run 'lsblk' and set NVME_DEV manually" + echo "Available devices:" + lsblk exit 1 fi fi diff --git a/infra/ec2-userdata.sh b/infra/ec2-userdata.sh index 56bce25..078c29b 100755 --- a/infra/ec2-userdata.sh +++ b/infra/ec2-userdata.sh @@ -2,9 +2,11 @@ set -euo pipefail # EveryTab EC2 Bootstrap -# Run this on the EC2 instance after first SSH connection. +# Runs automatically via cloud-init user_data on first boot. # Installs: Go, DuckDB, Unbound, psql, pg_dump +export HOME=/root + echo "=== EveryTab EC2 Bootstrap ===" # --- File descriptor limits --- @@ -44,7 +46,7 @@ sudo dnf install -y \ echo "--- Installing Go ---" GO_VERSION="1.22.4" if ! command -v go &>/dev/null; then - curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" | sudo tar -C /usr/local -xz + curl -fsSL "https://go.dev/dl/go$${GO_VERSION}.linux-amd64.tar.gz" | sudo tar -C /usr/local -xz echo 'export PATH=$PATH:/usr/local/go/bin:$HOME/go/bin' >> ~/.bashrc export PATH=$PATH:/usr/local/go/bin:$HOME/go/bin fi @@ -54,7 +56,7 @@ go version echo "--- Installing DuckDB ---" DUCKDB_VERSION="1.5.2" if ! command -v duckdb &>/dev/null; then - curl -fsSL "https://github.com/duckdb/duckdb/releases/download/v${DUCKDB_VERSION}/duckdb_cli-linux-amd64.zip" -o /tmp/duckdb.zip + curl -fsSL "https://github.com/duckdb/duckdb/releases/download/v$${DUCKDB_VERSION}/duckdb_cli-linux-amd64.zip" -o /tmp/duckdb.zip cd /tmp && unzip -o duckdb.zip && sudo mv duckdb /usr/local/bin/ && cd - fi duckdb -c "SELECT 'DuckDB OK';" @@ -151,7 +153,7 @@ if [ -n "$REPO_URL" ]; then cd /home/ec2-user/everytab echo "--- Building Go binaries ---" - sudo -u ec2-user bash -c 'export PATH=$PATH:/usr/local/go/bin && cd ~/everytab && go build -o ~/warc_parse ./pipeline/02_warc_parse/ && go build -o ~/icon_download ./pipeline/03_icon_download/ && go build -o ~/bundle_gen ./pipeline/05_bundle_gen/' + sudo -u ec2-user bash -c 'export PATH=/usr/local/go/bin:$PATH && cd /home/ec2-user/everytab && go build -o /home/ec2-user/warc_parse ./pipeline/02_warc_parse/ && go build -o /home/ec2-user/icon_download ./pipeline/03_icon_download/ && go build -o /home/ec2-user/bundle_gen ./pipeline/05_bundle_gen/' # Wait for DB to be ready, then apply schema echo "--- Waiting for database ---"