automated ec2 setup and build

infra/ec2-userdata.sh

@@ -136,11 +136,41 @@ echo -n "DuckDB: "; duckdb -c "SELECT version();" -noheader -csv
 echo -n "Unbound: "; dig +short example.com @127.0.0.1 | head -1
 echo -n "psql: "; psql --version
 
+echo ""
+
+# --- Database Connection ---
+DB_IP="${db_private_ip}"
+export DATABASE_URL="postgres://everytab@$${DB_IP}:5432/everytab"
+echo "export DATABASE_URL='postgres://everytab@$${DB_IP}:5432/everytab'" >> /home/ec2-user/.bashrc
+
+# --- Clone Repo + Build ---
+REPO_URL="${repo_url}"
+if [ -n "$REPO_URL" ]; then
+    echo "--- Cloning repo ---"
+    sudo -u ec2-user git clone "$REPO_URL" /home/ec2-user/everytab
+    cd /home/ec2-user/everytab
+
+    echo "--- Building Go binaries ---"
+    sudo -u ec2-user bash -c 'export PATH=$PATH:/usr/local/go/bin && cd ~/everytab && go build -o ~/warc_parse ./pipeline/02_warc_parse/ && go build -o ~/icon_download ./pipeline/03_icon_download/ && go build -o ~/bundle_gen ./pipeline/05_bundle_gen/'
+
+    # Wait for DB to be ready, then apply schema
+    echo "--- Waiting for database ---"
+    for i in $(seq 1 60); do
+        if pg_isready -h "$DB_IP" -q 2>/dev/null; then
+            echo "Database ready"
+            sudo -u ec2-user psql "$DATABASE_URL" -f /home/ec2-user/everytab/pipeline/01_cc_index/schema.sql
+            echo "Schema applied"
+            break
+        fi
+        sleep 5
+    done
+else
+    echo "No repo_url set — clone manually"
+fi
+
 echo ""
 echo "=== Bootstrap Complete ==="
 echo ""
-echo "Next: set up your database connection string."
-echo "  export DATABASE_URL='postgres://everytab:PASSWORD@RDS_ENDPOINT:5432/everytab'"
+echo "DATABASE_URL=$DATABASE_URL"
 echo ""
-echo "Test connection:"
-echo "  psql \$DATABASE_URL -c 'SELECT 1;'"
+echo "Ready to run the pipeline. See pipeline/README.md for usage."