joe/everytab
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix ec2 provisioning for cloud init

Browse source
This commit is contained in:
Joe Lothan 2026-05-25 21:43:12 -04:00
parent 7c5573c24d
commit a92c838d23
2 changed files with 10 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

9
infra/db-setup.sh
View file

@ -15,14 +15,13 @@ NVME_DEV="/dev/nvme1n1"
NVME_MOUNT="/data" NVME_MOUNT="/data"
if [ ! -d "$NVME_MOUNT" ]; then if [ ! -d "$NVME_MOUNT" ]; then
# Find the NVMe instance store (not the root EBS) # Find the NVMe instance store — look for unmounted nvme devices
# i3.large has one 475GB NVMe at /dev/nvme1n1 or similar
if [ ! -b "$NVME_DEV" ]; then if [ ! -b "$NVME_DEV" ]; then
# Try finding it NVME_DEV=$(lsblk -dpno NAME | grep nvme | head -1)
NVME_DEV=$(lsblk -dpno NAME,SIZE | grep -v "$(lsblk -dpno NAME /)" | head -1 | awk '{print $1}')
if [ -z "$NVME_DEV" ]; then if [ -z "$NVME_DEV" ]; then
echo "ERROR: Could not find NVMe instance store device" echo "ERROR: Could not find NVMe instance store device"
echo "Run 'lsblk' and set NVME_DEV manually" echo "Available devices:"
lsblk
exit 1 exit 1
fi fi
fi fi

10
infra/ec2-userdata.sh
View file

@ -2,9 +2,11 @@
set -euo pipefail set -euo pipefail
# EveryTab EC2 Bootstrap # EveryTab EC2 Bootstrap
# Run this on the EC2 instance after first SSH connection. # Runs automatically via cloud-init user_data on first boot.
# Installs: Go, DuckDB, Unbound, psql, pg_dump # Installs: Go, DuckDB, Unbound, psql, pg_dump
export HOME=/root
echo "=== EveryTab EC2 Bootstrap ===" echo "=== EveryTab EC2 Bootstrap ==="
# --- File descriptor limits --- # --- File descriptor limits ---
@ -44,7 +46,7 @@ sudo dnf install -y \
echo "--- Installing Go ---" echo "--- Installing Go ---"
GO_VERSION="1.22.4" GO_VERSION="1.22.4"
if ! command -v go &>/dev/null; then if ! command -v go &>/dev/null; then
curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" | sudo tar -C /usr/local -xz curl -fsSL "https://go.dev/dl/go$${GO_VERSION}.linux-amd64.tar.gz" | sudo tar -C /usr/local -xz
echo 'export PATH=$PATH:/usr/local/go/bin:$HOME/go/bin' >> ~/.bashrc echo 'export PATH=$PATH:/usr/local/go/bin:$HOME/go/bin' >> ~/.bashrc
export PATH=$PATH:/usr/local/go/bin:$HOME/go/bin export PATH=$PATH:/usr/local/go/bin:$HOME/go/bin
fi fi
@ -54,7 +56,7 @@ go version
echo "--- Installing DuckDB ---" echo "--- Installing DuckDB ---"
DUCKDB_VERSION="1.5.2" DUCKDB_VERSION="1.5.2"
if ! command -v duckdb &>/dev/null; then if ! command -v duckdb &>/dev/null; then
curl -fsSL "https://github.com/duckdb/duckdb/releases/download/v${DUCKDB_VERSION}/duckdb_cli-linux-amd64.zip" -o /tmp/duckdb.zip curl -fsSL "https://github.com/duckdb/duckdb/releases/download/v$${DUCKDB_VERSION}/duckdb_cli-linux-amd64.zip" -o /tmp/duckdb.zip
cd /tmp && unzip -o duckdb.zip && sudo mv duckdb /usr/local/bin/ && cd - cd /tmp && unzip -o duckdb.zip && sudo mv duckdb /usr/local/bin/ && cd -
fi fi
duckdb -c "SELECT 'DuckDB OK';" duckdb -c "SELECT 'DuckDB OK';"
@ -151,7 +153,7 @@ if [ -n "$REPO_URL" ]; then
cd /home/ec2-user/everytab cd /home/ec2-user/everytab
echo "--- Building Go binaries ---" echo "--- Building Go binaries ---"
sudo -u ec2-user bash -c 'export PATH=$PATH:/usr/local/go/bin && cd ~/everytab && go build -o ~/warc_parse ./pipeline/02_warc_parse/ && go build -o ~/icon_download ./pipeline/03_icon_download/ && go build -o ~/bundle_gen ./pipeline/05_bundle_gen/' sudo -u ec2-user bash -c 'export PATH=/usr/local/go/bin:$PATH && cd /home/ec2-user/everytab && go build -o /home/ec2-user/warc_parse ./pipeline/02_warc_parse/ && go build -o /home/ec2-user/icon_download ./pipeline/03_icon_download/ && go build -o /home/ec2-user/bundle_gen ./pipeline/05_bundle_gen/'
# Wait for DB to be ready, then apply schema # Wait for DB to be ready, then apply schema
echo "--- Waiting for database ---" echo "--- Waiting for database ---"