Compare commits
No commits in common. "79881fce4b40d2bdfe8ea0a29adf52e32a008b88" and "1343df1a41603d5238ad9516afb8c96d2238d7f7" have entirely different histories.
79881fce4b
...
1343df1a41
5 changed files with 37 additions and 168 deletions
|
|
@ -191,6 +191,9 @@
|
||||||
min-width: 80px;
|
min-width: 80px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.browser-safari .tab .tab-icon {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
|
||||||
.browser-safari .tab:hover {
|
.browser-safari .tab:hover {
|
||||||
background: var(--tab-hover);
|
background: var(--tab-hover);
|
||||||
|
|
@ -205,6 +208,9 @@
|
||||||
box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
|
box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.browser-safari .tab-active .tab-icon {
|
||||||
|
display: inline;
|
||||||
|
}
|
||||||
|
|
||||||
.tab-about .tab-title {
|
.tab-about .tab-title {
|
||||||
font-weight: 600;
|
font-weight: 600;
|
||||||
|
|
@ -297,11 +303,17 @@
|
||||||
border-radius: 8px;
|
border-radius: 8px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.browser-safari .iframe-urlbar .url-title {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
|
||||||
.iframe-urlbar .tab-icon {
|
.iframe-urlbar .tab-icon {
|
||||||
flex-shrink: 0;
|
flex-shrink: 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.url-title-mobile {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
|
||||||
.iframe-urlbar .url-text {
|
.iframe-urlbar .url-text {
|
||||||
font-size: 14px;
|
font-size: 14px;
|
||||||
|
|
@ -330,36 +342,26 @@
|
||||||
min-width: 0;
|
min-width: 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Mobile: title above address bar */
|
||||||
@media (max-width: 640px) {
|
@media (max-width: 640px) {
|
||||||
.iframe-urlbar .url-title {
|
.iframe-toolbar {
|
||||||
display: none;
|
flex-wrap: wrap;
|
||||||
}
|
}
|
||||||
}
|
.iframe-toolbar .url-title-mobile {
|
||||||
|
width: 100%;
|
||||||
/* Title bar above address bar — hidden on desktop (title goes in urlbar) */
|
|
||||||
.iframe-titlebar {
|
|
||||||
display: none;
|
|
||||||
padding: 6px 12px 2px;
|
|
||||||
background: var(--tab-bg);
|
|
||||||
font-size: 13px;
|
font-size: 13px;
|
||||||
color: var(--text);
|
color: var(--text);
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
text-overflow: ellipsis;
|
text-overflow: ellipsis;
|
||||||
white-space: nowrap;
|
white-space: nowrap;
|
||||||
|
padding: 0 4px 4px;
|
||||||
|
order: -1;
|
||||||
}
|
}
|
||||||
|
.iframe-urlbar .url-title {
|
||||||
@media (max-width: 640px) {
|
display: none;
|
||||||
.iframe-titlebar {
|
|
||||||
display: block;
|
|
||||||
text-align: center;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Paused marquee row */
|
|
||||||
.tab-row.paused {
|
|
||||||
animation-play-state: paused !important;
|
|
||||||
}
|
|
||||||
|
|
||||||
.iframe-close {
|
.iframe-close {
|
||||||
background: none;
|
background: none;
|
||||||
border: none;
|
border: none;
|
||||||
|
|
@ -377,7 +379,7 @@
|
||||||
|
|
||||||
.iframe-viewer iframe {
|
.iframe-viewer iframe {
|
||||||
width: 100%;
|
width: 100%;
|
||||||
height: calc(75vh - 78px);
|
height: calc(75vh - 56px);
|
||||||
border: none;
|
border: none;
|
||||||
background: white;
|
background: white;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -17,8 +17,6 @@ const loadingEl = document.getElementById("loading");
|
||||||
// Detect browser and OS, set classes on body for tab styling
|
// Detect browser and OS, set classes on body for tab styling
|
||||||
function detectBrowser() {
|
function detectBrowser() {
|
||||||
const ua = navigator.userAgent;
|
const ua = navigator.userAgent;
|
||||||
// Mobile defaults to chrome styling
|
|
||||||
if (/iPhone|iPad|Android/.test(ua)) return "chrome";
|
|
||||||
if (ua.includes("Firefox")) return "firefox";
|
if (ua.includes("Firefox")) return "firefox";
|
||||||
if (ua.includes("Edg/")) return "edge";
|
if (ua.includes("Edg/")) return "edge";
|
||||||
if (ua.includes("Safari") && !ua.includes("Chrome") && !ua.includes("CriOS")) return "safari";
|
if (ua.includes("Safari") && !ua.includes("Chrome") && !ua.includes("CriOS")) return "safari";
|
||||||
|
|
@ -157,9 +155,7 @@ function startRowAnimation(row) {
|
||||||
let pos = stagger * halfWidth;
|
let pos = stagger * halfWidth;
|
||||||
let lastTime = null;
|
let lastTime = null;
|
||||||
|
|
||||||
row._rafId = true;
|
|
||||||
function tick(now) {
|
function tick(now) {
|
||||||
if (row._paused) { lastTime = null; requestAnimationFrame(tick); return; }
|
|
||||||
if (lastTime === null) { lastTime = now; requestAnimationFrame(tick); return; }
|
if (lastTime === null) { lastTime = now; requestAnimationFrame(tick); return; }
|
||||||
pos += ((now - lastTime) / 1000) * pxPerSec;
|
pos += ((now - lastTime) / 1000) * pxPerSec;
|
||||||
lastTime = now;
|
lastTime = now;
|
||||||
|
|
@ -264,7 +260,6 @@ window.addEventListener("scroll", async () => {
|
||||||
// Inline iframe viewer
|
// Inline iframe viewer
|
||||||
let activeViewer = null;
|
let activeViewer = null;
|
||||||
let activeTab = null;
|
let activeTab = null;
|
||||||
let activeRow = null;
|
|
||||||
|
|
||||||
function openInlineViewer(tabEl, entry, url) {
|
function openInlineViewer(tabEl, entry, url) {
|
||||||
// Close existing viewer
|
// Close existing viewer
|
||||||
|
|
@ -277,25 +272,10 @@ function openInlineViewer(tabEl, entry, url) {
|
||||||
// Find the row this tab belongs to
|
// Find the row this tab belongs to
|
||||||
const row = tabEl.closest(".tab-row");
|
const row = tabEl.closest(".tab-row");
|
||||||
|
|
||||||
// Pause the marquee on the selected tab's row
|
|
||||||
row.classList.add("paused");
|
|
||||||
if (row._rafId) {
|
|
||||||
// Firefox rAF: store paused state
|
|
||||||
row._paused = true;
|
|
||||||
} else {
|
|
||||||
// CSS animation: getAnimations() to pause
|
|
||||||
row.getAnimations().forEach(a => a.pause());
|
|
||||||
}
|
|
||||||
|
|
||||||
// Build the viewer
|
// Build the viewer
|
||||||
const viewer = document.createElement("div");
|
const viewer = document.createElement("div");
|
||||||
viewer.className = "iframe-viewer";
|
viewer.className = "iframe-viewer";
|
||||||
|
|
||||||
// Title bar
|
|
||||||
const titlebar = document.createElement("div");
|
|
||||||
titlebar.className = "iframe-titlebar";
|
|
||||||
titlebar.textContent = entry.title || entry.url;
|
|
||||||
|
|
||||||
// Toolbar (address bar area)
|
// Toolbar (address bar area)
|
||||||
const toolbar = document.createElement("div");
|
const toolbar = document.createElement("div");
|
||||||
toolbar.className = "iframe-toolbar";
|
toolbar.className = "iframe-toolbar";
|
||||||
|
|
@ -326,6 +306,12 @@ function openInlineViewer(tabEl, entry, url) {
|
||||||
|
|
||||||
toolbar.appendChild(urlbar);
|
toolbar.appendChild(urlbar);
|
||||||
|
|
||||||
|
// Mobile: title shown above the URL bar (hidden on desktop via CSS)
|
||||||
|
const mobileTitle = document.createElement("span");
|
||||||
|
mobileTitle.className = "url-title-mobile";
|
||||||
|
mobileTitle.textContent = entry.title || "";
|
||||||
|
toolbar.appendChild(mobileTitle);
|
||||||
|
|
||||||
const close = document.createElement("button");
|
const close = document.createElement("button");
|
||||||
close.className = "iframe-close";
|
close.className = "iframe-close";
|
||||||
close.textContent = "✕";
|
close.textContent = "✕";
|
||||||
|
|
@ -336,14 +322,12 @@ function openInlineViewer(tabEl, entry, url) {
|
||||||
iframe.sandbox = "allow-scripts allow-same-origin allow-forms";
|
iframe.sandbox = "allow-scripts allow-same-origin allow-forms";
|
||||||
iframe.src = url;
|
iframe.src = url;
|
||||||
|
|
||||||
viewer.appendChild(titlebar);
|
|
||||||
viewer.appendChild(toolbar);
|
viewer.appendChild(toolbar);
|
||||||
viewer.appendChild(iframe);
|
viewer.appendChild(iframe);
|
||||||
|
|
||||||
// Insert after the row
|
// Insert after the row
|
||||||
row.after(viewer);
|
row.after(viewer);
|
||||||
activeViewer = viewer;
|
activeViewer = viewer;
|
||||||
activeRow = row;
|
|
||||||
|
|
||||||
// Scroll so the viewer is visible
|
// Scroll so the viewer is visible
|
||||||
viewer.scrollIntoView({ behavior: "smooth", block: "start" });
|
viewer.scrollIntoView({ behavior: "smooth", block: "start" });
|
||||||
|
|
@ -354,15 +338,6 @@ function closeInlineViewer() {
|
||||||
activeViewer.remove();
|
activeViewer.remove();
|
||||||
activeViewer = null;
|
activeViewer = null;
|
||||||
}
|
}
|
||||||
if (activeRow) {
|
|
||||||
activeRow.classList.remove("paused");
|
|
||||||
if (activeRow._paused) {
|
|
||||||
activeRow._paused = false;
|
|
||||||
} else {
|
|
||||||
activeRow.getAnimations().forEach(a => a.play());
|
|
||||||
}
|
|
||||||
activeRow = null;
|
|
||||||
}
|
|
||||||
if (activeTab) {
|
if (activeTab) {
|
||||||
activeTab.classList.remove("tab-active");
|
activeTab.classList.remove("tab-active");
|
||||||
activeTab = null;
|
activeTab = null;
|
||||||
|
|
|
||||||
|
|
@ -9,14 +9,6 @@ export HOME=/root
|
||||||
|
|
||||||
echo "=== EveryTab EC2 Bootstrap ==="
|
echo "=== EveryTab EC2 Bootstrap ==="
|
||||||
|
|
||||||
# --- EBS readahead ---
|
|
||||||
# Large readahead improves bundle gen throughput by prefetching icon files into page cache.
|
|
||||||
# 16MB readahead (32768 sectors × 512 bytes). Safe for all pipeline stages.
|
|
||||||
echo "--- Setting EBS readahead ---"
|
|
||||||
ROOT_DEV=$(findmnt -no SOURCE / | sed 's/p[0-9]*$//')
|
|
||||||
sudo blockdev --setra 32768 "$ROOT_DEV"
|
|
||||||
echo "Readahead: $(blockdev --getra "$ROOT_DEV") sectors on $ROOT_DEV"
|
|
||||||
|
|
||||||
# --- File descriptor limits ---
|
# --- File descriptor limits ---
|
||||||
echo "--- Raising file descriptor limits ---"
|
echo "--- Raising file descriptor limits ---"
|
||||||
echo '* soft nofile 65536' | sudo tee -a /etc/security/limits.conf
|
echo '* soft nofile 65536' | sudo tee -a /etc/security/limits.conf
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,7 @@ import (
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
|
"math/rand"
|
||||||
"os"
|
"os"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
|
|
@ -137,6 +138,9 @@ func main() {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
|
rand.Shuffle(len(icons), func(i, j int) {
|
||||||
|
icons[i], icons[j] = icons[j], icons[i]
|
||||||
|
})
|
||||||
for _, icon := range icons {
|
for _, icon := range icons {
|
||||||
iconCh <- icon
|
iconCh <- icon
|
||||||
}
|
}
|
||||||
|
|
|
||||||
104
pipeline/run.sh
104
pipeline/run.sh
|
|
@ -1,104 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
# Full pipeline run — chain all stages sequentially.
|
|
||||||
# Run in tmux. Monitor from another pane with psql or htop.
|
|
||||||
# Stops on any failure. Resume by commenting out completed stages.
|
|
||||||
|
|
||||||
usage() {
|
|
||||||
cat <<'EOF'
|
|
||||||
Usage: ./pipeline/run.sh --db-url DATABASE_URL [OPTIONS]
|
|
||||||
|
|
||||||
Required:
|
|
||||||
--db-url URL Postgres connection string
|
|
||||||
|
|
||||||
Optional:
|
|
||||||
--limit N CC-Index host limit (default: 0 = all)
|
|
||||||
--icons-dir DIR Icon storage directory (default: ~/icons)
|
|
||||||
--site-bucket NAME S3 bucket for bundles (default: everytab-site)
|
|
||||||
--help Show this help message
|
|
||||||
|
|
||||||
Example:
|
|
||||||
./pipeline/run.sh --db-url "$DATABASE_URL"
|
|
||||||
./pipeline/run.sh --db-url "$DATABASE_URL" --limit 3000000
|
|
||||||
EOF
|
|
||||||
exit 0
|
|
||||||
}
|
|
||||||
|
|
||||||
# Defaults
|
|
||||||
DB_URL=""
|
|
||||||
LIMIT=0
|
|
||||||
ICONS_DIR="$HOME/icons"
|
|
||||||
SITE_BUCKET="everytab-site"
|
|
||||||
|
|
||||||
if [ $# -eq 0 ]; then usage; fi
|
|
||||||
|
|
||||||
while [ $# -gt 0 ]; do
|
|
||||||
case "$1" in
|
|
||||||
--help) usage ;;
|
|
||||||
--db-url) DB_URL="$2"; shift 2 ;;
|
|
||||||
--limit) LIMIT="$2"; shift 2 ;;
|
|
||||||
--icons-dir) ICONS_DIR="$2"; shift 2 ;;
|
|
||||||
--site-bucket) SITE_BUCKET="$2"; shift 2 ;;
|
|
||||||
*) echo "Unknown option: $1"; usage ;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
|
|
||||||
if [ -z "$DB_URL" ]; then
|
|
||||||
echo "ERROR: --db-url is required"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
||||||
REPO_DIR="$(dirname "$SCRIPT_DIR")"
|
|
||||||
START_TIME=$(date +%s)
|
|
||||||
|
|
||||||
echo "=========================================="
|
|
||||||
echo " EveryTab Pipeline"
|
|
||||||
echo " $(date)"
|
|
||||||
echo " Limit: $LIMIT (0 = full run)"
|
|
||||||
echo "=========================================="
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
# --- Stage 1: CC-Index ---
|
|
||||||
echo ">>> Stage 1: CC-Index Query"
|
|
||||||
"$SCRIPT_DIR/01_cc_index/query.sh" --db-url "$DB_URL" --limit "$LIMIT"
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
# --- Stage 2: WARC Parsing ---
|
|
||||||
echo ">>> Stage 2: WARC Parsing"
|
|
||||||
~/warc_parse --db "$DB_URL" --log-file "$HOME/warc_parse.log" --log-errors-only
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
# --- Stage 3: Icon Download ---
|
|
||||||
echo ">>> Stage 3: Icon Download"
|
|
||||||
GOMEMLIMIT=12GiB ~/icon_download --db "$DB_URL" --icons-dir "$ICONS_DIR" --log-file "$HOME/icon_download.log" --log-errors-only
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
# --- Stage 4: Best Icon Selection ---
|
|
||||||
echo ">>> Stage 4: Best Icon Selection"
|
|
||||||
psql "$DB_URL" -f "$SCRIPT_DIR/04_best_icon/select.sql"
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
# --- Stage 5: Bundle Generation ---
|
|
||||||
echo ">>> Stage 5: Bundle Generation"
|
|
||||||
~/bundle_gen --db "$DB_URL" --icons-dir "$ICONS_DIR" --site-bucket "$SITE_BUCKET" --log-file "$HOME/bundle_gen.log" --log-errors-only
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
# --- Stage 6: Frontend Deploy ---
|
|
||||||
echo ">>> Stage 6: Frontend Deploy"
|
|
||||||
TOTAL_BUNDLES=$(jq -r '.bundles_created' stats/05_bundle_gen.json)
|
|
||||||
"$SCRIPT_DIR/06_frontend/deploy.sh" --total-bundles "$TOTAL_BUNDLES"
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
# --- Done ---
|
|
||||||
END_TIME=$(date +%s)
|
|
||||||
DURATION=$(( END_TIME - START_TIME ))
|
|
||||||
HOURS=$(( DURATION / 3600 ))
|
|
||||||
MINS=$(( (DURATION % 3600) / 60 ))
|
|
||||||
|
|
||||||
echo "=========================================="
|
|
||||||
echo " Pipeline Complete"
|
|
||||||
echo " $(date)"
|
|
||||||
echo " Total duration: ${HOURS}h ${MINS}m"
|
|
||||||
echo "=========================================="
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue