added bundle generation

pipeline/05_bundle_gen/db.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"context"
+
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+type HostRow struct {
+	Hostname       string
+	Protocol       string
+	HtmlTitle      string
+	IframeAllowed  bool
+	BestIconS3Key  string
+}
+
+// fetchHosts gets all hosts with titles, randomized order.
+func fetchHosts(ctx context.Context, pool *pgxpool.Pool, limit int) ([]HostRow, error) {
+	query := `
+		SELECT hostname, protocol, html_title, COALESCE(iframe_allowed, true), COALESCE(best_icon_s3_key, '')
+		FROM hosts
+		WHERE html_title IS NOT NULL
+		ORDER BY random()
+	`
+	if limit > 0 {
+		query += " LIMIT $1"
+	}
+
+	var rows interface{ Query(context.Context, string, ...interface{}) (interface{ Close(); Next() bool; Scan(...interface{}) error; Err() error }, error) }
+	_ = rows // unused, using pool directly
+
+	var hosts []HostRow
+
+	if limit > 0 {
+		pgRows, err := pool.Query(ctx, query, limit)
+		if err != nil {
+			return nil, err
+		}
+		defer pgRows.Close()
+		for pgRows.Next() {
+			var h HostRow
+			if err := pgRows.Scan(&h.Hostname, &h.Protocol, &h.HtmlTitle, &h.IframeAllowed, &h.BestIconS3Key); err != nil {
+				return nil, err
+			}
+			hosts = append(hosts, h)
+		}
+		return hosts, pgRows.Err()
+	}
+
+	pgRows, err := pool.Query(ctx, query)
+	if err != nil {
+		return nil, err
+	}
+	defer pgRows.Close()
+	for pgRows.Next() {
+		var h HostRow
+		if err := pgRows.Scan(&h.Hostname, &h.Protocol, &h.HtmlTitle, &h.IframeAllowed, &h.BestIconS3Key); err != nil {
+			return nil, err
+		}
+		hosts = append(hosts, h)
+	}
+	return hosts, pgRows.Err()
+}