joe/everytab
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

switched from s3 to disk for saving icons

Browse source
This commit is contained in:
Joe Lothan 2026-05-18 12:43:50 -04:00
parent 113a261dae
commit 5b3f6a6870
9 changed files with 84 additions and 112 deletions
Download patch file Download diff file Expand all files Collapse all files

13
pipeline/05_bundle_gen/bundle.go
View file

@ -1,7 +1,6 @@
package main
import (
"context"
"encoding/json"
"fmt"
"os"
@ -24,7 +23,7 @@ type Bundle struct {
}
// buildEntry creates a BundleEntry for a host, converting its icon if available.
func buildEntry(ctx context.Context, host HostRow, iconsBucket string, logWriter *LogWriter, stats *Stats) BundleEntry {
func buildEntry(host HostRow, iconsDir string, logWriter *LogWriter, stats *Stats) BundleEntry {
entry := BundleEntry{
Host: host.Hostname,
Title: host.HtmlTitle,
@ -36,7 +35,7 @@ func buildEntry(ctx context.Context, host HostRow, iconsBucket string, logWriter
return entry
}
encoded, w, h, convertErr := safeConvert(ctx, host.BestIconS3Key, iconsBucket)
encoded, w, h, convertErr := safeConvert(host.BestIconS3Key, iconsDir)
if convertErr != "" {
stats.ConvertErrors.Add(1)
logLine := fmt.Sprintf("CONVERT_ERROR: %s %s", host.Hostname, convertErr)
@ -54,7 +53,7 @@ func buildEntry(ctx context.Context, host HostRow, iconsBucket string, logWriter
}
// safeConvert wraps convertIconToBase64PNG with panic recovery.
func safeConvert(ctx context.Context, s3Key, iconsBucket string) (encoded string, w, h int, errMsg string) {
func safeConvert(hash, iconsDir string) (encoded string, w, h int, errMsg string) {
defer func() {
if r := recover(); r != nil {
errMsg = fmt.Sprintf("panic: %v", r)
@ -62,7 +61,7 @@ func safeConvert(ctx context.Context, s3Key, iconsBucket string) (encoded string
}()
var err error
encoded, w, h, err = convertIconToBase64PNG(ctx, s3Key, iconsBucket)
encoded, w, h, err = convertIconToBase64PNG(hash, iconsDir)
if err != nil {
return "", 0, 0, err.Error()
}
@ -79,7 +78,7 @@ func writeBundleLocal(outputDir string, index int, data []byte) error {
return os.WriteFile(path, data, 0644)
}
func writeBundleS3(ctx context.Context, bucket string, index int, data []byte) error {
func writeBundleS3(bucket string, index int, data []byte) error {
key := fmt.Sprintf("tabs/%04d.json", index)
return s3UploadBundle(ctx, bucket, key, data)
return s3UploadBundle(bucket, key, data)
}

9
pipeline/05_bundle_gen/convert.go
View file

@ -2,7 +2,6 @@ package main
import (
"bytes"
"context"
"encoding/base64"
"fmt"
"image"
@ -14,11 +13,11 @@ import (
_ "golang.org/x/image/webp"
)
// convertIconToBase64PNG downloads an icon from S3, converts it to PNG, and returns base64-encoded data.
func convertIconToBase64PNG(ctx context.Context, s3Key string, iconsBucket string) (encoded string, width, height int, err error) {
data, err := s3Download(ctx, iconsBucket, s3Key)
// convertIconToBase64PNG reads an icon from disk, converts it to PNG, and returns base64-encoded data.
func convertIconToBase64PNG(hash string, iconsDir string) (encoded string, width, height int, err error) {
data, err := readIconFromDisk(iconsDir, hash)
if err != nil {
return "", 0, 0, fmt.Errorf("s3 download: %w", err)
return "", 0, 0, fmt.Errorf("read icon: %w", err)
}
// Check for SVG (can't decode to raster without external deps)

10
pipeline/05_bundle_gen/main.go
View file

@ -15,7 +15,7 @@ import (
type Config struct {
DBUrl string
IconsBucket string
IconsDir string
SiteBucket string
EntriesPerBundle int
Concurrency int
@ -39,7 +39,7 @@ type Stats struct {
func main() {
cfg := Config{}
flag.StringVar(&cfg.DBUrl, "db", "", "Postgres connection string (required)")
flag.StringVar(&cfg.IconsBucket, "icons-bucket", "everytab-icons", "S3 bucket with downloaded icons")
flag.StringVar(&cfg.IconsDir, "icons-dir", "icons", "Directory with downloaded icons")
flag.StringVar(&cfg.SiteBucket, "site-bucket", "everytab-site", "S3 bucket for the static site")
flag.IntVar(&cfg.EntriesPerBundle, "entries-per-bundle", 120, "Tabs per bundle JSON file")
flag.BoolVar(&cfg.DryRun, "dry-run", false, "Write bundles to local disk instead of S3")
@ -122,7 +122,7 @@ func main() {
go func(idx int, h HostRow) {
defer wg.Done()
defer func() { <-sem }()
entries[idx] = buildEntry(ctx, h, cfg.IconsBucket, logWriter, stats)
entries[idx] = buildEntry(h, cfg.IconsDir, logWriter, stats)
n := processed.Add(1)
if n%5000 == 0 {
fmt.Printf(" processed %d/%d hosts\n", n, len(hosts))
@ -134,7 +134,7 @@ func main() {
// Clean old bundles before writing new ones (avoids orphans if count changed)
if !cfg.DryRun {
fmt.Println("\nCleaning old bundles from S3...")
if err := s3DeletePrefix(ctx, cfg.SiteBucket, "tabs/"); err != nil {
if err := s3DeletePrefix(cfg.SiteBucket, "tabs/"); err != nil {
log.Fatalf("Failed to clean old bundles: %v", err)
}
}
@ -160,7 +160,7 @@ func main() {
if cfg.DryRun {
err = writeBundleLocal(cfg.OutputDir, bundleIndex, data)
} else {
err = writeBundleS3(ctx, cfg.SiteBucket, bundleIndex, data)
err = writeBundleS3(cfg.SiteBucket, bundleIndex, data)
}
if err != nil {
log.Fatalf("Failed to write bundle %d: %v", bundleIndex, err)

30
pipeline/05_bundle_gen/s3.go
View file

@ -4,7 +4,8 @@ import (
"bytes"
"context"
"fmt"
"io"
"os"
"path/filepath"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
@ -23,22 +24,22 @@ func initS3() error {
return nil
}
// s3Download fetches an object from S3.
func s3Download(ctx context.Context, bucket, key string) ([]byte, error) {
resp, err := s3Client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
})
if err != nil {
return nil, err
// iconHashToPath converts a SHA-256 hash to a sharded file path.
func iconHashToPath(iconsDir, hash string) string {
if len(hash) < 6 {
return filepath.Join(iconsDir, hash)
}
defer resp.Body.Close()
return io.ReadAll(resp.Body)
return filepath.Join(iconsDir, hash[:2], hash[2:4], hash[4:6], hash)
}
// readIconFromDisk reads an icon file from the local sharded directory.
func readIconFromDisk(iconsDir, hash string) ([]byte, error) {
return os.ReadFile(iconHashToPath(iconsDir, hash))
}
// s3UploadBundle uploads a bundle JSON to S3.
func s3UploadBundle(ctx context.Context, bucket, key string, data []byte) error {
_, err := s3Client.PutObject(ctx, &s3.PutObjectInput{
func s3UploadBundle(bucket, key string, data []byte) error {
_, err := s3Client.PutObject(context.Background(), &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
Body: bytes.NewReader(data),
@ -48,7 +49,8 @@ func s3UploadBundle(ctx context.Context, bucket, key string, data []byte) error
}
// s3DeletePrefix deletes all objects under a prefix in S3.
func s3DeletePrefix(ctx context.Context, bucket, prefix string) error {
func s3DeletePrefix(bucket, prefix string) error {
ctx := context.Background()
paginator := s3.NewListObjectsV2Paginator(s3Client, &s3.ListObjectsV2Input{
Bucket: aws.String(bucket),
Prefix: aws.String(prefix),