joe/everytab
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

switched from s3 to disk for saving icons

Browse source
This commit is contained in:
Joe Lothan 2026-05-18 12:43:50 -04:00
parent 113a261dae
commit 5b3f6a6870
9 changed files with 84 additions and 112 deletions
Download patch file Download diff file Expand all files Collapse all files

10
pipeline/05_bundle_gen/main.go
View file

@ -15,7 +15,7 @@ import (
type Config struct {
DBUrl string
IconsBucket string
IconsDir string
SiteBucket string
EntriesPerBundle int
Concurrency int
@ -39,7 +39,7 @@ type Stats struct {
func main() {
cfg := Config{}
flag.StringVar(&cfg.DBUrl, "db", "", "Postgres connection string (required)")
flag.StringVar(&cfg.IconsBucket, "icons-bucket", "everytab-icons", "S3 bucket with downloaded icons")
flag.StringVar(&cfg.IconsDir, "icons-dir", "icons", "Directory with downloaded icons")
flag.StringVar(&cfg.SiteBucket, "site-bucket", "everytab-site", "S3 bucket for the static site")
flag.IntVar(&cfg.EntriesPerBundle, "entries-per-bundle", 120, "Tabs per bundle JSON file")
flag.BoolVar(&cfg.DryRun, "dry-run", false, "Write bundles to local disk instead of S3")
@ -122,7 +122,7 @@ func main() {
go func(idx int, h HostRow) {
defer wg.Done()
defer func() { <-sem }()
entries[idx] = buildEntry(ctx, h, cfg.IconsBucket, logWriter, stats)
entries[idx] = buildEntry(h, cfg.IconsDir, logWriter, stats)
n := processed.Add(1)
if n%5000 == 0 {
fmt.Printf(" processed %d/%d hosts\n", n, len(hosts))
@ -134,7 +134,7 @@ func main() {
// Clean old bundles before writing new ones (avoids orphans if count changed)
if !cfg.DryRun {
fmt.Println("\nCleaning old bundles from S3...")
if err := s3DeletePrefix(ctx, cfg.SiteBucket, "tabs/"); err != nil {
if err := s3DeletePrefix(cfg.SiteBucket, "tabs/"); err != nil {
log.Fatalf("Failed to clean old bundles: %v", err)
}
}
@ -160,7 +160,7 @@ func main() {
if cfg.DryRun {
err = writeBundleLocal(cfg.OutputDir, bundleIndex, data)
} else {
err = writeBundleS3(ctx, cfg.SiteBucket, bundleIndex, data)
err = writeBundleS3(cfg.SiteBucket, bundleIndex, data)
}
if err != nil {
log.Fatalf("Failed to write bundle %d: %v", bundleIndex, err)